vhost.c 42.6 KB
Newer Older
1 2 3 4 5 6
/* Copyright (C) 2009 Red Hat, Inc.
 * Copyright (C) 2006 Rusty Russell IBM Corporation
 *
 * Author: Michael S. Tsirkin <mst@redhat.com>
 *
 * Inspiration, some code, and most witty comments come from
R
Rob Landley 已提交
7
 * Documentation/virtual/lguest/lguest.c, by Rusty Russell
8 9 10 11 12 13 14 15
 *
 * This work is licensed under the terms of the GNU GPL, version 2.
 *
 * Generic code for virtio server in host kernel.
 */

#include <linux/eventfd.h>
#include <linux/vhost.h>
16
#include <linux/uio.h>
17
#include <linux/mm.h>
M
Michael S. Tsirkin 已提交
18
#include <linux/mmu_context.h>
19 20 21 22 23
#include <linux/miscdevice.h>
#include <linux/mutex.h>
#include <linux/poll.h>
#include <linux/file.h>
#include <linux/highmem.h>
24
#include <linux/slab.h>
25
#include <linux/vmalloc.h>
26
#include <linux/kthread.h>
27
#include <linux/cgroup.h>
A
Asias He 已提交
28
#include <linux/module.h>
29
#include <linux/sort.h>
30 31 32

#include "vhost.h"

33 34 35 36 37
static ushort max_mem_regions = 64;
module_param(max_mem_regions, ushort, 0444);
MODULE_PARM_DESC(max_mem_regions,
	"Maximum number of memory regions in memory map. (default: 64)");

38 39 40 41
enum {
	VHOST_MEMORY_F_LOG = 0x1,
};

42 43
#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
M
Michael S. Tsirkin 已提交
44

45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
#ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
static void vhost_vq_reset_user_be(struct vhost_virtqueue *vq)
{
	vq->user_be = !virtio_legacy_is_little_endian();
}

static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
{
	struct vhost_vring_state s;

	if (vq->private_data)
		return -EBUSY;

	if (copy_from_user(&s, argp, sizeof(s)))
		return -EFAULT;

	if (s.num != VHOST_VRING_LITTLE_ENDIAN &&
	    s.num != VHOST_VRING_BIG_ENDIAN)
		return -EINVAL;

	vq->user_be = s.num;

	return 0;
}

static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
				   int __user *argp)
{
	struct vhost_vring_state s = {
		.index = idx,
		.num = vq->user_be
	};

	if (copy_to_user(argp, &s, sizeof(s)))
		return -EFAULT;

	return 0;
}

static void vhost_init_is_le(struct vhost_virtqueue *vq)
{
	/* Note for legacy virtio: user_be is initialized at reset time
	 * according to the host endianness. If userspace does not set an
	 * explicit endianness, the default behavior is native endian, as
	 * expected by legacy virtio.
	 */
	vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be;
}
#else
static void vhost_vq_reset_user_be(struct vhost_virtqueue *vq)
{
}

static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
{
	return -ENOIOCTLCMD;
}

static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
				   int __user *argp)
{
	return -ENOIOCTLCMD;
}

static void vhost_init_is_le(struct vhost_virtqueue *vq)
{
	if (vhost_has_feature(vq, VIRTIO_F_VERSION_1))
		vq->is_le = true;
}
#endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */

116 117 118 119 120
static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
			    poll_table *pt)
{
	struct vhost_poll *poll;

K
Krishna Kumar 已提交
121
	poll = container_of(pt, struct vhost_poll, table);
122 123 124 125 126 127 128
	poll->wqh = wqh;
	add_wait_queue(wqh, &poll->wait);
}

static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync,
			     void *key)
{
129 130
	struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);

131 132 133
	if (!((unsigned long)key & poll->mask))
		return 0;

134
	vhost_poll_queue(poll);
135 136 137
	return 0;
}

138
void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
139 140 141 142 143 144 145
{
	INIT_LIST_HEAD(&work->node);
	work->fn = fn;
	init_waitqueue_head(&work->done);
	work->flushing = 0;
	work->queue_seq = work->done_seq = 0;
}
A
Asias He 已提交
146
EXPORT_SYMBOL_GPL(vhost_work_init);
147

148
/* Init poll structure */
149 150
void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
		     unsigned long mask, struct vhost_dev *dev)
151 152 153 154
{
	init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
	init_poll_funcptr(&poll->table, vhost_poll_func);
	poll->mask = mask;
155
	poll->dev = dev;
156
	poll->wqh = NULL;
157

158
	vhost_work_init(&poll->work, fn);
159
}
A
Asias He 已提交
160
EXPORT_SYMBOL_GPL(vhost_poll_init);
161 162 163

/* Start polling a file. We add ourselves to file's wait queue. The caller must
 * keep a reference to a file until after vhost_poll_stop is called. */
164
int vhost_poll_start(struct vhost_poll *poll, struct file *file)
165 166
{
	unsigned long mask;
167
	int ret = 0;
K
Krishna Kumar 已提交
168

J
Jason Wang 已提交
169 170 171
	if (poll->wqh)
		return 0;

172 173 174
	mask = file->f_op->poll(file, &poll->table);
	if (mask)
		vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask);
175 176 177 178 179 180 181
	if (mask & POLLERR) {
		if (poll->wqh)
			remove_wait_queue(poll->wqh, &poll->wait);
		ret = -EINVAL;
	}

	return ret;
182
}
A
Asias He 已提交
183
EXPORT_SYMBOL_GPL(vhost_poll_start);
184 185 186 187 188

/* Stop polling a file. After this function returns, it becomes safe to drop the
 * file reference. You must also flush afterwards. */
void vhost_poll_stop(struct vhost_poll *poll)
{
189 190 191 192
	if (poll->wqh) {
		remove_wait_queue(poll->wqh, &poll->wait);
		poll->wqh = NULL;
	}
193
}
A
Asias He 已提交
194
EXPORT_SYMBOL_GPL(vhost_poll_stop);
195

196 197 198 199
static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work,
				unsigned seq)
{
	int left;
K
Krishna Kumar 已提交
200

201 202 203 204 205 206
	spin_lock_irq(&dev->work_lock);
	left = seq - work->done_seq;
	spin_unlock_irq(&dev->work_lock);
	return left <= 0;
}

A
Asias He 已提交
207
void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
208
{
209 210 211
	unsigned seq;
	int flushing;

212
	spin_lock_irq(&dev->work_lock);
213 214
	seq = work->queue_seq;
	work->flushing++;
215
	spin_unlock_irq(&dev->work_lock);
216
	wait_event(work->done, vhost_work_seq_done(dev, work, seq));
217
	spin_lock_irq(&dev->work_lock);
218
	flushing = --work->flushing;
219
	spin_unlock_irq(&dev->work_lock);
220
	BUG_ON(flushing < 0);
221
}
A
Asias He 已提交
222
EXPORT_SYMBOL_GPL(vhost_work_flush);
223

224 225 226 227 228 229
/* Flush any work that has been scheduled. When calling this, don't hold any
 * locks that are also used by the callback. */
void vhost_poll_flush(struct vhost_poll *poll)
{
	vhost_work_flush(poll->dev, &poll->work);
}
A
Asias He 已提交
230
EXPORT_SYMBOL_GPL(vhost_poll_flush);
231

232
void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
233
{
234 235 236 237 238 239
	unsigned long flags;

	spin_lock_irqsave(&dev->work_lock, flags);
	if (list_empty(&work->node)) {
		list_add_tail(&work->node, &dev->work_list);
		work->queue_seq++;
240
		spin_unlock_irqrestore(&dev->work_lock, flags);
241
		wake_up_process(dev->worker);
242 243
	} else {
		spin_unlock_irqrestore(&dev->work_lock, flags);
244
	}
245
}
A
Asias He 已提交
246
EXPORT_SYMBOL_GPL(vhost_work_queue);
247

248 249 250 251
void vhost_poll_queue(struct vhost_poll *poll)
{
	vhost_work_queue(poll->dev, &poll->work);
}
A
Asias He 已提交
252
EXPORT_SYMBOL_GPL(vhost_poll_queue);
253

254 255 256 257 258 259 260 261 262 263
static void vhost_vq_reset(struct vhost_dev *dev,
			   struct vhost_virtqueue *vq)
{
	vq->num = 1;
	vq->desc = NULL;
	vq->avail = NULL;
	vq->used = NULL;
	vq->last_avail_idx = 0;
	vq->avail_idx = 0;
	vq->last_used_idx = 0;
M
Michael S. Tsirkin 已提交
264 265
	vq->signalled_used = 0;
	vq->signalled_used_valid = false;
266 267 268 269
	vq->used_flags = 0;
	vq->log_used = false;
	vq->log_addr = -1ull;
	vq->private_data = NULL;
270
	vq->acked_features = 0;
271 272 273 274 275 276
	vq->log_base = NULL;
	vq->error_ctx = NULL;
	vq->error = NULL;
	vq->kick = NULL;
	vq->call_ctx = NULL;
	vq->call = NULL;
277
	vq->log_ctx = NULL;
278
	vq->memory = NULL;
279 280
	vq->is_le = virtio_legacy_is_little_endian();
	vhost_vq_reset_user_be(vq);
281 282
}

283 284 285 286 287
static int vhost_worker(void *data)
{
	struct vhost_dev *dev = data;
	struct vhost_work *work = NULL;
	unsigned uninitialized_var(seq);
288
	mm_segment_t oldfs = get_fs();
289

290
	set_fs(USER_DS);
M
Michael S. Tsirkin 已提交
291 292
	use_mm(dev->mm);

293 294 295 296 297 298 299 300 301 302 303 304 305 306
	for (;;) {
		/* mb paired w/ kthread_stop */
		set_current_state(TASK_INTERRUPTIBLE);

		spin_lock_irq(&dev->work_lock);
		if (work) {
			work->done_seq = seq;
			if (work->flushing)
				wake_up_all(&work->done);
		}

		if (kthread_should_stop()) {
			spin_unlock_irq(&dev->work_lock);
			__set_current_state(TASK_RUNNING);
M
Michael S. Tsirkin 已提交
307
			break;
308 309 310 311 312 313 314 315 316 317 318 319 320
		}
		if (!list_empty(&dev->work_list)) {
			work = list_first_entry(&dev->work_list,
						struct vhost_work, node);
			list_del_init(&work->node);
			seq = work->queue_seq;
		} else
			work = NULL;
		spin_unlock_irq(&dev->work_lock);

		if (work) {
			__set_current_state(TASK_RUNNING);
			work->fn(work);
N
Nadav Har'El 已提交
321 322
			if (need_resched())
				schedule();
323 324 325 326
		} else
			schedule();

	}
M
Michael S. Tsirkin 已提交
327
	unuse_mm(dev->mm);
328
	set_fs(oldfs);
M
Michael S. Tsirkin 已提交
329
	return 0;
330 331
}

332 333 334 335 336 337 338 339 340 341
static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
{
	kfree(vq->indirect);
	vq->indirect = NULL;
	kfree(vq->log);
	vq->log = NULL;
	kfree(vq->heads);
	vq->heads = NULL;
}

J
Jason Wang 已提交
342 343 344
/* Helper to allocate iovec buffers for all vqs. */
static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
{
A
Asias He 已提交
345
	struct vhost_virtqueue *vq;
J
Jason Wang 已提交
346
	int i;
K
Krishna Kumar 已提交
347

J
Jason Wang 已提交
348
	for (i = 0; i < dev->nvqs; ++i) {
A
Asias He 已提交
349 350 351 352 353 354
		vq = dev->vqs[i];
		vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV,
				       GFP_KERNEL);
		vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL);
		vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL);
		if (!vq->indirect || !vq->log || !vq->heads)
J
Jason Wang 已提交
355 356 357
			goto err_nomem;
	}
	return 0;
K
Krishna Kumar 已提交
358

J
Jason Wang 已提交
359
err_nomem:
360
	for (; i >= 0; --i)
361
		vhost_vq_free_iovecs(dev->vqs[i]);
J
Jason Wang 已提交
362 363 364 365 366 367
	return -ENOMEM;
}

static void vhost_dev_free_iovecs(struct vhost_dev *dev)
{
	int i;
K
Krishna Kumar 已提交
368

369
	for (i = 0; i < dev->nvqs; ++i)
370
		vhost_vq_free_iovecs(dev->vqs[i]);
J
Jason Wang 已提交
371 372
}

Z
Zhi Yong Wu 已提交
373
void vhost_dev_init(struct vhost_dev *dev,
374
		    struct vhost_virtqueue **vqs, int nvqs)
375
{
A
Asias He 已提交
376
	struct vhost_virtqueue *vq;
377
	int i;
378

379 380 381 382 383 384 385
	dev->vqs = vqs;
	dev->nvqs = nvqs;
	mutex_init(&dev->mutex);
	dev->log_ctx = NULL;
	dev->log_file = NULL;
	dev->memory = NULL;
	dev->mm = NULL;
386 387 388
	spin_lock_init(&dev->work_lock);
	INIT_LIST_HEAD(&dev->work_list);
	dev->worker = NULL;
389 390

	for (i = 0; i < dev->nvqs; ++i) {
A
Asias He 已提交
391 392 393 394 395 396 397 398 399 400
		vq = dev->vqs[i];
		vq->log = NULL;
		vq->indirect = NULL;
		vq->heads = NULL;
		vq->dev = dev;
		mutex_init(&vq->mutex);
		vhost_vq_reset(dev, vq);
		if (vq->handle_kick)
			vhost_poll_init(&vq->poll, vq->handle_kick,
					POLLIN, dev);
401 402
	}
}
A
Asias He 已提交
403
EXPORT_SYMBOL_GPL(vhost_dev_init);
404 405 406 407 408 409 410

/* Caller should have device mutex */
long vhost_dev_check_owner(struct vhost_dev *dev)
{
	/* Are you the owner? If not, I don't think you mean to do that */
	return dev->mm == current->mm ? 0 : -EPERM;
}
A
Asias He 已提交
411
EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
412

413
struct vhost_attach_cgroups_struct {
K
Krishna Kumar 已提交
414 415 416
	struct vhost_work work;
	struct task_struct *owner;
	int ret;
417 418 419 420
};

static void vhost_attach_cgroups_work(struct vhost_work *work)
{
K
Krishna Kumar 已提交
421 422 423 424
	struct vhost_attach_cgroups_struct *s;

	s = container_of(work, struct vhost_attach_cgroups_struct, work);
	s->ret = cgroup_attach_task_all(s->owner, current);
425 426 427 428
}

static int vhost_attach_cgroups(struct vhost_dev *dev)
{
K
Krishna Kumar 已提交
429 430 431 432 433 434 435
	struct vhost_attach_cgroups_struct attach;

	attach.owner = current;
	vhost_work_init(&attach.work, vhost_attach_cgroups_work);
	vhost_work_queue(dev, &attach.work);
	vhost_work_flush(dev, &attach.work);
	return attach.ret;
436 437
}

438 439 440 441 442
/* Caller should have device mutex */
bool vhost_dev_has_owner(struct vhost_dev *dev)
{
	return dev->mm;
}
A
Asias He 已提交
443
EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
444

445
/* Caller should have device mutex */
A
Asias He 已提交
446
long vhost_dev_set_owner(struct vhost_dev *dev)
447
{
448 449
	struct task_struct *worker;
	int err;
K
Krishna Kumar 已提交
450

451
	/* Is there an owner already? */
452
	if (vhost_dev_has_owner(dev)) {
453 454 455
		err = -EBUSY;
		goto err_mm;
	}
K
Krishna Kumar 已提交
456

457 458
	/* No owner, become one */
	dev->mm = get_task_mm(current);
459 460 461 462 463 464 465
	worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
	if (IS_ERR(worker)) {
		err = PTR_ERR(worker);
		goto err_worker;
	}

	dev->worker = worker;
466 467 468
	wake_up_process(worker);	/* avoid contributing to loadavg */

	err = vhost_attach_cgroups(dev);
469 470
	if (err)
		goto err_cgroup;
471

J
Jason Wang 已提交
472 473 474 475
	err = vhost_dev_alloc_iovecs(dev);
	if (err)
		goto err_cgroup;

476
	return 0;
477 478
err_cgroup:
	kthread_stop(worker);
M
Michael S. Tsirkin 已提交
479
	dev->worker = NULL;
480 481 482 483 484 485
err_worker:
	if (dev->mm)
		mmput(dev->mm);
	dev->mm = NULL;
err_mm:
	return err;
486
}
A
Asias He 已提交
487
EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
488

489
struct vhost_memory *vhost_dev_reset_owner_prepare(void)
490
{
491 492
	return kmalloc(offsetof(struct vhost_memory, regions), GFP_KERNEL);
}
A
Asias He 已提交
493
EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
494

495 496 497
/* Caller should have device mutex */
void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_memory *memory)
{
498 499
	int i;

500
	vhost_dev_cleanup(dev, true);
501

502
	/* Restore memory to default empty mapping. */
503
	memory->nregions = 0;
504 505 506 507 508 509
	dev->memory = memory;
	/* We don't need VQ locks below since vhost_dev_cleanup makes sure
	 * VQs aren't running.
	 */
	for (i = 0; i < dev->nvqs; ++i)
		dev->vqs[i]->memory = memory;
510
}
A
Asias He 已提交
511
EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
512

513
void vhost_dev_stop(struct vhost_dev *dev)
514 515
{
	int i;
516 517

	for (i = 0; i < dev->nvqs; ++i) {
518 519 520
		if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) {
			vhost_poll_stop(&dev->vqs[i]->poll);
			vhost_poll_flush(&dev->vqs[i]->poll);
521
		}
522 523
	}
}
A
Asias He 已提交
524
EXPORT_SYMBOL_GPL(vhost_dev_stop);
525

526 527
/* Caller should have device mutex if and only if locked is set */
void vhost_dev_cleanup(struct vhost_dev *dev, bool locked)
528 529
{
	int i;
K
Krishna Kumar 已提交
530

531
	for (i = 0; i < dev->nvqs; ++i) {
532 533 534 535 536 537 538 539 540 541 542
		if (dev->vqs[i]->error_ctx)
			eventfd_ctx_put(dev->vqs[i]->error_ctx);
		if (dev->vqs[i]->error)
			fput(dev->vqs[i]->error);
		if (dev->vqs[i]->kick)
			fput(dev->vqs[i]->kick);
		if (dev->vqs[i]->call_ctx)
			eventfd_ctx_put(dev->vqs[i]->call_ctx);
		if (dev->vqs[i]->call)
			fput(dev->vqs[i]->call);
		vhost_vq_reset(dev, dev->vqs[i]);
543
	}
J
Jason Wang 已提交
544
	vhost_dev_free_iovecs(dev);
545 546 547 548 549 550 551
	if (dev->log_ctx)
		eventfd_ctx_put(dev->log_ctx);
	dev->log_ctx = NULL;
	if (dev->log_file)
		fput(dev->log_file);
	dev->log_file = NULL;
	/* No one will access memory at this point */
552
	kvfree(dev->memory);
553
	dev->memory = NULL;
554
	WARN_ON(!list_empty(&dev->work_list));
555 556 557 558
	if (dev->worker) {
		kthread_stop(dev->worker);
		dev->worker = NULL;
	}
559 560 561
	if (dev->mm)
		mmput(dev->mm);
	dev->mm = NULL;
562
}
A
Asias He 已提交
563
EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
564 565 566 567

static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
{
	u64 a = addr / VHOST_PAGE_SIZE / 8;
K
Krishna Kumar 已提交
568

569 570 571
	/* Make sure 64 bit math will not overflow. */
	if (a > ULONG_MAX - (unsigned long)log_base ||
	    a + (unsigned long)log_base > ULONG_MAX)
572
		return 0;
573 574 575 576 577 578 579 580 581 582

	return access_ok(VERIFY_WRITE, log_base + a,
			 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
}

/* Caller should have vq mutex and device mutex. */
static int vq_memory_access_ok(void __user *log_base, struct vhost_memory *mem,
			       int log_all)
{
	int i;
583

M
Michael S. Tsirkin 已提交
584 585
	if (!mem)
		return 0;
586

587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608
	for (i = 0; i < mem->nregions; ++i) {
		struct vhost_memory_region *m = mem->regions + i;
		unsigned long a = m->userspace_addr;
		if (m->memory_size > ULONG_MAX)
			return 0;
		else if (!access_ok(VERIFY_WRITE, (void __user *)a,
				    m->memory_size))
			return 0;
		else if (log_all && !log_access_ok(log_base,
						   m->guest_phys_addr,
						   m->memory_size))
			return 0;
	}
	return 1;
}

/* Can we switch to this memory table? */
/* Caller should have device mutex but not vq mutex */
static int memory_access_ok(struct vhost_dev *d, struct vhost_memory *mem,
			    int log_all)
{
	int i;
K
Krishna Kumar 已提交
609

610 611
	for (i = 0; i < d->nvqs; ++i) {
		int ok;
612 613
		bool log;

614
		mutex_lock(&d->vqs[i]->mutex);
615
		log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL);
616
		/* If ring is inactive, will check when it's enabled. */
617
		if (d->vqs[i]->private_data)
618
			ok = vq_memory_access_ok(d->vqs[i]->log_base, mem, log);
619 620
		else
			ok = 1;
621
		mutex_unlock(&d->vqs[i]->mutex);
622 623 624 625 626 627
		if (!ok)
			return 0;
	}
	return 1;
}

628
static int vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
629 630 631 632
			struct vring_desc __user *desc,
			struct vring_avail __user *avail,
			struct vring_used __user *used)
{
633
	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
634 635
	return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
	       access_ok(VERIFY_READ, avail,
M
Michael S. Tsirkin 已提交
636
			 sizeof *avail + num * sizeof *avail->ring + s) &&
637
	       access_ok(VERIFY_WRITE, used,
M
Michael S. Tsirkin 已提交
638
			sizeof *used + num * sizeof *used->ring + s);
639 640 641 642 643 644
}

/* Can we log writes? */
/* Caller should have device mutex but not vq mutex */
int vhost_log_access_ok(struct vhost_dev *dev)
{
645
	return memory_access_ok(dev, dev->memory, 1);
646
}
A
Asias He 已提交
647
EXPORT_SYMBOL_GPL(vhost_log_access_ok);
648 649 650

/* Verify access for write logging. */
/* Caller should have vq mutex and device mutex */
651
static int vq_log_access_ok(struct vhost_virtqueue *vq,
M
Michael S. Tsirkin 已提交
652
			    void __user *log_base)
653
{
654
	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
A
Arnd Bergmann 已提交
655

656
	return vq_memory_access_ok(log_base, vq->memory,
657
				   vhost_has_feature(vq, VHOST_F_LOG_ALL)) &&
658 659
		(!vq->log_used || log_access_ok(log_base, vq->log_addr,
					sizeof *vq->used +
M
Michael S. Tsirkin 已提交
660
					vq->num * sizeof *vq->used->ring + s));
661 662 663 664 665 666
}

/* Can we start vq? */
/* Caller should have vq mutex and device mutex */
int vhost_vq_access_ok(struct vhost_virtqueue *vq)
{
667 668
	return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used) &&
		vq_log_access_ok(vq, vq->log_base);
669
}
A
Asias He 已提交
670
EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
671

672 673 674 675 676 677 678 679 680 681
static int vhost_memory_reg_sort_cmp(const void *p1, const void *p2)
{
	const struct vhost_memory_region *r1 = p1, *r2 = p2;
	if (r1->guest_phys_addr < r2->guest_phys_addr)
		return 1;
	if (r1->guest_phys_addr > r2->guest_phys_addr)
		return -1;
	return 0;
}

682 683 684 685
static void *vhost_kvzalloc(unsigned long size)
{
	void *n = kzalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);

686
	if (!n)
687 688 689 690
		n = vzalloc(size);
	return n;
}

691 692 693 694
static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
{
	struct vhost_memory mem, *newmem, *oldmem;
	unsigned long size = offsetof(struct vhost_memory, regions);
695
	int i;
K
Krishna Kumar 已提交
696

697 698
	if (copy_from_user(&mem, m, size))
		return -EFAULT;
699 700
	if (mem.padding)
		return -EOPNOTSUPP;
701
	if (mem.nregions > max_mem_regions)
702
		return -E2BIG;
703
	newmem = vhost_kvzalloc(size + mem.nregions * sizeof(*m->regions));
704 705 706 707
	if (!newmem)
		return -ENOMEM;

	memcpy(newmem, &mem, size);
708 709
	if (copy_from_user(newmem->regions, m->regions,
			   mem.nregions * sizeof *m->regions)) {
710
		kvfree(newmem);
711
		return -EFAULT;
712
	}
713 714
	sort(newmem->regions, newmem->nregions, sizeof(*newmem->regions),
		vhost_memory_reg_sort_cmp, NULL);
715

716
	if (!memory_access_ok(d, newmem, 0)) {
717
		kvfree(newmem);
718
		return -EFAULT;
719
	}
720 721
	oldmem = d->memory;
	d->memory = newmem;
722

723
	/* All memory accesses are done under some VQ mutex. */
724 725
	for (i = 0; i < d->nvqs; ++i) {
		mutex_lock(&d->vqs[i]->mutex);
726
		d->vqs[i]->memory = newmem;
727 728
		mutex_unlock(&d->vqs[i]->mutex);
	}
729
	kvfree(oldmem);
730 731 732
	return 0;
}

733
long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp)
734
{
735 736
	struct file *eventfp, *filep = NULL;
	bool pollstart = false, pollstop = false;
737 738 739 740 741 742 743 744 745 746 747 748
	struct eventfd_ctx *ctx = NULL;
	u32 __user *idxp = argp;
	struct vhost_virtqueue *vq;
	struct vhost_vring_state s;
	struct vhost_vring_file f;
	struct vhost_vring_addr a;
	u32 idx;
	long r;

	r = get_user(idx, idxp);
	if (r < 0)
		return r;
749
	if (idx >= d->nvqs)
750 751
		return -ENOBUFS;

752
	vq = d->vqs[idx];
753 754 755 756 757 758 759 760 761 762 763

	mutex_lock(&vq->mutex);

	switch (ioctl) {
	case VHOST_SET_VRING_NUM:
		/* Resizing ring with an active backend?
		 * You don't want to do that. */
		if (vq->private_data) {
			r = -EBUSY;
			break;
		}
764 765
		if (copy_from_user(&s, argp, sizeof s)) {
			r = -EFAULT;
766
			break;
767
		}
768 769 770 771 772 773 774 775 776 777 778 779 780
		if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) {
			r = -EINVAL;
			break;
		}
		vq->num = s.num;
		break;
	case VHOST_SET_VRING_BASE:
		/* Moving base with an active backend?
		 * You don't want to do that. */
		if (vq->private_data) {
			r = -EBUSY;
			break;
		}
781 782
		if (copy_from_user(&s, argp, sizeof s)) {
			r = -EFAULT;
783
			break;
784
		}
785 786 787 788 789 790 791 792 793 794 795
		if (s.num > 0xffff) {
			r = -EINVAL;
			break;
		}
		vq->last_avail_idx = s.num;
		/* Forget the cached index value. */
		vq->avail_idx = vq->last_avail_idx;
		break;
	case VHOST_GET_VRING_BASE:
		s.index = idx;
		s.num = vq->last_avail_idx;
796 797
		if (copy_to_user(argp, &s, sizeof s))
			r = -EFAULT;
798 799
		break;
	case VHOST_SET_VRING_ADDR:
800 801
		if (copy_from_user(&a, argp, sizeof a)) {
			r = -EFAULT;
802
			break;
803
		}
804 805 806 807 808 809 810 811 812 813 814 815
		if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) {
			r = -EOPNOTSUPP;
			break;
		}
		/* For 32bit, verify that the top 32bits of the user
		   data are set to zero. */
		if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
		    (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
		    (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) {
			r = -EFAULT;
			break;
		}
816 817 818 819 820 821

		/* Make sure it's safe to cast pointers to vring types. */
		BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE);
		BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE);
		if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) ||
		    (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) ||
822
		    (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) {
823 824 825 826 827 828 829 830
			r = -EINVAL;
			break;
		}

		/* We only verify access here if backend is configured.
		 * If it is not, we don't as size might not have been setup.
		 * We will verify when backend is configured. */
		if (vq->private_data) {
831
			if (!vq_access_ok(vq, vq->num,
832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
				(void __user *)(unsigned long)a.desc_user_addr,
				(void __user *)(unsigned long)a.avail_user_addr,
				(void __user *)(unsigned long)a.used_user_addr)) {
				r = -EINVAL;
				break;
			}

			/* Also validate log access for used ring if enabled. */
			if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
			    !log_access_ok(vq->log_base, a.log_guest_addr,
					   sizeof *vq->used +
					   vq->num * sizeof *vq->used->ring)) {
				r = -EINVAL;
				break;
			}
		}

		vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
		vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
		vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
		vq->log_addr = a.log_guest_addr;
		vq->used = (void __user *)(unsigned long)a.used_user_addr;
		break;
	case VHOST_SET_VRING_KICK:
856 857
		if (copy_from_user(&f, argp, sizeof f)) {
			r = -EFAULT;
858
			break;
859
		}
860
		eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
861 862 863 864
		if (IS_ERR(eventfp)) {
			r = PTR_ERR(eventfp);
			break;
		}
865
		if (eventfp != vq->kick) {
866 867
			pollstop = (filep = vq->kick) != NULL;
			pollstart = (vq->kick = eventfp) != NULL;
868 869 870 871
		} else
			filep = eventfp;
		break;
	case VHOST_SET_VRING_CALL:
872 873
		if (copy_from_user(&f, argp, sizeof f)) {
			r = -EFAULT;
874
			break;
875
		}
876
		eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
877 878 879 880
		if (IS_ERR(eventfp)) {
			r = PTR_ERR(eventfp);
			break;
		}
881 882 883 884 885 886 887 888 889 890
		if (eventfp != vq->call) {
			filep = vq->call;
			ctx = vq->call_ctx;
			vq->call = eventfp;
			vq->call_ctx = eventfp ?
				eventfd_ctx_fileget(eventfp) : NULL;
		} else
			filep = eventfp;
		break;
	case VHOST_SET_VRING_ERR:
891 892
		if (copy_from_user(&f, argp, sizeof f)) {
			r = -EFAULT;
893
			break;
894
		}
895
		eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
896 897 898 899
		if (IS_ERR(eventfp)) {
			r = PTR_ERR(eventfp);
			break;
		}
900 901 902 903 904 905 906 907 908
		if (eventfp != vq->error) {
			filep = vq->error;
			vq->error = eventfp;
			ctx = vq->error_ctx;
			vq->error_ctx = eventfp ?
				eventfd_ctx_fileget(eventfp) : NULL;
		} else
			filep = eventfp;
		break;
909 910 911 912 913 914
	case VHOST_SET_VRING_ENDIAN:
		r = vhost_set_vring_endian(vq, argp);
		break;
	case VHOST_GET_VRING_ENDIAN:
		r = vhost_get_vring_endian(vq, idx, argp);
		break;
915 916 917 918 919 920 921 922 923 924 925 926 927
	default:
		r = -ENOIOCTLCMD;
	}

	if (pollstop && vq->handle_kick)
		vhost_poll_stop(&vq->poll);

	if (ctx)
		eventfd_ctx_put(ctx);
	if (filep)
		fput(filep);

	if (pollstart && vq->handle_kick)
928
		r = vhost_poll_start(&vq->poll, vq->kick);
929 930 931 932 933 934 935

	mutex_unlock(&vq->mutex);

	if (pollstop && vq->handle_kick)
		vhost_poll_flush(&vq->poll);
	return r;
}
A
Asias He 已提交
936
EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
937 938

/* Caller must have device mutex */
939
long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962
{
	struct file *eventfp, *filep = NULL;
	struct eventfd_ctx *ctx = NULL;
	u64 p;
	long r;
	int i, fd;

	/* If you are not the owner, you can become one */
	if (ioctl == VHOST_SET_OWNER) {
		r = vhost_dev_set_owner(d);
		goto done;
	}

	/* You must be the owner to do anything else */
	r = vhost_dev_check_owner(d);
	if (r)
		goto done;

	switch (ioctl) {
	case VHOST_SET_MEM_TABLE:
		r = vhost_set_memory(d, argp);
		break;
	case VHOST_SET_LOG_BASE:
963 964
		if (copy_from_user(&p, argp, sizeof p)) {
			r = -EFAULT;
965
			break;
966
		}
967 968 969 970 971 972 973
		if ((u64)(unsigned long)p != p) {
			r = -EFAULT;
			break;
		}
		for (i = 0; i < d->nvqs; ++i) {
			struct vhost_virtqueue *vq;
			void __user *base = (void __user *)(unsigned long)p;
974
			vq = d->vqs[i];
975 976
			mutex_lock(&vq->mutex);
			/* If ring is inactive, will check when it's enabled. */
977
			if (vq->private_data && !vq_log_access_ok(vq, base))
978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994
				r = -EFAULT;
			else
				vq->log_base = base;
			mutex_unlock(&vq->mutex);
		}
		break;
	case VHOST_SET_LOG_FD:
		r = get_user(fd, (int __user *)argp);
		if (r < 0)
			break;
		eventfp = fd == -1 ? NULL : eventfd_fget(fd);
		if (IS_ERR(eventfp)) {
			r = PTR_ERR(eventfp);
			break;
		}
		if (eventfp != d->log_file) {
			filep = d->log_file;
995
			d->log_file = eventfp;
996 997 998 999 1000 1001
			ctx = d->log_ctx;
			d->log_ctx = eventfp ?
				eventfd_ctx_fileget(eventfp) : NULL;
		} else
			filep = eventfp;
		for (i = 0; i < d->nvqs; ++i) {
1002 1003 1004
			mutex_lock(&d->vqs[i]->mutex);
			d->vqs[i]->log_ctx = d->log_ctx;
			mutex_unlock(&d->vqs[i]->mutex);
1005 1006 1007 1008 1009 1010 1011
		}
		if (ctx)
			eventfd_ctx_put(ctx);
		if (filep)
			fput(filep);
		break;
	default:
1012
		r = -ENOIOCTLCMD;
1013 1014 1015 1016 1017
		break;
	}
done:
	return r;
}
A
Asias He 已提交
1018
EXPORT_SYMBOL_GPL(vhost_dev_ioctl);
1019 1020 1021 1022

static const struct vhost_memory_region *find_region(struct vhost_memory *mem,
						     __u64 addr, __u32 len)
{
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
	const struct vhost_memory_region *reg;
	int start = 0, end = mem->nregions;

	while (start < end) {
		int slot = start + (end - start) / 2;
		reg = mem->regions + slot;
		if (addr >= reg->guest_phys_addr)
			end = slot;
		else
			start = slot + 1;
1033
	}
1034 1035 1036 1037 1038

	reg = mem->regions + start;
	if (addr >= reg->guest_phys_addr &&
		reg->guest_phys_addr + reg->memory_size > addr)
		return reg;
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
	return NULL;
}

/* TODO: This is really inefficient.  We need something like get_user()
 * (instruction directly accesses the data, with an exception table entry
 * returning -EFAULT). See Documentation/x86/exception-tables.txt.
 */
static int set_bit_to_user(int nr, void __user *addr)
{
	unsigned long log = (unsigned long)addr;
	struct page *page;
	void *base;
	int bit = nr + (log % PAGE_SIZE) * 8;
	int r;
K
Krishna Kumar 已提交
1053

1054
	r = get_user_pages_fast(log, 1, 1, &page);
1055
	if (r < 0)
1056
		return r;
1057
	BUG_ON(r != 1);
1058
	base = kmap_atomic(page);
1059
	set_bit(bit, base);
1060
	kunmap_atomic(base);
1061 1062 1063 1064 1065 1066 1067 1068
	set_page_dirty_lock(page);
	put_page(page);
	return 0;
}

static int log_write(void __user *log_base,
		     u64 write_address, u64 write_length)
{
1069
	u64 write_page = write_address / VHOST_PAGE_SIZE;
1070
	int r;
K
Krishna Kumar 已提交
1071

1072 1073
	if (!write_length)
		return 0;
1074
	write_length += write_address % VHOST_PAGE_SIZE;
1075 1076
	for (;;) {
		u64 base = (u64)(unsigned long)log_base;
1077 1078
		u64 log = base + write_page / 8;
		int bit = write_page % 8;
1079 1080 1081 1082 1083 1084 1085 1086
		if ((u64)(unsigned long)log != log)
			return -EFAULT;
		r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
		if (r < 0)
			return r;
		if (write_length <= VHOST_PAGE_SIZE)
			break;
		write_length -= VHOST_PAGE_SIZE;
1087
		write_page += 1;
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
	}
	return r;
}

int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
		    unsigned int log_num, u64 len)
{
	int i, r;

	/* Make sure data written is seen before log. */
1098
	smp_wmb();
1099 1100 1101 1102 1103 1104
	for (i = 0; i < log_num; ++i) {
		u64 l = min(log[i].len, len);
		r = log_write(vq->log_base, log[i].addr, l);
		if (r < 0)
			return r;
		len -= l;
1105 1106 1107
		if (!len) {
			if (vq->log_ctx)
				eventfd_signal(vq->log_ctx, 1);
1108
			return 0;
1109
		}
1110 1111 1112 1113 1114
	}
	/* Length written exceeds what we have stored. This is a bug. */
	BUG();
	return 0;
}
A
Asias He 已提交
1115
EXPORT_SYMBOL_GPL(vhost_log_write);
1116

1117 1118 1119
static int vhost_update_used_flags(struct vhost_virtqueue *vq)
{
	void __user *used;
1120
	if (__put_user(cpu_to_vhost16(vq, vq->used_flags), &vq->used->flags) < 0)
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137
		return -EFAULT;
	if (unlikely(vq->log_used)) {
		/* Make sure the flag is seen before log. */
		smp_wmb();
		/* Log used flag write. */
		used = &vq->used->flags;
		log_write(vq->log_base, vq->log_addr +
			  (used - (void __user *)vq->used),
			  sizeof vq->used->flags);
		if (vq->log_ctx)
			eventfd_signal(vq->log_ctx, 1);
	}
	return 0;
}

static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)
{
1138
	if (__put_user(cpu_to_vhost16(vq, vq->avail_idx), vhost_avail_event(vq)))
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156
		return -EFAULT;
	if (unlikely(vq->log_used)) {
		void __user *used;
		/* Make sure the event is seen before log. */
		smp_wmb();
		/* Log avail event write */
		used = vhost_avail_event(vq);
		log_write(vq->log_base, vq->log_addr +
			  (used - (void __user *)vq->used),
			  sizeof *vhost_avail_event(vq));
		if (vq->log_ctx)
			eventfd_signal(vq->log_ctx, 1);
	}
	return 0;
}

int vhost_init_used(struct vhost_virtqueue *vq)
{
1157
	__virtio16 last_used_idx;
1158
	int r;
1159 1160
	if (!vq->private_data) {
		vq->is_le = virtio_legacy_is_little_endian();
1161
		return 0;
1162 1163 1164
	}

	vhost_init_is_le(vq);
1165 1166 1167 1168 1169

	r = vhost_update_used_flags(vq);
	if (r)
		return r;
	vq->signalled_used_valid = false;
1170 1171 1172 1173 1174
	if (!access_ok(VERIFY_READ, &vq->used->idx, sizeof vq->used->idx))
		return -EFAULT;
	r = __get_user(last_used_idx, &vq->used->idx);
	if (r)
		return r;
1175
	vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx);
1176
	return 0;
1177
}
A
Asias He 已提交
1178
EXPORT_SYMBOL_GPL(vhost_init_used);
1179

1180
static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
C
Christoph Hellwig 已提交
1181
			  struct iovec iov[], int iov_size)
1182 1183 1184 1185 1186 1187 1188
{
	const struct vhost_memory_region *reg;
	struct vhost_memory *mem;
	struct iovec *_iov;
	u64 s = 0;
	int ret = 0;

1189
	mem = vq->memory;
1190 1191
	while ((u64)len > s) {
		u64 size;
1192
		if (unlikely(ret >= iov_size)) {
1193 1194 1195 1196
			ret = -ENOBUFS;
			break;
		}
		reg = find_region(mem, addr, len);
1197
		if (unlikely(!reg)) {
1198 1199 1200 1201 1202
			ret = -EFAULT;
			break;
		}
		_iov = iov + ret;
		size = reg->memory_size - addr + reg->guest_phys_addr;
1203
		_iov->iov_len = min((u64)len - s, size);
C
Christoph Hellwig 已提交
1204
		_iov->iov_base = (void __user *)(unsigned long)
1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216
			(reg->userspace_addr + addr - reg->guest_phys_addr);
		s += size;
		addr += size;
		++ret;
	}

	return ret;
}

/* Each buffer in the virtqueues is actually a chain of descriptors.  This
 * function returns the next descriptor in the chain,
 * or -1U if we're at the end. */
1217
static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc)
1218 1219 1220 1221
{
	unsigned int next;

	/* If this descriptor says it doesn't chain, we're done. */
1222
	if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT)))
1223 1224 1225
		return -1U;

	/* Check they're not leading us off end of descriptors. */
1226
	next = vhost16_to_cpu(vq, desc->next);
1227 1228 1229 1230 1231 1232 1233 1234
	/* Make sure compiler knows to grab that: we don't want it changing! */
	/* We will use the result as an index in an array, so most
	 * architectures only need a compiler barrier here. */
	read_barrier_depends();

	return next;
}

1235
static int get_indirect(struct vhost_virtqueue *vq,
1236 1237 1238 1239
			struct iovec iov[], unsigned int iov_size,
			unsigned int *out_num, unsigned int *in_num,
			struct vhost_log *log, unsigned int *log_num,
			struct vring_desc *indirect)
1240 1241 1242
{
	struct vring_desc desc;
	unsigned int i = 0, count, found = 0;
1243
	u32 len = vhost32_to_cpu(vq, indirect->len);
1244
	struct iov_iter from;
1245 1246 1247
	int ret;

	/* Sanity check */
1248
	if (unlikely(len % sizeof desc)) {
1249 1250
		vq_err(vq, "Invalid length in indirect descriptor: "
		       "len 0x%llx not multiple of 0x%zx\n",
1251
		       (unsigned long long)len,
1252 1253 1254 1255
		       sizeof desc);
		return -EINVAL;
	}

1256
	ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect,
J
Jason Wang 已提交
1257
			     UIO_MAXIOV);
1258
	if (unlikely(ret < 0)) {
1259 1260 1261
		vq_err(vq, "Translation failure %d in indirect.\n", ret);
		return ret;
	}
1262
	iov_iter_init(&from, READ, vq->indirect, ret, len);
1263 1264 1265 1266 1267

	/* We will use the result as an address to read from, so most
	 * architectures only need a compiler barrier here. */
	read_barrier_depends();

1268
	count = len / sizeof desc;
1269 1270
	/* Buffers are chained via a 16 bit next field, so
	 * we can have at most 2^16 of these. */
1271
	if (unlikely(count > USHRT_MAX + 1)) {
1272 1273 1274 1275 1276 1277 1278
		vq_err(vq, "Indirect buffer length too big: %d\n",
		       indirect->len);
		return -E2BIG;
	}

	do {
		unsigned iov_count = *in_num + *out_num;
1279
		if (unlikely(++found > count)) {
1280 1281 1282 1283 1284
			vq_err(vq, "Loop detected: last one at %u "
			       "indirect size %u\n",
			       i, count);
			return -EINVAL;
		}
1285 1286
		if (unlikely(copy_from_iter(&desc, sizeof(desc), &from) !=
			     sizeof(desc))) {
1287
			vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
1288
			       i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
1289 1290
			return -EINVAL;
		}
1291
		if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) {
1292
			vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
1293
			       i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
1294 1295 1296
			return -EINVAL;
		}

1297 1298
		ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
				     vhost32_to_cpu(vq, desc.len), iov + iov_count,
1299
				     iov_size - iov_count);
1300
		if (unlikely(ret < 0)) {
1301 1302 1303 1304 1305
			vq_err(vq, "Translation failure %d indirect idx %d\n",
			       ret, i);
			return ret;
		}
		/* If this is an input descriptor, increment that count. */
1306
		if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) {
1307 1308
			*in_num += ret;
			if (unlikely(log)) {
1309 1310
				log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
				log[*log_num].len = vhost32_to_cpu(vq, desc.len);
1311 1312 1313 1314 1315
				++*log_num;
			}
		} else {
			/* If it's an output descriptor, they're all supposed
			 * to come before any input descriptors. */
1316
			if (unlikely(*in_num)) {
1317 1318 1319 1320 1321 1322
				vq_err(vq, "Indirect descriptor "
				       "has out after in: idx %d\n", i);
				return -EINVAL;
			}
			*out_num += ret;
		}
1323
	} while ((i = next_desc(vq, &desc)) != -1);
1324 1325 1326 1327 1328 1329 1330 1331
	return 0;
}

/* This looks in the virtqueue and for the first available buffer, and converts
 * it to an iovec for convenient access.  Since descriptors consist of some
 * number of output then some number of input descriptors, it's actually two
 * iovecs, but we pack them into one and note how many of each there were.
 *
1332 1333 1334
 * This function returns the descriptor number found, or vq->num (which is
 * never a valid descriptor number) if none was found.  A negative code is
 * returned on error. */
1335
int vhost_get_vq_desc(struct vhost_virtqueue *vq,
1336 1337 1338
		      struct iovec iov[], unsigned int iov_size,
		      unsigned int *out_num, unsigned int *in_num,
		      struct vhost_log *log, unsigned int *log_num)
1339 1340 1341 1342
{
	struct vring_desc desc;
	unsigned int i, head, found = 0;
	u16 last_avail_idx;
1343 1344
	__virtio16 avail_idx;
	__virtio16 ring_head;
1345 1346 1347 1348
	int ret;

	/* Check it isn't doing very strange things with descriptor numbers. */
	last_avail_idx = vq->last_avail_idx;
1349
	if (unlikely(__get_user(avail_idx, &vq->avail->idx))) {
1350 1351
		vq_err(vq, "Failed to access avail idx at %p\n",
		       &vq->avail->idx);
1352
		return -EFAULT;
1353
	}
1354
	vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
1355

1356
	if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
1357 1358
		vq_err(vq, "Guest moved used index from %u to %u",
		       last_avail_idx, vq->avail_idx);
1359
		return -EFAULT;
1360 1361 1362 1363 1364 1365 1366
	}

	/* If there's nothing new since last we looked, return invalid. */
	if (vq->avail_idx == last_avail_idx)
		return vq->num;

	/* Only get avail ring entries after they have been exposed by guest. */
1367
	smp_rmb();
1368 1369 1370

	/* Grab the next descriptor number they're advertising, and increment
	 * the index we've seen. */
1371
	if (unlikely(__get_user(ring_head,
1372
				&vq->avail->ring[last_avail_idx & (vq->num - 1)]))) {
1373 1374 1375
		vq_err(vq, "Failed to read head: idx %d address %p\n",
		       last_avail_idx,
		       &vq->avail->ring[last_avail_idx % vq->num]);
1376
		return -EFAULT;
1377 1378
	}

1379 1380
	head = vhost16_to_cpu(vq, ring_head);

1381
	/* If their number is silly, that's an error. */
1382
	if (unlikely(head >= vq->num)) {
1383 1384
		vq_err(vq, "Guest says index %u > %u is available",
		       head, vq->num);
1385
		return -EINVAL;
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
	}

	/* When we start there are none of either input nor output. */
	*out_num = *in_num = 0;
	if (unlikely(log))
		*log_num = 0;

	i = head;
	do {
		unsigned iov_count = *in_num + *out_num;
1396
		if (unlikely(i >= vq->num)) {
1397 1398
			vq_err(vq, "Desc index is %u > %u, head = %u",
			       i, vq->num, head);
1399
			return -EINVAL;
1400
		}
1401
		if (unlikely(++found > vq->num)) {
1402 1403 1404
			vq_err(vq, "Loop detected: last one at %u "
			       "vq size %u head %u\n",
			       i, vq->num, head);
1405
			return -EINVAL;
1406
		}
1407
		ret = __copy_from_user(&desc, vq->desc + i, sizeof desc);
1408
		if (unlikely(ret)) {
1409 1410
			vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
			       i, vq->desc + i);
1411
			return -EFAULT;
1412
		}
1413
		if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) {
1414
			ret = get_indirect(vq, iov, iov_size,
1415 1416
					   out_num, in_num,
					   log, log_num, &desc);
1417
			if (unlikely(ret < 0)) {
1418 1419
				vq_err(vq, "Failure detected "
				       "in indirect descriptor at idx %d\n", i);
1420
				return ret;
1421 1422 1423 1424
			}
			continue;
		}

1425 1426
		ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
				     vhost32_to_cpu(vq, desc.len), iov + iov_count,
1427
				     iov_size - iov_count);
1428
		if (unlikely(ret < 0)) {
1429 1430
			vq_err(vq, "Translation failure %d descriptor idx %d\n",
			       ret, i);
1431
			return ret;
1432
		}
1433
		if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) {
1434 1435 1436 1437
			/* If this is an input descriptor,
			 * increment that count. */
			*in_num += ret;
			if (unlikely(log)) {
1438 1439
				log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
				log[*log_num].len = vhost32_to_cpu(vq, desc.len);
1440 1441 1442 1443 1444
				++*log_num;
			}
		} else {
			/* If it's an output descriptor, they're all supposed
			 * to come before any input descriptors. */
1445
			if (unlikely(*in_num)) {
1446 1447
				vq_err(vq, "Descriptor has out after in: "
				       "idx %d\n", i);
1448
				return -EINVAL;
1449 1450 1451
			}
			*out_num += ret;
		}
1452
	} while ((i = next_desc(vq, &desc)) != -1);
1453 1454 1455

	/* On success, increment avail index. */
	vq->last_avail_idx++;
M
Michael S. Tsirkin 已提交
1456 1457 1458 1459

	/* Assume notifications from guest are disabled at this point,
	 * if they aren't we would need to update avail_event index. */
	BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
1460 1461
	return head;
}
A
Asias He 已提交
1462
EXPORT_SYMBOL_GPL(vhost_get_vq_desc);
1463 1464

/* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */
1465
void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n)
1466
{
1467
	vq->last_avail_idx -= n;
1468
}
A
Asias He 已提交
1469
EXPORT_SYMBOL_GPL(vhost_discard_vq_desc);
1470 1471 1472 1473 1474

/* After we've used one of their buffers, we tell them about it.  We'll then
 * want to notify the guest, using eventfd. */
int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
{
1475 1476 1477 1478
	struct vring_used_elem heads = {
		cpu_to_vhost32(vq, head),
		cpu_to_vhost32(vq, len)
	};
1479

1480
	return vhost_add_used_n(vq, &heads, 1);
1481
}
A
Asias He 已提交
1482
EXPORT_SYMBOL_GPL(vhost_add_used);
1483

1484 1485 1486 1487 1488
static int __vhost_add_used_n(struct vhost_virtqueue *vq,
			    struct vring_used_elem *heads,
			    unsigned count)
{
	struct vring_used_elem __user *used;
M
Michael S. Tsirkin 已提交
1489
	u16 old, new;
1490 1491
	int start;

1492
	start = vq->last_used_idx & (vq->num - 1);
1493
	used = vq->used->ring + start;
1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
	if (count == 1) {
		if (__put_user(heads[0].id, &used->id)) {
			vq_err(vq, "Failed to write used id");
			return -EFAULT;
		}
		if (__put_user(heads[0].len, &used->len)) {
			vq_err(vq, "Failed to write used len");
			return -EFAULT;
		}
	} else if (__copy_to_user(used, heads, count * sizeof *used)) {
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
		vq_err(vq, "Failed to write used");
		return -EFAULT;
	}
	if (unlikely(vq->log_used)) {
		/* Make sure data is seen before log. */
		smp_wmb();
		/* Log used ring entry write. */
		log_write(vq->log_base,
			  vq->log_addr +
			   ((void __user *)used - (void __user *)vq->used),
			  count * sizeof *used);
	}
M
Michael S. Tsirkin 已提交
1516 1517 1518 1519 1520 1521 1522 1523
	old = vq->last_used_idx;
	new = (vq->last_used_idx += count);
	/* If the driver never bothers to signal in a very long while,
	 * used index might wrap around. If that happens, invalidate
	 * signalled_used index we stored. TODO: make sure driver
	 * signals at least once in 2^16 and remove this. */
	if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
		vq->signalled_used_valid = false;
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533
	return 0;
}

/* After we've used one of their buffers, we tell them about it.  We'll then
 * want to notify the guest, using eventfd. */
int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
		     unsigned count)
{
	int start, n, r;

1534
	start = vq->last_used_idx & (vq->num - 1);
1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546
	n = vq->num - start;
	if (n < count) {
		r = __vhost_add_used_n(vq, heads, n);
		if (r < 0)
			return r;
		heads += n;
		count -= n;
	}
	r = __vhost_add_used_n(vq, heads, count);

	/* Make sure buffer is written before we update index. */
	smp_wmb();
1547
	if (__put_user(cpu_to_vhost16(vq, vq->last_used_idx), &vq->used->idx)) {
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560
		vq_err(vq, "Failed to increment used idx");
		return -EFAULT;
	}
	if (unlikely(vq->log_used)) {
		/* Log used index update. */
		log_write(vq->log_base,
			  vq->log_addr + offsetof(struct vring_used, idx),
			  sizeof vq->used->idx);
		if (vq->log_ctx)
			eventfd_signal(vq->log_ctx, 1);
	}
	return r;
}
A
Asias He 已提交
1561
EXPORT_SYMBOL_GPL(vhost_add_used_n);
1562

M
Michael S. Tsirkin 已提交
1563
static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1564
{
1565 1566
	__u16 old, new;
	__virtio16 event;
M
Michael S. Tsirkin 已提交
1567
	bool v;
M
Michael S. Tsirkin 已提交
1568 1569 1570 1571 1572
	/* Flush out used index updates. This is paired
	 * with the barrier that the Guest executes when enabling
	 * interrupts. */
	smp_mb();

1573
	if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) &&
M
Michael S. Tsirkin 已提交
1574 1575 1576
	    unlikely(vq->avail_idx == vq->last_avail_idx))
		return true;

1577
	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
1578
		__virtio16 flags;
M
Michael S. Tsirkin 已提交
1579 1580 1581 1582
		if (__get_user(flags, &vq->avail->flags)) {
			vq_err(vq, "Failed to get flags");
			return true;
		}
1583
		return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT));
1584
	}
M
Michael S. Tsirkin 已提交
1585 1586 1587 1588
	old = vq->signalled_used;
	v = vq->signalled_used_valid;
	new = vq->signalled_used = vq->last_used_idx;
	vq->signalled_used_valid = true;
1589

M
Michael S. Tsirkin 已提交
1590 1591
	if (unlikely(!v))
		return true;
1592

1593
	if (__get_user(event, vhost_used_event(vq))) {
M
Michael S. Tsirkin 已提交
1594 1595 1596
		vq_err(vq, "Failed to get used event idx");
		return true;
	}
1597
	return vring_need_event(vhost16_to_cpu(vq, event), new, old);
M
Michael S. Tsirkin 已提交
1598 1599 1600 1601 1602
}

/* This actually signals the guest, using eventfd. */
void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
{
1603
	/* Signal the Guest tell them we used something up. */
M
Michael S. Tsirkin 已提交
1604
	if (vq->call_ctx && vhost_notify(dev, vq))
1605 1606
		eventfd_signal(vq->call_ctx, 1);
}
A
Asias He 已提交
1607
EXPORT_SYMBOL_GPL(vhost_signal);
1608 1609 1610 1611 1612 1613 1614 1615 1616

/* And here's the combo meal deal.  Supersize me! */
void vhost_add_used_and_signal(struct vhost_dev *dev,
			       struct vhost_virtqueue *vq,
			       unsigned int head, int len)
{
	vhost_add_used(vq, head, len);
	vhost_signal(dev, vq);
}
A
Asias He 已提交
1617
EXPORT_SYMBOL_GPL(vhost_add_used_and_signal);
1618

1619 1620 1621 1622 1623 1624 1625 1626
/* multi-buffer version of vhost_add_used_and_signal */
void vhost_add_used_and_signal_n(struct vhost_dev *dev,
				 struct vhost_virtqueue *vq,
				 struct vring_used_elem *heads, unsigned count)
{
	vhost_add_used_n(vq, heads, count);
	vhost_signal(dev, vq);
}
A
Asias He 已提交
1627
EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
1628

1629
/* OK, now we need to know about added descriptors. */
M
Michael S. Tsirkin 已提交
1630
bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1631
{
1632
	__virtio16 avail_idx;
1633
	int r;
K
Krishna Kumar 已提交
1634

1635 1636 1637
	if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
		return false;
	vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
1638
	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
1639
		r = vhost_update_used_flags(vq);
M
Michael S. Tsirkin 已提交
1640 1641 1642 1643 1644 1645
		if (r) {
			vq_err(vq, "Failed to enable notification at %p: %d\n",
			       &vq->used->flags, r);
			return false;
		}
	} else {
1646
		r = vhost_update_avail_event(vq, vq->avail_idx);
M
Michael S. Tsirkin 已提交
1647 1648 1649 1650 1651 1652
		if (r) {
			vq_err(vq, "Failed to update avail event index at %p: %d\n",
			       vhost_avail_event(vq), r);
			return false;
		}
	}
1653 1654
	/* They could have slipped one in as we were doing that: make
	 * sure it's written, then check again. */
1655
	smp_mb();
1656
	r = __get_user(avail_idx, &vq->avail->idx);
1657 1658 1659 1660 1661 1662
	if (r) {
		vq_err(vq, "Failed to check avail idx at %p: %d\n",
		       &vq->avail->idx, r);
		return false;
	}

1663
	return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx;
1664
}
A
Asias He 已提交
1665
EXPORT_SYMBOL_GPL(vhost_enable_notify);
1666 1667

/* We don't need to be notified again. */
M
Michael S. Tsirkin 已提交
1668
void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1669 1670
{
	int r;
K
Krishna Kumar 已提交
1671

1672 1673 1674
	if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
		return;
	vq->used_flags |= VRING_USED_F_NO_NOTIFY;
1675
	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
1676
		r = vhost_update_used_flags(vq);
M
Michael S. Tsirkin 已提交
1677 1678 1679 1680
		if (r)
			vq_err(vq, "Failed to enable notification at %p: %d\n",
			       &vq->used->flags, r);
	}
1681
}
A
Asias He 已提交
1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
EXPORT_SYMBOL_GPL(vhost_disable_notify);

static int __init vhost_init(void)
{
	return 0;
}

static void __exit vhost_exit(void)
{
}

module_init(vhost_init);
module_exit(vhost_exit);

MODULE_VERSION("0.0.1");
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Michael S. Tsirkin");
MODULE_DESCRIPTION("Host kernel accelerator for virtio");