kvm_main.c 53.3 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

18
#include "iodev.h"
A
Avi Kivity 已提交
19

20
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
21 22 23 24 25 26 27 28 29 30 31 32
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
33
#include <linux/sysdev.h>
A
Avi Kivity 已提交
34
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
35
#include <linux/sched.h>
36 37
#include <linux/cpumask.h>
#include <linux/smp.h>
38
#include <linux/anon_inodes.h>
39
#include <linux/profile.h>
40
#include <linux/kvm_para.h>
41
#include <linux/pagemap.h>
42
#include <linux/mman.h>
43
#include <linux/swap.h>
A
Avi Kivity 已提交
44

A
Avi Kivity 已提交
45 46 47
#include <asm/processor.h>
#include <asm/io.h>
#include <asm/uaccess.h>
48
#include <asm/pgtable.h>
A
Avi Kivity 已提交
49

50 51 52 53
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
#include "coalesced_mmio.h"
#endif

54 55 56 57 58 59
#ifdef KVM_CAP_DEVICE_ASSIGNMENT
#include <linux/pci.h>
#include <linux/interrupt.h>
#include "irq.h"
#endif

A
Avi Kivity 已提交
60 61 62
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

S
Sheng Yang 已提交
63 64 65
static int msi2intx = 1;
module_param(msi2intx, bool, 0);

66 67
DEFINE_SPINLOCK(kvm_lock);
LIST_HEAD(vm_list);
68

69
static cpumask_var_t cpus_hardware_enabled;
70

71 72
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
73

74 75
static __read_mostly struct preempt_ops kvm_preempt_ops;

76
struct dentry *kvm_debugfs_dir;
A
Avi Kivity 已提交
77

A
Avi Kivity 已提交
78 79 80
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);

H
Hannes Eder 已提交
81
static bool kvm_rebooting;
82

83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
#ifdef KVM_CAP_DEVICE_ASSIGNMENT
static struct kvm_assigned_dev_kernel *kvm_find_assigned_dev(struct list_head *head,
						      int assigned_dev_id)
{
	struct list_head *ptr;
	struct kvm_assigned_dev_kernel *match;

	list_for_each(ptr, head) {
		match = list_entry(ptr, struct kvm_assigned_dev_kernel, list);
		if (match->assigned_dev_id == assigned_dev_id)
			return match;
	}
	return NULL;
}

static void kvm_assigned_dev_interrupt_work_handler(struct work_struct *work)
{
	struct kvm_assigned_dev_kernel *assigned_dev;

	assigned_dev = container_of(work, struct kvm_assigned_dev_kernel,
				    interrupt_work);

	/* This is taken to safely inject irq inside the guest. When
	 * the interrupt injection (or the ioapic code) uses a
	 * finer-grained lock, update this
	 */
	mutex_lock(&assigned_dev->kvm->lock);
S
Sheng Yang 已提交
110 111 112 113
	kvm_set_irq(assigned_dev->kvm, assigned_dev->irq_source_id,
		    assigned_dev->guest_irq, 1);

	if (assigned_dev->irq_requested_type & KVM_ASSIGNED_DEV_GUEST_MSI) {
S
Sheng Yang 已提交
114
		enable_irq(assigned_dev->host_irq);
115
		assigned_dev->host_irq_disabled = false;
S
Sheng Yang 已提交
116
	}
117 118 119 120 121 122 123 124 125
	mutex_unlock(&assigned_dev->kvm->lock);
}

static irqreturn_t kvm_assigned_dev_intr(int irq, void *dev_id)
{
	struct kvm_assigned_dev_kernel *assigned_dev =
		(struct kvm_assigned_dev_kernel *) dev_id;

	schedule_work(&assigned_dev->interrupt_work);
126

127
	disable_irq_nosync(irq);
128 129
	assigned_dev->host_irq_disabled = true;

130 131 132 133 134 135 136 137 138 139 140 141 142
	return IRQ_HANDLED;
}

/* Ack the irq line for an assigned device */
static void kvm_assigned_dev_ack_irq(struct kvm_irq_ack_notifier *kian)
{
	struct kvm_assigned_dev_kernel *dev;

	if (kian->gsi == -1)
		return;

	dev = container_of(kian, struct kvm_assigned_dev_kernel,
			   ack_notifier);
143

144
	kvm_set_irq(dev->kvm, dev->irq_source_id, dev->guest_irq, 0);
145 146 147 148 149 150 151 152

	/* The guest irq may be shared so this ack may be
	 * from another device.
	 */
	if (dev->host_irq_disabled) {
		enable_irq(dev->host_irq);
		dev->host_irq_disabled = false;
	}
153 154
}

155
/* The function implicit hold kvm->lock mutex due to cancel_work_sync() */
156 157
static void kvm_free_assigned_irq(struct kvm *kvm,
				  struct kvm_assigned_dev_kernel *assigned_dev)
158
{
159 160
	if (!irqchip_in_kernel(kvm))
		return;
161

162
	kvm_unregister_irq_ack_notifier(&assigned_dev->ack_notifier);
163 164 165 166

	if (assigned_dev->irq_source_id != -1)
		kvm_free_irq_source_id(kvm, assigned_dev->irq_source_id);
	assigned_dev->irq_source_id = -1;
167

168 169 170
	if (!assigned_dev->irq_requested_type)
		return;

171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
	/*
	 * In kvm_free_device_irq, cancel_work_sync return true if:
	 * 1. work is scheduled, and then cancelled.
	 * 2. work callback is executed.
	 *
	 * The first one ensured that the irq is disabled and no more events
	 * would happen. But for the second one, the irq may be enabled (e.g.
	 * for MSI). So we disable irq here to prevent further events.
	 *
	 * Notice this maybe result in nested disable if the interrupt type is
	 * INTx, but it's OK for we are going to free it.
	 *
	 * If this function is a part of VM destroy, please ensure that till
	 * now, the kvm state is still legal for probably we also have to wait
	 * interrupt_work done.
	 */
	disable_irq_nosync(assigned_dev->host_irq);
	cancel_work_sync(&assigned_dev->interrupt_work);
189

190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
	free_irq(assigned_dev->host_irq, (void *)assigned_dev);

	if (assigned_dev->irq_requested_type & KVM_ASSIGNED_DEV_HOST_MSI)
		pci_disable_msi(assigned_dev->dev);

	assigned_dev->irq_requested_type = 0;
}


static void kvm_free_assigned_device(struct kvm *kvm,
				     struct kvm_assigned_dev_kernel
				     *assigned_dev)
{
	kvm_free_assigned_irq(kvm, assigned_dev);

205 206
	pci_reset_function(assigned_dev->dev);

207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
	pci_release_regions(assigned_dev->dev);
	pci_disable_device(assigned_dev->dev);
	pci_dev_put(assigned_dev->dev);

	list_del(&assigned_dev->list);
	kfree(assigned_dev);
}

void kvm_free_all_assigned_devices(struct kvm *kvm)
{
	struct list_head *ptr, *ptr2;
	struct kvm_assigned_dev_kernel *assigned_dev;

	list_for_each_safe(ptr, ptr2, &kvm->arch.assigned_dev_head) {
		assigned_dev = list_entry(ptr,
					  struct kvm_assigned_dev_kernel,
					  list);

		kvm_free_assigned_device(kvm, assigned_dev);
	}
}

229 230 231 232
static int assigned_device_update_intx(struct kvm *kvm,
			struct kvm_assigned_dev_kernel *adev,
			struct kvm_assigned_irq *airq)
{
233 234 235 236
	adev->guest_irq = airq->guest_irq;
	adev->ack_notifier.gsi = airq->guest_irq;

	if (adev->irq_requested_type & KVM_ASSIGNED_DEV_HOST_INTX)
237 238 239
		return 0;

	if (irqchip_in_kernel(kvm)) {
S
Sheng Yang 已提交
240
		if (!msi2intx &&
S
Sheng Yang 已提交
241 242
		    (adev->irq_requested_type & KVM_ASSIGNED_DEV_HOST_MSI)) {
			free_irq(adev->host_irq, (void *)adev);
S
Sheng Yang 已提交
243 244 245
			pci_disable_msi(adev->dev);
		}

246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
		if (!capable(CAP_SYS_RAWIO))
			return -EPERM;

		if (airq->host_irq)
			adev->host_irq = airq->host_irq;
		else
			adev->host_irq = adev->dev->irq;

		/* Even though this is PCI, we don't want to use shared
		 * interrupts. Sharing host devices with guest-assigned devices
		 * on the same interrupt line is not a happy situation: there
		 * are going to be long delays in accepting, acking, etc.
		 */
		if (request_irq(adev->host_irq, kvm_assigned_dev_intr,
				0, "kvm_assigned_intx_device", (void *)adev))
			return -EIO;
	}

264 265
	adev->irq_requested_type = KVM_ASSIGNED_DEV_GUEST_INTX |
				   KVM_ASSIGNED_DEV_HOST_INTX;
266 267 268
	return 0;
}

S
Sheng Yang 已提交
269 270 271 272 273 274 275
#ifdef CONFIG_X86
static int assigned_device_update_msi(struct kvm *kvm,
			struct kvm_assigned_dev_kernel *adev,
			struct kvm_assigned_irq *airq)
{
	int r;

S
Sheng Yang 已提交
276
	adev->guest_irq = airq->guest_irq;
S
Sheng Yang 已提交
277 278 279 280 281 282 283 284 285
	if (airq->flags & KVM_DEV_IRQ_ASSIGN_ENABLE_MSI) {
		/* x86 don't care upper address of guest msi message addr */
		adev->irq_requested_type |= KVM_ASSIGNED_DEV_GUEST_MSI;
		adev->irq_requested_type &= ~KVM_ASSIGNED_DEV_GUEST_INTX;
		adev->ack_notifier.gsi = -1;
	} else if (msi2intx) {
		adev->irq_requested_type |= KVM_ASSIGNED_DEV_GUEST_INTX;
		adev->irq_requested_type &= ~KVM_ASSIGNED_DEV_GUEST_MSI;
		adev->ack_notifier.gsi = airq->guest_irq;
286 287 288 289 290 291 292 293
	} else {
		/*
		 * Guest require to disable device MSI, we disable MSI and
		 * re-enable INTx by default again. Notice it's only for
		 * non-msi2intx.
		 */
		assigned_device_update_intx(kvm, adev, airq);
		return 0;
S
Sheng Yang 已提交
294
	}
S
Sheng Yang 已提交
295 296 297 298 299

	if (adev->irq_requested_type & KVM_ASSIGNED_DEV_HOST_MSI)
		return 0;

	if (irqchip_in_kernel(kvm)) {
S
Sheng Yang 已提交
300 301 302 303 304 305 306 307 308
		if (!msi2intx) {
			if (adev->irq_requested_type &
					KVM_ASSIGNED_DEV_HOST_INTX)
				free_irq(adev->host_irq, (void *)adev);

			r = pci_enable_msi(adev->dev);
			if (r)
				return r;
		}
S
Sheng Yang 已提交
309 310 311 312 313 314 315

		adev->host_irq = adev->dev->irq;
		if (request_irq(adev->host_irq, kvm_assigned_dev_intr, 0,
				"kvm_assigned_msi_device", (void *)adev))
			return -EIO;
	}

S
Sheng Yang 已提交
316 317 318 319
	if (!msi2intx)
		adev->irq_requested_type = KVM_ASSIGNED_DEV_GUEST_MSI;

	adev->irq_requested_type |= KVM_ASSIGNED_DEV_HOST_MSI;
S
Sheng Yang 已提交
320 321 322 323
	return 0;
}
#endif

324 325 326 327 328 329
static int kvm_vm_ioctl_assign_irq(struct kvm *kvm,
				   struct kvm_assigned_irq
				   *assigned_irq)
{
	int r = 0;
	struct kvm_assigned_dev_kernel *match;
330
	u32 current_flags = 0, changed_flags;
331 332 333 334 335 336 337 338 339 340

	mutex_lock(&kvm->lock);

	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
				      assigned_irq->assigned_dev_id);
	if (!match) {
		mutex_unlock(&kvm->lock);
		return -EINVAL;
	}

341
	if (!match->irq_requested_type) {
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357
		INIT_WORK(&match->interrupt_work,
				kvm_assigned_dev_interrupt_work_handler);
		if (irqchip_in_kernel(kvm)) {
			/* Register ack nofitier */
			match->ack_notifier.gsi = -1;
			match->ack_notifier.irq_acked =
					kvm_assigned_dev_ack_irq;
			kvm_register_irq_ack_notifier(kvm,
					&match->ack_notifier);

			/* Request IRQ source ID */
			r = kvm_request_irq_source_id(kvm);
			if (r < 0)
				goto out_release;
			else
				match->irq_source_id = r;
S
Sheng Yang 已提交
358 359 360 361 362 363 364

#ifdef CONFIG_X86
			/* Determine host device irq type, we can know the
			 * result from dev->msi_enabled */
			if (msi2intx)
				pci_enable_msi(match->dev);
#endif
365
		}
366 367
	}

368 369 370 371 372 373 374
	if ((match->irq_requested_type & KVM_ASSIGNED_DEV_HOST_MSI) &&
		 (match->irq_requested_type & KVM_ASSIGNED_DEV_GUEST_MSI))
		current_flags |= KVM_DEV_IRQ_ASSIGN_ENABLE_MSI;

	changed_flags = assigned_irq->flags ^ current_flags;

	if ((changed_flags & KVM_DEV_IRQ_ASSIGN_MSI_ACTION) ||
S
Sheng Yang 已提交
375
	    (msi2intx && match->dev->msi_enabled)) {
S
Sheng Yang 已提交
376 377 378 379 380 381 382 383 384 385 386 387
#ifdef CONFIG_X86
		r = assigned_device_update_msi(kvm, match, assigned_irq);
		if (r) {
			printk(KERN_WARNING "kvm: failed to enable "
					"MSI device!\n");
			goto out_release;
		}
#else
		r = -ENOTTY;
#endif
	} else if (assigned_irq->host_irq == 0 && match->dev->irq == 0) {
		/* Host device IRQ 0 means don't support INTx */
S
Sheng Yang 已提交
388 389 390 391 392 393 394 395 396 397
		if (!msi2intx) {
			printk(KERN_WARNING
			       "kvm: wait device to enable MSI!\n");
			r = 0;
		} else {
			printk(KERN_WARNING
			       "kvm: failed to enable MSI device!\n");
			r = -ENOTTY;
			goto out_release;
		}
S
Sheng Yang 已提交
398 399 400 401 402 403 404 405 406
	} else {
		/* Non-sharing INTx mode */
		r = assigned_device_update_intx(kvm, match, assigned_irq);
		if (r) {
			printk(KERN_WARNING "kvm: failed to enable "
					"INTx device!\n");
			goto out_release;
		}
	}
407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422

	mutex_unlock(&kvm->lock);
	return r;
out_release:
	mutex_unlock(&kvm->lock);
	kvm_free_assigned_device(kvm, match);
	return r;
}

static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
				      struct kvm_assigned_pci_dev *assigned_dev)
{
	int r = 0;
	struct kvm_assigned_dev_kernel *match;
	struct pci_dev *dev;

423
	down_read(&kvm->slots_lock);
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458
	mutex_lock(&kvm->lock);

	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
				      assigned_dev->assigned_dev_id);
	if (match) {
		/* device already assigned */
		r = -EINVAL;
		goto out;
	}

	match = kzalloc(sizeof(struct kvm_assigned_dev_kernel), GFP_KERNEL);
	if (match == NULL) {
		printk(KERN_INFO "%s: Couldn't allocate memory\n",
		       __func__);
		r = -ENOMEM;
		goto out;
	}
	dev = pci_get_bus_and_slot(assigned_dev->busnr,
				   assigned_dev->devfn);
	if (!dev) {
		printk(KERN_INFO "%s: host device not found\n", __func__);
		r = -EINVAL;
		goto out_free;
	}
	if (pci_enable_device(dev)) {
		printk(KERN_INFO "%s: Could not enable PCI device\n", __func__);
		r = -EBUSY;
		goto out_put;
	}
	r = pci_request_regions(dev, "kvm_assigned_device");
	if (r) {
		printk(KERN_INFO "%s: Could not get access to device regions\n",
		       __func__);
		goto out_disable;
	}
459 460 461

	pci_reset_function(dev);

462 463 464
	match->assigned_dev_id = assigned_dev->assigned_dev_id;
	match->host_busnr = assigned_dev->busnr;
	match->host_devfn = assigned_dev->devfn;
465
	match->flags = assigned_dev->flags;
466
	match->dev = dev;
467
	match->irq_source_id = -1;
468 469 470 471 472
	match->kvm = kvm;

	list_add(&match->list, &kvm->arch.assigned_dev_head);

	if (assigned_dev->flags & KVM_DEV_ASSIGN_ENABLE_IOMMU) {
J
Joerg Roedel 已提交
473
		if (!kvm->arch.iommu_domain) {
W
Weidong Han 已提交
474 475 476 477 478
			r = kvm_iommu_map_guest(kvm);
			if (r)
				goto out_list_del;
		}
		r = kvm_assign_device(kvm, match);
479 480 481 482 483 484
		if (r)
			goto out_list_del;
	}

out:
	mutex_unlock(&kvm->lock);
485
	up_read(&kvm->slots_lock);
486 487 488 489 490 491 492 493 494 495 496
	return r;
out_list_del:
	list_del(&match->list);
	pci_release_regions(dev);
out_disable:
	pci_disable_device(dev);
out_put:
	pci_dev_put(dev);
out_free:
	kfree(match);
	mutex_unlock(&kvm->lock);
497
	up_read(&kvm->slots_lock);
498 499 500 501
	return r;
}
#endif

W
Weidong Han 已提交
502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
#ifdef KVM_CAP_DEVICE_DEASSIGNMENT
static int kvm_vm_ioctl_deassign_device(struct kvm *kvm,
		struct kvm_assigned_pci_dev *assigned_dev)
{
	int r = 0;
	struct kvm_assigned_dev_kernel *match;

	mutex_lock(&kvm->lock);

	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
				      assigned_dev->assigned_dev_id);
	if (!match) {
		printk(KERN_INFO "%s: device hasn't been assigned before, "
		  "so cannot be deassigned\n", __func__);
		r = -EINVAL;
		goto out;
	}

520
	if (match->flags & KVM_DEV_ASSIGN_ENABLE_IOMMU)
W
Weidong Han 已提交
521 522 523 524 525 526 527 528 529 530
		kvm_deassign_device(kvm, match);

	kvm_free_assigned_device(kvm, match);

out:
	mutex_unlock(&kvm->lock);
	return r;
}
#endif

531 532 533 534 535
static inline int valid_vcpu(int n)
{
	return likely(n >= 0 && n < KVM_MAX_VCPUS);
}

536
inline int kvm_is_mmio_pfn(pfn_t pfn)
B
Ben-Ami Yassour 已提交
537
{
538 539 540 541
	if (pfn_valid(pfn)) {
		struct page *page = compound_head(pfn_to_page(pfn));
		return PageReserved(page);
	}
B
Ben-Ami Yassour 已提交
542 543 544 545

	return true;
}

A
Avi Kivity 已提交
546 547 548
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
549
void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
550
{
551 552
	int cpu;

A
Avi Kivity 已提交
553
	mutex_lock(&vcpu->mutex);
554 555
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
556
	kvm_arch_vcpu_load(vcpu, cpu);
557
	put_cpu();
A
Avi Kivity 已提交
558 559
}

560
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
561
{
562
	preempt_disable();
563
	kvm_arch_vcpu_put(vcpu);
564 565
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
566 567 568
	mutex_unlock(&vcpu->mutex);
}

569 570 571 572
static void ack_flush(void *_completed)
{
}

573
static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
574
{
575
	int i, cpu, me;
576 577
	cpumask_var_t cpus;
	bool called = true;
578 579
	struct kvm_vcpu *vcpu;

580 581 582
	if (alloc_cpumask_var(&cpus, GFP_ATOMIC))
		cpumask_clear(cpus);

583
	me = get_cpu();
R
Rusty Russell 已提交
584 585 586 587
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		vcpu = kvm->vcpus[i];
		if (!vcpu)
			continue;
588
		if (test_and_set_bit(req, &vcpu->requests))
589 590
			continue;
		cpu = vcpu->cpu;
591 592
		if (cpus != NULL && cpu != -1 && cpu != me)
			cpumask_set_cpu(cpu, cpus);
593
	}
594 595 596 597 598 599
	if (unlikely(cpus == NULL))
		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
	else if (!cpumask_empty(cpus))
		smp_call_function_many(cpus, ack_flush, NULL, 1);
	else
		called = false;
600
	put_cpu();
601
	free_cpumask_var(cpus);
602
	return called;
603 604
}

605
void kvm_flush_remote_tlbs(struct kvm *kvm)
606
{
607 608
	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
		++kvm->stat.remote_tlb_flush;
609 610
}

611 612 613 614
void kvm_reload_remote_mmus(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
}
615

R
Rusty Russell 已提交
616 617 618 619 620 621 622 623 624
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
E
Eddie Dong 已提交
625
	init_waitqueue_head(&vcpu->wq);
R
Rusty Russell 已提交
626 627 628 629 630 631 632 633

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

634
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
635
	if (r < 0)
636
		goto fail_free_run;
R
Rusty Russell 已提交
637 638 639 640 641
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
642
	return r;
R
Rusty Russell 已提交
643 644 645 646 647
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
648
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
649 650 651 652
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
					     struct mm_struct *mm,
					     unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int need_tlb_flush;

	/*
	 * When ->invalidate_page runs, the linux pte has been zapped
	 * already but the page is still allocated until
	 * ->invalidate_page returns. So if we increase the sequence
	 * here the kvm page fault will notice if the spte can't be
	 * established because the page is going to be freed. If
	 * instead the kvm page fault establishes the spte before
	 * ->invalidate_page runs, kvm_unmap_hva will release it
	 * before returning.
	 *
	 * The sequence increase only need to be seen at spin_unlock
	 * time, and not at spin_lock time.
	 *
	 * Increasing the sequence after the spin_unlock would be
	 * unsafe because the kvm page fault could then establish the
	 * pte after kvm_unmap_hva returned, without noticing the page
	 * is going to be freed.
	 */
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	need_tlb_flush = kvm_unmap_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);

	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);

}

static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
						    struct mm_struct *mm,
						    unsigned long start,
						    unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int need_tlb_flush = 0;

	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
	for (; start < end; start += PAGE_SIZE)
		need_tlb_flush |= kvm_unmap_hva(kvm, start);
	spin_unlock(&kvm->mmu_lock);

	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
						  struct mm_struct *mm,
						  unsigned long start,
						  unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
	/*
	 * The above sequence increase must be visible before the
	 * below count decrease but both values are read by the kvm
	 * page fault under mmu_lock spinlock so we don't need to add
	 * a smb_wmb() here in between the two.
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
					      unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young;

	spin_lock(&kvm->mmu_lock);
	young = kvm_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);

	if (young)
		kvm_flush_remote_tlbs(kvm);

	return young;
}

762 763 764 765 766 767 768
static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
				     struct mm_struct *mm)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	kvm_arch_flush_shadow(kvm);
}

769 770 771 772 773
static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
774
	.release		= kvm_mmu_notifier_release,
775 776 777
};
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

778
static struct kvm *kvm_create_vm(void)
A
Avi Kivity 已提交
779
{
780
	struct kvm *kvm = kvm_arch_create_vm();
781 782 783
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	struct page *page;
#endif
A
Avi Kivity 已提交
784

785 786
	if (IS_ERR(kvm))
		goto out;
787
#ifdef CONFIG_HAVE_KVM_IRQCHIP
788
	INIT_LIST_HEAD(&kvm->irq_routing);
789 790
	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
#endif
A
Avi Kivity 已提交
791

792 793 794 795 796 797 798 799 800 801
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		kfree(kvm);
		return ERR_PTR(-ENOMEM);
	}
	kvm->coalesced_mmio_ring =
			(struct kvm_coalesced_mmio_ring *)page_address(page);
#endif

802 803 804 805 806 807 808 809 810 811 812 813 814 815 816
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	{
		int err;
		kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
		err = mmu_notifier_register(&kvm->mmu_notifier, current->mm);
		if (err) {
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
			put_page(page);
#endif
			kfree(kvm);
			return ERR_PTR(err);
		}
	}
#endif

817 818
	kvm->mm = current->mm;
	atomic_inc(&kvm->mm->mm_count);
819
	spin_lock_init(&kvm->mmu_lock);
820
	kvm_io_bus_init(&kvm->pio_bus);
S
Shaohua Li 已提交
821
	mutex_init(&kvm->lock);
822
	kvm_io_bus_init(&kvm->mmio_bus);
823
	init_rwsem(&kvm->slots_lock);
I
Izik Eidus 已提交
824
	atomic_set(&kvm->users_count, 1);
825 826 827
	spin_lock(&kvm_lock);
	list_add(&kvm->vm_list, &vm_list);
	spin_unlock(&kvm_lock);
828 829 830
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	kvm_coalesced_mmio_init(kvm);
#endif
831
out:
832 833 834
	return kvm;
}

A
Avi Kivity 已提交
835 836 837 838 839 840
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
841 842
	if (!dont || free->rmap != dont->rmap)
		vfree(free->rmap);
A
Avi Kivity 已提交
843 844 845 846

	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
		vfree(free->dirty_bitmap);

M
Marcelo Tosatti 已提交
847 848 849
	if (!dont || free->lpage_info != dont->lpage_info)
		vfree(free->lpage_info);

A
Avi Kivity 已提交
850
	free->npages = 0;
A
Al Viro 已提交
851
	free->dirty_bitmap = NULL;
852
	free->rmap = NULL;
M
Marcelo Tosatti 已提交
853
	free->lpage_info = NULL;
A
Avi Kivity 已提交
854 855
}

856
void kvm_free_physmem(struct kvm *kvm)
A
Avi Kivity 已提交
857 858 859 860
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i)
A
Al Viro 已提交
861
		kvm_free_physmem_slot(&kvm->memslots[i], NULL);
A
Avi Kivity 已提交
862 863
}

864 865
static void kvm_destroy_vm(struct kvm *kvm)
{
866 867
	struct mm_struct *mm = kvm->mm;

868
	kvm_arch_sync_events(kvm);
869 870 871
	spin_lock(&kvm_lock);
	list_del(&kvm->vm_list);
	spin_unlock(&kvm_lock);
872
	kvm_free_irq_routing(kvm);
873
	kvm_io_bus_destroy(&kvm->pio_bus);
874
	kvm_io_bus_destroy(&kvm->mmio_bus);
875 876 877
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	if (kvm->coalesced_mmio_ring != NULL)
		free_page((unsigned long)kvm->coalesced_mmio_ring);
878 879 880
#endif
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
881
#endif
882
	kvm_arch_destroy_vm(kvm);
883
	mmdrop(mm);
884 885
}

I
Izik Eidus 已提交
886 887 888 889 890 891 892 893 894 895 896 897 898 899
void kvm_get_kvm(struct kvm *kvm)
{
	atomic_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
	if (atomic_dec_and_test(&kvm->users_count))
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


900 901 902 903
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

I
Izik Eidus 已提交
904
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
905 906 907 908 909 910 911 912
	return 0;
}

/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
913
 *
914
 * Must be called holding mmap_sem for write.
A
Avi Kivity 已提交
915
 */
916 917 918
int __kvm_set_memory_region(struct kvm *kvm,
			    struct kvm_userspace_memory_region *mem,
			    int user_alloc)
A
Avi Kivity 已提交
919 920 921 922
{
	int r;
	gfn_t base_gfn;
	unsigned long npages;
923
	int largepages;
A
Avi Kivity 已提交
924 925 926 927 928 929 930 931 932 933
	unsigned long i;
	struct kvm_memory_slot *memslot;
	struct kvm_memory_slot old, new;

	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
S
Sheng Yang 已提交
934
	if (user_alloc && (mem->userspace_addr & (PAGE_SIZE - 1)))
935
		goto out;
936
	if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
A
Avi Kivity 已提交
937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

	memslot = &kvm->memslots[mem->slot];
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

	new = old = *memslot;

	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	/* Disallow changing a memory slot's size. */
	r = -EINVAL;
	if (npages && old.npages && npages != old.npages)
957
		goto out_free;
A
Avi Kivity 已提交
958 959 960 961 962 963

	/* Check for overlaps */
	r = -EEXIST;
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *s = &kvm->memslots[i];

964
		if (s == memslot || !s->npages)
A
Avi Kivity 已提交
965 966 967
			continue;
		if (!((base_gfn + npages <= s->base_gfn) ||
		      (base_gfn >= s->base_gfn + s->npages)))
968
			goto out_free;
A
Avi Kivity 已提交
969 970 971 972
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
973
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
974 975 976 977

	r = -ENOMEM;

	/* Allocate if a slot is being created */
978
#ifndef CONFIG_S390
979
	if (npages && !new.rmap) {
M
Mike Day 已提交
980
		new.rmap = vmalloc(npages * sizeof(struct page *));
981 982

		if (!new.rmap)
983
			goto out_free;
984 985

		memset(new.rmap, 0, npages * sizeof(*new.rmap));
986

987
		new.user_alloc = user_alloc;
988 989 990 991 992 993 994 995 996
		/*
		 * hva_to_rmmap() serialzies with the mmu_lock and to be
		 * safe it has to ignore memslots with !user_alloc &&
		 * !userspace_addr.
		 */
		if (user_alloc)
			new.userspace_addr = mem->userspace_addr;
		else
			new.userspace_addr = 0;
A
Avi Kivity 已提交
997
	}
M
Marcelo Tosatti 已提交
998
	if (npages && !new.lpage_info) {
999 1000
		largepages = 1 + (base_gfn + npages - 1) / KVM_PAGES_PER_HPAGE;
		largepages -= base_gfn / KVM_PAGES_PER_HPAGE;
M
Marcelo Tosatti 已提交
1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013

		new.lpage_info = vmalloc(largepages * sizeof(*new.lpage_info));

		if (!new.lpage_info)
			goto out_free;

		memset(new.lpage_info, 0, largepages * sizeof(*new.lpage_info));

		if (base_gfn % KVM_PAGES_PER_HPAGE)
			new.lpage_info[0].write_count = 1;
		if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE)
			new.lpage_info[largepages-1].write_count = 1;
	}
A
Avi Kivity 已提交
1014 1015 1016 1017 1018 1019 1020

	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
		unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;

		new.dirty_bitmap = vmalloc(dirty_bytes);
		if (!new.dirty_bitmap)
1021
			goto out_free;
A
Avi Kivity 已提交
1022 1023
		memset(new.dirty_bitmap, 0, dirty_bytes);
	}
1024
#endif /* not defined CONFIG_S390 */
A
Avi Kivity 已提交
1025

1026 1027 1028
	if (!npages)
		kvm_arch_flush_shadow(kvm);

1029 1030 1031 1032
	spin_lock(&kvm->mmu_lock);
	if (mem->slot >= kvm->nmemslots)
		kvm->nmemslots = mem->slot + 1;

1033
	*memslot = new;
1034
	spin_unlock(&kvm->mmu_lock);
1035

1036 1037
	r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
	if (r) {
1038
		spin_lock(&kvm->mmu_lock);
1039
		*memslot = old;
1040
		spin_unlock(&kvm->mmu_lock);
1041
		goto out_free;
1042 1043
	}

1044 1045 1046 1047
	kvm_free_physmem_slot(&old, npages ? &new : NULL);
	/* Slot deletion case: we have to update the current slot */
	if (!npages)
		*memslot = old;
1048
#ifdef CONFIG_DMAR
B
Ben-Ami Yassour 已提交
1049 1050 1051 1052
	/* map the pages in iommu page table */
	r = kvm_iommu_map_pages(kvm, base_gfn, npages);
	if (r)
		goto out;
1053
#endif
A
Avi Kivity 已提交
1054 1055
	return 0;

1056
out_free:
A
Avi Kivity 已提交
1057 1058 1059
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
1060 1061

}
1062 1063 1064 1065 1066 1067 1068 1069
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
			  struct kvm_userspace_memory_region *mem,
			  int user_alloc)
{
	int r;

1070
	down_write(&kvm->slots_lock);
1071
	r = __kvm_set_memory_region(kvm, mem, user_alloc);
1072
	up_write(&kvm->slots_lock);
1073 1074
	return r;
}
1075 1076
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

1077 1078 1079 1080
int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
				   struct
				   kvm_userspace_memory_region *mem,
				   int user_alloc)
1081
{
1082 1083
	if (mem->slot >= KVM_MEMORY_SLOTS)
		return -EINVAL;
1084
	return kvm_set_memory_region(kvm, mem, user_alloc);
A
Avi Kivity 已提交
1085 1086
}

1087 1088
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
{
	struct kvm_memory_slot *memslot;
	int r, i;
	int n;
	unsigned long any = 0;

	r = -EINVAL;
	if (log->slot >= KVM_MEMORY_SLOTS)
		goto out;

	memslot = &kvm->memslots[log->slot];
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

1104
	n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
A
Avi Kivity 已提交
1105

1106
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
1107 1108 1109 1110 1111 1112
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

1113 1114
	if (any)
		*is_dirty = 1;
A
Avi Kivity 已提交
1115 1116 1117 1118 1119 1120

	r = 0;
out:
	return r;
}

1121 1122 1123 1124 1125 1126
int is_error_page(struct page *page)
{
	return page == bad_page;
}
EXPORT_SYMBOL_GPL(is_error_page);

1127 1128 1129 1130 1131 1132
int is_error_pfn(pfn_t pfn)
{
	return pfn == bad_pfn;
}
EXPORT_SYMBOL_GPL(is_error_pfn);

I
Izik Eidus 已提交
1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
static inline unsigned long bad_hva(void)
{
	return PAGE_OFFSET;
}

int kvm_is_error_hva(unsigned long addr)
{
	return addr == bad_hva();
}
EXPORT_SYMBOL_GPL(kvm_is_error_hva);

1144
struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return memslot;
	}
A
Al Viro 已提交
1155
	return NULL;
A
Avi Kivity 已提交
1156
}
1157
EXPORT_SYMBOL_GPL(gfn_to_memslot_unaliased);
1158 1159 1160 1161

struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	gfn = unalias_gfn(kvm, gfn);
1162
	return gfn_to_memslot_unaliased(kvm, gfn);
1163
}
A
Avi Kivity 已提交
1164

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
	int i;

	gfn = unalias_gfn(kvm, gfn);
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

M
Marcelo Tosatti 已提交
1181
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
I
Izik Eidus 已提交
1182 1183 1184 1185
{
	struct kvm_memory_slot *slot;

	gfn = unalias_gfn(kvm, gfn);
1186
	slot = gfn_to_memslot_unaliased(kvm, gfn);
I
Izik Eidus 已提交
1187 1188 1189 1190
	if (!slot)
		return bad_hva();
	return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
}
1191
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1192

1193
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
1194
{
1195
	struct page *page[1];
I
Izik Eidus 已提交
1196
	unsigned long addr;
1197
	int npages;
1198
	pfn_t pfn;
A
Avi Kivity 已提交
1199

1200 1201
	might_sleep();

I
Izik Eidus 已提交
1202 1203
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr)) {
1204
		get_page(bad_page);
1205
		return page_to_pfn(bad_page);
1206
	}
1207

1208
	npages = get_user_pages_fast(addr, 1, 1, page);
I
Izik Eidus 已提交
1209

1210 1211 1212
	if (unlikely(npages != 1)) {
		struct vm_area_struct *vma;

1213
		down_read(&current->mm->mmap_sem);
1214
		vma = find_vma(current->mm, addr);
1215

1216 1217
		if (vma == NULL || addr < vma->vm_start ||
		    !(vma->vm_flags & VM_PFNMAP)) {
1218
			up_read(&current->mm->mmap_sem);
1219 1220 1221 1222 1223
			get_page(bad_page);
			return page_to_pfn(bad_page);
		}

		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1224
		up_read(&current->mm->mmap_sem);
1225
		BUG_ON(!kvm_is_mmio_pfn(pfn));
1226 1227
	} else
		pfn = page_to_pfn(page[0]);
1228

1229
	return pfn;
1230 1231 1232 1233 1234 1235
}

EXPORT_SYMBOL_GPL(gfn_to_pfn);

struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
1236 1237 1238
	pfn_t pfn;

	pfn = gfn_to_pfn(kvm, gfn);
1239
	if (!kvm_is_mmio_pfn(pfn))
1240 1241
		return pfn_to_page(pfn);

1242
	WARN_ON(kvm_is_mmio_pfn(pfn));
1243 1244 1245

	get_page(bad_page);
	return bad_page;
A
Avi Kivity 已提交
1246
}
1247

A
Avi Kivity 已提交
1248 1249
EXPORT_SYMBOL_GPL(gfn_to_page);

1250 1251
void kvm_release_page_clean(struct page *page)
{
1252
	kvm_release_pfn_clean(page_to_pfn(page));
1253 1254 1255
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

1256 1257
void kvm_release_pfn_clean(pfn_t pfn)
{
1258
	if (!kvm_is_mmio_pfn(pfn))
1259
		put_page(pfn_to_page(pfn));
1260 1261 1262
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1263
void kvm_release_page_dirty(struct page *page)
1264
{
1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

void kvm_release_pfn_dirty(pfn_t pfn)
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);

void kvm_set_page_dirty(struct page *page)
{
	kvm_set_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_set_page_dirty);

void kvm_set_pfn_dirty(pfn_t pfn)
{
1284
	if (!kvm_is_mmio_pfn(pfn)) {
1285 1286 1287 1288
		struct page *page = pfn_to_page(pfn);
		if (!PageReserved(page))
			SetPageDirty(page);
	}
1289
}
1290 1291 1292 1293
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

void kvm_set_pfn_accessed(pfn_t pfn)
{
1294
	if (!kvm_is_mmio_pfn(pfn))
1295
		mark_page_accessed(pfn_to_page(pfn));
1296 1297 1298 1299 1300
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

void kvm_get_pfn(pfn_t pfn)
{
1301
	if (!kvm_is_mmio_pfn(pfn))
1302
		get_page(pfn_to_page(pfn));
1303 1304
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1305

1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
1317 1318
	int r;
	unsigned long addr;
1319

1320 1321 1322 1323 1324
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_from_user(data, (void __user *)addr + offset, len);
	if (r)
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	int r;
	unsigned long addr;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int offset = offset_in_page(gpa);

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1361
	pagefault_disable();
1362
	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1363
	pagefault_enable();
1364 1365 1366 1367 1368 1369
	if (r)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL(kvm_read_guest_atomic);

1370 1371 1372
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
1373 1374
	int r;
	unsigned long addr;
1375

1376 1377 1378 1379 1380
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_to_user((void __user *)addr + offset, data, len);
	if (r)
1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
1409
	return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

A
Avi Kivity 已提交
1432 1433
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
1434
	struct kvm_memory_slot *memslot;
A
Avi Kivity 已提交
1435

1436
	gfn = unalias_gfn(kvm, gfn);
1437
	memslot = gfn_to_memslot_unaliased(kvm, gfn);
R
Rusty Russell 已提交
1438 1439
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
1440

R
Rusty Russell 已提交
1441 1442 1443
		/* avoid RMW */
		if (!test_bit(rel_gfn, memslot->dirty_bitmap))
			set_bit(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
1444 1445 1446
	}
}

E
Eddie Dong 已提交
1447 1448 1449
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1450
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1451
{
1452 1453 1454 1455 1456
	DEFINE_WAIT(wait);

	for (;;) {
		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

1457 1458 1459 1460
		if (kvm_cpu_has_interrupt(vcpu) ||
		    kvm_cpu_has_pending_timer(vcpu) ||
		    kvm_arch_vcpu_runnable(vcpu)) {
			set_bit(KVM_REQ_UNHALT, &vcpu->requests);
1461
			break;
1462
		}
1463 1464 1465
		if (signal_pending(current))
			break;

E
Eddie Dong 已提交
1466 1467 1468 1469
		vcpu_put(vcpu);
		schedule();
		vcpu_load(vcpu);
	}
1470

1471
	finish_wait(&vcpu->wq, &wait);
E
Eddie Dong 已提交
1472 1473
}

A
Avi Kivity 已提交
1474 1475
void kvm_resched(struct kvm_vcpu *vcpu)
{
1476 1477
	if (!need_resched())
		return;
A
Avi Kivity 已提交
1478 1479 1480 1481
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

1482
static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1483 1484 1485 1486
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	struct page *page;

1487
	if (vmf->pgoff == 0)
1488
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
1489
#ifdef CONFIG_X86
1490
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1491
		page = virt_to_page(vcpu->arch.pio_data);
1492 1493 1494 1495
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
1496
#endif
1497
	else
1498
		return VM_FAULT_SIGBUS;
1499
	get_page(page);
1500 1501
	vmf->page = page;
	return 0;
1502 1503 1504
}

static struct vm_operations_struct kvm_vcpu_vm_ops = {
1505
	.fault = kvm_vcpu_fault,
1506 1507 1508 1509 1510 1511 1512 1513
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
1514 1515 1516 1517
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

A
Al Viro 已提交
1518
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
1519 1520 1521
	return 0;
}

1522
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
1523 1524 1525
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
	.compat_ioctl   = kvm_vcpu_ioctl,
1526
	.mmap           = kvm_vcpu_mmap,
A
Avi Kivity 已提交
1527 1528 1529 1530 1531 1532 1533
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
1534
	int fd = anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, 0);
A
Al Viro 已提交
1535
	if (fd < 0)
A
Al Viro 已提交
1536
		kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
1537 1538 1539
	return fd;
}

1540 1541 1542 1543 1544 1545 1546 1547 1548
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
{
	int r;
	struct kvm_vcpu *vcpu;

	if (!valid_vcpu(n))
R
Rusty Russell 已提交
1549
		return -EINVAL;
1550

1551
	vcpu = kvm_arch_vcpu_create(kvm, n);
R
Rusty Russell 已提交
1552 1553
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
1554

1555 1556
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

1557 1558
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
1559
		return r;
1560

S
Shaohua Li 已提交
1561
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
1562 1563
	if (kvm->vcpus[n]) {
		r = -EEXIST;
1564
		goto vcpu_destroy;
R
Rusty Russell 已提交
1565 1566
	}
	kvm->vcpus[n] = vcpu;
S
Shaohua Li 已提交
1567
	mutex_unlock(&kvm->lock);
1568

R
Rusty Russell 已提交
1569
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
1570
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
1571 1572
	r = create_vcpu_fd(vcpu);
	if (r < 0)
R
Rusty Russell 已提交
1573 1574
		goto unlink;
	return r;
1575

R
Rusty Russell 已提交
1576
unlink:
S
Shaohua Li 已提交
1577
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
1578
	kvm->vcpus[n] = NULL;
1579
vcpu_destroy:
1580
	mutex_unlock(&kvm->lock);
1581
	kvm_arch_vcpu_destroy(vcpu);
1582 1583 1584
	return r;
}

A
Avi Kivity 已提交
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
1596 1597
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
1598
{
A
Avi Kivity 已提交
1599
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
1600
	void __user *argp = (void __user *)arg;
1601
	int r;
1602 1603
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
1604

1605 1606
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
1607
	switch (ioctl) {
1608
	case KVM_RUN:
1609 1610 1611
		r = -EINVAL;
		if (arg)
			goto out;
1612
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
A
Avi Kivity 已提交
1613 1614
		break;
	case KVM_GET_REGS: {
1615
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1616

1617 1618 1619
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1620
			goto out;
1621 1622 1623
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
1624
		r = -EFAULT;
1625 1626
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
1627
		r = 0;
1628 1629
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
1630 1631 1632
		break;
	}
	case KVM_SET_REGS: {
1633
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1634

1635 1636 1637
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1638
			goto out;
1639 1640 1641 1642
		r = -EFAULT;
		if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
			goto out_free2;
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
A
Avi Kivity 已提交
1643
		if (r)
1644
			goto out_free2;
A
Avi Kivity 已提交
1645
		r = 0;
1646 1647
out_free2:
		kfree(kvm_regs);
A
Avi Kivity 已提交
1648 1649 1650
		break;
	}
	case KVM_GET_SREGS: {
1651 1652 1653 1654 1655
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
1656 1657 1658
		if (r)
			goto out;
		r = -EFAULT;
1659
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
1660 1661 1662 1663 1664
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
1665 1666 1667 1668
		kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
A
Avi Kivity 已提交
1669
		r = -EFAULT;
1670
		if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
1671
			goto out;
1672
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
1673 1674 1675 1676 1677
		if (r)
			goto out;
		r = 0;
		break;
	}
1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &mp_state, sizeof mp_state))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
		if (copy_from_user(&mp_state, argp, sizeof mp_state))
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1702 1703 1704 1705
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
1706
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
1707
			goto out;
1708
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
1709 1710 1711
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
1712
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
1713 1714 1715 1716
			goto out;
		r = 0;
		break;
	}
J
Jan Kiszka 已提交
1717 1718
	case KVM_SET_GUEST_DEBUG: {
		struct kvm_guest_debug dbg;
A
Avi Kivity 已提交
1719 1720

		r = -EFAULT;
A
Al Viro 已提交
1721
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
1722
			goto out;
J
Jan Kiszka 已提交
1723
		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
A
Avi Kivity 已提交
1724 1725 1726 1727 1728
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		break;
	}
A
Avi Kivity 已提交
1752
	case KVM_GET_FPU: {
1753 1754 1755 1756 1757
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
1758 1759 1760
		if (r)
			goto out;
		r = -EFAULT;
1761
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
1762 1763 1764 1765 1766
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
1767 1768 1769 1770
		fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
A
Avi Kivity 已提交
1771
		r = -EFAULT;
1772
		if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
1773
			goto out;
1774
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
1775 1776 1777 1778 1779
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1780
	default:
1781
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
1782 1783
	}
out:
1784 1785
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
1786 1787 1788 1789 1790 1791 1792 1793
	return r;
}

static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
1794
	int r;
A
Avi Kivity 已提交
1795

1796 1797
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
1798 1799 1800 1801 1802 1803
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		if (r < 0)
			goto out;
		break;
1804 1805 1806 1807 1808 1809 1810 1811 1812
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
A
Avi Kivity 已提交
1813 1814 1815 1816 1817 1818 1819 1820
		if (r)
			goto out;
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
1821
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
1822
			goto out;
1823
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
1824 1825 1826 1827
		if (r)
			goto out;
		break;
	}
1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = -ENXIO;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = -ENXIO;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		if (r)
			goto out;
		r = 0;
		break;
	}
1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876
#endif
#ifdef KVM_CAP_DEVICE_ASSIGNMENT
	case KVM_ASSIGN_PCI_DEVICE: {
		struct kvm_assigned_pci_dev assigned_dev;

		r = -EFAULT;
		if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
			goto out;
		r = kvm_vm_ioctl_assign_device(kvm, &assigned_dev);
		if (r)
			goto out;
		break;
	}
	case KVM_ASSIGN_IRQ: {
		struct kvm_assigned_irq assigned_irq;

		r = -EFAULT;
		if (copy_from_user(&assigned_irq, argp, sizeof assigned_irq))
			goto out;
		r = kvm_vm_ioctl_assign_irq(kvm, &assigned_irq);
		if (r)
			goto out;
		break;
	}
W
Weidong Han 已提交
1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889
#endif
#ifdef KVM_CAP_DEVICE_DEASSIGNMENT
	case KVM_DEASSIGN_PCI_DEVICE: {
		struct kvm_assigned_pci_dev assigned_dev;

		r = -EFAULT;
		if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
			goto out;
		r = kvm_vm_ioctl_deassign_device(kvm, &assigned_dev);
		if (r)
			goto out;
		break;
	}
1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919
#endif
#ifdef KVM_CAP_IRQ_ROUTING
	case KVM_SET_GSI_ROUTING: {
		struct kvm_irq_routing routing;
		struct kvm_irq_routing __user *urouting;
		struct kvm_irq_routing_entry *entries;

		r = -EFAULT;
		if (copy_from_user(&routing, argp, sizeof(routing)))
			goto out;
		r = -EINVAL;
		if (routing.nr >= KVM_MAX_IRQ_ROUTES)
			goto out;
		if (routing.flags)
			goto out;
		r = -ENOMEM;
		entries = vmalloc(routing.nr * sizeof(*entries));
		if (!entries)
			goto out;
		r = -EFAULT;
		urouting = argp;
		if (copy_from_user(entries, urouting->entries,
				   routing.nr * sizeof(*entries)))
			goto out_free_irq_routing;
		r = kvm_set_irq_routing(kvm, entries, routing.nr,
					routing.flags);
	out_free_irq_routing:
		vfree(entries);
		break;
	}
1920
#endif
1921
	default:
1922
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
1923 1924 1925 1926 1927
	}
out:
	return r;
}

1928
static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1929
{
1930 1931 1932 1933
	struct page *page[1];
	unsigned long addr;
	int npages;
	gfn_t gfn = vmf->pgoff;
1934 1935
	struct kvm *kvm = vma->vm_file->private_data;

1936 1937
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
1938
		return VM_FAULT_SIGBUS;
1939 1940 1941 1942

	npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
				NULL);
	if (unlikely(npages != 1))
1943
		return VM_FAULT_SIGBUS;
1944 1945

	vmf->page = page[0];
1946
	return 0;
1947 1948 1949
}

static struct vm_operations_struct kvm_vm_vm_ops = {
1950
	.fault = kvm_vm_fault,
1951 1952 1953 1954 1955 1956 1957 1958
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

1959
static struct file_operations kvm_vm_fops = {
1960 1961 1962 1963 1964 1965 1966 1967
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
	.compat_ioctl   = kvm_vm_ioctl,
	.mmap           = kvm_vm_mmap,
};

static int kvm_dev_ioctl_create_vm(void)
{
A
Al Viro 已提交
1968
	int fd;
1969 1970 1971
	struct kvm *kvm;

	kvm = kvm_create_vm();
1972 1973
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
1974
	fd = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, 0);
A
Al Viro 已提交
1975
	if (fd < 0)
A
Al Viro 已提交
1976
		kvm_put_kvm(kvm);
1977 1978 1979 1980

	return fd;
}

1981 1982 1983
static long kvm_dev_ioctl_check_extension_generic(long arg)
{
	switch (arg) {
1984
	case KVM_CAP_USER_MEMORY:
1985
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
1986
	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
1987
		return 1;
1988 1989
#ifdef CONFIG_HAVE_KVM_IRQCHIP
	case KVM_CAP_IRQ_ROUTING:
1990
		return KVM_MAX_IRQ_ROUTES;
1991
#endif
1992 1993 1994 1995 1996 1997
	default:
		break;
	}
	return kvm_dev_ioctl_check_extension(arg);
}

1998 1999 2000
static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
2001
	long r = -EINVAL;
2002 2003 2004

	switch (ioctl) {
	case KVM_GET_API_VERSION:
2005 2006 2007
		r = -EINVAL;
		if (arg)
			goto out;
2008 2009 2010
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
2011 2012 2013
		r = -EINVAL;
		if (arg)
			goto out;
2014 2015
		r = kvm_dev_ioctl_create_vm();
		break;
2016
	case KVM_CHECK_EXTENSION:
2017
		r = kvm_dev_ioctl_check_extension_generic(arg);
2018
		break;
2019 2020 2021 2022
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
2023 2024 2025
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
2026 2027 2028
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
		r += PAGE_SIZE;    /* coalesced mmio ring page */
2029
#endif
2030
		break;
2031 2032 2033 2034 2035
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
		r = kvm_trace_ioctl(ioctl, arg);
		break;
A
Avi Kivity 已提交
2036
	default:
2037
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
2049
	KVM_MINOR,
A
Avi Kivity 已提交
2050 2051 2052 2053
	"kvm",
	&kvm_chardev_ops,
};

2054 2055 2056 2057
static void hardware_enable(void *junk)
{
	int cpu = raw_smp_processor_id();

2058
	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2059
		return;
2060
	cpumask_set_cpu(cpu, cpus_hardware_enabled);
2061
	kvm_arch_hardware_enable(NULL);
2062 2063 2064 2065 2066 2067
}

static void hardware_disable(void *junk)
{
	int cpu = raw_smp_processor_id();

2068
	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2069
		return;
2070
	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2071
	kvm_arch_hardware_disable(NULL);
2072 2073
}

A
Avi Kivity 已提交
2074 2075 2076 2077 2078
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

2079
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
2080
	switch (val) {
2081
	case CPU_DYING:
2082 2083 2084 2085
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
A
Avi Kivity 已提交
2086
	case CPU_UP_CANCELED:
2087 2088
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
2089
		smp_call_function_single(cpu, hardware_disable, NULL, 1);
A
Avi Kivity 已提交
2090
		break;
2091 2092 2093
	case CPU_ONLINE:
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
2094
		smp_call_function_single(cpu, hardware_enable, NULL, 1);
A
Avi Kivity 已提交
2095 2096 2097 2098 2099
		break;
	}
	return NOTIFY_OK;
}

2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111

asmlinkage void kvm_handle_fault_on_reboot(void)
{
	if (kvm_rebooting)
		/* spin while reset goes on */
		while (true)
			;
	/* Fault while not rebooting.  We want the trace. */
	BUG();
}
EXPORT_SYMBOL_GPL(kvm_handle_fault_on_reboot);

2112
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
2113
		      void *v)
2114 2115 2116 2117 2118 2119 2120
{
	if (val == SYS_RESTART) {
		/*
		 * Some (well, at least mine) BIOSes hang on reboot if
		 * in vmx root mode.
		 */
		printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2121
		kvm_rebooting = true;
2122
		on_each_cpu(hardware_disable, NULL, 1);
2123 2124 2125 2126 2127 2128 2129 2130 2131
	}
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147
void kvm_io_bus_init(struct kvm_io_bus *bus)
{
	memset(bus, 0, sizeof(*bus));
}

void kvm_io_bus_destroy(struct kvm_io_bus *bus)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		kvm_iodevice_destructor(pos);
	}
}

2148 2149
struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus,
					  gpa_t addr, int len, int is_write)
2150 2151 2152 2153 2154 2155
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

2156
		if (pos->in_range(pos, addr, len, is_write))
2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169
			return pos;
	}

	return NULL;
}

void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
{
	BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));

	bus->devs[bus->dev_count++] = dev;
}

A
Avi Kivity 已提交
2170 2171 2172 2173 2174
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
	.priority = 20, /* must be > scheduler priority */
};

2175
static int vm_stat_get(void *_offset, u64 *val)
2176 2177 2178 2179
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

2180
	*val = 0;
2181 2182
	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
2183
		*val += *(u32 *)((void *)kvm + offset);
2184
	spin_unlock(&kvm_lock);
2185
	return 0;
2186 2187 2188 2189
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");

2190
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
2191 2192 2193 2194 2195 2196
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

2197
	*val = 0;
A
Avi Kivity 已提交
2198 2199 2200
	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
R
Rusty Russell 已提交
2201 2202
			vcpu = kvm->vcpus[i];
			if (vcpu)
2203
				*val += *(u32 *)((void *)vcpu + offset);
A
Avi Kivity 已提交
2204 2205
		}
	spin_unlock(&kvm_lock);
2206
	return 0;
A
Avi Kivity 已提交
2207 2208
}

2209 2210 2211 2212 2213 2214
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");

static struct file_operations *stat_fops[] = {
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
2215

2216
static void kvm_init_debug(void)
A
Avi Kivity 已提交
2217 2218 2219
{
	struct kvm_stats_debugfs_item *p;

2220
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
A
Avi Kivity 已提交
2221
	for (p = debugfs_entries; p->name; ++p)
2222
		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
A
Avi Kivity 已提交
2223
						(void *)(long)p->offset,
2224
						stat_fops[p->kind]);
A
Avi Kivity 已提交
2225 2226 2227 2228 2229 2230 2231 2232
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
2233
	debugfs_remove(kvm_debugfs_dir);
A
Avi Kivity 已提交
2234 2235
}

2236 2237
static int kvm_suspend(struct sys_device *dev, pm_message_t state)
{
A
Avi Kivity 已提交
2238
	hardware_disable(NULL);
2239 2240 2241 2242 2243
	return 0;
}

static int kvm_resume(struct sys_device *dev)
{
A
Avi Kivity 已提交
2244
	hardware_enable(NULL);
2245 2246 2247 2248
	return 0;
}

static struct sysdev_class kvm_sysdev_class = {
2249
	.name = "kvm",
2250 2251 2252 2253 2254 2255 2256 2257 2258
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

static struct sys_device kvm_sysdev = {
	.id = 0,
	.cls = &kvm_sysdev_class,
};

2259
struct page *bad_page;
2260
pfn_t bad_pfn;
A
Avi Kivity 已提交
2261

2262 2263 2264 2265 2266 2267 2268 2269 2270 2271
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2272
	kvm_arch_vcpu_load(vcpu, cpu);
2273 2274 2275 2276 2277 2278 2279
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2280
	kvm_arch_vcpu_put(vcpu);
2281 2282
}

2283
int kvm_init(void *opaque, unsigned int vcpu_size,
2284
		  struct module *module)
A
Avi Kivity 已提交
2285 2286
{
	int r;
Y
Yang, Sheng 已提交
2287
	int cpu;
A
Avi Kivity 已提交
2288

2289 2290
	kvm_init_debug();

2291 2292
	r = kvm_arch_init(opaque);
	if (r)
2293
		goto out_fail;
2294 2295 2296 2297 2298 2299 2300 2301

	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);

	if (bad_page == NULL) {
		r = -ENOMEM;
		goto out;
	}

2302 2303
	bad_pfn = page_to_pfn(bad_page);

2304 2305 2306 2307
	if (!alloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
		r = -ENOMEM;
		goto out_free_0;
	}
2308
	cpumask_clear(cpus_hardware_enabled);
2309

2310
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
2311
	if (r < 0)
2312
		goto out_free_0a;
A
Avi Kivity 已提交
2313

Y
Yang, Sheng 已提交
2314 2315
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
2316
				kvm_arch_check_processor_compat,
2317
				&r, 1);
Y
Yang, Sheng 已提交
2318
		if (r < 0)
2319
			goto out_free_1;
Y
Yang, Sheng 已提交
2320 2321
	}

2322
	on_each_cpu(hardware_enable, NULL, 1);
A
Avi Kivity 已提交
2323 2324
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
2325
		goto out_free_2;
A
Avi Kivity 已提交
2326 2327
	register_reboot_notifier(&kvm_reboot_notifier);

2328 2329
	r = sysdev_class_register(&kvm_sysdev_class);
	if (r)
2330
		goto out_free_3;
2331 2332 2333

	r = sysdev_register(&kvm_sysdev);
	if (r)
2334
		goto out_free_4;
2335

2336 2337
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
J
Joe Perches 已提交
2338 2339
					   __alignof__(struct kvm_vcpu),
					   0, NULL);
2340 2341
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
2342
		goto out_free_5;
2343 2344
	}

A
Avi Kivity 已提交
2345
	kvm_chardev_ops.owner = module;
2346 2347
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
2348 2349 2350

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
2351
		printk(KERN_ERR "kvm: misc device register failed\n");
A
Avi Kivity 已提交
2352 2353 2354
		goto out_free;
	}

2355 2356
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;
S
Sheng Yang 已提交
2357 2358 2359
#ifndef CONFIG_X86
	msi2intx = 0;
#endif
2360

2361
	return 0;
A
Avi Kivity 已提交
2362 2363

out_free:
2364
	kmem_cache_destroy(kvm_vcpu_cache);
2365
out_free_5:
2366
	sysdev_unregister(&kvm_sysdev);
2367
out_free_4:
2368
	sysdev_class_unregister(&kvm_sysdev_class);
2369
out_free_3:
A
Avi Kivity 已提交
2370
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
2371
	unregister_cpu_notifier(&kvm_cpu_notifier);
2372
out_free_2:
2373
	on_each_cpu(hardware_disable, NULL, 1);
2374
out_free_1:
2375
	kvm_arch_hardware_unsetup();
2376 2377
out_free_0a:
	free_cpumask_var(cpus_hardware_enabled);
2378 2379
out_free_0:
	__free_page(bad_page);
2380
out:
2381
	kvm_arch_exit();
2382
	kvm_exit_debug();
2383
out_fail:
A
Avi Kivity 已提交
2384 2385
	return r;
}
2386
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
2387

2388
void kvm_exit(void)
A
Avi Kivity 已提交
2389
{
2390
	kvm_trace_cleanup();
A
Avi Kivity 已提交
2391
	misc_deregister(&kvm_dev);
2392
	kmem_cache_destroy(kvm_vcpu_cache);
2393 2394
	sysdev_unregister(&kvm_sysdev);
	sysdev_class_unregister(&kvm_sysdev_class);
A
Avi Kivity 已提交
2395
	unregister_reboot_notifier(&kvm_reboot_notifier);
2396
	unregister_cpu_notifier(&kvm_cpu_notifier);
2397
	on_each_cpu(hardware_disable, NULL, 1);
2398
	kvm_arch_hardware_unsetup();
2399
	kvm_arch_exit();
A
Avi Kivity 已提交
2400
	kvm_exit_debug();
2401
	free_cpumask_var(cpus_hardware_enabled);
2402
	__free_page(bad_page);
A
Avi Kivity 已提交
2403
}
2404
EXPORT_SYMBOL_GPL(kvm_exit);