kvm_main.c 50.8 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

18
#include "iodev.h"
A
Avi Kivity 已提交
19

20
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
21 22 23 24 25 26 27 28 29 30 31 32
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
33
#include <linux/sysdev.h>
A
Avi Kivity 已提交
34
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
35
#include <linux/sched.h>
36 37
#include <linux/cpumask.h>
#include <linux/smp.h>
38
#include <linux/anon_inodes.h>
39
#include <linux/profile.h>
40
#include <linux/kvm_para.h>
41
#include <linux/pagemap.h>
42
#include <linux/mman.h>
43
#include <linux/swap.h>
A
Avi Kivity 已提交
44

A
Avi Kivity 已提交
45 46 47
#include <asm/processor.h>
#include <asm/io.h>
#include <asm/uaccess.h>
48
#include <asm/pgtable.h>
A
Avi Kivity 已提交
49

50 51 52 53
#ifdef CONFIG_X86
#include <asm/msidef.h>
#endif

54 55 56 57
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
#include "coalesced_mmio.h"
#endif

58 59 60 61 62 63
#ifdef KVM_CAP_DEVICE_ASSIGNMENT
#include <linux/pci.h>
#include <linux/interrupt.h>
#include "irq.h"
#endif

A
Avi Kivity 已提交
64 65 66
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

S
Sheng Yang 已提交
67 68 69
static int msi2intx = 1;
module_param(msi2intx, bool, 0);

70 71
DEFINE_SPINLOCK(kvm_lock);
LIST_HEAD(vm_list);
72

73 74
static cpumask_t cpus_hardware_enabled;

75 76
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
77

78 79
static __read_mostly struct preempt_ops kvm_preempt_ops;

80
struct dentry *kvm_debugfs_dir;
A
Avi Kivity 已提交
81

A
Avi Kivity 已提交
82 83 84
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);

H
Hannes Eder 已提交
85
static bool kvm_rebooting;
86

87
#ifdef KVM_CAP_DEVICE_ASSIGNMENT
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138

#ifdef CONFIG_X86
static void assigned_device_msi_dispatch(struct kvm_assigned_dev_kernel *dev)
{
	int vcpu_id;
	struct kvm_vcpu *vcpu;
	struct kvm_ioapic *ioapic = ioapic_irqchip(dev->kvm);
	int dest_id = (dev->guest_msi.address_lo & MSI_ADDR_DEST_ID_MASK)
			>> MSI_ADDR_DEST_ID_SHIFT;
	int vector = (dev->guest_msi.data & MSI_DATA_VECTOR_MASK)
			>> MSI_DATA_VECTOR_SHIFT;
	int dest_mode = test_bit(MSI_ADDR_DEST_MODE_SHIFT,
				(unsigned long *)&dev->guest_msi.address_lo);
	int trig_mode = test_bit(MSI_DATA_TRIGGER_SHIFT,
				(unsigned long *)&dev->guest_msi.data);
	int delivery_mode = test_bit(MSI_DATA_DELIVERY_MODE_SHIFT,
				(unsigned long *)&dev->guest_msi.data);
	u32 deliver_bitmask;

	BUG_ON(!ioapic);

	deliver_bitmask = kvm_ioapic_get_delivery_bitmask(ioapic,
				dest_id, dest_mode);
	/* IOAPIC delivery mode value is the same as MSI here */
	switch (delivery_mode) {
	case IOAPIC_LOWEST_PRIORITY:
		vcpu = kvm_get_lowest_prio_vcpu(ioapic->kvm, vector,
				deliver_bitmask);
		if (vcpu != NULL)
			kvm_apic_set_irq(vcpu, vector, trig_mode);
		else
			printk(KERN_INFO "kvm: null lowest priority vcpu!\n");
		break;
	case IOAPIC_FIXED:
		for (vcpu_id = 0; deliver_bitmask != 0; vcpu_id++) {
			if (!(deliver_bitmask & (1 << vcpu_id)))
				continue;
			deliver_bitmask &= ~(1 << vcpu_id);
			vcpu = ioapic->kvm->vcpus[vcpu_id];
			if (vcpu)
				kvm_apic_set_irq(vcpu, vector, trig_mode);
		}
		break;
	default:
		printk(KERN_INFO "kvm: unsupported MSI delivery mode\n");
	}
}
#else
static void assigned_device_msi_dispatch(struct kvm_assigned_dev_kernel *dev) {}
#endif

139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
static struct kvm_assigned_dev_kernel *kvm_find_assigned_dev(struct list_head *head,
						      int assigned_dev_id)
{
	struct list_head *ptr;
	struct kvm_assigned_dev_kernel *match;

	list_for_each(ptr, head) {
		match = list_entry(ptr, struct kvm_assigned_dev_kernel, list);
		if (match->assigned_dev_id == assigned_dev_id)
			return match;
	}
	return NULL;
}

static void kvm_assigned_dev_interrupt_work_handler(struct work_struct *work)
{
	struct kvm_assigned_dev_kernel *assigned_dev;

	assigned_dev = container_of(work, struct kvm_assigned_dev_kernel,
				    interrupt_work);

	/* This is taken to safely inject irq inside the guest. When
	 * the interrupt injection (or the ioapic code) uses a
	 * finer-grained lock, update this
	 */
	mutex_lock(&assigned_dev->kvm->lock);
S
Sheng Yang 已提交
165 166 167 168 169 170 171 172 173
	if (assigned_dev->irq_requested_type & KVM_ASSIGNED_DEV_GUEST_INTX)
		kvm_set_irq(assigned_dev->kvm,
			    assigned_dev->irq_source_id,
			    assigned_dev->guest_irq, 1);
	else if (assigned_dev->irq_requested_type &
				KVM_ASSIGNED_DEV_GUEST_MSI) {
		assigned_device_msi_dispatch(assigned_dev);
		enable_irq(assigned_dev->host_irq);
	}
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
	mutex_unlock(&assigned_dev->kvm->lock);
	kvm_put_kvm(assigned_dev->kvm);
}

static irqreturn_t kvm_assigned_dev_intr(int irq, void *dev_id)
{
	struct kvm_assigned_dev_kernel *assigned_dev =
		(struct kvm_assigned_dev_kernel *) dev_id;

	kvm_get_kvm(assigned_dev->kvm);
	schedule_work(&assigned_dev->interrupt_work);
	disable_irq_nosync(irq);
	return IRQ_HANDLED;
}

/* Ack the irq line for an assigned device */
static void kvm_assigned_dev_ack_irq(struct kvm_irq_ack_notifier *kian)
{
	struct kvm_assigned_dev_kernel *dev;

	if (kian->gsi == -1)
		return;

	dev = container_of(kian, struct kvm_assigned_dev_kernel,
			   ack_notifier);
199
	kvm_set_irq(dev->kvm, dev->irq_source_id, dev->guest_irq, 0);
200 201 202
	enable_irq(dev->host_irq);
}

203 204
static void kvm_free_assigned_irq(struct kvm *kvm,
				  struct kvm_assigned_dev_kernel *assigned_dev)
205
{
206 207
	if (!irqchip_in_kernel(kvm))
		return;
208

209
	kvm_unregister_irq_ack_notifier(&assigned_dev->ack_notifier);
210 211 212 213

	if (assigned_dev->irq_source_id != -1)
		kvm_free_irq_source_id(kvm, assigned_dev->irq_source_id);
	assigned_dev->irq_source_id = -1;
214

215 216 217
	if (!assigned_dev->irq_requested_type)
		return;

218 219 220 221 222 223
	if (cancel_work_sync(&assigned_dev->interrupt_work))
		/* We had pending work. That means we will have to take
		 * care of kvm_put_kvm.
		 */
		kvm_put_kvm(kvm);

224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
	free_irq(assigned_dev->host_irq, (void *)assigned_dev);

	if (assigned_dev->irq_requested_type & KVM_ASSIGNED_DEV_HOST_MSI)
		pci_disable_msi(assigned_dev->dev);

	assigned_dev->irq_requested_type = 0;
}


static void kvm_free_assigned_device(struct kvm *kvm,
				     struct kvm_assigned_dev_kernel
				     *assigned_dev)
{
	kvm_free_assigned_irq(kvm, assigned_dev);

239 240
	pci_reset_function(assigned_dev->dev);

241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
	pci_release_regions(assigned_dev->dev);
	pci_disable_device(assigned_dev->dev);
	pci_dev_put(assigned_dev->dev);

	list_del(&assigned_dev->list);
	kfree(assigned_dev);
}

void kvm_free_all_assigned_devices(struct kvm *kvm)
{
	struct list_head *ptr, *ptr2;
	struct kvm_assigned_dev_kernel *assigned_dev;

	list_for_each_safe(ptr, ptr2, &kvm->arch.assigned_dev_head) {
		assigned_dev = list_entry(ptr,
					  struct kvm_assigned_dev_kernel,
					  list);

		kvm_free_assigned_device(kvm, assigned_dev);
	}
}

263 264 265 266
static int assigned_device_update_intx(struct kvm *kvm,
			struct kvm_assigned_dev_kernel *adev,
			struct kvm_assigned_irq *airq)
{
267 268 269 270
	adev->guest_irq = airq->guest_irq;
	adev->ack_notifier.gsi = airq->guest_irq;

	if (adev->irq_requested_type & KVM_ASSIGNED_DEV_HOST_INTX)
271 272 273
		return 0;

	if (irqchip_in_kernel(kvm)) {
S
Sheng Yang 已提交
274 275
		if (!msi2intx &&
		    adev->irq_requested_type & KVM_ASSIGNED_DEV_HOST_MSI) {
S
Sheng Yang 已提交
276 277 278 279
			free_irq(adev->host_irq, (void *)kvm);
			pci_disable_msi(adev->dev);
		}

280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297
		if (!capable(CAP_SYS_RAWIO))
			return -EPERM;

		if (airq->host_irq)
			adev->host_irq = airq->host_irq;
		else
			adev->host_irq = adev->dev->irq;

		/* Even though this is PCI, we don't want to use shared
		 * interrupts. Sharing host devices with guest-assigned devices
		 * on the same interrupt line is not a happy situation: there
		 * are going to be long delays in accepting, acking, etc.
		 */
		if (request_irq(adev->host_irq, kvm_assigned_dev_intr,
				0, "kvm_assigned_intx_device", (void *)adev))
			return -EIO;
	}

298 299
	adev->irq_requested_type = KVM_ASSIGNED_DEV_GUEST_INTX |
				   KVM_ASSIGNED_DEV_HOST_INTX;
300 301 302
	return 0;
}

S
Sheng Yang 已提交
303 304 305 306 307 308 309
#ifdef CONFIG_X86
static int assigned_device_update_msi(struct kvm *kvm,
			struct kvm_assigned_dev_kernel *adev,
			struct kvm_assigned_irq *airq)
{
	int r;

S
Sheng Yang 已提交
310 311 312 313 314 315 316 317 318 319 320 321 322
	if (airq->flags & KVM_DEV_IRQ_ASSIGN_ENABLE_MSI) {
		/* x86 don't care upper address of guest msi message addr */
		adev->irq_requested_type |= KVM_ASSIGNED_DEV_GUEST_MSI;
		adev->irq_requested_type &= ~KVM_ASSIGNED_DEV_GUEST_INTX;
		adev->guest_msi.address_lo = airq->guest_msi.addr_lo;
		adev->guest_msi.data = airq->guest_msi.data;
		adev->ack_notifier.gsi = -1;
	} else if (msi2intx) {
		adev->irq_requested_type |= KVM_ASSIGNED_DEV_GUEST_INTX;
		adev->irq_requested_type &= ~KVM_ASSIGNED_DEV_GUEST_MSI;
		adev->guest_irq = airq->guest_irq;
		adev->ack_notifier.gsi = airq->guest_irq;
	}
S
Sheng Yang 已提交
323 324 325 326 327

	if (adev->irq_requested_type & KVM_ASSIGNED_DEV_HOST_MSI)
		return 0;

	if (irqchip_in_kernel(kvm)) {
S
Sheng Yang 已提交
328 329 330 331 332 333 334 335 336
		if (!msi2intx) {
			if (adev->irq_requested_type &
					KVM_ASSIGNED_DEV_HOST_INTX)
				free_irq(adev->host_irq, (void *)adev);

			r = pci_enable_msi(adev->dev);
			if (r)
				return r;
		}
S
Sheng Yang 已提交
337 338 339 340 341 342 343

		adev->host_irq = adev->dev->irq;
		if (request_irq(adev->host_irq, kvm_assigned_dev_intr, 0,
				"kvm_assigned_msi_device", (void *)adev))
			return -EIO;
	}

S
Sheng Yang 已提交
344 345 346 347
	if (!msi2intx)
		adev->irq_requested_type = KVM_ASSIGNED_DEV_GUEST_MSI;

	adev->irq_requested_type |= KVM_ASSIGNED_DEV_HOST_MSI;
S
Sheng Yang 已提交
348 349 350 351
	return 0;
}
#endif

352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
static int kvm_vm_ioctl_assign_irq(struct kvm *kvm,
				   struct kvm_assigned_irq
				   *assigned_irq)
{
	int r = 0;
	struct kvm_assigned_dev_kernel *match;

	mutex_lock(&kvm->lock);

	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
				      assigned_irq->assigned_dev_id);
	if (!match) {
		mutex_unlock(&kvm->lock);
		return -EINVAL;
	}

368
	if (!match->irq_requested_type) {
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
		INIT_WORK(&match->interrupt_work,
				kvm_assigned_dev_interrupt_work_handler);
		if (irqchip_in_kernel(kvm)) {
			/* Register ack nofitier */
			match->ack_notifier.gsi = -1;
			match->ack_notifier.irq_acked =
					kvm_assigned_dev_ack_irq;
			kvm_register_irq_ack_notifier(kvm,
					&match->ack_notifier);

			/* Request IRQ source ID */
			r = kvm_request_irq_source_id(kvm);
			if (r < 0)
				goto out_release;
			else
				match->irq_source_id = r;
S
Sheng Yang 已提交
385 386 387 388 389 390 391

#ifdef CONFIG_X86
			/* Determine host device irq type, we can know the
			 * result from dev->msi_enabled */
			if (msi2intx)
				pci_enable_msi(match->dev);
#endif
392
		}
393 394
	}

S
Sheng Yang 已提交
395 396 397
	if ((!msi2intx &&
	     (assigned_irq->flags & KVM_DEV_IRQ_ASSIGN_ENABLE_MSI)) ||
	    (msi2intx && match->dev->msi_enabled)) {
S
Sheng Yang 已提交
398 399 400 401 402 403 404 405 406 407 408 409
#ifdef CONFIG_X86
		r = assigned_device_update_msi(kvm, match, assigned_irq);
		if (r) {
			printk(KERN_WARNING "kvm: failed to enable "
					"MSI device!\n");
			goto out_release;
		}
#else
		r = -ENOTTY;
#endif
	} else if (assigned_irq->host_irq == 0 && match->dev->irq == 0) {
		/* Host device IRQ 0 means don't support INTx */
S
Sheng Yang 已提交
410 411 412 413 414 415 416 417 418 419
		if (!msi2intx) {
			printk(KERN_WARNING
			       "kvm: wait device to enable MSI!\n");
			r = 0;
		} else {
			printk(KERN_WARNING
			       "kvm: failed to enable MSI device!\n");
			r = -ENOTTY;
			goto out_release;
		}
S
Sheng Yang 已提交
420 421 422 423 424 425 426 427 428
	} else {
		/* Non-sharing INTx mode */
		r = assigned_device_update_intx(kvm, match, assigned_irq);
		if (r) {
			printk(KERN_WARNING "kvm: failed to enable "
					"INTx device!\n");
			goto out_release;
		}
	}
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479

	mutex_unlock(&kvm->lock);
	return r;
out_release:
	mutex_unlock(&kvm->lock);
	kvm_free_assigned_device(kvm, match);
	return r;
}

static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
				      struct kvm_assigned_pci_dev *assigned_dev)
{
	int r = 0;
	struct kvm_assigned_dev_kernel *match;
	struct pci_dev *dev;

	mutex_lock(&kvm->lock);

	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
				      assigned_dev->assigned_dev_id);
	if (match) {
		/* device already assigned */
		r = -EINVAL;
		goto out;
	}

	match = kzalloc(sizeof(struct kvm_assigned_dev_kernel), GFP_KERNEL);
	if (match == NULL) {
		printk(KERN_INFO "%s: Couldn't allocate memory\n",
		       __func__);
		r = -ENOMEM;
		goto out;
	}
	dev = pci_get_bus_and_slot(assigned_dev->busnr,
				   assigned_dev->devfn);
	if (!dev) {
		printk(KERN_INFO "%s: host device not found\n", __func__);
		r = -EINVAL;
		goto out_free;
	}
	if (pci_enable_device(dev)) {
		printk(KERN_INFO "%s: Could not enable PCI device\n", __func__);
		r = -EBUSY;
		goto out_put;
	}
	r = pci_request_regions(dev, "kvm_assigned_device");
	if (r) {
		printk(KERN_INFO "%s: Could not get access to device regions\n",
		       __func__);
		goto out_disable;
	}
480 481 482

	pci_reset_function(dev);

483 484 485 486
	match->assigned_dev_id = assigned_dev->assigned_dev_id;
	match->host_busnr = assigned_dev->busnr;
	match->host_devfn = assigned_dev->devfn;
	match->dev = dev;
487
	match->irq_source_id = -1;
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
	match->kvm = kvm;

	list_add(&match->list, &kvm->arch.assigned_dev_head);

	if (assigned_dev->flags & KVM_DEV_ASSIGN_ENABLE_IOMMU) {
		r = kvm_iommu_map_guest(kvm, match);
		if (r)
			goto out_list_del;
	}

out:
	mutex_unlock(&kvm->lock);
	return r;
out_list_del:
	list_del(&match->list);
	pci_release_regions(dev);
out_disable:
	pci_disable_device(dev);
out_put:
	pci_dev_put(dev);
out_free:
	kfree(match);
	mutex_unlock(&kvm->lock);
	return r;
}
#endif

515 516 517 518 519
static inline int valid_vcpu(int n)
{
	return likely(n >= 0 && n < KVM_MAX_VCPUS);
}

520
inline int kvm_is_mmio_pfn(pfn_t pfn)
B
Ben-Ami Yassour 已提交
521 522 523 524 525 526 527
{
	if (pfn_valid(pfn))
		return PageReserved(pfn_to_page(pfn));

	return true;
}

A
Avi Kivity 已提交
528 529 530
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
531
void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
532
{
533 534
	int cpu;

A
Avi Kivity 已提交
535
	mutex_lock(&vcpu->mutex);
536 537
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
538
	kvm_arch_vcpu_load(vcpu, cpu);
539
	put_cpu();
A
Avi Kivity 已提交
540 541
}

542
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
543
{
544
	preempt_disable();
545
	kvm_arch_vcpu_put(vcpu);
546 547
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
548 549 550
	mutex_unlock(&vcpu->mutex);
}

551 552 553 554
static void ack_flush(void *_completed)
{
}

555
static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
556
{
557
	int i, cpu, me;
558
	cpumask_t cpus;
559
	bool called = false;
560 561
	struct kvm_vcpu *vcpu;

562
	me = get_cpu();
563
	cpus_clear(cpus);
R
Rusty Russell 已提交
564 565 566 567
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		vcpu = kvm->vcpus[i];
		if (!vcpu)
			continue;
568
		if (test_and_set_bit(req, &vcpu->requests))
569 570
			continue;
		cpu = vcpu->cpu;
571
		if (cpu != -1 && cpu != me)
572
			cpu_set(cpu, cpus);
573
	}
574 575 576 577
	if (!cpus_empty(cpus)) {
		smp_call_function_mask(cpus, ack_flush, NULL, 1);
		called = true;
	}
578
	put_cpu();
579
	return called;
580 581
}

582
void kvm_flush_remote_tlbs(struct kvm *kvm)
583
{
584 585
	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
		++kvm->stat.remote_tlb_flush;
586 587
}

588 589 590 591
void kvm_reload_remote_mmus(struct kvm *kvm)
{
	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
}
592

R
Rusty Russell 已提交
593 594 595 596 597 598 599 600 601
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
E
Eddie Dong 已提交
602
	init_waitqueue_head(&vcpu->wq);
R
Rusty Russell 已提交
603 604 605 606 607 608 609 610

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

611
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
612
	if (r < 0)
613
		goto fail_free_run;
R
Rusty Russell 已提交
614 615 616 617 618
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
619
	return r;
R
Rusty Russell 已提交
620 621 622 623 624
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
625
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
626 627 628 629
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
	return container_of(mn, struct kvm, mmu_notifier);
}

static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
					     struct mm_struct *mm,
					     unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int need_tlb_flush;

	/*
	 * When ->invalidate_page runs, the linux pte has been zapped
	 * already but the page is still allocated until
	 * ->invalidate_page returns. So if we increase the sequence
	 * here the kvm page fault will notice if the spte can't be
	 * established because the page is going to be freed. If
	 * instead the kvm page fault establishes the spte before
	 * ->invalidate_page runs, kvm_unmap_hva will release it
	 * before returning.
	 *
	 * The sequence increase only need to be seen at spin_unlock
	 * time, and not at spin_lock time.
	 *
	 * Increasing the sequence after the spin_unlock would be
	 * unsafe because the kvm page fault could then establish the
	 * pte after kvm_unmap_hva returned, without noticing the page
	 * is going to be freed.
	 */
	spin_lock(&kvm->mmu_lock);
	kvm->mmu_notifier_seq++;
	need_tlb_flush = kvm_unmap_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);

	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);

}

static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
						    struct mm_struct *mm,
						    unsigned long start,
						    unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int need_tlb_flush = 0;

	spin_lock(&kvm->mmu_lock);
	/*
	 * The count increase must become visible at unlock time as no
	 * spte can be established without taking the mmu_lock and
	 * count is also read inside the mmu_lock critical section.
	 */
	kvm->mmu_notifier_count++;
	for (; start < end; start += PAGE_SIZE)
		need_tlb_flush |= kvm_unmap_hva(kvm, start);
	spin_unlock(&kvm->mmu_lock);

	/* we've to flush the tlb before the pages can be freed */
	if (need_tlb_flush)
		kvm_flush_remote_tlbs(kvm);
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
						  struct mm_struct *mm,
						  unsigned long start,
						  unsigned long end)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);

	spin_lock(&kvm->mmu_lock);
	/*
	 * This sequence increase will notify the kvm page fault that
	 * the page that is going to be mapped in the spte could have
	 * been freed.
	 */
	kvm->mmu_notifier_seq++;
	/*
	 * The above sequence increase must be visible before the
	 * below count decrease but both values are read by the kvm
	 * page fault under mmu_lock spinlock so we don't need to add
	 * a smb_wmb() here in between the two.
	 */
	kvm->mmu_notifier_count--;
	spin_unlock(&kvm->mmu_lock);

	BUG_ON(kvm->mmu_notifier_count < 0);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
					      struct mm_struct *mm,
					      unsigned long address)
{
	struct kvm *kvm = mmu_notifier_to_kvm(mn);
	int young;

	spin_lock(&kvm->mmu_lock);
	young = kvm_age_hva(kvm, address);
	spin_unlock(&kvm->mmu_lock);

	if (young)
		kvm_flush_remote_tlbs(kvm);

	return young;
}

static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
};
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */

747
static struct kvm *kvm_create_vm(void)
A
Avi Kivity 已提交
748
{
749
	struct kvm *kvm = kvm_arch_create_vm();
750 751 752
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	struct page *page;
#endif
A
Avi Kivity 已提交
753

754 755
	if (IS_ERR(kvm))
		goto out;
A
Avi Kivity 已提交
756

757 758 759 760 761 762 763 764 765 766
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		kfree(kvm);
		return ERR_PTR(-ENOMEM);
	}
	kvm->coalesced_mmio_ring =
			(struct kvm_coalesced_mmio_ring *)page_address(page);
#endif

767 768 769 770 771 772 773 774 775 776 777 778 779 780 781
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	{
		int err;
		kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
		err = mmu_notifier_register(&kvm->mmu_notifier, current->mm);
		if (err) {
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
			put_page(page);
#endif
			kfree(kvm);
			return ERR_PTR(err);
		}
	}
#endif

782 783
	kvm->mm = current->mm;
	atomic_inc(&kvm->mm->mm_count);
784
	spin_lock_init(&kvm->mmu_lock);
785
	kvm_io_bus_init(&kvm->pio_bus);
S
Shaohua Li 已提交
786
	mutex_init(&kvm->lock);
787
	kvm_io_bus_init(&kvm->mmio_bus);
788
	init_rwsem(&kvm->slots_lock);
I
Izik Eidus 已提交
789
	atomic_set(&kvm->users_count, 1);
790 791 792
	spin_lock(&kvm_lock);
	list_add(&kvm->vm_list, &vm_list);
	spin_unlock(&kvm_lock);
793 794 795
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	kvm_coalesced_mmio_init(kvm);
#endif
796
out:
797 798 799
	return kvm;
}

A
Avi Kivity 已提交
800 801 802 803 804 805
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
806 807
	if (!dont || free->rmap != dont->rmap)
		vfree(free->rmap);
A
Avi Kivity 已提交
808 809 810 811

	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
		vfree(free->dirty_bitmap);

M
Marcelo Tosatti 已提交
812 813 814
	if (!dont || free->lpage_info != dont->lpage_info)
		vfree(free->lpage_info);

A
Avi Kivity 已提交
815
	free->npages = 0;
A
Al Viro 已提交
816
	free->dirty_bitmap = NULL;
817
	free->rmap = NULL;
M
Marcelo Tosatti 已提交
818
	free->lpage_info = NULL;
A
Avi Kivity 已提交
819 820
}

821
void kvm_free_physmem(struct kvm *kvm)
A
Avi Kivity 已提交
822 823 824 825
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i)
A
Al Viro 已提交
826
		kvm_free_physmem_slot(&kvm->memslots[i], NULL);
A
Avi Kivity 已提交
827 828
}

829 830
static void kvm_destroy_vm(struct kvm *kvm)
{
831 832
	struct mm_struct *mm = kvm->mm;

833 834 835
	spin_lock(&kvm_lock);
	list_del(&kvm->vm_list);
	spin_unlock(&kvm_lock);
836
	kvm_io_bus_destroy(&kvm->pio_bus);
837
	kvm_io_bus_destroy(&kvm->mmio_bus);
838 839 840
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	if (kvm->coalesced_mmio_ring != NULL)
		free_page((unsigned long)kvm->coalesced_mmio_ring);
841 842 843
#endif
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
844
#endif
845
	kvm_arch_destroy_vm(kvm);
846
	mmdrop(mm);
847 848
}

I
Izik Eidus 已提交
849 850 851 852 853 854 855 856 857 858 859 860 861 862
void kvm_get_kvm(struct kvm *kvm)
{
	atomic_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
	if (atomic_dec_and_test(&kvm->users_count))
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


863 864 865 866
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

I
Izik Eidus 已提交
867
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
868 869 870 871 872 873 874 875
	return 0;
}

/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
876
 *
877
 * Must be called holding mmap_sem for write.
A
Avi Kivity 已提交
878
 */
879 880 881
int __kvm_set_memory_region(struct kvm *kvm,
			    struct kvm_userspace_memory_region *mem,
			    int user_alloc)
A
Avi Kivity 已提交
882 883 884 885 886 887 888 889 890 891 892 893 894 895
{
	int r;
	gfn_t base_gfn;
	unsigned long npages;
	unsigned long i;
	struct kvm_memory_slot *memslot;
	struct kvm_memory_slot old, new;

	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
S
Sheng Yang 已提交
896
	if (user_alloc && (mem->userspace_addr & (PAGE_SIZE - 1)))
897
		goto out;
898
	if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
A
Avi Kivity 已提交
899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

	memslot = &kvm->memslots[mem->slot];
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

	new = old = *memslot;

	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	/* Disallow changing a memory slot's size. */
	r = -EINVAL;
	if (npages && old.npages && npages != old.npages)
919
		goto out_free;
A
Avi Kivity 已提交
920 921 922 923 924 925 926 927 928 929

	/* Check for overlaps */
	r = -EEXIST;
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *s = &kvm->memslots[i];

		if (s == memslot)
			continue;
		if (!((base_gfn + npages <= s->base_gfn) ||
		      (base_gfn >= s->base_gfn + s->npages)))
930
			goto out_free;
A
Avi Kivity 已提交
931 932 933 934
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
935
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
936 937 938 939

	r = -ENOMEM;

	/* Allocate if a slot is being created */
940
#ifndef CONFIG_S390
941
	if (npages && !new.rmap) {
M
Mike Day 已提交
942
		new.rmap = vmalloc(npages * sizeof(struct page *));
943 944

		if (!new.rmap)
945
			goto out_free;
946 947

		memset(new.rmap, 0, npages * sizeof(*new.rmap));
948

949
		new.user_alloc = user_alloc;
950 951 952 953 954 955 956 957 958
		/*
		 * hva_to_rmmap() serialzies with the mmu_lock and to be
		 * safe it has to ignore memslots with !user_alloc &&
		 * !userspace_addr.
		 */
		if (user_alloc)
			new.userspace_addr = mem->userspace_addr;
		else
			new.userspace_addr = 0;
A
Avi Kivity 已提交
959
	}
M
Marcelo Tosatti 已提交
960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978
	if (npages && !new.lpage_info) {
		int largepages = npages / KVM_PAGES_PER_HPAGE;
		if (npages % KVM_PAGES_PER_HPAGE)
			largepages++;
		if (base_gfn % KVM_PAGES_PER_HPAGE)
			largepages++;

		new.lpage_info = vmalloc(largepages * sizeof(*new.lpage_info));

		if (!new.lpage_info)
			goto out_free;

		memset(new.lpage_info, 0, largepages * sizeof(*new.lpage_info));

		if (base_gfn % KVM_PAGES_PER_HPAGE)
			new.lpage_info[0].write_count = 1;
		if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE)
			new.lpage_info[largepages-1].write_count = 1;
	}
A
Avi Kivity 已提交
979 980 981 982 983 984 985

	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
		unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;

		new.dirty_bitmap = vmalloc(dirty_bytes);
		if (!new.dirty_bitmap)
986
			goto out_free;
A
Avi Kivity 已提交
987 988
		memset(new.dirty_bitmap, 0, dirty_bytes);
	}
989
#endif /* not defined CONFIG_S390 */
A
Avi Kivity 已提交
990

991 992 993
	if (!npages)
		kvm_arch_flush_shadow(kvm);

994 995 996 997
	spin_lock(&kvm->mmu_lock);
	if (mem->slot >= kvm->nmemslots)
		kvm->nmemslots = mem->slot + 1;

998
	*memslot = new;
999
	spin_unlock(&kvm->mmu_lock);
1000

1001 1002
	r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
	if (r) {
1003
		spin_lock(&kvm->mmu_lock);
1004
		*memslot = old;
1005
		spin_unlock(&kvm->mmu_lock);
1006
		goto out_free;
1007 1008
	}

1009 1010 1011 1012
	kvm_free_physmem_slot(&old, npages ? &new : NULL);
	/* Slot deletion case: we have to update the current slot */
	if (!npages)
		*memslot = old;
1013
#ifdef CONFIG_DMAR
B
Ben-Ami Yassour 已提交
1014 1015 1016 1017
	/* map the pages in iommu page table */
	r = kvm_iommu_map_pages(kvm, base_gfn, npages);
	if (r)
		goto out;
1018
#endif
A
Avi Kivity 已提交
1019 1020
	return 0;

1021
out_free:
A
Avi Kivity 已提交
1022 1023 1024
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
1025 1026

}
1027 1028 1029 1030 1031 1032 1033 1034
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
			  struct kvm_userspace_memory_region *mem,
			  int user_alloc)
{
	int r;

1035
	down_write(&kvm->slots_lock);
1036
	r = __kvm_set_memory_region(kvm, mem, user_alloc);
1037
	up_write(&kvm->slots_lock);
1038 1039
	return r;
}
1040 1041
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

1042 1043 1044 1045
int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
				   struct
				   kvm_userspace_memory_region *mem,
				   int user_alloc)
1046
{
1047 1048
	if (mem->slot >= KVM_MEMORY_SLOTS)
		return -EINVAL;
1049
	return kvm_set_memory_region(kvm, mem, user_alloc);
A
Avi Kivity 已提交
1050 1051
}

1052 1053
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068
{
	struct kvm_memory_slot *memslot;
	int r, i;
	int n;
	unsigned long any = 0;

	r = -EINVAL;
	if (log->slot >= KVM_MEMORY_SLOTS)
		goto out;

	memslot = &kvm->memslots[log->slot];
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

1069
	n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
A
Avi Kivity 已提交
1070

1071
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
1072 1073 1074 1075 1076 1077
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

1078 1079
	if (any)
		*is_dirty = 1;
A
Avi Kivity 已提交
1080 1081 1082 1083 1084 1085

	r = 0;
out:
	return r;
}

1086 1087 1088 1089 1090 1091
int is_error_page(struct page *page)
{
	return page == bad_page;
}
EXPORT_SYMBOL_GPL(is_error_page);

1092 1093 1094 1095 1096 1097
int is_error_pfn(pfn_t pfn)
{
	return pfn == bad_pfn;
}
EXPORT_SYMBOL_GPL(is_error_pfn);

I
Izik Eidus 已提交
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108
static inline unsigned long bad_hva(void)
{
	return PAGE_OFFSET;
}

int kvm_is_error_hva(unsigned long addr)
{
	return addr == bad_hva();
}
EXPORT_SYMBOL_GPL(kvm_is_error_hva);

1109
struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return memslot;
	}
A
Al Viro 已提交
1120
	return NULL;
A
Avi Kivity 已提交
1121
}
1122
EXPORT_SYMBOL_GPL(gfn_to_memslot_unaliased);
1123 1124 1125 1126

struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	gfn = unalias_gfn(kvm, gfn);
1127
	return gfn_to_memslot_unaliased(kvm, gfn);
1128
}
A
Avi Kivity 已提交
1129

1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
	int i;

	gfn = unalias_gfn(kvm, gfn);
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

M
Marcelo Tosatti 已提交
1146
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
I
Izik Eidus 已提交
1147 1148 1149 1150
{
	struct kvm_memory_slot *slot;

	gfn = unalias_gfn(kvm, gfn);
1151
	slot = gfn_to_memslot_unaliased(kvm, gfn);
I
Izik Eidus 已提交
1152 1153 1154 1155
	if (!slot)
		return bad_hva();
	return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
}
1156
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
1157

1158
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
1159
{
1160
	struct page *page[1];
I
Izik Eidus 已提交
1161
	unsigned long addr;
1162
	int npages;
1163
	pfn_t pfn;
A
Avi Kivity 已提交
1164

1165 1166
	might_sleep();

I
Izik Eidus 已提交
1167 1168
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr)) {
1169
		get_page(bad_page);
1170
		return page_to_pfn(bad_page);
1171
	}
1172

1173
	npages = get_user_pages_fast(addr, 1, 1, page);
I
Izik Eidus 已提交
1174

1175 1176 1177
	if (unlikely(npages != 1)) {
		struct vm_area_struct *vma;

1178
		down_read(&current->mm->mmap_sem);
1179
		vma = find_vma(current->mm, addr);
1180

1181 1182
		if (vma == NULL || addr < vma->vm_start ||
		    !(vma->vm_flags & VM_PFNMAP)) {
1183
			up_read(&current->mm->mmap_sem);
1184 1185 1186 1187 1188
			get_page(bad_page);
			return page_to_pfn(bad_page);
		}

		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1189
		up_read(&current->mm->mmap_sem);
1190
		BUG_ON(!kvm_is_mmio_pfn(pfn));
1191 1192
	} else
		pfn = page_to_pfn(page[0]);
1193

1194
	return pfn;
1195 1196 1197 1198 1199 1200
}

EXPORT_SYMBOL_GPL(gfn_to_pfn);

struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
1201 1202 1203
	pfn_t pfn;

	pfn = gfn_to_pfn(kvm, gfn);
1204
	if (!kvm_is_mmio_pfn(pfn))
1205 1206
		return pfn_to_page(pfn);

1207
	WARN_ON(kvm_is_mmio_pfn(pfn));
1208 1209 1210

	get_page(bad_page);
	return bad_page;
A
Avi Kivity 已提交
1211
}
1212

A
Avi Kivity 已提交
1213 1214
EXPORT_SYMBOL_GPL(gfn_to_page);

1215 1216
void kvm_release_page_clean(struct page *page)
{
1217
	kvm_release_pfn_clean(page_to_pfn(page));
1218 1219 1220
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

1221 1222
void kvm_release_pfn_clean(pfn_t pfn)
{
1223
	if (!kvm_is_mmio_pfn(pfn))
1224
		put_page(pfn_to_page(pfn));
1225 1226 1227
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

1228
void kvm_release_page_dirty(struct page *page)
1229
{
1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

void kvm_release_pfn_dirty(pfn_t pfn)
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);

void kvm_set_page_dirty(struct page *page)
{
	kvm_set_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_set_page_dirty);

void kvm_set_pfn_dirty(pfn_t pfn)
{
1249
	if (!kvm_is_mmio_pfn(pfn)) {
1250 1251 1252 1253
		struct page *page = pfn_to_page(pfn);
		if (!PageReserved(page))
			SetPageDirty(page);
	}
1254
}
1255 1256 1257 1258
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

void kvm_set_pfn_accessed(pfn_t pfn)
{
1259
	if (!kvm_is_mmio_pfn(pfn))
1260
		mark_page_accessed(pfn_to_page(pfn));
1261 1262 1263 1264 1265
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

void kvm_get_pfn(pfn_t pfn)
{
1266
	if (!kvm_is_mmio_pfn(pfn))
1267
		get_page(pfn_to_page(pfn));
1268 1269
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
1270

1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
1282 1283
	int r;
	unsigned long addr;
1284

1285 1286 1287 1288 1289
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_from_user(data, (void __user *)addr + offset, len);
	if (r)
1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	int r;
	unsigned long addr;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int offset = offset_in_page(gpa);

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
1326
	pagefault_disable();
1327
	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1328
	pagefault_enable();
1329 1330 1331 1332 1333 1334
	if (r)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL(kvm_read_guest_atomic);

1335 1336 1337
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
1338 1339
	int r;
	unsigned long addr;
1340

1341 1342 1343 1344 1345
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_to_user((void __user *)addr + offset, data, len);
	if (r)
1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
1374
	return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

A
Avi Kivity 已提交
1397 1398
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
1399
	struct kvm_memory_slot *memslot;
A
Avi Kivity 已提交
1400

1401
	gfn = unalias_gfn(kvm, gfn);
1402
	memslot = gfn_to_memslot_unaliased(kvm, gfn);
R
Rusty Russell 已提交
1403 1404
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
1405

R
Rusty Russell 已提交
1406 1407 1408
		/* avoid RMW */
		if (!test_bit(rel_gfn, memslot->dirty_bitmap))
			set_bit(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
1409 1410 1411
	}
}

E
Eddie Dong 已提交
1412 1413 1414
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1415
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1416
{
1417 1418 1419 1420 1421
	DEFINE_WAIT(wait);

	for (;;) {
		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

1422 1423 1424 1425
		if (kvm_cpu_has_interrupt(vcpu) ||
		    kvm_cpu_has_pending_timer(vcpu) ||
		    kvm_arch_vcpu_runnable(vcpu)) {
			set_bit(KVM_REQ_UNHALT, &vcpu->requests);
1426
			break;
1427
		}
1428 1429 1430
		if (signal_pending(current))
			break;

E
Eddie Dong 已提交
1431 1432 1433 1434
		vcpu_put(vcpu);
		schedule();
		vcpu_load(vcpu);
	}
1435

1436
	finish_wait(&vcpu->wq, &wait);
E
Eddie Dong 已提交
1437 1438
}

A
Avi Kivity 已提交
1439 1440
void kvm_resched(struct kvm_vcpu *vcpu)
{
1441 1442
	if (!need_resched())
		return;
A
Avi Kivity 已提交
1443 1444 1445 1446
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

1447
static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1448 1449 1450 1451
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	struct page *page;

1452
	if (vmf->pgoff == 0)
1453
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
1454
#ifdef CONFIG_X86
1455
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1456
		page = virt_to_page(vcpu->arch.pio_data);
1457 1458 1459 1460
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
A
Avi Kivity 已提交
1461
#endif
1462
	else
1463
		return VM_FAULT_SIGBUS;
1464
	get_page(page);
1465 1466
	vmf->page = page;
	return 0;
1467 1468 1469
}

static struct vm_operations_struct kvm_vcpu_vm_ops = {
1470
	.fault = kvm_vcpu_fault,
1471 1472 1473 1474 1475 1476 1477 1478
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
1479 1480 1481 1482
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

A
Al Viro 已提交
1483
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
1484 1485 1486
	return 0;
}

1487
static struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
1488 1489 1490
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
	.compat_ioctl   = kvm_vcpu_ioctl,
1491
	.mmap           = kvm_vcpu_mmap,
A
Avi Kivity 已提交
1492 1493 1494 1495 1496 1497 1498
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
1499
	int fd = anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, 0);
A
Al Viro 已提交
1500
	if (fd < 0)
A
Al Viro 已提交
1501
		kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
1502 1503 1504
	return fd;
}

1505 1506 1507 1508 1509 1510 1511 1512 1513
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
{
	int r;
	struct kvm_vcpu *vcpu;

	if (!valid_vcpu(n))
R
Rusty Russell 已提交
1514
		return -EINVAL;
1515

1516
	vcpu = kvm_arch_vcpu_create(kvm, n);
R
Rusty Russell 已提交
1517 1518
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
1519

1520 1521
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

1522 1523
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
1524
		return r;
1525

S
Shaohua Li 已提交
1526
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
1527 1528
	if (kvm->vcpus[n]) {
		r = -EEXIST;
1529
		goto vcpu_destroy;
R
Rusty Russell 已提交
1530 1531
	}
	kvm->vcpus[n] = vcpu;
S
Shaohua Li 已提交
1532
	mutex_unlock(&kvm->lock);
1533

R
Rusty Russell 已提交
1534
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
1535
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
1536 1537
	r = create_vcpu_fd(vcpu);
	if (r < 0)
R
Rusty Russell 已提交
1538 1539
		goto unlink;
	return r;
1540

R
Rusty Russell 已提交
1541
unlink:
S
Shaohua Li 已提交
1542
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
1543
	kvm->vcpus[n] = NULL;
1544
vcpu_destroy:
1545
	mutex_unlock(&kvm->lock);
1546
	kvm_arch_vcpu_destroy(vcpu);
1547 1548 1549
	return r;
}

A
Avi Kivity 已提交
1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
1561 1562
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
1563
{
A
Avi Kivity 已提交
1564
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
1565
	void __user *argp = (void __user *)arg;
1566
	int r;
1567 1568
	struct kvm_fpu *fpu = NULL;
	struct kvm_sregs *kvm_sregs = NULL;
A
Avi Kivity 已提交
1569

1570 1571
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
1572
	switch (ioctl) {
1573
	case KVM_RUN:
1574 1575 1576
		r = -EINVAL;
		if (arg)
			goto out;
1577
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
A
Avi Kivity 已提交
1578 1579
		break;
	case KVM_GET_REGS: {
1580
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1581

1582 1583 1584
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1585
			goto out;
1586 1587 1588
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
1589
		r = -EFAULT;
1590 1591
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
1592
		r = 0;
1593 1594
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
1595 1596 1597
		break;
	}
	case KVM_SET_REGS: {
1598
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
1599

1600 1601 1602
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
1603
			goto out;
1604 1605 1606 1607
		r = -EFAULT;
		if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
			goto out_free2;
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
A
Avi Kivity 已提交
1608
		if (r)
1609
			goto out_free2;
A
Avi Kivity 已提交
1610
		r = 0;
1611 1612
out_free2:
		kfree(kvm_regs);
A
Avi Kivity 已提交
1613 1614 1615
		break;
	}
	case KVM_GET_SREGS: {
1616 1617 1618 1619 1620
		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
1621 1622 1623
		if (r)
			goto out;
		r = -EFAULT;
1624
		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
1625 1626 1627 1628 1629
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
1630 1631 1632 1633
		kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
		r = -ENOMEM;
		if (!kvm_sregs)
			goto out;
A
Avi Kivity 已提交
1634
		r = -EFAULT;
1635
		if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
A
Avi Kivity 已提交
1636
			goto out;
1637
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
A
Avi Kivity 已提交
1638 1639 1640 1641 1642
		if (r)
			goto out;
		r = 0;
		break;
	}
1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &mp_state, sizeof mp_state))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
		if (copy_from_user(&mp_state, argp, sizeof mp_state))
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1667 1668 1669 1670
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
1671
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
1672
			goto out;
1673
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
1674 1675 1676
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
1677
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
1678 1679 1680 1681 1682 1683 1684 1685
			goto out;
		r = 0;
		break;
	}
	case KVM_DEBUG_GUEST: {
		struct kvm_debug_guest dbg;

		r = -EFAULT;
A
Al Viro 已提交
1686
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
1687
			goto out;
1688
		r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
A
Avi Kivity 已提交
1689 1690 1691 1692 1693
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		break;
	}
A
Avi Kivity 已提交
1717
	case KVM_GET_FPU: {
1718 1719 1720 1721 1722
		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
A
Avi Kivity 已提交
1723 1724 1725
		if (r)
			goto out;
		r = -EFAULT;
1726
		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
1727 1728 1729 1730 1731
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
1732 1733 1734 1735
		fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
		r = -ENOMEM;
		if (!fpu)
			goto out;
A
Avi Kivity 已提交
1736
		r = -EFAULT;
1737
		if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
A
Avi Kivity 已提交
1738
			goto out;
1739
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
A
Avi Kivity 已提交
1740 1741 1742 1743 1744
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1745
	default:
1746
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
1747 1748
	}
out:
1749 1750
	kfree(fpu);
	kfree(kvm_sregs);
A
Avi Kivity 已提交
1751 1752 1753 1754 1755 1756 1757 1758
	return r;
}

static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
1759
	int r;
A
Avi Kivity 已提交
1760

1761 1762
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
1763 1764 1765 1766 1767 1768
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		if (r < 0)
			goto out;
		break;
1769 1770 1771 1772 1773 1774 1775 1776 1777
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
A
Avi Kivity 已提交
1778 1779 1780 1781 1782 1783 1784 1785
		if (r)
			goto out;
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
1786
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
1787
			goto out;
1788
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
1789 1790 1791 1792
		if (r)
			goto out;
		break;
	}
1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
	case KVM_REGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = -ENXIO;
		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_UNREGISTER_COALESCED_MMIO: {
		struct kvm_coalesced_mmio_zone zone;
		r = -EFAULT;
		if (copy_from_user(&zone, argp, sizeof zone))
			goto out;
		r = -ENXIO;
		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
		if (r)
			goto out;
		r = 0;
		break;
	}
1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841
#endif
#ifdef KVM_CAP_DEVICE_ASSIGNMENT
	case KVM_ASSIGN_PCI_DEVICE: {
		struct kvm_assigned_pci_dev assigned_dev;

		r = -EFAULT;
		if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
			goto out;
		r = kvm_vm_ioctl_assign_device(kvm, &assigned_dev);
		if (r)
			goto out;
		break;
	}
	case KVM_ASSIGN_IRQ: {
		struct kvm_assigned_irq assigned_irq;

		r = -EFAULT;
		if (copy_from_user(&assigned_irq, argp, sizeof assigned_irq))
			goto out;
		r = kvm_vm_ioctl_assign_irq(kvm, &assigned_irq);
		if (r)
			goto out;
		break;
	}
1842
#endif
1843
	default:
1844
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
1845 1846 1847 1848 1849
	}
out:
	return r;
}

1850
static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1851
{
1852 1853 1854 1855
	struct page *page[1];
	unsigned long addr;
	int npages;
	gfn_t gfn = vmf->pgoff;
1856 1857
	struct kvm *kvm = vma->vm_file->private_data;

1858 1859
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
1860
		return VM_FAULT_SIGBUS;
1861 1862 1863 1864

	npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
				NULL);
	if (unlikely(npages != 1))
1865
		return VM_FAULT_SIGBUS;
1866 1867

	vmf->page = page[0];
1868
	return 0;
1869 1870 1871
}

static struct vm_operations_struct kvm_vm_vm_ops = {
1872
	.fault = kvm_vm_fault,
1873 1874 1875 1876 1877 1878 1879 1880
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

1881
static struct file_operations kvm_vm_fops = {
1882 1883 1884 1885 1886 1887 1888 1889
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
	.compat_ioctl   = kvm_vm_ioctl,
	.mmap           = kvm_vm_mmap,
};

static int kvm_dev_ioctl_create_vm(void)
{
A
Al Viro 已提交
1890
	int fd;
1891 1892 1893
	struct kvm *kvm;

	kvm = kvm_create_vm();
1894 1895
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
1896
	fd = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, 0);
A
Al Viro 已提交
1897
	if (fd < 0)
A
Al Viro 已提交
1898
		kvm_put_kvm(kvm);
1899 1900 1901 1902 1903 1904 1905

	return fd;
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
1906
	long r = -EINVAL;
1907 1908 1909

	switch (ioctl) {
	case KVM_GET_API_VERSION:
1910 1911 1912
		r = -EINVAL;
		if (arg)
			goto out;
1913 1914 1915
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
1916 1917 1918
		r = -EINVAL;
		if (arg)
			goto out;
1919 1920
		r = kvm_dev_ioctl_create_vm();
		break;
1921
	case KVM_CHECK_EXTENSION:
1922
		r = kvm_dev_ioctl_check_extension(arg);
1923
		break;
1924 1925 1926 1927
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
1928 1929 1930
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
1931 1932 1933
#endif
#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
		r += PAGE_SIZE;    /* coalesced mmio ring page */
1934
#endif
1935
		break;
1936 1937 1938 1939 1940
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
		r = kvm_trace_ioctl(ioctl, arg);
		break;
A
Avi Kivity 已提交
1941
	default:
1942
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
1954
	KVM_MINOR,
A
Avi Kivity 已提交
1955 1956 1957 1958
	"kvm",
	&kvm_chardev_ops,
};

1959 1960 1961 1962 1963 1964 1965
static void hardware_enable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_set(cpu, cpus_hardware_enabled);
1966
	kvm_arch_hardware_enable(NULL);
1967 1968 1969 1970 1971 1972 1973 1974 1975
}

static void hardware_disable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (!cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_clear(cpu, cpus_hardware_enabled);
1976
	kvm_arch_hardware_disable(NULL);
1977 1978
}

A
Avi Kivity 已提交
1979 1980 1981 1982 1983
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

1984
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
1985
	switch (val) {
1986
	case CPU_DYING:
1987 1988 1989 1990
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
A
Avi Kivity 已提交
1991
	case CPU_UP_CANCELED:
1992 1993
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
1994
		smp_call_function_single(cpu, hardware_disable, NULL, 1);
A
Avi Kivity 已提交
1995
		break;
1996 1997 1998
	case CPU_ONLINE:
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
1999
		smp_call_function_single(cpu, hardware_enable, NULL, 1);
A
Avi Kivity 已提交
2000 2001 2002 2003 2004
		break;
	}
	return NOTIFY_OK;
}

2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016

asmlinkage void kvm_handle_fault_on_reboot(void)
{
	if (kvm_rebooting)
		/* spin while reset goes on */
		while (true)
			;
	/* Fault while not rebooting.  We want the trace. */
	BUG();
}
EXPORT_SYMBOL_GPL(kvm_handle_fault_on_reboot);

2017
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
2018
		      void *v)
2019 2020 2021 2022 2023 2024 2025
{
	if (val == SYS_RESTART) {
		/*
		 * Some (well, at least mine) BIOSes hang on reboot if
		 * in vmx root mode.
		 */
		printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2026
		kvm_rebooting = true;
2027
		on_each_cpu(hardware_disable, NULL, 1);
2028 2029 2030 2031 2032 2033 2034 2035 2036
	}
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052
void kvm_io_bus_init(struct kvm_io_bus *bus)
{
	memset(bus, 0, sizeof(*bus));
}

void kvm_io_bus_destroy(struct kvm_io_bus *bus)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		kvm_iodevice_destructor(pos);
	}
}

2053 2054
struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus,
					  gpa_t addr, int len, int is_write)
2055 2056 2057 2058 2059 2060
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

2061
		if (pos->in_range(pos, addr, len, is_write))
2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074
			return pos;
	}

	return NULL;
}

void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
{
	BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));

	bus->devs[bus->dev_count++] = dev;
}

A
Avi Kivity 已提交
2075 2076 2077 2078 2079
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
	.priority = 20, /* must be > scheduler priority */
};

2080
static int vm_stat_get(void *_offset, u64 *val)
2081 2082 2083 2084
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

2085
	*val = 0;
2086 2087
	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
2088
		*val += *(u32 *)((void *)kvm + offset);
2089
	spin_unlock(&kvm_lock);
2090
	return 0;
2091 2092 2093 2094
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");

2095
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
2096 2097 2098 2099 2100 2101
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

2102
	*val = 0;
A
Avi Kivity 已提交
2103 2104 2105
	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
R
Rusty Russell 已提交
2106 2107
			vcpu = kvm->vcpus[i];
			if (vcpu)
2108
				*val += *(u32 *)((void *)vcpu + offset);
A
Avi Kivity 已提交
2109 2110
		}
	spin_unlock(&kvm_lock);
2111
	return 0;
A
Avi Kivity 已提交
2112 2113
}

2114 2115 2116 2117 2118 2119
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");

static struct file_operations *stat_fops[] = {
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
2120

2121
static void kvm_init_debug(void)
A
Avi Kivity 已提交
2122 2123 2124
{
	struct kvm_stats_debugfs_item *p;

2125
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
A
Avi Kivity 已提交
2126
	for (p = debugfs_entries; p->name; ++p)
2127
		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
A
Avi Kivity 已提交
2128
						(void *)(long)p->offset,
2129
						stat_fops[p->kind]);
A
Avi Kivity 已提交
2130 2131 2132 2133 2134 2135 2136 2137
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
2138
	debugfs_remove(kvm_debugfs_dir);
A
Avi Kivity 已提交
2139 2140
}

2141 2142
static int kvm_suspend(struct sys_device *dev, pm_message_t state)
{
A
Avi Kivity 已提交
2143
	hardware_disable(NULL);
2144 2145 2146 2147 2148
	return 0;
}

static int kvm_resume(struct sys_device *dev)
{
A
Avi Kivity 已提交
2149
	hardware_enable(NULL);
2150 2151 2152 2153
	return 0;
}

static struct sysdev_class kvm_sysdev_class = {
2154
	.name = "kvm",
2155 2156 2157 2158 2159 2160 2161 2162 2163
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

static struct sys_device kvm_sysdev = {
	.id = 0,
	.cls = &kvm_sysdev_class,
};

2164
struct page *bad_page;
2165
pfn_t bad_pfn;
A
Avi Kivity 已提交
2166

2167 2168 2169 2170 2171 2172 2173 2174 2175 2176
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2177
	kvm_arch_vcpu_load(vcpu, cpu);
2178 2179 2180 2181 2182 2183 2184
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

2185
	kvm_arch_vcpu_put(vcpu);
2186 2187
}

2188
int kvm_init(void *opaque, unsigned int vcpu_size,
2189
		  struct module *module)
A
Avi Kivity 已提交
2190 2191
{
	int r;
Y
Yang, Sheng 已提交
2192
	int cpu;
A
Avi Kivity 已提交
2193

2194 2195
	kvm_init_debug();

2196 2197
	r = kvm_arch_init(opaque);
	if (r)
2198
		goto out_fail;
2199 2200 2201 2202 2203 2204 2205 2206

	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);

	if (bad_page == NULL) {
		r = -ENOMEM;
		goto out;
	}

2207 2208
	bad_pfn = page_to_pfn(bad_page);

2209
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
2210
	if (r < 0)
2211
		goto out_free_0;
A
Avi Kivity 已提交
2212

Y
Yang, Sheng 已提交
2213 2214
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
2215
				kvm_arch_check_processor_compat,
2216
				&r, 1);
Y
Yang, Sheng 已提交
2217
		if (r < 0)
2218
			goto out_free_1;
Y
Yang, Sheng 已提交
2219 2220
	}

2221
	on_each_cpu(hardware_enable, NULL, 1);
A
Avi Kivity 已提交
2222 2223
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
2224
		goto out_free_2;
A
Avi Kivity 已提交
2225 2226
	register_reboot_notifier(&kvm_reboot_notifier);

2227 2228
	r = sysdev_class_register(&kvm_sysdev_class);
	if (r)
2229
		goto out_free_3;
2230 2231 2232

	r = sysdev_register(&kvm_sysdev);
	if (r)
2233
		goto out_free_4;
2234

2235 2236
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
J
Joe Perches 已提交
2237 2238
					   __alignof__(struct kvm_vcpu),
					   0, NULL);
2239 2240
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
2241
		goto out_free_5;
2242 2243
	}

A
Avi Kivity 已提交
2244
	kvm_chardev_ops.owner = module;
2245 2246
	kvm_vm_fops.owner = module;
	kvm_vcpu_fops.owner = module;
A
Avi Kivity 已提交
2247 2248 2249

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
2250
		printk(KERN_ERR "kvm: misc device register failed\n");
A
Avi Kivity 已提交
2251 2252 2253
		goto out_free;
	}

2254 2255
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;
S
Sheng Yang 已提交
2256 2257 2258
#ifndef CONFIG_X86
	msi2intx = 0;
#endif
2259

2260
	return 0;
A
Avi Kivity 已提交
2261 2262

out_free:
2263
	kmem_cache_destroy(kvm_vcpu_cache);
2264
out_free_5:
2265
	sysdev_unregister(&kvm_sysdev);
2266
out_free_4:
2267
	sysdev_class_unregister(&kvm_sysdev_class);
2268
out_free_3:
A
Avi Kivity 已提交
2269
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
2270
	unregister_cpu_notifier(&kvm_cpu_notifier);
2271
out_free_2:
2272
	on_each_cpu(hardware_disable, NULL, 1);
2273
out_free_1:
2274
	kvm_arch_hardware_unsetup();
2275 2276
out_free_0:
	__free_page(bad_page);
2277
out:
2278
	kvm_arch_exit();
2279
	kvm_exit_debug();
2280
out_fail:
A
Avi Kivity 已提交
2281 2282
	return r;
}
2283
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
2284

2285
void kvm_exit(void)
A
Avi Kivity 已提交
2286
{
2287
	kvm_trace_cleanup();
A
Avi Kivity 已提交
2288
	misc_deregister(&kvm_dev);
2289
	kmem_cache_destroy(kvm_vcpu_cache);
2290 2291
	sysdev_unregister(&kvm_sysdev);
	sysdev_class_unregister(&kvm_sysdev_class);
A
Avi Kivity 已提交
2292
	unregister_reboot_notifier(&kvm_reboot_notifier);
2293
	unregister_cpu_notifier(&kvm_cpu_notifier);
2294
	on_each_cpu(hardware_disable, NULL, 1);
2295
	kvm_arch_hardware_unsetup();
2296
	kvm_arch_exit();
A
Avi Kivity 已提交
2297
	kvm_exit_debug();
2298
	__free_page(bad_page);
A
Avi Kivity 已提交
2299
}
2300
EXPORT_SYMBOL_GPL(kvm_exit);