kvm_main.c 31.5 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

#include "kvm.h"
19
#include "x86.h"
20
#include "irq.h"
A
Avi Kivity 已提交
21 22 23 24 25 26 27 28 29 30 31 32 33

#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
34
#include <linux/sysdev.h>
A
Avi Kivity 已提交
35
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
36
#include <linux/sched.h>
37 38
#include <linux/cpumask.h>
#include <linux/smp.h>
39
#include <linux/anon_inodes.h>
40
#include <linux/profile.h>
41
#include <linux/kvm_para.h>
42
#include <linux/pagemap.h>
43
#include <linux/mman.h>
A
Avi Kivity 已提交
44

A
Avi Kivity 已提交
45 46 47 48
#include <asm/processor.h>
#include <asm/io.h>
#include <asm/uaccess.h>
#include <asm/desc.h>
A
Avi Kivity 已提交
49 50 51 52

MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

53 54
DEFINE_SPINLOCK(kvm_lock);
LIST_HEAD(vm_list);
55

56 57
static cpumask_t cpus_hardware_enabled;

58 59
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
60

61 62
static __read_mostly struct preempt_ops kvm_preempt_ops;

A
Avi Kivity 已提交
63 64
static struct dentry *debugfs_dir;

A
Avi Kivity 已提交
65 66 67
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);

68 69 70 71 72
static inline int valid_vcpu(int n)
{
	return likely(n >= 0 && n < KVM_MAX_VCPUS);
}

A
Avi Kivity 已提交
73 74 75
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
76
void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
77
{
78 79
	int cpu;

A
Avi Kivity 已提交
80
	mutex_lock(&vcpu->mutex);
81 82
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
83
	kvm_arch_vcpu_load(vcpu, cpu);
84
	put_cpu();
A
Avi Kivity 已提交
85 86
}

87
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
88
{
89
	preempt_disable();
90
	kvm_arch_vcpu_put(vcpu);
91 92
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
93 94 95
	mutex_unlock(&vcpu->mutex);
}

96 97 98 99 100 101
static void ack_flush(void *_completed)
{
}

void kvm_flush_remote_tlbs(struct kvm *kvm)
{
102
	int i, cpu;
103 104 105 106
	cpumask_t cpus;
	struct kvm_vcpu *vcpu;

	cpus_clear(cpus);
R
Rusty Russell 已提交
107 108 109 110
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		vcpu = kvm->vcpus[i];
		if (!vcpu)
			continue;
111
		if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
112 113 114
			continue;
		cpu = vcpu->cpu;
		if (cpu != -1 && cpu != raw_smp_processor_id())
115
			cpu_set(cpu, cpus);
116
	}
117
	smp_call_function_mask(cpus, ack_flush, NULL, 1);
118 119
}

R
Rusty Russell 已提交
120 121 122 123 124 125 126 127 128
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
E
Eddie Dong 已提交
129
	init_waitqueue_head(&vcpu->wq);
R
Rusty Russell 已提交
130 131 132 133 134 135 136 137

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

138
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
139
	if (r < 0)
140
		goto fail_free_run;
R
Rusty Russell 已提交
141 142 143 144 145
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
146
	return r;
R
Rusty Russell 已提交
147 148 149 150 151
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
152
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
153 154 155 156
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

157
static struct kvm *kvm_create_vm(void)
A
Avi Kivity 已提交
158 159 160 161
{
	struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);

	if (!kvm)
162
		return ERR_PTR(-ENOMEM);
A
Avi Kivity 已提交
163

164
	kvm_io_bus_init(&kvm->pio_bus);
S
Shaohua Li 已提交
165
	mutex_init(&kvm->lock);
A
Avi Kivity 已提交
166
	INIT_LIST_HEAD(&kvm->active_mmu_pages);
167
	kvm_io_bus_init(&kvm->mmio_bus);
168 169 170
	spin_lock(&kvm_lock);
	list_add(&kvm->vm_list, &vm_list);
	spin_unlock(&kvm_lock);
171 172 173
	return kvm;
}

A
Avi Kivity 已提交
174 175 176 177 178 179
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
180 181
	if (!dont || free->rmap != dont->rmap)
		vfree(free->rmap);
A
Avi Kivity 已提交
182 183 184 185 186

	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
		vfree(free->dirty_bitmap);

	free->npages = 0;
A
Al Viro 已提交
187
	free->dirty_bitmap = NULL;
188
	free->rmap = NULL;
A
Avi Kivity 已提交
189 190 191 192 193 194 195
}

static void kvm_free_physmem(struct kvm *kvm)
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i)
A
Al Viro 已提交
196
		kvm_free_physmem_slot(&kvm->memslots[i], NULL);
A
Avi Kivity 已提交
197 198
}

A
Avi Kivity 已提交
199 200 201 202 203 204 205
static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
	vcpu_load(vcpu);
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

A
Avi Kivity 已提交
206 207 208 209
static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;

A
Avi Kivity 已提交
210 211 212 213
	/*
	 * Unpin any mmu pages first.
	 */
	for (i = 0; i < KVM_MAX_VCPUS; ++i)
R
Rusty Russell 已提交
214 215 216 217
		if (kvm->vcpus[i])
			kvm_unload_vcpu_mmu(kvm->vcpus[i]);
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		if (kvm->vcpus[i]) {
218
			kvm_arch_vcpu_free(kvm->vcpus[i]);
R
Rusty Russell 已提交
219 220 221 222
			kvm->vcpus[i] = NULL;
		}
	}

A
Avi Kivity 已提交
223 224
}

225 226
static void kvm_destroy_vm(struct kvm *kvm)
{
227 228 229
	spin_lock(&kvm_lock);
	list_del(&kvm->vm_list);
	spin_unlock(&kvm_lock);
230
	kvm_io_bus_destroy(&kvm->pio_bus);
231
	kvm_io_bus_destroy(&kvm->mmio_bus);
232
	kfree(kvm->vpic);
E
Eddie Dong 已提交
233
	kfree(kvm->vioapic);
A
Avi Kivity 已提交
234 235 236
	kvm_free_vcpus(kvm);
	kvm_free_physmem(kvm);
	kfree(kvm);
237 238 239 240 241 242 243
}

static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

	kvm_destroy_vm(kvm);
A
Avi Kivity 已提交
244 245 246 247 248 249 250 251
	return 0;
}

/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
252 253
 *
 * Must be called holding kvm->lock.
A
Avi Kivity 已提交
254
 */
255 256 257
int __kvm_set_memory_region(struct kvm *kvm,
			    struct kvm_userspace_memory_region *mem,
			    int user_alloc)
A
Avi Kivity 已提交
258 259 260 261 262 263 264 265 266 267 268 269 270 271
{
	int r;
	gfn_t base_gfn;
	unsigned long npages;
	unsigned long i;
	struct kvm_memory_slot *memslot;
	struct kvm_memory_slot old, new;

	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
272
	if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
A
Avi Kivity 已提交
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

	memslot = &kvm->memslots[mem->slot];
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

	new = old = *memslot;

	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	/* Disallow changing a memory slot's size. */
	r = -EINVAL;
	if (npages && old.npages && npages != old.npages)
293
		goto out_free;
A
Avi Kivity 已提交
294 295 296 297 298 299 300 301 302 303

	/* Check for overlaps */
	r = -EEXIST;
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *s = &kvm->memslots[i];

		if (s == memslot)
			continue;
		if (!((base_gfn + npages <= s->base_gfn) ||
		      (base_gfn >= s->base_gfn + s->npages)))
304
			goto out_free;
A
Avi Kivity 已提交
305 306 307 308
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
309
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
310 311 312 313

	r = -ENOMEM;

	/* Allocate if a slot is being created */
314
	if (npages && !new.rmap) {
M
Mike Day 已提交
315
		new.rmap = vmalloc(npages * sizeof(struct page *));
316 317

		if (!new.rmap)
318
			goto out_free;
319 320

		memset(new.rmap, 0, npages * sizeof(*new.rmap));
321

322
		new.user_alloc = user_alloc;
323
		if (user_alloc)
324
			new.userspace_addr = mem->userspace_addr;
325 326 327 328 329 330 331 332 333 334
		else {
			down_write(&current->mm->mmap_sem);
			new.userspace_addr = do_mmap(NULL, 0,
						     npages * PAGE_SIZE,
						     PROT_READ | PROT_WRITE,
						     MAP_SHARED | MAP_ANONYMOUS,
						     0);
			up_write(&current->mm->mmap_sem);

			if (IS_ERR((void *)new.userspace_addr))
335
				goto out_free;
A
Avi Kivity 已提交
336
		}
337 338 339 340 341 342 343 344 345 346 347 348 349
	} else {
		if (!old.user_alloc && old.rmap) {
			int ret;

			down_write(&current->mm->mmap_sem);
			ret = do_munmap(current->mm, old.userspace_addr,
					old.npages * PAGE_SIZE);
			up_write(&current->mm->mmap_sem);
			if (ret < 0)
				printk(KERN_WARNING
				       "kvm_vm_ioctl_set_memory_region: "
				       "failed to munmap memory\n");
		}
A
Avi Kivity 已提交
350 351 352 353 354 355 356 357
	}

	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
		unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;

		new.dirty_bitmap = vmalloc(dirty_bytes);
		if (!new.dirty_bitmap)
358
			goto out_free;
A
Avi Kivity 已提交
359 360 361 362 363 364
		memset(new.dirty_bitmap, 0, dirty_bytes);
	}

	if (mem->slot >= kvm->nmemslots)
		kvm->nmemslots = mem->slot + 1;

365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382
	if (!kvm->n_requested_mmu_pages) {
		unsigned int n_pages;

		if (npages) {
			n_pages = npages * KVM_PERMILLE_MMU_PAGES / 1000;
			kvm_mmu_change_mmu_pages(kvm, kvm->n_alloc_mmu_pages +
						 n_pages);
		} else {
			unsigned int nr_mmu_pages;

			n_pages = old.npages * KVM_PERMILLE_MMU_PAGES / 1000;
			nr_mmu_pages = kvm->n_alloc_mmu_pages - n_pages;
			nr_mmu_pages = max(nr_mmu_pages,
				        (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
			kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
		}
	}

A
Avi Kivity 已提交
383 384
	*memslot = new;

385 386
	kvm_mmu_slot_remove_write_access(kvm, mem->slot);
	kvm_flush_remote_tlbs(kvm);
A
Avi Kivity 已提交
387 388 389 390

	kvm_free_physmem_slot(&old, &new);
	return 0;

391
out_free:
A
Avi Kivity 已提交
392 393 394
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
395 396

}
397 398 399 400 401 402 403 404 405 406 407 408 409
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
			  struct kvm_userspace_memory_region *mem,
			  int user_alloc)
{
	int r;

	mutex_lock(&kvm->lock);
	r = __kvm_set_memory_region(kvm, mem, user_alloc);
	mutex_unlock(&kvm->lock);
	return r;
}
410 411
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

412 413 414 415
int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
				   struct
				   kvm_userspace_memory_region *mem,
				   int user_alloc)
416
{
417 418
	if (mem->slot >= KVM_MEMORY_SLOTS)
		return -EINVAL;
419
	return kvm_set_memory_region(kvm, mem, user_alloc);
A
Avi Kivity 已提交
420 421 422 423 424
}

/*
 * Get (and clear) the dirty memory log for a memory slot.
 */
425 426
static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
				      struct kvm_dirty_log *log)
A
Avi Kivity 已提交
427 428 429 430 431 432
{
	struct kvm_memory_slot *memslot;
	int r, i;
	int n;
	unsigned long any = 0;

S
Shaohua Li 已提交
433
	mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
434 435 436 437 438 439 440 441 442 443

	r = -EINVAL;
	if (log->slot >= KVM_MEMORY_SLOTS)
		goto out;

	memslot = &kvm->memslots[log->slot];
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

444
	n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
A
Avi Kivity 已提交
445

446
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
447 448 449 450 451 452
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

453 454 455 456 457 458
	/* If nothing is dirty, don't bother messing with page tables. */
	if (any) {
		kvm_mmu_slot_remove_write_access(kvm, log->slot);
		kvm_flush_remote_tlbs(kvm);
		memset(memslot->dirty_bitmap, 0, n);
	}
A
Avi Kivity 已提交
459 460 461 462

	r = 0;

out:
S
Shaohua Li 已提交
463
	mutex_unlock(&kvm->lock);
A
Avi Kivity 已提交
464 465 466
	return r;
}

467 468 469 470 471 472
int is_error_page(struct page *page)
{
	return page == bad_page;
}
EXPORT_SYMBOL_GPL(is_error_page);

I
Izik Eidus 已提交
473 474 475 476 477 478 479 480 481 482 483
static inline unsigned long bad_hva(void)
{
	return PAGE_OFFSET;
}

int kvm_is_error_hva(unsigned long addr)
{
	return addr == bad_hva();
}
EXPORT_SYMBOL_GPL(kvm_is_error_hva);

484
gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
485 486 487 488 489 490 491 492 493 494 495 496 497 498
{
	int i;
	struct kvm_mem_alias *alias;

	for (i = 0; i < kvm->naliases; ++i) {
		alias = &kvm->aliases[i];
		if (gfn >= alias->base_gfn
		    && gfn < alias->base_gfn + alias->npages)
			return alias->target_gfn + gfn - alias->base_gfn;
	}
	return gfn;
}

static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
499 500 501 502 503 504 505 506 507 508
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return memslot;
	}
A
Al Viro 已提交
509
	return NULL;
A
Avi Kivity 已提交
510
}
511 512 513 514 515 516

struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	gfn = unalias_gfn(kvm, gfn);
	return __gfn_to_memslot(kvm, gfn);
}
A
Avi Kivity 已提交
517

518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
	int i;

	gfn = unalias_gfn(kvm, gfn);
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

I
Izik Eidus 已提交
534 535 536 537 538 539 540 541 542 543 544
static unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *slot;

	gfn = unalias_gfn(kvm, gfn);
	slot = __gfn_to_memslot(kvm, gfn);
	if (!slot)
		return bad_hva();
	return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
}

545 546 547 548
/*
 * Requires current->mm->mmap_sem to be held
 */
static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
549
{
550
	struct page *page[1];
I
Izik Eidus 已提交
551
	unsigned long addr;
552
	int npages;
A
Avi Kivity 已提交
553

554 555
	might_sleep();

I
Izik Eidus 已提交
556 557
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr)) {
558
		get_page(bad_page);
559
		return bad_page;
560
	}
561

I
Izik Eidus 已提交
562 563 564
	npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page,
				NULL);

565 566 567
	if (npages != 1) {
		get_page(bad_page);
		return bad_page;
568
	}
569 570

	return page[0];
A
Avi Kivity 已提交
571
}
572 573 574 575 576 577 578 579 580 581 582 583

struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
	struct page *page;

	down_read(&current->mm->mmap_sem);
	page = __gfn_to_page(kvm, gfn);
	up_read(&current->mm->mmap_sem);

	return page;
}

A
Avi Kivity 已提交
584 585
EXPORT_SYMBOL_GPL(gfn_to_page);

586 587 588 589 590 591 592 593
void kvm_release_page(struct page *page)
{
	if (!PageReserved(page))
		SetPageDirty(page);
	put_page(page);
}
EXPORT_SYMBOL_GPL(kvm_release_page);

594 595 596 597 598 599 600 601 602 603 604
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
605 606
	int r;
	unsigned long addr;
607

608 609 610 611 612
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_from_user(data, (void __user *)addr + offset, len);
	if (r)
613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
641 642
	int r;
	unsigned long addr;
643

644 645 646 647 648
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_to_user((void __user *)addr + offset, data, len);
	if (r)
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
	void *page_virt;
	struct page *page;

	page = gfn_to_page(kvm, gfn);
681 682
	if (is_error_page(page)) {
		kvm_release_page(page);
683
		return -EFAULT;
684
	}
685 686 687 688 689
	page_virt = kmap_atomic(page, KM_USER0);

	memset(page_virt + offset, 0, len);

	kunmap_atomic(page_virt, KM_USER0);
690
	kvm_release_page(page);
691
	mark_page_dirty(kvm, gfn);
692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

A
Avi Kivity 已提交
715 716
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
717
	struct kvm_memory_slot *memslot;
A
Avi Kivity 已提交
718

719
	gfn = unalias_gfn(kvm, gfn);
R
Rusty Russell 已提交
720 721 722
	memslot = __gfn_to_memslot(kvm, gfn);
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
723

R
Rusty Russell 已提交
724 725 726
		/* avoid RMW */
		if (!test_bit(rel_gfn, memslot->dirty_bitmap))
			set_bit(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
727 728 729
	}
}

E
Eddie Dong 已提交
730 731 732
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
733
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
734
{
E
Eddie Dong 已提交
735 736 737 738 739 740 741
	DECLARE_WAITQUEUE(wait, current);

	add_wait_queue(&vcpu->wq, &wait);

	/*
	 * We will block until either an interrupt or a signal wakes us up
	 */
742 743 744 745
	while (!kvm_cpu_has_interrupt(vcpu)
	       && !signal_pending(current)
	       && vcpu->mp_state != VCPU_MP_STATE_RUNNABLE
	       && vcpu->mp_state != VCPU_MP_STATE_SIPI_RECEIVED) {
E
Eddie Dong 已提交
746 747 748 749 750
		set_current_state(TASK_INTERRUPTIBLE);
		vcpu_put(vcpu);
		schedule();
		vcpu_load(vcpu);
	}
751

752
	__set_current_state(TASK_RUNNING);
E
Eddie Dong 已提交
753 754 755
	remove_wait_queue(&vcpu->wq, &wait);
}

A
Avi Kivity 已提交
756 757
void kvm_resched(struct kvm_vcpu *vcpu)
{
758 759
	if (!need_resched())
		return;
A
Avi Kivity 已提交
760 761 762 763 764 765 766
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

/*
 * Translate a guest virtual address to a guest physical address.
 */
A
Avi Kivity 已提交
767 768
static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
A
Avi Kivity 已提交
769 770 771 772
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;

A
Avi Kivity 已提交
773
	vcpu_load(vcpu);
S
Shaohua Li 已提交
774
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
775 776 777 778 779
	gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;
S
Shaohua Li 已提交
780
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
781 782 783 784 785
	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
786 787
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
A
Avi Kivity 已提交
788 789 790
{
	if (irq->irq < 0 || irq->irq >= 256)
		return -EINVAL;
E
Eddie Dong 已提交
791 792
	if (irqchip_in_kernel(vcpu->kvm))
		return -ENXIO;
A
Avi Kivity 已提交
793
	vcpu_load(vcpu);
A
Avi Kivity 已提交
794 795 796 797 798 799 800 801 802

	set_bit(irq->irq, vcpu->irq_pending);
	set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);

	vcpu_put(vcpu);

	return 0;
}

803 804 805 806 807 808 809 810 811
static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
				    unsigned long address,
				    int *type)
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	unsigned long pgoff;
	struct page *page;

	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
812 813 814 815 816
	if (pgoff == 0)
		page = virt_to_page(vcpu->run);
	else if (pgoff == KVM_PIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->pio_data);
	else
817 818
		return NOPAGE_SIGBUS;
	get_page(page);
819 820 821
	if (type != NULL)
		*type = VM_FAULT_MINOR;

822 823 824 825 826 827 828 829 830 831 832 833 834
	return page;
}

static struct vm_operations_struct kvm_vcpu_vm_ops = {
	.nopage = kvm_vcpu_nopage,
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
835 836 837 838 839 840 841 842 843 844 845 846
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

	fput(vcpu->kvm->filp);
	return 0;
}

static struct file_operations kvm_vcpu_fops = {
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
	.compat_ioctl   = kvm_vcpu_ioctl,
847
	.mmap           = kvm_vcpu_mmap,
A
Avi Kivity 已提交
848 849 850 851 852 853 854 855 856 857 858
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
	int fd, r;
	struct inode *inode;
	struct file *file;

859 860 861 862
	r = anon_inode_getfd(&fd, &inode, &file,
			     "kvm-vcpu", &kvm_vcpu_fops, vcpu);
	if (r)
		return r;
A
Avi Kivity 已提交
863 864 865 866
	atomic_inc(&vcpu->kvm->filp->f_count);
	return fd;
}

867 868 869 870 871 872 873 874 875
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
{
	int r;
	struct kvm_vcpu *vcpu;

	if (!valid_vcpu(n))
R
Rusty Russell 已提交
876
		return -EINVAL;
877

878
	vcpu = kvm_arch_vcpu_create(kvm, n);
R
Rusty Russell 已提交
879 880
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
881

882 883
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

S
Shaohua Li 已提交
884
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
885 886
	if (kvm->vcpus[n]) {
		r = -EEXIST;
S
Shaohua Li 已提交
887
		mutex_unlock(&kvm->lock);
888
		goto vcpu_destroy;
R
Rusty Russell 已提交
889 890
	}
	kvm->vcpus[n] = vcpu;
S
Shaohua Li 已提交
891
	mutex_unlock(&kvm->lock);
892

R
Rusty Russell 已提交
893
	/* Now it's all set up, let userspace reach it */
A
Avi Kivity 已提交
894 895
	r = create_vcpu_fd(vcpu);
	if (r < 0)
R
Rusty Russell 已提交
896 897
		goto unlink;
	return r;
898

R
Rusty Russell 已提交
899
unlink:
S
Shaohua Li 已提交
900
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
901
	kvm->vcpus[n] = NULL;
S
Shaohua Li 已提交
902
	mutex_unlock(&kvm->lock);
903 904
vcpu_destroy:
	kvm_arch_vcpu_destory(vcpu);
905 906 907
	return r;
}

A
Avi Kivity 已提交
908 909 910 911 912 913 914 915 916 917 918
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
919 920
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
921
{
A
Avi Kivity 已提交
922
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
923
	void __user *argp = (void __user *)arg;
924
	int r;
A
Avi Kivity 已提交
925 926

	switch (ioctl) {
927
	case KVM_RUN:
928 929 930
		r = -EINVAL;
		if (arg)
			goto out;
931
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
A
Avi Kivity 已提交
932 933 934 935
		break;
	case KVM_GET_REGS: {
		struct kvm_regs kvm_regs;

A
Avi Kivity 已提交
936
		memset(&kvm_regs, 0, sizeof kvm_regs);
937
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
A
Avi Kivity 已提交
938 939 940
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
941
		if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
A
Avi Kivity 已提交
942 943 944 945 946 947 948 949
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_REGS: {
		struct kvm_regs kvm_regs;

		r = -EFAULT;
A
Al Viro 已提交
950
		if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
A
Avi Kivity 已提交
951
			goto out;
952
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
A
Avi Kivity 已提交
953 954 955 956 957 958 959 960
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_SREGS: {
		struct kvm_sregs kvm_sregs;

A
Avi Kivity 已提交
961
		memset(&kvm_sregs, 0, sizeof kvm_sregs);
962
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
A
Avi Kivity 已提交
963 964 965
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
966
		if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
A
Avi Kivity 已提交
967 968 969 970 971 972 973 974
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
		struct kvm_sregs kvm_sregs;

		r = -EFAULT;
A
Al Viro 已提交
975
		if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
A
Avi Kivity 已提交
976
			goto out;
977
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
A
Avi Kivity 已提交
978 979 980 981 982 983 984 985 986
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
987
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
988
			goto out;
A
Avi Kivity 已提交
989
		r = kvm_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
990 991 992
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
993
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
994 995 996 997 998 999 1000 1001
			goto out;
		r = 0;
		break;
	}
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
A
Al Viro 已提交
1002
		if (copy_from_user(&irq, argp, sizeof irq))
A
Avi Kivity 已提交
1003
			goto out;
A
Avi Kivity 已提交
1004
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
A
Avi Kivity 已提交
1005 1006 1007 1008 1009 1010 1011 1012 1013
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_DEBUG_GUEST: {
		struct kvm_debug_guest dbg;

		r = -EFAULT;
A
Al Viro 已提交
1014
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
1015
			goto out;
1016
		r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
A
Avi Kivity 已提交
1017 1018 1019 1020 1021
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		break;
	}
A
Avi Kivity 已提交
1045 1046 1047 1048
	case KVM_GET_FPU: {
		struct kvm_fpu fpu;

		memset(&fpu, 0, sizeof fpu);
1049
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu);
A
Avi Kivity 已提交
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &fpu, sizeof fpu))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
		struct kvm_fpu fpu;

		r = -EFAULT;
		if (copy_from_user(&fpu, argp, sizeof fpu))
			goto out;
1064
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu);
A
Avi Kivity 已提交
1065 1066 1067 1068 1069
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1070
	default:
1071
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
	}
out:
	return r;
}

static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
1082
	int r;
A
Avi Kivity 已提交
1083 1084 1085 1086 1087 1088 1089

	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		if (r < 0)
			goto out;
		break;
1090 1091 1092 1093 1094 1095 1096 1097 1098
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
A
Avi Kivity 已提交
1099 1100 1101 1102 1103 1104 1105 1106
		if (r)
			goto out;
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
1107
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
1108
			goto out;
1109
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
1110 1111 1112 1113
		if (r)
			goto out;
		break;
	}
1114
	default:
1115
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
	}
out:
	return r;
}

static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
				  unsigned long address,
				  int *type)
{
	struct kvm *kvm = vma->vm_file->private_data;
	unsigned long pgoff;
	struct page *page;

	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1130 1131
	if (!kvm_is_visible_gfn(kvm, pgoff))
		return NOPAGE_SIGBUS;
1132 1133
	/* current->mm->mmap_sem is already held so call lockless version */
	page = __gfn_to_page(kvm, pgoff);
1134 1135
	if (is_error_page(page)) {
		kvm_release_page(page);
1136
		return NOPAGE_SIGBUS;
1137
	}
1138 1139 1140
	if (type != NULL)
		*type = VM_FAULT_MINOR;

1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
	return page;
}

static struct vm_operations_struct kvm_vm_vm_ops = {
	.nopage = kvm_vm_nopage,
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

static struct file_operations kvm_vm_fops = {
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
	.compat_ioctl   = kvm_vm_ioctl,
	.mmap           = kvm_vm_mmap,
};

static int kvm_dev_ioctl_create_vm(void)
{
	int fd, r;
	struct inode *inode;
	struct file *file;
	struct kvm *kvm;

	kvm = kvm_create_vm();
1169 1170 1171 1172 1173 1174
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
	r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
	if (r) {
		kvm_destroy_vm(kvm);
		return r;
1175 1176
	}

A
Avi Kivity 已提交
1177
	kvm->filp = file;
1178 1179 1180 1181 1182 1183 1184 1185

	return fd;
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
1186
	long r = -EINVAL;
1187 1188 1189

	switch (ioctl) {
	case KVM_GET_API_VERSION:
1190 1191 1192
		r = -EINVAL;
		if (arg)
			goto out;
1193 1194 1195
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
1196 1197 1198
		r = -EINVAL;
		if (arg)
			goto out;
1199 1200
		r = kvm_dev_ioctl_create_vm();
		break;
1201 1202 1203 1204 1205
	case KVM_CHECK_EXTENSION: {
		int ext = (long)argp;

		switch (ext) {
		case KVM_CAP_IRQCHIP:
E
Eddie Dong 已提交
1206
		case KVM_CAP_HLT:
1207
		case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
1208
		case KVM_CAP_USER_MEMORY:
1209
		case KVM_CAP_SET_TSS_ADDR:
1210 1211 1212 1213 1214 1215
			r = 1;
			break;
		default:
			r = 0;
			break;
		}
1216
		break;
1217
	}
1218 1219 1220 1221
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
1222
		r = 2 * PAGE_SIZE;
1223
		break;
A
Avi Kivity 已提交
1224
	default:
1225
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
1237
	KVM_MINOR,
A
Avi Kivity 已提交
1238 1239 1240 1241
	"kvm",
	&kvm_chardev_ops,
};

1242 1243 1244 1245 1246 1247 1248
static void hardware_enable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_set(cpu, cpus_hardware_enabled);
1249
	kvm_arch_hardware_enable(NULL);
1250 1251 1252 1253 1254 1255 1256 1257 1258 1259
}

static void hardware_disable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (!cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_clear(cpu, cpus_hardware_enabled);
	decache_vcpus_on_cpu(cpu);
1260
	kvm_arch_hardware_disable(NULL);
1261 1262
}

A
Avi Kivity 已提交
1263 1264 1265 1266 1267
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

1268
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
1269
	switch (val) {
1270
	case CPU_DYING:
1271 1272 1273 1274
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
A
Avi Kivity 已提交
1275
	case CPU_UP_CANCELED:
1276 1277
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
1278
		smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
A
Avi Kivity 已提交
1279
		break;
1280 1281 1282
	case CPU_ONLINE:
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
1283
		smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
A
Avi Kivity 已提交
1284 1285 1286 1287 1288
		break;
	}
	return NOTIFY_OK;
}

1289
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
1290
		      void *v)
1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
{
	if (val == SYS_RESTART) {
		/*
		 * Some (well, at least mine) BIOSes hang on reboot if
		 * in vmx root mode.
		 */
		printk(KERN_INFO "kvm: exiting hardware virtualization\n");
		on_each_cpu(hardware_disable, NULL, 0, 1);
	}
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
void kvm_io_bus_init(struct kvm_io_bus *bus)
{
	memset(bus, 0, sizeof(*bus));
}

void kvm_io_bus_destroy(struct kvm_io_bus *bus)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		kvm_iodevice_destructor(pos);
	}
}

struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		if (pos->in_range(pos, addr))
			return pos;
	}

	return NULL;
}

void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
{
	BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));

	bus->devs[bus->dev_count++] = dev;
}

A
Avi Kivity 已提交
1345 1346 1347 1348 1349
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
	.priority = 20, /* must be > scheduler priority */
};

A
Avi Kivity 已提交
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
static u64 stat_get(void *_offset)
{
	unsigned offset = (long)_offset;
	u64 total = 0;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
R
Rusty Russell 已提交
1361 1362 1363
			vcpu = kvm->vcpus[i];
			if (vcpu)
				total += *(u32 *)((void *)vcpu + offset);
A
Avi Kivity 已提交
1364 1365 1366 1367 1368
		}
	spin_unlock(&kvm_lock);
	return total;
}

R
Rusty Russell 已提交
1369
DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n");
A
Avi Kivity 已提交
1370

A
Avi Kivity 已提交
1371 1372 1373 1374
static __init void kvm_init_debug(void)
{
	struct kvm_stats_debugfs_item *p;

A
Al Viro 已提交
1375
	debugfs_dir = debugfs_create_dir("kvm", NULL);
A
Avi Kivity 已提交
1376
	for (p = debugfs_entries; p->name; ++p)
A
Avi Kivity 已提交
1377 1378 1379
		p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
						(void *)(long)p->offset,
						&stat_fops);
A
Avi Kivity 已提交
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
	debugfs_remove(debugfs_dir);
}

1391 1392
static int kvm_suspend(struct sys_device *dev, pm_message_t state)
{
A
Avi Kivity 已提交
1393
	hardware_disable(NULL);
1394 1395 1396 1397 1398
	return 0;
}

static int kvm_resume(struct sys_device *dev)
{
A
Avi Kivity 已提交
1399
	hardware_enable(NULL);
1400 1401 1402 1403
	return 0;
}

static struct sysdev_class kvm_sysdev_class = {
1404
	.name = "kvm",
1405 1406 1407 1408 1409 1410 1411 1412 1413
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

static struct sys_device kvm_sysdev = {
	.id = 0,
	.cls = &kvm_sysdev_class,
};

1414
struct page *bad_page;
A
Avi Kivity 已提交
1415

1416 1417 1418 1419 1420 1421 1422 1423 1424 1425
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

1426
	kvm_arch_vcpu_load(vcpu, cpu);
1427 1428 1429 1430 1431 1432 1433
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

1434
	kvm_arch_vcpu_put(vcpu);
1435 1436
}

1437
int kvm_init(void *opaque, unsigned int vcpu_size,
1438
		  struct module *module)
A
Avi Kivity 已提交
1439 1440
{
	int r;
Y
Yang, Sheng 已提交
1441
	int cpu;
A
Avi Kivity 已提交
1442

1443 1444 1445 1446 1447 1448
	r = kvm_mmu_module_init();
	if (r)
		goto out4;

	kvm_init_debug();

1449 1450 1451
	r = kvm_arch_init(opaque);
	if (r)
		goto out4;
1452 1453 1454 1455 1456 1457 1458 1459

	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);

	if (bad_page == NULL) {
		r = -ENOMEM;
		goto out;
	}

1460
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
1461
	if (r < 0)
1462
		goto out;
A
Avi Kivity 已提交
1463

Y
Yang, Sheng 已提交
1464 1465
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
1466
				kvm_arch_check_processor_compat,
Y
Yang, Sheng 已提交
1467 1468 1469 1470 1471
				&r, 0, 1);
		if (r < 0)
			goto out_free_0;
	}

1472
	on_each_cpu(hardware_enable, NULL, 0, 1);
A
Avi Kivity 已提交
1473 1474 1475
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
		goto out_free_1;
A
Avi Kivity 已提交
1476 1477
	register_reboot_notifier(&kvm_reboot_notifier);

1478 1479 1480 1481 1482 1483 1484 1485
	r = sysdev_class_register(&kvm_sysdev_class);
	if (r)
		goto out_free_2;

	r = sysdev_register(&kvm_sysdev);
	if (r)
		goto out_free_3;

1486 1487 1488 1489 1490 1491 1492 1493
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
					   __alignof__(struct kvm_vcpu), 0, 0);
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
		goto out_free_4;
	}

A
Avi Kivity 已提交
1494 1495 1496 1497
	kvm_chardev_ops.owner = module;

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
1498
		printk(KERN_ERR "kvm: misc device register failed\n");
A
Avi Kivity 已提交
1499 1500 1501
		goto out_free;
	}

1502 1503 1504
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

1505 1506 1507
	kvm_mmu_set_nonpresent_ptes(0ull, 0ull);

	return 0;
A
Avi Kivity 已提交
1508 1509

out_free:
1510 1511
	kmem_cache_destroy(kvm_vcpu_cache);
out_free_4:
1512 1513 1514 1515
	sysdev_unregister(&kvm_sysdev);
out_free_3:
	sysdev_class_unregister(&kvm_sysdev_class);
out_free_2:
A
Avi Kivity 已提交
1516
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
1517 1518
	unregister_cpu_notifier(&kvm_cpu_notifier);
out_free_1:
1519
	on_each_cpu(hardware_disable, NULL, 0, 1);
Y
Yang, Sheng 已提交
1520
out_free_0:
1521
	kvm_arch_hardware_unsetup();
1522
out:
1523
	kvm_arch_exit();
1524 1525 1526
	kvm_exit_debug();
	kvm_mmu_module_exit();
out4:
A
Avi Kivity 已提交
1527 1528
	return r;
}
1529
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
1530

1531
void kvm_exit(void)
A
Avi Kivity 已提交
1532 1533
{
	misc_deregister(&kvm_dev);
1534
	kmem_cache_destroy(kvm_vcpu_cache);
1535 1536
	sysdev_unregister(&kvm_sysdev);
	sysdev_class_unregister(&kvm_sysdev_class);
A
Avi Kivity 已提交
1537
	unregister_reboot_notifier(&kvm_reboot_notifier);
1538
	unregister_cpu_notifier(&kvm_cpu_notifier);
1539
	on_each_cpu(hardware_disable, NULL, 0, 1);
1540
	kvm_arch_hardware_unsetup();
1541
	kvm_arch_exit();
A
Avi Kivity 已提交
1542
	kvm_exit_debug();
1543
	__free_page(bad_page);
1544
	kvm_mmu_module_exit();
A
Avi Kivity 已提交
1545
}
1546
EXPORT_SYMBOL_GPL(kvm_exit);