kvm_main.c 31.7 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

18
#include "iodev.h"
A
Avi Kivity 已提交
19

20
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
21 22 23 24 25 26 27 28 29 30 31 32
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
33
#include <linux/sysdev.h>
A
Avi Kivity 已提交
34
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
35
#include <linux/sched.h>
36 37
#include <linux/cpumask.h>
#include <linux/smp.h>
38
#include <linux/anon_inodes.h>
39
#include <linux/profile.h>
40
#include <linux/kvm_para.h>
41
#include <linux/pagemap.h>
42
#include <linux/mman.h>
43
#include <linux/swap.h>
A
Avi Kivity 已提交
44

A
Avi Kivity 已提交
45 46 47
#include <asm/processor.h>
#include <asm/io.h>
#include <asm/uaccess.h>
48
#include <asm/pgtable.h>
A
Avi Kivity 已提交
49 50 51 52

MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

53 54
DEFINE_SPINLOCK(kvm_lock);
LIST_HEAD(vm_list);
55

56 57
static cpumask_t cpus_hardware_enabled;

58 59
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
60

61 62
static __read_mostly struct preempt_ops kvm_preempt_ops;

63
struct dentry *kvm_debugfs_dir;
A
Avi Kivity 已提交
64

A
Avi Kivity 已提交
65 66 67
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);

68 69 70 71 72
static inline int valid_vcpu(int n)
{
	return likely(n >= 0 && n < KVM_MAX_VCPUS);
}

A
Avi Kivity 已提交
73 74 75
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
76
void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
77
{
78 79
	int cpu;

A
Avi Kivity 已提交
80
	mutex_lock(&vcpu->mutex);
81 82
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
83
	kvm_arch_vcpu_load(vcpu, cpu);
84
	put_cpu();
A
Avi Kivity 已提交
85 86
}

87
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
88
{
89
	preempt_disable();
90
	kvm_arch_vcpu_put(vcpu);
91 92
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
93 94 95
	mutex_unlock(&vcpu->mutex);
}

96 97 98 99 100 101
static void ack_flush(void *_completed)
{
}

void kvm_flush_remote_tlbs(struct kvm *kvm)
{
102
	int i, cpu;
103 104 105 106
	cpumask_t cpus;
	struct kvm_vcpu *vcpu;

	cpus_clear(cpus);
R
Rusty Russell 已提交
107 108 109 110
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		vcpu = kvm->vcpus[i];
		if (!vcpu)
			continue;
111
		if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
112 113 114
			continue;
		cpu = vcpu->cpu;
		if (cpu != -1 && cpu != raw_smp_processor_id())
115
			cpu_set(cpu, cpus);
116
	}
117 118 119
	if (cpus_empty(cpus))
		return;
	++kvm->stat.remote_tlb_flush;
120
	smp_call_function_mask(cpus, ack_flush, NULL, 1);
121 122
}

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
void kvm_reload_remote_mmus(struct kvm *kvm)
{
	int i, cpu;
	cpumask_t cpus;
	struct kvm_vcpu *vcpu;

	cpus_clear(cpus);
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		vcpu = kvm->vcpus[i];
		if (!vcpu)
			continue;
		if (test_and_set_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
			continue;
		cpu = vcpu->cpu;
		if (cpu != -1 && cpu != raw_smp_processor_id())
			cpu_set(cpu, cpus);
	}
	if (cpus_empty(cpus))
		return;
	smp_call_function_mask(cpus, ack_flush, NULL, 1);
}


R
Rusty Russell 已提交
146 147 148 149 150 151 152 153 154
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
E
Eddie Dong 已提交
155
	init_waitqueue_head(&vcpu->wq);
R
Rusty Russell 已提交
156 157 158 159 160 161 162 163

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

164
	r = kvm_arch_vcpu_init(vcpu);
R
Rusty Russell 已提交
165
	if (r < 0)
166
		goto fail_free_run;
R
Rusty Russell 已提交
167 168 169 170 171
	return 0;

fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
172
	return r;
R
Rusty Russell 已提交
173 174 175 176 177
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
178
	kvm_arch_vcpu_uninit(vcpu);
R
Rusty Russell 已提交
179 180 181 182
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

183
static struct kvm *kvm_create_vm(void)
A
Avi Kivity 已提交
184
{
185
	struct kvm *kvm = kvm_arch_create_vm();
A
Avi Kivity 已提交
186

187 188
	if (IS_ERR(kvm))
		goto out;
A
Avi Kivity 已提交
189

190 191
	kvm->mm = current->mm;
	atomic_inc(&kvm->mm->mm_count);
192
	spin_lock_init(&kvm->mmu_lock);
193
	kvm_io_bus_init(&kvm->pio_bus);
S
Shaohua Li 已提交
194
	mutex_init(&kvm->lock);
195
	kvm_io_bus_init(&kvm->mmio_bus);
196
	init_rwsem(&kvm->slots_lock);
I
Izik Eidus 已提交
197
	atomic_set(&kvm->users_count, 1);
198 199 200
	spin_lock(&kvm_lock);
	list_add(&kvm->vm_list, &vm_list);
	spin_unlock(&kvm_lock);
201
out:
202 203 204
	return kvm;
}

A
Avi Kivity 已提交
205 206 207 208 209 210
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
211 212
	if (!dont || free->rmap != dont->rmap)
		vfree(free->rmap);
A
Avi Kivity 已提交
213 214 215 216

	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
		vfree(free->dirty_bitmap);

M
Marcelo Tosatti 已提交
217 218 219
	if (!dont || free->lpage_info != dont->lpage_info)
		vfree(free->lpage_info);

A
Avi Kivity 已提交
220
	free->npages = 0;
A
Al Viro 已提交
221
	free->dirty_bitmap = NULL;
222
	free->rmap = NULL;
M
Marcelo Tosatti 已提交
223
	free->lpage_info = NULL;
A
Avi Kivity 已提交
224 225
}

226
void kvm_free_physmem(struct kvm *kvm)
A
Avi Kivity 已提交
227 228 229 230
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i)
A
Al Viro 已提交
231
		kvm_free_physmem_slot(&kvm->memslots[i], NULL);
A
Avi Kivity 已提交
232 233
}

234 235
static void kvm_destroy_vm(struct kvm *kvm)
{
236 237
	struct mm_struct *mm = kvm->mm;

238 239 240
	spin_lock(&kvm_lock);
	list_del(&kvm->vm_list);
	spin_unlock(&kvm_lock);
241
	kvm_io_bus_destroy(&kvm->pio_bus);
242
	kvm_io_bus_destroy(&kvm->mmio_bus);
243
	kvm_arch_destroy_vm(kvm);
244
	mmdrop(mm);
245 246
}

I
Izik Eidus 已提交
247 248 249 250 251 252 253 254 255 256 257 258 259 260
void kvm_get_kvm(struct kvm *kvm)
{
	atomic_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

void kvm_put_kvm(struct kvm *kvm)
{
	if (atomic_dec_and_test(&kvm->users_count))
		kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);


261 262 263 264
static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

I
Izik Eidus 已提交
265
	kvm_put_kvm(kvm);
A
Avi Kivity 已提交
266 267 268 269 270 271 272 273
	return 0;
}

/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
274
 *
275
 * Must be called holding mmap_sem for write.
A
Avi Kivity 已提交
276
 */
277 278 279
int __kvm_set_memory_region(struct kvm *kvm,
			    struct kvm_userspace_memory_region *mem,
			    int user_alloc)
A
Avi Kivity 已提交
280 281 282 283 284 285 286 287 288 289 290 291 292 293
{
	int r;
	gfn_t base_gfn;
	unsigned long npages;
	unsigned long i;
	struct kvm_memory_slot *memslot;
	struct kvm_memory_slot old, new;

	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
294
	if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
A
Avi Kivity 已提交
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

	memslot = &kvm->memslots[mem->slot];
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

	new = old = *memslot;

	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	/* Disallow changing a memory slot's size. */
	r = -EINVAL;
	if (npages && old.npages && npages != old.npages)
315
		goto out_free;
A
Avi Kivity 已提交
316 317 318 319 320 321 322 323 324 325

	/* Check for overlaps */
	r = -EEXIST;
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *s = &kvm->memslots[i];

		if (s == memslot)
			continue;
		if (!((base_gfn + npages <= s->base_gfn) ||
		      (base_gfn >= s->base_gfn + s->npages)))
326
			goto out_free;
A
Avi Kivity 已提交
327 328 329 330
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
331
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
332 333 334 335

	r = -ENOMEM;

	/* Allocate if a slot is being created */
336
	if (npages && !new.rmap) {
M
Mike Day 已提交
337
		new.rmap = vmalloc(npages * sizeof(struct page *));
338 339

		if (!new.rmap)
340
			goto out_free;
341 342

		memset(new.rmap, 0, npages * sizeof(*new.rmap));
343

344
		new.user_alloc = user_alloc;
345
		new.userspace_addr = mem->userspace_addr;
A
Avi Kivity 已提交
346
	}
M
Marcelo Tosatti 已提交
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
	if (npages && !new.lpage_info) {
		int largepages = npages / KVM_PAGES_PER_HPAGE;
		if (npages % KVM_PAGES_PER_HPAGE)
			largepages++;
		if (base_gfn % KVM_PAGES_PER_HPAGE)
			largepages++;

		new.lpage_info = vmalloc(largepages * sizeof(*new.lpage_info));

		if (!new.lpage_info)
			goto out_free;

		memset(new.lpage_info, 0, largepages * sizeof(*new.lpage_info));

		if (base_gfn % KVM_PAGES_PER_HPAGE)
			new.lpage_info[0].write_count = 1;
		if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE)
			new.lpage_info[largepages-1].write_count = 1;
	}
A
Avi Kivity 已提交
366 367 368 369 370 371 372

	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
		unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;

		new.dirty_bitmap = vmalloc(dirty_bytes);
		if (!new.dirty_bitmap)
373
			goto out_free;
A
Avi Kivity 已提交
374 375 376 377 378 379
		memset(new.dirty_bitmap, 0, dirty_bytes);
	}

	if (mem->slot >= kvm->nmemslots)
		kvm->nmemslots = mem->slot + 1;

380 381
	*memslot = new;

382 383 384 385
	r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
	if (r) {
		*memslot = old;
		goto out_free;
386 387
	}

A
Avi Kivity 已提交
388 389 390
	kvm_free_physmem_slot(&old, &new);
	return 0;

391
out_free:
A
Avi Kivity 已提交
392 393 394
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
395 396

}
397 398 399 400 401 402 403 404
EXPORT_SYMBOL_GPL(__kvm_set_memory_region);

int kvm_set_memory_region(struct kvm *kvm,
			  struct kvm_userspace_memory_region *mem,
			  int user_alloc)
{
	int r;

405
	down_write(&kvm->slots_lock);
406
	r = __kvm_set_memory_region(kvm, mem, user_alloc);
407
	up_write(&kvm->slots_lock);
408 409
	return r;
}
410 411
EXPORT_SYMBOL_GPL(kvm_set_memory_region);

412 413 414 415
int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
				   struct
				   kvm_userspace_memory_region *mem,
				   int user_alloc)
416
{
417 418
	if (mem->slot >= KVM_MEMORY_SLOTS)
		return -EINVAL;
419
	return kvm_set_memory_region(kvm, mem, user_alloc);
A
Avi Kivity 已提交
420 421
}

422 423
int kvm_get_dirty_log(struct kvm *kvm,
			struct kvm_dirty_log *log, int *is_dirty)
A
Avi Kivity 已提交
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
{
	struct kvm_memory_slot *memslot;
	int r, i;
	int n;
	unsigned long any = 0;

	r = -EINVAL;
	if (log->slot >= KVM_MEMORY_SLOTS)
		goto out;

	memslot = &kvm->memslots[log->slot];
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

439
	n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
A
Avi Kivity 已提交
440

441
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
442 443 444 445 446 447
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

448 449
	if (any)
		*is_dirty = 1;
A
Avi Kivity 已提交
450 451 452 453 454 455

	r = 0;
out:
	return r;
}

456 457 458 459 460 461
int is_error_page(struct page *page)
{
	return page == bad_page;
}
EXPORT_SYMBOL_GPL(is_error_page);

462 463 464 465 466 467
int is_error_pfn(pfn_t pfn)
{
	return pfn == bad_pfn;
}
EXPORT_SYMBOL_GPL(is_error_pfn);

I
Izik Eidus 已提交
468 469 470 471 472 473 474 475 476 477 478
static inline unsigned long bad_hva(void)
{
	return PAGE_OFFSET;
}

int kvm_is_error_hva(unsigned long addr)
{
	return addr == bad_hva();
}
EXPORT_SYMBOL_GPL(kvm_is_error_hva);

479
static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
480 481 482 483 484 485 486 487 488 489
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return memslot;
	}
A
Al Viro 已提交
490
	return NULL;
A
Avi Kivity 已提交
491
}
492 493 494 495 496 497

struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	gfn = unalias_gfn(kvm, gfn);
	return __gfn_to_memslot(kvm, gfn);
}
A
Avi Kivity 已提交
498

499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
	int i;

	gfn = unalias_gfn(kvm, gfn);
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

M
Marcelo Tosatti 已提交
515
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
I
Izik Eidus 已提交
516 517 518 519 520 521 522 523 524
{
	struct kvm_memory_slot *slot;

	gfn = unalias_gfn(kvm, gfn);
	slot = __gfn_to_memslot(kvm, gfn);
	if (!slot)
		return bad_hva();
	return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
}
525
EXPORT_SYMBOL_GPL(gfn_to_hva);
I
Izik Eidus 已提交
526

527 528 529
/*
 * Requires current->mm->mmap_sem to be held
 */
530
pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
531
{
532
	struct page *page[1];
I
Izik Eidus 已提交
533
	unsigned long addr;
534
	int npages;
A
Avi Kivity 已提交
535

536 537
	might_sleep();

I
Izik Eidus 已提交
538 539
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr)) {
540
		get_page(bad_page);
541
		return page_to_pfn(bad_page);
542
	}
543

I
Izik Eidus 已提交
544 545 546
	npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page,
				NULL);

547 548
	if (npages != 1) {
		get_page(bad_page);
549
		return page_to_pfn(bad_page);
550
	}
551

552 553 554 555 556 557 558 559
	return page_to_pfn(page[0]);
}

EXPORT_SYMBOL_GPL(gfn_to_pfn);

struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
	return pfn_to_page(gfn_to_pfn(kvm, gfn));
A
Avi Kivity 已提交
560
}
561

A
Avi Kivity 已提交
562 563
EXPORT_SYMBOL_GPL(gfn_to_page);

564 565
void kvm_release_page_clean(struct page *page)
{
566
	kvm_release_pfn_clean(page_to_pfn(page));
567 568 569
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

570 571 572 573 574 575
void kvm_release_pfn_clean(pfn_t pfn)
{
	put_page(pfn_to_page(pfn));
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);

576
void kvm_release_page_dirty(struct page *page)
577
{
578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
	kvm_release_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

void kvm_release_pfn_dirty(pfn_t pfn)
{
	kvm_set_pfn_dirty(pfn);
	kvm_release_pfn_clean(pfn);
}
EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);

void kvm_set_page_dirty(struct page *page)
{
	kvm_set_pfn_dirty(page_to_pfn(page));
}
EXPORT_SYMBOL_GPL(kvm_set_page_dirty);

void kvm_set_pfn_dirty(pfn_t pfn)
{
	struct page *page = pfn_to_page(pfn);
598 599 600
	if (!PageReserved(page))
		SetPageDirty(page);
}
601 602 603 604 605 606 607 608 609 610 611 612 613
EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);

void kvm_set_pfn_accessed(pfn_t pfn)
{
	mark_page_accessed(pfn_to_page(pfn));
}
EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);

void kvm_get_pfn(pfn_t pfn)
{
	get_page(pfn_to_page(pfn));
}
EXPORT_SYMBOL_GPL(kvm_get_pfn);
614

615 616 617 618 619 620 621 622 623 624 625
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
626 627
	int r;
	unsigned long addr;
628

629 630 631 632 633
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_from_user(data, (void __user *)addr + offset, len);
	if (r)
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

659 660 661 662 663 664 665 666 667 668 669
int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
			  unsigned long len)
{
	int r;
	unsigned long addr;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int offset = offset_in_page(gpa);

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
670
	pagefault_disable();
671
	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
672
	pagefault_enable();
673 674 675 676 677 678
	if (r)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL(kvm_read_guest_atomic);

679 680 681
int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
682 683
	int r;
	unsigned long addr;
684

685 686 687 688 689
	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return -EFAULT;
	r = copy_to_user((void __user *)addr + offset, data, len);
	if (r)
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717
		return -EFAULT;
	mark_page_dirty(kvm, gfn);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
718
	return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

A
Avi Kivity 已提交
741 742
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
743
	struct kvm_memory_slot *memslot;
A
Avi Kivity 已提交
744

745
	gfn = unalias_gfn(kvm, gfn);
R
Rusty Russell 已提交
746 747 748
	memslot = __gfn_to_memslot(kvm, gfn);
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
749

R
Rusty Russell 已提交
750 751 752
		/* avoid RMW */
		if (!test_bit(rel_gfn, memslot->dirty_bitmap))
			set_bit(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
753 754 755
	}
}

E
Eddie Dong 已提交
756 757 758
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
759
void kvm_vcpu_block(struct kvm_vcpu *vcpu)
760
{
761 762 763 764 765 766 767 768 769 770 771 772 773 774
	DEFINE_WAIT(wait);

	for (;;) {
		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);

		if (kvm_cpu_has_interrupt(vcpu))
			break;
		if (kvm_cpu_has_pending_timer(vcpu))
			break;
		if (kvm_arch_vcpu_runnable(vcpu))
			break;
		if (signal_pending(current))
			break;

E
Eddie Dong 已提交
775 776 777 778
		vcpu_put(vcpu);
		schedule();
		vcpu_load(vcpu);
	}
779

780
	finish_wait(&vcpu->wq, &wait);
E
Eddie Dong 已提交
781 782
}

A
Avi Kivity 已提交
783 784
void kvm_resched(struct kvm_vcpu *vcpu)
{
785 786
	if (!need_resched())
		return;
A
Avi Kivity 已提交
787 788 789 790
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

791
static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
792 793 794 795
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	struct page *page;

796
	if (vmf->pgoff == 0)
797
		page = virt_to_page(vcpu->run);
A
Avi Kivity 已提交
798
#ifdef CONFIG_X86
799
	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
800
		page = virt_to_page(vcpu->arch.pio_data);
A
Avi Kivity 已提交
801
#endif
802
	else
803
		return VM_FAULT_SIGBUS;
804
	get_page(page);
805 806
	vmf->page = page;
	return 0;
807 808 809
}

static struct vm_operations_struct kvm_vcpu_vm_ops = {
810
	.fault = kvm_vcpu_fault,
811 812 813 814 815 816 817 818
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
819 820 821 822
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

A
Al Viro 已提交
823
	kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
824 825 826
	return 0;
}

827
static const struct file_operations kvm_vcpu_fops = {
A
Avi Kivity 已提交
828 829 830
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
	.compat_ioctl   = kvm_vcpu_ioctl,
831
	.mmap           = kvm_vcpu_mmap,
A
Avi Kivity 已提交
832 833 834 835 836 837 838
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
A
Al Viro 已提交
839 840
	int fd = anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu);
	if (fd < 0)
A
Al Viro 已提交
841
		kvm_put_kvm(vcpu->kvm);
A
Avi Kivity 已提交
842 843 844
	return fd;
}

845 846 847 848 849 850 851 852 853
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
{
	int r;
	struct kvm_vcpu *vcpu;

	if (!valid_vcpu(n))
R
Rusty Russell 已提交
854
		return -EINVAL;
855

856
	vcpu = kvm_arch_vcpu_create(kvm, n);
R
Rusty Russell 已提交
857 858
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
859

860 861
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

862 863 864 865
	r = kvm_arch_vcpu_setup(vcpu);
	if (r)
		goto vcpu_destroy;

S
Shaohua Li 已提交
866
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
867 868
	if (kvm->vcpus[n]) {
		r = -EEXIST;
S
Shaohua Li 已提交
869
		mutex_unlock(&kvm->lock);
870
		goto vcpu_destroy;
R
Rusty Russell 已提交
871 872
	}
	kvm->vcpus[n] = vcpu;
S
Shaohua Li 已提交
873
	mutex_unlock(&kvm->lock);
874

R
Rusty Russell 已提交
875
	/* Now it's all set up, let userspace reach it */
A
Al Viro 已提交
876
	kvm_get_kvm(kvm);
A
Avi Kivity 已提交
877 878
	r = create_vcpu_fd(vcpu);
	if (r < 0)
R
Rusty Russell 已提交
879 880
		goto unlink;
	return r;
881

R
Rusty Russell 已提交
882
unlink:
S
Shaohua Li 已提交
883
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
884
	kvm->vcpus[n] = NULL;
S
Shaohua Li 已提交
885
	mutex_unlock(&kvm->lock);
886
vcpu_destroy:
887
	kvm_arch_vcpu_destroy(vcpu);
888 889 890
	return r;
}

A
Avi Kivity 已提交
891 892 893 894 895 896 897 898 899 900 901
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
902 903
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
904
{
A
Avi Kivity 已提交
905
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
906
	void __user *argp = (void __user *)arg;
907
	int r;
A
Avi Kivity 已提交
908

909 910
	if (vcpu->kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
911
	switch (ioctl) {
912
	case KVM_RUN:
913 914 915
		r = -EINVAL;
		if (arg)
			goto out;
916
		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
A
Avi Kivity 已提交
917 918
		break;
	case KVM_GET_REGS: {
919
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
920

921 922 923
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
924
			goto out;
925 926 927
		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
		if (r)
			goto out_free1;
A
Avi Kivity 已提交
928
		r = -EFAULT;
929 930
		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
			goto out_free1;
A
Avi Kivity 已提交
931
		r = 0;
932 933
out_free1:
		kfree(kvm_regs);
A
Avi Kivity 已提交
934 935 936
		break;
	}
	case KVM_SET_REGS: {
937
		struct kvm_regs *kvm_regs;
A
Avi Kivity 已提交
938

939 940 941
		r = -ENOMEM;
		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
		if (!kvm_regs)
A
Avi Kivity 已提交
942
			goto out;
943 944 945 946
		r = -EFAULT;
		if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
			goto out_free2;
		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
A
Avi Kivity 已提交
947
		if (r)
948
			goto out_free2;
A
Avi Kivity 已提交
949
		r = 0;
950 951
out_free2:
		kfree(kvm_regs);
A
Avi Kivity 已提交
952 953 954 955 956
		break;
	}
	case KVM_GET_SREGS: {
		struct kvm_sregs kvm_sregs;

A
Avi Kivity 已提交
957
		memset(&kvm_sregs, 0, sizeof kvm_sregs);
958
		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
A
Avi Kivity 已提交
959 960 961
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
962
		if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
A
Avi Kivity 已提交
963 964 965 966 967 968 969 970
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
		struct kvm_sregs kvm_sregs;

		r = -EFAULT;
A
Al Viro 已提交
971
		if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
A
Avi Kivity 已提交
972
			goto out;
973
		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
A
Avi Kivity 已提交
974 975 976 977 978
		if (r)
			goto out;
		r = 0;
		break;
	}
979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
	case KVM_GET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &mp_state, sizeof mp_state))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_MP_STATE: {
		struct kvm_mp_state mp_state;

		r = -EFAULT;
		if (copy_from_user(&mp_state, argp, sizeof mp_state))
			goto out;
		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1003 1004 1005 1006
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
1007
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
1008
			goto out;
1009
		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
1010 1011 1012
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
1013
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
1014 1015 1016 1017 1018 1019 1020 1021
			goto out;
		r = 0;
		break;
	}
	case KVM_DEBUG_GUEST: {
		struct kvm_debug_guest dbg;

		r = -EFAULT;
A
Al Viro 已提交
1022
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
1023
			goto out;
1024
		r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
A
Avi Kivity 已提交
1025 1026 1027 1028 1029
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		break;
	}
A
Avi Kivity 已提交
1053 1054 1055 1056
	case KVM_GET_FPU: {
		struct kvm_fpu fpu;

		memset(&fpu, 0, sizeof fpu);
1057
		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu);
A
Avi Kivity 已提交
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &fpu, sizeof fpu))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
		struct kvm_fpu fpu;

		r = -EFAULT;
		if (copy_from_user(&fpu, argp, sizeof fpu))
			goto out;
1072
		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu);
A
Avi Kivity 已提交
1073 1074 1075 1076 1077
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
1078
	default:
1079
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
	}
out:
	return r;
}

static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
1090
	int r;
A
Avi Kivity 已提交
1091

1092 1093
	if (kvm->mm != current->mm)
		return -EIO;
A
Avi Kivity 已提交
1094 1095 1096 1097 1098 1099
	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		if (r < 0)
			goto out;
		break;
1100 1101 1102 1103 1104 1105 1106 1107 1108
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
A
Avi Kivity 已提交
1109 1110 1111 1112 1113 1114 1115 1116
		if (r)
			goto out;
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
1117
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
1118
			goto out;
1119
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
1120 1121 1122 1123
		if (r)
			goto out;
		break;
	}
1124
	default:
1125
		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
1126 1127 1128 1129 1130
	}
out:
	return r;
}

1131
static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1132 1133 1134 1135
{
	struct kvm *kvm = vma->vm_file->private_data;
	struct page *page;

1136 1137
	if (!kvm_is_visible_gfn(kvm, vmf->pgoff))
		return VM_FAULT_SIGBUS;
1138
	page = gfn_to_page(kvm, vmf->pgoff);
1139
	if (is_error_page(page)) {
1140
		kvm_release_page_clean(page);
1141
		return VM_FAULT_SIGBUS;
1142
	}
1143 1144
	vmf->page = page;
	return 0;
1145 1146 1147
}

static struct vm_operations_struct kvm_vm_vm_ops = {
1148
	.fault = kvm_vm_fault,
1149 1150 1151 1152 1153 1154 1155 1156
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

1157
static const struct file_operations kvm_vm_fops = {
1158 1159 1160 1161 1162 1163 1164 1165
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
	.compat_ioctl   = kvm_vm_ioctl,
	.mmap           = kvm_vm_mmap,
};

static int kvm_dev_ioctl_create_vm(void)
{
A
Al Viro 已提交
1166
	int fd;
1167 1168 1169
	struct kvm *kvm;

	kvm = kvm_create_vm();
1170 1171
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
A
Al Viro 已提交
1172 1173
	fd = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm);
	if (fd < 0)
A
Al Viro 已提交
1174
		kvm_put_kvm(kvm);
1175 1176 1177 1178 1179 1180 1181 1182

	return fd;
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
1183
	long r = -EINVAL;
1184 1185 1186

	switch (ioctl) {
	case KVM_GET_API_VERSION:
1187 1188 1189
		r = -EINVAL;
		if (arg)
			goto out;
1190 1191 1192
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
1193 1194 1195
		r = -EINVAL;
		if (arg)
			goto out;
1196 1197
		r = kvm_dev_ioctl_create_vm();
		break;
1198 1199
	case KVM_CHECK_EXTENSION:
		r = kvm_dev_ioctl_check_extension((long)argp);
1200
		break;
1201 1202 1203 1204
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
1205 1206 1207 1208
		r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
		r += PAGE_SIZE;    /* pio data page */
#endif
1209
		break;
1210 1211 1212 1213 1214
	case KVM_TRACE_ENABLE:
	case KVM_TRACE_PAUSE:
	case KVM_TRACE_DISABLE:
		r = kvm_trace_ioctl(ioctl, arg);
		break;
A
Avi Kivity 已提交
1215
	default:
1216
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
1228
	KVM_MINOR,
A
Avi Kivity 已提交
1229 1230 1231 1232
	"kvm",
	&kvm_chardev_ops,
};

1233 1234 1235 1236 1237 1238 1239
static void hardware_enable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_set(cpu, cpus_hardware_enabled);
1240
	kvm_arch_hardware_enable(NULL);
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
}

static void hardware_disable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (!cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_clear(cpu, cpus_hardware_enabled);
	decache_vcpus_on_cpu(cpu);
1251
	kvm_arch_hardware_disable(NULL);
1252 1253
}

A
Avi Kivity 已提交
1254 1255 1256 1257 1258
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

1259
	val &= ~CPU_TASKS_FROZEN;
A
Avi Kivity 已提交
1260
	switch (val) {
1261
	case CPU_DYING:
1262 1263 1264 1265
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
A
Avi Kivity 已提交
1266
	case CPU_UP_CANCELED:
1267 1268
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
1269
		smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
A
Avi Kivity 已提交
1270
		break;
1271 1272 1273
	case CPU_ONLINE:
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
1274
		smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
A
Avi Kivity 已提交
1275 1276 1277 1278 1279
		break;
	}
	return NOTIFY_OK;
}

1280
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
1281
		      void *v)
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298
{
	if (val == SYS_RESTART) {
		/*
		 * Some (well, at least mine) BIOSes hang on reboot if
		 * in vmx root mode.
		 */
		printk(KERN_INFO "kvm: exiting hardware virtualization\n");
		on_each_cpu(hardware_disable, NULL, 0, 1);
	}
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
void kvm_io_bus_init(struct kvm_io_bus *bus)
{
	memset(bus, 0, sizeof(*bus));
}

void kvm_io_bus_destroy(struct kvm_io_bus *bus)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		kvm_iodevice_destructor(pos);
	}
}

struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		if (pos->in_range(pos, addr))
			return pos;
	}

	return NULL;
}

void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
{
	BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));

	bus->devs[bus->dev_count++] = dev;
}

A
Avi Kivity 已提交
1336 1337 1338 1339 1340
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
	.priority = 20, /* must be > scheduler priority */
};

1341
static int vm_stat_get(void *_offset, u64 *val)
1342 1343 1344 1345
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;

1346
	*val = 0;
1347 1348
	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
1349
		*val += *(u32 *)((void *)kvm + offset);
1350
	spin_unlock(&kvm_lock);
1351
	return 0;
1352 1353 1354 1355
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");

1356
static int vcpu_stat_get(void *_offset, u64 *val)
A
Avi Kivity 已提交
1357 1358 1359 1360 1361 1362
{
	unsigned offset = (long)_offset;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

1363
	*val = 0;
A
Avi Kivity 已提交
1364 1365 1366
	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
R
Rusty Russell 已提交
1367 1368
			vcpu = kvm->vcpus[i];
			if (vcpu)
1369
				*val += *(u32 *)((void *)vcpu + offset);
A
Avi Kivity 已提交
1370 1371
		}
	spin_unlock(&kvm_lock);
1372
	return 0;
A
Avi Kivity 已提交
1373 1374
}

1375 1376 1377 1378 1379 1380
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");

static struct file_operations *stat_fops[] = {
	[KVM_STAT_VCPU] = &vcpu_stat_fops,
	[KVM_STAT_VM]   = &vm_stat_fops,
};
A
Avi Kivity 已提交
1381

1382
static void kvm_init_debug(void)
A
Avi Kivity 已提交
1383 1384 1385
{
	struct kvm_stats_debugfs_item *p;

1386
	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
A
Avi Kivity 已提交
1387
	for (p = debugfs_entries; p->name; ++p)
1388
		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
A
Avi Kivity 已提交
1389
						(void *)(long)p->offset,
1390
						stat_fops[p->kind]);
A
Avi Kivity 已提交
1391 1392 1393 1394 1395 1396 1397 1398
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
1399
	debugfs_remove(kvm_debugfs_dir);
A
Avi Kivity 已提交
1400 1401
}

1402 1403
static int kvm_suspend(struct sys_device *dev, pm_message_t state)
{
A
Avi Kivity 已提交
1404
	hardware_disable(NULL);
1405 1406 1407 1408 1409
	return 0;
}

static int kvm_resume(struct sys_device *dev)
{
A
Avi Kivity 已提交
1410
	hardware_enable(NULL);
1411 1412 1413 1414
	return 0;
}

static struct sysdev_class kvm_sysdev_class = {
1415
	.name = "kvm",
1416 1417 1418 1419 1420 1421 1422 1423 1424
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

static struct sys_device kvm_sysdev = {
	.id = 0,
	.cls = &kvm_sysdev_class,
};

1425
struct page *bad_page;
1426
pfn_t bad_pfn;
A
Avi Kivity 已提交
1427

1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

1438
	kvm_arch_vcpu_load(vcpu, cpu);
1439 1440 1441 1442 1443 1444 1445
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

1446
	kvm_arch_vcpu_put(vcpu);
1447 1448
}

1449
int kvm_init(void *opaque, unsigned int vcpu_size,
1450
		  struct module *module)
A
Avi Kivity 已提交
1451 1452
{
	int r;
Y
Yang, Sheng 已提交
1453
	int cpu;
A
Avi Kivity 已提交
1454

1455 1456
	kvm_init_debug();

1457 1458
	r = kvm_arch_init(opaque);
	if (r)
1459
		goto out_fail;
1460 1461 1462 1463 1464 1465 1466 1467

	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);

	if (bad_page == NULL) {
		r = -ENOMEM;
		goto out;
	}

1468 1469
	bad_pfn = page_to_pfn(bad_page);

1470
	r = kvm_arch_hardware_setup();
A
Avi Kivity 已提交
1471
	if (r < 0)
1472
		goto out_free_0;
A
Avi Kivity 已提交
1473

Y
Yang, Sheng 已提交
1474 1475
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
1476
				kvm_arch_check_processor_compat,
Y
Yang, Sheng 已提交
1477 1478
				&r, 0, 1);
		if (r < 0)
1479
			goto out_free_1;
Y
Yang, Sheng 已提交
1480 1481
	}

1482
	on_each_cpu(hardware_enable, NULL, 0, 1);
A
Avi Kivity 已提交
1483 1484
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
1485
		goto out_free_2;
A
Avi Kivity 已提交
1486 1487
	register_reboot_notifier(&kvm_reboot_notifier);

1488 1489
	r = sysdev_class_register(&kvm_sysdev_class);
	if (r)
1490
		goto out_free_3;
1491 1492 1493

	r = sysdev_register(&kvm_sysdev);
	if (r)
1494
		goto out_free_4;
1495

1496 1497
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
J
Joe Perches 已提交
1498 1499
					   __alignof__(struct kvm_vcpu),
					   0, NULL);
1500 1501
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
1502
		goto out_free_5;
1503 1504
	}

A
Avi Kivity 已提交
1505 1506 1507 1508
	kvm_chardev_ops.owner = module;

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
1509
		printk(KERN_ERR "kvm: misc device register failed\n");
A
Avi Kivity 已提交
1510 1511 1512
		goto out_free;
	}

1513 1514 1515
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

1516
	return 0;
A
Avi Kivity 已提交
1517 1518

out_free:
1519
	kmem_cache_destroy(kvm_vcpu_cache);
1520
out_free_5:
1521
	sysdev_unregister(&kvm_sysdev);
1522
out_free_4:
1523
	sysdev_class_unregister(&kvm_sysdev_class);
1524
out_free_3:
A
Avi Kivity 已提交
1525
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
1526
	unregister_cpu_notifier(&kvm_cpu_notifier);
1527
out_free_2:
1528
	on_each_cpu(hardware_disable, NULL, 0, 1);
1529
out_free_1:
1530
	kvm_arch_hardware_unsetup();
1531 1532
out_free_0:
	__free_page(bad_page);
1533
out:
1534
	kvm_arch_exit();
1535
	kvm_exit_debug();
1536
out_fail:
A
Avi Kivity 已提交
1537 1538
	return r;
}
1539
EXPORT_SYMBOL_GPL(kvm_init);
A
Avi Kivity 已提交
1540

1541
void kvm_exit(void)
A
Avi Kivity 已提交
1542
{
1543
	kvm_trace_cleanup();
A
Avi Kivity 已提交
1544
	misc_deregister(&kvm_dev);
1545
	kmem_cache_destroy(kvm_vcpu_cache);
1546 1547
	sysdev_unregister(&kvm_sysdev);
	sysdev_class_unregister(&kvm_sysdev_class);
A
Avi Kivity 已提交
1548
	unregister_reboot_notifier(&kvm_reboot_notifier);
1549
	unregister_cpu_notifier(&kvm_cpu_notifier);
1550
	on_each_cpu(hardware_disable, NULL, 0, 1);
1551
	kvm_arch_hardware_unsetup();
1552
	kvm_arch_exit();
A
Avi Kivity 已提交
1553
	kvm_exit_debug();
1554
	__free_page(bad_page);
A
Avi Kivity 已提交
1555
}
1556
EXPORT_SYMBOL_GPL(kvm_exit);