kvm_main.c 77.9 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

#include "kvm.h"
A
Avi Kivity 已提交
19 20
#include "x86_emulate.h"
#include "segment_descriptor.h"
21
#include "irq.h"
A
Avi Kivity 已提交
22 23 24 25 26 27 28 29 30 31 32 33 34

#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
35
#include <linux/sysdev.h>
A
Avi Kivity 已提交
36
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
37
#include <linux/sched.h>
38 39
#include <linux/cpumask.h>
#include <linux/smp.h>
40
#include <linux/anon_inodes.h>
41
#include <linux/profile.h>
42
#include <linux/kvm_para.h>
A
Avi Kivity 已提交
43

A
Avi Kivity 已提交
44 45 46 47 48
#include <asm/processor.h>
#include <asm/msr.h>
#include <asm/io.h>
#include <asm/uaccess.h>
#include <asm/desc.h>
A
Avi Kivity 已提交
49 50 51 52

MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

53 54 55
static DEFINE_SPINLOCK(kvm_lock);
static LIST_HEAD(vm_list);

56 57
static cpumask_t cpus_hardware_enabled;

58
struct kvm_x86_ops *kvm_x86_ops;
59 60
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
61

62 63
static __read_mostly struct preempt_ops kvm_preempt_ops;

A
Avi Kivity 已提交
64
#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x)
A
Avi Kivity 已提交
65 66 67

static struct kvm_stats_debugfs_item {
	const char *name;
A
Avi Kivity 已提交
68
	int offset;
A
Avi Kivity 已提交
69 70
	struct dentry *dentry;
} debugfs_entries[] = {
A
Avi Kivity 已提交
71 72 73 74 75 76 77 78 79 80
	{ "pf_fixed", STAT_OFFSET(pf_fixed) },
	{ "pf_guest", STAT_OFFSET(pf_guest) },
	{ "tlb_flush", STAT_OFFSET(tlb_flush) },
	{ "invlpg", STAT_OFFSET(invlpg) },
	{ "exits", STAT_OFFSET(exits) },
	{ "io_exits", STAT_OFFSET(io_exits) },
	{ "mmio_exits", STAT_OFFSET(mmio_exits) },
	{ "signal_exits", STAT_OFFSET(signal_exits) },
	{ "irq_window", STAT_OFFSET(irq_window_exits) },
	{ "halt_exits", STAT_OFFSET(halt_exits) },
E
Eddie Dong 已提交
81
	{ "halt_wakeup", STAT_OFFSET(halt_wakeup) },
A
Avi Kivity 已提交
82 83
	{ "request_irq", STAT_OFFSET(request_irq_exits) },
	{ "irq_exits", STAT_OFFSET(irq_exits) },
84
	{ "light_exits", STAT_OFFSET(light_exits) },
85
	{ "efer_reload", STAT_OFFSET(efer_reload) },
A
Avi Kivity 已提交
86
	{ NULL }
A
Avi Kivity 已提交
87 88 89 90 91 92
};

static struct dentry *debugfs_dir;

#define MAX_IO_MSRS 256

93 94 95 96
#define CR0_RESERVED_BITS						\
	(~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
			  | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
			  | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
97 98 99 100 101 102
#define CR4_RESERVED_BITS						\
	(~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
			  | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE	\
			  | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR	\
			  | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))

103
#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
A
Avi Kivity 已提交
104 105
#define EFER_RESERVED_BITS 0xfffffffffffff2fe

106
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
107 108 109 110 111 112 113 114 115
// LDT or TSS descriptor in the GDT. 16 bytes.
struct segment_descriptor_64 {
	struct segment_descriptor s;
	u32 base_higher;
	u32 pad_zero;
};

#endif

A
Avi Kivity 已提交
116 117 118
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);

A
Avi Kivity 已提交
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
unsigned long segment_base(u16 selector)
{
	struct descriptor_table gdt;
	struct segment_descriptor *d;
	unsigned long table_base;
	typedef unsigned long ul;
	unsigned long v;

	if (selector == 0)
		return 0;

	asm ("sgdt %0" : "=m"(gdt));
	table_base = gdt.base;

	if (selector & 4) {           /* from ldt */
		u16 ldt_selector;

		asm ("sldt %0" : "=g"(ldt_selector));
		table_base = segment_base(ldt_selector);
	}
	d = (struct segment_descriptor *)(table_base + (selector & ~7));
	v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
141
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
142 143 144 145 146 147 148 149
	if (d->system == 0
	    && (d->type == 2 || d->type == 9 || d->type == 11))
		v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
#endif
	return v;
}
EXPORT_SYMBOL_GPL(segment_base);

150 151 152 153 154
static inline int valid_vcpu(int n)
{
	return likely(n >= 0 && n < KVM_MAX_VCPUS);
}

155 156 157 158 159 160
void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
	if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
		return;

	vcpu->guest_fpu_loaded = 1;
161 162
	fx_save(&vcpu->host_fx_image);
	fx_restore(&vcpu->guest_fx_image);
163 164 165 166 167 168 169 170 171
}
EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);

void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
	if (!vcpu->guest_fpu_loaded)
		return;

	vcpu->guest_fpu_loaded = 0;
172 173
	fx_save(&vcpu->guest_fx_image);
	fx_restore(&vcpu->host_fx_image);
174 175 176
}
EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);

A
Avi Kivity 已提交
177 178 179 180
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
static void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
181
{
182 183
	int cpu;

A
Avi Kivity 已提交
184
	mutex_lock(&vcpu->mutex);
185 186
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
187
	kvm_x86_ops->vcpu_load(vcpu, cpu);
188
	put_cpu();
A
Avi Kivity 已提交
189 190 191 192
}

static void vcpu_put(struct kvm_vcpu *vcpu)
{
193
	preempt_disable();
194
	kvm_x86_ops->vcpu_put(vcpu);
195 196
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
197 198 199
	mutex_unlock(&vcpu->mutex);
}

200 201 202 203 204 205
static void ack_flush(void *_completed)
{
}

void kvm_flush_remote_tlbs(struct kvm *kvm)
{
206
	int i, cpu;
207 208 209 210
	cpumask_t cpus;
	struct kvm_vcpu *vcpu;

	cpus_clear(cpus);
R
Rusty Russell 已提交
211 212 213 214
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		vcpu = kvm->vcpus[i];
		if (!vcpu)
			continue;
215 216 217 218
		if (test_and_set_bit(KVM_TLB_FLUSH, &vcpu->requests))
			continue;
		cpu = vcpu->cpu;
		if (cpu != -1 && cpu != raw_smp_processor_id())
219
			cpu_set(cpu, cpus);
220
	}
221
	smp_call_function_mask(cpus, ack_flush, NULL, 1);
222 223
}

R
Rusty Russell 已提交
224 225 226 227 228 229 230 231 232 233
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->mmu.root_hpa = INVALID_PAGE;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
234 235 236 237
	if (!irqchip_in_kernel(kvm) || id == 0)
		vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
	else
		vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED;
E
Eddie Dong 已提交
238
	init_waitqueue_head(&vcpu->wq);
R
Rusty Russell 已提交
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail_free_run;
	}
	vcpu->pio_data = page_address(page);

	r = kvm_mmu_create(vcpu);
	if (r < 0)
		goto fail_free_pio_data;

	return 0;

fail_free_pio_data:
	free_page((unsigned long)vcpu->pio_data);
fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
	return -ENOMEM;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
	kvm_mmu_destroy(vcpu);
272 273
	if (vcpu->apic)
		hrtimer_cancel(&vcpu->apic->timer.dev);
E
Eddie Dong 已提交
274
	kvm_free_apic(vcpu->apic);
R
Rusty Russell 已提交
275 276 277 278 279
	free_page((unsigned long)vcpu->pio_data);
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

280
static struct kvm *kvm_create_vm(void)
A
Avi Kivity 已提交
281 282 283 284
{
	struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);

	if (!kvm)
285
		return ERR_PTR(-ENOMEM);
A
Avi Kivity 已提交
286

287
	kvm_io_bus_init(&kvm->pio_bus);
S
Shaohua Li 已提交
288
	mutex_init(&kvm->lock);
A
Avi Kivity 已提交
289
	INIT_LIST_HEAD(&kvm->active_mmu_pages);
290
	kvm_io_bus_init(&kvm->mmio_bus);
291 292 293
	spin_lock(&kvm_lock);
	list_add(&kvm->vm_list, &vm_list);
	spin_unlock(&kvm_lock);
294 295 296
	return kvm;
}

A
Avi Kivity 已提交
297 298 299 300 301 302 303 304 305 306 307
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
	int i;

	if (!dont || free->phys_mem != dont->phys_mem)
		if (free->phys_mem) {
			for (i = 0; i < free->npages; ++i)
A
Avi Kivity 已提交
308 309
				if (free->phys_mem[i])
					__free_page(free->phys_mem[i]);
A
Avi Kivity 已提交
310 311 312 313 314 315
			vfree(free->phys_mem);
		}

	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
		vfree(free->dirty_bitmap);

A
Al Viro 已提交
316
	free->phys_mem = NULL;
A
Avi Kivity 已提交
317
	free->npages = 0;
A
Al Viro 已提交
318
	free->dirty_bitmap = NULL;
A
Avi Kivity 已提交
319 320 321 322 323 324 325
}

static void kvm_free_physmem(struct kvm *kvm)
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i)
A
Al Viro 已提交
326
		kvm_free_physmem_slot(&kvm->memslots[i], NULL);
A
Avi Kivity 已提交
327 328
}

329 330 331 332
static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
{
	int i;

R
Rusty Russell 已提交
333
	for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i)
334 335 336 337 338 339
		if (vcpu->pio.guest_pages[i]) {
			__free_page(vcpu->pio.guest_pages[i]);
			vcpu->pio.guest_pages[i] = NULL;
		}
}

A
Avi Kivity 已提交
340 341 342 343 344 345 346
static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
	vcpu_load(vcpu);
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

A
Avi Kivity 已提交
347 348 349 350
static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;

A
Avi Kivity 已提交
351 352 353 354
	/*
	 * Unpin any mmu pages first.
	 */
	for (i = 0; i < KVM_MAX_VCPUS; ++i)
R
Rusty Russell 已提交
355 356 357 358
		if (kvm->vcpus[i])
			kvm_unload_vcpu_mmu(kvm->vcpus[i]);
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		if (kvm->vcpus[i]) {
359
			kvm_x86_ops->vcpu_free(kvm->vcpus[i]);
R
Rusty Russell 已提交
360 361 362 363
			kvm->vcpus[i] = NULL;
		}
	}

A
Avi Kivity 已提交
364 365
}

366 367
static void kvm_destroy_vm(struct kvm *kvm)
{
368 369 370
	spin_lock(&kvm_lock);
	list_del(&kvm->vm_list);
	spin_unlock(&kvm_lock);
371
	kvm_io_bus_destroy(&kvm->pio_bus);
372
	kvm_io_bus_destroy(&kvm->mmio_bus);
373
	kfree(kvm->vpic);
E
Eddie Dong 已提交
374
	kfree(kvm->vioapic);
A
Avi Kivity 已提交
375 376 377
	kvm_free_vcpus(kvm);
	kvm_free_physmem(kvm);
	kfree(kvm);
378 379 380 381 382 383 384
}

static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

	kvm_destroy_vm(kvm);
A
Avi Kivity 已提交
385 386 387 388 389
	return 0;
}

static void inject_gp(struct kvm_vcpu *vcpu)
{
390
	kvm_x86_ops->inject_gp(vcpu, 0);
A
Avi Kivity 已提交
391 392
}

393 394 395 396
/*
 * Load the pae pdptrs.  Return true is they are all valid.
 */
static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
A
Avi Kivity 已提交
397 398
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
399
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
A
Avi Kivity 已提交
400 401
	int i;
	u64 *pdpt;
402
	int ret;
A
Avi Kivity 已提交
403
	struct page *page;
R
Rusty Russell 已提交
404
	u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)];
A
Avi Kivity 已提交
405

S
Shaohua Li 已提交
406
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
407
	page = gfn_to_page(vcpu->kvm, pdpt_gfn);
R
Rusty Russell 已提交
408 409 410 411 412
	if (!page) {
		ret = 0;
		goto out;
	}

A
Avi Kivity 已提交
413
	pdpt = kmap_atomic(page, KM_USER0);
R
Rusty Russell 已提交
414 415
	memcpy(pdpte, pdpt+offset, sizeof(pdpte));
	kunmap_atomic(pdpt, KM_USER0);
A
Avi Kivity 已提交
416

R
Rusty Russell 已提交
417 418
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
		if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) {
419 420 421
			ret = 0;
			goto out;
		}
A
Avi Kivity 已提交
422
	}
R
Rusty Russell 已提交
423
	ret = 1;
A
Avi Kivity 已提交
424

R
Rusty Russell 已提交
425
	memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs));
426
out:
S
Shaohua Li 已提交
427
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
428

429
	return ret;
A
Avi Kivity 已提交
430 431 432 433
}

void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
434
	if (cr0 & CR0_RESERVED_BITS) {
A
Avi Kivity 已提交
435 436 437 438 439 440
		printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
		       cr0, vcpu->cr0);
		inject_gp(vcpu);
		return;
	}

441
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
A
Avi Kivity 已提交
442 443 444 445 446
		printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
		inject_gp(vcpu);
		return;
	}

447
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
A
Avi Kivity 已提交
448 449 450 451 452 453
		printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
		       "and a clear PE flag\n");
		inject_gp(vcpu);
		return;
	}

454
	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
455
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
456 457 458 459 460 461 462 463 464
		if ((vcpu->shadow_efer & EFER_LME)) {
			int cs_db, cs_l;

			if (!is_pae(vcpu)) {
				printk(KERN_DEBUG "set_cr0: #GP, start paging "
				       "in long mode while PAE is disabled\n");
				inject_gp(vcpu);
				return;
			}
465
			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
A
Avi Kivity 已提交
466 467 468 469 470 471 472 473 474
			if (cs_l) {
				printk(KERN_DEBUG "set_cr0: #GP, start paging "
				       "in long mode while CS.L == 1\n");
				inject_gp(vcpu);
				return;

			}
		} else
#endif
475
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
A
Avi Kivity 已提交
476 477 478 479 480 481 482 483
			printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
			       "reserved bits\n");
			inject_gp(vcpu);
			return;
		}

	}

484
	kvm_x86_ops->set_cr0(vcpu, cr0);
A
Avi Kivity 已提交
485 486
	vcpu->cr0 = cr0;

S
Shaohua Li 已提交
487
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
488
	kvm_mmu_reset_context(vcpu);
S
Shaohua Li 已提交
489
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
490 491 492 493 494 495 496 497 498 499 500 501
	return;
}
EXPORT_SYMBOL_GPL(set_cr0);

void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
{
	set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
}
EXPORT_SYMBOL_GPL(lmsw);

void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
502
	if (cr4 & CR4_RESERVED_BITS) {
A
Avi Kivity 已提交
503 504 505 506 507
		printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
		inject_gp(vcpu);
		return;
	}

A
Avi Kivity 已提交
508
	if (is_long_mode(vcpu)) {
509
		if (!(cr4 & X86_CR4_PAE)) {
A
Avi Kivity 已提交
510 511 512 513 514
			printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
			       "in long mode\n");
			inject_gp(vcpu);
			return;
		}
515
	} else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE)
516
		   && !load_pdptrs(vcpu, vcpu->cr3)) {
A
Avi Kivity 已提交
517 518
		printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
		inject_gp(vcpu);
519
		return;
A
Avi Kivity 已提交
520 521
	}

522
	if (cr4 & X86_CR4_VMXE) {
A
Avi Kivity 已提交
523 524 525 526
		printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
		inject_gp(vcpu);
		return;
	}
527
	kvm_x86_ops->set_cr4(vcpu, cr4);
R
Rusty Russell 已提交
528
	vcpu->cr4 = cr4;
S
Shaohua Li 已提交
529
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
530
	kvm_mmu_reset_context(vcpu);
S
Shaohua Li 已提交
531
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
532 533 534 535 536
}
EXPORT_SYMBOL_GPL(set_cr4);

void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
{
A
Avi Kivity 已提交
537
	if (is_long_mode(vcpu)) {
538
		if (cr3 & CR3_L_MODE_RESERVED_BITS) {
A
Avi Kivity 已提交
539 540 541 542 543
			printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
			inject_gp(vcpu);
			return;
		}
	} else {
544 545 546 547 548 549 550 551 552 553 554 555 556
		if (is_pae(vcpu)) {
			if (cr3 & CR3_PAE_RESERVED_BITS) {
				printk(KERN_DEBUG
				       "set_cr3: #GP, reserved bits\n");
				inject_gp(vcpu);
				return;
			}
			if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
				printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
				       "reserved bits\n");
				inject_gp(vcpu);
				return;
			}
A
Avi Kivity 已提交
557
		}
558 559 560 561
		/*
		 * We don't check reserved bits in nonpae mode, because
		 * this isn't enforced, and VMware depends on this.
		 */
A
Avi Kivity 已提交
562 563
	}

S
Shaohua Li 已提交
564
	mutex_lock(&vcpu->kvm->lock);
565 566 567 568 569 570 571 572 573 574 575
	/*
	 * Does the new cr3 value map to physical memory? (Note, we
	 * catch an invalid cr3 even in real-mode, because it would
	 * cause trouble later on when we turn on paging anyway.)
	 *
	 * A real CPU would silently accept an invalid cr3 and would
	 * attempt to use it - with largely undefined (and often hard
	 * to debug) behavior on the guest side.
	 */
	if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
		inject_gp(vcpu);
576 577
	else {
		vcpu->cr3 = cr3;
578
		vcpu->mmu.new_cr3(vcpu);
579
	}
S
Shaohua Li 已提交
580
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
581 582 583 584 585
}
EXPORT_SYMBOL_GPL(set_cr3);

void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
{
586
	if (cr8 & CR8_RESERVED_BITS) {
A
Avi Kivity 已提交
587 588 589 590
		printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
		inject_gp(vcpu);
		return;
	}
E
Eddie Dong 已提交
591 592 593 594
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_lapic_set_tpr(vcpu, cr8);
	else
		vcpu->cr8 = cr8;
A
Avi Kivity 已提交
595 596 597
}
EXPORT_SYMBOL_GPL(set_cr8);

598 599
unsigned long get_cr8(struct kvm_vcpu *vcpu)
{
E
Eddie Dong 已提交
600 601 602 603
	if (irqchip_in_kernel(vcpu->kvm))
		return kvm_lapic_get_cr8(vcpu);
	else
		return vcpu->cr8;
604 605 606 607 608
}
EXPORT_SYMBOL_GPL(get_cr8);

u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
E
Eddie Dong 已提交
609 610 611 612
	if (irqchip_in_kernel(vcpu->kvm))
		return vcpu->apic_base;
	else
		return vcpu->apic_base;
613 614 615 616 617
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
{
E
Eddie Dong 已提交
618 619 620 621 622
	/* TODO: reserve bits check */
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_lapic_set_base(vcpu, data);
	else
		vcpu->apic_base = data;
623 624 625
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

A
Avi Kivity 已提交
626 627
void fx_init(struct kvm_vcpu *vcpu)
{
628
	unsigned after_mxcsr_mask;
A
Avi Kivity 已提交
629

630 631
	/* Initialize guest FPU by resetting ours and saving into guest's */
	preempt_disable();
632
	fx_save(&vcpu->host_fx_image);
A
Avi Kivity 已提交
633
	fpu_init();
634 635
	fx_save(&vcpu->guest_fx_image);
	fx_restore(&vcpu->host_fx_image);
636
	preempt_enable();
A
Avi Kivity 已提交
637

638
	vcpu->cr0 |= X86_CR0_ET;
639 640 641 642
	after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
	vcpu->guest_fx_image.mxcsr = 0x1f80;
	memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask,
	       0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
A
Avi Kivity 已提交
643 644 645 646 647 648 649 650 651
}
EXPORT_SYMBOL_GPL(fx_init);

/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
 */
652 653
static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
					  struct kvm_memory_region *mem)
A
Avi Kivity 已提交
654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679
{
	int r;
	gfn_t base_gfn;
	unsigned long npages;
	unsigned long i;
	struct kvm_memory_slot *memslot;
	struct kvm_memory_slot old, new;

	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
	if (mem->slot >= KVM_MEMORY_SLOTS)
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

	memslot = &kvm->memslots[mem->slot];
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

S
Shaohua Li 已提交
680
	mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706

	new = old = *memslot;

	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	/* Disallow changing a memory slot's size. */
	r = -EINVAL;
	if (npages && old.npages && npages != old.npages)
		goto out_unlock;

	/* Check for overlaps */
	r = -EEXIST;
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *s = &kvm->memslots[i];

		if (s == memslot)
			continue;
		if (!((base_gfn + npages <= s->base_gfn) ||
		      (base_gfn >= s->base_gfn + s->npages)))
			goto out_unlock;
	}

	/* Deallocate if slot is being removed */
	if (!npages)
A
Al Viro 已提交
707
		new.phys_mem = NULL;
A
Avi Kivity 已提交
708 709 710

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
711
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
712 713 714 715 716 717 718 719

	r = -ENOMEM;

	/* Allocate if a slot is being created */
	if (npages && !new.phys_mem) {
		new.phys_mem = vmalloc(npages * sizeof(struct page *));

		if (!new.phys_mem)
L
Laurent Vivier 已提交
720
			goto out_unlock;
A
Avi Kivity 已提交
721 722 723 724 725 726

		memset(new.phys_mem, 0, npages * sizeof(struct page *));
		for (i = 0; i < npages; ++i) {
			new.phys_mem[i] = alloc_page(GFP_HIGHUSER
						     | __GFP_ZERO);
			if (!new.phys_mem[i])
L
Laurent Vivier 已提交
727
				goto out_unlock;
728
			set_page_private(new.phys_mem[i],0);
A
Avi Kivity 已提交
729 730 731 732 733 734 735 736 737
		}
	}

	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
		unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;

		new.dirty_bitmap = vmalloc(dirty_bytes);
		if (!new.dirty_bitmap)
L
Laurent Vivier 已提交
738
			goto out_unlock;
A
Avi Kivity 已提交
739 740 741 742 743 744 745 746
		memset(new.dirty_bitmap, 0, dirty_bytes);
	}

	if (mem->slot >= kvm->nmemslots)
		kvm->nmemslots = mem->slot + 1;

	*memslot = new;

747 748
	kvm_mmu_slot_remove_write_access(kvm, mem->slot);
	kvm_flush_remote_tlbs(kvm);
A
Avi Kivity 已提交
749

S
Shaohua Li 已提交
750
	mutex_unlock(&kvm->lock);
A
Avi Kivity 已提交
751 752 753 754 755

	kvm_free_physmem_slot(&old, &new);
	return 0;

out_unlock:
S
Shaohua Li 已提交
756
	mutex_unlock(&kvm->lock);
A
Avi Kivity 已提交
757 758 759 760 761 762 763 764
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
}

/*
 * Get (and clear) the dirty memory log for a memory slot.
 */
765 766
static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
				      struct kvm_dirty_log *log)
A
Avi Kivity 已提交
767 768 769 770 771 772
{
	struct kvm_memory_slot *memslot;
	int r, i;
	int n;
	unsigned long any = 0;

S
Shaohua Li 已提交
773
	mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
774 775 776 777 778 779 780 781 782 783

	r = -EINVAL;
	if (log->slot >= KVM_MEMORY_SLOTS)
		goto out;

	memslot = &kvm->memslots[log->slot];
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

784
	n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
A
Avi Kivity 已提交
785

786
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
787 788 789 790 791 792
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

793 794 795 796 797 798
	/* If nothing is dirty, don't bother messing with page tables. */
	if (any) {
		kvm_mmu_slot_remove_write_access(kvm, log->slot);
		kvm_flush_remote_tlbs(kvm);
		memset(memslot->dirty_bitmap, 0, n);
	}
A
Avi Kivity 已提交
799 800 801 802

	r = 0;

out:
S
Shaohua Li 已提交
803
	mutex_unlock(&kvm->lock);
A
Avi Kivity 已提交
804 805 806
	return r;
}

807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832
/*
 * Set a new alias region.  Aliases map a portion of physical memory into
 * another portion.  This is useful for memory windows, for example the PC
 * VGA region.
 */
static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
					 struct kvm_memory_alias *alias)
{
	int r, n;
	struct kvm_mem_alias *p;

	r = -EINVAL;
	/* General sanity checks */
	if (alias->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (alias->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
	if (alias->slot >= KVM_ALIAS_SLOTS)
		goto out;
	if (alias->guest_phys_addr + alias->memory_size
	    < alias->guest_phys_addr)
		goto out;
	if (alias->target_phys_addr + alias->memory_size
	    < alias->target_phys_addr)
		goto out;

S
Shaohua Li 已提交
833
	mutex_lock(&kvm->lock);
834 835 836 837 838 839 840 841 842 843 844

	p = &kvm->aliases[alias->slot];
	p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
	p->npages = alias->memory_size >> PAGE_SHIFT;
	p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;

	for (n = KVM_ALIAS_SLOTS; n > 0; --n)
		if (kvm->aliases[n - 1].npages)
			break;
	kvm->naliases = n;

845
	kvm_mmu_zap_all(kvm);
846

S
Shaohua Li 已提交
847
	mutex_unlock(&kvm->lock);
848 849 850 851 852 853 854

	return 0;

out:
	return r;
}

855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870
static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
		memcpy (&chip->chip.pic,
			&pic_irqchip(kvm)->pics[0],
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
		memcpy (&chip->chip.pic,
			&pic_irqchip(kvm)->pics[1],
			sizeof(struct kvm_pic_state));
		break;
871 872 873 874 875
	case KVM_IRQCHIP_IOAPIC:
		memcpy (&chip->chip.ioapic,
			ioapic_irqchip(kvm),
			sizeof(struct kvm_ioapic_state));
		break;
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
		memcpy (&pic_irqchip(kvm)->pics[0],
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
		memcpy (&pic_irqchip(kvm)->pics[1],
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
		break;
899 900 901 902 903
	case KVM_IRQCHIP_IOAPIC:
		memcpy (ioapic_irqchip(kvm),
			&chip->chip.ioapic,
			sizeof(struct kvm_ioapic_state));
		break;
904 905 906 907 908 909 910 911
	default:
		r = -EINVAL;
		break;
	}
	kvm_pic_update_irq(pic_irqchip(kvm));
	return r;
}

912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
static gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
{
	int i;
	struct kvm_mem_alias *alias;

	for (i = 0; i < kvm->naliases; ++i) {
		alias = &kvm->aliases[i];
		if (gfn >= alias->base_gfn
		    && gfn < alias->base_gfn + alias->npages)
			return alias->target_gfn + gfn - alias->base_gfn;
	}
	return gfn;
}

static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
927 928 929 930 931 932 933 934 935 936
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return memslot;
	}
A
Al Viro 已提交
937
	return NULL;
A
Avi Kivity 已提交
938
}
939 940 941 942 943 944

struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	gfn = unalias_gfn(kvm, gfn);
	return __gfn_to_memslot(kvm, gfn);
}
A
Avi Kivity 已提交
945

A
Avi Kivity 已提交
946 947 948 949
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *slot;

950 951
	gfn = unalias_gfn(kvm, gfn);
	slot = __gfn_to_memslot(kvm, gfn);
A
Avi Kivity 已提交
952 953 954 955 956 957
	if (!slot)
		return NULL;
	return slot->phys_mem[gfn - slot->base_gfn];
}
EXPORT_SYMBOL_GPL(gfn_to_page);

R
Rusty Russell 已提交
958
/* WARNING: Does not work on aliased pages. */
A
Avi Kivity 已提交
959 960
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
961
	struct kvm_memory_slot *memslot;
A
Avi Kivity 已提交
962

R
Rusty Russell 已提交
963 964 965
	memslot = __gfn_to_memslot(kvm, gfn);
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
966

R
Rusty Russell 已提交
967 968 969
		/* avoid RMW */
		if (!test_bit(rel_gfn, memslot->dirty_bitmap))
			set_bit(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
970 971 972
	}
}

973
int emulator_read_std(unsigned long addr,
974
			     void *val,
A
Avi Kivity 已提交
975
			     unsigned int bytes,
976
			     struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
977 978 979 980 981 982 983 984
{
	void *data = val;

	while (bytes) {
		gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
		unsigned long pfn;
A
Avi Kivity 已提交
985 986
		struct page *page;
		void *page_virt;
A
Avi Kivity 已提交
987 988 989 990

		if (gpa == UNMAPPED_GVA)
			return X86EMUL_PROPAGATE_FAULT;
		pfn = gpa >> PAGE_SHIFT;
A
Avi Kivity 已提交
991 992
		page = gfn_to_page(vcpu->kvm, pfn);
		if (!page)
A
Avi Kivity 已提交
993
			return X86EMUL_UNHANDLEABLE;
A
Avi Kivity 已提交
994
		page_virt = kmap_atomic(page, KM_USER0);
A
Avi Kivity 已提交
995

A
Avi Kivity 已提交
996
		memcpy(data, page_virt + offset, tocopy);
A
Avi Kivity 已提交
997

A
Avi Kivity 已提交
998
		kunmap_atomic(page_virt, KM_USER0);
A
Avi Kivity 已提交
999 1000 1001 1002 1003 1004 1005 1006

		bytes -= tocopy;
		data += tocopy;
		addr += tocopy;
	}

	return X86EMUL_CONTINUE;
}
1007
EXPORT_SYMBOL_GPL(emulator_read_std);
A
Avi Kivity 已提交
1008 1009

static int emulator_write_std(unsigned long addr,
1010
			      const void *val,
A
Avi Kivity 已提交
1011
			      unsigned int bytes,
1012
			      struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1013
{
1014
	pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes);
A
Avi Kivity 已提交
1015 1016 1017
	return X86EMUL_UNHANDLEABLE;
}

E
Eddie Dong 已提交
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
/*
 * Only apic need an MMIO device hook, so shortcut now..
 */
static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu,
						gpa_t addr)
{
	struct kvm_io_device *dev;

	if (vcpu->apic) {
		dev = &vcpu->apic->dev;
		if (dev->in_range(dev, addr))
			return dev;
	}
	return NULL;
}

1034 1035 1036
static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
						gpa_t addr)
{
E
Eddie Dong 已提交
1037 1038 1039 1040 1041 1042
	struct kvm_io_device *dev;

	dev = vcpu_find_pervcpu_dev(vcpu, addr);
	if (dev == NULL)
		dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr);
	return dev;
1043 1044
}

1045 1046 1047 1048 1049 1050
static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
					       gpa_t addr)
{
	return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr);
}

A
Avi Kivity 已提交
1051
static int emulator_read_emulated(unsigned long addr,
1052
				  void *val,
A
Avi Kivity 已提交
1053
				  unsigned int bytes,
1054
				  struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1055
{
1056 1057
	struct kvm_io_device *mmio_dev;
	gpa_t                 gpa;
A
Avi Kivity 已提交
1058 1059 1060 1061 1062

	if (vcpu->mmio_read_completed) {
		memcpy(val, vcpu->mmio_data, bytes);
		vcpu->mmio_read_completed = 0;
		return X86EMUL_CONTINUE;
1063
	} else if (emulator_read_std(addr, val, bytes, vcpu)
A
Avi Kivity 已提交
1064 1065
		   == X86EMUL_CONTINUE)
		return X86EMUL_CONTINUE;
A
Avi Kivity 已提交
1066

1067 1068 1069
	gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
	if (gpa == UNMAPPED_GVA)
		return X86EMUL_PROPAGATE_FAULT;
A
Avi Kivity 已提交
1070

1071 1072 1073 1074 1075 1076 1077
	/*
	 * Is this MMIO handled locally?
	 */
	mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
	if (mmio_dev) {
		kvm_iodevice_read(mmio_dev, gpa, bytes, val);
		return X86EMUL_CONTINUE;
A
Avi Kivity 已提交
1078
	}
1079 1080 1081 1082 1083 1084 1085

	vcpu->mmio_needed = 1;
	vcpu->mmio_phys_addr = gpa;
	vcpu->mmio_size = bytes;
	vcpu->mmio_is_write = 0;

	return X86EMUL_UNHANDLEABLE;
A
Avi Kivity 已提交
1086 1087
}

1088
static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
1089
			       const void *val, int bytes)
1090 1091 1092 1093 1094 1095
{
	struct page *page;
	void *virt;

	if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
		return 0;
A
Avi Kivity 已提交
1096 1097
	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
	if (!page)
1098
		return 0;
1099
	mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
1100
	virt = kmap_atomic(page, KM_USER0);
1101
	kvm_mmu_pte_write(vcpu, gpa, val, bytes);
1102
	memcpy(virt + offset_in_page(gpa), val, bytes);
1103 1104 1105 1106
	kunmap_atomic(virt, KM_USER0);
	return 1;
}

1107 1108 1109
static int emulator_write_emulated_onepage(unsigned long addr,
					   const void *val,
					   unsigned int bytes,
1110
					   struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1111
{
1112 1113
	struct kvm_io_device *mmio_dev;
	gpa_t                 gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
A
Avi Kivity 已提交
1114

1115
	if (gpa == UNMAPPED_GVA) {
1116
		kvm_x86_ops->inject_page_fault(vcpu, addr, 2);
A
Avi Kivity 已提交
1117
		return X86EMUL_PROPAGATE_FAULT;
1118
	}
A
Avi Kivity 已提交
1119

1120 1121 1122
	if (emulator_write_phys(vcpu, gpa, val, bytes))
		return X86EMUL_CONTINUE;

1123 1124 1125 1126 1127 1128 1129 1130 1131
	/*
	 * Is this MMIO handled locally?
	 */
	mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
	if (mmio_dev) {
		kvm_iodevice_write(mmio_dev, gpa, bytes, val);
		return X86EMUL_CONTINUE;
	}

A
Avi Kivity 已提交
1132 1133 1134 1135
	vcpu->mmio_needed = 1;
	vcpu->mmio_phys_addr = gpa;
	vcpu->mmio_size = bytes;
	vcpu->mmio_is_write = 1;
1136
	memcpy(vcpu->mmio_data, val, bytes);
A
Avi Kivity 已提交
1137 1138 1139 1140

	return X86EMUL_CONTINUE;
}

1141
int emulator_write_emulated(unsigned long addr,
1142 1143
				   const void *val,
				   unsigned int bytes,
1144
				   struct kvm_vcpu *vcpu)
1145 1146 1147 1148 1149 1150
{
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
		int rc, now;

		now = -addr & ~PAGE_MASK;
1151
		rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
1152 1153 1154 1155 1156 1157
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
		val += now;
		bytes -= now;
	}
1158
	return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
1159
}
1160
EXPORT_SYMBOL_GPL(emulator_write_emulated);
1161

A
Avi Kivity 已提交
1162
static int emulator_cmpxchg_emulated(unsigned long addr,
1163 1164
				     const void *old,
				     const void *new,
A
Avi Kivity 已提交
1165
				     unsigned int bytes,
1166
				     struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1167 1168 1169 1170 1171 1172 1173
{
	static int reported;

	if (!reported) {
		reported = 1;
		printk(KERN_WARNING "kvm: emulating exchange as write\n");
	}
1174
	return emulator_write_emulated(addr, new, bytes, vcpu);
A
Avi Kivity 已提交
1175 1176 1177 1178
}

static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
1179
	return kvm_x86_ops->get_segment_base(vcpu, seg);
A
Avi Kivity 已提交
1180 1181 1182 1183 1184 1185 1186 1187 1188
}

int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
{
	return X86EMUL_CONTINUE;
}

int emulate_clts(struct kvm_vcpu *vcpu)
{
1189
	kvm_x86_ops->set_cr0(vcpu, vcpu->cr0 & ~X86_CR0_TS);
A
Avi Kivity 已提交
1190 1191 1192 1193 1194 1195 1196 1197 1198
	return X86EMUL_CONTINUE;
}

int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
{
	struct kvm_vcpu *vcpu = ctxt->vcpu;

	switch (dr) {
	case 0 ... 3:
1199
		*dest = kvm_x86_ops->get_dr(vcpu, dr);
A
Avi Kivity 已提交
1200 1201
		return X86EMUL_CONTINUE;
	default:
1202
		pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr);
A
Avi Kivity 已提交
1203 1204 1205 1206 1207 1208 1209 1210 1211
		return X86EMUL_UNHANDLEABLE;
	}
}

int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
{
	unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
	int exception;

1212
	kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
A
Avi Kivity 已提交
1213 1214 1215 1216 1217 1218 1219
	if (exception) {
		/* FIXME: better handling */
		return X86EMUL_UNHANDLEABLE;
	}
	return X86EMUL_CONTINUE;
}

1220
void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context)
A
Avi Kivity 已提交
1221 1222 1223
{
	static int reported;
	u8 opcodes[4];
1224
	unsigned long rip = vcpu->rip;
A
Avi Kivity 已提交
1225 1226
	unsigned long rip_linear;

1227
	rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS);
A
Avi Kivity 已提交
1228 1229 1230 1231

	if (reported)
		return;

1232
	emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu);
A
Avi Kivity 已提交
1233

1234 1235
	printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n",
	       context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
A
Avi Kivity 已提交
1236 1237
	reported = 1;
}
1238
EXPORT_SYMBOL_GPL(kvm_report_emulation_failure);
A
Avi Kivity 已提交
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250

struct x86_emulate_ops emulate_ops = {
	.read_std            = emulator_read_std,
	.write_std           = emulator_write_std,
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
};

int emulate_instruction(struct kvm_vcpu *vcpu,
			struct kvm_run *run,
			unsigned long cr2,
1251 1252
			u16 error_code,
			int no_decode)
A
Avi Kivity 已提交
1253
{
1254
	int r = 0;
A
Avi Kivity 已提交
1255

1256
	vcpu->mmio_fault_cr2 = cr2;
1257
	kvm_x86_ops->cache_regs(vcpu);
A
Avi Kivity 已提交
1258 1259

	vcpu->mmio_is_write = 0;
1260
	vcpu->pio.string = 0;
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298

	if (!no_decode) {
		int cs_db, cs_l;
		kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);

		vcpu->emulate_ctxt.vcpu = vcpu;
		vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu);
		vcpu->emulate_ctxt.cr2 = cr2;
		vcpu->emulate_ctxt.mode =
			(vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM)
			? X86EMUL_MODE_REAL : cs_l
			? X86EMUL_MODE_PROT64 :	cs_db
			? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;

		if (vcpu->emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
			vcpu->emulate_ctxt.cs_base = 0;
			vcpu->emulate_ctxt.ds_base = 0;
			vcpu->emulate_ctxt.es_base = 0;
			vcpu->emulate_ctxt.ss_base = 0;
		} else {
			vcpu->emulate_ctxt.cs_base =
					get_segment_base(vcpu, VCPU_SREG_CS);
			vcpu->emulate_ctxt.ds_base =
					get_segment_base(vcpu, VCPU_SREG_DS);
			vcpu->emulate_ctxt.es_base =
					get_segment_base(vcpu, VCPU_SREG_ES);
			vcpu->emulate_ctxt.ss_base =
					get_segment_base(vcpu, VCPU_SREG_SS);
		}

		vcpu->emulate_ctxt.gs_base =
					get_segment_base(vcpu, VCPU_SREG_GS);
		vcpu->emulate_ctxt.fs_base =
					get_segment_base(vcpu, VCPU_SREG_FS);

		r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops);
	}

1299
	if (r == 0)
1300
		r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops);
1301

1302 1303
	if (vcpu->pio.string)
		return EMULATE_DO_MMIO;
A
Avi Kivity 已提交
1304 1305

	if ((r || vcpu->mmio_is_write) && run) {
1306
		run->exit_reason = KVM_EXIT_MMIO;
A
Avi Kivity 已提交
1307 1308 1309 1310 1311 1312 1313
		run->mmio.phys_addr = vcpu->mmio_phys_addr;
		memcpy(run->mmio.data, vcpu->mmio_data, 8);
		run->mmio.len = vcpu->mmio_size;
		run->mmio.is_write = vcpu->mmio_is_write;
	}

	if (r) {
1314 1315
		if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
			return EMULATE_DONE;
A
Avi Kivity 已提交
1316
		if (!vcpu->mmio_needed) {
1317
			kvm_report_emulation_failure(vcpu, "mmio");
A
Avi Kivity 已提交
1318 1319 1320 1321 1322
			return EMULATE_FAIL;
		}
		return EMULATE_DO_MMIO;
	}

1323
	kvm_x86_ops->decache_regs(vcpu);
1324
	kvm_x86_ops->set_rflags(vcpu, vcpu->emulate_ctxt.eflags);
A
Avi Kivity 已提交
1325

1326 1327
	if (vcpu->mmio_is_write) {
		vcpu->mmio_needed = 0;
A
Avi Kivity 已提交
1328
		return EMULATE_DO_MMIO;
1329
	}
A
Avi Kivity 已提交
1330 1331 1332 1333 1334

	return EMULATE_DONE;
}
EXPORT_SYMBOL_GPL(emulate_instruction);

E
Eddie Dong 已提交
1335 1336 1337
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1338
static void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1339
{
E
Eddie Dong 已提交
1340 1341 1342 1343 1344 1345 1346
	DECLARE_WAITQUEUE(wait, current);

	add_wait_queue(&vcpu->wq, &wait);

	/*
	 * We will block until either an interrupt or a signal wakes us up
	 */
1347 1348 1349 1350
	while (!kvm_cpu_has_interrupt(vcpu)
	       && !signal_pending(current)
	       && vcpu->mp_state != VCPU_MP_STATE_RUNNABLE
	       && vcpu->mp_state != VCPU_MP_STATE_SIPI_RECEIVED) {
E
Eddie Dong 已提交
1351 1352 1353 1354 1355
		set_current_state(TASK_INTERRUPTIBLE);
		vcpu_put(vcpu);
		schedule();
		vcpu_load(vcpu);
	}
1356

1357
	__set_current_state(TASK_RUNNING);
E
Eddie Dong 已提交
1358 1359 1360 1361 1362
	remove_wait_queue(&vcpu->wq, &wait);
}

int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
1363
	++vcpu->stat.halt_exits;
E
Eddie Dong 已提交
1364
	if (irqchip_in_kernel(vcpu->kvm)) {
1365 1366 1367 1368
		vcpu->mp_state = VCPU_MP_STATE_HALTED;
		kvm_vcpu_block(vcpu);
		if (vcpu->mp_state != VCPU_MP_STATE_RUNNABLE)
			return -EINTR;
E
Eddie Dong 已提交
1369 1370 1371 1372 1373
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
1374 1375 1376
}
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

1377
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
1378
{
1379
	unsigned long nr, a0, a1, a2, a3, ret;
1380

1381
	kvm_x86_ops->cache_regs(vcpu);
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394

	nr = vcpu->regs[VCPU_REGS_RAX];
	a0 = vcpu->regs[VCPU_REGS_RBX];
	a1 = vcpu->regs[VCPU_REGS_RCX];
	a2 = vcpu->regs[VCPU_REGS_RDX];
	a3 = vcpu->regs[VCPU_REGS_RSI];

	if (!is_long_mode(vcpu)) {
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
1395
	}
1396

1397 1398
	switch (nr) {
	default:
1399 1400
		ret = -KVM_ENOSYS;
		break;
1401 1402
	}
	vcpu->regs[VCPU_REGS_RAX] = ret;
1403
	kvm_x86_ops->decache_regs(vcpu);
1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
{
	char instruction[3];
	int ret = 0;

	mutex_lock(&vcpu->kvm->lock);

	/*
	 * Blow out the MMU to ensure that no other VCPU has an active mapping
	 * to ensure that the updated hypercall appears atomically across all
	 * VCPUs.
	 */
	kvm_mmu_zap_all(vcpu->kvm);

	kvm_x86_ops->cache_regs(vcpu);
	kvm_x86_ops->patch_hypercall(vcpu, instruction);
	if (emulator_write_emulated(vcpu->rip, instruction, 3, vcpu)
	    != X86EMUL_CONTINUE)
		ret = -EFAULT;

	mutex_unlock(&vcpu->kvm->lock);

	return ret;
1431 1432
}

A
Avi Kivity 已提交
1433 1434 1435 1436 1437 1438 1439 1440 1441
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
{
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
}

void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
{
	struct descriptor_table dt = { limit, base };

1442
	kvm_x86_ops->set_gdt(vcpu, &dt);
A
Avi Kivity 已提交
1443 1444 1445 1446 1447 1448
}

void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
{
	struct descriptor_table dt = { limit, base };

1449
	kvm_x86_ops->set_idt(vcpu, &dt);
A
Avi Kivity 已提交
1450 1451 1452 1453 1454 1455
}

void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
		   unsigned long *rflags)
{
	lmsw(vcpu, msw);
1456
	*rflags = kvm_x86_ops->get_rflags(vcpu);
A
Avi Kivity 已提交
1457 1458 1459 1460
}

unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
{
1461
	kvm_x86_ops->decache_cr4_guest_bits(vcpu);
A
Avi Kivity 已提交
1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482
	switch (cr) {
	case 0:
		return vcpu->cr0;
	case 2:
		return vcpu->cr2;
	case 3:
		return vcpu->cr3;
	case 4:
		return vcpu->cr4;
	default:
		vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
		return 0;
	}
}

void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
		     unsigned long *rflags)
{
	switch (cr) {
	case 0:
		set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1483
		*rflags = kvm_x86_ops->get_rflags(vcpu);
A
Avi Kivity 已提交
1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
		break;
	case 2:
		vcpu->cr2 = val;
		break;
	case 3:
		set_cr3(vcpu, val);
		break;
	case 4:
		set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
		break;
	default:
		vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
	}
}

1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517
int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data;

	switch (msr) {
	case 0xc0010010: /* SYSCFG */
	case 0xc0010015: /* HWCR */
	case MSR_IA32_PLATFORM_ID:
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MC0_CTL:
	case MSR_IA32_MCG_STATUS:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MC0_MISC:
	case MSR_IA32_MC0_MISC+4:
	case MSR_IA32_MC0_MISC+8:
	case MSR_IA32_MC0_MISC+12:
	case MSR_IA32_MC0_MISC+16:
	case MSR_IA32_UCODE_REV:
A
Avi Kivity 已提交
1518
	case MSR_IA32_PERF_STATUS:
1519
	case MSR_IA32_EBL_CR_POWERON:
1520 1521 1522 1523 1524
		/* MTRR registers */
	case 0xfe:
	case 0x200 ... 0x2ff:
		data = 0;
		break;
A
Avi Kivity 已提交
1525 1526 1527
	case 0xcd: /* fsb frequency */
		data = 3;
		break;
1528
	case MSR_IA32_APICBASE:
1529
		data = kvm_get_apic_base(vcpu);
1530
		break;
1531 1532 1533
	case MSR_IA32_MISC_ENABLE:
		data = vcpu->ia32_misc_enable_msr;
		break;
1534 1535 1536 1537 1538 1539
#ifdef CONFIG_X86_64
	case MSR_EFER:
		data = vcpu->shadow_efer;
		break;
#endif
	default:
1540
		pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
1541 1542 1543 1544 1545 1546 1547
		return 1;
	}
	*pdata = data;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

A
Avi Kivity 已提交
1548 1549 1550 1551 1552
/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1553
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
A
Avi Kivity 已提交
1554
{
1555
	return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
A
Avi Kivity 已提交
1556 1557
}

1558
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1559

1560
static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
A
Avi Kivity 已提交
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575
{
	if (efer & EFER_RESERVED_BITS) {
		printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
		       efer);
		inject_gp(vcpu);
		return;
	}

	if (is_paging(vcpu)
	    && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
		printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
		inject_gp(vcpu);
		return;
	}

1576
	kvm_x86_ops->set_efer(vcpu, efer);
1577

A
Avi Kivity 已提交
1578 1579 1580 1581 1582 1583 1584 1585
	efer &= ~EFER_LMA;
	efer |= vcpu->shadow_efer & EFER_LMA;

	vcpu->shadow_efer = efer;
}

#endif

1586 1587 1588 1589 1590 1591 1592 1593 1594
int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	switch (msr) {
#ifdef CONFIG_X86_64
	case MSR_EFER:
		set_efer(vcpu, data);
		break;
#endif
	case MSR_IA32_MC0_STATUS:
1595
		pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1596 1597
		       __FUNCTION__, data);
		break;
1598
	case MSR_IA32_MCG_STATUS:
1599
		pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
1600 1601
			__FUNCTION__, data);
		break;
1602 1603 1604 1605 1606
	case MSR_IA32_UCODE_REV:
	case MSR_IA32_UCODE_WRITE:
	case 0x200 ... 0x2ff: /* MTRRs */
		break;
	case MSR_IA32_APICBASE:
1607
		kvm_set_apic_base(vcpu, data);
1608
		break;
1609 1610 1611
	case MSR_IA32_MISC_ENABLE:
		vcpu->ia32_misc_enable_msr = data;
		break;
1612
	default:
1613
		pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr);
1614 1615 1616 1617 1618 1619
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);

A
Avi Kivity 已提交
1620 1621 1622 1623 1624
/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1625
int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
A
Avi Kivity 已提交
1626
{
1627
	return kvm_x86_ops->set_msr(vcpu, msr_index, data);
A
Avi Kivity 已提交
1628 1629 1630 1631
}

void kvm_resched(struct kvm_vcpu *vcpu)
{
1632 1633
	if (!need_resched())
		return;
A
Avi Kivity 已提交
1634 1635 1636 1637
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

1638 1639 1640 1641 1642 1643
void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
{
	int i;
	u32 function;
	struct kvm_cpuid_entry *e, *best;

1644
	kvm_x86_ops->cache_regs(vcpu);
1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669
	function = vcpu->regs[VCPU_REGS_RAX];
	vcpu->regs[VCPU_REGS_RAX] = 0;
	vcpu->regs[VCPU_REGS_RBX] = 0;
	vcpu->regs[VCPU_REGS_RCX] = 0;
	vcpu->regs[VCPU_REGS_RDX] = 0;
	best = NULL;
	for (i = 0; i < vcpu->cpuid_nent; ++i) {
		e = &vcpu->cpuid_entries[i];
		if (e->function == function) {
			best = e;
			break;
		}
		/*
		 * Both basic or both extended?
		 */
		if (((e->function ^ function) & 0x80000000) == 0)
			if (!best || e->function > best->function)
				best = e;
	}
	if (best) {
		vcpu->regs[VCPU_REGS_RAX] = best->eax;
		vcpu->regs[VCPU_REGS_RBX] = best->ebx;
		vcpu->regs[VCPU_REGS_RCX] = best->ecx;
		vcpu->regs[VCPU_REGS_RDX] = best->edx;
	}
1670 1671
	kvm_x86_ops->decache_regs(vcpu);
	kvm_x86_ops->skip_emulated_instruction(vcpu);
1672 1673 1674
}
EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);

1675
static int pio_copy_data(struct kvm_vcpu *vcpu)
1676
{
1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702
	void *p = vcpu->pio_data;
	void *q;
	unsigned bytes;
	int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1;

	q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
		 PAGE_KERNEL);
	if (!q) {
		free_pio_guest_pages(vcpu);
		return -ENOMEM;
	}
	q += vcpu->pio.guest_page_offset;
	bytes = vcpu->pio.size * vcpu->pio.cur_count;
	if (vcpu->pio.in)
		memcpy(q, p, bytes);
	else
		memcpy(p, q, bytes);
	q -= vcpu->pio.guest_page_offset;
	vunmap(q);
	free_pio_guest_pages(vcpu);
	return 0;
}

static int complete_pio(struct kvm_vcpu *vcpu)
{
	struct kvm_pio_request *io = &vcpu->pio;
1703
	long delta;
1704
	int r;
1705

1706
	kvm_x86_ops->cache_regs(vcpu);
1707 1708

	if (!io->string) {
1709 1710
		if (io->in)
			memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data,
1711 1712
			       io->size);
	} else {
1713 1714 1715
		if (io->in) {
			r = pio_copy_data(vcpu);
			if (r) {
1716
				kvm_x86_ops->cache_regs(vcpu);
1717 1718 1719 1720
				return r;
			}
		}

1721 1722
		delta = 1;
		if (io->rep) {
1723
			delta *= io->cur_count;
1724 1725 1726 1727 1728 1729
			/*
			 * The size of the register should really depend on
			 * current address size.
			 */
			vcpu->regs[VCPU_REGS_RCX] -= delta;
		}
1730
		if (io->down)
1731 1732
			delta = -delta;
		delta *= io->size;
1733
		if (io->in)
1734 1735 1736 1737 1738
			vcpu->regs[VCPU_REGS_RDI] += delta;
		else
			vcpu->regs[VCPU_REGS_RSI] += delta;
	}

1739
	kvm_x86_ops->decache_regs(vcpu);
1740

1741 1742 1743 1744
	io->count -= io->cur_count;
	io->cur_count = 0;

	return 0;
1745 1746
}

1747 1748 1749
static void kernel_pio(struct kvm_io_device *pio_dev,
		       struct kvm_vcpu *vcpu,
		       void *pd)
1750 1751 1752
{
	/* TODO: String I/O for in kernel device */

1753
	mutex_lock(&vcpu->kvm->lock);
1754 1755 1756
	if (vcpu->pio.in)
		kvm_iodevice_read(pio_dev, vcpu->pio.port,
				  vcpu->pio.size,
1757
				  pd);
1758 1759 1760
	else
		kvm_iodevice_write(pio_dev, vcpu->pio.port,
				   vcpu->pio.size,
1761
				   pd);
1762
	mutex_unlock(&vcpu->kvm->lock);
1763 1764 1765 1766 1767 1768 1769 1770 1771
}

static void pio_string_write(struct kvm_io_device *pio_dev,
			     struct kvm_vcpu *vcpu)
{
	struct kvm_pio_request *io = &vcpu->pio;
	void *pd = vcpu->pio_data;
	int i;

1772
	mutex_lock(&vcpu->kvm->lock);
1773 1774 1775 1776 1777 1778
	for (i = 0; i < io->cur_count; i++) {
		kvm_iodevice_write(pio_dev, io->port,
				   io->size,
				   pd);
		pd += io->size;
	}
1779
	mutex_unlock(&vcpu->kvm->lock);
1780 1781
}

L
Laurent Vivier 已提交
1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798
int kvm_emulate_pio (struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
		  int size, unsigned port)
{
	struct kvm_io_device *pio_dev;

	vcpu->run->exit_reason = KVM_EXIT_IO;
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
	vcpu->run->io.size = vcpu->pio.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1;
	vcpu->run->io.port = vcpu->pio.port = port;
	vcpu->pio.in = in;
	vcpu->pio.string = 0;
	vcpu->pio.down = 0;
	vcpu->pio.guest_page_offset = 0;
	vcpu->pio.rep = 0;

1799
	kvm_x86_ops->cache_regs(vcpu);
L
Laurent Vivier 已提交
1800
	memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4);
1801
	kvm_x86_ops->decache_regs(vcpu);
L
Laurent Vivier 已提交
1802

1803 1804
	kvm_x86_ops->skip_emulated_instruction(vcpu);

L
Laurent Vivier 已提交
1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816
	pio_dev = vcpu_find_pio_dev(vcpu, port);
	if (pio_dev) {
		kernel_pio(pio_dev, vcpu, vcpu->pio_data);
		complete_pio(vcpu);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_emulate_pio);

int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
		  int size, unsigned long count, int down,
1817 1818 1819
		  gva_t address, int rep, unsigned port)
{
	unsigned now, in_page;
1820
	int i, ret = 0;
1821 1822
	int nr_pages = 1;
	struct page *page;
1823
	struct kvm_io_device *pio_dev;
1824 1825 1826

	vcpu->run->exit_reason = KVM_EXIT_IO;
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
L
Laurent Vivier 已提交
1827
	vcpu->run->io.size = vcpu->pio.size = size;
1828
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
L
Laurent Vivier 已提交
1829 1830
	vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count;
	vcpu->run->io.port = vcpu->pio.port = port;
1831
	vcpu->pio.in = in;
L
Laurent Vivier 已提交
1832
	vcpu->pio.string = 1;
1833 1834 1835 1836 1837
	vcpu->pio.down = down;
	vcpu->pio.guest_page_offset = offset_in_page(address);
	vcpu->pio.rep = rep;

	if (!count) {
1838
		kvm_x86_ops->skip_emulated_instruction(vcpu);
1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
		return 1;
	}

	if (!down)
		in_page = PAGE_SIZE - offset_in_page(address);
	else
		in_page = offset_in_page(address) + size;
	now = min(count, (unsigned long)in_page / size);
	if (!now) {
		/*
		 * String I/O straddles page boundary.  Pin two guest pages
		 * so that we satisfy atomicity constraints.  Do just one
		 * transaction to avoid complexity.
		 */
		nr_pages = 2;
		now = 1;
	}
	if (down) {
		/*
		 * String I/O in reverse.  Yuck.  Kill the guest, fix later.
		 */
1860
		pr_unimpl(vcpu, "guest string pio down\n");
1861 1862 1863 1864 1865 1866
		inject_gp(vcpu);
		return 1;
	}
	vcpu->run->io.count = now;
	vcpu->pio.cur_count = now;

1867 1868 1869
	if (vcpu->pio.cur_count == vcpu->pio.count)
		kvm_x86_ops->skip_emulated_instruction(vcpu);

1870
	for (i = 0; i < nr_pages; ++i) {
S
Shaohua Li 已提交
1871
		mutex_lock(&vcpu->kvm->lock);
1872 1873 1874 1875
		page = gva_to_page(vcpu, address + i * PAGE_SIZE);
		if (page)
			get_page(page);
		vcpu->pio.guest_pages[i] = page;
S
Shaohua Li 已提交
1876
		mutex_unlock(&vcpu->kvm->lock);
1877 1878 1879 1880 1881 1882 1883
		if (!page) {
			inject_gp(vcpu);
			free_pio_guest_pages(vcpu);
			return 1;
		}
	}

L
Laurent Vivier 已提交
1884
	pio_dev = vcpu_find_pio_dev(vcpu, port);
1885 1886 1887 1888 1889 1890 1891 1892 1893 1894
	if (!vcpu->pio.in) {
		/* string PIO write */
		ret = pio_copy_data(vcpu);
		if (ret >= 0 && pio_dev) {
			pio_string_write(pio_dev, vcpu);
			complete_pio(vcpu);
			if (vcpu->pio.count == 0)
				ret = 1;
		}
	} else if (pio_dev)
1895
		pr_unimpl(vcpu, "no string pio read support yet, "
1896 1897 1898 1899
		       "port %x size %d count %ld\n",
			port, size, count);

	return ret;
1900
}
L
Laurent Vivier 已提交
1901
EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
1902

1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974
/*
 * Check if userspace requested an interrupt window, and that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
					  struct kvm_run *kvm_run)
{
	return (!vcpu->irq_summary &&
		kvm_run->request_interrupt_window &&
		vcpu->interrupt_window_open &&
		(kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF));
}

static void post_kvm_run_save(struct kvm_vcpu *vcpu,
			      struct kvm_run *kvm_run)
{
	kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
	kvm_run->cr8 = get_cr8(vcpu);
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_run->ready_for_interrupt_injection = 1;
	else
		kvm_run->ready_for_interrupt_injection =
					(vcpu->interrupt_window_open &&
					 vcpu->irq_summary == 0);
}

static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
{
	int r;

	if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) {
		printk("vcpu %d received sipi with vector # %x\n",
		       vcpu->vcpu_id, vcpu->sipi_vector);
		kvm_lapic_reset(vcpu);
		kvm_x86_ops->vcpu_reset(vcpu);
		vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
	}

preempted:
	if (vcpu->guest_debug.enabled)
		kvm_x86_ops->guest_debug_pre(vcpu);

again:
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		goto out;

	preempt_disable();

	kvm_x86_ops->prepare_guest_switch(vcpu);
	kvm_load_guest_fpu(vcpu);

	local_irq_disable();

	if (signal_pending(current)) {
		local_irq_enable();
		preempt_enable();
		r = -EINTR;
		kvm_run->exit_reason = KVM_EXIT_INTR;
		++vcpu->stat.signal_exits;
		goto out;
	}

	if (irqchip_in_kernel(vcpu->kvm))
		kvm_x86_ops->inject_pending_irq(vcpu);
	else if (!vcpu->mmio_read_completed)
		kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run);

	vcpu->guest_mode = 1;
1975
	kvm_guest_enter();
1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987

	if (vcpu->requests)
		if (test_and_clear_bit(KVM_TLB_FLUSH, &vcpu->requests))
			kvm_x86_ops->tlb_flush(vcpu);

	kvm_x86_ops->run(vcpu, kvm_run);

	vcpu->guest_mode = 0;
	local_irq_enable();

	++vcpu->stat.exits;

1988 1989 1990 1991 1992 1993 1994 1995 1996 1997
	/*
	 * We must have an instruction between local_irq_enable() and
	 * kvm_guest_exit(), so the timer interrupt isn't delayed by
	 * the interrupt shadow.  The stat.exits increment will do nicely.
	 * But we need to prevent reordering, hence this barrier():
	 */
	barrier();

	kvm_guest_exit();

1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034
	preempt_enable();

	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
		kvm_x86_ops->cache_regs(vcpu);
		profile_hit(KVM_PROFILING, (void *)vcpu->rip);
	}

	r = kvm_x86_ops->handle_exit(kvm_run, vcpu);

	if (r > 0) {
		if (dm_request_for_irq_injection(vcpu, kvm_run)) {
			r = -EINTR;
			kvm_run->exit_reason = KVM_EXIT_INTR;
			++vcpu->stat.request_irq_exits;
			goto out;
		}
		if (!need_resched()) {
			++vcpu->stat.light_exits;
			goto again;
		}
	}

out:
	if (r > 0) {
		kvm_resched(vcpu);
		goto preempted;
	}

	post_kvm_run_save(vcpu, kvm_run);

	return r;
}


A
Avi Kivity 已提交
2035
static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
2036 2037
{
	int r;
A
Avi Kivity 已提交
2038
	sigset_t sigsaved;
A
Avi Kivity 已提交
2039

A
Avi Kivity 已提交
2040
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2041

2042 2043 2044 2045 2046 2047
	if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) {
		kvm_vcpu_block(vcpu);
		vcpu_put(vcpu);
		return -EAGAIN;
	}

A
Avi Kivity 已提交
2048 2049 2050
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);

2051
	/* re-sync apic's tpr */
2052 2053
	if (!irqchip_in_kernel(vcpu->kvm))
		set_cr8(vcpu, kvm_run->cr8);
2054

2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
	if (vcpu->pio.cur_count) {
		r = complete_pio(vcpu);
		if (r)
			goto out;
	}

	if (vcpu->mmio_needed) {
		memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
		vcpu->mmio_read_completed = 1;
		vcpu->mmio_needed = 0;
		r = emulate_instruction(vcpu, kvm_run,
2066
					vcpu->mmio_fault_cr2, 0, 1);
2067 2068 2069 2070 2071 2072
		if (r == EMULATE_DO_MMIO) {
			/*
			 * Read-modify-write.  Back to userspace.
			 */
			r = 0;
			goto out;
2073
		}
A
Avi Kivity 已提交
2074 2075
	}

2076
	if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
2077
		kvm_x86_ops->cache_regs(vcpu);
2078
		vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
2079
		kvm_x86_ops->decache_regs(vcpu);
2080 2081
	}

2082
	r = __vcpu_run(vcpu, kvm_run);
A
Avi Kivity 已提交
2083

2084
out:
A
Avi Kivity 已提交
2085 2086 2087
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

A
Avi Kivity 已提交
2088 2089 2090 2091
	vcpu_put(vcpu);
	return r;
}

A
Avi Kivity 已提交
2092 2093
static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
				   struct kvm_regs *regs)
A
Avi Kivity 已提交
2094
{
A
Avi Kivity 已提交
2095
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2096

2097
	kvm_x86_ops->cache_regs(vcpu);
A
Avi Kivity 已提交
2098 2099 2100 2101 2102 2103 2104 2105 2106

	regs->rax = vcpu->regs[VCPU_REGS_RAX];
	regs->rbx = vcpu->regs[VCPU_REGS_RBX];
	regs->rcx = vcpu->regs[VCPU_REGS_RCX];
	regs->rdx = vcpu->regs[VCPU_REGS_RDX];
	regs->rsi = vcpu->regs[VCPU_REGS_RSI];
	regs->rdi = vcpu->regs[VCPU_REGS_RDI];
	regs->rsp = vcpu->regs[VCPU_REGS_RSP];
	regs->rbp = vcpu->regs[VCPU_REGS_RBP];
2107
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118
	regs->r8 = vcpu->regs[VCPU_REGS_R8];
	regs->r9 = vcpu->regs[VCPU_REGS_R9];
	regs->r10 = vcpu->regs[VCPU_REGS_R10];
	regs->r11 = vcpu->regs[VCPU_REGS_R11];
	regs->r12 = vcpu->regs[VCPU_REGS_R12];
	regs->r13 = vcpu->regs[VCPU_REGS_R13];
	regs->r14 = vcpu->regs[VCPU_REGS_R14];
	regs->r15 = vcpu->regs[VCPU_REGS_R15];
#endif

	regs->rip = vcpu->rip;
2119
	regs->rflags = kvm_x86_ops->get_rflags(vcpu);
A
Avi Kivity 已提交
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131

	/*
	 * Don't leak debug flags in case they were set for guest debugging
	 */
	if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
		regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);

	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2132 2133
static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
				   struct kvm_regs *regs)
A
Avi Kivity 已提交
2134
{
A
Avi Kivity 已提交
2135
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2136 2137 2138 2139 2140 2141 2142 2143 2144

	vcpu->regs[VCPU_REGS_RAX] = regs->rax;
	vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
	vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
	vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
	vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
	vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
	vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
	vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
2145
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156
	vcpu->regs[VCPU_REGS_R8] = regs->r8;
	vcpu->regs[VCPU_REGS_R9] = regs->r9;
	vcpu->regs[VCPU_REGS_R10] = regs->r10;
	vcpu->regs[VCPU_REGS_R11] = regs->r11;
	vcpu->regs[VCPU_REGS_R12] = regs->r12;
	vcpu->regs[VCPU_REGS_R13] = regs->r13;
	vcpu->regs[VCPU_REGS_R14] = regs->r14;
	vcpu->regs[VCPU_REGS_R15] = regs->r15;
#endif

	vcpu->rip = regs->rip;
2157
	kvm_x86_ops->set_rflags(vcpu, regs->rflags);
A
Avi Kivity 已提交
2158

2159
	kvm_x86_ops->decache_regs(vcpu);
A
Avi Kivity 已提交
2160 2161 2162 2163 2164 2165 2166 2167 2168

	vcpu_put(vcpu);

	return 0;
}

static void get_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
2169
	return kvm_x86_ops->get_segment(vcpu, var, seg);
A
Avi Kivity 已提交
2170 2171
}

A
Avi Kivity 已提交
2172 2173
static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				    struct kvm_sregs *sregs)
A
Avi Kivity 已提交
2174 2175
{
	struct descriptor_table dt;
E
Eddie Dong 已提交
2176
	int pending_vec;
A
Avi Kivity 已提交
2177

A
Avi Kivity 已提交
2178
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189

	get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);

	get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);

2190
	kvm_x86_ops->get_idt(vcpu, &dt);
A
Avi Kivity 已提交
2191 2192
	sregs->idt.limit = dt.limit;
	sregs->idt.base = dt.base;
2193
	kvm_x86_ops->get_gdt(vcpu, &dt);
A
Avi Kivity 已提交
2194 2195 2196
	sregs->gdt.limit = dt.limit;
	sregs->gdt.base = dt.base;

2197
	kvm_x86_ops->decache_cr4_guest_bits(vcpu);
A
Avi Kivity 已提交
2198 2199 2200 2201
	sregs->cr0 = vcpu->cr0;
	sregs->cr2 = vcpu->cr2;
	sregs->cr3 = vcpu->cr3;
	sregs->cr4 = vcpu->cr4;
2202
	sregs->cr8 = get_cr8(vcpu);
A
Avi Kivity 已提交
2203
	sregs->efer = vcpu->shadow_efer;
2204
	sregs->apic_base = kvm_get_apic_base(vcpu);
A
Avi Kivity 已提交
2205

E
Eddie Dong 已提交
2206
	if (irqchip_in_kernel(vcpu->kvm)) {
2207 2208
		memset(sregs->interrupt_bitmap, 0,
		       sizeof sregs->interrupt_bitmap);
2209
		pending_vec = kvm_x86_ops->get_irq(vcpu);
E
Eddie Dong 已提交
2210 2211 2212
		if (pending_vec >= 0)
			set_bit(pending_vec, (unsigned long *)sregs->interrupt_bitmap);
	} else
2213 2214
		memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
		       sizeof sregs->interrupt_bitmap);
A
Avi Kivity 已提交
2215 2216 2217 2218 2219 2220 2221 2222 2223

	vcpu_put(vcpu);

	return 0;
}

static void set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
2224
	return kvm_x86_ops->set_segment(vcpu, var, seg);
A
Avi Kivity 已提交
2225 2226
}

A
Avi Kivity 已提交
2227 2228
static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				    struct kvm_sregs *sregs)
A
Avi Kivity 已提交
2229 2230
{
	int mmu_reset_needed = 0;
E
Eddie Dong 已提交
2231
	int i, pending_vec, max_bits;
A
Avi Kivity 已提交
2232 2233
	struct descriptor_table dt;

A
Avi Kivity 已提交
2234
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2235 2236 2237

	dt.limit = sregs->idt.limit;
	dt.base = sregs->idt.base;
2238
	kvm_x86_ops->set_idt(vcpu, &dt);
A
Avi Kivity 已提交
2239 2240
	dt.limit = sregs->gdt.limit;
	dt.base = sregs->gdt.base;
2241
	kvm_x86_ops->set_gdt(vcpu, &dt);
A
Avi Kivity 已提交
2242 2243 2244 2245 2246

	vcpu->cr2 = sregs->cr2;
	mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
	vcpu->cr3 = sregs->cr3;

2247
	set_cr8(vcpu, sregs->cr8);
A
Avi Kivity 已提交
2248 2249

	mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
2250
#ifdef CONFIG_X86_64
2251
	kvm_x86_ops->set_efer(vcpu, sregs->efer);
A
Avi Kivity 已提交
2252
#endif
2253
	kvm_set_apic_base(vcpu, sregs->apic_base);
A
Avi Kivity 已提交
2254

2255
	kvm_x86_ops->decache_cr4_guest_bits(vcpu);
2256

A
Avi Kivity 已提交
2257
	mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
R
Rusty Russell 已提交
2258
	vcpu->cr0 = sregs->cr0;
2259
	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
A
Avi Kivity 已提交
2260 2261

	mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
2262
	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
2263 2264
	if (!is_long_mode(vcpu) && is_pae(vcpu))
		load_pdptrs(vcpu, vcpu->cr3);
A
Avi Kivity 已提交
2265 2266 2267 2268

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

2269 2270 2271 2272 2273 2274 2275
	if (!irqchip_in_kernel(vcpu->kvm)) {
		memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
		       sizeof vcpu->irq_pending);
		vcpu->irq_summary = 0;
		for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i)
			if (vcpu->irq_pending[i])
				__set_bit(i, &vcpu->irq_summary);
E
Eddie Dong 已提交
2276 2277 2278 2279 2280 2281 2282
	} else {
		max_bits = (sizeof sregs->interrupt_bitmap) << 3;
		pending_vec = find_first_bit(
			(const unsigned long *)sregs->interrupt_bitmap,
			max_bits);
		/* Only pending external irq is handled here */
		if (pending_vec < max_bits) {
2283
			kvm_x86_ops->set_irq(vcpu, pending_vec);
E
Eddie Dong 已提交
2284 2285
			printk("Set back pending irq %d\n", pending_vec);
		}
2286
	}
A
Avi Kivity 已提交
2287

2288 2289 2290 2291 2292 2293 2294 2295 2296 2297
	set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);

	set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);

A
Avi Kivity 已提交
2298 2299 2300 2301 2302
	vcpu_put(vcpu);

	return 0;
}

2303 2304 2305 2306 2307 2308 2309 2310 2311 2312
void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

	get_segment(vcpu, &cs, VCPU_SREG_CS);
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

A
Avi Kivity 已提交
2313 2314 2315
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
2316 2317 2318
 *
 * This list is modified at module load time to reflect the
 * capabilities of the host cpu.
A
Avi Kivity 已提交
2319 2320 2321 2322
 */
static u32 msrs_to_save[] = {
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
	MSR_K6_STAR,
2323
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2324 2325 2326 2327 2328
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
	MSR_IA32_TIME_STAMP_COUNTER,
};

2329 2330
static unsigned num_msrs_to_save;

2331 2332 2333 2334
static u32 emulated_msrs[] = {
	MSR_IA32_MISC_ENABLE,
};

2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348
static __init void kvm_init_msr_list(void)
{
	u32 dummy[2];
	unsigned i, j;

	for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
		if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
			continue;
		if (j < i)
			msrs_to_save[j] = msrs_to_save[i];
		j++;
	}
	num_msrs_to_save = j;
}
A
Avi Kivity 已提交
2349 2350 2351 2352 2353 2354

/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
2355
	return kvm_set_msr(vcpu, index, *data);
A
Avi Kivity 已提交
2356 2357 2358 2359 2360 2361 2362
}

/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
A
Avi Kivity 已提交
2363
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
A
Avi Kivity 已提交
2364 2365 2366 2367 2368 2369
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
	int i;

A
Avi Kivity 已提交
2370
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385

	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;

	vcpu_put(vcpu);

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
A
Avi Kivity 已提交
2386
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
A
Avi Kivity 已提交
2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
	if (copy_from_user(&msrs, user_msrs, sizeof msrs))
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	r = -ENOMEM;
	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
	entries = vmalloc(size);
	if (!entries)
		goto out;

	r = -EFAULT;
	if (copy_from_user(entries, user_msrs->entries, size))
		goto out_free;

A
Avi Kivity 已提交
2414
	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
A
Avi Kivity 已提交
2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
	vfree(entries);
out:
	return r;
}

/*
 * Translate a guest virtual address to a guest physical address.
 */
A
Avi Kivity 已提交
2433 2434
static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
A
Avi Kivity 已提交
2435 2436 2437 2438
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;

A
Avi Kivity 已提交
2439
	vcpu_load(vcpu);
S
Shaohua Li 已提交
2440
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
2441 2442 2443 2444 2445
	gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;
S
Shaohua Li 已提交
2446
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
2447 2448 2449 2450 2451
	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2452 2453
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
A
Avi Kivity 已提交
2454 2455 2456
{
	if (irq->irq < 0 || irq->irq >= 256)
		return -EINVAL;
E
Eddie Dong 已提交
2457 2458
	if (irqchip_in_kernel(vcpu->kvm))
		return -ENXIO;
A
Avi Kivity 已提交
2459
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2460 2461 2462 2463 2464 2465 2466 2467 2468

	set_bit(irq->irq, vcpu->irq_pending);
	set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);

	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2469 2470
static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
				      struct kvm_debug_guest *dbg)
A
Avi Kivity 已提交
2471 2472 2473
{
	int r;

A
Avi Kivity 已提交
2474
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2475

2476
	r = kvm_x86_ops->set_guest_debug(vcpu, dbg);
A
Avi Kivity 已提交
2477 2478 2479 2480 2481 2482

	vcpu_put(vcpu);

	return r;
}

2483 2484 2485 2486 2487 2488 2489 2490 2491
static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
				    unsigned long address,
				    int *type)
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	unsigned long pgoff;
	struct page *page;

	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2492 2493 2494 2495 2496
	if (pgoff == 0)
		page = virt_to_page(vcpu->run);
	else if (pgoff == KVM_PIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->pio_data);
	else
2497 2498
		return NOPAGE_SIGBUS;
	get_page(page);
2499 2500 2501
	if (type != NULL)
		*type = VM_FAULT_MINOR;

2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514
	return page;
}

static struct vm_operations_struct kvm_vcpu_vm_ops = {
	.nopage = kvm_vcpu_nopage,
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

	fput(vcpu->kvm->filp);
	return 0;
}

static struct file_operations kvm_vcpu_fops = {
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
	.compat_ioctl   = kvm_vcpu_ioctl,
2527
	.mmap           = kvm_vcpu_mmap,
A
Avi Kivity 已提交
2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
	int fd, r;
	struct inode *inode;
	struct file *file;

2539 2540 2541 2542
	r = anon_inode_getfd(&fd, &inode, &file,
			     "kvm-vcpu", &kvm_vcpu_fops, vcpu);
	if (r)
		return r;
A
Avi Kivity 已提交
2543 2544 2545 2546
	atomic_inc(&vcpu->kvm->filp->f_count);
	return fd;
}

2547 2548 2549 2550 2551 2552 2553 2554 2555
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
{
	int r;
	struct kvm_vcpu *vcpu;

	if (!valid_vcpu(n))
R
Rusty Russell 已提交
2556
		return -EINVAL;
2557

2558
	vcpu = kvm_x86_ops->vcpu_create(kvm, n);
R
Rusty Russell 已提交
2559 2560
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
2561

2562 2563
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

2564 2565 2566
	/* We do fxsave: this must be aligned. */
	BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF);

R
Rusty Russell 已提交
2567
	vcpu_load(vcpu);
2568 2569 2570
	r = kvm_mmu_setup(vcpu);
	vcpu_put(vcpu);
	if (r < 0)
R
Rusty Russell 已提交
2571 2572
		goto free_vcpu;

S
Shaohua Li 已提交
2573
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
2574 2575
	if (kvm->vcpus[n]) {
		r = -EEXIST;
S
Shaohua Li 已提交
2576
		mutex_unlock(&kvm->lock);
R
Rusty Russell 已提交
2577 2578 2579
		goto mmu_unload;
	}
	kvm->vcpus[n] = vcpu;
S
Shaohua Li 已提交
2580
	mutex_unlock(&kvm->lock);
2581

R
Rusty Russell 已提交
2582
	/* Now it's all set up, let userspace reach it */
A
Avi Kivity 已提交
2583 2584
	r = create_vcpu_fd(vcpu);
	if (r < 0)
R
Rusty Russell 已提交
2585 2586
		goto unlink;
	return r;
2587

R
Rusty Russell 已提交
2588
unlink:
S
Shaohua Li 已提交
2589
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
2590
	kvm->vcpus[n] = NULL;
S
Shaohua Li 已提交
2591
	mutex_unlock(&kvm->lock);
2592

R
Rusty Russell 已提交
2593 2594 2595 2596
mmu_unload:
	vcpu_load(vcpu);
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
2597

R
Rusty Russell 已提交
2598
free_vcpu:
2599
	kvm_x86_ops->vcpu_free(vcpu);
2600 2601 2602
	return r;
}

2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617
static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
{
	u64 efer;
	int i;
	struct kvm_cpuid_entry *e, *entry;

	rdmsrl(MSR_EFER, efer);
	entry = NULL;
	for (i = 0; i < vcpu->cpuid_nent; ++i) {
		e = &vcpu->cpuid_entries[i];
		if (e->function == 0x80000001) {
			entry = e;
			break;
		}
	}
2618
	if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) {
2619
		entry->edx &= ~(1 << 20);
2620
		printk(KERN_INFO "kvm: guest NX capability removed\n");
2621 2622 2623
	}
}

2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637
static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
				    struct kvm_cpuid *cpuid,
				    struct kvm_cpuid_entry __user *entries)
{
	int r;

	r = -E2BIG;
	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
		goto out;
	r = -EFAULT;
	if (copy_from_user(&vcpu->cpuid_entries, entries,
			   cpuid->nent * sizeof(struct kvm_cpuid_entry)))
		goto out;
	vcpu->cpuid_nent = cpuid->nent;
2638
	cpuid_fix_nx_cap(vcpu);
2639 2640 2641 2642 2643 2644
	return 0;

out:
	return r;
}

A
Avi Kivity 已提交
2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678
/*
 * fxsave fpu state.  Taken from x86_64/processor.h.  To be killed when
 * we have asm/x86/processor.h
 */
struct fxsave {
	u16	cwd;
	u16	swd;
	u16	twd;
	u16	fop;
	u64	rip;
	u64	rdp;
	u32	mxcsr;
	u32	mxcsr_mask;
	u32	st_space[32];	/* 8*16 bytes for each FP-reg = 128 bytes */
#ifdef CONFIG_X86_64
	u32	xmm_space[64];	/* 16*16 bytes for each XMM-reg = 256 bytes */
#else
	u32	xmm_space[32];	/* 8*16 bytes for each XMM-reg = 128 bytes */
#endif
};

static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
2679
	struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image;
A
Avi Kivity 已提交
2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698

	vcpu_load(vcpu);

	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);

	vcpu_put(vcpu);

	return 0;
}

static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
2699
	struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image;
A
Avi Kivity 已提交
2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716

	vcpu_load(vcpu);

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);

	vcpu_put(vcpu);

	return 0;
}

2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737
static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
	vcpu_load(vcpu);
	memcpy(s->regs, vcpu->apic->regs, sizeof *s);
	vcpu_put(vcpu);

	return 0;
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
	vcpu_load(vcpu);
	memcpy(vcpu->apic->regs, s->regs, sizeof *s);
	kvm_apic_post_state_restore(vcpu);
	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2738 2739
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
2740
{
A
Avi Kivity 已提交
2741
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
2742
	void __user *argp = (void __user *)arg;
A
Avi Kivity 已提交
2743 2744 2745
	int r = -EINVAL;

	switch (ioctl) {
2746
	case KVM_RUN:
2747 2748 2749
		r = -EINVAL;
		if (arg)
			goto out;
2750
		r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
A
Avi Kivity 已提交
2751 2752 2753 2754
		break;
	case KVM_GET_REGS: {
		struct kvm_regs kvm_regs;

A
Avi Kivity 已提交
2755 2756
		memset(&kvm_regs, 0, sizeof kvm_regs);
		r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
A
Avi Kivity 已提交
2757 2758 2759
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2760
		if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
A
Avi Kivity 已提交
2761 2762 2763 2764 2765 2766 2767 2768
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_REGS: {
		struct kvm_regs kvm_regs;

		r = -EFAULT;
A
Al Viro 已提交
2769
		if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
A
Avi Kivity 已提交
2770
			goto out;
A
Avi Kivity 已提交
2771
		r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
A
Avi Kivity 已提交
2772 2773 2774 2775 2776 2777 2778 2779
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_SREGS: {
		struct kvm_sregs kvm_sregs;

A
Avi Kivity 已提交
2780 2781
		memset(&kvm_sregs, 0, sizeof kvm_sregs);
		r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
A
Avi Kivity 已提交
2782 2783 2784
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2785
		if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
A
Avi Kivity 已提交
2786 2787 2788 2789 2790 2791 2792 2793
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
		struct kvm_sregs kvm_sregs;

		r = -EFAULT;
A
Al Viro 已提交
2794
		if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
A
Avi Kivity 已提交
2795
			goto out;
A
Avi Kivity 已提交
2796
		r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
A
Avi Kivity 已提交
2797 2798 2799 2800 2801 2802 2803 2804 2805
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
2806
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
2807
			goto out;
A
Avi Kivity 已提交
2808
		r = kvm_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
2809 2810 2811
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2812
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
2813 2814 2815 2816 2817 2818 2819 2820
			goto out;
		r = 0;
		break;
	}
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
A
Al Viro 已提交
2821
		if (copy_from_user(&irq, argp, sizeof irq))
A
Avi Kivity 已提交
2822
			goto out;
A
Avi Kivity 已提交
2823
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
A
Avi Kivity 已提交
2824 2825 2826 2827 2828 2829 2830 2831 2832
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_DEBUG_GUEST: {
		struct kvm_debug_guest dbg;

		r = -EFAULT;
A
Al Viro 已提交
2833
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
2834
			goto out;
A
Avi Kivity 已提交
2835
		r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
A
Avi Kivity 已提交
2836 2837 2838 2839 2840
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
2841
	case KVM_GET_MSRS:
2842
		r = msr_io(vcpu, argp, kvm_get_msr, 1);
A
Avi Kivity 已提交
2843 2844 2845 2846
		break;
	case KVM_SET_MSRS:
		r = msr_io(vcpu, argp, do_set_msr, 0);
		break;
2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		if (r)
			goto out;
		break;
	}
A
Avi Kivity 已提交
2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		break;
	}
A
Avi Kivity 已提交
2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906
	case KVM_GET_FPU: {
		struct kvm_fpu fpu;

		memset(&fpu, 0, sizeof fpu);
		r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &fpu, sizeof fpu))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
		struct kvm_fpu fpu;

		r = -EFAULT;
		if (copy_from_user(&fpu, argp, sizeof fpu))
			goto out;
		r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu);
		if (r)
			goto out;
		r = 0;
		break;
	}
2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931
	case KVM_GET_LAPIC: {
		struct kvm_lapic_state lapic;

		memset(&lapic, 0, sizeof lapic);
		r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &lapic, sizeof lapic))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
		struct kvm_lapic_state lapic;

		r = -EFAULT;
		if (copy_from_user(&lapic, argp, sizeof lapic))
			goto out;
		r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);;
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951
	default:
		;
	}
out:
	return r;
}

static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r = -EINVAL;

	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		if (r < 0)
			goto out;
		break;
A
Avi Kivity 已提交
2952 2953 2954 2955
	case KVM_SET_MEMORY_REGION: {
		struct kvm_memory_region kvm_mem;

		r = -EFAULT;
A
Al Viro 已提交
2956
		if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
A
Avi Kivity 已提交
2957
			goto out;
2958
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
A
Avi Kivity 已提交
2959 2960 2961 2962 2963 2964 2965 2966
		if (r)
			goto out;
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
2967
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
2968
			goto out;
2969
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
2970 2971 2972 2973
		if (r)
			goto out;
		break;
	}
2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984
	case KVM_SET_MEMORY_ALIAS: {
		struct kvm_memory_alias alias;

		r = -EFAULT;
		if (copy_from_user(&alias, argp, sizeof alias))
			goto out;
		r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
		if (r)
			goto out;
		break;
	}
2985 2986 2987
	case KVM_CREATE_IRQCHIP:
		r = -ENOMEM;
		kvm->vpic = kvm_create_pic(kvm);
E
Eddie Dong 已提交
2988 2989 2990 2991 2992 2993 2994 2995
		if (kvm->vpic) {
			r = kvm_ioapic_init(kvm);
			if (r) {
				kfree(kvm->vpic);
				kvm->vpic = NULL;
				goto out;
			}
		}
2996 2997 2998 2999 3000 3001 3002 3003 3004 3005
		else
			goto out;
		break;
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
		if (copy_from_user(&irq_event, argp, sizeof irq_event))
			goto out;
		if (irqchip_in_kernel(kvm)) {
3006
			mutex_lock(&kvm->lock);
3007 3008 3009 3010
			if (irq_event.irq < 16)
				kvm_pic_set_irq(pic_irqchip(kvm),
					irq_event.irq,
					irq_event.level);
E
Eddie Dong 已提交
3011 3012 3013
			kvm_ioapic_set_irq(kvm->vioapic,
					irq_event.irq,
					irq_event.level);
3014
			mutex_unlock(&kvm->lock);
3015 3016 3017 3018
			r = 0;
		}
		break;
	}
3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
		struct kvm_irqchip chip;

		r = -EFAULT;
		if (copy_from_user(&chip, argp, sizeof chip))
			goto out;
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
			goto out;
		r = kvm_vm_ioctl_get_irqchip(kvm, &chip);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &chip, sizeof chip))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
		struct kvm_irqchip chip;

		r = -EFAULT;
		if (copy_from_user(&chip, argp, sizeof chip))
			goto out;
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
			goto out;
		r = kvm_vm_ioctl_set_irqchip(kvm, &chip);
		if (r)
			goto out;
		r = 0;
		break;
	}
3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069
	default:
		;
	}
out:
	return r;
}

static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
				  unsigned long address,
				  int *type)
{
	struct kvm *kvm = vma->vm_file->private_data;
	unsigned long pgoff;
	struct page *page;

	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
A
Avi Kivity 已提交
3070
	page = gfn_to_page(kvm, pgoff);
3071 3072 3073
	if (!page)
		return NOPAGE_SIGBUS;
	get_page(page);
3074 3075 3076
	if (type != NULL)
		*type = VM_FAULT_MINOR;

3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104
	return page;
}

static struct vm_operations_struct kvm_vm_vm_ops = {
	.nopage = kvm_vm_nopage,
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

static struct file_operations kvm_vm_fops = {
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
	.compat_ioctl   = kvm_vm_ioctl,
	.mmap           = kvm_vm_mmap,
};

static int kvm_dev_ioctl_create_vm(void)
{
	int fd, r;
	struct inode *inode;
	struct file *file;
	struct kvm *kvm;

	kvm = kvm_create_vm();
3105 3106 3107 3108 3109 3110
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
	r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
	if (r) {
		kvm_destroy_vm(kvm);
		return r;
3111 3112
	}

A
Avi Kivity 已提交
3113
	kvm->filp = file;
3114 3115 3116 3117 3118 3119 3120 3121

	return fd;
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
3122
	long r = -EINVAL;
3123 3124 3125

	switch (ioctl) {
	case KVM_GET_API_VERSION:
3126 3127 3128
		r = -EINVAL;
		if (arg)
			goto out;
3129 3130 3131
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
3132 3133 3134
		r = -EINVAL;
		if (arg)
			goto out;
3135 3136
		r = kvm_dev_ioctl_create_vm();
		break;
A
Avi Kivity 已提交
3137
	case KVM_GET_MSR_INDEX_LIST: {
A
Al Viro 已提交
3138
		struct kvm_msr_list __user *user_msr_list = argp;
A
Avi Kivity 已提交
3139 3140 3141 3142 3143 3144 3145
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
			goto out;
		n = msr_list.nmsrs;
3146
		msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
A
Avi Kivity 已提交
3147 3148 3149
		if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
			goto out;
		r = -E2BIG;
3150
		if (n < num_msrs_to_save)
A
Avi Kivity 已提交
3151 3152 3153
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
3154
				 num_msrs_to_save * sizeof(u32)))
A
Avi Kivity 已提交
3155
			goto out;
3156 3157 3158 3159 3160
		if (copy_to_user(user_msr_list->indices
				 + num_msrs_to_save * sizeof(u32),
				 &emulated_msrs,
				 ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
			goto out;
A
Avi Kivity 已提交
3161
		r = 0;
A
Avi Kivity 已提交
3162
		break;
A
Avi Kivity 已提交
3163
	}
3164 3165 3166 3167 3168
	case KVM_CHECK_EXTENSION: {
		int ext = (long)argp;

		switch (ext) {
		case KVM_CAP_IRQCHIP:
E
Eddie Dong 已提交
3169
		case KVM_CAP_HLT:
3170 3171 3172 3173 3174 3175
			r = 1;
			break;
		default:
			r = 0;
			break;
		}
3176
		break;
3177
	}
3178 3179 3180 3181
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
3182
		r = 2 * PAGE_SIZE;
3183
		break;
A
Avi Kivity 已提交
3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196
	default:
		;
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
3197
	KVM_MINOR,
A
Avi Kivity 已提交
3198 3199 3200 3201
	"kvm",
	&kvm_chardev_ops,
};

A
Avi Kivity 已提交
3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212
/*
 * Make sure that a cpu that is being hot-unplugged does not have any vcpus
 * cached on it.
 */
static void decache_vcpus_on_cpu(int cpu)
{
	struct kvm *vm;
	struct kvm_vcpu *vcpu;
	int i;

	spin_lock(&kvm_lock);
S
Shaohua Li 已提交
3213
	list_for_each_entry(vm, &vm_list, vm_list)
A
Avi Kivity 已提交
3214
		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
R
Rusty Russell 已提交
3215 3216 3217
			vcpu = vm->vcpus[i];
			if (!vcpu)
				continue;
A
Avi Kivity 已提交
3218 3219 3220 3221 3222 3223 3224 3225 3226 3227
			/*
			 * If the vcpu is locked, then it is running on some
			 * other cpu and therefore it is not cached on the
			 * cpu in question.
			 *
			 * If it's not locked, check the last cpu it executed
			 * on.
			 */
			if (mutex_trylock(&vcpu->mutex)) {
				if (vcpu->cpu == cpu) {
3228
					kvm_x86_ops->vcpu_decache(vcpu);
A
Avi Kivity 已提交
3229 3230 3231 3232 3233 3234 3235 3236
					vcpu->cpu = -1;
				}
				mutex_unlock(&vcpu->mutex);
			}
		}
	spin_unlock(&kvm_lock);
}

3237 3238 3239 3240 3241 3242 3243
static void hardware_enable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_set(cpu, cpus_hardware_enabled);
3244
	kvm_x86_ops->hardware_enable(NULL);
3245 3246 3247 3248 3249 3250 3251 3252 3253 3254
}

static void hardware_disable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (!cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_clear(cpu, cpus_hardware_enabled);
	decache_vcpus_on_cpu(cpu);
3255
	kvm_x86_ops->hardware_disable(NULL);
3256 3257
}

A
Avi Kivity 已提交
3258 3259 3260 3261 3262 3263
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

	switch (val) {
3264 3265
	case CPU_DYING:
	case CPU_DYING_FROZEN:
3266 3267 3268 3269
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
A
Avi Kivity 已提交
3270
	case CPU_UP_CANCELED:
3271
	case CPU_UP_CANCELED_FROZEN:
3272 3273
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
3274
		smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
A
Avi Kivity 已提交
3275
		break;
3276
	case CPU_ONLINE:
3277
	case CPU_ONLINE_FROZEN:
3278 3279
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
3280
		smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
A
Avi Kivity 已提交
3281 3282 3283 3284 3285
		break;
	}
	return NOTIFY_OK;
}

3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
                       void *v)
{
	if (val == SYS_RESTART) {
		/*
		 * Some (well, at least mine) BIOSes hang on reboot if
		 * in vmx root mode.
		 */
		printk(KERN_INFO "kvm: exiting hardware virtualization\n");
		on_each_cpu(hardware_disable, NULL, 0, 1);
	}
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341
void kvm_io_bus_init(struct kvm_io_bus *bus)
{
	memset(bus, 0, sizeof(*bus));
}

void kvm_io_bus_destroy(struct kvm_io_bus *bus)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		kvm_iodevice_destructor(pos);
	}
}

struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		if (pos->in_range(pos, addr))
			return pos;
	}

	return NULL;
}

void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
{
	BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));

	bus->devs[bus->dev_count++] = dev;
}

A
Avi Kivity 已提交
3342 3343 3344 3345 3346
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
	.priority = 20, /* must be > scheduler priority */
};

A
Avi Kivity 已提交
3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357
static u64 stat_get(void *_offset)
{
	unsigned offset = (long)_offset;
	u64 total = 0;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
R
Rusty Russell 已提交
3358 3359 3360
			vcpu = kvm->vcpus[i];
			if (vcpu)
				total += *(u32 *)((void *)vcpu + offset);
A
Avi Kivity 已提交
3361 3362 3363 3364 3365
		}
	spin_unlock(&kvm_lock);
	return total;
}

R
Rusty Russell 已提交
3366
DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n");
A
Avi Kivity 已提交
3367

A
Avi Kivity 已提交
3368 3369 3370 3371
static __init void kvm_init_debug(void)
{
	struct kvm_stats_debugfs_item *p;

A
Al Viro 已提交
3372
	debugfs_dir = debugfs_create_dir("kvm", NULL);
A
Avi Kivity 已提交
3373
	for (p = debugfs_entries; p->name; ++p)
A
Avi Kivity 已提交
3374 3375 3376
		p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
						(void *)(long)p->offset,
						&stat_fops);
A
Avi Kivity 已提交
3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
	debugfs_remove(debugfs_dir);
}

3388 3389
static int kvm_suspend(struct sys_device *dev, pm_message_t state)
{
A
Avi Kivity 已提交
3390
	hardware_disable(NULL);
3391 3392 3393 3394 3395
	return 0;
}

static int kvm_resume(struct sys_device *dev)
{
A
Avi Kivity 已提交
3396
	hardware_enable(NULL);
3397 3398 3399 3400
	return 0;
}

static struct sysdev_class kvm_sysdev_class = {
3401
	.name = "kvm",
3402 3403 3404 3405 3406 3407 3408 3409 3410
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

static struct sys_device kvm_sysdev = {
	.id = 0,
	.cls = &kvm_sysdev_class,
};

A
Avi Kivity 已提交
3411 3412
hpa_t bad_page_address;

3413 3414 3415 3416 3417 3418 3419 3420 3421 3422
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

3423
	kvm_x86_ops->vcpu_load(vcpu, cpu);
3424 3425 3426 3427 3428 3429 3430
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

3431
	kvm_x86_ops->vcpu_put(vcpu);
3432 3433
}

3434
int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size,
3435
		  struct module *module)
A
Avi Kivity 已提交
3436 3437
{
	int r;
Y
Yang, Sheng 已提交
3438
	int cpu;
A
Avi Kivity 已提交
3439

3440
	if (kvm_x86_ops) {
3441 3442 3443 3444
		printk(KERN_ERR "kvm: already loaded the other module\n");
		return -EEXIST;
	}

3445
	if (!ops->cpu_has_kvm_support()) {
A
Avi Kivity 已提交
3446 3447 3448
		printk(KERN_ERR "kvm: no hardware support\n");
		return -EOPNOTSUPP;
	}
3449
	if (ops->disabled_by_bios()) {
A
Avi Kivity 已提交
3450 3451 3452 3453
		printk(KERN_ERR "kvm: disabled by bios\n");
		return -EOPNOTSUPP;
	}

3454
	kvm_x86_ops = ops;
3455

3456
	r = kvm_x86_ops->hardware_setup();
A
Avi Kivity 已提交
3457
	if (r < 0)
3458
		goto out;
A
Avi Kivity 已提交
3459

Y
Yang, Sheng 已提交
3460 3461
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
3462
				kvm_x86_ops->check_processor_compatibility,
Y
Yang, Sheng 已提交
3463 3464 3465 3466 3467
				&r, 0, 1);
		if (r < 0)
			goto out_free_0;
	}

3468
	on_each_cpu(hardware_enable, NULL, 0, 1);
A
Avi Kivity 已提交
3469 3470 3471
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
		goto out_free_1;
A
Avi Kivity 已提交
3472 3473
	register_reboot_notifier(&kvm_reboot_notifier);

3474 3475 3476 3477 3478 3479 3480 3481
	r = sysdev_class_register(&kvm_sysdev_class);
	if (r)
		goto out_free_2;

	r = sysdev_register(&kvm_sysdev);
	if (r)
		goto out_free_3;

3482 3483 3484 3485 3486 3487 3488 3489
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
					   __alignof__(struct kvm_vcpu), 0, 0);
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
		goto out_free_4;
	}

A
Avi Kivity 已提交
3490 3491 3492 3493 3494 3495 3496 3497
	kvm_chardev_ops.owner = module;

	r = misc_register(&kvm_dev);
	if (r) {
		printk (KERN_ERR "kvm: misc device register failed\n");
		goto out_free;
	}

3498 3499 3500
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

3501 3502 3503
	kvm_mmu_set_nonpresent_ptes(0ull, 0ull);

	return 0;
A
Avi Kivity 已提交
3504 3505

out_free:
3506 3507
	kmem_cache_destroy(kvm_vcpu_cache);
out_free_4:
3508 3509 3510 3511
	sysdev_unregister(&kvm_sysdev);
out_free_3:
	sysdev_class_unregister(&kvm_sysdev_class);
out_free_2:
A
Avi Kivity 已提交
3512
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
3513 3514
	unregister_cpu_notifier(&kvm_cpu_notifier);
out_free_1:
3515
	on_each_cpu(hardware_disable, NULL, 0, 1);
Y
Yang, Sheng 已提交
3516
out_free_0:
3517
	kvm_x86_ops->hardware_unsetup();
3518
out:
3519
	kvm_x86_ops = NULL;
A
Avi Kivity 已提交
3520 3521 3522
	return r;
}

3523
void kvm_exit_x86(void)
A
Avi Kivity 已提交
3524 3525
{
	misc_deregister(&kvm_dev);
3526
	kmem_cache_destroy(kvm_vcpu_cache);
3527 3528
	sysdev_unregister(&kvm_sysdev);
	sysdev_class_unregister(&kvm_sysdev_class);
A
Avi Kivity 已提交
3529
	unregister_reboot_notifier(&kvm_reboot_notifier);
3530
	unregister_cpu_notifier(&kvm_cpu_notifier);
3531
	on_each_cpu(hardware_disable, NULL, 0, 1);
3532 3533
	kvm_x86_ops->hardware_unsetup();
	kvm_x86_ops = NULL;
A
Avi Kivity 已提交
3534 3535 3536 3537 3538
}

static __init int kvm_init(void)
{
	static struct page *bad_page;
3539 3540
	int r;

3541 3542 3543 3544
	r = kvm_mmu_module_init();
	if (r)
		goto out4;

A
Avi Kivity 已提交
3545 3546
	kvm_init_debug();

3547 3548
	kvm_init_msr_list();

A
Avi Kivity 已提交
3549 3550 3551 3552 3553 3554 3555 3556
	if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
		r = -ENOMEM;
		goto out;
	}

	bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
	memset(__va(bad_page_address), 0, PAGE_SIZE);

3557
	return 0;
A
Avi Kivity 已提交
3558 3559 3560

out:
	kvm_exit_debug();
3561 3562
	kvm_mmu_module_exit();
out4:
A
Avi Kivity 已提交
3563 3564 3565 3566 3567 3568 3569
	return r;
}

static __exit void kvm_exit(void)
{
	kvm_exit_debug();
	__free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
3570
	kvm_mmu_module_exit();
A
Avi Kivity 已提交
3571 3572 3573 3574 3575
}

module_init(kvm_init)
module_exit(kvm_exit)

3576 3577
EXPORT_SYMBOL_GPL(kvm_init_x86);
EXPORT_SYMBOL_GPL(kvm_exit_x86);