kvm_main.c 79.9 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

#include "kvm.h"
A
Avi Kivity 已提交
19 20
#include "x86_emulate.h"
#include "segment_descriptor.h"
21
#include "irq.h"
A
Avi Kivity 已提交
22 23 24 25 26 27 28 29 30 31 32 33 34

#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
35
#include <linux/sysdev.h>
A
Avi Kivity 已提交
36
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
37
#include <linux/sched.h>
38 39
#include <linux/cpumask.h>
#include <linux/smp.h>
40
#include <linux/anon_inodes.h>
41
#include <linux/profile.h>
A
Avi Kivity 已提交
42

A
Avi Kivity 已提交
43 44 45 46 47
#include <asm/processor.h>
#include <asm/msr.h>
#include <asm/io.h>
#include <asm/uaccess.h>
#include <asm/desc.h>
A
Avi Kivity 已提交
48 49 50 51

MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

52 53 54
static DEFINE_SPINLOCK(kvm_lock);
static LIST_HEAD(vm_list);

55 56
static cpumask_t cpus_hardware_enabled;

57
struct kvm_x86_ops *kvm_x86_ops;
58 59
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
60

61 62
static __read_mostly struct preempt_ops kvm_preempt_ops;

A
Avi Kivity 已提交
63
#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x)
A
Avi Kivity 已提交
64 65 66

static struct kvm_stats_debugfs_item {
	const char *name;
A
Avi Kivity 已提交
67
	int offset;
A
Avi Kivity 已提交
68 69
	struct dentry *dentry;
} debugfs_entries[] = {
A
Avi Kivity 已提交
70 71 72 73 74 75 76 77 78 79
	{ "pf_fixed", STAT_OFFSET(pf_fixed) },
	{ "pf_guest", STAT_OFFSET(pf_guest) },
	{ "tlb_flush", STAT_OFFSET(tlb_flush) },
	{ "invlpg", STAT_OFFSET(invlpg) },
	{ "exits", STAT_OFFSET(exits) },
	{ "io_exits", STAT_OFFSET(io_exits) },
	{ "mmio_exits", STAT_OFFSET(mmio_exits) },
	{ "signal_exits", STAT_OFFSET(signal_exits) },
	{ "irq_window", STAT_OFFSET(irq_window_exits) },
	{ "halt_exits", STAT_OFFSET(halt_exits) },
E
Eddie Dong 已提交
80
	{ "halt_wakeup", STAT_OFFSET(halt_wakeup) },
A
Avi Kivity 已提交
81 82
	{ "request_irq", STAT_OFFSET(request_irq_exits) },
	{ "irq_exits", STAT_OFFSET(irq_exits) },
83
	{ "light_exits", STAT_OFFSET(light_exits) },
84
	{ "efer_reload", STAT_OFFSET(efer_reload) },
A
Avi Kivity 已提交
85
	{ NULL }
A
Avi Kivity 已提交
86 87 88 89 90 91
};

static struct dentry *debugfs_dir;

#define MAX_IO_MSRS 256

92 93 94 95
#define CR0_RESERVED_BITS						\
	(~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
			  | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
			  | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
96 97 98 99 100 101
#define CR4_RESERVED_BITS						\
	(~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
			  | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE	\
			  | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR	\
			  | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))

102
#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
A
Avi Kivity 已提交
103 104
#define EFER_RESERVED_BITS 0xfffffffffffff2fe

105
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
106 107 108 109 110 111 112 113 114
// LDT or TSS descriptor in the GDT. 16 bytes.
struct segment_descriptor_64 {
	struct segment_descriptor s;
	u32 base_higher;
	u32 pad_zero;
};

#endif

A
Avi Kivity 已提交
115 116 117
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);

A
Avi Kivity 已提交
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
unsigned long segment_base(u16 selector)
{
	struct descriptor_table gdt;
	struct segment_descriptor *d;
	unsigned long table_base;
	typedef unsigned long ul;
	unsigned long v;

	if (selector == 0)
		return 0;

	asm ("sgdt %0" : "=m"(gdt));
	table_base = gdt.base;

	if (selector & 4) {           /* from ldt */
		u16 ldt_selector;

		asm ("sldt %0" : "=g"(ldt_selector));
		table_base = segment_base(ldt_selector);
	}
	d = (struct segment_descriptor *)(table_base + (selector & ~7));
	v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
140
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
141 142 143 144 145 146 147 148
	if (d->system == 0
	    && (d->type == 2 || d->type == 9 || d->type == 11))
		v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
#endif
	return v;
}
EXPORT_SYMBOL_GPL(segment_base);

149 150 151 152 153
static inline int valid_vcpu(int n)
{
	return likely(n >= 0 && n < KVM_MAX_VCPUS);
}

154 155 156 157 158 159
void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
	if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
		return;

	vcpu->guest_fpu_loaded = 1;
160 161
	fx_save(&vcpu->host_fx_image);
	fx_restore(&vcpu->guest_fx_image);
162 163 164 165 166 167 168 169 170
}
EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);

void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
	if (!vcpu->guest_fpu_loaded)
		return;

	vcpu->guest_fpu_loaded = 0;
171 172
	fx_save(&vcpu->guest_fx_image);
	fx_restore(&vcpu->host_fx_image);
173 174 175
}
EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);

A
Avi Kivity 已提交
176 177 178 179
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
static void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
180
{
181 182
	int cpu;

A
Avi Kivity 已提交
183
	mutex_lock(&vcpu->mutex);
184 185
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
186
	kvm_x86_ops->vcpu_load(vcpu, cpu);
187
	put_cpu();
A
Avi Kivity 已提交
188 189 190 191
}

static void vcpu_put(struct kvm_vcpu *vcpu)
{
192
	preempt_disable();
193
	kvm_x86_ops->vcpu_put(vcpu);
194 195
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
196 197 198
	mutex_unlock(&vcpu->mutex);
}

199 200 201 202 203 204
static void ack_flush(void *_completed)
{
}

void kvm_flush_remote_tlbs(struct kvm *kvm)
{
205
	int i, cpu;
206 207 208 209
	cpumask_t cpus;
	struct kvm_vcpu *vcpu;

	cpus_clear(cpus);
R
Rusty Russell 已提交
210 211 212 213
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		vcpu = kvm->vcpus[i];
		if (!vcpu)
			continue;
214 215 216 217
		if (test_and_set_bit(KVM_TLB_FLUSH, &vcpu->requests))
			continue;
		cpu = vcpu->cpu;
		if (cpu != -1 && cpu != raw_smp_processor_id())
218
			cpu_set(cpu, cpus);
219
	}
220
	smp_call_function_mask(cpus, ack_flush, NULL, 1);
221 222
}

R
Rusty Russell 已提交
223 224 225 226 227 228 229 230 231 232
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->mmu.root_hpa = INVALID_PAGE;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
233 234 235 236
	if (!irqchip_in_kernel(kvm) || id == 0)
		vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
	else
		vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED;
E
Eddie Dong 已提交
237
	init_waitqueue_head(&vcpu->wq);
R
Rusty Russell 已提交
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail_free_run;
	}
	vcpu->pio_data = page_address(page);

	r = kvm_mmu_create(vcpu);
	if (r < 0)
		goto fail_free_pio_data;

	return 0;

fail_free_pio_data:
	free_page((unsigned long)vcpu->pio_data);
fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
	return -ENOMEM;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
	kvm_mmu_destroy(vcpu);
271 272
	if (vcpu->apic)
		hrtimer_cancel(&vcpu->apic->timer.dev);
E
Eddie Dong 已提交
273
	kvm_free_apic(vcpu->apic);
R
Rusty Russell 已提交
274 275 276 277 278
	free_page((unsigned long)vcpu->pio_data);
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

279
static struct kvm *kvm_create_vm(void)
A
Avi Kivity 已提交
280 281 282 283
{
	struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);

	if (!kvm)
284
		return ERR_PTR(-ENOMEM);
A
Avi Kivity 已提交
285

286
	kvm_io_bus_init(&kvm->pio_bus);
S
Shaohua Li 已提交
287
	mutex_init(&kvm->lock);
A
Avi Kivity 已提交
288
	INIT_LIST_HEAD(&kvm->active_mmu_pages);
289
	kvm_io_bus_init(&kvm->mmio_bus);
290 291 292
	spin_lock(&kvm_lock);
	list_add(&kvm->vm_list, &vm_list);
	spin_unlock(&kvm_lock);
293 294 295
	return kvm;
}

A
Avi Kivity 已提交
296 297 298 299 300 301 302 303 304 305 306
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
	int i;

	if (!dont || free->phys_mem != dont->phys_mem)
		if (free->phys_mem) {
			for (i = 0; i < free->npages; ++i)
A
Avi Kivity 已提交
307 308
				if (free->phys_mem[i])
					__free_page(free->phys_mem[i]);
A
Avi Kivity 已提交
309 310 311 312 313 314
			vfree(free->phys_mem);
		}

	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
		vfree(free->dirty_bitmap);

A
Al Viro 已提交
315
	free->phys_mem = NULL;
A
Avi Kivity 已提交
316
	free->npages = 0;
A
Al Viro 已提交
317
	free->dirty_bitmap = NULL;
A
Avi Kivity 已提交
318 319 320 321 322 323 324
}

static void kvm_free_physmem(struct kvm *kvm)
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i)
A
Al Viro 已提交
325
		kvm_free_physmem_slot(&kvm->memslots[i], NULL);
A
Avi Kivity 已提交
326 327
}

328 329 330 331
static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
{
	int i;

R
Rusty Russell 已提交
332
	for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i)
333 334 335 336 337 338
		if (vcpu->pio.guest_pages[i]) {
			__free_page(vcpu->pio.guest_pages[i]);
			vcpu->pio.guest_pages[i] = NULL;
		}
}

A
Avi Kivity 已提交
339 340 341 342 343 344 345
static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
	vcpu_load(vcpu);
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

A
Avi Kivity 已提交
346 347 348 349
static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;

A
Avi Kivity 已提交
350 351 352 353
	/*
	 * Unpin any mmu pages first.
	 */
	for (i = 0; i < KVM_MAX_VCPUS; ++i)
R
Rusty Russell 已提交
354 355 356 357
		if (kvm->vcpus[i])
			kvm_unload_vcpu_mmu(kvm->vcpus[i]);
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		if (kvm->vcpus[i]) {
358
			kvm_x86_ops->vcpu_free(kvm->vcpus[i]);
R
Rusty Russell 已提交
359 360 361 362
			kvm->vcpus[i] = NULL;
		}
	}

A
Avi Kivity 已提交
363 364
}

365 366
static void kvm_destroy_vm(struct kvm *kvm)
{
367 368 369
	spin_lock(&kvm_lock);
	list_del(&kvm->vm_list);
	spin_unlock(&kvm_lock);
370
	kvm_io_bus_destroy(&kvm->pio_bus);
371
	kvm_io_bus_destroy(&kvm->mmio_bus);
372
	kfree(kvm->vpic);
E
Eddie Dong 已提交
373
	kfree(kvm->vioapic);
A
Avi Kivity 已提交
374 375 376
	kvm_free_vcpus(kvm);
	kvm_free_physmem(kvm);
	kfree(kvm);
377 378 379 380 381 382 383
}

static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

	kvm_destroy_vm(kvm);
A
Avi Kivity 已提交
384 385 386 387 388
	return 0;
}

static void inject_gp(struct kvm_vcpu *vcpu)
{
389
	kvm_x86_ops->inject_gp(vcpu, 0);
A
Avi Kivity 已提交
390 391
}

392 393 394 395
/*
 * Load the pae pdptrs.  Return true is they are all valid.
 */
static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
A
Avi Kivity 已提交
396 397
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
398
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
A
Avi Kivity 已提交
399 400
	int i;
	u64 *pdpt;
401
	int ret;
A
Avi Kivity 已提交
402
	struct page *page;
R
Rusty Russell 已提交
403
	u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)];
A
Avi Kivity 已提交
404

S
Shaohua Li 已提交
405
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
406
	page = gfn_to_page(vcpu->kvm, pdpt_gfn);
R
Rusty Russell 已提交
407 408 409 410 411
	if (!page) {
		ret = 0;
		goto out;
	}

A
Avi Kivity 已提交
412
	pdpt = kmap_atomic(page, KM_USER0);
R
Rusty Russell 已提交
413 414
	memcpy(pdpte, pdpt+offset, sizeof(pdpte));
	kunmap_atomic(pdpt, KM_USER0);
A
Avi Kivity 已提交
415

R
Rusty Russell 已提交
416 417
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
		if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) {
418 419 420
			ret = 0;
			goto out;
		}
A
Avi Kivity 已提交
421
	}
R
Rusty Russell 已提交
422
	ret = 1;
A
Avi Kivity 已提交
423

R
Rusty Russell 已提交
424
	memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs));
425
out:
S
Shaohua Li 已提交
426
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
427

428
	return ret;
A
Avi Kivity 已提交
429 430 431 432
}

void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
433
	if (cr0 & CR0_RESERVED_BITS) {
A
Avi Kivity 已提交
434 435 436 437 438 439
		printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
		       cr0, vcpu->cr0);
		inject_gp(vcpu);
		return;
	}

440
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
A
Avi Kivity 已提交
441 442 443 444 445
		printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
		inject_gp(vcpu);
		return;
	}

446
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
A
Avi Kivity 已提交
447 448 449 450 451 452
		printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
		       "and a clear PE flag\n");
		inject_gp(vcpu);
		return;
	}

453
	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
454
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
455 456 457 458 459 460 461 462 463
		if ((vcpu->shadow_efer & EFER_LME)) {
			int cs_db, cs_l;

			if (!is_pae(vcpu)) {
				printk(KERN_DEBUG "set_cr0: #GP, start paging "
				       "in long mode while PAE is disabled\n");
				inject_gp(vcpu);
				return;
			}
464
			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
A
Avi Kivity 已提交
465 466 467 468 469 470 471 472 473
			if (cs_l) {
				printk(KERN_DEBUG "set_cr0: #GP, start paging "
				       "in long mode while CS.L == 1\n");
				inject_gp(vcpu);
				return;

			}
		} else
#endif
474
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
A
Avi Kivity 已提交
475 476 477 478 479 480 481 482
			printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
			       "reserved bits\n");
			inject_gp(vcpu);
			return;
		}

	}

483
	kvm_x86_ops->set_cr0(vcpu, cr0);
A
Avi Kivity 已提交
484 485
	vcpu->cr0 = cr0;

S
Shaohua Li 已提交
486
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
487
	kvm_mmu_reset_context(vcpu);
S
Shaohua Li 已提交
488
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
489 490 491 492 493 494 495 496 497 498 499 500
	return;
}
EXPORT_SYMBOL_GPL(set_cr0);

void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
{
	set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
}
EXPORT_SYMBOL_GPL(lmsw);

void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
501
	if (cr4 & CR4_RESERVED_BITS) {
A
Avi Kivity 已提交
502 503 504 505 506
		printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
		inject_gp(vcpu);
		return;
	}

A
Avi Kivity 已提交
507
	if (is_long_mode(vcpu)) {
508
		if (!(cr4 & X86_CR4_PAE)) {
A
Avi Kivity 已提交
509 510 511 512 513
			printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
			       "in long mode\n");
			inject_gp(vcpu);
			return;
		}
514
	} else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE)
515
		   && !load_pdptrs(vcpu, vcpu->cr3)) {
A
Avi Kivity 已提交
516 517
		printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
		inject_gp(vcpu);
518
		return;
A
Avi Kivity 已提交
519 520
	}

521
	if (cr4 & X86_CR4_VMXE) {
A
Avi Kivity 已提交
522 523 524 525
		printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
		inject_gp(vcpu);
		return;
	}
526
	kvm_x86_ops->set_cr4(vcpu, cr4);
R
Rusty Russell 已提交
527
	vcpu->cr4 = cr4;
S
Shaohua Li 已提交
528
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
529
	kvm_mmu_reset_context(vcpu);
S
Shaohua Li 已提交
530
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
531 532 533 534 535
}
EXPORT_SYMBOL_GPL(set_cr4);

void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
{
A
Avi Kivity 已提交
536
	if (is_long_mode(vcpu)) {
537
		if (cr3 & CR3_L_MODE_RESERVED_BITS) {
A
Avi Kivity 已提交
538 539 540 541 542
			printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
			inject_gp(vcpu);
			return;
		}
	} else {
543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562
		if (is_pae(vcpu)) {
			if (cr3 & CR3_PAE_RESERVED_BITS) {
				printk(KERN_DEBUG
				       "set_cr3: #GP, reserved bits\n");
				inject_gp(vcpu);
				return;
			}
			if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
				printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
				       "reserved bits\n");
				inject_gp(vcpu);
				return;
			}
		} else {
			if (cr3 & CR3_NONPAE_RESERVED_BITS) {
				printk(KERN_DEBUG
				       "set_cr3: #GP, reserved bits\n");
				inject_gp(vcpu);
				return;
			}
A
Avi Kivity 已提交
563 564 565
		}
	}

S
Shaohua Li 已提交
566
	mutex_lock(&vcpu->kvm->lock);
567 568 569 570 571 572 573 574 575 576 577
	/*
	 * Does the new cr3 value map to physical memory? (Note, we
	 * catch an invalid cr3 even in real-mode, because it would
	 * cause trouble later on when we turn on paging anyway.)
	 *
	 * A real CPU would silently accept an invalid cr3 and would
	 * attempt to use it - with largely undefined (and often hard
	 * to debug) behavior on the guest side.
	 */
	if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
		inject_gp(vcpu);
578 579
	else {
		vcpu->cr3 = cr3;
580
		vcpu->mmu.new_cr3(vcpu);
581
	}
S
Shaohua Li 已提交
582
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
583 584 585 586 587
}
EXPORT_SYMBOL_GPL(set_cr3);

void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
{
588
	if (cr8 & CR8_RESERVED_BITS) {
A
Avi Kivity 已提交
589 590 591 592
		printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
		inject_gp(vcpu);
		return;
	}
E
Eddie Dong 已提交
593 594 595 596
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_lapic_set_tpr(vcpu, cr8);
	else
		vcpu->cr8 = cr8;
A
Avi Kivity 已提交
597 598 599
}
EXPORT_SYMBOL_GPL(set_cr8);

600 601
unsigned long get_cr8(struct kvm_vcpu *vcpu)
{
E
Eddie Dong 已提交
602 603 604 605
	if (irqchip_in_kernel(vcpu->kvm))
		return kvm_lapic_get_cr8(vcpu);
	else
		return vcpu->cr8;
606 607 608 609 610
}
EXPORT_SYMBOL_GPL(get_cr8);

u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
E
Eddie Dong 已提交
611 612 613 614
	if (irqchip_in_kernel(vcpu->kvm))
		return vcpu->apic_base;
	else
		return vcpu->apic_base;
615 616 617 618 619
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
{
E
Eddie Dong 已提交
620 621 622 623 624
	/* TODO: reserve bits check */
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_lapic_set_base(vcpu, data);
	else
		vcpu->apic_base = data;
625 626 627
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

A
Avi Kivity 已提交
628 629
void fx_init(struct kvm_vcpu *vcpu)
{
630
	unsigned after_mxcsr_mask;
A
Avi Kivity 已提交
631

632 633
	/* Initialize guest FPU by resetting ours and saving into guest's */
	preempt_disable();
634
	fx_save(&vcpu->host_fx_image);
A
Avi Kivity 已提交
635
	fpu_init();
636 637
	fx_save(&vcpu->guest_fx_image);
	fx_restore(&vcpu->host_fx_image);
638
	preempt_enable();
A
Avi Kivity 已提交
639

640
	vcpu->cr0 |= X86_CR0_ET;
641 642 643 644
	after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
	vcpu->guest_fx_image.mxcsr = 0x1f80;
	memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask,
	       0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
A
Avi Kivity 已提交
645 646 647 648 649 650 651 652 653
}
EXPORT_SYMBOL_GPL(fx_init);

/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
 */
654 655
static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
					  struct kvm_memory_region *mem)
A
Avi Kivity 已提交
656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
{
	int r;
	gfn_t base_gfn;
	unsigned long npages;
	unsigned long i;
	struct kvm_memory_slot *memslot;
	struct kvm_memory_slot old, new;

	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
	if (mem->slot >= KVM_MEMORY_SLOTS)
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

	memslot = &kvm->memslots[mem->slot];
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

S
Shaohua Li 已提交
682
	mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708

	new = old = *memslot;

	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	/* Disallow changing a memory slot's size. */
	r = -EINVAL;
	if (npages && old.npages && npages != old.npages)
		goto out_unlock;

	/* Check for overlaps */
	r = -EEXIST;
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *s = &kvm->memslots[i];

		if (s == memslot)
			continue;
		if (!((base_gfn + npages <= s->base_gfn) ||
		      (base_gfn >= s->base_gfn + s->npages)))
			goto out_unlock;
	}

	/* Deallocate if slot is being removed */
	if (!npages)
A
Al Viro 已提交
709
		new.phys_mem = NULL;
A
Avi Kivity 已提交
710 711 712

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
713
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
714 715 716 717 718 719 720 721

	r = -ENOMEM;

	/* Allocate if a slot is being created */
	if (npages && !new.phys_mem) {
		new.phys_mem = vmalloc(npages * sizeof(struct page *));

		if (!new.phys_mem)
L
Laurent Vivier 已提交
722
			goto out_unlock;
A
Avi Kivity 已提交
723 724 725 726 727 728

		memset(new.phys_mem, 0, npages * sizeof(struct page *));
		for (i = 0; i < npages; ++i) {
			new.phys_mem[i] = alloc_page(GFP_HIGHUSER
						     | __GFP_ZERO);
			if (!new.phys_mem[i])
L
Laurent Vivier 已提交
729
				goto out_unlock;
730
			set_page_private(new.phys_mem[i],0);
A
Avi Kivity 已提交
731 732 733 734 735 736 737 738 739
		}
	}

	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
		unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;

		new.dirty_bitmap = vmalloc(dirty_bytes);
		if (!new.dirty_bitmap)
L
Laurent Vivier 已提交
740
			goto out_unlock;
A
Avi Kivity 已提交
741 742 743 744 745 746 747 748
		memset(new.dirty_bitmap, 0, dirty_bytes);
	}

	if (mem->slot >= kvm->nmemslots)
		kvm->nmemslots = mem->slot + 1;

	*memslot = new;

749 750
	kvm_mmu_slot_remove_write_access(kvm, mem->slot);
	kvm_flush_remote_tlbs(kvm);
A
Avi Kivity 已提交
751

S
Shaohua Li 已提交
752
	mutex_unlock(&kvm->lock);
A
Avi Kivity 已提交
753 754 755 756 757

	kvm_free_physmem_slot(&old, &new);
	return 0;

out_unlock:
S
Shaohua Li 已提交
758
	mutex_unlock(&kvm->lock);
A
Avi Kivity 已提交
759 760 761 762 763 764 765 766
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
}

/*
 * Get (and clear) the dirty memory log for a memory slot.
 */
767 768
static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
				      struct kvm_dirty_log *log)
A
Avi Kivity 已提交
769 770 771 772 773 774
{
	struct kvm_memory_slot *memslot;
	int r, i;
	int n;
	unsigned long any = 0;

S
Shaohua Li 已提交
775
	mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
776 777 778 779 780 781 782 783 784 785

	r = -EINVAL;
	if (log->slot >= KVM_MEMORY_SLOTS)
		goto out;

	memslot = &kvm->memslots[log->slot];
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

786
	n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
A
Avi Kivity 已提交
787

788
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
789 790 791 792 793 794
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

795 796 797 798 799 800
	/* If nothing is dirty, don't bother messing with page tables. */
	if (any) {
		kvm_mmu_slot_remove_write_access(kvm, log->slot);
		kvm_flush_remote_tlbs(kvm);
		memset(memslot->dirty_bitmap, 0, n);
	}
A
Avi Kivity 已提交
801 802 803 804

	r = 0;

out:
S
Shaohua Li 已提交
805
	mutex_unlock(&kvm->lock);
A
Avi Kivity 已提交
806 807 808
	return r;
}

809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834
/*
 * Set a new alias region.  Aliases map a portion of physical memory into
 * another portion.  This is useful for memory windows, for example the PC
 * VGA region.
 */
static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
					 struct kvm_memory_alias *alias)
{
	int r, n;
	struct kvm_mem_alias *p;

	r = -EINVAL;
	/* General sanity checks */
	if (alias->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (alias->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
	if (alias->slot >= KVM_ALIAS_SLOTS)
		goto out;
	if (alias->guest_phys_addr + alias->memory_size
	    < alias->guest_phys_addr)
		goto out;
	if (alias->target_phys_addr + alias->memory_size
	    < alias->target_phys_addr)
		goto out;

S
Shaohua Li 已提交
835
	mutex_lock(&kvm->lock);
836 837 838 839 840 841 842 843 844 845 846

	p = &kvm->aliases[alias->slot];
	p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
	p->npages = alias->memory_size >> PAGE_SHIFT;
	p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;

	for (n = KVM_ALIAS_SLOTS; n > 0; --n)
		if (kvm->aliases[n - 1].npages)
			break;
	kvm->naliases = n;

847
	kvm_mmu_zap_all(kvm);
848

S
Shaohua Li 已提交
849
	mutex_unlock(&kvm->lock);
850 851 852 853 854 855 856

	return 0;

out:
	return r;
}

857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872
static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
		memcpy (&chip->chip.pic,
			&pic_irqchip(kvm)->pics[0],
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
		memcpy (&chip->chip.pic,
			&pic_irqchip(kvm)->pics[1],
			sizeof(struct kvm_pic_state));
		break;
873 874 875 876 877
	case KVM_IRQCHIP_IOAPIC:
		memcpy (&chip->chip.ioapic,
			ioapic_irqchip(kvm),
			sizeof(struct kvm_ioapic_state));
		break;
878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
		memcpy (&pic_irqchip(kvm)->pics[0],
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
		memcpy (&pic_irqchip(kvm)->pics[1],
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
		break;
901 902 903 904 905
	case KVM_IRQCHIP_IOAPIC:
		memcpy (ioapic_irqchip(kvm),
			&chip->chip.ioapic,
			sizeof(struct kvm_ioapic_state));
		break;
906 907 908 909 910 911 912 913
	default:
		r = -EINVAL;
		break;
	}
	kvm_pic_update_irq(pic_irqchip(kvm));
	return r;
}

914 915 916 917 918 919 920 921 922 923 924 925 926 927 928
static gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
{
	int i;
	struct kvm_mem_alias *alias;

	for (i = 0; i < kvm->naliases; ++i) {
		alias = &kvm->aliases[i];
		if (gfn >= alias->base_gfn
		    && gfn < alias->base_gfn + alias->npages)
			return alias->target_gfn + gfn - alias->base_gfn;
	}
	return gfn;
}

static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
929 930 931 932 933 934 935 936 937 938
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return memslot;
	}
A
Al Viro 已提交
939
	return NULL;
A
Avi Kivity 已提交
940
}
941 942 943 944 945 946

struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	gfn = unalias_gfn(kvm, gfn);
	return __gfn_to_memslot(kvm, gfn);
}
A
Avi Kivity 已提交
947

A
Avi Kivity 已提交
948 949 950 951
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *slot;

952 953
	gfn = unalias_gfn(kvm, gfn);
	slot = __gfn_to_memslot(kvm, gfn);
A
Avi Kivity 已提交
954 955 956 957 958 959
	if (!slot)
		return NULL;
	return slot->phys_mem[gfn - slot->base_gfn];
}
EXPORT_SYMBOL_GPL(gfn_to_page);

R
Rusty Russell 已提交
960
/* WARNING: Does not work on aliased pages. */
A
Avi Kivity 已提交
961 962
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
963
	struct kvm_memory_slot *memslot;
A
Avi Kivity 已提交
964

R
Rusty Russell 已提交
965 966 967
	memslot = __gfn_to_memslot(kvm, gfn);
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
968

R
Rusty Russell 已提交
969 970 971
		/* avoid RMW */
		if (!test_bit(rel_gfn, memslot->dirty_bitmap))
			set_bit(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
972 973 974
	}
}

975
int emulator_read_std(unsigned long addr,
976
			     void *val,
A
Avi Kivity 已提交
977
			     unsigned int bytes,
978
			     struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
979 980 981 982 983 984 985 986
{
	void *data = val;

	while (bytes) {
		gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
		unsigned long pfn;
A
Avi Kivity 已提交
987 988
		struct page *page;
		void *page_virt;
A
Avi Kivity 已提交
989 990 991 992

		if (gpa == UNMAPPED_GVA)
			return X86EMUL_PROPAGATE_FAULT;
		pfn = gpa >> PAGE_SHIFT;
A
Avi Kivity 已提交
993 994
		page = gfn_to_page(vcpu->kvm, pfn);
		if (!page)
A
Avi Kivity 已提交
995
			return X86EMUL_UNHANDLEABLE;
A
Avi Kivity 已提交
996
		page_virt = kmap_atomic(page, KM_USER0);
A
Avi Kivity 已提交
997

A
Avi Kivity 已提交
998
		memcpy(data, page_virt + offset, tocopy);
A
Avi Kivity 已提交
999

A
Avi Kivity 已提交
1000
		kunmap_atomic(page_virt, KM_USER0);
A
Avi Kivity 已提交
1001 1002 1003 1004 1005 1006 1007 1008

		bytes -= tocopy;
		data += tocopy;
		addr += tocopy;
	}

	return X86EMUL_CONTINUE;
}
1009
EXPORT_SYMBOL_GPL(emulator_read_std);
A
Avi Kivity 已提交
1010 1011

static int emulator_write_std(unsigned long addr,
1012
			      const void *val,
A
Avi Kivity 已提交
1013
			      unsigned int bytes,
1014
			      struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1015
{
1016
	pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes);
A
Avi Kivity 已提交
1017 1018 1019
	return X86EMUL_UNHANDLEABLE;
}

E
Eddie Dong 已提交
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
/*
 * Only apic need an MMIO device hook, so shortcut now..
 */
static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu,
						gpa_t addr)
{
	struct kvm_io_device *dev;

	if (vcpu->apic) {
		dev = &vcpu->apic->dev;
		if (dev->in_range(dev, addr))
			return dev;
	}
	return NULL;
}

1036 1037 1038
static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
						gpa_t addr)
{
E
Eddie Dong 已提交
1039 1040 1041 1042 1043 1044
	struct kvm_io_device *dev;

	dev = vcpu_find_pervcpu_dev(vcpu, addr);
	if (dev == NULL)
		dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr);
	return dev;
1045 1046
}

1047 1048 1049 1050 1051 1052
static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
					       gpa_t addr)
{
	return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr);
}

A
Avi Kivity 已提交
1053
static int emulator_read_emulated(unsigned long addr,
1054
				  void *val,
A
Avi Kivity 已提交
1055
				  unsigned int bytes,
1056
				  struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1057
{
1058 1059
	struct kvm_io_device *mmio_dev;
	gpa_t                 gpa;
A
Avi Kivity 已提交
1060 1061 1062 1063 1064

	if (vcpu->mmio_read_completed) {
		memcpy(val, vcpu->mmio_data, bytes);
		vcpu->mmio_read_completed = 0;
		return X86EMUL_CONTINUE;
1065
	} else if (emulator_read_std(addr, val, bytes, vcpu)
A
Avi Kivity 已提交
1066 1067
		   == X86EMUL_CONTINUE)
		return X86EMUL_CONTINUE;
A
Avi Kivity 已提交
1068

1069 1070 1071
	gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
	if (gpa == UNMAPPED_GVA)
		return X86EMUL_PROPAGATE_FAULT;
A
Avi Kivity 已提交
1072

1073 1074 1075 1076 1077 1078 1079
	/*
	 * Is this MMIO handled locally?
	 */
	mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
	if (mmio_dev) {
		kvm_iodevice_read(mmio_dev, gpa, bytes, val);
		return X86EMUL_CONTINUE;
A
Avi Kivity 已提交
1080
	}
1081 1082 1083 1084 1085 1086 1087

	vcpu->mmio_needed = 1;
	vcpu->mmio_phys_addr = gpa;
	vcpu->mmio_size = bytes;
	vcpu->mmio_is_write = 0;

	return X86EMUL_UNHANDLEABLE;
A
Avi Kivity 已提交
1088 1089
}

1090
static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
1091
			       const void *val, int bytes)
1092 1093 1094 1095 1096 1097
{
	struct page *page;
	void *virt;

	if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
		return 0;
A
Avi Kivity 已提交
1098 1099
	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
	if (!page)
1100
		return 0;
1101
	mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
1102
	virt = kmap_atomic(page, KM_USER0);
1103
	kvm_mmu_pte_write(vcpu, gpa, val, bytes);
1104
	memcpy(virt + offset_in_page(gpa), val, bytes);
1105 1106 1107 1108
	kunmap_atomic(virt, KM_USER0);
	return 1;
}

1109 1110 1111
static int emulator_write_emulated_onepage(unsigned long addr,
					   const void *val,
					   unsigned int bytes,
1112
					   struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1113
{
1114 1115
	struct kvm_io_device *mmio_dev;
	gpa_t                 gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
A
Avi Kivity 已提交
1116

1117
	if (gpa == UNMAPPED_GVA) {
1118
		kvm_x86_ops->inject_page_fault(vcpu, addr, 2);
A
Avi Kivity 已提交
1119
		return X86EMUL_PROPAGATE_FAULT;
1120
	}
A
Avi Kivity 已提交
1121

1122 1123 1124
	if (emulator_write_phys(vcpu, gpa, val, bytes))
		return X86EMUL_CONTINUE;

1125 1126 1127 1128 1129 1130 1131 1132 1133
	/*
	 * Is this MMIO handled locally?
	 */
	mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
	if (mmio_dev) {
		kvm_iodevice_write(mmio_dev, gpa, bytes, val);
		return X86EMUL_CONTINUE;
	}

A
Avi Kivity 已提交
1134 1135 1136 1137
	vcpu->mmio_needed = 1;
	vcpu->mmio_phys_addr = gpa;
	vcpu->mmio_size = bytes;
	vcpu->mmio_is_write = 1;
1138
	memcpy(vcpu->mmio_data, val, bytes);
A
Avi Kivity 已提交
1139 1140 1141 1142

	return X86EMUL_CONTINUE;
}

1143
int emulator_write_emulated(unsigned long addr,
1144 1145
				   const void *val,
				   unsigned int bytes,
1146
				   struct kvm_vcpu *vcpu)
1147 1148 1149 1150 1151 1152
{
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
		int rc, now;

		now = -addr & ~PAGE_MASK;
1153
		rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
1154 1155 1156 1157 1158 1159
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
		val += now;
		bytes -= now;
	}
1160
	return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
1161
}
1162
EXPORT_SYMBOL_GPL(emulator_write_emulated);
1163

A
Avi Kivity 已提交
1164
static int emulator_cmpxchg_emulated(unsigned long addr,
1165 1166
				     const void *old,
				     const void *new,
A
Avi Kivity 已提交
1167
				     unsigned int bytes,
1168
				     struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1169 1170 1171 1172 1173 1174 1175
{
	static int reported;

	if (!reported) {
		reported = 1;
		printk(KERN_WARNING "kvm: emulating exchange as write\n");
	}
1176
	return emulator_write_emulated(addr, new, bytes, vcpu);
A
Avi Kivity 已提交
1177 1178 1179 1180
}

static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
1181
	return kvm_x86_ops->get_segment_base(vcpu, seg);
A
Avi Kivity 已提交
1182 1183 1184 1185 1186 1187 1188 1189 1190
}

int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
{
	return X86EMUL_CONTINUE;
}

int emulate_clts(struct kvm_vcpu *vcpu)
{
1191
	kvm_x86_ops->set_cr0(vcpu, vcpu->cr0 & ~X86_CR0_TS);
A
Avi Kivity 已提交
1192 1193 1194 1195 1196 1197 1198 1199 1200
	return X86EMUL_CONTINUE;
}

int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
{
	struct kvm_vcpu *vcpu = ctxt->vcpu;

	switch (dr) {
	case 0 ... 3:
1201
		*dest = kvm_x86_ops->get_dr(vcpu, dr);
A
Avi Kivity 已提交
1202 1203
		return X86EMUL_CONTINUE;
	default:
1204
		pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr);
A
Avi Kivity 已提交
1205 1206 1207 1208 1209 1210 1211 1212 1213
		return X86EMUL_UNHANDLEABLE;
	}
}

int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
{
	unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
	int exception;

1214
	kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
A
Avi Kivity 已提交
1215 1216 1217 1218 1219 1220 1221
	if (exception) {
		/* FIXME: better handling */
		return X86EMUL_UNHANDLEABLE;
	}
	return X86EMUL_CONTINUE;
}

1222
void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context)
A
Avi Kivity 已提交
1223 1224 1225
{
	static int reported;
	u8 opcodes[4];
1226
	unsigned long rip = vcpu->rip;
A
Avi Kivity 已提交
1227 1228
	unsigned long rip_linear;

1229
	rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS);
A
Avi Kivity 已提交
1230 1231 1232 1233

	if (reported)
		return;

1234
	emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu);
A
Avi Kivity 已提交
1235

1236 1237
	printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n",
	       context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
A
Avi Kivity 已提交
1238 1239
	reported = 1;
}
1240
EXPORT_SYMBOL_GPL(kvm_report_emulation_failure);
A
Avi Kivity 已提交
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258

struct x86_emulate_ops emulate_ops = {
	.read_std            = emulator_read_std,
	.write_std           = emulator_write_std,
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
};

int emulate_instruction(struct kvm_vcpu *vcpu,
			struct kvm_run *run,
			unsigned long cr2,
			u16 error_code)
{
	struct x86_emulate_ctxt emulate_ctxt;
	int r;
	int cs_db, cs_l;

1259
	vcpu->mmio_fault_cr2 = cr2;
1260
	kvm_x86_ops->cache_regs(vcpu);
A
Avi Kivity 已提交
1261

1262
	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
A
Avi Kivity 已提交
1263 1264

	emulate_ctxt.vcpu = vcpu;
1265
	emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu);
A
Avi Kivity 已提交
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
	emulate_ctxt.cr2 = cr2;
	emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
		? X86EMUL_MODE_REAL : cs_l
		? X86EMUL_MODE_PROT64 :	cs_db
		? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;

	if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
		emulate_ctxt.cs_base = 0;
		emulate_ctxt.ds_base = 0;
		emulate_ctxt.es_base = 0;
		emulate_ctxt.ss_base = 0;
	} else {
		emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
		emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
		emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
		emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
	}

	emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
	emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);

	vcpu->mmio_is_write = 0;
1288
	vcpu->pio.string = 0;
A
Avi Kivity 已提交
1289
	r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1290 1291
	if (vcpu->pio.string)
		return EMULATE_DO_MMIO;
A
Avi Kivity 已提交
1292 1293

	if ((r || vcpu->mmio_is_write) && run) {
1294
		run->exit_reason = KVM_EXIT_MMIO;
A
Avi Kivity 已提交
1295 1296 1297 1298 1299 1300 1301
		run->mmio.phys_addr = vcpu->mmio_phys_addr;
		memcpy(run->mmio.data, vcpu->mmio_data, 8);
		run->mmio.len = vcpu->mmio_size;
		run->mmio.is_write = vcpu->mmio_is_write;
	}

	if (r) {
1302 1303
		if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
			return EMULATE_DONE;
A
Avi Kivity 已提交
1304
		if (!vcpu->mmio_needed) {
1305
			kvm_report_emulation_failure(vcpu, "mmio");
A
Avi Kivity 已提交
1306 1307 1308 1309 1310
			return EMULATE_FAIL;
		}
		return EMULATE_DO_MMIO;
	}

1311 1312
	kvm_x86_ops->decache_regs(vcpu);
	kvm_x86_ops->set_rflags(vcpu, emulate_ctxt.eflags);
A
Avi Kivity 已提交
1313

1314 1315
	if (vcpu->mmio_is_write) {
		vcpu->mmio_needed = 0;
A
Avi Kivity 已提交
1316
		return EMULATE_DO_MMIO;
1317
	}
A
Avi Kivity 已提交
1318 1319 1320 1321 1322

	return EMULATE_DONE;
}
EXPORT_SYMBOL_GPL(emulate_instruction);

E
Eddie Dong 已提交
1323 1324 1325
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1326
static void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1327
{
E
Eddie Dong 已提交
1328 1329 1330 1331 1332 1333 1334
	DECLARE_WAITQUEUE(wait, current);

	add_wait_queue(&vcpu->wq, &wait);

	/*
	 * We will block until either an interrupt or a signal wakes us up
	 */
1335 1336 1337 1338
	while (!kvm_cpu_has_interrupt(vcpu)
	       && !signal_pending(current)
	       && vcpu->mp_state != VCPU_MP_STATE_RUNNABLE
	       && vcpu->mp_state != VCPU_MP_STATE_SIPI_RECEIVED) {
E
Eddie Dong 已提交
1339 1340 1341 1342 1343
		set_current_state(TASK_INTERRUPTIBLE);
		vcpu_put(vcpu);
		schedule();
		vcpu_load(vcpu);
	}
1344

1345
	__set_current_state(TASK_RUNNING);
E
Eddie Dong 已提交
1346 1347 1348 1349 1350
	remove_wait_queue(&vcpu->wq, &wait);
}

int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
1351
	++vcpu->stat.halt_exits;
E
Eddie Dong 已提交
1352
	if (irqchip_in_kernel(vcpu->kvm)) {
1353 1354 1355 1356
		vcpu->mp_state = VCPU_MP_STATE_HALTED;
		kvm_vcpu_block(vcpu);
		if (vcpu->mp_state != VCPU_MP_STATE_RUNNABLE)
			return -EINTR;
E
Eddie Dong 已提交
1357 1358 1359 1360 1361
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
1362 1363 1364
}
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

1365 1366 1367 1368
int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
	unsigned long nr, a0, a1, a2, a3, a4, a5, ret;

1369
	kvm_x86_ops->cache_regs(vcpu);
1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
	ret = -KVM_EINVAL;
#ifdef CONFIG_X86_64
	if (is_long_mode(vcpu)) {
		nr = vcpu->regs[VCPU_REGS_RAX];
		a0 = vcpu->regs[VCPU_REGS_RDI];
		a1 = vcpu->regs[VCPU_REGS_RSI];
		a2 = vcpu->regs[VCPU_REGS_RDX];
		a3 = vcpu->regs[VCPU_REGS_RCX];
		a4 = vcpu->regs[VCPU_REGS_R8];
		a5 = vcpu->regs[VCPU_REGS_R9];
	} else
#endif
	{
		nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
		a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
		a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
		a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
		a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
		a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
		a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
	}
	switch (nr) {
	default:
J
Jeff Dike 已提交
1393
		run->hypercall.nr = nr;
1394 1395 1396 1397 1398 1399 1400 1401
		run->hypercall.args[0] = a0;
		run->hypercall.args[1] = a1;
		run->hypercall.args[2] = a2;
		run->hypercall.args[3] = a3;
		run->hypercall.args[4] = a4;
		run->hypercall.args[5] = a5;
		run->hypercall.ret = ret;
		run->hypercall.longmode = is_long_mode(vcpu);
1402
		kvm_x86_ops->decache_regs(vcpu);
1403
		return 0;
1404 1405
	}
	vcpu->regs[VCPU_REGS_RAX] = ret;
1406
	kvm_x86_ops->decache_regs(vcpu);
1407 1408 1409 1410
	return 1;
}
EXPORT_SYMBOL_GPL(kvm_hypercall);

A
Avi Kivity 已提交
1411 1412 1413 1414 1415 1416 1417 1418 1419
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
{
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
}

void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
{
	struct descriptor_table dt = { limit, base };

1420
	kvm_x86_ops->set_gdt(vcpu, &dt);
A
Avi Kivity 已提交
1421 1422 1423 1424 1425 1426
}

void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
{
	struct descriptor_table dt = { limit, base };

1427
	kvm_x86_ops->set_idt(vcpu, &dt);
A
Avi Kivity 已提交
1428 1429 1430 1431 1432 1433
}

void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
		   unsigned long *rflags)
{
	lmsw(vcpu, msw);
1434
	*rflags = kvm_x86_ops->get_rflags(vcpu);
A
Avi Kivity 已提交
1435 1436 1437 1438
}

unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
{
1439
	kvm_x86_ops->decache_cr4_guest_bits(vcpu);
A
Avi Kivity 已提交
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
	switch (cr) {
	case 0:
		return vcpu->cr0;
	case 2:
		return vcpu->cr2;
	case 3:
		return vcpu->cr3;
	case 4:
		return vcpu->cr4;
	default:
		vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
		return 0;
	}
}

void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
		     unsigned long *rflags)
{
	switch (cr) {
	case 0:
		set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1461
		*rflags = kvm_x86_ops->get_rflags(vcpu);
A
Avi Kivity 已提交
1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476
		break;
	case 2:
		vcpu->cr2 = val;
		break;
	case 3:
		set_cr3(vcpu, val);
		break;
	case 4:
		set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
		break;
	default:
		vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
	}
}

I
Ingo Molnar 已提交
1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
/*
 * Register the para guest with the host:
 */
static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
{
	struct kvm_vcpu_para_state *para_state;
	hpa_t para_state_hpa, hypercall_hpa;
	struct page *para_state_page;
	unsigned char *hypercall;
	gpa_t hypercall_gpa;

	printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
	printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);

	/*
	 * Needs to be page aligned:
	 */
	if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
		goto err_gp;

	para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
	printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
	if (is_error_hpa(para_state_hpa))
		goto err_gp;

1502
	mark_page_dirty(vcpu->kvm, para_state_gpa >> PAGE_SHIFT);
I
Ingo Molnar 已提交
1503
	para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
1504
	para_state = kmap(para_state_page);
I
Ingo Molnar 已提交
1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531

	printk(KERN_DEBUG "....  guest version: %d\n", para_state->guest_version);
	printk(KERN_DEBUG "....           size: %d\n", para_state->size);

	para_state->host_version = KVM_PARA_API_VERSION;
	/*
	 * We cannot support guests that try to register themselves
	 * with a newer API version than the host supports:
	 */
	if (para_state->guest_version > KVM_PARA_API_VERSION) {
		para_state->ret = -KVM_EINVAL;
		goto err_kunmap_skip;
	}

	hypercall_gpa = para_state->hypercall_gpa;
	hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
	printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
	if (is_error_hpa(hypercall_hpa)) {
		para_state->ret = -KVM_EINVAL;
		goto err_kunmap_skip;
	}

	printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
	vcpu->para_state_page = para_state_page;
	vcpu->para_state_gpa = para_state_gpa;
	vcpu->hypercall_gpa = hypercall_gpa;

1532
	mark_page_dirty(vcpu->kvm, hypercall_gpa >> PAGE_SHIFT);
I
Ingo Molnar 已提交
1533 1534
	hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
				KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
1535
	kvm_x86_ops->patch_hypercall(vcpu, hypercall);
I
Ingo Molnar 已提交
1536 1537 1538 1539
	kunmap_atomic(hypercall, KM_USER1);

	para_state->ret = 0;
err_kunmap_skip:
1540
	kunmap(para_state_page);
I
Ingo Molnar 已提交
1541 1542 1543 1544 1545
	return 0;
err_gp:
	return 1;
}

1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564
int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data;

	switch (msr) {
	case 0xc0010010: /* SYSCFG */
	case 0xc0010015: /* HWCR */
	case MSR_IA32_PLATFORM_ID:
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MC0_CTL:
	case MSR_IA32_MCG_STATUS:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MC0_MISC:
	case MSR_IA32_MC0_MISC+4:
	case MSR_IA32_MC0_MISC+8:
	case MSR_IA32_MC0_MISC+12:
	case MSR_IA32_MC0_MISC+16:
	case MSR_IA32_UCODE_REV:
A
Avi Kivity 已提交
1565
	case MSR_IA32_PERF_STATUS:
1566
	case MSR_IA32_EBL_CR_POWERON:
1567 1568 1569 1570 1571
		/* MTRR registers */
	case 0xfe:
	case 0x200 ... 0x2ff:
		data = 0;
		break;
A
Avi Kivity 已提交
1572 1573 1574
	case 0xcd: /* fsb frequency */
		data = 3;
		break;
1575
	case MSR_IA32_APICBASE:
1576
		data = kvm_get_apic_base(vcpu);
1577
		break;
1578 1579 1580
	case MSR_IA32_MISC_ENABLE:
		data = vcpu->ia32_misc_enable_msr;
		break;
1581 1582 1583 1584 1585 1586
#ifdef CONFIG_X86_64
	case MSR_EFER:
		data = vcpu->shadow_efer;
		break;
#endif
	default:
1587
		pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
1588 1589 1590 1591 1592 1593 1594
		return 1;
	}
	*pdata = data;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

A
Avi Kivity 已提交
1595 1596 1597 1598 1599
/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1600
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
A
Avi Kivity 已提交
1601
{
1602
	return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
A
Avi Kivity 已提交
1603 1604
}

1605
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1606

1607
static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
A
Avi Kivity 已提交
1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
{
	if (efer & EFER_RESERVED_BITS) {
		printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
		       efer);
		inject_gp(vcpu);
		return;
	}

	if (is_paging(vcpu)
	    && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
		printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
		inject_gp(vcpu);
		return;
	}

1623
	kvm_x86_ops->set_efer(vcpu, efer);
1624

A
Avi Kivity 已提交
1625 1626 1627 1628 1629 1630 1631 1632
	efer &= ~EFER_LMA;
	efer |= vcpu->shadow_efer & EFER_LMA;

	vcpu->shadow_efer = efer;
}

#endif

1633 1634 1635 1636 1637 1638 1639 1640 1641
int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	switch (msr) {
#ifdef CONFIG_X86_64
	case MSR_EFER:
		set_efer(vcpu, data);
		break;
#endif
	case MSR_IA32_MC0_STATUS:
1642
		pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1643 1644
		       __FUNCTION__, data);
		break;
1645
	case MSR_IA32_MCG_STATUS:
1646
		pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
1647 1648
			__FUNCTION__, data);
		break;
1649 1650 1651 1652 1653
	case MSR_IA32_UCODE_REV:
	case MSR_IA32_UCODE_WRITE:
	case 0x200 ... 0x2ff: /* MTRRs */
		break;
	case MSR_IA32_APICBASE:
1654
		kvm_set_apic_base(vcpu, data);
1655
		break;
1656 1657 1658
	case MSR_IA32_MISC_ENABLE:
		vcpu->ia32_misc_enable_msr = data;
		break;
I
Ingo Molnar 已提交
1659 1660 1661 1662 1663 1664
	/*
	 * This is the 'probe whether the host is KVM' logic:
	 */
	case MSR_KVM_API_MAGIC:
		return vcpu_register_para(vcpu, data);

1665
	default:
1666
		pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr);
1667 1668 1669 1670 1671 1672
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);

A
Avi Kivity 已提交
1673 1674 1675 1676 1677
/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1678
int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
A
Avi Kivity 已提交
1679
{
1680
	return kvm_x86_ops->set_msr(vcpu, msr_index, data);
A
Avi Kivity 已提交
1681 1682 1683 1684
}

void kvm_resched(struct kvm_vcpu *vcpu)
{
1685 1686
	if (!need_resched())
		return;
A
Avi Kivity 已提交
1687 1688 1689 1690
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

1691 1692 1693 1694 1695 1696
void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
{
	int i;
	u32 function;
	struct kvm_cpuid_entry *e, *best;

1697
	kvm_x86_ops->cache_regs(vcpu);
1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722
	function = vcpu->regs[VCPU_REGS_RAX];
	vcpu->regs[VCPU_REGS_RAX] = 0;
	vcpu->regs[VCPU_REGS_RBX] = 0;
	vcpu->regs[VCPU_REGS_RCX] = 0;
	vcpu->regs[VCPU_REGS_RDX] = 0;
	best = NULL;
	for (i = 0; i < vcpu->cpuid_nent; ++i) {
		e = &vcpu->cpuid_entries[i];
		if (e->function == function) {
			best = e;
			break;
		}
		/*
		 * Both basic or both extended?
		 */
		if (((e->function ^ function) & 0x80000000) == 0)
			if (!best || e->function > best->function)
				best = e;
	}
	if (best) {
		vcpu->regs[VCPU_REGS_RAX] = best->eax;
		vcpu->regs[VCPU_REGS_RBX] = best->ebx;
		vcpu->regs[VCPU_REGS_RCX] = best->ecx;
		vcpu->regs[VCPU_REGS_RDX] = best->edx;
	}
1723 1724
	kvm_x86_ops->decache_regs(vcpu);
	kvm_x86_ops->skip_emulated_instruction(vcpu);
1725 1726 1727
}
EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);

1728
static int pio_copy_data(struct kvm_vcpu *vcpu)
1729
{
1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755
	void *p = vcpu->pio_data;
	void *q;
	unsigned bytes;
	int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1;

	q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
		 PAGE_KERNEL);
	if (!q) {
		free_pio_guest_pages(vcpu);
		return -ENOMEM;
	}
	q += vcpu->pio.guest_page_offset;
	bytes = vcpu->pio.size * vcpu->pio.cur_count;
	if (vcpu->pio.in)
		memcpy(q, p, bytes);
	else
		memcpy(p, q, bytes);
	q -= vcpu->pio.guest_page_offset;
	vunmap(q);
	free_pio_guest_pages(vcpu);
	return 0;
}

static int complete_pio(struct kvm_vcpu *vcpu)
{
	struct kvm_pio_request *io = &vcpu->pio;
1756
	long delta;
1757
	int r;
1758

1759
	kvm_x86_ops->cache_regs(vcpu);
1760 1761

	if (!io->string) {
1762 1763
		if (io->in)
			memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data,
1764 1765
			       io->size);
	} else {
1766 1767 1768
		if (io->in) {
			r = pio_copy_data(vcpu);
			if (r) {
1769
				kvm_x86_ops->cache_regs(vcpu);
1770 1771 1772 1773
				return r;
			}
		}

1774 1775
		delta = 1;
		if (io->rep) {
1776
			delta *= io->cur_count;
1777 1778 1779 1780 1781 1782
			/*
			 * The size of the register should really depend on
			 * current address size.
			 */
			vcpu->regs[VCPU_REGS_RCX] -= delta;
		}
1783
		if (io->down)
1784 1785
			delta = -delta;
		delta *= io->size;
1786
		if (io->in)
1787 1788 1789 1790 1791
			vcpu->regs[VCPU_REGS_RDI] += delta;
		else
			vcpu->regs[VCPU_REGS_RSI] += delta;
	}

1792
	kvm_x86_ops->decache_regs(vcpu);
1793

1794 1795 1796 1797
	io->count -= io->cur_count;
	io->cur_count = 0;

	return 0;
1798 1799
}

1800 1801 1802
static void kernel_pio(struct kvm_io_device *pio_dev,
		       struct kvm_vcpu *vcpu,
		       void *pd)
1803 1804 1805
{
	/* TODO: String I/O for in kernel device */

1806
	mutex_lock(&vcpu->kvm->lock);
1807 1808 1809
	if (vcpu->pio.in)
		kvm_iodevice_read(pio_dev, vcpu->pio.port,
				  vcpu->pio.size,
1810
				  pd);
1811 1812 1813
	else
		kvm_iodevice_write(pio_dev, vcpu->pio.port,
				   vcpu->pio.size,
1814
				   pd);
1815
	mutex_unlock(&vcpu->kvm->lock);
1816 1817 1818 1819 1820 1821 1822 1823 1824
}

static void pio_string_write(struct kvm_io_device *pio_dev,
			     struct kvm_vcpu *vcpu)
{
	struct kvm_pio_request *io = &vcpu->pio;
	void *pd = vcpu->pio_data;
	int i;

1825
	mutex_lock(&vcpu->kvm->lock);
1826 1827 1828 1829 1830 1831
	for (i = 0; i < io->cur_count; i++) {
		kvm_iodevice_write(pio_dev, io->port,
				   io->size,
				   pd);
		pd += io->size;
	}
1832
	mutex_unlock(&vcpu->kvm->lock);
1833 1834
}

L
Laurent Vivier 已提交
1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851
int kvm_emulate_pio (struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
		  int size, unsigned port)
{
	struct kvm_io_device *pio_dev;

	vcpu->run->exit_reason = KVM_EXIT_IO;
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
	vcpu->run->io.size = vcpu->pio.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1;
	vcpu->run->io.port = vcpu->pio.port = port;
	vcpu->pio.in = in;
	vcpu->pio.string = 0;
	vcpu->pio.down = 0;
	vcpu->pio.guest_page_offset = 0;
	vcpu->pio.rep = 0;

1852
	kvm_x86_ops->cache_regs(vcpu);
L
Laurent Vivier 已提交
1853
	memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4);
1854
	kvm_x86_ops->decache_regs(vcpu);
L
Laurent Vivier 已提交
1855

1856 1857
	kvm_x86_ops->skip_emulated_instruction(vcpu);

L
Laurent Vivier 已提交
1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
	pio_dev = vcpu_find_pio_dev(vcpu, port);
	if (pio_dev) {
		kernel_pio(pio_dev, vcpu, vcpu->pio_data);
		complete_pio(vcpu);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_emulate_pio);

int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
		  int size, unsigned long count, int down,
1870 1871 1872
		  gva_t address, int rep, unsigned port)
{
	unsigned now, in_page;
1873
	int i, ret = 0;
1874 1875
	int nr_pages = 1;
	struct page *page;
1876
	struct kvm_io_device *pio_dev;
1877 1878 1879

	vcpu->run->exit_reason = KVM_EXIT_IO;
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
L
Laurent Vivier 已提交
1880
	vcpu->run->io.size = vcpu->pio.size = size;
1881
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
L
Laurent Vivier 已提交
1882 1883
	vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count;
	vcpu->run->io.port = vcpu->pio.port = port;
1884
	vcpu->pio.in = in;
L
Laurent Vivier 已提交
1885
	vcpu->pio.string = 1;
1886 1887 1888 1889 1890
	vcpu->pio.down = down;
	vcpu->pio.guest_page_offset = offset_in_page(address);
	vcpu->pio.rep = rep;

	if (!count) {
1891
		kvm_x86_ops->skip_emulated_instruction(vcpu);
1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912
		return 1;
	}

	if (!down)
		in_page = PAGE_SIZE - offset_in_page(address);
	else
		in_page = offset_in_page(address) + size;
	now = min(count, (unsigned long)in_page / size);
	if (!now) {
		/*
		 * String I/O straddles page boundary.  Pin two guest pages
		 * so that we satisfy atomicity constraints.  Do just one
		 * transaction to avoid complexity.
		 */
		nr_pages = 2;
		now = 1;
	}
	if (down) {
		/*
		 * String I/O in reverse.  Yuck.  Kill the guest, fix later.
		 */
1913
		pr_unimpl(vcpu, "guest string pio down\n");
1914 1915 1916 1917 1918 1919
		inject_gp(vcpu);
		return 1;
	}
	vcpu->run->io.count = now;
	vcpu->pio.cur_count = now;

1920 1921 1922
	if (vcpu->pio.cur_count == vcpu->pio.count)
		kvm_x86_ops->skip_emulated_instruction(vcpu);

1923
	for (i = 0; i < nr_pages; ++i) {
S
Shaohua Li 已提交
1924
		mutex_lock(&vcpu->kvm->lock);
1925 1926 1927 1928
		page = gva_to_page(vcpu, address + i * PAGE_SIZE);
		if (page)
			get_page(page);
		vcpu->pio.guest_pages[i] = page;
S
Shaohua Li 已提交
1929
		mutex_unlock(&vcpu->kvm->lock);
1930 1931 1932 1933 1934 1935 1936
		if (!page) {
			inject_gp(vcpu);
			free_pio_guest_pages(vcpu);
			return 1;
		}
	}

L
Laurent Vivier 已提交
1937
	pio_dev = vcpu_find_pio_dev(vcpu, port);
1938 1939 1940 1941 1942 1943 1944 1945 1946 1947
	if (!vcpu->pio.in) {
		/* string PIO write */
		ret = pio_copy_data(vcpu);
		if (ret >= 0 && pio_dev) {
			pio_string_write(pio_dev, vcpu);
			complete_pio(vcpu);
			if (vcpu->pio.count == 0)
				ret = 1;
		}
	} else if (pio_dev)
1948
		pr_unimpl(vcpu, "no string pio read support yet, "
1949 1950 1951 1952
		       "port %x size %d count %ld\n",
			port, size, count);

	return ret;
1953
}
L
Laurent Vivier 已提交
1954
EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
1955

1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027
/*
 * Check if userspace requested an interrupt window, and that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
					  struct kvm_run *kvm_run)
{
	return (!vcpu->irq_summary &&
		kvm_run->request_interrupt_window &&
		vcpu->interrupt_window_open &&
		(kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF));
}

static void post_kvm_run_save(struct kvm_vcpu *vcpu,
			      struct kvm_run *kvm_run)
{
	kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
	kvm_run->cr8 = get_cr8(vcpu);
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_run->ready_for_interrupt_injection = 1;
	else
		kvm_run->ready_for_interrupt_injection =
					(vcpu->interrupt_window_open &&
					 vcpu->irq_summary == 0);
}

static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
{
	int r;

	if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) {
		printk("vcpu %d received sipi with vector # %x\n",
		       vcpu->vcpu_id, vcpu->sipi_vector);
		kvm_lapic_reset(vcpu);
		kvm_x86_ops->vcpu_reset(vcpu);
		vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
	}

preempted:
	if (vcpu->guest_debug.enabled)
		kvm_x86_ops->guest_debug_pre(vcpu);

again:
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		goto out;

	preempt_disable();

	kvm_x86_ops->prepare_guest_switch(vcpu);
	kvm_load_guest_fpu(vcpu);

	local_irq_disable();

	if (signal_pending(current)) {
		local_irq_enable();
		preempt_enable();
		r = -EINTR;
		kvm_run->exit_reason = KVM_EXIT_INTR;
		++vcpu->stat.signal_exits;
		goto out;
	}

	if (irqchip_in_kernel(vcpu->kvm))
		kvm_x86_ops->inject_pending_irq(vcpu);
	else if (!vcpu->mmio_read_completed)
		kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run);

	vcpu->guest_mode = 1;
2028
	kvm_guest_enter();
2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040

	if (vcpu->requests)
		if (test_and_clear_bit(KVM_TLB_FLUSH, &vcpu->requests))
			kvm_x86_ops->tlb_flush(vcpu);

	kvm_x86_ops->run(vcpu, kvm_run);

	vcpu->guest_mode = 0;
	local_irq_enable();

	++vcpu->stat.exits;

2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
	/*
	 * We must have an instruction between local_irq_enable() and
	 * kvm_guest_exit(), so the timer interrupt isn't delayed by
	 * the interrupt shadow.  The stat.exits increment will do nicely.
	 * But we need to prevent reordering, hence this barrier():
	 */
	barrier();

	kvm_guest_exit();

2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087
	preempt_enable();

	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
		kvm_x86_ops->cache_regs(vcpu);
		profile_hit(KVM_PROFILING, (void *)vcpu->rip);
	}

	r = kvm_x86_ops->handle_exit(kvm_run, vcpu);

	if (r > 0) {
		if (dm_request_for_irq_injection(vcpu, kvm_run)) {
			r = -EINTR;
			kvm_run->exit_reason = KVM_EXIT_INTR;
			++vcpu->stat.request_irq_exits;
			goto out;
		}
		if (!need_resched()) {
			++vcpu->stat.light_exits;
			goto again;
		}
	}

out:
	if (r > 0) {
		kvm_resched(vcpu);
		goto preempted;
	}

	post_kvm_run_save(vcpu, kvm_run);

	return r;
}


A
Avi Kivity 已提交
2088
static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
2089 2090
{
	int r;
A
Avi Kivity 已提交
2091
	sigset_t sigsaved;
A
Avi Kivity 已提交
2092

A
Avi Kivity 已提交
2093
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2094

2095 2096 2097 2098 2099 2100
	if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) {
		kvm_vcpu_block(vcpu);
		vcpu_put(vcpu);
		return -EAGAIN;
	}

A
Avi Kivity 已提交
2101 2102 2103
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);

2104
	/* re-sync apic's tpr */
2105 2106
	if (!irqchip_in_kernel(vcpu->kvm))
		set_cr8(vcpu, kvm_run->cr8);
2107

2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125
	if (vcpu->pio.cur_count) {
		r = complete_pio(vcpu);
		if (r)
			goto out;
	}

	if (vcpu->mmio_needed) {
		memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
		vcpu->mmio_read_completed = 1;
		vcpu->mmio_needed = 0;
		r = emulate_instruction(vcpu, kvm_run,
					vcpu->mmio_fault_cr2, 0);
		if (r == EMULATE_DO_MMIO) {
			/*
			 * Read-modify-write.  Back to userspace.
			 */
			r = 0;
			goto out;
2126
		}
A
Avi Kivity 已提交
2127 2128
	}

2129
	if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
2130
		kvm_x86_ops->cache_regs(vcpu);
2131
		vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
2132
		kvm_x86_ops->decache_regs(vcpu);
2133 2134
	}

2135
	r = __vcpu_run(vcpu, kvm_run);
A
Avi Kivity 已提交
2136

2137
out:
A
Avi Kivity 已提交
2138 2139 2140
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

A
Avi Kivity 已提交
2141 2142 2143 2144
	vcpu_put(vcpu);
	return r;
}

A
Avi Kivity 已提交
2145 2146
static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
				   struct kvm_regs *regs)
A
Avi Kivity 已提交
2147
{
A
Avi Kivity 已提交
2148
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2149

2150
	kvm_x86_ops->cache_regs(vcpu);
A
Avi Kivity 已提交
2151 2152 2153 2154 2155 2156 2157 2158 2159

	regs->rax = vcpu->regs[VCPU_REGS_RAX];
	regs->rbx = vcpu->regs[VCPU_REGS_RBX];
	regs->rcx = vcpu->regs[VCPU_REGS_RCX];
	regs->rdx = vcpu->regs[VCPU_REGS_RDX];
	regs->rsi = vcpu->regs[VCPU_REGS_RSI];
	regs->rdi = vcpu->regs[VCPU_REGS_RDI];
	regs->rsp = vcpu->regs[VCPU_REGS_RSP];
	regs->rbp = vcpu->regs[VCPU_REGS_RBP];
2160
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171
	regs->r8 = vcpu->regs[VCPU_REGS_R8];
	regs->r9 = vcpu->regs[VCPU_REGS_R9];
	regs->r10 = vcpu->regs[VCPU_REGS_R10];
	regs->r11 = vcpu->regs[VCPU_REGS_R11];
	regs->r12 = vcpu->regs[VCPU_REGS_R12];
	regs->r13 = vcpu->regs[VCPU_REGS_R13];
	regs->r14 = vcpu->regs[VCPU_REGS_R14];
	regs->r15 = vcpu->regs[VCPU_REGS_R15];
#endif

	regs->rip = vcpu->rip;
2172
	regs->rflags = kvm_x86_ops->get_rflags(vcpu);
A
Avi Kivity 已提交
2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184

	/*
	 * Don't leak debug flags in case they were set for guest debugging
	 */
	if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
		regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);

	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2185 2186
static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
				   struct kvm_regs *regs)
A
Avi Kivity 已提交
2187
{
A
Avi Kivity 已提交
2188
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2189 2190 2191 2192 2193 2194 2195 2196 2197

	vcpu->regs[VCPU_REGS_RAX] = regs->rax;
	vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
	vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
	vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
	vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
	vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
	vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
	vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
2198
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
	vcpu->regs[VCPU_REGS_R8] = regs->r8;
	vcpu->regs[VCPU_REGS_R9] = regs->r9;
	vcpu->regs[VCPU_REGS_R10] = regs->r10;
	vcpu->regs[VCPU_REGS_R11] = regs->r11;
	vcpu->regs[VCPU_REGS_R12] = regs->r12;
	vcpu->regs[VCPU_REGS_R13] = regs->r13;
	vcpu->regs[VCPU_REGS_R14] = regs->r14;
	vcpu->regs[VCPU_REGS_R15] = regs->r15;
#endif

	vcpu->rip = regs->rip;
2210
	kvm_x86_ops->set_rflags(vcpu, regs->rflags);
A
Avi Kivity 已提交
2211

2212
	kvm_x86_ops->decache_regs(vcpu);
A
Avi Kivity 已提交
2213 2214 2215 2216 2217 2218 2219 2220 2221

	vcpu_put(vcpu);

	return 0;
}

static void get_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
2222
	return kvm_x86_ops->get_segment(vcpu, var, seg);
A
Avi Kivity 已提交
2223 2224
}

A
Avi Kivity 已提交
2225 2226
static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				    struct kvm_sregs *sregs)
A
Avi Kivity 已提交
2227 2228
{
	struct descriptor_table dt;
E
Eddie Dong 已提交
2229
	int pending_vec;
A
Avi Kivity 已提交
2230

A
Avi Kivity 已提交
2231
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242

	get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);

	get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);

2243
	kvm_x86_ops->get_idt(vcpu, &dt);
A
Avi Kivity 已提交
2244 2245
	sregs->idt.limit = dt.limit;
	sregs->idt.base = dt.base;
2246
	kvm_x86_ops->get_gdt(vcpu, &dt);
A
Avi Kivity 已提交
2247 2248 2249
	sregs->gdt.limit = dt.limit;
	sregs->gdt.base = dt.base;

2250
	kvm_x86_ops->decache_cr4_guest_bits(vcpu);
A
Avi Kivity 已提交
2251 2252 2253 2254
	sregs->cr0 = vcpu->cr0;
	sregs->cr2 = vcpu->cr2;
	sregs->cr3 = vcpu->cr3;
	sregs->cr4 = vcpu->cr4;
2255
	sregs->cr8 = get_cr8(vcpu);
A
Avi Kivity 已提交
2256
	sregs->efer = vcpu->shadow_efer;
2257
	sregs->apic_base = kvm_get_apic_base(vcpu);
A
Avi Kivity 已提交
2258

E
Eddie Dong 已提交
2259
	if (irqchip_in_kernel(vcpu->kvm)) {
2260 2261
		memset(sregs->interrupt_bitmap, 0,
		       sizeof sregs->interrupt_bitmap);
2262
		pending_vec = kvm_x86_ops->get_irq(vcpu);
E
Eddie Dong 已提交
2263 2264 2265
		if (pending_vec >= 0)
			set_bit(pending_vec, (unsigned long *)sregs->interrupt_bitmap);
	} else
2266 2267
		memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
		       sizeof sregs->interrupt_bitmap);
A
Avi Kivity 已提交
2268 2269 2270 2271 2272 2273 2274 2275 2276

	vcpu_put(vcpu);

	return 0;
}

static void set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
2277
	return kvm_x86_ops->set_segment(vcpu, var, seg);
A
Avi Kivity 已提交
2278 2279
}

A
Avi Kivity 已提交
2280 2281
static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				    struct kvm_sregs *sregs)
A
Avi Kivity 已提交
2282 2283
{
	int mmu_reset_needed = 0;
E
Eddie Dong 已提交
2284
	int i, pending_vec, max_bits;
A
Avi Kivity 已提交
2285 2286
	struct descriptor_table dt;

A
Avi Kivity 已提交
2287
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2288 2289 2290

	dt.limit = sregs->idt.limit;
	dt.base = sregs->idt.base;
2291
	kvm_x86_ops->set_idt(vcpu, &dt);
A
Avi Kivity 已提交
2292 2293
	dt.limit = sregs->gdt.limit;
	dt.base = sregs->gdt.base;
2294
	kvm_x86_ops->set_gdt(vcpu, &dt);
A
Avi Kivity 已提交
2295 2296 2297 2298 2299

	vcpu->cr2 = sregs->cr2;
	mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
	vcpu->cr3 = sregs->cr3;

2300
	set_cr8(vcpu, sregs->cr8);
A
Avi Kivity 已提交
2301 2302

	mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
2303
#ifdef CONFIG_X86_64
2304
	kvm_x86_ops->set_efer(vcpu, sregs->efer);
A
Avi Kivity 已提交
2305
#endif
2306
	kvm_set_apic_base(vcpu, sregs->apic_base);
A
Avi Kivity 已提交
2307

2308
	kvm_x86_ops->decache_cr4_guest_bits(vcpu);
2309

A
Avi Kivity 已提交
2310
	mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
R
Rusty Russell 已提交
2311
	vcpu->cr0 = sregs->cr0;
2312
	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
A
Avi Kivity 已提交
2313 2314

	mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
2315
	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
2316 2317
	if (!is_long_mode(vcpu) && is_pae(vcpu))
		load_pdptrs(vcpu, vcpu->cr3);
A
Avi Kivity 已提交
2318 2319 2320 2321

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

2322 2323 2324 2325 2326 2327 2328
	if (!irqchip_in_kernel(vcpu->kvm)) {
		memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
		       sizeof vcpu->irq_pending);
		vcpu->irq_summary = 0;
		for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i)
			if (vcpu->irq_pending[i])
				__set_bit(i, &vcpu->irq_summary);
E
Eddie Dong 已提交
2329 2330 2331 2332 2333 2334 2335
	} else {
		max_bits = (sizeof sregs->interrupt_bitmap) << 3;
		pending_vec = find_first_bit(
			(const unsigned long *)sregs->interrupt_bitmap,
			max_bits);
		/* Only pending external irq is handled here */
		if (pending_vec < max_bits) {
2336
			kvm_x86_ops->set_irq(vcpu, pending_vec);
E
Eddie Dong 已提交
2337 2338
			printk("Set back pending irq %d\n", pending_vec);
		}
2339
	}
A
Avi Kivity 已提交
2340

2341 2342 2343 2344 2345 2346 2347 2348 2349 2350
	set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);

	set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);

A
Avi Kivity 已提交
2351 2352 2353 2354 2355
	vcpu_put(vcpu);

	return 0;
}

2356 2357 2358 2359 2360 2361 2362 2363 2364 2365
void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

	get_segment(vcpu, &cs, VCPU_SREG_CS);
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

A
Avi Kivity 已提交
2366 2367 2368
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
2369 2370 2371
 *
 * This list is modified at module load time to reflect the
 * capabilities of the host cpu.
A
Avi Kivity 已提交
2372 2373 2374 2375
 */
static u32 msrs_to_save[] = {
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
	MSR_K6_STAR,
2376
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2377 2378 2379 2380 2381
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
	MSR_IA32_TIME_STAMP_COUNTER,
};

2382 2383
static unsigned num_msrs_to_save;

2384 2385 2386 2387
static u32 emulated_msrs[] = {
	MSR_IA32_MISC_ENABLE,
};

2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
static __init void kvm_init_msr_list(void)
{
	u32 dummy[2];
	unsigned i, j;

	for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
		if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
			continue;
		if (j < i)
			msrs_to_save[j] = msrs_to_save[i];
		j++;
	}
	num_msrs_to_save = j;
}
A
Avi Kivity 已提交
2402 2403 2404 2405 2406 2407

/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
2408
	return kvm_set_msr(vcpu, index, *data);
A
Avi Kivity 已提交
2409 2410 2411 2412 2413 2414 2415
}

/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
A
Avi Kivity 已提交
2416
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
A
Avi Kivity 已提交
2417 2418 2419 2420 2421 2422
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
	int i;

A
Avi Kivity 已提交
2423
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438

	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;

	vcpu_put(vcpu);

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
A
Avi Kivity 已提交
2439
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
A
Avi Kivity 已提交
2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
	if (copy_from_user(&msrs, user_msrs, sizeof msrs))
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	r = -ENOMEM;
	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
	entries = vmalloc(size);
	if (!entries)
		goto out;

	r = -EFAULT;
	if (copy_from_user(entries, user_msrs->entries, size))
		goto out_free;

A
Avi Kivity 已提交
2467
	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
A
Avi Kivity 已提交
2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
	vfree(entries);
out:
	return r;
}

/*
 * Translate a guest virtual address to a guest physical address.
 */
A
Avi Kivity 已提交
2486 2487
static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
A
Avi Kivity 已提交
2488 2489 2490 2491
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;

A
Avi Kivity 已提交
2492
	vcpu_load(vcpu);
S
Shaohua Li 已提交
2493
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
2494 2495 2496 2497 2498
	gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;
S
Shaohua Li 已提交
2499
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
2500 2501 2502 2503 2504
	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2505 2506
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
A
Avi Kivity 已提交
2507 2508 2509
{
	if (irq->irq < 0 || irq->irq >= 256)
		return -EINVAL;
E
Eddie Dong 已提交
2510 2511
	if (irqchip_in_kernel(vcpu->kvm))
		return -ENXIO;
A
Avi Kivity 已提交
2512
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2513 2514 2515 2516 2517 2518 2519 2520 2521

	set_bit(irq->irq, vcpu->irq_pending);
	set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);

	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2522 2523
static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
				      struct kvm_debug_guest *dbg)
A
Avi Kivity 已提交
2524 2525 2526
{
	int r;

A
Avi Kivity 已提交
2527
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2528

2529
	r = kvm_x86_ops->set_guest_debug(vcpu, dbg);
A
Avi Kivity 已提交
2530 2531 2532 2533 2534 2535

	vcpu_put(vcpu);

	return r;
}

2536 2537 2538 2539 2540 2541 2542 2543 2544
static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
				    unsigned long address,
				    int *type)
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	unsigned long pgoff;
	struct page *page;

	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2545 2546 2547 2548 2549
	if (pgoff == 0)
		page = virt_to_page(vcpu->run);
	else if (pgoff == KVM_PIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->pio_data);
	else
2550 2551
		return NOPAGE_SIGBUS;
	get_page(page);
2552 2553 2554
	if (type != NULL)
		*type = VM_FAULT_MINOR;

2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567
	return page;
}

static struct vm_operations_struct kvm_vcpu_vm_ops = {
	.nopage = kvm_vcpu_nopage,
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

	fput(vcpu->kvm->filp);
	return 0;
}

static struct file_operations kvm_vcpu_fops = {
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
	.compat_ioctl   = kvm_vcpu_ioctl,
2580
	.mmap           = kvm_vcpu_mmap,
A
Avi Kivity 已提交
2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
	int fd, r;
	struct inode *inode;
	struct file *file;

2592 2593 2594 2595
	r = anon_inode_getfd(&fd, &inode, &file,
			     "kvm-vcpu", &kvm_vcpu_fops, vcpu);
	if (r)
		return r;
A
Avi Kivity 已提交
2596 2597 2598 2599
	atomic_inc(&vcpu->kvm->filp->f_count);
	return fd;
}

2600 2601 2602 2603 2604 2605 2606 2607 2608
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
{
	int r;
	struct kvm_vcpu *vcpu;

	if (!valid_vcpu(n))
R
Rusty Russell 已提交
2609
		return -EINVAL;
2610

2611
	vcpu = kvm_x86_ops->vcpu_create(kvm, n);
R
Rusty Russell 已提交
2612 2613
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
2614

2615 2616
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

2617 2618 2619
	/* We do fxsave: this must be aligned. */
	BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF);

R
Rusty Russell 已提交
2620
	vcpu_load(vcpu);
2621 2622 2623
	r = kvm_mmu_setup(vcpu);
	vcpu_put(vcpu);
	if (r < 0)
R
Rusty Russell 已提交
2624 2625
		goto free_vcpu;

S
Shaohua Li 已提交
2626
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
2627 2628
	if (kvm->vcpus[n]) {
		r = -EEXIST;
S
Shaohua Li 已提交
2629
		mutex_unlock(&kvm->lock);
R
Rusty Russell 已提交
2630 2631 2632
		goto mmu_unload;
	}
	kvm->vcpus[n] = vcpu;
S
Shaohua Li 已提交
2633
	mutex_unlock(&kvm->lock);
2634

R
Rusty Russell 已提交
2635
	/* Now it's all set up, let userspace reach it */
A
Avi Kivity 已提交
2636 2637
	r = create_vcpu_fd(vcpu);
	if (r < 0)
R
Rusty Russell 已提交
2638 2639
		goto unlink;
	return r;
2640

R
Rusty Russell 已提交
2641
unlink:
S
Shaohua Li 已提交
2642
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
2643
	kvm->vcpus[n] = NULL;
S
Shaohua Li 已提交
2644
	mutex_unlock(&kvm->lock);
2645

R
Rusty Russell 已提交
2646 2647 2648 2649
mmu_unload:
	vcpu_load(vcpu);
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
2650

R
Rusty Russell 已提交
2651
free_vcpu:
2652
	kvm_x86_ops->vcpu_free(vcpu);
2653 2654 2655
	return r;
}

2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670
static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
{
	u64 efer;
	int i;
	struct kvm_cpuid_entry *e, *entry;

	rdmsrl(MSR_EFER, efer);
	entry = NULL;
	for (i = 0; i < vcpu->cpuid_nent; ++i) {
		e = &vcpu->cpuid_entries[i];
		if (e->function == 0x80000001) {
			entry = e;
			break;
		}
	}
2671
	if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) {
2672
		entry->edx &= ~(1 << 20);
2673
		printk(KERN_INFO "kvm: guest NX capability removed\n");
2674 2675 2676
	}
}

2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690
static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
				    struct kvm_cpuid *cpuid,
				    struct kvm_cpuid_entry __user *entries)
{
	int r;

	r = -E2BIG;
	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
		goto out;
	r = -EFAULT;
	if (copy_from_user(&vcpu->cpuid_entries, entries,
			   cpuid->nent * sizeof(struct kvm_cpuid_entry)))
		goto out;
	vcpu->cpuid_nent = cpuid->nent;
2691
	cpuid_fix_nx_cap(vcpu);
2692 2693 2694 2695 2696 2697
	return 0;

out:
	return r;
}

A
Avi Kivity 已提交
2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731
/*
 * fxsave fpu state.  Taken from x86_64/processor.h.  To be killed when
 * we have asm/x86/processor.h
 */
struct fxsave {
	u16	cwd;
	u16	swd;
	u16	twd;
	u16	fop;
	u64	rip;
	u64	rdp;
	u32	mxcsr;
	u32	mxcsr_mask;
	u32	st_space[32];	/* 8*16 bytes for each FP-reg = 128 bytes */
#ifdef CONFIG_X86_64
	u32	xmm_space[64];	/* 16*16 bytes for each XMM-reg = 256 bytes */
#else
	u32	xmm_space[32];	/* 8*16 bytes for each XMM-reg = 128 bytes */
#endif
};

static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
2732
	struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image;
A
Avi Kivity 已提交
2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751

	vcpu_load(vcpu);

	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);

	vcpu_put(vcpu);

	return 0;
}

static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
2752
	struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image;
A
Avi Kivity 已提交
2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769

	vcpu_load(vcpu);

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);

	vcpu_put(vcpu);

	return 0;
}

2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790
static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
	vcpu_load(vcpu);
	memcpy(s->regs, vcpu->apic->regs, sizeof *s);
	vcpu_put(vcpu);

	return 0;
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
	vcpu_load(vcpu);
	memcpy(vcpu->apic->regs, s->regs, sizeof *s);
	kvm_apic_post_state_restore(vcpu);
	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2791 2792
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
2793
{
A
Avi Kivity 已提交
2794
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
2795
	void __user *argp = (void __user *)arg;
A
Avi Kivity 已提交
2796 2797 2798
	int r = -EINVAL;

	switch (ioctl) {
2799
	case KVM_RUN:
2800 2801 2802
		r = -EINVAL;
		if (arg)
			goto out;
2803
		r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
A
Avi Kivity 已提交
2804 2805 2806 2807
		break;
	case KVM_GET_REGS: {
		struct kvm_regs kvm_regs;

A
Avi Kivity 已提交
2808 2809
		memset(&kvm_regs, 0, sizeof kvm_regs);
		r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
A
Avi Kivity 已提交
2810 2811 2812
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2813
		if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
A
Avi Kivity 已提交
2814 2815 2816 2817 2818 2819 2820 2821
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_REGS: {
		struct kvm_regs kvm_regs;

		r = -EFAULT;
A
Al Viro 已提交
2822
		if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
A
Avi Kivity 已提交
2823
			goto out;
A
Avi Kivity 已提交
2824
		r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
A
Avi Kivity 已提交
2825 2826 2827 2828 2829 2830 2831 2832
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_SREGS: {
		struct kvm_sregs kvm_sregs;

A
Avi Kivity 已提交
2833 2834
		memset(&kvm_sregs, 0, sizeof kvm_sregs);
		r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
A
Avi Kivity 已提交
2835 2836 2837
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2838
		if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
A
Avi Kivity 已提交
2839 2840 2841 2842 2843 2844 2845 2846
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
		struct kvm_sregs kvm_sregs;

		r = -EFAULT;
A
Al Viro 已提交
2847
		if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
A
Avi Kivity 已提交
2848
			goto out;
A
Avi Kivity 已提交
2849
		r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
A
Avi Kivity 已提交
2850 2851 2852 2853 2854 2855 2856 2857 2858
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
2859
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
2860
			goto out;
A
Avi Kivity 已提交
2861
		r = kvm_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
2862 2863 2864
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2865
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
2866 2867 2868 2869 2870 2871 2872 2873
			goto out;
		r = 0;
		break;
	}
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
A
Al Viro 已提交
2874
		if (copy_from_user(&irq, argp, sizeof irq))
A
Avi Kivity 已提交
2875
			goto out;
A
Avi Kivity 已提交
2876
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
A
Avi Kivity 已提交
2877 2878 2879 2880 2881 2882 2883 2884 2885
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_DEBUG_GUEST: {
		struct kvm_debug_guest dbg;

		r = -EFAULT;
A
Al Viro 已提交
2886
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
2887
			goto out;
A
Avi Kivity 已提交
2888
		r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
A
Avi Kivity 已提交
2889 2890 2891 2892 2893
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
2894
	case KVM_GET_MSRS:
2895
		r = msr_io(vcpu, argp, kvm_get_msr, 1);
A
Avi Kivity 已提交
2896 2897 2898 2899
		break;
	case KVM_SET_MSRS:
		r = msr_io(vcpu, argp, do_set_msr, 0);
		break;
2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		if (r)
			goto out;
		break;
	}
A
Avi Kivity 已提交
2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		break;
	}
A
Avi Kivity 已提交
2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959
	case KVM_GET_FPU: {
		struct kvm_fpu fpu;

		memset(&fpu, 0, sizeof fpu);
		r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &fpu, sizeof fpu))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
		struct kvm_fpu fpu;

		r = -EFAULT;
		if (copy_from_user(&fpu, argp, sizeof fpu))
			goto out;
		r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu);
		if (r)
			goto out;
		r = 0;
		break;
	}
2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984
	case KVM_GET_LAPIC: {
		struct kvm_lapic_state lapic;

		memset(&lapic, 0, sizeof lapic);
		r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &lapic, sizeof lapic))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
		struct kvm_lapic_state lapic;

		r = -EFAULT;
		if (copy_from_user(&lapic, argp, sizeof lapic))
			goto out;
		r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);;
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004
	default:
		;
	}
out:
	return r;
}

static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r = -EINVAL;

	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		if (r < 0)
			goto out;
		break;
A
Avi Kivity 已提交
3005 3006 3007 3008
	case KVM_SET_MEMORY_REGION: {
		struct kvm_memory_region kvm_mem;

		r = -EFAULT;
A
Al Viro 已提交
3009
		if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
A
Avi Kivity 已提交
3010
			goto out;
3011
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
A
Avi Kivity 已提交
3012 3013 3014 3015 3016 3017 3018 3019
		if (r)
			goto out;
		break;
	}
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
3020
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
3021
			goto out;
3022
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
3023 3024 3025 3026
		if (r)
			goto out;
		break;
	}
3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037
	case KVM_SET_MEMORY_ALIAS: {
		struct kvm_memory_alias alias;

		r = -EFAULT;
		if (copy_from_user(&alias, argp, sizeof alias))
			goto out;
		r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
		if (r)
			goto out;
		break;
	}
3038 3039 3040
	case KVM_CREATE_IRQCHIP:
		r = -ENOMEM;
		kvm->vpic = kvm_create_pic(kvm);
E
Eddie Dong 已提交
3041 3042 3043 3044 3045 3046 3047 3048
		if (kvm->vpic) {
			r = kvm_ioapic_init(kvm);
			if (r) {
				kfree(kvm->vpic);
				kvm->vpic = NULL;
				goto out;
			}
		}
3049 3050 3051 3052 3053 3054 3055 3056 3057 3058
		else
			goto out;
		break;
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
		if (copy_from_user(&irq_event, argp, sizeof irq_event))
			goto out;
		if (irqchip_in_kernel(kvm)) {
3059
			mutex_lock(&kvm->lock);
3060 3061 3062 3063
			if (irq_event.irq < 16)
				kvm_pic_set_irq(pic_irqchip(kvm),
					irq_event.irq,
					irq_event.level);
E
Eddie Dong 已提交
3064 3065 3066
			kvm_ioapic_set_irq(kvm->vioapic,
					irq_event.irq,
					irq_event.level);
3067
			mutex_unlock(&kvm->lock);
3068 3069 3070 3071
			r = 0;
		}
		break;
	}
3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
		struct kvm_irqchip chip;

		r = -EFAULT;
		if (copy_from_user(&chip, argp, sizeof chip))
			goto out;
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
			goto out;
		r = kvm_vm_ioctl_get_irqchip(kvm, &chip);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &chip, sizeof chip))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
		struct kvm_irqchip chip;

		r = -EFAULT;
		if (copy_from_user(&chip, argp, sizeof chip))
			goto out;
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
			goto out;
		r = kvm_vm_ioctl_set_irqchip(kvm, &chip);
		if (r)
			goto out;
		r = 0;
		break;
	}
3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122
	default:
		;
	}
out:
	return r;
}

static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
				  unsigned long address,
				  int *type)
{
	struct kvm *kvm = vma->vm_file->private_data;
	unsigned long pgoff;
	struct page *page;

	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
A
Avi Kivity 已提交
3123
	page = gfn_to_page(kvm, pgoff);
3124 3125 3126
	if (!page)
		return NOPAGE_SIGBUS;
	get_page(page);
3127 3128 3129
	if (type != NULL)
		*type = VM_FAULT_MINOR;

3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157
	return page;
}

static struct vm_operations_struct kvm_vm_vm_ops = {
	.nopage = kvm_vm_nopage,
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

static struct file_operations kvm_vm_fops = {
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
	.compat_ioctl   = kvm_vm_ioctl,
	.mmap           = kvm_vm_mmap,
};

static int kvm_dev_ioctl_create_vm(void)
{
	int fd, r;
	struct inode *inode;
	struct file *file;
	struct kvm *kvm;

	kvm = kvm_create_vm();
3158 3159 3160 3161 3162 3163
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
	r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
	if (r) {
		kvm_destroy_vm(kvm);
		return r;
3164 3165
	}

A
Avi Kivity 已提交
3166
	kvm->filp = file;
3167 3168 3169 3170 3171 3172 3173 3174

	return fd;
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
3175
	long r = -EINVAL;
3176 3177 3178

	switch (ioctl) {
	case KVM_GET_API_VERSION:
3179 3180 3181
		r = -EINVAL;
		if (arg)
			goto out;
3182 3183 3184
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
3185 3186 3187
		r = -EINVAL;
		if (arg)
			goto out;
3188 3189
		r = kvm_dev_ioctl_create_vm();
		break;
A
Avi Kivity 已提交
3190
	case KVM_GET_MSR_INDEX_LIST: {
A
Al Viro 已提交
3191
		struct kvm_msr_list __user *user_msr_list = argp;
A
Avi Kivity 已提交
3192 3193 3194 3195 3196 3197 3198
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
			goto out;
		n = msr_list.nmsrs;
3199
		msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
A
Avi Kivity 已提交
3200 3201 3202
		if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
			goto out;
		r = -E2BIG;
3203
		if (n < num_msrs_to_save)
A
Avi Kivity 已提交
3204 3205 3206
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
3207
				 num_msrs_to_save * sizeof(u32)))
A
Avi Kivity 已提交
3208
			goto out;
3209 3210 3211 3212 3213
		if (copy_to_user(user_msr_list->indices
				 + num_msrs_to_save * sizeof(u32),
				 &emulated_msrs,
				 ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
			goto out;
A
Avi Kivity 已提交
3214
		r = 0;
A
Avi Kivity 已提交
3215
		break;
A
Avi Kivity 已提交
3216
	}
3217 3218 3219 3220 3221
	case KVM_CHECK_EXTENSION: {
		int ext = (long)argp;

		switch (ext) {
		case KVM_CAP_IRQCHIP:
E
Eddie Dong 已提交
3222
		case KVM_CAP_HLT:
3223 3224 3225 3226 3227 3228
			r = 1;
			break;
		default:
			r = 0;
			break;
		}
3229
		break;
3230
	}
3231 3232 3233 3234
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
3235
		r = 2 * PAGE_SIZE;
3236
		break;
A
Avi Kivity 已提交
3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249
	default:
		;
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
3250
	KVM_MINOR,
A
Avi Kivity 已提交
3251 3252 3253 3254
	"kvm",
	&kvm_chardev_ops,
};

A
Avi Kivity 已提交
3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265
/*
 * Make sure that a cpu that is being hot-unplugged does not have any vcpus
 * cached on it.
 */
static void decache_vcpus_on_cpu(int cpu)
{
	struct kvm *vm;
	struct kvm_vcpu *vcpu;
	int i;

	spin_lock(&kvm_lock);
S
Shaohua Li 已提交
3266
	list_for_each_entry(vm, &vm_list, vm_list)
A
Avi Kivity 已提交
3267
		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
R
Rusty Russell 已提交
3268 3269 3270
			vcpu = vm->vcpus[i];
			if (!vcpu)
				continue;
A
Avi Kivity 已提交
3271 3272 3273 3274 3275 3276 3277 3278 3279 3280
			/*
			 * If the vcpu is locked, then it is running on some
			 * other cpu and therefore it is not cached on the
			 * cpu in question.
			 *
			 * If it's not locked, check the last cpu it executed
			 * on.
			 */
			if (mutex_trylock(&vcpu->mutex)) {
				if (vcpu->cpu == cpu) {
3281
					kvm_x86_ops->vcpu_decache(vcpu);
A
Avi Kivity 已提交
3282 3283 3284 3285 3286 3287 3288 3289
					vcpu->cpu = -1;
				}
				mutex_unlock(&vcpu->mutex);
			}
		}
	spin_unlock(&kvm_lock);
}

3290 3291 3292 3293 3294 3295 3296
static void hardware_enable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_set(cpu, cpus_hardware_enabled);
3297
	kvm_x86_ops->hardware_enable(NULL);
3298 3299 3300 3301 3302 3303 3304 3305 3306 3307
}

static void hardware_disable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (!cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_clear(cpu, cpus_hardware_enabled);
	decache_vcpus_on_cpu(cpu);
3308
	kvm_x86_ops->hardware_disable(NULL);
3309 3310
}

A
Avi Kivity 已提交
3311 3312 3313 3314 3315 3316
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

	switch (val) {
3317 3318
	case CPU_DYING:
	case CPU_DYING_FROZEN:
3319 3320 3321 3322
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
A
Avi Kivity 已提交
3323
	case CPU_UP_CANCELED:
3324
	case CPU_UP_CANCELED_FROZEN:
3325 3326
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
3327
		smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
A
Avi Kivity 已提交
3328
		break;
3329
	case CPU_ONLINE:
3330
	case CPU_ONLINE_FROZEN:
3331 3332
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
3333
		smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
A
Avi Kivity 已提交
3334 3335 3336 3337 3338
		break;
	}
	return NOTIFY_OK;
}

3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
                       void *v)
{
	if (val == SYS_RESTART) {
		/*
		 * Some (well, at least mine) BIOSes hang on reboot if
		 * in vmx root mode.
		 */
		printk(KERN_INFO "kvm: exiting hardware virtualization\n");
		on_each_cpu(hardware_disable, NULL, 0, 1);
	}
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394
void kvm_io_bus_init(struct kvm_io_bus *bus)
{
	memset(bus, 0, sizeof(*bus));
}

void kvm_io_bus_destroy(struct kvm_io_bus *bus)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		kvm_iodevice_destructor(pos);
	}
}

struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		if (pos->in_range(pos, addr))
			return pos;
	}

	return NULL;
}

void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
{
	BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));

	bus->devs[bus->dev_count++] = dev;
}

A
Avi Kivity 已提交
3395 3396 3397 3398 3399
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
	.priority = 20, /* must be > scheduler priority */
};

A
Avi Kivity 已提交
3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410
static u64 stat_get(void *_offset)
{
	unsigned offset = (long)_offset;
	u64 total = 0;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
R
Rusty Russell 已提交
3411 3412 3413
			vcpu = kvm->vcpus[i];
			if (vcpu)
				total += *(u32 *)((void *)vcpu + offset);
A
Avi Kivity 已提交
3414 3415 3416 3417 3418
		}
	spin_unlock(&kvm_lock);
	return total;
}

R
Rusty Russell 已提交
3419
DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n");
A
Avi Kivity 已提交
3420

A
Avi Kivity 已提交
3421 3422 3423 3424
static __init void kvm_init_debug(void)
{
	struct kvm_stats_debugfs_item *p;

A
Al Viro 已提交
3425
	debugfs_dir = debugfs_create_dir("kvm", NULL);
A
Avi Kivity 已提交
3426
	for (p = debugfs_entries; p->name; ++p)
A
Avi Kivity 已提交
3427 3428 3429
		p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
						(void *)(long)p->offset,
						&stat_fops);
A
Avi Kivity 已提交
3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
	debugfs_remove(debugfs_dir);
}

3441 3442
static int kvm_suspend(struct sys_device *dev, pm_message_t state)
{
A
Avi Kivity 已提交
3443
	hardware_disable(NULL);
3444 3445 3446 3447 3448
	return 0;
}

static int kvm_resume(struct sys_device *dev)
{
A
Avi Kivity 已提交
3449
	hardware_enable(NULL);
3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463
	return 0;
}

static struct sysdev_class kvm_sysdev_class = {
	set_kset_name("kvm"),
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

static struct sys_device kvm_sysdev = {
	.id = 0,
	.cls = &kvm_sysdev_class,
};

A
Avi Kivity 已提交
3464 3465
hpa_t bad_page_address;

3466 3467 3468 3469 3470 3471 3472 3473 3474 3475
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

3476
	kvm_x86_ops->vcpu_load(vcpu, cpu);
3477 3478 3479 3480 3481 3482 3483
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

3484
	kvm_x86_ops->vcpu_put(vcpu);
3485 3486
}

3487
int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size,
3488
		  struct module *module)
A
Avi Kivity 已提交
3489 3490
{
	int r;
Y
Yang, Sheng 已提交
3491
	int cpu;
A
Avi Kivity 已提交
3492

3493
	if (kvm_x86_ops) {
3494 3495 3496 3497
		printk(KERN_ERR "kvm: already loaded the other module\n");
		return -EEXIST;
	}

3498
	if (!ops->cpu_has_kvm_support()) {
A
Avi Kivity 已提交
3499 3500 3501
		printk(KERN_ERR "kvm: no hardware support\n");
		return -EOPNOTSUPP;
	}
3502
	if (ops->disabled_by_bios()) {
A
Avi Kivity 已提交
3503 3504 3505 3506
		printk(KERN_ERR "kvm: disabled by bios\n");
		return -EOPNOTSUPP;
	}

3507
	kvm_x86_ops = ops;
3508

3509
	r = kvm_x86_ops->hardware_setup();
A
Avi Kivity 已提交
3510
	if (r < 0)
3511
		goto out;
A
Avi Kivity 已提交
3512

Y
Yang, Sheng 已提交
3513 3514
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
3515
				kvm_x86_ops->check_processor_compatibility,
Y
Yang, Sheng 已提交
3516 3517 3518 3519 3520
				&r, 0, 1);
		if (r < 0)
			goto out_free_0;
	}

3521
	on_each_cpu(hardware_enable, NULL, 0, 1);
A
Avi Kivity 已提交
3522 3523 3524
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
		goto out_free_1;
A
Avi Kivity 已提交
3525 3526
	register_reboot_notifier(&kvm_reboot_notifier);

3527 3528 3529 3530 3531 3532 3533 3534
	r = sysdev_class_register(&kvm_sysdev_class);
	if (r)
		goto out_free_2;

	r = sysdev_register(&kvm_sysdev);
	if (r)
		goto out_free_3;

3535 3536 3537 3538 3539 3540 3541 3542
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
					   __alignof__(struct kvm_vcpu), 0, 0);
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
		goto out_free_4;
	}

A
Avi Kivity 已提交
3543 3544 3545 3546 3547 3548 3549 3550
	kvm_chardev_ops.owner = module;

	r = misc_register(&kvm_dev);
	if (r) {
		printk (KERN_ERR "kvm: misc device register failed\n");
		goto out_free;
	}

3551 3552 3553
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

A
Avi Kivity 已提交
3554 3555 3556
	return r;

out_free:
3557 3558
	kmem_cache_destroy(kvm_vcpu_cache);
out_free_4:
3559 3560 3561 3562
	sysdev_unregister(&kvm_sysdev);
out_free_3:
	sysdev_class_unregister(&kvm_sysdev_class);
out_free_2:
A
Avi Kivity 已提交
3563
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
3564 3565
	unregister_cpu_notifier(&kvm_cpu_notifier);
out_free_1:
3566
	on_each_cpu(hardware_disable, NULL, 0, 1);
Y
Yang, Sheng 已提交
3567
out_free_0:
3568
	kvm_x86_ops->hardware_unsetup();
3569
out:
3570
	kvm_x86_ops = NULL;
A
Avi Kivity 已提交
3571 3572 3573
	return r;
}

3574
void kvm_exit_x86(void)
A
Avi Kivity 已提交
3575 3576
{
	misc_deregister(&kvm_dev);
3577
	kmem_cache_destroy(kvm_vcpu_cache);
3578 3579
	sysdev_unregister(&kvm_sysdev);
	sysdev_class_unregister(&kvm_sysdev_class);
A
Avi Kivity 已提交
3580
	unregister_reboot_notifier(&kvm_reboot_notifier);
3581
	unregister_cpu_notifier(&kvm_cpu_notifier);
3582
	on_each_cpu(hardware_disable, NULL, 0, 1);
3583 3584
	kvm_x86_ops->hardware_unsetup();
	kvm_x86_ops = NULL;
A
Avi Kivity 已提交
3585 3586 3587 3588 3589
}

static __init int kvm_init(void)
{
	static struct page *bad_page;
3590 3591
	int r;

3592 3593 3594 3595
	r = kvm_mmu_module_init();
	if (r)
		goto out4;

A
Avi Kivity 已提交
3596 3597
	kvm_init_debug();

3598 3599
	kvm_init_msr_list();

A
Avi Kivity 已提交
3600 3601 3602 3603 3604 3605 3606 3607
	if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
		r = -ENOMEM;
		goto out;
	}

	bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
	memset(__va(bad_page_address), 0, PAGE_SIZE);

3608
	return 0;
A
Avi Kivity 已提交
3609 3610 3611

out:
	kvm_exit_debug();
3612 3613
	kvm_mmu_module_exit();
out4:
A
Avi Kivity 已提交
3614 3615 3616 3617 3618 3619 3620
	return r;
}

static __exit void kvm_exit(void)
{
	kvm_exit_debug();
	__free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
3621
	kvm_mmu_module_exit();
A
Avi Kivity 已提交
3622 3623 3624 3625 3626
}

module_init(kvm_init)
module_exit(kvm_exit)

3627 3628
EXPORT_SYMBOL_GPL(kvm_init_x86);
EXPORT_SYMBOL_GPL(kvm_exit_x86);