kvm_main.c 77.2 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

#include "kvm.h"
19
#include "x86.h"
A
Avi Kivity 已提交
20 21
#include "x86_emulate.h"
#include "segment_descriptor.h"
22
#include "irq.h"
A
Avi Kivity 已提交
23 24 25 26 27 28 29 30 31 32 33 34 35

#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
36
#include <linux/sysdev.h>
A
Avi Kivity 已提交
37
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
38
#include <linux/sched.h>
39 40
#include <linux/cpumask.h>
#include <linux/smp.h>
41
#include <linux/anon_inodes.h>
42
#include <linux/profile.h>
43
#include <linux/kvm_para.h>
44
#include <linux/pagemap.h>
45
#include <linux/mman.h>
A
Avi Kivity 已提交
46

A
Avi Kivity 已提交
47 48 49 50 51
#include <asm/processor.h>
#include <asm/msr.h>
#include <asm/io.h>
#include <asm/uaccess.h>
#include <asm/desc.h>
A
Avi Kivity 已提交
52 53 54 55

MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

56 57 58
static DEFINE_SPINLOCK(kvm_lock);
static LIST_HEAD(vm_list);

59 60
static cpumask_t cpus_hardware_enabled;

61
struct kvm_x86_ops *kvm_x86_ops;
62 63
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
A
Avi Kivity 已提交
64

65 66
static __read_mostly struct preempt_ops kvm_preempt_ops;

A
Avi Kivity 已提交
67
#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x)
A
Avi Kivity 已提交
68 69 70

static struct kvm_stats_debugfs_item {
	const char *name;
A
Avi Kivity 已提交
71
	int offset;
A
Avi Kivity 已提交
72 73
	struct dentry *dentry;
} debugfs_entries[] = {
A
Avi Kivity 已提交
74 75 76 77 78 79 80 81 82 83
	{ "pf_fixed", STAT_OFFSET(pf_fixed) },
	{ "pf_guest", STAT_OFFSET(pf_guest) },
	{ "tlb_flush", STAT_OFFSET(tlb_flush) },
	{ "invlpg", STAT_OFFSET(invlpg) },
	{ "exits", STAT_OFFSET(exits) },
	{ "io_exits", STAT_OFFSET(io_exits) },
	{ "mmio_exits", STAT_OFFSET(mmio_exits) },
	{ "signal_exits", STAT_OFFSET(signal_exits) },
	{ "irq_window", STAT_OFFSET(irq_window_exits) },
	{ "halt_exits", STAT_OFFSET(halt_exits) },
E
Eddie Dong 已提交
84
	{ "halt_wakeup", STAT_OFFSET(halt_wakeup) },
A
Avi Kivity 已提交
85 86
	{ "request_irq", STAT_OFFSET(request_irq_exits) },
	{ "irq_exits", STAT_OFFSET(irq_exits) },
87
	{ "light_exits", STAT_OFFSET(light_exits) },
88
	{ "efer_reload", STAT_OFFSET(efer_reload) },
A
Avi Kivity 已提交
89
	{ NULL }
A
Avi Kivity 已提交
90 91 92 93
};

static struct dentry *debugfs_dir;

94 95 96 97
#define CR0_RESERVED_BITS						\
	(~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
			  | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
			  | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
98 99 100 101 102 103
#define CR4_RESERVED_BITS						\
	(~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
			  | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE	\
			  | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR	\
			  | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))

104
#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
A
Avi Kivity 已提交
105 106
#define EFER_RESERVED_BITS 0xfffffffffffff2fe

107
#ifdef CONFIG_X86_64
M
Mike Day 已提交
108
/* LDT or TSS descriptor in the GDT. 16 bytes. */
A
Avi Kivity 已提交
109 110 111 112 113 114 115 116
struct segment_descriptor_64 {
	struct segment_descriptor s;
	u32 base_higher;
	u32 pad_zero;
};

#endif

A
Avi Kivity 已提交
117 118 119
static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
			   unsigned long arg);

A
Avi Kivity 已提交
120 121 122 123 124 125 126 127 128 129
unsigned long segment_base(u16 selector)
{
	struct descriptor_table gdt;
	struct segment_descriptor *d;
	unsigned long table_base;
	unsigned long v;

	if (selector == 0)
		return 0;

M
Mike Day 已提交
130
	asm("sgdt %0" : "=m"(gdt));
A
Avi Kivity 已提交
131 132 133 134 135
	table_base = gdt.base;

	if (selector & 4) {           /* from ldt */
		u16 ldt_selector;

M
Mike Day 已提交
136
		asm("sldt %0" : "=g"(ldt_selector));
A
Avi Kivity 已提交
137 138 139
		table_base = segment_base(ldt_selector);
	}
	d = (struct segment_descriptor *)(table_base + (selector & ~7));
M
Mike Day 已提交
140 141
	v = d->base_low | ((unsigned long)d->base_mid << 16) |
		((unsigned long)d->base_high << 24);
142
#ifdef CONFIG_X86_64
M
Mike Day 已提交
143 144 145
	if (d->system == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
		v |= ((unsigned long) \
		      ((struct segment_descriptor_64 *)d)->base_higher) << 32;
A
Avi Kivity 已提交
146 147 148 149 150
#endif
	return v;
}
EXPORT_SYMBOL_GPL(segment_base);

151 152 153 154 155
static inline int valid_vcpu(int n)
{
	return likely(n >= 0 && n < KVM_MAX_VCPUS);
}

156 157 158 159 160 161
void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
	if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
		return;

	vcpu->guest_fpu_loaded = 1;
162 163
	fx_save(&vcpu->host_fx_image);
	fx_restore(&vcpu->guest_fx_image);
164 165 166 167 168 169 170 171 172
}
EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);

void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
	if (!vcpu->guest_fpu_loaded)
		return;

	vcpu->guest_fpu_loaded = 0;
173 174
	fx_save(&vcpu->guest_fx_image);
	fx_restore(&vcpu->host_fx_image);
175 176 177
}
EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);

A
Avi Kivity 已提交
178 179 180
/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
181
void vcpu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
182
{
183 184
	int cpu;

A
Avi Kivity 已提交
185
	mutex_lock(&vcpu->mutex);
186 187
	cpu = get_cpu();
	preempt_notifier_register(&vcpu->preempt_notifier);
188
	kvm_arch_vcpu_load(vcpu, cpu);
189
	put_cpu();
A
Avi Kivity 已提交
190 191
}

192
void vcpu_put(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
193
{
194
	preempt_disable();
195
	kvm_arch_vcpu_put(vcpu);
196 197
	preempt_notifier_unregister(&vcpu->preempt_notifier);
	preempt_enable();
A
Avi Kivity 已提交
198 199 200
	mutex_unlock(&vcpu->mutex);
}

201 202 203 204 205 206
static void ack_flush(void *_completed)
{
}

void kvm_flush_remote_tlbs(struct kvm *kvm)
{
207
	int i, cpu;
208 209 210 211
	cpumask_t cpus;
	struct kvm_vcpu *vcpu;

	cpus_clear(cpus);
R
Rusty Russell 已提交
212 213 214 215
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		vcpu = kvm->vcpus[i];
		if (!vcpu)
			continue;
216
		if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
217 218 219
			continue;
		cpu = vcpu->cpu;
		if (cpu != -1 && cpu != raw_smp_processor_id())
220
			cpu_set(cpu, cpus);
221
	}
222
	smp_call_function_mask(cpus, ack_flush, NULL, 1);
223 224
}

R
Rusty Russell 已提交
225 226 227 228 229 230 231 232 233 234
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
	struct page *page;
	int r;

	mutex_init(&vcpu->mutex);
	vcpu->cpu = -1;
	vcpu->mmu.root_hpa = INVALID_PAGE;
	vcpu->kvm = kvm;
	vcpu->vcpu_id = id;
235 236 237 238
	if (!irqchip_in_kernel(kvm) || id == 0)
		vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
	else
		vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED;
E
Eddie Dong 已提交
239
	init_waitqueue_head(&vcpu->wq);
R
Rusty Russell 已提交
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
	vcpu->run = page_address(page);

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail_free_run;
	}
	vcpu->pio_data = page_address(page);

	r = kvm_mmu_create(vcpu);
	if (r < 0)
		goto fail_free_pio_data;

259 260 261 262 263 264
	if (irqchip_in_kernel(kvm)) {
		r = kvm_create_lapic(vcpu);
		if (r < 0)
			goto fail_mmu_destroy;
	}

R
Rusty Russell 已提交
265 266
	return 0;

267 268
fail_mmu_destroy:
	kvm_mmu_destroy(vcpu);
R
Rusty Russell 已提交
269 270 271 272 273
fail_free_pio_data:
	free_page((unsigned long)vcpu->pio_data);
fail_free_run:
	free_page((unsigned long)vcpu->run);
fail:
274
	return r;
R
Rusty Russell 已提交
275 276 277 278 279
}
EXPORT_SYMBOL_GPL(kvm_vcpu_init);

void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
{
280
	kvm_free_lapic(vcpu);
R
Rusty Russell 已提交
281 282 283 284 285 286
	kvm_mmu_destroy(vcpu);
	free_page((unsigned long)vcpu->pio_data);
	free_page((unsigned long)vcpu->run);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);

287
static struct kvm *kvm_create_vm(void)
A
Avi Kivity 已提交
288 289 290 291
{
	struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);

	if (!kvm)
292
		return ERR_PTR(-ENOMEM);
A
Avi Kivity 已提交
293

294
	kvm_io_bus_init(&kvm->pio_bus);
S
Shaohua Li 已提交
295
	mutex_init(&kvm->lock);
A
Avi Kivity 已提交
296
	INIT_LIST_HEAD(&kvm->active_mmu_pages);
297
	kvm_io_bus_init(&kvm->mmio_bus);
298 299 300
	spin_lock(&kvm_lock);
	list_add(&kvm->vm_list, &vm_list);
	spin_unlock(&kvm_lock);
301 302 303
	return kvm;
}

A
Avi Kivity 已提交
304 305 306 307 308 309
/*
 * Free any memory in @free but not in @dont.
 */
static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
				  struct kvm_memory_slot *dont)
{
310 311
	if (!dont || free->rmap != dont->rmap)
		vfree(free->rmap);
A
Avi Kivity 已提交
312 313 314 315 316

	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
		vfree(free->dirty_bitmap);

	free->npages = 0;
A
Al Viro 已提交
317
	free->dirty_bitmap = NULL;
318
	free->rmap = NULL;
A
Avi Kivity 已提交
319 320 321 322 323 324 325
}

static void kvm_free_physmem(struct kvm *kvm)
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i)
A
Al Viro 已提交
326
		kvm_free_physmem_slot(&kvm->memslots[i], NULL);
A
Avi Kivity 已提交
327 328
}

329 330 331 332
static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
{
	int i;

R
Rusty Russell 已提交
333
	for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i)
334
		if (vcpu->pio.guest_pages[i]) {
335
			kvm_release_page(vcpu->pio.guest_pages[i]);
336 337 338 339
			vcpu->pio.guest_pages[i] = NULL;
		}
}

A
Avi Kivity 已提交
340 341 342 343 344 345 346
static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
	vcpu_load(vcpu);
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

A
Avi Kivity 已提交
347 348 349 350
static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;

A
Avi Kivity 已提交
351 352 353 354
	/*
	 * Unpin any mmu pages first.
	 */
	for (i = 0; i < KVM_MAX_VCPUS; ++i)
R
Rusty Russell 已提交
355 356 357 358
		if (kvm->vcpus[i])
			kvm_unload_vcpu_mmu(kvm->vcpus[i]);
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		if (kvm->vcpus[i]) {
359
			kvm_x86_ops->vcpu_free(kvm->vcpus[i]);
R
Rusty Russell 已提交
360 361 362 363
			kvm->vcpus[i] = NULL;
		}
	}

A
Avi Kivity 已提交
364 365
}

366 367
static void kvm_destroy_vm(struct kvm *kvm)
{
368 369 370
	spin_lock(&kvm_lock);
	list_del(&kvm->vm_list);
	spin_unlock(&kvm_lock);
371
	kvm_io_bus_destroy(&kvm->pio_bus);
372
	kvm_io_bus_destroy(&kvm->mmio_bus);
373
	kfree(kvm->vpic);
E
Eddie Dong 已提交
374
	kfree(kvm->vioapic);
A
Avi Kivity 已提交
375 376 377
	kvm_free_vcpus(kvm);
	kvm_free_physmem(kvm);
	kfree(kvm);
378 379 380 381 382 383 384
}

static int kvm_vm_release(struct inode *inode, struct file *filp)
{
	struct kvm *kvm = filp->private_data;

	kvm_destroy_vm(kvm);
A
Avi Kivity 已提交
385 386 387 388 389
	return 0;
}

static void inject_gp(struct kvm_vcpu *vcpu)
{
390
	kvm_x86_ops->inject_gp(vcpu, 0);
A
Avi Kivity 已提交
391 392
}

393 394 395 396
/*
 * Load the pae pdptrs.  Return true is they are all valid.
 */
static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
A
Avi Kivity 已提交
397 398
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
399
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
A
Avi Kivity 已提交
400
	int i;
401
	int ret;
R
Rusty Russell 已提交
402
	u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)];
A
Avi Kivity 已提交
403

S
Shaohua Li 已提交
404
	mutex_lock(&vcpu->kvm->lock);
405 406 407
	ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte,
				  offset * sizeof(u64), sizeof(pdpte));
	if (ret < 0) {
R
Rusty Russell 已提交
408 409 410 411 412
		ret = 0;
		goto out;
	}
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
		if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) {
413 414 415
			ret = 0;
			goto out;
		}
A
Avi Kivity 已提交
416
	}
R
Rusty Russell 已提交
417
	ret = 1;
A
Avi Kivity 已提交
418

R
Rusty Russell 已提交
419
	memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs));
420
out:
S
Shaohua Li 已提交
421
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
422

423
	return ret;
A
Avi Kivity 已提交
424 425 426 427
}

void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
428
	if (cr0 & CR0_RESERVED_BITS) {
A
Avi Kivity 已提交
429 430 431 432 433 434
		printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
		       cr0, vcpu->cr0);
		inject_gp(vcpu);
		return;
	}

435
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
A
Avi Kivity 已提交
436 437 438 439 440
		printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
		inject_gp(vcpu);
		return;
	}

441
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
A
Avi Kivity 已提交
442 443 444 445 446 447
		printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
		       "and a clear PE flag\n");
		inject_gp(vcpu);
		return;
	}

448
	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
449
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
450 451 452 453 454 455 456 457 458
		if ((vcpu->shadow_efer & EFER_LME)) {
			int cs_db, cs_l;

			if (!is_pae(vcpu)) {
				printk(KERN_DEBUG "set_cr0: #GP, start paging "
				       "in long mode while PAE is disabled\n");
				inject_gp(vcpu);
				return;
			}
459
			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
A
Avi Kivity 已提交
460 461 462 463 464 465 466 467 468
			if (cs_l) {
				printk(KERN_DEBUG "set_cr0: #GP, start paging "
				       "in long mode while CS.L == 1\n");
				inject_gp(vcpu);
				return;

			}
		} else
#endif
469
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
A
Avi Kivity 已提交
470 471 472 473 474 475 476 477
			printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
			       "reserved bits\n");
			inject_gp(vcpu);
			return;
		}

	}

478
	kvm_x86_ops->set_cr0(vcpu, cr0);
A
Avi Kivity 已提交
479 480
	vcpu->cr0 = cr0;

S
Shaohua Li 已提交
481
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
482
	kvm_mmu_reset_context(vcpu);
S
Shaohua Li 已提交
483
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
484 485 486 487 488 489 490 491 492 493 494 495
	return;
}
EXPORT_SYMBOL_GPL(set_cr0);

void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
{
	set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
}
EXPORT_SYMBOL_GPL(lmsw);

void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
496
	if (cr4 & CR4_RESERVED_BITS) {
A
Avi Kivity 已提交
497 498 499 500 501
		printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
		inject_gp(vcpu);
		return;
	}

A
Avi Kivity 已提交
502
	if (is_long_mode(vcpu)) {
503
		if (!(cr4 & X86_CR4_PAE)) {
A
Avi Kivity 已提交
504 505 506 507 508
			printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
			       "in long mode\n");
			inject_gp(vcpu);
			return;
		}
509
	} else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE)
510
		   && !load_pdptrs(vcpu, vcpu->cr3)) {
A
Avi Kivity 已提交
511 512
		printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
		inject_gp(vcpu);
513
		return;
A
Avi Kivity 已提交
514 515
	}

516
	if (cr4 & X86_CR4_VMXE) {
A
Avi Kivity 已提交
517 518 519 520
		printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
		inject_gp(vcpu);
		return;
	}
521
	kvm_x86_ops->set_cr4(vcpu, cr4);
R
Rusty Russell 已提交
522
	vcpu->cr4 = cr4;
S
Shaohua Li 已提交
523
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
524
	kvm_mmu_reset_context(vcpu);
S
Shaohua Li 已提交
525
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
526 527 528 529 530
}
EXPORT_SYMBOL_GPL(set_cr4);

void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
{
A
Avi Kivity 已提交
531
	if (is_long_mode(vcpu)) {
532
		if (cr3 & CR3_L_MODE_RESERVED_BITS) {
A
Avi Kivity 已提交
533 534 535 536 537
			printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
			inject_gp(vcpu);
			return;
		}
	} else {
538 539 540 541 542 543 544 545 546 547 548 549 550
		if (is_pae(vcpu)) {
			if (cr3 & CR3_PAE_RESERVED_BITS) {
				printk(KERN_DEBUG
				       "set_cr3: #GP, reserved bits\n");
				inject_gp(vcpu);
				return;
			}
			if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
				printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
				       "reserved bits\n");
				inject_gp(vcpu);
				return;
			}
A
Avi Kivity 已提交
551
		}
552 553 554 555
		/*
		 * We don't check reserved bits in nonpae mode, because
		 * this isn't enforced, and VMware depends on this.
		 */
A
Avi Kivity 已提交
556 557
	}

S
Shaohua Li 已提交
558
	mutex_lock(&vcpu->kvm->lock);
559 560 561 562 563 564 565 566 567 568 569
	/*
	 * Does the new cr3 value map to physical memory? (Note, we
	 * catch an invalid cr3 even in real-mode, because it would
	 * cause trouble later on when we turn on paging anyway.)
	 *
	 * A real CPU would silently accept an invalid cr3 and would
	 * attempt to use it - with largely undefined (and often hard
	 * to debug) behavior on the guest side.
	 */
	if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
		inject_gp(vcpu);
570 571
	else {
		vcpu->cr3 = cr3;
572
		vcpu->mmu.new_cr3(vcpu);
573
	}
S
Shaohua Li 已提交
574
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
575 576 577 578 579
}
EXPORT_SYMBOL_GPL(set_cr3);

void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
{
580
	if (cr8 & CR8_RESERVED_BITS) {
A
Avi Kivity 已提交
581 582 583 584
		printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
		inject_gp(vcpu);
		return;
	}
E
Eddie Dong 已提交
585 586 587 588
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_lapic_set_tpr(vcpu, cr8);
	else
		vcpu->cr8 = cr8;
A
Avi Kivity 已提交
589 590 591
}
EXPORT_SYMBOL_GPL(set_cr8);

592 593
unsigned long get_cr8(struct kvm_vcpu *vcpu)
{
E
Eddie Dong 已提交
594 595 596 597
	if (irqchip_in_kernel(vcpu->kvm))
		return kvm_lapic_get_cr8(vcpu);
	else
		return vcpu->cr8;
598 599 600 601 602
}
EXPORT_SYMBOL_GPL(get_cr8);

u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
E
Eddie Dong 已提交
603 604 605 606
	if (irqchip_in_kernel(vcpu->kvm))
		return vcpu->apic_base;
	else
		return vcpu->apic_base;
607 608 609 610 611
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
{
E
Eddie Dong 已提交
612 613 614 615 616
	/* TODO: reserve bits check */
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_lapic_set_base(vcpu, data);
	else
		vcpu->apic_base = data;
617 618 619
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

A
Avi Kivity 已提交
620 621
void fx_init(struct kvm_vcpu *vcpu)
{
622
	unsigned after_mxcsr_mask;
A
Avi Kivity 已提交
623

624 625
	/* Initialize guest FPU by resetting ours and saving into guest's */
	preempt_disable();
626
	fx_save(&vcpu->host_fx_image);
A
Avi Kivity 已提交
627
	fpu_init();
628 629
	fx_save(&vcpu->guest_fx_image);
	fx_restore(&vcpu->host_fx_image);
630
	preempt_enable();
A
Avi Kivity 已提交
631

632
	vcpu->cr0 |= X86_CR0_ET;
633 634 635 636
	after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
	vcpu->guest_fx_image.mxcsr = 0x1f80;
	memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask,
	       0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
A
Avi Kivity 已提交
637 638 639 640 641 642 643 644 645
}
EXPORT_SYMBOL_GPL(fx_init);

/*
 * Allocate some memory and give it an address in the guest physical address
 * space.
 *
 * Discontiguous memory is allowed, mostly for framebuffers.
 */
646
static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
647 648 649
					  struct
					  kvm_userspace_memory_region *mem,
					  int user_alloc)
A
Avi Kivity 已提交
650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675
{
	int r;
	gfn_t base_gfn;
	unsigned long npages;
	unsigned long i;
	struct kvm_memory_slot *memslot;
	struct kvm_memory_slot old, new;

	r = -EINVAL;
	/* General sanity checks */
	if (mem->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
	if (mem->slot >= KVM_MEMORY_SLOTS)
		goto out;
	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
		goto out;

	memslot = &kvm->memslots[mem->slot];
	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
	npages = mem->memory_size >> PAGE_SHIFT;

	if (!npages)
		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

S
Shaohua Li 已提交
676
	mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702

	new = old = *memslot;

	new.base_gfn = base_gfn;
	new.npages = npages;
	new.flags = mem->flags;

	/* Disallow changing a memory slot's size. */
	r = -EINVAL;
	if (npages && old.npages && npages != old.npages)
		goto out_unlock;

	/* Check for overlaps */
	r = -EEXIST;
	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *s = &kvm->memslots[i];

		if (s == memslot)
			continue;
		if (!((base_gfn + npages <= s->base_gfn) ||
		      (base_gfn >= s->base_gfn + s->npages)))
			goto out_unlock;
	}

	/* Free page dirty bitmap if unneeded */
	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
A
Al Viro 已提交
703
		new.dirty_bitmap = NULL;
A
Avi Kivity 已提交
704 705 706 707

	r = -ENOMEM;

	/* Allocate if a slot is being created */
708
	if (npages && !new.rmap) {
M
Mike Day 已提交
709
		new.rmap = vmalloc(npages * sizeof(struct page *));
710 711 712 713 714

		if (!new.rmap)
			goto out_unlock;

		memset(new.rmap, 0, npages * sizeof(*new.rmap));
715 716

		if (user_alloc)
717
			new.userspace_addr = mem->userspace_addr;
718 719 720 721 722 723 724 725 726 727 728
		else {
			down_write(&current->mm->mmap_sem);
			new.userspace_addr = do_mmap(NULL, 0,
						     npages * PAGE_SIZE,
						     PROT_READ | PROT_WRITE,
						     MAP_SHARED | MAP_ANONYMOUS,
						     0);
			up_write(&current->mm->mmap_sem);

			if (IS_ERR((void *)new.userspace_addr))
				goto out_unlock;
A
Avi Kivity 已提交
729 730 731 732 733 734 735 736 737
		}
	}

	/* Allocate page dirty bitmap if needed */
	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
		unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;

		new.dirty_bitmap = vmalloc(dirty_bytes);
		if (!new.dirty_bitmap)
L
Laurent Vivier 已提交
738
			goto out_unlock;
A
Avi Kivity 已提交
739 740 741 742 743 744
		memset(new.dirty_bitmap, 0, dirty_bytes);
	}

	if (mem->slot >= kvm->nmemslots)
		kvm->nmemslots = mem->slot + 1;

745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762
	if (!kvm->n_requested_mmu_pages) {
		unsigned int n_pages;

		if (npages) {
			n_pages = npages * KVM_PERMILLE_MMU_PAGES / 1000;
			kvm_mmu_change_mmu_pages(kvm, kvm->n_alloc_mmu_pages +
						 n_pages);
		} else {
			unsigned int nr_mmu_pages;

			n_pages = old.npages * KVM_PERMILLE_MMU_PAGES / 1000;
			nr_mmu_pages = kvm->n_alloc_mmu_pages - n_pages;
			nr_mmu_pages = max(nr_mmu_pages,
				        (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
			kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
		}
	}

A
Avi Kivity 已提交
763 764
	*memslot = new;

765 766
	kvm_mmu_slot_remove_write_access(kvm, mem->slot);
	kvm_flush_remote_tlbs(kvm);
A
Avi Kivity 已提交
767

S
Shaohua Li 已提交
768
	mutex_unlock(&kvm->lock);
A
Avi Kivity 已提交
769 770 771 772 773

	kvm_free_physmem_slot(&old, &new);
	return 0;

out_unlock:
S
Shaohua Li 已提交
774
	mutex_unlock(&kvm->lock);
A
Avi Kivity 已提交
775 776 777 778 779
	kvm_free_physmem_slot(&new, &old);
out:
	return r;
}

780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
					  u32 kvm_nr_mmu_pages)
{
	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
		return -EINVAL;

	mutex_lock(&kvm->lock);

	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
	kvm->n_requested_mmu_pages = kvm_nr_mmu_pages;

	mutex_unlock(&kvm->lock);
	return 0;
}

static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
{
	return kvm->n_alloc_mmu_pages;
}

A
Avi Kivity 已提交
800 801 802
/*
 * Get (and clear) the dirty memory log for a memory slot.
 */
803 804
static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
				      struct kvm_dirty_log *log)
A
Avi Kivity 已提交
805 806 807 808 809 810
{
	struct kvm_memory_slot *memslot;
	int r, i;
	int n;
	unsigned long any = 0;

S
Shaohua Li 已提交
811
	mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
812 813 814 815 816 817 818 819 820 821

	r = -EINVAL;
	if (log->slot >= KVM_MEMORY_SLOTS)
		goto out;

	memslot = &kvm->memslots[log->slot];
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

822
	n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
A
Avi Kivity 已提交
823

824
	for (i = 0; !any && i < n/sizeof(long); ++i)
A
Avi Kivity 已提交
825 826 827 828 829 830
		any = memslot->dirty_bitmap[i];

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
		goto out;

831 832 833 834 835 836
	/* If nothing is dirty, don't bother messing with page tables. */
	if (any) {
		kvm_mmu_slot_remove_write_access(kvm, log->slot);
		kvm_flush_remote_tlbs(kvm);
		memset(memslot->dirty_bitmap, 0, n);
	}
A
Avi Kivity 已提交
837 838 839 840

	r = 0;

out:
S
Shaohua Li 已提交
841
	mutex_unlock(&kvm->lock);
A
Avi Kivity 已提交
842 843 844
	return r;
}

845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870
/*
 * Set a new alias region.  Aliases map a portion of physical memory into
 * another portion.  This is useful for memory windows, for example the PC
 * VGA region.
 */
static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
					 struct kvm_memory_alias *alias)
{
	int r, n;
	struct kvm_mem_alias *p;

	r = -EINVAL;
	/* General sanity checks */
	if (alias->memory_size & (PAGE_SIZE - 1))
		goto out;
	if (alias->guest_phys_addr & (PAGE_SIZE - 1))
		goto out;
	if (alias->slot >= KVM_ALIAS_SLOTS)
		goto out;
	if (alias->guest_phys_addr + alias->memory_size
	    < alias->guest_phys_addr)
		goto out;
	if (alias->target_phys_addr + alias->memory_size
	    < alias->target_phys_addr)
		goto out;

S
Shaohua Li 已提交
871
	mutex_lock(&kvm->lock);
872 873 874 875 876 877 878 879 880 881 882

	p = &kvm->aliases[alias->slot];
	p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
	p->npages = alias->memory_size >> PAGE_SHIFT;
	p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;

	for (n = KVM_ALIAS_SLOTS; n > 0; --n)
		if (kvm->aliases[n - 1].npages)
			break;
	kvm->naliases = n;

883
	kvm_mmu_zap_all(kvm);
884

S
Shaohua Li 已提交
885
	mutex_unlock(&kvm->lock);
886 887 888 889 890 891 892

	return 0;

out:
	return r;
}

893 894 895 896 897 898 899
static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
M
Mike Day 已提交
900
		memcpy(&chip->chip.pic,
901 902 903 904
			&pic_irqchip(kvm)->pics[0],
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
M
Mike Day 已提交
905
		memcpy(&chip->chip.pic,
906 907 908
			&pic_irqchip(kvm)->pics[1],
			sizeof(struct kvm_pic_state));
		break;
909
	case KVM_IRQCHIP_IOAPIC:
M
Mike Day 已提交
910
		memcpy(&chip->chip.ioapic,
911 912 913
			ioapic_irqchip(kvm),
			sizeof(struct kvm_ioapic_state));
		break;
914 915 916 917 918 919 920 921 922 923 924 925 926 927
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
M
Mike Day 已提交
928
		memcpy(&pic_irqchip(kvm)->pics[0],
929 930 931 932
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
M
Mike Day 已提交
933
		memcpy(&pic_irqchip(kvm)->pics[1],
934 935 936
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
		break;
937
	case KVM_IRQCHIP_IOAPIC:
M
Mike Day 已提交
938
		memcpy(ioapic_irqchip(kvm),
939 940 941
			&chip->chip.ioapic,
			sizeof(struct kvm_ioapic_state));
		break;
942 943 944 945 946 947 948 949
	default:
		r = -EINVAL;
		break;
	}
	kvm_pic_update_irq(pic_irqchip(kvm));
	return r;
}

950 951 952 953 954 955
int is_error_page(struct page *page)
{
	return page == bad_page;
}
EXPORT_SYMBOL_GPL(is_error_page);

956
gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
957 958 959 960 961 962 963 964 965 966 967 968 969 970
{
	int i;
	struct kvm_mem_alias *alias;

	for (i = 0; i < kvm->naliases; ++i) {
		alias = &kvm->aliases[i];
		if (gfn >= alias->base_gfn
		    && gfn < alias->base_gfn + alias->npages)
			return alias->target_gfn + gfn - alias->base_gfn;
	}
	return gfn;
}

static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
A
Avi Kivity 已提交
971 972 973 974 975 976 977 978 979 980
{
	int i;

	for (i = 0; i < kvm->nmemslots; ++i) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];

		if (gfn >= memslot->base_gfn
		    && gfn < memslot->base_gfn + memslot->npages)
			return memslot;
	}
A
Al Viro 已提交
981
	return NULL;
A
Avi Kivity 已提交
982
}
983 984 985 986 987 988

struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
	gfn = unalias_gfn(kvm, gfn);
	return __gfn_to_memslot(kvm, gfn);
}
A
Avi Kivity 已提交
989

A
Avi Kivity 已提交
990 991 992
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *slot;
993 994
	struct page *page[1];
	int npages;
A
Avi Kivity 已提交
995

996 997
	might_sleep();

998 999
	gfn = unalias_gfn(kvm, gfn);
	slot = __gfn_to_memslot(kvm, gfn);
1000 1001
	if (!slot) {
		get_page(bad_page);
1002
		return bad_page;
1003
	}
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013

	down_read(&current->mm->mmap_sem);
	npages = get_user_pages(current, current->mm,
				slot->userspace_addr
				+ (gfn - slot->base_gfn) * PAGE_SIZE, 1,
				1, 1, page, NULL);
	up_read(&current->mm->mmap_sem);
	if (npages != 1) {
		get_page(bad_page);
		return bad_page;
1014
	}
1015 1016

	return page[0];
A
Avi Kivity 已提交
1017 1018 1019
}
EXPORT_SYMBOL_GPL(gfn_to_page);

1020 1021 1022 1023 1024 1025 1026 1027
void kvm_release_page(struct page *page)
{
	if (!PageReserved(page))
		SetPageDirty(page);
	put_page(page);
}
EXPORT_SYMBOL_GPL(kvm_release_page);

1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
static int next_segment(unsigned long len, int offset)
{
	if (len > PAGE_SIZE - offset)
		return PAGE_SIZE - offset;
	else
		return len;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
			int len)
{
	void *page_virt;
	struct page *page;

	page = gfn_to_page(kvm, gfn);
1043 1044
	if (is_error_page(page)) {
		kvm_release_page(page);
1045
		return -EFAULT;
1046
	}
1047 1048 1049 1050 1051
	page_virt = kmap_atomic(page, KM_USER0);

	memcpy(data, page_virt + offset, len);

	kunmap_atomic(page_virt, KM_USER0);
1052
	kvm_release_page(page);
1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
			 int offset, int len)
{
	void *page_virt;
	struct page *page;

	page = gfn_to_page(kvm, gfn);
1084 1085
	if (is_error_page(page)) {
		kvm_release_page(page);
1086
		return -EFAULT;
1087
	}
1088 1089 1090 1091 1092 1093
	page_virt = kmap_atomic(page, KM_USER0);

	memcpy(page_virt + offset, data, len);

	kunmap_atomic(page_virt, KM_USER0);
	mark_page_dirty(kvm, gfn);
1094
	kvm_release_page(page);
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
		    unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

	while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		data += seg;
		++gfn;
	}
	return 0;
}

int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
	void *page_virt;
	struct page *page;

	page = gfn_to_page(kvm, gfn);
1125 1126
	if (is_error_page(page)) {
		kvm_release_page(page);
1127
		return -EFAULT;
1128
	}
1129 1130 1131 1132 1133
	page_virt = kmap_atomic(page, KM_USER0);

	memset(page_virt + offset, 0, len);

	kunmap_atomic(page_virt, KM_USER0);
1134
	kvm_release_page(page);
1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	int seg;
	int offset = offset_in_page(gpa);
	int ret;

        while ((seg = next_segment(len, offset)) != 0) {
		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
		if (ret < 0)
			return ret;
		offset = 0;
		len -= seg;
		++gfn;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

R
Rusty Russell 已提交
1158
/* WARNING: Does not work on aliased pages. */
A
Avi Kivity 已提交
1159 1160
void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
1161
	struct kvm_memory_slot *memslot;
A
Avi Kivity 已提交
1162

R
Rusty Russell 已提交
1163 1164 1165
	memslot = __gfn_to_memslot(kvm, gfn);
	if (memslot && memslot->dirty_bitmap) {
		unsigned long rel_gfn = gfn - memslot->base_gfn;
A
Avi Kivity 已提交
1166

R
Rusty Russell 已提交
1167 1168 1169
		/* avoid RMW */
		if (!test_bit(rel_gfn, memslot->dirty_bitmap))
			set_bit(rel_gfn, memslot->dirty_bitmap);
A
Avi Kivity 已提交
1170 1171 1172
	}
}

1173
int emulator_read_std(unsigned long addr,
1174
			     void *val,
A
Avi Kivity 已提交
1175
			     unsigned int bytes,
1176
			     struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1177 1178 1179 1180 1181 1182 1183
{
	void *data = val;

	while (bytes) {
		gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
1184
		int ret;
A
Avi Kivity 已提交
1185 1186 1187

		if (gpa == UNMAPPED_GVA)
			return X86EMUL_PROPAGATE_FAULT;
1188 1189
		ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy);
		if (ret < 0)
A
Avi Kivity 已提交
1190 1191 1192 1193 1194 1195 1196 1197 1198
			return X86EMUL_UNHANDLEABLE;

		bytes -= tocopy;
		data += tocopy;
		addr += tocopy;
	}

	return X86EMUL_CONTINUE;
}
1199
EXPORT_SYMBOL_GPL(emulator_read_std);
A
Avi Kivity 已提交
1200 1201

static int emulator_write_std(unsigned long addr,
1202
			      const void *val,
A
Avi Kivity 已提交
1203
			      unsigned int bytes,
1204
			      struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1205
{
1206
	pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes);
A
Avi Kivity 已提交
1207 1208 1209
	return X86EMUL_UNHANDLEABLE;
}

E
Eddie Dong 已提交
1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225
/*
 * Only apic need an MMIO device hook, so shortcut now..
 */
static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu,
						gpa_t addr)
{
	struct kvm_io_device *dev;

	if (vcpu->apic) {
		dev = &vcpu->apic->dev;
		if (dev->in_range(dev, addr))
			return dev;
	}
	return NULL;
}

1226 1227 1228
static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
						gpa_t addr)
{
E
Eddie Dong 已提交
1229 1230 1231 1232 1233 1234
	struct kvm_io_device *dev;

	dev = vcpu_find_pervcpu_dev(vcpu, addr);
	if (dev == NULL)
		dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr);
	return dev;
1235 1236
}

1237 1238 1239 1240 1241 1242
static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
					       gpa_t addr)
{
	return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr);
}

A
Avi Kivity 已提交
1243
static int emulator_read_emulated(unsigned long addr,
1244
				  void *val,
A
Avi Kivity 已提交
1245
				  unsigned int bytes,
1246
				  struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1247
{
1248 1249
	struct kvm_io_device *mmio_dev;
	gpa_t                 gpa;
A
Avi Kivity 已提交
1250 1251 1252 1253 1254

	if (vcpu->mmio_read_completed) {
		memcpy(val, vcpu->mmio_data, bytes);
		vcpu->mmio_read_completed = 0;
		return X86EMUL_CONTINUE;
1255
	} else if (emulator_read_std(addr, val, bytes, vcpu)
A
Avi Kivity 已提交
1256 1257
		   == X86EMUL_CONTINUE)
		return X86EMUL_CONTINUE;
A
Avi Kivity 已提交
1258

1259 1260 1261
	gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
	if (gpa == UNMAPPED_GVA)
		return X86EMUL_PROPAGATE_FAULT;
A
Avi Kivity 已提交
1262

1263 1264 1265 1266 1267 1268 1269
	/*
	 * Is this MMIO handled locally?
	 */
	mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
	if (mmio_dev) {
		kvm_iodevice_read(mmio_dev, gpa, bytes, val);
		return X86EMUL_CONTINUE;
A
Avi Kivity 已提交
1270
	}
1271 1272 1273 1274 1275 1276 1277

	vcpu->mmio_needed = 1;
	vcpu->mmio_phys_addr = gpa;
	vcpu->mmio_size = bytes;
	vcpu->mmio_is_write = 0;

	return X86EMUL_UNHANDLEABLE;
A
Avi Kivity 已提交
1278 1279
}

1280
static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
1281
			       const void *val, int bytes)
1282
{
1283
	int ret;
1284

1285 1286
	ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
	if (ret < 0)
1287
		return 0;
1288
	kvm_mmu_pte_write(vcpu, gpa, val, bytes);
1289 1290 1291
	return 1;
}

1292 1293 1294
static int emulator_write_emulated_onepage(unsigned long addr,
					   const void *val,
					   unsigned int bytes,
1295
					   struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1296
{
1297 1298
	struct kvm_io_device *mmio_dev;
	gpa_t                 gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
A
Avi Kivity 已提交
1299

1300
	if (gpa == UNMAPPED_GVA) {
1301
		kvm_x86_ops->inject_page_fault(vcpu, addr, 2);
A
Avi Kivity 已提交
1302
		return X86EMUL_PROPAGATE_FAULT;
1303
	}
A
Avi Kivity 已提交
1304

1305 1306 1307
	if (emulator_write_phys(vcpu, gpa, val, bytes))
		return X86EMUL_CONTINUE;

1308 1309 1310 1311 1312 1313 1314 1315 1316
	/*
	 * Is this MMIO handled locally?
	 */
	mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
	if (mmio_dev) {
		kvm_iodevice_write(mmio_dev, gpa, bytes, val);
		return X86EMUL_CONTINUE;
	}

A
Avi Kivity 已提交
1317 1318 1319 1320
	vcpu->mmio_needed = 1;
	vcpu->mmio_phys_addr = gpa;
	vcpu->mmio_size = bytes;
	vcpu->mmio_is_write = 1;
1321
	memcpy(vcpu->mmio_data, val, bytes);
A
Avi Kivity 已提交
1322 1323 1324 1325

	return X86EMUL_CONTINUE;
}

1326
int emulator_write_emulated(unsigned long addr,
1327 1328
				   const void *val,
				   unsigned int bytes,
1329
				   struct kvm_vcpu *vcpu)
1330 1331 1332 1333 1334 1335
{
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
		int rc, now;

		now = -addr & ~PAGE_MASK;
1336
		rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
1337 1338 1339 1340 1341 1342
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
		val += now;
		bytes -= now;
	}
1343
	return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
1344
}
1345
EXPORT_SYMBOL_GPL(emulator_write_emulated);
1346

A
Avi Kivity 已提交
1347
static int emulator_cmpxchg_emulated(unsigned long addr,
1348 1349
				     const void *old,
				     const void *new,
A
Avi Kivity 已提交
1350
				     unsigned int bytes,
1351
				     struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1352 1353 1354 1355 1356 1357 1358
{
	static int reported;

	if (!reported) {
		reported = 1;
		printk(KERN_WARNING "kvm: emulating exchange as write\n");
	}
1359
	return emulator_write_emulated(addr, new, bytes, vcpu);
A
Avi Kivity 已提交
1360 1361 1362 1363
}

static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
1364
	return kvm_x86_ops->get_segment_base(vcpu, seg);
A
Avi Kivity 已提交
1365 1366 1367 1368 1369 1370 1371 1372 1373
}

int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
{
	return X86EMUL_CONTINUE;
}

int emulate_clts(struct kvm_vcpu *vcpu)
{
1374
	kvm_x86_ops->set_cr0(vcpu, vcpu->cr0 & ~X86_CR0_TS);
A
Avi Kivity 已提交
1375 1376 1377
	return X86EMUL_CONTINUE;
}

M
Mike Day 已提交
1378
int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
A
Avi Kivity 已提交
1379 1380 1381 1382 1383
{
	struct kvm_vcpu *vcpu = ctxt->vcpu;

	switch (dr) {
	case 0 ... 3:
1384
		*dest = kvm_x86_ops->get_dr(vcpu, dr);
A
Avi Kivity 已提交
1385 1386
		return X86EMUL_CONTINUE;
	default:
1387
		pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr);
A
Avi Kivity 已提交
1388 1389 1390 1391 1392 1393 1394 1395 1396
		return X86EMUL_UNHANDLEABLE;
	}
}

int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
{
	unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
	int exception;

1397
	kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
A
Avi Kivity 已提交
1398 1399 1400 1401 1402 1403 1404
	if (exception) {
		/* FIXME: better handling */
		return X86EMUL_UNHANDLEABLE;
	}
	return X86EMUL_CONTINUE;
}

1405
void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context)
A
Avi Kivity 已提交
1406 1407 1408
{
	static int reported;
	u8 opcodes[4];
1409
	unsigned long rip = vcpu->rip;
A
Avi Kivity 已提交
1410 1411
	unsigned long rip_linear;

1412
	rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS);
A
Avi Kivity 已提交
1413 1414 1415 1416

	if (reported)
		return;

1417
	emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu);
A
Avi Kivity 已提交
1418

1419 1420
	printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n",
	       context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
A
Avi Kivity 已提交
1421 1422
	reported = 1;
}
1423
EXPORT_SYMBOL_GPL(kvm_report_emulation_failure);
A
Avi Kivity 已提交
1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435

struct x86_emulate_ops emulate_ops = {
	.read_std            = emulator_read_std,
	.write_std           = emulator_write_std,
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
};

int emulate_instruction(struct kvm_vcpu *vcpu,
			struct kvm_run *run,
			unsigned long cr2,
1436 1437
			u16 error_code,
			int no_decode)
A
Avi Kivity 已提交
1438
{
1439
	int r;
A
Avi Kivity 已提交
1440

1441
	vcpu->mmio_fault_cr2 = cr2;
1442
	kvm_x86_ops->cache_regs(vcpu);
A
Avi Kivity 已提交
1443 1444

	vcpu->mmio_is_write = 0;
1445
	vcpu->pio.string = 0;
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481

	if (!no_decode) {
		int cs_db, cs_l;
		kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);

		vcpu->emulate_ctxt.vcpu = vcpu;
		vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu);
		vcpu->emulate_ctxt.cr2 = cr2;
		vcpu->emulate_ctxt.mode =
			(vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM)
			? X86EMUL_MODE_REAL : cs_l
			? X86EMUL_MODE_PROT64 :	cs_db
			? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;

		if (vcpu->emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
			vcpu->emulate_ctxt.cs_base = 0;
			vcpu->emulate_ctxt.ds_base = 0;
			vcpu->emulate_ctxt.es_base = 0;
			vcpu->emulate_ctxt.ss_base = 0;
		} else {
			vcpu->emulate_ctxt.cs_base =
					get_segment_base(vcpu, VCPU_SREG_CS);
			vcpu->emulate_ctxt.ds_base =
					get_segment_base(vcpu, VCPU_SREG_DS);
			vcpu->emulate_ctxt.es_base =
					get_segment_base(vcpu, VCPU_SREG_ES);
			vcpu->emulate_ctxt.ss_base =
					get_segment_base(vcpu, VCPU_SREG_SS);
		}

		vcpu->emulate_ctxt.gs_base =
					get_segment_base(vcpu, VCPU_SREG_GS);
		vcpu->emulate_ctxt.fs_base =
					get_segment_base(vcpu, VCPU_SREG_FS);

		r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops);
1482 1483 1484 1485 1486
		if (r)  {
			if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
				return EMULATE_DONE;
			return EMULATE_FAIL;
		}
1487 1488
	}

1489
	r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops);
1490

1491 1492
	if (vcpu->pio.string)
		return EMULATE_DO_MMIO;
A
Avi Kivity 已提交
1493 1494

	if ((r || vcpu->mmio_is_write) && run) {
1495
		run->exit_reason = KVM_EXIT_MMIO;
A
Avi Kivity 已提交
1496 1497 1498 1499 1500 1501 1502
		run->mmio.phys_addr = vcpu->mmio_phys_addr;
		memcpy(run->mmio.data, vcpu->mmio_data, 8);
		run->mmio.len = vcpu->mmio_size;
		run->mmio.is_write = vcpu->mmio_is_write;
	}

	if (r) {
1503 1504
		if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
			return EMULATE_DONE;
A
Avi Kivity 已提交
1505
		if (!vcpu->mmio_needed) {
1506
			kvm_report_emulation_failure(vcpu, "mmio");
A
Avi Kivity 已提交
1507 1508 1509 1510 1511
			return EMULATE_FAIL;
		}
		return EMULATE_DO_MMIO;
	}

1512
	kvm_x86_ops->decache_regs(vcpu);
1513
	kvm_x86_ops->set_rflags(vcpu, vcpu->emulate_ctxt.eflags);
A
Avi Kivity 已提交
1514

1515 1516
	if (vcpu->mmio_is_write) {
		vcpu->mmio_needed = 0;
A
Avi Kivity 已提交
1517
		return EMULATE_DO_MMIO;
1518
	}
A
Avi Kivity 已提交
1519 1520 1521 1522 1523

	return EMULATE_DONE;
}
EXPORT_SYMBOL_GPL(emulate_instruction);

E
Eddie Dong 已提交
1524 1525 1526
/*
 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
 */
1527
static void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1528
{
E
Eddie Dong 已提交
1529 1530 1531 1532 1533 1534 1535
	DECLARE_WAITQUEUE(wait, current);

	add_wait_queue(&vcpu->wq, &wait);

	/*
	 * We will block until either an interrupt or a signal wakes us up
	 */
1536 1537 1538 1539
	while (!kvm_cpu_has_interrupt(vcpu)
	       && !signal_pending(current)
	       && vcpu->mp_state != VCPU_MP_STATE_RUNNABLE
	       && vcpu->mp_state != VCPU_MP_STATE_SIPI_RECEIVED) {
E
Eddie Dong 已提交
1540 1541 1542 1543 1544
		set_current_state(TASK_INTERRUPTIBLE);
		vcpu_put(vcpu);
		schedule();
		vcpu_load(vcpu);
	}
1545

1546
	__set_current_state(TASK_RUNNING);
E
Eddie Dong 已提交
1547 1548 1549 1550 1551
	remove_wait_queue(&vcpu->wq, &wait);
}

int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
1552
	++vcpu->stat.halt_exits;
E
Eddie Dong 已提交
1553
	if (irqchip_in_kernel(vcpu->kvm)) {
1554 1555 1556 1557
		vcpu->mp_state = VCPU_MP_STATE_HALTED;
		kvm_vcpu_block(vcpu);
		if (vcpu->mp_state != VCPU_MP_STATE_RUNNABLE)
			return -EINTR;
E
Eddie Dong 已提交
1558 1559 1560 1561 1562
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
1563 1564 1565
}
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

1566
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
1567
{
1568
	unsigned long nr, a0, a1, a2, a3, ret;
1569

1570
	kvm_x86_ops->cache_regs(vcpu);
1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583

	nr = vcpu->regs[VCPU_REGS_RAX];
	a0 = vcpu->regs[VCPU_REGS_RBX];
	a1 = vcpu->regs[VCPU_REGS_RCX];
	a2 = vcpu->regs[VCPU_REGS_RDX];
	a3 = vcpu->regs[VCPU_REGS_RSI];

	if (!is_long_mode(vcpu)) {
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
1584
	}
1585

1586 1587
	switch (nr) {
	default:
1588 1589
		ret = -KVM_ENOSYS;
		break;
1590 1591
	}
	vcpu->regs[VCPU_REGS_RAX] = ret;
1592
	kvm_x86_ops->decache_regs(vcpu);
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
{
	char instruction[3];
	int ret = 0;

	mutex_lock(&vcpu->kvm->lock);

	/*
	 * Blow out the MMU to ensure that no other VCPU has an active mapping
	 * to ensure that the updated hypercall appears atomically across all
	 * VCPUs.
	 */
	kvm_mmu_zap_all(vcpu->kvm);

	kvm_x86_ops->cache_regs(vcpu);
	kvm_x86_ops->patch_hypercall(vcpu, instruction);
	if (emulator_write_emulated(vcpu->rip, instruction, 3, vcpu)
	    != X86EMUL_CONTINUE)
		ret = -EFAULT;

	mutex_unlock(&vcpu->kvm->lock);

	return ret;
1620 1621
}

A
Avi Kivity 已提交
1622 1623 1624 1625 1626 1627 1628 1629 1630
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
{
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
}

void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
{
	struct descriptor_table dt = { limit, base };

1631
	kvm_x86_ops->set_gdt(vcpu, &dt);
A
Avi Kivity 已提交
1632 1633 1634 1635 1636 1637
}

void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
{
	struct descriptor_table dt = { limit, base };

1638
	kvm_x86_ops->set_idt(vcpu, &dt);
A
Avi Kivity 已提交
1639 1640 1641 1642 1643 1644
}

void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
		   unsigned long *rflags)
{
	lmsw(vcpu, msw);
1645
	*rflags = kvm_x86_ops->get_rflags(vcpu);
A
Avi Kivity 已提交
1646 1647 1648 1649
}

unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
{
1650
	kvm_x86_ops->decache_cr4_guest_bits(vcpu);
A
Avi Kivity 已提交
1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671
	switch (cr) {
	case 0:
		return vcpu->cr0;
	case 2:
		return vcpu->cr2;
	case 3:
		return vcpu->cr3;
	case 4:
		return vcpu->cr4;
	default:
		vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
		return 0;
	}
}

void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
		     unsigned long *rflags)
{
	switch (cr) {
	case 0:
		set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1672
		*rflags = kvm_x86_ops->get_rflags(vcpu);
A
Avi Kivity 已提交
1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687
		break;
	case 2:
		vcpu->cr2 = val;
		break;
	case 3:
		set_cr3(vcpu, val);
		break;
	case 4:
		set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
		break;
	default:
		vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
	}
}

1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706
int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data;

	switch (msr) {
	case 0xc0010010: /* SYSCFG */
	case 0xc0010015: /* HWCR */
	case MSR_IA32_PLATFORM_ID:
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MC0_CTL:
	case MSR_IA32_MCG_STATUS:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MC0_MISC:
	case MSR_IA32_MC0_MISC+4:
	case MSR_IA32_MC0_MISC+8:
	case MSR_IA32_MC0_MISC+12:
	case MSR_IA32_MC0_MISC+16:
	case MSR_IA32_UCODE_REV:
A
Avi Kivity 已提交
1707
	case MSR_IA32_PERF_STATUS:
1708
	case MSR_IA32_EBL_CR_POWERON:
1709 1710 1711 1712 1713
		/* MTRR registers */
	case 0xfe:
	case 0x200 ... 0x2ff:
		data = 0;
		break;
A
Avi Kivity 已提交
1714 1715 1716
	case 0xcd: /* fsb frequency */
		data = 3;
		break;
1717
	case MSR_IA32_APICBASE:
1718
		data = kvm_get_apic_base(vcpu);
1719
		break;
1720 1721 1722
	case MSR_IA32_MISC_ENABLE:
		data = vcpu->ia32_misc_enable_msr;
		break;
1723 1724 1725 1726 1727 1728
#ifdef CONFIG_X86_64
	case MSR_EFER:
		data = vcpu->shadow_efer;
		break;
#endif
	default:
1729
		pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
1730 1731 1732 1733 1734 1735 1736
		return 1;
	}
	*pdata = data;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

A
Avi Kivity 已提交
1737 1738 1739 1740 1741
/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1742
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
A
Avi Kivity 已提交
1743
{
1744
	return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
A
Avi Kivity 已提交
1745 1746
}

1747
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1748

1749
static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
A
Avi Kivity 已提交
1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764
{
	if (efer & EFER_RESERVED_BITS) {
		printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
		       efer);
		inject_gp(vcpu);
		return;
	}

	if (is_paging(vcpu)
	    && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
		printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
		inject_gp(vcpu);
		return;
	}

1765
	kvm_x86_ops->set_efer(vcpu, efer);
1766

A
Avi Kivity 已提交
1767 1768 1769 1770 1771 1772 1773 1774
	efer &= ~EFER_LMA;
	efer |= vcpu->shadow_efer & EFER_LMA;

	vcpu->shadow_efer = efer;
}

#endif

1775 1776 1777 1778 1779 1780 1781 1782 1783
int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	switch (msr) {
#ifdef CONFIG_X86_64
	case MSR_EFER:
		set_efer(vcpu, data);
		break;
#endif
	case MSR_IA32_MC0_STATUS:
1784
		pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1785 1786
		       __FUNCTION__, data);
		break;
1787
	case MSR_IA32_MCG_STATUS:
1788
		pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
1789 1790
			__FUNCTION__, data);
		break;
1791 1792 1793 1794 1795
	case MSR_IA32_UCODE_REV:
	case MSR_IA32_UCODE_WRITE:
	case 0x200 ... 0x2ff: /* MTRRs */
		break;
	case MSR_IA32_APICBASE:
1796
		kvm_set_apic_base(vcpu, data);
1797
		break;
1798 1799 1800
	case MSR_IA32_MISC_ENABLE:
		vcpu->ia32_misc_enable_msr = data;
		break;
1801
	default:
1802
		pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr);
1803 1804 1805 1806 1807 1808
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);

A
Avi Kivity 已提交
1809 1810 1811 1812 1813
/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1814
int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
A
Avi Kivity 已提交
1815
{
1816
	return kvm_x86_ops->set_msr(vcpu, msr_index, data);
A
Avi Kivity 已提交
1817 1818 1819 1820
}

void kvm_resched(struct kvm_vcpu *vcpu)
{
1821 1822
	if (!need_resched())
		return;
A
Avi Kivity 已提交
1823 1824 1825 1826
	cond_resched();
}
EXPORT_SYMBOL_GPL(kvm_resched);

1827 1828 1829 1830 1831 1832
void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
{
	int i;
	u32 function;
	struct kvm_cpuid_entry *e, *best;

1833
	kvm_x86_ops->cache_regs(vcpu);
1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858
	function = vcpu->regs[VCPU_REGS_RAX];
	vcpu->regs[VCPU_REGS_RAX] = 0;
	vcpu->regs[VCPU_REGS_RBX] = 0;
	vcpu->regs[VCPU_REGS_RCX] = 0;
	vcpu->regs[VCPU_REGS_RDX] = 0;
	best = NULL;
	for (i = 0; i < vcpu->cpuid_nent; ++i) {
		e = &vcpu->cpuid_entries[i];
		if (e->function == function) {
			best = e;
			break;
		}
		/*
		 * Both basic or both extended?
		 */
		if (((e->function ^ function) & 0x80000000) == 0)
			if (!best || e->function > best->function)
				best = e;
	}
	if (best) {
		vcpu->regs[VCPU_REGS_RAX] = best->eax;
		vcpu->regs[VCPU_REGS_RBX] = best->ebx;
		vcpu->regs[VCPU_REGS_RCX] = best->ecx;
		vcpu->regs[VCPU_REGS_RDX] = best->edx;
	}
1859 1860
	kvm_x86_ops->decache_regs(vcpu);
	kvm_x86_ops->skip_emulated_instruction(vcpu);
1861 1862 1863
}
EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);

1864
static int pio_copy_data(struct kvm_vcpu *vcpu)
1865
{
1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891
	void *p = vcpu->pio_data;
	void *q;
	unsigned bytes;
	int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1;

	q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
		 PAGE_KERNEL);
	if (!q) {
		free_pio_guest_pages(vcpu);
		return -ENOMEM;
	}
	q += vcpu->pio.guest_page_offset;
	bytes = vcpu->pio.size * vcpu->pio.cur_count;
	if (vcpu->pio.in)
		memcpy(q, p, bytes);
	else
		memcpy(p, q, bytes);
	q -= vcpu->pio.guest_page_offset;
	vunmap(q);
	free_pio_guest_pages(vcpu);
	return 0;
}

static int complete_pio(struct kvm_vcpu *vcpu)
{
	struct kvm_pio_request *io = &vcpu->pio;
1892
	long delta;
1893
	int r;
1894

1895
	kvm_x86_ops->cache_regs(vcpu);
1896 1897

	if (!io->string) {
1898 1899
		if (io->in)
			memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data,
1900 1901
			       io->size);
	} else {
1902 1903 1904
		if (io->in) {
			r = pio_copy_data(vcpu);
			if (r) {
1905
				kvm_x86_ops->cache_regs(vcpu);
1906 1907 1908 1909
				return r;
			}
		}

1910 1911
		delta = 1;
		if (io->rep) {
1912
			delta *= io->cur_count;
1913 1914 1915 1916 1917 1918
			/*
			 * The size of the register should really depend on
			 * current address size.
			 */
			vcpu->regs[VCPU_REGS_RCX] -= delta;
		}
1919
		if (io->down)
1920 1921
			delta = -delta;
		delta *= io->size;
1922
		if (io->in)
1923 1924 1925 1926 1927
			vcpu->regs[VCPU_REGS_RDI] += delta;
		else
			vcpu->regs[VCPU_REGS_RSI] += delta;
	}

1928
	kvm_x86_ops->decache_regs(vcpu);
1929

1930 1931 1932 1933
	io->count -= io->cur_count;
	io->cur_count = 0;

	return 0;
1934 1935
}

1936 1937 1938
static void kernel_pio(struct kvm_io_device *pio_dev,
		       struct kvm_vcpu *vcpu,
		       void *pd)
1939 1940 1941
{
	/* TODO: String I/O for in kernel device */

1942
	mutex_lock(&vcpu->kvm->lock);
1943 1944 1945
	if (vcpu->pio.in)
		kvm_iodevice_read(pio_dev, vcpu->pio.port,
				  vcpu->pio.size,
1946
				  pd);
1947 1948 1949
	else
		kvm_iodevice_write(pio_dev, vcpu->pio.port,
				   vcpu->pio.size,
1950
				   pd);
1951
	mutex_unlock(&vcpu->kvm->lock);
1952 1953 1954 1955 1956 1957 1958 1959 1960
}

static void pio_string_write(struct kvm_io_device *pio_dev,
			     struct kvm_vcpu *vcpu)
{
	struct kvm_pio_request *io = &vcpu->pio;
	void *pd = vcpu->pio_data;
	int i;

1961
	mutex_lock(&vcpu->kvm->lock);
1962 1963 1964 1965 1966 1967
	for (i = 0; i < io->cur_count; i++) {
		kvm_iodevice_write(pio_dev, io->port,
				   io->size,
				   pd);
		pd += io->size;
	}
1968
	mutex_unlock(&vcpu->kvm->lock);
1969 1970
}

M
Mike Day 已提交
1971
int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
L
Laurent Vivier 已提交
1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987
		  int size, unsigned port)
{
	struct kvm_io_device *pio_dev;

	vcpu->run->exit_reason = KVM_EXIT_IO;
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
	vcpu->run->io.size = vcpu->pio.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1;
	vcpu->run->io.port = vcpu->pio.port = port;
	vcpu->pio.in = in;
	vcpu->pio.string = 0;
	vcpu->pio.down = 0;
	vcpu->pio.guest_page_offset = 0;
	vcpu->pio.rep = 0;

1988
	kvm_x86_ops->cache_regs(vcpu);
L
Laurent Vivier 已提交
1989
	memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4);
1990
	kvm_x86_ops->decache_regs(vcpu);
L
Laurent Vivier 已提交
1991

1992 1993
	kvm_x86_ops->skip_emulated_instruction(vcpu);

L
Laurent Vivier 已提交
1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005
	pio_dev = vcpu_find_pio_dev(vcpu, port);
	if (pio_dev) {
		kernel_pio(pio_dev, vcpu, vcpu->pio_data);
		complete_pio(vcpu);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_emulate_pio);

int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
		  int size, unsigned long count, int down,
2006 2007 2008
		  gva_t address, int rep, unsigned port)
{
	unsigned now, in_page;
2009
	int i, ret = 0;
2010 2011
	int nr_pages = 1;
	struct page *page;
2012
	struct kvm_io_device *pio_dev;
2013 2014 2015

	vcpu->run->exit_reason = KVM_EXIT_IO;
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
L
Laurent Vivier 已提交
2016
	vcpu->run->io.size = vcpu->pio.size = size;
2017
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
L
Laurent Vivier 已提交
2018 2019
	vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count;
	vcpu->run->io.port = vcpu->pio.port = port;
2020
	vcpu->pio.in = in;
L
Laurent Vivier 已提交
2021
	vcpu->pio.string = 1;
2022 2023 2024 2025 2026
	vcpu->pio.down = down;
	vcpu->pio.guest_page_offset = offset_in_page(address);
	vcpu->pio.rep = rep;

	if (!count) {
2027
		kvm_x86_ops->skip_emulated_instruction(vcpu);
2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048
		return 1;
	}

	if (!down)
		in_page = PAGE_SIZE - offset_in_page(address);
	else
		in_page = offset_in_page(address) + size;
	now = min(count, (unsigned long)in_page / size);
	if (!now) {
		/*
		 * String I/O straddles page boundary.  Pin two guest pages
		 * so that we satisfy atomicity constraints.  Do just one
		 * transaction to avoid complexity.
		 */
		nr_pages = 2;
		now = 1;
	}
	if (down) {
		/*
		 * String I/O in reverse.  Yuck.  Kill the guest, fix later.
		 */
2049
		pr_unimpl(vcpu, "guest string pio down\n");
2050 2051 2052 2053 2054 2055
		inject_gp(vcpu);
		return 1;
	}
	vcpu->run->io.count = now;
	vcpu->pio.cur_count = now;

2056 2057 2058
	if (vcpu->pio.cur_count == vcpu->pio.count)
		kvm_x86_ops->skip_emulated_instruction(vcpu);

2059
	for (i = 0; i < nr_pages; ++i) {
S
Shaohua Li 已提交
2060
		mutex_lock(&vcpu->kvm->lock);
2061 2062
		page = gva_to_page(vcpu, address + i * PAGE_SIZE);
		vcpu->pio.guest_pages[i] = page;
S
Shaohua Li 已提交
2063
		mutex_unlock(&vcpu->kvm->lock);
2064 2065 2066 2067 2068 2069 2070
		if (!page) {
			inject_gp(vcpu);
			free_pio_guest_pages(vcpu);
			return 1;
		}
	}

L
Laurent Vivier 已提交
2071
	pio_dev = vcpu_find_pio_dev(vcpu, port);
2072 2073 2074 2075 2076 2077 2078 2079 2080 2081
	if (!vcpu->pio.in) {
		/* string PIO write */
		ret = pio_copy_data(vcpu);
		if (ret >= 0 && pio_dev) {
			pio_string_write(pio_dev, vcpu);
			complete_pio(vcpu);
			if (vcpu->pio.count == 0)
				ret = 1;
		}
	} else if (pio_dev)
2082
		pr_unimpl(vcpu, "no string pio read support yet, "
2083 2084 2085 2086
		       "port %x size %d count %ld\n",
			port, size, count);

	return ret;
2087
}
L
Laurent Vivier 已提交
2088
EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
2089

2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123
/*
 * Check if userspace requested an interrupt window, and that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
					  struct kvm_run *kvm_run)
{
	return (!vcpu->irq_summary &&
		kvm_run->request_interrupt_window &&
		vcpu->interrupt_window_open &&
		(kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF));
}

static void post_kvm_run_save(struct kvm_vcpu *vcpu,
			      struct kvm_run *kvm_run)
{
	kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
	kvm_run->cr8 = get_cr8(vcpu);
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_run->ready_for_interrupt_injection = 1;
	else
		kvm_run->ready_for_interrupt_injection =
					(vcpu->interrupt_window_open &&
					 vcpu->irq_summary == 0);
}

static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
{
	int r;

	if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) {
M
Mike Day 已提交
2124
		pr_debug("vcpu %d received sipi with vector # %x\n",
2125 2126
		       vcpu->vcpu_id, vcpu->sipi_vector);
		kvm_lapic_reset(vcpu);
2127 2128 2129
		r = kvm_x86_ops->vcpu_reset(vcpu);
		if (r)
			return r;
2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141
		vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
	}

preempted:
	if (vcpu->guest_debug.enabled)
		kvm_x86_ops->guest_debug_pre(vcpu);

again:
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		goto out;

2142 2143
	kvm_inject_pending_timer_irqs(vcpu);

2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165
	preempt_disable();

	kvm_x86_ops->prepare_guest_switch(vcpu);
	kvm_load_guest_fpu(vcpu);

	local_irq_disable();

	if (signal_pending(current)) {
		local_irq_enable();
		preempt_enable();
		r = -EINTR;
		kvm_run->exit_reason = KVM_EXIT_INTR;
		++vcpu->stat.signal_exits;
		goto out;
	}

	if (irqchip_in_kernel(vcpu->kvm))
		kvm_x86_ops->inject_pending_irq(vcpu);
	else if (!vcpu->mmio_read_completed)
		kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run);

	vcpu->guest_mode = 1;
2166
	kvm_guest_enter();
2167 2168

	if (vcpu->requests)
2169
		if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
2170 2171 2172 2173 2174 2175 2176 2177 2178
			kvm_x86_ops->tlb_flush(vcpu);

	kvm_x86_ops->run(vcpu, kvm_run);

	vcpu->guest_mode = 0;
	local_irq_enable();

	++vcpu->stat.exits;

2179 2180 2181 2182 2183 2184 2185 2186 2187 2188
	/*
	 * We must have an instruction between local_irq_enable() and
	 * kvm_guest_exit(), so the timer interrupt isn't delayed by
	 * the interrupt shadow.  The stat.exits increment will do nicely.
	 * But we need to prevent reordering, hence this barrier():
	 */
	barrier();

	kvm_guest_exit();

2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225
	preempt_enable();

	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
		kvm_x86_ops->cache_regs(vcpu);
		profile_hit(KVM_PROFILING, (void *)vcpu->rip);
	}

	r = kvm_x86_ops->handle_exit(kvm_run, vcpu);

	if (r > 0) {
		if (dm_request_for_irq_injection(vcpu, kvm_run)) {
			r = -EINTR;
			kvm_run->exit_reason = KVM_EXIT_INTR;
			++vcpu->stat.request_irq_exits;
			goto out;
		}
		if (!need_resched()) {
			++vcpu->stat.light_exits;
			goto again;
		}
	}

out:
	if (r > 0) {
		kvm_resched(vcpu);
		goto preempted;
	}

	post_kvm_run_save(vcpu, kvm_run);

	return r;
}


A
Avi Kivity 已提交
2226
static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
2227 2228
{
	int r;
A
Avi Kivity 已提交
2229
	sigset_t sigsaved;
A
Avi Kivity 已提交
2230

A
Avi Kivity 已提交
2231
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2232

2233 2234 2235 2236 2237 2238
	if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) {
		kvm_vcpu_block(vcpu);
		vcpu_put(vcpu);
		return -EAGAIN;
	}

A
Avi Kivity 已提交
2239 2240 2241
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);

2242
	/* re-sync apic's tpr */
2243 2244
	if (!irqchip_in_kernel(vcpu->kvm))
		set_cr8(vcpu, kvm_run->cr8);
2245

2246 2247 2248 2249 2250
	if (vcpu->pio.cur_count) {
		r = complete_pio(vcpu);
		if (r)
			goto out;
	}
2251
#if CONFIG_HAS_IOMEM
2252 2253 2254 2255 2256
	if (vcpu->mmio_needed) {
		memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
		vcpu->mmio_read_completed = 1;
		vcpu->mmio_needed = 0;
		r = emulate_instruction(vcpu, kvm_run,
2257
					vcpu->mmio_fault_cr2, 0, 1);
2258 2259 2260 2261 2262 2263
		if (r == EMULATE_DO_MMIO) {
			/*
			 * Read-modify-write.  Back to userspace.
			 */
			r = 0;
			goto out;
2264
		}
A
Avi Kivity 已提交
2265
	}
2266
#endif
2267
	if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
2268
		kvm_x86_ops->cache_regs(vcpu);
2269
		vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
2270
		kvm_x86_ops->decache_regs(vcpu);
2271 2272
	}

2273
	r = __vcpu_run(vcpu, kvm_run);
A
Avi Kivity 已提交
2274

2275
out:
A
Avi Kivity 已提交
2276 2277 2278
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

A
Avi Kivity 已提交
2279 2280 2281 2282
	vcpu_put(vcpu);
	return r;
}

A
Avi Kivity 已提交
2283 2284
static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
				   struct kvm_regs *regs)
A
Avi Kivity 已提交
2285
{
A
Avi Kivity 已提交
2286
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2287

2288
	kvm_x86_ops->cache_regs(vcpu);
A
Avi Kivity 已提交
2289 2290 2291 2292 2293 2294 2295 2296 2297

	regs->rax = vcpu->regs[VCPU_REGS_RAX];
	regs->rbx = vcpu->regs[VCPU_REGS_RBX];
	regs->rcx = vcpu->regs[VCPU_REGS_RCX];
	regs->rdx = vcpu->regs[VCPU_REGS_RDX];
	regs->rsi = vcpu->regs[VCPU_REGS_RSI];
	regs->rdi = vcpu->regs[VCPU_REGS_RDI];
	regs->rsp = vcpu->regs[VCPU_REGS_RSP];
	regs->rbp = vcpu->regs[VCPU_REGS_RBP];
2298
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309
	regs->r8 = vcpu->regs[VCPU_REGS_R8];
	regs->r9 = vcpu->regs[VCPU_REGS_R9];
	regs->r10 = vcpu->regs[VCPU_REGS_R10];
	regs->r11 = vcpu->regs[VCPU_REGS_R11];
	regs->r12 = vcpu->regs[VCPU_REGS_R12];
	regs->r13 = vcpu->regs[VCPU_REGS_R13];
	regs->r14 = vcpu->regs[VCPU_REGS_R14];
	regs->r15 = vcpu->regs[VCPU_REGS_R15];
#endif

	regs->rip = vcpu->rip;
2310
	regs->rflags = kvm_x86_ops->get_rflags(vcpu);
A
Avi Kivity 已提交
2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322

	/*
	 * Don't leak debug flags in case they were set for guest debugging
	 */
	if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
		regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);

	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2323 2324
static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
				   struct kvm_regs *regs)
A
Avi Kivity 已提交
2325
{
A
Avi Kivity 已提交
2326
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2327 2328 2329 2330 2331 2332 2333 2334 2335

	vcpu->regs[VCPU_REGS_RAX] = regs->rax;
	vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
	vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
	vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
	vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
	vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
	vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
	vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
2336
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347
	vcpu->regs[VCPU_REGS_R8] = regs->r8;
	vcpu->regs[VCPU_REGS_R9] = regs->r9;
	vcpu->regs[VCPU_REGS_R10] = regs->r10;
	vcpu->regs[VCPU_REGS_R11] = regs->r11;
	vcpu->regs[VCPU_REGS_R12] = regs->r12;
	vcpu->regs[VCPU_REGS_R13] = regs->r13;
	vcpu->regs[VCPU_REGS_R14] = regs->r14;
	vcpu->regs[VCPU_REGS_R15] = regs->r15;
#endif

	vcpu->rip = regs->rip;
2348
	kvm_x86_ops->set_rflags(vcpu, regs->rflags);
A
Avi Kivity 已提交
2349

2350
	kvm_x86_ops->decache_regs(vcpu);
A
Avi Kivity 已提交
2351 2352 2353 2354 2355 2356 2357 2358 2359

	vcpu_put(vcpu);

	return 0;
}

static void get_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
2360
	return kvm_x86_ops->get_segment(vcpu, var, seg);
A
Avi Kivity 已提交
2361 2362
}

A
Avi Kivity 已提交
2363 2364
static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				    struct kvm_sregs *sregs)
A
Avi Kivity 已提交
2365 2366
{
	struct descriptor_table dt;
E
Eddie Dong 已提交
2367
	int pending_vec;
A
Avi Kivity 已提交
2368

A
Avi Kivity 已提交
2369
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380

	get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);

	get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);

2381
	kvm_x86_ops->get_idt(vcpu, &dt);
A
Avi Kivity 已提交
2382 2383
	sregs->idt.limit = dt.limit;
	sregs->idt.base = dt.base;
2384
	kvm_x86_ops->get_gdt(vcpu, &dt);
A
Avi Kivity 已提交
2385 2386 2387
	sregs->gdt.limit = dt.limit;
	sregs->gdt.base = dt.base;

2388
	kvm_x86_ops->decache_cr4_guest_bits(vcpu);
A
Avi Kivity 已提交
2389 2390 2391 2392
	sregs->cr0 = vcpu->cr0;
	sregs->cr2 = vcpu->cr2;
	sregs->cr3 = vcpu->cr3;
	sregs->cr4 = vcpu->cr4;
2393
	sregs->cr8 = get_cr8(vcpu);
A
Avi Kivity 已提交
2394
	sregs->efer = vcpu->shadow_efer;
2395
	sregs->apic_base = kvm_get_apic_base(vcpu);
A
Avi Kivity 已提交
2396

E
Eddie Dong 已提交
2397
	if (irqchip_in_kernel(vcpu->kvm)) {
2398 2399
		memset(sregs->interrupt_bitmap, 0,
		       sizeof sregs->interrupt_bitmap);
2400
		pending_vec = kvm_x86_ops->get_irq(vcpu);
E
Eddie Dong 已提交
2401
		if (pending_vec >= 0)
M
Mike Day 已提交
2402 2403
			set_bit(pending_vec,
				(unsigned long *)sregs->interrupt_bitmap);
E
Eddie Dong 已提交
2404
	} else
2405 2406
		memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
		       sizeof sregs->interrupt_bitmap);
A
Avi Kivity 已提交
2407 2408 2409 2410 2411 2412 2413 2414 2415

	vcpu_put(vcpu);

	return 0;
}

static void set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
2416
	return kvm_x86_ops->set_segment(vcpu, var, seg);
A
Avi Kivity 已提交
2417 2418
}

A
Avi Kivity 已提交
2419 2420
static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				    struct kvm_sregs *sregs)
A
Avi Kivity 已提交
2421 2422
{
	int mmu_reset_needed = 0;
E
Eddie Dong 已提交
2423
	int i, pending_vec, max_bits;
A
Avi Kivity 已提交
2424 2425
	struct descriptor_table dt;

A
Avi Kivity 已提交
2426
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2427 2428 2429

	dt.limit = sregs->idt.limit;
	dt.base = sregs->idt.base;
2430
	kvm_x86_ops->set_idt(vcpu, &dt);
A
Avi Kivity 已提交
2431 2432
	dt.limit = sregs->gdt.limit;
	dt.base = sregs->gdt.base;
2433
	kvm_x86_ops->set_gdt(vcpu, &dt);
A
Avi Kivity 已提交
2434 2435 2436 2437 2438

	vcpu->cr2 = sregs->cr2;
	mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
	vcpu->cr3 = sregs->cr3;

2439
	set_cr8(vcpu, sregs->cr8);
A
Avi Kivity 已提交
2440 2441

	mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
2442
#ifdef CONFIG_X86_64
2443
	kvm_x86_ops->set_efer(vcpu, sregs->efer);
A
Avi Kivity 已提交
2444
#endif
2445
	kvm_set_apic_base(vcpu, sregs->apic_base);
A
Avi Kivity 已提交
2446

2447
	kvm_x86_ops->decache_cr4_guest_bits(vcpu);
2448

A
Avi Kivity 已提交
2449
	mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
R
Rusty Russell 已提交
2450
	vcpu->cr0 = sregs->cr0;
2451
	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
A
Avi Kivity 已提交
2452 2453

	mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
2454
	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
2455 2456
	if (!is_long_mode(vcpu) && is_pae(vcpu))
		load_pdptrs(vcpu, vcpu->cr3);
A
Avi Kivity 已提交
2457 2458 2459 2460

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

2461 2462 2463 2464 2465 2466 2467
	if (!irqchip_in_kernel(vcpu->kvm)) {
		memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
		       sizeof vcpu->irq_pending);
		vcpu->irq_summary = 0;
		for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i)
			if (vcpu->irq_pending[i])
				__set_bit(i, &vcpu->irq_summary);
E
Eddie Dong 已提交
2468 2469 2470 2471 2472 2473 2474
	} else {
		max_bits = (sizeof sregs->interrupt_bitmap) << 3;
		pending_vec = find_first_bit(
			(const unsigned long *)sregs->interrupt_bitmap,
			max_bits);
		/* Only pending external irq is handled here */
		if (pending_vec < max_bits) {
2475
			kvm_x86_ops->set_irq(vcpu, pending_vec);
M
Mike Day 已提交
2476 2477
			pr_debug("Set back pending irq %d\n",
				 pending_vec);
E
Eddie Dong 已提交
2478
		}
2479
	}
A
Avi Kivity 已提交
2480

2481 2482 2483 2484 2485 2486 2487 2488 2489 2490
	set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);

	set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);

A
Avi Kivity 已提交
2491 2492 2493 2494 2495
	vcpu_put(vcpu);

	return 0;
}

2496 2497 2498 2499 2500 2501 2502 2503 2504 2505
void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

	get_segment(vcpu, &cs, VCPU_SREG_CS);
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

A
Avi Kivity 已提交
2506 2507 2508
/*
 * Translate a guest virtual address to a guest physical address.
 */
A
Avi Kivity 已提交
2509 2510
static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
A
Avi Kivity 已提交
2511 2512 2513 2514
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;

A
Avi Kivity 已提交
2515
	vcpu_load(vcpu);
S
Shaohua Li 已提交
2516
	mutex_lock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
2517 2518 2519 2520 2521
	gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;
S
Shaohua Li 已提交
2522
	mutex_unlock(&vcpu->kvm->lock);
A
Avi Kivity 已提交
2523 2524 2525 2526 2527
	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2528 2529
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
A
Avi Kivity 已提交
2530 2531 2532
{
	if (irq->irq < 0 || irq->irq >= 256)
		return -EINVAL;
E
Eddie Dong 已提交
2533 2534
	if (irqchip_in_kernel(vcpu->kvm))
		return -ENXIO;
A
Avi Kivity 已提交
2535
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2536 2537 2538 2539 2540 2541 2542 2543 2544

	set_bit(irq->irq, vcpu->irq_pending);
	set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);

	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2545 2546
static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
				      struct kvm_debug_guest *dbg)
A
Avi Kivity 已提交
2547 2548 2549
{
	int r;

A
Avi Kivity 已提交
2550
	vcpu_load(vcpu);
A
Avi Kivity 已提交
2551

2552
	r = kvm_x86_ops->set_guest_debug(vcpu, dbg);
A
Avi Kivity 已提交
2553 2554 2555 2556 2557 2558

	vcpu_put(vcpu);

	return r;
}

2559 2560 2561 2562 2563 2564 2565 2566 2567
static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
				    unsigned long address,
				    int *type)
{
	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
	unsigned long pgoff;
	struct page *page;

	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2568 2569 2570 2571 2572
	if (pgoff == 0)
		page = virt_to_page(vcpu->run);
	else if (pgoff == KVM_PIO_PAGE_OFFSET)
		page = virt_to_page(vcpu->pio_data);
	else
2573 2574
		return NOPAGE_SIGBUS;
	get_page(page);
2575 2576 2577
	if (type != NULL)
		*type = VM_FAULT_MINOR;

2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590
	return page;
}

static struct vm_operations_struct kvm_vcpu_vm_ops = {
	.nopage = kvm_vcpu_nopage,
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vcpu_vm_ops;
	return 0;
}

A
Avi Kivity 已提交
2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602
static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
	struct kvm_vcpu *vcpu = filp->private_data;

	fput(vcpu->kvm->filp);
	return 0;
}

static struct file_operations kvm_vcpu_fops = {
	.release        = kvm_vcpu_release,
	.unlocked_ioctl = kvm_vcpu_ioctl,
	.compat_ioctl   = kvm_vcpu_ioctl,
2603
	.mmap           = kvm_vcpu_mmap,
A
Avi Kivity 已提交
2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
	int fd, r;
	struct inode *inode;
	struct file *file;

2615 2616 2617 2618
	r = anon_inode_getfd(&fd, &inode, &file,
			     "kvm-vcpu", &kvm_vcpu_fops, vcpu);
	if (r)
		return r;
A
Avi Kivity 已提交
2619 2620 2621 2622
	atomic_inc(&vcpu->kvm->filp->f_count);
	return fd;
}

2623 2624 2625 2626 2627 2628 2629 2630 2631
/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
{
	int r;
	struct kvm_vcpu *vcpu;

	if (!valid_vcpu(n))
R
Rusty Russell 已提交
2632
		return -EINVAL;
2633

2634
	vcpu = kvm_x86_ops->vcpu_create(kvm, n);
R
Rusty Russell 已提交
2635 2636
	if (IS_ERR(vcpu))
		return PTR_ERR(vcpu);
2637

2638 2639
	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);

2640 2641 2642
	/* We do fxsave: this must be aligned. */
	BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF);

R
Rusty Russell 已提交
2643
	vcpu_load(vcpu);
2644 2645 2646
	r = kvm_x86_ops->vcpu_reset(vcpu);
	if (r == 0)
		r = kvm_mmu_setup(vcpu);
2647 2648
	vcpu_put(vcpu);
	if (r < 0)
R
Rusty Russell 已提交
2649 2650
		goto free_vcpu;

S
Shaohua Li 已提交
2651
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
2652 2653
	if (kvm->vcpus[n]) {
		r = -EEXIST;
S
Shaohua Li 已提交
2654
		mutex_unlock(&kvm->lock);
R
Rusty Russell 已提交
2655 2656 2657
		goto mmu_unload;
	}
	kvm->vcpus[n] = vcpu;
S
Shaohua Li 已提交
2658
	mutex_unlock(&kvm->lock);
2659

R
Rusty Russell 已提交
2660
	/* Now it's all set up, let userspace reach it */
A
Avi Kivity 已提交
2661 2662
	r = create_vcpu_fd(vcpu);
	if (r < 0)
R
Rusty Russell 已提交
2663 2664
		goto unlink;
	return r;
2665

R
Rusty Russell 已提交
2666
unlink:
S
Shaohua Li 已提交
2667
	mutex_lock(&kvm->lock);
R
Rusty Russell 已提交
2668
	kvm->vcpus[n] = NULL;
S
Shaohua Li 已提交
2669
	mutex_unlock(&kvm->lock);
2670

R
Rusty Russell 已提交
2671 2672 2673 2674
mmu_unload:
	vcpu_load(vcpu);
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
2675

R
Rusty Russell 已提交
2676
free_vcpu:
2677
	kvm_x86_ops->vcpu_free(vcpu);
2678 2679 2680
	return r;
}

A
Avi Kivity 已提交
2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691
static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
	if (sigset) {
		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
		vcpu->sigset_active = 1;
		vcpu->sigset = *sigset;
	} else
		vcpu->sigset_active = 0;
	return 0;
}

A
Avi Kivity 已提交
2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714
/*
 * fxsave fpu state.  Taken from x86_64/processor.h.  To be killed when
 * we have asm/x86/processor.h
 */
struct fxsave {
	u16	cwd;
	u16	swd;
	u16	twd;
	u16	fop;
	u64	rip;
	u64	rdp;
	u32	mxcsr;
	u32	mxcsr_mask;
	u32	st_space[32];	/* 8*16 bytes for each FP-reg = 128 bytes */
#ifdef CONFIG_X86_64
	u32	xmm_space[64];	/* 16*16 bytes for each XMM-reg = 256 bytes */
#else
	u32	xmm_space[32];	/* 8*16 bytes for each XMM-reg = 128 bytes */
#endif
};

static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
2715
	struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image;
A
Avi Kivity 已提交
2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734

	vcpu_load(vcpu);

	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);

	vcpu_put(vcpu);

	return 0;
}

static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
2735
	struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image;
A
Avi Kivity 已提交
2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752

	vcpu_load(vcpu);

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);

	vcpu_put(vcpu);

	return 0;
}

A
Avi Kivity 已提交
2753 2754
static long kvm_vcpu_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
A
Avi Kivity 已提交
2755
{
A
Avi Kivity 已提交
2756
	struct kvm_vcpu *vcpu = filp->private_data;
A
Al Viro 已提交
2757
	void __user *argp = (void __user *)arg;
2758
	int r;
A
Avi Kivity 已提交
2759 2760

	switch (ioctl) {
2761
	case KVM_RUN:
2762 2763 2764
		r = -EINVAL;
		if (arg)
			goto out;
2765
		r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
A
Avi Kivity 已提交
2766 2767 2768 2769
		break;
	case KVM_GET_REGS: {
		struct kvm_regs kvm_regs;

A
Avi Kivity 已提交
2770 2771
		memset(&kvm_regs, 0, sizeof kvm_regs);
		r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
A
Avi Kivity 已提交
2772 2773 2774
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2775
		if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
A
Avi Kivity 已提交
2776 2777 2778 2779 2780 2781 2782 2783
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_REGS: {
		struct kvm_regs kvm_regs;

		r = -EFAULT;
A
Al Viro 已提交
2784
		if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
A
Avi Kivity 已提交
2785
			goto out;
A
Avi Kivity 已提交
2786
		r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
A
Avi Kivity 已提交
2787 2788 2789 2790 2791 2792 2793 2794
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_SREGS: {
		struct kvm_sregs kvm_sregs;

A
Avi Kivity 已提交
2795 2796
		memset(&kvm_sregs, 0, sizeof kvm_sregs);
		r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
A
Avi Kivity 已提交
2797 2798 2799
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2800
		if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
A
Avi Kivity 已提交
2801 2802 2803 2804 2805 2806 2807 2808
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS: {
		struct kvm_sregs kvm_sregs;

		r = -EFAULT;
A
Al Viro 已提交
2809
		if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
A
Avi Kivity 已提交
2810
			goto out;
A
Avi Kivity 已提交
2811
		r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
A
Avi Kivity 已提交
2812 2813 2814 2815 2816 2817 2818 2819 2820
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_TRANSLATE: {
		struct kvm_translation tr;

		r = -EFAULT;
A
Al Viro 已提交
2821
		if (copy_from_user(&tr, argp, sizeof tr))
A
Avi Kivity 已提交
2822
			goto out;
A
Avi Kivity 已提交
2823
		r = kvm_vcpu_ioctl_translate(vcpu, &tr);
A
Avi Kivity 已提交
2824 2825 2826
		if (r)
			goto out;
		r = -EFAULT;
A
Al Viro 已提交
2827
		if (copy_to_user(argp, &tr, sizeof tr))
A
Avi Kivity 已提交
2828 2829 2830 2831 2832 2833 2834 2835
			goto out;
		r = 0;
		break;
	}
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
A
Al Viro 已提交
2836
		if (copy_from_user(&irq, argp, sizeof irq))
A
Avi Kivity 已提交
2837
			goto out;
A
Avi Kivity 已提交
2838
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
A
Avi Kivity 已提交
2839 2840 2841 2842 2843 2844 2845 2846 2847
		if (r)
			goto out;
		r = 0;
		break;
	}
	case KVM_DEBUG_GUEST: {
		struct kvm_debug_guest dbg;

		r = -EFAULT;
A
Al Viro 已提交
2848
		if (copy_from_user(&dbg, argp, sizeof dbg))
A
Avi Kivity 已提交
2849
			goto out;
A
Avi Kivity 已提交
2850
		r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
A
Avi Kivity 已提交
2851 2852 2853 2854 2855
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878
	case KVM_SET_SIGNAL_MASK: {
		struct kvm_signal_mask __user *sigmask_arg = argp;
		struct kvm_signal_mask kvm_sigmask;
		sigset_t sigset, *p;

		p = NULL;
		if (argp) {
			r = -EFAULT;
			if (copy_from_user(&kvm_sigmask, argp,
					   sizeof kvm_sigmask))
				goto out;
			r = -EINVAL;
			if (kvm_sigmask.len != sizeof sigset)
				goto out;
			r = -EFAULT;
			if (copy_from_user(&sigset, sigmask_arg->sigset,
					   sizeof sigset))
				goto out;
			p = &sigset;
		}
		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
		break;
	}
A
Avi Kivity 已提交
2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903
	case KVM_GET_FPU: {
		struct kvm_fpu fpu;

		memset(&fpu, 0, sizeof fpu);
		r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &fpu, sizeof fpu))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_FPU: {
		struct kvm_fpu fpu;

		r = -EFAULT;
		if (copy_from_user(&fpu, argp, sizeof fpu))
			goto out;
		r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu);
		if (r)
			goto out;
		r = 0;
		break;
	}
A
Avi Kivity 已提交
2904
	default:
2905
		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923
	}
out:
	return r;
}

static long kvm_vm_ioctl(struct file *filp,
			   unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r = -EINVAL;

	switch (ioctl) {
	case KVM_CREATE_VCPU:
		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
		if (r < 0)
			goto out;
		break;
A
Avi Kivity 已提交
2924 2925
	case KVM_SET_MEMORY_REGION: {
		struct kvm_memory_region kvm_mem;
2926
		struct kvm_userspace_memory_region kvm_userspace_mem;
A
Avi Kivity 已提交
2927 2928

		r = -EFAULT;
A
Al Viro 已提交
2929
		if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
A
Avi Kivity 已提交
2930
			goto out;
2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948
		kvm_userspace_mem.slot = kvm_mem.slot;
		kvm_userspace_mem.flags = kvm_mem.flags;
		kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr;
		kvm_userspace_mem.memory_size = kvm_mem.memory_size;
		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0);
		if (r)
			goto out;
		break;
	}
	case KVM_SET_USER_MEMORY_REGION: {
		struct kvm_userspace_memory_region kvm_userspace_mem;

		r = -EFAULT;
		if (copy_from_user(&kvm_userspace_mem, argp,
						sizeof kvm_userspace_mem))
			goto out;

		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
A
Avi Kivity 已提交
2949 2950 2951 2952
		if (r)
			goto out;
		break;
	}
2953 2954 2955 2956 2957 2958 2959 2960
	case KVM_SET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
		if (r)
			goto out;
		break;
	case KVM_GET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
		break;
A
Avi Kivity 已提交
2961 2962 2963 2964
	case KVM_GET_DIRTY_LOG: {
		struct kvm_dirty_log log;

		r = -EFAULT;
A
Al Viro 已提交
2965
		if (copy_from_user(&log, argp, sizeof log))
A
Avi Kivity 已提交
2966
			goto out;
2967
		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
A
Avi Kivity 已提交
2968 2969 2970 2971
		if (r)
			goto out;
		break;
	}
2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982
	case KVM_SET_MEMORY_ALIAS: {
		struct kvm_memory_alias alias;

		r = -EFAULT;
		if (copy_from_user(&alias, argp, sizeof alias))
			goto out;
		r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
		if (r)
			goto out;
		break;
	}
2983 2984 2985
	case KVM_CREATE_IRQCHIP:
		r = -ENOMEM;
		kvm->vpic = kvm_create_pic(kvm);
E
Eddie Dong 已提交
2986 2987 2988 2989 2990 2991 2992
		if (kvm->vpic) {
			r = kvm_ioapic_init(kvm);
			if (r) {
				kfree(kvm->vpic);
				kvm->vpic = NULL;
				goto out;
			}
M
Mike Day 已提交
2993
		} else
2994 2995 2996 2997 2998 2999 3000 3001 3002
			goto out;
		break;
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
		if (copy_from_user(&irq_event, argp, sizeof irq_event))
			goto out;
		if (irqchip_in_kernel(kvm)) {
3003
			mutex_lock(&kvm->lock);
3004 3005 3006 3007
			if (irq_event.irq < 16)
				kvm_pic_set_irq(pic_irqchip(kvm),
					irq_event.irq,
					irq_event.level);
E
Eddie Dong 已提交
3008 3009 3010
			kvm_ioapic_set_irq(kvm->vioapic,
					irq_event.irq,
					irq_event.level);
3011
			mutex_unlock(&kvm->lock);
3012 3013 3014 3015
			r = 0;
		}
		break;
	}
3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
		struct kvm_irqchip chip;

		r = -EFAULT;
		if (copy_from_user(&chip, argp, sizeof chip))
			goto out;
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
			goto out;
		r = kvm_vm_ioctl_get_irqchip(kvm, &chip);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &chip, sizeof chip))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
		struct kvm_irqchip chip;

		r = -EFAULT;
		if (copy_from_user(&chip, argp, sizeof chip))
			goto out;
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
			goto out;
		r = kvm_vm_ioctl_set_irqchip(kvm, &chip);
		if (r)
			goto out;
		r = 0;
		break;
	}
3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066
	default:
		;
	}
out:
	return r;
}

static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
				  unsigned long address,
				  int *type)
{
	struct kvm *kvm = vma->vm_file->private_data;
	unsigned long pgoff;
	struct page *page;

	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
A
Avi Kivity 已提交
3067
	page = gfn_to_page(kvm, pgoff);
3068 3069
	if (is_error_page(page)) {
		kvm_release_page(page);
3070
		return NOPAGE_SIGBUS;
3071
	}
3072 3073 3074
	if (type != NULL)
		*type = VM_FAULT_MINOR;

3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102
	return page;
}

static struct vm_operations_struct kvm_vm_vm_ops = {
	.nopage = kvm_vm_nopage,
};

static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_vm_vm_ops;
	return 0;
}

static struct file_operations kvm_vm_fops = {
	.release        = kvm_vm_release,
	.unlocked_ioctl = kvm_vm_ioctl,
	.compat_ioctl   = kvm_vm_ioctl,
	.mmap           = kvm_vm_mmap,
};

static int kvm_dev_ioctl_create_vm(void)
{
	int fd, r;
	struct inode *inode;
	struct file *file;
	struct kvm *kvm;

	kvm = kvm_create_vm();
3103 3104 3105 3106 3107 3108
	if (IS_ERR(kvm))
		return PTR_ERR(kvm);
	r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
	if (r) {
		kvm_destroy_vm(kvm);
		return r;
3109 3110
	}

A
Avi Kivity 已提交
3111
	kvm->filp = file;
3112 3113 3114 3115 3116 3117 3118 3119

	return fd;
}

static long kvm_dev_ioctl(struct file *filp,
			  unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
3120
	long r = -EINVAL;
3121 3122 3123

	switch (ioctl) {
	case KVM_GET_API_VERSION:
3124 3125 3126
		r = -EINVAL;
		if (arg)
			goto out;
3127 3128 3129
		r = KVM_API_VERSION;
		break;
	case KVM_CREATE_VM:
3130 3131 3132
		r = -EINVAL;
		if (arg)
			goto out;
3133 3134
		r = kvm_dev_ioctl_create_vm();
		break;
3135 3136 3137 3138 3139
	case KVM_CHECK_EXTENSION: {
		int ext = (long)argp;

		switch (ext) {
		case KVM_CAP_IRQCHIP:
E
Eddie Dong 已提交
3140
		case KVM_CAP_HLT:
3141
		case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
3142
		case KVM_CAP_USER_MEMORY:
3143 3144 3145 3146 3147 3148
			r = 1;
			break;
		default:
			r = 0;
			break;
		}
3149
		break;
3150
	}
3151 3152 3153 3154
	case KVM_GET_VCPU_MMAP_SIZE:
		r = -EINVAL;
		if (arg)
			goto out;
3155
		r = 2 * PAGE_SIZE;
3156
		break;
A
Avi Kivity 已提交
3157
	default:
3158
		return kvm_arch_dev_ioctl(filp, ioctl, arg);
A
Avi Kivity 已提交
3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169
	}
out:
	return r;
}

static struct file_operations kvm_chardev_ops = {
	.unlocked_ioctl = kvm_dev_ioctl,
	.compat_ioctl   = kvm_dev_ioctl,
};

static struct miscdevice kvm_dev = {
A
Avi Kivity 已提交
3170
	KVM_MINOR,
A
Avi Kivity 已提交
3171 3172 3173 3174
	"kvm",
	&kvm_chardev_ops,
};

A
Avi Kivity 已提交
3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185
/*
 * Make sure that a cpu that is being hot-unplugged does not have any vcpus
 * cached on it.
 */
static void decache_vcpus_on_cpu(int cpu)
{
	struct kvm *vm;
	struct kvm_vcpu *vcpu;
	int i;

	spin_lock(&kvm_lock);
S
Shaohua Li 已提交
3186
	list_for_each_entry(vm, &vm_list, vm_list)
A
Avi Kivity 已提交
3187
		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
R
Rusty Russell 已提交
3188 3189 3190
			vcpu = vm->vcpus[i];
			if (!vcpu)
				continue;
A
Avi Kivity 已提交
3191 3192 3193 3194 3195 3196 3197 3198 3199 3200
			/*
			 * If the vcpu is locked, then it is running on some
			 * other cpu and therefore it is not cached on the
			 * cpu in question.
			 *
			 * If it's not locked, check the last cpu it executed
			 * on.
			 */
			if (mutex_trylock(&vcpu->mutex)) {
				if (vcpu->cpu == cpu) {
3201
					kvm_x86_ops->vcpu_decache(vcpu);
A
Avi Kivity 已提交
3202 3203 3204 3205 3206 3207 3208 3209
					vcpu->cpu = -1;
				}
				mutex_unlock(&vcpu->mutex);
			}
		}
	spin_unlock(&kvm_lock);
}

3210 3211 3212 3213 3214 3215 3216
static void hardware_enable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_set(cpu, cpus_hardware_enabled);
3217
	kvm_x86_ops->hardware_enable(NULL);
3218 3219 3220 3221 3222 3223 3224 3225 3226 3227
}

static void hardware_disable(void *junk)
{
	int cpu = raw_smp_processor_id();

	if (!cpu_isset(cpu, cpus_hardware_enabled))
		return;
	cpu_clear(cpu, cpus_hardware_enabled);
	decache_vcpus_on_cpu(cpu);
3228
	kvm_x86_ops->hardware_disable(NULL);
3229 3230
}

A
Avi Kivity 已提交
3231 3232 3233 3234 3235 3236
static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
			   void *v)
{
	int cpu = (long)v;

	switch (val) {
3237 3238
	case CPU_DYING:
	case CPU_DYING_FROZEN:
3239 3240 3241 3242
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
		hardware_disable(NULL);
		break;
A
Avi Kivity 已提交
3243
	case CPU_UP_CANCELED:
3244
	case CPU_UP_CANCELED_FROZEN:
3245 3246
		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
		       cpu);
3247
		smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
A
Avi Kivity 已提交
3248
		break;
3249
	case CPU_ONLINE:
3250
	case CPU_ONLINE_FROZEN:
3251 3252
		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
		       cpu);
3253
		smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
A
Avi Kivity 已提交
3254 3255 3256 3257 3258
		break;
	}
	return NOTIFY_OK;
}

3259
static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
M
Mike Day 已提交
3260
		      void *v)
3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277
{
	if (val == SYS_RESTART) {
		/*
		 * Some (well, at least mine) BIOSes hang on reboot if
		 * in vmx root mode.
		 */
		printk(KERN_INFO "kvm: exiting hardware virtualization\n");
		on_each_cpu(hardware_disable, NULL, 0, 1);
	}
	return NOTIFY_OK;
}

static struct notifier_block kvm_reboot_notifier = {
	.notifier_call = kvm_reboot,
	.priority = 0,
};

3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314
void kvm_io_bus_init(struct kvm_io_bus *bus)
{
	memset(bus, 0, sizeof(*bus));
}

void kvm_io_bus_destroy(struct kvm_io_bus *bus)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		kvm_iodevice_destructor(pos);
	}
}

struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
{
	int i;

	for (i = 0; i < bus->dev_count; i++) {
		struct kvm_io_device *pos = bus->devs[i];

		if (pos->in_range(pos, addr))
			return pos;
	}

	return NULL;
}

void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
{
	BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));

	bus->devs[bus->dev_count++] = dev;
}

A
Avi Kivity 已提交
3315 3316 3317 3318 3319
static struct notifier_block kvm_cpu_notifier = {
	.notifier_call = kvm_cpu_hotplug,
	.priority = 20, /* must be > scheduler priority */
};

A
Avi Kivity 已提交
3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330
static u64 stat_get(void *_offset)
{
	unsigned offset = (long)_offset;
	u64 total = 0;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;

	spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
R
Rusty Russell 已提交
3331 3332 3333
			vcpu = kvm->vcpus[i];
			if (vcpu)
				total += *(u32 *)((void *)vcpu + offset);
A
Avi Kivity 已提交
3334 3335 3336 3337 3338
		}
	spin_unlock(&kvm_lock);
	return total;
}

R
Rusty Russell 已提交
3339
DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n");
A
Avi Kivity 已提交
3340

A
Avi Kivity 已提交
3341 3342 3343 3344
static __init void kvm_init_debug(void)
{
	struct kvm_stats_debugfs_item *p;

A
Al Viro 已提交
3345
	debugfs_dir = debugfs_create_dir("kvm", NULL);
A
Avi Kivity 已提交
3346
	for (p = debugfs_entries; p->name; ++p)
A
Avi Kivity 已提交
3347 3348 3349
		p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
						(void *)(long)p->offset,
						&stat_fops);
A
Avi Kivity 已提交
3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360
}

static void kvm_exit_debug(void)
{
	struct kvm_stats_debugfs_item *p;

	for (p = debugfs_entries; p->name; ++p)
		debugfs_remove(p->dentry);
	debugfs_remove(debugfs_dir);
}

3361 3362
static int kvm_suspend(struct sys_device *dev, pm_message_t state)
{
A
Avi Kivity 已提交
3363
	hardware_disable(NULL);
3364 3365 3366 3367 3368
	return 0;
}

static int kvm_resume(struct sys_device *dev)
{
A
Avi Kivity 已提交
3369
	hardware_enable(NULL);
3370 3371 3372 3373
	return 0;
}

static struct sysdev_class kvm_sysdev_class = {
3374
	.name = "kvm",
3375 3376 3377 3378 3379 3380 3381 3382 3383
	.suspend = kvm_suspend,
	.resume = kvm_resume,
};

static struct sys_device kvm_sysdev = {
	.id = 0,
	.cls = &kvm_sysdev_class,
};

3384
struct page *bad_page;
A
Avi Kivity 已提交
3385

3386 3387 3388 3389 3390 3391 3392 3393 3394 3395
static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
	return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

3396
	kvm_x86_ops->vcpu_load(vcpu, cpu);
3397 3398 3399 3400 3401 3402 3403
}

static void kvm_sched_out(struct preempt_notifier *pn,
			  struct task_struct *next)
{
	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

3404
	kvm_x86_ops->vcpu_put(vcpu);
3405 3406
}

3407
int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size,
3408
		  struct module *module)
A
Avi Kivity 已提交
3409 3410
{
	int r;
Y
Yang, Sheng 已提交
3411
	int cpu;
A
Avi Kivity 已提交
3412

3413
	if (kvm_x86_ops) {
3414 3415 3416 3417
		printk(KERN_ERR "kvm: already loaded the other module\n");
		return -EEXIST;
	}

3418
	if (!ops->cpu_has_kvm_support()) {
A
Avi Kivity 已提交
3419 3420 3421
		printk(KERN_ERR "kvm: no hardware support\n");
		return -EOPNOTSUPP;
	}
3422
	if (ops->disabled_by_bios()) {
A
Avi Kivity 已提交
3423 3424 3425 3426
		printk(KERN_ERR "kvm: disabled by bios\n");
		return -EOPNOTSUPP;
	}

3427
	kvm_x86_ops = ops;
3428

3429
	r = kvm_x86_ops->hardware_setup();
A
Avi Kivity 已提交
3430
	if (r < 0)
3431
		goto out;
A
Avi Kivity 已提交
3432

Y
Yang, Sheng 已提交
3433 3434
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu,
3435
				kvm_x86_ops->check_processor_compatibility,
Y
Yang, Sheng 已提交
3436 3437 3438 3439 3440
				&r, 0, 1);
		if (r < 0)
			goto out_free_0;
	}

3441
	on_each_cpu(hardware_enable, NULL, 0, 1);
A
Avi Kivity 已提交
3442 3443 3444
	r = register_cpu_notifier(&kvm_cpu_notifier);
	if (r)
		goto out_free_1;
A
Avi Kivity 已提交
3445 3446
	register_reboot_notifier(&kvm_reboot_notifier);

3447 3448 3449 3450 3451 3452 3453 3454
	r = sysdev_class_register(&kvm_sysdev_class);
	if (r)
		goto out_free_2;

	r = sysdev_register(&kvm_sysdev);
	if (r)
		goto out_free_3;

3455 3456 3457 3458 3459 3460 3461 3462
	/* A kmem cache lets us meet the alignment requirements of fx_save. */
	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
					   __alignof__(struct kvm_vcpu), 0, 0);
	if (!kvm_vcpu_cache) {
		r = -ENOMEM;
		goto out_free_4;
	}

A
Avi Kivity 已提交
3463 3464 3465 3466
	kvm_chardev_ops.owner = module;

	r = misc_register(&kvm_dev);
	if (r) {
M
Mike Day 已提交
3467
		printk(KERN_ERR "kvm: misc device register failed\n");
A
Avi Kivity 已提交
3468 3469 3470
		goto out_free;
	}

3471 3472 3473
	kvm_preempt_ops.sched_in = kvm_sched_in;
	kvm_preempt_ops.sched_out = kvm_sched_out;

3474 3475 3476
	kvm_mmu_set_nonpresent_ptes(0ull, 0ull);

	return 0;
A
Avi Kivity 已提交
3477 3478

out_free:
3479 3480
	kmem_cache_destroy(kvm_vcpu_cache);
out_free_4:
3481 3482 3483 3484
	sysdev_unregister(&kvm_sysdev);
out_free_3:
	sysdev_class_unregister(&kvm_sysdev_class);
out_free_2:
A
Avi Kivity 已提交
3485
	unregister_reboot_notifier(&kvm_reboot_notifier);
A
Avi Kivity 已提交
3486 3487
	unregister_cpu_notifier(&kvm_cpu_notifier);
out_free_1:
3488
	on_each_cpu(hardware_disable, NULL, 0, 1);
Y
Yang, Sheng 已提交
3489
out_free_0:
3490
	kvm_x86_ops->hardware_unsetup();
3491
out:
3492
	kvm_x86_ops = NULL;
A
Avi Kivity 已提交
3493 3494
	return r;
}
M
Mike Day 已提交
3495
EXPORT_SYMBOL_GPL(kvm_init_x86);
A
Avi Kivity 已提交
3496

3497
void kvm_exit_x86(void)
A
Avi Kivity 已提交
3498 3499
{
	misc_deregister(&kvm_dev);
3500
	kmem_cache_destroy(kvm_vcpu_cache);
3501 3502
	sysdev_unregister(&kvm_sysdev);
	sysdev_class_unregister(&kvm_sysdev_class);
A
Avi Kivity 已提交
3503
	unregister_reboot_notifier(&kvm_reboot_notifier);
3504
	unregister_cpu_notifier(&kvm_cpu_notifier);
3505
	on_each_cpu(hardware_disable, NULL, 0, 1);
3506 3507
	kvm_x86_ops->hardware_unsetup();
	kvm_x86_ops = NULL;
A
Avi Kivity 已提交
3508
}
M
Mike Day 已提交
3509
EXPORT_SYMBOL_GPL(kvm_exit_x86);
A
Avi Kivity 已提交
3510 3511 3512

static __init int kvm_init(void)
{
3513 3514
	int r;

3515 3516 3517 3518
	r = kvm_mmu_module_init();
	if (r)
		goto out4;

A
Avi Kivity 已提交
3519 3520
	kvm_init_debug();

3521
	kvm_arch_init();
3522

3523
	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
M
Mike Day 已提交
3524 3525

	if (bad_page == NULL) {
A
Avi Kivity 已提交
3526 3527 3528 3529
		r = -ENOMEM;
		goto out;
	}

3530
	return 0;
A
Avi Kivity 已提交
3531 3532 3533

out:
	kvm_exit_debug();
3534 3535
	kvm_mmu_module_exit();
out4:
A
Avi Kivity 已提交
3536 3537 3538 3539 3540 3541
	return r;
}

static __exit void kvm_exit(void)
{
	kvm_exit_debug();
3542
	__free_page(bad_page);
3543
	kvm_mmu_module_exit();
A
Avi Kivity 已提交
3544 3545 3546 3547
}

module_init(kvm_init)
module_exit(kvm_exit)