x86.c 157.2 KB
Newer Older
1 2 3 4 5 6
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
B
Ben-Ami Yassour 已提交
7 8
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
N
Nicolas Kaiser 已提交
9
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
10 11 12 13
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
B
Ben-Ami Yassour 已提交
14 15
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
16 17 18 19 20 21
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

22
#include <linux/kvm_host.h>
23
#include "irq.h"
24
#include "mmu.h"
S
Sheng Yang 已提交
25
#include "i8254.h"
26
#include "tss.h"
27
#include "kvm_cache_regs.h"
28
#include "x86.h"
A
Avi Kivity 已提交
29
#include "cpuid.h"
30

31
#include <linux/clocksource.h>
B
Ben-Ami Yassour 已提交
32
#include <linux/interrupt.h>
33 34 35
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
36
#include <linux/module.h>
37
#include <linux/mman.h>
38
#include <linux/highmem.h>
J
Joerg Roedel 已提交
39
#include <linux/iommu.h>
B
Ben-Ami Yassour 已提交
40
#include <linux/intel-iommu.h>
41
#include <linux/cpufreq.h>
A
Avi Kivity 已提交
42
#include <linux/user-return-notifier.h>
43
#include <linux/srcu.h>
44
#include <linux/slab.h>
45
#include <linux/perf_event.h>
46
#include <linux/uaccess.h>
47
#include <linux/hash.h>
48
#include <linux/pci.h>
A
Avi Kivity 已提交
49
#include <trace/events/kvm.h>
X
Xiao Guangrong 已提交
50

51 52
#define CREATE_TRACE_POINTS
#include "trace.h"
53

54
#include <asm/debugreg.h>
55
#include <asm/msr.h>
56
#include <asm/desc.h>
S
Sheng Yang 已提交
57
#include <asm/mtrr.h>
H
Huang Ying 已提交
58
#include <asm/mce.h>
59
#include <asm/i387.h>
S
Sheng Yang 已提交
60
#include <asm/xcr.h>
61
#include <asm/pvclock.h>
62
#include <asm/div64.h>
63

64
#define MAX_IO_MSRS 256
H
Huang Ying 已提交
65
#define KVM_MAX_MCE_BANKS 32
66
#define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P)
H
Huang Ying 已提交
67

68 69 70
#define emul_to_vcpu(ctxt) \
	container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)

71 72 73 74 75
/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
76 77
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
78
#else
79
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
80
#endif
81

82 83
#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
84

85
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
A
Avi Kivity 已提交
86
static void process_nmi(struct kvm_vcpu *vcpu);
87

88
struct kvm_x86_ops *kvm_x86_ops;
89
EXPORT_SYMBOL_GPL(kvm_x86_ops);
90

91 92
static bool ignore_msrs = 0;
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
93

94 95 96 97 98
bool kvm_has_tsc_control;
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
u32  kvm_max_guest_tsc_khz;
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);

A
Avi Kivity 已提交
99 100 101 102
#define KVM_NR_SHARED_MSRS 16

struct kvm_shared_msrs_global {
	int nr;
103
	u32 msrs[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
104 105 106 107 108
};

struct kvm_shared_msrs {
	struct user_return_notifier urn;
	bool registered;
109 110 111 112
	struct kvm_shared_msr_values {
		u64 host;
		u64 curr;
	} values[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
113 114 115 116 117
};

static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
static DEFINE_PER_CPU(struct kvm_shared_msrs, shared_msrs);

118
struct kvm_stats_debugfs_item debugfs_entries[] = {
119 120 121 122 123 124 125 126 127
	{ "pf_fixed", VCPU_STAT(pf_fixed) },
	{ "pf_guest", VCPU_STAT(pf_guest) },
	{ "tlb_flush", VCPU_STAT(tlb_flush) },
	{ "invlpg", VCPU_STAT(invlpg) },
	{ "exits", VCPU_STAT(exits) },
	{ "io_exits", VCPU_STAT(io_exits) },
	{ "mmio_exits", VCPU_STAT(mmio_exits) },
	{ "signal_exits", VCPU_STAT(signal_exits) },
	{ "irq_window", VCPU_STAT(irq_window_exits) },
128
	{ "nmi_window", VCPU_STAT(nmi_window_exits) },
129 130
	{ "halt_exits", VCPU_STAT(halt_exits) },
	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
A
Amit Shah 已提交
131
	{ "hypercalls", VCPU_STAT(hypercalls) },
132 133 134 135 136 137 138
	{ "request_irq", VCPU_STAT(request_irq_exits) },
	{ "irq_exits", VCPU_STAT(irq_exits) },
	{ "host_state_reload", VCPU_STAT(host_state_reload) },
	{ "efer_reload", VCPU_STAT(efer_reload) },
	{ "fpu_reload", VCPU_STAT(fpu_reload) },
	{ "insn_emulation", VCPU_STAT(insn_emulation) },
	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
139
	{ "irq_injections", VCPU_STAT(irq_injections) },
140
	{ "nmi_injections", VCPU_STAT(nmi_injections) },
A
Avi Kivity 已提交
141 142 143 144 145 146
	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
	{ "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
	{ "mmu_flooded", VM_STAT(mmu_flooded) },
	{ "mmu_recycled", VM_STAT(mmu_recycled) },
A
Avi Kivity 已提交
147
	{ "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
148
	{ "mmu_unsync", VM_STAT(mmu_unsync) },
149
	{ "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
M
Marcelo Tosatti 已提交
150
	{ "largepages", VM_STAT(lpages) },
151 152 153
	{ NULL }
};

154 155
u64 __read_mostly host_xcr0;

156 157
int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);

158 159 160 161 162 163 164
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
	int i;
	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
		vcpu->arch.apf.gfns[i] = ~0;
}

A
Avi Kivity 已提交
165 166 167 168 169
static void kvm_on_user_return(struct user_return_notifier *urn)
{
	unsigned slot;
	struct kvm_shared_msrs *locals
		= container_of(urn, struct kvm_shared_msrs, urn);
170
	struct kvm_shared_msr_values *values;
A
Avi Kivity 已提交
171 172

	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
173 174 175 176
		values = &locals->values[slot];
		if (values->host != values->curr) {
			wrmsrl(shared_msrs_global.msrs[slot], values->host);
			values->curr = values->host;
A
Avi Kivity 已提交
177 178 179 180 181 182
		}
	}
	locals->registered = false;
	user_return_notifier_unregister(urn);
}

183
static void shared_msr_update(unsigned slot, u32 msr)
A
Avi Kivity 已提交
184
{
185
	struct kvm_shared_msrs *smsr;
A
Avi Kivity 已提交
186 187
	u64 value;

188 189 190 191 192 193 194 195 196 197 198 199 200 201
	smsr = &__get_cpu_var(shared_msrs);
	/* only read, and nobody should modify it at this time,
	 * so don't need lock */
	if (slot >= shared_msrs_global.nr) {
		printk(KERN_ERR "kvm: invalid MSR slot!");
		return;
	}
	rdmsrl_safe(msr, &value);
	smsr->values[slot].host = value;
	smsr->values[slot].curr = value;
}

void kvm_define_shared_msr(unsigned slot, u32 msr)
{
A
Avi Kivity 已提交
202 203
	if (slot >= shared_msrs_global.nr)
		shared_msrs_global.nr = slot + 1;
204 205 206
	shared_msrs_global.msrs[slot] = msr;
	/* we need ensured the shared_msr_global have been updated */
	smp_wmb();
A
Avi Kivity 已提交
207 208 209 210 211 212 213 214
}
EXPORT_SYMBOL_GPL(kvm_define_shared_msr);

static void kvm_shared_msr_cpu_online(void)
{
	unsigned i;

	for (i = 0; i < shared_msrs_global.nr; ++i)
215
		shared_msr_update(i, shared_msrs_global.msrs[i]);
A
Avi Kivity 已提交
216 217
}

218
void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
A
Avi Kivity 已提交
219 220 221
{
	struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);

222
	if (((value ^ smsr->values[slot].curr) & mask) == 0)
A
Avi Kivity 已提交
223
		return;
224 225
	smsr->values[slot].curr = value;
	wrmsrl(shared_msrs_global.msrs[slot], value);
A
Avi Kivity 已提交
226 227 228 229 230 231 232 233
	if (!smsr->registered) {
		smsr->urn.on_user_return = kvm_on_user_return;
		user_return_notifier_register(&smsr->urn);
		smsr->registered = true;
	}
}
EXPORT_SYMBOL_GPL(kvm_set_shared_msr);

234 235 236 237 238 239 240 241
static void drop_user_return_notifiers(void *ignore)
{
	struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);

	if (smsr->registered)
		kvm_on_user_return(&smsr->urn);
}

242 243 244
u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
	if (irqchip_in_kernel(vcpu->kvm))
245
		return vcpu->arch.apic_base;
246
	else
247
		return vcpu->arch.apic_base;
248 249 250 251 252 253 254 255 256
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
{
	/* TODO: reserve bits check */
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_lapic_set_base(vcpu, data);
	else
257
		vcpu->arch.apic_base = data;
258 259 260
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
#define EXCPT_BENIGN		0
#define EXCPT_CONTRIBUTORY	1
#define EXCPT_PF		2

static int exception_class(int vector)
{
	switch (vector) {
	case PF_VECTOR:
		return EXCPT_PF;
	case DE_VECTOR:
	case TS_VECTOR:
	case NP_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
		return EXCPT_CONTRIBUTORY;
	default:
		break;
	}
	return EXCPT_BENIGN;
}

static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
283 284
		unsigned nr, bool has_error, u32 error_code,
		bool reinject)
285 286 287 288
{
	u32 prev_nr;
	int class1, class2;

289 290
	kvm_make_request(KVM_REQ_EVENT, vcpu);

291 292 293 294 295 296
	if (!vcpu->arch.exception.pending) {
	queue:
		vcpu->arch.exception.pending = true;
		vcpu->arch.exception.has_error_code = has_error;
		vcpu->arch.exception.nr = nr;
		vcpu->arch.exception.error_code = error_code;
297
		vcpu->arch.exception.reinject = reinject;
298 299 300 301 302 303 304
		return;
	}

	/* to check exception */
	prev_nr = vcpu->arch.exception.nr;
	if (prev_nr == DF_VECTOR) {
		/* triple fault -> shutdown */
305
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
		return;
	}
	class1 = exception_class(prev_nr);
	class2 = exception_class(nr);
	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
		/* generate double fault per SDM Table 5-5 */
		vcpu->arch.exception.pending = true;
		vcpu->arch.exception.has_error_code = true;
		vcpu->arch.exception.nr = DF_VECTOR;
		vcpu->arch.exception.error_code = 0;
	} else
		/* replace previous exception with a new one in a hope
		   that instruction re-execution will regenerate lost
		   exception */
		goto queue;
}

324 325
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
326
	kvm_multiple_exception(vcpu, nr, false, 0, false);
327 328 329
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

330 331 332 333 334 335
void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
	kvm_multiple_exception(vcpu, nr, false, 0, true);
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

336
void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
337
{
338 339 340 341 342 343
	if (err)
		kvm_inject_gp(vcpu, 0);
	else
		kvm_x86_ops->skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
344

345
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
346 347
{
	++vcpu->stat.pf_guest;
348 349
	vcpu->arch.cr2 = fault->address;
	kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
350
}
N
Nadav Har'El 已提交
351
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
352

353
void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
354
{
355 356
	if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
		vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
357
	else
358
		vcpu->arch.mmu.inject_page_fault(vcpu, fault);
359 360
}

361 362
void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
363 364
	atomic_inc(&vcpu->arch.nmi_queued);
	kvm_make_request(KVM_REQ_NMI, vcpu);
365 366 367
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

368 369
void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
370
	kvm_multiple_exception(vcpu, nr, true, error_code, false);
371 372 373
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

374 375 376 377 378 379
void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
	kvm_multiple_exception(vcpu, nr, true, error_code, true);
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

380 381 382 383 384
/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
385
{
386 387 388 389
	if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
		return true;
	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
	return false;
390
}
391
EXPORT_SYMBOL_GPL(kvm_require_cpl);
392

393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
/*
 * This function will be used to read from the physical memory of the currently
 * running guest. The difference to kvm_read_guest_page is that this function
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gfn_t ngfn, void *data, int offset, int len,
			    u32 access)
{
	gfn_t real_gfn;
	gpa_t ngpa;

	ngpa     = gfn_to_gpa(ngfn);
	real_gfn = mmu->translate_gpa(vcpu, ngpa, access);
	if (real_gfn == UNMAPPED_GVA)
		return -EFAULT;

	real_gfn = gpa_to_gfn(real_gfn);

	return kvm_read_guest_page(vcpu->kvm, real_gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

416 417 418 419 420 421 422
int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
			       void *data, int offset, int len, u32 access)
{
	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
				       data, offset, len, access);
}

423 424 425
/*
 * Load the pae pdptrs.  Return true is they are all valid.
 */
426
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
427 428 429 430 431
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
	int i;
	int ret;
432
	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
433

434 435 436
	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
				      offset * sizeof(u64), sizeof(pdpte),
				      PFERR_USER_MASK|PFERR_WRITE_MASK);
437 438 439 440 441
	if (ret < 0) {
		ret = 0;
		goto out;
	}
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
442
		if (is_present_gpte(pdpte[i]) &&
443
		    (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
444 445 446 447 448 449
			ret = 0;
			goto out;
		}
	}
	ret = 1;

450
	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
A
Avi Kivity 已提交
451 452 453 454
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
455 456 457 458
out:

	return ret;
}
459
EXPORT_SYMBOL_GPL(load_pdptrs);
460

461 462
static bool pdptrs_changed(struct kvm_vcpu *vcpu)
{
463
	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
464
	bool changed = true;
465 466
	int offset;
	gfn_t gfn;
467 468 469 470 471
	int r;

	if (is_long_mode(vcpu) || !is_pae(vcpu))
		return false;

A
Avi Kivity 已提交
472 473 474 475
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_avail))
		return true;

476 477
	gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT;
	offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1);
478 479
	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
				       PFERR_USER_MASK | PFERR_WRITE_MASK);
480 481
	if (r < 0)
		goto out;
482
	changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
483 484 485 486 487
out:

	return changed;
}

488
int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
489
{
490 491 492 493
	unsigned long old_cr0 = kvm_read_cr0(vcpu);
	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP |
				    X86_CR0_CD | X86_CR0_NW;

494 495
	cr0 |= X86_CR0_ET;

496
#ifdef CONFIG_X86_64
497 498
	if (cr0 & 0xffffffff00000000UL)
		return 1;
499 500 501
#endif

	cr0 &= ~CR0_RESERVED_BITS;
502

503 504
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
		return 1;
505

506 507
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
		return 1;
508 509 510

	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
#ifdef CONFIG_X86_64
511
		if ((vcpu->arch.efer & EFER_LME)) {
512 513
			int cs_db, cs_l;

514 515
			if (!is_pae(vcpu))
				return 1;
516
			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
517 518
			if (cs_l)
				return 1;
519 520
		} else
#endif
521
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
522
						 kvm_read_cr3(vcpu)))
523
			return 1;
524 525 526 527
	}

	kvm_x86_ops->set_cr0(vcpu, cr0);

528
	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
529
		kvm_clear_async_pf_completion_queue(vcpu);
530 531
		kvm_async_pf_hash_reset(vcpu);
	}
532

533 534
	if ((cr0 ^ old_cr0) & update_bits)
		kvm_mmu_reset_context(vcpu);
535 536
	return 0;
}
537
EXPORT_SYMBOL_GPL(kvm_set_cr0);
538

539
void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
540
{
541
	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
542
}
543
EXPORT_SYMBOL_GPL(kvm_lmsw);
544

545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
	u64 xcr0;

	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
	if (index != XCR_XFEATURE_ENABLED_MASK)
		return 1;
	xcr0 = xcr;
	if (kvm_x86_ops->get_cpl(vcpu) != 0)
		return 1;
	if (!(xcr0 & XSTATE_FP))
		return 1;
	if ((xcr0 & XSTATE_YMM) && !(xcr0 & XSTATE_SSE))
		return 1;
	if (xcr0 & ~host_xcr0)
		return 1;
	vcpu->arch.xcr0 = xcr0;
	vcpu->guest_xcr0_loaded = 0;
	return 0;
}

int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
	if (__kvm_set_xcr(vcpu, index, xcr)) {
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_xcr);

576
int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
577
{
578
	unsigned long old_cr4 = kvm_read_cr4(vcpu);
579 580
	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE |
				   X86_CR4_PAE | X86_CR4_SMEP;
581 582
	if (cr4 & CR4_RESERVED_BITS)
		return 1;
583

584 585 586
	if (!guest_cpuid_has_xsave(vcpu) && (cr4 & X86_CR4_OSXSAVE))
		return 1;

587 588 589
	if (!guest_cpuid_has_smep(vcpu) && (cr4 & X86_CR4_SMEP))
		return 1;

590 591 592
	if (!guest_cpuid_has_fsgsbase(vcpu) && (cr4 & X86_CR4_RDWRGSFS))
		return 1;

593
	if (is_long_mode(vcpu)) {
594 595
		if (!(cr4 & X86_CR4_PAE))
			return 1;
596 597
	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
		   && ((cr4 ^ old_cr4) & pdptr_bits)
598 599
		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
				   kvm_read_cr3(vcpu)))
600 601
		return 1;

602
	if (kvm_x86_ops->set_cr4(vcpu, cr4))
603
		return 1;
604

605 606
	if ((cr4 ^ old_cr4) & pdptr_bits)
		kvm_mmu_reset_context(vcpu);
607

608
	if ((cr4 ^ old_cr4) & X86_CR4_OSXSAVE)
A
Avi Kivity 已提交
609
		kvm_update_cpuid(vcpu);
610

611 612
	return 0;
}
613
EXPORT_SYMBOL_GPL(kvm_set_cr4);
614

615
int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
616
{
617
	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
618
		kvm_mmu_sync_roots(vcpu);
619
		kvm_mmu_flush_tlb(vcpu);
620
		return 0;
621 622
	}

623
	if (is_long_mode(vcpu)) {
624 625
		if (cr3 & CR3_L_MODE_RESERVED_BITS)
			return 1;
626 627
	} else {
		if (is_pae(vcpu)) {
628 629
			if (cr3 & CR3_PAE_RESERVED_BITS)
				return 1;
630 631
			if (is_paging(vcpu) &&
			    !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
632
				return 1;
633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649
		}
		/*
		 * We don't check reserved bits in nonpae mode, because
		 * this isn't enforced, and VMware depends on this.
		 */
	}

	/*
	 * Does the new cr3 value map to physical memory? (Note, we
	 * catch an invalid cr3 even in real-mode, because it would
	 * cause trouble later on when we turn on paging anyway.)
	 *
	 * A real CPU would silently accept an invalid cr3 and would
	 * attempt to use it - with largely undefined (and often hard
	 * to debug) behavior on the guest side.
	 */
	if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
650 651
		return 1;
	vcpu->arch.cr3 = cr3;
652
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
653 654 655
	vcpu->arch.mmu.new_cr3(vcpu);
	return 0;
}
656
EXPORT_SYMBOL_GPL(kvm_set_cr3);
657

A
Andre Przywara 已提交
658
int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
659
{
660 661
	if (cr8 & CR8_RESERVED_BITS)
		return 1;
662 663 664
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_lapic_set_tpr(vcpu, cr8);
	else
665
		vcpu->arch.cr8 = cr8;
666 667
	return 0;
}
668
EXPORT_SYMBOL_GPL(kvm_set_cr8);
669

670
unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
671 672 673 674
{
	if (irqchip_in_kernel(vcpu->kvm))
		return kvm_lapic_get_cr8(vcpu);
	else
675
		return vcpu->arch.cr8;
676
}
677
EXPORT_SYMBOL_GPL(kvm_get_cr8);
678

679
static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
680 681 682 683 684 685 686 687
{
	switch (dr) {
	case 0 ... 3:
		vcpu->arch.db[dr] = val;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = val;
		break;
	case 4:
688 689
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
			return 1; /* #UD */
690 691
		/* fall through */
	case 6:
692 693
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
694 695 696
		vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
		break;
	case 5:
697 698
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
			return 1; /* #UD */
699 700
		/* fall through */
	default: /* 7 */
701 702
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
703 704 705 706 707 708 709 710 711 712
		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
			kvm_x86_ops->set_dr7(vcpu, vcpu->arch.dr7);
			vcpu->arch.switch_db_regs = (val & DR7_BP_EN_MASK);
		}
		break;
	}

	return 0;
}
713 714 715 716 717 718 719 720 721 722 723 724 725

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
	int res;

	res = __kvm_set_dr(vcpu, dr, val);
	if (res > 0)
		kvm_queue_exception(vcpu, UD_VECTOR);
	else if (res < 0)
		kvm_inject_gp(vcpu, 0);

	return res;
}
726 727
EXPORT_SYMBOL_GPL(kvm_set_dr);

728
static int _kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
729 730 731 732 733 734
{
	switch (dr) {
	case 0 ... 3:
		*val = vcpu->arch.db[dr];
		break;
	case 4:
735
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
736 737 738 739 740 741
			return 1;
		/* fall through */
	case 6:
		*val = vcpu->arch.dr6;
		break;
	case 5:
742
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
743 744 745 746 747 748 749 750 751
			return 1;
		/* fall through */
	default: /* 7 */
		*val = vcpu->arch.dr7;
		break;
	}

	return 0;
}
752 753 754 755 756 757 758 759 760

int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
{
	if (_kvm_get_dr(vcpu, dr, val)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}
	return 0;
}
761 762
EXPORT_SYMBOL_GPL(kvm_get_dr);

A
Avi Kivity 已提交
763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
bool kvm_rdpmc(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
	u64 data;
	int err;

	err = kvm_pmu_read_pmc(vcpu, ecx, &data);
	if (err)
		return err;
	kvm_register_write(vcpu, VCPU_REGS_RAX, (u32)data);
	kvm_register_write(vcpu, VCPU_REGS_RDX, data >> 32);
	return err;
}
EXPORT_SYMBOL_GPL(kvm_rdpmc);

778 779 780 781 782
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
 * This list is modified at module load time to reflect the
783 784
 * capabilities of the host cpu. This capabilities test skips MSRs that are
 * kvm-specific. Those are put in the beginning of the list.
785
 */
786

G
Glauber Costa 已提交
787
#define KVM_SAVE_MSRS_BEGIN	9
788
static u32 msrs_to_save[] = {
789
	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
790
	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
791
	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
G
Glauber Costa 已提交
792
	HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
793
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
B
Brian Gerst 已提交
794
	MSR_STAR,
795 796 797
#ifdef CONFIG_X86_64
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
798
	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA
799 800 801 802 803
};

static unsigned num_msrs_to_save;

static u32 emulated_msrs[] = {
804
	MSR_IA32_TSCDEADLINE,
805
	MSR_IA32_MISC_ENABLE,
806 807
	MSR_IA32_MCG_STATUS,
	MSR_IA32_MCG_CTL,
808 809
};

810
static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
811
{
812 813
	u64 old_efer = vcpu->arch.efer;

814 815
	if (efer & efer_reserved_bits)
		return 1;
816 817

	if (is_paging(vcpu)
818 819
	    && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
		return 1;
820

A
Alexander Graf 已提交
821 822 823 824
	if (efer & EFER_FFXSR) {
		struct kvm_cpuid_entry2 *feat;

		feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
825 826
		if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT)))
			return 1;
A
Alexander Graf 已提交
827 828
	}

829 830 831 832
	if (efer & EFER_SVME) {
		struct kvm_cpuid_entry2 *feat;

		feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
833 834
		if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM)))
			return 1;
835 836
	}

837
	efer &= ~EFER_LMA;
838
	efer |= vcpu->arch.efer & EFER_LMA;
839

840 841
	kvm_x86_ops->set_efer(vcpu, efer);

842
	vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
843

844 845 846 847
	/* Update reserved bits */
	if ((efer ^ old_efer) & EFER_NX)
		kvm_mmu_reset_context(vcpu);

848
	return 0;
849 850
}

851 852 853 854 855 856 857
void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);


858 859 860 861 862 863 864 865 866 867
/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
{
	return kvm_x86_ops->set_msr(vcpu, msr_index, data);
}

868 869 870 871 872 873 874 875
/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return kvm_set_msr(vcpu, index, *data);
}

876 877
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
{
878 879
	int version;
	int r;
880
	struct pvclock_wall_clock wc;
881
	struct timespec boot;
882 883 884 885

	if (!wall_clock)
		return;

886 887 888 889 890 891 892 893
	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
	if (r)
		return;

	if (version & 1)
		++version;  /* first time write, random junk */

	++version;
894 895 896

	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));

897 898
	/*
	 * The guest calculates current wall clock time by adding
Z
Zachary Amsden 已提交
899
	 * system time (updated by kvm_guest_time_update below) to the
900 901 902
	 * wall clock specified here.  guest system time equals host
	 * system time for us, thus we must fill in host boot time here.
	 */
903
	getboottime(&boot);
904 905 906 907

	wc.sec = boot.tv_sec;
	wc.nsec = boot.tv_nsec;
	wc.version = version;
908 909 910 911 912 913 914

	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));

	version++;
	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
}

915 916 917 918 919 920 921 922 923 924 925 926
static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
	uint32_t quotient, remainder;

	/* Don't try to replace with do_div(), this one calculates
	 * "(dividend << 32) / divisor" */
	__asm__ ( "divl %4"
		  : "=a" (quotient), "=d" (remainder)
		  : "0" (0), "1" (dividend), "r" (divisor) );
	return quotient;
}

927 928
static void kvm_get_time_scale(uint32_t scaled_khz, uint32_t base_khz,
			       s8 *pshift, u32 *pmultiplier)
929
{
930
	uint64_t scaled64;
931 932 933 934
	int32_t  shift = 0;
	uint64_t tps64;
	uint32_t tps32;

935 936
	tps64 = base_khz * 1000LL;
	scaled64 = scaled_khz * 1000LL;
937
	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
938 939 940 941 942
		tps64 >>= 1;
		shift--;
	}

	tps32 = (uint32_t)tps64;
943 944
	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
945 946 947
			scaled64 >>= 1;
		else
			tps32 <<= 1;
948 949 950
		shift++;
	}

951 952
	*pshift = shift;
	*pmultiplier = div_frac(scaled64, tps32);
953

954 955
	pr_debug("%s: base_khz %u => %u, shift %d, mul %u\n",
		 __func__, base_khz, scaled_khz, shift, *pmultiplier);
956 957
}

958 959 960 961 962 963 964 965
static inline u64 get_kernel_ns(void)
{
	struct timespec ts;

	WARN_ON(preemptible());
	ktime_get_ts(&ts);
	monotonic_to_bootbased(&ts);
	return timespec_to_ns(&ts);
966 967
}

968
static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
Z
Zachary Amsden 已提交
969
unsigned long max_tsc_khz;
970

971 972 973 974 975 976 977 978 979
static inline int kvm_tsc_changes_freq(void)
{
	int cpu = get_cpu();
	int ret = !boot_cpu_has(X86_FEATURE_CONSTANT_TSC) &&
		  cpufreq_quick_get(cpu) != 0;
	put_cpu();
	return ret;
}

980
u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu)
981 982 983 984 985 986 987
{
	if (vcpu->arch.virtual_tsc_khz)
		return vcpu->arch.virtual_tsc_khz;
	else
		return __this_cpu_read(cpu_tsc_khz);
}

988
static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
989
{
990 991
	u64 ret;

992 993 994 995
	WARN_ON(preemptible());
	if (kvm_tsc_changes_freq())
		printk_once(KERN_WARNING
		 "kvm: unreliable cycle conversion on adjustable rate TSC\n");
996
	ret = nsec * vcpu_tsc_khz(vcpu);
997 998
	do_div(ret, USEC_PER_SEC);
	return ret;
999 1000
}

1001
static void kvm_init_tsc_catchup(struct kvm_vcpu *vcpu, u32 this_tsc_khz)
Z
Zachary Amsden 已提交
1002 1003 1004
{
	/* Compute a scale to convert nanoseconds in TSC cycles */
	kvm_get_time_scale(this_tsc_khz, NSEC_PER_SEC / 1000,
1005 1006
			   &vcpu->arch.tsc_catchup_shift,
			   &vcpu->arch.tsc_catchup_mult);
Z
Zachary Amsden 已提交
1007 1008 1009 1010 1011
}

static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.last_tsc_nsec,
1012 1013
				      vcpu->arch.tsc_catchup_mult,
				      vcpu->arch.tsc_catchup_shift);
Z
Zachary Amsden 已提交
1014 1015 1016 1017
	tsc += vcpu->arch.last_tsc_write;
	return tsc;
}

1018 1019 1020
void kvm_write_tsc(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
Z
Zachary Amsden 已提交
1021
	u64 offset, ns, elapsed;
1022
	unsigned long flags;
Z
Zachary Amsden 已提交
1023
	s64 sdiff;
1024

1025
	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
1026
	offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
1027
	ns = get_kernel_ns();
Z
Zachary Amsden 已提交
1028
	elapsed = ns - kvm->arch.last_tsc_nsec;
Z
Zachary Amsden 已提交
1029 1030 1031
	sdiff = data - kvm->arch.last_tsc_write;
	if (sdiff < 0)
		sdiff = -sdiff;
Z
Zachary Amsden 已提交
1032 1033

	/*
Z
Zachary Amsden 已提交
1034
	 * Special case: close write to TSC within 5 seconds of
Z
Zachary Amsden 已提交
1035
	 * another CPU is interpreted as an attempt to synchronize
L
Lucas De Marchi 已提交
1036
	 * The 5 seconds is to accommodate host load / swapping as
Z
Zachary Amsden 已提交
1037
	 * well as any reset of TSC during the boot process.
Z
Zachary Amsden 已提交
1038 1039
	 *
	 * In that case, for a reliable TSC, we can match TSC offsets,
Z
Zachary Amsden 已提交
1040
	 * or make a best guest using elapsed value.
Z
Zachary Amsden 已提交
1041
	 */
1042
	if (sdiff < nsec_to_cycles(vcpu, 5ULL * NSEC_PER_SEC) &&
Z
Zachary Amsden 已提交
1043
	    elapsed < 5ULL * NSEC_PER_SEC) {
Z
Zachary Amsden 已提交
1044 1045 1046 1047
		if (!check_tsc_unstable()) {
			offset = kvm->arch.last_tsc_offset;
			pr_debug("kvm: matched tsc offset for %llu\n", data);
		} else {
1048
			u64 delta = nsec_to_cycles(vcpu, elapsed);
1049 1050
			offset += delta;
			pr_debug("kvm: adjusted tsc offset by %llu\n", delta);
Z
Zachary Amsden 已提交
1051 1052 1053 1054 1055 1056
		}
		ns = kvm->arch.last_tsc_nsec;
	}
	kvm->arch.last_tsc_nsec = ns;
	kvm->arch.last_tsc_write = data;
	kvm->arch.last_tsc_offset = offset;
1057
	kvm_x86_ops->write_tsc_offset(vcpu, offset);
1058
	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
1059 1060 1061

	/* Reset of TSC must disable overshoot protection below */
	vcpu->arch.hv_clock.tsc_timestamp = 0;
Z
Zachary Amsden 已提交
1062 1063
	vcpu->arch.last_tsc_write = data;
	vcpu->arch.last_tsc_nsec = ns;
1064 1065 1066
}
EXPORT_SYMBOL_GPL(kvm_write_tsc);

Z
Zachary Amsden 已提交
1067
static int kvm_guest_time_update(struct kvm_vcpu *v)
1068 1069 1070 1071
{
	unsigned long flags;
	struct kvm_vcpu_arch *vcpu = &v->arch;
	void *shared_kaddr;
1072
	unsigned long this_tsc_khz;
1073 1074
	s64 kernel_ns, max_kernel_ns;
	u64 tsc_timestamp;
1075 1076 1077

	/* Keep irq disabled to prevent changes to the clock */
	local_irq_save(flags);
N
Nadav Har'El 已提交
1078
	tsc_timestamp = kvm_x86_ops->read_l1_tsc(v);
1079
	kernel_ns = get_kernel_ns();
1080
	this_tsc_khz = vcpu_tsc_khz(v);
1081
	if (unlikely(this_tsc_khz == 0)) {
Z
Zachary Amsden 已提交
1082
		local_irq_restore(flags);
Z
Zachary Amsden 已提交
1083
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
1084 1085
		return 1;
	}
1086

Z
Zachary Amsden 已提交
1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
	/*
	 * We may have to catch up the TSC to match elapsed wall clock
	 * time for two reasons, even if kvmclock is used.
	 *   1) CPU could have been running below the maximum TSC rate
	 *   2) Broken TSC compensation resets the base at each VCPU
	 *      entry to avoid unknown leaps of TSC even when running
	 *      again on the same CPU.  This may cause apparent elapsed
	 *      time to disappear, and the guest to stand still or run
	 *	very slowly.
	 */
	if (vcpu->tsc_catchup) {
		u64 tsc = compute_guest_tsc(v, kernel_ns);
		if (tsc > tsc_timestamp) {
			kvm_x86_ops->adjust_tsc_offset(v, tsc - tsc_timestamp);
			tsc_timestamp = tsc;
		}
1103 1104
	}

1105 1106
	local_irq_restore(flags);

Z
Zachary Amsden 已提交
1107 1108
	if (!vcpu->time_page)
		return 0;
1109

1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139
	/*
	 * Time as measured by the TSC may go backwards when resetting the base
	 * tsc_timestamp.  The reason for this is that the TSC resolution is
	 * higher than the resolution of the other clock scales.  Thus, many
	 * possible measurments of the TSC correspond to one measurement of any
	 * other clock, and so a spread of values is possible.  This is not a
	 * problem for the computation of the nanosecond clock; with TSC rates
	 * around 1GHZ, there can only be a few cycles which correspond to one
	 * nanosecond value, and any path through this code will inevitably
	 * take longer than that.  However, with the kernel_ns value itself,
	 * the precision may be much lower, down to HZ granularity.  If the
	 * first sampling of TSC against kernel_ns ends in the low part of the
	 * range, and the second in the high end of the range, we can get:
	 *
	 * (TSC - offset_low) * S + kns_old > (TSC - offset_high) * S + kns_new
	 *
	 * As the sampling errors potentially range in the thousands of cycles,
	 * it is possible such a time value has already been observed by the
	 * guest.  To protect against this, we must compute the system time as
	 * observed by the guest and ensure the new system time is greater.
	 */
	max_kernel_ns = 0;
	if (vcpu->hv_clock.tsc_timestamp && vcpu->last_guest_tsc) {
		max_kernel_ns = vcpu->last_guest_tsc -
				vcpu->hv_clock.tsc_timestamp;
		max_kernel_ns = pvclock_scale_delta(max_kernel_ns,
				    vcpu->hv_clock.tsc_to_system_mul,
				    vcpu->hv_clock.tsc_shift);
		max_kernel_ns += vcpu->last_kernel_ns;
	}
1140

Z
Zachary Amsden 已提交
1141
	if (unlikely(vcpu->hw_tsc_khz != this_tsc_khz)) {
1142 1143 1144
		kvm_get_time_scale(NSEC_PER_SEC / 1000, this_tsc_khz,
				   &vcpu->hv_clock.tsc_shift,
				   &vcpu->hv_clock.tsc_to_system_mul);
Z
Zachary Amsden 已提交
1145
		vcpu->hw_tsc_khz = this_tsc_khz;
1146 1147
	}

1148 1149 1150
	if (max_kernel_ns > kernel_ns)
		kernel_ns = max_kernel_ns;

1151
	/* With all the info we got, fill in the values */
1152
	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
1153
	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
1154
	vcpu->last_kernel_ns = kernel_ns;
Z
Zachary Amsden 已提交
1155
	vcpu->last_guest_tsc = tsc_timestamp;
1156 1157
	vcpu->hv_clock.flags = 0;

1158 1159 1160
	/*
	 * The interface expects us to write an even number signaling that the
	 * update is finished. Since the guest won't see the intermediate
1161
	 * state, we just increase by 2 at the end.
1162
	 */
1163
	vcpu->hv_clock.version += 2;
1164 1165 1166 1167

	shared_kaddr = kmap_atomic(vcpu->time_page, KM_USER0);

	memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock,
1168
	       sizeof(vcpu->hv_clock));
1169 1170 1171 1172

	kunmap_atomic(shared_kaddr, KM_USER0);

	mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT);
1173
	return 0;
1174 1175
}

A
Avi Kivity 已提交
1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199
static bool msr_mtrr_valid(unsigned msr)
{
	switch (msr) {
	case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
	case MSR_MTRRfix64K_00000:
	case MSR_MTRRfix16K_80000:
	case MSR_MTRRfix16K_A0000:
	case MSR_MTRRfix4K_C0000:
	case MSR_MTRRfix4K_C8000:
	case MSR_MTRRfix4K_D0000:
	case MSR_MTRRfix4K_D8000:
	case MSR_MTRRfix4K_E0000:
	case MSR_MTRRfix4K_E8000:
	case MSR_MTRRfix4K_F0000:
	case MSR_MTRRfix4K_F8000:
	case MSR_MTRRdefType:
	case MSR_IA32_CR_PAT:
		return true;
	case 0x2f8:
		return true;
	}
	return false;
}

1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
static bool valid_pat_type(unsigned t)
{
	return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
}

static bool valid_mtrr_type(unsigned t)
{
	return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
}

static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	int i;

	if (!msr_mtrr_valid(msr))
		return false;

	if (msr == MSR_IA32_CR_PAT) {
		for (i = 0; i < 8; i++)
			if (!valid_pat_type((data >> (i * 8)) & 0xff))
				return false;
		return true;
	} else if (msr == MSR_MTRRdefType) {
		if (data & ~0xcff)
			return false;
		return valid_mtrr_type(data & 0xff);
	} else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
		for (i = 0; i < 8 ; i++)
			if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
				return false;
		return true;
	}

	/* variable MTRRs */
	return valid_mtrr_type(data & 0xff);
}

A
Avi Kivity 已提交
1237 1238
static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
S
Sheng Yang 已提交
1239 1240
	u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;

1241
	if (!mtrr_valid(vcpu, msr, data))
A
Avi Kivity 已提交
1242 1243
		return 1;

S
Sheng Yang 已提交
1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
	if (msr == MSR_MTRRdefType) {
		vcpu->arch.mtrr_state.def_type = data;
		vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
	} else if (msr == MSR_MTRRfix64K_00000)
		p[0] = data;
	else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
		p[1 + msr - MSR_MTRRfix16K_80000] = data;
	else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
		p[3 + msr - MSR_MTRRfix4K_C0000] = data;
	else if (msr == MSR_IA32_CR_PAT)
		vcpu->arch.pat = data;
	else {	/* Variable MTRRs */
		int idx, is_mtrr_mask;
		u64 *pt;

		idx = (msr - 0x200) / 2;
		is_mtrr_mask = msr - 0x200 - 2 * idx;
		if (!is_mtrr_mask)
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
		else
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
		*pt = data;
	}

	kvm_mmu_reset_context(vcpu);
A
Avi Kivity 已提交
1271 1272
	return 0;
}
1273

H
Huang Ying 已提交
1274
static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1275
{
H
Huang Ying 已提交
1276 1277 1278
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;

1279 1280
	switch (msr) {
	case MSR_IA32_MCG_STATUS:
H
Huang Ying 已提交
1281
		vcpu->arch.mcg_status = data;
1282
		break;
1283
	case MSR_IA32_MCG_CTL:
H
Huang Ying 已提交
1284 1285 1286 1287 1288 1289 1290 1291 1292 1293
		if (!(mcg_cap & MCG_CTL_P))
			return 1;
		if (data != 0 && data != ~(u64)0)
			return -1;
		vcpu->arch.mcg_ctl = data;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
		    msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
			u32 offset = msr - MSR_IA32_MC0_CTL;
1294 1295 1296 1297 1298
			/* only 0 or all 1s can be written to IA32_MCi_CTL
			 * some Linux kernels though clear bit 10 in bank 4 to
			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
			 * this to avoid an uncatched #GP in the guest
			 */
H
Huang Ying 已提交
1299
			if ((offset & 0x3) == 0 &&
1300
			    data != 0 && (data | (1 << 10)) != ~(u64)0)
H
Huang Ying 已提交
1301 1302 1303 1304 1305 1306 1307 1308 1309
				return -1;
			vcpu->arch.mce_banks[offset] = data;
			break;
		}
		return 1;
	}
	return 0;
}

E
Ed Swierk 已提交
1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
	int lm = is_long_mode(vcpu);
	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
		: kvm->arch.xen_hvm_config.blob_size_32;
	u32 page_num = data & ~PAGE_MASK;
	u64 page_addr = data & PAGE_MASK;
	u8 *page;
	int r;

	r = -E2BIG;
	if (page_num >= blob_size)
		goto out;
	r = -ENOMEM;
1327 1328 1329
	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
	if (IS_ERR(page)) {
		r = PTR_ERR(page);
E
Ed Swierk 已提交
1330
		goto out;
1331
	}
E
Ed Swierk 已提交
1332 1333 1334 1335 1336 1337 1338 1339 1340
	if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
		goto out_free;
	r = 0;
out_free:
	kfree(page);
out:
	return r;
}

1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387
static bool kvm_hv_hypercall_enabled(struct kvm *kvm)
{
	return kvm->arch.hv_hypercall & HV_X64_MSR_HYPERCALL_ENABLE;
}

static bool kvm_hv_msr_partition_wide(u32 msr)
{
	bool r = false;
	switch (msr) {
	case HV_X64_MSR_GUEST_OS_ID:
	case HV_X64_MSR_HYPERCALL:
		r = true;
		break;
	}

	return r;
}

static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	struct kvm *kvm = vcpu->kvm;

	switch (msr) {
	case HV_X64_MSR_GUEST_OS_ID:
		kvm->arch.hv_guest_os_id = data;
		/* setting guest os id to zero disables hypercall page */
		if (!kvm->arch.hv_guest_os_id)
			kvm->arch.hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
		break;
	case HV_X64_MSR_HYPERCALL: {
		u64 gfn;
		unsigned long addr;
		u8 instructions[4];

		/* if guest os id is not set hypercall should remain disabled */
		if (!kvm->arch.hv_guest_os_id)
			break;
		if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) {
			kvm->arch.hv_hypercall = data;
			break;
		}
		gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;
		addr = gfn_to_hva(kvm, gfn);
		if (kvm_is_error_hva(addr))
			return 1;
		kvm_x86_ops->patch_hypercall(vcpu, instructions);
		((unsigned char *)instructions)[3] = 0xc3; /* ret */
1388
		if (__copy_to_user((void __user *)addr, instructions, 4))
1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402
			return 1;
		kvm->arch.hv_hypercall = data;
		break;
	}
	default:
		pr_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
			  "data 0x%llx\n", msr, data);
		return 1;
	}
	return 0;
}

static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
G
Gleb Natapov 已提交
1403 1404 1405
	switch (msr) {
	case HV_X64_MSR_APIC_ASSIST_PAGE: {
		unsigned long addr;
1406

G
Gleb Natapov 已提交
1407 1408 1409 1410 1411 1412 1413 1414
		if (!(data & HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE)) {
			vcpu->arch.hv_vapic = data;
			break;
		}
		addr = gfn_to_hva(vcpu->kvm, data >>
				  HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT);
		if (kvm_is_error_hva(addr))
			return 1;
1415
		if (__clear_user((void __user *)addr, PAGE_SIZE))
G
Gleb Natapov 已提交
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
			return 1;
		vcpu->arch.hv_vapic = data;
		break;
	}
	case HV_X64_MSR_EOI:
		return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data);
	case HV_X64_MSR_ICR:
		return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data);
	case HV_X64_MSR_TPR:
		return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data);
	default:
		pr_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
			  "data 0x%llx\n", msr, data);
		return 1;
	}

	return 0;
1433 1434
}

1435 1436 1437 1438
static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
{
	gpa_t gpa = data & ~0x3f;

1439 1440
	/* Bits 2:5 are resrved, Should be zero */
	if (data & 0x3c)
1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453
		return 1;

	vcpu->arch.apf.msr_val = data;

	if (!(data & KVM_ASYNC_PF_ENABLED)) {
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_async_pf_hash_reset(vcpu);
		return 0;
	}

	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa))
		return 1;

1454
	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
1455 1456 1457 1458
	kvm_async_pf_wakeup_all(vcpu);
	return 0;
}

1459 1460 1461 1462 1463 1464 1465 1466
static void kvmclock_reset(struct kvm_vcpu *vcpu)
{
	if (vcpu->arch.time_page) {
		kvm_release_page_dirty(vcpu->arch.time_page);
		vcpu->arch.time_page = NULL;
	}
}

G
Glauber Costa 已提交
1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495
static void accumulate_steal_time(struct kvm_vcpu *vcpu)
{
	u64 delta;

	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

	delta = current->sched_info.run_delay - vcpu->arch.st.last_steal;
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
	vcpu->arch.st.accum_steal = delta;
}

static void record_steal_time(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

	if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
		return;

	vcpu->arch.st.steal.steal += vcpu->arch.st.accum_steal;
	vcpu->arch.st.steal.version += 2;
	vcpu->arch.st.accum_steal = 0;

	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
}

1496 1497
int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
1498 1499
	bool pr = false;

1500 1501
	switch (msr) {
	case MSR_EFER:
1502
		return set_efer(vcpu, data);
1503 1504
	case MSR_K7_HWCR:
		data &= ~(u64)0x40;	/* ignore flush filter disable */
1505
		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
1506 1507 1508 1509 1510
		if (data != 0) {
			pr_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
				data);
			return 1;
		}
1511
		break;
1512 1513 1514 1515 1516 1517
	case MSR_FAM10H_MMIO_CONF_BASE:
		if (data != 0) {
			pr_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
				"0x%llx\n", data);
			return 1;
		}
1518
		break;
1519
	case MSR_AMD64_NB_CFG:
1520
		break;
1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532
	case MSR_IA32_DEBUGCTLMSR:
		if (!data) {
			/* We support the non-activated case already */
			break;
		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
			/* Values other than LBR and BTF are vendor-specific,
			   thus reserved and should throw a #GP */
			return 1;
		}
		pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
			__func__, data);
		break;
1533 1534
	case MSR_IA32_UCODE_REV:
	case MSR_IA32_UCODE_WRITE:
1535
	case MSR_VM_HSAVE_PA:
A
Andre Przywara 已提交
1536
	case MSR_AMD64_PATCH_LOADER:
1537
		break;
A
Avi Kivity 已提交
1538 1539
	case 0x200 ... 0x2ff:
		return set_msr_mtrr(vcpu, msr, data);
1540 1541 1542
	case MSR_IA32_APICBASE:
		kvm_set_apic_base(vcpu, data);
		break;
G
Gleb Natapov 已提交
1543 1544
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_write(vcpu, msr, data);
1545 1546 1547
	case MSR_IA32_TSCDEADLINE:
		kvm_set_lapic_tscdeadline_msr(vcpu, data);
		break;
1548
	case MSR_IA32_MISC_ENABLE:
1549
		vcpu->arch.ia32_misc_enable_msr = data;
1550
		break;
1551
	case MSR_KVM_WALL_CLOCK_NEW:
1552 1553 1554 1555
	case MSR_KVM_WALL_CLOCK:
		vcpu->kvm->arch.wall_clock = data;
		kvm_write_wall_clock(vcpu->kvm, data);
		break;
1556
	case MSR_KVM_SYSTEM_TIME_NEW:
1557
	case MSR_KVM_SYSTEM_TIME: {
1558
		kvmclock_reset(vcpu);
1559 1560

		vcpu->arch.time = data;
Z
Zachary Amsden 已提交
1561
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578

		/* we verify if the enable bit is set... */
		if (!(data & 1))
			break;

		/* ...but clean it before doing the actual write */
		vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);

		vcpu->arch.time_page =
				gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);

		if (is_error_page(vcpu->arch.time_page)) {
			kvm_release_page_clean(vcpu->arch.time_page);
			vcpu->arch.time_page = NULL;
		}
		break;
	}
1579 1580 1581 1582
	case MSR_KVM_ASYNC_PF_EN:
		if (kvm_pv_enable_async_pf(vcpu, data))
			return 1;
		break;
G
Glauber Costa 已提交
1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609
	case MSR_KVM_STEAL_TIME:

		if (unlikely(!sched_info_on()))
			return 1;

		if (data & KVM_STEAL_RESERVED_MASK)
			return 1;

		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
							data & KVM_STEAL_VALID_BITS))
			return 1;

		vcpu->arch.st.msr_val = data;

		if (!(data & KVM_MSR_ENABLED))
			break;

		vcpu->arch.st.last_steal = current->sched_info.run_delay;

		preempt_disable();
		accumulate_steal_time(vcpu);
		preempt_enable();

		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);

		break;

H
Huang Ying 已提交
1610 1611 1612 1613
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
	case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
		return set_msr_mce(vcpu, msr, data);
1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639

	/* Performance counters are not protected by a CPUID bit,
	 * so we should check all of them in the generic path for the sake of
	 * cross vendor migration.
	 * Writing a zero into the event select MSRs disables them,
	 * which we perfectly emulate ;-). Any other value should be at least
	 * reported, some guests depend on them.
	 */
	case MSR_K7_EVNTSEL0:
	case MSR_K7_EVNTSEL1:
	case MSR_K7_EVNTSEL2:
	case MSR_K7_EVNTSEL3:
		if (data != 0)
			pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
				"0x%x data 0x%llx\n", msr, data);
		break;
	/* at least RHEL 4 unconditionally writes to the perfctr registers,
	 * so we ignore writes to make it happy.
	 */
	case MSR_K7_PERFCTR0:
	case MSR_K7_PERFCTR1:
	case MSR_K7_PERFCTR2:
	case MSR_K7_PERFCTR3:
		pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
			"0x%x data 0x%llx\n", msr, data);
		break;
1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651
	case MSR_P6_PERFCTR0:
	case MSR_P6_PERFCTR1:
		pr = true;
	case MSR_P6_EVNTSEL0:
	case MSR_P6_EVNTSEL1:
		if (kvm_pmu_msr(vcpu, msr))
			return kvm_pmu_set_msr(vcpu, msr, data);

		if (pr || data != 0)
			pr_unimpl(vcpu, "disabled perfctr wrmsr: "
				"0x%x data 0x%llx\n", msr, data);
		break;
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
	case MSR_K7_CLK_CTL:
		/*
		 * Ignore all writes to this no longer documented MSR.
		 * Writes are only relevant for old K7 processors,
		 * all pre-dating SVM, but a recommended workaround from
		 * AMD for these chips. It is possible to speicify the
		 * affected processor models on the command line, hence
		 * the need to ignore the workaround.
		 */
		break;
1662 1663 1664 1665 1666 1667 1668 1669 1670 1671
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
		if (kvm_hv_msr_partition_wide(msr)) {
			int r;
			mutex_lock(&vcpu->kvm->lock);
			r = set_msr_hyperv_pw(vcpu, msr, data);
			mutex_unlock(&vcpu->kvm->lock);
			return r;
		} else
			return set_msr_hyperv(vcpu, msr, data);
		break;
1672 1673 1674 1675 1676 1677
	case MSR_IA32_BBL_CR_CTL3:
		/* Drop writes to this legacy MSR -- see rdmsr
		 * counterpart for further detail.
		 */
		pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", msr, data);
		break;
1678 1679 1680 1681 1682 1683 1684 1685 1686 1687
	case MSR_AMD64_OSVW_ID_LENGTH:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		vcpu->arch.osvw.length = data;
		break;
	case MSR_AMD64_OSVW_STATUS:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		vcpu->arch.osvw.status = data;
		break;
1688
	default:
E
Ed Swierk 已提交
1689 1690
		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
			return xen_hvm_config(vcpu, data);
1691 1692
		if (kvm_pmu_msr(vcpu, msr))
			return kvm_pmu_set_msr(vcpu, msr, data);
1693 1694 1695 1696 1697 1698 1699 1700 1701
		if (!ignore_msrs) {
			pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
				msr, data);
			return 1;
		} else {
			pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
				msr, data);
			break;
		}
1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);


/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
	return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
}

A
Avi Kivity 已提交
1718 1719
static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
S
Sheng Yang 已提交
1720 1721
	u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;

A
Avi Kivity 已提交
1722 1723 1724
	if (!msr_mtrr_valid(msr))
		return 1;

S
Sheng Yang 已提交
1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750
	if (msr == MSR_MTRRdefType)
		*pdata = vcpu->arch.mtrr_state.def_type +
			 (vcpu->arch.mtrr_state.enabled << 10);
	else if (msr == MSR_MTRRfix64K_00000)
		*pdata = p[0];
	else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
		*pdata = p[1 + msr - MSR_MTRRfix16K_80000];
	else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
		*pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
	else if (msr == MSR_IA32_CR_PAT)
		*pdata = vcpu->arch.pat;
	else {	/* Variable MTRRs */
		int idx, is_mtrr_mask;
		u64 *pt;

		idx = (msr - 0x200) / 2;
		is_mtrr_mask = msr - 0x200 - 2 * idx;
		if (!is_mtrr_mask)
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
		else
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
		*pdata = *pt;
	}

A
Avi Kivity 已提交
1751 1752 1753
	return 0;
}

H
Huang Ying 已提交
1754
static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1755 1756
{
	u64 data;
H
Huang Ying 已提交
1757 1758
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
1759 1760 1761 1762

	switch (msr) {
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
H
Huang Ying 已提交
1763 1764
		data = 0;
		break;
1765
	case MSR_IA32_MCG_CAP:
H
Huang Ying 已提交
1766 1767
		data = vcpu->arch.mcg_cap;
		break;
1768
	case MSR_IA32_MCG_CTL:
H
Huang Ying 已提交
1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788
		if (!(mcg_cap & MCG_CTL_P))
			return 1;
		data = vcpu->arch.mcg_ctl;
		break;
	case MSR_IA32_MCG_STATUS:
		data = vcpu->arch.mcg_status;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
		    msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
			u32 offset = msr - MSR_IA32_MC0_CTL;
			data = vcpu->arch.mce_banks[offset];
			break;
		}
		return 1;
	}
	*pdata = data;
	return 0;
}

1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822
static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data = 0;
	struct kvm *kvm = vcpu->kvm;

	switch (msr) {
	case HV_X64_MSR_GUEST_OS_ID:
		data = kvm->arch.hv_guest_os_id;
		break;
	case HV_X64_MSR_HYPERCALL:
		data = kvm->arch.hv_hypercall;
		break;
	default:
		pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
		return 1;
	}

	*pdata = data;
	return 0;
}

static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data = 0;

	switch (msr) {
	case HV_X64_MSR_VP_INDEX: {
		int r;
		struct kvm_vcpu *v;
		kvm_for_each_vcpu(r, v, vcpu->kvm)
			if (v == vcpu)
				data = r;
		break;
	}
G
Gleb Natapov 已提交
1823 1824 1825 1826 1827 1828
	case HV_X64_MSR_EOI:
		return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata);
	case HV_X64_MSR_ICR:
		return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
	case HV_X64_MSR_TPR:
		return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
1829
	case HV_X64_MSR_APIC_ASSIST_PAGE:
1830 1831
		data = vcpu->arch.hv_vapic;
		break;
1832 1833 1834 1835 1836 1837 1838 1839
	default:
		pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
		return 1;
	}
	*pdata = data;
	return 0;
}

H
Huang Ying 已提交
1840 1841 1842 1843 1844 1845
int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data;

	switch (msr) {
	case MSR_IA32_PLATFORM_ID:
1846
	case MSR_IA32_EBL_CR_POWERON:
1847 1848 1849 1850 1851
	case MSR_IA32_DEBUGCTLMSR:
	case MSR_IA32_LASTBRANCHFROMIP:
	case MSR_IA32_LASTBRANCHTOIP:
	case MSR_IA32_LASTINTFROMIP:
	case MSR_IA32_LASTINTTOIP:
1852 1853
	case MSR_K8_SYSCFG:
	case MSR_K7_HWCR:
1854
	case MSR_VM_HSAVE_PA:
A
Amit Shah 已提交
1855
	case MSR_K7_EVNTSEL0:
A
Amit Shah 已提交
1856
	case MSR_K7_PERFCTR0:
1857
	case MSR_K8_INT_PENDING_MSG:
1858
	case MSR_AMD64_NB_CFG:
1859
	case MSR_FAM10H_MMIO_CONF_BASE:
1860 1861
		data = 0;
		break;
1862 1863 1864 1865 1866 1867 1868 1869
	case MSR_P6_PERFCTR0:
	case MSR_P6_PERFCTR1:
	case MSR_P6_EVNTSEL0:
	case MSR_P6_EVNTSEL1:
		if (kvm_pmu_msr(vcpu, msr))
			return kvm_pmu_get_msr(vcpu, msr, pdata);
		data = 0;
		break;
1870 1871 1872
	case MSR_IA32_UCODE_REV:
		data = 0x100000000ULL;
		break;
A
Avi Kivity 已提交
1873 1874 1875 1876 1877
	case MSR_MTRRcap:
		data = 0x500 | KVM_NR_VAR_MTRR;
		break;
	case 0x200 ... 0x2ff:
		return get_msr_mtrr(vcpu, msr, pdata);
1878 1879 1880
	case 0xcd: /* fsb frequency */
		data = 3;
		break;
1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894
		/*
		 * MSR_EBC_FREQUENCY_ID
		 * Conservative value valid for even the basic CPU models.
		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
		 * and 266MHz for model 3, or 4. Set Core Clock
		 * Frequency to System Bus Frequency Ratio to 1 (bits
		 * 31:24) even though these are only valid for CPU
		 * models > 2, however guests may end up dividing or
		 * multiplying by zero otherwise.
		 */
	case MSR_EBC_FREQUENCY_ID:
		data = 1 << 24;
		break;
1895 1896 1897
	case MSR_IA32_APICBASE:
		data = kvm_get_apic_base(vcpu);
		break;
G
Gleb Natapov 已提交
1898 1899 1900
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_read(vcpu, msr, pdata);
		break;
1901 1902 1903
	case MSR_IA32_TSCDEADLINE:
		data = kvm_get_lapic_tscdeadline_msr(vcpu);
		break;
1904
	case MSR_IA32_MISC_ENABLE:
1905
		data = vcpu->arch.ia32_misc_enable_msr;
1906
		break;
1907 1908 1909 1910 1911 1912
	case MSR_IA32_PERF_STATUS:
		/* TSC increment by tick */
		data = 1000ULL;
		/* CPU multiplier */
		data |= (((uint64_t)4ULL) << 40);
		break;
1913
	case MSR_EFER:
1914
		data = vcpu->arch.efer;
1915
		break;
1916
	case MSR_KVM_WALL_CLOCK:
1917
	case MSR_KVM_WALL_CLOCK_NEW:
1918 1919 1920
		data = vcpu->kvm->arch.wall_clock;
		break;
	case MSR_KVM_SYSTEM_TIME:
1921
	case MSR_KVM_SYSTEM_TIME_NEW:
1922 1923
		data = vcpu->arch.time;
		break;
1924 1925 1926
	case MSR_KVM_ASYNC_PF_EN:
		data = vcpu->arch.apf.msr_val;
		break;
G
Glauber Costa 已提交
1927 1928 1929
	case MSR_KVM_STEAL_TIME:
		data = vcpu->arch.st.msr_val;
		break;
H
Huang Ying 已提交
1930 1931 1932 1933 1934 1935 1936
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
	case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
		return get_msr_mce(vcpu, msr, pdata);
1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948
	case MSR_K7_CLK_CTL:
		/*
		 * Provide expected ramp-up count for K7. All other
		 * are set to zero, indicating minimum divisors for
		 * every field.
		 *
		 * This prevents guest kernels on AMD host with CPU
		 * type 6, model 8 and higher from exploding due to
		 * the rdmsr failing.
		 */
		data = 0x20000000;
		break;
1949 1950 1951 1952 1953 1954 1955 1956 1957 1958
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
		if (kvm_hv_msr_partition_wide(msr)) {
			int r;
			mutex_lock(&vcpu->kvm->lock);
			r = get_msr_hyperv_pw(vcpu, msr, pdata);
			mutex_unlock(&vcpu->kvm->lock);
			return r;
		} else
			return get_msr_hyperv(vcpu, msr, pdata);
		break;
1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
	case MSR_IA32_BBL_CR_CTL3:
		/* This legacy MSR exists but isn't fully documented in current
		 * silicon.  It is however accessed by winxp in very narrow
		 * scenarios where it sets bit #19, itself documented as
		 * a "reserved" bit.  Best effort attempt to source coherent
		 * read data here should the balance of the register be
		 * interpreted by the guest:
		 *
		 * L2 cache control register 3: 64GB range, 256KB size,
		 * enabled, latency 0x1, configured
		 */
		data = 0xbe702111;
		break;
1972 1973 1974 1975 1976 1977 1978 1979 1980 1981
	case MSR_AMD64_OSVW_ID_LENGTH:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		data = vcpu->arch.osvw.length;
		break;
	case MSR_AMD64_OSVW_STATUS:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		data = vcpu->arch.osvw.status;
		break;
1982
	default:
1983 1984
		if (kvm_pmu_msr(vcpu, msr))
			return kvm_pmu_get_msr(vcpu, msr, pdata);
1985 1986 1987 1988 1989 1990 1991 1992
		if (!ignore_msrs) {
			pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
			return 1;
		} else {
			pr_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
			data = 0;
		}
		break;
1993 1994 1995 1996 1997 1998
	}
	*pdata = data;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

1999 2000 2001 2002 2003 2004 2005 2006 2007 2008
/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
2009
	int i, idx;
2010

2011
	idx = srcu_read_lock(&vcpu->kvm->srcu);
2012 2013 2014
	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;
2015
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
	if (copy_from_user(&msrs, user_msrs, sizeof msrs))
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
2044 2045 2046
	entries = memdup_user(user_msrs->entries, size);
	if (IS_ERR(entries)) {
		r = PTR_ERR(entries);
2047
		goto out;
2048
	}
2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060

	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
2061
	kfree(entries);
2062 2063 2064 2065
out:
	return r;
}

2066 2067 2068 2069 2070 2071 2072 2073 2074
int kvm_dev_ioctl_check_extension(long ext)
{
	int r;

	switch (ext) {
	case KVM_CAP_IRQCHIP:
	case KVM_CAP_HLT:
	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
	case KVM_CAP_SET_TSS_ADDR:
2075
	case KVM_CAP_EXT_CPUID:
2076
	case KVM_CAP_CLOCKSOURCE:
S
Sheng Yang 已提交
2077
	case KVM_CAP_PIT:
2078
	case KVM_CAP_NOP_IO_DELAY:
2079
	case KVM_CAP_MP_STATE:
2080
	case KVM_CAP_SYNC_MMU:
2081
	case KVM_CAP_USER_NMI:
2082
	case KVM_CAP_REINJECT_CONTROL:
2083
	case KVM_CAP_IRQ_INJECT_STATUS:
2084
	case KVM_CAP_ASSIGN_DEV_IRQ:
G
Gregory Haskins 已提交
2085
	case KVM_CAP_IRQFD:
G
Gregory Haskins 已提交
2086
	case KVM_CAP_IOEVENTFD:
2087
	case KVM_CAP_PIT2:
B
Beth Kon 已提交
2088
	case KVM_CAP_PIT_STATE2:
2089
	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
E
Ed Swierk 已提交
2090
	case KVM_CAP_XEN_HVM:
2091
	case KVM_CAP_ADJUST_CLOCK:
J
Jan Kiszka 已提交
2092
	case KVM_CAP_VCPU_EVENTS:
2093
	case KVM_CAP_HYPERV:
G
Gleb Natapov 已提交
2094
	case KVM_CAP_HYPERV_VAPIC:
2095
	case KVM_CAP_HYPERV_SPIN:
2096
	case KVM_CAP_PCI_SEGMENT:
2097
	case KVM_CAP_DEBUGREGS:
2098
	case KVM_CAP_X86_ROBUST_SINGLESTEP:
2099
	case KVM_CAP_XSAVE:
2100
	case KVM_CAP_ASYNC_PF:
2101
	case KVM_CAP_GET_TSC_KHZ:
2102 2103
		r = 1;
		break;
2104 2105 2106
	case KVM_CAP_COALESCED_MMIO:
		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
		break;
2107 2108 2109
	case KVM_CAP_VAPIC:
		r = !kvm_x86_ops->cpu_has_accelerated_tpr();
		break;
2110
	case KVM_CAP_NR_VCPUS:
2111 2112 2113
		r = KVM_SOFT_MAX_VCPUS;
		break;
	case KVM_CAP_MAX_VCPUS:
2114 2115
		r = KVM_MAX_VCPUS;
		break;
2116 2117 2118
	case KVM_CAP_NR_MEMSLOTS:
		r = KVM_MEMORY_SLOTS;
		break;
2119 2120
	case KVM_CAP_PV_MMU:	/* obsolete */
		r = 0;
2121
		break;
B
Ben-Ami Yassour 已提交
2122
	case KVM_CAP_IOMMU:
2123
		r = iommu_present(&pci_bus_type);
B
Ben-Ami Yassour 已提交
2124
		break;
H
Huang Ying 已提交
2125 2126 2127
	case KVM_CAP_MCE:
		r = KVM_MAX_MCE_BANKS;
		break;
2128 2129 2130
	case KVM_CAP_XCRS:
		r = cpu_has_xsave;
		break;
2131 2132 2133
	case KVM_CAP_TSC_CONTROL:
		r = kvm_has_tsc_control;
		break;
2134 2135 2136
	case KVM_CAP_TSC_DEADLINE_TIMER:
		r = boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER);
		break;
2137 2138 2139 2140 2141 2142 2143 2144
	default:
		r = 0;
		break;
	}
	return r;

}

2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164
long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {
	case KVM_GET_MSR_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
			goto out;
		n = msr_list.nmsrs;
		msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
		if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
			goto out;
		r = -E2BIG;
J
Jan Kiszka 已提交
2165
		if (n < msr_list.nmsrs)
2166 2167 2168 2169 2170
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
				 num_msrs_to_save * sizeof(u32)))
			goto out;
J
Jan Kiszka 已提交
2171
		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
2172 2173 2174 2175 2176 2177
				 &emulated_msrs,
				 ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
			goto out;
		r = 0;
		break;
	}
2178 2179 2180 2181 2182 2183 2184 2185
	case KVM_GET_SUPPORTED_CPUID: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
2186
						      cpuid_arg->entries);
2187 2188 2189 2190 2191 2192 2193 2194 2195
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
			goto out;
		r = 0;
		break;
	}
H
Huang Ying 已提交
2196 2197 2198 2199 2200 2201 2202 2203 2204 2205
	case KVM_X86_GET_MCE_CAP_SUPPORTED: {
		u64 mce_cap;

		mce_cap = KVM_MCE_CAP_SUPPORTED;
		r = -EFAULT;
		if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
			goto out;
		r = 0;
		break;
	}
2206 2207 2208 2209 2210 2211 2212
	default:
		r = -EINVAL;
	}
out:
	return r;
}

2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223
static void wbinvd_ipi(void *garbage)
{
	wbinvd();
}

static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
	return vcpu->kvm->arch.iommu_domain &&
		!(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY);
}

2224 2225
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
2226 2227 2228 2229 2230 2231 2232 2233 2234
	/* Address WBINVD may be executed by guest */
	if (need_emulate_wbinvd(vcpu)) {
		if (kvm_x86_ops->has_wbinvd_exit())
			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
			smp_call_function_single(vcpu->cpu,
					wbinvd_ipi, NULL, 1);
	}

2235
	kvm_x86_ops->vcpu_load(vcpu, cpu);
2236
	if (unlikely(vcpu->cpu != cpu) || check_tsc_unstable()) {
Z
Zachary Amsden 已提交
2237
		/* Make sure TSC doesn't go backwards */
2238 2239 2240
		s64 tsc_delta;
		u64 tsc;

N
Nadav Har'El 已提交
2241
		tsc = kvm_x86_ops->read_l1_tsc(vcpu);
2242 2243 2244
		tsc_delta = !vcpu->arch.last_guest_tsc ? 0 :
			     tsc - vcpu->arch.last_guest_tsc;

Z
Zachary Amsden 已提交
2245 2246
		if (tsc_delta < 0)
			mark_tsc_unstable("KVM discovered backwards TSC");
Z
Zachary Amsden 已提交
2247
		if (check_tsc_unstable()) {
Z
Zachary Amsden 已提交
2248
			kvm_x86_ops->adjust_tsc_offset(vcpu, -tsc_delta);
Z
Zachary Amsden 已提交
2249 2250
			vcpu->arch.tsc_catchup = 1;
		}
2251
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
Z
Zachary Amsden 已提交
2252 2253
		if (vcpu->cpu != cpu)
			kvm_migrate_timers(vcpu);
Z
Zachary Amsden 已提交
2254
		vcpu->cpu = cpu;
Z
Zachary Amsden 已提交
2255
	}
G
Glauber Costa 已提交
2256 2257 2258

	accumulate_steal_time(vcpu);
	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
2259 2260 2261 2262
}

void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
2263
	kvm_x86_ops->vcpu_put(vcpu);
2264
	kvm_put_guest_fpu(vcpu);
N
Nadav Har'El 已提交
2265
	vcpu->arch.last_guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu);
2266 2267 2268 2269 2270
}

static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
2271
	memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
2272 2273 2274 2275 2276 2277 2278

	return 0;
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
2279
	memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
2280
	kvm_apic_post_state_restore(vcpu);
2281
	update_cr8_intercept(vcpu);
2282 2283 2284 2285

	return 0;
}

2286 2287 2288 2289 2290 2291 2292 2293
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
{
	if (irq->irq < 0 || irq->irq >= 256)
		return -EINVAL;
	if (irqchip_in_kernel(vcpu->kvm))
		return -ENXIO;

2294
	kvm_queue_interrupt(vcpu, irq->irq, false);
2295
	kvm_make_request(KVM_REQ_EVENT, vcpu);
2296 2297 2298 2299

	return 0;
}

2300 2301 2302 2303 2304 2305 2306
static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
{
	kvm_inject_nmi(vcpu);

	return 0;
}

2307 2308 2309 2310 2311 2312 2313 2314 2315
static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
					   struct kvm_tpr_access_ctl *tac)
{
	if (tac->flags)
		return -EINVAL;
	vcpu->arch.tpr_access_reporting = !!tac->enabled;
	return 0;
}

H
Huang Ying 已提交
2316 2317 2318 2319 2320 2321 2322
static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
					u64 mcg_cap)
{
	int r;
	unsigned bank_num = mcg_cap & 0xff, bank;

	r = -EINVAL;
2323
	if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
H
Huang Ying 已提交
2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363
		goto out;
	if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
		goto out;
	r = 0;
	vcpu->arch.mcg_cap = mcg_cap;
	/* Init IA32_MCG_CTL to all 1s */
	if (mcg_cap & MCG_CTL_P)
		vcpu->arch.mcg_ctl = ~(u64)0;
	/* Init IA32_MCi_CTL to all 1s */
	for (bank = 0; bank < bank_num; bank++)
		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
out:
	return r;
}

static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
				      struct kvm_x86_mce *mce)
{
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
	u64 *banks = vcpu->arch.mce_banks;

	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
		return -EINVAL;
	/*
	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
	 * reporting is disabled
	 */
	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
	    vcpu->arch.mcg_ctl != ~(u64)0)
		return 0;
	banks += 4 * mce->bank;
	/*
	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
	 * reporting is disabled for the bank
	 */
	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
		return 0;
	if (mce->status & MCI_STATUS_UC) {
		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
2364
		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
2365
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
H
Huang Ying 已提交
2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386
			return 0;
		}
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		vcpu->arch.mcg_status = mce->mcg_status;
		banks[1] = mce->status;
		kvm_queue_exception(vcpu, MC_VECTOR);
	} else if (!(banks[1] & MCI_STATUS_VAL)
		   || !(banks[1] & MCI_STATUS_UC)) {
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		banks[1] = mce->status;
	} else
		banks[1] |= MCI_STATUS_OVER;
	return 0;
}

J
Jan Kiszka 已提交
2387 2388 2389
static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
					       struct kvm_vcpu_events *events)
{
A
Avi Kivity 已提交
2390
	process_nmi(vcpu);
2391 2392 2393
	events->exception.injected =
		vcpu->arch.exception.pending &&
		!kvm_exception_is_soft(vcpu->arch.exception.nr);
J
Jan Kiszka 已提交
2394 2395
	events->exception.nr = vcpu->arch.exception.nr;
	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
2396
	events->exception.pad = 0;
J
Jan Kiszka 已提交
2397 2398
	events->exception.error_code = vcpu->arch.exception.error_code;

2399 2400
	events->interrupt.injected =
		vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft;
J
Jan Kiszka 已提交
2401
	events->interrupt.nr = vcpu->arch.interrupt.nr;
2402
	events->interrupt.soft = 0;
2403 2404 2405
	events->interrupt.shadow =
		kvm_x86_ops->get_interrupt_shadow(vcpu,
			KVM_X86_SHADOW_INT_MOV_SS | KVM_X86_SHADOW_INT_STI);
J
Jan Kiszka 已提交
2406 2407

	events->nmi.injected = vcpu->arch.nmi_injected;
A
Avi Kivity 已提交
2408
	events->nmi.pending = vcpu->arch.nmi_pending != 0;
J
Jan Kiszka 已提交
2409
	events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
2410
	events->nmi.pad = 0;
J
Jan Kiszka 已提交
2411 2412 2413

	events->sipi_vector = vcpu->arch.sipi_vector;

2414
	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
2415 2416
			 | KVM_VCPUEVENT_VALID_SIPI_VECTOR
			 | KVM_VCPUEVENT_VALID_SHADOW);
2417
	memset(&events->reserved, 0, sizeof(events->reserved));
J
Jan Kiszka 已提交
2418 2419 2420 2421 2422
}

static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
					      struct kvm_vcpu_events *events)
{
2423
	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
2424 2425
			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
			      | KVM_VCPUEVENT_VALID_SHADOW))
J
Jan Kiszka 已提交
2426 2427
		return -EINVAL;

A
Avi Kivity 已提交
2428
	process_nmi(vcpu);
J
Jan Kiszka 已提交
2429 2430 2431 2432 2433 2434 2435 2436
	vcpu->arch.exception.pending = events->exception.injected;
	vcpu->arch.exception.nr = events->exception.nr;
	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
	vcpu->arch.exception.error_code = events->exception.error_code;

	vcpu->arch.interrupt.pending = events->interrupt.injected;
	vcpu->arch.interrupt.nr = events->interrupt.nr;
	vcpu->arch.interrupt.soft = events->interrupt.soft;
2437 2438 2439
	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
		kvm_x86_ops->set_interrupt_shadow(vcpu,
						  events->interrupt.shadow);
J
Jan Kiszka 已提交
2440 2441

	vcpu->arch.nmi_injected = events->nmi.injected;
2442 2443
	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
		vcpu->arch.nmi_pending = events->nmi.pending;
J
Jan Kiszka 已提交
2444 2445
	kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);

2446 2447
	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR)
		vcpu->arch.sipi_vector = events->sipi_vector;
J
Jan Kiszka 已提交
2448

2449 2450
	kvm_make_request(KVM_REQ_EVENT, vcpu);

J
Jan Kiszka 已提交
2451 2452 2453
	return 0;
}

2454 2455 2456 2457 2458 2459 2460
static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
					     struct kvm_debugregs *dbgregs)
{
	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
	dbgregs->dr6 = vcpu->arch.dr6;
	dbgregs->dr7 = vcpu->arch.dr7;
	dbgregs->flags = 0;
2461
	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476
}

static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
					    struct kvm_debugregs *dbgregs)
{
	if (dbgregs->flags)
		return -EINVAL;

	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
	vcpu->arch.dr6 = dbgregs->dr6;
	vcpu->arch.dr7 = dbgregs->dr7;

	return 0;
}

2477 2478 2479 2480 2481 2482
static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
					 struct kvm_xsave *guest_xsave)
{
	if (cpu_has_xsave)
		memcpy(guest_xsave->region,
			&vcpu->arch.guest_fpu.state->xsave,
2483
			xstate_size);
2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500
	else {
		memcpy(guest_xsave->region,
			&vcpu->arch.guest_fpu.state->fxsave,
			sizeof(struct i387_fxsave_struct));
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
			XSTATE_FPSSE;
	}
}

static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
					struct kvm_xsave *guest_xsave)
{
	u64 xstate_bv =
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];

	if (cpu_has_xsave)
		memcpy(&vcpu->arch.guest_fpu.state->xsave,
2501
			guest_xsave->region, xstate_size);
2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547
	else {
		if (xstate_bv & ~XSTATE_FPSSE)
			return -EINVAL;
		memcpy(&vcpu->arch.guest_fpu.state->fxsave,
			guest_xsave->region, sizeof(struct i387_fxsave_struct));
	}
	return 0;
}

static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
					struct kvm_xcrs *guest_xcrs)
{
	if (!cpu_has_xsave) {
		guest_xcrs->nr_xcrs = 0;
		return;
	}

	guest_xcrs->nr_xcrs = 1;
	guest_xcrs->flags = 0;
	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
}

static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
				       struct kvm_xcrs *guest_xcrs)
{
	int i, r = 0;

	if (!cpu_has_xsave)
		return -EINVAL;

	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
		return -EINVAL;

	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
		/* Only support XCR0 currently */
		if (guest_xcrs->xcrs[0].xcr == XCR_XFEATURE_ENABLED_MASK) {
			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
				guest_xcrs->xcrs[0].value);
			break;
		}
	if (r)
		r = -EINVAL;
	return r;
}

2548 2549 2550 2551 2552 2553
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r;
2554 2555 2556 2557 2558 2559 2560 2561
	union {
		struct kvm_lapic_state *lapic;
		struct kvm_xsave *xsave;
		struct kvm_xcrs *xcrs;
		void *buffer;
	} u;

	u.buffer = NULL;
2562 2563
	switch (ioctl) {
	case KVM_GET_LAPIC: {
2564 2565 2566
		r = -EINVAL;
		if (!vcpu->arch.apic)
			goto out;
2567
		u.lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
2568

2569
		r = -ENOMEM;
2570
		if (!u.lapic)
2571
			goto out;
2572
		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
2573 2574 2575
		if (r)
			goto out;
		r = -EFAULT;
2576
		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
2577 2578 2579 2580 2581
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
2582 2583 2584
		r = -EINVAL;
		if (!vcpu->arch.apic)
			goto out;
2585 2586 2587
		u.lapic = memdup_user(argp, sizeof(*u.lapic));
		if (IS_ERR(u.lapic)) {
			r = PTR_ERR(u.lapic);
2588
			goto out;
2589 2590
		}

2591
		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
2592 2593 2594 2595 2596
		if (r)
			goto out;
		r = 0;
		break;
	}
2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
		if (copy_from_user(&irq, argp, sizeof irq))
			goto out;
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
		if (r)
			goto out;
		r = 0;
		break;
	}
2609 2610 2611 2612 2613 2614 2615
	case KVM_NMI: {
		r = kvm_vcpu_ioctl_nmi(vcpu);
		if (r)
			goto out;
		r = 0;
		break;
	}
2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		if (r)
			goto out;
		break;
	}
2628 2629 2630 2631 2632 2633 2634 2635
	case KVM_SET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
2636
					      cpuid_arg->entries);
2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648
		if (r)
			goto out;
		break;
	}
	case KVM_GET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
2649
					      cpuid_arg->entries);
2650 2651 2652 2653 2654 2655 2656 2657
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
			goto out;
		r = 0;
		break;
	}
2658 2659 2660 2661 2662 2663
	case KVM_GET_MSRS:
		r = msr_io(vcpu, argp, kvm_get_msr, 1);
		break;
	case KVM_SET_MSRS:
		r = msr_io(vcpu, argp, do_set_msr, 0);
		break;
2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678
	case KVM_TPR_ACCESS_REPORTING: {
		struct kvm_tpr_access_ctl tac;

		r = -EFAULT;
		if (copy_from_user(&tac, argp, sizeof tac))
			goto out;
		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &tac, sizeof tac))
			goto out;
		r = 0;
		break;
	};
A
Avi Kivity 已提交
2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691
	case KVM_SET_VAPIC_ADDR: {
		struct kvm_vapic_addr va;

		r = -EINVAL;
		if (!irqchip_in_kernel(vcpu->kvm))
			goto out;
		r = -EFAULT;
		if (copy_from_user(&va, argp, sizeof va))
			goto out;
		r = 0;
		kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
		break;
	}
H
Huang Ying 已提交
2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709
	case KVM_X86_SETUP_MCE: {
		u64 mcg_cap;

		r = -EFAULT;
		if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
			goto out;
		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
		break;
	}
	case KVM_X86_SET_MCE: {
		struct kvm_x86_mce mce;

		r = -EFAULT;
		if (copy_from_user(&mce, argp, sizeof mce))
			goto out;
		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
		break;
	}
J
Jan Kiszka 已提交
2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);

		r = -EFAULT;
		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		r = -EFAULT;
		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
			break;

		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
		break;
	}
2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753
	case KVM_GET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);

		r = -EFAULT;
		if (copy_to_user(argp, &dbgregs,
				 sizeof(struct kvm_debugregs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		r = -EFAULT;
		if (copy_from_user(&dbgregs, argp,
				   sizeof(struct kvm_debugregs)))
			break;

		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
		break;
	}
2754
	case KVM_GET_XSAVE: {
2755
		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL);
2756
		r = -ENOMEM;
2757
		if (!u.xsave)
2758 2759
			break;

2760
		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
2761 2762

		r = -EFAULT;
2763
		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
2764 2765 2766 2767 2768
			break;
		r = 0;
		break;
	}
	case KVM_SET_XSAVE: {
2769 2770 2771 2772 2773
		u.xsave = memdup_user(argp, sizeof(*u.xsave));
		if (IS_ERR(u.xsave)) {
			r = PTR_ERR(u.xsave);
			goto out;
		}
2774

2775
		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
2776 2777 2778
		break;
	}
	case KVM_GET_XCRS: {
2779
		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL);
2780
		r = -ENOMEM;
2781
		if (!u.xcrs)
2782 2783
			break;

2784
		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
2785 2786

		r = -EFAULT;
2787
		if (copy_to_user(argp, u.xcrs,
2788 2789 2790 2791 2792 2793
				 sizeof(struct kvm_xcrs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XCRS: {
2794 2795 2796 2797 2798
		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
		if (IS_ERR(u.xcrs)) {
			r = PTR_ERR(u.xcrs);
			goto out;
		}
2799

2800
		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
2801 2802
		break;
	}
2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828
	case KVM_SET_TSC_KHZ: {
		u32 user_tsc_khz;

		r = -EINVAL;
		if (!kvm_has_tsc_control)
			break;

		user_tsc_khz = (u32)arg;

		if (user_tsc_khz >= kvm_max_guest_tsc_khz)
			goto out;

		kvm_x86_ops->set_tsc_khz(vcpu, user_tsc_khz);

		r = 0;
		goto out;
	}
	case KVM_GET_TSC_KHZ: {
		r = -EIO;
		if (check_tsc_unstable())
			goto out;

		r = vcpu_tsc_khz(vcpu);

		goto out;
	}
2829 2830 2831 2832
	default:
		r = -EINVAL;
	}
out:
2833
	kfree(u.buffer);
2834 2835 2836
	return r;
}

2837 2838 2839 2840 2841
int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
{
	return VM_FAULT_SIGBUS;
}

2842 2843 2844 2845 2846 2847 2848 2849 2850 2851
static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
{
	int ret;

	if (addr > (unsigned int)(-3 * PAGE_SIZE))
		return -1;
	ret = kvm_x86_ops->set_tss_addr(kvm, addr);
	return ret;
}

2852 2853 2854 2855 2856 2857 2858
static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
					      u64 ident_addr)
{
	kvm->arch.ept_identity_map_addr = ident_addr;
	return 0;
}

2859 2860 2861 2862 2863 2864
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
					  u32 kvm_nr_mmu_pages)
{
	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
		return -EINVAL;

2865
	mutex_lock(&kvm->slots_lock);
2866
	spin_lock(&kvm->mmu_lock);
2867 2868

	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
2869
	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
2870

2871
	spin_unlock(&kvm->mmu_lock);
2872
	mutex_unlock(&kvm->slots_lock);
2873 2874 2875 2876 2877
	return 0;
}

static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
{
2878
	return kvm->arch.n_max_mmu_pages;
2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897
}

static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
		memcpy(&chip->chip.pic,
			&pic_irqchip(kvm)->pics[0],
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
		memcpy(&chip->chip.pic,
			&pic_irqchip(kvm)->pics[1],
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_IOAPIC:
G
Gleb Natapov 已提交
2898
		r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913
		break;
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
2914
		spin_lock(&pic_irqchip(kvm)->lock);
2915 2916 2917
		memcpy(&pic_irqchip(kvm)->pics[0],
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
2918
		spin_unlock(&pic_irqchip(kvm)->lock);
2919 2920
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
2921
		spin_lock(&pic_irqchip(kvm)->lock);
2922 2923 2924
		memcpy(&pic_irqchip(kvm)->pics[1],
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
2925
		spin_unlock(&pic_irqchip(kvm)->lock);
2926 2927
		break;
	case KVM_IRQCHIP_IOAPIC:
G
Gleb Natapov 已提交
2928
		r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
2929 2930 2931 2932 2933 2934 2935 2936 2937
		break;
	default:
		r = -EINVAL;
		break;
	}
	kvm_pic_update_irq(pic_irqchip(kvm));
	return r;
}

2938 2939 2940 2941
static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
	int r = 0;

2942
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
2943
	memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
2944
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
2945 2946 2947 2948 2949 2950 2951
	return r;
}

static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
	int r = 0;

2952
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
2953
	memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
B
Beth Kon 已提交
2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967
	kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
	return r;
}

static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	int r = 0;

	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
		sizeof(ps->channels));
	ps->flags = kvm->arch.vpit->pit_state.flags;
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
2968
	memset(&ps->reserved, 0, sizeof(ps->reserved));
B
Beth Kon 已提交
2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984
	return r;
}

static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	int r = 0, start = 0;
	u32 prev_legacy, cur_legacy;
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
	if (!prev_legacy && cur_legacy)
		start = 1;
	memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
	       sizeof(kvm->arch.vpit->pit_state.channels));
	kvm->arch.vpit->pit_state.flags = ps->flags;
	kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
2985
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
2986 2987 2988
	return r;
}

2989 2990 2991 2992 2993
static int kvm_vm_ioctl_reinject(struct kvm *kvm,
				 struct kvm_reinject_control *control)
{
	if (!kvm->arch.vpit)
		return -ENXIO;
2994
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
2995
	kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject;
2996
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
2997 2998 2999
	return 0;
}

3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043
/**
 * write_protect_slot - write protect a slot for dirty logging
 * @kvm: the kvm instance
 * @memslot: the slot we protect
 * @dirty_bitmap: the bitmap indicating which pages are dirty
 * @nr_dirty_pages: the number of dirty pages
 *
 * We have two ways to find all sptes to protect:
 * 1. Use kvm_mmu_slot_remove_write_access() which walks all shadow pages and
 *    checks ones that have a spte mapping a page in the slot.
 * 2. Use kvm_mmu_rmap_write_protect() for each gfn found in the bitmap.
 *
 * Generally speaking, if there are not so many dirty pages compared to the
 * number of shadow pages, we should use the latter.
 *
 * Note that letting others write into a page marked dirty in the old bitmap
 * by using the remaining tlb entry is not a problem.  That page will become
 * write protected again when we flush the tlb and then be reported dirty to
 * the user space by copying the old bitmap.
 */
static void write_protect_slot(struct kvm *kvm,
			       struct kvm_memory_slot *memslot,
			       unsigned long *dirty_bitmap,
			       unsigned long nr_dirty_pages)
{
	/* Not many dirty pages compared to # of shadow pages. */
	if (nr_dirty_pages < kvm->arch.n_used_mmu_pages) {
		unsigned long gfn_offset;

		for_each_set_bit(gfn_offset, dirty_bitmap, memslot->npages) {
			unsigned long gfn = memslot->base_gfn + gfn_offset;

			spin_lock(&kvm->mmu_lock);
			kvm_mmu_rmap_write_protect(kvm, gfn, memslot);
			spin_unlock(&kvm->mmu_lock);
		}
		kvm_flush_remote_tlbs(kvm);
	} else {
		spin_lock(&kvm->mmu_lock);
		kvm_mmu_slot_remove_write_access(kvm, memslot->id);
		spin_unlock(&kvm->mmu_lock);
	}
}

3044 3045 3046 3047 3048 3049
/*
 * Get (and clear) the dirty memory log for a memory slot.
 */
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
				      struct kvm_dirty_log *log)
{
3050
	int r;
3051
	struct kvm_memory_slot *memslot;
3052
	unsigned long n, nr_dirty_pages;
3053

3054
	mutex_lock(&kvm->slots_lock);
3055

M
Marcelo Tosatti 已提交
3056 3057 3058 3059
	r = -EINVAL;
	if (log->slot >= KVM_MEMORY_SLOTS)
		goto out;

3060
	memslot = id_to_memslot(kvm->memslots, log->slot);
M
Marcelo Tosatti 已提交
3061 3062 3063 3064
	r = -ENOENT;
	if (!memslot->dirty_bitmap)
		goto out;

3065
	n = kvm_dirty_bitmap_bytes(memslot);
3066
	nr_dirty_pages = memslot->nr_dirty_pages;
M
Marcelo Tosatti 已提交
3067

3068
	/* If nothing is dirty, don't bother messing with page tables. */
3069
	if (nr_dirty_pages) {
M
Marcelo Tosatti 已提交
3070
		struct kvm_memslots *slots, *old_slots;
3071
		unsigned long *dirty_bitmap, *dirty_bitmap_head;
M
Marcelo Tosatti 已提交
3072

3073 3074 3075 3076 3077
		dirty_bitmap = memslot->dirty_bitmap;
		dirty_bitmap_head = memslot->dirty_bitmap_head;
		if (dirty_bitmap == dirty_bitmap_head)
			dirty_bitmap_head += n / sizeof(long);
		memset(dirty_bitmap_head, 0, n);
M
Marcelo Tosatti 已提交
3078

3079
		r = -ENOMEM;
3080
		slots = kmemdup(kvm->memslots, sizeof(*kvm->memslots), GFP_KERNEL);
3081
		if (!slots)
3082
			goto out;
3083

3084
		memslot = id_to_memslot(slots, log->slot);
3085
		memslot->nr_dirty_pages = 0;
3086
		memslot->dirty_bitmap = dirty_bitmap_head;
3087
		update_memslots(slots, NULL);
M
Marcelo Tosatti 已提交
3088 3089 3090 3091 3092

		old_slots = kvm->memslots;
		rcu_assign_pointer(kvm->memslots, slots);
		synchronize_srcu_expedited(&kvm->srcu);
		kfree(old_slots);
3093

3094
		write_protect_slot(kvm, memslot, dirty_bitmap, nr_dirty_pages);
3095

3096
		r = -EFAULT;
3097
		if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n))
3098 3099 3100 3101 3102
			goto out;
	} else {
		r = -EFAULT;
		if (clear_user(log->dirty_bitmap, n))
			goto out;
3103
	}
M
Marcelo Tosatti 已提交
3104

3105 3106
	r = 0;
out:
3107
	mutex_unlock(&kvm->slots_lock);
3108 3109 3110
	return r;
}

3111 3112 3113 3114 3115
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
3116
	int r = -ENOTTY;
3117 3118 3119 3120 3121 3122 3123
	/*
	 * This union makes it completely explicit to gcc-3.x
	 * that these two variables' stack usage should be
	 * combined, not added together.
	 */
	union {
		struct kvm_pit_state ps;
B
Beth Kon 已提交
3124
		struct kvm_pit_state2 ps2;
3125
		struct kvm_pit_config pit_config;
3126
	} u;
3127 3128 3129 3130 3131 3132 3133

	switch (ioctl) {
	case KVM_SET_TSS_ADDR:
		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
		if (r < 0)
			goto out;
		break;
3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144
	case KVM_SET_IDENTITY_MAP_ADDR: {
		u64 ident_addr;

		r = -EFAULT;
		if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
			goto out;
		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
		if (r < 0)
			goto out;
		break;
	}
3145 3146 3147 3148 3149 3150 3151 3152
	case KVM_SET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
		if (r)
			goto out;
		break;
	case KVM_GET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
		break;
3153 3154 3155 3156 3157 3158 3159
	case KVM_CREATE_IRQCHIP: {
		struct kvm_pic *vpic;

		mutex_lock(&kvm->lock);
		r = -EEXIST;
		if (kvm->arch.vpic)
			goto create_irqchip_unlock;
3160
		r = -ENOMEM;
3161 3162
		vpic = kvm_create_pic(kvm);
		if (vpic) {
3163 3164
			r = kvm_ioapic_init(kvm);
			if (r) {
3165
				mutex_lock(&kvm->slots_lock);
3166
				kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
3167 3168 3169 3170 3171
							  &vpic->dev_master);
				kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
							  &vpic->dev_slave);
				kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
							  &vpic->dev_eclr);
3172
				mutex_unlock(&kvm->slots_lock);
3173 3174
				kfree(vpic);
				goto create_irqchip_unlock;
3175 3176
			}
		} else
3177 3178 3179 3180
			goto create_irqchip_unlock;
		smp_wmb();
		kvm->arch.vpic = vpic;
		smp_wmb();
3181 3182
		r = kvm_setup_default_irq_routing(kvm);
		if (r) {
3183
			mutex_lock(&kvm->slots_lock);
3184
			mutex_lock(&kvm->irq_lock);
3185 3186
			kvm_ioapic_destroy(kvm);
			kvm_destroy_pic(kvm);
3187
			mutex_unlock(&kvm->irq_lock);
3188
			mutex_unlock(&kvm->slots_lock);
3189
		}
3190 3191
	create_irqchip_unlock:
		mutex_unlock(&kvm->lock);
3192
		break;
3193
	}
S
Sheng Yang 已提交
3194
	case KVM_CREATE_PIT:
3195 3196 3197 3198 3199 3200 3201 3202
		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
		goto create_pit;
	case KVM_CREATE_PIT2:
		r = -EFAULT;
		if (copy_from_user(&u.pit_config, argp,
				   sizeof(struct kvm_pit_config)))
			goto out;
	create_pit:
3203
		mutex_lock(&kvm->slots_lock);
A
Avi Kivity 已提交
3204 3205 3206
		r = -EEXIST;
		if (kvm->arch.vpit)
			goto create_pit_unlock;
S
Sheng Yang 已提交
3207
		r = -ENOMEM;
3208
		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
S
Sheng Yang 已提交
3209 3210
		if (kvm->arch.vpit)
			r = 0;
A
Avi Kivity 已提交
3211
	create_pit_unlock:
3212
		mutex_unlock(&kvm->slots_lock);
S
Sheng Yang 已提交
3213
		break;
3214
	case KVM_IRQ_LINE_STATUS:
3215 3216 3217 3218 3219 3220
	case KVM_IRQ_LINE: {
		struct kvm_irq_level irq_event;

		r = -EFAULT;
		if (copy_from_user(&irq_event, argp, sizeof irq_event))
			goto out;
3221
		r = -ENXIO;
3222
		if (irqchip_in_kernel(kvm)) {
3223 3224 3225 3226
			__s32 status;
			status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
					irq_event.irq, irq_event.level);
			if (ioctl == KVM_IRQ_LINE_STATUS) {
3227
				r = -EFAULT;
3228 3229 3230 3231 3232
				irq_event.status = status;
				if (copy_to_user(argp, &irq_event,
							sizeof irq_event))
					goto out;
			}
3233 3234 3235 3236 3237 3238
			r = 0;
		}
		break;
	}
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3239
		struct kvm_irqchip *chip;
3240

3241 3242 3243
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
3244
			goto out;
3245 3246
		}

3247 3248
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
3249 3250
			goto get_irqchip_out;
		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
3251
		if (r)
3252
			goto get_irqchip_out;
3253
		r = -EFAULT;
3254 3255
		if (copy_to_user(argp, chip, sizeof *chip))
			goto get_irqchip_out;
3256
		r = 0;
3257 3258 3259 3260
	get_irqchip_out:
		kfree(chip);
		if (r)
			goto out;
3261 3262 3263 3264
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3265
		struct kvm_irqchip *chip;
3266

3267 3268 3269
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
3270
			goto out;
3271 3272
		}

3273 3274
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
3275 3276
			goto set_irqchip_out;
		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
3277
		if (r)
3278
			goto set_irqchip_out;
3279
		r = 0;
3280 3281 3282 3283
	set_irqchip_out:
		kfree(chip);
		if (r)
			goto out;
3284 3285
		break;
	}
3286 3287
	case KVM_GET_PIT: {
		r = -EFAULT;
3288
		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
3289 3290 3291 3292
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
3293
		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
3294 3295 3296
		if (r)
			goto out;
		r = -EFAULT;
3297
		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
3298 3299 3300 3301 3302 3303
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT: {
		r = -EFAULT;
3304
		if (copy_from_user(&u.ps, argp, sizeof u.ps))
3305 3306 3307 3308
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
3309
		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
3310 3311 3312 3313 3314
		if (r)
			goto out;
		r = 0;
		break;
	}
B
Beth Kon 已提交
3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340
	case KVM_GET_PIT2: {
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT2: {
		r = -EFAULT;
		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
		if (r)
			goto out;
		r = 0;
		break;
	}
3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351
	case KVM_REINJECT_CONTROL: {
		struct kvm_reinject_control control;
		r =  -EFAULT;
		if (copy_from_user(&control, argp, sizeof(control)))
			goto out;
		r = kvm_vm_ioctl_reinject(kvm, &control);
		if (r)
			goto out;
		r = 0;
		break;
	}
E
Ed Swierk 已提交
3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362
	case KVM_XEN_HVM_CONFIG: {
		r = -EFAULT;
		if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
				   sizeof(struct kvm_xen_hvm_config)))
			goto out;
		r = -EINVAL;
		if (kvm->arch.xen_hvm_config.flags)
			goto out;
		r = 0;
		break;
	}
3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376
	case KVM_SET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;
		s64 delta;

		r = -EFAULT;
		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
			goto out;

		r = -EINVAL;
		if (user_ns.flags)
			goto out;

		r = 0;
3377
		local_irq_disable();
3378
		now_ns = get_kernel_ns();
3379
		delta = user_ns.clock - now_ns;
3380
		local_irq_enable();
3381 3382 3383 3384 3385 3386 3387
		kvm->arch.kvmclock_offset = delta;
		break;
	}
	case KVM_GET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

3388
		local_irq_disable();
3389
		now_ns = get_kernel_ns();
3390
		user_ns.clock = kvm->arch.kvmclock_offset + now_ns;
3391
		local_irq_enable();
3392
		user_ns.flags = 0;
3393
		memset(&user_ns.pad, 0, sizeof(user_ns.pad));
3394 3395 3396 3397 3398 3399 3400 3401

		r = -EFAULT;
		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
			goto out;
		r = 0;
		break;
	}

3402 3403 3404 3405 3406 3407 3408
	default:
		;
	}
out:
	return r;
}

3409
static void kvm_init_msr_list(void)
3410 3411 3412 3413
{
	u32 dummy[2];
	unsigned i, j;

3414 3415
	/* skip the first msrs in the list. KVM-specific */
	for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) {
3416 3417 3418 3419 3420 3421 3422 3423 3424
		if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
			continue;
		if (j < i)
			msrs_to_save[j] = msrs_to_save[i];
		j++;
	}
	num_msrs_to_save = j;
}

3425 3426
static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
			   const void *v)
3427
{
3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
		if (!(vcpu->arch.apic &&
		      !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
			break;
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
3442

3443
	return handled;
3444 3445
}

3446
static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
3447
{
3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
		if (!(vcpu->arch.apic &&
		      !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
			break;
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v);
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
3463

3464
	return handled;
3465 3466
}

3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478
static void kvm_set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
	kvm_x86_ops->set_segment(vcpu, var, seg);
}

void kvm_get_segment(struct kvm_vcpu *vcpu,
		     struct kvm_segment *var, int seg)
{
	kvm_x86_ops->get_segment(vcpu, var, seg);
}

3479
gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
3480 3481
{
	gpa_t t_gpa;
3482
	struct x86_exception exception;
3483 3484 3485 3486 3487

	BUG_ON(!mmu_is_nested(vcpu));

	/* NPT walks are always user-walks */
	access |= PFERR_USER_MASK;
3488
	t_gpa  = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &exception);
3489 3490 3491 3492

	return t_gpa;
}

3493 3494
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
			      struct x86_exception *exception)
3495 3496
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
3497
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
3498 3499
}

3500 3501
 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
3502 3503 3504
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_FETCH_MASK;
3505
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
3506 3507
}

3508 3509
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
			       struct x86_exception *exception)
3510 3511 3512
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_WRITE_MASK;
3513
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
3514 3515 3516
}

/* uses this to access any guest's mapped memory without checking CPL */
3517 3518
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
3519
{
3520
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
3521 3522 3523 3524
}

static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
3525
				      struct x86_exception *exception)
3526 3527
{
	void *data = val;
3528
	int r = X86EMUL_CONTINUE;
3529 3530

	while (bytes) {
3531
		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
3532
							    exception);
3533
		unsigned offset = addr & (PAGE_SIZE-1);
3534
		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
3535 3536
		int ret;

3537
		if (gpa == UNMAPPED_GVA)
3538
			return X86EMUL_PROPAGATE_FAULT;
3539
		ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
3540
		if (ret < 0) {
3541
			r = X86EMUL_IO_NEEDED;
3542 3543
			goto out;
		}
3544

3545 3546 3547
		bytes -= toread;
		data += toread;
		addr += toread;
3548
	}
3549 3550
out:
	return r;
3551
}
3552

3553
/* used for instruction fetching */
3554 3555
static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
				gva_t addr, void *val, unsigned int bytes,
3556
				struct x86_exception *exception)
3557
{
3558
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3559
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
3560

3561
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu,
3562 3563
					  access | PFERR_FETCH_MASK,
					  exception);
3564 3565
}

3566
int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
3567
			       gva_t addr, void *val, unsigned int bytes,
3568
			       struct x86_exception *exception)
3569
{
3570
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3571
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
3572

3573
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
3574
					  exception);
3575
}
3576
EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
3577

3578 3579
static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt,
				      gva_t addr, void *val, unsigned int bytes,
3580
				      struct x86_exception *exception)
3581
{
3582
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3583
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception);
3584 3585
}

N
Nadav Har'El 已提交
3586
int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
3587
				       gva_t addr, void *val,
3588
				       unsigned int bytes,
3589
				       struct x86_exception *exception)
3590
{
3591
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3592 3593 3594 3595
	void *data = val;
	int r = X86EMUL_CONTINUE;

	while (bytes) {
3596 3597
		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
							     PFERR_WRITE_MASK,
3598
							     exception);
3599 3600 3601 3602
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
		int ret;

3603
		if (gpa == UNMAPPED_GVA)
3604
			return X86EMUL_PROPAGATE_FAULT;
3605 3606
		ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
		if (ret < 0) {
3607
			r = X86EMUL_IO_NEEDED;
3608 3609 3610 3611 3612 3613 3614 3615 3616 3617
			goto out;
		}

		bytes -= towrite;
		data += towrite;
		addr += towrite;
	}
out:
	return r;
}
N
Nadav Har'El 已提交
3618
EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
3619

3620 3621 3622 3623 3624 3625
static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
				gpa_t *gpa, struct x86_exception *exception,
				bool write)
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;

3626 3627 3628 3629 3630
	if (vcpu_match_mmio_gva(vcpu, gva) &&
		  check_write_user_access(vcpu, write, access,
		  vcpu->arch.access)) {
		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
					(gva & (PAGE_SIZE - 1));
X
Xiao Guangrong 已提交
3631
		trace_vcpu_match_mmio(gva, *gpa, write, false);
3632 3633 3634
		return 1;
	}

3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646
	if (write)
		access |= PFERR_WRITE_MASK;

	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);

	if (*gpa == UNMAPPED_GVA)
		return -1;

	/* For APIC access vmexit */
	if ((*gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		return 1;

X
Xiao Guangrong 已提交
3647 3648
	if (vcpu_match_mmio_gpa(vcpu, *gpa)) {
		trace_vcpu_match_mmio(gva, *gpa, write, true);
3649
		return 1;
X
Xiao Guangrong 已提交
3650
	}
3651

3652 3653 3654
	return 0;
}

3655
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
3656
			const void *val, int bytes)
3657 3658 3659 3660
{
	int ret;

	ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
3661
	if (ret < 0)
3662
		return 0;
3663
	kvm_mmu_pte_write(vcpu, gpa, val, bytes);
3664 3665 3666
	return 1;
}

3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738
struct read_write_emulator_ops {
	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
				  int bytes);
	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
				  void *val, int bytes);
	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
			       int bytes, void *val);
	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
				    void *val, int bytes);
	bool write;
};

static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
{
	if (vcpu->mmio_read_completed) {
		memcpy(val, vcpu->mmio_data, bytes);
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
			       vcpu->mmio_phys_addr, *(u64 *)val);
		vcpu->mmio_read_completed = 0;
		return 1;
	}

	return 0;
}

static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			void *val, int bytes)
{
	return !kvm_read_guest(vcpu->kvm, gpa, val, bytes);
}

static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			 void *val, int bytes)
{
	return emulator_write_phys(vcpu, gpa, val, bytes);
}

static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
{
	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
	return vcpu_mmio_write(vcpu, gpa, bytes, val);
}

static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			  void *val, int bytes)
{
	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
	return X86EMUL_IO_NEEDED;
}

static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			   void *val, int bytes)
{
	memcpy(vcpu->mmio_data, val, bytes);
	memcpy(vcpu->run->mmio.data, vcpu->mmio_data, 8);
	return X86EMUL_CONTINUE;
}

static struct read_write_emulator_ops read_emultor = {
	.read_write_prepare = read_prepare,
	.read_write_emulate = read_emulate,
	.read_write_mmio = vcpu_mmio_read,
	.read_write_exit_mmio = read_exit_mmio,
};

static struct read_write_emulator_ops write_emultor = {
	.read_write_emulate = write_emulate,
	.read_write_mmio = write_mmio,
	.read_write_exit_mmio = write_exit_mmio,
	.write = true,
};

3739 3740 3741 3742 3743
static int emulator_read_write_onepage(unsigned long addr, void *val,
				       unsigned int bytes,
				       struct x86_exception *exception,
				       struct kvm_vcpu *vcpu,
				       struct read_write_emulator_ops *ops)
3744
{
3745 3746
	gpa_t gpa;
	int handled, ret;
3747 3748 3749 3750 3751
	bool write = ops->write;

	if (ops->read_write_prepare &&
		  ops->read_write_prepare(vcpu, val, bytes))
		return X86EMUL_CONTINUE;
3752

3753
	ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
3754

3755
	if (ret < 0)
3756 3757 3758
		return X86EMUL_PROPAGATE_FAULT;

	/* For APIC access vmexit */
3759
	if (ret)
3760 3761
		goto mmio;

3762
	if (ops->read_write_emulate(vcpu, gpa, val, bytes))
3763 3764 3765 3766 3767 3768
		return X86EMUL_CONTINUE;

mmio:
	/*
	 * Is this MMIO handled locally?
	 */
3769
	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
3770
	if (handled == bytes)
3771 3772
		return X86EMUL_CONTINUE;

3773 3774 3775 3776
	gpa += handled;
	bytes -= handled;
	val += handled;

3777
	vcpu->mmio_needed = 1;
3778 3779
	vcpu->run->exit_reason = KVM_EXIT_MMIO;
	vcpu->run->mmio.phys_addr = vcpu->mmio_phys_addr = gpa;
A
Avi Kivity 已提交
3780 3781
	vcpu->mmio_size = bytes;
	vcpu->run->mmio.len = min(vcpu->mmio_size, 8);
3782
	vcpu->run->mmio.is_write = vcpu->mmio_is_write = write;
A
Avi Kivity 已提交
3783
	vcpu->mmio_index = 0;
3784

3785
	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
3786 3787
}

3788 3789 3790 3791
int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr,
			void *val, unsigned int bytes,
			struct x86_exception *exception,
			struct read_write_emulator_ops *ops)
3792
{
3793 3794
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

3795 3796 3797 3798 3799
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
		int rc, now;

		now = -addr & ~PAGE_MASK;
3800 3801 3802
		rc = emulator_read_write_onepage(addr, val, now, exception,
						 vcpu, ops);

3803 3804 3805 3806 3807 3808
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
		val += now;
		bytes -= now;
	}
3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831

	return emulator_read_write_onepage(addr, val, bytes, exception,
					   vcpu, ops);
}

static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
				  unsigned long addr,
				  void *val,
				  unsigned int bytes,
				  struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, val, bytes,
				   exception, &read_emultor);
}

int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
			    unsigned long addr,
			    const void *val,
			    unsigned int bytes,
			    struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, (void *)val, bytes,
				   exception, &write_emultor);
3832 3833
}

3834 3835 3836 3837 3838 3839 3840
#define CMPXCHG_TYPE(t, ptr, old, new) \
	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))

#ifdef CONFIG_X86_64
#  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
#else
#  define CMPXCHG64(ptr, old, new) \
3841
	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
3842 3843
#endif

3844 3845
static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
				     unsigned long addr,
3846 3847 3848
				     const void *old,
				     const void *new,
				     unsigned int bytes,
3849
				     struct x86_exception *exception)
3850
{
3851
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3852 3853 3854 3855
	gpa_t gpa;
	struct page *page;
	char *kaddr;
	bool exchanged;
3856

3857 3858 3859
	/* guests cmpxchg8b have to be emulated atomically */
	if (bytes > 8 || (bytes & (bytes - 1)))
		goto emul_write;
3860

3861
	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
3862

3863 3864 3865
	if (gpa == UNMAPPED_GVA ||
	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		goto emul_write;
3866

3867 3868
	if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
		goto emul_write;
3869

3870
	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
3871 3872 3873 3874
	if (is_error_page(page)) {
		kvm_release_page_clean(page);
		goto emul_write;
	}
3875

3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892
	kaddr = kmap_atomic(page, KM_USER0);
	kaddr += offset_in_page(gpa);
	switch (bytes) {
	case 1:
		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
		break;
	case 2:
		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
		break;
	case 4:
		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
		break;
	case 8:
		exchanged = CMPXCHG64(kaddr, old, new);
		break;
	default:
		BUG();
3893
	}
3894 3895 3896 3897 3898 3899
	kunmap_atomic(kaddr, KM_USER0);
	kvm_release_page_dirty(page);

	if (!exchanged)
		return X86EMUL_CMPXCHG_FAILED;

3900
	kvm_mmu_pte_write(vcpu, gpa, new, bytes);
3901 3902

	return X86EMUL_CONTINUE;
3903

3904
emul_write:
3905
	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
3906

3907
	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
3908 3909
}

3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924
static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
{
	/* TODO: String I/O for in kernel device */
	int r;

	if (vcpu->arch.pio.in)
		r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
				    vcpu->arch.pio.size, pd);
	else
		r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
				     vcpu->arch.pio.port, vcpu->arch.pio.size,
				     pd);
	return r;
}

3925 3926 3927
static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
			       unsigned short port, void *val,
			       unsigned int count, bool in)
3928
{
3929
	trace_kvm_pio(!in, port, size, count);
3930 3931

	vcpu->arch.pio.port = port;
3932
	vcpu->arch.pio.in = in;
3933
	vcpu->arch.pio.count  = count;
3934 3935 3936
	vcpu->arch.pio.size = size;

	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
3937
		vcpu->arch.pio.count = 0;
3938 3939 3940 3941
		return 1;
	}

	vcpu->run->exit_reason = KVM_EXIT_IO;
3942
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
3943 3944 3945 3946 3947 3948 3949 3950
	vcpu->run->io.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = count;
	vcpu->run->io.port = port;

	return 0;
}

3951 3952 3953
static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
				    int size, unsigned short port, void *val,
				    unsigned int count)
3954
{
3955
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3956
	int ret;
3957

3958 3959
	if (vcpu->arch.pio.count)
		goto data_avail;
3960

3961 3962 3963 3964
	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
	if (ret) {
data_avail:
		memcpy(val, vcpu->arch.pio_data, size * count);
3965
		vcpu->arch.pio.count = 0;
3966 3967 3968 3969 3970 3971
		return 1;
	}

	return 0;
}

3972 3973 3974 3975 3976 3977 3978 3979 3980 3981
static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
				     int size, unsigned short port,
				     const void *val, unsigned int count)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	memcpy(vcpu->arch.pio_data, val, size * count);
	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
}

3982 3983 3984 3985 3986
static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	return kvm_x86_ops->get_segment_base(vcpu, seg);
}

3987
static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
3988
{
3989
	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
3990 3991
}

3992 3993 3994 3995 3996 3997
int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
	if (!need_emulate_wbinvd(vcpu))
		return X86EMUL_CONTINUE;

	if (kvm_x86_ops->has_wbinvd_exit()) {
3998 3999 4000
		int cpu = get_cpu();

		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
4001 4002
		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
				wbinvd_ipi, NULL, 1);
4003
		put_cpu();
4004
		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
4005 4006
	} else
		wbinvd();
4007 4008 4009 4010
	return X86EMUL_CONTINUE;
}
EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);

4011 4012 4013 4014 4015
static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
{
	kvm_emulate_wbinvd(emul_to_vcpu(ctxt));
}

4016
int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
4017
{
4018
	return _kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
4019 4020
}

4021
int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
4022
{
4023

4024
	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
4025 4026
}

4027
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
4028
{
4029
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
4030 4031
}

4032
static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
4033
{
4034
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4035 4036 4037 4038 4039 4040 4041 4042 4043 4044
	unsigned long value;

	switch (cr) {
	case 0:
		value = kvm_read_cr0(vcpu);
		break;
	case 2:
		value = vcpu->arch.cr2;
		break;
	case 3:
4045
		value = kvm_read_cr3(vcpu);
4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060
		break;
	case 4:
		value = kvm_read_cr4(vcpu);
		break;
	case 8:
		value = kvm_get_cr8(vcpu);
		break;
	default:
		vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
		return 0;
	}

	return value;
}

4061
static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
4062
{
4063
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4064 4065
	int res = 0;

4066 4067
	switch (cr) {
	case 0:
4068
		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
4069 4070 4071 4072 4073
		break;
	case 2:
		vcpu->arch.cr2 = val;
		break;
	case 3:
4074
		res = kvm_set_cr3(vcpu, val);
4075 4076
		break;
	case 4:
4077
		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
4078 4079
		break;
	case 8:
A
Andre Przywara 已提交
4080
		res = kvm_set_cr8(vcpu, val);
4081 4082 4083
		break;
	default:
		vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
4084
		res = -1;
4085
	}
4086 4087

	return res;
4088 4089
}

4090
static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
4091
{
4092
	return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
4093 4094
}

4095
static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
4096
{
4097
	kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
4098 4099
}

4100
static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
4101
{
4102
	kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
4103 4104
}

4105 4106 4107 4108 4109 4110 4111 4112 4113 4114
static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
}

static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
}

4115 4116
static unsigned long emulator_get_cached_segment_base(
	struct x86_emulate_ctxt *ctxt, int seg)
4117
{
4118
	return get_segment_base(emul_to_vcpu(ctxt), seg);
4119 4120
}

4121 4122 4123
static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
				 struct desc_struct *desc, u32 *base3,
				 int seg)
4124 4125 4126
{
	struct kvm_segment var;

4127
	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
4128
	*selector = var.selector;
4129 4130 4131 4132 4133 4134 4135 4136

	if (var.unusable)
		return false;

	if (var.g)
		var.limit >>= 12;
	set_desc_limit(desc, var.limit);
	set_desc_base(desc, (unsigned long)var.base);
4137 4138 4139 4140
#ifdef CONFIG_X86_64
	if (base3)
		*base3 = var.base >> 32;
#endif
4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152
	desc->type = var.type;
	desc->s = var.s;
	desc->dpl = var.dpl;
	desc->p = var.present;
	desc->avl = var.avl;
	desc->l = var.l;
	desc->d = var.db;
	desc->g = var.g;

	return true;
}

4153 4154 4155
static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
				 struct desc_struct *desc, u32 base3,
				 int seg)
4156
{
4157
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4158 4159
	struct kvm_segment var;

4160
	var.selector = selector;
4161
	var.base = get_desc_base(desc);
4162 4163 4164
#ifdef CONFIG_X86_64
	var.base |= ((u64)base3) << 32;
#endif
4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183
	var.limit = get_desc_limit(desc);
	if (desc->g)
		var.limit = (var.limit << 12) | 0xfff;
	var.type = desc->type;
	var.present = desc->p;
	var.dpl = desc->dpl;
	var.db = desc->d;
	var.s = desc->s;
	var.l = desc->l;
	var.g = desc->g;
	var.avl = desc->avl;
	var.present = desc->p;
	var.unusable = !var.present;
	var.padding = 0;

	kvm_set_segment(vcpu, &var, seg);
	return;
}

4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195
static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 *pdata)
{
	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
}

static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 data)
{
	return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
}

4196 4197 4198 4199 4200 4201
static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
			     u32 pmc, u64 *pdata)
{
	return kvm_pmu_read_pmc(emul_to_vcpu(ctxt), pmc, pdata);
}

4202 4203 4204 4205 4206
static void emulator_halt(struct x86_emulate_ctxt *ctxt)
{
	emul_to_vcpu(ctxt)->arch.halt_request = 1;
}

4207 4208 4209
static void emulator_get_fpu(struct x86_emulate_ctxt *ctxt)
{
	preempt_disable();
4210
	kvm_load_guest_fpu(emul_to_vcpu(ctxt));
4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222
	/*
	 * CR0.TS may reference the host fpu state, not the guest fpu state,
	 * so it may be clear at this point.
	 */
	clts();
}

static void emulator_put_fpu(struct x86_emulate_ctxt *ctxt)
{
	preempt_enable();
}

4223
static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
4224
			      struct x86_instruction_info *info,
4225 4226
			      enum x86_intercept_stage stage)
{
4227
	return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
4228 4229
}

4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251
static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
			       u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
{
	struct kvm_cpuid_entry2 *cpuid = NULL;

	if (eax && ecx)
		cpuid = kvm_find_cpuid_entry(emul_to_vcpu(ctxt),
					    *eax, *ecx);

	if (cpuid) {
		*eax = cpuid->eax;
		*ecx = cpuid->ecx;
		if (ebx)
			*ebx = cpuid->ebx;
		if (edx)
			*edx = cpuid->edx;
		return true;
	}

	return false;
}

4252
static struct x86_emulate_ops emulate_ops = {
4253
	.read_std            = kvm_read_guest_virt_system,
4254
	.write_std           = kvm_write_guest_virt_system,
4255
	.fetch               = kvm_fetch_guest_virt,
4256 4257 4258
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
4259
	.invlpg              = emulator_invlpg,
4260 4261
	.pio_in_emulated     = emulator_pio_in_emulated,
	.pio_out_emulated    = emulator_pio_out_emulated,
4262 4263
	.get_segment         = emulator_get_segment,
	.set_segment         = emulator_set_segment,
4264
	.get_cached_segment_base = emulator_get_cached_segment_base,
4265
	.get_gdt             = emulator_get_gdt,
4266
	.get_idt	     = emulator_get_idt,
4267 4268
	.set_gdt             = emulator_set_gdt,
	.set_idt	     = emulator_set_idt,
4269 4270
	.get_cr              = emulator_get_cr,
	.set_cr              = emulator_set_cr,
4271
	.cpl                 = emulator_get_cpl,
4272 4273
	.get_dr              = emulator_get_dr,
	.set_dr              = emulator_set_dr,
4274 4275
	.set_msr             = emulator_set_msr,
	.get_msr             = emulator_get_msr,
4276
	.read_pmc            = emulator_read_pmc,
4277
	.halt                = emulator_halt,
4278
	.wbinvd              = emulator_wbinvd,
4279
	.fix_hypercall       = emulator_fix_hypercall,
4280 4281
	.get_fpu             = emulator_get_fpu,
	.put_fpu             = emulator_put_fpu,
4282
	.intercept           = emulator_intercept,
4283
	.get_cpuid           = emulator_get_cpuid,
4284 4285
};

4286 4287 4288 4289 4290 4291 4292 4293
static void cache_all_regs(struct kvm_vcpu *vcpu)
{
	kvm_register_read(vcpu, VCPU_REGS_RAX);
	kvm_register_read(vcpu, VCPU_REGS_RSP);
	kvm_register_read(vcpu, VCPU_REGS_RIP);
	vcpu->arch.regs_dirty = ~0;
}

4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307
static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
{
	u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask);
	/*
	 * an sti; sti; sequence only disable interrupts for the first
	 * instruction. So, if the last instruction, be it emulated or
	 * not, left the system with the INT_STI flag enabled, it
	 * means that the last instruction is an sti. We should not
	 * leave the flag on in this case. The same goes for mov ss
	 */
	if (!(int_shadow & mask))
		kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
}

4308 4309 4310
static void inject_emulated_exception(struct kvm_vcpu *vcpu)
{
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4311
	if (ctxt->exception.vector == PF_VECTOR)
4312
		kvm_propagate_fault(vcpu, &ctxt->exception);
4313 4314 4315
	else if (ctxt->exception.error_code_valid)
		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
				      ctxt->exception.error_code);
4316
	else
4317
		kvm_queue_exception(vcpu, ctxt->exception.vector);
4318 4319
}

4320
static void init_decode_cache(struct x86_emulate_ctxt *ctxt,
4321 4322
			      const unsigned long *regs)
{
4323 4324 4325
	memset(&ctxt->twobyte, 0,
	       (void *)&ctxt->regs - (void *)&ctxt->twobyte);
	memcpy(ctxt->regs, regs, sizeof(ctxt->regs));
4326

4327 4328 4329 4330 4331 4332
	ctxt->fetch.start = 0;
	ctxt->fetch.end = 0;
	ctxt->io_read.pos = 0;
	ctxt->io_read.end = 0;
	ctxt->mem_read.pos = 0;
	ctxt->mem_read.end = 0;
4333 4334
}

4335 4336
static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
{
4337
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4338 4339
	int cs_db, cs_l;

4340 4341 4342 4343 4344 4345
	/*
	 * TODO: fix emulate.c to use guest_read/write_register
	 * instead of direct ->regs accesses, can save hundred cycles
	 * on Intel for instructions that don't read/change RSP, for
	 * for example.
	 */
4346 4347 4348 4349
	cache_all_regs(vcpu);

	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);

4350 4351 4352 4353 4354 4355 4356 4357 4358
	ctxt->eflags = kvm_get_rflags(vcpu);
	ctxt->eip = kvm_rip_read(vcpu);
	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
		     cs_l				? X86EMUL_MODE_PROT64 :
		     cs_db				? X86EMUL_MODE_PROT32 :
							  X86EMUL_MODE_PROT16;
	ctxt->guest_mode = is_guest_mode(vcpu);

4359
	init_decode_cache(ctxt, vcpu->arch.regs);
4360
	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
4361 4362
}

4363
int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
4364
{
4365
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4366 4367 4368 4369
	int ret;

	init_emulate_ctxt(vcpu);

4370 4371 4372
	ctxt->op_bytes = 2;
	ctxt->ad_bytes = 2;
	ctxt->_eip = ctxt->eip + inc_eip;
4373
	ret = emulate_int_real(ctxt, irq);
4374 4375 4376 4377

	if (ret != X86EMUL_CONTINUE)
		return EMULATE_FAIL;

4378 4379
	ctxt->eip = ctxt->_eip;
	memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
4380 4381
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
4382 4383

	if (irq == NMI_VECTOR)
A
Avi Kivity 已提交
4384
		vcpu->arch.nmi_pending = 0;
4385 4386 4387 4388 4389 4390 4391
	else
		vcpu->arch.interrupt.pending = false;

	return EMULATE_DONE;
}
EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);

4392 4393
static int handle_emulation_failure(struct kvm_vcpu *vcpu)
{
4394 4395
	int r = EMULATE_DONE;

4396 4397
	++vcpu->stat.insn_emulation_fail;
	trace_kvm_emulate_insn_failed(vcpu);
4398 4399 4400 4401 4402 4403
	if (!is_guest_mode(vcpu)) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
		r = EMULATE_FAIL;
	}
4404
	kvm_queue_exception(vcpu, UD_VECTOR);
4405 4406

	return r;
4407 4408
}

4409 4410 4411 4412
static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
{
	gpa_t gpa;

4413 4414 4415
	if (tdp_enabled)
		return false;

4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434
	/*
	 * if emulation was due to access to shadowed page table
	 * and it failed try to unshadow page and re-entetr the
	 * guest to let CPU execute the instruction.
	 */
	if (kvm_mmu_unprotect_page_virt(vcpu, gva))
		return true;

	gpa = kvm_mmu_gva_to_gpa_system(vcpu, gva, NULL);

	if (gpa == UNMAPPED_GVA)
		return true; /* let cpu generate fault */

	if (!kvm_is_error_hva(gfn_to_hva(vcpu->kvm, gpa >> PAGE_SHIFT)))
		return true;

	return false;
}

4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478
static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
			      unsigned long cr2,  int emulation_type)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	unsigned long last_retry_eip, last_retry_addr, gpa = cr2;

	last_retry_eip = vcpu->arch.last_retry_eip;
	last_retry_addr = vcpu->arch.last_retry_addr;

	/*
	 * If the emulation is caused by #PF and it is non-page_table
	 * writing instruction, it means the VM-EXIT is caused by shadow
	 * page protected, we can zap the shadow page and retry this
	 * instruction directly.
	 *
	 * Note: if the guest uses a non-page-table modifying instruction
	 * on the PDE that points to the instruction, then we will unmap
	 * the instruction and go to an infinite loop. So, we cache the
	 * last retried eip and the last fault address, if we meet the eip
	 * and the address again, we can break out of the potential infinite
	 * loop.
	 */
	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;

	if (!(emulation_type & EMULTYPE_RETRY))
		return false;

	if (x86_page_table_writing_insn(ctxt))
		return false;

	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2)
		return false;

	vcpu->arch.last_retry_eip = ctxt->eip;
	vcpu->arch.last_retry_addr = cr2;

	if (!vcpu->arch.mmu.direct_map)
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);

	kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);

	return true;
}

4479 4480
int x86_emulate_instruction(struct kvm_vcpu *vcpu,
			    unsigned long cr2,
4481 4482 4483
			    int emulation_type,
			    void *insn,
			    int insn_len)
4484
{
4485
	int r;
4486
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4487
	bool writeback = true;
4488

4489
	kvm_clear_exception_queue(vcpu);
G
Gleb Natapov 已提交
4490

4491
	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
4492
		init_emulate_ctxt(vcpu);
4493 4494 4495
		ctxt->interruptibility = 0;
		ctxt->have_exception = false;
		ctxt->perm_ok = false;
4496

4497
		ctxt->only_vendor_specific_insn
4498 4499
			= emulation_type & EMULTYPE_TRAP_UD;

4500
		r = x86_decode_insn(ctxt, insn, insn_len);
4501

A
Avi Kivity 已提交
4502
		trace_kvm_emulate_insn_start(vcpu);
4503
		++vcpu->stat.insn_emulation;
4504
		if (r != EMULATION_OK)  {
4505 4506
			if (emulation_type & EMULTYPE_TRAP_UD)
				return EMULATE_FAIL;
4507
			if (reexecute_instruction(vcpu, cr2))
4508
				return EMULATE_DONE;
4509 4510 4511
			if (emulation_type & EMULTYPE_SKIP)
				return EMULATE_FAIL;
			return handle_emulation_failure(vcpu);
4512 4513 4514
		}
	}

4515
	if (emulation_type & EMULTYPE_SKIP) {
4516
		kvm_rip_write(vcpu, ctxt->_eip);
4517 4518 4519
		return EMULATE_DONE;
	}

4520 4521 4522
	if (retry_instruction(ctxt, cr2, emulation_type))
		return EMULATE_DONE;

4523
	/* this is needed for vmware backdoor interface to work since it
4524
	   changes registers values  during IO operation */
4525 4526
	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
4527
		memcpy(ctxt->regs, vcpu->arch.regs, sizeof ctxt->regs);
4528
	}
4529

4530
restart:
4531
	r = x86_emulate_insn(ctxt);
4532

4533 4534 4535
	if (r == EMULATION_INTERCEPTED)
		return EMULATE_DONE;

4536
	if (r == EMULATION_FAILED) {
4537
		if (reexecute_instruction(vcpu, cr2))
4538 4539
			return EMULATE_DONE;

4540
		return handle_emulation_failure(vcpu);
4541 4542
	}

4543
	if (ctxt->have_exception) {
4544
		inject_emulated_exception(vcpu);
4545 4546
		r = EMULATE_DONE;
	} else if (vcpu->arch.pio.count) {
4547 4548
		if (!vcpu->arch.pio.in)
			vcpu->arch.pio.count = 0;
4549 4550
		else
			writeback = false;
4551
		r = EMULATE_DO_MMIO;
4552 4553 4554
	} else if (vcpu->mmio_needed) {
		if (!vcpu->mmio_is_write)
			writeback = false;
4555
		r = EMULATE_DO_MMIO;
4556
	} else if (r == EMULATION_RESTART)
4557
		goto restart;
4558 4559
	else
		r = EMULATE_DONE;
4560

4561
	if (writeback) {
4562 4563
		toggle_interruptibility(vcpu, ctxt->interruptibility);
		kvm_set_rflags(vcpu, ctxt->eflags);
4564
		kvm_make_request(KVM_REQ_EVENT, vcpu);
4565
		memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
4566
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
4567
		kvm_rip_write(vcpu, ctxt->eip);
4568 4569
	} else
		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
4570 4571

	return r;
4572
}
4573
EXPORT_SYMBOL_GPL(x86_emulate_instruction);
4574

4575
int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
4576
{
4577
	unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX);
4578 4579
	int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
					    size, port, &val, 1);
4580
	/* do not return to emulator after return from userspace */
4581
	vcpu->arch.pio.count = 0;
4582 4583
	return ret;
}
4584
EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
4585

4586 4587
static void tsc_bad(void *info)
{
T
Tejun Heo 已提交
4588
	__this_cpu_write(cpu_tsc_khz, 0);
4589 4590 4591
}

static void tsc_khz_changed(void *data)
4592
{
4593 4594 4595 4596 4597 4598 4599 4600 4601
	struct cpufreq_freqs *freq = data;
	unsigned long khz = 0;

	if (data)
		khz = freq->new;
	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		khz = cpufreq_quick_get(raw_smp_processor_id());
	if (!khz)
		khz = tsc_khz;
T
Tejun Heo 已提交
4602
	__this_cpu_write(cpu_tsc_khz, khz);
4603 4604 4605 4606 4607 4608 4609 4610 4611 4612
}

static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
				     void *data)
{
	struct cpufreq_freqs *freq = data;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i, send_ipi = 0;

4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651
	/*
	 * We allow guests to temporarily run on slowing clocks,
	 * provided we notify them after, or to run on accelerating
	 * clocks, provided we notify them before.  Thus time never
	 * goes backwards.
	 *
	 * However, we have a problem.  We can't atomically update
	 * the frequency of a given CPU from this function; it is
	 * merely a notifier, which can be called from any CPU.
	 * Changing the TSC frequency at arbitrary points in time
	 * requires a recomputation of local variables related to
	 * the TSC for each VCPU.  We must flag these local variables
	 * to be updated and be sure the update takes place with the
	 * new frequency before any guests proceed.
	 *
	 * Unfortunately, the combination of hotplug CPU and frequency
	 * change creates an intractable locking scenario; the order
	 * of when these callouts happen is undefined with respect to
	 * CPU hotplug, and they can race with each other.  As such,
	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
	 * undefined; you can actually have a CPU frequency change take
	 * place in between the computation of X and the setting of the
	 * variable.  To protect against this problem, all updates of
	 * the per_cpu tsc_khz variable are done in an interrupt
	 * protected IPI, and all callers wishing to update the value
	 * must wait for a synchronous IPI to complete (which is trivial
	 * if the caller is on the CPU already).  This establishes the
	 * necessary total order on variable updates.
	 *
	 * Note that because a guest time update may take place
	 * anytime after the setting of the VCPU's request bit, the
	 * correct TSC value must be set before the request.  However,
	 * to ensure the update actually makes it to any guest which
	 * starts running in hardware virtualization between the set
	 * and the acquisition of the spinlock, we must also ping the
	 * CPU after setting the request bit.
	 *
	 */

4652 4653 4654 4655
	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
		return 0;
	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
		return 0;
4656 4657

	smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
4658

4659
	raw_spin_lock(&kvm_lock);
4660
	list_for_each_entry(kvm, &vm_list, vm_list) {
4661
		kvm_for_each_vcpu(i, vcpu, kvm) {
4662 4663
			if (vcpu->cpu != freq->cpu)
				continue;
Z
Zachary Amsden 已提交
4664
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
4665
			if (vcpu->cpu != smp_processor_id())
4666
				send_ipi = 1;
4667 4668
		}
	}
4669
	raw_spin_unlock(&kvm_lock);
4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683

	if (freq->old < freq->new && send_ipi) {
		/*
		 * We upscale the frequency.  Must make the guest
		 * doesn't see old kvmclock values while running with
		 * the new frequency, otherwise we risk the guest sees
		 * time go backwards.
		 *
		 * In case we update the frequency for another cpu
		 * (which might be in guest context) send an interrupt
		 * to kick the cpu out of guest context.  Next time
		 * guest context is entered kvmclock will be updated,
		 * so the guest will not see stale values.
		 */
4684
		smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
4685 4686 4687 4688 4689
	}
	return 0;
}

static struct notifier_block kvmclock_cpufreq_notifier_block = {
4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712
	.notifier_call  = kvmclock_cpufreq_notifier
};

static int kvmclock_cpu_notifier(struct notifier_block *nfb,
					unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned long)hcpu;

	switch (action) {
		case CPU_ONLINE:
		case CPU_DOWN_FAILED:
			smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
			break;
		case CPU_DOWN_PREPARE:
			smp_call_function_single(cpu, tsc_bad, NULL, 1);
			break;
	}
	return NOTIFY_OK;
}

static struct notifier_block kvmclock_cpu_notifier_block = {
	.notifier_call  = kvmclock_cpu_notifier,
	.priority = -INT_MAX
4713 4714
};

4715 4716 4717 4718
static void kvm_timer_init(void)
{
	int cpu;

Z
Zachary Amsden 已提交
4719
	max_tsc_khz = tsc_khz;
4720
	register_hotcpu_notifier(&kvmclock_cpu_notifier_block);
4721
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
Z
Zachary Amsden 已提交
4722 4723 4724
#ifdef CONFIG_CPU_FREQ
		struct cpufreq_policy policy;
		memset(&policy, 0, sizeof(policy));
4725 4726
		cpu = get_cpu();
		cpufreq_get_policy(&policy, cpu);
Z
Zachary Amsden 已提交
4727 4728
		if (policy.cpuinfo.max_freq)
			max_tsc_khz = policy.cpuinfo.max_freq;
4729
		put_cpu();
Z
Zachary Amsden 已提交
4730
#endif
4731 4732 4733
		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
					  CPUFREQ_TRANSITION_NOTIFIER);
	}
Z
Zachary Amsden 已提交
4734
	pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz);
4735 4736
	for_each_online_cpu(cpu)
		smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
4737 4738
}

4739 4740
static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);

4741
int kvm_is_in_guest(void)
4742
{
4743
	return __this_cpu_read(current_vcpu) != NULL;
4744 4745 4746 4747 4748
}

static int kvm_is_user_mode(void)
{
	int user_mode = 3;
4749

4750 4751
	if (__this_cpu_read(current_vcpu))
		user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
4752

4753 4754 4755 4756 4757 4758
	return user_mode != 0;
}

static unsigned long kvm_get_guest_ip(void)
{
	unsigned long ip = 0;
4759

4760 4761
	if (__this_cpu_read(current_vcpu))
		ip = kvm_rip_read(__this_cpu_read(current_vcpu));
4762

4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773
	return ip;
}

static struct perf_guest_info_callbacks kvm_guest_cbs = {
	.is_in_guest		= kvm_is_in_guest,
	.is_user_mode		= kvm_is_user_mode,
	.get_guest_ip		= kvm_get_guest_ip,
};

void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
{
4774
	__this_cpu_write(current_vcpu, vcpu);
4775 4776 4777 4778 4779
}
EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);

void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
{
4780
	__this_cpu_write(current_vcpu, NULL);
4781 4782 4783
}
EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);

4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807
static void kvm_set_mmio_spte_mask(void)
{
	u64 mask;
	int maxphyaddr = boot_cpu_data.x86_phys_bits;

	/*
	 * Set the reserved bits and the present bit of an paging-structure
	 * entry to generate page fault with PFER.RSV = 1.
	 */
	mask = ((1ull << (62 - maxphyaddr + 1)) - 1) << maxphyaddr;
	mask |= 1ull;

#ifdef CONFIG_X86_64
	/*
	 * If reserved bit is not supported, clear the present bit to disable
	 * mmio page fault.
	 */
	if (maxphyaddr == 52)
		mask &= ~1ull;
#endif

	kvm_mmu_set_mmio_spte_mask(mask);
}

4808
int kvm_arch_init(void *opaque)
4809
{
4810
	int r;
4811 4812 4813 4814
	struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;

	if (kvm_x86_ops) {
		printk(KERN_ERR "kvm: already loaded the other module\n");
4815 4816
		r = -EEXIST;
		goto out;
4817 4818 4819 4820
	}

	if (!ops->cpu_has_kvm_support()) {
		printk(KERN_ERR "kvm: no hardware support\n");
4821 4822
		r = -EOPNOTSUPP;
		goto out;
4823 4824 4825
	}
	if (ops->disabled_by_bios()) {
		printk(KERN_ERR "kvm: disabled by bios\n");
4826 4827
		r = -EOPNOTSUPP;
		goto out;
4828 4829
	}

4830 4831 4832 4833
	r = kvm_mmu_module_init();
	if (r)
		goto out;

4834
	kvm_set_mmio_spte_mask();
4835 4836
	kvm_init_msr_list();

4837
	kvm_x86_ops = ops;
S
Sheng Yang 已提交
4838
	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
4839
			PT_DIRTY_MASK, PT64_NX_MASK, 0);
4840

4841
	kvm_timer_init();
4842

4843 4844
	perf_register_guest_info_callbacks(&kvm_guest_cbs);

4845 4846 4847
	if (cpu_has_xsave)
		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);

4848
	return 0;
4849 4850 4851

out:
	return r;
4852
}
4853

4854 4855
void kvm_arch_exit(void)
{
4856 4857
	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);

4858 4859 4860
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
					    CPUFREQ_TRANSITION_NOTIFIER);
4861
	unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block);
4862
	kvm_x86_ops = NULL;
4863 4864
	kvm_mmu_module_exit();
}
4865

4866 4867 4868 4869
int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
	++vcpu->stat.halt_exits;
	if (irqchip_in_kernel(vcpu->kvm)) {
4870
		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
4871 4872 4873 4874 4875 4876 4877 4878
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
}
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889
int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
{
	u64 param, ingpa, outgpa, ret;
	uint16_t code, rep_idx, rep_cnt, res = HV_STATUS_SUCCESS, rep_done = 0;
	bool fast, longmode;
	int cs_db, cs_l;

	/*
	 * hypercall generates UD from non zero cpl and real mode
	 * per HYPER-V spec
	 */
4890
	if (kvm_x86_ops->get_cpl(vcpu) != 0 || !is_protmode(vcpu)) {
4891 4892 4893 4894 4895 4896 4897 4898
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 0;
	}

	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
	longmode = is_long_mode(vcpu) && cs_l == 1;

	if (!longmode) {
4899 4900 4901 4902 4903 4904
		param = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDX) << 32) |
			(kvm_register_read(vcpu, VCPU_REGS_RAX) & 0xffffffff);
		ingpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RBX) << 32) |
			(kvm_register_read(vcpu, VCPU_REGS_RCX) & 0xffffffff);
		outgpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDI) << 32) |
			(kvm_register_read(vcpu, VCPU_REGS_RSI) & 0xffffffff);
4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920
	}
#ifdef CONFIG_X86_64
	else {
		param = kvm_register_read(vcpu, VCPU_REGS_RCX);
		ingpa = kvm_register_read(vcpu, VCPU_REGS_RDX);
		outgpa = kvm_register_read(vcpu, VCPU_REGS_R8);
	}
#endif

	code = param & 0xffff;
	fast = (param >> 16) & 0x1;
	rep_cnt = (param >> 32) & 0xfff;
	rep_idx = (param >> 48) & 0xfff;

	trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa);

4921 4922 4923 4924 4925 4926 4927 4928
	switch (code) {
	case HV_X64_HV_NOTIFY_LONG_SPIN_WAIT:
		kvm_vcpu_on_spin(vcpu);
		break;
	default:
		res = HV_STATUS_INVALID_HYPERCALL_CODE;
		break;
	}
4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940

	ret = res | (((u64)rep_done & 0xfff) << 32);
	if (longmode) {
		kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
	} else {
		kvm_register_write(vcpu, VCPU_REGS_RDX, ret >> 32);
		kvm_register_write(vcpu, VCPU_REGS_RAX, ret & 0xffffffff);
	}

	return 1;
}

4941 4942 4943
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
{
	unsigned long nr, a0, a1, a2, a3, ret;
4944
	int r = 1;
4945

4946 4947 4948
	if (kvm_hv_hypercall_enabled(vcpu->kvm))
		return kvm_hv_hypercall(vcpu);

4949 4950 4951 4952 4953
	nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
	a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
	a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
	a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
	a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
4954

4955
	trace_kvm_hypercall(nr, a0, a1, a2, a3);
F
Feng (Eric) Liu 已提交
4956

4957 4958 4959 4960 4961 4962 4963 4964
	if (!is_long_mode(vcpu)) {
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
	}

4965 4966 4967 4968 4969
	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
		ret = -KVM_EPERM;
		goto out;
	}

4970
	switch (nr) {
A
Avi Kivity 已提交
4971 4972 4973
	case KVM_HC_VAPIC_POLL_IRQ:
		ret = 0;
		break;
4974 4975 4976 4977
	default:
		ret = -KVM_ENOSYS;
		break;
	}
4978
out:
4979
	kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
A
Amit Shah 已提交
4980
	++vcpu->stat.hypercalls;
4981
	return r;
4982 4983 4984
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

4985
int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
4986
{
4987
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4988
	char instruction[3];
4989
	unsigned long rip = kvm_rip_read(vcpu);
4990 4991 4992 4993 4994 4995 4996 4997 4998 4999

	/*
	 * Blow out the MMU to ensure that no other VCPU has an active mapping
	 * to ensure that the updated hypercall appears atomically across all
	 * VCPUs.
	 */
	kvm_mmu_zap_all(vcpu->kvm);

	kvm_x86_ops->patch_hypercall(vcpu, instruction);

5000
	return emulator_write_emulated(ctxt, rip, instruction, 3, NULL);
5001 5002
}

5003 5004 5005 5006 5007 5008
/*
 * Check if userspace requested an interrupt window, and that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
A
Avi Kivity 已提交
5009
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
5010
{
5011
	return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
A
Avi Kivity 已提交
5012
		vcpu->run->request_interrupt_window &&
5013
		kvm_arch_interrupt_allowed(vcpu));
5014 5015
}

A
Avi Kivity 已提交
5016
static void post_kvm_run_save(struct kvm_vcpu *vcpu)
5017
{
A
Avi Kivity 已提交
5018 5019
	struct kvm_run *kvm_run = vcpu->run;

5020
	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
5021
	kvm_run->cr8 = kvm_get_cr8(vcpu);
5022
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
5023
	if (irqchip_in_kernel(vcpu->kvm))
5024
		kvm_run->ready_for_interrupt_injection = 1;
5025
	else
5026
		kvm_run->ready_for_interrupt_injection =
5027 5028 5029
			kvm_arch_interrupt_allowed(vcpu) &&
			!kvm_cpu_has_interrupt(vcpu) &&
			!kvm_event_needs_reinjection(vcpu);
5030 5031
}

A
Avi Kivity 已提交
5032 5033 5034 5035 5036 5037 5038 5039 5040
static void vapic_enter(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	struct page *page;

	if (!apic || !apic->vapic_addr)
		return;

	page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
5041 5042

	vcpu->arch.apic->vapic_page = page;
A
Avi Kivity 已提交
5043 5044 5045 5046 5047
}

static void vapic_exit(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
5048
	int idx;
A
Avi Kivity 已提交
5049 5050 5051 5052

	if (!apic || !apic->vapic_addr)
		return;

5053
	idx = srcu_read_lock(&vcpu->kvm->srcu);
A
Avi Kivity 已提交
5054 5055
	kvm_release_page_dirty(apic->vapic_page);
	mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
5056
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
A
Avi Kivity 已提交
5057 5058
}

5059 5060 5061 5062 5063 5064 5065
static void update_cr8_intercept(struct kvm_vcpu *vcpu)
{
	int max_irr, tpr;

	if (!kvm_x86_ops->update_cr8_intercept)
		return;

5066 5067 5068
	if (!vcpu->arch.apic)
		return;

5069 5070 5071 5072
	if (!vcpu->arch.apic->vapic_addr)
		max_irr = kvm_lapic_find_highest_irr(vcpu);
	else
		max_irr = -1;
5073 5074 5075 5076 5077 5078 5079 5080 5081

	if (max_irr != -1)
		max_irr >>= 4;

	tpr = kvm_lapic_get_cr8(vcpu);

	kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
}

A
Avi Kivity 已提交
5082
static void inject_pending_event(struct kvm_vcpu *vcpu)
5083 5084
{
	/* try to reinject previous events if any */
5085
	if (vcpu->arch.exception.pending) {
A
Avi Kivity 已提交
5086 5087 5088
		trace_kvm_inj_exception(vcpu->arch.exception.nr,
					vcpu->arch.exception.has_error_code,
					vcpu->arch.exception.error_code);
5089 5090
		kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
					  vcpu->arch.exception.has_error_code,
5091 5092
					  vcpu->arch.exception.error_code,
					  vcpu->arch.exception.reinject);
5093 5094 5095
		return;
	}

5096 5097 5098 5099 5100 5101
	if (vcpu->arch.nmi_injected) {
		kvm_x86_ops->set_nmi(vcpu);
		return;
	}

	if (vcpu->arch.interrupt.pending) {
5102
		kvm_x86_ops->set_irq(vcpu);
5103 5104 5105 5106 5107 5108
		return;
	}

	/* try to inject new event if pending */
	if (vcpu->arch.nmi_pending) {
		if (kvm_x86_ops->nmi_allowed(vcpu)) {
A
Avi Kivity 已提交
5109
			--vcpu->arch.nmi_pending;
5110 5111 5112 5113 5114
			vcpu->arch.nmi_injected = true;
			kvm_x86_ops->set_nmi(vcpu);
		}
	} else if (kvm_cpu_has_interrupt(vcpu)) {
		if (kvm_x86_ops->interrupt_allowed(vcpu)) {
5115 5116 5117
			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
					    false);
			kvm_x86_ops->set_irq(vcpu);
5118 5119 5120 5121
		}
	}
}

5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140
static void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
{
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
			!vcpu->guest_xcr0_loaded) {
		/* kvm_set_xcr() also depends on this */
		xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
		vcpu->guest_xcr0_loaded = 1;
	}
}

static void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
{
	if (vcpu->guest_xcr0_loaded) {
		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
		vcpu->guest_xcr0_loaded = 0;
	}
}

A
Avi Kivity 已提交
5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157
static void process_nmi(struct kvm_vcpu *vcpu)
{
	unsigned limit = 2;

	/*
	 * x86 is limited to one NMI running, and one NMI pending after it.
	 * If an NMI is already in progress, limit further NMIs to just one.
	 * Otherwise, allow two (and we'll inject the first one immediately).
	 */
	if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
		limit = 1;

	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

A
Avi Kivity 已提交
5158
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
5159 5160
{
	int r;
5161
	bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
A
Avi Kivity 已提交
5162
		vcpu->run->request_interrupt_window;
5163
	bool req_immediate_exit = 0;
5164

5165
	if (vcpu->requests) {
5166
		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
5167
			kvm_mmu_unload(vcpu);
5168
		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
M
Marcelo Tosatti 已提交
5169
			__kvm_migrate_timers(vcpu);
Z
Zachary Amsden 已提交
5170 5171
		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
			r = kvm_guest_time_update(vcpu);
5172 5173 5174
			if (unlikely(r))
				goto out;
		}
5175
		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
5176
			kvm_mmu_sync_roots(vcpu);
5177
		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
5178
			kvm_x86_ops->tlb_flush(vcpu);
5179
		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
A
Avi Kivity 已提交
5180
			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
A
Avi Kivity 已提交
5181 5182 5183
			r = 0;
			goto out;
		}
5184
		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
A
Avi Kivity 已提交
5185
			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
J
Joerg Roedel 已提交
5186 5187 5188
			r = 0;
			goto out;
		}
5189
		if (kvm_check_request(KVM_REQ_DEACTIVATE_FPU, vcpu)) {
5190 5191 5192
			vcpu->fpu_active = 0;
			kvm_x86_ops->fpu_deactivate(vcpu);
		}
5193 5194 5195 5196 5197 5198
		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
			/* Page is swapped out. Do synthetic halt */
			vcpu->arch.apf.halted = true;
			r = 1;
			goto out;
		}
G
Glauber Costa 已提交
5199 5200
		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
			record_steal_time(vcpu);
A
Avi Kivity 已提交
5201 5202
		if (kvm_check_request(KVM_REQ_NMI, vcpu))
			process_nmi(vcpu);
5203 5204
		req_immediate_exit =
			kvm_check_request(KVM_REQ_IMMEDIATE_EXIT, vcpu);
5205 5206 5207 5208
		if (kvm_check_request(KVM_REQ_PMU, vcpu))
			kvm_handle_pmu_event(vcpu);
		if (kvm_check_request(KVM_REQ_PMI, vcpu))
			kvm_deliver_pmi(vcpu);
5209
	}
A
Avi Kivity 已提交
5210

5211 5212 5213 5214
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		goto out;

A
Avi Kivity 已提交
5215 5216 5217 5218
	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
		inject_pending_event(vcpu);

		/* enable NMI/IRQ window open exits if needed */
A
Avi Kivity 已提交
5219
		if (vcpu->arch.nmi_pending)
A
Avi Kivity 已提交
5220 5221 5222 5223 5224 5225 5226 5227 5228 5229
			kvm_x86_ops->enable_nmi_window(vcpu);
		else if (kvm_cpu_has_interrupt(vcpu) || req_int_win)
			kvm_x86_ops->enable_irq_window(vcpu);

		if (kvm_lapic_enabled(vcpu)) {
			update_cr8_intercept(vcpu);
			kvm_lapic_sync_to_vapic(vcpu);
		}
	}

5230 5231 5232
	preempt_disable();

	kvm_x86_ops->prepare_guest_switch(vcpu);
5233 5234
	if (vcpu->fpu_active)
		kvm_load_guest_fpu(vcpu);
5235
	kvm_load_guest_xcr0(vcpu);
5236

5237 5238 5239 5240 5241 5242
	vcpu->mode = IN_GUEST_MODE;

	/* We should set ->mode before check ->requests,
	 * see the comment in make_all_cpus_request.
	 */
	smp_mb();
5243

A
Avi Kivity 已提交
5244
	local_irq_disable();
5245

5246
	if (vcpu->mode == EXITING_GUEST_MODE || vcpu->requests
A
Avi Kivity 已提交
5247
	    || need_resched() || signal_pending(current)) {
5248
		vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
5249
		smp_wmb();
5250 5251
		local_irq_enable();
		preempt_enable();
A
Avi Kivity 已提交
5252
		kvm_x86_ops->cancel_injection(vcpu);
5253 5254 5255 5256
		r = 1;
		goto out;
	}

5257
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
5258

5259 5260 5261
	if (req_immediate_exit)
		smp_send_reschedule(vcpu->cpu);

5262 5263
	kvm_guest_enter();

5264 5265 5266 5267 5268 5269 5270
	if (unlikely(vcpu->arch.switch_db_regs)) {
		set_debugreg(0, 7);
		set_debugreg(vcpu->arch.eff_db[0], 0);
		set_debugreg(vcpu->arch.eff_db[1], 1);
		set_debugreg(vcpu->arch.eff_db[2], 2);
		set_debugreg(vcpu->arch.eff_db[3], 3);
	}
5271

5272
	trace_kvm_entry(vcpu->vcpu_id);
A
Avi Kivity 已提交
5273
	kvm_x86_ops->run(vcpu);
5274

5275 5276 5277 5278 5279 5280 5281
	/*
	 * If the guest has used debug registers, at least dr7
	 * will be disabled while returning to the host.
	 * If we don't have active breakpoints in the host, we don't
	 * care about the messed up debug address registers. But if
	 * we have some of them active, restore the old state.
	 */
5282
	if (hw_breakpoint_active())
5283
		hw_breakpoint_restore();
5284

N
Nadav Har'El 已提交
5285
	vcpu->arch.last_guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu);
5286

5287
	vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
5288
	smp_wmb();
5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304
	local_irq_enable();

	++vcpu->stat.exits;

	/*
	 * We must have an instruction between local_irq_enable() and
	 * kvm_guest_exit(), so the timer interrupt isn't delayed by
	 * the interrupt shadow.  The stat.exits increment will do nicely.
	 * But we need to prevent reordering, hence this barrier():
	 */
	barrier();

	kvm_guest_exit();

	preempt_enable();

5305
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
5306

5307 5308 5309 5310
	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
5311 5312
		unsigned long rip = kvm_rip_read(vcpu);
		profile_hit(KVM_PROFILING, (void *)rip);
5313 5314
	}

5315

A
Avi Kivity 已提交
5316 5317
	kvm_lapic_sync_from_vapic(vcpu);

A
Avi Kivity 已提交
5318
	r = kvm_x86_ops->handle_exit(vcpu);
5319 5320 5321
out:
	return r;
}
5322

5323

A
Avi Kivity 已提交
5324
static int __vcpu_run(struct kvm_vcpu *vcpu)
5325 5326
{
	int r;
5327
	struct kvm *kvm = vcpu->kvm;
5328 5329

	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) {
5330 5331
		pr_debug("vcpu %d received sipi with vector # %x\n",
			 vcpu->vcpu_id, vcpu->arch.sipi_vector);
5332
		kvm_lapic_reset(vcpu);
5333
		r = kvm_arch_vcpu_reset(vcpu);
5334 5335 5336
		if (r)
			return r;
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
5337 5338
	}

5339
	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
5340 5341 5342 5343
	vapic_enter(vcpu);

	r = 1;
	while (r > 0) {
5344 5345
		if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		    !vcpu->arch.apf.halted)
A
Avi Kivity 已提交
5346
			r = vcpu_enter_guest(vcpu);
5347
		else {
5348
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
5349
			kvm_vcpu_block(vcpu);
5350
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
5351
			if (kvm_check_request(KVM_REQ_UNHALT, vcpu))
5352 5353 5354
			{
				switch(vcpu->arch.mp_state) {
				case KVM_MP_STATE_HALTED:
5355
					vcpu->arch.mp_state =
5356 5357
						KVM_MP_STATE_RUNNABLE;
				case KVM_MP_STATE_RUNNABLE:
5358
					vcpu->arch.apf.halted = false;
5359 5360 5361 5362 5363 5364 5365
					break;
				case KVM_MP_STATE_SIPI_RECEIVED:
				default:
					r = -EINTR;
					break;
				}
			}
5366 5367
		}

5368 5369 5370 5371 5372 5373 5374
		if (r <= 0)
			break;

		clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
		if (kvm_cpu_has_pending_timer(vcpu))
			kvm_inject_pending_timer_irqs(vcpu);

A
Avi Kivity 已提交
5375
		if (dm_request_for_irq_injection(vcpu)) {
5376
			r = -EINTR;
A
Avi Kivity 已提交
5377
			vcpu->run->exit_reason = KVM_EXIT_INTR;
5378 5379
			++vcpu->stat.request_irq_exits;
		}
5380 5381 5382

		kvm_check_async_pf_completion(vcpu);

5383 5384
		if (signal_pending(current)) {
			r = -EINTR;
A
Avi Kivity 已提交
5385
			vcpu->run->exit_reason = KVM_EXIT_INTR;
5386 5387 5388
			++vcpu->stat.signal_exits;
		}
		if (need_resched()) {
5389
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
5390
			kvm_resched(vcpu);
5391
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
5392
		}
5393 5394
	}

5395
	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
5396

A
Avi Kivity 已提交
5397 5398
	vapic_exit(vcpu);

5399 5400 5401
	return r;
}

5402 5403 5404 5405 5406 5407 5408 5409 5410 5411
static int complete_mmio(struct kvm_vcpu *vcpu)
{
	struct kvm_run *run = vcpu->run;
	int r;

	if (!(vcpu->arch.pio.count || vcpu->mmio_needed))
		return 1;

	if (vcpu->mmio_needed) {
		vcpu->mmio_needed = 0;
A
Avi Kivity 已提交
5412
		if (!vcpu->mmio_is_write)
G
Gleb Natapov 已提交
5413 5414
			memcpy(vcpu->mmio_data + vcpu->mmio_index,
			       run->mmio.data, 8);
A
Avi Kivity 已提交
5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427
		vcpu->mmio_index += 8;
		if (vcpu->mmio_index < vcpu->mmio_size) {
			run->exit_reason = KVM_EXIT_MMIO;
			run->mmio.phys_addr = vcpu->mmio_phys_addr + vcpu->mmio_index;
			memcpy(run->mmio.data, vcpu->mmio_data + vcpu->mmio_index, 8);
			run->mmio.len = min(vcpu->mmio_size - vcpu->mmio_index, 8);
			run->mmio.is_write = vcpu->mmio_is_write;
			vcpu->mmio_needed = 1;
			return 0;
		}
		if (vcpu->mmio_is_write)
			return 1;
		vcpu->mmio_read_completed = 1;
5428 5429 5430 5431 5432 5433 5434 5435 5436
	}
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
	r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
	if (r != EMULATE_DONE)
		return 0;
	return 1;
}

5437 5438 5439 5440 5441
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
{
	int r;
	sigset_t sigsaved;

5442 5443 5444
	if (!tsk_used_math(current) && init_fpu(current))
		return -ENOMEM;

5445 5446 5447
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);

5448
	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
5449
		kvm_vcpu_block(vcpu);
5450
		clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
5451 5452
		r = -EAGAIN;
		goto out;
5453 5454 5455
	}

	/* re-sync apic's tpr */
A
Andre Przywara 已提交
5456 5457 5458 5459 5460 5461
	if (!irqchip_in_kernel(vcpu->kvm)) {
		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
			r = -EINVAL;
			goto out;
		}
	}
5462

5463 5464 5465 5466
	r = complete_mmio(vcpu);
	if (r <= 0)
		goto out;

A
Avi Kivity 已提交
5467
	r = __vcpu_run(vcpu);
5468 5469

out:
5470
	post_kvm_run_save(vcpu);
5471 5472 5473 5474 5475 5476 5477 5478
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

	return r;
}

int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
5479 5480 5481 5482 5483 5484 5485 5486
	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
		/*
		 * We are here if userspace calls get_regs() in the middle of
		 * instruction emulation. Registers state needs to be copied
		 * back from emulation context to vcpu. Usrapace shouldn't do
		 * that usually, but some bad designed PV devices (vmware
		 * backdoor interface) need this to work
		 */
5487 5488
		struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
		memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
5489 5490
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
	}
5491 5492 5493 5494 5495 5496 5497 5498
	regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
	regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
	regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
	regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
	regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
	regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
	regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
	regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
5499
#ifdef CONFIG_X86_64
5500 5501 5502 5503 5504 5505 5506 5507
	regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
	regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
	regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
	regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
	regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
	regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
	regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
	regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
5508 5509
#endif

5510
	regs->rip = kvm_rip_read(vcpu);
5511
	regs->rflags = kvm_get_rflags(vcpu);
5512 5513 5514 5515 5516 5517

	return 0;
}

int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
5518 5519 5520
	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;

5521 5522 5523 5524 5525 5526 5527 5528
	kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
	kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
	kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
	kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
	kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
	kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
	kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
	kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
5529
#ifdef CONFIG_X86_64
5530 5531 5532 5533 5534 5535 5536 5537
	kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
	kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
	kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
	kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
	kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
	kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
	kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
	kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
5538 5539
#endif

5540
	kvm_rip_write(vcpu, regs->rip);
5541
	kvm_set_rflags(vcpu, regs->rflags);
5542

5543 5544
	vcpu->arch.exception.pending = false;

5545 5546
	kvm_make_request(KVM_REQ_EVENT, vcpu);

5547 5548 5549 5550 5551 5552 5553
	return 0;
}

void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

5554
	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
5555 5556 5557 5558 5559 5560 5561 5562
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
5563
	struct desc_ptr dt;
5564

5565 5566 5567 5568 5569 5570
	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
5571

5572 5573
	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
5574 5575

	kvm_x86_ops->get_idt(vcpu, &dt);
5576 5577
	sregs->idt.limit = dt.size;
	sregs->idt.base = dt.address;
5578
	kvm_x86_ops->get_gdt(vcpu, &dt);
5579 5580
	sregs->gdt.limit = dt.size;
	sregs->gdt.base = dt.address;
5581

5582
	sregs->cr0 = kvm_read_cr0(vcpu);
5583
	sregs->cr2 = vcpu->arch.cr2;
5584
	sregs->cr3 = kvm_read_cr3(vcpu);
5585
	sregs->cr4 = kvm_read_cr4(vcpu);
5586
	sregs->cr8 = kvm_get_cr8(vcpu);
5587
	sregs->efer = vcpu->arch.efer;
5588 5589
	sregs->apic_base = kvm_get_apic_base(vcpu);

G
Gleb Natapov 已提交
5590
	memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
5591

5592
	if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
5593 5594
		set_bit(vcpu->arch.interrupt.nr,
			(unsigned long *)sregs->interrupt_bitmap);
5595

5596 5597 5598
	return 0;
}

5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
	mp_state->mp_state = vcpu->arch.mp_state;
	return 0;
}

int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
	vcpu->arch.mp_state = mp_state->mp_state;
5610
	kvm_make_request(KVM_REQ_EVENT, vcpu);
5611 5612 5613
	return 0;
}

5614 5615
int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason,
		    bool has_error_code, u32 error_code)
5616
{
5617
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
5618
	int ret;
5619

5620
	init_emulate_ctxt(vcpu);
5621

5622 5623
	ret = emulator_task_switch(ctxt, tss_selector, reason,
				   has_error_code, error_code);
5624 5625

	if (ret)
5626
		return EMULATE_FAIL;
5627

5628
	memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
5629 5630
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
5631
	kvm_make_request(KVM_REQ_EVENT, vcpu);
5632
	return EMULATE_DONE;
5633 5634 5635
}
EXPORT_SYMBOL_GPL(kvm_task_switch);

5636 5637 5638 5639
int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	int mmu_reset_needed = 0;
5640
	int pending_vec, max_bits, idx;
5641
	struct desc_ptr dt;
5642

5643 5644
	dt.size = sregs->idt.limit;
	dt.address = sregs->idt.base;
5645
	kvm_x86_ops->set_idt(vcpu, &dt);
5646 5647
	dt.size = sregs->gdt.limit;
	dt.address = sregs->gdt.base;
5648 5649
	kvm_x86_ops->set_gdt(vcpu, &dt);

5650
	vcpu->arch.cr2 = sregs->cr2;
5651
	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
5652
	vcpu->arch.cr3 = sregs->cr3;
5653
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
5654

5655
	kvm_set_cr8(vcpu, sregs->cr8);
5656

5657
	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
5658 5659 5660
	kvm_x86_ops->set_efer(vcpu, sregs->efer);
	kvm_set_apic_base(vcpu, sregs->apic_base);

5661
	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
5662
	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
5663
	vcpu->arch.cr0 = sregs->cr0;
5664

5665
	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
5666
	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
S
Sheng Yang 已提交
5667
	if (sregs->cr4 & X86_CR4_OSXSAVE)
A
Avi Kivity 已提交
5668
		kvm_update_cpuid(vcpu);
5669 5670

	idx = srcu_read_lock(&vcpu->kvm->srcu);
5671
	if (!is_long_mode(vcpu) && is_pae(vcpu)) {
5672
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
5673 5674
		mmu_reset_needed = 1;
	}
5675
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
5676 5677 5678 5679

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

G
Gleb Natapov 已提交
5680 5681 5682 5683
	max_bits = (sizeof sregs->interrupt_bitmap) << 3;
	pending_vec = find_first_bit(
		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
	if (pending_vec < max_bits) {
5684
		kvm_queue_interrupt(vcpu, pending_vec, false);
G
Gleb Natapov 已提交
5685
		pr_debug("Set back pending irq %d\n", pending_vec);
5686 5687
	}

5688 5689 5690 5691 5692 5693
	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
5694

5695 5696
	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
5697

5698 5699
	update_cr8_intercept(vcpu);

M
Marcelo Tosatti 已提交
5700
	/* Older userspace won't unhalt the vcpu on reset. */
5701
	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
M
Marcelo Tosatti 已提交
5702
	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
5703
	    !is_protmode(vcpu))
M
Marcelo Tosatti 已提交
5704 5705
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;

5706 5707
	kvm_make_request(KVM_REQ_EVENT, vcpu);

5708 5709 5710
	return 0;
}

J
Jan Kiszka 已提交
5711 5712
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg)
5713
{
5714
	unsigned long rflags;
5715
	int i, r;
5716

5717 5718 5719
	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
		r = -EBUSY;
		if (vcpu->arch.exception.pending)
5720
			goto out;
5721 5722 5723 5724 5725 5726
		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
			kvm_queue_exception(vcpu, DB_VECTOR);
		else
			kvm_queue_exception(vcpu, BP_VECTOR);
	}

5727 5728 5729 5730 5731
	/*
	 * Read rflags as long as potentially injected trace flags are still
	 * filtered out.
	 */
	rflags = kvm_get_rflags(vcpu);
5732 5733 5734 5735 5736 5737

	vcpu->guest_debug = dbg->control;
	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
		vcpu->guest_debug = 0;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
5738 5739 5740 5741 5742 5743 5744 5745 5746 5747
		for (i = 0; i < KVM_NR_DB_REGS; ++i)
			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
		vcpu->arch.switch_db_regs =
			(dbg->arch.debugreg[7] & DR7_BP_EN_MASK);
	} else {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
		vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK);
	}

J
Jan Kiszka 已提交
5748 5749 5750
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
			get_segment_base(vcpu, VCPU_SREG_CS);
5751

5752 5753 5754 5755 5756
	/*
	 * Trigger an rflags update that will inject or remove the trace
	 * flags.
	 */
	kvm_set_rflags(vcpu, rflags);
5757

5758
	kvm_x86_ops->set_guest_debug(vcpu, dbg);
5759

5760
	r = 0;
J
Jan Kiszka 已提交
5761

5762
out:
5763 5764 5765 5766

	return r;
}

5767 5768 5769 5770 5771 5772 5773 5774
/*
 * Translate a guest virtual address to a guest physical address.
 */
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;
5775
	int idx;
5776

5777
	idx = srcu_read_lock(&vcpu->kvm->srcu);
5778
	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
5779
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
5780 5781 5782 5783 5784 5785 5786 5787
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;

	return 0;
}

5788 5789
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
S
Sheng Yang 已提交
5790 5791
	struct i387_fxsave_struct *fxsave =
			&vcpu->arch.guest_fpu.state->fxsave;
5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806

	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);

	return 0;
}

int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
S
Sheng Yang 已提交
5807 5808
	struct i387_fxsave_struct *fxsave =
			&vcpu->arch.guest_fpu.state->fxsave;
5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);

	return 0;
}

5822
int fx_init(struct kvm_vcpu *vcpu)
5823
{
5824 5825 5826 5827 5828 5829
	int err;

	err = fpu_alloc(&vcpu->arch.guest_fpu);
	if (err)
		return err;

S
Sheng Yang 已提交
5830
	fpu_finit(&vcpu->arch.guest_fpu);
5831

5832 5833 5834 5835 5836
	/*
	 * Ensure guest xcr0 is valid for loading
	 */
	vcpu->arch.xcr0 = XSTATE_FP;

5837
	vcpu->arch.cr0 |= X86_CR0_ET;
5838 5839

	return 0;
5840 5841 5842
}
EXPORT_SYMBOL_GPL(fx_init);

S
Sheng Yang 已提交
5843 5844 5845 5846 5847
static void fx_free(struct kvm_vcpu *vcpu)
{
	fpu_free(&vcpu->arch.guest_fpu);
}

5848 5849
void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
5850
	if (vcpu->guest_fpu_loaded)
5851 5852
		return;

5853 5854 5855 5856 5857 5858
	/*
	 * Restore all possible states in the guest,
	 * and assume host would use all available bits.
	 * Guest xcr0 would be loaded later.
	 */
	kvm_put_guest_xcr0(vcpu);
5859
	vcpu->guest_fpu_loaded = 1;
5860
	unlazy_fpu(current);
S
Sheng Yang 已提交
5861
	fpu_restore_checking(&vcpu->arch.guest_fpu);
5862
	trace_kvm_fpu(1);
5863 5864 5865 5866
}

void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
5867 5868
	kvm_put_guest_xcr0(vcpu);

5869 5870 5871 5872
	if (!vcpu->guest_fpu_loaded)
		return;

	vcpu->guest_fpu_loaded = 0;
S
Sheng Yang 已提交
5873
	fpu_save_init(&vcpu->arch.guest_fpu);
A
Avi Kivity 已提交
5874
	++vcpu->stat.fpu_reload;
5875
	kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu);
5876
	trace_kvm_fpu(0);
5877
}
5878 5879 5880

void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
{
5881
	kvmclock_reset(vcpu);
5882

5883
	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
S
Sheng Yang 已提交
5884
	fx_free(vcpu);
5885 5886 5887 5888 5889 5890
	kvm_x86_ops->vcpu_free(vcpu);
}

struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
						unsigned int id)
{
Z
Zachary Amsden 已提交
5891 5892 5893 5894
	if (check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
		printk_once(KERN_WARNING
		"kvm: SMP vm created on host with unstable TSC; "
		"guest TSC will not be reliable\n");
5895 5896
	return kvm_x86_ops->vcpu_create(kvm, id);
}
5897

5898 5899 5900
int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
{
	int r;
5901

S
Sheng Yang 已提交
5902
	vcpu->arch.mtrr_state.have_fixed = 1;
5903 5904 5905 5906 5907 5908
	vcpu_load(vcpu);
	r = kvm_arch_vcpu_reset(vcpu);
	if (r == 0)
		r = kvm_mmu_setup(vcpu);
	vcpu_put(vcpu);

5909
	return r;
5910 5911
}

5912
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
5913
{
5914 5915
	vcpu->arch.apf.msr_val = 0;

5916 5917 5918 5919
	vcpu_load(vcpu);
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);

S
Sheng Yang 已提交
5920
	fx_free(vcpu);
5921 5922 5923 5924 5925
	kvm_x86_ops->vcpu_free(vcpu);
}

int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
5926 5927
	atomic_set(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = 0;
5928 5929
	vcpu->arch.nmi_injected = false;

5930 5931 5932 5933 5934
	vcpu->arch.switch_db_regs = 0;
	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
	vcpu->arch.dr6 = DR6_FIXED_1;
	vcpu->arch.dr7 = DR7_FIXED_1;

5935
	kvm_make_request(KVM_REQ_EVENT, vcpu);
5936
	vcpu->arch.apf.msr_val = 0;
G
Glauber Costa 已提交
5937
	vcpu->arch.st.msr_val = 0;
5938

5939 5940
	kvmclock_reset(vcpu);

5941 5942 5943
	kvm_clear_async_pf_completion_queue(vcpu);
	kvm_async_pf_hash_reset(vcpu);
	vcpu->arch.apf.halted = false;
5944

5945 5946
	kvm_pmu_reset(vcpu);

5947 5948 5949
	return kvm_x86_ops->vcpu_reset(vcpu);
}

5950
int kvm_arch_hardware_enable(void *garbage)
5951
{
5952 5953 5954
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;
A
Avi Kivity 已提交
5955 5956

	kvm_shared_msr_cpu_online();
5957 5958 5959
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_for_each_vcpu(i, vcpu, kvm)
			if (vcpu->cpu == smp_processor_id())
Z
Zachary Amsden 已提交
5960
				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5961
	return kvm_x86_ops->hardware_enable(garbage);
5962 5963 5964 5965 5966
}

void kvm_arch_hardware_disable(void *garbage)
{
	kvm_x86_ops->hardware_disable(garbage);
5967
	drop_user_return_notifiers(garbage);
5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993
}

int kvm_arch_hardware_setup(void)
{
	return kvm_x86_ops->hardware_setup();
}

void kvm_arch_hardware_unsetup(void)
{
	kvm_x86_ops->hardware_unsetup();
}

void kvm_arch_check_processor_compat(void *rtn)
{
	kvm_x86_ops->check_processor_compatibility(rtn);
}

int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
	struct page *page;
	struct kvm *kvm;
	int r;

	BUG_ON(vcpu->kvm == NULL);
	kvm = vcpu->kvm;

5994
	vcpu->arch.emulate_ctxt.ops = &emulate_ops;
5995
	if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
5996
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
5997
	else
5998
		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
5999 6000 6001 6002 6003 6004

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
6005
	vcpu->arch.pio_data = page_address(page);
6006

6007
	kvm_init_tsc_catchup(vcpu, max_tsc_khz);
Z
Zachary Amsden 已提交
6008

6009 6010 6011 6012 6013 6014 6015 6016 6017 6018
	r = kvm_mmu_create(vcpu);
	if (r < 0)
		goto fail_free_pio_data;

	if (irqchip_in_kernel(kvm)) {
		r = kvm_create_lapic(vcpu);
		if (r < 0)
			goto fail_mmu_destroy;
	}

H
Huang Ying 已提交
6019 6020 6021 6022
	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
				       GFP_KERNEL);
	if (!vcpu->arch.mce_banks) {
		r = -ENOMEM;
6023
		goto fail_free_lapic;
H
Huang Ying 已提交
6024 6025 6026
	}
	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;

6027 6028 6029
	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL))
		goto fail_free_mce_banks;

6030
	kvm_async_pf_hash_reset(vcpu);
6031
	kvm_pmu_init(vcpu);
6032

6033
	return 0;
6034 6035
fail_free_mce_banks:
	kfree(vcpu->arch.mce_banks);
6036 6037
fail_free_lapic:
	kvm_free_lapic(vcpu);
6038 6039 6040
fail_mmu_destroy:
	kvm_mmu_destroy(vcpu);
fail_free_pio_data:
6041
	free_page((unsigned long)vcpu->arch.pio_data);
6042 6043 6044 6045 6046 6047
fail:
	return r;
}

void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
{
6048 6049
	int idx;

6050
	kvm_pmu_destroy(vcpu);
6051
	kfree(vcpu->arch.mce_banks);
6052
	kvm_free_lapic(vcpu);
6053
	idx = srcu_read_lock(&vcpu->kvm->srcu);
6054
	kvm_mmu_destroy(vcpu);
6055
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
6056
	free_page((unsigned long)vcpu->arch.pio_data);
6057
}
6058

6059
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
6060
{
6061 6062 6063
	if (type)
		return -EINVAL;

6064
	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
B
Ben-Ami Yassour 已提交
6065
	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
6066

6067 6068 6069
	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);

6070
	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
6071

6072
	return 0;
6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084
}

static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
	vcpu_load(vcpu);
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;
6085
	struct kvm_vcpu *vcpu;
6086 6087 6088 6089

	/*
	 * Unpin any mmu pages first.
	 */
6090 6091
	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_clear_async_pf_completion_queue(vcpu);
6092
		kvm_unload_vcpu_mmu(vcpu);
6093
	}
6094 6095 6096 6097 6098 6099
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_arch_vcpu_free(vcpu);

	mutex_lock(&kvm->lock);
	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
		kvm->vcpus[i] = NULL;
6100

6101 6102
	atomic_set(&kvm->online_vcpus, 0);
	mutex_unlock(&kvm->lock);
6103 6104
}

6105 6106
void kvm_arch_sync_events(struct kvm *kvm)
{
6107
	kvm_free_all_assigned_devices(kvm);
6108
	kvm_free_pit(kvm);
6109 6110
}

6111 6112
void kvm_arch_destroy_vm(struct kvm *kvm)
{
6113
	kvm_iommu_unmap_guest(kvm);
6114 6115
	kfree(kvm->arch.vpic);
	kfree(kvm->arch.vioapic);
6116
	kvm_free_vcpus(kvm);
6117 6118
	if (kvm->arch.apic_access_page)
		put_page(kvm->arch.apic_access_page);
6119 6120
	if (kvm->arch.ept_identity_pagetable)
		put_page(kvm->arch.ept_identity_pagetable);
6121
}
6122

6123 6124
int kvm_arch_prepare_memory_region(struct kvm *kvm,
				struct kvm_memory_slot *memslot,
6125
				struct kvm_memory_slot old,
6126
				struct kvm_userspace_memory_region *mem,
6127 6128
				int user_alloc)
{
6129
	int npages = memslot->npages;
6130 6131 6132 6133 6134
	int map_flags = MAP_PRIVATE | MAP_ANONYMOUS;

	/* Prevent internal slot pages from being moved by fork()/COW. */
	if (memslot->id >= KVM_MEMORY_SLOTS)
		map_flags = MAP_SHARED | MAP_ANONYMOUS;
6135 6136 6137 6138 6139 6140

	/*To keep backward compatibility with older userspace,
	 *x86 needs to hanlde !user_alloc case.
	 */
	if (!user_alloc) {
		if (npages && !old.rmap) {
6141 6142
			unsigned long userspace_addr;

6143
			down_write(&current->mm->mmap_sem);
6144 6145 6146
			userspace_addr = do_mmap(NULL, 0,
						 npages * PAGE_SIZE,
						 PROT_READ | PROT_WRITE,
6147
						 map_flags,
6148
						 0);
6149
			up_write(&current->mm->mmap_sem);
6150

6151 6152 6153 6154
			if (IS_ERR((void *)userspace_addr))
				return PTR_ERR((void *)userspace_addr);

			memslot->userspace_addr = userspace_addr;
6155 6156 6157
		}
	}

6158 6159 6160 6161 6162 6163 6164 6165 6166 6167

	return 0;
}

void kvm_arch_commit_memory_region(struct kvm *kvm,
				struct kvm_userspace_memory_region *mem,
				struct kvm_memory_slot old,
				int user_alloc)
{

6168
	int nr_mmu_pages = 0, npages = mem->memory_size >> PAGE_SHIFT;
6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182

	if (!user_alloc && !old.user_alloc && old.rmap && !npages) {
		int ret;

		down_write(&current->mm->mmap_sem);
		ret = do_munmap(current->mm, old.userspace_addr,
				old.npages * PAGE_SIZE);
		up_write(&current->mm->mmap_sem);
		if (ret < 0)
			printk(KERN_WARNING
			       "kvm_vm_ioctl_set_memory_region: "
			       "failed to munmap memory\n");
	}

6183 6184 6185
	if (!kvm->arch.n_requested_mmu_pages)
		nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);

6186
	spin_lock(&kvm->mmu_lock);
6187
	if (nr_mmu_pages)
6188 6189
		kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
	kvm_mmu_slot_remove_write_access(kvm, mem->slot);
6190
	spin_unlock(&kvm->mmu_lock);
6191
}
6192

6193 6194 6195
void kvm_arch_flush_shadow(struct kvm *kvm)
{
	kvm_mmu_zap_all(kvm);
6196
	kvm_reload_remote_mmus(kvm);
6197 6198
}

6199 6200
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
6201 6202 6203
	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		!vcpu->arch.apf.halted)
		|| !list_empty_careful(&vcpu->async_pf.done)
6204
		|| vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
A
Avi Kivity 已提交
6205
		|| atomic_read(&vcpu->arch.nmi_queued) ||
6206 6207
		(kvm_arch_interrupt_allowed(vcpu) &&
		 kvm_cpu_has_interrupt(vcpu));
6208
}
6209 6210 6211

void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
6212 6213
	int me;
	int cpu = vcpu->cpu;
6214 6215 6216 6217 6218

	if (waitqueue_active(&vcpu->wq)) {
		wake_up_interruptible(&vcpu->wq);
		++vcpu->stat.halt_wakeup;
	}
6219 6220 6221

	me = get_cpu();
	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
6222
		if (kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE)
6223
			smp_send_reschedule(cpu);
6224
	put_cpu();
6225
}
6226 6227 6228 6229 6230

int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	return kvm_x86_ops->interrupt_allowed(vcpu);
}
6231

J
Jan Kiszka 已提交
6232 6233 6234 6235 6236 6237 6238 6239 6240
bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
{
	unsigned long current_rip = kvm_rip_read(vcpu) +
		get_segment_base(vcpu, VCPU_SREG_CS);

	return current_rip == linear_rip;
}
EXPORT_SYMBOL_GPL(kvm_is_linear_rip);

6241 6242 6243 6244 6245 6246
unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
{
	unsigned long rflags;

	rflags = kvm_x86_ops->get_rflags(vcpu);
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
6247
		rflags &= ~X86_EFLAGS_TF;
6248 6249 6250 6251 6252 6253 6254
	return rflags;
}
EXPORT_SYMBOL_GPL(kvm_get_rflags);

void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
J
Jan Kiszka 已提交
6255
	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
6256
		rflags |= X86_EFLAGS_TF;
6257
	kvm_x86_ops->set_rflags(vcpu, rflags);
6258
	kvm_make_request(KVM_REQ_EVENT, vcpu);
6259 6260 6261
}
EXPORT_SYMBOL_GPL(kvm_set_rflags);

G
Gleb Natapov 已提交
6262 6263 6264 6265
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
{
	int r;

X
Xiao Guangrong 已提交
6266
	if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) ||
6267
	      is_error_page(work->page))
G
Gleb Natapov 已提交
6268 6269 6270 6271 6272 6273
		return;

	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		return;

X
Xiao Guangrong 已提交
6274 6275 6276 6277
	if (!vcpu->arch.mmu.direct_map &&
	      work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu))
		return;

G
Gleb Natapov 已提交
6278 6279 6280
	vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true);
}

6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306
static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
{
	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
}

static inline u32 kvm_async_pf_next_probe(u32 key)
{
	return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
}

static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 key = kvm_async_pf_hash_fn(gfn);

	while (vcpu->arch.apf.gfns[key] != ~0)
		key = kvm_async_pf_next_probe(key);

	vcpu->arch.apf.gfns[key] = gfn;
}

static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	int i;
	u32 key = kvm_async_pf_hash_fn(gfn);

	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
6307 6308
		     (vcpu->arch.apf.gfns[key] != gfn &&
		      vcpu->arch.apf.gfns[key] != ~0); i++)
6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341
		key = kvm_async_pf_next_probe(key);

	return key;
}

bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
}

static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 i, j, k;

	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
	while (true) {
		vcpu->arch.apf.gfns[i] = ~0;
		do {
			j = kvm_async_pf_next_probe(j);
			if (vcpu->arch.apf.gfns[j] == ~0)
				return;
			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
			/*
			 * k lies cyclically in ]i,j]
			 * |    i.k.j |
			 * |....j i.k.| or  |.k..j i...|
			 */
		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
		i = j;
	}
}

6342 6343 6344 6345 6346 6347 6348
static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
{

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
				      sizeof(val));
}

6349 6350 6351
void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
				     struct kvm_async_pf *work)
{
6352 6353
	struct x86_exception fault;

6354
	trace_kvm_async_pf_not_present(work->arch.token, work->gva);
6355
	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
6356 6357

	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
6358 6359
	    (vcpu->arch.apf.send_user_only &&
	     kvm_x86_ops->get_cpl(vcpu) == 0))
6360 6361
		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
	else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
6362 6363 6364 6365 6366 6367
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
		kvm_inject_page_fault(vcpu, &fault);
6368
	}
6369 6370 6371 6372 6373
}

void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
				 struct kvm_async_pf *work)
{
6374 6375
	struct x86_exception fault;

6376 6377 6378 6379 6380 6381 6382 6383
	trace_kvm_async_pf_ready(work->arch.token, work->gva);
	if (is_error_page(work->page))
		work->arch.token = ~0; /* broadcast wakeup */
	else
		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);

	if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) &&
	    !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
6384 6385 6386 6387 6388 6389
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
		kvm_inject_page_fault(vcpu, &fault);
6390
	}
6391
	vcpu->arch.apf.halted = false;
6392 6393 6394 6395 6396 6397 6398 6399 6400
}

bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
		return true;
	else
		return !kvm_event_needs_reinjection(vcpu) &&
			kvm_x86_ops->interrupt_allowed(vcpu);
6401 6402
}

6403 6404 6405 6406 6407
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
6408
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
6409
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
6410
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
6411
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
6412
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
6413
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
6414
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);