x86.c 179.8 KB
Newer Older
1 2 3 4 5 6
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
B
Ben-Ami Yassour 已提交
7 8
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
N
Nicolas Kaiser 已提交
9
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
10 11 12 13
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
B
Ben-Ami Yassour 已提交
14 15
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
16 17 18 19 20 21
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

22
#include <linux/kvm_host.h>
23
#include "irq.h"
24
#include "mmu.h"
S
Sheng Yang 已提交
25
#include "i8254.h"
26
#include "tss.h"
27
#include "kvm_cache_regs.h"
28
#include "x86.h"
A
Avi Kivity 已提交
29
#include "cpuid.h"
30

31
#include <linux/clocksource.h>
B
Ben-Ami Yassour 已提交
32
#include <linux/interrupt.h>
33 34 35
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
36
#include <linux/module.h>
37
#include <linux/mman.h>
38
#include <linux/highmem.h>
J
Joerg Roedel 已提交
39
#include <linux/iommu.h>
B
Ben-Ami Yassour 已提交
40
#include <linux/intel-iommu.h>
41
#include <linux/cpufreq.h>
A
Avi Kivity 已提交
42
#include <linux/user-return-notifier.h>
43
#include <linux/srcu.h>
44
#include <linux/slab.h>
45
#include <linux/perf_event.h>
46
#include <linux/uaccess.h>
47
#include <linux/hash.h>
48
#include <linux/pci.h>
49 50
#include <linux/timekeeper_internal.h>
#include <linux/pvclock_gtod.h>
A
Avi Kivity 已提交
51
#include <trace/events/kvm.h>
X
Xiao Guangrong 已提交
52

53 54
#define CREATE_TRACE_POINTS
#include "trace.h"
55

56
#include <asm/debugreg.h>
57
#include <asm/msr.h>
58
#include <asm/desc.h>
S
Sheng Yang 已提交
59
#include <asm/mtrr.h>
H
Huang Ying 已提交
60
#include <asm/mce.h>
61
#include <asm/i387.h>
62
#include <asm/fpu-internal.h> /* Ugh! */
S
Sheng Yang 已提交
63
#include <asm/xcr.h>
64
#include <asm/pvclock.h>
65
#include <asm/div64.h>
66

67
#define MAX_IO_MSRS 256
H
Huang Ying 已提交
68
#define KVM_MAX_MCE_BANKS 32
69
#define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P)
H
Huang Ying 已提交
70

71 72 73
#define emul_to_vcpu(ctxt) \
	container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)

74 75 76 77 78
/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
79 80
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
81
#else
82
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
83
#endif
84

85 86
#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
87

88
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
A
Avi Kivity 已提交
89
static void process_nmi(struct kvm_vcpu *vcpu);
90

91
struct kvm_x86_ops *kvm_x86_ops;
92
EXPORT_SYMBOL_GPL(kvm_x86_ops);
93

94 95
static bool ignore_msrs = 0;
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
96

97 98 99 100 101
bool kvm_has_tsc_control;
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
u32  kvm_max_guest_tsc_khz;
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);

102 103 104 105
/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
static u32 tsc_tolerance_ppm = 250;
module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);

A
Avi Kivity 已提交
106 107 108 109
#define KVM_NR_SHARED_MSRS 16

struct kvm_shared_msrs_global {
	int nr;
110
	u32 msrs[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
111 112 113 114 115
};

struct kvm_shared_msrs {
	struct user_return_notifier urn;
	bool registered;
116 117 118 119
	struct kvm_shared_msr_values {
		u64 host;
		u64 curr;
	} values[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
120 121 122
};

static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
123
static struct kvm_shared_msrs __percpu *shared_msrs;
A
Avi Kivity 已提交
124

125
struct kvm_stats_debugfs_item debugfs_entries[] = {
126 127 128 129 130 131 132 133 134
	{ "pf_fixed", VCPU_STAT(pf_fixed) },
	{ "pf_guest", VCPU_STAT(pf_guest) },
	{ "tlb_flush", VCPU_STAT(tlb_flush) },
	{ "invlpg", VCPU_STAT(invlpg) },
	{ "exits", VCPU_STAT(exits) },
	{ "io_exits", VCPU_STAT(io_exits) },
	{ "mmio_exits", VCPU_STAT(mmio_exits) },
	{ "signal_exits", VCPU_STAT(signal_exits) },
	{ "irq_window", VCPU_STAT(irq_window_exits) },
135
	{ "nmi_window", VCPU_STAT(nmi_window_exits) },
136 137
	{ "halt_exits", VCPU_STAT(halt_exits) },
	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
A
Amit Shah 已提交
138
	{ "hypercalls", VCPU_STAT(hypercalls) },
139 140 141 142 143 144 145
	{ "request_irq", VCPU_STAT(request_irq_exits) },
	{ "irq_exits", VCPU_STAT(irq_exits) },
	{ "host_state_reload", VCPU_STAT(host_state_reload) },
	{ "efer_reload", VCPU_STAT(efer_reload) },
	{ "fpu_reload", VCPU_STAT(fpu_reload) },
	{ "insn_emulation", VCPU_STAT(insn_emulation) },
	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
146
	{ "irq_injections", VCPU_STAT(irq_injections) },
147
	{ "nmi_injections", VCPU_STAT(nmi_injections) },
A
Avi Kivity 已提交
148 149 150 151 152 153
	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
	{ "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
	{ "mmu_flooded", VM_STAT(mmu_flooded) },
	{ "mmu_recycled", VM_STAT(mmu_recycled) },
A
Avi Kivity 已提交
154
	{ "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
155
	{ "mmu_unsync", VM_STAT(mmu_unsync) },
156
	{ "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
M
Marcelo Tosatti 已提交
157
	{ "largepages", VM_STAT(lpages) },
158 159 160
	{ NULL }
};

161 162
u64 __read_mostly host_xcr0;

163
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
164

165
static int kvm_vcpu_reset(struct kvm_vcpu *vcpu);
166

167 168 169 170 171 172 173
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
	int i;
	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
		vcpu->arch.apf.gfns[i] = ~0;
}

A
Avi Kivity 已提交
174 175 176 177 178
static void kvm_on_user_return(struct user_return_notifier *urn)
{
	unsigned slot;
	struct kvm_shared_msrs *locals
		= container_of(urn, struct kvm_shared_msrs, urn);
179
	struct kvm_shared_msr_values *values;
A
Avi Kivity 已提交
180 181

	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
182 183 184 185
		values = &locals->values[slot];
		if (values->host != values->curr) {
			wrmsrl(shared_msrs_global.msrs[slot], values->host);
			values->curr = values->host;
A
Avi Kivity 已提交
186 187 188 189 190 191
		}
	}
	locals->registered = false;
	user_return_notifier_unregister(urn);
}

192
static void shared_msr_update(unsigned slot, u32 msr)
A
Avi Kivity 已提交
193 194
{
	u64 value;
195 196
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
A
Avi Kivity 已提交
197

198 199 200 201 202 203 204 205 206 207 208 209 210
	/* only read, and nobody should modify it at this time,
	 * so don't need lock */
	if (slot >= shared_msrs_global.nr) {
		printk(KERN_ERR "kvm: invalid MSR slot!");
		return;
	}
	rdmsrl_safe(msr, &value);
	smsr->values[slot].host = value;
	smsr->values[slot].curr = value;
}

void kvm_define_shared_msr(unsigned slot, u32 msr)
{
A
Avi Kivity 已提交
211 212
	if (slot >= shared_msrs_global.nr)
		shared_msrs_global.nr = slot + 1;
213 214 215
	shared_msrs_global.msrs[slot] = msr;
	/* we need ensured the shared_msr_global have been updated */
	smp_wmb();
A
Avi Kivity 已提交
216 217 218 219 220 221 222 223
}
EXPORT_SYMBOL_GPL(kvm_define_shared_msr);

static void kvm_shared_msr_cpu_online(void)
{
	unsigned i;

	for (i = 0; i < shared_msrs_global.nr; ++i)
224
		shared_msr_update(i, shared_msrs_global.msrs[i]);
A
Avi Kivity 已提交
225 226
}

227
void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
A
Avi Kivity 已提交
228
{
229 230
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
A
Avi Kivity 已提交
231

232
	if (((value ^ smsr->values[slot].curr) & mask) == 0)
A
Avi Kivity 已提交
233
		return;
234 235
	smsr->values[slot].curr = value;
	wrmsrl(shared_msrs_global.msrs[slot], value);
A
Avi Kivity 已提交
236 237 238 239 240 241 242 243
	if (!smsr->registered) {
		smsr->urn.on_user_return = kvm_on_user_return;
		user_return_notifier_register(&smsr->urn);
		smsr->registered = true;
	}
}
EXPORT_SYMBOL_GPL(kvm_set_shared_msr);

244 245
static void drop_user_return_notifiers(void *ignore)
{
246 247
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
248 249 250 251 252

	if (smsr->registered)
		kvm_on_user_return(&smsr->urn);
}

253 254
u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
255
	return vcpu->arch.apic_base;
256 257 258 259 260 261
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
{
	/* TODO: reserve bits check */
262
	kvm_lapic_set_base(vcpu, data);
263 264 265
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
#define EXCPT_BENIGN		0
#define EXCPT_CONTRIBUTORY	1
#define EXCPT_PF		2

static int exception_class(int vector)
{
	switch (vector) {
	case PF_VECTOR:
		return EXCPT_PF;
	case DE_VECTOR:
	case TS_VECTOR:
	case NP_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
		return EXCPT_CONTRIBUTORY;
	default:
		break;
	}
	return EXCPT_BENIGN;
}

static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
288 289
		unsigned nr, bool has_error, u32 error_code,
		bool reinject)
290 291 292 293
{
	u32 prev_nr;
	int class1, class2;

294 295
	kvm_make_request(KVM_REQ_EVENT, vcpu);

296 297 298 299 300 301
	if (!vcpu->arch.exception.pending) {
	queue:
		vcpu->arch.exception.pending = true;
		vcpu->arch.exception.has_error_code = has_error;
		vcpu->arch.exception.nr = nr;
		vcpu->arch.exception.error_code = error_code;
302
		vcpu->arch.exception.reinject = reinject;
303 304 305 306 307 308 309
		return;
	}

	/* to check exception */
	prev_nr = vcpu->arch.exception.nr;
	if (prev_nr == DF_VECTOR) {
		/* triple fault -> shutdown */
310
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
		return;
	}
	class1 = exception_class(prev_nr);
	class2 = exception_class(nr);
	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
		/* generate double fault per SDM Table 5-5 */
		vcpu->arch.exception.pending = true;
		vcpu->arch.exception.has_error_code = true;
		vcpu->arch.exception.nr = DF_VECTOR;
		vcpu->arch.exception.error_code = 0;
	} else
		/* replace previous exception with a new one in a hope
		   that instruction re-execution will regenerate lost
		   exception */
		goto queue;
}

329 330
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
331
	kvm_multiple_exception(vcpu, nr, false, 0, false);
332 333 334
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

335 336 337 338 339 340
void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
	kvm_multiple_exception(vcpu, nr, false, 0, true);
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

341
void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
342
{
343 344 345 346 347 348
	if (err)
		kvm_inject_gp(vcpu, 0);
	else
		kvm_x86_ops->skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
349

350
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
351 352
{
	++vcpu->stat.pf_guest;
353 354
	vcpu->arch.cr2 = fault->address;
	kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
355
}
N
Nadav Har'El 已提交
356
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
357

358
void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
359
{
360 361
	if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
		vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
362
	else
363
		vcpu->arch.mmu.inject_page_fault(vcpu, fault);
364 365
}

366 367
void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
368 369
	atomic_inc(&vcpu->arch.nmi_queued);
	kvm_make_request(KVM_REQ_NMI, vcpu);
370 371 372
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

373 374
void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
375
	kvm_multiple_exception(vcpu, nr, true, error_code, false);
376 377 378
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

379 380 381 382 383 384
void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
	kvm_multiple_exception(vcpu, nr, true, error_code, true);
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

385 386 387 388 389
/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
390
{
391 392 393 394
	if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
		return true;
	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
	return false;
395
}
396
EXPORT_SYMBOL_GPL(kvm_require_cpl);
397

398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
/*
 * This function will be used to read from the physical memory of the currently
 * running guest. The difference to kvm_read_guest_page is that this function
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gfn_t ngfn, void *data, int offset, int len,
			    u32 access)
{
	gfn_t real_gfn;
	gpa_t ngpa;

	ngpa     = gfn_to_gpa(ngfn);
	real_gfn = mmu->translate_gpa(vcpu, ngpa, access);
	if (real_gfn == UNMAPPED_GVA)
		return -EFAULT;

	real_gfn = gpa_to_gfn(real_gfn);

	return kvm_read_guest_page(vcpu->kvm, real_gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

421 422 423 424 425 426 427
int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
			       void *data, int offset, int len, u32 access)
{
	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
				       data, offset, len, access);
}

428 429 430
/*
 * Load the pae pdptrs.  Return true is they are all valid.
 */
431
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
432 433 434 435 436
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
	int i;
	int ret;
437
	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
438

439 440 441
	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
				      offset * sizeof(u64), sizeof(pdpte),
				      PFERR_USER_MASK|PFERR_WRITE_MASK);
442 443 444 445 446
	if (ret < 0) {
		ret = 0;
		goto out;
	}
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
447
		if (is_present_gpte(pdpte[i]) &&
448
		    (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
449 450 451 452 453 454
			ret = 0;
			goto out;
		}
	}
	ret = 1;

455
	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
A
Avi Kivity 已提交
456 457 458 459
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
460 461 462 463
out:

	return ret;
}
464
EXPORT_SYMBOL_GPL(load_pdptrs);
465

466 467
static bool pdptrs_changed(struct kvm_vcpu *vcpu)
{
468
	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
469
	bool changed = true;
470 471
	int offset;
	gfn_t gfn;
472 473 474 475 476
	int r;

	if (is_long_mode(vcpu) || !is_pae(vcpu))
		return false;

A
Avi Kivity 已提交
477 478 479 480
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_avail))
		return true;

481 482
	gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT;
	offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1);
483 484
	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
				       PFERR_USER_MASK | PFERR_WRITE_MASK);
485 486
	if (r < 0)
		goto out;
487
	changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
488 489 490 491 492
out:

	return changed;
}

493
int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
494
{
495 496 497 498
	unsigned long old_cr0 = kvm_read_cr0(vcpu);
	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP |
				    X86_CR0_CD | X86_CR0_NW;

499 500
	cr0 |= X86_CR0_ET;

501
#ifdef CONFIG_X86_64
502 503
	if (cr0 & 0xffffffff00000000UL)
		return 1;
504 505 506
#endif

	cr0 &= ~CR0_RESERVED_BITS;
507

508 509
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
		return 1;
510

511 512
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
		return 1;
513 514 515

	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
#ifdef CONFIG_X86_64
516
		if ((vcpu->arch.efer & EFER_LME)) {
517 518
			int cs_db, cs_l;

519 520
			if (!is_pae(vcpu))
				return 1;
521
			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
522 523
			if (cs_l)
				return 1;
524 525
		} else
#endif
526
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
527
						 kvm_read_cr3(vcpu)))
528
			return 1;
529 530
	}

531 532 533
	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
		return 1;

534 535
	kvm_x86_ops->set_cr0(vcpu, cr0);

536
	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
537
		kvm_clear_async_pf_completion_queue(vcpu);
538 539
		kvm_async_pf_hash_reset(vcpu);
	}
540

541 542
	if ((cr0 ^ old_cr0) & update_bits)
		kvm_mmu_reset_context(vcpu);
543 544
	return 0;
}
545
EXPORT_SYMBOL_GPL(kvm_set_cr0);
546

547
void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
548
{
549
	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
550
}
551
EXPORT_SYMBOL_GPL(kvm_lmsw);
552

553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583
int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
	u64 xcr0;

	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
	if (index != XCR_XFEATURE_ENABLED_MASK)
		return 1;
	xcr0 = xcr;
	if (kvm_x86_ops->get_cpl(vcpu) != 0)
		return 1;
	if (!(xcr0 & XSTATE_FP))
		return 1;
	if ((xcr0 & XSTATE_YMM) && !(xcr0 & XSTATE_SSE))
		return 1;
	if (xcr0 & ~host_xcr0)
		return 1;
	vcpu->arch.xcr0 = xcr0;
	vcpu->guest_xcr0_loaded = 0;
	return 0;
}

int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
	if (__kvm_set_xcr(vcpu, index, xcr)) {
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_xcr);

584
int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
585
{
586
	unsigned long old_cr4 = kvm_read_cr4(vcpu);
587 588
	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE |
				   X86_CR4_PAE | X86_CR4_SMEP;
589 590
	if (cr4 & CR4_RESERVED_BITS)
		return 1;
591

592 593 594
	if (!guest_cpuid_has_xsave(vcpu) && (cr4 & X86_CR4_OSXSAVE))
		return 1;

595 596 597
	if (!guest_cpuid_has_smep(vcpu) && (cr4 & X86_CR4_SMEP))
		return 1;

598 599 600
	if (!guest_cpuid_has_fsgsbase(vcpu) && (cr4 & X86_CR4_RDWRGSFS))
		return 1;

601
	if (is_long_mode(vcpu)) {
602 603
		if (!(cr4 & X86_CR4_PAE))
			return 1;
604 605
	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
		   && ((cr4 ^ old_cr4) & pdptr_bits)
606 607
		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
				   kvm_read_cr3(vcpu)))
608 609
		return 1;

610 611 612 613 614 615 616 617 618
	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
		if (!guest_cpuid_has_pcid(vcpu))
			return 1;

		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
			return 1;
	}

619
	if (kvm_x86_ops->set_cr4(vcpu, cr4))
620
		return 1;
621

622 623
	if (((cr4 ^ old_cr4) & pdptr_bits) ||
	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
624
		kvm_mmu_reset_context(vcpu);
625

626
	if ((cr4 ^ old_cr4) & X86_CR4_OSXSAVE)
A
Avi Kivity 已提交
627
		kvm_update_cpuid(vcpu);
628

629 630
	return 0;
}
631
EXPORT_SYMBOL_GPL(kvm_set_cr4);
632

633
int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
634
{
635
	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
636
		kvm_mmu_sync_roots(vcpu);
637
		kvm_mmu_flush_tlb(vcpu);
638
		return 0;
639 640
	}

641
	if (is_long_mode(vcpu)) {
642
		if (kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)) {
643 644 645 646 647
			if (cr3 & CR3_PCID_ENABLED_RESERVED_BITS)
				return 1;
		} else
			if (cr3 & CR3_L_MODE_RESERVED_BITS)
				return 1;
648 649
	} else {
		if (is_pae(vcpu)) {
650 651
			if (cr3 & CR3_PAE_RESERVED_BITS)
				return 1;
652 653
			if (is_paging(vcpu) &&
			    !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
654
				return 1;
655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
		}
		/*
		 * We don't check reserved bits in nonpae mode, because
		 * this isn't enforced, and VMware depends on this.
		 */
	}

	/*
	 * Does the new cr3 value map to physical memory? (Note, we
	 * catch an invalid cr3 even in real-mode, because it would
	 * cause trouble later on when we turn on paging anyway.)
	 *
	 * A real CPU would silently accept an invalid cr3 and would
	 * attempt to use it - with largely undefined (and often hard
	 * to debug) behavior on the guest side.
	 */
	if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
672 673
		return 1;
	vcpu->arch.cr3 = cr3;
674
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
675 676 677
	vcpu->arch.mmu.new_cr3(vcpu);
	return 0;
}
678
EXPORT_SYMBOL_GPL(kvm_set_cr3);
679

A
Andre Przywara 已提交
680
int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
681
{
682 683
	if (cr8 & CR8_RESERVED_BITS)
		return 1;
684 685 686
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_lapic_set_tpr(vcpu, cr8);
	else
687
		vcpu->arch.cr8 = cr8;
688 689
	return 0;
}
690
EXPORT_SYMBOL_GPL(kvm_set_cr8);
691

692
unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
693 694 695 696
{
	if (irqchip_in_kernel(vcpu->kvm))
		return kvm_lapic_get_cr8(vcpu);
	else
697
		return vcpu->arch.cr8;
698
}
699
EXPORT_SYMBOL_GPL(kvm_get_cr8);
700

701 702 703 704 705 706 707 708 709 710 711 712
static void kvm_update_dr7(struct kvm_vcpu *vcpu)
{
	unsigned long dr7;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		dr7 = vcpu->arch.guest_debug_dr7;
	else
		dr7 = vcpu->arch.dr7;
	kvm_x86_ops->set_dr7(vcpu, dr7);
	vcpu->arch.switch_db_regs = (dr7 & DR7_BP_EN_MASK);
}

713
static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
714 715 716 717 718 719 720 721
{
	switch (dr) {
	case 0 ... 3:
		vcpu->arch.db[dr] = val;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = val;
		break;
	case 4:
722 723
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
			return 1; /* #UD */
724 725
		/* fall through */
	case 6:
726 727
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
728 729 730
		vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
		break;
	case 5:
731 732
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
			return 1; /* #UD */
733 734
		/* fall through */
	default: /* 7 */
735 736
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
737
		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
738
		kvm_update_dr7(vcpu);
739 740 741 742 743
		break;
	}

	return 0;
}
744 745 746 747 748 749 750 751 752 753 754 755 756

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
	int res;

	res = __kvm_set_dr(vcpu, dr, val);
	if (res > 0)
		kvm_queue_exception(vcpu, UD_VECTOR);
	else if (res < 0)
		kvm_inject_gp(vcpu, 0);

	return res;
}
757 758
EXPORT_SYMBOL_GPL(kvm_set_dr);

759
static int _kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
760 761 762 763 764 765
{
	switch (dr) {
	case 0 ... 3:
		*val = vcpu->arch.db[dr];
		break;
	case 4:
766
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
767 768 769 770 771 772
			return 1;
		/* fall through */
	case 6:
		*val = vcpu->arch.dr6;
		break;
	case 5:
773
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
774 775 776 777 778 779 780 781 782
			return 1;
		/* fall through */
	default: /* 7 */
		*val = vcpu->arch.dr7;
		break;
	}

	return 0;
}
783 784 785 786 787 788 789 790 791

int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
{
	if (_kvm_get_dr(vcpu, dr, val)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}
	return 0;
}
792 793
EXPORT_SYMBOL_GPL(kvm_get_dr);

A
Avi Kivity 已提交
794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
bool kvm_rdpmc(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
	u64 data;
	int err;

	err = kvm_pmu_read_pmc(vcpu, ecx, &data);
	if (err)
		return err;
	kvm_register_write(vcpu, VCPU_REGS_RAX, (u32)data);
	kvm_register_write(vcpu, VCPU_REGS_RDX, data >> 32);
	return err;
}
EXPORT_SYMBOL_GPL(kvm_rdpmc);

809 810 811 812 813
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
 * This list is modified at module load time to reflect the
814 815
 * capabilities of the host cpu. This capabilities test skips MSRs that are
 * kvm-specific. Those are put in the beginning of the list.
816
 */
817

818
#define KVM_SAVE_MSRS_BEGIN	10
819
static u32 msrs_to_save[] = {
820
	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
821
	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
822
	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
G
Glauber Costa 已提交
823
	HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
824
	MSR_KVM_PV_EOI_EN,
825
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
B
Brian Gerst 已提交
826
	MSR_STAR,
827 828 829
#ifdef CONFIG_X86_64
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
830
	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA
831 832 833 834
};

static unsigned num_msrs_to_save;

M
Mathias Krause 已提交
835
static const u32 emulated_msrs[] = {
W
Will Auld 已提交
836
	MSR_IA32_TSC_ADJUST,
837
	MSR_IA32_TSCDEADLINE,
838
	MSR_IA32_MISC_ENABLE,
839 840
	MSR_IA32_MCG_STATUS,
	MSR_IA32_MCG_CTL,
841 842
};

843
static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
844
{
845 846
	u64 old_efer = vcpu->arch.efer;

847 848
	if (efer & efer_reserved_bits)
		return 1;
849 850

	if (is_paging(vcpu)
851 852
	    && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
		return 1;
853

A
Alexander Graf 已提交
854 855 856 857
	if (efer & EFER_FFXSR) {
		struct kvm_cpuid_entry2 *feat;

		feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
858 859
		if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT)))
			return 1;
A
Alexander Graf 已提交
860 861
	}

862 863 864 865
	if (efer & EFER_SVME) {
		struct kvm_cpuid_entry2 *feat;

		feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
866 867
		if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM)))
			return 1;
868 869
	}

870
	efer &= ~EFER_LMA;
871
	efer |= vcpu->arch.efer & EFER_LMA;
872

873 874
	kvm_x86_ops->set_efer(vcpu, efer);

875 876 877 878
	/* Update reserved bits */
	if ((efer ^ old_efer) & EFER_NX)
		kvm_mmu_reset_context(vcpu);

879
	return 0;
880 881
}

882 883 884 885 886 887 888
void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);


889 890 891 892 893
/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
894
int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
895
{
896
	return kvm_x86_ops->set_msr(vcpu, msr);
897 898
}

899 900 901 902 903
/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
904 905 906 907 908 909
	struct msr_data msr;

	msr.data = *data;
	msr.index = index;
	msr.host_initiated = true;
	return kvm_set_msr(vcpu, &msr);
910 911
}

912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960
#ifdef CONFIG_X86_64
struct pvclock_gtod_data {
	seqcount_t	seq;

	struct { /* extract of a clocksource struct */
		int vclock_mode;
		cycle_t	cycle_last;
		cycle_t	mask;
		u32	mult;
		u32	shift;
	} clock;

	/* open coded 'struct timespec' */
	u64		monotonic_time_snsec;
	time_t		monotonic_time_sec;
};

static struct pvclock_gtod_data pvclock_gtod_data;

static void update_pvclock_gtod(struct timekeeper *tk)
{
	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;

	write_seqcount_begin(&vdata->seq);

	/* copy pvclock gtod data */
	vdata->clock.vclock_mode	= tk->clock->archdata.vclock_mode;
	vdata->clock.cycle_last		= tk->clock->cycle_last;
	vdata->clock.mask		= tk->clock->mask;
	vdata->clock.mult		= tk->mult;
	vdata->clock.shift		= tk->shift;

	vdata->monotonic_time_sec	= tk->xtime_sec
					+ tk->wall_to_monotonic.tv_sec;
	vdata->monotonic_time_snsec	= tk->xtime_nsec
					+ (tk->wall_to_monotonic.tv_nsec
						<< tk->shift);
	while (vdata->monotonic_time_snsec >=
					(((u64)NSEC_PER_SEC) << tk->shift)) {
		vdata->monotonic_time_snsec -=
					((u64)NSEC_PER_SEC) << tk->shift;
		vdata->monotonic_time_sec++;
	}

	write_seqcount_end(&vdata->seq);
}
#endif


961 962
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
{
963 964
	int version;
	int r;
965
	struct pvclock_wall_clock wc;
966
	struct timespec boot;
967 968 969 970

	if (!wall_clock)
		return;

971 972 973 974 975 976 977 978
	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
	if (r)
		return;

	if (version & 1)
		++version;  /* first time write, random junk */

	++version;
979 980 981

	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));

982 983
	/*
	 * The guest calculates current wall clock time by adding
Z
Zachary Amsden 已提交
984
	 * system time (updated by kvm_guest_time_update below) to the
985 986 987
	 * wall clock specified here.  guest system time equals host
	 * system time for us, thus we must fill in host boot time here.
	 */
988
	getboottime(&boot);
989

990 991 992 993
	if (kvm->arch.kvmclock_offset) {
		struct timespec ts = ns_to_timespec(kvm->arch.kvmclock_offset);
		boot = timespec_sub(boot, ts);
	}
994 995 996
	wc.sec = boot.tv_sec;
	wc.nsec = boot.tv_nsec;
	wc.version = version;
997 998 999 1000 1001 1002 1003

	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));

	version++;
	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
}

1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
	uint32_t quotient, remainder;

	/* Don't try to replace with do_div(), this one calculates
	 * "(dividend << 32) / divisor" */
	__asm__ ( "divl %4"
		  : "=a" (quotient), "=d" (remainder)
		  : "0" (0), "1" (dividend), "r" (divisor) );
	return quotient;
}

1016 1017
static void kvm_get_time_scale(uint32_t scaled_khz, uint32_t base_khz,
			       s8 *pshift, u32 *pmultiplier)
1018
{
1019
	uint64_t scaled64;
1020 1021 1022 1023
	int32_t  shift = 0;
	uint64_t tps64;
	uint32_t tps32;

1024 1025
	tps64 = base_khz * 1000LL;
	scaled64 = scaled_khz * 1000LL;
1026
	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
1027 1028 1029 1030 1031
		tps64 >>= 1;
		shift--;
	}

	tps32 = (uint32_t)tps64;
1032 1033
	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
1034 1035 1036
			scaled64 >>= 1;
		else
			tps32 <<= 1;
1037 1038 1039
		shift++;
	}

1040 1041
	*pshift = shift;
	*pmultiplier = div_frac(scaled64, tps32);
1042

1043 1044
	pr_debug("%s: base_khz %u => %u, shift %d, mul %u\n",
		 __func__, base_khz, scaled_khz, shift, *pmultiplier);
1045 1046
}

1047 1048 1049 1050 1051 1052 1053 1054
static inline u64 get_kernel_ns(void)
{
	struct timespec ts;

	WARN_ON(preemptible());
	ktime_get_ts(&ts);
	monotonic_to_bootbased(&ts);
	return timespec_to_ns(&ts);
1055 1056
}

1057
#ifdef CONFIG_X86_64
1058
static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
1059
#endif
1060

1061
static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
Z
Zachary Amsden 已提交
1062
unsigned long max_tsc_khz;
1063

1064
static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
1065
{
1066 1067
	return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
				   vcpu->arch.virtual_tsc_shift);
1068 1069
}

1070
static u32 adjust_tsc_khz(u32 khz, s32 ppm)
1071
{
1072 1073 1074
	u64 v = (u64)khz * (1000000 + ppm);
	do_div(v, 1000000);
	return v;
1075 1076
}

1077
static void kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 this_tsc_khz)
1078
{
1079 1080
	u32 thresh_lo, thresh_hi;
	int use_scaling = 0;
1081

Z
Zachary Amsden 已提交
1082 1083
	/* Compute a scale to convert nanoseconds in TSC cycles */
	kvm_get_time_scale(this_tsc_khz, NSEC_PER_SEC / 1000,
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
			   &vcpu->arch.virtual_tsc_shift,
			   &vcpu->arch.virtual_tsc_mult);
	vcpu->arch.virtual_tsc_khz = this_tsc_khz;

	/*
	 * Compute the variation in TSC rate which is acceptable
	 * within the range of tolerance and decide if the
	 * rate being applied is within that bounds of the hardware
	 * rate.  If so, no scaling or compensation need be done.
	 */
	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
	if (this_tsc_khz < thresh_lo || this_tsc_khz > thresh_hi) {
		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", this_tsc_khz, thresh_lo, thresh_hi);
		use_scaling = 1;
	}
	kvm_x86_ops->set_tsc_khz(vcpu, this_tsc_khz, use_scaling);
Z
Zachary Amsden 已提交
1101 1102 1103 1104
}

static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
1105
	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
1106 1107
				      vcpu->arch.virtual_tsc_mult,
				      vcpu->arch.virtual_tsc_shift);
1108
	tsc += vcpu->arch.this_tsc_write;
Z
Zachary Amsden 已提交
1109 1110 1111
	return tsc;
}

1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
	bool vcpus_matched;
	bool do_request = false;
	struct kvm_arch *ka = &vcpu->kvm->arch;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			 atomic_read(&vcpu->kvm->online_vcpus));

	if (vcpus_matched && gtod->clock.vclock_mode == VCLOCK_TSC)
		if (!ka->use_master_clock)
			do_request = 1;

	if (!vcpus_matched && ka->use_master_clock)
			do_request = 1;

	if (do_request)
		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);

	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
			    atomic_read(&vcpu->kvm->online_vcpus),
		            ka->use_master_clock, gtod->clock.vclock_mode);
#endif
}

W
Will Auld 已提交
1139 1140 1141 1142 1143 1144
static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
{
	u64 curr_offset = kvm_x86_ops->read_tsc_offset(vcpu);
	vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
}

1145
void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
1146 1147
{
	struct kvm *kvm = vcpu->kvm;
Z
Zachary Amsden 已提交
1148
	u64 offset, ns, elapsed;
1149
	unsigned long flags;
1150
	s64 usdiff;
1151
	bool matched;
1152
	u64 data = msr->data;
1153

1154
	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
1155
	offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
1156
	ns = get_kernel_ns();
Z
Zachary Amsden 已提交
1157
	elapsed = ns - kvm->arch.last_tsc_nsec;
1158 1159

	/* n.b - signed multiplication and division required */
1160
	usdiff = data - kvm->arch.last_tsc_write;
1161
#ifdef CONFIG_X86_64
1162
	usdiff = (usdiff * 1000) / vcpu->arch.virtual_tsc_khz;
1163 1164 1165
#else
	/* do_div() only does unsigned */
	asm("idivl %2; xor %%edx, %%edx"
1166 1167
	    : "=A"(usdiff)
	    : "A"(usdiff * 1000), "rm"(vcpu->arch.virtual_tsc_khz));
1168
#endif
1169 1170 1171 1172
	do_div(elapsed, 1000);
	usdiff -= elapsed;
	if (usdiff < 0)
		usdiff = -usdiff;
Z
Zachary Amsden 已提交
1173 1174

	/*
1175 1176 1177 1178 1179 1180 1181 1182 1183
	 * Special case: TSC write with a small delta (1 second) of virtual
	 * cycle time against real time is interpreted as an attempt to
	 * synchronize the CPU.
         *
	 * For a reliable TSC, we can match TSC offsets, and for an unstable
	 * TSC, we add elapsed time in this computation.  We could let the
	 * compensation code attempt to catch up if we fall behind, but
	 * it's better to try to match offsets from the beginning.
         */
1184
	if (usdiff < USEC_PER_SEC &&
1185
	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
Z
Zachary Amsden 已提交
1186
		if (!check_tsc_unstable()) {
1187
			offset = kvm->arch.cur_tsc_offset;
Z
Zachary Amsden 已提交
1188 1189
			pr_debug("kvm: matched tsc offset for %llu\n", data);
		} else {
1190
			u64 delta = nsec_to_cycles(vcpu, elapsed);
1191 1192
			data += delta;
			offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
1193
			pr_debug("kvm: adjusted tsc offset by %llu\n", delta);
Z
Zachary Amsden 已提交
1194
		}
1195
		matched = true;
1196 1197 1198 1199 1200 1201
	} else {
		/*
		 * We split periods of matched TSC writes into generations.
		 * For each generation, we track the original measured
		 * nanosecond time, offset, and write, so if TSCs are in
		 * sync, we can match exact offset, and if not, we can match
G
Guo Chao 已提交
1202
		 * exact software computation in compute_guest_tsc()
1203 1204 1205 1206 1207 1208 1209
		 *
		 * These values are tracked in kvm->arch.cur_xxx variables.
		 */
		kvm->arch.cur_tsc_generation++;
		kvm->arch.cur_tsc_nsec = ns;
		kvm->arch.cur_tsc_write = data;
		kvm->arch.cur_tsc_offset = offset;
1210
		matched = false;
1211 1212
		pr_debug("kvm: new tsc generation %u, clock %llu\n",
			 kvm->arch.cur_tsc_generation, data);
Z
Zachary Amsden 已提交
1213
	}
1214 1215 1216 1217 1218

	/*
	 * We also track th most recent recorded KHZ, write and time to
	 * allow the matching interval to be extended at each write.
	 */
Z
Zachary Amsden 已提交
1219 1220
	kvm->arch.last_tsc_nsec = ns;
	kvm->arch.last_tsc_write = data;
1221
	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
1222 1223 1224

	/* Reset of TSC must disable overshoot protection below */
	vcpu->arch.hv_clock.tsc_timestamp = 0;
1225
	vcpu->arch.last_guest_tsc = data;
1226 1227 1228 1229 1230 1231

	/* Keep track of which generation this VCPU has synchronized to */
	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;

W
Will Auld 已提交
1232 1233
	if (guest_cpuid_has_tsc_adjust(vcpu) && !msr->host_initiated)
		update_ia32_tsc_adjust_msr(vcpu, offset);
1234 1235
	kvm_x86_ops->write_tsc_offset(vcpu, offset);
	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
1236 1237 1238 1239 1240 1241 1242 1243 1244

	spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
	if (matched)
		kvm->arch.nr_vcpus_matched_tsc++;
	else
		kvm->arch.nr_vcpus_matched_tsc = 0;

	kvm_track_tsc_matching(vcpu);
	spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
1245
}
1246

1247 1248
EXPORT_SYMBOL_GPL(kvm_write_tsc);

1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
#ifdef CONFIG_X86_64

static cycle_t read_tsc(void)
{
	cycle_t ret;
	u64 last;

	/*
	 * Empirically, a fence (of type that depends on the CPU)
	 * before rdtsc is enough to ensure that rdtsc is ordered
	 * with respect to loads.  The various CPU manuals are unclear
	 * as to whether rdtsc can be reordered with later loads,
	 * but no one has ever seen it happen.
	 */
	rdtsc_barrier();
	ret = (cycle_t)vget_cycles();

	last = pvclock_gtod_data.clock.cycle_last;

	if (likely(ret >= last))
		return ret;

	/*
	 * GCC likes to generate cmov here, but this branch is extremely
	 * predictable (it's just a funciton of time and the likely is
	 * very likely) and there's a data dependence, so force GCC
	 * to generate a branch instead.  I don't barrier() because
	 * we don't actually need a barrier, and if this function
	 * ever gets inlined it will generate worse code.
	 */
	asm volatile ("");
	return last;
}

static inline u64 vgettsc(cycle_t *cycle_now)
{
	long v;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	*cycle_now = read_tsc();

	v = (*cycle_now - gtod->clock.cycle_last) & gtod->clock.mask;
	return v * gtod->clock.mult;
}

static int do_monotonic(struct timespec *ts, cycle_t *cycle_now)
{
	unsigned long seq;
	u64 ns;
	int mode;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	ts->tv_nsec = 0;
	do {
		seq = read_seqcount_begin(&gtod->seq);
		mode = gtod->clock.vclock_mode;
		ts->tv_sec = gtod->monotonic_time_sec;
		ns = gtod->monotonic_time_snsec;
		ns += vgettsc(cycle_now);
		ns >>= gtod->clock.shift;
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
	timespec_add_ns(ts, ns);

	return mode;
}

/* returns true if host is using tsc clocksource */
static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now)
{
	struct timespec ts;

	/* checked again under seqlock below */
	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
		return false;

	if (do_monotonic(&ts, cycle_now) != VCLOCK_TSC)
		return false;

	monotonic_to_bootbased(&ts);
	*kernel_ns = timespec_to_ns(&ts);

	return true;
}
#endif

/*
 *
1336 1337 1338
 * Assuming a stable TSC across physical CPUS, and a stable TSC
 * across virtual CPUs, the following condition is possible.
 * Each numbered line represents an event visible to both
1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
 * CPUs at the next numbered event.
 *
 * "timespecX" represents host monotonic time. "tscX" represents
 * RDTSC value.
 *
 * 		VCPU0 on CPU0		|	VCPU1 on CPU1
 *
 * 1.  read timespec0,tsc0
 * 2.					| timespec1 = timespec0 + N
 * 					| tsc1 = tsc0 + M
 * 3. transition to guest		| transition to guest
 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
 * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
 * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
 *
 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
 *
 * 	- ret0 < ret1
 *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
 *		...
 *	- 0 < N - M => M < N
 *
 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
 * always the case (the difference between two distinct xtime instances
 * might be smaller then the difference between corresponding TSC reads,
 * when updating guest vcpus pvclock areas).
 *
 * To avoid that problem, do not allow visibility of distinct
 * system_timestamp/tsc_timestamp values simultaneously: use a master
 * copy of host monotonic time values. Update that master copy
 * in lockstep.
 *
1371
 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
1372 1373 1374 1375 1376 1377 1378 1379
 *
 */

static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	struct kvm_arch *ka = &kvm->arch;
	int vclock_mode;
1380 1381 1382 1383
	bool host_tsc_clocksource, vcpus_matched;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			atomic_read(&kvm->online_vcpus));
1384 1385 1386 1387 1388

	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
1389
	host_tsc_clocksource = kvm_get_time_and_clockread(
1390 1391 1392
					&ka->master_kernel_ns,
					&ka->master_cycle_now);

1393 1394
	ka->use_master_clock = host_tsc_clocksource & vcpus_matched;

1395 1396 1397 1398
	if (ka->use_master_clock)
		atomic_set(&kvm_guest_has_master_clock, 1);

	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
1399 1400
	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
					vcpus_matched);
1401 1402 1403
#endif
}

Z
Zachary Amsden 已提交
1404
static int kvm_guest_time_update(struct kvm_vcpu *v)
1405
{
1406
	unsigned long flags, this_tsc_khz;
1407
	struct kvm_vcpu_arch *vcpu = &v->arch;
1408
	struct kvm_arch *ka = &v->kvm->arch;
1409
	void *shared_kaddr;
1410
	s64 kernel_ns, max_kernel_ns;
1411
	u64 tsc_timestamp, host_tsc;
1412
	struct pvclock_vcpu_time_info *guest_hv_clock;
1413
	u8 pvclock_flags;
1414 1415 1416 1417
	bool use_master_clock;

	kernel_ns = 0;
	host_tsc = 0;
1418 1419 1420

	/* Keep irq disabled to prevent changes to the clock */
	local_irq_save(flags);
1421
	this_tsc_khz = __get_cpu_var(cpu_tsc_khz);
1422
	if (unlikely(this_tsc_khz == 0)) {
Z
Zachary Amsden 已提交
1423
		local_irq_restore(flags);
Z
Zachary Amsden 已提交
1424
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
1425 1426
		return 1;
	}
1427

1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
	spin_lock(&ka->pvclock_gtod_sync_lock);
	use_master_clock = ka->use_master_clock;
	if (use_master_clock) {
		host_tsc = ka->master_cycle_now;
		kernel_ns = ka->master_kernel_ns;
	}
	spin_unlock(&ka->pvclock_gtod_sync_lock);
	if (!use_master_clock) {
		host_tsc = native_read_tsc();
		kernel_ns = get_kernel_ns();
	}

	tsc_timestamp = kvm_x86_ops->read_l1_tsc(v, host_tsc);

Z
Zachary Amsden 已提交
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458
	/*
	 * We may have to catch up the TSC to match elapsed wall clock
	 * time for two reasons, even if kvmclock is used.
	 *   1) CPU could have been running below the maximum TSC rate
	 *   2) Broken TSC compensation resets the base at each VCPU
	 *      entry to avoid unknown leaps of TSC even when running
	 *      again on the same CPU.  This may cause apparent elapsed
	 *      time to disappear, and the guest to stand still or run
	 *	very slowly.
	 */
	if (vcpu->tsc_catchup) {
		u64 tsc = compute_guest_tsc(v, kernel_ns);
		if (tsc > tsc_timestamp) {
1459
			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
Z
Zachary Amsden 已提交
1460 1461
			tsc_timestamp = tsc;
		}
1462 1463
	}

1464 1465
	local_irq_restore(flags);

Z
Zachary Amsden 已提交
1466 1467
	if (!vcpu->time_page)
		return 0;
1468

1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490
	/*
	 * Time as measured by the TSC may go backwards when resetting the base
	 * tsc_timestamp.  The reason for this is that the TSC resolution is
	 * higher than the resolution of the other clock scales.  Thus, many
	 * possible measurments of the TSC correspond to one measurement of any
	 * other clock, and so a spread of values is possible.  This is not a
	 * problem for the computation of the nanosecond clock; with TSC rates
	 * around 1GHZ, there can only be a few cycles which correspond to one
	 * nanosecond value, and any path through this code will inevitably
	 * take longer than that.  However, with the kernel_ns value itself,
	 * the precision may be much lower, down to HZ granularity.  If the
	 * first sampling of TSC against kernel_ns ends in the low part of the
	 * range, and the second in the high end of the range, we can get:
	 *
	 * (TSC - offset_low) * S + kns_old > (TSC - offset_high) * S + kns_new
	 *
	 * As the sampling errors potentially range in the thousands of cycles,
	 * it is possible such a time value has already been observed by the
	 * guest.  To protect against this, we must compute the system time as
	 * observed by the guest and ensure the new system time is greater.
	 */
	max_kernel_ns = 0;
1491
	if (vcpu->hv_clock.tsc_timestamp) {
1492 1493 1494 1495 1496 1497 1498
		max_kernel_ns = vcpu->last_guest_tsc -
				vcpu->hv_clock.tsc_timestamp;
		max_kernel_ns = pvclock_scale_delta(max_kernel_ns,
				    vcpu->hv_clock.tsc_to_system_mul,
				    vcpu->hv_clock.tsc_shift);
		max_kernel_ns += vcpu->last_kernel_ns;
	}
1499

Z
Zachary Amsden 已提交
1500
	if (unlikely(vcpu->hw_tsc_khz != this_tsc_khz)) {
1501 1502 1503
		kvm_get_time_scale(NSEC_PER_SEC / 1000, this_tsc_khz,
				   &vcpu->hv_clock.tsc_shift,
				   &vcpu->hv_clock.tsc_to_system_mul);
Z
Zachary Amsden 已提交
1504
		vcpu->hw_tsc_khz = this_tsc_khz;
1505 1506
	}

1507 1508 1509 1510 1511 1512 1513 1514
	/* with a master <monotonic time, tsc value> tuple,
	 * pvclock clock reads always increase at the (scaled) rate
	 * of guest TSC - no need to deal with sampling errors.
	 */
	if (!use_master_clock) {
		if (max_kernel_ns > kernel_ns)
			kernel_ns = max_kernel_ns;
	}
1515
	/* With all the info we got, fill in the values */
1516
	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
1517
	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
1518
	vcpu->last_kernel_ns = kernel_ns;
Z
Zachary Amsden 已提交
1519
	vcpu->last_guest_tsc = tsc_timestamp;
1520

1521 1522 1523
	/*
	 * The interface expects us to write an even number signaling that the
	 * update is finished. Since the guest won't see the intermediate
1524
	 * state, we just increase by 2 at the end.
1525
	 */
1526
	vcpu->hv_clock.version += 2;
1527

1528
	shared_kaddr = kmap_atomic(vcpu->time_page);
1529

1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
	guest_hv_clock = shared_kaddr + vcpu->time_offset;

	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
	pvclock_flags = (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);

	if (vcpu->pvclock_set_guest_stopped_request) {
		pvclock_flags |= PVCLOCK_GUEST_STOPPED;
		vcpu->pvclock_set_guest_stopped_request = false;
	}

1540 1541 1542 1543
	/* If the host uses TSC clocksource, then it is stable */
	if (use_master_clock)
		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;

1544 1545
	vcpu->hv_clock.flags = pvclock_flags;

1546
	memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock,
1547
	       sizeof(vcpu->hv_clock));
1548

1549
	kunmap_atomic(shared_kaddr);
1550 1551

	mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT);
1552
	return 0;
1553 1554
}

A
Avi Kivity 已提交
1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578
static bool msr_mtrr_valid(unsigned msr)
{
	switch (msr) {
	case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
	case MSR_MTRRfix64K_00000:
	case MSR_MTRRfix16K_80000:
	case MSR_MTRRfix16K_A0000:
	case MSR_MTRRfix4K_C0000:
	case MSR_MTRRfix4K_C8000:
	case MSR_MTRRfix4K_D0000:
	case MSR_MTRRfix4K_D8000:
	case MSR_MTRRfix4K_E0000:
	case MSR_MTRRfix4K_E8000:
	case MSR_MTRRfix4K_F0000:
	case MSR_MTRRfix4K_F8000:
	case MSR_MTRRdefType:
	case MSR_IA32_CR_PAT:
		return true;
	case 0x2f8:
		return true;
	}
	return false;
}

1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
static bool valid_pat_type(unsigned t)
{
	return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
}

static bool valid_mtrr_type(unsigned t)
{
	return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
}

static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	int i;

	if (!msr_mtrr_valid(msr))
		return false;

	if (msr == MSR_IA32_CR_PAT) {
		for (i = 0; i < 8; i++)
			if (!valid_pat_type((data >> (i * 8)) & 0xff))
				return false;
		return true;
	} else if (msr == MSR_MTRRdefType) {
		if (data & ~0xcff)
			return false;
		return valid_mtrr_type(data & 0xff);
	} else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
		for (i = 0; i < 8 ; i++)
			if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
				return false;
		return true;
	}

	/* variable MTRRs */
	return valid_mtrr_type(data & 0xff);
}

A
Avi Kivity 已提交
1616 1617
static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
S
Sheng Yang 已提交
1618 1619
	u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;

1620
	if (!mtrr_valid(vcpu, msr, data))
A
Avi Kivity 已提交
1621 1622
		return 1;

S
Sheng Yang 已提交
1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649
	if (msr == MSR_MTRRdefType) {
		vcpu->arch.mtrr_state.def_type = data;
		vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
	} else if (msr == MSR_MTRRfix64K_00000)
		p[0] = data;
	else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
		p[1 + msr - MSR_MTRRfix16K_80000] = data;
	else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
		p[3 + msr - MSR_MTRRfix4K_C0000] = data;
	else if (msr == MSR_IA32_CR_PAT)
		vcpu->arch.pat = data;
	else {	/* Variable MTRRs */
		int idx, is_mtrr_mask;
		u64 *pt;

		idx = (msr - 0x200) / 2;
		is_mtrr_mask = msr - 0x200 - 2 * idx;
		if (!is_mtrr_mask)
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
		else
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
		*pt = data;
	}

	kvm_mmu_reset_context(vcpu);
A
Avi Kivity 已提交
1650 1651
	return 0;
}
1652

H
Huang Ying 已提交
1653
static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1654
{
H
Huang Ying 已提交
1655 1656 1657
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;

1658 1659
	switch (msr) {
	case MSR_IA32_MCG_STATUS:
H
Huang Ying 已提交
1660
		vcpu->arch.mcg_status = data;
1661
		break;
1662
	case MSR_IA32_MCG_CTL:
H
Huang Ying 已提交
1663 1664 1665 1666 1667 1668 1669 1670 1671 1672
		if (!(mcg_cap & MCG_CTL_P))
			return 1;
		if (data != 0 && data != ~(u64)0)
			return -1;
		vcpu->arch.mcg_ctl = data;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
		    msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
			u32 offset = msr - MSR_IA32_MC0_CTL;
1673 1674 1675 1676 1677
			/* only 0 or all 1s can be written to IA32_MCi_CTL
			 * some Linux kernels though clear bit 10 in bank 4 to
			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
			 * this to avoid an uncatched #GP in the guest
			 */
H
Huang Ying 已提交
1678
			if ((offset & 0x3) == 0 &&
1679
			    data != 0 && (data | (1 << 10)) != ~(u64)0)
H
Huang Ying 已提交
1680 1681 1682 1683 1684 1685 1686 1687 1688
				return -1;
			vcpu->arch.mce_banks[offset] = data;
			break;
		}
		return 1;
	}
	return 0;
}

E
Ed Swierk 已提交
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705
static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
	int lm = is_long_mode(vcpu);
	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
		: kvm->arch.xen_hvm_config.blob_size_32;
	u32 page_num = data & ~PAGE_MASK;
	u64 page_addr = data & PAGE_MASK;
	u8 *page;
	int r;

	r = -E2BIG;
	if (page_num >= blob_size)
		goto out;
	r = -ENOMEM;
1706 1707 1708
	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
	if (IS_ERR(page)) {
		r = PTR_ERR(page);
E
Ed Swierk 已提交
1709
		goto out;
1710
	}
E
Ed Swierk 已提交
1711 1712 1713 1714 1715 1716 1717 1718 1719
	if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
		goto out_free;
	r = 0;
out_free:
	kfree(page);
out:
	return r;
}

1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766
static bool kvm_hv_hypercall_enabled(struct kvm *kvm)
{
	return kvm->arch.hv_hypercall & HV_X64_MSR_HYPERCALL_ENABLE;
}

static bool kvm_hv_msr_partition_wide(u32 msr)
{
	bool r = false;
	switch (msr) {
	case HV_X64_MSR_GUEST_OS_ID:
	case HV_X64_MSR_HYPERCALL:
		r = true;
		break;
	}

	return r;
}

static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	struct kvm *kvm = vcpu->kvm;

	switch (msr) {
	case HV_X64_MSR_GUEST_OS_ID:
		kvm->arch.hv_guest_os_id = data;
		/* setting guest os id to zero disables hypercall page */
		if (!kvm->arch.hv_guest_os_id)
			kvm->arch.hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
		break;
	case HV_X64_MSR_HYPERCALL: {
		u64 gfn;
		unsigned long addr;
		u8 instructions[4];

		/* if guest os id is not set hypercall should remain disabled */
		if (!kvm->arch.hv_guest_os_id)
			break;
		if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) {
			kvm->arch.hv_hypercall = data;
			break;
		}
		gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;
		addr = gfn_to_hva(kvm, gfn);
		if (kvm_is_error_hva(addr))
			return 1;
		kvm_x86_ops->patch_hypercall(vcpu, instructions);
		((unsigned char *)instructions)[3] = 0xc3; /* ret */
1767
		if (__copy_to_user((void __user *)addr, instructions, 4))
1768 1769 1770 1771 1772
			return 1;
		kvm->arch.hv_hypercall = data;
		break;
	}
	default:
1773 1774
		vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
			    "data 0x%llx\n", msr, data);
1775 1776 1777 1778 1779 1780 1781
		return 1;
	}
	return 0;
}

static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
G
Gleb Natapov 已提交
1782 1783 1784
	switch (msr) {
	case HV_X64_MSR_APIC_ASSIST_PAGE: {
		unsigned long addr;
1785

G
Gleb Natapov 已提交
1786 1787 1788 1789 1790 1791 1792 1793
		if (!(data & HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE)) {
			vcpu->arch.hv_vapic = data;
			break;
		}
		addr = gfn_to_hva(vcpu->kvm, data >>
				  HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT);
		if (kvm_is_error_hva(addr))
			return 1;
1794
		if (__clear_user((void __user *)addr, PAGE_SIZE))
G
Gleb Natapov 已提交
1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805
			return 1;
		vcpu->arch.hv_vapic = data;
		break;
	}
	case HV_X64_MSR_EOI:
		return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data);
	case HV_X64_MSR_ICR:
		return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data);
	case HV_X64_MSR_TPR:
		return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data);
	default:
1806 1807
		vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
			    "data 0x%llx\n", msr, data);
G
Gleb Natapov 已提交
1808 1809 1810 1811
		return 1;
	}

	return 0;
1812 1813
}

1814 1815 1816 1817
static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
{
	gpa_t gpa = data & ~0x3f;

G
Guo Chao 已提交
1818
	/* Bits 2:5 are reserved, Should be zero */
1819
	if (data & 0x3c)
1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
		return 1;

	vcpu->arch.apf.msr_val = data;

	if (!(data & KVM_ASYNC_PF_ENABLED)) {
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_async_pf_hash_reset(vcpu);
		return 0;
	}

	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa))
		return 1;

1833
	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
1834 1835 1836 1837
	kvm_async_pf_wakeup_all(vcpu);
	return 0;
}

1838 1839 1840 1841 1842 1843 1844 1845
static void kvmclock_reset(struct kvm_vcpu *vcpu)
{
	if (vcpu->arch.time_page) {
		kvm_release_page_dirty(vcpu->arch.time_page);
		vcpu->arch.time_page = NULL;
	}
}

G
Glauber Costa 已提交
1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874
static void accumulate_steal_time(struct kvm_vcpu *vcpu)
{
	u64 delta;

	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

	delta = current->sched_info.run_delay - vcpu->arch.st.last_steal;
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
	vcpu->arch.st.accum_steal = delta;
}

static void record_steal_time(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

	if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
		return;

	vcpu->arch.st.steal.steal += vcpu->arch.st.accum_steal;
	vcpu->arch.st.steal.version += 2;
	vcpu->arch.st.accum_steal = 0;

	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
}

1875
int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1876
{
1877
	bool pr = false;
1878 1879
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
1880

1881
	switch (msr) {
1882 1883 1884 1885 1886 1887 1888 1889
	case MSR_AMD64_NB_CFG:
	case MSR_IA32_UCODE_REV:
	case MSR_IA32_UCODE_WRITE:
	case MSR_VM_HSAVE_PA:
	case MSR_AMD64_PATCH_LOADER:
	case MSR_AMD64_BU_CFG2:
		break;

1890
	case MSR_EFER:
1891
		return set_efer(vcpu, data);
1892 1893
	case MSR_K7_HWCR:
		data &= ~(u64)0x40;	/* ignore flush filter disable */
1894
		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
1895
		data &= ~(u64)0x8;	/* ignore TLB cache disable */
1896
		if (data != 0) {
1897 1898
			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
				    data);
1899 1900
			return 1;
		}
1901
		break;
1902 1903
	case MSR_FAM10H_MMIO_CONF_BASE:
		if (data != 0) {
1904 1905
			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
				    "0x%llx\n", data);
1906 1907
			return 1;
		}
1908
		break;
1909 1910 1911 1912 1913 1914 1915 1916 1917
	case MSR_IA32_DEBUGCTLMSR:
		if (!data) {
			/* We support the non-activated case already */
			break;
		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
			/* Values other than LBR and BTF are vendor-specific,
			   thus reserved and should throw a #GP */
			return 1;
		}
1918 1919
		vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
			    __func__, data);
1920
		break;
A
Avi Kivity 已提交
1921 1922
	case 0x200 ... 0x2ff:
		return set_msr_mtrr(vcpu, msr, data);
1923 1924 1925
	case MSR_IA32_APICBASE:
		kvm_set_apic_base(vcpu, data);
		break;
G
Gleb Natapov 已提交
1926 1927
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_write(vcpu, msr, data);
1928 1929 1930
	case MSR_IA32_TSCDEADLINE:
		kvm_set_lapic_tscdeadline_msr(vcpu, data);
		break;
W
Will Auld 已提交
1931 1932 1933 1934 1935 1936 1937 1938 1939
	case MSR_IA32_TSC_ADJUST:
		if (guest_cpuid_has_tsc_adjust(vcpu)) {
			if (!msr_info->host_initiated) {
				u64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
				kvm_x86_ops->adjust_tsc_offset(vcpu, adj, true);
			}
			vcpu->arch.ia32_tsc_adjust_msr = data;
		}
		break;
1940
	case MSR_IA32_MISC_ENABLE:
1941
		vcpu->arch.ia32_misc_enable_msr = data;
1942
		break;
1943
	case MSR_KVM_WALL_CLOCK_NEW:
1944 1945 1946 1947
	case MSR_KVM_WALL_CLOCK:
		vcpu->kvm->arch.wall_clock = data;
		kvm_write_wall_clock(vcpu->kvm, data);
		break;
1948
	case MSR_KVM_SYSTEM_TIME_NEW:
1949
	case MSR_KVM_SYSTEM_TIME: {
1950
		kvmclock_reset(vcpu);
1951 1952

		vcpu->arch.time = data;
Z
Zachary Amsden 已提交
1953
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964

		/* we verify if the enable bit is set... */
		if (!(data & 1))
			break;

		/* ...but clean it before doing the actual write */
		vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);

		vcpu->arch.time_page =
				gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);

1965
		if (is_error_page(vcpu->arch.time_page))
1966
			vcpu->arch.time_page = NULL;
1967

1968 1969
		break;
	}
1970 1971 1972 1973
	case MSR_KVM_ASYNC_PF_EN:
		if (kvm_pv_enable_async_pf(vcpu, data))
			return 1;
		break;
G
Glauber Costa 已提交
1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
	case MSR_KVM_STEAL_TIME:

		if (unlikely(!sched_info_on()))
			return 1;

		if (data & KVM_STEAL_RESERVED_MASK)
			return 1;

		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
							data & KVM_STEAL_VALID_BITS))
			return 1;

		vcpu->arch.st.msr_val = data;

		if (!(data & KVM_MSR_ENABLED))
			break;

		vcpu->arch.st.last_steal = current->sched_info.run_delay;

		preempt_disable();
		accumulate_steal_time(vcpu);
		preempt_enable();

		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);

		break;
2000 2001 2002 2003
	case MSR_KVM_PV_EOI_EN:
		if (kvm_lapic_enable_pv_eoi(vcpu, data))
			return 1;
		break;
G
Glauber Costa 已提交
2004

H
Huang Ying 已提交
2005 2006 2007 2008
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
	case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
		return set_msr_mce(vcpu, msr, data);
2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021

	/* Performance counters are not protected by a CPUID bit,
	 * so we should check all of them in the generic path for the sake of
	 * cross vendor migration.
	 * Writing a zero into the event select MSRs disables them,
	 * which we perfectly emulate ;-). Any other value should be at least
	 * reported, some guests depend on them.
	 */
	case MSR_K7_EVNTSEL0:
	case MSR_K7_EVNTSEL1:
	case MSR_K7_EVNTSEL2:
	case MSR_K7_EVNTSEL3:
		if (data != 0)
2022 2023
			vcpu_unimpl(vcpu, "unimplemented perfctr wrmsr: "
				    "0x%x data 0x%llx\n", msr, data);
2024 2025 2026 2027 2028 2029 2030 2031
		break;
	/* at least RHEL 4 unconditionally writes to the perfctr registers,
	 * so we ignore writes to make it happy.
	 */
	case MSR_K7_PERFCTR0:
	case MSR_K7_PERFCTR1:
	case MSR_K7_PERFCTR2:
	case MSR_K7_PERFCTR3:
2032 2033
		vcpu_unimpl(vcpu, "unimplemented perfctr wrmsr: "
			    "0x%x data 0x%llx\n", msr, data);
2034
		break;
2035 2036 2037 2038 2039 2040 2041 2042 2043
	case MSR_P6_PERFCTR0:
	case MSR_P6_PERFCTR1:
		pr = true;
	case MSR_P6_EVNTSEL0:
	case MSR_P6_EVNTSEL1:
		if (kvm_pmu_msr(vcpu, msr))
			return kvm_pmu_set_msr(vcpu, msr, data);

		if (pr || data != 0)
2044 2045
			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
				    "0x%x data 0x%llx\n", msr, data);
2046
		break;
2047 2048 2049 2050 2051
	case MSR_K7_CLK_CTL:
		/*
		 * Ignore all writes to this no longer documented MSR.
		 * Writes are only relevant for old K7 processors,
		 * all pre-dating SVM, but a recommended workaround from
G
Guo Chao 已提交
2052
		 * AMD for these chips. It is possible to specify the
2053 2054 2055 2056
		 * affected processor models on the command line, hence
		 * the need to ignore the workaround.
		 */
		break;
2057 2058 2059 2060 2061 2062 2063 2064 2065 2066
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
		if (kvm_hv_msr_partition_wide(msr)) {
			int r;
			mutex_lock(&vcpu->kvm->lock);
			r = set_msr_hyperv_pw(vcpu, msr, data);
			mutex_unlock(&vcpu->kvm->lock);
			return r;
		} else
			return set_msr_hyperv(vcpu, msr, data);
		break;
2067 2068 2069 2070
	case MSR_IA32_BBL_CR_CTL3:
		/* Drop writes to this legacy MSR -- see rdmsr
		 * counterpart for further detail.
		 */
2071
		vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", msr, data);
2072
		break;
2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
	case MSR_AMD64_OSVW_ID_LENGTH:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		vcpu->arch.osvw.length = data;
		break;
	case MSR_AMD64_OSVW_STATUS:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		vcpu->arch.osvw.status = data;
		break;
2083
	default:
E
Ed Swierk 已提交
2084 2085
		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
			return xen_hvm_config(vcpu, data);
2086 2087
		if (kvm_pmu_msr(vcpu, msr))
			return kvm_pmu_set_msr(vcpu, msr, data);
2088
		if (!ignore_msrs) {
2089 2090
			vcpu_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
				    msr, data);
2091 2092
			return 1;
		} else {
2093 2094
			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
				    msr, data);
2095 2096
			break;
		}
2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);


/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
	return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
}

A
Avi Kivity 已提交
2113 2114
static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
S
Sheng Yang 已提交
2115 2116
	u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;

A
Avi Kivity 已提交
2117 2118 2119
	if (!msr_mtrr_valid(msr))
		return 1;

S
Sheng Yang 已提交
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145
	if (msr == MSR_MTRRdefType)
		*pdata = vcpu->arch.mtrr_state.def_type +
			 (vcpu->arch.mtrr_state.enabled << 10);
	else if (msr == MSR_MTRRfix64K_00000)
		*pdata = p[0];
	else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
		*pdata = p[1 + msr - MSR_MTRRfix16K_80000];
	else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
		*pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
	else if (msr == MSR_IA32_CR_PAT)
		*pdata = vcpu->arch.pat;
	else {	/* Variable MTRRs */
		int idx, is_mtrr_mask;
		u64 *pt;

		idx = (msr - 0x200) / 2;
		is_mtrr_mask = msr - 0x200 - 2 * idx;
		if (!is_mtrr_mask)
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
		else
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
		*pdata = *pt;
	}

A
Avi Kivity 已提交
2146 2147 2148
	return 0;
}

H
Huang Ying 已提交
2149
static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
2150 2151
{
	u64 data;
H
Huang Ying 已提交
2152 2153
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2154 2155 2156 2157

	switch (msr) {
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
H
Huang Ying 已提交
2158 2159
		data = 0;
		break;
2160
	case MSR_IA32_MCG_CAP:
H
Huang Ying 已提交
2161 2162
		data = vcpu->arch.mcg_cap;
		break;
2163
	case MSR_IA32_MCG_CTL:
H
Huang Ying 已提交
2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183
		if (!(mcg_cap & MCG_CTL_P))
			return 1;
		data = vcpu->arch.mcg_ctl;
		break;
	case MSR_IA32_MCG_STATUS:
		data = vcpu->arch.mcg_status;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
		    msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
			u32 offset = msr - MSR_IA32_MC0_CTL;
			data = vcpu->arch.mce_banks[offset];
			break;
		}
		return 1;
	}
	*pdata = data;
	return 0;
}

2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196
static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data = 0;
	struct kvm *kvm = vcpu->kvm;

	switch (msr) {
	case HV_X64_MSR_GUEST_OS_ID:
		data = kvm->arch.hv_guest_os_id;
		break;
	case HV_X64_MSR_HYPERCALL:
		data = kvm->arch.hv_hypercall;
		break;
	default:
2197
		vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217
		return 1;
	}

	*pdata = data;
	return 0;
}

static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data = 0;

	switch (msr) {
	case HV_X64_MSR_VP_INDEX: {
		int r;
		struct kvm_vcpu *v;
		kvm_for_each_vcpu(r, v, vcpu->kvm)
			if (v == vcpu)
				data = r;
		break;
	}
G
Gleb Natapov 已提交
2218 2219 2220 2221 2222 2223
	case HV_X64_MSR_EOI:
		return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata);
	case HV_X64_MSR_ICR:
		return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
	case HV_X64_MSR_TPR:
		return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
2224
	case HV_X64_MSR_APIC_ASSIST_PAGE:
2225 2226
		data = vcpu->arch.hv_vapic;
		break;
2227
	default:
2228
		vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
2229 2230 2231 2232 2233 2234
		return 1;
	}
	*pdata = data;
	return 0;
}

H
Huang Ying 已提交
2235 2236 2237 2238 2239 2240
int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data;

	switch (msr) {
	case MSR_IA32_PLATFORM_ID:
2241
	case MSR_IA32_EBL_CR_POWERON:
2242 2243 2244 2245 2246
	case MSR_IA32_DEBUGCTLMSR:
	case MSR_IA32_LASTBRANCHFROMIP:
	case MSR_IA32_LASTBRANCHTOIP:
	case MSR_IA32_LASTINTFROMIP:
	case MSR_IA32_LASTINTTOIP:
2247 2248
	case MSR_K8_SYSCFG:
	case MSR_K7_HWCR:
2249
	case MSR_VM_HSAVE_PA:
A
Amit Shah 已提交
2250
	case MSR_K7_EVNTSEL0:
A
Amit Shah 已提交
2251
	case MSR_K7_PERFCTR0:
2252
	case MSR_K8_INT_PENDING_MSG:
2253
	case MSR_AMD64_NB_CFG:
2254
	case MSR_FAM10H_MMIO_CONF_BASE:
2255
	case MSR_AMD64_BU_CFG2:
2256 2257
		data = 0;
		break;
2258 2259 2260 2261 2262 2263 2264 2265
	case MSR_P6_PERFCTR0:
	case MSR_P6_PERFCTR1:
	case MSR_P6_EVNTSEL0:
	case MSR_P6_EVNTSEL1:
		if (kvm_pmu_msr(vcpu, msr))
			return kvm_pmu_get_msr(vcpu, msr, pdata);
		data = 0;
		break;
2266 2267 2268
	case MSR_IA32_UCODE_REV:
		data = 0x100000000ULL;
		break;
A
Avi Kivity 已提交
2269 2270 2271 2272 2273
	case MSR_MTRRcap:
		data = 0x500 | KVM_NR_VAR_MTRR;
		break;
	case 0x200 ... 0x2ff:
		return get_msr_mtrr(vcpu, msr, pdata);
2274 2275 2276
	case 0xcd: /* fsb frequency */
		data = 3;
		break;
2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290
		/*
		 * MSR_EBC_FREQUENCY_ID
		 * Conservative value valid for even the basic CPU models.
		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
		 * and 266MHz for model 3, or 4. Set Core Clock
		 * Frequency to System Bus Frequency Ratio to 1 (bits
		 * 31:24) even though these are only valid for CPU
		 * models > 2, however guests may end up dividing or
		 * multiplying by zero otherwise.
		 */
	case MSR_EBC_FREQUENCY_ID:
		data = 1 << 24;
		break;
2291 2292 2293
	case MSR_IA32_APICBASE:
		data = kvm_get_apic_base(vcpu);
		break;
G
Gleb Natapov 已提交
2294 2295 2296
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_read(vcpu, msr, pdata);
		break;
2297 2298 2299
	case MSR_IA32_TSCDEADLINE:
		data = kvm_get_lapic_tscdeadline_msr(vcpu);
		break;
W
Will Auld 已提交
2300 2301 2302
	case MSR_IA32_TSC_ADJUST:
		data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
		break;
2303
	case MSR_IA32_MISC_ENABLE:
2304
		data = vcpu->arch.ia32_misc_enable_msr;
2305
		break;
2306 2307 2308 2309 2310 2311
	case MSR_IA32_PERF_STATUS:
		/* TSC increment by tick */
		data = 1000ULL;
		/* CPU multiplier */
		data |= (((uint64_t)4ULL) << 40);
		break;
2312
	case MSR_EFER:
2313
		data = vcpu->arch.efer;
2314
		break;
2315
	case MSR_KVM_WALL_CLOCK:
2316
	case MSR_KVM_WALL_CLOCK_NEW:
2317 2318 2319
		data = vcpu->kvm->arch.wall_clock;
		break;
	case MSR_KVM_SYSTEM_TIME:
2320
	case MSR_KVM_SYSTEM_TIME_NEW:
2321 2322
		data = vcpu->arch.time;
		break;
2323 2324 2325
	case MSR_KVM_ASYNC_PF_EN:
		data = vcpu->arch.apf.msr_val;
		break;
G
Glauber Costa 已提交
2326 2327 2328
	case MSR_KVM_STEAL_TIME:
		data = vcpu->arch.st.msr_val;
		break;
2329 2330 2331
	case MSR_KVM_PV_EOI_EN:
		data = vcpu->arch.pv_eoi.msr_val;
		break;
H
Huang Ying 已提交
2332 2333 2334 2335 2336 2337 2338
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
	case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
		return get_msr_mce(vcpu, msr, pdata);
2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350
	case MSR_K7_CLK_CTL:
		/*
		 * Provide expected ramp-up count for K7. All other
		 * are set to zero, indicating minimum divisors for
		 * every field.
		 *
		 * This prevents guest kernels on AMD host with CPU
		 * type 6, model 8 and higher from exploding due to
		 * the rdmsr failing.
		 */
		data = 0x20000000;
		break;
2351 2352 2353 2354 2355 2356 2357 2358 2359 2360
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
		if (kvm_hv_msr_partition_wide(msr)) {
			int r;
			mutex_lock(&vcpu->kvm->lock);
			r = get_msr_hyperv_pw(vcpu, msr, pdata);
			mutex_unlock(&vcpu->kvm->lock);
			return r;
		} else
			return get_msr_hyperv(vcpu, msr, pdata);
		break;
2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373
	case MSR_IA32_BBL_CR_CTL3:
		/* This legacy MSR exists but isn't fully documented in current
		 * silicon.  It is however accessed by winxp in very narrow
		 * scenarios where it sets bit #19, itself documented as
		 * a "reserved" bit.  Best effort attempt to source coherent
		 * read data here should the balance of the register be
		 * interpreted by the guest:
		 *
		 * L2 cache control register 3: 64GB range, 256KB size,
		 * enabled, latency 0x1, configured
		 */
		data = 0xbe702111;
		break;
2374 2375 2376 2377 2378 2379 2380 2381 2382 2383
	case MSR_AMD64_OSVW_ID_LENGTH:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		data = vcpu->arch.osvw.length;
		break;
	case MSR_AMD64_OSVW_STATUS:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		data = vcpu->arch.osvw.status;
		break;
2384
	default:
2385 2386
		if (kvm_pmu_msr(vcpu, msr))
			return kvm_pmu_get_msr(vcpu, msr, pdata);
2387
		if (!ignore_msrs) {
2388
			vcpu_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
2389 2390
			return 1;
		} else {
2391
			vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
2392 2393 2394
			data = 0;
		}
		break;
2395 2396 2397 2398 2399 2400
	}
	*pdata = data;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

2401 2402 2403 2404 2405 2406 2407 2408 2409 2410
/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
2411
	int i, idx;
2412

2413
	idx = srcu_read_lock(&vcpu->kvm->srcu);
2414 2415 2416
	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;
2417
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
	if (copy_from_user(&msrs, user_msrs, sizeof msrs))
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
2446 2447 2448
	entries = memdup_user(user_msrs->entries, size);
	if (IS_ERR(entries)) {
		r = PTR_ERR(entries);
2449
		goto out;
2450
	}
2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462

	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
2463
	kfree(entries);
2464 2465 2466 2467
out:
	return r;
}

2468 2469 2470 2471 2472 2473 2474 2475 2476
int kvm_dev_ioctl_check_extension(long ext)
{
	int r;

	switch (ext) {
	case KVM_CAP_IRQCHIP:
	case KVM_CAP_HLT:
	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
	case KVM_CAP_SET_TSS_ADDR:
2477
	case KVM_CAP_EXT_CPUID:
2478
	case KVM_CAP_CLOCKSOURCE:
S
Sheng Yang 已提交
2479
	case KVM_CAP_PIT:
2480
	case KVM_CAP_NOP_IO_DELAY:
2481
	case KVM_CAP_MP_STATE:
2482
	case KVM_CAP_SYNC_MMU:
2483
	case KVM_CAP_USER_NMI:
2484
	case KVM_CAP_REINJECT_CONTROL:
2485
	case KVM_CAP_IRQ_INJECT_STATUS:
2486
	case KVM_CAP_ASSIGN_DEV_IRQ:
G
Gregory Haskins 已提交
2487
	case KVM_CAP_IRQFD:
G
Gregory Haskins 已提交
2488
	case KVM_CAP_IOEVENTFD:
2489
	case KVM_CAP_PIT2:
B
Beth Kon 已提交
2490
	case KVM_CAP_PIT_STATE2:
2491
	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
E
Ed Swierk 已提交
2492
	case KVM_CAP_XEN_HVM:
2493
	case KVM_CAP_ADJUST_CLOCK:
J
Jan Kiszka 已提交
2494
	case KVM_CAP_VCPU_EVENTS:
2495
	case KVM_CAP_HYPERV:
G
Gleb Natapov 已提交
2496
	case KVM_CAP_HYPERV_VAPIC:
2497
	case KVM_CAP_HYPERV_SPIN:
2498
	case KVM_CAP_PCI_SEGMENT:
2499
	case KVM_CAP_DEBUGREGS:
2500
	case KVM_CAP_X86_ROBUST_SINGLESTEP:
2501
	case KVM_CAP_XSAVE:
2502
	case KVM_CAP_ASYNC_PF:
2503
	case KVM_CAP_GET_TSC_KHZ:
2504
	case KVM_CAP_PCI_2_3:
2505
	case KVM_CAP_KVMCLOCK_CTRL:
X
Xiao Guangrong 已提交
2506
	case KVM_CAP_READONLY_MEM:
2507
	case KVM_CAP_IRQFD_RESAMPLE:
2508 2509
		r = 1;
		break;
2510 2511 2512
	case KVM_CAP_COALESCED_MMIO:
		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
		break;
2513 2514 2515
	case KVM_CAP_VAPIC:
		r = !kvm_x86_ops->cpu_has_accelerated_tpr();
		break;
2516
	case KVM_CAP_NR_VCPUS:
2517 2518 2519
		r = KVM_SOFT_MAX_VCPUS;
		break;
	case KVM_CAP_MAX_VCPUS:
2520 2521
		r = KVM_MAX_VCPUS;
		break;
2522
	case KVM_CAP_NR_MEMSLOTS:
2523
		r = KVM_USER_MEM_SLOTS;
2524
		break;
2525 2526
	case KVM_CAP_PV_MMU:	/* obsolete */
		r = 0;
2527
		break;
B
Ben-Ami Yassour 已提交
2528
	case KVM_CAP_IOMMU:
2529
		r = iommu_present(&pci_bus_type);
B
Ben-Ami Yassour 已提交
2530
		break;
H
Huang Ying 已提交
2531 2532 2533
	case KVM_CAP_MCE:
		r = KVM_MAX_MCE_BANKS;
		break;
2534 2535 2536
	case KVM_CAP_XCRS:
		r = cpu_has_xsave;
		break;
2537 2538 2539
	case KVM_CAP_TSC_CONTROL:
		r = kvm_has_tsc_control;
		break;
2540 2541 2542
	case KVM_CAP_TSC_DEADLINE_TIMER:
		r = boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER);
		break;
2543 2544 2545 2546 2547 2548 2549 2550
	default:
		r = 0;
		break;
	}
	return r;

}

2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570
long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {
	case KVM_GET_MSR_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
			goto out;
		n = msr_list.nmsrs;
		msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
		if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
			goto out;
		r = -E2BIG;
J
Jan Kiszka 已提交
2571
		if (n < msr_list.nmsrs)
2572 2573 2574 2575 2576
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
				 num_msrs_to_save * sizeof(u32)))
			goto out;
J
Jan Kiszka 已提交
2577
		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
2578 2579 2580 2581 2582 2583
				 &emulated_msrs,
				 ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
			goto out;
		r = 0;
		break;
	}
2584 2585 2586 2587 2588 2589 2590 2591
	case KVM_GET_SUPPORTED_CPUID: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
2592
						      cpuid_arg->entries);
2593 2594 2595 2596 2597 2598 2599 2600 2601
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
			goto out;
		r = 0;
		break;
	}
H
Huang Ying 已提交
2602 2603 2604 2605 2606 2607 2608 2609 2610 2611
	case KVM_X86_GET_MCE_CAP_SUPPORTED: {
		u64 mce_cap;

		mce_cap = KVM_MCE_CAP_SUPPORTED;
		r = -EFAULT;
		if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
			goto out;
		r = 0;
		break;
	}
2612 2613 2614 2615 2616 2617 2618
	default:
		r = -EINVAL;
	}
out:
	return r;
}

2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629
static void wbinvd_ipi(void *garbage)
{
	wbinvd();
}

static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
	return vcpu->kvm->arch.iommu_domain &&
		!(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY);
}

2630 2631
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
2632 2633 2634 2635 2636 2637 2638 2639 2640
	/* Address WBINVD may be executed by guest */
	if (need_emulate_wbinvd(vcpu)) {
		if (kvm_x86_ops->has_wbinvd_exit())
			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
			smp_call_function_single(vcpu->cpu,
					wbinvd_ipi, NULL, 1);
	}

2641
	kvm_x86_ops->vcpu_load(vcpu, cpu);
2642

2643 2644 2645 2646 2647 2648
	/* Apply any externally detected TSC adjustments (due to suspend) */
	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
		vcpu->arch.tsc_offset_adjustment = 0;
		set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
	}
2649

2650
	if (unlikely(vcpu->cpu != cpu) || check_tsc_unstable()) {
2651 2652
		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
				native_read_tsc() - vcpu->arch.last_host_tsc;
Z
Zachary Amsden 已提交
2653 2654
		if (tsc_delta < 0)
			mark_tsc_unstable("KVM discovered backwards TSC");
Z
Zachary Amsden 已提交
2655
		if (check_tsc_unstable()) {
2656 2657 2658
			u64 offset = kvm_x86_ops->compute_tsc_offset(vcpu,
						vcpu->arch.last_guest_tsc);
			kvm_x86_ops->write_tsc_offset(vcpu, offset);
Z
Zachary Amsden 已提交
2659 2660
			vcpu->arch.tsc_catchup = 1;
		}
2661 2662 2663 2664 2665 2666
		/*
		 * On a host with synchronized TSC, there is no need to update
		 * kvmclock on vcpu->cpu migration
		 */
		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
Z
Zachary Amsden 已提交
2667 2668
		if (vcpu->cpu != cpu)
			kvm_migrate_timers(vcpu);
Z
Zachary Amsden 已提交
2669
		vcpu->cpu = cpu;
Z
Zachary Amsden 已提交
2670
	}
G
Glauber Costa 已提交
2671 2672 2673

	accumulate_steal_time(vcpu);
	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
2674 2675 2676 2677
}

void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
2678
	kvm_x86_ops->vcpu_put(vcpu);
2679
	kvm_put_guest_fpu(vcpu);
2680
	vcpu->arch.last_host_tsc = native_read_tsc();
2681 2682 2683 2684 2685
}

static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
2686
	memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
2687 2688 2689 2690 2691 2692 2693

	return 0;
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
2694
	kvm_apic_post_state_restore(vcpu, s);
2695
	update_cr8_intercept(vcpu);
2696 2697 2698 2699

	return 0;
}

2700 2701 2702
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
{
2703
	if (irq->irq >= KVM_NR_INTERRUPTS)
2704 2705 2706 2707
		return -EINVAL;
	if (irqchip_in_kernel(vcpu->kvm))
		return -ENXIO;

2708
	kvm_queue_interrupt(vcpu, irq->irq, false);
2709
	kvm_make_request(KVM_REQ_EVENT, vcpu);
2710 2711 2712 2713

	return 0;
}

2714 2715 2716 2717 2718 2719 2720
static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
{
	kvm_inject_nmi(vcpu);

	return 0;
}

2721 2722 2723 2724 2725 2726 2727 2728 2729
static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
					   struct kvm_tpr_access_ctl *tac)
{
	if (tac->flags)
		return -EINVAL;
	vcpu->arch.tpr_access_reporting = !!tac->enabled;
	return 0;
}

H
Huang Ying 已提交
2730 2731 2732 2733 2734 2735 2736
static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
					u64 mcg_cap)
{
	int r;
	unsigned bank_num = mcg_cap & 0xff, bank;

	r = -EINVAL;
2737
	if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
H
Huang Ying 已提交
2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777
		goto out;
	if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
		goto out;
	r = 0;
	vcpu->arch.mcg_cap = mcg_cap;
	/* Init IA32_MCG_CTL to all 1s */
	if (mcg_cap & MCG_CTL_P)
		vcpu->arch.mcg_ctl = ~(u64)0;
	/* Init IA32_MCi_CTL to all 1s */
	for (bank = 0; bank < bank_num; bank++)
		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
out:
	return r;
}

static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
				      struct kvm_x86_mce *mce)
{
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
	u64 *banks = vcpu->arch.mce_banks;

	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
		return -EINVAL;
	/*
	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
	 * reporting is disabled
	 */
	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
	    vcpu->arch.mcg_ctl != ~(u64)0)
		return 0;
	banks += 4 * mce->bank;
	/*
	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
	 * reporting is disabled for the bank
	 */
	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
		return 0;
	if (mce->status & MCI_STATUS_UC) {
		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
2778
		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
2779
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
H
Huang Ying 已提交
2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800
			return 0;
		}
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		vcpu->arch.mcg_status = mce->mcg_status;
		banks[1] = mce->status;
		kvm_queue_exception(vcpu, MC_VECTOR);
	} else if (!(banks[1] & MCI_STATUS_VAL)
		   || !(banks[1] & MCI_STATUS_UC)) {
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		banks[1] = mce->status;
	} else
		banks[1] |= MCI_STATUS_OVER;
	return 0;
}

J
Jan Kiszka 已提交
2801 2802 2803
static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
					       struct kvm_vcpu_events *events)
{
A
Avi Kivity 已提交
2804
	process_nmi(vcpu);
2805 2806 2807
	events->exception.injected =
		vcpu->arch.exception.pending &&
		!kvm_exception_is_soft(vcpu->arch.exception.nr);
J
Jan Kiszka 已提交
2808 2809
	events->exception.nr = vcpu->arch.exception.nr;
	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
2810
	events->exception.pad = 0;
J
Jan Kiszka 已提交
2811 2812
	events->exception.error_code = vcpu->arch.exception.error_code;

2813 2814
	events->interrupt.injected =
		vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft;
J
Jan Kiszka 已提交
2815
	events->interrupt.nr = vcpu->arch.interrupt.nr;
2816
	events->interrupt.soft = 0;
2817 2818 2819
	events->interrupt.shadow =
		kvm_x86_ops->get_interrupt_shadow(vcpu,
			KVM_X86_SHADOW_INT_MOV_SS | KVM_X86_SHADOW_INT_STI);
J
Jan Kiszka 已提交
2820 2821

	events->nmi.injected = vcpu->arch.nmi_injected;
A
Avi Kivity 已提交
2822
	events->nmi.pending = vcpu->arch.nmi_pending != 0;
J
Jan Kiszka 已提交
2823
	events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
2824
	events->nmi.pad = 0;
J
Jan Kiszka 已提交
2825 2826 2827

	events->sipi_vector = vcpu->arch.sipi_vector;

2828
	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
2829 2830
			 | KVM_VCPUEVENT_VALID_SIPI_VECTOR
			 | KVM_VCPUEVENT_VALID_SHADOW);
2831
	memset(&events->reserved, 0, sizeof(events->reserved));
J
Jan Kiszka 已提交
2832 2833 2834 2835 2836
}

static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
					      struct kvm_vcpu_events *events)
{
2837
	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
2838 2839
			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
			      | KVM_VCPUEVENT_VALID_SHADOW))
J
Jan Kiszka 已提交
2840 2841
		return -EINVAL;

A
Avi Kivity 已提交
2842
	process_nmi(vcpu);
J
Jan Kiszka 已提交
2843 2844 2845 2846 2847 2848 2849 2850
	vcpu->arch.exception.pending = events->exception.injected;
	vcpu->arch.exception.nr = events->exception.nr;
	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
	vcpu->arch.exception.error_code = events->exception.error_code;

	vcpu->arch.interrupt.pending = events->interrupt.injected;
	vcpu->arch.interrupt.nr = events->interrupt.nr;
	vcpu->arch.interrupt.soft = events->interrupt.soft;
2851 2852 2853
	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
		kvm_x86_ops->set_interrupt_shadow(vcpu,
						  events->interrupt.shadow);
J
Jan Kiszka 已提交
2854 2855

	vcpu->arch.nmi_injected = events->nmi.injected;
2856 2857
	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
		vcpu->arch.nmi_pending = events->nmi.pending;
J
Jan Kiszka 已提交
2858 2859
	kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);

2860 2861
	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR)
		vcpu->arch.sipi_vector = events->sipi_vector;
J
Jan Kiszka 已提交
2862

2863 2864
	kvm_make_request(KVM_REQ_EVENT, vcpu);

J
Jan Kiszka 已提交
2865 2866 2867
	return 0;
}

2868 2869 2870 2871 2872 2873 2874
static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
					     struct kvm_debugregs *dbgregs)
{
	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
	dbgregs->dr6 = vcpu->arch.dr6;
	dbgregs->dr7 = vcpu->arch.dr7;
	dbgregs->flags = 0;
2875
	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890
}

static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
					    struct kvm_debugregs *dbgregs)
{
	if (dbgregs->flags)
		return -EINVAL;

	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
	vcpu->arch.dr6 = dbgregs->dr6;
	vcpu->arch.dr7 = dbgregs->dr7;

	return 0;
}

2891 2892 2893 2894 2895 2896
static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
					 struct kvm_xsave *guest_xsave)
{
	if (cpu_has_xsave)
		memcpy(guest_xsave->region,
			&vcpu->arch.guest_fpu.state->xsave,
2897
			xstate_size);
2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914
	else {
		memcpy(guest_xsave->region,
			&vcpu->arch.guest_fpu.state->fxsave,
			sizeof(struct i387_fxsave_struct));
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
			XSTATE_FPSSE;
	}
}

static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
					struct kvm_xsave *guest_xsave)
{
	u64 xstate_bv =
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];

	if (cpu_has_xsave)
		memcpy(&vcpu->arch.guest_fpu.state->xsave,
2915
			guest_xsave->region, xstate_size);
2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961
	else {
		if (xstate_bv & ~XSTATE_FPSSE)
			return -EINVAL;
		memcpy(&vcpu->arch.guest_fpu.state->fxsave,
			guest_xsave->region, sizeof(struct i387_fxsave_struct));
	}
	return 0;
}

static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
					struct kvm_xcrs *guest_xcrs)
{
	if (!cpu_has_xsave) {
		guest_xcrs->nr_xcrs = 0;
		return;
	}

	guest_xcrs->nr_xcrs = 1;
	guest_xcrs->flags = 0;
	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
}

static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
				       struct kvm_xcrs *guest_xcrs)
{
	int i, r = 0;

	if (!cpu_has_xsave)
		return -EINVAL;

	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
		return -EINVAL;

	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
		/* Only support XCR0 currently */
		if (guest_xcrs->xcrs[0].xcr == XCR_XFEATURE_ENABLED_MASK) {
			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
				guest_xcrs->xcrs[0].value);
			break;
		}
	if (r)
		r = -EINVAL;
	return r;
}

2962 2963 2964 2965 2966 2967 2968 2969 2970 2971
/*
 * kvm_set_guest_paused() indicates to the guest kernel that it has been
 * stopped by the hypervisor.  This function will be called from the host only.
 * EINVAL is returned when the host attempts to set the flag for a guest that
 * does not support pv clocks.
 */
static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
{
	if (!vcpu->arch.time_page)
		return -EINVAL;
2972
	vcpu->arch.pvclock_set_guest_stopped_request = true;
2973 2974 2975 2976
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	return 0;
}

2977 2978 2979 2980 2981 2982
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r;
2983 2984 2985 2986 2987 2988 2989 2990
	union {
		struct kvm_lapic_state *lapic;
		struct kvm_xsave *xsave;
		struct kvm_xcrs *xcrs;
		void *buffer;
	} u;

	u.buffer = NULL;
2991 2992
	switch (ioctl) {
	case KVM_GET_LAPIC: {
2993 2994 2995
		r = -EINVAL;
		if (!vcpu->arch.apic)
			goto out;
2996
		u.lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
2997

2998
		r = -ENOMEM;
2999
		if (!u.lapic)
3000
			goto out;
3001
		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
3002 3003 3004
		if (r)
			goto out;
		r = -EFAULT;
3005
		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
3006 3007 3008 3009 3010
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
3011 3012 3013
		r = -EINVAL;
		if (!vcpu->arch.apic)
			goto out;
3014
		u.lapic = memdup_user(argp, sizeof(*u.lapic));
G
Guo Chao 已提交
3015 3016
		if (IS_ERR(u.lapic))
			return PTR_ERR(u.lapic);
3017

3018
		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
3019 3020
		break;
	}
3021 3022 3023 3024 3025 3026 3027 3028 3029
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
		if (copy_from_user(&irq, argp, sizeof irq))
			goto out;
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
		break;
	}
3030 3031 3032 3033
	case KVM_NMI: {
		r = kvm_vcpu_ioctl_nmi(vcpu);
		break;
	}
3034 3035 3036 3037 3038 3039 3040 3041 3042 3043
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		break;
	}
3044 3045 3046 3047 3048 3049 3050 3051
	case KVM_SET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
3052
					      cpuid_arg->entries);
3053 3054 3055 3056 3057 3058 3059 3060 3061 3062
		break;
	}
	case KVM_GET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
3063
					      cpuid_arg->entries);
3064 3065 3066 3067 3068 3069 3070 3071
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
			goto out;
		r = 0;
		break;
	}
3072 3073 3074 3075 3076 3077
	case KVM_GET_MSRS:
		r = msr_io(vcpu, argp, kvm_get_msr, 1);
		break;
	case KVM_SET_MSRS:
		r = msr_io(vcpu, argp, do_set_msr, 0);
		break;
3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092
	case KVM_TPR_ACCESS_REPORTING: {
		struct kvm_tpr_access_ctl tac;

		r = -EFAULT;
		if (copy_from_user(&tac, argp, sizeof tac))
			goto out;
		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &tac, sizeof tac))
			goto out;
		r = 0;
		break;
	};
A
Avi Kivity 已提交
3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105
	case KVM_SET_VAPIC_ADDR: {
		struct kvm_vapic_addr va;

		r = -EINVAL;
		if (!irqchip_in_kernel(vcpu->kvm))
			goto out;
		r = -EFAULT;
		if (copy_from_user(&va, argp, sizeof va))
			goto out;
		r = 0;
		kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
		break;
	}
H
Huang Ying 已提交
3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
	case KVM_X86_SETUP_MCE: {
		u64 mcg_cap;

		r = -EFAULT;
		if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
			goto out;
		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
		break;
	}
	case KVM_X86_SET_MCE: {
		struct kvm_x86_mce mce;

		r = -EFAULT;
		if (copy_from_user(&mce, argp, sizeof mce))
			goto out;
		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
		break;
	}
J
Jan Kiszka 已提交
3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);

		r = -EFAULT;
		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		r = -EFAULT;
		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
			break;

		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
		break;
	}
3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167
	case KVM_GET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);

		r = -EFAULT;
		if (copy_to_user(argp, &dbgregs,
				 sizeof(struct kvm_debugregs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		r = -EFAULT;
		if (copy_from_user(&dbgregs, argp,
				   sizeof(struct kvm_debugregs)))
			break;

		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
		break;
	}
3168
	case KVM_GET_XSAVE: {
3169
		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL);
3170
		r = -ENOMEM;
3171
		if (!u.xsave)
3172 3173
			break;

3174
		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
3175 3176

		r = -EFAULT;
3177
		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
3178 3179 3180 3181 3182
			break;
		r = 0;
		break;
	}
	case KVM_SET_XSAVE: {
3183
		u.xsave = memdup_user(argp, sizeof(*u.xsave));
G
Guo Chao 已提交
3184 3185
		if (IS_ERR(u.xsave))
			return PTR_ERR(u.xsave);
3186

3187
		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
3188 3189 3190
		break;
	}
	case KVM_GET_XCRS: {
3191
		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL);
3192
		r = -ENOMEM;
3193
		if (!u.xcrs)
3194 3195
			break;

3196
		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
3197 3198

		r = -EFAULT;
3199
		if (copy_to_user(argp, u.xcrs,
3200 3201 3202 3203 3204 3205
				 sizeof(struct kvm_xcrs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XCRS: {
3206
		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
G
Guo Chao 已提交
3207 3208
		if (IS_ERR(u.xcrs))
			return PTR_ERR(u.xcrs);
3209

3210
		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
3211 3212
		break;
	}
3213 3214 3215 3216 3217 3218 3219 3220 3221
	case KVM_SET_TSC_KHZ: {
		u32 user_tsc_khz;

		r = -EINVAL;
		user_tsc_khz = (u32)arg;

		if (user_tsc_khz >= kvm_max_guest_tsc_khz)
			goto out;

3222 3223 3224 3225
		if (user_tsc_khz == 0)
			user_tsc_khz = tsc_khz;

		kvm_set_tsc_khz(vcpu, user_tsc_khz);
3226 3227 3228 3229 3230

		r = 0;
		goto out;
	}
	case KVM_GET_TSC_KHZ: {
3231
		r = vcpu->arch.virtual_tsc_khz;
3232 3233
		goto out;
	}
3234 3235 3236 3237
	case KVM_KVMCLOCK_CTRL: {
		r = kvm_set_guest_paused(vcpu);
		goto out;
	}
3238 3239 3240 3241
	default:
		r = -EINVAL;
	}
out:
3242
	kfree(u.buffer);
3243 3244 3245
	return r;
}

3246 3247 3248 3249 3250
int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
{
	return VM_FAULT_SIGBUS;
}

3251 3252 3253 3254 3255
static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
{
	int ret;

	if (addr > (unsigned int)(-3 * PAGE_SIZE))
3256
		return -EINVAL;
3257 3258 3259 3260
	ret = kvm_x86_ops->set_tss_addr(kvm, addr);
	return ret;
}

3261 3262 3263 3264 3265 3266 3267
static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
					      u64 ident_addr)
{
	kvm->arch.ept_identity_map_addr = ident_addr;
	return 0;
}

3268 3269 3270 3271 3272 3273
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
					  u32 kvm_nr_mmu_pages)
{
	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
		return -EINVAL;

3274
	mutex_lock(&kvm->slots_lock);
3275 3276

	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
3277
	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
3278

3279
	mutex_unlock(&kvm->slots_lock);
3280 3281 3282 3283 3284
	return 0;
}

static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
{
3285
	return kvm->arch.n_max_mmu_pages;
3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304
}

static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
		memcpy(&chip->chip.pic,
			&pic_irqchip(kvm)->pics[0],
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
		memcpy(&chip->chip.pic,
			&pic_irqchip(kvm)->pics[1],
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_IOAPIC:
G
Gleb Natapov 已提交
3305
		r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320
		break;
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
3321
		spin_lock(&pic_irqchip(kvm)->lock);
3322 3323 3324
		memcpy(&pic_irqchip(kvm)->pics[0],
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
3325
		spin_unlock(&pic_irqchip(kvm)->lock);
3326 3327
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
3328
		spin_lock(&pic_irqchip(kvm)->lock);
3329 3330 3331
		memcpy(&pic_irqchip(kvm)->pics[1],
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
3332
		spin_unlock(&pic_irqchip(kvm)->lock);
3333 3334
		break;
	case KVM_IRQCHIP_IOAPIC:
G
Gleb Natapov 已提交
3335
		r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
3336 3337 3338 3339 3340 3341 3342 3343 3344
		break;
	default:
		r = -EINVAL;
		break;
	}
	kvm_pic_update_irq(pic_irqchip(kvm));
	return r;
}

3345 3346 3347 3348
static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
	int r = 0;

3349
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
3350
	memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
3351
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3352 3353 3354 3355 3356 3357 3358
	return r;
}

static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
	int r = 0;

3359
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
3360
	memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
B
Beth Kon 已提交
3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374
	kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
	return r;
}

static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	int r = 0;

	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
		sizeof(ps->channels));
	ps->flags = kvm->arch.vpit->pit_state.flags;
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3375
	memset(&ps->reserved, 0, sizeof(ps->reserved));
B
Beth Kon 已提交
3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391
	return r;
}

static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	int r = 0, start = 0;
	u32 prev_legacy, cur_legacy;
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
	if (!prev_legacy && cur_legacy)
		start = 1;
	memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
	       sizeof(kvm->arch.vpit->pit_state.channels));
	kvm->arch.vpit->pit_state.flags = ps->flags;
	kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
3392
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3393 3394 3395
	return r;
}

3396 3397 3398 3399 3400
static int kvm_vm_ioctl_reinject(struct kvm *kvm,
				 struct kvm_reinject_control *control)
{
	if (!kvm->arch.vpit)
		return -ENXIO;
3401
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
3402
	kvm->arch.vpit->pit_state.reinject = control->pit_reinject;
3403
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3404 3405 3406
	return 0;
}

3407
/**
3408 3409 3410
 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
 * @kvm: kvm instance
 * @log: slot id and address to which we copy the log
3411
 *
3412 3413 3414
 * We need to keep it in mind that VCPU threads can write to the bitmap
 * concurrently.  So, to avoid losing data, we keep the following order for
 * each bit:
3415
 *
3416 3417 3418 3419
 *   1. Take a snapshot of the bit and clear it if needed.
 *   2. Write protect the corresponding page.
 *   3. Flush TLB's if needed.
 *   4. Copy the snapshot to the userspace.
3420
 *
3421 3422 3423 3424
 * Between 2 and 3, the guest may write to the page using the remaining TLB
 * entry.  This is not a problem because the page will be reported dirty at
 * step 4 using the snapshot taken before and step 3 ensures that successive
 * writes will be logged for the next call.
3425
 */
3426
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
3427
{
3428
	int r;
3429
	struct kvm_memory_slot *memslot;
3430 3431 3432 3433
	unsigned long n, i;
	unsigned long *dirty_bitmap;
	unsigned long *dirty_bitmap_buffer;
	bool is_dirty = false;
3434

3435
	mutex_lock(&kvm->slots_lock);
3436

M
Marcelo Tosatti 已提交
3437
	r = -EINVAL;
3438
	if (log->slot >= KVM_USER_MEM_SLOTS)
M
Marcelo Tosatti 已提交
3439 3440
		goto out;

3441
	memslot = id_to_memslot(kvm->memslots, log->slot);
3442 3443

	dirty_bitmap = memslot->dirty_bitmap;
M
Marcelo Tosatti 已提交
3444
	r = -ENOENT;
3445
	if (!dirty_bitmap)
M
Marcelo Tosatti 已提交
3446 3447
		goto out;

3448
	n = kvm_dirty_bitmap_bytes(memslot);
M
Marcelo Tosatti 已提交
3449

3450 3451
	dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
	memset(dirty_bitmap_buffer, 0, n);
M
Marcelo Tosatti 已提交
3452

3453
	spin_lock(&kvm->mmu_lock);
M
Marcelo Tosatti 已提交
3454

3455 3456 3457
	for (i = 0; i < n / sizeof(long); i++) {
		unsigned long mask;
		gfn_t offset;
3458

3459 3460
		if (!dirty_bitmap[i])
			continue;
M
Marcelo Tosatti 已提交
3461

3462
		is_dirty = true;
3463

3464 3465
		mask = xchg(&dirty_bitmap[i], 0);
		dirty_bitmap_buffer[i] = mask;
3466

3467 3468
		offset = i * BITS_PER_LONG;
		kvm_mmu_write_protect_pt_masked(kvm, memslot, offset, mask);
3469
	}
3470 3471 3472 3473 3474 3475 3476 3477
	if (is_dirty)
		kvm_flush_remote_tlbs(kvm);

	spin_unlock(&kvm->mmu_lock);

	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
		goto out;
M
Marcelo Tosatti 已提交
3478

3479 3480
	r = 0;
out:
3481
	mutex_unlock(&kvm->slots_lock);
3482 3483 3484
	return r;
}

3485 3486 3487 3488 3489 3490 3491 3492 3493 3494
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event)
{
	if (!irqchip_in_kernel(kvm))
		return -ENXIO;

	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
					irq_event->irq, irq_event->level);
	return 0;
}

3495 3496 3497 3498 3499
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
3500
	int r = -ENOTTY;
3501 3502 3503 3504 3505 3506 3507
	/*
	 * This union makes it completely explicit to gcc-3.x
	 * that these two variables' stack usage should be
	 * combined, not added together.
	 */
	union {
		struct kvm_pit_state ps;
B
Beth Kon 已提交
3508
		struct kvm_pit_state2 ps2;
3509
		struct kvm_pit_config pit_config;
3510
	} u;
3511 3512 3513 3514 3515

	switch (ioctl) {
	case KVM_SET_TSS_ADDR:
		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
		break;
3516 3517 3518 3519 3520 3521 3522 3523 3524
	case KVM_SET_IDENTITY_MAP_ADDR: {
		u64 ident_addr;

		r = -EFAULT;
		if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
			goto out;
		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
		break;
	}
3525 3526 3527 3528 3529 3530
	case KVM_SET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
		break;
	case KVM_GET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
		break;
3531 3532 3533 3534 3535 3536 3537
	case KVM_CREATE_IRQCHIP: {
		struct kvm_pic *vpic;

		mutex_lock(&kvm->lock);
		r = -EEXIST;
		if (kvm->arch.vpic)
			goto create_irqchip_unlock;
3538 3539 3540
		r = -EINVAL;
		if (atomic_read(&kvm->online_vcpus))
			goto create_irqchip_unlock;
3541
		r = -ENOMEM;
3542 3543
		vpic = kvm_create_pic(kvm);
		if (vpic) {
3544 3545
			r = kvm_ioapic_init(kvm);
			if (r) {
3546
				mutex_lock(&kvm->slots_lock);
3547
				kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
3548 3549 3550 3551 3552
							  &vpic->dev_master);
				kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
							  &vpic->dev_slave);
				kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
							  &vpic->dev_eclr);
3553
				mutex_unlock(&kvm->slots_lock);
3554 3555
				kfree(vpic);
				goto create_irqchip_unlock;
3556 3557
			}
		} else
3558 3559 3560 3561
			goto create_irqchip_unlock;
		smp_wmb();
		kvm->arch.vpic = vpic;
		smp_wmb();
3562 3563
		r = kvm_setup_default_irq_routing(kvm);
		if (r) {
3564
			mutex_lock(&kvm->slots_lock);
3565
			mutex_lock(&kvm->irq_lock);
3566 3567
			kvm_ioapic_destroy(kvm);
			kvm_destroy_pic(kvm);
3568
			mutex_unlock(&kvm->irq_lock);
3569
			mutex_unlock(&kvm->slots_lock);
3570
		}
3571 3572
	create_irqchip_unlock:
		mutex_unlock(&kvm->lock);
3573
		break;
3574
	}
S
Sheng Yang 已提交
3575
	case KVM_CREATE_PIT:
3576 3577 3578 3579 3580 3581 3582 3583
		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
		goto create_pit;
	case KVM_CREATE_PIT2:
		r = -EFAULT;
		if (copy_from_user(&u.pit_config, argp,
				   sizeof(struct kvm_pit_config)))
			goto out;
	create_pit:
3584
		mutex_lock(&kvm->slots_lock);
A
Avi Kivity 已提交
3585 3586 3587
		r = -EEXIST;
		if (kvm->arch.vpit)
			goto create_pit_unlock;
S
Sheng Yang 已提交
3588
		r = -ENOMEM;
3589
		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
S
Sheng Yang 已提交
3590 3591
		if (kvm->arch.vpit)
			r = 0;
A
Avi Kivity 已提交
3592
	create_pit_unlock:
3593
		mutex_unlock(&kvm->slots_lock);
S
Sheng Yang 已提交
3594
		break;
3595 3596
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3597
		struct kvm_irqchip *chip;
3598

3599 3600 3601
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
3602
			goto out;
3603 3604
		}

3605 3606
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
3607 3608
			goto get_irqchip_out;
		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
3609
		if (r)
3610
			goto get_irqchip_out;
3611
		r = -EFAULT;
3612 3613
		if (copy_to_user(argp, chip, sizeof *chip))
			goto get_irqchip_out;
3614
		r = 0;
3615 3616
	get_irqchip_out:
		kfree(chip);
3617 3618 3619 3620
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3621
		struct kvm_irqchip *chip;
3622

3623 3624 3625
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
3626
			goto out;
3627 3628
		}

3629 3630
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
3631 3632
			goto set_irqchip_out;
		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
3633
		if (r)
3634
			goto set_irqchip_out;
3635
		r = 0;
3636 3637
	set_irqchip_out:
		kfree(chip);
3638 3639
		break;
	}
3640 3641
	case KVM_GET_PIT: {
		r = -EFAULT;
3642
		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
3643 3644 3645 3646
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
3647
		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
3648 3649 3650
		if (r)
			goto out;
		r = -EFAULT;
3651
		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
3652 3653 3654 3655 3656 3657
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT: {
		r = -EFAULT;
3658
		if (copy_from_user(&u.ps, argp, sizeof u.ps))
3659 3660 3661 3662
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
3663
		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
3664 3665
		break;
	}
B
Beth Kon 已提交
3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688
	case KVM_GET_PIT2: {
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT2: {
		r = -EFAULT;
		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
		break;
	}
3689 3690 3691 3692 3693 3694 3695 3696
	case KVM_REINJECT_CONTROL: {
		struct kvm_reinject_control control;
		r =  -EFAULT;
		if (copy_from_user(&control, argp, sizeof(control)))
			goto out;
		r = kvm_vm_ioctl_reinject(kvm, &control);
		break;
	}
E
Ed Swierk 已提交
3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707
	case KVM_XEN_HVM_CONFIG: {
		r = -EFAULT;
		if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
				   sizeof(struct kvm_xen_hvm_config)))
			goto out;
		r = -EINVAL;
		if (kvm->arch.xen_hvm_config.flags)
			goto out;
		r = 0;
		break;
	}
3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721
	case KVM_SET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;
		s64 delta;

		r = -EFAULT;
		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
			goto out;

		r = -EINVAL;
		if (user_ns.flags)
			goto out;

		r = 0;
3722
		local_irq_disable();
3723
		now_ns = get_kernel_ns();
3724
		delta = user_ns.clock - now_ns;
3725
		local_irq_enable();
3726 3727 3728 3729 3730 3731 3732
		kvm->arch.kvmclock_offset = delta;
		break;
	}
	case KVM_GET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

3733
		local_irq_disable();
3734
		now_ns = get_kernel_ns();
3735
		user_ns.clock = kvm->arch.kvmclock_offset + now_ns;
3736
		local_irq_enable();
3737
		user_ns.flags = 0;
3738
		memset(&user_ns.pad, 0, sizeof(user_ns.pad));
3739 3740 3741 3742 3743 3744 3745 3746

		r = -EFAULT;
		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
			goto out;
		r = 0;
		break;
	}

3747 3748 3749 3750 3751 3752 3753
	default:
		;
	}
out:
	return r;
}

3754
static void kvm_init_msr_list(void)
3755 3756 3757 3758
{
	u32 dummy[2];
	unsigned i, j;

3759 3760
	/* skip the first msrs in the list. KVM-specific */
	for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) {
3761 3762 3763 3764 3765 3766 3767 3768 3769
		if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
			continue;
		if (j < i)
			msrs_to_save[j] = msrs_to_save[i];
		j++;
	}
	num_msrs_to_save = j;
}

3770 3771
static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
			   const void *v)
3772
{
3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
		if (!(vcpu->arch.apic &&
		      !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
			break;
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
3787

3788
	return handled;
3789 3790
}

3791
static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
3792
{
3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
		if (!(vcpu->arch.apic &&
		      !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
			break;
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v);
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
3808

3809
	return handled;
3810 3811
}

3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823
static void kvm_set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
	kvm_x86_ops->set_segment(vcpu, var, seg);
}

void kvm_get_segment(struct kvm_vcpu *vcpu,
		     struct kvm_segment *var, int seg)
{
	kvm_x86_ops->get_segment(vcpu, var, seg);
}

3824
gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
3825 3826
{
	gpa_t t_gpa;
3827
	struct x86_exception exception;
3828 3829 3830 3831 3832

	BUG_ON(!mmu_is_nested(vcpu));

	/* NPT walks are always user-walks */
	access |= PFERR_USER_MASK;
3833
	t_gpa  = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &exception);
3834 3835 3836 3837

	return t_gpa;
}

3838 3839
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
			      struct x86_exception *exception)
3840 3841
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
3842
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
3843 3844
}

3845 3846
 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
3847 3848 3849
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_FETCH_MASK;
3850
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
3851 3852
}

3853 3854
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
			       struct x86_exception *exception)
3855 3856 3857
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_WRITE_MASK;
3858
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
3859 3860 3861
}

/* uses this to access any guest's mapped memory without checking CPL */
3862 3863
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
3864
{
3865
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
3866 3867 3868 3869
}

static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
3870
				      struct x86_exception *exception)
3871 3872
{
	void *data = val;
3873
	int r = X86EMUL_CONTINUE;
3874 3875

	while (bytes) {
3876
		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
3877
							    exception);
3878
		unsigned offset = addr & (PAGE_SIZE-1);
3879
		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
3880 3881
		int ret;

3882
		if (gpa == UNMAPPED_GVA)
3883
			return X86EMUL_PROPAGATE_FAULT;
3884
		ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
3885
		if (ret < 0) {
3886
			r = X86EMUL_IO_NEEDED;
3887 3888
			goto out;
		}
3889

3890 3891 3892
		bytes -= toread;
		data += toread;
		addr += toread;
3893
	}
3894 3895
out:
	return r;
3896
}
3897

3898
/* used for instruction fetching */
3899 3900
static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
				gva_t addr, void *val, unsigned int bytes,
3901
				struct x86_exception *exception)
3902
{
3903
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3904
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
3905

3906
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu,
3907 3908
					  access | PFERR_FETCH_MASK,
					  exception);
3909 3910
}

3911
int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
3912
			       gva_t addr, void *val, unsigned int bytes,
3913
			       struct x86_exception *exception)
3914
{
3915
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3916
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
3917

3918
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
3919
					  exception);
3920
}
3921
EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
3922

3923 3924
static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt,
				      gva_t addr, void *val, unsigned int bytes,
3925
				      struct x86_exception *exception)
3926
{
3927
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3928
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception);
3929 3930
}

N
Nadav Har'El 已提交
3931
int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
3932
				       gva_t addr, void *val,
3933
				       unsigned int bytes,
3934
				       struct x86_exception *exception)
3935
{
3936
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3937 3938 3939 3940
	void *data = val;
	int r = X86EMUL_CONTINUE;

	while (bytes) {
3941 3942
		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
							     PFERR_WRITE_MASK,
3943
							     exception);
3944 3945 3946 3947
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
		int ret;

3948
		if (gpa == UNMAPPED_GVA)
3949
			return X86EMUL_PROPAGATE_FAULT;
3950 3951
		ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
		if (ret < 0) {
3952
			r = X86EMUL_IO_NEEDED;
3953 3954 3955 3956 3957 3958 3959 3960 3961 3962
			goto out;
		}

		bytes -= towrite;
		data += towrite;
		addr += towrite;
	}
out:
	return r;
}
N
Nadav Har'El 已提交
3963
EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
3964

3965 3966 3967 3968
static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
				gpa_t *gpa, struct x86_exception *exception,
				bool write)
{
3969 3970
	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
		| (write ? PFERR_WRITE_MASK : 0);
3971

3972 3973
	if (vcpu_match_mmio_gva(vcpu, gva)
	    && !permission_fault(vcpu->arch.walk_mmu, vcpu->arch.access, access)) {
3974 3975
		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
					(gva & (PAGE_SIZE - 1));
X
Xiao Guangrong 已提交
3976
		trace_vcpu_match_mmio(gva, *gpa, write, false);
3977 3978 3979
		return 1;
	}

3980 3981 3982 3983 3984 3985 3986 3987 3988
	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);

	if (*gpa == UNMAPPED_GVA)
		return -1;

	/* For APIC access vmexit */
	if ((*gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		return 1;

X
Xiao Guangrong 已提交
3989 3990
	if (vcpu_match_mmio_gpa(vcpu, *gpa)) {
		trace_vcpu_match_mmio(gva, *gpa, write, true);
3991
		return 1;
X
Xiao Guangrong 已提交
3992
	}
3993

3994 3995 3996
	return 0;
}

3997
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
3998
			const void *val, int bytes)
3999 4000 4001 4002
{
	int ret;

	ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
4003
	if (ret < 0)
4004
		return 0;
4005
	kvm_mmu_pte_write(vcpu, gpa, val, bytes);
4006 4007 4008
	return 1;
}

4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024
struct read_write_emulator_ops {
	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
				  int bytes);
	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
				  void *val, int bytes);
	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
			       int bytes, void *val);
	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
				    void *val, int bytes);
	bool write;
};

static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
{
	if (vcpu->mmio_read_completed) {
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
A
Avi Kivity 已提交
4025
			       vcpu->mmio_fragments[0].gpa, *(u64 *)val);
4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060
		vcpu->mmio_read_completed = 0;
		return 1;
	}

	return 0;
}

static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			void *val, int bytes)
{
	return !kvm_read_guest(vcpu->kvm, gpa, val, bytes);
}

static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			 void *val, int bytes)
{
	return emulator_write_phys(vcpu, gpa, val, bytes);
}

static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
{
	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
	return vcpu_mmio_write(vcpu, gpa, bytes, val);
}

static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			  void *val, int bytes)
{
	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
	return X86EMUL_IO_NEEDED;
}

static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			   void *val, int bytes)
{
A
Avi Kivity 已提交
4061 4062
	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];

4063
	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
4064 4065 4066
	return X86EMUL_CONTINUE;
}

4067
static const struct read_write_emulator_ops read_emultor = {
4068 4069 4070 4071 4072 4073
	.read_write_prepare = read_prepare,
	.read_write_emulate = read_emulate,
	.read_write_mmio = vcpu_mmio_read,
	.read_write_exit_mmio = read_exit_mmio,
};

4074
static const struct read_write_emulator_ops write_emultor = {
4075 4076 4077 4078 4079 4080
	.read_write_emulate = write_emulate,
	.read_write_mmio = write_mmio,
	.read_write_exit_mmio = write_exit_mmio,
	.write = true,
};

4081 4082 4083 4084
static int emulator_read_write_onepage(unsigned long addr, void *val,
				       unsigned int bytes,
				       struct x86_exception *exception,
				       struct kvm_vcpu *vcpu,
4085
				       const struct read_write_emulator_ops *ops)
4086
{
4087 4088
	gpa_t gpa;
	int handled, ret;
4089
	bool write = ops->write;
A
Avi Kivity 已提交
4090
	struct kvm_mmio_fragment *frag;
4091

4092
	ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
4093

4094
	if (ret < 0)
4095 4096 4097
		return X86EMUL_PROPAGATE_FAULT;

	/* For APIC access vmexit */
4098
	if (ret)
4099 4100
		goto mmio;

4101
	if (ops->read_write_emulate(vcpu, gpa, val, bytes))
4102 4103 4104 4105 4106 4107
		return X86EMUL_CONTINUE;

mmio:
	/*
	 * Is this MMIO handled locally?
	 */
4108
	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
4109
	if (handled == bytes)
4110 4111
		return X86EMUL_CONTINUE;

4112 4113 4114 4115
	gpa += handled;
	bytes -= handled;
	val += handled;

4116 4117 4118 4119 4120
	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
	frag->gpa = gpa;
	frag->data = val;
	frag->len = bytes;
A
Avi Kivity 已提交
4121
	return X86EMUL_CONTINUE;
4122 4123
}

4124 4125 4126
int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr,
			void *val, unsigned int bytes,
			struct x86_exception *exception,
4127
			const struct read_write_emulator_ops *ops)
4128
{
4129
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
A
Avi Kivity 已提交
4130 4131 4132 4133 4134 4135 4136 4137
	gpa_t gpa;
	int rc;

	if (ops->read_write_prepare &&
		  ops->read_write_prepare(vcpu, val, bytes))
		return X86EMUL_CONTINUE;

	vcpu->mmio_nr_fragments = 0;
4138

4139 4140
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
A
Avi Kivity 已提交
4141
		int now;
4142 4143

		now = -addr & ~PAGE_MASK;
4144 4145 4146
		rc = emulator_read_write_onepage(addr, val, now, exception,
						 vcpu, ops);

4147 4148 4149 4150 4151 4152
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
		val += now;
		bytes -= now;
	}
4153

A
Avi Kivity 已提交
4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166
	rc = emulator_read_write_onepage(addr, val, bytes, exception,
					 vcpu, ops);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	if (!vcpu->mmio_nr_fragments)
		return rc;

	gpa = vcpu->mmio_fragments[0].gpa;

	vcpu->mmio_needed = 1;
	vcpu->mmio_cur_fragment = 0;

4167
	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
A
Avi Kivity 已提交
4168 4169 4170 4171 4172
	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
	vcpu->run->exit_reason = KVM_EXIT_MMIO;
	vcpu->run->mmio.phys_addr = gpa;

	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192
}

static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
				  unsigned long addr,
				  void *val,
				  unsigned int bytes,
				  struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, val, bytes,
				   exception, &read_emultor);
}

int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
			    unsigned long addr,
			    const void *val,
			    unsigned int bytes,
			    struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, (void *)val, bytes,
				   exception, &write_emultor);
4193 4194
}

4195 4196 4197 4198 4199 4200 4201
#define CMPXCHG_TYPE(t, ptr, old, new) \
	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))

#ifdef CONFIG_X86_64
#  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
#else
#  define CMPXCHG64(ptr, old, new) \
4202
	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
4203 4204
#endif

4205 4206
static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
				     unsigned long addr,
4207 4208 4209
				     const void *old,
				     const void *new,
				     unsigned int bytes,
4210
				     struct x86_exception *exception)
4211
{
4212
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4213 4214 4215 4216
	gpa_t gpa;
	struct page *page;
	char *kaddr;
	bool exchanged;
4217

4218 4219 4220
	/* guests cmpxchg8b have to be emulated atomically */
	if (bytes > 8 || (bytes & (bytes - 1)))
		goto emul_write;
4221

4222
	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
4223

4224 4225 4226
	if (gpa == UNMAPPED_GVA ||
	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		goto emul_write;
4227

4228 4229
	if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
		goto emul_write;
4230

4231
	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
4232
	if (is_error_page(page))
4233
		goto emul_write;
4234

4235
	kaddr = kmap_atomic(page);
4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251
	kaddr += offset_in_page(gpa);
	switch (bytes) {
	case 1:
		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
		break;
	case 2:
		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
		break;
	case 4:
		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
		break;
	case 8:
		exchanged = CMPXCHG64(kaddr, old, new);
		break;
	default:
		BUG();
4252
	}
4253
	kunmap_atomic(kaddr);
4254 4255 4256 4257 4258
	kvm_release_page_dirty(page);

	if (!exchanged)
		return X86EMUL_CMPXCHG_FAILED;

4259
	kvm_mmu_pte_write(vcpu, gpa, new, bytes);
4260 4261

	return X86EMUL_CONTINUE;
4262

4263
emul_write:
4264
	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
4265

4266
	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
4267 4268
}

4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283
static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
{
	/* TODO: String I/O for in kernel device */
	int r;

	if (vcpu->arch.pio.in)
		r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
				    vcpu->arch.pio.size, pd);
	else
		r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
				     vcpu->arch.pio.port, vcpu->arch.pio.size,
				     pd);
	return r;
}

4284 4285 4286
static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
			       unsigned short port, void *val,
			       unsigned int count, bool in)
4287
{
4288
	trace_kvm_pio(!in, port, size, count);
4289 4290

	vcpu->arch.pio.port = port;
4291
	vcpu->arch.pio.in = in;
4292
	vcpu->arch.pio.count  = count;
4293 4294 4295
	vcpu->arch.pio.size = size;

	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
4296
		vcpu->arch.pio.count = 0;
4297 4298 4299 4300
		return 1;
	}

	vcpu->run->exit_reason = KVM_EXIT_IO;
4301
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
4302 4303 4304 4305 4306 4307 4308 4309
	vcpu->run->io.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = count;
	vcpu->run->io.port = port;

	return 0;
}

4310 4311 4312
static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
				    int size, unsigned short port, void *val,
				    unsigned int count)
4313
{
4314
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4315
	int ret;
4316

4317 4318
	if (vcpu->arch.pio.count)
		goto data_avail;
4319

4320 4321 4322 4323
	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
	if (ret) {
data_avail:
		memcpy(val, vcpu->arch.pio_data, size * count);
4324
		vcpu->arch.pio.count = 0;
4325 4326 4327 4328 4329 4330
		return 1;
	}

	return 0;
}

4331 4332 4333 4334 4335 4336 4337 4338 4339 4340
static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
				     int size, unsigned short port,
				     const void *val, unsigned int count)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	memcpy(vcpu->arch.pio_data, val, size * count);
	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
}

4341 4342 4343 4344 4345
static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	return kvm_x86_ops->get_segment_base(vcpu, seg);
}

4346
static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
4347
{
4348
	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
4349 4350
}

4351 4352 4353 4354 4355 4356
int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
	if (!need_emulate_wbinvd(vcpu))
		return X86EMUL_CONTINUE;

	if (kvm_x86_ops->has_wbinvd_exit()) {
4357 4358 4359
		int cpu = get_cpu();

		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
4360 4361
		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
				wbinvd_ipi, NULL, 1);
4362
		put_cpu();
4363
		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
4364 4365
	} else
		wbinvd();
4366 4367 4368 4369
	return X86EMUL_CONTINUE;
}
EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);

4370 4371 4372 4373 4374
static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
{
	kvm_emulate_wbinvd(emul_to_vcpu(ctxt));
}

4375
int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
4376
{
4377
	return _kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
4378 4379
}

4380
int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
4381
{
4382

4383
	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
4384 4385
}

4386
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
4387
{
4388
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
4389 4390
}

4391
static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
4392
{
4393
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4394 4395 4396 4397 4398 4399 4400 4401 4402 4403
	unsigned long value;

	switch (cr) {
	case 0:
		value = kvm_read_cr0(vcpu);
		break;
	case 2:
		value = vcpu->arch.cr2;
		break;
	case 3:
4404
		value = kvm_read_cr3(vcpu);
4405 4406 4407 4408 4409 4410 4411 4412
		break;
	case 4:
		value = kvm_read_cr4(vcpu);
		break;
	case 8:
		value = kvm_get_cr8(vcpu);
		break;
	default:
4413
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
4414 4415 4416 4417 4418 4419
		return 0;
	}

	return value;
}

4420
static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
4421
{
4422
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4423 4424
	int res = 0;

4425 4426
	switch (cr) {
	case 0:
4427
		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
4428 4429 4430 4431 4432
		break;
	case 2:
		vcpu->arch.cr2 = val;
		break;
	case 3:
4433
		res = kvm_set_cr3(vcpu, val);
4434 4435
		break;
	case 4:
4436
		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
4437 4438
		break;
	case 8:
A
Andre Przywara 已提交
4439
		res = kvm_set_cr8(vcpu, val);
4440 4441
		break;
	default:
4442
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
4443
		res = -1;
4444
	}
4445 4446

	return res;
4447 4448
}

4449 4450 4451 4452 4453
static void emulator_set_rflags(struct x86_emulate_ctxt *ctxt, ulong val)
{
	kvm_set_rflags(emul_to_vcpu(ctxt), val);
}

4454
static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
4455
{
4456
	return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
4457 4458
}

4459
static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
4460
{
4461
	kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
4462 4463
}

4464
static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
4465
{
4466
	kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
4467 4468
}

4469 4470 4471 4472 4473 4474 4475 4476 4477 4478
static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
}

static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
}

4479 4480
static unsigned long emulator_get_cached_segment_base(
	struct x86_emulate_ctxt *ctxt, int seg)
4481
{
4482
	return get_segment_base(emul_to_vcpu(ctxt), seg);
4483 4484
}

4485 4486 4487
static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
				 struct desc_struct *desc, u32 *base3,
				 int seg)
4488 4489 4490
{
	struct kvm_segment var;

4491
	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
4492
	*selector = var.selector;
4493

4494 4495
	if (var.unusable) {
		memset(desc, 0, sizeof(*desc));
4496
		return false;
4497
	}
4498 4499 4500 4501 4502

	if (var.g)
		var.limit >>= 12;
	set_desc_limit(desc, var.limit);
	set_desc_base(desc, (unsigned long)var.base);
4503 4504 4505 4506
#ifdef CONFIG_X86_64
	if (base3)
		*base3 = var.base >> 32;
#endif
4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518
	desc->type = var.type;
	desc->s = var.s;
	desc->dpl = var.dpl;
	desc->p = var.present;
	desc->avl = var.avl;
	desc->l = var.l;
	desc->d = var.db;
	desc->g = var.g;

	return true;
}

4519 4520 4521
static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
				 struct desc_struct *desc, u32 base3,
				 int seg)
4522
{
4523
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4524 4525
	struct kvm_segment var;

4526
	var.selector = selector;
4527
	var.base = get_desc_base(desc);
4528 4529 4530
#ifdef CONFIG_X86_64
	var.base |= ((u64)base3) << 32;
#endif
4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549
	var.limit = get_desc_limit(desc);
	if (desc->g)
		var.limit = (var.limit << 12) | 0xfff;
	var.type = desc->type;
	var.present = desc->p;
	var.dpl = desc->dpl;
	var.db = desc->d;
	var.s = desc->s;
	var.l = desc->l;
	var.g = desc->g;
	var.avl = desc->avl;
	var.present = desc->p;
	var.unusable = !var.present;
	var.padding = 0;

	kvm_set_segment(vcpu, &var, seg);
	return;
}

4550 4551 4552 4553 4554 4555 4556 4557 4558
static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 *pdata)
{
	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
}

static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 data)
{
4559 4560 4561 4562 4563 4564
	struct msr_data msr;

	msr.data = data;
	msr.index = msr_index;
	msr.host_initiated = false;
	return kvm_set_msr(emul_to_vcpu(ctxt), &msr);
4565 4566
}

4567 4568 4569 4570 4571 4572
static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
			     u32 pmc, u64 *pdata)
{
	return kvm_pmu_read_pmc(emul_to_vcpu(ctxt), pmc, pdata);
}

4573 4574 4575 4576 4577
static void emulator_halt(struct x86_emulate_ctxt *ctxt)
{
	emul_to_vcpu(ctxt)->arch.halt_request = 1;
}

4578 4579 4580
static void emulator_get_fpu(struct x86_emulate_ctxt *ctxt)
{
	preempt_disable();
4581
	kvm_load_guest_fpu(emul_to_vcpu(ctxt));
4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593
	/*
	 * CR0.TS may reference the host fpu state, not the guest fpu state,
	 * so it may be clear at this point.
	 */
	clts();
}

static void emulator_put_fpu(struct x86_emulate_ctxt *ctxt)
{
	preempt_enable();
}

4594
static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
4595
			      struct x86_instruction_info *info,
4596 4597
			      enum x86_intercept_stage stage)
{
4598
	return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
4599 4600
}

4601
static void emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
4602 4603
			       u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
{
4604
	kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx);
4605 4606
}

4607 4608 4609 4610 4611 4612 4613 4614 4615 4616
static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
{
	return kvm_register_read(emul_to_vcpu(ctxt), reg);
}

static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
{
	kvm_register_write(emul_to_vcpu(ctxt), reg, val);
}

4617
static const struct x86_emulate_ops emulate_ops = {
4618 4619
	.read_gpr            = emulator_read_gpr,
	.write_gpr           = emulator_write_gpr,
4620
	.read_std            = kvm_read_guest_virt_system,
4621
	.write_std           = kvm_write_guest_virt_system,
4622
	.fetch               = kvm_fetch_guest_virt,
4623 4624 4625
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
4626
	.invlpg              = emulator_invlpg,
4627 4628
	.pio_in_emulated     = emulator_pio_in_emulated,
	.pio_out_emulated    = emulator_pio_out_emulated,
4629 4630
	.get_segment         = emulator_get_segment,
	.set_segment         = emulator_set_segment,
4631
	.get_cached_segment_base = emulator_get_cached_segment_base,
4632
	.get_gdt             = emulator_get_gdt,
4633
	.get_idt	     = emulator_get_idt,
4634 4635
	.set_gdt             = emulator_set_gdt,
	.set_idt	     = emulator_set_idt,
4636 4637
	.get_cr              = emulator_get_cr,
	.set_cr              = emulator_set_cr,
4638
	.set_rflags          = emulator_set_rflags,
4639
	.cpl                 = emulator_get_cpl,
4640 4641
	.get_dr              = emulator_get_dr,
	.set_dr              = emulator_set_dr,
4642 4643
	.set_msr             = emulator_set_msr,
	.get_msr             = emulator_get_msr,
4644
	.read_pmc            = emulator_read_pmc,
4645
	.halt                = emulator_halt,
4646
	.wbinvd              = emulator_wbinvd,
4647
	.fix_hypercall       = emulator_fix_hypercall,
4648 4649
	.get_fpu             = emulator_get_fpu,
	.put_fpu             = emulator_put_fpu,
4650
	.intercept           = emulator_intercept,
4651
	.get_cpuid           = emulator_get_cpuid,
4652 4653
};

4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667
static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
{
	u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask);
	/*
	 * an sti; sti; sequence only disable interrupts for the first
	 * instruction. So, if the last instruction, be it emulated or
	 * not, left the system with the INT_STI flag enabled, it
	 * means that the last instruction is an sti. We should not
	 * leave the flag on in this case. The same goes for mov ss
	 */
	if (!(int_shadow & mask))
		kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
}

4668 4669 4670
static void inject_emulated_exception(struct kvm_vcpu *vcpu)
{
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4671
	if (ctxt->exception.vector == PF_VECTOR)
4672
		kvm_propagate_fault(vcpu, &ctxt->exception);
4673 4674 4675
	else if (ctxt->exception.error_code_valid)
		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
				      ctxt->exception.error_code);
4676
	else
4677
		kvm_queue_exception(vcpu, ctxt->exception.vector);
4678 4679
}

4680
static void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4681
{
4682
	memset(&ctxt->twobyte, 0,
4683
	       (void *)&ctxt->_regs - (void *)&ctxt->twobyte);
4684

4685 4686 4687 4688 4689 4690
	ctxt->fetch.start = 0;
	ctxt->fetch.end = 0;
	ctxt->io_read.pos = 0;
	ctxt->io_read.end = 0;
	ctxt->mem_read.pos = 0;
	ctxt->mem_read.end = 0;
4691 4692
}

4693 4694
static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
{
4695
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4696 4697 4698 4699
	int cs_db, cs_l;

	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);

4700 4701 4702 4703 4704 4705 4706 4707 4708
	ctxt->eflags = kvm_get_rflags(vcpu);
	ctxt->eip = kvm_rip_read(vcpu);
	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
		     cs_l				? X86EMUL_MODE_PROT64 :
		     cs_db				? X86EMUL_MODE_PROT32 :
							  X86EMUL_MODE_PROT16;
	ctxt->guest_mode = is_guest_mode(vcpu);

4709
	init_decode_cache(ctxt);
4710
	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
4711 4712
}

4713
int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
4714
{
4715
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4716 4717 4718 4719
	int ret;

	init_emulate_ctxt(vcpu);

4720 4721 4722
	ctxt->op_bytes = 2;
	ctxt->ad_bytes = 2;
	ctxt->_eip = ctxt->eip + inc_eip;
4723
	ret = emulate_int_real(ctxt, irq);
4724 4725 4726 4727

	if (ret != X86EMUL_CONTINUE)
		return EMULATE_FAIL;

4728
	ctxt->eip = ctxt->_eip;
4729 4730
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
4731 4732

	if (irq == NMI_VECTOR)
A
Avi Kivity 已提交
4733
		vcpu->arch.nmi_pending = 0;
4734 4735 4736 4737 4738 4739 4740
	else
		vcpu->arch.interrupt.pending = false;

	return EMULATE_DONE;
}
EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);

4741 4742
static int handle_emulation_failure(struct kvm_vcpu *vcpu)
{
4743 4744
	int r = EMULATE_DONE;

4745 4746
	++vcpu->stat.insn_emulation_fail;
	trace_kvm_emulate_insn_failed(vcpu);
4747 4748 4749 4750 4751 4752
	if (!is_guest_mode(vcpu)) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
		r = EMULATE_FAIL;
	}
4753
	kvm_queue_exception(vcpu, UD_VECTOR);
4754 4755

	return r;
4756 4757
}

4758 4759
static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,
				  bool write_fault_to_shadow_pgtable)
4760
{
4761
	gpa_t gpa = cr2;
4762
	pfn_t pfn;
4763

4764 4765 4766 4767 4768 4769
	if (!vcpu->arch.mmu.direct_map) {
		/*
		 * Write permission should be allowed since only
		 * write access need to be emulated.
		 */
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
4770

4771 4772 4773 4774 4775 4776 4777
		/*
		 * If the mapping is invalid in guest, let cpu retry
		 * it to generate fault.
		 */
		if (gpa == UNMAPPED_GVA)
			return true;
	}
4778

4779 4780 4781 4782 4783 4784 4785
	/*
	 * Do not retry the unhandleable instruction if it faults on the
	 * readonly host memory, otherwise it will goto a infinite loop:
	 * retry instruction -> write #PF -> emulation fail -> retry
	 * instruction -> ...
	 */
	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806

	/*
	 * If the instruction failed on the error pfn, it can not be fixed,
	 * report the error to userspace.
	 */
	if (is_error_noslot_pfn(pfn))
		return false;

	kvm_release_pfn_clean(pfn);

	/* The instructions are well-emulated on direct mmu. */
	if (vcpu->arch.mmu.direct_map) {
		unsigned int indirect_shadow_pages;

		spin_lock(&vcpu->kvm->mmu_lock);
		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
		spin_unlock(&vcpu->kvm->mmu_lock);

		if (indirect_shadow_pages)
			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));

4807
		return true;
4808
	}
4809

4810 4811 4812 4813 4814 4815
	/*
	 * if emulation was due to access to shadowed page table
	 * and it failed try to unshadow page and re-enter the
	 * guest to let CPU execute the instruction.
	 */
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
4816 4817 4818 4819 4820 4821 4822

	/*
	 * If the access faults on its page table, it can not
	 * be fixed by unprotecting shadow page and it should
	 * be reported to userspace.
	 */
	return !write_fault_to_shadow_pgtable;
4823 4824
}

4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863
static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
			      unsigned long cr2,  int emulation_type)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	unsigned long last_retry_eip, last_retry_addr, gpa = cr2;

	last_retry_eip = vcpu->arch.last_retry_eip;
	last_retry_addr = vcpu->arch.last_retry_addr;

	/*
	 * If the emulation is caused by #PF and it is non-page_table
	 * writing instruction, it means the VM-EXIT is caused by shadow
	 * page protected, we can zap the shadow page and retry this
	 * instruction directly.
	 *
	 * Note: if the guest uses a non-page-table modifying instruction
	 * on the PDE that points to the instruction, then we will unmap
	 * the instruction and go to an infinite loop. So, we cache the
	 * last retried eip and the last fault address, if we meet the eip
	 * and the address again, we can break out of the potential infinite
	 * loop.
	 */
	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;

	if (!(emulation_type & EMULTYPE_RETRY))
		return false;

	if (x86_page_table_writing_insn(ctxt))
		return false;

	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2)
		return false;

	vcpu->arch.last_retry_eip = ctxt->eip;
	vcpu->arch.last_retry_addr = cr2;

	if (!vcpu->arch.mmu.direct_map)
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);

4864
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
4865 4866 4867 4868

	return true;
}

4869 4870 4871
static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
static int complete_emulated_pio(struct kvm_vcpu *vcpu);

4872 4873
int x86_emulate_instruction(struct kvm_vcpu *vcpu,
			    unsigned long cr2,
4874 4875 4876
			    int emulation_type,
			    void *insn,
			    int insn_len)
4877
{
4878
	int r;
4879
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4880
	bool writeback = true;
4881
	bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
4882

4883 4884 4885 4886 4887
	/*
	 * Clear write_fault_to_shadow_pgtable here to ensure it is
	 * never reused.
	 */
	vcpu->arch.write_fault_to_shadow_pgtable = false;
4888
	kvm_clear_exception_queue(vcpu);
G
Gleb Natapov 已提交
4889

4890
	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
4891
		init_emulate_ctxt(vcpu);
4892 4893 4894
		ctxt->interruptibility = 0;
		ctxt->have_exception = false;
		ctxt->perm_ok = false;
4895

4896
		ctxt->only_vendor_specific_insn
4897 4898
			= emulation_type & EMULTYPE_TRAP_UD;

4899
		r = x86_decode_insn(ctxt, insn, insn_len);
4900

A
Avi Kivity 已提交
4901
		trace_kvm_emulate_insn_start(vcpu);
4902
		++vcpu->stat.insn_emulation;
4903
		if (r != EMULATION_OK)  {
4904 4905
			if (emulation_type & EMULTYPE_TRAP_UD)
				return EMULATE_FAIL;
4906 4907
			if (reexecute_instruction(vcpu, cr2,
						  write_fault_to_spt))
4908
				return EMULATE_DONE;
4909 4910 4911
			if (emulation_type & EMULTYPE_SKIP)
				return EMULATE_FAIL;
			return handle_emulation_failure(vcpu);
4912 4913 4914
		}
	}

4915
	if (emulation_type & EMULTYPE_SKIP) {
4916
		kvm_rip_write(vcpu, ctxt->_eip);
4917 4918 4919
		return EMULATE_DONE;
	}

4920 4921 4922
	if (retry_instruction(ctxt, cr2, emulation_type))
		return EMULATE_DONE;

4923
	/* this is needed for vmware backdoor interface to work since it
4924
	   changes registers values  during IO operation */
4925 4926
	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
4927
		emulator_invalidate_register_cache(ctxt);
4928
	}
4929

4930
restart:
4931
	r = x86_emulate_insn(ctxt);
4932

4933 4934 4935
	if (r == EMULATION_INTERCEPTED)
		return EMULATE_DONE;

4936
	if (r == EMULATION_FAILED) {
4937
		if (reexecute_instruction(vcpu, cr2, write_fault_to_spt))
4938 4939
			return EMULATE_DONE;

4940
		return handle_emulation_failure(vcpu);
4941 4942
	}

4943
	if (ctxt->have_exception) {
4944
		inject_emulated_exception(vcpu);
4945 4946
		r = EMULATE_DONE;
	} else if (vcpu->arch.pio.count) {
4947 4948
		if (!vcpu->arch.pio.in)
			vcpu->arch.pio.count = 0;
4949
		else {
4950
			writeback = false;
4951 4952
			vcpu->arch.complete_userspace_io = complete_emulated_pio;
		}
4953
		r = EMULATE_DO_MMIO;
4954 4955 4956
	} else if (vcpu->mmio_needed) {
		if (!vcpu->mmio_is_write)
			writeback = false;
4957
		r = EMULATE_DO_MMIO;
4958
		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
4959
	} else if (r == EMULATION_RESTART)
4960
		goto restart;
4961 4962
	else
		r = EMULATE_DONE;
4963

4964
	if (writeback) {
4965 4966
		toggle_interruptibility(vcpu, ctxt->interruptibility);
		kvm_set_rflags(vcpu, ctxt->eflags);
4967 4968
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
4969
		kvm_rip_write(vcpu, ctxt->eip);
4970 4971
	} else
		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
4972 4973

	return r;
4974
}
4975
EXPORT_SYMBOL_GPL(x86_emulate_instruction);
4976

4977
int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
4978
{
4979
	unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX);
4980 4981
	int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
					    size, port, &val, 1);
4982
	/* do not return to emulator after return from userspace */
4983
	vcpu->arch.pio.count = 0;
4984 4985
	return ret;
}
4986
EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
4987

4988 4989
static void tsc_bad(void *info)
{
T
Tejun Heo 已提交
4990
	__this_cpu_write(cpu_tsc_khz, 0);
4991 4992 4993
}

static void tsc_khz_changed(void *data)
4994
{
4995 4996 4997 4998 4999 5000 5001 5002 5003
	struct cpufreq_freqs *freq = data;
	unsigned long khz = 0;

	if (data)
		khz = freq->new;
	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		khz = cpufreq_quick_get(raw_smp_processor_id());
	if (!khz)
		khz = tsc_khz;
T
Tejun Heo 已提交
5004
	__this_cpu_write(cpu_tsc_khz, khz);
5005 5006 5007 5008 5009 5010 5011 5012 5013 5014
}

static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
				     void *data)
{
	struct cpufreq_freqs *freq = data;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i, send_ipi = 0;

5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053
	/*
	 * We allow guests to temporarily run on slowing clocks,
	 * provided we notify them after, or to run on accelerating
	 * clocks, provided we notify them before.  Thus time never
	 * goes backwards.
	 *
	 * However, we have a problem.  We can't atomically update
	 * the frequency of a given CPU from this function; it is
	 * merely a notifier, which can be called from any CPU.
	 * Changing the TSC frequency at arbitrary points in time
	 * requires a recomputation of local variables related to
	 * the TSC for each VCPU.  We must flag these local variables
	 * to be updated and be sure the update takes place with the
	 * new frequency before any guests proceed.
	 *
	 * Unfortunately, the combination of hotplug CPU and frequency
	 * change creates an intractable locking scenario; the order
	 * of when these callouts happen is undefined with respect to
	 * CPU hotplug, and they can race with each other.  As such,
	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
	 * undefined; you can actually have a CPU frequency change take
	 * place in between the computation of X and the setting of the
	 * variable.  To protect against this problem, all updates of
	 * the per_cpu tsc_khz variable are done in an interrupt
	 * protected IPI, and all callers wishing to update the value
	 * must wait for a synchronous IPI to complete (which is trivial
	 * if the caller is on the CPU already).  This establishes the
	 * necessary total order on variable updates.
	 *
	 * Note that because a guest time update may take place
	 * anytime after the setting of the VCPU's request bit, the
	 * correct TSC value must be set before the request.  However,
	 * to ensure the update actually makes it to any guest which
	 * starts running in hardware virtualization between the set
	 * and the acquisition of the spinlock, we must also ping the
	 * CPU after setting the request bit.
	 *
	 */

5054 5055 5056 5057
	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
		return 0;
	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
		return 0;
5058 5059

	smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
5060

5061
	raw_spin_lock(&kvm_lock);
5062
	list_for_each_entry(kvm, &vm_list, vm_list) {
5063
		kvm_for_each_vcpu(i, vcpu, kvm) {
5064 5065
			if (vcpu->cpu != freq->cpu)
				continue;
Z
Zachary Amsden 已提交
5066
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5067
			if (vcpu->cpu != smp_processor_id())
5068
				send_ipi = 1;
5069 5070
		}
	}
5071
	raw_spin_unlock(&kvm_lock);
5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085

	if (freq->old < freq->new && send_ipi) {
		/*
		 * We upscale the frequency.  Must make the guest
		 * doesn't see old kvmclock values while running with
		 * the new frequency, otherwise we risk the guest sees
		 * time go backwards.
		 *
		 * In case we update the frequency for another cpu
		 * (which might be in guest context) send an interrupt
		 * to kick the cpu out of guest context.  Next time
		 * guest context is entered kvmclock will be updated,
		 * so the guest will not see stale values.
		 */
5086
		smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
5087 5088 5089 5090 5091
	}
	return 0;
}

static struct notifier_block kvmclock_cpufreq_notifier_block = {
5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114
	.notifier_call  = kvmclock_cpufreq_notifier
};

static int kvmclock_cpu_notifier(struct notifier_block *nfb,
					unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned long)hcpu;

	switch (action) {
		case CPU_ONLINE:
		case CPU_DOWN_FAILED:
			smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
			break;
		case CPU_DOWN_PREPARE:
			smp_call_function_single(cpu, tsc_bad, NULL, 1);
			break;
	}
	return NOTIFY_OK;
}

static struct notifier_block kvmclock_cpu_notifier_block = {
	.notifier_call  = kvmclock_cpu_notifier,
	.priority = -INT_MAX
5115 5116
};

5117 5118 5119 5120
static void kvm_timer_init(void)
{
	int cpu;

Z
Zachary Amsden 已提交
5121
	max_tsc_khz = tsc_khz;
5122
	register_hotcpu_notifier(&kvmclock_cpu_notifier_block);
5123
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
Z
Zachary Amsden 已提交
5124 5125 5126
#ifdef CONFIG_CPU_FREQ
		struct cpufreq_policy policy;
		memset(&policy, 0, sizeof(policy));
5127 5128
		cpu = get_cpu();
		cpufreq_get_policy(&policy, cpu);
Z
Zachary Amsden 已提交
5129 5130
		if (policy.cpuinfo.max_freq)
			max_tsc_khz = policy.cpuinfo.max_freq;
5131
		put_cpu();
Z
Zachary Amsden 已提交
5132
#endif
5133 5134 5135
		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
					  CPUFREQ_TRANSITION_NOTIFIER);
	}
Z
Zachary Amsden 已提交
5136
	pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz);
5137 5138
	for_each_online_cpu(cpu)
		smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
5139 5140
}

5141 5142
static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);

5143
int kvm_is_in_guest(void)
5144
{
5145
	return __this_cpu_read(current_vcpu) != NULL;
5146 5147 5148 5149 5150
}

static int kvm_is_user_mode(void)
{
	int user_mode = 3;
5151

5152 5153
	if (__this_cpu_read(current_vcpu))
		user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
5154

5155 5156 5157 5158 5159 5160
	return user_mode != 0;
}

static unsigned long kvm_get_guest_ip(void)
{
	unsigned long ip = 0;
5161

5162 5163
	if (__this_cpu_read(current_vcpu))
		ip = kvm_rip_read(__this_cpu_read(current_vcpu));
5164

5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175
	return ip;
}

static struct perf_guest_info_callbacks kvm_guest_cbs = {
	.is_in_guest		= kvm_is_in_guest,
	.is_user_mode		= kvm_is_user_mode,
	.get_guest_ip		= kvm_get_guest_ip,
};

void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
{
5176
	__this_cpu_write(current_vcpu, vcpu);
5177 5178 5179 5180 5181
}
EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);

void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
{
5182
	__this_cpu_write(current_vcpu, NULL);
5183 5184 5185
}
EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);

5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209
static void kvm_set_mmio_spte_mask(void)
{
	u64 mask;
	int maxphyaddr = boot_cpu_data.x86_phys_bits;

	/*
	 * Set the reserved bits and the present bit of an paging-structure
	 * entry to generate page fault with PFER.RSV = 1.
	 */
	mask = ((1ull << (62 - maxphyaddr + 1)) - 1) << maxphyaddr;
	mask |= 1ull;

#ifdef CONFIG_X86_64
	/*
	 * If reserved bit is not supported, clear the present bit to disable
	 * mmio page fault.
	 */
	if (maxphyaddr == 52)
		mask &= ~1ull;
#endif

	kvm_mmu_set_mmio_spte_mask(mask);
}

5210 5211 5212
#ifdef CONFIG_X86_64
static void pvclock_gtod_update_fn(struct work_struct *work)
{
5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223
	struct kvm *kvm;

	struct kvm_vcpu *vcpu;
	int i;

	raw_spin_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_for_each_vcpu(i, vcpu, kvm)
			set_bit(KVM_REQ_MASTERCLOCK_UPDATE, &vcpu->requests);
	atomic_set(&kvm_guest_has_master_clock, 0);
	raw_spin_unlock(&kvm_lock);
5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253
}

static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);

/*
 * Notification about pvclock gtod data update.
 */
static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
			       void *priv)
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	struct timekeeper *tk = priv;

	update_pvclock_gtod(tk);

	/* disable master clock if host does not trust, or does not
	 * use, TSC clocksource
	 */
	if (gtod->clock.vclock_mode != VCLOCK_TSC &&
	    atomic_read(&kvm_guest_has_master_clock) != 0)
		queue_work(system_long_wq, &pvclock_gtod_work);

	return 0;
}

static struct notifier_block pvclock_gtod_notifier = {
	.notifier_call = pvclock_gtod_notify,
};
#endif

5254
int kvm_arch_init(void *opaque)
5255
{
5256
	int r;
5257 5258 5259 5260
	struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;

	if (kvm_x86_ops) {
		printk(KERN_ERR "kvm: already loaded the other module\n");
5261 5262
		r = -EEXIST;
		goto out;
5263 5264 5265 5266
	}

	if (!ops->cpu_has_kvm_support()) {
		printk(KERN_ERR "kvm: no hardware support\n");
5267 5268
		r = -EOPNOTSUPP;
		goto out;
5269 5270 5271
	}
	if (ops->disabled_by_bios()) {
		printk(KERN_ERR "kvm: disabled by bios\n");
5272 5273
		r = -EOPNOTSUPP;
		goto out;
5274 5275
	}

5276 5277 5278 5279 5280 5281 5282
	r = -ENOMEM;
	shared_msrs = alloc_percpu(struct kvm_shared_msrs);
	if (!shared_msrs) {
		printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
		goto out;
	}

5283 5284
	r = kvm_mmu_module_init();
	if (r)
5285
		goto out_free_percpu;
5286

5287
	kvm_set_mmio_spte_mask();
5288 5289
	kvm_init_msr_list();

5290
	kvm_x86_ops = ops;
S
Sheng Yang 已提交
5291
	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
5292
			PT_DIRTY_MASK, PT64_NX_MASK, 0);
5293

5294
	kvm_timer_init();
5295

5296 5297
	perf_register_guest_info_callbacks(&kvm_guest_cbs);

5298 5299 5300
	if (cpu_has_xsave)
		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);

5301
	kvm_lapic_init();
5302 5303 5304 5305
#ifdef CONFIG_X86_64
	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
#endif

5306
	return 0;
5307

5308 5309
out_free_percpu:
	free_percpu(shared_msrs);
5310 5311
out:
	return r;
5312
}
5313

5314 5315
void kvm_arch_exit(void)
{
5316 5317
	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);

5318 5319 5320
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
					    CPUFREQ_TRANSITION_NOTIFIER);
5321
	unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block);
5322 5323 5324
#ifdef CONFIG_X86_64
	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
#endif
5325
	kvm_x86_ops = NULL;
5326
	kvm_mmu_module_exit();
5327
	free_percpu(shared_msrs);
5328
}
5329

5330 5331 5332 5333
int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
	++vcpu->stat.halt_exits;
	if (irqchip_in_kernel(vcpu->kvm)) {
5334
		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
5335 5336 5337 5338 5339 5340 5341 5342
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
}
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353
int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
{
	u64 param, ingpa, outgpa, ret;
	uint16_t code, rep_idx, rep_cnt, res = HV_STATUS_SUCCESS, rep_done = 0;
	bool fast, longmode;
	int cs_db, cs_l;

	/*
	 * hypercall generates UD from non zero cpl and real mode
	 * per HYPER-V spec
	 */
5354
	if (kvm_x86_ops->get_cpl(vcpu) != 0 || !is_protmode(vcpu)) {
5355 5356 5357 5358 5359 5360 5361 5362
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 0;
	}

	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
	longmode = is_long_mode(vcpu) && cs_l == 1;

	if (!longmode) {
5363 5364 5365 5366 5367 5368
		param = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDX) << 32) |
			(kvm_register_read(vcpu, VCPU_REGS_RAX) & 0xffffffff);
		ingpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RBX) << 32) |
			(kvm_register_read(vcpu, VCPU_REGS_RCX) & 0xffffffff);
		outgpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDI) << 32) |
			(kvm_register_read(vcpu, VCPU_REGS_RSI) & 0xffffffff);
5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384
	}
#ifdef CONFIG_X86_64
	else {
		param = kvm_register_read(vcpu, VCPU_REGS_RCX);
		ingpa = kvm_register_read(vcpu, VCPU_REGS_RDX);
		outgpa = kvm_register_read(vcpu, VCPU_REGS_R8);
	}
#endif

	code = param & 0xffff;
	fast = (param >> 16) & 0x1;
	rep_cnt = (param >> 32) & 0xfff;
	rep_idx = (param >> 48) & 0xfff;

	trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa);

5385 5386 5387 5388 5389 5390 5391 5392
	switch (code) {
	case HV_X64_HV_NOTIFY_LONG_SPIN_WAIT:
		kvm_vcpu_on_spin(vcpu);
		break;
	default:
		res = HV_STATUS_INVALID_HYPERCALL_CODE;
		break;
	}
5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404

	ret = res | (((u64)rep_done & 0xfff) << 32);
	if (longmode) {
		kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
	} else {
		kvm_register_write(vcpu, VCPU_REGS_RDX, ret >> 32);
		kvm_register_write(vcpu, VCPU_REGS_RAX, ret & 0xffffffff);
	}

	return 1;
}

5405 5406 5407
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
{
	unsigned long nr, a0, a1, a2, a3, ret;
5408
	int r = 1;
5409

5410 5411 5412
	if (kvm_hv_hypercall_enabled(vcpu->kvm))
		return kvm_hv_hypercall(vcpu);

5413 5414 5415 5416 5417
	nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
	a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
	a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
	a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
	a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
5418

5419
	trace_kvm_hypercall(nr, a0, a1, a2, a3);
F
Feng (Eric) Liu 已提交
5420

5421 5422 5423 5424 5425 5426 5427 5428
	if (!is_long_mode(vcpu)) {
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
	}

5429 5430 5431 5432 5433
	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
		ret = -KVM_EPERM;
		goto out;
	}

5434
	switch (nr) {
A
Avi Kivity 已提交
5435 5436 5437
	case KVM_HC_VAPIC_POLL_IRQ:
		ret = 0;
		break;
5438 5439 5440 5441
	default:
		ret = -KVM_ENOSYS;
		break;
	}
5442
out:
5443
	kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
A
Amit Shah 已提交
5444
	++vcpu->stat.hypercalls;
5445
	return r;
5446 5447 5448
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

5449
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
5450
{
5451
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5452
	char instruction[3];
5453
	unsigned long rip = kvm_rip_read(vcpu);
5454 5455 5456 5457 5458 5459 5460 5461 5462 5463

	/*
	 * Blow out the MMU to ensure that no other VCPU has an active mapping
	 * to ensure that the updated hypercall appears atomically across all
	 * VCPUs.
	 */
	kvm_mmu_zap_all(vcpu->kvm);

	kvm_x86_ops->patch_hypercall(vcpu, instruction);

5464
	return emulator_write_emulated(ctxt, rip, instruction, 3, NULL);
5465 5466
}

5467 5468 5469 5470 5471 5472
/*
 * Check if userspace requested an interrupt window, and that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
A
Avi Kivity 已提交
5473
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
5474
{
5475
	return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
A
Avi Kivity 已提交
5476
		vcpu->run->request_interrupt_window &&
5477
		kvm_arch_interrupt_allowed(vcpu));
5478 5479
}

A
Avi Kivity 已提交
5480
static void post_kvm_run_save(struct kvm_vcpu *vcpu)
5481
{
A
Avi Kivity 已提交
5482 5483
	struct kvm_run *kvm_run = vcpu->run;

5484
	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
5485
	kvm_run->cr8 = kvm_get_cr8(vcpu);
5486
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
5487
	if (irqchip_in_kernel(vcpu->kvm))
5488
		kvm_run->ready_for_interrupt_injection = 1;
5489
	else
5490
		kvm_run->ready_for_interrupt_injection =
5491 5492 5493
			kvm_arch_interrupt_allowed(vcpu) &&
			!kvm_cpu_has_interrupt(vcpu) &&
			!kvm_event_needs_reinjection(vcpu);
5494 5495
}

5496
static int vapic_enter(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5497 5498 5499 5500 5501
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	struct page *page;

	if (!apic || !apic->vapic_addr)
5502
		return 0;
A
Avi Kivity 已提交
5503 5504

	page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
5505 5506
	if (is_error_page(page))
		return -EFAULT;
5507 5508

	vcpu->arch.apic->vapic_page = page;
5509
	return 0;
A
Avi Kivity 已提交
5510 5511 5512 5513 5514
}

static void vapic_exit(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
5515
	int idx;
A
Avi Kivity 已提交
5516 5517 5518 5519

	if (!apic || !apic->vapic_addr)
		return;

5520
	idx = srcu_read_lock(&vcpu->kvm->srcu);
A
Avi Kivity 已提交
5521 5522
	kvm_release_page_dirty(apic->vapic_page);
	mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
5523
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
A
Avi Kivity 已提交
5524 5525
}

5526 5527 5528 5529 5530 5531 5532
static void update_cr8_intercept(struct kvm_vcpu *vcpu)
{
	int max_irr, tpr;

	if (!kvm_x86_ops->update_cr8_intercept)
		return;

5533 5534 5535
	if (!vcpu->arch.apic)
		return;

5536 5537 5538 5539
	if (!vcpu->arch.apic->vapic_addr)
		max_irr = kvm_lapic_find_highest_irr(vcpu);
	else
		max_irr = -1;
5540 5541 5542 5543 5544 5545 5546 5547 5548

	if (max_irr != -1)
		max_irr >>= 4;

	tpr = kvm_lapic_get_cr8(vcpu);

	kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
}

A
Avi Kivity 已提交
5549
static void inject_pending_event(struct kvm_vcpu *vcpu)
5550 5551
{
	/* try to reinject previous events if any */
5552
	if (vcpu->arch.exception.pending) {
A
Avi Kivity 已提交
5553 5554 5555
		trace_kvm_inj_exception(vcpu->arch.exception.nr,
					vcpu->arch.exception.has_error_code,
					vcpu->arch.exception.error_code);
5556 5557
		kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
					  vcpu->arch.exception.has_error_code,
5558 5559
					  vcpu->arch.exception.error_code,
					  vcpu->arch.exception.reinject);
5560 5561 5562
		return;
	}

5563 5564 5565 5566 5567 5568
	if (vcpu->arch.nmi_injected) {
		kvm_x86_ops->set_nmi(vcpu);
		return;
	}

	if (vcpu->arch.interrupt.pending) {
5569
		kvm_x86_ops->set_irq(vcpu);
5570 5571 5572 5573 5574 5575
		return;
	}

	/* try to inject new event if pending */
	if (vcpu->arch.nmi_pending) {
		if (kvm_x86_ops->nmi_allowed(vcpu)) {
A
Avi Kivity 已提交
5576
			--vcpu->arch.nmi_pending;
5577 5578 5579
			vcpu->arch.nmi_injected = true;
			kvm_x86_ops->set_nmi(vcpu);
		}
5580
	} else if (kvm_cpu_has_injectable_intr(vcpu)) {
5581
		if (kvm_x86_ops->interrupt_allowed(vcpu)) {
5582 5583 5584
			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
					    false);
			kvm_x86_ops->set_irq(vcpu);
5585 5586 5587 5588
		}
	}
}

5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607
static void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
{
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
			!vcpu->guest_xcr0_loaded) {
		/* kvm_set_xcr() also depends on this */
		xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
		vcpu->guest_xcr0_loaded = 1;
	}
}

static void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
{
	if (vcpu->guest_xcr0_loaded) {
		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
		vcpu->guest_xcr0_loaded = 0;
	}
}

A
Avi Kivity 已提交
5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624
static void process_nmi(struct kvm_vcpu *vcpu)
{
	unsigned limit = 2;

	/*
	 * x86 is limited to one NMI running, and one NMI pending after it.
	 * If an NMI is already in progress, limit further NMIs to just one.
	 * Otherwise, allow two (and we'll inject the first one immediately).
	 */
	if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
		limit = 1;

	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647
static void kvm_gen_update_masterclock(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	int i;
	struct kvm_vcpu *vcpu;
	struct kvm_arch *ka = &kvm->arch;

	spin_lock(&ka->pvclock_gtod_sync_lock);
	kvm_make_mclock_inprogress_request(kvm);
	/* no guest entries from this point */
	pvclock_update_vm_gtod_copy(kvm);

	kvm_for_each_vcpu(i, vcpu, kvm)
		set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);

	/* guest entries allowed */
	kvm_for_each_vcpu(i, vcpu, kvm)
		clear_bit(KVM_REQ_MCLOCK_INPROGRESS, &vcpu->requests);

	spin_unlock(&ka->pvclock_gtod_sync_lock);
#endif
}

5648 5649 5650 5651 5652 5653 5654 5655 5656 5657
static void update_eoi_exitmap(struct kvm_vcpu *vcpu)
{
	u64 eoi_exit_bitmap[4];

	memset(eoi_exit_bitmap, 0, 32);

	kvm_ioapic_calculate_eoi_exitmap(vcpu, eoi_exit_bitmap);
	kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
}

A
Avi Kivity 已提交
5658
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
5659 5660
{
	int r;
5661
	bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
A
Avi Kivity 已提交
5662
		vcpu->run->request_interrupt_window;
5663
	bool req_immediate_exit = 0;
5664

5665
	if (vcpu->requests) {
5666
		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
5667
			kvm_mmu_unload(vcpu);
5668
		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
M
Marcelo Tosatti 已提交
5669
			__kvm_migrate_timers(vcpu);
5670 5671
		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
			kvm_gen_update_masterclock(vcpu->kvm);
Z
Zachary Amsden 已提交
5672 5673
		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
			r = kvm_guest_time_update(vcpu);
5674 5675 5676
			if (unlikely(r))
				goto out;
		}
5677
		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
5678
			kvm_mmu_sync_roots(vcpu);
5679
		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
5680
			kvm_x86_ops->tlb_flush(vcpu);
5681
		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
A
Avi Kivity 已提交
5682
			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
A
Avi Kivity 已提交
5683 5684 5685
			r = 0;
			goto out;
		}
5686
		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
A
Avi Kivity 已提交
5687
			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
J
Joerg Roedel 已提交
5688 5689 5690
			r = 0;
			goto out;
		}
5691
		if (kvm_check_request(KVM_REQ_DEACTIVATE_FPU, vcpu)) {
5692 5693 5694
			vcpu->fpu_active = 0;
			kvm_x86_ops->fpu_deactivate(vcpu);
		}
5695 5696 5697 5698 5699 5700
		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
			/* Page is swapped out. Do synthetic halt */
			vcpu->arch.apf.halted = true;
			r = 1;
			goto out;
		}
G
Glauber Costa 已提交
5701 5702
		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
			record_steal_time(vcpu);
A
Avi Kivity 已提交
5703 5704
		if (kvm_check_request(KVM_REQ_NMI, vcpu))
			process_nmi(vcpu);
5705 5706
		req_immediate_exit =
			kvm_check_request(KVM_REQ_IMMEDIATE_EXIT, vcpu);
5707 5708 5709 5710
		if (kvm_check_request(KVM_REQ_PMU, vcpu))
			kvm_handle_pmu_event(vcpu);
		if (kvm_check_request(KVM_REQ_PMI, vcpu))
			kvm_deliver_pmi(vcpu);
5711 5712
		if (kvm_check_request(KVM_REQ_EOIBITMAP, vcpu))
			update_eoi_exitmap(vcpu);
5713
	}
A
Avi Kivity 已提交
5714

A
Avi Kivity 已提交
5715 5716 5717 5718
	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
		inject_pending_event(vcpu);

		/* enable NMI/IRQ window open exits if needed */
A
Avi Kivity 已提交
5719
		if (vcpu->arch.nmi_pending)
A
Avi Kivity 已提交
5720
			kvm_x86_ops->enable_nmi_window(vcpu);
5721
		else if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
A
Avi Kivity 已提交
5722 5723 5724
			kvm_x86_ops->enable_irq_window(vcpu);

		if (kvm_lapic_enabled(vcpu)) {
5725 5726 5727 5728 5729 5730 5731
			/*
			 * Update architecture specific hints for APIC
			 * virtual interrupt delivery.
			 */
			if (kvm_x86_ops->hwapic_irr_update)
				kvm_x86_ops->hwapic_irr_update(vcpu,
					kvm_lapic_find_highest_irr(vcpu));
A
Avi Kivity 已提交
5732 5733 5734 5735 5736
			update_cr8_intercept(vcpu);
			kvm_lapic_sync_to_vapic(vcpu);
		}
	}

5737 5738
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r)) {
5739
		goto cancel_injection;
5740 5741
	}

5742 5743 5744
	preempt_disable();

	kvm_x86_ops->prepare_guest_switch(vcpu);
5745 5746
	if (vcpu->fpu_active)
		kvm_load_guest_fpu(vcpu);
5747
	kvm_load_guest_xcr0(vcpu);
5748

5749 5750 5751 5752 5753 5754
	vcpu->mode = IN_GUEST_MODE;

	/* We should set ->mode before check ->requests,
	 * see the comment in make_all_cpus_request.
	 */
	smp_mb();
5755

A
Avi Kivity 已提交
5756
	local_irq_disable();
5757

5758
	if (vcpu->mode == EXITING_GUEST_MODE || vcpu->requests
A
Avi Kivity 已提交
5759
	    || need_resched() || signal_pending(current)) {
5760
		vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
5761
		smp_wmb();
5762 5763 5764
		local_irq_enable();
		preempt_enable();
		r = 1;
5765
		goto cancel_injection;
5766 5767
	}

5768
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
5769

5770 5771 5772
	if (req_immediate_exit)
		smp_send_reschedule(vcpu->cpu);

5773 5774
	kvm_guest_enter();

5775 5776 5777 5778 5779 5780 5781
	if (unlikely(vcpu->arch.switch_db_regs)) {
		set_debugreg(0, 7);
		set_debugreg(vcpu->arch.eff_db[0], 0);
		set_debugreg(vcpu->arch.eff_db[1], 1);
		set_debugreg(vcpu->arch.eff_db[2], 2);
		set_debugreg(vcpu->arch.eff_db[3], 3);
	}
5782

5783
	trace_kvm_entry(vcpu->vcpu_id);
A
Avi Kivity 已提交
5784
	kvm_x86_ops->run(vcpu);
5785

5786 5787 5788 5789 5790 5791 5792
	/*
	 * If the guest has used debug registers, at least dr7
	 * will be disabled while returning to the host.
	 * If we don't have active breakpoints in the host, we don't
	 * care about the messed up debug address registers. But if
	 * we have some of them active, restore the old state.
	 */
5793
	if (hw_breakpoint_active())
5794
		hw_breakpoint_restore();
5795

5796 5797
	vcpu->arch.last_guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu,
							   native_read_tsc());
5798

5799
	vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
5800
	smp_wmb();
5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816
	local_irq_enable();

	++vcpu->stat.exits;

	/*
	 * We must have an instruction between local_irq_enable() and
	 * kvm_guest_exit(), so the timer interrupt isn't delayed by
	 * the interrupt shadow.  The stat.exits increment will do nicely.
	 * But we need to prevent reordering, hence this barrier():
	 */
	barrier();

	kvm_guest_exit();

	preempt_enable();

5817
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
5818

5819 5820 5821 5822
	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
5823 5824
		unsigned long rip = kvm_rip_read(vcpu);
		profile_hit(KVM_PROFILING, (void *)rip);
5825 5826
	}

5827 5828
	if (unlikely(vcpu->arch.tsc_always_catchup))
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5829

5830 5831
	if (vcpu->arch.apic_attention)
		kvm_lapic_sync_from_vapic(vcpu);
A
Avi Kivity 已提交
5832

A
Avi Kivity 已提交
5833
	r = kvm_x86_ops->handle_exit(vcpu);
5834 5835 5836 5837
	return r;

cancel_injection:
	kvm_x86_ops->cancel_injection(vcpu);
5838 5839
	if (unlikely(vcpu->arch.apic_attention))
		kvm_lapic_sync_from_vapic(vcpu);
5840 5841 5842
out:
	return r;
}
5843

5844

A
Avi Kivity 已提交
5845
static int __vcpu_run(struct kvm_vcpu *vcpu)
5846 5847
{
	int r;
5848
	struct kvm *kvm = vcpu->kvm;
5849 5850

	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) {
5851 5852
		pr_debug("vcpu %d received sipi with vector # %x\n",
			 vcpu->vcpu_id, vcpu->arch.sipi_vector);
5853
		kvm_lapic_reset(vcpu);
5854
		r = kvm_vcpu_reset(vcpu);
5855 5856 5857
		if (r)
			return r;
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
5858 5859
	}

5860
	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
5861 5862 5863 5864 5865
	r = vapic_enter(vcpu);
	if (r) {
		srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
		return r;
	}
5866 5867 5868

	r = 1;
	while (r > 0) {
5869 5870
		if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		    !vcpu->arch.apf.halted)
A
Avi Kivity 已提交
5871
			r = vcpu_enter_guest(vcpu);
5872
		else {
5873
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
5874
			kvm_vcpu_block(vcpu);
5875
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
5876
			if (kvm_check_request(KVM_REQ_UNHALT, vcpu))
5877 5878 5879
			{
				switch(vcpu->arch.mp_state) {
				case KVM_MP_STATE_HALTED:
5880
					vcpu->arch.mp_state =
5881 5882
						KVM_MP_STATE_RUNNABLE;
				case KVM_MP_STATE_RUNNABLE:
5883
					vcpu->arch.apf.halted = false;
5884 5885 5886 5887 5888 5889 5890
					break;
				case KVM_MP_STATE_SIPI_RECEIVED:
				default:
					r = -EINTR;
					break;
				}
			}
5891 5892
		}

5893 5894 5895 5896 5897 5898 5899
		if (r <= 0)
			break;

		clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
		if (kvm_cpu_has_pending_timer(vcpu))
			kvm_inject_pending_timer_irqs(vcpu);

A
Avi Kivity 已提交
5900
		if (dm_request_for_irq_injection(vcpu)) {
5901
			r = -EINTR;
A
Avi Kivity 已提交
5902
			vcpu->run->exit_reason = KVM_EXIT_INTR;
5903 5904
			++vcpu->stat.request_irq_exits;
		}
5905 5906 5907

		kvm_check_async_pf_completion(vcpu);

5908 5909
		if (signal_pending(current)) {
			r = -EINTR;
A
Avi Kivity 已提交
5910
			vcpu->run->exit_reason = KVM_EXIT_INTR;
5911 5912 5913
			++vcpu->stat.signal_exits;
		}
		if (need_resched()) {
5914
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
5915
			kvm_resched(vcpu);
5916
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
5917
		}
5918 5919
	}

5920
	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
5921

A
Avi Kivity 已提交
5922 5923
	vapic_exit(vcpu);

5924 5925 5926
	return r;
}

5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944
static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
{
	int r;
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
	r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
	if (r != EMULATE_DONE)
		return 0;
	return 1;
}

static int complete_emulated_pio(struct kvm_vcpu *vcpu)
{
	BUG_ON(!vcpu->arch.pio.count);

	return complete_emulated_io(vcpu);
}

A
Avi Kivity 已提交
5945 5946 5947 5948 5949
/*
 * Implements the following, as a state machine:
 *
 * read:
 *   for each fragment
5950 5951 5952 5953
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       exit
 *       copy data
A
Avi Kivity 已提交
5954 5955 5956 5957
 *   execute insn
 *
 * write:
 *   for each fragment
5958 5959 5960 5961
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       copy data
 *       exit
A
Avi Kivity 已提交
5962
 */
5963
static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
5964 5965
{
	struct kvm_run *run = vcpu->run;
A
Avi Kivity 已提交
5966
	struct kvm_mmio_fragment *frag;
5967
	unsigned len;
5968

5969
	BUG_ON(!vcpu->mmio_needed);
5970

5971
	/* Complete previous fragment */
5972 5973
	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
	len = min(8u, frag->len);
5974
	if (!vcpu->mmio_is_write)
5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987
		memcpy(frag->data, run->mmio.data, len);

	if (frag->len <= 8) {
		/* Switch to the next fragment. */
		frag++;
		vcpu->mmio_cur_fragment++;
	} else {
		/* Go forward to the next mmio piece. */
		frag->data += len;
		frag->gpa += len;
		frag->len -= len;
	}

5988 5989
	if (vcpu->mmio_cur_fragment == vcpu->mmio_nr_fragments) {
		vcpu->mmio_needed = 0;
A
Avi Kivity 已提交
5990
		if (vcpu->mmio_is_write)
5991 5992 5993 5994
			return 1;
		vcpu->mmio_read_completed = 1;
		return complete_emulated_io(vcpu);
	}
5995

5996 5997 5998
	run->exit_reason = KVM_EXIT_MMIO;
	run->mmio.phys_addr = frag->gpa;
	if (vcpu->mmio_is_write)
5999 6000
		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
	run->mmio.len = min(8u, frag->len);
6001 6002 6003
	run->mmio.is_write = vcpu->mmio_is_write;
	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
	return 0;
6004 6005
}

6006

6007 6008 6009 6010 6011
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
{
	int r;
	sigset_t sigsaved;

6012 6013 6014
	if (!tsk_used_math(current) && init_fpu(current))
		return -ENOMEM;

6015 6016 6017
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);

6018
	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
6019
		kvm_vcpu_block(vcpu);
6020
		clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
6021 6022
		r = -EAGAIN;
		goto out;
6023 6024 6025
	}

	/* re-sync apic's tpr */
A
Andre Przywara 已提交
6026 6027 6028 6029 6030 6031
	if (!irqchip_in_kernel(vcpu->kvm)) {
		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
			r = -EINVAL;
			goto out;
		}
	}
6032

6033 6034 6035 6036 6037 6038 6039 6040
	if (unlikely(vcpu->arch.complete_userspace_io)) {
		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
		vcpu->arch.complete_userspace_io = NULL;
		r = cui(vcpu);
		if (r <= 0)
			goto out;
	} else
		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
6041

A
Avi Kivity 已提交
6042
	r = __vcpu_run(vcpu);
6043 6044

out:
6045
	post_kvm_run_save(vcpu);
6046 6047 6048 6049 6050 6051 6052 6053
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

	return r;
}

int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
6054 6055 6056 6057
	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
		/*
		 * We are here if userspace calls get_regs() in the middle of
		 * instruction emulation. Registers state needs to be copied
G
Guo Chao 已提交
6058
		 * back from emulation context to vcpu. Userspace shouldn't do
6059 6060 6061
		 * that usually, but some bad designed PV devices (vmware
		 * backdoor interface) need this to work
		 */
6062
		emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
6063 6064
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
	}
6065 6066 6067 6068 6069 6070 6071 6072
	regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
	regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
	regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
	regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
	regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
	regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
	regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
	regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
6073
#ifdef CONFIG_X86_64
6074 6075 6076 6077 6078 6079 6080 6081
	regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
	regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
	regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
	regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
	regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
	regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
	regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
	regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
6082 6083
#endif

6084
	regs->rip = kvm_rip_read(vcpu);
6085
	regs->rflags = kvm_get_rflags(vcpu);
6086 6087 6088 6089 6090 6091

	return 0;
}

int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
6092 6093 6094
	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;

6095 6096 6097 6098 6099 6100 6101 6102
	kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
	kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
	kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
	kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
	kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
	kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
	kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
	kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
6103
#ifdef CONFIG_X86_64
6104 6105 6106 6107 6108 6109 6110 6111
	kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
	kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
	kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
	kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
	kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
	kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
	kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
	kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
6112 6113
#endif

6114
	kvm_rip_write(vcpu, regs->rip);
6115
	kvm_set_rflags(vcpu, regs->rflags);
6116

6117 6118
	vcpu->arch.exception.pending = false;

6119 6120
	kvm_make_request(KVM_REQ_EVENT, vcpu);

6121 6122 6123 6124 6125 6126 6127
	return 0;
}

void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

6128
	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
6129 6130 6131 6132 6133 6134 6135 6136
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
6137
	struct desc_ptr dt;
6138

6139 6140 6141 6142 6143 6144
	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
6145

6146 6147
	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
6148 6149

	kvm_x86_ops->get_idt(vcpu, &dt);
6150 6151
	sregs->idt.limit = dt.size;
	sregs->idt.base = dt.address;
6152
	kvm_x86_ops->get_gdt(vcpu, &dt);
6153 6154
	sregs->gdt.limit = dt.size;
	sregs->gdt.base = dt.address;
6155

6156
	sregs->cr0 = kvm_read_cr0(vcpu);
6157
	sregs->cr2 = vcpu->arch.cr2;
6158
	sregs->cr3 = kvm_read_cr3(vcpu);
6159
	sregs->cr4 = kvm_read_cr4(vcpu);
6160
	sregs->cr8 = kvm_get_cr8(vcpu);
6161
	sregs->efer = vcpu->arch.efer;
6162 6163
	sregs->apic_base = kvm_get_apic_base(vcpu);

G
Gleb Natapov 已提交
6164
	memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
6165

6166
	if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
6167 6168
		set_bit(vcpu->arch.interrupt.nr,
			(unsigned long *)sregs->interrupt_bitmap);
6169

6170 6171 6172
	return 0;
}

6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
	mp_state->mp_state = vcpu->arch.mp_state;
	return 0;
}

int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
	vcpu->arch.mp_state = mp_state->mp_state;
6184
	kvm_make_request(KVM_REQ_EVENT, vcpu);
6185 6186 6187
	return 0;
}

6188 6189
int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
		    int reason, bool has_error_code, u32 error_code)
6190
{
6191
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6192
	int ret;
6193

6194
	init_emulate_ctxt(vcpu);
6195

6196
	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
6197
				   has_error_code, error_code);
6198 6199

	if (ret)
6200
		return EMULATE_FAIL;
6201

6202 6203
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
6204
	kvm_make_request(KVM_REQ_EVENT, vcpu);
6205
	return EMULATE_DONE;
6206 6207 6208
}
EXPORT_SYMBOL_GPL(kvm_task_switch);

6209 6210 6211 6212
int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	int mmu_reset_needed = 0;
6213
	int pending_vec, max_bits, idx;
6214
	struct desc_ptr dt;
6215

6216 6217 6218
	if (!guest_cpuid_has_xsave(vcpu) && (sregs->cr4 & X86_CR4_OSXSAVE))
		return -EINVAL;

6219 6220
	dt.size = sregs->idt.limit;
	dt.address = sregs->idt.base;
6221
	kvm_x86_ops->set_idt(vcpu, &dt);
6222 6223
	dt.size = sregs->gdt.limit;
	dt.address = sregs->gdt.base;
6224 6225
	kvm_x86_ops->set_gdt(vcpu, &dt);

6226
	vcpu->arch.cr2 = sregs->cr2;
6227
	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
6228
	vcpu->arch.cr3 = sregs->cr3;
6229
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
6230

6231
	kvm_set_cr8(vcpu, sregs->cr8);
6232

6233
	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
6234 6235 6236
	kvm_x86_ops->set_efer(vcpu, sregs->efer);
	kvm_set_apic_base(vcpu, sregs->apic_base);

6237
	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
6238
	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
6239
	vcpu->arch.cr0 = sregs->cr0;
6240

6241
	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
6242
	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
S
Sheng Yang 已提交
6243
	if (sregs->cr4 & X86_CR4_OSXSAVE)
A
Avi Kivity 已提交
6244
		kvm_update_cpuid(vcpu);
6245 6246

	idx = srcu_read_lock(&vcpu->kvm->srcu);
6247
	if (!is_long_mode(vcpu) && is_pae(vcpu)) {
6248
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
6249 6250
		mmu_reset_needed = 1;
	}
6251
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
6252 6253 6254 6255

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

6256
	max_bits = KVM_NR_INTERRUPTS;
G
Gleb Natapov 已提交
6257 6258 6259
	pending_vec = find_first_bit(
		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
	if (pending_vec < max_bits) {
6260
		kvm_queue_interrupt(vcpu, pending_vec, false);
G
Gleb Natapov 已提交
6261
		pr_debug("Set back pending irq %d\n", pending_vec);
6262 6263
	}

6264 6265 6266 6267 6268 6269
	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
6270

6271 6272
	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
6273

6274 6275
	update_cr8_intercept(vcpu);

M
Marcelo Tosatti 已提交
6276
	/* Older userspace won't unhalt the vcpu on reset. */
6277
	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
M
Marcelo Tosatti 已提交
6278
	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
6279
	    !is_protmode(vcpu))
M
Marcelo Tosatti 已提交
6280 6281
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;

6282 6283
	kvm_make_request(KVM_REQ_EVENT, vcpu);

6284 6285 6286
	return 0;
}

J
Jan Kiszka 已提交
6287 6288
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg)
6289
{
6290
	unsigned long rflags;
6291
	int i, r;
6292

6293 6294 6295
	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
		r = -EBUSY;
		if (vcpu->arch.exception.pending)
6296
			goto out;
6297 6298 6299 6300 6301 6302
		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
			kvm_queue_exception(vcpu, DB_VECTOR);
		else
			kvm_queue_exception(vcpu, BP_VECTOR);
	}

6303 6304 6305 6306 6307
	/*
	 * Read rflags as long as potentially injected trace flags are still
	 * filtered out.
	 */
	rflags = kvm_get_rflags(vcpu);
6308 6309 6310 6311 6312 6313

	vcpu->guest_debug = dbg->control;
	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
		vcpu->guest_debug = 0;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
6314 6315
		for (i = 0; i < KVM_NR_DB_REGS; ++i)
			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
6316
		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
6317 6318 6319 6320
	} else {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
	}
6321
	kvm_update_dr7(vcpu);
6322

J
Jan Kiszka 已提交
6323 6324 6325
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
			get_segment_base(vcpu, VCPU_SREG_CS);
6326

6327 6328 6329 6330 6331
	/*
	 * Trigger an rflags update that will inject or remove the trace
	 * flags.
	 */
	kvm_set_rflags(vcpu, rflags);
6332

6333
	kvm_x86_ops->update_db_bp_intercept(vcpu);
6334

6335
	r = 0;
J
Jan Kiszka 已提交
6336

6337
out:
6338 6339 6340 6341

	return r;
}

6342 6343 6344 6345 6346 6347 6348 6349
/*
 * Translate a guest virtual address to a guest physical address.
 */
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;
6350
	int idx;
6351

6352
	idx = srcu_read_lock(&vcpu->kvm->srcu);
6353
	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
6354
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
6355 6356 6357 6358 6359 6360 6361 6362
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;

	return 0;
}

6363 6364
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
S
Sheng Yang 已提交
6365 6366
	struct i387_fxsave_struct *fxsave =
			&vcpu->arch.guest_fpu.state->fxsave;
6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381

	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);

	return 0;
}

int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
S
Sheng Yang 已提交
6382 6383
	struct i387_fxsave_struct *fxsave =
			&vcpu->arch.guest_fpu.state->fxsave;
6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);

	return 0;
}

6397
int fx_init(struct kvm_vcpu *vcpu)
6398
{
6399 6400 6401 6402 6403 6404
	int err;

	err = fpu_alloc(&vcpu->arch.guest_fpu);
	if (err)
		return err;

S
Sheng Yang 已提交
6405
	fpu_finit(&vcpu->arch.guest_fpu);
6406

6407 6408 6409 6410 6411
	/*
	 * Ensure guest xcr0 is valid for loading
	 */
	vcpu->arch.xcr0 = XSTATE_FP;

6412
	vcpu->arch.cr0 |= X86_CR0_ET;
6413 6414

	return 0;
6415 6416 6417
}
EXPORT_SYMBOL_GPL(fx_init);

S
Sheng Yang 已提交
6418 6419 6420 6421 6422
static void fx_free(struct kvm_vcpu *vcpu)
{
	fpu_free(&vcpu->arch.guest_fpu);
}

6423 6424
void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
6425
	if (vcpu->guest_fpu_loaded)
6426 6427
		return;

6428 6429 6430 6431 6432 6433
	/*
	 * Restore all possible states in the guest,
	 * and assume host would use all available bits.
	 * Guest xcr0 would be loaded later.
	 */
	kvm_put_guest_xcr0(vcpu);
6434
	vcpu->guest_fpu_loaded = 1;
6435
	__kernel_fpu_begin();
S
Sheng Yang 已提交
6436
	fpu_restore_checking(&vcpu->arch.guest_fpu);
6437
	trace_kvm_fpu(1);
6438 6439 6440 6441
}

void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
6442 6443
	kvm_put_guest_xcr0(vcpu);

6444 6445 6446 6447
	if (!vcpu->guest_fpu_loaded)
		return;

	vcpu->guest_fpu_loaded = 0;
S
Sheng Yang 已提交
6448
	fpu_save_init(&vcpu->arch.guest_fpu);
6449
	__kernel_fpu_end();
A
Avi Kivity 已提交
6450
	++vcpu->stat.fpu_reload;
6451
	kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu);
6452
	trace_kvm_fpu(0);
6453
}
6454 6455 6456

void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
{
6457
	kvmclock_reset(vcpu);
6458

6459
	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
S
Sheng Yang 已提交
6460
	fx_free(vcpu);
6461 6462 6463 6464 6465 6466
	kvm_x86_ops->vcpu_free(vcpu);
}

struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
						unsigned int id)
{
Z
Zachary Amsden 已提交
6467 6468 6469 6470
	if (check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
		printk_once(KERN_WARNING
		"kvm: SMP vm created on host with unstable TSC; "
		"guest TSC will not be reliable\n");
6471 6472
	return kvm_x86_ops->vcpu_create(kvm, id);
}
6473

6474 6475 6476
int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
{
	int r;
6477

S
Sheng Yang 已提交
6478
	vcpu->arch.mtrr_state.have_fixed = 1;
6479 6480 6481
	r = vcpu_load(vcpu);
	if (r)
		return r;
6482
	r = kvm_vcpu_reset(vcpu);
6483 6484 6485 6486
	if (r == 0)
		r = kvm_mmu_setup(vcpu);
	vcpu_put(vcpu);

6487
	return r;
6488 6489
}

6490 6491 6492
int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
{
	int r;
6493
	struct msr_data msr;
6494 6495 6496 6497

	r = vcpu_load(vcpu);
	if (r)
		return r;
6498 6499 6500 6501
	msr.data = 0x0;
	msr.index = MSR_IA32_TSC;
	msr.host_initiated = true;
	kvm_write_tsc(vcpu, &msr);
6502 6503 6504 6505 6506
	vcpu_put(vcpu);

	return r;
}

6507
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
6508
{
6509
	int r;
6510 6511
	vcpu->arch.apf.msr_val = 0;

6512 6513
	r = vcpu_load(vcpu);
	BUG_ON(r);
6514 6515 6516
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);

S
Sheng Yang 已提交
6517
	fx_free(vcpu);
6518 6519 6520
	kvm_x86_ops->vcpu_free(vcpu);
}

6521
static int kvm_vcpu_reset(struct kvm_vcpu *vcpu)
6522
{
A
Avi Kivity 已提交
6523 6524
	atomic_set(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = 0;
6525 6526
	vcpu->arch.nmi_injected = false;

6527 6528 6529
	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
	vcpu->arch.dr6 = DR6_FIXED_1;
	vcpu->arch.dr7 = DR7_FIXED_1;
6530
	kvm_update_dr7(vcpu);
6531

6532
	kvm_make_request(KVM_REQ_EVENT, vcpu);
6533
	vcpu->arch.apf.msr_val = 0;
G
Glauber Costa 已提交
6534
	vcpu->arch.st.msr_val = 0;
6535

6536 6537
	kvmclock_reset(vcpu);

6538 6539 6540
	kvm_clear_async_pf_completion_queue(vcpu);
	kvm_async_pf_hash_reset(vcpu);
	vcpu->arch.apf.halted = false;
6541

6542 6543
	kvm_pmu_reset(vcpu);

6544 6545 6546 6547
	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;

6548 6549 6550
	return kvm_x86_ops->vcpu_reset(vcpu);
}

6551
int kvm_arch_hardware_enable(void *garbage)
6552
{
6553 6554 6555
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;
6556 6557 6558 6559
	int ret;
	u64 local_tsc;
	u64 max_tsc = 0;
	bool stable, backwards_tsc = false;
A
Avi Kivity 已提交
6560 6561

	kvm_shared_msr_cpu_online();
6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612
	ret = kvm_x86_ops->hardware_enable(garbage);
	if (ret != 0)
		return ret;

	local_tsc = native_read_tsc();
	stable = !check_tsc_unstable();
	list_for_each_entry(kvm, &vm_list, vm_list) {
		kvm_for_each_vcpu(i, vcpu, kvm) {
			if (!stable && vcpu->cpu == smp_processor_id())
				set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
				backwards_tsc = true;
				if (vcpu->arch.last_host_tsc > max_tsc)
					max_tsc = vcpu->arch.last_host_tsc;
			}
		}
	}

	/*
	 * Sometimes, even reliable TSCs go backwards.  This happens on
	 * platforms that reset TSC during suspend or hibernate actions, but
	 * maintain synchronization.  We must compensate.  Fortunately, we can
	 * detect that condition here, which happens early in CPU bringup,
	 * before any KVM threads can be running.  Unfortunately, we can't
	 * bring the TSCs fully up to date with real time, as we aren't yet far
	 * enough into CPU bringup that we know how much real time has actually
	 * elapsed; our helper function, get_kernel_ns() will be using boot
	 * variables that haven't been updated yet.
	 *
	 * So we simply find the maximum observed TSC above, then record the
	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
	 * the adjustment will be applied.  Note that we accumulate
	 * adjustments, in case multiple suspend cycles happen before some VCPU
	 * gets a chance to run again.  In the event that no KVM threads get a
	 * chance to run, we will miss the entire elapsed period, as we'll have
	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
	 * loose cycle time.  This isn't too big a deal, since the loss will be
	 * uniform across all VCPUs (not to mention the scenario is extremely
	 * unlikely). It is possible that a second hibernate recovery happens
	 * much faster than a first, causing the observed TSC here to be
	 * smaller; this would require additional padding adjustment, which is
	 * why we set last_host_tsc to the local tsc observed here.
	 *
	 * N.B. - this code below runs only on platforms with reliable TSC,
	 * as that is the only way backwards_tsc is set above.  Also note
	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
	 * have the same delta_cyc adjustment applied if backwards_tsc
	 * is detected.  Note further, this adjustment is only done once,
	 * as we reset last_host_tsc on all VCPUs to stop this from being
	 * called multiple times (one for each physical CPU bringup).
	 *
G
Guo Chao 已提交
6613
	 * Platforms with unreliable TSCs don't have to deal with this, they
6614 6615 6616 6617 6618 6619 6620 6621 6622 6623
	 * will be compensated by the logic in vcpu_load, which sets the TSC to
	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
	 * guarantee that they stay in perfect synchronization.
	 */
	if (backwards_tsc) {
		u64 delta_cyc = max_tsc - local_tsc;
		list_for_each_entry(kvm, &vm_list, vm_list) {
			kvm_for_each_vcpu(i, vcpu, kvm) {
				vcpu->arch.tsc_offset_adjustment += delta_cyc;
				vcpu->arch.last_host_tsc = local_tsc;
6624 6625
				set_bit(KVM_REQ_MASTERCLOCK_UPDATE,
					&vcpu->requests);
6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639
			}

			/*
			 * We have to disable TSC offset matching.. if you were
			 * booting a VM while issuing an S4 host suspend....
			 * you may have some problem.  Solving this issue is
			 * left as an exercise to the reader.
			 */
			kvm->arch.last_tsc_nsec = 0;
			kvm->arch.last_tsc_write = 0;
		}

	}
	return 0;
6640 6641 6642 6643 6644
}

void kvm_arch_hardware_disable(void *garbage)
{
	kvm_x86_ops->hardware_disable(garbage);
6645
	drop_user_return_notifiers(garbage);
6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662
}

int kvm_arch_hardware_setup(void)
{
	return kvm_x86_ops->hardware_setup();
}

void kvm_arch_hardware_unsetup(void)
{
	kvm_x86_ops->hardware_unsetup();
}

void kvm_arch_check_processor_compat(void *rtn)
{
	kvm_x86_ops->check_processor_compatibility(rtn);
}

6663 6664 6665 6666 6667
bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu)
{
	return irqchip_in_kernel(vcpu->kvm) == (vcpu->arch.apic != NULL);
}

6668 6669
struct static_key kvm_no_apic_vcpu __read_mostly;

6670 6671 6672 6673 6674 6675 6676 6677 6678
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
	struct page *page;
	struct kvm *kvm;
	int r;

	BUG_ON(vcpu->kvm == NULL);
	kvm = vcpu->kvm;

6679
	vcpu->arch.emulate_ctxt.ops = &emulate_ops;
6680
	if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
6681
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
6682
	else
6683
		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
6684 6685 6686 6687 6688 6689

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
6690
	vcpu->arch.pio_data = page_address(page);
6691

6692
	kvm_set_tsc_khz(vcpu, max_tsc_khz);
Z
Zachary Amsden 已提交
6693

6694 6695 6696 6697 6698 6699 6700 6701
	r = kvm_mmu_create(vcpu);
	if (r < 0)
		goto fail_free_pio_data;

	if (irqchip_in_kernel(kvm)) {
		r = kvm_create_lapic(vcpu);
		if (r < 0)
			goto fail_mmu_destroy;
6702 6703
	} else
		static_key_slow_inc(&kvm_no_apic_vcpu);
6704

H
Huang Ying 已提交
6705 6706 6707 6708
	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
				       GFP_KERNEL);
	if (!vcpu->arch.mce_banks) {
		r = -ENOMEM;
6709
		goto fail_free_lapic;
H
Huang Ying 已提交
6710 6711 6712
	}
	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;

6713 6714 6715
	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL))
		goto fail_free_mce_banks;

6716 6717 6718 6719
	r = fx_init(vcpu);
	if (r)
		goto fail_free_wbinvd_dirty_mask;

W
Will Auld 已提交
6720
	vcpu->arch.ia32_tsc_adjust_msr = 0x0;
6721
	kvm_async_pf_hash_reset(vcpu);
6722
	kvm_pmu_init(vcpu);
6723

6724
	return 0;
6725 6726
fail_free_wbinvd_dirty_mask:
	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
6727 6728
fail_free_mce_banks:
	kfree(vcpu->arch.mce_banks);
6729 6730
fail_free_lapic:
	kvm_free_lapic(vcpu);
6731 6732 6733
fail_mmu_destroy:
	kvm_mmu_destroy(vcpu);
fail_free_pio_data:
6734
	free_page((unsigned long)vcpu->arch.pio_data);
6735 6736 6737 6738 6739 6740
fail:
	return r;
}

void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
{
6741 6742
	int idx;

6743
	kvm_pmu_destroy(vcpu);
6744
	kfree(vcpu->arch.mce_banks);
6745
	kvm_free_lapic(vcpu);
6746
	idx = srcu_read_lock(&vcpu->kvm->srcu);
6747
	kvm_mmu_destroy(vcpu);
6748
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
6749
	free_page((unsigned long)vcpu->arch.pio_data);
6750 6751
	if (!irqchip_in_kernel(vcpu->kvm))
		static_key_slow_dec(&kvm_no_apic_vcpu);
6752
}
6753

6754
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
6755
{
6756 6757 6758
	if (type)
		return -EINVAL;

6759
	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
B
Ben-Ami Yassour 已提交
6760
	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
6761

6762 6763
	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
6764 6765 6766
	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
		&kvm->arch.irq_sources_bitmap);
6767

6768
	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
6769
	mutex_init(&kvm->arch.apic_map_lock);
6770 6771 6772
	spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);

	pvclock_update_vm_gtod_copy(kvm);
6773

6774
	return 0;
6775 6776 6777 6778
}

static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
6779 6780 6781
	int r;
	r = vcpu_load(vcpu);
	BUG_ON(r);
6782 6783 6784 6785 6786 6787 6788
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;
6789
	struct kvm_vcpu *vcpu;
6790 6791 6792 6793

	/*
	 * Unpin any mmu pages first.
	 */
6794 6795
	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_clear_async_pf_completion_queue(vcpu);
6796
		kvm_unload_vcpu_mmu(vcpu);
6797
	}
6798 6799 6800 6801 6802 6803
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_arch_vcpu_free(vcpu);

	mutex_lock(&kvm->lock);
	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
		kvm->vcpus[i] = NULL;
6804

6805 6806
	atomic_set(&kvm->online_vcpus, 0);
	mutex_unlock(&kvm->lock);
6807 6808
}

6809 6810
void kvm_arch_sync_events(struct kvm *kvm)
{
6811
	kvm_free_all_assigned_devices(kvm);
6812
	kvm_free_pit(kvm);
6813 6814
}

6815 6816
void kvm_arch_destroy_vm(struct kvm *kvm)
{
6817
	kvm_iommu_unmap_guest(kvm);
6818 6819
	kfree(kvm->arch.vpic);
	kfree(kvm->arch.vioapic);
6820
	kvm_free_vcpus(kvm);
6821 6822
	if (kvm->arch.apic_access_page)
		put_page(kvm->arch.apic_access_page);
6823 6824
	if (kvm->arch.ept_identity_pagetable)
		put_page(kvm->arch.ept_identity_pagetable);
6825
	kfree(rcu_dereference_check(kvm->arch.apic_map, 1));
6826
}
6827

6828 6829 6830 6831 6832
void kvm_arch_free_memslot(struct kvm_memory_slot *free,
			   struct kvm_memory_slot *dont)
{
	int i;

6833 6834 6835 6836
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
		if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
			kvm_kvfree(free->arch.rmap[i]);
			free->arch.rmap[i] = NULL;
6837
		}
6838 6839 6840 6841 6842 6843 6844
		if (i == 0)
			continue;

		if (!dont || free->arch.lpage_info[i - 1] !=
			     dont->arch.lpage_info[i - 1]) {
			kvm_kvfree(free->arch.lpage_info[i - 1]);
			free->arch.lpage_info[i - 1] = NULL;
6845 6846 6847 6848 6849 6850 6851 6852
		}
	}
}

int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
{
	int i;

6853
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
6854 6855
		unsigned long ugfn;
		int lpages;
6856
		int level = i + 1;
6857 6858 6859 6860

		lpages = gfn_to_index(slot->base_gfn + npages - 1,
				      slot->base_gfn, level) + 1;

6861 6862 6863
		slot->arch.rmap[i] =
			kvm_kvzalloc(lpages * sizeof(*slot->arch.rmap[i]));
		if (!slot->arch.rmap[i])
6864
			goto out_free;
6865 6866
		if (i == 0)
			continue;
6867

6868 6869 6870
		slot->arch.lpage_info[i - 1] = kvm_kvzalloc(lpages *
					sizeof(*slot->arch.lpage_info[i - 1]));
		if (!slot->arch.lpage_info[i - 1])
6871 6872 6873
			goto out_free;

		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
6874
			slot->arch.lpage_info[i - 1][0].write_count = 1;
6875
		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
6876
			slot->arch.lpage_info[i - 1][lpages - 1].write_count = 1;
6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887
		ugfn = slot->userspace_addr >> PAGE_SHIFT;
		/*
		 * If the gfn and userspace address are not aligned wrt each
		 * other, or if explicitly asked to, disable large page
		 * support for this slot
		 */
		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
		    !kvm_largepages_enabled()) {
			unsigned long j;

			for (j = 0; j < lpages; ++j)
6888
				slot->arch.lpage_info[i - 1][j].write_count = 1;
6889 6890 6891 6892 6893 6894
		}
	}

	return 0;

out_free:
6895 6896 6897 6898 6899 6900 6901 6902
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
		kvm_kvfree(slot->arch.rmap[i]);
		slot->arch.rmap[i] = NULL;
		if (i == 0)
			continue;

		kvm_kvfree(slot->arch.lpage_info[i - 1]);
		slot->arch.lpage_info[i - 1] = NULL;
6903 6904 6905 6906
	}
	return -ENOMEM;
}

6907 6908
int kvm_arch_prepare_memory_region(struct kvm *kvm,
				struct kvm_memory_slot *memslot,
6909
				struct kvm_memory_slot old,
6910
				struct kvm_userspace_memory_region *mem)
6911
{
6912
	int npages = memslot->npages;
6913

6914 6915 6916
	/*
	 * Only private memory slots need to be mapped here since
	 * KVM_SET_MEMORY_REGION ioctl is no longer supported.
6917
	 */
6918 6919
	if ((memslot->id >= KVM_USER_MEM_SLOTS) && npages && !old.npages) {
		unsigned long userspace_addr;
6920

6921 6922 6923 6924 6925 6926 6927
		/*
		 * MAP_SHARED to prevent internal slot pages from being moved
		 * by fork()/COW.
		 */
		userspace_addr = vm_mmap(NULL, 0, npages * PAGE_SIZE,
					 PROT_READ | PROT_WRITE,
					 MAP_SHARED | MAP_ANONYMOUS, 0);
6928

6929 6930
		if (IS_ERR((void *)userspace_addr))
			return PTR_ERR((void *)userspace_addr);
6931

6932
		memslot->userspace_addr = userspace_addr;
6933 6934
	}

6935 6936 6937 6938 6939
	return 0;
}

void kvm_arch_commit_memory_region(struct kvm *kvm,
				struct kvm_userspace_memory_region *mem,
6940
				struct kvm_memory_slot old)
6941 6942
{

6943
	int nr_mmu_pages = 0, npages = mem->memory_size >> PAGE_SHIFT;
6944

6945
	if ((mem->slot >= KVM_USER_MEM_SLOTS) && old.npages && !npages) {
6946 6947
		int ret;

A
Al Viro 已提交
6948
		ret = vm_munmap(old.userspace_addr,
6949 6950 6951 6952 6953 6954 6955
				old.npages * PAGE_SIZE);
		if (ret < 0)
			printk(KERN_WARNING
			       "kvm_vm_ioctl_set_memory_region: "
			       "failed to munmap memory\n");
	}

6956 6957 6958 6959
	if (!kvm->arch.n_requested_mmu_pages)
		nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);

	if (nr_mmu_pages)
6960
		kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
6961 6962 6963 6964 6965
	/*
	 * Write protect all pages for dirty logging.
	 * Existing largepage mappings are destroyed here and new ones will
	 * not be created until the end of the logging.
	 */
6966
	if (npages && (mem->flags & KVM_MEM_LOG_DIRTY_PAGES))
6967
		kvm_mmu_slot_remove_write_access(kvm, mem->slot);
6968 6969 6970 6971 6972 6973 6974 6975
	/*
	 * If memory slot is created, or moved, we need to clear all
	 * mmio sptes.
	 */
	if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT) {
		kvm_mmu_zap_all(kvm);
		kvm_reload_remote_mmus(kvm);
	}
6976
}
6977

6978
void kvm_arch_flush_shadow_all(struct kvm *kvm)
6979 6980
{
	kvm_mmu_zap_all(kvm);
6981
	kvm_reload_remote_mmus(kvm);
6982 6983
}

6984 6985 6986 6987 6988 6989
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
	kvm_arch_flush_shadow_all(kvm);
}

6990 6991
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
6992 6993 6994
	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		!vcpu->arch.apf.halted)
		|| !list_empty_careful(&vcpu->async_pf.done)
6995
		|| vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
A
Avi Kivity 已提交
6996
		|| atomic_read(&vcpu->arch.nmi_queued) ||
6997 6998
		(kvm_arch_interrupt_allowed(vcpu) &&
		 kvm_cpu_has_interrupt(vcpu));
6999
}
7000

7001
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
7002
{
7003
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
7004
}
7005 7006 7007 7008 7009

int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	return kvm_x86_ops->interrupt_allowed(vcpu);
}
7010

J
Jan Kiszka 已提交
7011 7012 7013 7014 7015 7016 7017 7018 7019
bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
{
	unsigned long current_rip = kvm_rip_read(vcpu) +
		get_segment_base(vcpu, VCPU_SREG_CS);

	return current_rip == linear_rip;
}
EXPORT_SYMBOL_GPL(kvm_is_linear_rip);

7020 7021 7022 7023 7024 7025
unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
{
	unsigned long rflags;

	rflags = kvm_x86_ops->get_rflags(vcpu);
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
7026
		rflags &= ~X86_EFLAGS_TF;
7027 7028 7029 7030 7031 7032 7033
	return rflags;
}
EXPORT_SYMBOL_GPL(kvm_get_rflags);

void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
J
Jan Kiszka 已提交
7034
	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
7035
		rflags |= X86_EFLAGS_TF;
7036
	kvm_x86_ops->set_rflags(vcpu, rflags);
7037
	kvm_make_request(KVM_REQ_EVENT, vcpu);
7038 7039 7040
}
EXPORT_SYMBOL_GPL(kvm_set_rflags);

G
Gleb Natapov 已提交
7041 7042 7043 7044
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
{
	int r;

X
Xiao Guangrong 已提交
7045
	if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) ||
7046
	      is_error_page(work->page))
G
Gleb Natapov 已提交
7047 7048 7049 7050 7051 7052
		return;

	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		return;

X
Xiao Guangrong 已提交
7053 7054 7055 7056
	if (!vcpu->arch.mmu.direct_map &&
	      work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu))
		return;

G
Gleb Natapov 已提交
7057 7058 7059
	vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true);
}

7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085
static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
{
	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
}

static inline u32 kvm_async_pf_next_probe(u32 key)
{
	return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
}

static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 key = kvm_async_pf_hash_fn(gfn);

	while (vcpu->arch.apf.gfns[key] != ~0)
		key = kvm_async_pf_next_probe(key);

	vcpu->arch.apf.gfns[key] = gfn;
}

static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	int i;
	u32 key = kvm_async_pf_hash_fn(gfn);

	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
7086 7087
		     (vcpu->arch.apf.gfns[key] != gfn &&
		      vcpu->arch.apf.gfns[key] != ~0); i++)
7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120
		key = kvm_async_pf_next_probe(key);

	return key;
}

bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
}

static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 i, j, k;

	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
	while (true) {
		vcpu->arch.apf.gfns[i] = ~0;
		do {
			j = kvm_async_pf_next_probe(j);
			if (vcpu->arch.apf.gfns[j] == ~0)
				return;
			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
			/*
			 * k lies cyclically in ]i,j]
			 * |    i.k.j |
			 * |....j i.k.| or  |.k..j i...|
			 */
		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
		i = j;
	}
}

7121 7122 7123 7124 7125 7126 7127
static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
{

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
				      sizeof(val));
}

7128 7129 7130
void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
				     struct kvm_async_pf *work)
{
7131 7132
	struct x86_exception fault;

7133
	trace_kvm_async_pf_not_present(work->arch.token, work->gva);
7134
	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
7135 7136

	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
7137 7138
	    (vcpu->arch.apf.send_user_only &&
	     kvm_x86_ops->get_cpl(vcpu) == 0))
7139 7140
		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
	else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
7141 7142 7143 7144 7145 7146
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
		kvm_inject_page_fault(vcpu, &fault);
7147
	}
7148 7149 7150 7151 7152
}

void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
				 struct kvm_async_pf *work)
{
7153 7154
	struct x86_exception fault;

7155 7156 7157 7158 7159 7160 7161 7162
	trace_kvm_async_pf_ready(work->arch.token, work->gva);
	if (is_error_page(work->page))
		work->arch.token = ~0; /* broadcast wakeup */
	else
		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);

	if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) &&
	    !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
7163 7164 7165 7166 7167 7168
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
		kvm_inject_page_fault(vcpu, &fault);
7169
	}
7170
	vcpu->arch.apf.halted = false;
7171
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
7172 7173 7174 7175 7176 7177 7178 7179 7180
}

bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
		return true;
	else
		return !kvm_event_needs_reinjection(vcpu) &&
			kvm_x86_ops->interrupt_allowed(vcpu);
7181 7182
}

7183 7184 7185 7186 7187
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
7188
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
7189
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
7190
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
7191
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
7192
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
7193
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
7194
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);