x86.c 195.0 KB
Newer Older
1 2 3 4 5 6
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
B
Ben-Ami Yassour 已提交
7 8
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
N
Nicolas Kaiser 已提交
9
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
10 11 12 13
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
B
Ben-Ami Yassour 已提交
14 15
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
16 17 18 19 20 21
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

22
#include <linux/kvm_host.h>
23
#include "irq.h"
24
#include "mmu.h"
S
Sheng Yang 已提交
25
#include "i8254.h"
26
#include "tss.h"
27
#include "kvm_cache_regs.h"
28
#include "x86.h"
A
Avi Kivity 已提交
29
#include "cpuid.h"
30

31
#include <linux/clocksource.h>
B
Ben-Ami Yassour 已提交
32
#include <linux/interrupt.h>
33 34 35
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
36
#include <linux/module.h>
37
#include <linux/mman.h>
38
#include <linux/highmem.h>
J
Joerg Roedel 已提交
39
#include <linux/iommu.h>
B
Ben-Ami Yassour 已提交
40
#include <linux/intel-iommu.h>
41
#include <linux/cpufreq.h>
A
Avi Kivity 已提交
42
#include <linux/user-return-notifier.h>
43
#include <linux/srcu.h>
44
#include <linux/slab.h>
45
#include <linux/perf_event.h>
46
#include <linux/uaccess.h>
47
#include <linux/hash.h>
48
#include <linux/pci.h>
49 50
#include <linux/timekeeper_internal.h>
#include <linux/pvclock_gtod.h>
A
Avi Kivity 已提交
51
#include <trace/events/kvm.h>
X
Xiao Guangrong 已提交
52

53 54
#define CREATE_TRACE_POINTS
#include "trace.h"
55

56
#include <asm/debugreg.h>
57
#include <asm/msr.h>
58
#include <asm/desc.h>
S
Sheng Yang 已提交
59
#include <asm/mtrr.h>
H
Huang Ying 已提交
60
#include <asm/mce.h>
61
#include <asm/i387.h>
62
#include <asm/fpu-internal.h> /* Ugh! */
S
Sheng Yang 已提交
63
#include <asm/xcr.h>
64
#include <asm/pvclock.h>
65
#include <asm/div64.h>
66

67
#define MAX_IO_MSRS 256
H
Huang Ying 已提交
68
#define KVM_MAX_MCE_BANKS 32
69
#define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P)
H
Huang Ying 已提交
70

71 72 73
#define emul_to_vcpu(ctxt) \
	container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)

74 75 76 77 78
/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
79 80
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
81
#else
82
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
83
#endif
84

85 86
#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
87

88
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
A
Avi Kivity 已提交
89
static void process_nmi(struct kvm_vcpu *vcpu);
90
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
91

92
struct kvm_x86_ops *kvm_x86_ops;
93
EXPORT_SYMBOL_GPL(kvm_x86_ops);
94

95 96
static bool ignore_msrs = 0;
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
97

98 99 100
unsigned int min_timer_period_us = 500;
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

101 102 103 104 105
bool kvm_has_tsc_control;
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
u32  kvm_max_guest_tsc_khz;
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);

106 107 108 109
/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
static u32 tsc_tolerance_ppm = 250;
module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);

110 111
static bool backwards_tsc_observed = false;

A
Avi Kivity 已提交
112 113 114 115
#define KVM_NR_SHARED_MSRS 16

struct kvm_shared_msrs_global {
	int nr;
116
	u32 msrs[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
117 118 119 120 121
};

struct kvm_shared_msrs {
	struct user_return_notifier urn;
	bool registered;
122 123 124 125
	struct kvm_shared_msr_values {
		u64 host;
		u64 curr;
	} values[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
126 127 128
};

static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
129
static struct kvm_shared_msrs __percpu *shared_msrs;
A
Avi Kivity 已提交
130

131
struct kvm_stats_debugfs_item debugfs_entries[] = {
132 133 134 135 136 137 138 139 140
	{ "pf_fixed", VCPU_STAT(pf_fixed) },
	{ "pf_guest", VCPU_STAT(pf_guest) },
	{ "tlb_flush", VCPU_STAT(tlb_flush) },
	{ "invlpg", VCPU_STAT(invlpg) },
	{ "exits", VCPU_STAT(exits) },
	{ "io_exits", VCPU_STAT(io_exits) },
	{ "mmio_exits", VCPU_STAT(mmio_exits) },
	{ "signal_exits", VCPU_STAT(signal_exits) },
	{ "irq_window", VCPU_STAT(irq_window_exits) },
141
	{ "nmi_window", VCPU_STAT(nmi_window_exits) },
142 143
	{ "halt_exits", VCPU_STAT(halt_exits) },
	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
A
Amit Shah 已提交
144
	{ "hypercalls", VCPU_STAT(hypercalls) },
145 146 147 148 149 150 151
	{ "request_irq", VCPU_STAT(request_irq_exits) },
	{ "irq_exits", VCPU_STAT(irq_exits) },
	{ "host_state_reload", VCPU_STAT(host_state_reload) },
	{ "efer_reload", VCPU_STAT(efer_reload) },
	{ "fpu_reload", VCPU_STAT(fpu_reload) },
	{ "insn_emulation", VCPU_STAT(insn_emulation) },
	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
152
	{ "irq_injections", VCPU_STAT(irq_injections) },
153
	{ "nmi_injections", VCPU_STAT(nmi_injections) },
A
Avi Kivity 已提交
154 155 156 157 158 159
	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
	{ "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
	{ "mmu_flooded", VM_STAT(mmu_flooded) },
	{ "mmu_recycled", VM_STAT(mmu_recycled) },
A
Avi Kivity 已提交
160
	{ "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
161
	{ "mmu_unsync", VM_STAT(mmu_unsync) },
162
	{ "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
M
Marcelo Tosatti 已提交
163
	{ "largepages", VM_STAT(lpages) },
164 165 166
	{ NULL }
};

167 168
u64 __read_mostly host_xcr0;

169
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
170

171 172 173 174 175 176 177
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
	int i;
	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
		vcpu->arch.apf.gfns[i] = ~0;
}

A
Avi Kivity 已提交
178 179 180 181 182
static void kvm_on_user_return(struct user_return_notifier *urn)
{
	unsigned slot;
	struct kvm_shared_msrs *locals
		= container_of(urn, struct kvm_shared_msrs, urn);
183
	struct kvm_shared_msr_values *values;
A
Avi Kivity 已提交
184 185

	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
186 187 188 189
		values = &locals->values[slot];
		if (values->host != values->curr) {
			wrmsrl(shared_msrs_global.msrs[slot], values->host);
			values->curr = values->host;
A
Avi Kivity 已提交
190 191 192 193 194 195
		}
	}
	locals->registered = false;
	user_return_notifier_unregister(urn);
}

196
static void shared_msr_update(unsigned slot, u32 msr)
A
Avi Kivity 已提交
197 198
{
	u64 value;
199 200
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
A
Avi Kivity 已提交
201

202 203 204 205 206 207 208 209 210 211 212 213 214
	/* only read, and nobody should modify it at this time,
	 * so don't need lock */
	if (slot >= shared_msrs_global.nr) {
		printk(KERN_ERR "kvm: invalid MSR slot!");
		return;
	}
	rdmsrl_safe(msr, &value);
	smsr->values[slot].host = value;
	smsr->values[slot].curr = value;
}

void kvm_define_shared_msr(unsigned slot, u32 msr)
{
215
	BUG_ON(slot >= KVM_NR_SHARED_MSRS);
A
Avi Kivity 已提交
216 217
	if (slot >= shared_msrs_global.nr)
		shared_msrs_global.nr = slot + 1;
218 219 220
	shared_msrs_global.msrs[slot] = msr;
	/* we need ensured the shared_msr_global have been updated */
	smp_wmb();
A
Avi Kivity 已提交
221 222 223 224 225 226 227 228
}
EXPORT_SYMBOL_GPL(kvm_define_shared_msr);

static void kvm_shared_msr_cpu_online(void)
{
	unsigned i;

	for (i = 0; i < shared_msrs_global.nr; ++i)
229
		shared_msr_update(i, shared_msrs_global.msrs[i]);
A
Avi Kivity 已提交
230 231
}

232
int kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
A
Avi Kivity 已提交
233
{
234 235
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
236
	int err;
A
Avi Kivity 已提交
237

238
	if (((value ^ smsr->values[slot].curr) & mask) == 0)
239
		return 0;
240
	smsr->values[slot].curr = value;
241 242 243 244
	err = wrmsrl_safe(shared_msrs_global.msrs[slot], value);
	if (err)
		return 1;

A
Avi Kivity 已提交
245 246 247 248 249
	if (!smsr->registered) {
		smsr->urn.on_user_return = kvm_on_user_return;
		user_return_notifier_register(&smsr->urn);
		smsr->registered = true;
	}
250
	return 0;
A
Avi Kivity 已提交
251 252 253
}
EXPORT_SYMBOL_GPL(kvm_set_shared_msr);

254
static void drop_user_return_notifiers(void)
255
{
256 257
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
258 259 260 261 262

	if (smsr->registered)
		kvm_on_user_return(&smsr->urn);
}

263 264
u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
265
	return vcpu->arch.apic_base;
266 267 268
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
	u64 old_state = vcpu->arch.apic_base &
		(MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
	u64 new_state = msr_info->data &
		(MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) |
		0x2ff | (guest_cpuid_has_x2apic(vcpu) ? 0 : X2APIC_ENABLE);

	if (!msr_info->host_initiated &&
	    ((msr_info->data & reserved_bits) != 0 ||
	     new_state == X2APIC_ENABLE ||
	     (new_state == MSR_IA32_APICBASE_ENABLE &&
	      old_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) ||
	     (new_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE) &&
	      old_state == 0)))
		return 1;

	kvm_lapic_set_base(vcpu, msr_info->data);
	return 0;
289 290 291
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

292
asmlinkage __visible void kvm_spurious_fault(void)
293 294 295 296 297 298
{
	/* Fault while not rebooting.  We want the trace. */
	BUG();
}
EXPORT_SYMBOL_GPL(kvm_spurious_fault);

299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
#define EXCPT_BENIGN		0
#define EXCPT_CONTRIBUTORY	1
#define EXCPT_PF		2

static int exception_class(int vector)
{
	switch (vector) {
	case PF_VECTOR:
		return EXCPT_PF;
	case DE_VECTOR:
	case TS_VECTOR:
	case NP_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
		return EXCPT_CONTRIBUTORY;
	default:
		break;
	}
	return EXCPT_BENIGN;
}

320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
#define EXCPT_FAULT		0
#define EXCPT_TRAP		1
#define EXCPT_ABORT		2
#define EXCPT_INTERRUPT		3

static int exception_type(int vector)
{
	unsigned int mask;

	if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
		return EXCPT_INTERRUPT;

	mask = 1 << vector;

	/* #DB is trap, as instruction watchpoints are handled elsewhere */
	if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR)))
		return EXCPT_TRAP;

	if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
		return EXCPT_ABORT;

	/* Reserved exceptions will result in fault */
	return EXCPT_FAULT;
}

345
static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
346 347
		unsigned nr, bool has_error, u32 error_code,
		bool reinject)
348 349 350 351
{
	u32 prev_nr;
	int class1, class2;

352 353
	kvm_make_request(KVM_REQ_EVENT, vcpu);

354 355
	if (!vcpu->arch.exception.pending) {
	queue:
356 357
		if (has_error && !is_protmode(vcpu))
			has_error = false;
358 359 360 361
		vcpu->arch.exception.pending = true;
		vcpu->arch.exception.has_error_code = has_error;
		vcpu->arch.exception.nr = nr;
		vcpu->arch.exception.error_code = error_code;
362
		vcpu->arch.exception.reinject = reinject;
363 364 365 366 367 368 369
		return;
	}

	/* to check exception */
	prev_nr = vcpu->arch.exception.nr;
	if (prev_nr == DF_VECTOR) {
		/* triple fault -> shutdown */
370
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
		return;
	}
	class1 = exception_class(prev_nr);
	class2 = exception_class(nr);
	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
		/* generate double fault per SDM Table 5-5 */
		vcpu->arch.exception.pending = true;
		vcpu->arch.exception.has_error_code = true;
		vcpu->arch.exception.nr = DF_VECTOR;
		vcpu->arch.exception.error_code = 0;
	} else
		/* replace previous exception with a new one in a hope
		   that instruction re-execution will regenerate lost
		   exception */
		goto queue;
}

389 390
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
391
	kvm_multiple_exception(vcpu, nr, false, 0, false);
392 393 394
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

395 396 397 398 399 400
void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
	kvm_multiple_exception(vcpu, nr, false, 0, true);
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

401
void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
402
{
403 404 405 406 407 408
	if (err)
		kvm_inject_gp(vcpu, 0);
	else
		kvm_x86_ops->skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
409

410
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
411 412
{
	++vcpu->stat.pf_guest;
413 414
	vcpu->arch.cr2 = fault->address;
	kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
415
}
N
Nadav Har'El 已提交
416
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
417

418
static bool kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
419
{
420 421
	if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
		vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
422
	else
423
		vcpu->arch.mmu.inject_page_fault(vcpu, fault);
424 425

	return fault->nested_page_fault;
426 427
}

428 429
void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
430 431
	atomic_inc(&vcpu->arch.nmi_queued);
	kvm_make_request(KVM_REQ_NMI, vcpu);
432 433 434
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

435 436
void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
437
	kvm_multiple_exception(vcpu, nr, true, error_code, false);
438 439 440
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

441 442 443 444 445 446
void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
	kvm_multiple_exception(vcpu, nr, true, error_code, true);
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

447 448 449 450 451
/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
452
{
453 454 455 456
	if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
		return true;
	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
	return false;
457
}
458
EXPORT_SYMBOL_GPL(kvm_require_cpl);
459

460 461 462 463 464 465 466 467 468 469
bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
{
	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
		return true;

	kvm_queue_exception(vcpu, UD_VECTOR);
	return false;
}
EXPORT_SYMBOL_GPL(kvm_require_dr);

470 471 472 473 474 475 476 477 478
/*
 * This function will be used to read from the physical memory of the currently
 * running guest. The difference to kvm_read_guest_page is that this function
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gfn_t ngfn, void *data, int offset, int len,
			    u32 access)
{
479
	struct x86_exception exception;
480 481 482 483
	gfn_t real_gfn;
	gpa_t ngpa;

	ngpa     = gfn_to_gpa(ngfn);
484
	real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);
485 486 487 488 489 490 491 492 493
	if (real_gfn == UNMAPPED_GVA)
		return -EFAULT;

	real_gfn = gpa_to_gfn(real_gfn);

	return kvm_read_guest_page(vcpu->kvm, real_gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

494 495 496 497 498 499 500
int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
			       void *data, int offset, int len, u32 access)
{
	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
				       data, offset, len, access);
}

501 502 503
/*
 * Load the pae pdptrs.  Return true is they are all valid.
 */
504
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
505 506 507 508 509
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
	int i;
	int ret;
510
	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
511

512 513 514
	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
				      offset * sizeof(u64), sizeof(pdpte),
				      PFERR_USER_MASK|PFERR_WRITE_MASK);
515 516 517 518 519
	if (ret < 0) {
		ret = 0;
		goto out;
	}
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
520
		if (is_present_gpte(pdpte[i]) &&
521
		    (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
522 523 524 525 526 527
			ret = 0;
			goto out;
		}
	}
	ret = 1;

528
	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
A
Avi Kivity 已提交
529 530 531 532
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
533 534 535 536
out:

	return ret;
}
537
EXPORT_SYMBOL_GPL(load_pdptrs);
538

539 540
static bool pdptrs_changed(struct kvm_vcpu *vcpu)
{
541
	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
542
	bool changed = true;
543 544
	int offset;
	gfn_t gfn;
545 546 547 548 549
	int r;

	if (is_long_mode(vcpu) || !is_pae(vcpu))
		return false;

A
Avi Kivity 已提交
550 551 552 553
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_avail))
		return true;

554 555
	gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT;
	offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1);
556 557
	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
				       PFERR_USER_MASK | PFERR_WRITE_MASK);
558 559
	if (r < 0)
		goto out;
560
	changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
561 562 563 564 565
out:

	return changed;
}

566
int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
567
{
568 569 570 571
	unsigned long old_cr0 = kvm_read_cr0(vcpu);
	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP |
				    X86_CR0_CD | X86_CR0_NW;

572 573
	cr0 |= X86_CR0_ET;

574
#ifdef CONFIG_X86_64
575 576
	if (cr0 & 0xffffffff00000000UL)
		return 1;
577 578 579
#endif

	cr0 &= ~CR0_RESERVED_BITS;
580

581 582
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
		return 1;
583

584 585
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
		return 1;
586 587 588

	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
#ifdef CONFIG_X86_64
589
		if ((vcpu->arch.efer & EFER_LME)) {
590 591
			int cs_db, cs_l;

592 593
			if (!is_pae(vcpu))
				return 1;
594
			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
595 596
			if (cs_l)
				return 1;
597 598
		} else
#endif
599
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
600
						 kvm_read_cr3(vcpu)))
601
			return 1;
602 603
	}

604 605 606
	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
		return 1;

607 608
	kvm_x86_ops->set_cr0(vcpu, cr0);

609
	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
610
		kvm_clear_async_pf_completion_queue(vcpu);
611 612
		kvm_async_pf_hash_reset(vcpu);
	}
613

614 615
	if ((cr0 ^ old_cr0) & update_bits)
		kvm_mmu_reset_context(vcpu);
616 617
	return 0;
}
618
EXPORT_SYMBOL_GPL(kvm_set_cr0);
619

620
void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
621
{
622
	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
623
}
624
EXPORT_SYMBOL_GPL(kvm_lmsw);
625

626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644
static void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
{
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
			!vcpu->guest_xcr0_loaded) {
		/* kvm_set_xcr() also depends on this */
		xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
		vcpu->guest_xcr0_loaded = 1;
	}
}

static void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
{
	if (vcpu->guest_xcr0_loaded) {
		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
		vcpu->guest_xcr0_loaded = 0;
	}
}

645 646
int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
647 648
	u64 xcr0 = xcr;
	u64 old_xcr0 = vcpu->arch.xcr0;
649
	u64 valid_bits;
650 651 652 653 654 655 656 657

	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
	if (index != XCR_XFEATURE_ENABLED_MASK)
		return 1;
	if (!(xcr0 & XSTATE_FP))
		return 1;
	if ((xcr0 & XSTATE_YMM) && !(xcr0 & XSTATE_SSE))
		return 1;
658 659 660 661 662 663 664 665

	/*
	 * Do not allow the guest to set bits that we do not support
	 * saving.  However, xcr0 bit 0 is always set, even if the
	 * emulated CPU does not support XSAVE (see fx_init).
	 */
	valid_bits = vcpu->arch.guest_supported_xcr0 | XSTATE_FP;
	if (xcr0 & ~valid_bits)
666
		return 1;
667

668 669 670
	if ((!(xcr0 & XSTATE_BNDREGS)) != (!(xcr0 & XSTATE_BNDCSR)))
		return 1;

671 672 673 674 675 676
	if (xcr0 & XSTATE_AVX512) {
		if (!(xcr0 & XSTATE_YMM))
			return 1;
		if ((xcr0 & XSTATE_AVX512) != XSTATE_AVX512)
			return 1;
	}
677
	kvm_put_guest_xcr0(vcpu);
678
	vcpu->arch.xcr0 = xcr0;
679 680 681

	if ((xcr0 ^ old_xcr0) & XSTATE_EXTEND_MASK)
		kvm_update_cpuid(vcpu);
682 683 684 685 686
	return 0;
}

int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
687 688
	if (kvm_x86_ops->get_cpl(vcpu) != 0 ||
	    __kvm_set_xcr(vcpu, index, xcr)) {
689 690 691 692 693 694 695
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_xcr);

696
int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
697
{
698
	unsigned long old_cr4 = kvm_read_cr4(vcpu);
699 700
	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE |
				   X86_CR4_PAE | X86_CR4_SMEP;
701 702
	if (cr4 & CR4_RESERVED_BITS)
		return 1;
703

704 705 706
	if (!guest_cpuid_has_xsave(vcpu) && (cr4 & X86_CR4_OSXSAVE))
		return 1;

707 708 709
	if (!guest_cpuid_has_smep(vcpu) && (cr4 & X86_CR4_SMEP))
		return 1;

F
Feng Wu 已提交
710 711 712
	if (!guest_cpuid_has_smap(vcpu) && (cr4 & X86_CR4_SMAP))
		return 1;

713
	if (!guest_cpuid_has_fsgsbase(vcpu) && (cr4 & X86_CR4_FSGSBASE))
714 715
		return 1;

716
	if (is_long_mode(vcpu)) {
717 718
		if (!(cr4 & X86_CR4_PAE))
			return 1;
719 720
	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
		   && ((cr4 ^ old_cr4) & pdptr_bits)
721 722
		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
				   kvm_read_cr3(vcpu)))
723 724
		return 1;

725 726 727 728 729 730 731 732 733
	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
		if (!guest_cpuid_has_pcid(vcpu))
			return 1;

		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
			return 1;
	}

734
	if (kvm_x86_ops->set_cr4(vcpu, cr4))
735
		return 1;
736

737 738
	if (((cr4 ^ old_cr4) & pdptr_bits) ||
	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
739
		kvm_mmu_reset_context(vcpu);
740

F
Feng Wu 已提交
741 742 743
	if ((cr4 ^ old_cr4) & X86_CR4_SMAP)
		update_permission_bitmask(vcpu, vcpu->arch.walk_mmu, false);

744
	if ((cr4 ^ old_cr4) & X86_CR4_OSXSAVE)
A
Avi Kivity 已提交
745
		kvm_update_cpuid(vcpu);
746

747 748
	return 0;
}
749
EXPORT_SYMBOL_GPL(kvm_set_cr4);
750

751
int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
752
{
753
	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
754
		kvm_mmu_sync_roots(vcpu);
755
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
756
		return 0;
757 758
	}

759
	if (is_long_mode(vcpu)) {
760 761 762 763
		if (cr3 & CR3_L_MODE_RESERVED_BITS)
			return 1;
	} else if (is_pae(vcpu) && is_paging(vcpu) &&
		   !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
N
Nadav Amit 已提交
764
		return 1;
765

766
	vcpu->arch.cr3 = cr3;
767
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
768
	kvm_mmu_new_cr3(vcpu);
769 770
	return 0;
}
771
EXPORT_SYMBOL_GPL(kvm_set_cr3);
772

A
Andre Przywara 已提交
773
int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
774
{
775 776
	if (cr8 & CR8_RESERVED_BITS)
		return 1;
777 778 779
	if (irqchip_in_kernel(vcpu->kvm))
		kvm_lapic_set_tpr(vcpu, cr8);
	else
780
		vcpu->arch.cr8 = cr8;
781 782
	return 0;
}
783
EXPORT_SYMBOL_GPL(kvm_set_cr8);
784

785
unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
786 787 788 789
{
	if (irqchip_in_kernel(vcpu->kvm))
		return kvm_lapic_get_cr8(vcpu);
	else
790
		return vcpu->arch.cr8;
791
}
792
EXPORT_SYMBOL_GPL(kvm_get_cr8);
793

J
Jan Kiszka 已提交
794 795 796 797 798 799
static void kvm_update_dr6(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
		kvm_x86_ops->set_dr6(vcpu, vcpu->arch.dr6);
}

800 801 802 803 804 805 806 807 808
static void kvm_update_dr7(struct kvm_vcpu *vcpu)
{
	unsigned long dr7;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		dr7 = vcpu->arch.guest_debug_dr7;
	else
		dr7 = vcpu->arch.dr7;
	kvm_x86_ops->set_dr7(vcpu, dr7);
809 810 811
	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
	if (dr7 & DR7_BP_EN_MASK)
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
812 813
}

814 815 816 817 818 819 820 821 822
static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
{
	u64 fixed = DR6_FIXED_1;

	if (!guest_cpuid_has_rtm(vcpu))
		fixed |= DR6_RTM;
	return fixed;
}

823
static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
824 825 826 827 828 829 830 831 832 833
{
	switch (dr) {
	case 0 ... 3:
		vcpu->arch.db[dr] = val;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = val;
		break;
	case 4:
		/* fall through */
	case 6:
834 835
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
836
		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
J
Jan Kiszka 已提交
837
		kvm_update_dr6(vcpu);
838 839 840 841
		break;
	case 5:
		/* fall through */
	default: /* 7 */
842 843
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
844
		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
845
		kvm_update_dr7(vcpu);
846 847 848 849 850
		break;
	}

	return 0;
}
851 852 853

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
854
	if (__kvm_set_dr(vcpu, dr, val)) {
855
		kvm_inject_gp(vcpu, 0);
856 857 858
		return 1;
	}
	return 0;
859
}
860 861
EXPORT_SYMBOL_GPL(kvm_set_dr);

862
int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
863 864 865 866 867 868 869 870
{
	switch (dr) {
	case 0 ... 3:
		*val = vcpu->arch.db[dr];
		break;
	case 4:
		/* fall through */
	case 6:
J
Jan Kiszka 已提交
871 872 873 874
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
			*val = vcpu->arch.dr6;
		else
			*val = kvm_x86_ops->get_dr6(vcpu);
875 876 877 878 879 880 881
		break;
	case 5:
		/* fall through */
	default: /* 7 */
		*val = vcpu->arch.dr7;
		break;
	}
882 883
	return 0;
}
884 885
EXPORT_SYMBOL_GPL(kvm_get_dr);

A
Avi Kivity 已提交
886 887 888 889 890 891 892 893 894 895 896 897 898 899 900
bool kvm_rdpmc(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
	u64 data;
	int err;

	err = kvm_pmu_read_pmc(vcpu, ecx, &data);
	if (err)
		return err;
	kvm_register_write(vcpu, VCPU_REGS_RAX, (u32)data);
	kvm_register_write(vcpu, VCPU_REGS_RDX, data >> 32);
	return err;
}
EXPORT_SYMBOL_GPL(kvm_rdpmc);

901 902 903 904 905
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
 * This list is modified at module load time to reflect the
906 907
 * capabilities of the host cpu. This capabilities test skips MSRs that are
 * kvm-specific. Those are put in the beginning of the list.
908
 */
909

910
#define KVM_SAVE_MSRS_BEGIN	12
911
static u32 msrs_to_save[] = {
912
	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
913
	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
914
	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
915
	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
G
Glauber Costa 已提交
916
	HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
917
	MSR_KVM_PV_EOI_EN,
918
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
B
Brian Gerst 已提交
919
	MSR_STAR,
920 921 922
#ifdef CONFIG_X86_64
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
923
	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
924
	MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS
925 926 927 928
};

static unsigned num_msrs_to_save;

M
Mathias Krause 已提交
929
static const u32 emulated_msrs[] = {
W
Will Auld 已提交
930
	MSR_IA32_TSC_ADJUST,
931
	MSR_IA32_TSCDEADLINE,
932
	MSR_IA32_MISC_ENABLE,
933 934
	MSR_IA32_MCG_STATUS,
	MSR_IA32_MCG_CTL,
935 936
};

937
bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
938
{
939
	if (efer & efer_reserved_bits)
940
		return false;
941

A
Alexander Graf 已提交
942 943 944 945
	if (efer & EFER_FFXSR) {
		struct kvm_cpuid_entry2 *feat;

		feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
946
		if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT)))
947
			return false;
A
Alexander Graf 已提交
948 949
	}

950 951 952 953
	if (efer & EFER_SVME) {
		struct kvm_cpuid_entry2 *feat;

		feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
954
		if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM)))
955
			return false;
956 957
	}

958 959 960 961 962 963 964 965 966 967 968 969 970 971 972
	return true;
}
EXPORT_SYMBOL_GPL(kvm_valid_efer);

static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	u64 old_efer = vcpu->arch.efer;

	if (!kvm_valid_efer(vcpu, efer))
		return 1;

	if (is_paging(vcpu)
	    && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
		return 1;

973
	efer &= ~EFER_LMA;
974
	efer |= vcpu->arch.efer & EFER_LMA;
975

976 977
	kvm_x86_ops->set_efer(vcpu, efer);

978 979 980 981
	/* Update reserved bits */
	if ((efer ^ old_efer) & EFER_NX)
		kvm_mmu_reset_context(vcpu);

982
	return 0;
983 984
}

985 986 987 988 989 990
void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);

991 992 993 994 995
/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
996
int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
997
{
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
	switch (msr->index) {
	case MSR_FS_BASE:
	case MSR_GS_BASE:
	case MSR_KERNEL_GS_BASE:
	case MSR_CSTAR:
	case MSR_LSTAR:
		if (is_noncanonical_address(msr->data))
			return 1;
		break;
	case MSR_IA32_SYSENTER_EIP:
	case MSR_IA32_SYSENTER_ESP:
		/*
		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
		 * non-canonical address is written on Intel but not on
		 * AMD (which ignores the top 32-bits, because it does
		 * not implement 64-bit SYSENTER).
		 *
		 * 64-bit code should hence be able to write a non-canonical
		 * value on AMD.  Making the address canonical ensures that
		 * vmentry does not fail on Intel after writing a non-canonical
		 * value, and that something deterministic happens if the guest
		 * invokes 64-bit SYSENTER.
		 */
		msr->data = get_canonical(msr->data);
	}
1023
	return kvm_x86_ops->set_msr(vcpu, msr);
1024
}
1025
EXPORT_SYMBOL_GPL(kvm_set_msr);
1026

1027 1028 1029 1030 1031
/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
1032 1033 1034 1035 1036 1037
	struct msr_data msr;

	msr.data = *data;
	msr.index = index;
	msr.host_initiated = true;
	return kvm_set_msr(vcpu, &msr);
1038 1039
}

1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
#ifdef CONFIG_X86_64
struct pvclock_gtod_data {
	seqcount_t	seq;

	struct { /* extract of a clocksource struct */
		int vclock_mode;
		cycle_t	cycle_last;
		cycle_t	mask;
		u32	mult;
		u32	shift;
	} clock;

1052 1053
	u64		boot_ns;
	u64		nsec_base;
1054 1055 1056 1057 1058 1059 1060
};

static struct pvclock_gtod_data pvclock_gtod_data;

static void update_pvclock_gtod(struct timekeeper *tk)
{
	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
1061 1062
	u64 boot_ns;

1063
	boot_ns = ktime_to_ns(ktime_add(tk->tkr.base_mono, tk->offs_boot));
1064 1065 1066 1067

	write_seqcount_begin(&vdata->seq);

	/* copy pvclock gtod data */
1068 1069 1070 1071 1072
	vdata->clock.vclock_mode	= tk->tkr.clock->archdata.vclock_mode;
	vdata->clock.cycle_last		= tk->tkr.cycle_last;
	vdata->clock.mask		= tk->tkr.mask;
	vdata->clock.mult		= tk->tkr.mult;
	vdata->clock.shift		= tk->tkr.shift;
1073

1074
	vdata->boot_ns			= boot_ns;
1075
	vdata->nsec_base		= tk->tkr.xtime_nsec;
1076 1077 1078 1079 1080 1081

	write_seqcount_end(&vdata->seq);
}
#endif


1082 1083
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
{
1084 1085
	int version;
	int r;
1086
	struct pvclock_wall_clock wc;
1087
	struct timespec boot;
1088 1089 1090 1091

	if (!wall_clock)
		return;

1092 1093 1094 1095 1096 1097 1098 1099
	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
	if (r)
		return;

	if (version & 1)
		++version;  /* first time write, random junk */

	++version;
1100 1101 1102

	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));

1103 1104
	/*
	 * The guest calculates current wall clock time by adding
Z
Zachary Amsden 已提交
1105
	 * system time (updated by kvm_guest_time_update below) to the
1106 1107 1108
	 * wall clock specified here.  guest system time equals host
	 * system time for us, thus we must fill in host boot time here.
	 */
1109
	getboottime(&boot);
1110

1111 1112 1113 1114
	if (kvm->arch.kvmclock_offset) {
		struct timespec ts = ns_to_timespec(kvm->arch.kvmclock_offset);
		boot = timespec_sub(boot, ts);
	}
1115 1116 1117
	wc.sec = boot.tv_sec;
	wc.nsec = boot.tv_nsec;
	wc.version = version;
1118 1119 1120 1121 1122 1123 1124

	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));

	version++;
	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
}

1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
	uint32_t quotient, remainder;

	/* Don't try to replace with do_div(), this one calculates
	 * "(dividend << 32) / divisor" */
	__asm__ ( "divl %4"
		  : "=a" (quotient), "=d" (remainder)
		  : "0" (0), "1" (dividend), "r" (divisor) );
	return quotient;
}

1137 1138
static void kvm_get_time_scale(uint32_t scaled_khz, uint32_t base_khz,
			       s8 *pshift, u32 *pmultiplier)
1139
{
1140
	uint64_t scaled64;
1141 1142 1143 1144
	int32_t  shift = 0;
	uint64_t tps64;
	uint32_t tps32;

1145 1146
	tps64 = base_khz * 1000LL;
	scaled64 = scaled_khz * 1000LL;
1147
	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
1148 1149 1150 1151 1152
		tps64 >>= 1;
		shift--;
	}

	tps32 = (uint32_t)tps64;
1153 1154
	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
1155 1156 1157
			scaled64 >>= 1;
		else
			tps32 <<= 1;
1158 1159 1160
		shift++;
	}

1161 1162
	*pshift = shift;
	*pmultiplier = div_frac(scaled64, tps32);
1163

1164 1165
	pr_debug("%s: base_khz %u => %u, shift %d, mul %u\n",
		 __func__, base_khz, scaled_khz, shift, *pmultiplier);
1166 1167
}

1168 1169
static inline u64 get_kernel_ns(void)
{
1170
	return ktime_get_boot_ns();
1171 1172
}

1173
#ifdef CONFIG_X86_64
1174
static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
1175
#endif
1176

1177
static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
Z
Zachary Amsden 已提交
1178
unsigned long max_tsc_khz;
1179

1180
static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
1181
{
1182 1183
	return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
				   vcpu->arch.virtual_tsc_shift);
1184 1185
}

1186
static u32 adjust_tsc_khz(u32 khz, s32 ppm)
1187
{
1188 1189 1190
	u64 v = (u64)khz * (1000000 + ppm);
	do_div(v, 1000000);
	return v;
1191 1192
}

1193
static void kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 this_tsc_khz)
1194
{
1195 1196
	u32 thresh_lo, thresh_hi;
	int use_scaling = 0;
1197

1198 1199 1200 1201
	/* tsc_khz can be zero if TSC calibration fails */
	if (this_tsc_khz == 0)
		return;

Z
Zachary Amsden 已提交
1202 1203
	/* Compute a scale to convert nanoseconds in TSC cycles */
	kvm_get_time_scale(this_tsc_khz, NSEC_PER_SEC / 1000,
1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220
			   &vcpu->arch.virtual_tsc_shift,
			   &vcpu->arch.virtual_tsc_mult);
	vcpu->arch.virtual_tsc_khz = this_tsc_khz;

	/*
	 * Compute the variation in TSC rate which is acceptable
	 * within the range of tolerance and decide if the
	 * rate being applied is within that bounds of the hardware
	 * rate.  If so, no scaling or compensation need be done.
	 */
	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
	if (this_tsc_khz < thresh_lo || this_tsc_khz > thresh_hi) {
		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", this_tsc_khz, thresh_lo, thresh_hi);
		use_scaling = 1;
	}
	kvm_x86_ops->set_tsc_khz(vcpu, this_tsc_khz, use_scaling);
Z
Zachary Amsden 已提交
1221 1222 1223 1224
}

static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
1225
	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
1226 1227
				      vcpu->arch.virtual_tsc_mult,
				      vcpu->arch.virtual_tsc_shift);
1228
	tsc += vcpu->arch.this_tsc_write;
Z
Zachary Amsden 已提交
1229 1230 1231
	return tsc;
}

1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258
void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
	bool vcpus_matched;
	bool do_request = false;
	struct kvm_arch *ka = &vcpu->kvm->arch;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			 atomic_read(&vcpu->kvm->online_vcpus));

	if (vcpus_matched && gtod->clock.vclock_mode == VCLOCK_TSC)
		if (!ka->use_master_clock)
			do_request = 1;

	if (!vcpus_matched && ka->use_master_clock)
			do_request = 1;

	if (do_request)
		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);

	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
			    atomic_read(&vcpu->kvm->online_vcpus),
		            ka->use_master_clock, gtod->clock.vclock_mode);
#endif
}

W
Will Auld 已提交
1259 1260 1261 1262 1263 1264
static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
{
	u64 curr_offset = kvm_x86_ops->read_tsc_offset(vcpu);
	vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
}

1265
void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
1266 1267
{
	struct kvm *kvm = vcpu->kvm;
Z
Zachary Amsden 已提交
1268
	u64 offset, ns, elapsed;
1269
	unsigned long flags;
1270
	s64 usdiff;
1271
	bool matched;
T
Tomasz Grabiec 已提交
1272
	bool already_matched;
1273
	u64 data = msr->data;
1274

1275
	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
1276
	offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
1277
	ns = get_kernel_ns();
Z
Zachary Amsden 已提交
1278
	elapsed = ns - kvm->arch.last_tsc_nsec;
1279

1280
	if (vcpu->arch.virtual_tsc_khz) {
1281 1282
		int faulted = 0;

1283 1284
		/* n.b - signed multiplication and division required */
		usdiff = data - kvm->arch.last_tsc_write;
1285
#ifdef CONFIG_X86_64
1286
		usdiff = (usdiff * 1000) / vcpu->arch.virtual_tsc_khz;
1287
#else
1288
		/* do_div() only does unsigned */
1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
		asm("1: idivl %[divisor]\n"
		    "2: xor %%edx, %%edx\n"
		    "   movl $0, %[faulted]\n"
		    "3:\n"
		    ".section .fixup,\"ax\"\n"
		    "4: movl $1, %[faulted]\n"
		    "   jmp  3b\n"
		    ".previous\n"

		_ASM_EXTABLE(1b, 4b)

		: "=A"(usdiff), [faulted] "=r" (faulted)
		: "A"(usdiff * 1000), [divisor] "rm"(vcpu->arch.virtual_tsc_khz));

1303
#endif
1304 1305 1306 1307
		do_div(elapsed, 1000);
		usdiff -= elapsed;
		if (usdiff < 0)
			usdiff = -usdiff;
1308 1309 1310 1311

		/* idivl overflow => difference is larger than USEC_PER_SEC */
		if (faulted)
			usdiff = USEC_PER_SEC;
1312 1313
	} else
		usdiff = USEC_PER_SEC; /* disable TSC match window below */
Z
Zachary Amsden 已提交
1314 1315

	/*
1316 1317 1318 1319 1320 1321 1322 1323 1324
	 * Special case: TSC write with a small delta (1 second) of virtual
	 * cycle time against real time is interpreted as an attempt to
	 * synchronize the CPU.
         *
	 * For a reliable TSC, we can match TSC offsets, and for an unstable
	 * TSC, we add elapsed time in this computation.  We could let the
	 * compensation code attempt to catch up if we fall behind, but
	 * it's better to try to match offsets from the beginning.
         */
1325
	if (usdiff < USEC_PER_SEC &&
1326
	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
Z
Zachary Amsden 已提交
1327
		if (!check_tsc_unstable()) {
1328
			offset = kvm->arch.cur_tsc_offset;
Z
Zachary Amsden 已提交
1329 1330
			pr_debug("kvm: matched tsc offset for %llu\n", data);
		} else {
1331
			u64 delta = nsec_to_cycles(vcpu, elapsed);
1332 1333
			data += delta;
			offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
1334
			pr_debug("kvm: adjusted tsc offset by %llu\n", delta);
Z
Zachary Amsden 已提交
1335
		}
1336
		matched = true;
T
Tomasz Grabiec 已提交
1337
		already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
1338 1339 1340 1341 1342 1343
	} else {
		/*
		 * We split periods of matched TSC writes into generations.
		 * For each generation, we track the original measured
		 * nanosecond time, offset, and write, so if TSCs are in
		 * sync, we can match exact offset, and if not, we can match
G
Guo Chao 已提交
1344
		 * exact software computation in compute_guest_tsc()
1345 1346 1347 1348 1349 1350 1351
		 *
		 * These values are tracked in kvm->arch.cur_xxx variables.
		 */
		kvm->arch.cur_tsc_generation++;
		kvm->arch.cur_tsc_nsec = ns;
		kvm->arch.cur_tsc_write = data;
		kvm->arch.cur_tsc_offset = offset;
1352
		matched = false;
T
Tomasz Grabiec 已提交
1353
		pr_debug("kvm: new tsc generation %llu, clock %llu\n",
1354
			 kvm->arch.cur_tsc_generation, data);
Z
Zachary Amsden 已提交
1355
	}
1356 1357 1358 1359 1360

	/*
	 * We also track th most recent recorded KHZ, write and time to
	 * allow the matching interval to be extended at each write.
	 */
Z
Zachary Amsden 已提交
1361 1362
	kvm->arch.last_tsc_nsec = ns;
	kvm->arch.last_tsc_write = data;
1363
	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
1364

1365
	vcpu->arch.last_guest_tsc = data;
1366 1367 1368 1369 1370 1371

	/* Keep track of which generation this VCPU has synchronized to */
	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;

W
Will Auld 已提交
1372 1373
	if (guest_cpuid_has_tsc_adjust(vcpu) && !msr->host_initiated)
		update_ia32_tsc_adjust_msr(vcpu, offset);
1374 1375
	kvm_x86_ops->write_tsc_offset(vcpu, offset);
	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
1376 1377

	spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
T
Tomasz Grabiec 已提交
1378
	if (!matched) {
1379
		kvm->arch.nr_vcpus_matched_tsc = 0;
T
Tomasz Grabiec 已提交
1380 1381 1382
	} else if (!already_matched) {
		kvm->arch.nr_vcpus_matched_tsc++;
	}
1383 1384 1385

	kvm_track_tsc_matching(vcpu);
	spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
1386
}
1387

1388 1389
EXPORT_SYMBOL_GPL(kvm_write_tsc);

1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434
#ifdef CONFIG_X86_64

static cycle_t read_tsc(void)
{
	cycle_t ret;
	u64 last;

	/*
	 * Empirically, a fence (of type that depends on the CPU)
	 * before rdtsc is enough to ensure that rdtsc is ordered
	 * with respect to loads.  The various CPU manuals are unclear
	 * as to whether rdtsc can be reordered with later loads,
	 * but no one has ever seen it happen.
	 */
	rdtsc_barrier();
	ret = (cycle_t)vget_cycles();

	last = pvclock_gtod_data.clock.cycle_last;

	if (likely(ret >= last))
		return ret;

	/*
	 * GCC likes to generate cmov here, but this branch is extremely
	 * predictable (it's just a funciton of time and the likely is
	 * very likely) and there's a data dependence, so force GCC
	 * to generate a branch instead.  I don't barrier() because
	 * we don't actually need a barrier, and if this function
	 * ever gets inlined it will generate worse code.
	 */
	asm volatile ("");
	return last;
}

static inline u64 vgettsc(cycle_t *cycle_now)
{
	long v;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	*cycle_now = read_tsc();

	v = (*cycle_now - gtod->clock.cycle_last) & gtod->clock.mask;
	return v * gtod->clock.mult;
}

1435
static int do_monotonic_boot(s64 *t, cycle_t *cycle_now)
1436
{
1437
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
1438 1439
	unsigned long seq;
	int mode;
1440
	u64 ns;
1441 1442 1443 1444

	do {
		seq = read_seqcount_begin(&gtod->seq);
		mode = gtod->clock.vclock_mode;
1445
		ns = gtod->nsec_base;
1446 1447
		ns += vgettsc(cycle_now);
		ns >>= gtod->clock.shift;
1448
		ns += gtod->boot_ns;
1449
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
1450
	*t = ns;
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461

	return mode;
}

/* returns true if host is using tsc clocksource */
static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now)
{
	/* checked again under seqlock below */
	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
		return false;

1462
	return do_monotonic_boot(kernel_ns, cycle_now) == VCLOCK_TSC;
1463 1464 1465 1466 1467
}
#endif

/*
 *
1468 1469 1470
 * Assuming a stable TSC across physical CPUS, and a stable TSC
 * across virtual CPUs, the following condition is possible.
 * Each numbered line represents an event visible to both
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502
 * CPUs at the next numbered event.
 *
 * "timespecX" represents host monotonic time. "tscX" represents
 * RDTSC value.
 *
 * 		VCPU0 on CPU0		|	VCPU1 on CPU1
 *
 * 1.  read timespec0,tsc0
 * 2.					| timespec1 = timespec0 + N
 * 					| tsc1 = tsc0 + M
 * 3. transition to guest		| transition to guest
 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
 * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
 * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
 *
 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
 *
 * 	- ret0 < ret1
 *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
 *		...
 *	- 0 < N - M => M < N
 *
 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
 * always the case (the difference between two distinct xtime instances
 * might be smaller then the difference between corresponding TSC reads,
 * when updating guest vcpus pvclock areas).
 *
 * To avoid that problem, do not allow visibility of distinct
 * system_timestamp/tsc_timestamp values simultaneously: use a master
 * copy of host monotonic time values. Update that master copy
 * in lockstep.
 *
1503
 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
1504 1505 1506 1507 1508 1509 1510 1511
 *
 */

static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	struct kvm_arch *ka = &kvm->arch;
	int vclock_mode;
1512 1513 1514 1515
	bool host_tsc_clocksource, vcpus_matched;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			atomic_read(&kvm->online_vcpus));
1516 1517 1518 1519 1520

	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
1521
	host_tsc_clocksource = kvm_get_time_and_clockread(
1522 1523 1524
					&ka->master_kernel_ns,
					&ka->master_cycle_now);

1525 1526
	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
				&& !backwards_tsc_observed;
1527

1528 1529 1530 1531
	if (ka->use_master_clock)
		atomic_set(&kvm_guest_has_master_clock, 1);

	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
1532 1533
	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
					vcpus_matched);
1534 1535 1536
#endif
}

1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
static void kvm_gen_update_masterclock(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	int i;
	struct kvm_vcpu *vcpu;
	struct kvm_arch *ka = &kvm->arch;

	spin_lock(&ka->pvclock_gtod_sync_lock);
	kvm_make_mclock_inprogress_request(kvm);
	/* no guest entries from this point */
	pvclock_update_vm_gtod_copy(kvm);

	kvm_for_each_vcpu(i, vcpu, kvm)
1550
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
1551 1552 1553 1554 1555 1556 1557 1558 1559

	/* guest entries allowed */
	kvm_for_each_vcpu(i, vcpu, kvm)
		clear_bit(KVM_REQ_MCLOCK_INPROGRESS, &vcpu->requests);

	spin_unlock(&ka->pvclock_gtod_sync_lock);
#endif
}

Z
Zachary Amsden 已提交
1560
static int kvm_guest_time_update(struct kvm_vcpu *v)
1561
{
1562
	unsigned long flags, this_tsc_khz;
1563
	struct kvm_vcpu_arch *vcpu = &v->arch;
1564
	struct kvm_arch *ka = &v->kvm->arch;
1565
	s64 kernel_ns;
1566
	u64 tsc_timestamp, host_tsc;
1567
	struct pvclock_vcpu_time_info guest_hv_clock;
1568
	u8 pvclock_flags;
1569 1570 1571 1572
	bool use_master_clock;

	kernel_ns = 0;
	host_tsc = 0;
1573

1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584
	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
	spin_lock(&ka->pvclock_gtod_sync_lock);
	use_master_clock = ka->use_master_clock;
	if (use_master_clock) {
		host_tsc = ka->master_cycle_now;
		kernel_ns = ka->master_kernel_ns;
	}
	spin_unlock(&ka->pvclock_gtod_sync_lock);
1585 1586 1587

	/* Keep irq disabled to prevent changes to the clock */
	local_irq_save(flags);
1588
	this_tsc_khz = __this_cpu_read(cpu_tsc_khz);
1589 1590 1591 1592 1593
	if (unlikely(this_tsc_khz == 0)) {
		local_irq_restore(flags);
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
		return 1;
	}
1594 1595 1596 1597 1598 1599 1600
	if (!use_master_clock) {
		host_tsc = native_read_tsc();
		kernel_ns = get_kernel_ns();
	}

	tsc_timestamp = kvm_x86_ops->read_l1_tsc(v, host_tsc);

Z
Zachary Amsden 已提交
1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613
	/*
	 * We may have to catch up the TSC to match elapsed wall clock
	 * time for two reasons, even if kvmclock is used.
	 *   1) CPU could have been running below the maximum TSC rate
	 *   2) Broken TSC compensation resets the base at each VCPU
	 *      entry to avoid unknown leaps of TSC even when running
	 *      again on the same CPU.  This may cause apparent elapsed
	 *      time to disappear, and the guest to stand still or run
	 *	very slowly.
	 */
	if (vcpu->tsc_catchup) {
		u64 tsc = compute_guest_tsc(v, kernel_ns);
		if (tsc > tsc_timestamp) {
1614
			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
Z
Zachary Amsden 已提交
1615 1616
			tsc_timestamp = tsc;
		}
1617 1618
	}

1619 1620
	local_irq_restore(flags);

1621
	if (!vcpu->pv_time_enabled)
Z
Zachary Amsden 已提交
1622
		return 0;
1623

Z
Zachary Amsden 已提交
1624
	if (unlikely(vcpu->hw_tsc_khz != this_tsc_khz)) {
1625 1626 1627
		kvm_get_time_scale(NSEC_PER_SEC / 1000, this_tsc_khz,
				   &vcpu->hv_clock.tsc_shift,
				   &vcpu->hv_clock.tsc_to_system_mul);
Z
Zachary Amsden 已提交
1628
		vcpu->hw_tsc_khz = this_tsc_khz;
1629 1630 1631
	}

	/* With all the info we got, fill in the values */
1632
	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
1633
	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
Z
Zachary Amsden 已提交
1634
	vcpu->last_guest_tsc = tsc_timestamp;
1635

1636 1637 1638
	/*
	 * The interface expects us to write an even number signaling that the
	 * update is finished. Since the guest won't see the intermediate
1639
	 * state, we just increase by 2 at the end.
1640
	 */
1641
	vcpu->hv_clock.version += 2;
1642

1643 1644 1645
	if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
		&guest_hv_clock, sizeof(guest_hv_clock))))
		return 0;
1646 1647

	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
1648
	pvclock_flags = (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);
1649 1650 1651 1652 1653 1654

	if (vcpu->pvclock_set_guest_stopped_request) {
		pvclock_flags |= PVCLOCK_GUEST_STOPPED;
		vcpu->pvclock_set_guest_stopped_request = false;
	}

1655 1656 1657 1658
	/* If the host uses TSC clocksource, then it is stable */
	if (use_master_clock)
		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;

1659 1660
	vcpu->hv_clock.flags = pvclock_flags;

1661 1662 1663
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock));
1664
	return 0;
1665 1666
}

1667 1668 1669 1670 1671 1672 1673 1674
/*
 * kvmclock updates which are isolated to a given vcpu, such as
 * vcpu->cpu migration, should not allow system_timestamp from
 * the rest of the vcpus to remain static. Otherwise ntp frequency
 * correction applies to one vcpu's system_timestamp but not
 * the others.
 *
 * So in those cases, request a kvmclock update for all vcpus.
1675 1676 1677 1678
 * We need to rate-limit these requests though, as they can
 * considerably slow guests that have a large number of vcpus.
 * The time for a remote vcpu to update its kvmclock is bound
 * by the delay we use to rate-limit the updates.
1679 1680
 */

1681 1682 1683
#define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)

static void kvmclock_update_fn(struct work_struct *work)
1684 1685
{
	int i;
1686 1687 1688 1689
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_update_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);
1690 1691 1692
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm) {
1693
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
1694 1695 1696 1697
		kvm_vcpu_kick(vcpu);
	}
}

1698 1699 1700 1701
static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
{
	struct kvm *kvm = v->kvm;

1702
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
1703 1704 1705 1706
	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
					KVMCLOCK_UPDATE_DELAY);
}

1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720
#define KVMCLOCK_SYNC_PERIOD (300 * HZ)

static void kvmclock_sync_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_sync_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);

	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
}

A
Avi Kivity 已提交
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
static bool msr_mtrr_valid(unsigned msr)
{
	switch (msr) {
	case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
	case MSR_MTRRfix64K_00000:
	case MSR_MTRRfix16K_80000:
	case MSR_MTRRfix16K_A0000:
	case MSR_MTRRfix4K_C0000:
	case MSR_MTRRfix4K_C8000:
	case MSR_MTRRfix4K_D0000:
	case MSR_MTRRfix4K_D8000:
	case MSR_MTRRfix4K_E0000:
	case MSR_MTRRfix4K_E8000:
	case MSR_MTRRfix4K_F0000:
	case MSR_MTRRfix4K_F8000:
	case MSR_MTRRdefType:
	case MSR_IA32_CR_PAT:
		return true;
	case 0x2f8:
		return true;
	}
	return false;
}

1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
static bool valid_pat_type(unsigned t)
{
	return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
}

static bool valid_mtrr_type(unsigned t)
{
	return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
}

1755
bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1756 1757
{
	int i;
1758
	u64 mask;
1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779

	if (!msr_mtrr_valid(msr))
		return false;

	if (msr == MSR_IA32_CR_PAT) {
		for (i = 0; i < 8; i++)
			if (!valid_pat_type((data >> (i * 8)) & 0xff))
				return false;
		return true;
	} else if (msr == MSR_MTRRdefType) {
		if (data & ~0xcff)
			return false;
		return valid_mtrr_type(data & 0xff);
	} else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
		for (i = 0; i < 8 ; i++)
			if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
				return false;
		return true;
	}

	/* variable MTRRs */
1780 1781
	WARN_ON(!(msr >= 0x200 && msr < 0x200 + 2 * KVM_NR_VAR_MTRR));

1782
	mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
1783
	if ((msr & 1) == 0) {
1784
		/* MTRR base */
1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795
		if (!valid_mtrr_type(data & 0xff))
			return false;
		mask |= 0xf00;
	} else
		/* MTRR mask */
		mask |= 0x7ff;
	if (data & mask) {
		kvm_inject_gp(vcpu, 0);
		return false;
	}

1796
	return true;
1797
}
1798
EXPORT_SYMBOL_GPL(kvm_mtrr_valid);
1799

A
Avi Kivity 已提交
1800 1801
static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
S
Sheng Yang 已提交
1802 1803
	u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;

1804
	if (!kvm_mtrr_valid(vcpu, msr, data))
A
Avi Kivity 已提交
1805 1806
		return 1;

S
Sheng Yang 已提交
1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833
	if (msr == MSR_MTRRdefType) {
		vcpu->arch.mtrr_state.def_type = data;
		vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
	} else if (msr == MSR_MTRRfix64K_00000)
		p[0] = data;
	else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
		p[1 + msr - MSR_MTRRfix16K_80000] = data;
	else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
		p[3 + msr - MSR_MTRRfix4K_C0000] = data;
	else if (msr == MSR_IA32_CR_PAT)
		vcpu->arch.pat = data;
	else {	/* Variable MTRRs */
		int idx, is_mtrr_mask;
		u64 *pt;

		idx = (msr - 0x200) / 2;
		is_mtrr_mask = msr - 0x200 - 2 * idx;
		if (!is_mtrr_mask)
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
		else
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
		*pt = data;
	}

	kvm_mmu_reset_context(vcpu);
A
Avi Kivity 已提交
1834 1835
	return 0;
}
1836

H
Huang Ying 已提交
1837
static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1838
{
H
Huang Ying 已提交
1839 1840 1841
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;

1842 1843
	switch (msr) {
	case MSR_IA32_MCG_STATUS:
H
Huang Ying 已提交
1844
		vcpu->arch.mcg_status = data;
1845
		break;
1846
	case MSR_IA32_MCG_CTL:
H
Huang Ying 已提交
1847 1848 1849 1850 1851 1852 1853 1854
		if (!(mcg_cap & MCG_CTL_P))
			return 1;
		if (data != 0 && data != ~(u64)0)
			return -1;
		vcpu->arch.mcg_ctl = data;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
1855
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
H
Huang Ying 已提交
1856
			u32 offset = msr - MSR_IA32_MC0_CTL;
1857 1858 1859 1860 1861
			/* only 0 or all 1s can be written to IA32_MCi_CTL
			 * some Linux kernels though clear bit 10 in bank 4 to
			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
			 * this to avoid an uncatched #GP in the guest
			 */
H
Huang Ying 已提交
1862
			if ((offset & 0x3) == 0 &&
1863
			    data != 0 && (data | (1 << 10)) != ~(u64)0)
H
Huang Ying 已提交
1864 1865 1866 1867 1868 1869 1870 1871 1872
				return -1;
			vcpu->arch.mce_banks[offset] = data;
			break;
		}
		return 1;
	}
	return 0;
}

E
Ed Swierk 已提交
1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889
static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
	int lm = is_long_mode(vcpu);
	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
		: kvm->arch.xen_hvm_config.blob_size_32;
	u32 page_num = data & ~PAGE_MASK;
	u64 page_addr = data & PAGE_MASK;
	u8 *page;
	int r;

	r = -E2BIG;
	if (page_num >= blob_size)
		goto out;
	r = -ENOMEM;
1890 1891 1892
	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
	if (IS_ERR(page)) {
		r = PTR_ERR(page);
E
Ed Swierk 已提交
1893
		goto out;
1894
	}
E
Ed Swierk 已提交
1895 1896 1897 1898 1899 1900 1901 1902 1903
	if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
		goto out_free;
	r = 0;
out_free:
	kfree(page);
out:
	return r;
}

1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914
static bool kvm_hv_hypercall_enabled(struct kvm *kvm)
{
	return kvm->arch.hv_hypercall & HV_X64_MSR_HYPERCALL_ENABLE;
}

static bool kvm_hv_msr_partition_wide(u32 msr)
{
	bool r = false;
	switch (msr) {
	case HV_X64_MSR_GUEST_OS_ID:
	case HV_X64_MSR_HYPERCALL:
1915 1916
	case HV_X64_MSR_REFERENCE_TSC:
	case HV_X64_MSR_TIME_REF_COUNT:
1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952
		r = true;
		break;
	}

	return r;
}

static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	struct kvm *kvm = vcpu->kvm;

	switch (msr) {
	case HV_X64_MSR_GUEST_OS_ID:
		kvm->arch.hv_guest_os_id = data;
		/* setting guest os id to zero disables hypercall page */
		if (!kvm->arch.hv_guest_os_id)
			kvm->arch.hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
		break;
	case HV_X64_MSR_HYPERCALL: {
		u64 gfn;
		unsigned long addr;
		u8 instructions[4];

		/* if guest os id is not set hypercall should remain disabled */
		if (!kvm->arch.hv_guest_os_id)
			break;
		if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) {
			kvm->arch.hv_hypercall = data;
			break;
		}
		gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;
		addr = gfn_to_hva(kvm, gfn);
		if (kvm_is_error_hva(addr))
			return 1;
		kvm_x86_ops->patch_hypercall(vcpu, instructions);
		((unsigned char *)instructions)[3] = 0xc3; /* ret */
1953
		if (__copy_to_user((void __user *)addr, instructions, 4))
1954 1955
			return 1;
		kvm->arch.hv_hypercall = data;
1956
		mark_page_dirty(kvm, gfn);
1957 1958
		break;
	}
1959 1960 1961 1962 1963 1964 1965 1966
	case HV_X64_MSR_REFERENCE_TSC: {
		u64 gfn;
		HV_REFERENCE_TSC_PAGE tsc_ref;
		memset(&tsc_ref, 0, sizeof(tsc_ref));
		kvm->arch.hv_tsc_page = data;
		if (!(data & HV_X64_MSR_TSC_REFERENCE_ENABLE))
			break;
		gfn = data >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT;
1967
		if (kvm_write_guest(kvm, gfn << HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT,
1968 1969 1970 1971 1972
			&tsc_ref, sizeof(tsc_ref)))
			return 1;
		mark_page_dirty(kvm, gfn);
		break;
	}
1973
	default:
1974 1975
		vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
			    "data 0x%llx\n", msr, data);
1976 1977 1978 1979 1980 1981 1982
		return 1;
	}
	return 0;
}

static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
G
Gleb Natapov 已提交
1983 1984
	switch (msr) {
	case HV_X64_MSR_APIC_ASSIST_PAGE: {
1985
		u64 gfn;
G
Gleb Natapov 已提交
1986
		unsigned long addr;
1987

G
Gleb Natapov 已提交
1988 1989
		if (!(data & HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE)) {
			vcpu->arch.hv_vapic = data;
1990 1991
			if (kvm_lapic_enable_pv_eoi(vcpu, 0))
				return 1;
G
Gleb Natapov 已提交
1992 1993
			break;
		}
1994 1995
		gfn = data >> HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT;
		addr = gfn_to_hva(vcpu->kvm, gfn);
G
Gleb Natapov 已提交
1996 1997
		if (kvm_is_error_hva(addr))
			return 1;
1998
		if (__clear_user((void __user *)addr, PAGE_SIZE))
G
Gleb Natapov 已提交
1999 2000
			return 1;
		vcpu->arch.hv_vapic = data;
2001
		mark_page_dirty(vcpu->kvm, gfn);
2002 2003
		if (kvm_lapic_enable_pv_eoi(vcpu, gfn_to_gpa(gfn) | KVM_MSR_ENABLED))
			return 1;
G
Gleb Natapov 已提交
2004 2005 2006 2007 2008 2009 2010 2011 2012
		break;
	}
	case HV_X64_MSR_EOI:
		return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data);
	case HV_X64_MSR_ICR:
		return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data);
	case HV_X64_MSR_TPR:
		return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data);
	default:
2013 2014
		vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
			    "data 0x%llx\n", msr, data);
G
Gleb Natapov 已提交
2015 2016 2017 2018
		return 1;
	}

	return 0;
2019 2020
}

2021 2022 2023 2024
static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
{
	gpa_t gpa = data & ~0x3f;

G
Guo Chao 已提交
2025
	/* Bits 2:5 are reserved, Should be zero */
2026
	if (data & 0x3c)
2027 2028 2029 2030 2031 2032 2033 2034 2035 2036
		return 1;

	vcpu->arch.apf.msr_val = data;

	if (!(data & KVM_ASYNC_PF_ENABLED)) {
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_async_pf_hash_reset(vcpu);
		return 0;
	}

2037 2038
	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
					sizeof(u32)))
2039 2040
		return 1;

2041
	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
2042 2043 2044 2045
	kvm_async_pf_wakeup_all(vcpu);
	return 0;
}

2046 2047
static void kvmclock_reset(struct kvm_vcpu *vcpu)
{
2048
	vcpu->arch.pv_time_enabled = false;
2049 2050
}

G
Glauber Costa 已提交
2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079
static void accumulate_steal_time(struct kvm_vcpu *vcpu)
{
	u64 delta;

	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

	delta = current->sched_info.run_delay - vcpu->arch.st.last_steal;
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
	vcpu->arch.st.accum_steal = delta;
}

static void record_steal_time(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

	if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
		return;

	vcpu->arch.st.steal.steal += vcpu->arch.st.accum_steal;
	vcpu->arch.st.steal.version += 2;
	vcpu->arch.st.accum_steal = 0;

	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
}

2080
int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2081
{
2082
	bool pr = false;
2083 2084
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
2085

2086
	switch (msr) {
2087 2088 2089 2090 2091 2092 2093 2094
	case MSR_AMD64_NB_CFG:
	case MSR_IA32_UCODE_REV:
	case MSR_IA32_UCODE_WRITE:
	case MSR_VM_HSAVE_PA:
	case MSR_AMD64_PATCH_LOADER:
	case MSR_AMD64_BU_CFG2:
		break;

2095
	case MSR_EFER:
2096
		return set_efer(vcpu, data);
2097 2098
	case MSR_K7_HWCR:
		data &= ~(u64)0x40;	/* ignore flush filter disable */
2099
		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
2100
		data &= ~(u64)0x8;	/* ignore TLB cache disable */
2101
		data &= ~(u64)0x40000;  /* ignore Mc status write enable */
2102
		if (data != 0) {
2103 2104
			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
				    data);
2105 2106
			return 1;
		}
2107
		break;
2108 2109
	case MSR_FAM10H_MMIO_CONF_BASE:
		if (data != 0) {
2110 2111
			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
				    "0x%llx\n", data);
2112 2113
			return 1;
		}
2114
		break;
2115 2116 2117 2118 2119 2120 2121 2122 2123
	case MSR_IA32_DEBUGCTLMSR:
		if (!data) {
			/* We support the non-activated case already */
			break;
		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
			/* Values other than LBR and BTF are vendor-specific,
			   thus reserved and should throw a #GP */
			return 1;
		}
2124 2125
		vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
			    __func__, data);
2126
		break;
A
Avi Kivity 已提交
2127 2128
	case 0x200 ... 0x2ff:
		return set_msr_mtrr(vcpu, msr, data);
2129
	case MSR_IA32_APICBASE:
2130
		return kvm_set_apic_base(vcpu, msr_info);
G
Gleb Natapov 已提交
2131 2132
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_write(vcpu, msr, data);
2133 2134 2135
	case MSR_IA32_TSCDEADLINE:
		kvm_set_lapic_tscdeadline_msr(vcpu, data);
		break;
W
Will Auld 已提交
2136 2137 2138 2139 2140 2141 2142 2143 2144
	case MSR_IA32_TSC_ADJUST:
		if (guest_cpuid_has_tsc_adjust(vcpu)) {
			if (!msr_info->host_initiated) {
				u64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
				kvm_x86_ops->adjust_tsc_offset(vcpu, adj, true);
			}
			vcpu->arch.ia32_tsc_adjust_msr = data;
		}
		break;
2145
	case MSR_IA32_MISC_ENABLE:
2146
		vcpu->arch.ia32_misc_enable_msr = data;
2147
		break;
2148
	case MSR_KVM_WALL_CLOCK_NEW:
2149 2150 2151 2152
	case MSR_KVM_WALL_CLOCK:
		vcpu->kvm->arch.wall_clock = data;
		kvm_write_wall_clock(vcpu->kvm, data);
		break;
2153
	case MSR_KVM_SYSTEM_TIME_NEW:
2154
	case MSR_KVM_SYSTEM_TIME: {
2155
		u64 gpa_offset;
2156
		kvmclock_reset(vcpu);
2157 2158

		vcpu->arch.time = data;
2159
		kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2160 2161 2162 2163 2164

		/* we verify if the enable bit is set... */
		if (!(data & 1))
			break;

2165
		gpa_offset = data & ~(PAGE_MASK | 1);
2166

2167
		if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2168 2169
		     &vcpu->arch.pv_time, data & ~1ULL,
		     sizeof(struct pvclock_vcpu_time_info)))
2170 2171 2172
			vcpu->arch.pv_time_enabled = false;
		else
			vcpu->arch.pv_time_enabled = true;
2173

2174 2175
		break;
	}
2176 2177 2178 2179
	case MSR_KVM_ASYNC_PF_EN:
		if (kvm_pv_enable_async_pf(vcpu, data))
			return 1;
		break;
G
Glauber Costa 已提交
2180 2181 2182 2183 2184 2185 2186 2187 2188
	case MSR_KVM_STEAL_TIME:

		if (unlikely(!sched_info_on()))
			return 1;

		if (data & KVM_STEAL_RESERVED_MASK)
			return 1;

		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
2189 2190
						data & KVM_STEAL_VALID_BITS,
						sizeof(struct kvm_steal_time)))
G
Glauber Costa 已提交
2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206
			return 1;

		vcpu->arch.st.msr_val = data;

		if (!(data & KVM_MSR_ENABLED))
			break;

		vcpu->arch.st.last_steal = current->sched_info.run_delay;

		preempt_disable();
		accumulate_steal_time(vcpu);
		preempt_enable();

		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);

		break;
2207 2208 2209 2210
	case MSR_KVM_PV_EOI_EN:
		if (kvm_lapic_enable_pv_eoi(vcpu, data))
			return 1;
		break;
G
Glauber Costa 已提交
2211

H
Huang Ying 已提交
2212 2213
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
2214
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
H
Huang Ying 已提交
2215
		return set_msr_mce(vcpu, msr, data);
2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228

	/* Performance counters are not protected by a CPUID bit,
	 * so we should check all of them in the generic path for the sake of
	 * cross vendor migration.
	 * Writing a zero into the event select MSRs disables them,
	 * which we perfectly emulate ;-). Any other value should be at least
	 * reported, some guests depend on them.
	 */
	case MSR_K7_EVNTSEL0:
	case MSR_K7_EVNTSEL1:
	case MSR_K7_EVNTSEL2:
	case MSR_K7_EVNTSEL3:
		if (data != 0)
2229 2230
			vcpu_unimpl(vcpu, "unimplemented perfctr wrmsr: "
				    "0x%x data 0x%llx\n", msr, data);
2231 2232 2233 2234 2235 2236 2237 2238
		break;
	/* at least RHEL 4 unconditionally writes to the perfctr registers,
	 * so we ignore writes to make it happy.
	 */
	case MSR_K7_PERFCTR0:
	case MSR_K7_PERFCTR1:
	case MSR_K7_PERFCTR2:
	case MSR_K7_PERFCTR3:
2239 2240
		vcpu_unimpl(vcpu, "unimplemented perfctr wrmsr: "
			    "0x%x data 0x%llx\n", msr, data);
2241
		break;
2242 2243 2244 2245 2246 2247
	case MSR_P6_PERFCTR0:
	case MSR_P6_PERFCTR1:
		pr = true;
	case MSR_P6_EVNTSEL0:
	case MSR_P6_EVNTSEL1:
		if (kvm_pmu_msr(vcpu, msr))
2248
			return kvm_pmu_set_msr(vcpu, msr_info);
2249 2250

		if (pr || data != 0)
2251 2252
			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
				    "0x%x data 0x%llx\n", msr, data);
2253
		break;
2254 2255 2256 2257 2258
	case MSR_K7_CLK_CTL:
		/*
		 * Ignore all writes to this no longer documented MSR.
		 * Writes are only relevant for old K7 processors,
		 * all pre-dating SVM, but a recommended workaround from
G
Guo Chao 已提交
2259
		 * AMD for these chips. It is possible to specify the
2260 2261 2262 2263
		 * affected processor models on the command line, hence
		 * the need to ignore the workaround.
		 */
		break;
2264 2265 2266 2267 2268 2269 2270 2271 2272 2273
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
		if (kvm_hv_msr_partition_wide(msr)) {
			int r;
			mutex_lock(&vcpu->kvm->lock);
			r = set_msr_hyperv_pw(vcpu, msr, data);
			mutex_unlock(&vcpu->kvm->lock);
			return r;
		} else
			return set_msr_hyperv(vcpu, msr, data);
		break;
2274 2275 2276 2277
	case MSR_IA32_BBL_CR_CTL3:
		/* Drop writes to this legacy MSR -- see rdmsr
		 * counterpart for further detail.
		 */
2278
		vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", msr, data);
2279
		break;
2280 2281 2282 2283 2284 2285 2286 2287 2288 2289
	case MSR_AMD64_OSVW_ID_LENGTH:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		vcpu->arch.osvw.length = data;
		break;
	case MSR_AMD64_OSVW_STATUS:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		vcpu->arch.osvw.status = data;
		break;
2290
	default:
E
Ed Swierk 已提交
2291 2292
		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
			return xen_hvm_config(vcpu, data);
2293
		if (kvm_pmu_msr(vcpu, msr))
2294
			return kvm_pmu_set_msr(vcpu, msr_info);
2295
		if (!ignore_msrs) {
2296 2297
			vcpu_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
				    msr, data);
2298 2299
			return 1;
		} else {
2300 2301
			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
				    msr, data);
2302 2303
			break;
		}
2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);


/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
	return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
}

A
Avi Kivity 已提交
2320 2321
static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
S
Sheng Yang 已提交
2322 2323
	u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;

A
Avi Kivity 已提交
2324 2325 2326
	if (!msr_mtrr_valid(msr))
		return 1;

S
Sheng Yang 已提交
2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352
	if (msr == MSR_MTRRdefType)
		*pdata = vcpu->arch.mtrr_state.def_type +
			 (vcpu->arch.mtrr_state.enabled << 10);
	else if (msr == MSR_MTRRfix64K_00000)
		*pdata = p[0];
	else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
		*pdata = p[1 + msr - MSR_MTRRfix16K_80000];
	else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
		*pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
	else if (msr == MSR_IA32_CR_PAT)
		*pdata = vcpu->arch.pat;
	else {	/* Variable MTRRs */
		int idx, is_mtrr_mask;
		u64 *pt;

		idx = (msr - 0x200) / 2;
		is_mtrr_mask = msr - 0x200 - 2 * idx;
		if (!is_mtrr_mask)
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
		else
			pt =
			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
		*pdata = *pt;
	}

A
Avi Kivity 已提交
2353 2354 2355
	return 0;
}

H
Huang Ying 已提交
2356
static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
2357 2358
{
	u64 data;
H
Huang Ying 已提交
2359 2360
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2361 2362 2363 2364

	switch (msr) {
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
H
Huang Ying 已提交
2365 2366
		data = 0;
		break;
2367
	case MSR_IA32_MCG_CAP:
H
Huang Ying 已提交
2368 2369
		data = vcpu->arch.mcg_cap;
		break;
2370
	case MSR_IA32_MCG_CTL:
H
Huang Ying 已提交
2371 2372 2373 2374 2375 2376 2377 2378 2379
		if (!(mcg_cap & MCG_CTL_P))
			return 1;
		data = vcpu->arch.mcg_ctl;
		break;
	case MSR_IA32_MCG_STATUS:
		data = vcpu->arch.mcg_status;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2380
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
H
Huang Ying 已提交
2381 2382 2383 2384 2385 2386 2387 2388 2389 2390
			u32 offset = msr - MSR_IA32_MC0_CTL;
			data = vcpu->arch.mce_banks[offset];
			break;
		}
		return 1;
	}
	*pdata = data;
	return 0;
}

2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402
static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data = 0;
	struct kvm *kvm = vcpu->kvm;

	switch (msr) {
	case HV_X64_MSR_GUEST_OS_ID:
		data = kvm->arch.hv_guest_os_id;
		break;
	case HV_X64_MSR_HYPERCALL:
		data = kvm->arch.hv_hypercall;
		break;
2403 2404 2405 2406 2407 2408 2409 2410
	case HV_X64_MSR_TIME_REF_COUNT: {
		data =
		     div_u64(get_kernel_ns() + kvm->arch.kvmclock_offset, 100);
		break;
	}
	case HV_X64_MSR_REFERENCE_TSC:
		data = kvm->arch.hv_tsc_page;
		break;
2411
	default:
2412
		vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427
		return 1;
	}

	*pdata = data;
	return 0;
}

static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data = 0;

	switch (msr) {
	case HV_X64_MSR_VP_INDEX: {
		int r;
		struct kvm_vcpu *v;
2428 2429
		kvm_for_each_vcpu(r, v, vcpu->kvm) {
			if (v == vcpu) {
2430
				data = r;
2431 2432 2433
				break;
			}
		}
2434 2435
		break;
	}
G
Gleb Natapov 已提交
2436 2437 2438 2439 2440 2441
	case HV_X64_MSR_EOI:
		return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata);
	case HV_X64_MSR_ICR:
		return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
	case HV_X64_MSR_TPR:
		return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
2442
	case HV_X64_MSR_APIC_ASSIST_PAGE:
2443 2444
		data = vcpu->arch.hv_vapic;
		break;
2445
	default:
2446
		vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
2447 2448 2449 2450 2451 2452
		return 1;
	}
	*pdata = data;
	return 0;
}

H
Huang Ying 已提交
2453 2454 2455 2456 2457 2458
int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
	u64 data;

	switch (msr) {
	case MSR_IA32_PLATFORM_ID:
2459
	case MSR_IA32_EBL_CR_POWERON:
2460 2461 2462 2463 2464
	case MSR_IA32_DEBUGCTLMSR:
	case MSR_IA32_LASTBRANCHFROMIP:
	case MSR_IA32_LASTBRANCHTOIP:
	case MSR_IA32_LASTINTFROMIP:
	case MSR_IA32_LASTINTTOIP:
2465 2466
	case MSR_K8_SYSCFG:
	case MSR_K7_HWCR:
2467
	case MSR_VM_HSAVE_PA:
A
Amit Shah 已提交
2468
	case MSR_K7_EVNTSEL0:
2469 2470 2471
	case MSR_K7_EVNTSEL1:
	case MSR_K7_EVNTSEL2:
	case MSR_K7_EVNTSEL3:
A
Amit Shah 已提交
2472
	case MSR_K7_PERFCTR0:
2473 2474 2475
	case MSR_K7_PERFCTR1:
	case MSR_K7_PERFCTR2:
	case MSR_K7_PERFCTR3:
2476
	case MSR_K8_INT_PENDING_MSG:
2477
	case MSR_AMD64_NB_CFG:
2478
	case MSR_FAM10H_MMIO_CONF_BASE:
2479
	case MSR_AMD64_BU_CFG2:
2480 2481
		data = 0;
		break;
2482 2483 2484 2485 2486 2487 2488 2489
	case MSR_P6_PERFCTR0:
	case MSR_P6_PERFCTR1:
	case MSR_P6_EVNTSEL0:
	case MSR_P6_EVNTSEL1:
		if (kvm_pmu_msr(vcpu, msr))
			return kvm_pmu_get_msr(vcpu, msr, pdata);
		data = 0;
		break;
2490 2491 2492
	case MSR_IA32_UCODE_REV:
		data = 0x100000000ULL;
		break;
A
Avi Kivity 已提交
2493 2494 2495 2496 2497
	case MSR_MTRRcap:
		data = 0x500 | KVM_NR_VAR_MTRR;
		break;
	case 0x200 ... 0x2ff:
		return get_msr_mtrr(vcpu, msr, pdata);
2498 2499 2500
	case 0xcd: /* fsb frequency */
		data = 3;
		break;
2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514
		/*
		 * MSR_EBC_FREQUENCY_ID
		 * Conservative value valid for even the basic CPU models.
		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
		 * and 266MHz for model 3, or 4. Set Core Clock
		 * Frequency to System Bus Frequency Ratio to 1 (bits
		 * 31:24) even though these are only valid for CPU
		 * models > 2, however guests may end up dividing or
		 * multiplying by zero otherwise.
		 */
	case MSR_EBC_FREQUENCY_ID:
		data = 1 << 24;
		break;
2515 2516 2517
	case MSR_IA32_APICBASE:
		data = kvm_get_apic_base(vcpu);
		break;
G
Gleb Natapov 已提交
2518 2519 2520
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_read(vcpu, msr, pdata);
		break;
2521 2522 2523
	case MSR_IA32_TSCDEADLINE:
		data = kvm_get_lapic_tscdeadline_msr(vcpu);
		break;
W
Will Auld 已提交
2524 2525 2526
	case MSR_IA32_TSC_ADJUST:
		data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
		break;
2527
	case MSR_IA32_MISC_ENABLE:
2528
		data = vcpu->arch.ia32_misc_enable_msr;
2529
		break;
2530 2531 2532 2533 2534 2535
	case MSR_IA32_PERF_STATUS:
		/* TSC increment by tick */
		data = 1000ULL;
		/* CPU multiplier */
		data |= (((uint64_t)4ULL) << 40);
		break;
2536
	case MSR_EFER:
2537
		data = vcpu->arch.efer;
2538
		break;
2539
	case MSR_KVM_WALL_CLOCK:
2540
	case MSR_KVM_WALL_CLOCK_NEW:
2541 2542 2543
		data = vcpu->kvm->arch.wall_clock;
		break;
	case MSR_KVM_SYSTEM_TIME:
2544
	case MSR_KVM_SYSTEM_TIME_NEW:
2545 2546
		data = vcpu->arch.time;
		break;
2547 2548 2549
	case MSR_KVM_ASYNC_PF_EN:
		data = vcpu->arch.apf.msr_val;
		break;
G
Glauber Costa 已提交
2550 2551 2552
	case MSR_KVM_STEAL_TIME:
		data = vcpu->arch.st.msr_val;
		break;
2553 2554 2555
	case MSR_KVM_PV_EOI_EN:
		data = vcpu->arch.pv_eoi.msr_val;
		break;
H
Huang Ying 已提交
2556 2557 2558 2559 2560
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
2561
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
H
Huang Ying 已提交
2562
		return get_msr_mce(vcpu, msr, pdata);
2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574
	case MSR_K7_CLK_CTL:
		/*
		 * Provide expected ramp-up count for K7. All other
		 * are set to zero, indicating minimum divisors for
		 * every field.
		 *
		 * This prevents guest kernels on AMD host with CPU
		 * type 6, model 8 and higher from exploding due to
		 * the rdmsr failing.
		 */
		data = 0x20000000;
		break;
2575 2576 2577 2578 2579 2580 2581 2582 2583 2584
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
		if (kvm_hv_msr_partition_wide(msr)) {
			int r;
			mutex_lock(&vcpu->kvm->lock);
			r = get_msr_hyperv_pw(vcpu, msr, pdata);
			mutex_unlock(&vcpu->kvm->lock);
			return r;
		} else
			return get_msr_hyperv(vcpu, msr, pdata);
		break;
2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597
	case MSR_IA32_BBL_CR_CTL3:
		/* This legacy MSR exists but isn't fully documented in current
		 * silicon.  It is however accessed by winxp in very narrow
		 * scenarios where it sets bit #19, itself documented as
		 * a "reserved" bit.  Best effort attempt to source coherent
		 * read data here should the balance of the register be
		 * interpreted by the guest:
		 *
		 * L2 cache control register 3: 64GB range, 256KB size,
		 * enabled, latency 0x1, configured
		 */
		data = 0xbe702111;
		break;
2598 2599 2600 2601 2602 2603 2604 2605 2606 2607
	case MSR_AMD64_OSVW_ID_LENGTH:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		data = vcpu->arch.osvw.length;
		break;
	case MSR_AMD64_OSVW_STATUS:
		if (!guest_cpuid_has_osvw(vcpu))
			return 1;
		data = vcpu->arch.osvw.status;
		break;
2608
	default:
2609 2610
		if (kvm_pmu_msr(vcpu, msr))
			return kvm_pmu_get_msr(vcpu, msr, pdata);
2611
		if (!ignore_msrs) {
2612
			vcpu_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
2613 2614
			return 1;
		} else {
2615
			vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
2616 2617 2618
			data = 0;
		}
		break;
2619 2620 2621 2622 2623 2624
	}
	*pdata = data;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

2625 2626 2627 2628 2629 2630 2631 2632 2633 2634
/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
2635
	int i, idx;
2636

2637
	idx = srcu_read_lock(&vcpu->kvm->srcu);
2638 2639 2640
	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;
2641
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
	if (copy_from_user(&msrs, user_msrs, sizeof msrs))
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
2670 2671 2672
	entries = memdup_user(user_msrs->entries, size);
	if (IS_ERR(entries)) {
		r = PTR_ERR(entries);
2673
		goto out;
2674
	}
2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686

	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
2687
	kfree(entries);
2688 2689 2690 2691
out:
	return r;
}

2692
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
2693 2694 2695 2696 2697 2698 2699 2700
{
	int r;

	switch (ext) {
	case KVM_CAP_IRQCHIP:
	case KVM_CAP_HLT:
	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
	case KVM_CAP_SET_TSS_ADDR:
2701
	case KVM_CAP_EXT_CPUID:
B
Borislav Petkov 已提交
2702
	case KVM_CAP_EXT_EMUL_CPUID:
2703
	case KVM_CAP_CLOCKSOURCE:
S
Sheng Yang 已提交
2704
	case KVM_CAP_PIT:
2705
	case KVM_CAP_NOP_IO_DELAY:
2706
	case KVM_CAP_MP_STATE:
2707
	case KVM_CAP_SYNC_MMU:
2708
	case KVM_CAP_USER_NMI:
2709
	case KVM_CAP_REINJECT_CONTROL:
2710
	case KVM_CAP_IRQ_INJECT_STATUS:
G
Gregory Haskins 已提交
2711
	case KVM_CAP_IRQFD:
G
Gregory Haskins 已提交
2712
	case KVM_CAP_IOEVENTFD:
2713
	case KVM_CAP_IOEVENTFD_NO_LENGTH:
2714
	case KVM_CAP_PIT2:
B
Beth Kon 已提交
2715
	case KVM_CAP_PIT_STATE2:
2716
	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
E
Ed Swierk 已提交
2717
	case KVM_CAP_XEN_HVM:
2718
	case KVM_CAP_ADJUST_CLOCK:
J
Jan Kiszka 已提交
2719
	case KVM_CAP_VCPU_EVENTS:
2720
	case KVM_CAP_HYPERV:
G
Gleb Natapov 已提交
2721
	case KVM_CAP_HYPERV_VAPIC:
2722
	case KVM_CAP_HYPERV_SPIN:
2723
	case KVM_CAP_PCI_SEGMENT:
2724
	case KVM_CAP_DEBUGREGS:
2725
	case KVM_CAP_X86_ROBUST_SINGLESTEP:
2726
	case KVM_CAP_XSAVE:
2727
	case KVM_CAP_ASYNC_PF:
2728
	case KVM_CAP_GET_TSC_KHZ:
2729
	case KVM_CAP_KVMCLOCK_CTRL:
X
Xiao Guangrong 已提交
2730
	case KVM_CAP_READONLY_MEM:
2731
	case KVM_CAP_HYPERV_TIME:
2732
	case KVM_CAP_IOAPIC_POLARITY_IGNORED:
2733 2734 2735 2736
#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
	case KVM_CAP_ASSIGN_DEV_IRQ:
	case KVM_CAP_PCI_2_3:
#endif
2737 2738
		r = 1;
		break;
2739 2740 2741
	case KVM_CAP_COALESCED_MMIO:
		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
		break;
2742 2743 2744
	case KVM_CAP_VAPIC:
		r = !kvm_x86_ops->cpu_has_accelerated_tpr();
		break;
2745
	case KVM_CAP_NR_VCPUS:
2746 2747 2748
		r = KVM_SOFT_MAX_VCPUS;
		break;
	case KVM_CAP_MAX_VCPUS:
2749 2750
		r = KVM_MAX_VCPUS;
		break;
2751
	case KVM_CAP_NR_MEMSLOTS:
2752
		r = KVM_USER_MEM_SLOTS;
2753
		break;
2754 2755
	case KVM_CAP_PV_MMU:	/* obsolete */
		r = 0;
2756
		break;
2757
#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
B
Ben-Ami Yassour 已提交
2758
	case KVM_CAP_IOMMU:
2759
		r = iommu_present(&pci_bus_type);
B
Ben-Ami Yassour 已提交
2760
		break;
2761
#endif
H
Huang Ying 已提交
2762 2763 2764
	case KVM_CAP_MCE:
		r = KVM_MAX_MCE_BANKS;
		break;
2765 2766 2767
	case KVM_CAP_XCRS:
		r = cpu_has_xsave;
		break;
2768 2769 2770
	case KVM_CAP_TSC_CONTROL:
		r = kvm_has_tsc_control;
		break;
2771 2772 2773
	case KVM_CAP_TSC_DEADLINE_TIMER:
		r = boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER);
		break;
2774 2775 2776 2777 2778 2779 2780 2781
	default:
		r = 0;
		break;
	}
	return r;

}

2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801
long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {
	case KVM_GET_MSR_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
			goto out;
		n = msr_list.nmsrs;
		msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
		if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
			goto out;
		r = -E2BIG;
J
Jan Kiszka 已提交
2802
		if (n < msr_list.nmsrs)
2803 2804 2805 2806 2807
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
				 num_msrs_to_save * sizeof(u32)))
			goto out;
J
Jan Kiszka 已提交
2808
		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
2809 2810 2811 2812 2813 2814
				 &emulated_msrs,
				 ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
			goto out;
		r = 0;
		break;
	}
B
Borislav Petkov 已提交
2815 2816
	case KVM_GET_SUPPORTED_CPUID:
	case KVM_GET_EMULATED_CPUID: {
2817 2818 2819 2820 2821 2822
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
B
Borislav Petkov 已提交
2823 2824 2825

		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
					    ioctl);
2826 2827 2828 2829 2830 2831 2832 2833 2834
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
			goto out;
		r = 0;
		break;
	}
H
Huang Ying 已提交
2835 2836 2837 2838 2839 2840 2841 2842 2843 2844
	case KVM_X86_GET_MCE_CAP_SUPPORTED: {
		u64 mce_cap;

		mce_cap = KVM_MCE_CAP_SUPPORTED;
		r = -EFAULT;
		if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
			goto out;
		r = 0;
		break;
	}
2845 2846 2847 2848 2849 2850 2851
	default:
		r = -EINVAL;
	}
out:
	return r;
}

2852 2853 2854 2855 2856 2857 2858
static void wbinvd_ipi(void *garbage)
{
	wbinvd();
}

static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
2859
	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
2860 2861
}

2862 2863
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
2864 2865 2866 2867 2868 2869 2870 2871 2872
	/* Address WBINVD may be executed by guest */
	if (need_emulate_wbinvd(vcpu)) {
		if (kvm_x86_ops->has_wbinvd_exit())
			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
			smp_call_function_single(vcpu->cpu,
					wbinvd_ipi, NULL, 1);
	}

2873
	kvm_x86_ops->vcpu_load(vcpu, cpu);
2874

2875 2876 2877 2878
	/* Apply any externally detected TSC adjustments (due to suspend) */
	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
		vcpu->arch.tsc_offset_adjustment = 0;
2879
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2880
	}
2881

2882
	if (unlikely(vcpu->cpu != cpu) || check_tsc_unstable()) {
2883 2884
		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
				native_read_tsc() - vcpu->arch.last_host_tsc;
Z
Zachary Amsden 已提交
2885 2886
		if (tsc_delta < 0)
			mark_tsc_unstable("KVM discovered backwards TSC");
Z
Zachary Amsden 已提交
2887
		if (check_tsc_unstable()) {
2888 2889 2890
			u64 offset = kvm_x86_ops->compute_tsc_offset(vcpu,
						vcpu->arch.last_guest_tsc);
			kvm_x86_ops->write_tsc_offset(vcpu, offset);
Z
Zachary Amsden 已提交
2891 2892
			vcpu->arch.tsc_catchup = 1;
		}
2893 2894 2895 2896 2897
		/*
		 * On a host with synchronized TSC, there is no need to update
		 * kvmclock on vcpu->cpu migration
		 */
		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
2898
			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
Z
Zachary Amsden 已提交
2899 2900
		if (vcpu->cpu != cpu)
			kvm_migrate_timers(vcpu);
Z
Zachary Amsden 已提交
2901
		vcpu->cpu = cpu;
Z
Zachary Amsden 已提交
2902
	}
G
Glauber Costa 已提交
2903 2904 2905

	accumulate_steal_time(vcpu);
	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
2906 2907 2908 2909
}

void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
2910
	kvm_x86_ops->vcpu_put(vcpu);
2911
	kvm_put_guest_fpu(vcpu);
2912
	vcpu->arch.last_host_tsc = native_read_tsc();
2913 2914 2915 2916 2917
}

static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
2918
	kvm_x86_ops->sync_pir_to_irr(vcpu);
2919
	memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
2920 2921 2922 2923 2924 2925 2926

	return 0;
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
2927
	kvm_apic_post_state_restore(vcpu, s);
2928
	update_cr8_intercept(vcpu);
2929 2930 2931 2932

	return 0;
}

2933 2934 2935
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
{
2936
	if (irq->irq >= KVM_NR_INTERRUPTS)
2937 2938 2939 2940
		return -EINVAL;
	if (irqchip_in_kernel(vcpu->kvm))
		return -ENXIO;

2941
	kvm_queue_interrupt(vcpu, irq->irq, false);
2942
	kvm_make_request(KVM_REQ_EVENT, vcpu);
2943 2944 2945 2946

	return 0;
}

2947 2948 2949 2950 2951 2952 2953
static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
{
	kvm_inject_nmi(vcpu);

	return 0;
}

2954 2955 2956 2957 2958 2959 2960 2961 2962
static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
					   struct kvm_tpr_access_ctl *tac)
{
	if (tac->flags)
		return -EINVAL;
	vcpu->arch.tpr_access_reporting = !!tac->enabled;
	return 0;
}

H
Huang Ying 已提交
2963 2964 2965 2966 2967 2968 2969
static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
					u64 mcg_cap)
{
	int r;
	unsigned bank_num = mcg_cap & 0xff, bank;

	r = -EINVAL;
2970
	if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
H
Huang Ying 已提交
2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010
		goto out;
	if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
		goto out;
	r = 0;
	vcpu->arch.mcg_cap = mcg_cap;
	/* Init IA32_MCG_CTL to all 1s */
	if (mcg_cap & MCG_CTL_P)
		vcpu->arch.mcg_ctl = ~(u64)0;
	/* Init IA32_MCi_CTL to all 1s */
	for (bank = 0; bank < bank_num; bank++)
		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
out:
	return r;
}

static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
				      struct kvm_x86_mce *mce)
{
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
	u64 *banks = vcpu->arch.mce_banks;

	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
		return -EINVAL;
	/*
	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
	 * reporting is disabled
	 */
	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
	    vcpu->arch.mcg_ctl != ~(u64)0)
		return 0;
	banks += 4 * mce->bank;
	/*
	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
	 * reporting is disabled for the bank
	 */
	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
		return 0;
	if (mce->status & MCI_STATUS_UC) {
		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
3011
		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
3012
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
H
Huang Ying 已提交
3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033
			return 0;
		}
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		vcpu->arch.mcg_status = mce->mcg_status;
		banks[1] = mce->status;
		kvm_queue_exception(vcpu, MC_VECTOR);
	} else if (!(banks[1] & MCI_STATUS_VAL)
		   || !(banks[1] & MCI_STATUS_UC)) {
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		banks[1] = mce->status;
	} else
		banks[1] |= MCI_STATUS_OVER;
	return 0;
}

J
Jan Kiszka 已提交
3034 3035 3036
static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
					       struct kvm_vcpu_events *events)
{
A
Avi Kivity 已提交
3037
	process_nmi(vcpu);
3038 3039 3040
	events->exception.injected =
		vcpu->arch.exception.pending &&
		!kvm_exception_is_soft(vcpu->arch.exception.nr);
J
Jan Kiszka 已提交
3041 3042
	events->exception.nr = vcpu->arch.exception.nr;
	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
3043
	events->exception.pad = 0;
J
Jan Kiszka 已提交
3044 3045
	events->exception.error_code = vcpu->arch.exception.error_code;

3046 3047
	events->interrupt.injected =
		vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft;
J
Jan Kiszka 已提交
3048
	events->interrupt.nr = vcpu->arch.interrupt.nr;
3049
	events->interrupt.soft = 0;
3050
	events->interrupt.shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
J
Jan Kiszka 已提交
3051 3052

	events->nmi.injected = vcpu->arch.nmi_injected;
A
Avi Kivity 已提交
3053
	events->nmi.pending = vcpu->arch.nmi_pending != 0;
J
Jan Kiszka 已提交
3054
	events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
3055
	events->nmi.pad = 0;
J
Jan Kiszka 已提交
3056

3057
	events->sipi_vector = 0; /* never valid when reporting to user space */
J
Jan Kiszka 已提交
3058

3059
	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
3060
			 | KVM_VCPUEVENT_VALID_SHADOW);
3061
	memset(&events->reserved, 0, sizeof(events->reserved));
J
Jan Kiszka 已提交
3062 3063 3064 3065 3066
}

static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
					      struct kvm_vcpu_events *events)
{
3067
	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
3068 3069
			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
			      | KVM_VCPUEVENT_VALID_SHADOW))
J
Jan Kiszka 已提交
3070 3071
		return -EINVAL;

A
Avi Kivity 已提交
3072
	process_nmi(vcpu);
J
Jan Kiszka 已提交
3073 3074 3075 3076 3077 3078 3079 3080
	vcpu->arch.exception.pending = events->exception.injected;
	vcpu->arch.exception.nr = events->exception.nr;
	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
	vcpu->arch.exception.error_code = events->exception.error_code;

	vcpu->arch.interrupt.pending = events->interrupt.injected;
	vcpu->arch.interrupt.nr = events->interrupt.nr;
	vcpu->arch.interrupt.soft = events->interrupt.soft;
3081 3082 3083
	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
		kvm_x86_ops->set_interrupt_shadow(vcpu,
						  events->interrupt.shadow);
J
Jan Kiszka 已提交
3084 3085

	vcpu->arch.nmi_injected = events->nmi.injected;
3086 3087
	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
		vcpu->arch.nmi_pending = events->nmi.pending;
J
Jan Kiszka 已提交
3088 3089
	kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);

3090 3091 3092
	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
	    kvm_vcpu_has_lapic(vcpu))
		vcpu->arch.apic->sipi_vector = events->sipi_vector;
J
Jan Kiszka 已提交
3093

3094 3095
	kvm_make_request(KVM_REQ_EVENT, vcpu);

J
Jan Kiszka 已提交
3096 3097 3098
	return 0;
}

3099 3100 3101
static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
					     struct kvm_debugregs *dbgregs)
{
J
Jan Kiszka 已提交
3102 3103
	unsigned long val;

3104
	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
3105
	kvm_get_dr(vcpu, 6, &val);
J
Jan Kiszka 已提交
3106
	dbgregs->dr6 = val;
3107 3108
	dbgregs->dr7 = vcpu->arch.dr7;
	dbgregs->flags = 0;
3109
	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
3110 3111 3112 3113 3114 3115 3116 3117 3118 3119
}

static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
					    struct kvm_debugregs *dbgregs)
{
	if (dbgregs->flags)
		return -EINVAL;

	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
	vcpu->arch.dr6 = dbgregs->dr6;
J
Jan Kiszka 已提交
3120
	kvm_update_dr6(vcpu);
3121
	vcpu->arch.dr7 = dbgregs->dr7;
3122
	kvm_update_dr7(vcpu);
3123 3124 3125 3126

	return 0;
}

3127 3128 3129
static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
					 struct kvm_xsave *guest_xsave)
{
3130
	if (cpu_has_xsave) {
3131 3132
		memcpy(guest_xsave->region,
			&vcpu->arch.guest_fpu.state->xsave,
3133 3134 3135 3136
			vcpu->arch.guest_xstate_size);
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] &=
			vcpu->arch.guest_supported_xcr0 | XSTATE_FPSSE;
	} else {
3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150
		memcpy(guest_xsave->region,
			&vcpu->arch.guest_fpu.state->fxsave,
			sizeof(struct i387_fxsave_struct));
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
			XSTATE_FPSSE;
	}
}

static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
					struct kvm_xsave *guest_xsave)
{
	u64 xstate_bv =
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];

3151 3152 3153 3154 3155 3156
	if (cpu_has_xsave) {
		/*
		 * Here we allow setting states that are not present in
		 * CPUID leaf 0xD, index 0, EDX:EAX.  This is for compatibility
		 * with old userspace.
		 */
3157
		if (xstate_bv & ~kvm_supported_xcr0())
3158
			return -EINVAL;
3159
		memcpy(&vcpu->arch.guest_fpu.state->xsave,
3160
			guest_xsave->region, vcpu->arch.guest_xstate_size);
3161
	} else {
3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196
		if (xstate_bv & ~XSTATE_FPSSE)
			return -EINVAL;
		memcpy(&vcpu->arch.guest_fpu.state->fxsave,
			guest_xsave->region, sizeof(struct i387_fxsave_struct));
	}
	return 0;
}

static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
					struct kvm_xcrs *guest_xcrs)
{
	if (!cpu_has_xsave) {
		guest_xcrs->nr_xcrs = 0;
		return;
	}

	guest_xcrs->nr_xcrs = 1;
	guest_xcrs->flags = 0;
	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
}

static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
				       struct kvm_xcrs *guest_xcrs)
{
	int i, r = 0;

	if (!cpu_has_xsave)
		return -EINVAL;

	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
		return -EINVAL;

	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
		/* Only support XCR0 currently */
P
Paolo Bonzini 已提交
3197
		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
3198
			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
P
Paolo Bonzini 已提交
3199
				guest_xcrs->xcrs[i].value);
3200 3201 3202 3203 3204 3205 3206
			break;
		}
	if (r)
		r = -EINVAL;
	return r;
}

3207 3208 3209 3210 3211 3212 3213 3214
/*
 * kvm_set_guest_paused() indicates to the guest kernel that it has been
 * stopped by the hypervisor.  This function will be called from the host only.
 * EINVAL is returned when the host attempts to set the flag for a guest that
 * does not support pv clocks.
 */
static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
{
3215
	if (!vcpu->arch.pv_time_enabled)
3216
		return -EINVAL;
3217
	vcpu->arch.pvclock_set_guest_stopped_request = true;
3218 3219 3220 3221
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	return 0;
}

3222 3223 3224 3225 3226 3227
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r;
3228 3229 3230 3231 3232 3233 3234 3235
	union {
		struct kvm_lapic_state *lapic;
		struct kvm_xsave *xsave;
		struct kvm_xcrs *xcrs;
		void *buffer;
	} u;

	u.buffer = NULL;
3236 3237
	switch (ioctl) {
	case KVM_GET_LAPIC: {
3238 3239 3240
		r = -EINVAL;
		if (!vcpu->arch.apic)
			goto out;
3241
		u.lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
3242

3243
		r = -ENOMEM;
3244
		if (!u.lapic)
3245
			goto out;
3246
		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
3247 3248 3249
		if (r)
			goto out;
		r = -EFAULT;
3250
		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
3251 3252 3253 3254 3255
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
3256 3257 3258
		r = -EINVAL;
		if (!vcpu->arch.apic)
			goto out;
3259
		u.lapic = memdup_user(argp, sizeof(*u.lapic));
G
Guo Chao 已提交
3260 3261
		if (IS_ERR(u.lapic))
			return PTR_ERR(u.lapic);
3262

3263
		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
3264 3265
		break;
	}
3266 3267 3268 3269 3270 3271 3272 3273 3274
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
		if (copy_from_user(&irq, argp, sizeof irq))
			goto out;
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
		break;
	}
3275 3276 3277 3278
	case KVM_NMI: {
		r = kvm_vcpu_ioctl_nmi(vcpu);
		break;
	}
3279 3280 3281 3282 3283 3284 3285 3286 3287 3288
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		break;
	}
3289 3290 3291 3292 3293 3294 3295 3296
	case KVM_SET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
3297
					      cpuid_arg->entries);
3298 3299 3300 3301 3302 3303 3304 3305 3306 3307
		break;
	}
	case KVM_GET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
			goto out;
		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
3308
					      cpuid_arg->entries);
3309 3310 3311 3312 3313 3314 3315 3316
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
			goto out;
		r = 0;
		break;
	}
3317 3318 3319 3320 3321 3322
	case KVM_GET_MSRS:
		r = msr_io(vcpu, argp, kvm_get_msr, 1);
		break;
	case KVM_SET_MSRS:
		r = msr_io(vcpu, argp, do_set_msr, 0);
		break;
3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337
	case KVM_TPR_ACCESS_REPORTING: {
		struct kvm_tpr_access_ctl tac;

		r = -EFAULT;
		if (copy_from_user(&tac, argp, sizeof tac))
			goto out;
		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &tac, sizeof tac))
			goto out;
		r = 0;
		break;
	};
A
Avi Kivity 已提交
3338 3339 3340 3341 3342 3343 3344 3345 3346
	case KVM_SET_VAPIC_ADDR: {
		struct kvm_vapic_addr va;

		r = -EINVAL;
		if (!irqchip_in_kernel(vcpu->kvm))
			goto out;
		r = -EFAULT;
		if (copy_from_user(&va, argp, sizeof va))
			goto out;
3347
		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
A
Avi Kivity 已提交
3348 3349
		break;
	}
H
Huang Ying 已提交
3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367
	case KVM_X86_SETUP_MCE: {
		u64 mcg_cap;

		r = -EFAULT;
		if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
			goto out;
		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
		break;
	}
	case KVM_X86_SET_MCE: {
		struct kvm_x86_mce mce;

		r = -EFAULT;
		if (copy_from_user(&mce, argp, sizeof mce))
			goto out;
		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
		break;
	}
J
Jan Kiszka 已提交
3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);

		r = -EFAULT;
		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		r = -EFAULT;
		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
			break;

		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
		break;
	}
3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411
	case KVM_GET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);

		r = -EFAULT;
		if (copy_to_user(argp, &dbgregs,
				 sizeof(struct kvm_debugregs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		r = -EFAULT;
		if (copy_from_user(&dbgregs, argp,
				   sizeof(struct kvm_debugregs)))
			break;

		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
		break;
	}
3412
	case KVM_GET_XSAVE: {
3413
		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL);
3414
		r = -ENOMEM;
3415
		if (!u.xsave)
3416 3417
			break;

3418
		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
3419 3420

		r = -EFAULT;
3421
		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
3422 3423 3424 3425 3426
			break;
		r = 0;
		break;
	}
	case KVM_SET_XSAVE: {
3427
		u.xsave = memdup_user(argp, sizeof(*u.xsave));
G
Guo Chao 已提交
3428 3429
		if (IS_ERR(u.xsave))
			return PTR_ERR(u.xsave);
3430

3431
		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
3432 3433 3434
		break;
	}
	case KVM_GET_XCRS: {
3435
		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL);
3436
		r = -ENOMEM;
3437
		if (!u.xcrs)
3438 3439
			break;

3440
		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
3441 3442

		r = -EFAULT;
3443
		if (copy_to_user(argp, u.xcrs,
3444 3445 3446 3447 3448 3449
				 sizeof(struct kvm_xcrs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XCRS: {
3450
		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
G
Guo Chao 已提交
3451 3452
		if (IS_ERR(u.xcrs))
			return PTR_ERR(u.xcrs);
3453

3454
		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
3455 3456
		break;
	}
3457 3458 3459 3460 3461 3462 3463 3464 3465
	case KVM_SET_TSC_KHZ: {
		u32 user_tsc_khz;

		r = -EINVAL;
		user_tsc_khz = (u32)arg;

		if (user_tsc_khz >= kvm_max_guest_tsc_khz)
			goto out;

3466 3467 3468 3469
		if (user_tsc_khz == 0)
			user_tsc_khz = tsc_khz;

		kvm_set_tsc_khz(vcpu, user_tsc_khz);
3470 3471 3472 3473 3474

		r = 0;
		goto out;
	}
	case KVM_GET_TSC_KHZ: {
3475
		r = vcpu->arch.virtual_tsc_khz;
3476 3477
		goto out;
	}
3478 3479 3480 3481
	case KVM_KVMCLOCK_CTRL: {
		r = kvm_set_guest_paused(vcpu);
		goto out;
	}
3482 3483 3484 3485
	default:
		r = -EINVAL;
	}
out:
3486
	kfree(u.buffer);
3487 3488 3489
	return r;
}

3490 3491 3492 3493 3494
int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
{
	return VM_FAULT_SIGBUS;
}

3495 3496 3497 3498 3499
static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
{
	int ret;

	if (addr > (unsigned int)(-3 * PAGE_SIZE))
3500
		return -EINVAL;
3501 3502 3503 3504
	ret = kvm_x86_ops->set_tss_addr(kvm, addr);
	return ret;
}

3505 3506 3507 3508 3509 3510 3511
static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
					      u64 ident_addr)
{
	kvm->arch.ept_identity_map_addr = ident_addr;
	return 0;
}

3512 3513 3514 3515 3516 3517
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
					  u32 kvm_nr_mmu_pages)
{
	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
		return -EINVAL;

3518
	mutex_lock(&kvm->slots_lock);
3519 3520

	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
3521
	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
3522

3523
	mutex_unlock(&kvm->slots_lock);
3524 3525 3526 3527 3528
	return 0;
}

static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
{
3529
	return kvm->arch.n_max_mmu_pages;
3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548
}

static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
		memcpy(&chip->chip.pic,
			&pic_irqchip(kvm)->pics[0],
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
		memcpy(&chip->chip.pic,
			&pic_irqchip(kvm)->pics[1],
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_IOAPIC:
G
Gleb Natapov 已提交
3549
		r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564
		break;
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
3565
		spin_lock(&pic_irqchip(kvm)->lock);
3566 3567 3568
		memcpy(&pic_irqchip(kvm)->pics[0],
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
3569
		spin_unlock(&pic_irqchip(kvm)->lock);
3570 3571
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
3572
		spin_lock(&pic_irqchip(kvm)->lock);
3573 3574 3575
		memcpy(&pic_irqchip(kvm)->pics[1],
			&chip->chip.pic,
			sizeof(struct kvm_pic_state));
3576
		spin_unlock(&pic_irqchip(kvm)->lock);
3577 3578
		break;
	case KVM_IRQCHIP_IOAPIC:
G
Gleb Natapov 已提交
3579
		r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
3580 3581 3582 3583 3584 3585 3586 3587 3588
		break;
	default:
		r = -EINVAL;
		break;
	}
	kvm_pic_update_irq(pic_irqchip(kvm));
	return r;
}

3589 3590 3591 3592
static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
	int r = 0;

3593
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
3594
	memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
3595
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3596 3597 3598 3599 3600 3601 3602
	return r;
}

static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
	int r = 0;

3603
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
3604
	memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
B
Beth Kon 已提交
3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618
	kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
	return r;
}

static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	int r = 0;

	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
		sizeof(ps->channels));
	ps->flags = kvm->arch.vpit->pit_state.flags;
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3619
	memset(&ps->reserved, 0, sizeof(ps->reserved));
B
Beth Kon 已提交
3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635
	return r;
}

static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	int r = 0, start = 0;
	u32 prev_legacy, cur_legacy;
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
	if (!prev_legacy && cur_legacy)
		start = 1;
	memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
	       sizeof(kvm->arch.vpit->pit_state.channels));
	kvm->arch.vpit->pit_state.flags = ps->flags;
	kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
3636
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3637 3638 3639
	return r;
}

3640 3641 3642 3643 3644
static int kvm_vm_ioctl_reinject(struct kvm *kvm,
				 struct kvm_reinject_control *control)
{
	if (!kvm->arch.vpit)
		return -ENXIO;
3645
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
3646
	kvm->arch.vpit->pit_state.reinject = control->pit_reinject;
3647
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3648 3649 3650
	return 0;
}

3651
/**
3652 3653 3654
 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
 * @kvm: kvm instance
 * @log: slot id and address to which we copy the log
3655
 *
3656 3657 3658
 * We need to keep it in mind that VCPU threads can write to the bitmap
 * concurrently.  So, to avoid losing data, we keep the following order for
 * each bit:
3659
 *
3660 3661 3662 3663
 *   1. Take a snapshot of the bit and clear it if needed.
 *   2. Write protect the corresponding page.
 *   3. Flush TLB's if needed.
 *   4. Copy the snapshot to the userspace.
3664
 *
3665 3666 3667 3668
 * Between 2 and 3, the guest may write to the page using the remaining TLB
 * entry.  This is not a problem because the page will be reported dirty at
 * step 4 using the snapshot taken before and step 3 ensures that successive
 * writes will be logged for the next call.
3669
 */
3670
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
3671
{
3672
	int r;
3673
	struct kvm_memory_slot *memslot;
3674 3675 3676 3677
	unsigned long n, i;
	unsigned long *dirty_bitmap;
	unsigned long *dirty_bitmap_buffer;
	bool is_dirty = false;
3678

3679
	mutex_lock(&kvm->slots_lock);
3680

M
Marcelo Tosatti 已提交
3681
	r = -EINVAL;
3682
	if (log->slot >= KVM_USER_MEM_SLOTS)
M
Marcelo Tosatti 已提交
3683 3684
		goto out;

3685
	memslot = id_to_memslot(kvm->memslots, log->slot);
3686 3687

	dirty_bitmap = memslot->dirty_bitmap;
M
Marcelo Tosatti 已提交
3688
	r = -ENOENT;
3689
	if (!dirty_bitmap)
M
Marcelo Tosatti 已提交
3690 3691
		goto out;

3692
	n = kvm_dirty_bitmap_bytes(memslot);
M
Marcelo Tosatti 已提交
3693

3694 3695
	dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
	memset(dirty_bitmap_buffer, 0, n);
M
Marcelo Tosatti 已提交
3696

3697
	spin_lock(&kvm->mmu_lock);
M
Marcelo Tosatti 已提交
3698

3699 3700 3701
	for (i = 0; i < n / sizeof(long); i++) {
		unsigned long mask;
		gfn_t offset;
3702

3703 3704
		if (!dirty_bitmap[i])
			continue;
M
Marcelo Tosatti 已提交
3705

3706
		is_dirty = true;
3707

3708 3709
		mask = xchg(&dirty_bitmap[i], 0);
		dirty_bitmap_buffer[i] = mask;
3710

3711 3712
		offset = i * BITS_PER_LONG;
		kvm_mmu_write_protect_pt_masked(kvm, memslot, offset, mask);
3713
	}
3714 3715 3716

	spin_unlock(&kvm->mmu_lock);

3717 3718 3719 3720 3721 3722 3723 3724 3725 3726
	/* See the comments in kvm_mmu_slot_remove_write_access(). */
	lockdep_assert_held(&kvm->slots_lock);

	/*
	 * All the TLBs can be flushed out of mmu lock, see the comments in
	 * kvm_mmu_slot_remove_write_access().
	 */
	if (is_dirty)
		kvm_flush_remote_tlbs(kvm);

3727 3728 3729
	r = -EFAULT;
	if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
		goto out;
M
Marcelo Tosatti 已提交
3730

3731 3732
	r = 0;
out:
3733
	mutex_unlock(&kvm->slots_lock);
3734 3735 3736
	return r;
}

3737 3738
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
			bool line_status)
3739 3740 3741 3742 3743
{
	if (!irqchip_in_kernel(kvm))
		return -ENXIO;

	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
3744 3745
					irq_event->irq, irq_event->level,
					line_status);
3746 3747 3748
	return 0;
}

3749 3750 3751 3752 3753
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
3754
	int r = -ENOTTY;
3755 3756 3757 3758 3759 3760 3761
	/*
	 * This union makes it completely explicit to gcc-3.x
	 * that these two variables' stack usage should be
	 * combined, not added together.
	 */
	union {
		struct kvm_pit_state ps;
B
Beth Kon 已提交
3762
		struct kvm_pit_state2 ps2;
3763
		struct kvm_pit_config pit_config;
3764
	} u;
3765 3766 3767 3768 3769

	switch (ioctl) {
	case KVM_SET_TSS_ADDR:
		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
		break;
3770 3771 3772 3773 3774 3775 3776 3777 3778
	case KVM_SET_IDENTITY_MAP_ADDR: {
		u64 ident_addr;

		r = -EFAULT;
		if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
			goto out;
		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
		break;
	}
3779 3780 3781 3782 3783 3784
	case KVM_SET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
		break;
	case KVM_GET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
		break;
3785 3786 3787 3788 3789 3790 3791
	case KVM_CREATE_IRQCHIP: {
		struct kvm_pic *vpic;

		mutex_lock(&kvm->lock);
		r = -EEXIST;
		if (kvm->arch.vpic)
			goto create_irqchip_unlock;
3792 3793 3794
		r = -EINVAL;
		if (atomic_read(&kvm->online_vcpus))
			goto create_irqchip_unlock;
3795
		r = -ENOMEM;
3796 3797
		vpic = kvm_create_pic(kvm);
		if (vpic) {
3798 3799
			r = kvm_ioapic_init(kvm);
			if (r) {
3800
				mutex_lock(&kvm->slots_lock);
3801
				kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
3802 3803 3804 3805 3806
							  &vpic->dev_master);
				kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
							  &vpic->dev_slave);
				kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
							  &vpic->dev_eclr);
3807
				mutex_unlock(&kvm->slots_lock);
3808 3809
				kfree(vpic);
				goto create_irqchip_unlock;
3810 3811
			}
		} else
3812 3813 3814 3815
			goto create_irqchip_unlock;
		smp_wmb();
		kvm->arch.vpic = vpic;
		smp_wmb();
3816 3817
		r = kvm_setup_default_irq_routing(kvm);
		if (r) {
3818
			mutex_lock(&kvm->slots_lock);
3819
			mutex_lock(&kvm->irq_lock);
3820 3821
			kvm_ioapic_destroy(kvm);
			kvm_destroy_pic(kvm);
3822
			mutex_unlock(&kvm->irq_lock);
3823
			mutex_unlock(&kvm->slots_lock);
3824
		}
3825 3826
	create_irqchip_unlock:
		mutex_unlock(&kvm->lock);
3827
		break;
3828
	}
S
Sheng Yang 已提交
3829
	case KVM_CREATE_PIT:
3830 3831 3832 3833 3834 3835 3836 3837
		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
		goto create_pit;
	case KVM_CREATE_PIT2:
		r = -EFAULT;
		if (copy_from_user(&u.pit_config, argp,
				   sizeof(struct kvm_pit_config)))
			goto out;
	create_pit:
3838
		mutex_lock(&kvm->slots_lock);
A
Avi Kivity 已提交
3839 3840 3841
		r = -EEXIST;
		if (kvm->arch.vpit)
			goto create_pit_unlock;
S
Sheng Yang 已提交
3842
		r = -ENOMEM;
3843
		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
S
Sheng Yang 已提交
3844 3845
		if (kvm->arch.vpit)
			r = 0;
A
Avi Kivity 已提交
3846
	create_pit_unlock:
3847
		mutex_unlock(&kvm->slots_lock);
S
Sheng Yang 已提交
3848
		break;
3849 3850
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3851
		struct kvm_irqchip *chip;
3852

3853 3854 3855
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
3856
			goto out;
3857 3858
		}

3859 3860
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
3861 3862
			goto get_irqchip_out;
		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
3863
		if (r)
3864
			goto get_irqchip_out;
3865
		r = -EFAULT;
3866 3867
		if (copy_to_user(argp, chip, sizeof *chip))
			goto get_irqchip_out;
3868
		r = 0;
3869 3870
	get_irqchip_out:
		kfree(chip);
3871 3872 3873 3874
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3875
		struct kvm_irqchip *chip;
3876

3877 3878 3879
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
3880
			goto out;
3881 3882
		}

3883 3884
		r = -ENXIO;
		if (!irqchip_in_kernel(kvm))
3885 3886
			goto set_irqchip_out;
		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
3887
		if (r)
3888
			goto set_irqchip_out;
3889
		r = 0;
3890 3891
	set_irqchip_out:
		kfree(chip);
3892 3893
		break;
	}
3894 3895
	case KVM_GET_PIT: {
		r = -EFAULT;
3896
		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
3897 3898 3899 3900
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
3901
		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
3902 3903 3904
		if (r)
			goto out;
		r = -EFAULT;
3905
		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
3906 3907 3908 3909 3910 3911
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT: {
		r = -EFAULT;
3912
		if (copy_from_user(&u.ps, argp, sizeof u.ps))
3913 3914 3915 3916
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
3917
		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
3918 3919
		break;
	}
B
Beth Kon 已提交
3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942
	case KVM_GET_PIT2: {
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT2: {
		r = -EFAULT;
		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
		break;
	}
3943 3944 3945 3946 3947 3948 3949 3950
	case KVM_REINJECT_CONTROL: {
		struct kvm_reinject_control control;
		r =  -EFAULT;
		if (copy_from_user(&control, argp, sizeof(control)))
			goto out;
		r = kvm_vm_ioctl_reinject(kvm, &control);
		break;
	}
E
Ed Swierk 已提交
3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961
	case KVM_XEN_HVM_CONFIG: {
		r = -EFAULT;
		if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
				   sizeof(struct kvm_xen_hvm_config)))
			goto out;
		r = -EINVAL;
		if (kvm->arch.xen_hvm_config.flags)
			goto out;
		r = 0;
		break;
	}
3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975
	case KVM_SET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;
		s64 delta;

		r = -EFAULT;
		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
			goto out;

		r = -EINVAL;
		if (user_ns.flags)
			goto out;

		r = 0;
3976
		local_irq_disable();
3977
		now_ns = get_kernel_ns();
3978
		delta = user_ns.clock - now_ns;
3979
		local_irq_enable();
3980
		kvm->arch.kvmclock_offset = delta;
3981
		kvm_gen_update_masterclock(kvm);
3982 3983 3984 3985 3986 3987
		break;
	}
	case KVM_GET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

3988
		local_irq_disable();
3989
		now_ns = get_kernel_ns();
3990
		user_ns.clock = kvm->arch.kvmclock_offset + now_ns;
3991
		local_irq_enable();
3992
		user_ns.flags = 0;
3993
		memset(&user_ns.pad, 0, sizeof(user_ns.pad));
3994 3995 3996 3997 3998 3999 4000 4001

		r = -EFAULT;
		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
			goto out;
		r = 0;
		break;
	}

4002 4003 4004 4005 4006 4007 4008
	default:
		;
	}
out:
	return r;
}

4009
static void kvm_init_msr_list(void)
4010 4011 4012 4013
{
	u32 dummy[2];
	unsigned i, j;

4014 4015
	/* skip the first msrs in the list. KVM-specific */
	for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) {
4016 4017
		if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
			continue;
4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034

		/*
		 * Even MSRs that are valid in the host may not be exposed
		 * to the guests in some cases.  We could work around this
		 * in VMX with the generic MSR save/load machinery, but it
		 * is not really worthwhile since it will really only
		 * happen with nested virtualization.
		 */
		switch (msrs_to_save[i]) {
		case MSR_IA32_BNDCFGS:
			if (!kvm_x86_ops->mpx_supported())
				continue;
			break;
		default:
			break;
		}

4035 4036 4037 4038 4039 4040 4041
		if (j < i)
			msrs_to_save[j] = msrs_to_save[i];
		j++;
	}
	num_msrs_to_save = j;
}

4042 4043
static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
			   const void *v)
4044
{
4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
		if (!(vcpu->arch.apic &&
		      !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
			break;
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
4059

4060
	return handled;
4061 4062
}

4063
static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
4064
{
4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
		if (!(vcpu->arch.apic &&
		      !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
			break;
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v);
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
4080

4081
	return handled;
4082 4083
}

4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095
static void kvm_set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
	kvm_x86_ops->set_segment(vcpu, var, seg);
}

void kvm_get_segment(struct kvm_vcpu *vcpu,
		     struct kvm_segment *var, int seg)
{
	kvm_x86_ops->get_segment(vcpu, var, seg);
}

4096 4097
gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
			   struct x86_exception *exception)
4098 4099 4100 4101 4102 4103 4104
{
	gpa_t t_gpa;

	BUG_ON(!mmu_is_nested(vcpu));

	/* NPT walks are always user-walks */
	access |= PFERR_USER_MASK;
4105
	t_gpa  = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, exception);
4106 4107 4108 4109

	return t_gpa;
}

4110 4111
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
			      struct x86_exception *exception)
4112 4113
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
4114
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
4115 4116
}

4117 4118
 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
4119 4120 4121
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_FETCH_MASK;
4122
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
4123 4124
}

4125 4126
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
			       struct x86_exception *exception)
4127 4128 4129
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_WRITE_MASK;
4130
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
4131 4132 4133
}

/* uses this to access any guest's mapped memory without checking CPL */
4134 4135
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
4136
{
4137
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
4138 4139 4140 4141
}

static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
4142
				      struct x86_exception *exception)
4143 4144
{
	void *data = val;
4145
	int r = X86EMUL_CONTINUE;
4146 4147

	while (bytes) {
4148
		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
4149
							    exception);
4150
		unsigned offset = addr & (PAGE_SIZE-1);
4151
		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
4152 4153
		int ret;

4154
		if (gpa == UNMAPPED_GVA)
4155
			return X86EMUL_PROPAGATE_FAULT;
4156 4157
		ret = kvm_read_guest_page(vcpu->kvm, gpa >> PAGE_SHIFT, data,
					  offset, toread);
4158
		if (ret < 0) {
4159
			r = X86EMUL_IO_NEEDED;
4160 4161
			goto out;
		}
4162

4163 4164 4165
		bytes -= toread;
		data += toread;
		addr += toread;
4166
	}
4167 4168
out:
	return r;
4169
}
4170

4171
/* used for instruction fetching */
4172 4173
static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
				gva_t addr, void *val, unsigned int bytes,
4174
				struct x86_exception *exception)
4175
{
4176
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4177
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
4178 4179
	unsigned offset;
	int ret;
4180

4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195
	/* Inline kvm_read_guest_virt_helper for speed.  */
	gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,
						    exception);
	if (unlikely(gpa == UNMAPPED_GVA))
		return X86EMUL_PROPAGATE_FAULT;

	offset = addr & (PAGE_SIZE-1);
	if (WARN_ON(offset + bytes > PAGE_SIZE))
		bytes = (unsigned)PAGE_SIZE - offset;
	ret = kvm_read_guest_page(vcpu->kvm, gpa >> PAGE_SHIFT, val,
				  offset, bytes);
	if (unlikely(ret < 0))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
4196 4197
}

4198
int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
4199
			       gva_t addr, void *val, unsigned int bytes,
4200
			       struct x86_exception *exception)
4201
{
4202
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4203
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
4204

4205
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
4206
					  exception);
4207
}
4208
EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
4209

4210 4211
static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt,
				      gva_t addr, void *val, unsigned int bytes,
4212
				      struct x86_exception *exception)
4213
{
4214
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4215
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception);
4216 4217
}

N
Nadav Har'El 已提交
4218
int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
4219
				       gva_t addr, void *val,
4220
				       unsigned int bytes,
4221
				       struct x86_exception *exception)
4222
{
4223
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4224 4225 4226 4227
	void *data = val;
	int r = X86EMUL_CONTINUE;

	while (bytes) {
4228 4229
		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
							     PFERR_WRITE_MASK,
4230
							     exception);
4231 4232 4233 4234
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
		int ret;

4235
		if (gpa == UNMAPPED_GVA)
4236
			return X86EMUL_PROPAGATE_FAULT;
4237 4238
		ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
		if (ret < 0) {
4239
			r = X86EMUL_IO_NEEDED;
4240 4241 4242 4243 4244 4245 4246 4247 4248 4249
			goto out;
		}

		bytes -= towrite;
		data += towrite;
		addr += towrite;
	}
out:
	return r;
}
N
Nadav Har'El 已提交
4250
EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
4251

4252 4253 4254 4255
static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
				gpa_t *gpa, struct x86_exception *exception,
				bool write)
{
4256 4257
	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
		| (write ? PFERR_WRITE_MASK : 0);
4258

4259
	if (vcpu_match_mmio_gva(vcpu, gva)
F
Feng Wu 已提交
4260 4261
	    && !permission_fault(vcpu, vcpu->arch.walk_mmu,
				 vcpu->arch.access, access)) {
4262 4263
		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
					(gva & (PAGE_SIZE - 1));
X
Xiao Guangrong 已提交
4264
		trace_vcpu_match_mmio(gva, *gpa, write, false);
4265 4266 4267
		return 1;
	}

4268 4269 4270 4271 4272 4273 4274 4275 4276
	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);

	if (*gpa == UNMAPPED_GVA)
		return -1;

	/* For APIC access vmexit */
	if ((*gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		return 1;

X
Xiao Guangrong 已提交
4277 4278
	if (vcpu_match_mmio_gpa(vcpu, *gpa)) {
		trace_vcpu_match_mmio(gva, *gpa, write, true);
4279
		return 1;
X
Xiao Guangrong 已提交
4280
	}
4281

4282 4283 4284
	return 0;
}

4285
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
4286
			const void *val, int bytes)
4287 4288 4289 4290
{
	int ret;

	ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
4291
	if (ret < 0)
4292
		return 0;
4293
	kvm_mmu_pte_write(vcpu, gpa, val, bytes);
4294 4295 4296
	return 1;
}

4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312
struct read_write_emulator_ops {
	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
				  int bytes);
	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
				  void *val, int bytes);
	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
			       int bytes, void *val);
	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
				    void *val, int bytes);
	bool write;
};

static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
{
	if (vcpu->mmio_read_completed) {
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
A
Avi Kivity 已提交
4313
			       vcpu->mmio_fragments[0].gpa, *(u64 *)val);
4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348
		vcpu->mmio_read_completed = 0;
		return 1;
	}

	return 0;
}

static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			void *val, int bytes)
{
	return !kvm_read_guest(vcpu->kvm, gpa, val, bytes);
}

static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			 void *val, int bytes)
{
	return emulator_write_phys(vcpu, gpa, val, bytes);
}

static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
{
	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
	return vcpu_mmio_write(vcpu, gpa, bytes, val);
}

static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			  void *val, int bytes)
{
	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
	return X86EMUL_IO_NEEDED;
}

static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			   void *val, int bytes)
{
A
Avi Kivity 已提交
4349 4350
	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];

4351
	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
4352 4353 4354
	return X86EMUL_CONTINUE;
}

4355
static const struct read_write_emulator_ops read_emultor = {
4356 4357 4358 4359 4360 4361
	.read_write_prepare = read_prepare,
	.read_write_emulate = read_emulate,
	.read_write_mmio = vcpu_mmio_read,
	.read_write_exit_mmio = read_exit_mmio,
};

4362
static const struct read_write_emulator_ops write_emultor = {
4363 4364 4365 4366 4367 4368
	.read_write_emulate = write_emulate,
	.read_write_mmio = write_mmio,
	.read_write_exit_mmio = write_exit_mmio,
	.write = true,
};

4369 4370 4371 4372
static int emulator_read_write_onepage(unsigned long addr, void *val,
				       unsigned int bytes,
				       struct x86_exception *exception,
				       struct kvm_vcpu *vcpu,
4373
				       const struct read_write_emulator_ops *ops)
4374
{
4375 4376
	gpa_t gpa;
	int handled, ret;
4377
	bool write = ops->write;
A
Avi Kivity 已提交
4378
	struct kvm_mmio_fragment *frag;
4379

4380
	ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
4381

4382
	if (ret < 0)
4383 4384 4385
		return X86EMUL_PROPAGATE_FAULT;

	/* For APIC access vmexit */
4386
	if (ret)
4387 4388
		goto mmio;

4389
	if (ops->read_write_emulate(vcpu, gpa, val, bytes))
4390 4391 4392 4393 4394 4395
		return X86EMUL_CONTINUE;

mmio:
	/*
	 * Is this MMIO handled locally?
	 */
4396
	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
4397
	if (handled == bytes)
4398 4399
		return X86EMUL_CONTINUE;

4400 4401 4402 4403
	gpa += handled;
	bytes -= handled;
	val += handled;

4404 4405 4406 4407 4408
	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
	frag->gpa = gpa;
	frag->data = val;
	frag->len = bytes;
A
Avi Kivity 已提交
4409
	return X86EMUL_CONTINUE;
4410 4411
}

4412 4413 4414
int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr,
			void *val, unsigned int bytes,
			struct x86_exception *exception,
4415
			const struct read_write_emulator_ops *ops)
4416
{
4417
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
A
Avi Kivity 已提交
4418 4419 4420 4421 4422 4423 4424 4425
	gpa_t gpa;
	int rc;

	if (ops->read_write_prepare &&
		  ops->read_write_prepare(vcpu, val, bytes))
		return X86EMUL_CONTINUE;

	vcpu->mmio_nr_fragments = 0;
4426

4427 4428
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
A
Avi Kivity 已提交
4429
		int now;
4430 4431

		now = -addr & ~PAGE_MASK;
4432 4433 4434
		rc = emulator_read_write_onepage(addr, val, now, exception,
						 vcpu, ops);

4435 4436 4437 4438 4439 4440
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
		val += now;
		bytes -= now;
	}
4441

A
Avi Kivity 已提交
4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454
	rc = emulator_read_write_onepage(addr, val, bytes, exception,
					 vcpu, ops);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	if (!vcpu->mmio_nr_fragments)
		return rc;

	gpa = vcpu->mmio_fragments[0].gpa;

	vcpu->mmio_needed = 1;
	vcpu->mmio_cur_fragment = 0;

4455
	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
A
Avi Kivity 已提交
4456 4457 4458 4459 4460
	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
	vcpu->run->exit_reason = KVM_EXIT_MMIO;
	vcpu->run->mmio.phys_addr = gpa;

	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480
}

static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
				  unsigned long addr,
				  void *val,
				  unsigned int bytes,
				  struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, val, bytes,
				   exception, &read_emultor);
}

int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
			    unsigned long addr,
			    const void *val,
			    unsigned int bytes,
			    struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, (void *)val, bytes,
				   exception, &write_emultor);
4481 4482
}

4483 4484 4485 4486 4487 4488 4489
#define CMPXCHG_TYPE(t, ptr, old, new) \
	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))

#ifdef CONFIG_X86_64
#  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
#else
#  define CMPXCHG64(ptr, old, new) \
4490
	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
4491 4492
#endif

4493 4494
static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
				     unsigned long addr,
4495 4496 4497
				     const void *old,
				     const void *new,
				     unsigned int bytes,
4498
				     struct x86_exception *exception)
4499
{
4500
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4501 4502 4503 4504
	gpa_t gpa;
	struct page *page;
	char *kaddr;
	bool exchanged;
4505

4506 4507 4508
	/* guests cmpxchg8b have to be emulated atomically */
	if (bytes > 8 || (bytes & (bytes - 1)))
		goto emul_write;
4509

4510
	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
4511

4512 4513 4514
	if (gpa == UNMAPPED_GVA ||
	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		goto emul_write;
4515

4516 4517
	if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
		goto emul_write;
4518

4519
	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
4520
	if (is_error_page(page))
4521
		goto emul_write;
4522

4523
	kaddr = kmap_atomic(page);
4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539
	kaddr += offset_in_page(gpa);
	switch (bytes) {
	case 1:
		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
		break;
	case 2:
		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
		break;
	case 4:
		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
		break;
	case 8:
		exchanged = CMPXCHG64(kaddr, old, new);
		break;
	default:
		BUG();
4540
	}
4541
	kunmap_atomic(kaddr);
4542 4543 4544 4545 4546
	kvm_release_page_dirty(page);

	if (!exchanged)
		return X86EMUL_CMPXCHG_FAILED;

4547
	mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
4548
	kvm_mmu_pte_write(vcpu, gpa, new, bytes);
4549 4550

	return X86EMUL_CONTINUE;
4551

4552
emul_write:
4553
	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
4554

4555
	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
4556 4557
}

4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572
static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
{
	/* TODO: String I/O for in kernel device */
	int r;

	if (vcpu->arch.pio.in)
		r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
				    vcpu->arch.pio.size, pd);
	else
		r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
				     vcpu->arch.pio.port, vcpu->arch.pio.size,
				     pd);
	return r;
}

4573 4574 4575
static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
			       unsigned short port, void *val,
			       unsigned int count, bool in)
4576 4577
{
	vcpu->arch.pio.port = port;
4578
	vcpu->arch.pio.in = in;
4579
	vcpu->arch.pio.count  = count;
4580 4581 4582
	vcpu->arch.pio.size = size;

	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
4583
		vcpu->arch.pio.count = 0;
4584 4585 4586 4587
		return 1;
	}

	vcpu->run->exit_reason = KVM_EXIT_IO;
4588
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
4589 4590 4591 4592 4593 4594 4595 4596
	vcpu->run->io.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = count;
	vcpu->run->io.port = port;

	return 0;
}

4597 4598 4599
static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
				    int size, unsigned short port, void *val,
				    unsigned int count)
4600
{
4601
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4602
	int ret;
4603

4604 4605
	if (vcpu->arch.pio.count)
		goto data_avail;
4606

4607 4608 4609 4610
	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
	if (ret) {
data_avail:
		memcpy(val, vcpu->arch.pio_data, size * count);
4611
		trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
4612
		vcpu->arch.pio.count = 0;
4613 4614 4615 4616 4617 4618
		return 1;
	}

	return 0;
}

4619 4620 4621 4622 4623 4624 4625
static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
				     int size, unsigned short port,
				     const void *val, unsigned int count)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	memcpy(vcpu->arch.pio_data, val, size * count);
4626
	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
4627 4628 4629
	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
}

4630 4631 4632 4633 4634
static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	return kvm_x86_ops->get_segment_base(vcpu, seg);
}

4635
static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
4636
{
4637
	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
4638 4639
}

4640 4641 4642 4643 4644 4645
int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
	if (!need_emulate_wbinvd(vcpu))
		return X86EMUL_CONTINUE;

	if (kvm_x86_ops->has_wbinvd_exit()) {
4646 4647 4648
		int cpu = get_cpu();

		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
4649 4650
		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
				wbinvd_ipi, NULL, 1);
4651
		put_cpu();
4652
		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
4653 4654
	} else
		wbinvd();
4655 4656 4657 4658
	return X86EMUL_CONTINUE;
}
EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);

4659 4660 4661 4662 4663
static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
{
	kvm_emulate_wbinvd(emul_to_vcpu(ctxt));
}

4664
int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
4665
{
4666
	return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
4667 4668
}

4669
int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
4670
{
4671

4672
	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
4673 4674
}

4675
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
4676
{
4677
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
4678 4679
}

4680
static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
4681
{
4682
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4683 4684 4685 4686 4687 4688 4689 4690 4691 4692
	unsigned long value;

	switch (cr) {
	case 0:
		value = kvm_read_cr0(vcpu);
		break;
	case 2:
		value = vcpu->arch.cr2;
		break;
	case 3:
4693
		value = kvm_read_cr3(vcpu);
4694 4695 4696 4697 4698 4699 4700 4701
		break;
	case 4:
		value = kvm_read_cr4(vcpu);
		break;
	case 8:
		value = kvm_get_cr8(vcpu);
		break;
	default:
4702
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
4703 4704 4705 4706 4707 4708
		return 0;
	}

	return value;
}

4709
static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
4710
{
4711
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4712 4713
	int res = 0;

4714 4715
	switch (cr) {
	case 0:
4716
		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
4717 4718 4719 4720 4721
		break;
	case 2:
		vcpu->arch.cr2 = val;
		break;
	case 3:
4722
		res = kvm_set_cr3(vcpu, val);
4723 4724
		break;
	case 4:
4725
		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
4726 4727
		break;
	case 8:
A
Andre Przywara 已提交
4728
		res = kvm_set_cr8(vcpu, val);
4729 4730
		break;
	default:
4731
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
4732
		res = -1;
4733
	}
4734 4735

	return res;
4736 4737
}

4738
static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
4739
{
4740
	return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
4741 4742
}

4743
static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
4744
{
4745
	kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
4746 4747
}

4748
static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
4749
{
4750
	kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
4751 4752
}

4753 4754 4755 4756 4757 4758 4759 4760 4761 4762
static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
}

static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
}

4763 4764
static unsigned long emulator_get_cached_segment_base(
	struct x86_emulate_ctxt *ctxt, int seg)
4765
{
4766
	return get_segment_base(emul_to_vcpu(ctxt), seg);
4767 4768
}

4769 4770 4771
static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
				 struct desc_struct *desc, u32 *base3,
				 int seg)
4772 4773 4774
{
	struct kvm_segment var;

4775
	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
4776
	*selector = var.selector;
4777

4778 4779
	if (var.unusable) {
		memset(desc, 0, sizeof(*desc));
4780
		return false;
4781
	}
4782 4783 4784 4785 4786

	if (var.g)
		var.limit >>= 12;
	set_desc_limit(desc, var.limit);
	set_desc_base(desc, (unsigned long)var.base);
4787 4788 4789 4790
#ifdef CONFIG_X86_64
	if (base3)
		*base3 = var.base >> 32;
#endif
4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802
	desc->type = var.type;
	desc->s = var.s;
	desc->dpl = var.dpl;
	desc->p = var.present;
	desc->avl = var.avl;
	desc->l = var.l;
	desc->d = var.db;
	desc->g = var.g;

	return true;
}

4803 4804 4805
static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
				 struct desc_struct *desc, u32 base3,
				 int seg)
4806
{
4807
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4808 4809
	struct kvm_segment var;

4810
	var.selector = selector;
4811
	var.base = get_desc_base(desc);
4812 4813 4814
#ifdef CONFIG_X86_64
	var.base |= ((u64)base3) << 32;
#endif
4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832
	var.limit = get_desc_limit(desc);
	if (desc->g)
		var.limit = (var.limit << 12) | 0xfff;
	var.type = desc->type;
	var.dpl = desc->dpl;
	var.db = desc->d;
	var.s = desc->s;
	var.l = desc->l;
	var.g = desc->g;
	var.avl = desc->avl;
	var.present = desc->p;
	var.unusable = !var.present;
	var.padding = 0;

	kvm_set_segment(vcpu, &var, seg);
	return;
}

4833 4834 4835 4836 4837 4838 4839 4840 4841
static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 *pdata)
{
	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
}

static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 data)
{
4842 4843 4844 4845 4846 4847
	struct msr_data msr;

	msr.data = data;
	msr.index = msr_index;
	msr.host_initiated = false;
	return kvm_set_msr(emul_to_vcpu(ctxt), &msr);
4848 4849
}

4850 4851 4852 4853 4854 4855
static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
			      u32 pmc)
{
	return kvm_pmu_check_pmc(emul_to_vcpu(ctxt), pmc);
}

4856 4857 4858 4859 4860 4861
static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
			     u32 pmc, u64 *pdata)
{
	return kvm_pmu_read_pmc(emul_to_vcpu(ctxt), pmc, pdata);
}

4862 4863 4864 4865 4866
static void emulator_halt(struct x86_emulate_ctxt *ctxt)
{
	emul_to_vcpu(ctxt)->arch.halt_request = 1;
}

4867 4868 4869
static void emulator_get_fpu(struct x86_emulate_ctxt *ctxt)
{
	preempt_disable();
4870
	kvm_load_guest_fpu(emul_to_vcpu(ctxt));
4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882
	/*
	 * CR0.TS may reference the host fpu state, not the guest fpu state,
	 * so it may be clear at this point.
	 */
	clts();
}

static void emulator_put_fpu(struct x86_emulate_ctxt *ctxt)
{
	preempt_enable();
}

4883
static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
4884
			      struct x86_instruction_info *info,
4885 4886
			      enum x86_intercept_stage stage)
{
4887
	return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
4888 4889
}

4890
static void emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
4891 4892
			       u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
{
4893
	kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx);
4894 4895
}

4896 4897 4898 4899 4900 4901 4902 4903 4904 4905
static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
{
	return kvm_register_read(emul_to_vcpu(ctxt), reg);
}

static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
{
	kvm_register_write(emul_to_vcpu(ctxt), reg, val);
}

4906
static const struct x86_emulate_ops emulate_ops = {
4907 4908
	.read_gpr            = emulator_read_gpr,
	.write_gpr           = emulator_write_gpr,
4909
	.read_std            = kvm_read_guest_virt_system,
4910
	.write_std           = kvm_write_guest_virt_system,
4911
	.fetch               = kvm_fetch_guest_virt,
4912 4913 4914
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
4915
	.invlpg              = emulator_invlpg,
4916 4917
	.pio_in_emulated     = emulator_pio_in_emulated,
	.pio_out_emulated    = emulator_pio_out_emulated,
4918 4919
	.get_segment         = emulator_get_segment,
	.set_segment         = emulator_set_segment,
4920
	.get_cached_segment_base = emulator_get_cached_segment_base,
4921
	.get_gdt             = emulator_get_gdt,
4922
	.get_idt	     = emulator_get_idt,
4923 4924
	.set_gdt             = emulator_set_gdt,
	.set_idt	     = emulator_set_idt,
4925 4926
	.get_cr              = emulator_get_cr,
	.set_cr              = emulator_set_cr,
4927
	.cpl                 = emulator_get_cpl,
4928 4929
	.get_dr              = emulator_get_dr,
	.set_dr              = emulator_set_dr,
4930 4931
	.set_msr             = emulator_set_msr,
	.get_msr             = emulator_get_msr,
4932
	.check_pmc	     = emulator_check_pmc,
4933
	.read_pmc            = emulator_read_pmc,
4934
	.halt                = emulator_halt,
4935
	.wbinvd              = emulator_wbinvd,
4936
	.fix_hypercall       = emulator_fix_hypercall,
4937 4938
	.get_fpu             = emulator_get_fpu,
	.put_fpu             = emulator_put_fpu,
4939
	.intercept           = emulator_intercept,
4940
	.get_cpuid           = emulator_get_cpuid,
4941 4942
};

4943 4944
static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
{
4945
	u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
4946 4947 4948 4949 4950 4951 4952
	/*
	 * an sti; sti; sequence only disable interrupts for the first
	 * instruction. So, if the last instruction, be it emulated or
	 * not, left the system with the INT_STI flag enabled, it
	 * means that the last instruction is an sti. We should not
	 * leave the flag on in this case. The same goes for mov ss
	 */
4953 4954
	if (int_shadow & mask)
		mask = 0;
4955
	if (unlikely(int_shadow || mask)) {
4956
		kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
4957 4958 4959
		if (!mask)
			kvm_make_request(KVM_REQ_EVENT, vcpu);
	}
4960 4961
}

4962
static bool inject_emulated_exception(struct kvm_vcpu *vcpu)
4963 4964
{
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4965
	if (ctxt->exception.vector == PF_VECTOR)
4966 4967 4968
		return kvm_propagate_fault(vcpu, &ctxt->exception);

	if (ctxt->exception.error_code_valid)
4969 4970
		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
				      ctxt->exception.error_code);
4971
	else
4972
		kvm_queue_exception(vcpu, ctxt->exception.vector);
4973
	return false;
4974 4975
}

4976 4977
static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
{
4978
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4979 4980 4981 4982
	int cs_db, cs_l;

	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);

4983 4984 4985 4986
	ctxt->eflags = kvm_get_rflags(vcpu);
	ctxt->eip = kvm_rip_read(vcpu);
	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
4987
		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :
4988 4989 4990 4991
		     cs_db				? X86EMUL_MODE_PROT32 :
							  X86EMUL_MODE_PROT16;
	ctxt->guest_mode = is_guest_mode(vcpu);

4992
	init_decode_cache(ctxt);
4993
	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
4994 4995
}

4996
int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
4997
{
4998
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4999 5000 5001 5002
	int ret;

	init_emulate_ctxt(vcpu);

5003 5004 5005
	ctxt->op_bytes = 2;
	ctxt->ad_bytes = 2;
	ctxt->_eip = ctxt->eip + inc_eip;
5006
	ret = emulate_int_real(ctxt, irq);
5007 5008 5009 5010

	if (ret != X86EMUL_CONTINUE)
		return EMULATE_FAIL;

5011
	ctxt->eip = ctxt->_eip;
5012 5013
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
5014 5015

	if (irq == NMI_VECTOR)
A
Avi Kivity 已提交
5016
		vcpu->arch.nmi_pending = 0;
5017 5018 5019 5020 5021 5022 5023
	else
		vcpu->arch.interrupt.pending = false;

	return EMULATE_DONE;
}
EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);

5024 5025
static int handle_emulation_failure(struct kvm_vcpu *vcpu)
{
5026 5027
	int r = EMULATE_DONE;

5028 5029
	++vcpu->stat.insn_emulation_fail;
	trace_kvm_emulate_insn_failed(vcpu);
5030
	if (!is_guest_mode(vcpu) && kvm_x86_ops->get_cpl(vcpu) == 0) {
5031 5032 5033 5034 5035
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
		r = EMULATE_FAIL;
	}
5036
	kvm_queue_exception(vcpu, UD_VECTOR);
5037 5038

	return r;
5039 5040
}

5041
static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,
5042 5043
				  bool write_fault_to_shadow_pgtable,
				  int emulation_type)
5044
{
5045
	gpa_t gpa = cr2;
5046
	pfn_t pfn;
5047

5048 5049 5050
	if (emulation_type & EMULTYPE_NO_REEXECUTE)
		return false;

5051 5052 5053 5054 5055 5056
	if (!vcpu->arch.mmu.direct_map) {
		/*
		 * Write permission should be allowed since only
		 * write access need to be emulated.
		 */
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
5057

5058 5059 5060 5061 5062 5063 5064
		/*
		 * If the mapping is invalid in guest, let cpu retry
		 * it to generate fault.
		 */
		if (gpa == UNMAPPED_GVA)
			return true;
	}
5065

5066 5067 5068 5069 5070 5071 5072
	/*
	 * Do not retry the unhandleable instruction if it faults on the
	 * readonly host memory, otherwise it will goto a infinite loop:
	 * retry instruction -> write #PF -> emulation fail -> retry
	 * instruction -> ...
	 */
	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093

	/*
	 * If the instruction failed on the error pfn, it can not be fixed,
	 * report the error to userspace.
	 */
	if (is_error_noslot_pfn(pfn))
		return false;

	kvm_release_pfn_clean(pfn);

	/* The instructions are well-emulated on direct mmu. */
	if (vcpu->arch.mmu.direct_map) {
		unsigned int indirect_shadow_pages;

		spin_lock(&vcpu->kvm->mmu_lock);
		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
		spin_unlock(&vcpu->kvm->mmu_lock);

		if (indirect_shadow_pages)
			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));

5094
		return true;
5095
	}
5096

5097 5098 5099 5100 5101 5102
	/*
	 * if emulation was due to access to shadowed page table
	 * and it failed try to unshadow page and re-enter the
	 * guest to let CPU execute the instruction.
	 */
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
5103 5104 5105 5106 5107 5108 5109

	/*
	 * If the access faults on its page table, it can not
	 * be fixed by unprotecting shadow page and it should
	 * be reported to userspace.
	 */
	return !write_fault_to_shadow_pgtable;
5110 5111
}

5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150
static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
			      unsigned long cr2,  int emulation_type)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	unsigned long last_retry_eip, last_retry_addr, gpa = cr2;

	last_retry_eip = vcpu->arch.last_retry_eip;
	last_retry_addr = vcpu->arch.last_retry_addr;

	/*
	 * If the emulation is caused by #PF and it is non-page_table
	 * writing instruction, it means the VM-EXIT is caused by shadow
	 * page protected, we can zap the shadow page and retry this
	 * instruction directly.
	 *
	 * Note: if the guest uses a non-page-table modifying instruction
	 * on the PDE that points to the instruction, then we will unmap
	 * the instruction and go to an infinite loop. So, we cache the
	 * last retried eip and the last fault address, if we meet the eip
	 * and the address again, we can break out of the potential infinite
	 * loop.
	 */
	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;

	if (!(emulation_type & EMULTYPE_RETRY))
		return false;

	if (x86_page_table_writing_insn(ctxt))
		return false;

	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2)
		return false;

	vcpu->arch.last_retry_eip = ctxt->eip;
	vcpu->arch.last_retry_addr = cr2;

	if (!vcpu->arch.mmu.direct_map)
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);

5151
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
5152 5153 5154 5155

	return true;
}

5156 5157 5158
static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
static int complete_emulated_pio(struct kvm_vcpu *vcpu);

5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173
static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
				unsigned long *db)
{
	u32 dr6 = 0;
	int i;
	u32 enable, rwlen;

	enable = dr7;
	rwlen = dr7 >> 16;
	for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
		if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
			dr6 |= (1 << i);
	return dr6;
}

5174
static void kvm_vcpu_check_singlestep(struct kvm_vcpu *vcpu, unsigned long rflags, int *r)
5175 5176 5177 5178
{
	struct kvm_run *kvm_run = vcpu->run;

	/*
5179 5180
	 * rflags is the old, "raw" value of the flags.  The new value has
	 * not been saved yet.
5181 5182 5183 5184 5185 5186 5187
	 *
	 * This is correct even for TF set by the guest, because "the
	 * processor will not generate this exception after the instruction
	 * that sets the TF flag".
	 */
	if (unlikely(rflags & X86_EFLAGS_TF)) {
		if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
5188 5189
			kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1 |
						  DR6_RTM;
5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201
			kvm_run->debug.arch.pc = vcpu->arch.singlestep_rip;
			kvm_run->debug.arch.exception = DB_VECTOR;
			kvm_run->exit_reason = KVM_EXIT_DEBUG;
			*r = EMULATE_USER_EXIT;
		} else {
			vcpu->arch.emulate_ctxt.eflags &= ~X86_EFLAGS_TF;
			/*
			 * "Certain debug exceptions may clear bit 0-3.  The
			 * remaining contents of the DR6 register are never
			 * cleared by the processor".
			 */
			vcpu->arch.dr6 &= ~15;
5202
			vcpu->arch.dr6 |= DR6_BS | DR6_RTM;
5203 5204 5205 5206 5207
			kvm_queue_exception(vcpu, DB_VECTOR);
		}
	}
}

5208 5209 5210 5211
static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
{
	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
5212 5213 5214
		struct kvm_run *kvm_run = vcpu->run;
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
5215 5216 5217 5218
					   vcpu->arch.guest_debug_dr7,
					   vcpu->arch.eff_db);

		if (dr6 != 0) {
5219
			kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
5220
			kvm_run->debug.arch.pc = eip;
5221 5222 5223 5224 5225 5226 5227
			kvm_run->debug.arch.exception = DB_VECTOR;
			kvm_run->exit_reason = KVM_EXIT_DEBUG;
			*r = EMULATE_USER_EXIT;
			return true;
		}
	}

5228 5229
	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
	    !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {
5230 5231
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
5232 5233 5234 5235 5236
					   vcpu->arch.dr7,
					   vcpu->arch.db);

		if (dr6 != 0) {
			vcpu->arch.dr6 &= ~15;
5237
			vcpu->arch.dr6 |= dr6 | DR6_RTM;
5238 5239 5240 5241 5242 5243 5244 5245 5246
			kvm_queue_exception(vcpu, DB_VECTOR);
			*r = EMULATE_DONE;
			return true;
		}
	}

	return false;
}

5247 5248
int x86_emulate_instruction(struct kvm_vcpu *vcpu,
			    unsigned long cr2,
5249 5250 5251
			    int emulation_type,
			    void *insn,
			    int insn_len)
5252
{
5253
	int r;
5254
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
5255
	bool writeback = true;
5256
	bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
5257

5258 5259 5260 5261 5262
	/*
	 * Clear write_fault_to_shadow_pgtable here to ensure it is
	 * never reused.
	 */
	vcpu->arch.write_fault_to_shadow_pgtable = false;
5263
	kvm_clear_exception_queue(vcpu);
G
Gleb Natapov 已提交
5264

5265
	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
5266
		init_emulate_ctxt(vcpu);
5267 5268 5269 5270 5271 5272 5273 5274 5275 5276

		/*
		 * We will reenter on the same instruction since
		 * we do not set complete_userspace_io.  This does not
		 * handle watchpoints yet, those would be handled in
		 * the emulate_ops.
		 */
		if (kvm_vcpu_check_breakpoint(vcpu, &r))
			return r;

5277 5278
		ctxt->interruptibility = 0;
		ctxt->have_exception = false;
5279
		ctxt->exception.vector = -1;
5280
		ctxt->perm_ok = false;
5281

5282
		ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
5283

5284
		r = x86_decode_insn(ctxt, insn, insn_len);
5285

A
Avi Kivity 已提交
5286
		trace_kvm_emulate_insn_start(vcpu);
5287
		++vcpu->stat.insn_emulation;
5288
		if (r != EMULATION_OK)  {
5289 5290
			if (emulation_type & EMULTYPE_TRAP_UD)
				return EMULATE_FAIL;
5291 5292
			if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
						emulation_type))
5293
				return EMULATE_DONE;
5294 5295 5296
			if (emulation_type & EMULTYPE_SKIP)
				return EMULATE_FAIL;
			return handle_emulation_failure(vcpu);
5297 5298 5299
		}
	}

5300
	if (emulation_type & EMULTYPE_SKIP) {
5301
		kvm_rip_write(vcpu, ctxt->_eip);
5302 5303
		if (ctxt->eflags & X86_EFLAGS_RF)
			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
5304 5305 5306
		return EMULATE_DONE;
	}

5307 5308 5309
	if (retry_instruction(ctxt, cr2, emulation_type))
		return EMULATE_DONE;

5310
	/* this is needed for vmware backdoor interface to work since it
5311
	   changes registers values  during IO operation */
5312 5313
	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
5314
		emulator_invalidate_register_cache(ctxt);
5315
	}
5316

5317
restart:
5318
	r = x86_emulate_insn(ctxt);
5319

5320 5321 5322
	if (r == EMULATION_INTERCEPTED)
		return EMULATE_DONE;

5323
	if (r == EMULATION_FAILED) {
5324 5325
		if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
					emulation_type))
5326 5327
			return EMULATE_DONE;

5328
		return handle_emulation_failure(vcpu);
5329 5330
	}

5331
	if (ctxt->have_exception) {
5332
		r = EMULATE_DONE;
5333 5334
		if (inject_emulated_exception(vcpu))
			return r;
5335
	} else if (vcpu->arch.pio.count) {
5336 5337
		if (!vcpu->arch.pio.in) {
			/* FIXME: return into emulator if single-stepping.  */
5338
			vcpu->arch.pio.count = 0;
5339
		} else {
5340
			writeback = false;
5341 5342
			vcpu->arch.complete_userspace_io = complete_emulated_pio;
		}
P
Paolo Bonzini 已提交
5343
		r = EMULATE_USER_EXIT;
5344 5345 5346
	} else if (vcpu->mmio_needed) {
		if (!vcpu->mmio_is_write)
			writeback = false;
P
Paolo Bonzini 已提交
5347
		r = EMULATE_USER_EXIT;
5348
		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
5349
	} else if (r == EMULATION_RESTART)
5350
		goto restart;
5351 5352
	else
		r = EMULATE_DONE;
5353

5354
	if (writeback) {
5355
		unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
5356
		toggle_interruptibility(vcpu, ctxt->interruptibility);
5357
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
5358
		kvm_rip_write(vcpu, ctxt->eip);
5359
		if (r == EMULATE_DONE)
5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370
			kvm_vcpu_check_singlestep(vcpu, rflags, &r);
		__kvm_set_rflags(vcpu, ctxt->eflags);

		/*
		 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
		 * do nothing, and it will be requested again as soon as
		 * the shadow expires.  But we still need to check here,
		 * because POPF has no interrupt shadow.
		 */
		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
			kvm_make_request(KVM_REQ_EVENT, vcpu);
5371 5372
	} else
		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
5373 5374

	return r;
5375
}
5376
EXPORT_SYMBOL_GPL(x86_emulate_instruction);
5377

5378
int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
5379
{
5380
	unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX);
5381 5382
	int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
					    size, port, &val, 1);
5383
	/* do not return to emulator after return from userspace */
5384
	vcpu->arch.pio.count = 0;
5385 5386
	return ret;
}
5387
EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
5388

5389 5390
static void tsc_bad(void *info)
{
T
Tejun Heo 已提交
5391
	__this_cpu_write(cpu_tsc_khz, 0);
5392 5393 5394
}

static void tsc_khz_changed(void *data)
5395
{
5396 5397 5398 5399 5400 5401 5402 5403 5404
	struct cpufreq_freqs *freq = data;
	unsigned long khz = 0;

	if (data)
		khz = freq->new;
	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		khz = cpufreq_quick_get(raw_smp_processor_id());
	if (!khz)
		khz = tsc_khz;
T
Tejun Heo 已提交
5405
	__this_cpu_write(cpu_tsc_khz, khz);
5406 5407 5408 5409 5410 5411 5412 5413 5414 5415
}

static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
				     void *data)
{
	struct cpufreq_freqs *freq = data;
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i, send_ipi = 0;

5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454
	/*
	 * We allow guests to temporarily run on slowing clocks,
	 * provided we notify them after, or to run on accelerating
	 * clocks, provided we notify them before.  Thus time never
	 * goes backwards.
	 *
	 * However, we have a problem.  We can't atomically update
	 * the frequency of a given CPU from this function; it is
	 * merely a notifier, which can be called from any CPU.
	 * Changing the TSC frequency at arbitrary points in time
	 * requires a recomputation of local variables related to
	 * the TSC for each VCPU.  We must flag these local variables
	 * to be updated and be sure the update takes place with the
	 * new frequency before any guests proceed.
	 *
	 * Unfortunately, the combination of hotplug CPU and frequency
	 * change creates an intractable locking scenario; the order
	 * of when these callouts happen is undefined with respect to
	 * CPU hotplug, and they can race with each other.  As such,
	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
	 * undefined; you can actually have a CPU frequency change take
	 * place in between the computation of X and the setting of the
	 * variable.  To protect against this problem, all updates of
	 * the per_cpu tsc_khz variable are done in an interrupt
	 * protected IPI, and all callers wishing to update the value
	 * must wait for a synchronous IPI to complete (which is trivial
	 * if the caller is on the CPU already).  This establishes the
	 * necessary total order on variable updates.
	 *
	 * Note that because a guest time update may take place
	 * anytime after the setting of the VCPU's request bit, the
	 * correct TSC value must be set before the request.  However,
	 * to ensure the update actually makes it to any guest which
	 * starts running in hardware virtualization between the set
	 * and the acquisition of the spinlock, we must also ping the
	 * CPU after setting the request bit.
	 *
	 */

5455 5456 5457 5458
	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
		return 0;
	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
		return 0;
5459 5460

	smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
5461

5462
	spin_lock(&kvm_lock);
5463
	list_for_each_entry(kvm, &vm_list, vm_list) {
5464
		kvm_for_each_vcpu(i, vcpu, kvm) {
5465 5466
			if (vcpu->cpu != freq->cpu)
				continue;
Z
Zachary Amsden 已提交
5467
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5468
			if (vcpu->cpu != smp_processor_id())
5469
				send_ipi = 1;
5470 5471
		}
	}
5472
	spin_unlock(&kvm_lock);
5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486

	if (freq->old < freq->new && send_ipi) {
		/*
		 * We upscale the frequency.  Must make the guest
		 * doesn't see old kvmclock values while running with
		 * the new frequency, otherwise we risk the guest sees
		 * time go backwards.
		 *
		 * In case we update the frequency for another cpu
		 * (which might be in guest context) send an interrupt
		 * to kick the cpu out of guest context.  Next time
		 * guest context is entered kvmclock will be updated,
		 * so the guest will not see stale values.
		 */
5487
		smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
5488 5489 5490 5491 5492
	}
	return 0;
}

static struct notifier_block kvmclock_cpufreq_notifier_block = {
5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515
	.notifier_call  = kvmclock_cpufreq_notifier
};

static int kvmclock_cpu_notifier(struct notifier_block *nfb,
					unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned long)hcpu;

	switch (action) {
		case CPU_ONLINE:
		case CPU_DOWN_FAILED:
			smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
			break;
		case CPU_DOWN_PREPARE:
			smp_call_function_single(cpu, tsc_bad, NULL, 1);
			break;
	}
	return NOTIFY_OK;
}

static struct notifier_block kvmclock_cpu_notifier_block = {
	.notifier_call  = kvmclock_cpu_notifier,
	.priority = -INT_MAX
5516 5517
};

5518 5519 5520 5521
static void kvm_timer_init(void)
{
	int cpu;

Z
Zachary Amsden 已提交
5522
	max_tsc_khz = tsc_khz;
5523 5524

	cpu_notifier_register_begin();
5525
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
Z
Zachary Amsden 已提交
5526 5527 5528
#ifdef CONFIG_CPU_FREQ
		struct cpufreq_policy policy;
		memset(&policy, 0, sizeof(policy));
5529 5530
		cpu = get_cpu();
		cpufreq_get_policy(&policy, cpu);
Z
Zachary Amsden 已提交
5531 5532
		if (policy.cpuinfo.max_freq)
			max_tsc_khz = policy.cpuinfo.max_freq;
5533
		put_cpu();
Z
Zachary Amsden 已提交
5534
#endif
5535 5536 5537
		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
					  CPUFREQ_TRANSITION_NOTIFIER);
	}
Z
Zachary Amsden 已提交
5538
	pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz);
5539 5540
	for_each_online_cpu(cpu)
		smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
5541 5542 5543 5544

	__register_hotcpu_notifier(&kvmclock_cpu_notifier_block);
	cpu_notifier_register_done();

5545 5546
}

5547 5548
static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);

5549
int kvm_is_in_guest(void)
5550
{
5551
	return __this_cpu_read(current_vcpu) != NULL;
5552 5553 5554 5555 5556
}

static int kvm_is_user_mode(void)
{
	int user_mode = 3;
5557

5558 5559
	if (__this_cpu_read(current_vcpu))
		user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
5560

5561 5562 5563 5564 5565 5566
	return user_mode != 0;
}

static unsigned long kvm_get_guest_ip(void)
{
	unsigned long ip = 0;
5567

5568 5569
	if (__this_cpu_read(current_vcpu))
		ip = kvm_rip_read(__this_cpu_read(current_vcpu));
5570

5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581
	return ip;
}

static struct perf_guest_info_callbacks kvm_guest_cbs = {
	.is_in_guest		= kvm_is_in_guest,
	.is_user_mode		= kvm_is_user_mode,
	.get_guest_ip		= kvm_get_guest_ip,
};

void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
{
5582
	__this_cpu_write(current_vcpu, vcpu);
5583 5584 5585 5586 5587
}
EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);

void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
{
5588
	__this_cpu_write(current_vcpu, NULL);
5589 5590 5591
}
EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);

5592 5593 5594 5595 5596 5597 5598 5599 5600
static void kvm_set_mmio_spte_mask(void)
{
	u64 mask;
	int maxphyaddr = boot_cpu_data.x86_phys_bits;

	/*
	 * Set the reserved bits and the present bit of an paging-structure
	 * entry to generate page fault with PFER.RSV = 1.
	 */
5601
	 /* Mask the reserved physical address bits. */
5602
	mask = rsvd_bits(maxphyaddr, 51);
5603 5604 5605 5606 5607

	/* Bit 62 is always reserved for 32bit host. */
	mask |= 0x3ull << 62;

	/* Set the present bit. */
5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621
	mask |= 1ull;

#ifdef CONFIG_X86_64
	/*
	 * If reserved bit is not supported, clear the present bit to disable
	 * mmio page fault.
	 */
	if (maxphyaddr == 52)
		mask &= ~1ull;
#endif

	kvm_mmu_set_mmio_spte_mask(mask);
}

5622 5623 5624
#ifdef CONFIG_X86_64
static void pvclock_gtod_update_fn(struct work_struct *work)
{
5625 5626 5627 5628 5629
	struct kvm *kvm;

	struct kvm_vcpu *vcpu;
	int i;

5630
	spin_lock(&kvm_lock);
5631 5632
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_for_each_vcpu(i, vcpu, kvm)
5633
			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
5634
	atomic_set(&kvm_guest_has_master_clock, 0);
5635
	spin_unlock(&kvm_lock);
5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665
}

static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);

/*
 * Notification about pvclock gtod data update.
 */
static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
			       void *priv)
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	struct timekeeper *tk = priv;

	update_pvclock_gtod(tk);

	/* disable master clock if host does not trust, or does not
	 * use, TSC clocksource
	 */
	if (gtod->clock.vclock_mode != VCLOCK_TSC &&
	    atomic_read(&kvm_guest_has_master_clock) != 0)
		queue_work(system_long_wq, &pvclock_gtod_work);

	return 0;
}

static struct notifier_block pvclock_gtod_notifier = {
	.notifier_call = pvclock_gtod_notify,
};
#endif

5666
int kvm_arch_init(void *opaque)
5667
{
5668
	int r;
M
Mathias Krause 已提交
5669
	struct kvm_x86_ops *ops = opaque;
5670 5671 5672

	if (kvm_x86_ops) {
		printk(KERN_ERR "kvm: already loaded the other module\n");
5673 5674
		r = -EEXIST;
		goto out;
5675 5676 5677 5678
	}

	if (!ops->cpu_has_kvm_support()) {
		printk(KERN_ERR "kvm: no hardware support\n");
5679 5680
		r = -EOPNOTSUPP;
		goto out;
5681 5682 5683
	}
	if (ops->disabled_by_bios()) {
		printk(KERN_ERR "kvm: disabled by bios\n");
5684 5685
		r = -EOPNOTSUPP;
		goto out;
5686 5687
	}

5688 5689 5690 5691 5692 5693 5694
	r = -ENOMEM;
	shared_msrs = alloc_percpu(struct kvm_shared_msrs);
	if (!shared_msrs) {
		printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
		goto out;
	}

5695 5696
	r = kvm_mmu_module_init();
	if (r)
5697
		goto out_free_percpu;
5698

5699
	kvm_set_mmio_spte_mask();
5700

5701
	kvm_x86_ops = ops;
P
Paolo Bonzini 已提交
5702 5703
	kvm_init_msr_list();

S
Sheng Yang 已提交
5704
	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
5705
			PT_DIRTY_MASK, PT64_NX_MASK, 0);
5706

5707
	kvm_timer_init();
5708

5709 5710
	perf_register_guest_info_callbacks(&kvm_guest_cbs);

5711 5712 5713
	if (cpu_has_xsave)
		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);

5714
	kvm_lapic_init();
5715 5716 5717 5718
#ifdef CONFIG_X86_64
	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
#endif

5719
	return 0;
5720

5721 5722
out_free_percpu:
	free_percpu(shared_msrs);
5723 5724
out:
	return r;
5725
}
5726

5727 5728
void kvm_arch_exit(void)
{
5729 5730
	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);

5731 5732 5733
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
					    CPUFREQ_TRANSITION_NOTIFIER);
5734
	unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block);
5735 5736 5737
#ifdef CONFIG_X86_64
	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
#endif
5738
	kvm_x86_ops = NULL;
5739
	kvm_mmu_module_exit();
5740
	free_percpu(shared_msrs);
5741
}
5742

5743 5744 5745 5746
int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
	++vcpu->stat.halt_exits;
	if (irqchip_in_kernel(vcpu->kvm)) {
5747
		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
5748 5749 5750 5751 5752 5753 5754 5755
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
}
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

5756 5757 5758 5759 5760 5761 5762 5763 5764 5765
int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
{
	u64 param, ingpa, outgpa, ret;
	uint16_t code, rep_idx, rep_cnt, res = HV_STATUS_SUCCESS, rep_done = 0;
	bool fast, longmode;

	/*
	 * hypercall generates UD from non zero cpl and real mode
	 * per HYPER-V spec
	 */
5766
	if (kvm_x86_ops->get_cpl(vcpu) != 0 || !is_protmode(vcpu)) {
5767 5768 5769 5770
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 0;
	}

5771
	longmode = is_64_bit_mode(vcpu);
5772 5773

	if (!longmode) {
5774 5775 5776 5777 5778 5779
		param = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDX) << 32) |
			(kvm_register_read(vcpu, VCPU_REGS_RAX) & 0xffffffff);
		ingpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RBX) << 32) |
			(kvm_register_read(vcpu, VCPU_REGS_RCX) & 0xffffffff);
		outgpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDI) << 32) |
			(kvm_register_read(vcpu, VCPU_REGS_RSI) & 0xffffffff);
5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795
	}
#ifdef CONFIG_X86_64
	else {
		param = kvm_register_read(vcpu, VCPU_REGS_RCX);
		ingpa = kvm_register_read(vcpu, VCPU_REGS_RDX);
		outgpa = kvm_register_read(vcpu, VCPU_REGS_R8);
	}
#endif

	code = param & 0xffff;
	fast = (param >> 16) & 0x1;
	rep_cnt = (param >> 32) & 0xfff;
	rep_idx = (param >> 48) & 0xfff;

	trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa);

5796 5797 5798 5799 5800 5801 5802 5803
	switch (code) {
	case HV_X64_HV_NOTIFY_LONG_SPIN_WAIT:
		kvm_vcpu_on_spin(vcpu);
		break;
	default:
		res = HV_STATUS_INVALID_HYPERCALL_CODE;
		break;
	}
5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815

	ret = res | (((u64)rep_done & 0xfff) << 32);
	if (longmode) {
		kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
	} else {
		kvm_register_write(vcpu, VCPU_REGS_RDX, ret >> 32);
		kvm_register_write(vcpu, VCPU_REGS_RAX, ret & 0xffffffff);
	}

	return 1;
}

5816 5817 5818 5819 5820 5821 5822
/*
 * kvm_pv_kick_cpu_op:  Kick a vcpu.
 *
 * @apicid - apicid of vcpu to be kicked.
 */
static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
{
5823
	struct kvm_lapic_irq lapic_irq;
5824

5825 5826 5827
	lapic_irq.shorthand = 0;
	lapic_irq.dest_mode = 0;
	lapic_irq.dest_id = apicid;
5828

5829 5830
	lapic_irq.delivery_mode = APIC_DM_REMRD;
	kvm_irq_delivery_to_apic(kvm, 0, &lapic_irq, NULL);
5831 5832
}

5833 5834 5835
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
{
	unsigned long nr, a0, a1, a2, a3, ret;
5836
	int op_64_bit, r = 1;
5837

5838 5839 5840
	if (kvm_hv_hypercall_enabled(vcpu->kvm))
		return kvm_hv_hypercall(vcpu);

5841 5842 5843 5844 5845
	nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
	a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
	a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
	a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
	a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
5846

5847
	trace_kvm_hypercall(nr, a0, a1, a2, a3);
F
Feng (Eric) Liu 已提交
5848

5849 5850
	op_64_bit = is_64_bit_mode(vcpu);
	if (!op_64_bit) {
5851 5852 5853 5854 5855 5856 5857
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
	}

5858 5859 5860 5861 5862
	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
		ret = -KVM_EPERM;
		goto out;
	}

5863
	switch (nr) {
A
Avi Kivity 已提交
5864 5865 5866
	case KVM_HC_VAPIC_POLL_IRQ:
		ret = 0;
		break;
5867 5868 5869 5870
	case KVM_HC_KICK_CPU:
		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
		ret = 0;
		break;
5871 5872 5873 5874
	default:
		ret = -KVM_ENOSYS;
		break;
	}
5875
out:
5876 5877
	if (!op_64_bit)
		ret = (u32)ret;
5878
	kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
A
Amit Shah 已提交
5879
	++vcpu->stat.hypercalls;
5880
	return r;
5881 5882 5883
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

5884
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
5885
{
5886
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5887
	char instruction[3];
5888
	unsigned long rip = kvm_rip_read(vcpu);
5889 5890 5891

	kvm_x86_ops->patch_hypercall(vcpu, instruction);

5892
	return emulator_write_emulated(ctxt, rip, instruction, 3, NULL);
5893 5894
}

5895 5896 5897 5898 5899 5900
/*
 * Check if userspace requested an interrupt window, and that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
A
Avi Kivity 已提交
5901
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
5902
{
5903
	return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
A
Avi Kivity 已提交
5904
		vcpu->run->request_interrupt_window &&
5905
		kvm_arch_interrupt_allowed(vcpu));
5906 5907
}

A
Avi Kivity 已提交
5908
static void post_kvm_run_save(struct kvm_vcpu *vcpu)
5909
{
A
Avi Kivity 已提交
5910 5911
	struct kvm_run *kvm_run = vcpu->run;

5912
	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
5913
	kvm_run->cr8 = kvm_get_cr8(vcpu);
5914
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
5915
	if (irqchip_in_kernel(vcpu->kvm))
5916
		kvm_run->ready_for_interrupt_injection = 1;
5917
	else
5918
		kvm_run->ready_for_interrupt_injection =
5919 5920 5921
			kvm_arch_interrupt_allowed(vcpu) &&
			!kvm_cpu_has_interrupt(vcpu) &&
			!kvm_event_needs_reinjection(vcpu);
5922 5923
}

5924 5925 5926 5927 5928 5929 5930
static void update_cr8_intercept(struct kvm_vcpu *vcpu)
{
	int max_irr, tpr;

	if (!kvm_x86_ops->update_cr8_intercept)
		return;

5931 5932 5933
	if (!vcpu->arch.apic)
		return;

5934 5935 5936 5937
	if (!vcpu->arch.apic->vapic_addr)
		max_irr = kvm_lapic_find_highest_irr(vcpu);
	else
		max_irr = -1;
5938 5939 5940 5941 5942 5943 5944 5945 5946

	if (max_irr != -1)
		max_irr >>= 4;

	tpr = kvm_lapic_get_cr8(vcpu);

	kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
}

5947
static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win)
5948
{
5949 5950
	int r;

5951
	/* try to reinject previous events if any */
5952
	if (vcpu->arch.exception.pending) {
A
Avi Kivity 已提交
5953 5954 5955
		trace_kvm_inj_exception(vcpu->arch.exception.nr,
					vcpu->arch.exception.has_error_code,
					vcpu->arch.exception.error_code);
5956 5957 5958 5959 5960

		if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
					     X86_EFLAGS_RF);

5961 5962 5963 5964 5965 5966
		if (vcpu->arch.exception.nr == DB_VECTOR &&
		    (vcpu->arch.dr7 & DR7_GD)) {
			vcpu->arch.dr7 &= ~DR7_GD;
			kvm_update_dr7(vcpu);
		}

5967 5968
		kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
					  vcpu->arch.exception.has_error_code,
5969 5970
					  vcpu->arch.exception.error_code,
					  vcpu->arch.exception.reinject);
5971
		return 0;
5972 5973
	}

5974 5975
	if (vcpu->arch.nmi_injected) {
		kvm_x86_ops->set_nmi(vcpu);
5976
		return 0;
5977 5978 5979
	}

	if (vcpu->arch.interrupt.pending) {
5980
		kvm_x86_ops->set_irq(vcpu);
5981 5982 5983 5984 5985 5986 5987
		return 0;
	}

	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
		r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
		if (r != 0)
			return r;
5988 5989 5990 5991 5992
	}

	/* try to inject new event if pending */
	if (vcpu->arch.nmi_pending) {
		if (kvm_x86_ops->nmi_allowed(vcpu)) {
A
Avi Kivity 已提交
5993
			--vcpu->arch.nmi_pending;
5994 5995 5996
			vcpu->arch.nmi_injected = true;
			kvm_x86_ops->set_nmi(vcpu);
		}
5997
	} else if (kvm_cpu_has_injectable_intr(vcpu)) {
5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009
		/*
		 * Because interrupts can be injected asynchronously, we are
		 * calling check_nested_events again here to avoid a race condition.
		 * See https://lkml.org/lkml/2014/7/2/60 for discussion about this
		 * proposal and current concerns.  Perhaps we should be setting
		 * KVM_REQ_EVENT only on certain events and not unconditionally?
		 */
		if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
			r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
			if (r != 0)
				return r;
		}
6010
		if (kvm_x86_ops->interrupt_allowed(vcpu)) {
6011 6012 6013
			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
					    false);
			kvm_x86_ops->set_irq(vcpu);
6014 6015
		}
	}
6016
	return 0;
6017 6018
}

A
Avi Kivity 已提交
6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035
static void process_nmi(struct kvm_vcpu *vcpu)
{
	unsigned limit = 2;

	/*
	 * x86 is limited to one NMI running, and one NMI pending after it.
	 * If an NMI is already in progress, limit further NMIs to just one.
	 * Otherwise, allow two (and we'll inject the first one immediately).
	 */
	if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
		limit = 1;

	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

6036
static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
6037 6038
{
	u64 eoi_exit_bitmap[4];
6039
	u32 tmr[8];
6040

6041 6042
	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
		return;
6043 6044

	memset(eoi_exit_bitmap, 0, 32);
6045
	memset(tmr, 0, 32);
6046

6047
	kvm_ioapic_scan_entry(vcpu, eoi_exit_bitmap, tmr);
6048
	kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
6049
	kvm_apic_update_tmr(vcpu, tmr);
6050 6051
}

6052 6053 6054 6055 6056 6057
static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu)
{
	++vcpu->stat.tlb_flush;
	kvm_x86_ops->tlb_flush(vcpu);
}

6058 6059
void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
{
6060 6061
	struct page *page = NULL;

6062 6063 6064
	if (!irqchip_in_kernel(vcpu->kvm))
		return;

6065 6066 6067
	if (!kvm_x86_ops->set_apic_access_page_addr)
		return;

6068 6069 6070 6071 6072 6073 6074 6075
	page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
	kvm_x86_ops->set_apic_access_page_addr(vcpu, page_to_phys(page));

	/*
	 * Do not pin apic access page in memory, the MMU notifier
	 * will call us again if it is migrated or swapped out.
	 */
	put_page(page);
6076 6077 6078
}
EXPORT_SYMBOL_GPL(kvm_vcpu_reload_apic_access_page);

6079 6080 6081
void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
					   unsigned long address)
{
6082 6083 6084 6085 6086 6087
	/*
	 * The physical address of apic access page is stored in the VMCS.
	 * Update it when it becomes invalid.
	 */
	if (address == gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT))
		kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
6088 6089
}

6090 6091 6092 6093 6094
/*
 * Returns 1 to let __vcpu_run() continue the guest execution loop without
 * exiting to the userspace.  Otherwise, the value will be returned to the
 * userspace.
 */
A
Avi Kivity 已提交
6095
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
6096 6097
{
	int r;
6098
	bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
A
Avi Kivity 已提交
6099
		vcpu->run->request_interrupt_window;
6100
	bool req_immediate_exit = false;
6101

6102
	if (vcpu->requests) {
6103
		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
6104
			kvm_mmu_unload(vcpu);
6105
		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
M
Marcelo Tosatti 已提交
6106
			__kvm_migrate_timers(vcpu);
6107 6108
		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
			kvm_gen_update_masterclock(vcpu->kvm);
6109 6110
		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
			kvm_gen_kvmclock_update(vcpu);
Z
Zachary Amsden 已提交
6111 6112
		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
			r = kvm_guest_time_update(vcpu);
6113 6114 6115
			if (unlikely(r))
				goto out;
		}
6116
		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
6117
			kvm_mmu_sync_roots(vcpu);
6118
		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
6119
			kvm_vcpu_flush_tlb(vcpu);
6120
		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
A
Avi Kivity 已提交
6121
			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
A
Avi Kivity 已提交
6122 6123 6124
			r = 0;
			goto out;
		}
6125
		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
A
Avi Kivity 已提交
6126
			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
J
Joerg Roedel 已提交
6127 6128 6129
			r = 0;
			goto out;
		}
6130
		if (kvm_check_request(KVM_REQ_DEACTIVATE_FPU, vcpu)) {
6131 6132 6133
			vcpu->fpu_active = 0;
			kvm_x86_ops->fpu_deactivate(vcpu);
		}
6134 6135 6136 6137 6138 6139
		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
			/* Page is swapped out. Do synthetic halt */
			vcpu->arch.apf.halted = true;
			r = 1;
			goto out;
		}
G
Glauber Costa 已提交
6140 6141
		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
			record_steal_time(vcpu);
A
Avi Kivity 已提交
6142 6143
		if (kvm_check_request(KVM_REQ_NMI, vcpu))
			process_nmi(vcpu);
6144 6145 6146 6147
		if (kvm_check_request(KVM_REQ_PMU, vcpu))
			kvm_handle_pmu_event(vcpu);
		if (kvm_check_request(KVM_REQ_PMI, vcpu))
			kvm_deliver_pmi(vcpu);
6148 6149
		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
			vcpu_scan_ioapic(vcpu);
6150 6151
		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
			kvm_vcpu_reload_apic_access_page(vcpu);
6152
	}
A
Avi Kivity 已提交
6153

A
Avi Kivity 已提交
6154
	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
6155 6156 6157 6158 6159 6160
		kvm_apic_accept_events(vcpu);
		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
			r = 1;
			goto out;
		}

6161 6162
		if (inject_pending_event(vcpu, req_int_win) != 0)
			req_immediate_exit = true;
A
Avi Kivity 已提交
6163
		/* enable NMI/IRQ window open exits if needed */
6164
		else if (vcpu->arch.nmi_pending)
6165
			kvm_x86_ops->enable_nmi_window(vcpu);
6166
		else if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
6167
			kvm_x86_ops->enable_irq_window(vcpu);
A
Avi Kivity 已提交
6168 6169

		if (kvm_lapic_enabled(vcpu)) {
6170 6171 6172 6173 6174 6175 6176
			/*
			 * Update architecture specific hints for APIC
			 * virtual interrupt delivery.
			 */
			if (kvm_x86_ops->hwapic_irr_update)
				kvm_x86_ops->hwapic_irr_update(vcpu,
					kvm_lapic_find_highest_irr(vcpu));
A
Avi Kivity 已提交
6177 6178 6179 6180 6181
			update_cr8_intercept(vcpu);
			kvm_lapic_sync_to_vapic(vcpu);
		}
	}

6182 6183
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r)) {
6184
		goto cancel_injection;
6185 6186
	}

6187 6188 6189
	preempt_disable();

	kvm_x86_ops->prepare_guest_switch(vcpu);
6190 6191
	if (vcpu->fpu_active)
		kvm_load_guest_fpu(vcpu);
6192
	kvm_load_guest_xcr0(vcpu);
6193

6194 6195
	vcpu->mode = IN_GUEST_MODE;

6196 6197
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);

6198 6199 6200
	/* We should set ->mode before check ->requests,
	 * see the comment in make_all_cpus_request.
	 */
6201
	smp_mb__after_srcu_read_unlock();
6202

A
Avi Kivity 已提交
6203
	local_irq_disable();
6204

6205
	if (vcpu->mode == EXITING_GUEST_MODE || vcpu->requests
A
Avi Kivity 已提交
6206
	    || need_resched() || signal_pending(current)) {
6207
		vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
6208
		smp_wmb();
6209 6210
		local_irq_enable();
		preempt_enable();
6211
		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
6212
		r = 1;
6213
		goto cancel_injection;
6214 6215
	}

6216 6217 6218
	if (req_immediate_exit)
		smp_send_reschedule(vcpu->cpu);

6219 6220
	kvm_guest_enter();

6221 6222 6223 6224 6225 6226
	if (unlikely(vcpu->arch.switch_db_regs)) {
		set_debugreg(0, 7);
		set_debugreg(vcpu->arch.eff_db[0], 0);
		set_debugreg(vcpu->arch.eff_db[1], 1);
		set_debugreg(vcpu->arch.eff_db[2], 2);
		set_debugreg(vcpu->arch.eff_db[3], 3);
6227
		set_debugreg(vcpu->arch.dr6, 6);
6228
	}
6229

6230
	trace_kvm_entry(vcpu->vcpu_id);
A
Avi Kivity 已提交
6231
	kvm_x86_ops->run(vcpu);
6232

6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247
	/*
	 * Do this here before restoring debug registers on the host.  And
	 * since we do this before handling the vmexit, a DR access vmexit
	 * can (a) read the correct value of the debug registers, (b) set
	 * KVM_DEBUGREG_WONT_EXIT again.
	 */
	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
		int i;

		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
		kvm_x86_ops->sync_dirty_debug_regs(vcpu);
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
	}

6248 6249 6250 6251 6252 6253 6254
	/*
	 * If the guest has used debug registers, at least dr7
	 * will be disabled while returning to the host.
	 * If we don't have active breakpoints in the host, we don't
	 * care about the messed up debug address registers. But if
	 * we have some of them active, restore the old state.
	 */
6255
	if (hw_breakpoint_active())
6256
		hw_breakpoint_restore();
6257

6258 6259
	vcpu->arch.last_guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu,
							   native_read_tsc());
6260

6261
	vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
6262
	smp_wmb();
6263 6264 6265

	/* Interrupt is enabled by handle_external_intr() */
	kvm_x86_ops->handle_external_intr(vcpu);
6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280

	++vcpu->stat.exits;

	/*
	 * We must have an instruction between local_irq_enable() and
	 * kvm_guest_exit(), so the timer interrupt isn't delayed by
	 * the interrupt shadow.  The stat.exits increment will do nicely.
	 * But we need to prevent reordering, hence this barrier():
	 */
	barrier();

	kvm_guest_exit();

	preempt_enable();

6281
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
6282

6283 6284 6285 6286
	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
6287 6288
		unsigned long rip = kvm_rip_read(vcpu);
		profile_hit(KVM_PROFILING, (void *)rip);
6289 6290
	}

6291 6292
	if (unlikely(vcpu->arch.tsc_always_catchup))
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
6293

6294 6295
	if (vcpu->arch.apic_attention)
		kvm_lapic_sync_from_vapic(vcpu);
A
Avi Kivity 已提交
6296

A
Avi Kivity 已提交
6297
	r = kvm_x86_ops->handle_exit(vcpu);
6298 6299 6300 6301
	return r;

cancel_injection:
	kvm_x86_ops->cancel_injection(vcpu);
6302 6303
	if (unlikely(vcpu->arch.apic_attention))
		kvm_lapic_sync_from_vapic(vcpu);
6304 6305 6306
out:
	return r;
}
6307

6308

A
Avi Kivity 已提交
6309
static int __vcpu_run(struct kvm_vcpu *vcpu)
6310 6311
{
	int r;
6312
	struct kvm *kvm = vcpu->kvm;
6313

6314
	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6315 6316 6317

	r = 1;
	while (r > 0) {
6318 6319
		if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		    !vcpu->arch.apf.halted)
A
Avi Kivity 已提交
6320
			r = vcpu_enter_guest(vcpu);
6321
		else {
6322
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
6323
			kvm_vcpu_block(vcpu);
6324
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6325 6326
			if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
				kvm_apic_accept_events(vcpu);
6327 6328
				switch(vcpu->arch.mp_state) {
				case KVM_MP_STATE_HALTED:
6329
					vcpu->arch.pv.pv_unhalted = false;
6330
					vcpu->arch.mp_state =
6331 6332
						KVM_MP_STATE_RUNNABLE;
				case KVM_MP_STATE_RUNNABLE:
6333
					vcpu->arch.apf.halted = false;
6334
					break;
6335 6336
				case KVM_MP_STATE_INIT_RECEIVED:
					break;
6337 6338 6339 6340 6341
				default:
					r = -EINTR;
					break;
				}
			}
6342 6343
		}

6344 6345 6346 6347 6348 6349 6350
		if (r <= 0)
			break;

		clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
		if (kvm_cpu_has_pending_timer(vcpu))
			kvm_inject_pending_timer_irqs(vcpu);

A
Avi Kivity 已提交
6351
		if (dm_request_for_irq_injection(vcpu)) {
6352
			r = -EINTR;
A
Avi Kivity 已提交
6353
			vcpu->run->exit_reason = KVM_EXIT_INTR;
6354 6355
			++vcpu->stat.request_irq_exits;
		}
6356 6357 6358

		kvm_check_async_pf_completion(vcpu);

6359 6360
		if (signal_pending(current)) {
			r = -EINTR;
A
Avi Kivity 已提交
6361
			vcpu->run->exit_reason = KVM_EXIT_INTR;
6362 6363 6364
			++vcpu->stat.signal_exits;
		}
		if (need_resched()) {
6365
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
6366
			cond_resched();
6367
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6368
		}
6369 6370
	}

6371
	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
6372 6373 6374 6375

	return r;
}

6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393
static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
{
	int r;
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
	r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
	if (r != EMULATE_DONE)
		return 0;
	return 1;
}

static int complete_emulated_pio(struct kvm_vcpu *vcpu)
{
	BUG_ON(!vcpu->arch.pio.count);

	return complete_emulated_io(vcpu);
}

A
Avi Kivity 已提交
6394 6395 6396 6397 6398
/*
 * Implements the following, as a state machine:
 *
 * read:
 *   for each fragment
6399 6400 6401 6402
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       exit
 *       copy data
A
Avi Kivity 已提交
6403 6404 6405 6406
 *   execute insn
 *
 * write:
 *   for each fragment
6407 6408 6409 6410
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       copy data
 *       exit
A
Avi Kivity 已提交
6411
 */
6412
static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
6413 6414
{
	struct kvm_run *run = vcpu->run;
A
Avi Kivity 已提交
6415
	struct kvm_mmio_fragment *frag;
6416
	unsigned len;
6417

6418
	BUG_ON(!vcpu->mmio_needed);
6419

6420
	/* Complete previous fragment */
6421 6422
	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
	len = min(8u, frag->len);
6423
	if (!vcpu->mmio_is_write)
6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436
		memcpy(frag->data, run->mmio.data, len);

	if (frag->len <= 8) {
		/* Switch to the next fragment. */
		frag++;
		vcpu->mmio_cur_fragment++;
	} else {
		/* Go forward to the next mmio piece. */
		frag->data += len;
		frag->gpa += len;
		frag->len -= len;
	}

6437
	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
6438
		vcpu->mmio_needed = 0;
6439 6440

		/* FIXME: return into emulator if single-stepping.  */
A
Avi Kivity 已提交
6441
		if (vcpu->mmio_is_write)
6442 6443 6444 6445
			return 1;
		vcpu->mmio_read_completed = 1;
		return complete_emulated_io(vcpu);
	}
6446

6447 6448 6449
	run->exit_reason = KVM_EXIT_MMIO;
	run->mmio.phys_addr = frag->gpa;
	if (vcpu->mmio_is_write)
6450 6451
		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
	run->mmio.len = min(8u, frag->len);
6452 6453 6454
	run->mmio.is_write = vcpu->mmio_is_write;
	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
	return 0;
6455 6456
}

6457

6458 6459 6460 6461 6462
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
{
	int r;
	sigset_t sigsaved;

6463 6464 6465
	if (!tsk_used_math(current) && init_fpu(current))
		return -ENOMEM;

6466 6467 6468
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);

6469
	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
6470
		kvm_vcpu_block(vcpu);
6471
		kvm_apic_accept_events(vcpu);
6472
		clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
6473 6474
		r = -EAGAIN;
		goto out;
6475 6476 6477
	}

	/* re-sync apic's tpr */
A
Andre Przywara 已提交
6478 6479 6480 6481 6482 6483
	if (!irqchip_in_kernel(vcpu->kvm)) {
		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
			r = -EINVAL;
			goto out;
		}
	}
6484

6485 6486 6487 6488 6489 6490 6491 6492
	if (unlikely(vcpu->arch.complete_userspace_io)) {
		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
		vcpu->arch.complete_userspace_io = NULL;
		r = cui(vcpu);
		if (r <= 0)
			goto out;
	} else
		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
6493

A
Avi Kivity 已提交
6494
	r = __vcpu_run(vcpu);
6495 6496

out:
6497
	post_kvm_run_save(vcpu);
6498 6499 6500 6501 6502 6503 6504 6505
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

	return r;
}

int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
6506 6507 6508 6509
	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
		/*
		 * We are here if userspace calls get_regs() in the middle of
		 * instruction emulation. Registers state needs to be copied
G
Guo Chao 已提交
6510
		 * back from emulation context to vcpu. Userspace shouldn't do
6511 6512 6513
		 * that usually, but some bad designed PV devices (vmware
		 * backdoor interface) need this to work
		 */
6514
		emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
6515 6516
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
	}
6517 6518 6519 6520 6521 6522 6523 6524
	regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
	regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
	regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
	regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
	regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
	regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
	regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
	regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
6525
#ifdef CONFIG_X86_64
6526 6527 6528 6529 6530 6531 6532 6533
	regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
	regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
	regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
	regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
	regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
	regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
	regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
	regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
6534 6535
#endif

6536
	regs->rip = kvm_rip_read(vcpu);
6537
	regs->rflags = kvm_get_rflags(vcpu);
6538 6539 6540 6541 6542 6543

	return 0;
}

int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
6544 6545 6546
	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;

6547 6548 6549 6550 6551 6552 6553 6554
	kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
	kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
	kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
	kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
	kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
	kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
	kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
	kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
6555
#ifdef CONFIG_X86_64
6556 6557 6558 6559 6560 6561 6562 6563
	kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
	kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
	kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
	kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
	kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
	kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
	kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
	kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
6564 6565
#endif

6566
	kvm_rip_write(vcpu, regs->rip);
6567
	kvm_set_rflags(vcpu, regs->rflags);
6568

6569 6570
	vcpu->arch.exception.pending = false;

6571 6572
	kvm_make_request(KVM_REQ_EVENT, vcpu);

6573 6574 6575 6576 6577 6578 6579
	return 0;
}

void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

6580
	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
6581 6582 6583 6584 6585 6586 6587 6588
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
6589
	struct desc_ptr dt;
6590

6591 6592 6593 6594 6595 6596
	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
6597

6598 6599
	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
6600 6601

	kvm_x86_ops->get_idt(vcpu, &dt);
6602 6603
	sregs->idt.limit = dt.size;
	sregs->idt.base = dt.address;
6604
	kvm_x86_ops->get_gdt(vcpu, &dt);
6605 6606
	sregs->gdt.limit = dt.size;
	sregs->gdt.base = dt.address;
6607

6608
	sregs->cr0 = kvm_read_cr0(vcpu);
6609
	sregs->cr2 = vcpu->arch.cr2;
6610
	sregs->cr3 = kvm_read_cr3(vcpu);
6611
	sregs->cr4 = kvm_read_cr4(vcpu);
6612
	sregs->cr8 = kvm_get_cr8(vcpu);
6613
	sregs->efer = vcpu->arch.efer;
6614 6615
	sregs->apic_base = kvm_get_apic_base(vcpu);

G
Gleb Natapov 已提交
6616
	memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
6617

6618
	if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
6619 6620
		set_bit(vcpu->arch.interrupt.nr,
			(unsigned long *)sregs->interrupt_bitmap);
6621

6622 6623 6624
	return 0;
}

6625 6626 6627
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
6628
	kvm_apic_accept_events(vcpu);
6629 6630 6631 6632 6633 6634
	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
					vcpu->arch.pv.pv_unhalted)
		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
	else
		mp_state->mp_state = vcpu->arch.mp_state;

6635 6636 6637 6638 6639 6640
	return 0;
}

int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
6641 6642 6643 6644 6645 6646 6647 6648 6649
	if (!kvm_vcpu_has_lapic(vcpu) &&
	    mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
		return -EINVAL;

	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
	} else
		vcpu->arch.mp_state = mp_state->mp_state;
6650
	kvm_make_request(KVM_REQ_EVENT, vcpu);
6651 6652 6653
	return 0;
}

6654 6655
int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
		    int reason, bool has_error_code, u32 error_code)
6656
{
6657
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6658
	int ret;
6659

6660
	init_emulate_ctxt(vcpu);
6661

6662
	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
6663
				   has_error_code, error_code);
6664 6665

	if (ret)
6666
		return EMULATE_FAIL;
6667

6668 6669
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
6670
	kvm_make_request(KVM_REQ_EVENT, vcpu);
6671
	return EMULATE_DONE;
6672 6673 6674
}
EXPORT_SYMBOL_GPL(kvm_task_switch);

6675 6676 6677
int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
6678
	struct msr_data apic_base_msr;
6679
	int mmu_reset_needed = 0;
6680
	int pending_vec, max_bits, idx;
6681
	struct desc_ptr dt;
6682

6683 6684 6685
	if (!guest_cpuid_has_xsave(vcpu) && (sregs->cr4 & X86_CR4_OSXSAVE))
		return -EINVAL;

6686 6687
	dt.size = sregs->idt.limit;
	dt.address = sregs->idt.base;
6688
	kvm_x86_ops->set_idt(vcpu, &dt);
6689 6690
	dt.size = sregs->gdt.limit;
	dt.address = sregs->gdt.base;
6691 6692
	kvm_x86_ops->set_gdt(vcpu, &dt);

6693
	vcpu->arch.cr2 = sregs->cr2;
6694
	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
6695
	vcpu->arch.cr3 = sregs->cr3;
6696
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
6697

6698
	kvm_set_cr8(vcpu, sregs->cr8);
6699

6700
	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
6701
	kvm_x86_ops->set_efer(vcpu, sregs->efer);
6702 6703 6704
	apic_base_msr.data = sregs->apic_base;
	apic_base_msr.host_initiated = true;
	kvm_set_apic_base(vcpu, &apic_base_msr);
6705

6706
	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
6707
	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
6708
	vcpu->arch.cr0 = sregs->cr0;
6709

6710
	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
6711
	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
S
Sheng Yang 已提交
6712
	if (sregs->cr4 & X86_CR4_OSXSAVE)
A
Avi Kivity 已提交
6713
		kvm_update_cpuid(vcpu);
6714 6715

	idx = srcu_read_lock(&vcpu->kvm->srcu);
6716
	if (!is_long_mode(vcpu) && is_pae(vcpu)) {
6717
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
6718 6719
		mmu_reset_needed = 1;
	}
6720
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
6721 6722 6723 6724

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

6725
	max_bits = KVM_NR_INTERRUPTS;
G
Gleb Natapov 已提交
6726 6727 6728
	pending_vec = find_first_bit(
		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
	if (pending_vec < max_bits) {
6729
		kvm_queue_interrupt(vcpu, pending_vec, false);
G
Gleb Natapov 已提交
6730
		pr_debug("Set back pending irq %d\n", pending_vec);
6731 6732
	}

6733 6734 6735 6736 6737 6738
	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
6739

6740 6741
	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
6742

6743 6744
	update_cr8_intercept(vcpu);

M
Marcelo Tosatti 已提交
6745
	/* Older userspace won't unhalt the vcpu on reset. */
6746
	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
M
Marcelo Tosatti 已提交
6747
	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
6748
	    !is_protmode(vcpu))
M
Marcelo Tosatti 已提交
6749 6750
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;

6751 6752
	kvm_make_request(KVM_REQ_EVENT, vcpu);

6753 6754 6755
	return 0;
}

J
Jan Kiszka 已提交
6756 6757
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg)
6758
{
6759
	unsigned long rflags;
6760
	int i, r;
6761

6762 6763 6764
	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
		r = -EBUSY;
		if (vcpu->arch.exception.pending)
6765
			goto out;
6766 6767 6768 6769 6770 6771
		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
			kvm_queue_exception(vcpu, DB_VECTOR);
		else
			kvm_queue_exception(vcpu, BP_VECTOR);
	}

6772 6773 6774 6775 6776
	/*
	 * Read rflags as long as potentially injected trace flags are still
	 * filtered out.
	 */
	rflags = kvm_get_rflags(vcpu);
6777 6778 6779 6780 6781 6782

	vcpu->guest_debug = dbg->control;
	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
		vcpu->guest_debug = 0;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
6783 6784
		for (i = 0; i < KVM_NR_DB_REGS; ++i)
			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
6785
		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
6786 6787 6788 6789
	} else {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
	}
6790
	kvm_update_dr7(vcpu);
6791

J
Jan Kiszka 已提交
6792 6793 6794
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
			get_segment_base(vcpu, VCPU_SREG_CS);
6795

6796 6797 6798 6799 6800
	/*
	 * Trigger an rflags update that will inject or remove the trace
	 * flags.
	 */
	kvm_set_rflags(vcpu, rflags);
6801

6802
	kvm_x86_ops->update_db_bp_intercept(vcpu);
6803

6804
	r = 0;
J
Jan Kiszka 已提交
6805

6806
out:
6807 6808 6809 6810

	return r;
}

6811 6812 6813 6814 6815 6816 6817 6818
/*
 * Translate a guest virtual address to a guest physical address.
 */
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;
6819
	int idx;
6820

6821
	idx = srcu_read_lock(&vcpu->kvm->srcu);
6822
	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
6823
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
6824 6825 6826 6827 6828 6829 6830 6831
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;

	return 0;
}

6832 6833
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
S
Sheng Yang 已提交
6834 6835
	struct i387_fxsave_struct *fxsave =
			&vcpu->arch.guest_fpu.state->fxsave;
6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849 6850

	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);

	return 0;
}

int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
S
Sheng Yang 已提交
6851 6852
	struct i387_fxsave_struct *fxsave =
			&vcpu->arch.guest_fpu.state->fxsave;
6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);

	return 0;
}

6866
int fx_init(struct kvm_vcpu *vcpu)
6867
{
6868 6869 6870 6871 6872 6873
	int err;

	err = fpu_alloc(&vcpu->arch.guest_fpu);
	if (err)
		return err;

S
Sheng Yang 已提交
6874
	fpu_finit(&vcpu->arch.guest_fpu);
6875

6876 6877 6878 6879 6880
	/*
	 * Ensure guest xcr0 is valid for loading
	 */
	vcpu->arch.xcr0 = XSTATE_FP;

6881
	vcpu->arch.cr0 |= X86_CR0_ET;
6882 6883

	return 0;
6884 6885 6886
}
EXPORT_SYMBOL_GPL(fx_init);

S
Sheng Yang 已提交
6887 6888 6889 6890 6891
static void fx_free(struct kvm_vcpu *vcpu)
{
	fpu_free(&vcpu->arch.guest_fpu);
}

6892 6893
void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
6894
	if (vcpu->guest_fpu_loaded)
6895 6896
		return;

6897 6898 6899 6900 6901 6902
	/*
	 * Restore all possible states in the guest,
	 * and assume host would use all available bits.
	 * Guest xcr0 would be loaded later.
	 */
	kvm_put_guest_xcr0(vcpu);
6903
	vcpu->guest_fpu_loaded = 1;
6904
	__kernel_fpu_begin();
S
Sheng Yang 已提交
6905
	fpu_restore_checking(&vcpu->arch.guest_fpu);
6906
	trace_kvm_fpu(1);
6907 6908 6909 6910
}

void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
6911 6912
	kvm_put_guest_xcr0(vcpu);

6913 6914 6915 6916
	if (!vcpu->guest_fpu_loaded)
		return;

	vcpu->guest_fpu_loaded = 0;
S
Sheng Yang 已提交
6917
	fpu_save_init(&vcpu->arch.guest_fpu);
6918
	__kernel_fpu_end();
A
Avi Kivity 已提交
6919
	++vcpu->stat.fpu_reload;
6920
	kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu);
6921
	trace_kvm_fpu(0);
6922
}
6923 6924 6925

void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
{
6926
	kvmclock_reset(vcpu);
6927

6928
	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
S
Sheng Yang 已提交
6929
	fx_free(vcpu);
6930 6931 6932 6933 6934 6935
	kvm_x86_ops->vcpu_free(vcpu);
}

struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
						unsigned int id)
{
Z
Zachary Amsden 已提交
6936 6937 6938 6939
	if (check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
		printk_once(KERN_WARNING
		"kvm: SMP vm created on host with unstable TSC; "
		"guest TSC will not be reliable\n");
6940 6941
	return kvm_x86_ops->vcpu_create(kvm, id);
}
6942

6943 6944 6945
int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
{
	int r;
6946

S
Sheng Yang 已提交
6947
	vcpu->arch.mtrr_state.have_fixed = 1;
6948 6949 6950
	r = vcpu_load(vcpu);
	if (r)
		return r;
6951
	kvm_vcpu_reset(vcpu);
6952
	kvm_mmu_setup(vcpu);
6953 6954
	vcpu_put(vcpu);

6955
	return r;
6956 6957
}

6958 6959 6960
int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
{
	int r;
6961
	struct msr_data msr;
6962
	struct kvm *kvm = vcpu->kvm;
6963 6964 6965 6966

	r = vcpu_load(vcpu);
	if (r)
		return r;
6967 6968 6969 6970
	msr.data = 0x0;
	msr.index = MSR_IA32_TSC;
	msr.host_initiated = true;
	kvm_write_tsc(vcpu, &msr);
6971 6972
	vcpu_put(vcpu);

6973 6974 6975
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);

6976 6977 6978
	return r;
}

6979
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
6980
{
6981
	int r;
6982 6983
	vcpu->arch.apf.msr_val = 0;

6984 6985
	r = vcpu_load(vcpu);
	BUG_ON(r);
6986 6987 6988
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);

S
Sheng Yang 已提交
6989
	fx_free(vcpu);
6990 6991 6992
	kvm_x86_ops->vcpu_free(vcpu);
}

6993
void kvm_vcpu_reset(struct kvm_vcpu *vcpu)
6994
{
A
Avi Kivity 已提交
6995 6996
	atomic_set(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = 0;
6997
	vcpu->arch.nmi_injected = false;
6998 6999
	kvm_clear_interrupt_queue(vcpu);
	kvm_clear_exception_queue(vcpu);
7000

7001
	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
7002
	vcpu->arch.dr6 = DR6_INIT;
J
Jan Kiszka 已提交
7003
	kvm_update_dr6(vcpu);
7004
	vcpu->arch.dr7 = DR7_FIXED_1;
7005
	kvm_update_dr7(vcpu);
7006

7007
	kvm_make_request(KVM_REQ_EVENT, vcpu);
7008
	vcpu->arch.apf.msr_val = 0;
G
Glauber Costa 已提交
7009
	vcpu->arch.st.msr_val = 0;
7010

7011 7012
	kvmclock_reset(vcpu);

7013 7014 7015
	kvm_clear_async_pf_completion_queue(vcpu);
	kvm_async_pf_hash_reset(vcpu);
	vcpu->arch.apf.halted = false;
7016

7017 7018
	kvm_pmu_reset(vcpu);

7019 7020 7021 7022
	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;

7023
	kvm_x86_ops->vcpu_reset(vcpu);
7024 7025
}

7026 7027 7028 7029 7030 7031 7032 7033 7034
void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, unsigned int vector)
{
	struct kvm_segment cs;

	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs.selector = vector << 8;
	cs.base = vector << 12;
	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_rip_write(vcpu, 0);
7035 7036
}

7037
int kvm_arch_hardware_enable(void)
7038
{
7039 7040 7041
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;
7042 7043 7044 7045
	int ret;
	u64 local_tsc;
	u64 max_tsc = 0;
	bool stable, backwards_tsc = false;
A
Avi Kivity 已提交
7046 7047

	kvm_shared_msr_cpu_online();
7048
	ret = kvm_x86_ops->hardware_enable();
7049 7050 7051 7052 7053 7054 7055 7056
	if (ret != 0)
		return ret;

	local_tsc = native_read_tsc();
	stable = !check_tsc_unstable();
	list_for_each_entry(kvm, &vm_list, vm_list) {
		kvm_for_each_vcpu(i, vcpu, kvm) {
			if (!stable && vcpu->cpu == smp_processor_id())
7057
				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098
			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
				backwards_tsc = true;
				if (vcpu->arch.last_host_tsc > max_tsc)
					max_tsc = vcpu->arch.last_host_tsc;
			}
		}
	}

	/*
	 * Sometimes, even reliable TSCs go backwards.  This happens on
	 * platforms that reset TSC during suspend or hibernate actions, but
	 * maintain synchronization.  We must compensate.  Fortunately, we can
	 * detect that condition here, which happens early in CPU bringup,
	 * before any KVM threads can be running.  Unfortunately, we can't
	 * bring the TSCs fully up to date with real time, as we aren't yet far
	 * enough into CPU bringup that we know how much real time has actually
	 * elapsed; our helper function, get_kernel_ns() will be using boot
	 * variables that haven't been updated yet.
	 *
	 * So we simply find the maximum observed TSC above, then record the
	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
	 * the adjustment will be applied.  Note that we accumulate
	 * adjustments, in case multiple suspend cycles happen before some VCPU
	 * gets a chance to run again.  In the event that no KVM threads get a
	 * chance to run, we will miss the entire elapsed period, as we'll have
	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
	 * loose cycle time.  This isn't too big a deal, since the loss will be
	 * uniform across all VCPUs (not to mention the scenario is extremely
	 * unlikely). It is possible that a second hibernate recovery happens
	 * much faster than a first, causing the observed TSC here to be
	 * smaller; this would require additional padding adjustment, which is
	 * why we set last_host_tsc to the local tsc observed here.
	 *
	 * N.B. - this code below runs only on platforms with reliable TSC,
	 * as that is the only way backwards_tsc is set above.  Also note
	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
	 * have the same delta_cyc adjustment applied if backwards_tsc
	 * is detected.  Note further, this adjustment is only done once,
	 * as we reset last_host_tsc on all VCPUs to stop this from being
	 * called multiple times (one for each physical CPU bringup).
	 *
G
Guo Chao 已提交
7099
	 * Platforms with unreliable TSCs don't have to deal with this, they
7100 7101 7102 7103 7104 7105
	 * will be compensated by the logic in vcpu_load, which sets the TSC to
	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
	 * guarantee that they stay in perfect synchronization.
	 */
	if (backwards_tsc) {
		u64 delta_cyc = max_tsc - local_tsc;
7106
		backwards_tsc_observed = true;
7107 7108 7109 7110
		list_for_each_entry(kvm, &vm_list, vm_list) {
			kvm_for_each_vcpu(i, vcpu, kvm) {
				vcpu->arch.tsc_offset_adjustment += delta_cyc;
				vcpu->arch.last_host_tsc = local_tsc;
7111
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125
			}

			/*
			 * We have to disable TSC offset matching.. if you were
			 * booting a VM while issuing an S4 host suspend....
			 * you may have some problem.  Solving this issue is
			 * left as an exercise to the reader.
			 */
			kvm->arch.last_tsc_nsec = 0;
			kvm->arch.last_tsc_write = 0;
		}

	}
	return 0;
7126 7127
}

7128
void kvm_arch_hardware_disable(void)
7129
{
7130 7131
	kvm_x86_ops->hardware_disable();
	drop_user_return_notifiers();
7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148
}

int kvm_arch_hardware_setup(void)
{
	return kvm_x86_ops->hardware_setup();
}

void kvm_arch_hardware_unsetup(void)
{
	kvm_x86_ops->hardware_unsetup();
}

void kvm_arch_check_processor_compat(void *rtn)
{
	kvm_x86_ops->check_processor_compatibility(rtn);
}

7149 7150 7151 7152 7153
bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu)
{
	return irqchip_in_kernel(vcpu->kvm) == (vcpu->arch.apic != NULL);
}

7154 7155
struct static_key kvm_no_apic_vcpu __read_mostly;

7156 7157 7158 7159 7160 7161 7162 7163 7164
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
	struct page *page;
	struct kvm *kvm;
	int r;

	BUG_ON(vcpu->kvm == NULL);
	kvm = vcpu->kvm;

7165
	vcpu->arch.pv.pv_unhalted = false;
7166
	vcpu->arch.emulate_ctxt.ops = &emulate_ops;
7167
	if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
7168
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
7169
	else
7170
		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
7171 7172 7173 7174 7175 7176

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
7177
	vcpu->arch.pio_data = page_address(page);
7178

7179
	kvm_set_tsc_khz(vcpu, max_tsc_khz);
Z
Zachary Amsden 已提交
7180

7181 7182 7183 7184 7185 7186 7187 7188
	r = kvm_mmu_create(vcpu);
	if (r < 0)
		goto fail_free_pio_data;

	if (irqchip_in_kernel(kvm)) {
		r = kvm_create_lapic(vcpu);
		if (r < 0)
			goto fail_mmu_destroy;
7189 7190
	} else
		static_key_slow_inc(&kvm_no_apic_vcpu);
7191

H
Huang Ying 已提交
7192 7193 7194 7195
	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
				       GFP_KERNEL);
	if (!vcpu->arch.mce_banks) {
		r = -ENOMEM;
7196
		goto fail_free_lapic;
H
Huang Ying 已提交
7197 7198 7199
	}
	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;

7200 7201
	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL)) {
		r = -ENOMEM;
7202
		goto fail_free_mce_banks;
7203
	}
7204

7205 7206 7207 7208
	r = fx_init(vcpu);
	if (r)
		goto fail_free_wbinvd_dirty_mask;

W
Will Auld 已提交
7209
	vcpu->arch.ia32_tsc_adjust_msr = 0x0;
7210
	vcpu->arch.pv_time_enabled = false;
7211 7212

	vcpu->arch.guest_supported_xcr0 = 0;
7213
	vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
7214

7215
	kvm_async_pf_hash_reset(vcpu);
7216
	kvm_pmu_init(vcpu);
7217

7218
	return 0;
7219 7220
fail_free_wbinvd_dirty_mask:
	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
7221 7222
fail_free_mce_banks:
	kfree(vcpu->arch.mce_banks);
7223 7224
fail_free_lapic:
	kvm_free_lapic(vcpu);
7225 7226 7227
fail_mmu_destroy:
	kvm_mmu_destroy(vcpu);
fail_free_pio_data:
7228
	free_page((unsigned long)vcpu->arch.pio_data);
7229 7230 7231 7232 7233 7234
fail:
	return r;
}

void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
{
7235 7236
	int idx;

7237
	kvm_pmu_destroy(vcpu);
7238
	kfree(vcpu->arch.mce_banks);
7239
	kvm_free_lapic(vcpu);
7240
	idx = srcu_read_lock(&vcpu->kvm->srcu);
7241
	kvm_mmu_destroy(vcpu);
7242
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
7243
	free_page((unsigned long)vcpu->arch.pio_data);
7244 7245
	if (!irqchip_in_kernel(vcpu->kvm))
		static_key_slow_dec(&kvm_no_apic_vcpu);
7246
}
7247

R
Radim Krčmář 已提交
7248 7249
void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
{
7250
	kvm_x86_ops->sched_in(vcpu, cpu);
R
Radim Krčmář 已提交
7251 7252
}

7253
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
7254
{
7255 7256 7257
	if (type)
		return -EINVAL;

7258
	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
7259
	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
B
Ben-Ami Yassour 已提交
7260
	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
7261
	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
7262

7263 7264
	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
7265 7266 7267
	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
		&kvm->arch.irq_sources_bitmap);
7268

7269
	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
7270
	mutex_init(&kvm->arch.apic_map_lock);
7271 7272 7273
	spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);

	pvclock_update_vm_gtod_copy(kvm);
7274

7275
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
7276
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
7277

7278
	return 0;
7279 7280 7281 7282
}

static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
7283 7284 7285
	int r;
	r = vcpu_load(vcpu);
	BUG_ON(r);
7286 7287 7288 7289 7290 7291 7292
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;
7293
	struct kvm_vcpu *vcpu;
7294 7295 7296 7297

	/*
	 * Unpin any mmu pages first.
	 */
7298 7299
	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_clear_async_pf_completion_queue(vcpu);
7300
		kvm_unload_vcpu_mmu(vcpu);
7301
	}
7302 7303 7304 7305 7306 7307
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_arch_vcpu_free(vcpu);

	mutex_lock(&kvm->lock);
	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
		kvm->vcpus[i] = NULL;
7308

7309 7310
	atomic_set(&kvm->online_vcpus, 0);
	mutex_unlock(&kvm->lock);
7311 7312
}

7313 7314
void kvm_arch_sync_events(struct kvm *kvm)
{
7315
	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
7316
	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
7317
	kvm_free_all_assigned_devices(kvm);
7318
	kvm_free_pit(kvm);
7319 7320
}

7321 7322
void kvm_arch_destroy_vm(struct kvm *kvm)
{
7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339
	if (current->mm == kvm->mm) {
		/*
		 * Free memory regions allocated on behalf of userspace,
		 * unless the the memory map has changed due to process exit
		 * or fd copying.
		 */
		struct kvm_userspace_memory_region mem;
		memset(&mem, 0, sizeof(mem));
		mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
		kvm_set_memory_region(kvm, &mem);

		mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
		kvm_set_memory_region(kvm, &mem);

		mem.slot = TSS_PRIVATE_MEMSLOT;
		kvm_set_memory_region(kvm, &mem);
	}
7340
	kvm_iommu_unmap_guest(kvm);
7341 7342
	kfree(kvm->arch.vpic);
	kfree(kvm->arch.vioapic);
7343
	kvm_free_vcpus(kvm);
7344
	kfree(rcu_dereference_check(kvm->arch.apic_map, 1));
7345
}
7346

7347
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
7348 7349 7350 7351
			   struct kvm_memory_slot *dont)
{
	int i;

7352 7353 7354 7355
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
		if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
			kvm_kvfree(free->arch.rmap[i]);
			free->arch.rmap[i] = NULL;
7356
		}
7357 7358 7359 7360 7361 7362 7363
		if (i == 0)
			continue;

		if (!dont || free->arch.lpage_info[i - 1] !=
			     dont->arch.lpage_info[i - 1]) {
			kvm_kvfree(free->arch.lpage_info[i - 1]);
			free->arch.lpage_info[i - 1] = NULL;
7364 7365 7366 7367
		}
	}
}

7368 7369
int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
			    unsigned long npages)
7370 7371 7372
{
	int i;

7373
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
7374 7375
		unsigned long ugfn;
		int lpages;
7376
		int level = i + 1;
7377 7378 7379 7380

		lpages = gfn_to_index(slot->base_gfn + npages - 1,
				      slot->base_gfn, level) + 1;

7381 7382 7383
		slot->arch.rmap[i] =
			kvm_kvzalloc(lpages * sizeof(*slot->arch.rmap[i]));
		if (!slot->arch.rmap[i])
7384
			goto out_free;
7385 7386
		if (i == 0)
			continue;
7387

7388 7389 7390
		slot->arch.lpage_info[i - 1] = kvm_kvzalloc(lpages *
					sizeof(*slot->arch.lpage_info[i - 1]));
		if (!slot->arch.lpage_info[i - 1])
7391 7392 7393
			goto out_free;

		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
7394
			slot->arch.lpage_info[i - 1][0].write_count = 1;
7395
		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
7396
			slot->arch.lpage_info[i - 1][lpages - 1].write_count = 1;
7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407
		ugfn = slot->userspace_addr >> PAGE_SHIFT;
		/*
		 * If the gfn and userspace address are not aligned wrt each
		 * other, or if explicitly asked to, disable large page
		 * support for this slot
		 */
		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
		    !kvm_largepages_enabled()) {
			unsigned long j;

			for (j = 0; j < lpages; ++j)
7408
				slot->arch.lpage_info[i - 1][j].write_count = 1;
7409 7410 7411 7412 7413 7414
		}
	}

	return 0;

out_free:
7415 7416 7417 7418 7419 7420 7421 7422
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
		kvm_kvfree(slot->arch.rmap[i]);
		slot->arch.rmap[i] = NULL;
		if (i == 0)
			continue;

		kvm_kvfree(slot->arch.lpage_info[i - 1]);
		slot->arch.lpage_info[i - 1] = NULL;
7423 7424 7425 7426
	}
	return -ENOMEM;
}

7427 7428
void kvm_arch_memslots_updated(struct kvm *kvm)
{
7429 7430 7431 7432 7433
	/*
	 * memslots->generation has been incremented.
	 * mmio generation may have reached its maximum value.
	 */
	kvm_mmu_invalidate_mmio_sptes(kvm);
7434 7435
}

7436 7437 7438
int kvm_arch_prepare_memory_region(struct kvm *kvm,
				struct kvm_memory_slot *memslot,
				struct kvm_userspace_memory_region *mem,
7439
				enum kvm_mr_change change)
7440
{
7441 7442 7443
	/*
	 * Only private memory slots need to be mapped here since
	 * KVM_SET_MEMORY_REGION ioctl is no longer supported.
7444
	 */
7445
	if ((memslot->id >= KVM_USER_MEM_SLOTS) && (change == KVM_MR_CREATE)) {
7446
		unsigned long userspace_addr;
7447

7448 7449 7450 7451
		/*
		 * MAP_SHARED to prevent internal slot pages from being moved
		 * by fork()/COW.
		 */
7452
		userspace_addr = vm_mmap(NULL, 0, memslot->npages * PAGE_SIZE,
7453 7454
					 PROT_READ | PROT_WRITE,
					 MAP_SHARED | MAP_ANONYMOUS, 0);
7455

7456 7457
		if (IS_ERR((void *)userspace_addr))
			return PTR_ERR((void *)userspace_addr);
7458

7459
		memslot->userspace_addr = userspace_addr;
7460 7461
	}

7462 7463 7464 7465 7466
	return 0;
}

void kvm_arch_commit_memory_region(struct kvm *kvm,
				struct kvm_userspace_memory_region *mem,
7467 7468
				const struct kvm_memory_slot *old,
				enum kvm_mr_change change)
7469 7470
{

7471
	int nr_mmu_pages = 0;
7472

7473
	if ((mem->slot >= KVM_USER_MEM_SLOTS) && (change == KVM_MR_DELETE)) {
7474 7475
		int ret;

7476 7477
		ret = vm_munmap(old->userspace_addr,
				old->npages * PAGE_SIZE);
7478 7479 7480 7481 7482 7483
		if (ret < 0)
			printk(KERN_WARNING
			       "kvm_vm_ioctl_set_memory_region: "
			       "failed to munmap memory\n");
	}

7484 7485 7486 7487
	if (!kvm->arch.n_requested_mmu_pages)
		nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);

	if (nr_mmu_pages)
7488
		kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
7489 7490
	/*
	 * Write protect all pages for dirty logging.
7491 7492 7493 7494 7495 7496
	 *
	 * All the sptes including the large sptes which point to this
	 * slot are set to readonly. We can not create any new large
	 * spte on this slot until the end of the logging.
	 *
	 * See the comments in fast_page_fault().
7497
	 */
7498
	if ((change != KVM_MR_DELETE) && (mem->flags & KVM_MEM_LOG_DIRTY_PAGES))
7499
		kvm_mmu_slot_remove_write_access(kvm, mem->slot);
7500
}
7501

7502
void kvm_arch_flush_shadow_all(struct kvm *kvm)
7503
{
7504
	kvm_mmu_invalidate_zap_all_pages(kvm);
7505 7506
}

7507 7508 7509
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
7510
	kvm_mmu_invalidate_zap_all_pages(kvm);
7511 7512
}

7513 7514
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
7515 7516 7517
	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events)
		kvm_x86_ops->check_nested_events(vcpu, false);

7518 7519 7520
	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		!vcpu->arch.apf.halted)
		|| !list_empty_careful(&vcpu->async_pf.done)
7521
		|| kvm_apic_has_events(vcpu)
7522
		|| vcpu->arch.pv.pv_unhalted
A
Avi Kivity 已提交
7523
		|| atomic_read(&vcpu->arch.nmi_queued) ||
7524 7525
		(kvm_arch_interrupt_allowed(vcpu) &&
		 kvm_cpu_has_interrupt(vcpu));
7526
}
7527

7528
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
7529
{
7530
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
7531
}
7532 7533 7534 7535 7536

int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	return kvm_x86_ops->interrupt_allowed(vcpu);
}
7537

7538
unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
J
Jan Kiszka 已提交
7539
{
7540 7541 7542 7543 7544 7545
	if (is_64_bit_mode(vcpu))
		return kvm_rip_read(vcpu);
	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
		     kvm_rip_read(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
J
Jan Kiszka 已提交
7546

7547 7548 7549
bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
{
	return kvm_get_linear_rip(vcpu) == linear_rip;
J
Jan Kiszka 已提交
7550 7551 7552
}
EXPORT_SYMBOL_GPL(kvm_is_linear_rip);

7553 7554 7555 7556 7557 7558
unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
{
	unsigned long rflags;

	rflags = kvm_x86_ops->get_rflags(vcpu);
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
7559
		rflags &= ~X86_EFLAGS_TF;
7560 7561 7562 7563
	return rflags;
}
EXPORT_SYMBOL_GPL(kvm_get_rflags);

7564
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
7565 7566
{
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
J
Jan Kiszka 已提交
7567
	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
7568
		rflags |= X86_EFLAGS_TF;
7569
	kvm_x86_ops->set_rflags(vcpu, rflags);
7570 7571 7572 7573 7574
}

void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
	__kvm_set_rflags(vcpu, rflags);
7575
	kvm_make_request(KVM_REQ_EVENT, vcpu);
7576 7577 7578
}
EXPORT_SYMBOL_GPL(kvm_set_rflags);

G
Gleb Natapov 已提交
7579 7580 7581 7582
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
{
	int r;

X
Xiao Guangrong 已提交
7583
	if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) ||
7584
	      work->wakeup_all)
G
Gleb Natapov 已提交
7585 7586 7587 7588 7589 7590
		return;

	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		return;

X
Xiao Guangrong 已提交
7591 7592 7593 7594
	if (!vcpu->arch.mmu.direct_map &&
	      work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu))
		return;

G
Gleb Natapov 已提交
7595 7596 7597
	vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true);
}

7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623
static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
{
	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
}

static inline u32 kvm_async_pf_next_probe(u32 key)
{
	return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
}

static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 key = kvm_async_pf_hash_fn(gfn);

	while (vcpu->arch.apf.gfns[key] != ~0)
		key = kvm_async_pf_next_probe(key);

	vcpu->arch.apf.gfns[key] = gfn;
}

static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	int i;
	u32 key = kvm_async_pf_hash_fn(gfn);

	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
7624 7625
		     (vcpu->arch.apf.gfns[key] != gfn &&
		      vcpu->arch.apf.gfns[key] != ~0); i++)
7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658
		key = kvm_async_pf_next_probe(key);

	return key;
}

bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
}

static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 i, j, k;

	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
	while (true) {
		vcpu->arch.apf.gfns[i] = ~0;
		do {
			j = kvm_async_pf_next_probe(j);
			if (vcpu->arch.apf.gfns[j] == ~0)
				return;
			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
			/*
			 * k lies cyclically in ]i,j]
			 * |    i.k.j |
			 * |....j i.k.| or  |.k..j i...|
			 */
		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
		i = j;
	}
}

7659 7660 7661 7662 7663 7664 7665
static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
{

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
				      sizeof(val));
}

7666 7667 7668
void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
				     struct kvm_async_pf *work)
{
7669 7670
	struct x86_exception fault;

7671
	trace_kvm_async_pf_not_present(work->arch.token, work->gva);
7672
	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
7673 7674

	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
7675 7676
	    (vcpu->arch.apf.send_user_only &&
	     kvm_x86_ops->get_cpl(vcpu) == 0))
7677 7678
		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
	else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
7679 7680 7681 7682 7683 7684
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
		kvm_inject_page_fault(vcpu, &fault);
7685
	}
7686 7687 7688 7689 7690
}

void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
				 struct kvm_async_pf *work)
{
7691 7692
	struct x86_exception fault;

7693
	trace_kvm_async_pf_ready(work->arch.token, work->gva);
7694
	if (work->wakeup_all)
7695 7696 7697 7698 7699 7700
		work->arch.token = ~0; /* broadcast wakeup */
	else
		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);

	if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) &&
	    !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
7701 7702 7703 7704 7705 7706
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
		kvm_inject_page_fault(vcpu, &fault);
7707
	}
7708
	vcpu->arch.apf.halted = false;
7709
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
7710 7711 7712 7713 7714 7715 7716 7717 7718
}

bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
		return true;
	else
		return !kvm_event_needs_reinjection(vcpu) &&
			kvm_x86_ops->interrupt_allowed(vcpu);
7719 7720
}

7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738
void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);

void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);

bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);

7739 7740 7741 7742 7743
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
7744
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
7745
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
7746
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
7747
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
7748
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
7749
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
7750
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
7751
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
7752
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window);