x86.c 277.6 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
B
Ben-Ami Yassour 已提交
8 9
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
N
Nicolas Kaiser 已提交
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11 12 13 14
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
B
Ben-Ami Yassour 已提交
15 16
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
17 18
 */

19
#include <linux/kvm_host.h>
20
#include "irq.h"
21
#include "mmu.h"
S
Sheng Yang 已提交
22
#include "i8254.h"
23
#include "tss.h"
24
#include "kvm_cache_regs.h"
25
#include "kvm_emulate.h"
26
#include "x86.h"
A
Avi Kivity 已提交
27
#include "cpuid.h"
28
#include "pmu.h"
29
#include "hyperv.h"
30
#include "lapic.h"
31

32
#include <linux/clocksource.h>
B
Ben-Ami Yassour 已提交
33
#include <linux/interrupt.h>
34 35 36
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
37 38
#include <linux/export.h>
#include <linux/moduleparam.h>
39
#include <linux/mman.h>
40
#include <linux/highmem.h>
J
Joerg Roedel 已提交
41
#include <linux/iommu.h>
B
Ben-Ami Yassour 已提交
42
#include <linux/intel-iommu.h>
43
#include <linux/cpufreq.h>
A
Avi Kivity 已提交
44
#include <linux/user-return-notifier.h>
45
#include <linux/srcu.h>
46
#include <linux/slab.h>
47
#include <linux/perf_event.h>
48
#include <linux/uaccess.h>
49
#include <linux/hash.h>
50
#include <linux/pci.h>
51 52
#include <linux/timekeeper_internal.h>
#include <linux/pvclock_gtod.h>
F
Feng Wu 已提交
53 54
#include <linux/kvm_irqfd.h>
#include <linux/irqbypass.h>
55
#include <linux/sched/stat.h>
56
#include <linux/sched/isolation.h>
57
#include <linux/mem_encrypt.h>
58

A
Avi Kivity 已提交
59
#include <trace/events/kvm.h>
X
Xiao Guangrong 已提交
60

61
#include <asm/debugreg.h>
62
#include <asm/msr.h>
63
#include <asm/desc.h>
H
Huang Ying 已提交
64
#include <asm/mce.h>
65
#include <linux/kernel_stat.h>
66
#include <asm/fpu/internal.h> /* Ugh! */
67
#include <asm/pvclock.h>
68
#include <asm/div64.h>
69
#include <asm/irq_remapping.h>
70
#include <asm/mshyperv.h>
71
#include <asm/hypervisor.h>
72
#include <asm/intel_pt.h>
73
#include <asm/emulate_prefix.h>
74
#include <clocksource/hyperv_timer.h>
75

76 77 78
#define CREATE_TRACE_POINTS
#include "trace.h"

79
#define MAX_IO_MSRS 256
H
Huang Ying 已提交
80
#define KVM_MAX_MCE_BANKS 32
81 82
u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P;
EXPORT_SYMBOL_GPL(kvm_mce_cap_supported);
H
Huang Ying 已提交
83

84
#define emul_to_vcpu(ctxt) \
85
	((struct kvm_vcpu *)(ctxt)->vcpu)
86

87 88 89 90 91
/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
92 93
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
94
#else
95
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
96
#endif
97

98 99
static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;

100 101
#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
                                    KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
102

103
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
A
Avi Kivity 已提交
104
static void process_nmi(struct kvm_vcpu *vcpu);
105
static void enter_smm(struct kvm_vcpu *vcpu);
106
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
K
Ken Hofsass 已提交
107 108
static void store_regs(struct kvm_vcpu *vcpu);
static int sync_regs(struct kvm_vcpu *vcpu);
109

110
struct kvm_x86_ops kvm_x86_ops __read_mostly;
111
EXPORT_SYMBOL_GPL(kvm_x86_ops);
112

113
static bool __read_mostly ignore_msrs = 0;
114
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
115

116 117 118
static bool __read_mostly report_ignored_msrs = true;
module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);

119
unsigned int min_timer_period_us = 200;
120 121
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

122 123 124
static bool __read_mostly kvmclock_periodic_sync = true;
module_param(kvmclock_periodic_sync, bool, S_IRUGO);

125
bool __read_mostly kvm_has_tsc_control;
126
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
127
u32  __read_mostly kvm_max_guest_tsc_khz;
128
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
129 130 131 132
u8   __read_mostly kvm_tsc_scaling_ratio_frac_bits;
EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits);
u64  __read_mostly kvm_max_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
133 134
u64 __read_mostly kvm_default_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
135

136
/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
137
static u32 __read_mostly tsc_tolerance_ppm = 250;
138 139
module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);

140 141 142 143 144 145 146
/*
 * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
 * adaptive tuning starting from default advancment of 1000ns.  '0' disables
 * advancement entirely.  Any other value is used as-is and disables adaptive
 * tuning, i.e. allows priveleged userspace to set an exact advancement time.
 */
static int __read_mostly lapic_timer_advance_ns = -1;
147
module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
148

149 150 151
static bool __read_mostly vector_hashing = true;
module_param(vector_hashing, bool, S_IRUGO);

152 153 154 155
bool __read_mostly enable_vmware_backdoor = false;
module_param(enable_vmware_backdoor, bool, S_IRUGO);
EXPORT_SYMBOL_GPL(enable_vmware_backdoor);

156 157 158
static bool __read_mostly force_emulation_prefix = false;
module_param(force_emulation_prefix, bool, S_IRUGO);

159 160 161
int __read_mostly pi_inject_timer = -1;
module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR);

A
Avi Kivity 已提交
162 163 164 165
#define KVM_NR_SHARED_MSRS 16

struct kvm_shared_msrs_global {
	int nr;
166
	u32 msrs[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
167 168 169 170 171
};

struct kvm_shared_msrs {
	struct user_return_notifier urn;
	bool registered;
172 173 174 175
	struct kvm_shared_msr_values {
		u64 host;
		u64 curr;
	} values[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
176 177 178
};

static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
179
static struct kvm_shared_msrs __percpu *shared_msrs;
A
Avi Kivity 已提交
180

181 182 183 184 185
#define KVM_SUPPORTED_XCR0     (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \
				| XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \
				| XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \
				| XFEATURE_MASK_PKRU)

186 187 188
u64 __read_mostly host_efer;
EXPORT_SYMBOL_GPL(host_efer);

189
static u64 __read_mostly host_xss;
190 191
u64 __read_mostly supported_xss;
EXPORT_SYMBOL_GPL(supported_xss);
192

193
struct kvm_stats_debugfs_item debugfs_entries[] = {
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
	VCPU_STAT("pf_fixed", pf_fixed),
	VCPU_STAT("pf_guest", pf_guest),
	VCPU_STAT("tlb_flush", tlb_flush),
	VCPU_STAT("invlpg", invlpg),
	VCPU_STAT("exits", exits),
	VCPU_STAT("io_exits", io_exits),
	VCPU_STAT("mmio_exits", mmio_exits),
	VCPU_STAT("signal_exits", signal_exits),
	VCPU_STAT("irq_window", irq_window_exits),
	VCPU_STAT("nmi_window", nmi_window_exits),
	VCPU_STAT("halt_exits", halt_exits),
	VCPU_STAT("halt_successful_poll", halt_successful_poll),
	VCPU_STAT("halt_attempted_poll", halt_attempted_poll),
	VCPU_STAT("halt_poll_invalid", halt_poll_invalid),
	VCPU_STAT("halt_wakeup", halt_wakeup),
	VCPU_STAT("hypercalls", hypercalls),
	VCPU_STAT("request_irq", request_irq_exits),
	VCPU_STAT("irq_exits", irq_exits),
	VCPU_STAT("host_state_reload", host_state_reload),
	VCPU_STAT("fpu_reload", fpu_reload),
	VCPU_STAT("insn_emulation", insn_emulation),
	VCPU_STAT("insn_emulation_fail", insn_emulation_fail),
	VCPU_STAT("irq_injections", irq_injections),
	VCPU_STAT("nmi_injections", nmi_injections),
	VCPU_STAT("req_event", req_event),
	VCPU_STAT("l1d_flush", l1d_flush),
	VM_STAT("mmu_shadow_zapped", mmu_shadow_zapped),
	VM_STAT("mmu_pte_write", mmu_pte_write),
	VM_STAT("mmu_pte_updated", mmu_pte_updated),
	VM_STAT("mmu_pde_zapped", mmu_pde_zapped),
	VM_STAT("mmu_flooded", mmu_flooded),
	VM_STAT("mmu_recycled", mmu_recycled),
	VM_STAT("mmu_cache_miss", mmu_cache_miss),
	VM_STAT("mmu_unsync", mmu_unsync),
	VM_STAT("remote_tlb_flush", remote_tlb_flush),
	VM_STAT("largepages", lpages, .mode = 0444),
	VM_STAT("nx_largepages_splitted", nx_lpage_splits, .mode = 0444),
	VM_STAT("max_mmu_page_hash_collisions", max_mmu_page_hash_collisions),
232 233 234
	{ NULL }
};

235
u64 __read_mostly host_xcr0;
236 237
u64 __read_mostly supported_xcr0;
EXPORT_SYMBOL_GPL(supported_xcr0);
238

239 240 241
struct kmem_cache *x86_fpu_cache;
EXPORT_SYMBOL_GPL(x86_fpu_cache);

242 243 244 245
static struct kmem_cache *x86_emulator_cache;

static struct kmem_cache *kvm_alloc_emulator_cache(void)
{
246 247 248 249
	unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src);
	unsigned int size = sizeof(struct x86_emulate_ctxt);

	return kmem_cache_create_usercopy("x86_emulator", size,
250
					  __alignof__(struct x86_emulate_ctxt),
251 252
					  SLAB_ACCOUNT, useroffset,
					  size - useroffset, NULL);
253 254
}

255
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
256

257 258 259 260 261 262 263
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
	int i;
	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
		vcpu->arch.apf.gfns[i] = ~0;
}

A
Avi Kivity 已提交
264 265 266 267 268
static void kvm_on_user_return(struct user_return_notifier *urn)
{
	unsigned slot;
	struct kvm_shared_msrs *locals
		= container_of(urn, struct kvm_shared_msrs, urn);
269
	struct kvm_shared_msr_values *values;
270 271 272 273 274 275 276 277 278 279 280 281
	unsigned long flags;

	/*
	 * Disabling irqs at this point since the following code could be
	 * interrupted and executed through kvm_arch_hardware_disable()
	 */
	local_irq_save(flags);
	if (locals->registered) {
		locals->registered = false;
		user_return_notifier_unregister(urn);
	}
	local_irq_restore(flags);
A
Avi Kivity 已提交
282
	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
283 284 285 286
		values = &locals->values[slot];
		if (values->host != values->curr) {
			wrmsrl(shared_msrs_global.msrs[slot], values->host);
			values->curr = values->host;
A
Avi Kivity 已提交
287 288 289 290
		}
	}
}

291 292
void kvm_define_shared_msr(unsigned slot, u32 msr)
{
293
	BUG_ON(slot >= KVM_NR_SHARED_MSRS);
294
	shared_msrs_global.msrs[slot] = msr;
A
Avi Kivity 已提交
295 296 297 298 299 300 301
	if (slot >= shared_msrs_global.nr)
		shared_msrs_global.nr = slot + 1;
}
EXPORT_SYMBOL_GPL(kvm_define_shared_msr);

static void kvm_shared_msr_cpu_online(void)
{
302 303 304 305
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
	u64 value;
	int i;
A
Avi Kivity 已提交
306

307 308 309 310 311
	for (i = 0; i < shared_msrs_global.nr; ++i) {
		rdmsrl_safe(shared_msrs_global.msrs[i], &value);
		smsr->values[i].host = value;
		smsr->values[i].curr = value;
	}
A
Avi Kivity 已提交
312 313
}

314
int kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
A
Avi Kivity 已提交
315
{
316 317
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
318
	int err;
A
Avi Kivity 已提交
319

320 321
	value = (value & mask) | (smsr->values[slot].host & ~mask);
	if (value == smsr->values[slot].curr)
322 323 324 325 326
		return 0;
	err = wrmsrl_safe(shared_msrs_global.msrs[slot], value);
	if (err)
		return 1;

327
	smsr->values[slot].curr = value;
A
Avi Kivity 已提交
328 329 330 331 332
	if (!smsr->registered) {
		smsr->urn.on_user_return = kvm_on_user_return;
		user_return_notifier_register(&smsr->urn);
		smsr->registered = true;
	}
333
	return 0;
A
Avi Kivity 已提交
334 335 336
}
EXPORT_SYMBOL_GPL(kvm_set_shared_msr);

337
static void drop_user_return_notifiers(void)
338
{
339 340
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
341 342 343 344 345

	if (smsr->registered)
		kvm_on_user_return(&smsr->urn);
}

346 347
u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
348
	return vcpu->arch.apic_base;
349 350 351
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

352 353 354 355 356 357
enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
{
	return kvm_apic_mode(kvm_get_apic_base(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_apic_mode);

358 359
int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
360 361
	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
362 363
	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |
		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
364

365
	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
366
		return 1;
367 368 369 370 371 372
	if (!msr_info->host_initiated) {
		if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
			return 1;
		if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
			return 1;
	}
373 374

	kvm_lapic_set_base(vcpu, msr_info->data);
375
	kvm_recalculate_apic_map(vcpu->kvm);
376
	return 0;
377 378 379
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

380
asmlinkage __visible void kvm_spurious_fault(void)
381 382
{
	/* Fault while not rebooting.  We want the trace. */
383
	BUG_ON(!kvm_rebooting);
384 385 386
}
EXPORT_SYMBOL_GPL(kvm_spurious_fault);

387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
#define EXCPT_BENIGN		0
#define EXCPT_CONTRIBUTORY	1
#define EXCPT_PF		2

static int exception_class(int vector)
{
	switch (vector) {
	case PF_VECTOR:
		return EXCPT_PF;
	case DE_VECTOR:
	case TS_VECTOR:
	case NP_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
		return EXCPT_CONTRIBUTORY;
	default:
		break;
	}
	return EXCPT_BENIGN;
}

408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
#define EXCPT_FAULT		0
#define EXCPT_TRAP		1
#define EXCPT_ABORT		2
#define EXCPT_INTERRUPT		3

static int exception_type(int vector)
{
	unsigned int mask;

	if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
		return EXCPT_INTERRUPT;

	mask = 1 << vector;

	/* #DB is trap, as instruction watchpoints are handled elsewhere */
	if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR)))
		return EXCPT_TRAP;

	if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
		return EXCPT_ABORT;

	/* Reserved exceptions will result in fault */
	return EXCPT_FAULT;
}

433 434 435 436 437 438 439 440 441 442
void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
{
	unsigned nr = vcpu->arch.exception.nr;
	bool has_payload = vcpu->arch.exception.has_payload;
	unsigned long payload = vcpu->arch.exception.payload;

	if (!has_payload)
		return;

	switch (nr) {
443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463
	case DB_VECTOR:
		/*
		 * "Certain debug exceptions may clear bit 0-3.  The
		 * remaining contents of the DR6 register are never
		 * cleared by the processor".
		 */
		vcpu->arch.dr6 &= ~DR_TRAP_BITS;
		/*
		 * DR6.RTM is set by all #DB exceptions that don't clear it.
		 */
		vcpu->arch.dr6 |= DR6_RTM;
		vcpu->arch.dr6 |= payload;
		/*
		 * Bit 16 should be set in the payload whenever the #DB
		 * exception should clear DR6.RTM. This makes the payload
		 * compatible with the pending debug exceptions under VMX.
		 * Though not currently documented in the SDM, this also
		 * makes the payload compatible with the exit qualification
		 * for #DB exceptions under VMX.
		 */
		vcpu->arch.dr6 ^= payload & DR6_RTM;
464 465 466 467 468 469 470 471

		/*
		 * The #DB payload is defined as compatible with the 'pending
		 * debug exceptions' field under VMX, not DR6. While bit 12 is
		 * defined in the 'pending debug exceptions' field (enabled
		 * breakpoint), it is reserved and must be zero in DR6.
		 */
		vcpu->arch.dr6 &= ~BIT(12);
472
		break;
473 474 475 476 477 478 479 480 481 482
	case PF_VECTOR:
		vcpu->arch.cr2 = payload;
		break;
	}

	vcpu->arch.exception.has_payload = false;
	vcpu->arch.exception.payload = 0;
}
EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);

483
static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
484
		unsigned nr, bool has_error, u32 error_code,
485
	        bool has_payload, unsigned long payload, bool reinject)
486 487 488 489
{
	u32 prev_nr;
	int class1, class2;

490 491
	kvm_make_request(KVM_REQ_EVENT, vcpu);

492
	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
493
	queue:
494 495
		if (has_error && !is_protmode(vcpu))
			has_error = false;
496 497 498 499 500 501 502 503 504 505 506
		if (reinject) {
			/*
			 * On vmentry, vcpu->arch.exception.pending is only
			 * true if an event injection was blocked by
			 * nested_run_pending.  In that case, however,
			 * vcpu_enter_guest requests an immediate exit,
			 * and the guest shouldn't proceed far enough to
			 * need reinjection.
			 */
			WARN_ON_ONCE(vcpu->arch.exception.pending);
			vcpu->arch.exception.injected = true;
507 508 509 510 511 512 513 514
			if (WARN_ON_ONCE(has_payload)) {
				/*
				 * A reinjected event has already
				 * delivered its payload.
				 */
				has_payload = false;
				payload = 0;
			}
515 516 517 518
		} else {
			vcpu->arch.exception.pending = true;
			vcpu->arch.exception.injected = false;
		}
519 520 521
		vcpu->arch.exception.has_error_code = has_error;
		vcpu->arch.exception.nr = nr;
		vcpu->arch.exception.error_code = error_code;
522 523
		vcpu->arch.exception.has_payload = has_payload;
		vcpu->arch.exception.payload = payload;
524
		if (!is_guest_mode(vcpu))
525
			kvm_deliver_exception_payload(vcpu);
526 527 528 529 530 531 532
		return;
	}

	/* to check exception */
	prev_nr = vcpu->arch.exception.nr;
	if (prev_nr == DF_VECTOR) {
		/* triple fault -> shutdown */
533
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
534 535 536 537 538 539
		return;
	}
	class1 = exception_class(prev_nr);
	class2 = exception_class(nr);
	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
540 541 542 543 544
		/*
		 * Generate double fault per SDM Table 5-5.  Set
		 * exception.pending = true so that the double fault
		 * can trigger a nested vmexit.
		 */
545
		vcpu->arch.exception.pending = true;
546
		vcpu->arch.exception.injected = false;
547 548 549
		vcpu->arch.exception.has_error_code = true;
		vcpu->arch.exception.nr = DF_VECTOR;
		vcpu->arch.exception.error_code = 0;
550 551
		vcpu->arch.exception.has_payload = false;
		vcpu->arch.exception.payload = 0;
552 553 554 555 556 557 558
	} else
		/* replace previous exception with a new one in a hope
		   that instruction re-execution will regenerate lost
		   exception */
		goto queue;
}

559 560
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
561
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
562 563 564
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

565 566
void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
567
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
568 569 570
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

571 572
void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
			   unsigned long payload)
573 574 575
{
	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
}
576
EXPORT_SYMBOL_GPL(kvm_queue_exception_p);
577

578 579 580 581 582 583 584
static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
				    u32 error_code, unsigned long payload)
{
	kvm_multiple_exception(vcpu, nr, true, error_code,
			       true, payload, false);
}

585
int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
586
{
587 588 589
	if (err)
		kvm_inject_gp(vcpu, 0);
	else
590 591 592
		return kvm_skip_emulated_instruction(vcpu);

	return 1;
593 594
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
595

596
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
597 598
{
	++vcpu->stat.pf_guest;
599 600
	vcpu->arch.exception.nested_apf =
		is_guest_mode(vcpu) && fault->async_page_fault;
601
	if (vcpu->arch.exception.nested_apf) {
602
		vcpu->arch.apf.nested_apf_token = fault->address;
603 604 605 606 607
		kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
	} else {
		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
					fault->address);
	}
608
}
N
Nadav Har'El 已提交
609
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
610

611 612
bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
				    struct x86_exception *fault)
613
{
614
	struct kvm_mmu *fault_mmu;
615 616
	WARN_ON_ONCE(fault->vector != PF_VECTOR);

617 618
	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
					       vcpu->arch.walk_mmu;
619

620 621 622 623 624 625 626 627 628 629
	/*
	 * Invalidate the TLB entry for the faulting address, if it exists,
	 * else the access will fault indefinitely (and to emulate hardware).
	 */
	if ((fault->error_code & PFERR_PRESENT_MASK) &&
	    !(fault->error_code & PFERR_RSVD_MASK))
		kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,
				       fault_mmu->root_hpa);

	fault_mmu->inject_page_fault(vcpu, fault);
630
	return fault->nested_page_fault;
631
}
632
EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault);
633

634 635
void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
636 637
	atomic_inc(&vcpu->arch.nmi_queued);
	kvm_make_request(KVM_REQ_NMI, vcpu);
638 639 640
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

641 642
void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
643
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
644 645 646
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

647 648
void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
649
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
650 651 652
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

653 654 655 656 657
/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
658
{
659
	if (kvm_x86_ops.get_cpl(vcpu) <= required_cpl)
660 661 662
		return true;
	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
	return false;
663
}
664
EXPORT_SYMBOL_GPL(kvm_require_cpl);
665

666 667 668 669 670 671 672 673 674 675
bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
{
	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
		return true;

	kvm_queue_exception(vcpu, UD_VECTOR);
	return false;
}
EXPORT_SYMBOL_GPL(kvm_require_dr);

676 677
/*
 * This function will be used to read from the physical memory of the currently
678
 * running guest. The difference to kvm_vcpu_read_guest_page is that this function
679 680 681 682 683 684
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gfn_t ngfn, void *data, int offset, int len,
			    u32 access)
{
685
	struct x86_exception exception;
686 687 688 689
	gfn_t real_gfn;
	gpa_t ngpa;

	ngpa     = gfn_to_gpa(ngfn);
690
	real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);
691 692 693 694 695
	if (real_gfn == UNMAPPED_GVA)
		return -EFAULT;

	real_gfn = gpa_to_gfn(real_gfn);

696
	return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len);
697 698 699
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

700
static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
701 702 703 704 705 706
			       void *data, int offset, int len, u32 access)
{
	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
				       data, offset, len, access);
}

707 708 709 710 711 712
static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
{
	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) | rsvd_bits(5, 8) |
	       rsvd_bits(1, 2);
}

713
/*
714
 * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
715
 */
716
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
717 718 719 720 721
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
	int i;
	int ret;
722
	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
723

724 725 726
	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
				      offset * sizeof(u64), sizeof(pdpte),
				      PFERR_USER_MASK|PFERR_WRITE_MASK);
727 728 729 730 731
	if (ret < 0) {
		ret = 0;
		goto out;
	}
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
B
Bandan Das 已提交
732
		if ((pdpte[i] & PT_PRESENT_MASK) &&
733
		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
734 735 736 737 738 739
			ret = 0;
			goto out;
		}
	}
	ret = 1;

740
	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
741 742
	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);

743 744 745 746
out:

	return ret;
}
747
EXPORT_SYMBOL_GPL(load_pdptrs);
748

749
bool pdptrs_changed(struct kvm_vcpu *vcpu)
750
{
751
	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
752 753
	int offset;
	gfn_t gfn;
754 755
	int r;

756
	if (!is_pae_paging(vcpu))
757 758
		return false;

759
	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
A
Avi Kivity 已提交
760 761
		return true;

762 763
	gfn = (kvm_read_cr3(vcpu) & 0xffffffe0ul) >> PAGE_SHIFT;
	offset = (kvm_read_cr3(vcpu) & 0xffffffe0ul) & (PAGE_SIZE - 1);
764 765
	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
				       PFERR_USER_MASK | PFERR_WRITE_MASK);
766
	if (r < 0)
767
		return true;
768

769
	return memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
770
}
771
EXPORT_SYMBOL_GPL(pdptrs_changed);
772

773
int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
774
{
775
	unsigned long old_cr0 = kvm_read_cr0(vcpu);
776
	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP;
777

778 779
	cr0 |= X86_CR0_ET;

780
#ifdef CONFIG_X86_64
781 782
	if (cr0 & 0xffffffff00000000UL)
		return 1;
783 784 785
#endif

	cr0 &= ~CR0_RESERVED_BITS;
786

787 788
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
		return 1;
789

790 791
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
		return 1;
792 793 794

	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
#ifdef CONFIG_X86_64
795
		if ((vcpu->arch.efer & EFER_LME)) {
796 797
			int cs_db, cs_l;

798 799
			if (!is_pae(vcpu))
				return 1;
800
			kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
801 802
			if (cs_l)
				return 1;
803 804
		} else
#endif
805
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
806
						 kvm_read_cr3(vcpu)))
807
			return 1;
808 809
	}

810 811 812
	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
		return 1;

813
	kvm_x86_ops.set_cr0(vcpu, cr0);
814

815
	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
816
		kvm_clear_async_pf_completion_queue(vcpu);
817 818
		kvm_async_pf_hash_reset(vcpu);
	}
819

820 821
	if ((cr0 ^ old_cr0) & update_bits)
		kvm_mmu_reset_context(vcpu);
822

823 824 825
	if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
826 827
		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);

828 829
	return 0;
}
830
EXPORT_SYMBOL_GPL(kvm_set_cr0);
831

832
void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
833
{
834
	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
835
}
836
EXPORT_SYMBOL_GPL(kvm_lmsw);
837

838
void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
839
{
840 841 842 843 844 845 846 847 848
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {

		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);

		if (vcpu->arch.xsaves_enabled &&
		    vcpu->arch.ia32_xss != host_xss)
			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
	}
849 850 851 852 853 854

	if (static_cpu_has(X86_FEATURE_PKU) &&
	    (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||
	     (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU)) &&
	    vcpu->arch.pkru != vcpu->arch.host_pkru)
		__write_pkru(vcpu->arch.pkru);
855
}
856
EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state);
857

858
void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
859
{
860 861 862 863 864 865 866 867
	if (static_cpu_has(X86_FEATURE_PKU) &&
	    (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||
	     (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU))) {
		vcpu->arch.pkru = rdpkru();
		if (vcpu->arch.pkru != vcpu->arch.host_pkru)
			__write_pkru(vcpu->arch.host_pkru);
	}

868 869 870 871 872 873 874 875 876 877
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {

		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);

		if (vcpu->arch.xsaves_enabled &&
		    vcpu->arch.ia32_xss != host_xss)
			wrmsrl(MSR_IA32_XSS, host_xss);
	}

878
}
879
EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state);
880

881
static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
882
{
883 884
	u64 xcr0 = xcr;
	u64 old_xcr0 = vcpu->arch.xcr0;
885
	u64 valid_bits;
886 887 888 889

	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
	if (index != XCR_XFEATURE_ENABLED_MASK)
		return 1;
D
Dave Hansen 已提交
890
	if (!(xcr0 & XFEATURE_MASK_FP))
891
		return 1;
D
Dave Hansen 已提交
892
	if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
893
		return 1;
894 895 896 897 898 899

	/*
	 * Do not allow the guest to set bits that we do not support
	 * saving.  However, xcr0 bit 0 is always set, even if the
	 * emulated CPU does not support XSAVE (see fx_init).
	 */
D
Dave Hansen 已提交
900
	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
901
	if (xcr0 & ~valid_bits)
902
		return 1;
903

D
Dave Hansen 已提交
904 905
	if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
	    (!(xcr0 & XFEATURE_MASK_BNDCSR)))
906 907
		return 1;

D
Dave Hansen 已提交
908 909
	if (xcr0 & XFEATURE_MASK_AVX512) {
		if (!(xcr0 & XFEATURE_MASK_YMM))
910
			return 1;
D
Dave Hansen 已提交
911
		if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
912 913
			return 1;
	}
914
	vcpu->arch.xcr0 = xcr0;
915

D
Dave Hansen 已提交
916
	if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
917
		kvm_update_cpuid(vcpu);
918 919 920 921 922
	return 0;
}

int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
923
	if (kvm_x86_ops.get_cpl(vcpu) != 0 ||
924
	    __kvm_set_xcr(vcpu, index, xcr)) {
925 926 927 928 929 930 931
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_xcr);

932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947
#define __cr4_reserved_bits(__cpu_has, __c)		\
({							\
	u64 __reserved_bits = CR4_RESERVED_BITS;	\
							\
	if (!__cpu_has(__c, X86_FEATURE_XSAVE))		\
		__reserved_bits |= X86_CR4_OSXSAVE;	\
	if (!__cpu_has(__c, X86_FEATURE_SMEP))		\
		__reserved_bits |= X86_CR4_SMEP;	\
	if (!__cpu_has(__c, X86_FEATURE_SMAP))		\
		__reserved_bits |= X86_CR4_SMAP;	\
	if (!__cpu_has(__c, X86_FEATURE_FSGSBASE))	\
		__reserved_bits |= X86_CR4_FSGSBASE;	\
	if (!__cpu_has(__c, X86_FEATURE_PKU))		\
		__reserved_bits |= X86_CR4_PKE;		\
	if (!__cpu_has(__c, X86_FEATURE_LA57))		\
		__reserved_bits |= X86_CR4_LA57;	\
948 949
	if (!__cpu_has(__c, X86_FEATURE_UMIP))		\
		__reserved_bits |= X86_CR4_UMIP;	\
950 951
	__reserved_bits;				\
})
952

953
static int kvm_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
954
{
955
	if (cr4 & cr4_reserved_bits)
956
		return -EINVAL;
957

958
	if (cr4 & __cr4_reserved_bits(guest_cpuid_has, vcpu))
959 960 961 962 963 964 965 966 967 968 969 970
		return -EINVAL;

	return 0;
}

int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
	unsigned long old_cr4 = kvm_read_cr4(vcpu);
	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE |
				   X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE;

	if (kvm_valid_cr4(vcpu, cr4))
P
Paolo Bonzini 已提交
971 972
		return 1;

973
	if (is_long_mode(vcpu)) {
974 975
		if (!(cr4 & X86_CR4_PAE))
			return 1;
976 977
	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
		   && ((cr4 ^ old_cr4) & pdptr_bits)
978 979
		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
				   kvm_read_cr3(vcpu)))
980 981
		return 1;

982
	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
983
		if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
984 985 986 987 988 989 990
			return 1;

		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
			return 1;
	}

991
	if (kvm_x86_ops.set_cr4(vcpu, cr4))
992
		return 1;
993

994 995
	if (((cr4 ^ old_cr4) & pdptr_bits) ||
	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
996
		kvm_mmu_reset_context(vcpu);
997

998
	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
A
Avi Kivity 已提交
999
		kvm_update_cpuid(vcpu);
1000

1001 1002
	return 0;
}
1003
EXPORT_SYMBOL_GPL(kvm_set_cr4);
1004

1005
int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1006
{
1007
	bool skip_tlb_flush = false;
1008
#ifdef CONFIG_X86_64
1009 1010
	bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);

1011
	if (pcid_enabled) {
1012 1013
		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
		cr3 &= ~X86_CR3_PCID_NOFLUSH;
1014
	}
1015
#endif
N
Nadav Amit 已提交
1016

1017
	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
1018 1019
		if (!skip_tlb_flush) {
			kvm_mmu_sync_roots(vcpu);
1020
			kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1021
		}
1022
		return 0;
1023 1024
	}

1025
	if (is_long_mode(vcpu) &&
1026
	    (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63)))
1027
		return 1;
1028 1029
	else if (is_pae_paging(vcpu) &&
		 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
N
Nadav Amit 已提交
1030
		return 1;
1031

1032
	kvm_mmu_new_pgd(vcpu, cr3, skip_tlb_flush, skip_tlb_flush);
1033
	vcpu->arch.cr3 = cr3;
1034
	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
1035

1036 1037
	return 0;
}
1038
EXPORT_SYMBOL_GPL(kvm_set_cr3);
1039

A
Andre Przywara 已提交
1040
int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
1041
{
1042 1043
	if (cr8 & CR8_RESERVED_BITS)
		return 1;
1044
	if (lapic_in_kernel(vcpu))
1045 1046
		kvm_lapic_set_tpr(vcpu, cr8);
	else
1047
		vcpu->arch.cr8 = cr8;
1048 1049
	return 0;
}
1050
EXPORT_SYMBOL_GPL(kvm_set_cr8);
1051

1052
unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1053
{
1054
	if (lapic_in_kernel(vcpu))
1055 1056
		return kvm_lapic_get_cr8(vcpu);
	else
1057
		return vcpu->arch.cr8;
1058
}
1059
EXPORT_SYMBOL_GPL(kvm_get_cr8);
1060

1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
{
	int i;

	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
	}
}

1072 1073 1074 1075 1076 1077 1078 1079
static void kvm_update_dr7(struct kvm_vcpu *vcpu)
{
	unsigned long dr7;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		dr7 = vcpu->arch.guest_debug_dr7;
	else
		dr7 = vcpu->arch.dr7;
1080
	kvm_x86_ops.set_dr7(vcpu, dr7);
1081 1082 1083
	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
	if (dr7 & DR7_BP_EN_MASK)
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1084 1085
}

1086 1087 1088 1089
static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
{
	u64 fixed = DR6_FIXED_1;

1090
	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1091 1092 1093 1094
		fixed |= DR6_RTM;
	return fixed;
}

1095
static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1096
{
1097 1098
	size_t size = ARRAY_SIZE(vcpu->arch.db);

1099 1100
	switch (dr) {
	case 0 ... 3:
1101
		vcpu->arch.db[array_index_nospec(dr, size)] = val;
1102 1103 1104 1105 1106 1107
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = val;
		break;
	case 4:
		/* fall through */
	case 6:
1108 1109
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
1110
		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
1111 1112 1113 1114
		break;
	case 5:
		/* fall through */
	default: /* 7 */
1115
		if (!kvm_dr7_valid(val))
1116
			return -1; /* #GP */
1117
		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1118
		kvm_update_dr7(vcpu);
1119 1120 1121 1122 1123
		break;
	}

	return 0;
}
1124 1125 1126

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
1127
	if (__kvm_set_dr(vcpu, dr, val)) {
1128
		kvm_inject_gp(vcpu, 0);
1129 1130 1131
		return 1;
	}
	return 0;
1132
}
1133 1134
EXPORT_SYMBOL_GPL(kvm_set_dr);

1135
int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1136
{
1137 1138
	size_t size = ARRAY_SIZE(vcpu->arch.db);

1139 1140
	switch (dr) {
	case 0 ... 3:
1141
		*val = vcpu->arch.db[array_index_nospec(dr, size)];
1142 1143 1144 1145
		break;
	case 4:
		/* fall through */
	case 6:
1146
		*val = vcpu->arch.dr6;
1147 1148 1149 1150 1151 1152 1153
		break;
	case 5:
		/* fall through */
	default: /* 7 */
		*val = vcpu->arch.dr7;
		break;
	}
1154 1155
	return 0;
}
1156 1157
EXPORT_SYMBOL_GPL(kvm_get_dr);

A
Avi Kivity 已提交
1158 1159
bool kvm_rdpmc(struct kvm_vcpu *vcpu)
{
1160
	u32 ecx = kvm_rcx_read(vcpu);
A
Avi Kivity 已提交
1161 1162 1163
	u64 data;
	int err;

1164
	err = kvm_pmu_rdpmc(vcpu, ecx, &data);
A
Avi Kivity 已提交
1165 1166
	if (err)
		return err;
1167 1168
	kvm_rax_write(vcpu, (u32)data);
	kvm_rdx_write(vcpu, data >> 32);
A
Avi Kivity 已提交
1169 1170 1171 1172
	return err;
}
EXPORT_SYMBOL_GPL(kvm_rdpmc);

1173 1174 1175 1176
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
1177 1178 1179
 * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features)
 * extract the supported MSRs from the related const lists.
 * msrs_to_save is selected from the msrs_to_save_all to reflect the
1180
 * capabilities of the host cpu. This capabilities test skips MSRs that are
1181
 * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs
1182
 * may depend on host virtualization features rather than host cpu features.
1183
 */
1184

1185
static const u32 msrs_to_save_all[] = {
1186
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
B
Brian Gerst 已提交
1187
	MSR_STAR,
1188 1189 1190
#ifdef CONFIG_X86_64
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
1191
	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
1192
	MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
1193
	MSR_IA32_SPEC_CTRL,
1194 1195 1196 1197 1198 1199
	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
1200 1201
	MSR_IA32_UMWAIT_CONTROL,

1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223
	MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
	MSR_ARCH_PERFMON_FIXED_CTR0 + 2, MSR_ARCH_PERFMON_FIXED_CTR0 + 3,
	MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
	MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
	MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
	MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
	MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
	MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
	MSR_ARCH_PERFMON_PERFCTR0 + 8, MSR_ARCH_PERFMON_PERFCTR0 + 9,
	MSR_ARCH_PERFMON_PERFCTR0 + 10, MSR_ARCH_PERFMON_PERFCTR0 + 11,
	MSR_ARCH_PERFMON_PERFCTR0 + 12, MSR_ARCH_PERFMON_PERFCTR0 + 13,
	MSR_ARCH_PERFMON_PERFCTR0 + 14, MSR_ARCH_PERFMON_PERFCTR0 + 15,
	MSR_ARCH_PERFMON_PERFCTR0 + 16, MSR_ARCH_PERFMON_PERFCTR0 + 17,
	MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
	MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
	MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
	MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
	MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
	MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
	MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
	MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
	MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
1224 1225
};

1226
static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_all)];
1227 1228
static unsigned num_msrs_to_save;

1229
static const u32 emulated_msrs_all[] = {
1230 1231 1232 1233
	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
1234
	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
1235 1236
	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
1237
	HV_X64_MSR_RESET,
1238
	HV_X64_MSR_VP_INDEX,
1239
	HV_X64_MSR_VP_RUNTIME,
1240
	HV_X64_MSR_SCONTROL,
A
Andrey Smetanin 已提交
1241
	HV_X64_MSR_STIMER0_CONFIG,
1242
	HV_X64_MSR_VP_ASSIST_PAGE,
1243 1244 1245 1246
	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
	HV_X64_MSR_TSC_EMULATION_STATUS,

	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
1247 1248
	MSR_KVM_PV_EOI_EN,

W
Will Auld 已提交
1249
	MSR_IA32_TSC_ADJUST,
1250
	MSR_IA32_TSCDEADLINE,
1251
	MSR_IA32_ARCH_CAPABILITIES,
1252
	MSR_IA32_MISC_ENABLE,
1253 1254
	MSR_IA32_MCG_STATUS,
	MSR_IA32_MCG_CTL,
1255
	MSR_IA32_MCG_EXT_CTL,
P
Paolo Bonzini 已提交
1256
	MSR_IA32_SMBASE,
1257
	MSR_SMI_COUNT,
K
Kyle Huey 已提交
1258 1259
	MSR_PLATFORM_INFO,
	MSR_MISC_FEATURES_ENABLES,
1260
	MSR_AMD64_VIRT_SPEC_CTRL,
1261
	MSR_IA32_POWER_CTL,
1262
	MSR_IA32_UCODE_REV,
1263

1264 1265 1266 1267 1268
	/*
	 * The following list leaves out MSRs whose values are determined
	 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
	 * We always support the "true" VMX control MSRs, even if the host
	 * processor does not, so I am putting these registers here rather
1269
	 * than in msrs_to_save_all.
1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
	 */
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1284
	MSR_K7_HWCR,
1285
	MSR_KVM_POLL_CONTROL,
1286 1287
};

1288
static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
1289 1290
static unsigned num_emulated_msrs;

1291 1292 1293 1294
/*
 * List of msr numbers which are used to expose MSR-based features that
 * can be used by a hypervisor to validate requested CPU features.
 */
1295
static const u32 msr_based_features_all[] = {
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR0_FIXED1,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_CR4_FIXED1,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1315
	MSR_F10H_DECFG,
1316
	MSR_IA32_UCODE_REV,
1317
	MSR_IA32_ARCH_CAPABILITIES,
1318 1319
};

1320
static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)];
1321 1322
static unsigned int num_msr_based_features;

1323
static u64 kvm_get_arch_capabilities(void)
1324
{
1325
	u64 data = 0;
1326

1327 1328
	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);
1329

P
Paolo Bonzini 已提交
1330 1331 1332 1333 1334 1335 1336 1337
	/*
	 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
	 * the nested hypervisor runs with NX huge pages.  If it is not,
	 * L1 is anyway vulnerable to ITLB_MULTIHIT explots from other
	 * L1 guests, so it need not worry about its own (L2) guests.
	 */
	data |= ARCH_CAP_PSCHANGE_MC_NO;

1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
	/*
	 * If we're doing cache flushes (either "always" or "cond")
	 * we will do one whenever the guest does a vmlaunch/vmresume.
	 * If an outer hypervisor is doing the cache flush for us
	 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
	 * capability to the guest too, and if EPT is disabled we're not
	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
	 * require a nested hypervisor to do a flush of its own.
	 */
	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;

1350 1351 1352 1353 1354 1355 1356
	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
		data |= ARCH_CAP_RDCL_NO;
	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
		data |= ARCH_CAP_SSB_NO;
	if (!boot_cpu_has_bug(X86_BUG_MDS))
		data |= ARCH_CAP_MDS_NO;

1357
	/*
1358 1359 1360 1361
	 * On TAA affected systems:
	 *      - nothing to do if TSX is disabled on the host.
	 *      - we emulate TSX_CTRL if present on the host.
	 *	  This lets the guest use VERW to clear CPU buffers.
1362
	 */
1363
	if (!boot_cpu_has(X86_FEATURE_RTM))
1364
		data &= ~(ARCH_CAP_TAA_NO | ARCH_CAP_TSX_CTRL_MSR);
1365 1366
	else if (!boot_cpu_has_bug(X86_BUG_TAA))
		data |= ARCH_CAP_TAA_NO;
1367

1368 1369 1370
	return data;
}

1371 1372 1373
static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
{
	switch (msr->index) {
1374
	case MSR_IA32_ARCH_CAPABILITIES:
1375 1376 1377
		msr->data = kvm_get_arch_capabilities();
		break;
	case MSR_IA32_UCODE_REV:
1378
		rdmsrl_safe(msr->index, &msr->data);
1379
		break;
1380
	default:
1381
		if (kvm_x86_ops.get_msr_feature(msr))
1382 1383 1384 1385 1386
			return 1;
	}
	return 0;
}

1387 1388 1389
static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	struct kvm_msr_entry msr;
1390
	int r;
1391 1392

	msr.index = index;
1393 1394 1395
	r = kvm_get_msr_feature(&msr);
	if (r)
		return r;
1396 1397 1398 1399 1400 1401

	*data = msr.data;

	return 0;
}

1402
static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1403
{
1404
	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1405
		return false;
A
Alexander Graf 已提交
1406

1407
	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1408
		return false;
1409

1410 1411 1412 1413 1414 1415
	if (efer & (EFER_LME | EFER_LMA) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))
		return false;

	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
		return false;
1416

1417
	return true;
1418 1419 1420 1421 1422 1423 1424 1425

}
bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	if (efer & efer_reserved_bits)
		return false;

	return __kvm_valid_efer(vcpu, efer);
1426 1427 1428
}
EXPORT_SYMBOL_GPL(kvm_valid_efer);

1429
static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1430 1431
{
	u64 old_efer = vcpu->arch.efer;
1432
	u64 efer = msr_info->data;
1433

1434
	if (efer & efer_reserved_bits)
1435
		return 1;
1436

1437 1438 1439 1440 1441 1442 1443 1444
	if (!msr_info->host_initiated) {
		if (!__kvm_valid_efer(vcpu, efer))
			return 1;

		if (is_paging(vcpu) &&
		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
			return 1;
	}
1445

1446
	efer &= ~EFER_LMA;
1447
	efer |= vcpu->arch.efer & EFER_LMA;
1448

1449
	kvm_x86_ops.set_efer(vcpu, efer);
1450

1451 1452 1453 1454
	/* Update reserved bits */
	if ((efer ^ old_efer) & EFER_NX)
		kvm_mmu_reset_context(vcpu);

1455
	return 0;
1456 1457
}

1458 1459 1460 1461 1462 1463
void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);

1464
/*
1465 1466
 * Write @data into the MSR specified by @index.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
1467 1468 1469
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1470 1471
static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
			 bool host_initiated)
1472
{
1473 1474 1475
	struct msr_data msr;

	switch (index) {
1476 1477 1478 1479 1480
	case MSR_FS_BASE:
	case MSR_GS_BASE:
	case MSR_KERNEL_GS_BASE:
	case MSR_CSTAR:
	case MSR_LSTAR:
1481
		if (is_noncanonical_address(data, vcpu))
1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
			return 1;
		break;
	case MSR_IA32_SYSENTER_EIP:
	case MSR_IA32_SYSENTER_ESP:
		/*
		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
		 * non-canonical address is written on Intel but not on
		 * AMD (which ignores the top 32-bits, because it does
		 * not implement 64-bit SYSENTER).
		 *
		 * 64-bit code should hence be able to write a non-canonical
		 * value on AMD.  Making the address canonical ensures that
		 * vmentry does not fail on Intel after writing a non-canonical
		 * value, and that something deterministic happens if the guest
		 * invokes 64-bit SYSENTER.
		 */
1498
		data = get_canonical(data, vcpu_virt_addr_bits(vcpu));
1499
	}
1500 1501 1502 1503 1504

	msr.data = data;
	msr.index = index;
	msr.host_initiated = host_initiated;

1505
	return kvm_x86_ops.set_msr(vcpu, &msr);
1506 1507
}

1508
/*
1509 1510 1511 1512
 * Read the MSR specified by @index into @data.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
1513
 */
1514 1515
int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
		  bool host_initiated)
1516 1517
{
	struct msr_data msr;
1518
	int ret;
1519 1520

	msr.index = index;
1521
	msr.host_initiated = host_initiated;
1522

1523
	ret = kvm_x86_ops.get_msr(vcpu, &msr);
1524 1525 1526
	if (!ret)
		*data = msr.data;
	return ret;
1527 1528
}

1529
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1530
{
1531 1532 1533
	return __kvm_get_msr(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_get_msr);
1534

1535 1536 1537 1538 1539 1540
int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
{
	return __kvm_set_msr(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_set_msr);

1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575
int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data;

	if (kvm_get_msr(vcpu, ecx, &data)) {
		trace_kvm_msr_read_ex(ecx);
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

	trace_kvm_msr_read(ecx, data);

	kvm_rax_write(vcpu, data & -1u);
	kvm_rdx_write(vcpu, (data >> 32) & -1u);
	return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);

int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data = kvm_read_edx_eax(vcpu);

	if (kvm_set_msr(vcpu, ecx, data)) {
		trace_kvm_msr_write_ex(ecx, data);
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

	trace_kvm_msr_write(ecx, data);
	return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);

1576 1577 1578 1579 1580 1581 1582 1583 1584
/*
 * The fast path for frequent and performance sensitive wrmsr emulation,
 * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces
 * the latency of virtual IPI by avoiding the expensive bits of transitioning
 * from guest to host, e.g. reacquiring KVM's SRCU lock. In contrast to the
 * other cases which must be called after interrupts are enabled on the host.
 */
static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
{
1585 1586 1587 1588
	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
		return 1;

	if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) &&
1589
		((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) &&
1590 1591
		((data & APIC_MODE_MASK) == APIC_DM_FIXED) &&
		((u32)(data >> 32) != X2APIC_BROADCAST)) {
1592

1593 1594
		data &= ~(1 << 12);
		kvm_apic_send_ipi(vcpu->arch.apic, (u32)data, (u32)(data >> 32));
1595
		kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR2, (u32)(data >> 32));
1596 1597 1598
		kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR, (u32)data);
		trace_kvm_apic_write(APIC_ICR, (u32)data);
		return 0;
1599 1600 1601 1602 1603 1604 1605 1606
	}

	return 1;
}

enum exit_fastpath_completion handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
{
	u32 msr = kvm_rcx_read(vcpu);
1607
	u64 data;
1608 1609 1610 1611
	int ret = 0;

	switch (msr) {
	case APIC_BASE_MSR + (APIC_ICR >> 4):
1612
		data = kvm_read_edx_eax(vcpu);
1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627
		ret = handle_fastpath_set_x2apic_icr_irqoff(vcpu, data);
		break;
	default:
		return EXIT_FASTPATH_NONE;
	}

	if (!ret) {
		trace_kvm_msr_write(msr, data);
		return EXIT_FASTPATH_SKIP_EMUL_INS;
	}

	return EXIT_FASTPATH_NONE;
}
EXPORT_SYMBOL_GPL(handle_fastpath_set_msr_irqoff);

1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return __kvm_get_msr(vcpu, index, data, true);
}

static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return __kvm_set_msr(vcpu, index, *data, true);
1639 1640
}

1641
#ifdef CONFIG_X86_64
1642 1643 1644 1645 1646 1647
struct pvclock_clock {
	int vclock_mode;
	u64 cycle_last;
	u64 mask;
	u32 mult;
	u32 shift;
1648 1649
	u64 base_cycles;
	u64 offset;
1650 1651
};

1652 1653 1654
struct pvclock_gtod_data {
	seqcount_t	seq;

1655 1656
	struct pvclock_clock clock; /* extract of a clocksource struct */
	struct pvclock_clock raw_clock; /* extract of a clocksource struct */
1657

1658
	ktime_t		offs_boot;
1659
	u64		wall_time_sec;
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670
};

static struct pvclock_gtod_data pvclock_gtod_data;

static void update_pvclock_gtod(struct timekeeper *tk)
{
	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;

	write_seqcount_begin(&vdata->seq);

	/* copy pvclock gtod data */
1671
	vdata->clock.vclock_mode	= tk->tkr_mono.clock->vdso_clock_mode;
1672 1673 1674 1675
	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
	vdata->clock.mask		= tk->tkr_mono.mask;
	vdata->clock.mult		= tk->tkr_mono.mult;
	vdata->clock.shift		= tk->tkr_mono.shift;
1676 1677
	vdata->clock.base_cycles	= tk->tkr_mono.xtime_nsec;
	vdata->clock.offset		= tk->tkr_mono.base;
1678

1679
	vdata->raw_clock.vclock_mode	= tk->tkr_raw.clock->vdso_clock_mode;
1680 1681 1682 1683
	vdata->raw_clock.cycle_last	= tk->tkr_raw.cycle_last;
	vdata->raw_clock.mask		= tk->tkr_raw.mask;
	vdata->raw_clock.mult		= tk->tkr_raw.mult;
	vdata->raw_clock.shift		= tk->tkr_raw.shift;
1684 1685
	vdata->raw_clock.base_cycles	= tk->tkr_raw.xtime_nsec;
	vdata->raw_clock.offset		= tk->tkr_raw.base;
1686

1687 1688
	vdata->wall_time_sec            = tk->xtime_sec;

1689
	vdata->offs_boot		= tk->offs_boot;
1690

1691 1692
	write_seqcount_end(&vdata->seq);
}
1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704

static s64 get_kvmclock_base_ns(void)
{
	/* Count up from boot time, but with the frequency of the raw clock.  */
	return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
}
#else
static s64 get_kvmclock_base_ns(void)
{
	/* Master clock not used, so we can just use CLOCK_BOOTTIME.  */
	return ktime_get_boottime_ns();
}
1705 1706
#endif

1707 1708 1709
void kvm_set_pending_timer(struct kvm_vcpu *vcpu)
{
	kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
1710
	kvm_vcpu_kick(vcpu);
1711
}
1712

1713 1714
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
{
1715 1716
	int version;
	int r;
1717
	struct pvclock_wall_clock wc;
1718
	u64 wall_nsec;
1719 1720 1721 1722

	if (!wall_clock)
		return;

1723 1724 1725 1726 1727 1728 1729 1730
	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
	if (r)
		return;

	if (version & 1)
		++version;  /* first time write, random junk */

	++version;
1731

1732 1733
	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
		return;
1734

1735 1736
	/*
	 * The guest calculates current wall clock time by adding
Z
Zachary Amsden 已提交
1737
	 * system time (updated by kvm_guest_time_update below) to the
1738
	 * wall clock specified here.  We do the reverse here.
1739
	 */
1740
	wall_nsec = ktime_get_real_ns() - get_kvmclock_ns(kvm);
1741

1742 1743
	wc.nsec = do_div(wall_nsec, 1000000000);
	wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */
1744
	wc.version = version;
1745 1746 1747 1748 1749 1750 1751

	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));

	version++;
	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
}

1752 1753
static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
1754 1755
	do_shl32_div32(dividend, divisor);
	return dividend;
1756 1757
}

1758
static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
1759
			       s8 *pshift, u32 *pmultiplier)
1760
{
1761
	uint64_t scaled64;
1762 1763 1764 1765
	int32_t  shift = 0;
	uint64_t tps64;
	uint32_t tps32;

1766 1767
	tps64 = base_hz;
	scaled64 = scaled_hz;
1768
	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
1769 1770 1771 1772 1773
		tps64 >>= 1;
		shift--;
	}

	tps32 = (uint32_t)tps64;
1774 1775
	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
1776 1777 1778
			scaled64 >>= 1;
		else
			tps32 <<= 1;
1779 1780 1781
		shift++;
	}

1782 1783
	*pshift = shift;
	*pmultiplier = div_frac(scaled64, tps32);
1784 1785
}

1786
#ifdef CONFIG_X86_64
1787
static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
1788
#endif
1789

1790
static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
1791
static unsigned long max_tsc_khz;
1792

1793
static u32 adjust_tsc_khz(u32 khz, s32 ppm)
1794
{
1795 1796 1797
	u64 v = (u64)khz * (1000000 + ppm);
	do_div(v, 1000000);
	return v;
1798 1799
}

1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816
static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
{
	u64 ratio;

	/* Guest TSC same frequency as host TSC? */
	if (!scale) {
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
		return 0;
	}

	/* TSC scaling supported? */
	if (!kvm_has_tsc_control) {
		if (user_tsc_khz > tsc_khz) {
			vcpu->arch.tsc_catchup = 1;
			vcpu->arch.tsc_always_catchup = 1;
			return 0;
		} else {
1817
			pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
1818 1819 1820 1821 1822 1823 1824 1825 1826
			return -1;
		}
	}

	/* TSC scaling required  - calculate ratio */
	ratio = mul_u64_u32_div(1ULL << kvm_tsc_scaling_ratio_frac_bits,
				user_tsc_khz, tsc_khz);

	if (ratio == 0 || ratio >= kvm_max_tsc_scaling_ratio) {
1827 1828
		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
			            user_tsc_khz);
1829 1830 1831 1832 1833 1834 1835
		return -1;
	}

	vcpu->arch.tsc_scaling_ratio = ratio;
	return 0;
}

1836
static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
1837
{
1838 1839
	u32 thresh_lo, thresh_hi;
	int use_scaling = 0;
1840

1841
	/* tsc_khz can be zero if TSC calibration fails */
1842
	if (user_tsc_khz == 0) {
1843 1844
		/* set tsc_scaling_ratio to a safe value */
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
1845
		return -1;
1846
	}
1847

Z
Zachary Amsden 已提交
1848
	/* Compute a scale to convert nanoseconds in TSC cycles */
1849
	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
1850 1851
			   &vcpu->arch.virtual_tsc_shift,
			   &vcpu->arch.virtual_tsc_mult);
1852
	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
1853 1854 1855 1856 1857 1858 1859 1860 1861

	/*
	 * Compute the variation in TSC rate which is acceptable
	 * within the range of tolerance and decide if the
	 * rate being applied is within that bounds of the hardware
	 * rate.  If so, no scaling or compensation need be done.
	 */
	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
1862 1863
	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", user_tsc_khz, thresh_lo, thresh_hi);
1864 1865
		use_scaling = 1;
	}
1866
	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
Z
Zachary Amsden 已提交
1867 1868 1869 1870
}

static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
1871
	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
1872 1873
				      vcpu->arch.virtual_tsc_mult,
				      vcpu->arch.virtual_tsc_shift);
1874
	tsc += vcpu->arch.this_tsc_write;
Z
Zachary Amsden 已提交
1875 1876 1877
	return tsc;
}

1878 1879
static inline int gtod_is_based_on_tsc(int mode)
{
1880
	return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK;
1881 1882
}

1883
static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
1884 1885 1886 1887 1888 1889 1890 1891 1892
{
#ifdef CONFIG_X86_64
	bool vcpus_matched;
	struct kvm_arch *ka = &vcpu->kvm->arch;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			 atomic_read(&vcpu->kvm->online_vcpus));

1893 1894 1895 1896 1897 1898 1899 1900 1901
	/*
	 * Once the masterclock is enabled, always perform request in
	 * order to update it.
	 *
	 * In order to enable masterclock, the host clocksource must be TSC
	 * and the vcpus need to have matched TSCs.  When that happens,
	 * perform request to enable masterclock.
	 */
	if (ka->use_master_clock ||
1902
	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
1903 1904 1905 1906 1907 1908 1909 1910
		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);

	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
			    atomic_read(&vcpu->kvm->online_vcpus),
		            ka->use_master_clock, gtod->clock.vclock_mode);
#endif
}

W
Will Auld 已提交
1911 1912
static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
{
1913
	u64 curr_offset = kvm_x86_ops.read_l1_tsc_offset(vcpu);
W
Will Auld 已提交
1914 1915 1916
	vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
}

1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943
/*
 * Multiply tsc by a fixed point number represented by ratio.
 *
 * The most significant 64-N bits (mult) of ratio represent the
 * integral part of the fixed point number; the remaining N bits
 * (frac) represent the fractional part, ie. ratio represents a fixed
 * point number (mult + frac * 2^(-N)).
 *
 * N equals to kvm_tsc_scaling_ratio_frac_bits.
 */
static inline u64 __scale_tsc(u64 ratio, u64 tsc)
{
	return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits);
}

u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc)
{
	u64 _tsc = tsc;
	u64 ratio = vcpu->arch.tsc_scaling_ratio;

	if (ratio != kvm_default_tsc_scaling_ratio)
		_tsc = __scale_tsc(ratio, tsc);

	return _tsc;
}
EXPORT_SYMBOL_GPL(kvm_scale_tsc);

1944 1945 1946 1947 1948 1949 1950 1951 1952
static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
{
	u64 tsc;

	tsc = kvm_scale_tsc(vcpu, rdtsc());

	return target_tsc - tsc;
}

1953 1954
u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
{
1955
	u64 tsc_offset = kvm_x86_ops.read_l1_tsc_offset(vcpu);
1956 1957

	return tsc_offset + kvm_scale_tsc(vcpu, host_tsc);
1958 1959 1960
}
EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);

1961 1962
static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
{
1963
	vcpu->arch.tsc_offset = kvm_x86_ops.write_l1_tsc_offset(vcpu, offset);
1964 1965
}

1966 1967 1968 1969 1970 1971 1972
static inline bool kvm_check_tsc_unstable(void)
{
#ifdef CONFIG_X86_64
	/*
	 * TSC is marked unstable when we're running on Hyper-V,
	 * 'TSC page' clocksource is good.
	 */
1973
	if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK)
1974 1975 1976 1977 1978
		return false;
#endif
	return check_tsc_unstable();
}

1979
void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
1980 1981
{
	struct kvm *kvm = vcpu->kvm;
Z
Zachary Amsden 已提交
1982
	u64 offset, ns, elapsed;
1983
	unsigned long flags;
1984
	bool matched;
T
Tomasz Grabiec 已提交
1985
	bool already_matched;
1986
	u64 data = msr->data;
1987
	bool synchronizing = false;
1988

1989
	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
1990
	offset = kvm_compute_tsc_offset(vcpu, data);
1991
	ns = get_kvmclock_base_ns();
Z
Zachary Amsden 已提交
1992
	elapsed = ns - kvm->arch.last_tsc_nsec;
1993

1994
	if (vcpu->arch.virtual_tsc_khz) {
1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013
		if (data == 0 && msr->host_initiated) {
			/*
			 * detection of vcpu initialization -- need to sync
			 * with other vCPUs. This particularly helps to keep
			 * kvm_clock stable after CPU hotplug
			 */
			synchronizing = true;
		} else {
			u64 tsc_exp = kvm->arch.last_tsc_write +
						nsec_to_cycles(vcpu, elapsed);
			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
			/*
			 * Special case: TSC write with a small delta (1 second)
			 * of virtual cycle time against real time is
			 * interpreted as an attempt to synchronize the CPU.
			 */
			synchronizing = data < tsc_exp + tsc_hz &&
					data + tsc_hz > tsc_exp;
		}
2014
	}
Z
Zachary Amsden 已提交
2015 2016

	/*
2017 2018 2019 2020 2021
	 * For a reliable TSC, we can match TSC offsets, and for an unstable
	 * TSC, we add elapsed time in this computation.  We could let the
	 * compensation code attempt to catch up if we fall behind, but
	 * it's better to try to match offsets from the beginning.
         */
2022
	if (synchronizing &&
2023
	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
2024
		if (!kvm_check_tsc_unstable()) {
2025
			offset = kvm->arch.cur_tsc_offset;
Z
Zachary Amsden 已提交
2026
		} else {
2027
			u64 delta = nsec_to_cycles(vcpu, elapsed);
2028
			data += delta;
2029
			offset = kvm_compute_tsc_offset(vcpu, data);
Z
Zachary Amsden 已提交
2030
		}
2031
		matched = true;
T
Tomasz Grabiec 已提交
2032
		already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
2033 2034 2035 2036 2037 2038
	} else {
		/*
		 * We split periods of matched TSC writes into generations.
		 * For each generation, we track the original measured
		 * nanosecond time, offset, and write, so if TSCs are in
		 * sync, we can match exact offset, and if not, we can match
G
Guo Chao 已提交
2039
		 * exact software computation in compute_guest_tsc()
2040 2041 2042 2043 2044 2045 2046
		 *
		 * These values are tracked in kvm->arch.cur_xxx variables.
		 */
		kvm->arch.cur_tsc_generation++;
		kvm->arch.cur_tsc_nsec = ns;
		kvm->arch.cur_tsc_write = data;
		kvm->arch.cur_tsc_offset = offset;
2047
		matched = false;
Z
Zachary Amsden 已提交
2048
	}
2049 2050 2051 2052 2053

	/*
	 * We also track th most recent recorded KHZ, write and time to
	 * allow the matching interval to be extended at each write.
	 */
Z
Zachary Amsden 已提交
2054 2055
	kvm->arch.last_tsc_nsec = ns;
	kvm->arch.last_tsc_write = data;
2056
	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
2057

2058
	vcpu->arch.last_guest_tsc = data;
2059 2060 2061 2062 2063 2064

	/* Keep track of which generation this VCPU has synchronized to */
	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;

2065
	if (!msr->host_initiated && guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST))
W
Will Auld 已提交
2066
		update_ia32_tsc_adjust_msr(vcpu, offset);
2067

2068
	kvm_vcpu_write_tsc_offset(vcpu, offset);
2069
	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
2070 2071

	spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
T
Tomasz Grabiec 已提交
2072
	if (!matched) {
2073
		kvm->arch.nr_vcpus_matched_tsc = 0;
T
Tomasz Grabiec 已提交
2074 2075 2076
	} else if (!already_matched) {
		kvm->arch.nr_vcpus_matched_tsc++;
	}
2077 2078 2079

	kvm_track_tsc_matching(vcpu);
	spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
2080
}
2081

2082 2083
EXPORT_SYMBOL_GPL(kvm_write_tsc);

2084 2085 2086
static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
					   s64 adjustment)
{
2087
	u64 tsc_offset = kvm_x86_ops.read_l1_tsc_offset(vcpu);
2088
	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2089 2090 2091 2092 2093 2094 2095
}

static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
{
	if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
		WARN_ON(adjustment < 0);
	adjustment = kvm_scale_tsc(vcpu, (u64) adjustment);
2096
	adjust_tsc_offset_guest(vcpu, adjustment);
2097 2098
}

2099 2100
#ifdef CONFIG_X86_64

2101
static u64 read_tsc(void)
2102
{
2103
	u64 ret = (u64)rdtsc_ordered();
2104
	u64 last = pvclock_gtod_data.clock.cycle_last;
2105 2106 2107 2108 2109 2110

	if (likely(ret >= last))
		return ret;

	/*
	 * GCC likes to generate cmov here, but this branch is extremely
2111
	 * predictable (it's just a function of time and the likely is
2112 2113 2114 2115 2116 2117 2118 2119 2120
	 * very likely) and there's a data dependence, so force GCC
	 * to generate a branch instead.  I don't barrier() because
	 * we don't actually need a barrier, and if this function
	 * ever gets inlined it will generate worse code.
	 */
	asm volatile ("");
	return last;
}

2121 2122
static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
			  int *mode)
2123 2124
{
	long v;
2125 2126
	u64 tsc_pg_val;

2127
	switch (clock->vclock_mode) {
2128
	case VDSO_CLOCKMODE_HVCLOCK:
2129 2130 2131 2132
		tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
						  tsc_timestamp);
		if (tsc_pg_val != U64_MAX) {
			/* TSC page valid */
2133
			*mode = VDSO_CLOCKMODE_HVCLOCK;
2134 2135
			v = (tsc_pg_val - clock->cycle_last) &
				clock->mask;
2136 2137
		} else {
			/* TSC page invalid */
2138
			*mode = VDSO_CLOCKMODE_NONE;
2139 2140
		}
		break;
2141 2142
	case VDSO_CLOCKMODE_TSC:
		*mode = VDSO_CLOCKMODE_TSC;
2143
		*tsc_timestamp = read_tsc();
2144 2145
		v = (*tsc_timestamp - clock->cycle_last) &
			clock->mask;
2146 2147
		break;
	default:
2148
		*mode = VDSO_CLOCKMODE_NONE;
2149
	}
2150

2151
	if (*mode == VDSO_CLOCKMODE_NONE)
2152
		*tsc_timestamp = v = 0;
2153

2154
	return v * clock->mult;
2155 2156
}

2157
static int do_monotonic_raw(s64 *t, u64 *tsc_timestamp)
2158
{
2159
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2160 2161
	unsigned long seq;
	int mode;
2162
	u64 ns;
2163 2164 2165

	do {
		seq = read_seqcount_begin(&gtod->seq);
2166
		ns = gtod->raw_clock.base_cycles;
2167
		ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
2168 2169
		ns >>= gtod->raw_clock.shift;
		ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
2170
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2171
	*t = ns;
2172 2173 2174 2175

	return mode;
}

2176
static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
2177 2178 2179 2180 2181 2182 2183 2184 2185
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	unsigned long seq;
	int mode;
	u64 ns;

	do {
		seq = read_seqcount_begin(&gtod->seq);
		ts->tv_sec = gtod->wall_time_sec;
2186
		ns = gtod->clock.base_cycles;
2187
		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
2188 2189 2190 2191 2192 2193 2194 2195 2196
		ns >>= gtod->clock.shift;
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));

	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return mode;
}

2197 2198
/* returns true if host is using TSC based clocksource */
static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
2199 2200
{
	/* checked again under seqlock below */
2201
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2202 2203
		return false;

2204
	return gtod_is_based_on_tsc(do_monotonic_raw(kernel_ns,
2205
						      tsc_timestamp));
2206
}
2207

2208
/* returns true if host is using TSC based clocksource */
2209
static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
2210
					   u64 *tsc_timestamp)
2211 2212
{
	/* checked again under seqlock below */
2213
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2214 2215
		return false;

2216
	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
2217
}
2218 2219 2220 2221
#endif

/*
 *
2222 2223 2224
 * Assuming a stable TSC across physical CPUS, and a stable TSC
 * across virtual CPUs, the following condition is possible.
 * Each numbered line represents an event visible to both
2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
 * CPUs at the next numbered event.
 *
 * "timespecX" represents host monotonic time. "tscX" represents
 * RDTSC value.
 *
 * 		VCPU0 on CPU0		|	VCPU1 on CPU1
 *
 * 1.  read timespec0,tsc0
 * 2.					| timespec1 = timespec0 + N
 * 					| tsc1 = tsc0 + M
 * 3. transition to guest		| transition to guest
 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
 * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
 * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
 *
 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
 *
 * 	- ret0 < ret1
 *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
 *		...
 *	- 0 < N - M => M < N
 *
 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
 * always the case (the difference between two distinct xtime instances
 * might be smaller then the difference between corresponding TSC reads,
 * when updating guest vcpus pvclock areas).
 *
 * To avoid that problem, do not allow visibility of distinct
 * system_timestamp/tsc_timestamp values simultaneously: use a master
 * copy of host monotonic time values. Update that master copy
 * in lockstep.
 *
2257
 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
2258 2259 2260 2261 2262 2263 2264 2265
 *
 */

static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	struct kvm_arch *ka = &kvm->arch;
	int vclock_mode;
2266 2267 2268 2269
	bool host_tsc_clocksource, vcpus_matched;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			atomic_read(&kvm->online_vcpus));
2270 2271 2272 2273 2274

	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
2275
	host_tsc_clocksource = kvm_get_time_and_clockread(
2276 2277 2278
					&ka->master_kernel_ns,
					&ka->master_cycle_now);

2279
	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
2280
				&& !ka->backwards_tsc_observed
2281
				&& !ka->boot_vcpu_runs_old_kvmclock;
2282

2283 2284 2285 2286
	if (ka->use_master_clock)
		atomic_set(&kvm_guest_has_master_clock, 1);

	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
2287 2288
	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
					vcpus_matched);
2289 2290 2291
#endif
}

2292 2293 2294 2295 2296
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
}

2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309
static void kvm_gen_update_masterclock(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	int i;
	struct kvm_vcpu *vcpu;
	struct kvm_arch *ka = &kvm->arch;

	spin_lock(&ka->pvclock_gtod_sync_lock);
	kvm_make_mclock_inprogress_request(kvm);
	/* no guest entries from this point */
	pvclock_update_vm_gtod_copy(kvm);

	kvm_for_each_vcpu(i, vcpu, kvm)
2310
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2311 2312 2313

	/* guest entries allowed */
	kvm_for_each_vcpu(i, vcpu, kvm)
2314
		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
2315 2316 2317 2318 2319

	spin_unlock(&ka->pvclock_gtod_sync_lock);
#endif
}

2320
u64 get_kvmclock_ns(struct kvm *kvm)
2321 2322
{
	struct kvm_arch *ka = &kvm->arch;
2323
	struct pvclock_vcpu_time_info hv_clock;
2324
	u64 ret;
2325

2326 2327 2328
	spin_lock(&ka->pvclock_gtod_sync_lock);
	if (!ka->use_master_clock) {
		spin_unlock(&ka->pvclock_gtod_sync_lock);
2329
		return get_kvmclock_base_ns() + ka->kvmclock_offset;
2330 2331
	}

2332 2333 2334 2335
	hv_clock.tsc_timestamp = ka->master_cycle_now;
	hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
	spin_unlock(&ka->pvclock_gtod_sync_lock);

2336 2337 2338
	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
	get_cpu();

2339 2340 2341 2342 2343 2344
	if (__this_cpu_read(cpu_tsc_khz)) {
		kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL,
				   &hv_clock.tsc_shift,
				   &hv_clock.tsc_to_system_mul);
		ret = __pvclock_read_cycles(&hv_clock, rdtsc());
	} else
2345
		ret = get_kvmclock_base_ns() + ka->kvmclock_offset;
2346 2347 2348 2349

	put_cpu();

	return ret;
2350 2351
}

2352 2353 2354 2355 2356
static void kvm_setup_pvclock_page(struct kvm_vcpu *v)
{
	struct kvm_vcpu_arch *vcpu = &v->arch;
	struct pvclock_vcpu_time_info guest_hv_clock;

2357
	if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376
		&guest_hv_clock, sizeof(guest_hv_clock))))
		return;

	/* This VCPU is paused, but it's legal for a guest to read another
	 * VCPU's kvmclock, so we really have to follow the specification where
	 * it says that version is odd if data is being modified, and even after
	 * it is consistent.
	 *
	 * Version field updates must be kept separate.  This is because
	 * kvm_write_guest_cached might use a "rep movs" instruction, and
	 * writes within a string instruction are weakly ordered.  So there
	 * are three writes overall.
	 *
	 * As a small optimization, only write the version field in the first
	 * and third write.  The vcpu->pv_time cache is still valid, because the
	 * version field is the first in the struct.
	 */
	BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);

2377 2378 2379
	if (guest_hv_clock.version & 1)
		++guest_hv_clock.version;  /* first time write, random junk */

2380
	vcpu->hv_clock.version = guest_hv_clock.version + 1;
2381 2382 2383
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396

	smp_wmb();

	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
	vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);

	if (vcpu->pvclock_set_guest_stopped_request) {
		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
		vcpu->pvclock_set_guest_stopped_request = false;
	}

	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);

2397 2398 2399
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock));
2400 2401 2402 2403

	smp_wmb();

	vcpu->hv_clock.version++;
2404 2405 2406
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2407 2408
}

Z
Zachary Amsden 已提交
2409
static int kvm_guest_time_update(struct kvm_vcpu *v)
2410
{
2411
	unsigned long flags, tgt_tsc_khz;
2412
	struct kvm_vcpu_arch *vcpu = &v->arch;
2413
	struct kvm_arch *ka = &v->kvm->arch;
2414
	s64 kernel_ns;
2415
	u64 tsc_timestamp, host_tsc;
2416
	u8 pvclock_flags;
2417 2418 2419 2420
	bool use_master_clock;

	kernel_ns = 0;
	host_tsc = 0;
2421

2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432
	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
	spin_lock(&ka->pvclock_gtod_sync_lock);
	use_master_clock = ka->use_master_clock;
	if (use_master_clock) {
		host_tsc = ka->master_cycle_now;
		kernel_ns = ka->master_kernel_ns;
	}
	spin_unlock(&ka->pvclock_gtod_sync_lock);
2433 2434 2435

	/* Keep irq disabled to prevent changes to the clock */
	local_irq_save(flags);
2436 2437
	tgt_tsc_khz = __this_cpu_read(cpu_tsc_khz);
	if (unlikely(tgt_tsc_khz == 0)) {
2438 2439 2440 2441
		local_irq_restore(flags);
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
		return 1;
	}
2442
	if (!use_master_clock) {
2443
		host_tsc = rdtsc();
2444
		kernel_ns = get_kvmclock_base_ns();
2445 2446
	}

2447
	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
2448

Z
Zachary Amsden 已提交
2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461
	/*
	 * We may have to catch up the TSC to match elapsed wall clock
	 * time for two reasons, even if kvmclock is used.
	 *   1) CPU could have been running below the maximum TSC rate
	 *   2) Broken TSC compensation resets the base at each VCPU
	 *      entry to avoid unknown leaps of TSC even when running
	 *      again on the same CPU.  This may cause apparent elapsed
	 *      time to disappear, and the guest to stand still or run
	 *	very slowly.
	 */
	if (vcpu->tsc_catchup) {
		u64 tsc = compute_guest_tsc(v, kernel_ns);
		if (tsc > tsc_timestamp) {
2462
			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
Z
Zachary Amsden 已提交
2463 2464
			tsc_timestamp = tsc;
		}
2465 2466
	}

2467 2468
	local_irq_restore(flags);

2469
	/* With all the info we got, fill in the values */
2470

2471 2472 2473 2474
	if (kvm_has_tsc_control)
		tgt_tsc_khz = kvm_scale_tsc(v, tgt_tsc_khz);

	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
2475
		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
2476 2477
				   &vcpu->hv_clock.tsc_shift,
				   &vcpu->hv_clock.tsc_to_system_mul);
2478
		vcpu->hw_tsc_khz = tgt_tsc_khz;
2479 2480
	}

2481
	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
2482
	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
Z
Zachary Amsden 已提交
2483
	vcpu->last_guest_tsc = tsc_timestamp;
2484

2485
	/* If the host uses TSC clocksource, then it is stable */
2486
	pvclock_flags = 0;
2487 2488 2489
	if (use_master_clock)
		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;

2490 2491
	vcpu->hv_clock.flags = pvclock_flags;

P
Paolo Bonzini 已提交
2492 2493 2494 2495
	if (vcpu->pv_time_enabled)
		kvm_setup_pvclock_page(v);
	if (v == kvm_get_vcpu(v->kvm, 0))
		kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
2496
	return 0;
2497 2498
}

2499 2500 2501 2502 2503 2504 2505 2506
/*
 * kvmclock updates which are isolated to a given vcpu, such as
 * vcpu->cpu migration, should not allow system_timestamp from
 * the rest of the vcpus to remain static. Otherwise ntp frequency
 * correction applies to one vcpu's system_timestamp but not
 * the others.
 *
 * So in those cases, request a kvmclock update for all vcpus.
2507 2508 2509 2510
 * We need to rate-limit these requests though, as they can
 * considerably slow guests that have a large number of vcpus.
 * The time for a remote vcpu to update its kvmclock is bound
 * by the delay we use to rate-limit the updates.
2511 2512
 */

2513 2514 2515
#define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)

static void kvmclock_update_fn(struct work_struct *work)
2516 2517
{
	int i;
2518 2519 2520 2521
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_update_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);
2522 2523 2524
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm) {
2525
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2526 2527 2528 2529
		kvm_vcpu_kick(vcpu);
	}
}

2530 2531 2532 2533
static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
{
	struct kvm *kvm = v->kvm;

2534
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
2535 2536 2537 2538
	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
					KVMCLOCK_UPDATE_DELAY);
}

2539 2540 2541 2542 2543 2544 2545 2546 2547
#define KVMCLOCK_SYNC_PERIOD (300 * HZ)

static void kvmclock_sync_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_sync_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);

2548 2549 2550
	if (!kvmclock_periodic_sync)
		return;

2551 2552 2553 2554 2555
	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
}

2556 2557 2558 2559 2560 2561
/*
 * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
 */
static bool can_set_mci_status(struct kvm_vcpu *vcpu)
{
	/* McStatusWrEn enabled? */
2562
	if (guest_cpuid_is_amd_or_hygon(vcpu))
2563 2564 2565 2566 2567
		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));

	return false;
}

2568
static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2569
{
H
Huang Ying 已提交
2570 2571
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2572 2573
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
H
Huang Ying 已提交
2574

2575 2576
	switch (msr) {
	case MSR_IA32_MCG_STATUS:
H
Huang Ying 已提交
2577
		vcpu->arch.mcg_status = data;
2578
		break;
2579
	case MSR_IA32_MCG_CTL:
2580 2581
		if (!(mcg_cap & MCG_CTL_P) &&
		    (data || !msr_info->host_initiated))
H
Huang Ying 已提交
2582 2583
			return 1;
		if (data != 0 && data != ~(u64)0)
2584
			return 1;
H
Huang Ying 已提交
2585 2586 2587 2588
		vcpu->arch.mcg_ctl = data;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2589
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
2590 2591 2592 2593
			u32 offset = array_index_nospec(
				msr - MSR_IA32_MC0_CTL,
				MSR_IA32_MCx_CTL(bank_num) - MSR_IA32_MC0_CTL);

2594 2595 2596 2597 2598
			/* only 0 or all 1s can be written to IA32_MCi_CTL
			 * some Linux kernels though clear bit 10 in bank 4 to
			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
			 * this to avoid an uncatched #GP in the guest
			 */
H
Huang Ying 已提交
2599
			if ((offset & 0x3) == 0 &&
2600
			    data != 0 && (data | (1 << 10)) != ~(u64)0)
H
Huang Ying 已提交
2601
				return -1;
2602 2603

			/* MCi_STATUS */
2604
			if (!msr_info->host_initiated &&
2605 2606 2607 2608 2609
			    (offset & 0x3) == 1 && data != 0) {
				if (!can_set_mci_status(vcpu))
					return -1;
			}

H
Huang Ying 已提交
2610 2611 2612 2613 2614 2615 2616 2617
			vcpu->arch.mce_banks[offset] = data;
			break;
		}
		return 1;
	}
	return 0;
}

E
Ed Swierk 已提交
2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634
static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
	int lm = is_long_mode(vcpu);
	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
		: kvm->arch.xen_hvm_config.blob_size_32;
	u32 page_num = data & ~PAGE_MASK;
	u64 page_addr = data & PAGE_MASK;
	u8 *page;
	int r;

	r = -E2BIG;
	if (page_num >= blob_size)
		goto out;
	r = -ENOMEM;
2635 2636 2637
	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
	if (IS_ERR(page)) {
		r = PTR_ERR(page);
E
Ed Swierk 已提交
2638
		goto out;
2639
	}
2640
	if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE))
E
Ed Swierk 已提交
2641 2642 2643 2644 2645 2646 2647 2648
		goto out_free;
	r = 0;
out_free:
	kfree(page);
out:
	return r;
}

2649 2650 2651 2652
static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
{
	gpa_t gpa = data & ~0x3f;

2653 2654
	/* Bits 3:5 are reserved, Should be zero */
	if (data & 0x38)
2655 2656 2657 2658 2659 2660 2661 2662 2663 2664
		return 1;

	vcpu->arch.apf.msr_val = data;

	if (!(data & KVM_ASYNC_PF_ENABLED)) {
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_async_pf_hash_reset(vcpu);
		return 0;
	}

2665
	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
2666
					sizeof(u32)))
2667 2668
		return 1;

2669
	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
2670
	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
2671 2672 2673 2674
	kvm_async_pf_wakeup_all(vcpu);
	return 0;
}

2675 2676
static void kvmclock_reset(struct kvm_vcpu *vcpu)
{
2677
	vcpu->arch.pv_time_enabled = false;
P
Paolo Bonzini 已提交
2678
	vcpu->arch.time = 0;
2679 2680
}

2681
static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
2682 2683
{
	++vcpu->stat.tlb_flush;
2684
	kvm_x86_ops.tlb_flush_all(vcpu);
2685 2686
}

2687 2688 2689 2690 2691 2692
static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
{
	++vcpu->stat.tlb_flush;
	kvm_x86_ops.tlb_flush_guest(vcpu);
}

G
Glauber Costa 已提交
2693 2694
static void record_steal_time(struct kvm_vcpu *vcpu)
{
2695 2696 2697
	struct kvm_host_map map;
	struct kvm_steal_time *st;

G
Glauber Costa 已提交
2698 2699 2700
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

2701 2702 2703
	/* -EAGAIN is returned in atomic context so we can just return. */
	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT,
			&map, &vcpu->arch.st.cache, false))
G
Glauber Costa 已提交
2704 2705
		return;

2706 2707 2708
	st = map.hva +
		offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);

2709 2710 2711 2712
	/*
	 * Doing a TLB flush here, on the guest's behalf, can avoid
	 * expensive IPIs.
	 */
2713
	trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
2714 2715
		st->preempted & KVM_VCPU_FLUSH_TLB);
	if (xchg(&st->preempted, 0) & KVM_VCPU_FLUSH_TLB)
2716
		kvm_vcpu_flush_tlb_guest(vcpu);
2717

2718
	vcpu->arch.st.preempted = 0;
W
Wanpeng Li 已提交
2719

2720 2721
	if (st->version & 1)
		st->version += 1;  /* first time write, random junk */
W
Wanpeng Li 已提交
2722

2723
	st->version += 1;
W
Wanpeng Li 已提交
2724 2725 2726

	smp_wmb();

2727
	st->steal += current->sched_info.run_delay -
2728 2729
		vcpu->arch.st.last_steal;
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
W
Wanpeng Li 已提交
2730 2731 2732

	smp_wmb();

2733
	st->version += 1;
G
Glauber Costa 已提交
2734

2735
	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, false);
G
Glauber Costa 已提交
2736 2737
}

2738
int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2739
{
2740
	bool pr = false;
2741 2742
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
2743

2744
	switch (msr) {
2745 2746 2747 2748 2749
	case MSR_AMD64_NB_CFG:
	case MSR_IA32_UCODE_WRITE:
	case MSR_VM_HSAVE_PA:
	case MSR_AMD64_PATCH_LOADER:
	case MSR_AMD64_BU_CFG2:
2750
	case MSR_AMD64_DC_CFG:
2751
	case MSR_F15H_EX_CFG:
2752 2753
		break;

2754 2755 2756 2757
	case MSR_IA32_UCODE_REV:
		if (msr_info->host_initiated)
			vcpu->arch.microcode_version = data;
		break;
2758 2759 2760 2761 2762
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.arch_capabilities = data;
		break;
2763
	case MSR_EFER:
2764
		return set_efer(vcpu, msr_info);
2765 2766
	case MSR_K7_HWCR:
		data &= ~(u64)0x40;	/* ignore flush filter disable */
2767
		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
2768
		data &= ~(u64)0x8;	/* ignore TLB cache disable */
2769 2770 2771 2772 2773

		/* Handle McStatusWrEn */
		if (data == BIT_ULL(18)) {
			vcpu->arch.msr_hwcr = data;
		} else if (data != 0) {
2774 2775
			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
				    data);
2776 2777
			return 1;
		}
2778
		break;
2779 2780
	case MSR_FAM10H_MMIO_CONF_BASE:
		if (data != 0) {
2781 2782
			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
				    "0x%llx\n", data);
2783 2784
			return 1;
		}
2785
		break;
2786 2787 2788 2789 2790 2791 2792 2793 2794
	case MSR_IA32_DEBUGCTLMSR:
		if (!data) {
			/* We support the non-activated case already */
			break;
		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
			/* Values other than LBR and BTF are vendor-specific,
			   thus reserved and should throw a #GP */
			return 1;
		}
2795 2796
		vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
			    __func__, data);
2797
		break;
A
Avi Kivity 已提交
2798
	case 0x200 ... 0x2ff:
2799
		return kvm_mtrr_set_msr(vcpu, msr, data);
2800
	case MSR_IA32_APICBASE:
2801
		return kvm_set_apic_base(vcpu, msr_info);
G
Gleb Natapov 已提交
2802 2803
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_write(vcpu, msr, data);
2804 2805 2806
	case MSR_IA32_TSCDEADLINE:
		kvm_set_lapic_tscdeadline_msr(vcpu, data);
		break;
W
Will Auld 已提交
2807
	case MSR_IA32_TSC_ADJUST:
2808
		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
W
Will Auld 已提交
2809
			if (!msr_info->host_initiated) {
2810
				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
2811
				adjust_tsc_offset_guest(vcpu, adj);
W
Will Auld 已提交
2812 2813 2814 2815
			}
			vcpu->arch.ia32_tsc_adjust_msr = data;
		}
		break;
2816
	case MSR_IA32_MISC_ENABLE:
2817 2818 2819 2820 2821 2822 2823 2824 2825
		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
		    ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
				return 1;
			vcpu->arch.ia32_misc_enable_msr = data;
			kvm_update_cpuid(vcpu);
		} else {
			vcpu->arch.ia32_misc_enable_msr = data;
		}
2826
		break;
P
Paolo Bonzini 已提交
2827 2828 2829 2830 2831
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smbase = data;
		break;
2832 2833 2834
	case MSR_IA32_POWER_CTL:
		vcpu->arch.msr_ia32_power_ctl = data;
		break;
2835 2836 2837
	case MSR_IA32_TSC:
		kvm_write_tsc(vcpu, msr_info);
		break;
2838 2839 2840 2841 2842
	case MSR_IA32_XSS:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
			return 1;
		/*
2843 2844 2845
		 * KVM supports exposing PT to the guest, but does not support
		 * IA32_XSS[bit 8]. Guests have to use RDMSR/WRMSR rather than
		 * XSAVES/XRSTORS to save/restore PT MSRs.
2846
		 */
2847
		if (data & ~supported_xss)
2848 2849 2850
			return 1;
		vcpu->arch.ia32_xss = data;
		break;
2851 2852 2853 2854 2855
	case MSR_SMI_COUNT:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smi_count = data;
		break;
2856
	case MSR_KVM_WALL_CLOCK_NEW:
2857 2858 2859 2860
	case MSR_KVM_WALL_CLOCK:
		vcpu->kvm->arch.wall_clock = data;
		kvm_write_wall_clock(vcpu->kvm, data);
		break;
2861
	case MSR_KVM_SYSTEM_TIME_NEW:
2862
	case MSR_KVM_SYSTEM_TIME: {
2863 2864 2865 2866 2867 2868
		struct kvm_arch *ka = &vcpu->kvm->arch;

		if (vcpu->vcpu_id == 0 && !msr_info->host_initiated) {
			bool tmp = (msr == MSR_KVM_SYSTEM_TIME);

			if (ka->boot_vcpu_runs_old_kvmclock != tmp)
2869
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2870 2871 2872 2873

			ka->boot_vcpu_runs_old_kvmclock = tmp;
		}

2874
		vcpu->arch.time = data;
2875
		kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2876 2877

		/* we verify if the enable bit is set... */
P
Paolo Bonzini 已提交
2878
		vcpu->arch.pv_time_enabled = false;
2879 2880 2881
		if (!(data & 1))
			break;

P
Paolo Bonzini 已提交
2882
		if (!kvm_gfn_to_hva_cache_init(vcpu->kvm,
2883 2884
		     &vcpu->arch.pv_time, data & ~1ULL,
		     sizeof(struct pvclock_vcpu_time_info)))
2885
			vcpu->arch.pv_time_enabled = true;
2886

2887 2888
		break;
	}
2889 2890 2891 2892
	case MSR_KVM_ASYNC_PF_EN:
		if (kvm_pv_enable_async_pf(vcpu, data))
			return 1;
		break;
G
Glauber Costa 已提交
2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908
	case MSR_KVM_STEAL_TIME:

		if (unlikely(!sched_info_on()))
			return 1;

		if (data & KVM_STEAL_RESERVED_MASK)
			return 1;

		vcpu->arch.st.msr_val = data;

		if (!(data & KVM_MSR_ENABLED))
			break;

		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);

		break;
2909
	case MSR_KVM_PV_EOI_EN:
2910
		if (kvm_lapic_enable_pv_eoi(vcpu, data, sizeof(u8)))
2911 2912
			return 1;
		break;
G
Glauber Costa 已提交
2913

2914 2915 2916 2917 2918 2919 2920 2921
	case MSR_KVM_POLL_CONTROL:
		/* only enable bit supported */
		if (data & (-1ULL << 1))
			return 1;

		vcpu->arch.msr_kvm_poll_control = data;
		break;

H
Huang Ying 已提交
2922 2923
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
2924
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
2925
		return set_msr_mce(vcpu, msr_info);
2926

2927 2928 2929 2930 2931
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
		pr = true; /* fall through */
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
2932
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2933
			return kvm_pmu_set_msr(vcpu, msr_info);
2934 2935

		if (pr || data != 0)
2936 2937
			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
				    "0x%x data 0x%llx\n", msr, data);
2938
		break;
2939 2940 2941 2942 2943
	case MSR_K7_CLK_CTL:
		/*
		 * Ignore all writes to this no longer documented MSR.
		 * Writes are only relevant for old K7 processors,
		 * all pre-dating SVM, but a recommended workaround from
G
Guo Chao 已提交
2944
		 * AMD for these chips. It is possible to specify the
2945 2946 2947 2948
		 * affected processor models on the command line, hence
		 * the need to ignore the workaround.
		 */
		break;
2949
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
2950 2951
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
A
Andrey Smetanin 已提交
2952
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
2953 2954 2955
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
2956 2957
		return kvm_hv_set_msr_common(vcpu, msr, data,
					     msr_info->host_initiated);
2958 2959 2960 2961
	case MSR_IA32_BBL_CR_CTL3:
		/* Drop writes to this legacy MSR -- see rdmsr
		 * counterpart for further detail.
		 */
2962 2963 2964
		if (report_ignored_msrs)
			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
				msr, data);
2965
		break;
2966
	case MSR_AMD64_OSVW_ID_LENGTH:
2967
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2968 2969 2970 2971
			return 1;
		vcpu->arch.osvw.length = data;
		break;
	case MSR_AMD64_OSVW_STATUS:
2972
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2973 2974 2975
			return 1;
		vcpu->arch.osvw.status = data;
		break;
K
Kyle Huey 已提交
2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989
	case MSR_PLATFORM_INFO:
		if (!msr_info->host_initiated ||
		    (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
		     cpuid_fault_enabled(vcpu)))
			return 1;
		vcpu->arch.msr_platform_info = data;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
		     !supports_cpuid_fault(vcpu)))
			return 1;
		vcpu->arch.msr_misc_features_enables = data;
		break;
2990
	default:
E
Ed Swierk 已提交
2991 2992
		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
			return xen_hvm_config(vcpu, data);
2993
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2994
			return kvm_pmu_set_msr(vcpu, msr_info);
2995
		if (!ignore_msrs) {
2996
			vcpu_debug_ratelimited(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n",
2997
				    msr, data);
2998 2999
			return 1;
		} else {
3000 3001 3002 3003
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu,
					"ignored wrmsr: 0x%x data 0x%llx\n",
					msr, data);
3004 3005
			break;
		}
3006 3007 3008 3009 3010
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);

3011
static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
3012 3013
{
	u64 data;
H
Huang Ying 已提交
3014 3015
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
3016 3017 3018 3019

	switch (msr) {
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
H
Huang Ying 已提交
3020 3021
		data = 0;
		break;
3022
	case MSR_IA32_MCG_CAP:
H
Huang Ying 已提交
3023 3024
		data = vcpu->arch.mcg_cap;
		break;
3025
	case MSR_IA32_MCG_CTL:
3026
		if (!(mcg_cap & MCG_CTL_P) && !host)
H
Huang Ying 已提交
3027 3028 3029 3030 3031 3032 3033 3034
			return 1;
		data = vcpu->arch.mcg_ctl;
		break;
	case MSR_IA32_MCG_STATUS:
		data = vcpu->arch.mcg_status;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
3035
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
3036 3037 3038 3039
			u32 offset = array_index_nospec(
				msr - MSR_IA32_MC0_CTL,
				MSR_IA32_MCx_CTL(bank_num) - MSR_IA32_MC0_CTL);

H
Huang Ying 已提交
3040 3041 3042 3043 3044 3045 3046 3047 3048
			data = vcpu->arch.mce_banks[offset];
			break;
		}
		return 1;
	}
	*pdata = data;
	return 0;
}

3049
int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
H
Huang Ying 已提交
3050
{
3051
	switch (msr_info->index) {
H
Huang Ying 已提交
3052
	case MSR_IA32_PLATFORM_ID:
3053
	case MSR_IA32_EBL_CR_POWERON:
3054 3055 3056 3057 3058
	case MSR_IA32_DEBUGCTLMSR:
	case MSR_IA32_LASTBRANCHFROMIP:
	case MSR_IA32_LASTBRANCHTOIP:
	case MSR_IA32_LASTINTFROMIP:
	case MSR_IA32_LASTINTTOIP:
3059
	case MSR_K8_SYSCFG:
3060 3061
	case MSR_K8_TSEG_ADDR:
	case MSR_K8_TSEG_MASK:
3062
	case MSR_VM_HSAVE_PA:
3063
	case MSR_K8_INT_PENDING_MSG:
3064
	case MSR_AMD64_NB_CFG:
3065
	case MSR_FAM10H_MMIO_CONF_BASE:
3066
	case MSR_AMD64_BU_CFG2:
D
Dmitry Bilunov 已提交
3067
	case MSR_IA32_PERF_CTL:
3068
	case MSR_AMD64_DC_CFG:
3069
	case MSR_F15H_EX_CFG:
3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080
	/*
	 * Intel Sandy Bridge CPUs must support the RAPL (running average power
	 * limit) MSRs. Just return 0, as we do not want to expose the host
	 * data here. Do not conditionalize this on CPUID, as KVM does not do
	 * so for existing CPU-specific MSRs.
	 */
	case MSR_RAPL_POWER_UNIT:
	case MSR_PP0_ENERGY_STATUS:	/* Power plane 0 (core) */
	case MSR_PP1_ENERGY_STATUS:	/* Power plane 1 (graphics uncore) */
	case MSR_PKG_ENERGY_STATUS:	/* Total package */
	case MSR_DRAM_ENERGY_STATUS:	/* DRAM controller */
3081
		msr_info->data = 0;
3082
		break;
3083
	case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
3084 3085 3086 3087
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3088
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3089 3090
			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
		msr_info->data = 0;
3091
		break;
3092
	case MSR_IA32_UCODE_REV:
3093
		msr_info->data = vcpu->arch.microcode_version;
3094
		break;
3095 3096 3097 3098 3099 3100
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
			return 1;
		msr_info->data = vcpu->arch.arch_capabilities;
		break;
3101 3102 3103
	case MSR_IA32_POWER_CTL:
		msr_info->data = vcpu->arch.msr_ia32_power_ctl;
		break;
3104 3105 3106
	case MSR_IA32_TSC:
		msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + vcpu->arch.tsc_offset;
		break;
A
Avi Kivity 已提交
3107 3108
	case MSR_MTRRcap:
	case 0x200 ... 0x2ff:
3109
		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
3110
	case 0xcd: /* fsb frequency */
3111
		msr_info->data = 3;
3112
		break;
3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124
		/*
		 * MSR_EBC_FREQUENCY_ID
		 * Conservative value valid for even the basic CPU models.
		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
		 * and 266MHz for model 3, or 4. Set Core Clock
		 * Frequency to System Bus Frequency Ratio to 1 (bits
		 * 31:24) even though these are only valid for CPU
		 * models > 2, however guests may end up dividing or
		 * multiplying by zero otherwise.
		 */
	case MSR_EBC_FREQUENCY_ID:
3125
		msr_info->data = 1 << 24;
3126
		break;
3127
	case MSR_IA32_APICBASE:
3128
		msr_info->data = kvm_get_apic_base(vcpu);
3129
		break;
G
Gleb Natapov 已提交
3130
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
3131
		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
3132
	case MSR_IA32_TSCDEADLINE:
3133
		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
3134
		break;
W
Will Auld 已提交
3135
	case MSR_IA32_TSC_ADJUST:
3136
		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
W
Will Auld 已提交
3137
		break;
3138
	case MSR_IA32_MISC_ENABLE:
3139
		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
3140
		break;
P
Paolo Bonzini 已提交
3141 3142 3143 3144
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		msr_info->data = vcpu->arch.smbase;
3145
		break;
3146 3147 3148
	case MSR_SMI_COUNT:
		msr_info->data = vcpu->arch.smi_count;
		break;
3149 3150
	case MSR_IA32_PERF_STATUS:
		/* TSC increment by tick */
3151
		msr_info->data = 1000ULL;
3152
		/* CPU multiplier */
3153
		msr_info->data |= (((uint64_t)4ULL) << 40);
3154
		break;
3155
	case MSR_EFER:
3156
		msr_info->data = vcpu->arch.efer;
3157
		break;
3158
	case MSR_KVM_WALL_CLOCK:
3159
	case MSR_KVM_WALL_CLOCK_NEW:
3160
		msr_info->data = vcpu->kvm->arch.wall_clock;
3161 3162
		break;
	case MSR_KVM_SYSTEM_TIME:
3163
	case MSR_KVM_SYSTEM_TIME_NEW:
3164
		msr_info->data = vcpu->arch.time;
3165
		break;
3166
	case MSR_KVM_ASYNC_PF_EN:
3167
		msr_info->data = vcpu->arch.apf.msr_val;
3168
		break;
G
Glauber Costa 已提交
3169
	case MSR_KVM_STEAL_TIME:
3170
		msr_info->data = vcpu->arch.st.msr_val;
G
Glauber Costa 已提交
3171
		break;
3172
	case MSR_KVM_PV_EOI_EN:
3173
		msr_info->data = vcpu->arch.pv_eoi.msr_val;
3174
		break;
3175 3176 3177
	case MSR_KVM_POLL_CONTROL:
		msr_info->data = vcpu->arch.msr_kvm_poll_control;
		break;
H
Huang Ying 已提交
3178 3179 3180 3181 3182
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
3183
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3184 3185
		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
				   msr_info->host_initiated);
3186 3187 3188 3189 3190 3191
	case MSR_IA32_XSS:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
			return 1;
		msr_info->data = vcpu->arch.ia32_xss;
		break;
3192 3193 3194 3195 3196 3197 3198 3199 3200 3201
	case MSR_K7_CLK_CTL:
		/*
		 * Provide expected ramp-up count for K7. All other
		 * are set to zero, indicating minimum divisors for
		 * every field.
		 *
		 * This prevents guest kernels on AMD host with CPU
		 * type 6, model 8 and higher from exploding due to
		 * the rdmsr failing.
		 */
3202
		msr_info->data = 0x20000000;
3203
		break;
3204
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
3205 3206
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
A
Andrey Smetanin 已提交
3207
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
3208 3209 3210
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
3211
		return kvm_hv_get_msr_common(vcpu,
3212 3213
					     msr_info->index, &msr_info->data,
					     msr_info->host_initiated);
3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224
	case MSR_IA32_BBL_CR_CTL3:
		/* This legacy MSR exists but isn't fully documented in current
		 * silicon.  It is however accessed by winxp in very narrow
		 * scenarios where it sets bit #19, itself documented as
		 * a "reserved" bit.  Best effort attempt to source coherent
		 * read data here should the balance of the register be
		 * interpreted by the guest:
		 *
		 * L2 cache control register 3: 64GB range, 256KB size,
		 * enabled, latency 0x1, configured
		 */
3225
		msr_info->data = 0xbe702111;
3226
		break;
3227
	case MSR_AMD64_OSVW_ID_LENGTH:
3228
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3229
			return 1;
3230
		msr_info->data = vcpu->arch.osvw.length;
3231 3232
		break;
	case MSR_AMD64_OSVW_STATUS:
3233
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3234
			return 1;
3235
		msr_info->data = vcpu->arch.osvw.status;
3236
		break;
K
Kyle Huey 已提交
3237
	case MSR_PLATFORM_INFO:
3238 3239 3240
		if (!msr_info->host_initiated &&
		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
			return 1;
K
Kyle Huey 已提交
3241 3242 3243 3244 3245
		msr_info->data = vcpu->arch.msr_platform_info;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		msr_info->data = vcpu->arch.msr_misc_features_enables;
		break;
3246 3247 3248
	case MSR_K7_HWCR:
		msr_info->data = vcpu->arch.msr_hwcr;
		break;
3249
	default:
3250
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3251
			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
3252
		if (!ignore_msrs) {
3253 3254
			vcpu_debug_ratelimited(vcpu, "unhandled rdmsr: 0x%x\n",
					       msr_info->index);
3255 3256
			return 1;
		} else {
3257 3258 3259
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n",
					msr_info->index);
3260
			msr_info->data = 0;
3261 3262
		}
		break;
3263 3264 3265 3266 3267
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

3268 3269 3270 3271 3272 3273 3274 3275 3276 3277
/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
3278
	int i;
3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302

	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
3303
	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
3304 3305 3306 3307 3308 3309 3310
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
3311 3312 3313
	entries = memdup_user(user_msrs->entries, size);
	if (IS_ERR(entries)) {
		r = PTR_ERR(entries);
3314
		goto out;
3315
	}
3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327

	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
3328
	kfree(entries);
3329 3330 3331 3332
out:
	return r;
}

3333 3334 3335
static inline bool kvm_can_mwait_in_guest(void)
{
	return boot_cpu_has(X86_FEATURE_MWAIT) &&
3336 3337
		!boot_cpu_has_bug(X86_BUG_MONITOR) &&
		boot_cpu_has(X86_FEATURE_ARAT);
3338 3339
}

3340
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
3341
{
3342
	int r = 0;
3343 3344 3345 3346 3347 3348

	switch (ext) {
	case KVM_CAP_IRQCHIP:
	case KVM_CAP_HLT:
	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
	case KVM_CAP_SET_TSS_ADDR:
3349
	case KVM_CAP_EXT_CPUID:
B
Borislav Petkov 已提交
3350
	case KVM_CAP_EXT_EMUL_CPUID:
3351
	case KVM_CAP_CLOCKSOURCE:
S
Sheng Yang 已提交
3352
	case KVM_CAP_PIT:
3353
	case KVM_CAP_NOP_IO_DELAY:
3354
	case KVM_CAP_MP_STATE:
3355
	case KVM_CAP_SYNC_MMU:
3356
	case KVM_CAP_USER_NMI:
3357
	case KVM_CAP_REINJECT_CONTROL:
3358
	case KVM_CAP_IRQ_INJECT_STATUS:
G
Gregory Haskins 已提交
3359
	case KVM_CAP_IOEVENTFD:
3360
	case KVM_CAP_IOEVENTFD_NO_LENGTH:
3361
	case KVM_CAP_PIT2:
B
Beth Kon 已提交
3362
	case KVM_CAP_PIT_STATE2:
3363
	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
E
Ed Swierk 已提交
3364
	case KVM_CAP_XEN_HVM:
J
Jan Kiszka 已提交
3365
	case KVM_CAP_VCPU_EVENTS:
3366
	case KVM_CAP_HYPERV:
G
Gleb Natapov 已提交
3367
	case KVM_CAP_HYPERV_VAPIC:
3368
	case KVM_CAP_HYPERV_SPIN:
3369
	case KVM_CAP_HYPERV_SYNIC:
3370
	case KVM_CAP_HYPERV_SYNIC2:
3371
	case KVM_CAP_HYPERV_VP_INDEX:
3372
	case KVM_CAP_HYPERV_EVENTFD:
3373
	case KVM_CAP_HYPERV_TLBFLUSH:
3374
	case KVM_CAP_HYPERV_SEND_IPI:
3375
	case KVM_CAP_HYPERV_CPUID:
3376
	case KVM_CAP_PCI_SEGMENT:
3377
	case KVM_CAP_DEBUGREGS:
3378
	case KVM_CAP_X86_ROBUST_SINGLESTEP:
3379
	case KVM_CAP_XSAVE:
3380
	case KVM_CAP_ASYNC_PF:
3381
	case KVM_CAP_GET_TSC_KHZ:
3382
	case KVM_CAP_KVMCLOCK_CTRL:
X
Xiao Guangrong 已提交
3383
	case KVM_CAP_READONLY_MEM:
3384
	case KVM_CAP_HYPERV_TIME:
3385
	case KVM_CAP_IOAPIC_POLARITY_IGNORED:
3386
	case KVM_CAP_TSC_DEADLINE_TIMER:
3387
	case KVM_CAP_DISABLE_QUIRKS:
3388
	case KVM_CAP_SET_BOOT_CPU_ID:
3389
 	case KVM_CAP_SPLIT_IRQCHIP:
3390
	case KVM_CAP_IMMEDIATE_EXIT:
E
Eric Hankland 已提交
3391
	case KVM_CAP_PMU_EVENT_FILTER:
3392
	case KVM_CAP_GET_MSR_FEATURES:
3393
	case KVM_CAP_MSR_PLATFORM_INFO:
3394
	case KVM_CAP_EXCEPTION_PAYLOAD:
3395
	case KVM_CAP_SET_GUEST_DEBUG:
3396 3397
		r = 1;
		break;
K
Ken Hofsass 已提交
3398 3399 3400
	case KVM_CAP_SYNC_REGS:
		r = KVM_SYNC_X86_VALID_FIELDS;
		break;
3401 3402 3403
	case KVM_CAP_ADJUST_CLOCK:
		r = KVM_CLOCK_TSC_STABLE;
		break;
3404
	case KVM_CAP_X86_DISABLE_EXITS:
3405 3406
		r |=  KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE |
		      KVM_X86_DISABLE_EXITS_CSTATE;
3407 3408
		if(kvm_can_mwait_in_guest())
			r |= KVM_X86_DISABLE_EXITS_MWAIT;
3409
		break;
3410 3411 3412 3413 3414 3415 3416 3417 3418
	case KVM_CAP_X86_SMM:
		/* SMBASE is usually relocated above 1M on modern chipsets,
		 * and SMM handlers might indeed rely on 4G segment limits,
		 * so do not report SMM to be available if real mode is
		 * emulated via vm86 mode.  Still, do not go to great lengths
		 * to avoid userspace's usage of the feature, because it is a
		 * fringe case that is not enabled except via specific settings
		 * of the module parameters.
		 */
3419
		r = kvm_x86_ops.has_emulated_msr(MSR_IA32_SMBASE);
3420
		break;
3421
	case KVM_CAP_VAPIC:
3422
		r = !kvm_x86_ops.cpu_has_accelerated_tpr();
3423
		break;
3424
	case KVM_CAP_NR_VCPUS:
3425 3426 3427
		r = KVM_SOFT_MAX_VCPUS;
		break;
	case KVM_CAP_MAX_VCPUS:
3428 3429
		r = KVM_MAX_VCPUS;
		break;
3430 3431 3432
	case KVM_CAP_MAX_VCPU_ID:
		r = KVM_MAX_VCPU_ID;
		break;
3433 3434
	case KVM_CAP_PV_MMU:	/* obsolete */
		r = 0;
3435
		break;
H
Huang Ying 已提交
3436 3437 3438
	case KVM_CAP_MCE:
		r = KVM_MAX_MCE_BANKS;
		break;
3439
	case KVM_CAP_XCRS:
3440
		r = boot_cpu_has(X86_FEATURE_XSAVE);
3441
		break;
3442 3443 3444
	case KVM_CAP_TSC_CONTROL:
		r = kvm_has_tsc_control;
		break;
3445 3446 3447
	case KVM_CAP_X2APIC_API:
		r = KVM_X2APIC_API_VALID_FLAGS;
		break;
3448
	case KVM_CAP_NESTED_STATE:
3449 3450
		r = kvm_x86_ops.nested_ops->get_state ?
			kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;
3451
		break;
3452
	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
3453
		r = kvm_x86_ops.enable_direct_tlbflush != NULL;
3454 3455
		break;
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
3456
		r = kvm_x86_ops.nested_ops->enable_evmcs != NULL;
3457
		break;
3458 3459 3460 3461 3462 3463 3464
	default:
		break;
	}
	return r;

}

3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477
long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {
	case KVM_GET_MSR_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
3478
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
3479 3480
			goto out;
		n = msr_list.nmsrs;
3481
		msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
3482
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
3483 3484
			goto out;
		r = -E2BIG;
J
Jan Kiszka 已提交
3485
		if (n < msr_list.nmsrs)
3486 3487 3488 3489 3490
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
				 num_msrs_to_save * sizeof(u32)))
			goto out;
J
Jan Kiszka 已提交
3491
		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
3492
				 &emulated_msrs,
3493
				 num_emulated_msrs * sizeof(u32)))
3494 3495 3496 3497
			goto out;
		r = 0;
		break;
	}
B
Borislav Petkov 已提交
3498 3499
	case KVM_GET_SUPPORTED_CPUID:
	case KVM_GET_EMULATED_CPUID: {
3500 3501 3502 3503
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
3504
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
3505
			goto out;
B
Borislav Petkov 已提交
3506 3507 3508

		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
					    ioctl);
3509 3510 3511 3512
		if (r)
			goto out;

		r = -EFAULT;
3513
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
3514 3515 3516 3517
			goto out;
		r = 0;
		break;
	}
3518
	case KVM_X86_GET_MCE_CAP_SUPPORTED:
H
Huang Ying 已提交
3519
		r = -EFAULT;
3520 3521
		if (copy_to_user(argp, &kvm_mce_cap_supported,
				 sizeof(kvm_mce_cap_supported)))
H
Huang Ying 已提交
3522 3523 3524
			goto out;
		r = 0;
		break;
3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549
	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned int n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
			goto out;
		n = msr_list.nmsrs;
		msr_list.nmsrs = num_msr_based_features;
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
			goto out;
		r = -E2BIG;
		if (n < msr_list.nmsrs)
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msr_based_features,
				 num_msr_based_features * sizeof(u32)))
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_MSRS:
		r = msr_io(NULL, argp, do_get_msr_feature, 1);
		break;
3550 3551
	default:
		r = -EINVAL;
3552
		break;
3553 3554 3555 3556 3557
	}
out:
	return r;
}

3558 3559 3560 3561 3562 3563 3564
static void wbinvd_ipi(void *garbage)
{
	wbinvd();
}

static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
3565
	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
3566 3567
}

3568 3569
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
3570 3571
	/* Address WBINVD may be executed by guest */
	if (need_emulate_wbinvd(vcpu)) {
3572
		if (kvm_x86_ops.has_wbinvd_exit())
3573 3574 3575 3576 3577 3578
			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
			smp_call_function_single(vcpu->cpu,
					wbinvd_ipi, NULL, 1);
	}

3579
	kvm_x86_ops.vcpu_load(vcpu, cpu);
3580

3581 3582 3583
	/* Save host pkru register if supported */
	vcpu->arch.host_pkru = read_pkru();

3584 3585 3586 3587
	/* Apply any externally detected TSC adjustments (due to suspend) */
	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
		vcpu->arch.tsc_offset_adjustment = 0;
3588
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3589
	}
3590

3591
	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
3592
		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
3593
				rdtsc() - vcpu->arch.last_host_tsc;
Z
Zachary Amsden 已提交
3594 3595
		if (tsc_delta < 0)
			mark_tsc_unstable("KVM discovered backwards TSC");
3596

3597
		if (kvm_check_tsc_unstable()) {
3598
			u64 offset = kvm_compute_tsc_offset(vcpu,
3599
						vcpu->arch.last_guest_tsc);
3600
			kvm_vcpu_write_tsc_offset(vcpu, offset);
Z
Zachary Amsden 已提交
3601 3602
			vcpu->arch.tsc_catchup = 1;
		}
3603 3604 3605 3606

		if (kvm_lapic_hv_timer_in_use(vcpu))
			kvm_lapic_restart_hv_timer(vcpu);

3607 3608 3609 3610 3611
		/*
		 * On a host with synchronized TSC, there is no need to update
		 * kvmclock on vcpu->cpu migration
		 */
		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
3612
			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
Z
Zachary Amsden 已提交
3613
		if (vcpu->cpu != cpu)
3614
			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
Z
Zachary Amsden 已提交
3615
		vcpu->cpu = cpu;
Z
Zachary Amsden 已提交
3616
	}
G
Glauber Costa 已提交
3617 3618

	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3619 3620
}

3621 3622
static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
{
3623 3624 3625
	struct kvm_host_map map;
	struct kvm_steal_time *st;

3626 3627 3628
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

3629
	if (vcpu->arch.st.preempted)
3630 3631
		return;

3632 3633 3634 3635 3636 3637
	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT, &map,
			&vcpu->arch.st.cache, true))
		return;

	st = map.hva +
		offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);
3638

3639
	st->preempted = vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
3640

3641
	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, true);
3642 3643
}

3644 3645
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
3646
	int idx;
3647 3648

	if (vcpu->preempted)
3649
		vcpu->arch.preempted_in_kernel = !kvm_x86_ops.get_cpl(vcpu);
3650

3651 3652 3653 3654 3655 3656 3657 3658 3659
	/*
	 * Disable page faults because we're in atomic context here.
	 * kvm_write_guest_offset_cached() would call might_fault()
	 * that relies on pagefault_disable() to tell if there's a
	 * bug. NOTE: the write to guest memory may not go through if
	 * during postcopy live migration or if there's heavy guest
	 * paging.
	 */
	pagefault_disable();
3660 3661 3662 3663 3664
	/*
	 * kvm_memslots() will be called by
	 * kvm_write_guest_offset_cached() so take the srcu lock.
	 */
	idx = srcu_read_lock(&vcpu->kvm->srcu);
3665
	kvm_steal_time_set_preempted(vcpu);
3666
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
3667
	pagefault_enable();
3668
	kvm_x86_ops.vcpu_put(vcpu);
3669
	vcpu->arch.last_host_tsc = rdtsc();
3670
	/*
3671 3672 3673
	 * If userspace has set any breakpoints or watchpoints, dr6 is restored
	 * on every vmexit, but if not, we might have a stale dr6 from the
	 * guest. do_debug expects dr6 to be cleared after it runs, do the same.
3674
	 */
3675
	set_debugreg(0, 6);
3676 3677 3678 3679 3680
}

static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3681
	if (vcpu->arch.apicv_active)
3682
		kvm_x86_ops.sync_pir_to_irr(vcpu);
3683

3684
	return kvm_apic_get_state(vcpu, s);
3685 3686 3687 3688 3689
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3690 3691 3692 3693 3694
	int r;

	r = kvm_apic_set_state(vcpu, s);
	if (r)
		return r;
3695
	update_cr8_intercept(vcpu);
3696 3697 3698 3699

	return 0;
}

3700 3701 3702 3703 3704 3705
static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
{
	return (!lapic_in_kernel(vcpu) ||
		kvm_apic_accept_pic_intr(vcpu));
}

3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719
/*
 * if userspace requested an interrupt window, check that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
{
	return kvm_arch_interrupt_allowed(vcpu) &&
		!kvm_cpu_has_interrupt(vcpu) &&
		!kvm_event_needs_reinjection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);
}

3720 3721 3722
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
{
3723
	if (irq->irq >= KVM_NR_INTERRUPTS)
3724
		return -EINVAL;
3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736

	if (!irqchip_in_kernel(vcpu->kvm)) {
		kvm_queue_interrupt(vcpu, irq->irq, false);
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		return 0;
	}

	/*
	 * With in-kernel LAPIC, we only use this to inject EXTINT, so
	 * fail for in-kernel 8259.
	 */
	if (pic_in_kernel(vcpu->kvm))
3737 3738
		return -ENXIO;

3739 3740
	if (vcpu->arch.pending_external_vector != -1)
		return -EEXIST;
3741

3742
	vcpu->arch.pending_external_vector = irq->irq;
3743
	kvm_make_request(KVM_REQ_EVENT, vcpu);
3744 3745 3746
	return 0;
}

3747 3748 3749 3750 3751 3752 3753
static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
{
	kvm_inject_nmi(vcpu);

	return 0;
}

3754 3755
static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
{
P
Paolo Bonzini 已提交
3756 3757
	kvm_make_request(KVM_REQ_SMI, vcpu);

3758 3759 3760
	return 0;
}

3761 3762 3763 3764 3765 3766 3767 3768 3769
static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
					   struct kvm_tpr_access_ctl *tac)
{
	if (tac->flags)
		return -EINVAL;
	vcpu->arch.tpr_access_reporting = !!tac->enabled;
	return 0;
}

H
Huang Ying 已提交
3770 3771 3772 3773 3774 3775 3776
static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
					u64 mcg_cap)
{
	int r;
	unsigned bank_num = mcg_cap & 0xff, bank;

	r = -EINVAL;
3777
	if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
H
Huang Ying 已提交
3778
		goto out;
3779
	if (mcg_cap & ~(kvm_mce_cap_supported | 0xff | 0xff0000))
H
Huang Ying 已提交
3780 3781 3782 3783 3784 3785 3786 3787 3788
		goto out;
	r = 0;
	vcpu->arch.mcg_cap = mcg_cap;
	/* Init IA32_MCG_CTL to all 1s */
	if (mcg_cap & MCG_CTL_P)
		vcpu->arch.mcg_ctl = ~(u64)0;
	/* Init IA32_MCi_CTL to all 1s */
	for (bank = 0; bank < bank_num; bank++)
		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
3789

3790
	kvm_x86_ops.setup_mce(vcpu);
H
Huang Ying 已提交
3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819
out:
	return r;
}

static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
				      struct kvm_x86_mce *mce)
{
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
	u64 *banks = vcpu->arch.mce_banks;

	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
		return -EINVAL;
	/*
	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
	 * reporting is disabled
	 */
	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
	    vcpu->arch.mcg_ctl != ~(u64)0)
		return 0;
	banks += 4 * mce->bank;
	/*
	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
	 * reporting is disabled for the bank
	 */
	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
		return 0;
	if (mce->status & MCI_STATUS_UC) {
		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
3820
		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
3821
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
H
Huang Ying 已提交
3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842
			return 0;
		}
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		vcpu->arch.mcg_status = mce->mcg_status;
		banks[1] = mce->status;
		kvm_queue_exception(vcpu, MC_VECTOR);
	} else if (!(banks[1] & MCI_STATUS_VAL)
		   || !(banks[1] & MCI_STATUS_UC)) {
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		banks[1] = mce->status;
	} else
		banks[1] |= MCI_STATUS_OVER;
	return 0;
}

J
Jan Kiszka 已提交
3843 3844 3845
static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
					       struct kvm_vcpu_events *events)
{
A
Avi Kivity 已提交
3846
	process_nmi(vcpu);
3847

3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862
	/*
	 * In guest mode, payload delivery should be deferred,
	 * so that the L1 hypervisor can intercept #PF before
	 * CR2 is modified (or intercept #DB before DR6 is
	 * modified under nVMX). Unless the per-VM capability,
	 * KVM_CAP_EXCEPTION_PAYLOAD, is set, we may not defer the delivery of
	 * an exception payload and handle after a KVM_GET_VCPU_EVENTS. Since we
	 * opportunistically defer the exception payload, deliver it if the
	 * capability hasn't been requested before processing a
	 * KVM_GET_VCPU_EVENTS.
	 */
	if (!vcpu->kvm->arch.exception_payload_enabled &&
	    vcpu->arch.exception.pending && vcpu->arch.exception.has_payload)
		kvm_deliver_exception_payload(vcpu);

3863
	/*
3864 3865 3866 3867
	 * The API doesn't provide the instruction length for software
	 * exceptions, so don't report them. As long as the guest RIP
	 * isn't advanced, we should expect to encounter the exception
	 * again.
3868
	 */
3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883
	if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {
		events->exception.injected = 0;
		events->exception.pending = 0;
	} else {
		events->exception.injected = vcpu->arch.exception.injected;
		events->exception.pending = vcpu->arch.exception.pending;
		/*
		 * For ABI compatibility, deliberately conflate
		 * pending and injected exceptions when
		 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
		 */
		if (!vcpu->kvm->arch.exception_payload_enabled)
			events->exception.injected |=
				vcpu->arch.exception.pending;
	}
J
Jan Kiszka 已提交
3884 3885 3886
	events->exception.nr = vcpu->arch.exception.nr;
	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
	events->exception.error_code = vcpu->arch.exception.error_code;
3887 3888
	events->exception_has_payload = vcpu->arch.exception.has_payload;
	events->exception_payload = vcpu->arch.exception.payload;
J
Jan Kiszka 已提交
3889

3890
	events->interrupt.injected =
3891
		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
J
Jan Kiszka 已提交
3892
	events->interrupt.nr = vcpu->arch.interrupt.nr;
3893
	events->interrupt.soft = 0;
3894
	events->interrupt.shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);
J
Jan Kiszka 已提交
3895 3896

	events->nmi.injected = vcpu->arch.nmi_injected;
A
Avi Kivity 已提交
3897
	events->nmi.pending = vcpu->arch.nmi_pending != 0;
3898
	events->nmi.masked = kvm_x86_ops.get_nmi_mask(vcpu);
3899
	events->nmi.pad = 0;
J
Jan Kiszka 已提交
3900

3901
	events->sipi_vector = 0; /* never valid when reporting to user space */
J
Jan Kiszka 已提交
3902

3903 3904 3905 3906 3907 3908
	events->smi.smm = is_smm(vcpu);
	events->smi.pending = vcpu->arch.smi_pending;
	events->smi.smm_inside_nmi =
		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
	events->smi.latched_init = kvm_lapic_latched_init(vcpu);

3909
	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
3910 3911
			 | KVM_VCPUEVENT_VALID_SHADOW
			 | KVM_VCPUEVENT_VALID_SMM);
3912 3913 3914
	if (vcpu->kvm->arch.exception_payload_enabled)
		events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;

3915
	memset(&events->reserved, 0, sizeof(events->reserved));
J
Jan Kiszka 已提交
3916 3917
}

3918
static void kvm_smm_changed(struct kvm_vcpu *vcpu);
3919

J
Jan Kiszka 已提交
3920 3921 3922
static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
					      struct kvm_vcpu_events *events)
{
3923
	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
3924
			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
3925
			      | KVM_VCPUEVENT_VALID_SHADOW
3926 3927
			      | KVM_VCPUEVENT_VALID_SMM
			      | KVM_VCPUEVENT_VALID_PAYLOAD))
J
Jan Kiszka 已提交
3928 3929
		return -EINVAL;

3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943
	if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
		if (!vcpu->kvm->arch.exception_payload_enabled)
			return -EINVAL;
		if (events->exception.pending)
			events->exception.injected = 0;
		else
			events->exception_has_payload = 0;
	} else {
		events->exception.pending = 0;
		events->exception_has_payload = 0;
	}

	if ((events->exception.injected || events->exception.pending) &&
	    (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
3944 3945
		return -EINVAL;

3946 3947 3948 3949 3950 3951
	/* INITs are latched while in SMM */
	if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
	    (events->smi.smm || events->smi.pending) &&
	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
		return -EINVAL;

A
Avi Kivity 已提交
3952
	process_nmi(vcpu);
3953 3954
	vcpu->arch.exception.injected = events->exception.injected;
	vcpu->arch.exception.pending = events->exception.pending;
J
Jan Kiszka 已提交
3955 3956 3957
	vcpu->arch.exception.nr = events->exception.nr;
	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
	vcpu->arch.exception.error_code = events->exception.error_code;
3958 3959
	vcpu->arch.exception.has_payload = events->exception_has_payload;
	vcpu->arch.exception.payload = events->exception_payload;
J
Jan Kiszka 已提交
3960

3961
	vcpu->arch.interrupt.injected = events->interrupt.injected;
J
Jan Kiszka 已提交
3962 3963
	vcpu->arch.interrupt.nr = events->interrupt.nr;
	vcpu->arch.interrupt.soft = events->interrupt.soft;
3964
	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
3965
		kvm_x86_ops.set_interrupt_shadow(vcpu,
3966
						  events->interrupt.shadow);
J
Jan Kiszka 已提交
3967 3968

	vcpu->arch.nmi_injected = events->nmi.injected;
3969 3970
	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
		vcpu->arch.nmi_pending = events->nmi.pending;
3971
	kvm_x86_ops.set_nmi_mask(vcpu, events->nmi.masked);
J
Jan Kiszka 已提交
3972

3973
	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
3974
	    lapic_in_kernel(vcpu))
3975
		vcpu->arch.apic->sipi_vector = events->sipi_vector;
J
Jan Kiszka 已提交
3976

3977
	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
3978 3979 3980 3981 3982 3983 3984
		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
			if (events->smi.smm)
				vcpu->arch.hflags |= HF_SMM_MASK;
			else
				vcpu->arch.hflags &= ~HF_SMM_MASK;
			kvm_smm_changed(vcpu);
		}
3985

3986
		vcpu->arch.smi_pending = events->smi.pending;
3987 3988 3989 3990

		if (events->smi.smm) {
			if (events->smi.smm_inside_nmi)
				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
3991
			else
3992
				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
3993 3994 3995 3996 3997 3998 3999
		}

		if (lapic_in_kernel(vcpu)) {
			if (events->smi.latched_init)
				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
			else
				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
4000 4001 4002
		}
	}

4003 4004
	kvm_make_request(KVM_REQ_EVENT, vcpu);

J
Jan Kiszka 已提交
4005 4006 4007
	return 0;
}

4008 4009 4010
static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
					     struct kvm_debugregs *dbgregs)
{
J
Jan Kiszka 已提交
4011 4012
	unsigned long val;

4013
	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
4014
	kvm_get_dr(vcpu, 6, &val);
J
Jan Kiszka 已提交
4015
	dbgregs->dr6 = val;
4016 4017
	dbgregs->dr7 = vcpu->arch.dr7;
	dbgregs->flags = 0;
4018
	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
4019 4020 4021 4022 4023 4024 4025 4026
}

static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
					    struct kvm_debugregs *dbgregs)
{
	if (dbgregs->flags)
		return -EINVAL;

4027 4028 4029 4030 4031
	if (dbgregs->dr6 & ~0xffffffffull)
		return -EINVAL;
	if (dbgregs->dr7 & ~0xffffffffull)
		return -EINVAL;

4032
	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
4033
	kvm_update_dr0123(vcpu);
4034 4035
	vcpu->arch.dr6 = dbgregs->dr6;
	vcpu->arch.dr7 = dbgregs->dr7;
4036
	kvm_update_dr7(vcpu);
4037 4038 4039 4040

	return 0;
}

4041 4042 4043 4044
#define XSTATE_COMPACTION_ENABLED (1ULL << 63)

static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu)
{
4045
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
4046
	u64 xstate_bv = xsave->header.xfeatures;
4047 4048 4049 4050 4051 4052 4053 4054 4055
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(dest, xsave, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV */
4056
	xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;
4057 4058 4059 4060 4061 4062
	*(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv;

	/*
	 * Copy each region from the possibly compacted offset to the
	 * non-compacted offset.
	 */
D
Dave Hansen 已提交
4063
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
4064
	while (valid) {
4065 4066 4067
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *src = get_xsave_addr(xsave, xfeature_nr);
4068 4069 4070

		if (src) {
			u32 size, offset, ecx, edx;
4071
			cpuid_count(XSTATE_CPUID, xfeature_nr,
4072
				    &size, &offset, &ecx, &edx);
4073
			if (xfeature_nr == XFEATURE_PKRU)
4074 4075 4076 4077 4078
				memcpy(dest + offset, &vcpu->arch.pkru,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest + offset, src, size);

4079 4080
		}

4081
		valid -= xfeature_mask;
4082 4083 4084 4085 4086
	}
}

static void load_xsave(struct kvm_vcpu *vcpu, u8 *src)
{
4087
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
4088 4089 4090 4091 4092 4093 4094 4095 4096 4097
	u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET);
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(xsave, src, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV and possibly XCOMP_BV.  */
4098
	xsave->header.xfeatures = xstate_bv;
4099
	if (boot_cpu_has(X86_FEATURE_XSAVES))
4100
		xsave->header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED;
4101 4102 4103 4104 4105

	/*
	 * Copy each region from the non-compacted offset to the
	 * possibly compacted offset.
	 */
D
Dave Hansen 已提交
4106
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
4107
	while (valid) {
4108 4109 4110
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *dest = get_xsave_addr(xsave, xfeature_nr);
4111 4112 4113

		if (dest) {
			u32 size, offset, ecx, edx;
4114
			cpuid_count(XSTATE_CPUID, xfeature_nr,
4115
				    &size, &offset, &ecx, &edx);
4116
			if (xfeature_nr == XFEATURE_PKRU)
4117 4118 4119 4120
				memcpy(&vcpu->arch.pkru, src + offset,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest, src + offset, size);
4121
		}
4122

4123
		valid -= xfeature_mask;
4124 4125 4126
	}
}

4127 4128 4129
static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
					 struct kvm_xsave *guest_xsave)
{
4130
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
4131 4132
		memset(guest_xsave, 0, sizeof(struct kvm_xsave));
		fill_xsave((u8 *) guest_xsave->region, vcpu);
4133
	} else {
4134
		memcpy(guest_xsave->region,
4135
			&vcpu->arch.guest_fpu->state.fxsave,
4136
			sizeof(struct fxregs_state));
4137
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
D
Dave Hansen 已提交
4138
			XFEATURE_MASK_FPSSE;
4139 4140 4141
	}
}

4142 4143
#define XSAVE_MXCSR_OFFSET 24

4144 4145 4146 4147 4148
static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
					struct kvm_xsave *guest_xsave)
{
	u64 xstate_bv =
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
4149
	u32 mxcsr = *(u32 *)&guest_xsave->region[XSAVE_MXCSR_OFFSET / sizeof(u32)];
4150

4151
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
4152 4153 4154 4155 4156
		/*
		 * Here we allow setting states that are not present in
		 * CPUID leaf 0xD, index 0, EDX:EAX.  This is for compatibility
		 * with old userspace.
		 */
4157
		if (xstate_bv & ~supported_xcr0 || mxcsr & ~mxcsr_feature_mask)
4158
			return -EINVAL;
4159
		load_xsave(vcpu, (u8 *)guest_xsave->region);
4160
	} else {
4161 4162
		if (xstate_bv & ~XFEATURE_MASK_FPSSE ||
			mxcsr & ~mxcsr_feature_mask)
4163
			return -EINVAL;
4164
		memcpy(&vcpu->arch.guest_fpu->state.fxsave,
4165
			guest_xsave->region, sizeof(struct fxregs_state));
4166 4167 4168 4169 4170 4171 4172
	}
	return 0;
}

static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
					struct kvm_xcrs *guest_xcrs)
{
4173
	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188
		guest_xcrs->nr_xcrs = 0;
		return;
	}

	guest_xcrs->nr_xcrs = 1;
	guest_xcrs->flags = 0;
	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
}

static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
				       struct kvm_xcrs *guest_xcrs)
{
	int i, r = 0;

4189
	if (!boot_cpu_has(X86_FEATURE_XSAVE))
4190 4191 4192 4193 4194 4195 4196
		return -EINVAL;

	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
		return -EINVAL;

	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
		/* Only support XCR0 currently */
P
Paolo Bonzini 已提交
4197
		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
4198
			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
P
Paolo Bonzini 已提交
4199
				guest_xcrs->xcrs[i].value);
4200 4201 4202 4203 4204 4205 4206
			break;
		}
	if (r)
		r = -EINVAL;
	return r;
}

4207 4208 4209 4210 4211 4212 4213 4214
/*
 * kvm_set_guest_paused() indicates to the guest kernel that it has been
 * stopped by the hypervisor.  This function will be called from the host only.
 * EINVAL is returned when the host attempts to set the flag for a guest that
 * does not support pv clocks.
 */
static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
{
4215
	if (!vcpu->arch.pv_time_enabled)
4216
		return -EINVAL;
4217
	vcpu->arch.pvclock_set_guest_stopped_request = true;
4218 4219 4220 4221
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	return 0;
}

4222 4223 4224
static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
				     struct kvm_enable_cap *cap)
{
4225 4226 4227 4228
	int r;
	uint16_t vmcs_version;
	void __user *user_ptr;

4229 4230 4231 4232
	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
4233 4234 4235
	case KVM_CAP_HYPERV_SYNIC2:
		if (cap->args[0])
			return -EINVAL;
4236 4237
		/* fall through */

4238
	case KVM_CAP_HYPERV_SYNIC:
4239 4240
		if (!irqchip_in_kernel(vcpu->kvm))
			return -EINVAL;
4241 4242
		return kvm_hv_activate_synic(vcpu, cap->cap ==
					     KVM_CAP_HYPERV_SYNIC2);
4243
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
4244
		if (!kvm_x86_ops.nested_ops->enable_evmcs)
4245
			return -ENOTTY;
4246
		r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);
4247 4248 4249 4250 4251 4252 4253
		if (!r) {
			user_ptr = (void __user *)(uintptr_t)cap->args[0];
			if (copy_to_user(user_ptr, &vmcs_version,
					 sizeof(vmcs_version)))
				r = -EFAULT;
		}
		return r;
4254
	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
4255
		if (!kvm_x86_ops.enable_direct_tlbflush)
4256 4257
			return -ENOTTY;

4258
		return kvm_x86_ops.enable_direct_tlbflush(vcpu);
4259

4260 4261 4262 4263 4264
	default:
		return -EINVAL;
	}
}

4265 4266 4267 4268 4269 4270
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r;
4271 4272 4273 4274 4275 4276 4277
	union {
		struct kvm_lapic_state *lapic;
		struct kvm_xsave *xsave;
		struct kvm_xcrs *xcrs;
		void *buffer;
	} u;

4278 4279
	vcpu_load(vcpu);

4280
	u.buffer = NULL;
4281 4282
	switch (ioctl) {
	case KVM_GET_LAPIC: {
4283
		r = -EINVAL;
4284
		if (!lapic_in_kernel(vcpu))
4285
			goto out;
4286 4287
		u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
				GFP_KERNEL_ACCOUNT);
4288

4289
		r = -ENOMEM;
4290
		if (!u.lapic)
4291
			goto out;
4292
		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
4293 4294 4295
		if (r)
			goto out;
		r = -EFAULT;
4296
		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
4297 4298 4299 4300 4301
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
4302
		r = -EINVAL;
4303
		if (!lapic_in_kernel(vcpu))
4304
			goto out;
4305
		u.lapic = memdup_user(argp, sizeof(*u.lapic));
4306 4307 4308 4309
		if (IS_ERR(u.lapic)) {
			r = PTR_ERR(u.lapic);
			goto out_nofree;
		}
4310

4311
		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
4312 4313
		break;
	}
4314 4315 4316 4317
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
4318
		if (copy_from_user(&irq, argp, sizeof(irq)))
4319 4320 4321 4322
			goto out;
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
		break;
	}
4323 4324 4325 4326
	case KVM_NMI: {
		r = kvm_vcpu_ioctl_nmi(vcpu);
		break;
	}
4327 4328 4329 4330
	case KVM_SMI: {
		r = kvm_vcpu_ioctl_smi(vcpu);
		break;
	}
4331 4332 4333 4334 4335
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
4336
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4337 4338 4339 4340
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		break;
	}
4341 4342 4343 4344 4345
	case KVM_SET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4346
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4347 4348
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
4349
					      cpuid_arg->entries);
4350 4351 4352 4353 4354 4355 4356
		break;
	}
	case KVM_GET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4357
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4358 4359
			goto out;
		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
4360
					      cpuid_arg->entries);
4361 4362 4363
		if (r)
			goto out;
		r = -EFAULT;
4364
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4365 4366 4367 4368
			goto out;
		r = 0;
		break;
	}
4369 4370
	case KVM_GET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4371
		r = msr_io(vcpu, argp, do_get_msr, 1);
4372
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4373
		break;
4374 4375 4376
	}
	case KVM_SET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4377
		r = msr_io(vcpu, argp, do_set_msr, 0);
4378
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4379
		break;
4380
	}
4381 4382 4383 4384
	case KVM_TPR_ACCESS_REPORTING: {
		struct kvm_tpr_access_ctl tac;

		r = -EFAULT;
4385
		if (copy_from_user(&tac, argp, sizeof(tac)))
4386 4387 4388 4389 4390
			goto out;
		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
		if (r)
			goto out;
		r = -EFAULT;
4391
		if (copy_to_user(argp, &tac, sizeof(tac)))
4392 4393 4394 4395
			goto out;
		r = 0;
		break;
	};
A
Avi Kivity 已提交
4396 4397
	case KVM_SET_VAPIC_ADDR: {
		struct kvm_vapic_addr va;
4398
		int idx;
A
Avi Kivity 已提交
4399 4400

		r = -EINVAL;
4401
		if (!lapic_in_kernel(vcpu))
A
Avi Kivity 已提交
4402 4403
			goto out;
		r = -EFAULT;
4404
		if (copy_from_user(&va, argp, sizeof(va)))
A
Avi Kivity 已提交
4405
			goto out;
4406
		idx = srcu_read_lock(&vcpu->kvm->srcu);
4407
		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
4408
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
A
Avi Kivity 已提交
4409 4410
		break;
	}
H
Huang Ying 已提交
4411 4412 4413 4414
	case KVM_X86_SETUP_MCE: {
		u64 mcg_cap;

		r = -EFAULT;
4415
		if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
H
Huang Ying 已提交
4416 4417 4418 4419 4420 4421 4422 4423
			goto out;
		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
		break;
	}
	case KVM_X86_SET_MCE: {
		struct kvm_x86_mce mce;

		r = -EFAULT;
4424
		if (copy_from_user(&mce, argp, sizeof(mce)))
H
Huang Ying 已提交
4425 4426 4427 4428
			goto out;
		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
		break;
	}
J
Jan Kiszka 已提交
4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);

		r = -EFAULT;
		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		r = -EFAULT;
		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
			break;

		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
		break;
	}
4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472
	case KVM_GET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);

		r = -EFAULT;
		if (copy_to_user(argp, &dbgregs,
				 sizeof(struct kvm_debugregs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		r = -EFAULT;
		if (copy_from_user(&dbgregs, argp,
				   sizeof(struct kvm_debugregs)))
			break;

		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
		break;
	}
4473
	case KVM_GET_XSAVE: {
4474
		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
4475
		r = -ENOMEM;
4476
		if (!u.xsave)
4477 4478
			break;

4479
		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
4480 4481

		r = -EFAULT;
4482
		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
4483 4484 4485 4486 4487
			break;
		r = 0;
		break;
	}
	case KVM_SET_XSAVE: {
4488
		u.xsave = memdup_user(argp, sizeof(*u.xsave));
4489 4490 4491 4492
		if (IS_ERR(u.xsave)) {
			r = PTR_ERR(u.xsave);
			goto out_nofree;
		}
4493

4494
		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
4495 4496 4497
		break;
	}
	case KVM_GET_XCRS: {
4498
		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
4499
		r = -ENOMEM;
4500
		if (!u.xcrs)
4501 4502
			break;

4503
		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
4504 4505

		r = -EFAULT;
4506
		if (copy_to_user(argp, u.xcrs,
4507 4508 4509 4510 4511 4512
				 sizeof(struct kvm_xcrs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XCRS: {
4513
		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
4514 4515 4516 4517
		if (IS_ERR(u.xcrs)) {
			r = PTR_ERR(u.xcrs);
			goto out_nofree;
		}
4518

4519
		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
4520 4521
		break;
	}
4522 4523 4524 4525 4526 4527 4528 4529 4530
	case KVM_SET_TSC_KHZ: {
		u32 user_tsc_khz;

		r = -EINVAL;
		user_tsc_khz = (u32)arg;

		if (user_tsc_khz >= kvm_max_guest_tsc_khz)
			goto out;

4531 4532 4533
		if (user_tsc_khz == 0)
			user_tsc_khz = tsc_khz;

4534 4535
		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
			r = 0;
4536 4537 4538 4539

		goto out;
	}
	case KVM_GET_TSC_KHZ: {
4540
		r = vcpu->arch.virtual_tsc_khz;
4541 4542
		goto out;
	}
4543 4544 4545 4546
	case KVM_KVMCLOCK_CTRL: {
		r = kvm_set_guest_paused(vcpu);
		goto out;
	}
4547 4548 4549 4550 4551 4552 4553 4554 4555
	case KVM_ENABLE_CAP: {
		struct kvm_enable_cap cap;

		r = -EFAULT;
		if (copy_from_user(&cap, argp, sizeof(cap)))
			goto out;
		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
		break;
	}
4556 4557 4558 4559 4560
	case KVM_GET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		u32 user_data_size;

		r = -EINVAL;
4561
		if (!kvm_x86_ops.nested_ops->get_state)
4562 4563 4564
			break;

		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
4565
		r = -EFAULT;
4566
		if (get_user(user_data_size, &user_kvm_nested_state->size))
4567
			break;
4568

4569 4570
		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
						     user_data_size);
4571
		if (r < 0)
4572
			break;
4573 4574 4575

		if (r > user_data_size) {
			if (put_user(r, &user_kvm_nested_state->size))
4576 4577 4578 4579
				r = -EFAULT;
			else
				r = -E2BIG;
			break;
4580
		}
4581

4582 4583 4584 4585 4586 4587
		r = 0;
		break;
	}
	case KVM_SET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		struct kvm_nested_state kvm_state;
4588
		int idx;
4589 4590

		r = -EINVAL;
4591
		if (!kvm_x86_ops.nested_ops->set_state)
4592 4593
			break;

4594
		r = -EFAULT;
4595
		if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
4596
			break;
4597

4598
		r = -EINVAL;
4599
		if (kvm_state.size < sizeof(kvm_state))
4600
			break;
4601 4602

		if (kvm_state.flags &
4603 4604
		    ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
		      | KVM_STATE_NESTED_EVMCS))
4605
			break;
4606 4607

		/* nested_run_pending implies guest_mode.  */
4608 4609
		if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
		    && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
4610
			break;
4611

4612
		idx = srcu_read_lock(&vcpu->kvm->srcu);
4613
		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
4614
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4615 4616
		break;
	}
4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635
	case KVM_GET_SUPPORTED_HV_CPUID: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
			goto out;

		r = kvm_vcpu_ioctl_get_hv_cpuid(vcpu, &cpuid,
						cpuid_arg->entries);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
			goto out;
		r = 0;
		break;
	}
4636 4637 4638 4639
	default:
		r = -EINVAL;
	}
out:
4640
	kfree(u.buffer);
4641 4642
out_nofree:
	vcpu_put(vcpu);
4643 4644 4645
	return r;
}

4646
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4647 4648 4649 4650
{
	return VM_FAULT_SIGBUS;
}

4651 4652 4653 4654 4655
static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
{
	int ret;

	if (addr > (unsigned int)(-3 * PAGE_SIZE))
4656
		return -EINVAL;
4657
	ret = kvm_x86_ops.set_tss_addr(kvm, addr);
4658 4659 4660
	return ret;
}

4661 4662 4663
static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
					      u64 ident_addr)
{
4664
	return kvm_x86_ops.set_identity_map_addr(kvm, ident_addr);
4665 4666
}

4667
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
4668
					 unsigned long kvm_nr_mmu_pages)
4669 4670 4671 4672
{
	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
		return -EINVAL;

4673
	mutex_lock(&kvm->slots_lock);
4674 4675

	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
4676
	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
4677

4678
	mutex_unlock(&kvm->slots_lock);
4679 4680 4681
	return 0;
}

4682
static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
4683
{
4684
	return kvm->arch.n_max_mmu_pages;
4685 4686 4687 4688
}

static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4689
	struct kvm_pic *pic = kvm->arch.vpic;
4690 4691 4692 4693 4694
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4695
		memcpy(&chip->chip.pic, &pic->pics[0],
4696 4697 4698
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4699
		memcpy(&chip->chip.pic, &pic->pics[1],
4700 4701 4702
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_IOAPIC:
4703
		kvm_get_ioapic(kvm, &chip->chip.ioapic);
4704 4705 4706 4707 4708 4709 4710 4711 4712 4713
		break;
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4714
	struct kvm_pic *pic = kvm->arch.vpic;
4715 4716 4717 4718 4719
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4720 4721
		spin_lock(&pic->lock);
		memcpy(&pic->pics[0], &chip->chip.pic,
4722
			sizeof(struct kvm_pic_state));
4723
		spin_unlock(&pic->lock);
4724 4725
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4726 4727
		spin_lock(&pic->lock);
		memcpy(&pic->pics[1], &chip->chip.pic,
4728
			sizeof(struct kvm_pic_state));
4729
		spin_unlock(&pic->lock);
4730 4731
		break;
	case KVM_IRQCHIP_IOAPIC:
4732
		kvm_set_ioapic(kvm, &chip->chip.ioapic);
4733 4734 4735 4736 4737
		break;
	default:
		r = -EINVAL;
		break;
	}
4738
	kvm_pic_update_irq(pic);
4739 4740 4741
	return r;
}

4742 4743
static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4744 4745 4746 4747 4748 4749 4750
	struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;

	BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));

	mutex_lock(&kps->lock);
	memcpy(ps, &kps->channels, sizeof(*ps));
	mutex_unlock(&kps->lock);
4751
	return 0;
4752 4753 4754 4755
}

static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4756
	int i;
4757 4758 4759
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
4760
	memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
4761
	for (i = 0; i < 3; i++)
4762 4763
		kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
	mutex_unlock(&pit->pit_state.lock);
4764
	return 0;
B
Beth Kon 已提交
4765 4766 4767 4768 4769 4770 4771 4772 4773
}

static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
		sizeof(ps->channels));
	ps->flags = kvm->arch.vpit->pit_state.flags;
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
4774
	memset(&ps->reserved, 0, sizeof(ps->reserved));
4775
	return 0;
B
Beth Kon 已提交
4776 4777 4778 4779
}

static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
4780
	int start = 0;
4781
	int i;
B
Beth Kon 已提交
4782
	u32 prev_legacy, cur_legacy;
4783 4784 4785 4786
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
	prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
B
Beth Kon 已提交
4787 4788 4789
	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
	if (!prev_legacy && cur_legacy)
		start = 1;
4790 4791 4792
	memcpy(&pit->pit_state.channels, &ps->channels,
	       sizeof(pit->pit_state.channels));
	pit->pit_state.flags = ps->flags;
4793
	for (i = 0; i < 3; i++)
4794
		kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
4795
				   start && i == 0);
4796
	mutex_unlock(&pit->pit_state.lock);
4797
	return 0;
4798 4799
}

4800 4801 4802
static int kvm_vm_ioctl_reinject(struct kvm *kvm,
				 struct kvm_reinject_control *control)
{
4803 4804 4805 4806 4807 4808 4809 4810 4811
	struct kvm_pit *pit = kvm->arch.vpit;

	/* pit->pit_state.lock was overloaded to prevent userspace from getting
	 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
	 * ioctls in parallel.  Use a separate lock if that ioctl isn't rare.
	 */
	mutex_lock(&pit->pit_state.lock);
	kvm_pit_set_reinject(pit, control->pit_reinject);
	mutex_unlock(&pit->pit_state.lock);
4812

4813 4814 4815
	return 0;
}

4816
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
4817
{
4818 4819 4820
	/*
	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
	 */
4821 4822
	if (kvm_x86_ops.flush_log_dirty)
		kvm_x86_ops.flush_log_dirty(kvm);
4823 4824
}

4825 4826
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
			bool line_status)
4827 4828 4829 4830 4831
{
	if (!irqchip_in_kernel(kvm))
		return -ENXIO;

	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
4832 4833
					irq_event->irq, irq_event->level,
					line_status);
4834 4835 4836
	return 0;
}

4837 4838
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
			    struct kvm_enable_cap *cap)
4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849
{
	int r;

	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
	case KVM_CAP_DISABLE_QUIRKS:
		kvm->arch.disabled_quirks = cap->args[0];
		r = 0;
		break;
4850 4851
	case KVM_CAP_SPLIT_IRQCHIP: {
		mutex_lock(&kvm->lock);
4852 4853 4854
		r = -EINVAL;
		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
			goto split_irqchip_unlock;
4855 4856 4857
		r = -EEXIST;
		if (irqchip_in_kernel(kvm))
			goto split_irqchip_unlock;
P
Paolo Bonzini 已提交
4858
		if (kvm->created_vcpus)
4859 4860
			goto split_irqchip_unlock;
		r = kvm_setup_empty_irq_routing(kvm);
4861
		if (r)
4862 4863 4864
			goto split_irqchip_unlock;
		/* Pairs with irqchip_in_kernel. */
		smp_wmb();
4865
		kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
4866
		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
4867 4868 4869 4870 4871
		r = 0;
split_irqchip_unlock:
		mutex_unlock(&kvm->lock);
		break;
	}
4872 4873 4874 4875 4876 4877 4878
	case KVM_CAP_X2APIC_API:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
			break;

		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
			kvm->arch.x2apic_format = true;
4879 4880
		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
			kvm->arch.x2apic_broadcast_quirk_disabled = true;
4881 4882 4883

		r = 0;
		break;
4884 4885 4886 4887 4888 4889 4890 4891
	case KVM_CAP_X86_DISABLE_EXITS:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
			break;

		if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
			kvm_can_mwait_in_guest())
			kvm->arch.mwait_in_guest = true;
M
Michael S. Tsirkin 已提交
4892
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
4893
			kvm->arch.hlt_in_guest = true;
4894 4895
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
			kvm->arch.pause_in_guest = true;
4896 4897
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
			kvm->arch.cstate_in_guest = true;
4898 4899
		r = 0;
		break;
4900 4901 4902
	case KVM_CAP_MSR_PLATFORM_INFO:
		kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
		r = 0;
4903 4904 4905 4906
		break;
	case KVM_CAP_EXCEPTION_PAYLOAD:
		kvm->arch.exception_payload_enabled = cap->args[0];
		r = 0;
4907
		break;
4908 4909 4910 4911 4912 4913 4914
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

4915 4916 4917 4918 4919
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
4920
	int r = -ENOTTY;
4921 4922 4923 4924 4925 4926 4927
	/*
	 * This union makes it completely explicit to gcc-3.x
	 * that these two variables' stack usage should be
	 * combined, not added together.
	 */
	union {
		struct kvm_pit_state ps;
B
Beth Kon 已提交
4928
		struct kvm_pit_state2 ps2;
4929
		struct kvm_pit_config pit_config;
4930
	} u;
4931 4932 4933 4934 4935

	switch (ioctl) {
	case KVM_SET_TSS_ADDR:
		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
		break;
4936 4937 4938
	case KVM_SET_IDENTITY_MAP_ADDR: {
		u64 ident_addr;

4939 4940 4941 4942
		mutex_lock(&kvm->lock);
		r = -EINVAL;
		if (kvm->created_vcpus)
			goto set_identity_unlock;
4943
		r = -EFAULT;
4944
		if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
4945
			goto set_identity_unlock;
4946
		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
4947 4948
set_identity_unlock:
		mutex_unlock(&kvm->lock);
4949 4950
		break;
	}
4951 4952 4953 4954 4955 4956
	case KVM_SET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
		break;
	case KVM_GET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
		break;
4957 4958
	case KVM_CREATE_IRQCHIP: {
		mutex_lock(&kvm->lock);
4959

4960
		r = -EEXIST;
4961
		if (irqchip_in_kernel(kvm))
4962
			goto create_irqchip_unlock;
4963

4964
		r = -EINVAL;
P
Paolo Bonzini 已提交
4965
		if (kvm->created_vcpus)
4966
			goto create_irqchip_unlock;
4967 4968 4969

		r = kvm_pic_init(kvm);
		if (r)
4970
			goto create_irqchip_unlock;
4971 4972 4973 4974

		r = kvm_ioapic_init(kvm);
		if (r) {
			kvm_pic_destroy(kvm);
4975
			goto create_irqchip_unlock;
4976 4977
		}

4978 4979
		r = kvm_setup_default_irq_routing(kvm);
		if (r) {
4980
			kvm_ioapic_destroy(kvm);
4981
			kvm_pic_destroy(kvm);
4982
			goto create_irqchip_unlock;
4983
		}
4984
		/* Write kvm->irq_routing before enabling irqchip_in_kernel. */
4985
		smp_wmb();
4986
		kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
4987 4988
	create_irqchip_unlock:
		mutex_unlock(&kvm->lock);
4989
		break;
4990
	}
S
Sheng Yang 已提交
4991
	case KVM_CREATE_PIT:
4992 4993 4994 4995 4996 4997 4998 4999
		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
		goto create_pit;
	case KVM_CREATE_PIT2:
		r = -EFAULT;
		if (copy_from_user(&u.pit_config, argp,
				   sizeof(struct kvm_pit_config)))
			goto out;
	create_pit:
5000
		mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
5001 5002 5003
		r = -EEXIST;
		if (kvm->arch.vpit)
			goto create_pit_unlock;
S
Sheng Yang 已提交
5004
		r = -ENOMEM;
5005
		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
S
Sheng Yang 已提交
5006 5007
		if (kvm->arch.vpit)
			r = 0;
A
Avi Kivity 已提交
5008
	create_pit_unlock:
5009
		mutex_unlock(&kvm->lock);
S
Sheng Yang 已提交
5010
		break;
5011 5012
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
5013
		struct kvm_irqchip *chip;
5014

5015 5016 5017
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
5018
			goto out;
5019 5020
		}

5021
		r = -ENXIO;
5022
		if (!irqchip_kernel(kvm))
5023 5024
			goto get_irqchip_out;
		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
5025
		if (r)
5026
			goto get_irqchip_out;
5027
		r = -EFAULT;
5028
		if (copy_to_user(argp, chip, sizeof(*chip)))
5029
			goto get_irqchip_out;
5030
		r = 0;
5031 5032
	get_irqchip_out:
		kfree(chip);
5033 5034 5035 5036
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
5037
		struct kvm_irqchip *chip;
5038

5039 5040 5041
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
5042
			goto out;
5043 5044
		}

5045
		r = -ENXIO;
5046
		if (!irqchip_kernel(kvm))
5047 5048 5049 5050
			goto set_irqchip_out;
		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
	set_irqchip_out:
		kfree(chip);
5051 5052
		break;
	}
5053 5054
	case KVM_GET_PIT: {
		r = -EFAULT;
5055
		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
5056 5057 5058 5059
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
5060
		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
5061 5062 5063
		if (r)
			goto out;
		r = -EFAULT;
5064
		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
5065 5066 5067 5068 5069 5070
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT: {
		r = -EFAULT;
5071
		if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
5072
			goto out;
5073
		mutex_lock(&kvm->lock);
5074 5075
		r = -ENXIO;
		if (!kvm->arch.vpit)
5076
			goto set_pit_out;
5077
		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
5078 5079
set_pit_out:
		mutex_unlock(&kvm->lock);
5080 5081
		break;
	}
B
Beth Kon 已提交
5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098
	case KVM_GET_PIT2: {
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT2: {
		r = -EFAULT;
		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
			goto out;
5099
		mutex_lock(&kvm->lock);
B
Beth Kon 已提交
5100 5101
		r = -ENXIO;
		if (!kvm->arch.vpit)
5102
			goto set_pit2_out;
B
Beth Kon 已提交
5103
		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
5104 5105
set_pit2_out:
		mutex_unlock(&kvm->lock);
B
Beth Kon 已提交
5106 5107
		break;
	}
5108 5109 5110 5111 5112
	case KVM_REINJECT_CONTROL: {
		struct kvm_reinject_control control;
		r =  -EFAULT;
		if (copy_from_user(&control, argp, sizeof(control)))
			goto out;
5113 5114 5115
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
5116 5117 5118
		r = kvm_vm_ioctl_reinject(kvm, &control);
		break;
	}
5119 5120 5121
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
		mutex_lock(&kvm->lock);
P
Paolo Bonzini 已提交
5122
		if (kvm->created_vcpus)
5123 5124 5125 5126 5127
			r = -EBUSY;
		else
			kvm->arch.bsp_vcpu_id = arg;
		mutex_unlock(&kvm->lock);
		break;
E
Ed Swierk 已提交
5128
	case KVM_XEN_HVM_CONFIG: {
5129
		struct kvm_xen_hvm_config xhc;
E
Ed Swierk 已提交
5130
		r = -EFAULT;
5131
		if (copy_from_user(&xhc, argp, sizeof(xhc)))
E
Ed Swierk 已提交
5132 5133
			goto out;
		r = -EINVAL;
5134
		if (xhc.flags)
E
Ed Swierk 已提交
5135
			goto out;
5136
		memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));
E
Ed Swierk 已提交
5137 5138 5139
		r = 0;
		break;
	}
5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152
	case KVM_SET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

		r = -EFAULT;
		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
			goto out;

		r = -EINVAL;
		if (user_ns.flags)
			goto out;

		r = 0;
5153 5154 5155 5156 5157 5158
		/*
		 * TODO: userspace has to take care of races with VCPU_RUN, so
		 * kvm_gen_update_masterclock() can be cut down to locked
		 * pvclock_update_vm_gtod_copy().
		 */
		kvm_gen_update_masterclock(kvm);
5159
		now_ns = get_kvmclock_ns(kvm);
5160
		kvm->arch.kvmclock_offset += user_ns.clock - now_ns;
5161
		kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE);
5162 5163 5164 5165 5166 5167
		break;
	}
	case KVM_GET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

5168
		now_ns = get_kvmclock_ns(kvm);
5169
		user_ns.clock = now_ns;
5170
		user_ns.flags = kvm->arch.use_master_clock ? KVM_CLOCK_TSC_STABLE : 0;
5171
		memset(&user_ns.pad, 0, sizeof(user_ns.pad));
5172 5173 5174 5175 5176 5177 5178

		r = -EFAULT;
		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
			goto out;
		r = 0;
		break;
	}
5179 5180
	case KVM_MEMORY_ENCRYPT_OP: {
		r = -ENOTTY;
5181 5182
		if (kvm_x86_ops.mem_enc_op)
			r = kvm_x86_ops.mem_enc_op(kvm, argp);
5183 5184
		break;
	}
5185 5186 5187 5188 5189 5190 5191 5192
	case KVM_MEMORY_ENCRYPT_REG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
5193 5194
		if (kvm_x86_ops.mem_enc_reg_region)
			r = kvm_x86_ops.mem_enc_reg_region(kvm, &region);
5195 5196 5197 5198 5199 5200 5201 5202 5203 5204
		break;
	}
	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
5205 5206
		if (kvm_x86_ops.mem_enc_unreg_region)
			r = kvm_x86_ops.mem_enc_unreg_region(kvm, &region);
5207 5208
		break;
	}
5209 5210 5211 5212 5213 5214 5215 5216 5217
	case KVM_HYPERV_EVENTFD: {
		struct kvm_hyperv_eventfd hvevfd;

		r = -EFAULT;
		if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
			goto out;
		r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
		break;
	}
E
Eric Hankland 已提交
5218 5219 5220
	case KVM_SET_PMU_EVENT_FILTER:
		r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
		break;
5221
	default:
5222
		r = -ENOTTY;
5223 5224 5225 5226 5227
	}
out:
	return r;
}

5228
static void kvm_init_msr_list(void)
5229
{
5230
	struct x86_pmu_capability x86_pmu;
5231
	u32 dummy[2];
5232
	unsigned i;
5233

5234
	BUILD_BUG_ON_MSG(INTEL_PMC_MAX_FIXED != 4,
5235
			 "Please update the fixed PMCs in msrs_to_saved_all[]");
5236 5237

	perf_get_x86_pmu_capability(&x86_pmu);
5238

5239 5240 5241 5242
	num_msrs_to_save = 0;
	num_emulated_msrs = 0;
	num_msr_based_features = 0;

5243 5244
	for (i = 0; i < ARRAY_SIZE(msrs_to_save_all); i++) {
		if (rdmsr_safe(msrs_to_save_all[i], &dummy[0], &dummy[1]) < 0)
5245
			continue;
5246 5247 5248

		/*
		 * Even MSRs that are valid in the host may not be exposed
5249
		 * to the guests in some cases.
5250
		 */
5251
		switch (msrs_to_save_all[i]) {
5252
		case MSR_IA32_BNDCFGS:
5253
			if (!kvm_mpx_supported())
5254 5255
				continue;
			break;
5256
		case MSR_TSC_AUX:
5257
			if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP))
5258 5259
				continue;
			break;
5260 5261
		case MSR_IA32_RTIT_CTL:
		case MSR_IA32_RTIT_STATUS:
5262
			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT))
5263 5264 5265
				continue;
			break;
		case MSR_IA32_RTIT_CR3_MATCH:
5266
			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
5267 5268 5269 5270 5271
			    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
				continue;
			break;
		case MSR_IA32_RTIT_OUTPUT_BASE:
		case MSR_IA32_RTIT_OUTPUT_MASK:
5272
			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
5273 5274 5275 5276 5277
				(!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
				 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
				continue;
			break;
		case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: {
5278
			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
5279
				msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >=
5280 5281 5282
				intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
				continue;
			break;
5283
		case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17:
5284
			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
5285 5286 5287
			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
				continue;
			break;
5288
		case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 17:
5289
			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >=
5290 5291
			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
				continue;
5292
		}
5293 5294 5295 5296
		default:
			break;
		}

5297
		msrs_to_save[num_msrs_to_save++] = msrs_to_save_all[i];
5298
	}
5299

5300
	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
5301
		if (!kvm_x86_ops.has_emulated_msr(emulated_msrs_all[i]))
5302
			continue;
5303

5304
		emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
5305
	}
5306

5307
	for (i = 0; i < ARRAY_SIZE(msr_based_features_all); i++) {
5308 5309
		struct kvm_msr_entry msr;

5310
		msr.index = msr_based_features_all[i];
5311
		if (kvm_get_msr_feature(&msr))
5312 5313
			continue;

5314
		msr_based_features[num_msr_based_features++] = msr_based_features_all[i];
5315
	}
5316 5317
}

5318 5319
static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
			   const void *v)
5320
{
5321 5322 5323 5324 5325
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5326
		if (!(lapic_in_kernel(vcpu) &&
5327 5328
		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
5329 5330 5331 5332 5333 5334
			break;
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5335

5336
	return handled;
5337 5338
}

5339
static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
5340
{
5341 5342 5343 5344 5345
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5346
		if (!(lapic_in_kernel(vcpu) &&
5347 5348 5349
		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
					 addr, n, v))
		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
5350
			break;
5351
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
5352 5353 5354 5355 5356
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5357

5358
	return handled;
5359 5360
}

5361 5362 5363
static void kvm_set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
5364
	kvm_x86_ops.set_segment(vcpu, var, seg);
5365 5366 5367 5368 5369
}

void kvm_get_segment(struct kvm_vcpu *vcpu,
		     struct kvm_segment *var, int seg)
{
5370
	kvm_x86_ops.get_segment(vcpu, var, seg);
5371 5372
}

5373 5374
gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
			   struct x86_exception *exception)
5375 5376 5377 5378 5379 5380 5381
{
	gpa_t t_gpa;

	BUG_ON(!mmu_is_nested(vcpu));

	/* NPT walks are always user-walks */
	access |= PFERR_USER_MASK;
5382
	t_gpa  = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception);
5383 5384 5385 5386

	return t_gpa;
}

5387 5388
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
			      struct x86_exception *exception)
5389
{
5390
	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5391
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5392 5393
}

5394 5395
 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5396
{
5397
	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5398
	access |= PFERR_FETCH_MASK;
5399
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5400 5401
}

5402 5403
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
			       struct x86_exception *exception)
5404
{
5405
	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5406
	access |= PFERR_WRITE_MASK;
5407
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5408 5409 5410
}

/* uses this to access any guest's mapped memory without checking CPL */
5411 5412
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5413
{
5414
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
5415 5416 5417 5418
}

static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
5419
				      struct x86_exception *exception)
5420 5421
{
	void *data = val;
5422
	int r = X86EMUL_CONTINUE;
5423 5424

	while (bytes) {
5425
		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
5426
							    exception);
5427
		unsigned offset = addr & (PAGE_SIZE-1);
5428
		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
5429 5430
		int ret;

5431
		if (gpa == UNMAPPED_GVA)
5432
			return X86EMUL_PROPAGATE_FAULT;
5433 5434
		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
					       offset, toread);
5435
		if (ret < 0) {
5436
			r = X86EMUL_IO_NEEDED;
5437 5438
			goto out;
		}
5439

5440 5441 5442
		bytes -= toread;
		data += toread;
		addr += toread;
5443
	}
5444 5445
out:
	return r;
5446
}
5447

5448
/* used for instruction fetching */
5449 5450
static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
				gva_t addr, void *val, unsigned int bytes,
5451
				struct x86_exception *exception)
5452
{
5453
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5454
	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5455 5456
	unsigned offset;
	int ret;
5457

5458 5459 5460 5461 5462 5463 5464 5465 5466
	/* Inline kvm_read_guest_virt_helper for speed.  */
	gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,
						    exception);
	if (unlikely(gpa == UNMAPPED_GVA))
		return X86EMUL_PROPAGATE_FAULT;

	offset = addr & (PAGE_SIZE-1);
	if (WARN_ON(offset + bytes > PAGE_SIZE))
		bytes = (unsigned)PAGE_SIZE - offset;
5467 5468
	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
				       offset, bytes);
5469 5470 5471 5472
	if (unlikely(ret < 0))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
5473 5474
}

5475
int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
5476
			       gva_t addr, void *val, unsigned int bytes,
5477
			       struct x86_exception *exception)
5478
{
5479
	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5480

5481 5482 5483 5484 5485 5486 5487
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5488
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
5489
					  exception);
5490
}
5491
EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
5492

5493 5494
static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
			     gva_t addr, void *val, unsigned int bytes,
5495
			     struct x86_exception *exception, bool system)
5496
{
5497
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5498 5499
	u32 access = 0;

5500
	if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)
5501 5502 5503
		access |= PFERR_USER_MASK;

	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
5504 5505
}

5506 5507 5508 5509 5510 5511 5512 5513 5514
static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
		unsigned long addr, void *val, unsigned int bytes)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);

	return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
}

5515 5516 5517
static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
				      struct x86_exception *exception)
5518 5519 5520 5521 5522
{
	void *data = val;
	int r = X86EMUL_CONTINUE;

	while (bytes) {
5523
		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
5524
							     access,
5525
							     exception);
5526 5527 5528 5529
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
		int ret;

5530
		if (gpa == UNMAPPED_GVA)
5531
			return X86EMUL_PROPAGATE_FAULT;
5532
		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
5533
		if (ret < 0) {
5534
			r = X86EMUL_IO_NEEDED;
5535 5536 5537 5538 5539 5540 5541 5542 5543 5544
			goto out;
		}

		bytes -= towrite;
		data += towrite;
		addr += towrite;
	}
out:
	return r;
}
5545 5546

static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
5547 5548
			      unsigned int bytes, struct x86_exception *exception,
			      bool system)
5549 5550
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5551 5552
	u32 access = PFERR_WRITE_MASK;

5553
	if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)
5554
		access |= PFERR_USER_MASK;
5555 5556

	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
5557
					   access, exception);
5558 5559 5560 5561 5562
}

int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
				unsigned int bytes, struct x86_exception *exception)
{
P
Paolo Bonzini 已提交
5563 5564 5565
	/* kvm_write_guest_virt_system can pull in tons of pages. */
	vcpu->arch.l1tf_flush_l1d = true;

5566 5567 5568 5569 5570 5571 5572
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5573 5574 5575
	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
					   PFERR_WRITE_MASK, exception);
}
N
Nadav Har'El 已提交
5576
EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
5577

W
Wanpeng Li 已提交
5578 5579
int handle_ud(struct kvm_vcpu *vcpu)
{
5580
	static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX };
5581 5582 5583 5584 5585
	int emul_type = EMULTYPE_TRAP_UD;
	char sig[5]; /* ud2; .ascii "kvm" */
	struct x86_exception e;

	if (force_emulation_prefix &&
5586 5587
	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
				sig, sizeof(sig), &e) == 0 &&
5588
	    memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) {
5589
		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
5590
		emul_type = EMULTYPE_TRAP_UD_FORCED;
5591
	}
W
Wanpeng Li 已提交
5592

5593
	return kvm_emulate_instruction(vcpu, emul_type);
W
Wanpeng Li 已提交
5594 5595 5596
}
EXPORT_SYMBOL_GPL(handle_ud);

5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611
static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
			    gpa_t gpa, bool write)
{
	/* For APIC access vmexit */
	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		return 1;

	if (vcpu_match_mmio_gpa(vcpu, gpa)) {
		trace_vcpu_match_mmio(gva, gpa, write, true);
		return 1;
	}

	return 0;
}

5612 5613 5614 5615
static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
				gpa_t *gpa, struct x86_exception *exception,
				bool write)
{
5616
	u32 access = ((kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
5617
		| (write ? PFERR_WRITE_MASK : 0);
5618

5619 5620 5621 5622 5623
	/*
	 * currently PKRU is only applied to ept enabled guest so
	 * there is no pkey in EPT page table for L1 guest or EPT
	 * shadow page table for L2 guest.
	 */
5624
	if (vcpu_match_mmio_gva(vcpu, gva)
F
Feng Wu 已提交
5625
	    && !permission_fault(vcpu, vcpu->arch.walk_mmu,
5626
				 vcpu->arch.mmio_access, 0, access)) {
5627 5628
		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
					(gva & (PAGE_SIZE - 1));
X
Xiao Guangrong 已提交
5629
		trace_vcpu_match_mmio(gva, *gpa, write, false);
5630 5631 5632
		return 1;
	}

5633 5634 5635 5636 5637
	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);

	if (*gpa == UNMAPPED_GVA)
		return -1;

5638
	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
5639 5640
}

5641
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
5642
			const void *val, int bytes)
5643 5644 5645
{
	int ret;

5646
	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
5647
	if (ret < 0)
5648
		return 0;
5649
	kvm_page_track_write(vcpu, gpa, val, bytes);
5650 5651 5652
	return 1;
}

5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668
struct read_write_emulator_ops {
	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
				  int bytes);
	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
				  void *val, int bytes);
	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
			       int bytes, void *val);
	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
				    void *val, int bytes);
	bool write;
};

static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
{
	if (vcpu->mmio_read_completed) {
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
5669
			       vcpu->mmio_fragments[0].gpa, val);
5670 5671 5672 5673 5674 5675 5676 5677 5678 5679
		vcpu->mmio_read_completed = 0;
		return 1;
	}

	return 0;
}

static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			void *val, int bytes)
{
5680
	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
5681 5682 5683 5684 5685 5686 5687 5688 5689 5690
}

static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			 void *val, int bytes)
{
	return emulator_write_phys(vcpu, gpa, val, bytes);
}

static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
{
5691
	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
5692 5693 5694 5695 5696 5697
	return vcpu_mmio_write(vcpu, gpa, bytes, val);
}

static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			  void *val, int bytes)
{
5698
	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
5699 5700 5701 5702 5703 5704
	return X86EMUL_IO_NEEDED;
}

static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			   void *val, int bytes)
{
A
Avi Kivity 已提交
5705 5706
	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];

5707
	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
5708 5709 5710
	return X86EMUL_CONTINUE;
}

5711
static const struct read_write_emulator_ops read_emultor = {
5712 5713 5714 5715 5716 5717
	.read_write_prepare = read_prepare,
	.read_write_emulate = read_emulate,
	.read_write_mmio = vcpu_mmio_read,
	.read_write_exit_mmio = read_exit_mmio,
};

5718
static const struct read_write_emulator_ops write_emultor = {
5719 5720 5721 5722 5723 5724
	.read_write_emulate = write_emulate,
	.read_write_mmio = write_mmio,
	.read_write_exit_mmio = write_exit_mmio,
	.write = true,
};

5725 5726 5727 5728
static int emulator_read_write_onepage(unsigned long addr, void *val,
				       unsigned int bytes,
				       struct x86_exception *exception,
				       struct kvm_vcpu *vcpu,
5729
				       const struct read_write_emulator_ops *ops)
5730
{
5731 5732
	gpa_t gpa;
	int handled, ret;
5733
	bool write = ops->write;
A
Avi Kivity 已提交
5734
	struct kvm_mmio_fragment *frag;
5735
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
5736 5737 5738 5739 5740 5741 5742 5743

	/*
	 * If the exit was due to a NPF we may already have a GPA.
	 * If the GPA is present, use it to avoid the GVA to GPA table walk.
	 * Note, this cannot be used on string operations since string
	 * operation using rep will only have the initial GPA from the NPF
	 * occurred.
	 */
5744 5745 5746
	if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) &&
	    (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) {
		gpa = ctxt->gpa_val;
5747 5748 5749 5750 5751
		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
	} else {
		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
		if (ret < 0)
			return X86EMUL_PROPAGATE_FAULT;
5752
	}
5753

5754
	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
5755 5756 5757 5758 5759
		return X86EMUL_CONTINUE;

	/*
	 * Is this MMIO handled locally?
	 */
5760
	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
5761
	if (handled == bytes)
5762 5763
		return X86EMUL_CONTINUE;

5764 5765 5766 5767
	gpa += handled;
	bytes -= handled;
	val += handled;

5768 5769 5770 5771 5772
	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
	frag->gpa = gpa;
	frag->data = val;
	frag->len = bytes;
A
Avi Kivity 已提交
5773
	return X86EMUL_CONTINUE;
5774 5775
}

5776 5777
static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
			unsigned long addr,
5778 5779
			void *val, unsigned int bytes,
			struct x86_exception *exception,
5780
			const struct read_write_emulator_ops *ops)
5781
{
5782
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
A
Avi Kivity 已提交
5783 5784 5785 5786 5787 5788 5789 5790
	gpa_t gpa;
	int rc;

	if (ops->read_write_prepare &&
		  ops->read_write_prepare(vcpu, val, bytes))
		return X86EMUL_CONTINUE;

	vcpu->mmio_nr_fragments = 0;
5791

5792 5793
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
A
Avi Kivity 已提交
5794
		int now;
5795 5796

		now = -addr & ~PAGE_MASK;
5797 5798 5799
		rc = emulator_read_write_onepage(addr, val, now, exception,
						 vcpu, ops);

5800 5801 5802
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
5803 5804
		if (ctxt->mode != X86EMUL_MODE_PROT64)
			addr = (u32)addr;
5805 5806 5807
		val += now;
		bytes -= now;
	}
5808

A
Avi Kivity 已提交
5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821
	rc = emulator_read_write_onepage(addr, val, bytes, exception,
					 vcpu, ops);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	if (!vcpu->mmio_nr_fragments)
		return rc;

	gpa = vcpu->mmio_fragments[0].gpa;

	vcpu->mmio_needed = 1;
	vcpu->mmio_cur_fragment = 0;

5822
	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
A
Avi Kivity 已提交
5823 5824 5825 5826 5827
	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
	vcpu->run->exit_reason = KVM_EXIT_MMIO;
	vcpu->run->mmio.phys_addr = gpa;

	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839
}

static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
				  unsigned long addr,
				  void *val,
				  unsigned int bytes,
				  struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, val, bytes,
				   exception, &read_emultor);
}

5840
static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
5841 5842 5843 5844 5845 5846 5847
			    unsigned long addr,
			    const void *val,
			    unsigned int bytes,
			    struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, (void *)val, bytes,
				   exception, &write_emultor);
5848 5849
}

5850 5851 5852 5853 5854 5855 5856
#define CMPXCHG_TYPE(t, ptr, old, new) \
	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))

#ifdef CONFIG_X86_64
#  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
#else
#  define CMPXCHG64(ptr, old, new) \
5857
	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
5858 5859
#endif

5860 5861
static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
				     unsigned long addr,
5862 5863 5864
				     const void *old,
				     const void *new,
				     unsigned int bytes,
5865
				     struct x86_exception *exception)
5866
{
5867
	struct kvm_host_map map;
5868
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5869
	u64 page_line_mask;
5870 5871 5872
	gpa_t gpa;
	char *kaddr;
	bool exchanged;
5873

5874 5875 5876
	/* guests cmpxchg8b have to be emulated atomically */
	if (bytes > 8 || (bytes & (bytes - 1)))
		goto emul_write;
5877

5878
	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
5879

5880 5881 5882
	if (gpa == UNMAPPED_GVA ||
	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		goto emul_write;
5883

5884 5885 5886 5887 5888 5889 5890 5891 5892 5893
	/*
	 * Emulate the atomic as a straight write to avoid #AC if SLD is
	 * enabled in the host and the access splits a cache line.
	 */
	if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
		page_line_mask = ~(cache_line_size() - 1);
	else
		page_line_mask = PAGE_MASK;

	if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask))
5894
		goto emul_write;
5895

5896
	if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))
5897
		goto emul_write;
5898

5899 5900
	kaddr = map.hva + offset_in_page(gpa);

5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915
	switch (bytes) {
	case 1:
		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
		break;
	case 2:
		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
		break;
	case 4:
		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
		break;
	case 8:
		exchanged = CMPXCHG64(kaddr, old, new);
		break;
	default:
		BUG();
5916
	}
5917 5918

	kvm_vcpu_unmap(vcpu, &map, true);
5919 5920 5921 5922

	if (!exchanged)
		return X86EMUL_CMPXCHG_FAILED;

5923
	kvm_page_track_write(vcpu, gpa, new, bytes);
5924 5925

	return X86EMUL_CONTINUE;
5926

5927
emul_write:
5928
	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
5929

5930
	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
5931 5932
}

5933 5934
static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
{
5935
	int r = 0, i;
5936

5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948
	for (i = 0; i < vcpu->arch.pio.count; i++) {
		if (vcpu->arch.pio.in)
			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
					    vcpu->arch.pio.size, pd);
		else
			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
					     vcpu->arch.pio.port, vcpu->arch.pio.size,
					     pd);
		if (r)
			break;
		pd += vcpu->arch.pio.size;
	}
5949 5950 5951
	return r;
}

5952 5953 5954
static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
			       unsigned short port, void *val,
			       unsigned int count, bool in)
5955 5956
{
	vcpu->arch.pio.port = port;
5957
	vcpu->arch.pio.in = in;
5958
	vcpu->arch.pio.count  = count;
5959 5960 5961
	vcpu->arch.pio.size = size;

	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
5962
		vcpu->arch.pio.count = 0;
5963 5964 5965 5966
		return 1;
	}

	vcpu->run->exit_reason = KVM_EXIT_IO;
5967
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
5968 5969 5970 5971 5972 5973 5974 5975
	vcpu->run->io.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = count;
	vcpu->run->io.port = port;

	return 0;
}

5976 5977
static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
			   unsigned short port, void *val, unsigned int count)
5978
{
5979
	int ret;
5980

5981 5982
	if (vcpu->arch.pio.count)
		goto data_avail;
5983

5984 5985
	memset(vcpu->arch.pio_data, 0, size * count);

5986 5987 5988 5989
	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
	if (ret) {
data_avail:
		memcpy(val, vcpu->arch.pio_data, size * count);
5990
		trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
5991
		vcpu->arch.pio.count = 0;
5992 5993 5994 5995 5996 5997
		return 1;
	}

	return 0;
}

5998 5999 6000
static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
				    int size, unsigned short port, void *val,
				    unsigned int count)
6001
{
6002
	return emulator_pio_in(emul_to_vcpu(ctxt), size, port, val, count);
6003

6004
}
6005

6006 6007 6008 6009
static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,
			    unsigned short port, const void *val,
			    unsigned int count)
{
6010
	memcpy(vcpu->arch.pio_data, val, size * count);
6011
	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
6012 6013 6014
	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
}

6015 6016 6017 6018 6019 6020 6021
static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
				     int size, unsigned short port,
				     const void *val, unsigned int count)
{
	return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count);
}

6022 6023
static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
6024
	return kvm_x86_ops.get_segment_base(vcpu, seg);
6025 6026
}

6027
static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
6028
{
6029
	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
6030 6031
}

6032
static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
6033 6034 6035 6036
{
	if (!need_emulate_wbinvd(vcpu))
		return X86EMUL_CONTINUE;

6037
	if (kvm_x86_ops.has_wbinvd_exit()) {
6038 6039 6040
		int cpu = get_cpu();

		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
6041 6042
		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
				wbinvd_ipi, NULL, 1);
6043
		put_cpu();
6044
		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
6045 6046
	} else
		wbinvd();
6047 6048
	return X86EMUL_CONTINUE;
}
6049 6050 6051

int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
6052 6053
	kvm_emulate_wbinvd_noskip(vcpu);
	return kvm_skip_emulated_instruction(vcpu);
6054
}
6055 6056
EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);

6057 6058


6059 6060
static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
{
6061
	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
6062 6063
}

6064 6065
static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long *dest)
6066
{
6067
	return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
6068 6069
}

6070 6071
static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long value)
6072
{
6073

6074
	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
6075 6076
}

6077
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
6078
{
6079
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
6080 6081
}

6082
static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
6083
{
6084
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6085 6086 6087 6088 6089 6090 6091 6092 6093 6094
	unsigned long value;

	switch (cr) {
	case 0:
		value = kvm_read_cr0(vcpu);
		break;
	case 2:
		value = vcpu->arch.cr2;
		break;
	case 3:
6095
		value = kvm_read_cr3(vcpu);
6096 6097 6098 6099 6100 6101 6102 6103
		break;
	case 4:
		value = kvm_read_cr4(vcpu);
		break;
	case 8:
		value = kvm_get_cr8(vcpu);
		break;
	default:
6104
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
6105 6106 6107 6108 6109 6110
		return 0;
	}

	return value;
}

6111
static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
6112
{
6113
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6114 6115
	int res = 0;

6116 6117
	switch (cr) {
	case 0:
6118
		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
6119 6120 6121 6122 6123
		break;
	case 2:
		vcpu->arch.cr2 = val;
		break;
	case 3:
6124
		res = kvm_set_cr3(vcpu, val);
6125 6126
		break;
	case 4:
6127
		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
6128 6129
		break;
	case 8:
A
Andre Przywara 已提交
6130
		res = kvm_set_cr8(vcpu, val);
6131 6132
		break;
	default:
6133
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
6134
		res = -1;
6135
	}
6136 6137

	return res;
6138 6139
}

6140
static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
6141
{
6142
	return kvm_x86_ops.get_cpl(emul_to_vcpu(ctxt));
6143 6144
}

6145
static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6146
{
6147
	kvm_x86_ops.get_gdt(emul_to_vcpu(ctxt), dt);
6148 6149
}

6150
static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6151
{
6152
	kvm_x86_ops.get_idt(emul_to_vcpu(ctxt), dt);
6153 6154
}

6155 6156
static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
6157
	kvm_x86_ops.set_gdt(emul_to_vcpu(ctxt), dt);
6158 6159 6160 6161
}

static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
6162
	kvm_x86_ops.set_idt(emul_to_vcpu(ctxt), dt);
6163 6164
}

6165 6166
static unsigned long emulator_get_cached_segment_base(
	struct x86_emulate_ctxt *ctxt, int seg)
6167
{
6168
	return get_segment_base(emul_to_vcpu(ctxt), seg);
6169 6170
}

6171 6172 6173
static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
				 struct desc_struct *desc, u32 *base3,
				 int seg)
6174 6175 6176
{
	struct kvm_segment var;

6177
	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
6178
	*selector = var.selector;
6179

6180 6181
	if (var.unusable) {
		memset(desc, 0, sizeof(*desc));
6182 6183
		if (base3)
			*base3 = 0;
6184
		return false;
6185
	}
6186 6187 6188 6189 6190

	if (var.g)
		var.limit >>= 12;
	set_desc_limit(desc, var.limit);
	set_desc_base(desc, (unsigned long)var.base);
6191 6192 6193 6194
#ifdef CONFIG_X86_64
	if (base3)
		*base3 = var.base >> 32;
#endif
6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206
	desc->type = var.type;
	desc->s = var.s;
	desc->dpl = var.dpl;
	desc->p = var.present;
	desc->avl = var.avl;
	desc->l = var.l;
	desc->d = var.db;
	desc->g = var.g;

	return true;
}

6207 6208 6209
static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
				 struct desc_struct *desc, u32 base3,
				 int seg)
6210
{
6211
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6212 6213
	struct kvm_segment var;

6214
	var.selector = selector;
6215
	var.base = get_desc_base(desc);
6216 6217 6218
#ifdef CONFIG_X86_64
	var.base |= ((u64)base3) << 32;
#endif
6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236
	var.limit = get_desc_limit(desc);
	if (desc->g)
		var.limit = (var.limit << 12) | 0xfff;
	var.type = desc->type;
	var.dpl = desc->dpl;
	var.db = desc->d;
	var.s = desc->s;
	var.l = desc->l;
	var.g = desc->g;
	var.avl = desc->avl;
	var.present = desc->p;
	var.unusable = !var.present;
	var.padding = 0;

	kvm_set_segment(vcpu, &var, seg);
	return;
}

6237 6238 6239
static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 *pdata)
{
6240
	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
6241 6242 6243 6244 6245
}

static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 data)
{
6246
	return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
6247 6248
}

P
Paolo Bonzini 已提交
6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262
static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	return vcpu->arch.smbase;
}

static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	vcpu->arch.smbase = smbase;
}

6263 6264 6265
static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
			      u32 pmc)
{
6266
	return kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc);
6267 6268
}

6269 6270 6271
static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
			     u32 pmc, u64 *pdata)
{
6272
	return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
6273 6274
}

6275 6276 6277 6278 6279
static void emulator_halt(struct x86_emulate_ctxt *ctxt)
{
	emul_to_vcpu(ctxt)->arch.halt_request = 1;
}

6280
static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
6281
			      struct x86_instruction_info *info,
6282 6283
			      enum x86_intercept_stage stage)
{
6284
	return kvm_x86_ops.check_intercept(emul_to_vcpu(ctxt), info, stage,
6285
					    &ctxt->exception);
6286 6287
}

6288
static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
6289 6290
			      u32 *eax, u32 *ebx, u32 *ecx, u32 *edx,
			      bool exact_only)
6291
{
6292
	return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only);
6293 6294
}

6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309
static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_LM);
}

static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE);
}

static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR);
}

6310 6311 6312 6313 6314 6315 6316 6317 6318 6319
static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
{
	return kvm_register_read(emul_to_vcpu(ctxt), reg);
}

static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
{
	kvm_register_write(emul_to_vcpu(ctxt), reg, val);
}

6320 6321
static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
{
6322
	kvm_x86_ops.set_nmi_mask(emul_to_vcpu(ctxt), masked);
6323 6324
}

6325 6326 6327 6328 6329 6330 6331
static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
{
	return emul_to_vcpu(ctxt)->arch.hflags;
}

static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_flags)
{
6332
	emul_to_vcpu(ctxt)->arch.hflags = emul_flags;
6333 6334
}

6335 6336
static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt,
				  const char *smstate)
6337
{
6338
	return kvm_x86_ops.pre_leave_smm(emul_to_vcpu(ctxt), smstate);
6339 6340
}

6341 6342 6343 6344 6345
static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt)
{
	kvm_smm_changed(emul_to_vcpu(ctxt));
}

6346 6347 6348 6349 6350
static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
{
	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
}

6351
static const struct x86_emulate_ops emulate_ops = {
6352 6353
	.read_gpr            = emulator_read_gpr,
	.write_gpr           = emulator_write_gpr,
6354 6355
	.read_std            = emulator_read_std,
	.write_std           = emulator_write_std,
6356
	.read_phys           = kvm_read_guest_phys_system,
6357
	.fetch               = kvm_fetch_guest_virt,
6358 6359 6360
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
6361
	.invlpg              = emulator_invlpg,
6362 6363
	.pio_in_emulated     = emulator_pio_in_emulated,
	.pio_out_emulated    = emulator_pio_out_emulated,
6364 6365
	.get_segment         = emulator_get_segment,
	.set_segment         = emulator_set_segment,
6366
	.get_cached_segment_base = emulator_get_cached_segment_base,
6367
	.get_gdt             = emulator_get_gdt,
6368
	.get_idt	     = emulator_get_idt,
6369 6370
	.set_gdt             = emulator_set_gdt,
	.set_idt	     = emulator_set_idt,
6371 6372
	.get_cr              = emulator_get_cr,
	.set_cr              = emulator_set_cr,
6373
	.cpl                 = emulator_get_cpl,
6374 6375
	.get_dr              = emulator_get_dr,
	.set_dr              = emulator_set_dr,
P
Paolo Bonzini 已提交
6376 6377
	.get_smbase          = emulator_get_smbase,
	.set_smbase          = emulator_set_smbase,
6378 6379
	.set_msr             = emulator_set_msr,
	.get_msr             = emulator_get_msr,
6380
	.check_pmc	     = emulator_check_pmc,
6381
	.read_pmc            = emulator_read_pmc,
6382
	.halt                = emulator_halt,
6383
	.wbinvd              = emulator_wbinvd,
6384
	.fix_hypercall       = emulator_fix_hypercall,
6385
	.intercept           = emulator_intercept,
6386
	.get_cpuid           = emulator_get_cpuid,
6387 6388 6389
	.guest_has_long_mode = emulator_guest_has_long_mode,
	.guest_has_movbe     = emulator_guest_has_movbe,
	.guest_has_fxsr      = emulator_guest_has_fxsr,
6390
	.set_nmi_mask        = emulator_set_nmi_mask,
6391 6392
	.get_hflags          = emulator_get_hflags,
	.set_hflags          = emulator_set_hflags,
6393
	.pre_leave_smm       = emulator_pre_leave_smm,
6394
	.post_leave_smm      = emulator_post_leave_smm,
6395
	.set_xcr             = emulator_set_xcr,
6396 6397
};

6398 6399
static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
{
6400
	u32 int_shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);
6401 6402 6403 6404 6405 6406 6407
	/*
	 * an sti; sti; sequence only disable interrupts for the first
	 * instruction. So, if the last instruction, be it emulated or
	 * not, left the system with the INT_STI flag enabled, it
	 * means that the last instruction is an sti. We should not
	 * leave the flag on in this case. The same goes for mov ss
	 */
6408 6409
	if (int_shadow & mask)
		mask = 0;
6410
	if (unlikely(int_shadow || mask)) {
6411
		kvm_x86_ops.set_interrupt_shadow(vcpu, mask);
6412 6413 6414
		if (!mask)
			kvm_make_request(KVM_REQ_EVENT, vcpu);
	}
6415 6416
}

6417
static bool inject_emulated_exception(struct kvm_vcpu *vcpu)
6418
{
6419
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6420
	if (ctxt->exception.vector == PF_VECTOR)
6421
		return kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);
6422 6423

	if (ctxt->exception.error_code_valid)
6424 6425
		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
				      ctxt->exception.error_code);
6426
	else
6427
		kvm_queue_exception(vcpu, ctxt->exception.vector);
6428
	return false;
6429 6430
}

6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447
static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)
{
	struct x86_emulate_ctxt *ctxt;

	ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT);
	if (!ctxt) {
		pr_err("kvm: failed to allocate vcpu's emulator\n");
		return NULL;
	}

	ctxt->vcpu = vcpu;
	ctxt->ops = &emulate_ops;
	vcpu->arch.emulate_ctxt = ctxt;

	return ctxt;
}

6448 6449
static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
{
6450
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6451 6452
	int cs_db, cs_l;

6453
	kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
6454

6455
	ctxt->gpa_available = false;
6456
	ctxt->eflags = kvm_get_rflags(vcpu);
6457 6458
	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;

6459 6460 6461
	ctxt->eip = kvm_rip_read(vcpu);
	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
6462
		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :
6463 6464
		     cs_db				? X86EMUL_MODE_PROT32 :
							  X86EMUL_MODE_PROT16;
6465
	BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK);
P
Paolo Bonzini 已提交
6466 6467
	BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK);
	BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK);
6468

6469
	init_decode_cache(ctxt);
6470
	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6471 6472
}

6473
void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
6474
{
6475
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6476 6477 6478 6479
	int ret;

	init_emulate_ctxt(vcpu);

6480 6481 6482
	ctxt->op_bytes = 2;
	ctxt->ad_bytes = 2;
	ctxt->_eip = ctxt->eip + inc_eip;
6483
	ret = emulate_int_real(ctxt, irq);
6484

6485 6486 6487 6488 6489 6490 6491
	if (ret != X86EMUL_CONTINUE) {
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
	} else {
		ctxt->eip = ctxt->_eip;
		kvm_rip_write(vcpu, ctxt->eip);
		kvm_set_rflags(vcpu, ctxt->eflags);
	}
6492 6493 6494
}
EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);

6495
static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
6496 6497 6498
{
	++vcpu->stat.insn_emulation_fail;
	trace_kvm_emulate_insn_failed(vcpu);
6499

6500 6501
	if (emulation_type & EMULTYPE_VMWARE_GP) {
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6502
		return 1;
6503
	}
6504

6505 6506 6507 6508
	if (emulation_type & EMULTYPE_SKIP) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6509
		return 0;
6510 6511
	}

6512 6513
	kvm_queue_exception(vcpu, UD_VECTOR);

6514
	if (!is_guest_mode(vcpu) && kvm_x86_ops.get_cpl(vcpu) == 0) {
6515 6516 6517
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6518
		return 0;
6519
	}
6520

6521
	return 1;
6522 6523
}

6524
static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
6525 6526
				  bool write_fault_to_shadow_pgtable,
				  int emulation_type)
6527
{
6528
	gpa_t gpa = cr2_or_gpa;
D
Dan Williams 已提交
6529
	kvm_pfn_t pfn;
6530

6531
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
6532 6533
		return false;

6534 6535
	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
	    WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
6536 6537
		return false;

6538
	if (!vcpu->arch.mmu->direct_map) {
6539 6540 6541 6542
		/*
		 * Write permission should be allowed since only
		 * write access need to be emulated.
		 */
6543
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
6544

6545 6546 6547 6548 6549 6550 6551
		/*
		 * If the mapping is invalid in guest, let cpu retry
		 * it to generate fault.
		 */
		if (gpa == UNMAPPED_GVA)
			return true;
	}
6552

6553 6554 6555 6556 6557 6558 6559
	/*
	 * Do not retry the unhandleable instruction if it faults on the
	 * readonly host memory, otherwise it will goto a infinite loop:
	 * retry instruction -> write #PF -> emulation fail -> retry
	 * instruction -> ...
	 */
	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570

	/*
	 * If the instruction failed on the error pfn, it can not be fixed,
	 * report the error to userspace.
	 */
	if (is_error_noslot_pfn(pfn))
		return false;

	kvm_release_pfn_clean(pfn);

	/* The instructions are well-emulated on direct mmu. */
6571
	if (vcpu->arch.mmu->direct_map) {
6572 6573 6574 6575 6576 6577 6578 6579 6580
		unsigned int indirect_shadow_pages;

		spin_lock(&vcpu->kvm->mmu_lock);
		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
		spin_unlock(&vcpu->kvm->mmu_lock);

		if (indirect_shadow_pages)
			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));

6581
		return true;
6582
	}
6583

6584 6585 6586 6587 6588 6589
	/*
	 * if emulation was due to access to shadowed page table
	 * and it failed try to unshadow page and re-enter the
	 * guest to let CPU execute the instruction.
	 */
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6590 6591 6592 6593 6594 6595 6596

	/*
	 * If the access faults on its page table, it can not
	 * be fixed by unprotecting shadow page and it should
	 * be reported to userspace.
	 */
	return !write_fault_to_shadow_pgtable;
6597 6598
}

6599
static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
6600
			      gpa_t cr2_or_gpa,  int emulation_type)
6601 6602
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6603
	unsigned long last_retry_eip, last_retry_addr, gpa = cr2_or_gpa;
6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622

	last_retry_eip = vcpu->arch.last_retry_eip;
	last_retry_addr = vcpu->arch.last_retry_addr;

	/*
	 * If the emulation is caused by #PF and it is non-page_table
	 * writing instruction, it means the VM-EXIT is caused by shadow
	 * page protected, we can zap the shadow page and retry this
	 * instruction directly.
	 *
	 * Note: if the guest uses a non-page-table modifying instruction
	 * on the PDE that points to the instruction, then we will unmap
	 * the instruction and go to an infinite loop. So, we cache the
	 * last retried eip and the last fault address, if we meet the eip
	 * and the address again, we can break out of the potential infinite
	 * loop.
	 */
	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;

6623
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
6624 6625
		return false;

6626 6627
	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
	    WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
6628 6629
		return false;

6630 6631 6632
	if (x86_page_table_writing_insn(ctxt))
		return false;

6633
	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa)
6634 6635 6636
		return false;

	vcpu->arch.last_retry_eip = ctxt->eip;
6637
	vcpu->arch.last_retry_addr = cr2_or_gpa;
6638

6639
	if (!vcpu->arch.mmu->direct_map)
6640
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
6641

6642
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6643 6644 6645 6646

	return true;
}

6647 6648 6649
static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
static int complete_emulated_pio(struct kvm_vcpu *vcpu);

P
Paolo Bonzini 已提交
6650
static void kvm_smm_changed(struct kvm_vcpu *vcpu)
6651
{
P
Paolo Bonzini 已提交
6652
	if (!(vcpu->arch.hflags & HF_SMM_MASK)) {
6653 6654 6655
		/* This is a good place to trace that we are exiting SMM.  */
		trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false);

6656 6657
		/* Process a latched INIT or SMI, if any.  */
		kvm_make_request(KVM_REQ_EVENT, vcpu);
P
Paolo Bonzini 已提交
6658
	}
6659 6660

	kvm_mmu_reset_context(vcpu);
P
Paolo Bonzini 已提交
6661 6662
}

6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677
static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
				unsigned long *db)
{
	u32 dr6 = 0;
	int i;
	u32 enable, rwlen;

	enable = dr7;
	rwlen = dr7 >> 16;
	for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
		if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
			dr6 |= (1 << i);
	return dr6;
}

6678
static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
6679 6680 6681
{
	struct kvm_run *kvm_run = vcpu->run;

6682 6683
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
		kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1 | DR6_RTM;
6684
		kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
6685 6686
		kvm_run->debug.arch.exception = DB_VECTOR;
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
6687
		return 0;
6688
	}
6689
	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
6690
	return 1;
6691 6692
}

6693 6694
int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
6695
	unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
6696
	int r;
6697

6698
	r = kvm_x86_ops.skip_emulated_instruction(vcpu);
6699
	if (unlikely(!r))
6700
		return 0;
6701 6702 6703 6704 6705 6706 6707 6708 6709 6710

	/*
	 * rflags is the old, "raw" value of the flags.  The new value has
	 * not been saved yet.
	 *
	 * This is correct even for TF set by the guest, because "the
	 * processor will not generate this exception after the instruction
	 * that sets the TF flag".
	 */
	if (unlikely(rflags & X86_EFLAGS_TF))
6711
		r = kvm_vcpu_do_singlestep(vcpu);
6712
	return r;
6713 6714 6715
}
EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);

6716 6717 6718 6719
static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
{
	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
6720 6721 6722
		struct kvm_run *kvm_run = vcpu->run;
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6723 6724 6725 6726
					   vcpu->arch.guest_debug_dr7,
					   vcpu->arch.eff_db);

		if (dr6 != 0) {
6727
			kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
6728
			kvm_run->debug.arch.pc = eip;
6729 6730
			kvm_run->debug.arch.exception = DB_VECTOR;
			kvm_run->exit_reason = KVM_EXIT_DEBUG;
6731
			*r = 0;
6732 6733 6734 6735
			return true;
		}
	}

6736 6737
	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
	    !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {
6738 6739
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6740 6741 6742 6743
					   vcpu->arch.dr7,
					   vcpu->arch.db);

		if (dr6 != 0) {
6744
			kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
6745
			*r = 1;
6746 6747 6748 6749 6750 6751 6752
			return true;
		}
	}

	return false;
}

6753 6754
static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
{
6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778
	switch (ctxt->opcode_len) {
	case 1:
		switch (ctxt->b) {
		case 0xe4:	/* IN */
		case 0xe5:
		case 0xec:
		case 0xed:
		case 0xe6:	/* OUT */
		case 0xe7:
		case 0xee:
		case 0xef:
		case 0x6c:	/* INS */
		case 0x6d:
		case 0x6e:	/* OUTS */
		case 0x6f:
			return true;
		}
		break;
	case 2:
		switch (ctxt->b) {
		case 0x33:	/* RDPMC */
			return true;
		}
		break;
6779 6780 6781 6782 6783
	}

	return false;
}

6784 6785
int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
			    int emulation_type, void *insn, int insn_len)
6786
{
6787
	int r;
6788
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6789
	bool writeback = true;
6790
	bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
6791

P
Paolo Bonzini 已提交
6792 6793
	vcpu->arch.l1tf_flush_l1d = true;

6794 6795 6796 6797 6798
	/*
	 * Clear write_fault_to_shadow_pgtable here to ensure it is
	 * never reused.
	 */
	vcpu->arch.write_fault_to_shadow_pgtable = false;
6799
	kvm_clear_exception_queue(vcpu);
G
Gleb Natapov 已提交
6800

6801
	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
6802
		init_emulate_ctxt(vcpu);
6803 6804 6805 6806 6807 6808 6809

		/*
		 * We will reenter on the same instruction since
		 * we do not set complete_userspace_io.  This does not
		 * handle watchpoints yet, those would be handled in
		 * the emulate_ops.
		 */
6810 6811
		if (!(emulation_type & EMULTYPE_SKIP) &&
		    kvm_vcpu_check_breakpoint(vcpu, &r))
6812 6813
			return r;

6814 6815
		ctxt->interruptibility = 0;
		ctxt->have_exception = false;
6816
		ctxt->exception.vector = -1;
6817
		ctxt->perm_ok = false;
6818

6819
		ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
6820

6821
		r = x86_decode_insn(ctxt, insn, insn_len);
6822

A
Avi Kivity 已提交
6823
		trace_kvm_emulate_insn_start(vcpu);
6824
		++vcpu->stat.insn_emulation;
6825
		if (r != EMULATION_OK)  {
6826
			if ((emulation_type & EMULTYPE_TRAP_UD) ||
6827 6828
			    (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
				kvm_queue_exception(vcpu, UD_VECTOR);
6829
				return 1;
6830
			}
6831 6832 6833
			if (reexecute_instruction(vcpu, cr2_or_gpa,
						  write_fault_to_spt,
						  emulation_type))
6834
				return 1;
6835
			if (ctxt->have_exception) {
6836 6837 6838 6839 6840 6841
				/*
				 * #UD should result in just EMULATION_FAILED, and trap-like
				 * exception should not be encountered during decode.
				 */
				WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
					     exception_type(ctxt->exception.vector) == EXCPT_TRAP);
6842
				inject_emulated_exception(vcpu);
6843
				return 1;
6844
			}
6845
			return handle_emulation_failure(vcpu, emulation_type);
6846 6847 6848
		}
	}

6849 6850 6851
	if ((emulation_type & EMULTYPE_VMWARE_GP) &&
	    !is_vmware_backdoor_opcode(ctxt)) {
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6852
		return 1;
6853
	}
6854

6855 6856 6857 6858 6859
	/*
	 * Note, EMULTYPE_SKIP is intended for use *only* by vendor callbacks
	 * for kvm_skip_emulated_instruction().  The caller is responsible for
	 * updating interruptibility state and injecting single-step #DBs.
	 */
6860
	if (emulation_type & EMULTYPE_SKIP) {
6861
		kvm_rip_write(vcpu, ctxt->_eip);
6862 6863
		if (ctxt->eflags & X86_EFLAGS_RF)
			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
6864
		return 1;
6865 6866
	}

6867
	if (retry_instruction(ctxt, cr2_or_gpa, emulation_type))
6868
		return 1;
6869

6870
	/* this is needed for vmware backdoor interface to work since it
6871
	   changes registers values  during IO operation */
6872 6873
	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6874
		emulator_invalidate_register_cache(ctxt);
6875
	}
6876

6877
restart:
6878 6879 6880 6881 6882 6883
	if (emulation_type & EMULTYPE_PF) {
		/* Save the faulting GPA (cr2) in the address field */
		ctxt->exception.address = cr2_or_gpa;

		/* With shadow page tables, cr2 contains a GVA or nGPA. */
		if (vcpu->arch.mmu->direct_map) {
6884 6885
			ctxt->gpa_available = true;
			ctxt->gpa_val = cr2_or_gpa;
6886 6887 6888 6889 6890
		}
	} else {
		/* Sanitize the address out of an abundance of paranoia. */
		ctxt->exception.address = 0;
	}
6891

6892
	r = x86_emulate_insn(ctxt);
6893

6894
	if (r == EMULATION_INTERCEPTED)
6895
		return 1;
6896

6897
	if (r == EMULATION_FAILED) {
6898
		if (reexecute_instruction(vcpu, cr2_or_gpa, write_fault_to_spt,
6899
					emulation_type))
6900
			return 1;
6901

6902
		return handle_emulation_failure(vcpu, emulation_type);
6903 6904
	}

6905
	if (ctxt->have_exception) {
6906
		r = 1;
6907 6908
		if (inject_emulated_exception(vcpu))
			return r;
6909
	} else if (vcpu->arch.pio.count) {
6910 6911
		if (!vcpu->arch.pio.in) {
			/* FIXME: return into emulator if single-stepping.  */
6912
			vcpu->arch.pio.count = 0;
6913
		} else {
6914
			writeback = false;
6915 6916
			vcpu->arch.complete_userspace_io = complete_emulated_pio;
		}
6917
		r = 0;
6918
	} else if (vcpu->mmio_needed) {
6919 6920
		++vcpu->stat.mmio_exits;

6921 6922
		if (!vcpu->mmio_is_write)
			writeback = false;
6923
		r = 0;
6924
		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
6925
	} else if (r == EMULATION_RESTART)
6926
		goto restart;
6927
	else
6928
		r = 1;
6929

6930
	if (writeback) {
6931
		unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
6932
		toggle_interruptibility(vcpu, ctxt->interruptibility);
6933
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
6934
		if (!ctxt->have_exception ||
6935 6936
		    exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
			kvm_rip_write(vcpu, ctxt->eip);
6937
			if (r && ctxt->tf)
6938
				r = kvm_vcpu_do_singlestep(vcpu);
6939 6940
			if (kvm_x86_ops.update_emulated_instruction)
				kvm_x86_ops.update_emulated_instruction(vcpu);
6941
			__kvm_set_rflags(vcpu, ctxt->eflags);
6942
		}
6943 6944 6945 6946 6947 6948 6949 6950 6951

		/*
		 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
		 * do nothing, and it will be requested again as soon as
		 * the shadow expires.  But we still need to check here,
		 * because POPF has no interrupt shadow.
		 */
		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
			kvm_make_request(KVM_REQ_EVENT, vcpu);
6952 6953
	} else
		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
6954 6955

	return r;
6956
}
6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969

int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
{
	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction);

int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
					void *insn, int insn_len)
{
	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
6970

6971 6972 6973 6974 6975 6976
static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;
	return 1;
}

6977 6978 6979 6980 6981 6982 6983 6984 6985 6986
static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;

	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
		return 1;

	return kvm_skip_emulated_instruction(vcpu);
}

6987 6988
static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
			    unsigned short port)
6989
{
6990
	unsigned long val = kvm_rax_read(vcpu);
6991 6992
	int ret = emulator_pio_out(vcpu, size, port, &val, 1);

6993 6994
	if (ret)
		return ret;
6995

6996 6997 6998 6999 7000 7001 7002 7003 7004 7005
	/*
	 * Workaround userspace that relies on old KVM behavior of %rip being
	 * incremented prior to exiting to userspace to handle "OUT 0x7e".
	 */
	if (port == 0x7e &&
	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
		vcpu->arch.complete_userspace_io =
			complete_fast_pio_out_port_0x7e;
		kvm_skip_emulated_instruction(vcpu);
	} else {
7006 7007 7008
		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
		vcpu->arch.complete_userspace_io = complete_fast_pio_out;
	}
7009
	return 0;
7010 7011
}

7012 7013 7014 7015 7016 7017 7018
static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
{
	unsigned long val;

	/* We should only ever be called with arch.pio.count equal to 1 */
	BUG_ON(vcpu->arch.pio.count != 1);

7019 7020 7021 7022 7023
	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
		vcpu->arch.pio.count = 0;
		return 1;
	}

7024
	/* For size less than 4 we merge, else we zero extend */
7025
	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
7026 7027

	/*
7028
	 * Since vcpu->arch.pio.count == 1 let emulator_pio_in perform
7029 7030
	 * the copy and tracing
	 */
7031
	emulator_pio_in(vcpu, vcpu->arch.pio.size, vcpu->arch.pio.port, &val, 1);
7032
	kvm_rax_write(vcpu, val);
7033

7034
	return kvm_skip_emulated_instruction(vcpu);
7035 7036
}

7037 7038
static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
			   unsigned short port)
7039 7040 7041 7042 7043
{
	unsigned long val;
	int ret;

	/* For size less than 4 we merge, else we zero extend */
7044
	val = (size < 4) ? kvm_rax_read(vcpu) : 0;
7045

7046
	ret = emulator_pio_in(vcpu, size, port, &val, 1);
7047
	if (ret) {
7048
		kvm_rax_write(vcpu, val);
7049 7050 7051
		return ret;
	}

7052
	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
7053 7054 7055 7056
	vcpu->arch.complete_userspace_io = complete_fast_pio_in;

	return 0;
}
7057 7058 7059

int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
{
7060
	int ret;
7061 7062

	if (in)
7063
		ret = kvm_fast_pio_in(vcpu, size, port);
7064
	else
7065 7066
		ret = kvm_fast_pio_out(vcpu, size, port);
	return ret && kvm_skip_emulated_instruction(vcpu);
7067 7068
}
EXPORT_SYMBOL_GPL(kvm_fast_pio);
7069

7070
static int kvmclock_cpu_down_prep(unsigned int cpu)
7071
{
T
Tejun Heo 已提交
7072
	__this_cpu_write(cpu_tsc_khz, 0);
7073
	return 0;
7074 7075 7076
}

static void tsc_khz_changed(void *data)
7077
{
7078 7079 7080 7081 7082 7083 7084 7085 7086
	struct cpufreq_freqs *freq = data;
	unsigned long khz = 0;

	if (data)
		khz = freq->new;
	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		khz = cpufreq_quick_get(raw_smp_processor_id());
	if (!khz)
		khz = tsc_khz;
T
Tejun Heo 已提交
7087
	__this_cpu_write(cpu_tsc_khz, khz);
7088 7089
}

7090
#ifdef CONFIG_X86_64
7091 7092 7093 7094 7095 7096
static void kvm_hyperv_tsc_notifier(void)
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int cpu;

J
Junaid Shahid 已提交
7097
	mutex_lock(&kvm_lock);
7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_make_mclock_inprogress_request(kvm);

	hyperv_stop_tsc_emulation();

	/* TSC frequency always matches when on Hyper-V */
	for_each_present_cpu(cpu)
		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
	kvm_max_guest_tsc_khz = tsc_khz;

	list_for_each_entry(kvm, &vm_list, vm_list) {
		struct kvm_arch *ka = &kvm->arch;

		spin_lock(&ka->pvclock_gtod_sync_lock);

		pvclock_update_vm_gtod_copy(kvm);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);

		spin_unlock(&ka->pvclock_gtod_sync_lock);
	}
J
Junaid Shahid 已提交
7123
	mutex_unlock(&kvm_lock);
7124
}
7125
#endif
7126

7127
static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
7128 7129 7130 7131 7132
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i, send_ipi = 0;

7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171
	/*
	 * We allow guests to temporarily run on slowing clocks,
	 * provided we notify them after, or to run on accelerating
	 * clocks, provided we notify them before.  Thus time never
	 * goes backwards.
	 *
	 * However, we have a problem.  We can't atomically update
	 * the frequency of a given CPU from this function; it is
	 * merely a notifier, which can be called from any CPU.
	 * Changing the TSC frequency at arbitrary points in time
	 * requires a recomputation of local variables related to
	 * the TSC for each VCPU.  We must flag these local variables
	 * to be updated and be sure the update takes place with the
	 * new frequency before any guests proceed.
	 *
	 * Unfortunately, the combination of hotplug CPU and frequency
	 * change creates an intractable locking scenario; the order
	 * of when these callouts happen is undefined with respect to
	 * CPU hotplug, and they can race with each other.  As such,
	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
	 * undefined; you can actually have a CPU frequency change take
	 * place in between the computation of X and the setting of the
	 * variable.  To protect against this problem, all updates of
	 * the per_cpu tsc_khz variable are done in an interrupt
	 * protected IPI, and all callers wishing to update the value
	 * must wait for a synchronous IPI to complete (which is trivial
	 * if the caller is on the CPU already).  This establishes the
	 * necessary total order on variable updates.
	 *
	 * Note that because a guest time update may take place
	 * anytime after the setting of the VCPU's request bit, the
	 * correct TSC value must be set before the request.  However,
	 * to ensure the update actually makes it to any guest which
	 * starts running in hardware virtualization between the set
	 * and the acquisition of the spinlock, we must also ping the
	 * CPU after setting the request bit.
	 *
	 */

7172
	smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
7173

J
Junaid Shahid 已提交
7174
	mutex_lock(&kvm_lock);
7175
	list_for_each_entry(kvm, &vm_list, vm_list) {
7176
		kvm_for_each_vcpu(i, vcpu, kvm) {
7177
			if (vcpu->cpu != cpu)
7178
				continue;
Z
Zachary Amsden 已提交
7179
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
J
Junaid Shahid 已提交
7180
			if (vcpu->cpu != raw_smp_processor_id())
7181
				send_ipi = 1;
7182 7183
		}
	}
J
Junaid Shahid 已提交
7184
	mutex_unlock(&kvm_lock);
7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198

	if (freq->old < freq->new && send_ipi) {
		/*
		 * We upscale the frequency.  Must make the guest
		 * doesn't see old kvmclock values while running with
		 * the new frequency, otherwise we risk the guest sees
		 * time go backwards.
		 *
		 * In case we update the frequency for another cpu
		 * (which might be in guest context) send an interrupt
		 * to kick the cpu out of guest context.  Next time
		 * guest context is entered kvmclock will be updated,
		 * so the guest will not see stale values.
		 */
7199
		smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
7200
	}
7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216
}

static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
				     void *data)
{
	struct cpufreq_freqs *freq = data;
	int cpu;

	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
		return 0;
	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
		return 0;

	for_each_cpu(cpu, freq->policy->cpus)
		__kvmclock_cpufreq_notifier(freq, cpu);

7217 7218 7219 7220
	return 0;
}

static struct notifier_block kvmclock_cpufreq_notifier_block = {
7221 7222 7223
	.notifier_call  = kvmclock_cpufreq_notifier
};

7224
static int kvmclock_cpu_online(unsigned int cpu)
7225
{
7226 7227
	tsc_khz_changed(NULL);
	return 0;
7228 7229
}

7230 7231
static void kvm_timer_init(void)
{
Z
Zachary Amsden 已提交
7232
	max_tsc_khz = tsc_khz;
7233

7234
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
Z
Zachary Amsden 已提交
7235
#ifdef CONFIG_CPU_FREQ
7236
		struct cpufreq_policy *policy;
7237 7238
		int cpu;

7239
		cpu = get_cpu();
7240
		policy = cpufreq_cpu_get(cpu);
7241 7242 7243 7244 7245
		if (policy) {
			if (policy->cpuinfo.max_freq)
				max_tsc_khz = policy->cpuinfo.max_freq;
			cpufreq_cpu_put(policy);
		}
7246
		put_cpu();
Z
Zachary Amsden 已提交
7247
#endif
7248 7249 7250
		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
					  CPUFREQ_TRANSITION_NOTIFIER);
	}
7251

T
Thomas Gleixner 已提交
7252
	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
7253
			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
7254 7255
}

7256 7257
DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu);
7258

7259
int kvm_is_in_guest(void)
7260
{
7261
	return __this_cpu_read(current_vcpu) != NULL;
7262 7263 7264 7265 7266
}

static int kvm_is_user_mode(void)
{
	int user_mode = 3;
7267

7268
	if (__this_cpu_read(current_vcpu))
7269
		user_mode = kvm_x86_ops.get_cpl(__this_cpu_read(current_vcpu));
7270

7271 7272 7273 7274 7275 7276
	return user_mode != 0;
}

static unsigned long kvm_get_guest_ip(void)
{
	unsigned long ip = 0;
7277

7278 7279
	if (__this_cpu_read(current_vcpu))
		ip = kvm_rip_read(__this_cpu_read(current_vcpu));
7280

7281 7282 7283
	return ip;
}

L
Luwei Kang 已提交
7284 7285 7286 7287 7288 7289 7290 7291 7292
static void kvm_handle_intel_pt_intr(void)
{
	struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);

	kvm_make_request(KVM_REQ_PMI, vcpu);
	__set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
			(unsigned long *)&vcpu->arch.pmu.global_status);
}

7293 7294 7295 7296
static struct perf_guest_info_callbacks kvm_guest_cbs = {
	.is_in_guest		= kvm_is_in_guest,
	.is_user_mode		= kvm_is_user_mode,
	.get_guest_ip		= kvm_get_guest_ip,
L
Luwei Kang 已提交
7297
	.handle_intel_pt_intr	= kvm_handle_intel_pt_intr,
7298 7299
};

7300 7301 7302
#ifdef CONFIG_X86_64
static void pvclock_gtod_update_fn(struct work_struct *work)
{
7303 7304 7305 7306 7307
	struct kvm *kvm;

	struct kvm_vcpu *vcpu;
	int i;

J
Junaid Shahid 已提交
7308
	mutex_lock(&kvm_lock);
7309 7310
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_for_each_vcpu(i, vcpu, kvm)
7311
			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
7312
	atomic_set(&kvm_guest_has_master_clock, 0);
J
Junaid Shahid 已提交
7313
	mutex_unlock(&kvm_lock);
7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329
}

static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);

/*
 * Notification about pvclock gtod data update.
 */
static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
			       void *priv)
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	struct timekeeper *tk = priv;

	update_pvclock_gtod(tk);

	/* disable master clock if host does not trust, or does not
7330
	 * use, TSC based clocksource.
7331
	 */
7332
	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343
	    atomic_read(&kvm_guest_has_master_clock) != 0)
		queue_work(system_long_wq, &pvclock_gtod_work);

	return 0;
}

static struct notifier_block pvclock_gtod_notifier = {
	.notifier_call = pvclock_gtod_notify,
};
#endif

7344
int kvm_arch_init(void *opaque)
7345
{
7346
	struct kvm_x86_init_ops *ops = opaque;
7347
	int r;
7348

7349
	if (kvm_x86_ops.hardware_enable) {
7350
		printk(KERN_ERR "kvm: already loaded the other module\n");
7351 7352
		r = -EEXIST;
		goto out;
7353 7354 7355
	}

	if (!ops->cpu_has_kvm_support()) {
7356
		pr_err_ratelimited("kvm: no hardware support\n");
7357 7358
		r = -EOPNOTSUPP;
		goto out;
7359 7360
	}
	if (ops->disabled_by_bios()) {
7361
		pr_err_ratelimited("kvm: disabled by bios\n");
7362 7363
		r = -EOPNOTSUPP;
		goto out;
7364 7365
	}

7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376
	/*
	 * KVM explicitly assumes that the guest has an FPU and
	 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
	 * vCPU's FPU state as a fxregs_state struct.
	 */
	if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
		printk(KERN_ERR "kvm: inadequate fpu\n");
		r = -EOPNOTSUPP;
		goto out;
	}

7377
	r = -ENOMEM;
7378
	x86_fpu_cache = kmem_cache_create("x86_fpu", sizeof(struct fpu),
7379 7380 7381 7382 7383 7384 7385
					  __alignof__(struct fpu), SLAB_ACCOUNT,
					  NULL);
	if (!x86_fpu_cache) {
		printk(KERN_ERR "kvm: failed to allocate cache for x86 fpu\n");
		goto out;
	}

7386 7387 7388 7389 7390 7391
	x86_emulator_cache = kvm_alloc_emulator_cache();
	if (!x86_emulator_cache) {
		pr_err("kvm: failed to allocate cache for x86 emulator\n");
		goto out_free_x86_fpu_cache;
	}

7392 7393 7394
	shared_msrs = alloc_percpu(struct kvm_shared_msrs);
	if (!shared_msrs) {
		printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
7395
		goto out_free_x86_emulator_cache;
7396 7397
	}

7398 7399
	r = kvm_mmu_module_init();
	if (r)
7400
		goto out_free_percpu;
7401

S
Sheng Yang 已提交
7402
	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
7403
			PT_DIRTY_MASK, PT64_NX_MASK, 0,
7404
			PT_PRESENT_MASK, 0, sme_me_mask);
7405
	kvm_timer_init();
7406

7407 7408
	perf_register_guest_info_callbacks(&kvm_guest_cbs);

7409
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
7410
		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
7411 7412
		supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
	}
7413

7414
	kvm_lapic_init();
7415 7416
	if (pi_inject_timer == -1)
		pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);
7417 7418
#ifdef CONFIG_X86_64
	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
7419

7420
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7421
		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
7422 7423
#endif

7424
	return 0;
7425

7426 7427
out_free_percpu:
	free_percpu(shared_msrs);
7428 7429
out_free_x86_emulator_cache:
	kmem_cache_destroy(x86_emulator_cache);
7430 7431
out_free_x86_fpu_cache:
	kmem_cache_destroy(x86_fpu_cache);
7432 7433
out:
	return r;
7434
}
7435

7436 7437
void kvm_arch_exit(void)
{
7438
#ifdef CONFIG_X86_64
7439
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7440 7441
		clear_hv_tscchange_cb();
#endif
7442
	kvm_lapic_exit();
7443 7444
	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);

7445 7446 7447
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
					    CPUFREQ_TRANSITION_NOTIFIER);
7448
	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
7449 7450 7451
#ifdef CONFIG_X86_64
	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
#endif
7452
	kvm_x86_ops.hardware_enable = NULL;
7453
	kvm_mmu_module_exit();
7454
	free_percpu(shared_msrs);
7455
	kmem_cache_destroy(x86_fpu_cache);
7456
}
7457

7458
int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
7459 7460
{
	++vcpu->stat.halt_exits;
7461
	if (lapic_in_kernel(vcpu)) {
7462
		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
7463 7464 7465 7466 7467 7468
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
}
7469 7470 7471 7472
EXPORT_SYMBOL_GPL(kvm_vcpu_halt);

int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
7473 7474 7475 7476 7477 7478
	int ret = kvm_skip_emulated_instruction(vcpu);
	/*
	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
	 * KVM_EXIT_DEBUG here.
	 */
	return kvm_vcpu_halt(vcpu) && ret;
7479
}
7480 7481
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

7482
#ifdef CONFIG_X86_64
7483 7484 7485 7486
static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
			        unsigned long clock_type)
{
	struct kvm_clock_pairing clock_pairing;
7487
	struct timespec64 ts;
P
Paolo Bonzini 已提交
7488
	u64 cycle;
7489 7490 7491 7492 7493 7494 7495 7496 7497 7498 7499 7500
	int ret;

	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
		return -KVM_EOPNOTSUPP;

	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)
		return -KVM_EOPNOTSUPP;

	clock_pairing.sec = ts.tv_sec;
	clock_pairing.nsec = ts.tv_nsec;
	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
	clock_pairing.flags = 0;
7501
	memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
7502 7503 7504 7505 7506 7507 7508 7509

	ret = 0;
	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
			    sizeof(struct kvm_clock_pairing)))
		ret = -KVM_EFAULT;

	return ret;
}
7510
#endif
7511

7512 7513 7514 7515 7516 7517 7518
/*
 * kvm_pv_kick_cpu_op:  Kick a vcpu.
 *
 * @apicid - apicid of vcpu to be kicked.
 */
static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
{
7519
	struct kvm_lapic_irq lapic_irq;
7520

7521
	lapic_irq.shorthand = APIC_DEST_NOSHORT;
7522
	lapic_irq.dest_mode = APIC_DEST_PHYSICAL;
7523
	lapic_irq.level = 0;
7524
	lapic_irq.dest_id = apicid;
7525
	lapic_irq.msi_redir_hint = false;
7526

7527
	lapic_irq.delivery_mode = APIC_DM_REMRD;
7528
	kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
7529 7530
}

7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547
bool kvm_apicv_activated(struct kvm *kvm)
{
	return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0);
}
EXPORT_SYMBOL_GPL(kvm_apicv_activated);

void kvm_apicv_init(struct kvm *kvm, bool enable)
{
	if (enable)
		clear_bit(APICV_INHIBIT_REASON_DISABLE,
			  &kvm->arch.apicv_inhibit_reasons);
	else
		set_bit(APICV_INHIBIT_REASON_DISABLE,
			&kvm->arch.apicv_inhibit_reasons);
}
EXPORT_SYMBOL_GPL(kvm_apicv_init);

7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560
static void kvm_sched_yield(struct kvm *kvm, unsigned long dest_id)
{
	struct kvm_vcpu *target = NULL;
	struct kvm_apic_map *map;

	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);

	if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])
		target = map->phys_map[dest_id]->vcpu;

	rcu_read_unlock();

7561
	if (target && READ_ONCE(target->ready))
7562 7563 7564
		kvm_vcpu_yield_to(target);
}

7565 7566 7567
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
{
	unsigned long nr, a0, a1, a2, a3, ret;
7568
	int op_64_bit;
7569

7570 7571
	if (kvm_hv_hypercall_enabled(vcpu->kvm))
		return kvm_hv_hypercall(vcpu);
7572

7573 7574 7575 7576 7577
	nr = kvm_rax_read(vcpu);
	a0 = kvm_rbx_read(vcpu);
	a1 = kvm_rcx_read(vcpu);
	a2 = kvm_rdx_read(vcpu);
	a3 = kvm_rsi_read(vcpu);
7578

7579
	trace_kvm_hypercall(nr, a0, a1, a2, a3);
F
Feng (Eric) Liu 已提交
7580

7581 7582
	op_64_bit = is_64_bit_mode(vcpu);
	if (!op_64_bit) {
7583 7584 7585 7586 7587 7588 7589
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
	}

7590
	if (kvm_x86_ops.get_cpl(vcpu) != 0) {
7591
		ret = -KVM_EPERM;
7592
		goto out;
7593 7594
	}

7595
	switch (nr) {
A
Avi Kivity 已提交
7596 7597 7598
	case KVM_HC_VAPIC_POLL_IRQ:
		ret = 0;
		break;
7599 7600
	case KVM_HC_KICK_CPU:
		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
7601
		kvm_sched_yield(vcpu->kvm, a1);
7602 7603
		ret = 0;
		break;
7604
#ifdef CONFIG_X86_64
7605 7606 7607
	case KVM_HC_CLOCK_PAIRING:
		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
		break;
7608
#endif
7609 7610 7611
	case KVM_HC_SEND_IPI:
		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
		break;
7612 7613 7614 7615
	case KVM_HC_SCHED_YIELD:
		kvm_sched_yield(vcpu->kvm, a0);
		ret = 0;
		break;
7616 7617 7618 7619
	default:
		ret = -KVM_ENOSYS;
		break;
	}
7620
out:
7621 7622
	if (!op_64_bit)
		ret = (u32)ret;
7623
	kvm_rax_write(vcpu, ret);
7624

A
Amit Shah 已提交
7625
	++vcpu->stat.hypercalls;
7626
	return kvm_skip_emulated_instruction(vcpu);
7627 7628 7629
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

7630
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
7631
{
7632
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7633
	char instruction[3];
7634
	unsigned long rip = kvm_rip_read(vcpu);
7635

7636
	kvm_x86_ops.patch_hypercall(vcpu, instruction);
7637

7638 7639
	return emulator_write_emulated(ctxt, rip, instruction, 3,
		&ctxt->exception);
7640 7641
}

A
Avi Kivity 已提交
7642
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
7643
{
7644 7645
	return vcpu->run->request_interrupt_window &&
		likely(!pic_in_kernel(vcpu->kvm));
7646 7647
}

A
Avi Kivity 已提交
7648
static void post_kvm_run_save(struct kvm_vcpu *vcpu)
7649
{
A
Avi Kivity 已提交
7650 7651
	struct kvm_run *kvm_run = vcpu->run;

7652
	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
7653
	kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;
7654
	kvm_run->cr8 = kvm_get_cr8(vcpu);
7655
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
7656 7657
	kvm_run->ready_for_interrupt_injection =
		pic_in_kernel(vcpu->kvm) ||
7658
		kvm_vcpu_ready_for_interrupt_injection(vcpu);
7659 7660
}

7661 7662 7663 7664
static void update_cr8_intercept(struct kvm_vcpu *vcpu)
{
	int max_irr, tpr;

7665
	if (!kvm_x86_ops.update_cr8_intercept)
7666 7667
		return;

7668
	if (!lapic_in_kernel(vcpu))
7669 7670
		return;

7671 7672 7673
	if (vcpu->arch.apicv_active)
		return;

7674 7675 7676 7677
	if (!vcpu->arch.apic->vapic_addr)
		max_irr = kvm_lapic_find_highest_irr(vcpu);
	else
		max_irr = -1;
7678 7679 7680 7681 7682 7683

	if (max_irr != -1)
		max_irr >>= 4;

	tpr = kvm_lapic_get_cr8(vcpu);

7684
	kvm_x86_ops.update_cr8_intercept(vcpu, tpr, max_irr);
7685 7686
}

7687
static int inject_pending_event(struct kvm_vcpu *vcpu)
7688
{
7689 7690
	int r;

7691
	/* try to reinject previous events if any */
7692

7693
	if (vcpu->arch.exception.injected)
7694
		kvm_x86_ops.queue_exception(vcpu);
7695
	/*
7696 7697 7698 7699 7700 7701 7702 7703 7704 7705 7706 7707
	 * Do not inject an NMI or interrupt if there is a pending
	 * exception.  Exceptions and interrupts are recognized at
	 * instruction boundaries, i.e. the start of an instruction.
	 * Trap-like exceptions, e.g. #DB, have higher priority than
	 * NMIs and interrupts, i.e. traps are recognized before an
	 * NMI/interrupt that's pending on the same instruction.
	 * Fault-like exceptions, e.g. #GP and #PF, are the lowest
	 * priority, but are only generated (pended) during instruction
	 * execution, i.e. a pending fault-like exception means the
	 * fault occurred on the *previous* instruction and must be
	 * serviced prior to recognizing any new events in order to
	 * fully complete the previous instruction.
7708
	 */
7709 7710
	else if (!vcpu->arch.exception.pending) {
		if (vcpu->arch.nmi_injected)
7711
			kvm_x86_ops.set_nmi(vcpu);
7712
		else if (vcpu->arch.interrupt.injected)
7713
			kvm_x86_ops.set_irq(vcpu);
7714 7715
	}

7716 7717 7718 7719 7720 7721
	/*
	 * Call check_nested_events() even if we reinjected a previous event
	 * in order for caller to determine if it should require immediate-exit
	 * from L2 to L1 due to pending L1 events which require exit
	 * from L2 to L1.
	 */
7722
	if (is_guest_mode(vcpu)) {
7723
		r = kvm_x86_ops.nested_ops->check_events(vcpu);
7724 7725 7726 7727 7728
		if (r != 0)
			return r;
	}

	/* try to inject new event if pending */
7729
	if (vcpu->arch.exception.pending) {
A
Avi Kivity 已提交
7730 7731 7732
		trace_kvm_inj_exception(vcpu->arch.exception.nr,
					vcpu->arch.exception.has_error_code,
					vcpu->arch.exception.error_code);
7733

7734
		WARN_ON_ONCE(vcpu->arch.exception.injected);
7735 7736 7737
		vcpu->arch.exception.pending = false;
		vcpu->arch.exception.injected = true;

7738 7739 7740 7741
		if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
					     X86_EFLAGS_RF);

7742 7743 7744 7745 7746 7747 7748 7749 7750 7751 7752 7753 7754 7755 7756 7757
		if (vcpu->arch.exception.nr == DB_VECTOR) {
			/*
			 * This code assumes that nSVM doesn't use
			 * check_nested_events(). If it does, the
			 * DR6/DR7 changes should happen before L1
			 * gets a #VMEXIT for an intercepted #DB in
			 * L2.  (Under VMX, on the other hand, the
			 * DR6/DR7 changes should not happen in the
			 * event of a VM-exit to L1 for an intercepted
			 * #DB in L2.)
			 */
			kvm_deliver_exception_payload(vcpu);
			if (vcpu->arch.dr7 & DR7_GD) {
				vcpu->arch.dr7 &= ~DR7_GD;
				kvm_update_dr7(vcpu);
			}
7758 7759
		}

7760
		kvm_x86_ops.queue_exception(vcpu);
7761 7762 7763 7764 7765 7766
	}

	/* Don't consider new event if we re-injected an event */
	if (kvm_event_needs_reinjection(vcpu))
		return 0;

7767
	if (vcpu->arch.smi_pending && kvm_x86_ops.smi_allowed(vcpu)) {
7768
		vcpu->arch.smi_pending = false;
7769
		++vcpu->arch.smi_count;
7770
		enter_smm(vcpu);
7771
	} else if (vcpu->arch.nmi_pending && kvm_x86_ops.nmi_allowed(vcpu)) {
7772 7773
		--vcpu->arch.nmi_pending;
		vcpu->arch.nmi_injected = true;
7774
		kvm_x86_ops.set_nmi(vcpu);
7775
	} else if (kvm_cpu_has_injectable_intr(vcpu)) {
7776 7777 7778 7779 7780 7781 7782
		/*
		 * Because interrupts can be injected asynchronously, we are
		 * calling check_nested_events again here to avoid a race condition.
		 * See https://lkml.org/lkml/2014/7/2/60 for discussion about this
		 * proposal and current concerns.  Perhaps we should be setting
		 * KVM_REQ_EVENT only on certain events and not unconditionally?
		 */
7783
		if (is_guest_mode(vcpu)) {
7784
			r = kvm_x86_ops.nested_ops->check_events(vcpu);
7785 7786 7787
			if (r != 0)
				return r;
		}
7788
		if (kvm_x86_ops.interrupt_allowed(vcpu)) {
7789 7790
			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
					    false);
7791
			kvm_x86_ops.set_irq(vcpu);
7792 7793
		}
	}
7794

7795
	return 0;
7796 7797
}

A
Avi Kivity 已提交
7798 7799 7800 7801 7802 7803 7804 7805 7806
static void process_nmi(struct kvm_vcpu *vcpu)
{
	unsigned limit = 2;

	/*
	 * x86 is limited to one NMI running, and one NMI pending after it.
	 * If an NMI is already in progress, limit further NMIs to just one.
	 * Otherwise, allow two (and we'll inject the first one immediately).
	 */
7807
	if (kvm_x86_ops.get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
A
Avi Kivity 已提交
7808 7809 7810 7811 7812 7813 7814
		limit = 1;

	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

7815
static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828
{
	u32 flags = 0;
	flags |= seg->g       << 23;
	flags |= seg->db      << 22;
	flags |= seg->l       << 21;
	flags |= seg->avl     << 20;
	flags |= seg->present << 15;
	flags |= seg->dpl     << 13;
	flags |= seg->s       << 12;
	flags |= seg->type    << 8;
	return flags;
}

7829
static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843
{
	struct kvm_segment seg;
	int offset;

	kvm_get_segment(vcpu, &seg, n);
	put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector);

	if (n < 3)
		offset = 0x7f84 + n * 12;
	else
		offset = 0x7f2c + (n - 3) * 12;

	put_smstate(u32, buf, offset + 8, seg.base);
	put_smstate(u32, buf, offset + 4, seg.limit);
7844
	put_smstate(u32, buf, offset, enter_smm_get_segment_flags(&seg));
7845 7846
}

7847
#ifdef CONFIG_X86_64
7848
static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
7849 7850 7851 7852 7853 7854 7855 7856
{
	struct kvm_segment seg;
	int offset;
	u16 flags;

	kvm_get_segment(vcpu, &seg, n);
	offset = 0x7e00 + n * 16;

7857
	flags = enter_smm_get_segment_flags(&seg) >> 8;
7858 7859 7860 7861 7862
	put_smstate(u16, buf, offset, seg.selector);
	put_smstate(u16, buf, offset + 2, flags);
	put_smstate(u32, buf, offset + 4, seg.limit);
	put_smstate(u64, buf, offset + 8, seg.base);
}
7863
#endif
7864

7865
static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
	put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
	put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
	put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));

	for (i = 0; i < 8; i++)
		put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read(vcpu, i));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u32, buf, 0x7fcc, (u32)val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u32, buf, 0x7fc8, (u32)val);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u32, buf, 0x7fc4, seg.selector);
	put_smstate(u32, buf, 0x7f64, seg.base);
	put_smstate(u32, buf, 0x7f60, seg.limit);
7889
	put_smstate(u32, buf, 0x7f5c, enter_smm_get_segment_flags(&seg));
7890 7891 7892 7893 7894

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u32, buf, 0x7fc0, seg.selector);
	put_smstate(u32, buf, 0x7f80, seg.base);
	put_smstate(u32, buf, 0x7f7c, seg.limit);
7895
	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
7896

7897
	kvm_x86_ops.get_gdt(vcpu, &dt);
7898 7899 7900
	put_smstate(u32, buf, 0x7f74, dt.address);
	put_smstate(u32, buf, 0x7f70, dt.size);

7901
	kvm_x86_ops.get_idt(vcpu, &dt);
7902 7903 7904 7905
	put_smstate(u32, buf, 0x7f58, dt.address);
	put_smstate(u32, buf, 0x7f54, dt.size);

	for (i = 0; i < 6; i++)
7906
		enter_smm_save_seg_32(vcpu, buf, i);
7907 7908 7909 7910 7911 7912 7913 7914

	put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020000);
	put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
}

7915
#ifdef CONFIG_X86_64
7916
static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	for (i = 0; i < 16; i++)
		put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read(vcpu, i));

	put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
	put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u64, buf, 0x7f68, val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u64, buf, 0x7f60, val);

	put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
	put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
	put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));

	put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020064);

	put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u16, buf, 0x7e90, seg.selector);
7947
	put_smstate(u16, buf, 0x7e92, enter_smm_get_segment_flags(&seg) >> 8);
7948 7949 7950
	put_smstate(u32, buf, 0x7e94, seg.limit);
	put_smstate(u64, buf, 0x7e98, seg.base);

7951
	kvm_x86_ops.get_idt(vcpu, &dt);
7952 7953 7954 7955 7956
	put_smstate(u32, buf, 0x7e84, dt.size);
	put_smstate(u64, buf, 0x7e88, dt.address);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u16, buf, 0x7e70, seg.selector);
7957
	put_smstate(u16, buf, 0x7e72, enter_smm_get_segment_flags(&seg) >> 8);
7958 7959 7960
	put_smstate(u32, buf, 0x7e74, seg.limit);
	put_smstate(u64, buf, 0x7e78, seg.base);

7961
	kvm_x86_ops.get_gdt(vcpu, &dt);
7962 7963 7964 7965
	put_smstate(u32, buf, 0x7e64, dt.size);
	put_smstate(u64, buf, 0x7e68, dt.address);

	for (i = 0; i < 6; i++)
7966
		enter_smm_save_seg_64(vcpu, buf, i);
7967
}
7968
#endif
7969

7970
static void enter_smm(struct kvm_vcpu *vcpu)
P
Paolo Bonzini 已提交
7971
{
7972
	struct kvm_segment cs, ds;
7973
	struct desc_ptr dt;
7974 7975 7976 7977 7978
	char buf[512];
	u32 cr0;

	trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true);
	memset(buf, 0, 512);
7979
#ifdef CONFIG_X86_64
7980
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7981
		enter_smm_save_state_64(vcpu, buf);
7982
	else
7983
#endif
7984
		enter_smm_save_state_32(vcpu, buf);
7985

7986 7987 7988 7989 7990
	/*
	 * Give pre_enter_smm() a chance to make ISA-specific changes to the
	 * vCPU state (e.g. leave guest mode) after we've saved the state into
	 * the SMM state-save area.
	 */
7991
	kvm_x86_ops.pre_enter_smm(vcpu, buf);
7992 7993

	vcpu->arch.hflags |= HF_SMM_MASK;
7994
	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
7995

7996
	if (kvm_x86_ops.get_nmi_mask(vcpu))
7997 7998
		vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
	else
7999
		kvm_x86_ops.set_nmi_mask(vcpu, true);
8000 8001 8002 8003 8004

	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
	kvm_rip_write(vcpu, 0x8000);

	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
8005
	kvm_x86_ops.set_cr0(vcpu, cr0);
8006 8007
	vcpu->arch.cr0 = cr0;

8008
	kvm_x86_ops.set_cr4(vcpu, 0);
8009

8010 8011
	/* Undocumented: IDT limit is set to zero on entry to SMM.  */
	dt.address = dt.size = 0;
8012
	kvm_x86_ops.set_idt(vcpu, &dt);
8013

8014 8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032 8033 8034 8035 8036 8037 8038 8039 8040
	__kvm_set_dr(vcpu, 7, DR7_FIXED_1);

	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
	cs.base = vcpu->arch.smbase;

	ds.selector = 0;
	ds.base = 0;

	cs.limit    = ds.limit = 0xffffffff;
	cs.type     = ds.type = 0x3;
	cs.dpl      = ds.dpl = 0;
	cs.db       = ds.db = 0;
	cs.s        = ds.s = 1;
	cs.l        = ds.l = 0;
	cs.g        = ds.g = 1;
	cs.avl      = ds.avl = 0;
	cs.present  = ds.present = 1;
	cs.unusable = ds.unusable = 0;
	cs.padding  = ds.padding = 0;

	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);

8041
#ifdef CONFIG_X86_64
8042
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
8043
		kvm_x86_ops.set_efer(vcpu, 0);
8044
#endif
8045 8046 8047

	kvm_update_cpuid(vcpu);
	kvm_mmu_reset_context(vcpu);
P
Paolo Bonzini 已提交
8048 8049
}

8050
static void process_smi(struct kvm_vcpu *vcpu)
8051 8052 8053 8054 8055
{
	vcpu->arch.smi_pending = true;
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

8056 8057 8058 8059 8060 8061 8062
void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
				       unsigned long *vcpu_bitmap)
{
	cpumask_var_t cpus;

	zalloc_cpumask_var(&cpus, GFP_ATOMIC);

8063
	kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC,
8064
				    NULL, vcpu_bitmap, cpus);
8065 8066 8067 8068

	free_cpumask_var(cpus);
}

8069 8070 8071 8072 8073
void kvm_make_scan_ioapic_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
}

8074 8075 8076 8077 8078 8079 8080
void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
{
	if (!lapic_in_kernel(vcpu))
		return;

	vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm);
	kvm_apic_update_apicv(vcpu);
8081
	kvm_x86_ops.refresh_apicv_exec_ctrl(vcpu);
8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093
}
EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv);

/*
 * NOTE: Do not hold any lock prior to calling this.
 *
 * In particular, kvm_request_apicv_update() expects kvm->srcu not to be
 * locked, because it calls __x86_set_memory_region() which does
 * synchronize_srcu(&kvm->srcu).
 */
void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit)
{
8094
	struct kvm_vcpu *except;
8095 8096
	unsigned long old, new, expected;

8097 8098
	if (!kvm_x86_ops.check_apicv_inhibit_reasons ||
	    !kvm_x86_ops.check_apicv_inhibit_reasons(bit))
8099 8100
		return;

8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112 8113 8114
	old = READ_ONCE(kvm->arch.apicv_inhibit_reasons);
	do {
		expected = new = old;
		if (activate)
			__clear_bit(bit, &new);
		else
			__set_bit(bit, &new);
		if (new == old)
			break;
		old = cmpxchg(&kvm->arch.apicv_inhibit_reasons, expected, new);
	} while (old != expected);

	if (!!old == !!new)
		return;
8115

8116
	trace_kvm_apicv_update_request(activate, bit);
8117 8118
	if (kvm_x86_ops.pre_update_apicv_exec_ctrl)
		kvm_x86_ops.pre_update_apicv_exec_ctrl(kvm, activate);
8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129

	/*
	 * Sending request to update APICV for all other vcpus,
	 * while update the calling vcpu immediately instead of
	 * waiting for another #VMEXIT to handle the request.
	 */
	except = kvm_get_running_vcpu();
	kvm_make_all_cpus_request_except(kvm, KVM_REQ_APICV_UPDATE,
					 except);
	if (except)
		kvm_vcpu_update_apicv(except);
8130 8131 8132
}
EXPORT_SYMBOL_GPL(kvm_request_apicv_update);

8133
static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
8134
{
8135
	if (!kvm_apic_present(vcpu))
8136
		return;
8137

8138
	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
8139

8140
	if (irqchip_split(vcpu->kvm))
8141
		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
8142
	else {
8143
		if (vcpu->arch.apicv_active)
8144
			kvm_x86_ops.sync_pir_to_irr(vcpu);
8145 8146
		if (ioapic_in_kernel(vcpu->kvm))
			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
8147
	}
8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161

	if (is_guest_mode(vcpu))
		vcpu->arch.load_eoi_exitmap_pending = true;
	else
		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
}

static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
{
	u64 eoi_exit_bitmap[4];

	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
		return;

8162 8163
	bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
		  vcpu_to_synic(vcpu)->vec_bitmap, 256);
8164
	kvm_x86_ops.load_eoi_exitmap(vcpu, eoi_exit_bitmap);
8165 8166
}

8167 8168 8169
int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
		unsigned long start, unsigned long end,
		bool blockable)
8170 8171 8172 8173 8174 8175 8176 8177 8178 8179
{
	unsigned long apic_address;

	/*
	 * The physical address of apic access page is stored in the VMCS.
	 * Update it when it becomes invalid.
	 */
	apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
	if (start <= apic_address && apic_address < end)
		kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
8180 8181

	return 0;
8182 8183
}

8184 8185
void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
{
8186
	if (!lapic_in_kernel(vcpu))
8187 8188
		return;

8189
	if (!kvm_x86_ops.set_apic_access_page_addr)
8190 8191
		return;

8192
	kvm_x86_ops.set_apic_access_page_addr(vcpu);
8193 8194
}

8195 8196 8197 8198 8199 8200
void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
{
	smp_send_reschedule(vcpu->cpu);
}
EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);

8201
/*
8202
 * Returns 1 to let vcpu_run() continue the guest execution loop without
8203 8204 8205
 * exiting to the userspace.  Otherwise, the value will be returned to the
 * userspace.
 */
A
Avi Kivity 已提交
8206
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
8207 8208
{
	int r;
8209 8210 8211
	bool req_int_win =
		dm_request_for_irq_injection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);
8212
	enum exit_fastpath_completion exit_fastpath;
8213

8214
	bool req_immediate_exit = false;
8215

R
Radim Krčmář 已提交
8216
	if (kvm_request_pending(vcpu)) {
8217
		if (kvm_check_request(KVM_REQ_GET_VMCS12_PAGES, vcpu)) {
8218
			if (unlikely(!kvm_x86_ops.nested_ops->get_vmcs12_pages(vcpu))) {
8219 8220 8221 8222
				r = 0;
				goto out;
			}
		}
8223
		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
8224
			kvm_mmu_unload(vcpu);
8225
		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
M
Marcelo Tosatti 已提交
8226
			__kvm_migrate_timers(vcpu);
8227 8228
		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
			kvm_gen_update_masterclock(vcpu->kvm);
8229 8230
		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
			kvm_gen_kvmclock_update(vcpu);
Z
Zachary Amsden 已提交
8231 8232
		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
			r = kvm_guest_time_update(vcpu);
8233 8234 8235
			if (unlikely(r))
				goto out;
		}
8236
		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
8237
			kvm_mmu_sync_roots(vcpu);
8238 8239
		if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))
			kvm_mmu_load_pgd(vcpu);
8240
		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
8241
			kvm_vcpu_flush_tlb_all(vcpu);
8242 8243 8244 8245 8246 8247

			/* Flushing all ASIDs flushes the current ASID... */
			kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
		}
		if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
			kvm_vcpu_flush_tlb_current(vcpu);
8248 8249
		if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu))
			kvm_vcpu_flush_tlb_guest(vcpu);
8250

8251
		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
A
Avi Kivity 已提交
8252
			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
A
Avi Kivity 已提交
8253 8254 8255
			r = 0;
			goto out;
		}
8256
		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
A
Avi Kivity 已提交
8257
			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
8258
			vcpu->mmio_needed = 0;
J
Joerg Roedel 已提交
8259 8260 8261
			r = 0;
			goto out;
		}
8262 8263 8264 8265 8266 8267
		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
			/* Page is swapped out. Do synthetic halt */
			vcpu->arch.apf.halted = true;
			r = 1;
			goto out;
		}
G
Glauber Costa 已提交
8268 8269
		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
			record_steal_time(vcpu);
P
Paolo Bonzini 已提交
8270 8271
		if (kvm_check_request(KVM_REQ_SMI, vcpu))
			process_smi(vcpu);
A
Avi Kivity 已提交
8272 8273
		if (kvm_check_request(KVM_REQ_NMI, vcpu))
			process_nmi(vcpu);
8274
		if (kvm_check_request(KVM_REQ_PMU, vcpu))
8275
			kvm_pmu_handle_event(vcpu);
8276
		if (kvm_check_request(KVM_REQ_PMI, vcpu))
8277
			kvm_pmu_deliver_pmi(vcpu);
8278 8279 8280
		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
			if (test_bit(vcpu->arch.pending_ioapic_eoi,
8281
				     vcpu->arch.ioapic_handled_vectors)) {
8282 8283 8284 8285 8286 8287 8288
				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
				vcpu->run->eoi.vector =
						vcpu->arch.pending_ioapic_eoi;
				r = 0;
				goto out;
			}
		}
8289 8290
		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
			vcpu_scan_ioapic(vcpu);
8291 8292
		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
			vcpu_load_eoi_exitmap(vcpu);
8293 8294
		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
			kvm_vcpu_reload_apic_access_page(vcpu);
8295 8296 8297 8298 8299 8300
		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
			r = 0;
			goto out;
		}
8301 8302 8303 8304 8305 8306
		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
			r = 0;
			goto out;
		}
A
Andrey Smetanin 已提交
8307 8308 8309 8310 8311 8312
		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_HYPERV;
			vcpu->run->hyperv = vcpu->arch.hyperv.exit;
			r = 0;
			goto out;
		}
8313 8314 8315 8316 8317 8318

		/*
		 * KVM_REQ_HV_STIMER has to be processed after
		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
		 * depend on the guest clock being up-to-date
		 */
A
Andrey Smetanin 已提交
8319 8320
		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
			kvm_hv_process_stimers(vcpu);
8321 8322
		if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))
			kvm_vcpu_update_apicv(vcpu);
8323
	}
A
Avi Kivity 已提交
8324

A
Avi Kivity 已提交
8325
	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
8326
		++vcpu->stat.req_event;
8327 8328 8329 8330 8331 8332
		kvm_apic_accept_events(vcpu);
		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
			r = 1;
			goto out;
		}

8333
		if (inject_pending_event(vcpu) != 0)
8334
			req_immediate_exit = true;
8335
		else {
8336
			/* Enable SMI/NMI/IRQ window open exits if needed.
8337
			 *
8338 8339 8340 8341 8342 8343 8344 8345 8346 8347 8348
			 * SMIs have three cases:
			 * 1) They can be nested, and then there is nothing to
			 *    do here because RSM will cause a vmexit anyway.
			 * 2) There is an ISA-specific reason why SMI cannot be
			 *    injected, and the moment when this changes can be
			 *    intercepted.
			 * 3) Or the SMI can be pending because
			 *    inject_pending_event has completed the injection
			 *    of an IRQ or NMI from the previous vmexit, and
			 *    then we request an immediate exit to inject the
			 *    SMI.
8349 8350
			 */
			if (vcpu->arch.smi_pending && !is_smm(vcpu))
8351
				if (!kvm_x86_ops.enable_smi_window(vcpu))
8352
					req_immediate_exit = true;
8353
			if (vcpu->arch.nmi_pending)
8354
				kvm_x86_ops.enable_nmi_window(vcpu);
8355
			if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
8356
				kvm_x86_ops.enable_irq_window(vcpu);
8357 8358 8359 8360
			if (is_guest_mode(vcpu) &&
			    kvm_x86_ops.nested_ops->hv_timer_pending &&
			    kvm_x86_ops.nested_ops->hv_timer_pending(vcpu))
				req_immediate_exit = true;
8361
			WARN_ON(vcpu->arch.exception.pending);
8362
		}
A
Avi Kivity 已提交
8363 8364 8365 8366 8367 8368 8369

		if (kvm_lapic_enabled(vcpu)) {
			update_cr8_intercept(vcpu);
			kvm_lapic_sync_to_vapic(vcpu);
		}
	}

8370 8371
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r)) {
8372
		goto cancel_injection;
8373 8374
	}

8375 8376
	preempt_disable();

8377
	kvm_x86_ops.prepare_guest_switch(vcpu);
8378 8379 8380 8381 8382 8383 8384

	/*
	 * Disable IRQs before setting IN_GUEST_MODE.  Posted interrupt
	 * IPI are then delayed after guest entry, which ensures that they
	 * result in virtual interrupt delivery.
	 */
	local_irq_disable();
8385 8386
	vcpu->mode = IN_GUEST_MODE;

8387 8388
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);

8389
	/*
8390
	 * 1) We should set ->mode before checking ->requests.  Please see
8391
	 * the comment in kvm_vcpu_exiting_guest_mode().
8392
	 *
8393
	 * 2) For APICv, we should set ->mode before checking PID.ON. This
8394 8395 8396 8397 8398 8399
	 * pairs with the memory barrier implicit in pi_test_and_set_on
	 * (see vmx_deliver_posted_interrupt).
	 *
	 * 3) This also orders the write to mode from any reads to the page
	 * tables done while the VCPU is running.  Please see the comment
	 * in kvm_flush_remote_tlbs.
8400
	 */
8401
	smp_mb__after_srcu_read_unlock();
8402

8403 8404 8405 8406
	/*
	 * This handles the case where a posted interrupt was
	 * notified with kvm_vcpu_kick.
	 */
8407
	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
8408
		kvm_x86_ops.sync_pir_to_irr(vcpu);
8409

R
Radim Krčmář 已提交
8410
	if (vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu)
A
Avi Kivity 已提交
8411
	    || need_resched() || signal_pending(current)) {
8412
		vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
8413
		smp_wmb();
8414 8415
		local_irq_enable();
		preempt_enable();
8416
		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8417
		r = 1;
8418
		goto cancel_injection;
8419 8420
	}

8421 8422
	if (req_immediate_exit) {
		kvm_make_request(KVM_REQ_EVENT, vcpu);
8423
		kvm_x86_ops.request_immediate_exit(vcpu);
8424
	}
8425

8426
	trace_kvm_entry(vcpu->vcpu_id);
8427
	guest_enter_irqoff();
8428

8429 8430 8431
	fpregs_assert_state_consistent();
	if (test_thread_flag(TIF_NEED_FPU_LOAD))
		switch_fpu_return();
8432

8433 8434 8435 8436 8437 8438
	if (unlikely(vcpu->arch.switch_db_regs)) {
		set_debugreg(0, 7);
		set_debugreg(vcpu->arch.eff_db[0], 0);
		set_debugreg(vcpu->arch.eff_db[1], 1);
		set_debugreg(vcpu->arch.eff_db[2], 2);
		set_debugreg(vcpu->arch.eff_db[3], 3);
8439
		set_debugreg(vcpu->arch.dr6, 6);
8440
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8441
	}
8442

8443
	exit_fastpath = kvm_x86_ops.run(vcpu);
8444

8445 8446 8447 8448 8449 8450 8451 8452
	/*
	 * Do this here before restoring debug registers on the host.  And
	 * since we do this before handling the vmexit, a DR access vmexit
	 * can (a) read the correct value of the debug registers, (b) set
	 * KVM_DEBUGREG_WONT_EXIT again.
	 */
	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
8453
		kvm_x86_ops.sync_dirty_debug_regs(vcpu);
8454 8455 8456
		kvm_update_dr0123(vcpu);
		kvm_update_dr7(vcpu);
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8457 8458
	}

8459 8460 8461 8462 8463 8464 8465
	/*
	 * If the guest has used debug registers, at least dr7
	 * will be disabled while returning to the host.
	 * If we don't have active breakpoints in the host, we don't
	 * care about the messed up debug address registers. But if
	 * we have some of them active, restore the old state.
	 */
8466
	if (hw_breakpoint_active())
8467
		hw_breakpoint_restore();
8468

8469
	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
8470

8471
	vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
8472
	smp_wmb();
8473

8474
	kvm_x86_ops.handle_exit_irqoff(vcpu);
8475

8476 8477 8478 8479 8480 8481 8482 8483 8484
	/*
	 * Consume any pending interrupts, including the possible source of
	 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
	 * An instruction is required after local_irq_enable() to fully unblock
	 * interrupts on processors that implement an interrupt shadow, the
	 * stat.exits increment will do nicely.
	 */
	kvm_before_interrupt(vcpu);
	local_irq_enable();
8485
	++vcpu->stat.exits;
8486 8487
	local_irq_disable();
	kvm_after_interrupt(vcpu);
8488

P
Paolo Bonzini 已提交
8489
	guest_exit_irqoff();
8490 8491 8492 8493 8494 8495 8496
	if (lapic_in_kernel(vcpu)) {
		s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;
		if (delta != S64_MIN) {
			trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta);
			vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;
		}
	}
8497

P
Paolo Bonzini 已提交
8498
	local_irq_enable();
8499 8500
	preempt_enable();

8501
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8502

8503 8504 8505 8506
	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
8507 8508
		unsigned long rip = kvm_rip_read(vcpu);
		profile_hit(KVM_PROFILING, (void *)rip);
8509 8510
	}

8511 8512
	if (unlikely(vcpu->arch.tsc_always_catchup))
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
8513

8514 8515
	if (vcpu->arch.apic_attention)
		kvm_lapic_sync_from_vapic(vcpu);
A
Avi Kivity 已提交
8516

8517
	r = kvm_x86_ops.handle_exit(vcpu, exit_fastpath);
8518 8519 8520
	return r;

cancel_injection:
8521 8522
	if (req_immediate_exit)
		kvm_make_request(KVM_REQ_EVENT, vcpu);
8523
	kvm_x86_ops.cancel_injection(vcpu);
8524 8525
	if (unlikely(vcpu->arch.apic_attention))
		kvm_lapic_sync_from_vapic(vcpu);
8526 8527 8528
out:
	return r;
}
8529

8530 8531
static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
{
8532
	if (!kvm_arch_vcpu_runnable(vcpu) &&
8533
	    (!kvm_x86_ops.pre_block || kvm_x86_ops.pre_block(vcpu) == 0)) {
8534 8535 8536
		srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
		kvm_vcpu_block(vcpu);
		vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8537

8538 8539
		if (kvm_x86_ops.post_block)
			kvm_x86_ops.post_block(vcpu);
8540

8541 8542 8543
		if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
			return 1;
	}
8544 8545 8546 8547 8548 8549 8550

	kvm_apic_accept_events(vcpu);
	switch(vcpu->arch.mp_state) {
	case KVM_MP_STATE_HALTED:
		vcpu->arch.pv.pv_unhalted = false;
		vcpu->arch.mp_state =
			KVM_MP_STATE_RUNNABLE;
8551
		/* fall through */
8552 8553 8554 8555 8556 8557 8558 8559 8560 8561
	case KVM_MP_STATE_RUNNABLE:
		vcpu->arch.apf.halted = false;
		break;
	case KVM_MP_STATE_INIT_RECEIVED:
		break;
	default:
		return -EINTR;
	}
	return 1;
}
8562

8563 8564
static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
{
8565
	if (is_guest_mode(vcpu))
8566
		kvm_x86_ops.nested_ops->check_events(vcpu);
8567

8568 8569 8570 8571
	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		!vcpu->arch.apf.halted);
}

8572
static int vcpu_run(struct kvm_vcpu *vcpu)
8573 8574
{
	int r;
8575
	struct kvm *kvm = vcpu->kvm;
8576

8577
	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
P
Paolo Bonzini 已提交
8578
	vcpu->arch.l1tf_flush_l1d = true;
8579

8580
	for (;;) {
8581
		if (kvm_vcpu_running(vcpu)) {
A
Avi Kivity 已提交
8582
			r = vcpu_enter_guest(vcpu);
8583
		} else {
8584
			r = vcpu_block(kvm, vcpu);
8585 8586
		}

8587 8588 8589
		if (r <= 0)
			break;

8590
		kvm_clear_request(KVM_REQ_PENDING_TIMER, vcpu);
8591 8592 8593
		if (kvm_cpu_has_pending_timer(vcpu))
			kvm_inject_pending_timer_irqs(vcpu);

8594 8595
		if (dm_request_for_irq_injection(vcpu) &&
			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
8596 8597
			r = 0;
			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
8598
			++vcpu->stat.request_irq_exits;
8599
			break;
8600
		}
8601 8602 8603

		kvm_check_async_pf_completion(vcpu);

8604 8605
		if (signal_pending(current)) {
			r = -EINTR;
A
Avi Kivity 已提交
8606
			vcpu->run->exit_reason = KVM_EXIT_INTR;
8607
			++vcpu->stat.signal_exits;
8608
			break;
8609 8610
		}
		if (need_resched()) {
8611
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8612
			cond_resched();
8613
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8614
		}
8615 8616
	}

8617
	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8618 8619 8620 8621

	return r;
}

8622 8623 8624
static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
{
	int r;
8625

8626
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8627
	r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
8628
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
8629
	return r;
8630 8631 8632 8633 8634 8635 8636 8637 8638
}

static int complete_emulated_pio(struct kvm_vcpu *vcpu)
{
	BUG_ON(!vcpu->arch.pio.count);

	return complete_emulated_io(vcpu);
}

A
Avi Kivity 已提交
8639 8640 8641 8642 8643
/*
 * Implements the following, as a state machine:
 *
 * read:
 *   for each fragment
8644 8645 8646 8647
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       exit
 *       copy data
A
Avi Kivity 已提交
8648 8649 8650 8651
 *   execute insn
 *
 * write:
 *   for each fragment
8652 8653 8654 8655
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       copy data
 *       exit
A
Avi Kivity 已提交
8656
 */
8657
static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
8658 8659
{
	struct kvm_run *run = vcpu->run;
A
Avi Kivity 已提交
8660
	struct kvm_mmio_fragment *frag;
8661
	unsigned len;
8662

8663
	BUG_ON(!vcpu->mmio_needed);
8664

8665
	/* Complete previous fragment */
8666 8667
	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
	len = min(8u, frag->len);
8668
	if (!vcpu->mmio_is_write)
8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680 8681
		memcpy(frag->data, run->mmio.data, len);

	if (frag->len <= 8) {
		/* Switch to the next fragment. */
		frag++;
		vcpu->mmio_cur_fragment++;
	} else {
		/* Go forward to the next mmio piece. */
		frag->data += len;
		frag->gpa += len;
		frag->len -= len;
	}

8682
	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
8683
		vcpu->mmio_needed = 0;
8684 8685

		/* FIXME: return into emulator if single-stepping.  */
A
Avi Kivity 已提交
8686
		if (vcpu->mmio_is_write)
8687 8688 8689 8690
			return 1;
		vcpu->mmio_read_completed = 1;
		return complete_emulated_io(vcpu);
	}
8691

8692 8693 8694
	run->exit_reason = KVM_EXIT_MMIO;
	run->mmio.phys_addr = frag->gpa;
	if (vcpu->mmio_is_write)
8695 8696
		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
	run->mmio.len = min(8u, frag->len);
8697 8698 8699
	run->mmio.is_write = vcpu->mmio_is_write;
	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
	return 0;
8700 8701
}

8702 8703 8704 8705 8706 8707 8708 8709 8710 8711 8712 8713 8714
static void kvm_save_current_fpu(struct fpu *fpu)
{
	/*
	 * If the target FPU state is not resident in the CPU registers, just
	 * memcpy() from current, else save CPU state directly to the target.
	 */
	if (test_thread_flag(TIF_NEED_FPU_LOAD))
		memcpy(&fpu->state, &current->thread.fpu.state,
		       fpu_kernel_xstate_size);
	else
		copy_fpregs_to_fpstate(fpu);
}

8715 8716 8717
/* Swap (qemu) user FPU context for the guest FPU context. */
static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
8718 8719
	fpregs_lock();

8720 8721
	kvm_save_current_fpu(vcpu->arch.user_fpu);

8722
	/* PKRU is separately restored in kvm_x86_ops.run.  */
8723
	__copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,
8724
				~XFEATURE_MASK_PKRU);
8725 8726 8727 8728

	fpregs_mark_activate();
	fpregs_unlock();

8729 8730 8731 8732 8733 8734
	trace_kvm_fpu(1);
}

/* When vcpu_run ends, restore user space FPU context. */
static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
8735 8736
	fpregs_lock();

8737 8738
	kvm_save_current_fpu(vcpu->arch.guest_fpu);

8739
	copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);
8740 8741 8742 8743

	fpregs_mark_activate();
	fpregs_unlock();

8744 8745 8746 8747
	++vcpu->stat.fpu_reload;
	trace_kvm_fpu(0);
}

8748
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
8749
{
8750
	struct kvm_run *kvm_run = vcpu->run;
8751 8752
	int r;

8753
	vcpu_load(vcpu);
8754
	kvm_sigset_activate(vcpu);
8755 8756
	kvm_load_guest_fpu(vcpu);

8757
	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
8758 8759 8760 8761
		if (kvm_run->immediate_exit) {
			r = -EINTR;
			goto out;
		}
8762
		kvm_vcpu_block(vcpu);
8763
		kvm_apic_accept_events(vcpu);
8764
		kvm_clear_request(KVM_REQ_UNHALT, vcpu);
8765
		r = -EAGAIN;
8766 8767
		if (signal_pending(current)) {
			r = -EINTR;
8768
			kvm_run->exit_reason = KVM_EXIT_INTR;
8769 8770
			++vcpu->stat.signal_exits;
		}
8771
		goto out;
8772 8773
	}

8774
	if (kvm_run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) {
K
Ken Hofsass 已提交
8775 8776 8777 8778
		r = -EINVAL;
		goto out;
	}

8779
	if (kvm_run->kvm_dirty_regs) {
K
Ken Hofsass 已提交
8780 8781 8782 8783 8784
		r = sync_regs(vcpu);
		if (r != 0)
			goto out;
	}

8785
	/* re-sync apic's tpr */
8786
	if (!lapic_in_kernel(vcpu)) {
A
Andre Przywara 已提交
8787 8788 8789 8790 8791
		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
			r = -EINVAL;
			goto out;
		}
	}
8792

8793 8794 8795 8796 8797
	if (unlikely(vcpu->arch.complete_userspace_io)) {
		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
		vcpu->arch.complete_userspace_io = NULL;
		r = cui(vcpu);
		if (r <= 0)
8798
			goto out;
8799 8800
	} else
		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
8801

8802 8803 8804 8805
	if (kvm_run->immediate_exit)
		r = -EINTR;
	else
		r = vcpu_run(vcpu);
8806 8807

out:
8808
	kvm_put_guest_fpu(vcpu);
8809
	if (kvm_run->kvm_valid_regs)
K
Ken Hofsass 已提交
8810
		store_regs(vcpu);
8811
	post_kvm_run_save(vcpu);
8812
	kvm_sigset_deactivate(vcpu);
8813

8814
	vcpu_put(vcpu);
8815 8816 8817
	return r;
}

K
Ken Hofsass 已提交
8818
static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8819
{
8820 8821 8822 8823
	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
		/*
		 * We are here if userspace calls get_regs() in the middle of
		 * instruction emulation. Registers state needs to be copied
G
Guo Chao 已提交
8824
		 * back from emulation context to vcpu. Userspace shouldn't do
8825 8826 8827
		 * that usually, but some bad designed PV devices (vmware
		 * backdoor interface) need this to work
		 */
8828
		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
8829 8830
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
	}
8831 8832 8833 8834 8835 8836
	regs->rax = kvm_rax_read(vcpu);
	regs->rbx = kvm_rbx_read(vcpu);
	regs->rcx = kvm_rcx_read(vcpu);
	regs->rdx = kvm_rdx_read(vcpu);
	regs->rsi = kvm_rsi_read(vcpu);
	regs->rdi = kvm_rdi_read(vcpu);
8837
	regs->rsp = kvm_rsp_read(vcpu);
8838
	regs->rbp = kvm_rbp_read(vcpu);
8839
#ifdef CONFIG_X86_64
8840 8841 8842 8843 8844 8845 8846 8847
	regs->r8 = kvm_r8_read(vcpu);
	regs->r9 = kvm_r9_read(vcpu);
	regs->r10 = kvm_r10_read(vcpu);
	regs->r11 = kvm_r11_read(vcpu);
	regs->r12 = kvm_r12_read(vcpu);
	regs->r13 = kvm_r13_read(vcpu);
	regs->r14 = kvm_r14_read(vcpu);
	regs->r15 = kvm_r15_read(vcpu);
8848 8849
#endif

8850
	regs->rip = kvm_rip_read(vcpu);
8851
	regs->rflags = kvm_get_rflags(vcpu);
K
Ken Hofsass 已提交
8852
}
8853

K
Ken Hofsass 已提交
8854 8855 8856 8857
int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__get_regs(vcpu, regs);
8858
	vcpu_put(vcpu);
8859 8860 8861
	return 0;
}

K
Ken Hofsass 已提交
8862
static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8863
{
8864 8865 8866
	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;

8867 8868 8869 8870 8871 8872
	kvm_rax_write(vcpu, regs->rax);
	kvm_rbx_write(vcpu, regs->rbx);
	kvm_rcx_write(vcpu, regs->rcx);
	kvm_rdx_write(vcpu, regs->rdx);
	kvm_rsi_write(vcpu, regs->rsi);
	kvm_rdi_write(vcpu, regs->rdi);
8873
	kvm_rsp_write(vcpu, regs->rsp);
8874
	kvm_rbp_write(vcpu, regs->rbp);
8875
#ifdef CONFIG_X86_64
8876 8877 8878 8879 8880 8881 8882 8883
	kvm_r8_write(vcpu, regs->r8);
	kvm_r9_write(vcpu, regs->r9);
	kvm_r10_write(vcpu, regs->r10);
	kvm_r11_write(vcpu, regs->r11);
	kvm_r12_write(vcpu, regs->r12);
	kvm_r13_write(vcpu, regs->r13);
	kvm_r14_write(vcpu, regs->r14);
	kvm_r15_write(vcpu, regs->r15);
8884 8885
#endif

8886
	kvm_rip_write(vcpu, regs->rip);
8887
	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
8888

8889 8890
	vcpu->arch.exception.pending = false;

8891
	kvm_make_request(KVM_REQ_EVENT, vcpu);
K
Ken Hofsass 已提交
8892
}
8893

K
Ken Hofsass 已提交
8894 8895 8896 8897
int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__set_regs(vcpu, regs);
8898
	vcpu_put(vcpu);
8899 8900 8901 8902 8903 8904 8905
	return 0;
}

void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

8906
	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
8907 8908 8909 8910 8911
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

K
Ken Hofsass 已提交
8912
static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8913
{
8914
	struct desc_ptr dt;
8915

8916 8917 8918 8919 8920 8921
	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8922

8923 8924
	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
8925

8926
	kvm_x86_ops.get_idt(vcpu, &dt);
8927 8928
	sregs->idt.limit = dt.size;
	sregs->idt.base = dt.address;
8929
	kvm_x86_ops.get_gdt(vcpu, &dt);
8930 8931
	sregs->gdt.limit = dt.size;
	sregs->gdt.base = dt.address;
8932

8933
	sregs->cr0 = kvm_read_cr0(vcpu);
8934
	sregs->cr2 = vcpu->arch.cr2;
8935
	sregs->cr3 = kvm_read_cr3(vcpu);
8936
	sregs->cr4 = kvm_read_cr4(vcpu);
8937
	sregs->cr8 = kvm_get_cr8(vcpu);
8938
	sregs->efer = vcpu->arch.efer;
8939 8940
	sregs->apic_base = kvm_get_apic_base(vcpu);

8941
	memset(sregs->interrupt_bitmap, 0, sizeof(sregs->interrupt_bitmap));
8942

8943
	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
8944 8945
		set_bit(vcpu->arch.interrupt.nr,
			(unsigned long *)sregs->interrupt_bitmap);
K
Ken Hofsass 已提交
8946
}
8947

K
Ken Hofsass 已提交
8948 8949 8950 8951 8952
int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	vcpu_load(vcpu);
	__get_sregs(vcpu, sregs);
8953
	vcpu_put(vcpu);
8954 8955 8956
	return 0;
}

8957 8958 8959
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
8960
	vcpu_load(vcpu);
8961 8962
	if (kvm_mpx_supported())
		kvm_load_guest_fpu(vcpu);
8963

8964
	kvm_apic_accept_events(vcpu);
8965 8966 8967 8968 8969 8970
	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
					vcpu->arch.pv.pv_unhalted)
		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
	else
		mp_state->mp_state = vcpu->arch.mp_state;

8971 8972
	if (kvm_mpx_supported())
		kvm_put_guest_fpu(vcpu);
8973
	vcpu_put(vcpu);
8974 8975 8976 8977 8978 8979
	return 0;
}

int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
8980 8981 8982 8983
	int ret = -EINVAL;

	vcpu_load(vcpu);

8984
	if (!lapic_in_kernel(vcpu) &&
8985
	    mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
8986
		goto out;
8987

8988 8989 8990 8991 8992 8993
	/*
	 * KVM_MP_STATE_INIT_RECEIVED means the processor is in
	 * INIT state; latched init should be reported using
	 * KVM_SET_VCPU_EVENTS, so reject it here.
	 */
	if ((kvm_vcpu_latch_init(vcpu) || vcpu->arch.smi_pending) &&
8994 8995
	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
8996
		goto out;
8997

8998 8999 9000 9001 9002
	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
	} else
		vcpu->arch.mp_state = mp_state->mp_state;
9003
	kvm_make_request(KVM_REQ_EVENT, vcpu);
9004 9005 9006 9007 9008

	ret = 0;
out:
	vcpu_put(vcpu);
	return ret;
9009 9010
}

9011 9012
int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
		    int reason, bool has_error_code, u32 error_code)
9013
{
9014
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
9015
	int ret;
9016

9017
	init_emulate_ctxt(vcpu);
9018

9019
	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
9020
				   has_error_code, error_code);
9021 9022 9023 9024
	if (ret) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
9025
		return 0;
9026
	}
9027

9028 9029
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
9030
	return 1;
9031 9032 9033
}
EXPORT_SYMBOL_GPL(kvm_task_switch);

P
Peng Hao 已提交
9034
static int kvm_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
9035
{
9036
	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
9037 9038 9039 9040 9041
		/*
		 * When EFER.LME and CR0.PG are set, the processor is in
		 * 64-bit mode (though maybe in a 32-bit code segment).
		 * CR4.PAE and EFER.LMA must be set.
		 */
9042
		if (!(sregs->cr4 & X86_CR4_PAE)
9043 9044 9045 9046 9047 9048 9049 9050 9051 9052 9053
		    || !(sregs->efer & EFER_LMA))
			return -EINVAL;
	} else {
		/*
		 * Not in 64-bit mode: EFER.LMA is clear and the code
		 * segment cannot be 64-bit.
		 */
		if (sregs->efer & EFER_LMA || sregs->cs.l)
			return -EINVAL;
	}

9054
	return kvm_valid_cr4(vcpu, sregs->cr4);
9055 9056
}

K
Ken Hofsass 已提交
9057
static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
9058
{
9059
	struct msr_data apic_base_msr;
9060
	int mmu_reset_needed = 0;
9061
	int cpuid_update_needed = 0;
9062
	int pending_vec, max_bits, idx;
9063
	struct desc_ptr dt;
9064 9065
	int ret = -EINVAL;

9066
	if (kvm_valid_sregs(vcpu, sregs))
9067
		goto out;
9068

9069 9070 9071
	apic_base_msr.data = sregs->apic_base;
	apic_base_msr.host_initiated = true;
	if (kvm_set_apic_base(vcpu, &apic_base_msr))
9072
		goto out;
9073

9074 9075
	dt.size = sregs->idt.limit;
	dt.address = sregs->idt.base;
9076
	kvm_x86_ops.set_idt(vcpu, &dt);
9077 9078
	dt.size = sregs->gdt.limit;
	dt.address = sregs->gdt.base;
9079
	kvm_x86_ops.set_gdt(vcpu, &dt);
9080

9081
	vcpu->arch.cr2 = sregs->cr2;
9082
	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
9083
	vcpu->arch.cr3 = sregs->cr3;
9084
	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
9085

9086
	kvm_set_cr8(vcpu, sregs->cr8);
9087

9088
	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
9089
	kvm_x86_ops.set_efer(vcpu, sregs->efer);
9090

9091
	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
9092
	kvm_x86_ops.set_cr0(vcpu, sregs->cr0);
9093
	vcpu->arch.cr0 = sregs->cr0;
9094

9095
	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
9096 9097
	cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) &
				(X86_CR4_OSXSAVE | X86_CR4_PKE));
9098
	kvm_x86_ops.set_cr4(vcpu, sregs->cr4);
9099
	if (cpuid_update_needed)
A
Avi Kivity 已提交
9100
		kvm_update_cpuid(vcpu);
9101 9102

	idx = srcu_read_lock(&vcpu->kvm->srcu);
9103
	if (is_pae_paging(vcpu)) {
9104
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
9105 9106
		mmu_reset_needed = 1;
	}
9107
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9108 9109 9110 9111

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

9112
	max_bits = KVM_NR_INTERRUPTS;
G
Gleb Natapov 已提交
9113 9114 9115
	pending_vec = find_first_bit(
		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
	if (pending_vec < max_bits) {
9116
		kvm_queue_interrupt(vcpu, pending_vec, false);
G
Gleb Natapov 已提交
9117
		pr_debug("Set back pending irq %d\n", pending_vec);
9118 9119
	}

9120 9121 9122 9123 9124 9125
	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
9126

9127 9128
	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
9129

9130 9131
	update_cr8_intercept(vcpu);

M
Marcelo Tosatti 已提交
9132
	/* Older userspace won't unhalt the vcpu on reset. */
9133
	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
M
Marcelo Tosatti 已提交
9134
	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
9135
	    !is_protmode(vcpu))
M
Marcelo Tosatti 已提交
9136 9137
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;

9138 9139
	kvm_make_request(KVM_REQ_EVENT, vcpu);

9140 9141
	ret = 0;
out:
K
Ken Hofsass 已提交
9142 9143 9144 9145 9146 9147 9148 9149 9150 9151
	return ret;
}

int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	int ret;

	vcpu_load(vcpu);
	ret = __set_sregs(vcpu, sregs);
9152 9153
	vcpu_put(vcpu);
	return ret;
9154 9155
}

J
Jan Kiszka 已提交
9156 9157
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg)
9158
{
9159
	unsigned long rflags;
9160
	int i, r;
9161

9162 9163
	vcpu_load(vcpu);

9164 9165 9166
	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
		r = -EBUSY;
		if (vcpu->arch.exception.pending)
9167
			goto out;
9168 9169 9170 9171 9172 9173
		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
			kvm_queue_exception(vcpu, DB_VECTOR);
		else
			kvm_queue_exception(vcpu, BP_VECTOR);
	}

9174 9175 9176 9177 9178
	/*
	 * Read rflags as long as potentially injected trace flags are still
	 * filtered out.
	 */
	rflags = kvm_get_rflags(vcpu);
9179 9180 9181 9182 9183 9184

	vcpu->guest_debug = dbg->control;
	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
		vcpu->guest_debug = 0;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
9185 9186
		for (i = 0; i < KVM_NR_DB_REGS; ++i)
			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
9187
		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
9188 9189 9190 9191
	} else {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
	}
9192
	kvm_update_dr7(vcpu);
9193

J
Jan Kiszka 已提交
9194 9195 9196
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
			get_segment_base(vcpu, VCPU_SREG_CS);
9197

9198 9199 9200 9201 9202
	/*
	 * Trigger an rflags update that will inject or remove the trace
	 * flags.
	 */
	kvm_set_rflags(vcpu, rflags);
9203

9204
	kvm_x86_ops.update_bp_intercept(vcpu);
9205

9206
	r = 0;
J
Jan Kiszka 已提交
9207

9208
out:
9209
	vcpu_put(vcpu);
9210 9211 9212
	return r;
}

9213 9214 9215 9216 9217 9218 9219 9220
/*
 * Translate a guest virtual address to a guest physical address.
 */
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;
9221
	int idx;
9222

9223 9224
	vcpu_load(vcpu);

9225
	idx = srcu_read_lock(&vcpu->kvm->srcu);
9226
	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
9227
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9228 9229 9230 9231 9232
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;

9233
	vcpu_put(vcpu);
9234 9235 9236
	return 0;
}

9237 9238
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
9239
	struct fxregs_state *fxsave;
9240

9241
	vcpu_load(vcpu);
9242

9243
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
9244 9245 9246 9247 9248 9249 9250
	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
9251
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
9252

9253
	vcpu_put(vcpu);
9254 9255 9256 9257 9258
	return 0;
}

int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
9259 9260 9261 9262
	struct fxregs_state *fxsave;

	vcpu_load(vcpu);

9263
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
9264 9265 9266 9267 9268 9269 9270 9271

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
9272
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
9273

9274
	vcpu_put(vcpu);
9275 9276 9277
	return 0;
}

K
Ken Hofsass 已提交
9278 9279 9280 9281 9282 9283 9284 9285 9286 9287 9288 9289 9290 9291 9292 9293 9294 9295 9296 9297 9298 9299 9300 9301 9302 9303 9304 9305 9306 9307 9308 9309 9310 9311 9312 9313 9314 9315 9316
static void store_regs(struct kvm_vcpu *vcpu)
{
	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
		__get_regs(vcpu, &vcpu->run->s.regs.regs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
		kvm_vcpu_ioctl_x86_get_vcpu_events(
				vcpu, &vcpu->run->s.regs.events);
}

static int sync_regs(struct kvm_vcpu *vcpu)
{
	if (vcpu->run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)
		return -EINVAL;

	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
		__set_regs(vcpu, &vcpu->run->s.regs.regs);
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
		if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
		if (kvm_vcpu_ioctl_x86_set_vcpu_events(
				vcpu, &vcpu->run->s.regs.events))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
	}

	return 0;
}

I
Ingo Molnar 已提交
9317
static void fx_init(struct kvm_vcpu *vcpu)
9318
{
9319
	fpstate_init(&vcpu->arch.guest_fpu->state);
9320
	if (boot_cpu_has(X86_FEATURE_XSAVES))
9321
		vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =
9322
			host_xcr0 | XSTATE_COMPACTION_ENABLED;
9323

9324 9325 9326
	/*
	 * Ensure guest xcr0 is valid for loading
	 */
D
Dave Hansen 已提交
9327
	vcpu->arch.xcr0 = XFEATURE_MASK_FP;
9328

9329
	vcpu->arch.cr0 |= X86_CR0_ET;
9330 9331
}

9332
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
9333
{
9334 9335 9336
	if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
		pr_warn_once("kvm: SMP vm created on host with unstable TSC; "
			     "guest TSC will not be reliable\n");
9337

9338
	return 0;
9339 9340
}

9341
int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
9342
{
9343 9344
	struct page *page;
	int r;
9345

9346 9347 9348 9349
	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
	else
		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
9350

9351
	kvm_set_tsc_khz(vcpu, max_tsc_khz);
9352

9353 9354 9355 9356 9357 9358 9359 9360
	r = kvm_mmu_create(vcpu);
	if (r < 0)
		return r;

	if (irqchip_in_kernel(vcpu->kvm)) {
		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
		if (r < 0)
			goto fail_mmu_destroy;
9361 9362
		if (kvm_apicv_activated(vcpu->kvm))
			vcpu->arch.apicv_active = true;
9363 9364 9365 9366 9367 9368 9369 9370 9371 9372 9373 9374 9375 9376 9377 9378 9379 9380 9381 9382
	} else
		static_key_slow_inc(&kvm_no_apic_vcpu);

	r = -ENOMEM;

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page)
		goto fail_free_lapic;
	vcpu->arch.pio_data = page_address(page);

	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
				       GFP_KERNEL_ACCOUNT);
	if (!vcpu->arch.mce_banks)
		goto fail_free_pio_data;
	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;

	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
				GFP_KERNEL_ACCOUNT))
		goto fail_free_mce_banks;

9383 9384 9385
	if (!alloc_emulate_ctxt(vcpu))
		goto free_wbinvd_dirty_mask;

9386 9387 9388 9389
	vcpu->arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,
						GFP_KERNEL_ACCOUNT);
	if (!vcpu->arch.user_fpu) {
		pr_err("kvm: failed to allocate userspace's fpu\n");
9390
		goto free_emulate_ctxt;
9391 9392 9393 9394 9395 9396 9397 9398 9399 9400 9401 9402 9403 9404 9405 9406 9407 9408 9409 9410 9411 9412 9413 9414
	}

	vcpu->arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,
						 GFP_KERNEL_ACCOUNT);
	if (!vcpu->arch.guest_fpu) {
		pr_err("kvm: failed to allocate vcpu's fpu\n");
		goto free_user_fpu;
	}
	fx_init(vcpu);

	vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;

	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);

	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;

	kvm_async_pf_hash_reset(vcpu);
	kvm_pmu_init(vcpu);

	vcpu->arch.pending_external_vector = -1;
	vcpu->arch.preempted_in_kernel = false;

	kvm_hv_vcpu_init(vcpu);

9415
	r = kvm_x86_ops.vcpu_create(vcpu);
9416 9417
	if (r)
		goto free_guest_fpu;
9418

9419
	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
9420
	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
X
Xiao Guangrong 已提交
9421
	kvm_vcpu_mtrr_init(vcpu);
9422
	vcpu_load(vcpu);
9423
	kvm_vcpu_reset(vcpu, false);
9424
	kvm_init_mmu(vcpu, false);
9425
	vcpu_put(vcpu);
9426
	return 0;
9427 9428 9429 9430 9431

free_guest_fpu:
	kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
free_user_fpu:
	kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
9432 9433
free_emulate_ctxt:
	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444
free_wbinvd_dirty_mask:
	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
fail_free_mce_banks:
	kfree(vcpu->arch.mce_banks);
fail_free_pio_data:
	free_page((unsigned long)vcpu->arch.pio_data);
fail_free_lapic:
	kvm_free_lapic(vcpu);
fail_mmu_destroy:
	kvm_mmu_destroy(vcpu);
	return r;
9445 9446
}

9447
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
9448
{
9449
	struct msr_data msr;
9450
	struct kvm *kvm = vcpu->kvm;
9451

9452 9453
	kvm_hv_vcpu_postcreate(vcpu);

9454
	if (mutex_lock_killable(&vcpu->mutex))
9455
		return;
9456
	vcpu_load(vcpu);
9457 9458 9459 9460
	msr.data = 0x0;
	msr.index = MSR_IA32_TSC;
	msr.host_initiated = true;
	kvm_write_tsc(vcpu, &msr);
9461
	vcpu_put(vcpu);
9462 9463 9464 9465

	/* poll control enabled by default */
	vcpu->arch.msr_kvm_poll_control = 1;

9466
	mutex_unlock(&vcpu->mutex);
9467

9468 9469 9470
	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)
		schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
						KVMCLOCK_SYNC_PERIOD);
9471 9472
}

9473
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
9474
{
9475
	struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache;
9476
	int idx;
9477

9478 9479
	kvm_release_pfn(cache->pfn, cache->dirty, cache);

9480
	kvmclock_reset(vcpu);
9481

9482
	kvm_x86_ops.vcpu_free(vcpu);
9483

9484
	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
9485 9486 9487
	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
	kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
	kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
9488 9489 9490 9491 9492 9493 9494 9495 9496 9497 9498

	kvm_hv_vcpu_uninit(vcpu);
	kvm_pmu_destroy(vcpu);
	kfree(vcpu->arch.mce_banks);
	kvm_free_lapic(vcpu);
	idx = srcu_read_lock(&vcpu->kvm->srcu);
	kvm_mmu_destroy(vcpu);
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	free_page((unsigned long)vcpu->arch.pio_data);
	if (!lapic_in_kernel(vcpu))
		static_key_slow_dec(&kvm_no_apic_vcpu);
9499 9500
}

9501
void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
9502
{
9503 9504
	kvm_lapic_reset(vcpu, init_event);

9505 9506
	vcpu->arch.hflags = 0;

9507
	vcpu->arch.smi_pending = 0;
9508
	vcpu->arch.smi_count = 0;
A
Avi Kivity 已提交
9509 9510
	atomic_set(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = 0;
9511
	vcpu->arch.nmi_injected = false;
9512 9513
	kvm_clear_interrupt_queue(vcpu);
	kvm_clear_exception_queue(vcpu);
9514

9515
	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
9516
	kvm_update_dr0123(vcpu);
9517
	vcpu->arch.dr6 = DR6_INIT;
9518
	vcpu->arch.dr7 = DR7_FIXED_1;
9519
	kvm_update_dr7(vcpu);
9520

N
Nadav Amit 已提交
9521 9522
	vcpu->arch.cr2 = 0;

9523
	kvm_make_request(KVM_REQ_EVENT, vcpu);
9524
	vcpu->arch.apf.msr_val = 0;
G
Glauber Costa 已提交
9525
	vcpu->arch.st.msr_val = 0;
9526

9527 9528
	kvmclock_reset(vcpu);

9529 9530 9531
	kvm_clear_async_pf_completion_queue(vcpu);
	kvm_async_pf_hash_reset(vcpu);
	vcpu->arch.apf.halted = false;
9532

9533 9534 9535 9536 9537 9538 9539
	if (kvm_mpx_supported()) {
		void *mpx_state_buffer;

		/*
		 * To avoid have the INIT path from kvm_apic_has_events() that be
		 * called with loaded FPU and does not let userspace fix the state.
		 */
9540 9541
		if (init_event)
			kvm_put_guest_fpu(vcpu);
9542
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9543
					XFEATURE_BNDREGS);
9544 9545
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndreg_state));
9546
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9547
					XFEATURE_BNDCSR);
9548 9549
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndcsr));
9550 9551
		if (init_event)
			kvm_load_guest_fpu(vcpu);
9552 9553
	}

P
Paolo Bonzini 已提交
9554
	if (!init_event) {
9555
		kvm_pmu_reset(vcpu);
P
Paolo Bonzini 已提交
9556
		vcpu->arch.smbase = 0x30000;
K
Kyle Huey 已提交
9557 9558

		vcpu->arch.msr_misc_features_enables = 0;
9559 9560

		vcpu->arch.xcr0 = XFEATURE_MASK_FP;
P
Paolo Bonzini 已提交
9561
	}
9562

9563 9564 9565 9566
	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;

9567 9568
	vcpu->arch.ia32_xss = 0;

9569
	kvm_x86_ops.vcpu_reset(vcpu, init_event);
9570 9571
}

9572
void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
9573 9574 9575 9576 9577 9578 9579 9580
{
	struct kvm_segment cs;

	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs.selector = vector << 8;
	cs.base = vector << 12;
	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_rip_write(vcpu, 0);
9581 9582
}

9583
int kvm_arch_hardware_enable(void)
9584
{
9585 9586 9587
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;
9588 9589 9590 9591
	int ret;
	u64 local_tsc;
	u64 max_tsc = 0;
	bool stable, backwards_tsc = false;
A
Avi Kivity 已提交
9592 9593

	kvm_shared_msr_cpu_online();
9594
	ret = kvm_x86_ops.hardware_enable();
9595 9596 9597
	if (ret != 0)
		return ret;

9598
	local_tsc = rdtsc();
9599
	stable = !kvm_check_tsc_unstable();
9600 9601 9602
	list_for_each_entry(kvm, &vm_list, vm_list) {
		kvm_for_each_vcpu(i, vcpu, kvm) {
			if (!stable && vcpu->cpu == smp_processor_id())
9603
				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
9604 9605 9606 9607 9608 9609 9610 9611 9612 9613 9614 9615 9616 9617 9618 9619
			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
				backwards_tsc = true;
				if (vcpu->arch.last_host_tsc > max_tsc)
					max_tsc = vcpu->arch.last_host_tsc;
			}
		}
	}

	/*
	 * Sometimes, even reliable TSCs go backwards.  This happens on
	 * platforms that reset TSC during suspend or hibernate actions, but
	 * maintain synchronization.  We must compensate.  Fortunately, we can
	 * detect that condition here, which happens early in CPU bringup,
	 * before any KVM threads can be running.  Unfortunately, we can't
	 * bring the TSCs fully up to date with real time, as we aren't yet far
	 * enough into CPU bringup that we know how much real time has actually
9620
	 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot
9621 9622 9623 9624 9625 9626 9627 9628 9629 9630 9631 9632 9633 9634 9635 9636 9637 9638 9639 9640 9641 9642 9643 9644
	 * variables that haven't been updated yet.
	 *
	 * So we simply find the maximum observed TSC above, then record the
	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
	 * the adjustment will be applied.  Note that we accumulate
	 * adjustments, in case multiple suspend cycles happen before some VCPU
	 * gets a chance to run again.  In the event that no KVM threads get a
	 * chance to run, we will miss the entire elapsed period, as we'll have
	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
	 * loose cycle time.  This isn't too big a deal, since the loss will be
	 * uniform across all VCPUs (not to mention the scenario is extremely
	 * unlikely). It is possible that a second hibernate recovery happens
	 * much faster than a first, causing the observed TSC here to be
	 * smaller; this would require additional padding adjustment, which is
	 * why we set last_host_tsc to the local tsc observed here.
	 *
	 * N.B. - this code below runs only on platforms with reliable TSC,
	 * as that is the only way backwards_tsc is set above.  Also note
	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
	 * have the same delta_cyc adjustment applied if backwards_tsc
	 * is detected.  Note further, this adjustment is only done once,
	 * as we reset last_host_tsc on all VCPUs to stop this from being
	 * called multiple times (one for each physical CPU bringup).
	 *
G
Guo Chao 已提交
9645
	 * Platforms with unreliable TSCs don't have to deal with this, they
9646 9647 9648 9649 9650 9651 9652
	 * will be compensated by the logic in vcpu_load, which sets the TSC to
	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
	 * guarantee that they stay in perfect synchronization.
	 */
	if (backwards_tsc) {
		u64 delta_cyc = max_tsc - local_tsc;
		list_for_each_entry(kvm, &vm_list, vm_list) {
9653
			kvm->arch.backwards_tsc_observed = true;
9654 9655 9656
			kvm_for_each_vcpu(i, vcpu, kvm) {
				vcpu->arch.tsc_offset_adjustment += delta_cyc;
				vcpu->arch.last_host_tsc = local_tsc;
9657
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
9658 9659 9660 9661 9662 9663 9664 9665 9666 9667 9668 9669 9670 9671
			}

			/*
			 * We have to disable TSC offset matching.. if you were
			 * booting a VM while issuing an S4 host suspend....
			 * you may have some problem.  Solving this issue is
			 * left as an exercise to the reader.
			 */
			kvm->arch.last_tsc_nsec = 0;
			kvm->arch.last_tsc_write = 0;
		}

	}
	return 0;
9672 9673
}

9674
void kvm_arch_hardware_disable(void)
9675
{
9676
	kvm_x86_ops.hardware_disable();
9677
	drop_user_return_notifiers();
9678 9679
}

9680
int kvm_arch_hardware_setup(void *opaque)
9681
{
9682
	struct kvm_x86_init_ops *ops = opaque;
9683 9684
	int r;

9685 9686
	rdmsrl_safe(MSR_EFER, &host_efer);

9687 9688 9689
	if (boot_cpu_has(X86_FEATURE_XSAVES))
		rdmsrl(MSR_IA32_XSS, host_xss);

9690
	r = ops->hardware_setup();
9691 9692 9693
	if (r != 0)
		return r;

9694
	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
9695

9696 9697 9698
	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
		supported_xss = 0;

9699 9700 9701
#define __kvm_cpu_cap_has(UNUSED_, f) kvm_cpu_cap_has(f)
	cr4_reserved_bits = __cr4_reserved_bits(__kvm_cpu_cap_has, UNUSED_);
#undef __kvm_cpu_cap_has
9702

9703 9704 9705 9706
	if (kvm_has_tsc_control) {
		/*
		 * Make sure the user can only configure tsc_khz values that
		 * fit into a signed integer.
9707
		 * A min value is not calculated because it will always
9708 9709 9710 9711 9712 9713
		 * be 1 on all machines.
		 */
		u64 max = min(0x7fffffffULL,
			      __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz));
		kvm_max_guest_tsc_khz = max;

9714
		kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits;
9715
	}
9716

9717 9718
	kvm_init_msr_list();
	return 0;
9719 9720 9721 9722
}

void kvm_arch_hardware_unsetup(void)
{
9723
	kvm_x86_ops.hardware_unsetup();
9724 9725
}

9726
int kvm_arch_check_processor_compat(void *opaque)
9727
{
9728
	struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
9729
	struct kvm_x86_init_ops *ops = opaque;
9730 9731 9732

	WARN_ON(!irqs_disabled());

9733 9734
	if (__cr4_reserved_bits(cpu_has, c) !=
	    __cr4_reserved_bits(cpu_has, &boot_cpu_data))
9735 9736
		return -EIO;

9737
	return ops->check_processor_compatibility();
9738 9739 9740 9741 9742 9743 9744 9745 9746 9747 9748
}

bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
{
	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);

bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
9749 9750
}

9751
struct static_key kvm_no_apic_vcpu __read_mostly;
9752
EXPORT_SYMBOL_GPL(kvm_no_apic_vcpu);
9753

R
Radim Krčmář 已提交
9754 9755
void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
{
9756 9757
	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);

P
Paolo Bonzini 已提交
9758
	vcpu->arch.l1tf_flush_l1d = true;
9759 9760 9761 9762
	if (pmu->version && unlikely(pmu->event_count)) {
		pmu->need_cleanup = true;
		kvm_make_request(KVM_REQ_PMU, vcpu);
	}
9763
	kvm_x86_ops.sched_in(vcpu, cpu);
R
Radim Krčmář 已提交
9764 9765
}

9766 9767 9768 9769
void kvm_arch_free_vm(struct kvm *kvm)
{
	kfree(kvm->arch.hyperv.hv_pa_pg);
	vfree(kvm);
R
Radim Krčmář 已提交
9770 9771
}

9772

9773
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
9774
{
9775 9776 9777
	if (type)
		return -EINVAL;

9778
	INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
9779
	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
9780
	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
9781
	INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages);
B
Ben-Ami Yassour 已提交
9782
	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
9783
	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
9784

9785 9786
	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
9787 9788 9789
	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
		&kvm->arch.irq_sources_bitmap);
9790

9791
	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
9792
	mutex_init(&kvm->arch.apic_map_lock);
9793 9794
	spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);

9795
	kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
9796
	pvclock_update_vm_gtod_copy(kvm);
9797

9798 9799
	kvm->arch.guest_can_read_msr_platform_info = true;

9800
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
9801
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
9802

9803
	kvm_hv_init_vm(kvm);
9804
	kvm_page_track_init(kvm);
9805
	kvm_mmu_init_vm(kvm);
9806

9807
	return kvm_x86_ops.vm_init(kvm);
9808 9809
}

9810 9811 9812 9813 9814
int kvm_arch_post_init_vm(struct kvm *kvm)
{
	return kvm_mmu_post_init_vm(kvm);
}

9815 9816
static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
9817
	vcpu_load(vcpu);
9818 9819 9820 9821 9822 9823 9824
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;
9825
	struct kvm_vcpu *vcpu;
9826 9827 9828 9829

	/*
	 * Unpin any mmu pages first.
	 */
9830 9831
	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_clear_async_pf_completion_queue(vcpu);
9832
		kvm_unload_vcpu_mmu(vcpu);
9833
	}
9834
	kvm_for_each_vcpu(i, vcpu, kvm)
9835
		kvm_vcpu_destroy(vcpu);
9836 9837 9838 9839

	mutex_lock(&kvm->lock);
	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
		kvm->vcpus[i] = NULL;
9840

9841 9842
	atomic_set(&kvm->online_vcpus, 0);
	mutex_unlock(&kvm->lock);
9843 9844
}

9845 9846
void kvm_arch_sync_events(struct kvm *kvm)
{
9847
	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
9848
	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
9849
	kvm_free_pit(kvm);
9850 9851
}

9852
int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9853 9854
{
	int i, r;
9855
	unsigned long hva, uninitialized_var(old_npages);
9856
	struct kvm_memslots *slots = kvm_memslots(kvm);
9857
	struct kvm_memory_slot *slot;
9858 9859

	/* Called with kvm->slots_lock held.  */
9860 9861
	if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
		return -EINVAL;
9862

9863 9864
	slot = id_to_memslot(slots, id);
	if (size) {
9865
		if (slot && slot->npages)
9866 9867 9868 9869 9870 9871 9872 9873 9874 9875 9876
			return -EEXIST;

		/*
		 * MAP_SHARED to prevent internal slot pages from being moved
		 * by fork()/COW.
		 */
		hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
			      MAP_SHARED | MAP_ANONYMOUS, 0);
		if (IS_ERR((void *)hva))
			return PTR_ERR((void *)hva);
	} else {
9877
		if (!slot || !slot->npages)
9878 9879
			return 0;

9880 9881 9882 9883 9884 9885
		/*
		 * Stuff a non-canonical value to catch use-after-delete.  This
		 * ends up being 0 on 32-bit KVM, but there's no better
		 * alternative.
		 */
		hva = (unsigned long)(0xdeadull << 48);
9886
		old_npages = slot->npages;
9887 9888
	}

9889
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
9890
		struct kvm_userspace_memory_region m;
9891

9892 9893 9894
		m.slot = id | (i << 16);
		m.flags = 0;
		m.guest_phys_addr = gpa;
9895
		m.userspace_addr = hva;
9896
		m.memory_size = size;
9897 9898 9899 9900 9901
		r = __kvm_set_memory_region(kvm, &m);
		if (r < 0)
			return r;
	}

9902
	if (!size)
9903
		vm_munmap(hva, old_npages * PAGE_SIZE);
9904

9905 9906 9907 9908
	return 0;
}
EXPORT_SYMBOL_GPL(__x86_set_memory_region);

9909 9910 9911 9912 9913
void kvm_arch_pre_destroy_vm(struct kvm *kvm)
{
	kvm_mmu_pre_destroy_vm(kvm);
}

9914 9915
void kvm_arch_destroy_vm(struct kvm *kvm)
{
9916 9917 9918 9919 9920 9921
	if (current->mm == kvm->mm) {
		/*
		 * Free memory regions allocated on behalf of userspace,
		 * unless the the memory map has changed due to process exit
		 * or fd copying.
		 */
9922 9923 9924 9925 9926 9927 9928
		mutex_lock(&kvm->slots_lock);
		__x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
					0, 0);
		__x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
					0, 0);
		__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
		mutex_unlock(&kvm->slots_lock);
9929
	}
9930 9931
	if (kvm_x86_ops.vm_destroy)
		kvm_x86_ops.vm_destroy(kvm);
9932 9933
	kvm_pic_destroy(kvm);
	kvm_ioapic_destroy(kvm);
9934
	kvm_free_vcpus(kvm);
9935
	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
E
Eric Hankland 已提交
9936
	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
9937
	kvm_mmu_uninit_vm(kvm);
9938
	kvm_page_track_cleanup(kvm);
9939
	kvm_hv_destroy_vm(kvm);
9940
}
9941

9942
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
9943 9944 9945
{
	int i;

9946
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
9947 9948 9949
		kvfree(slot->arch.rmap[i]);
		slot->arch.rmap[i] = NULL;

9950 9951 9952
		if (i == 0)
			continue;

9953 9954
		kvfree(slot->arch.lpage_info[i - 1]);
		slot->arch.lpage_info[i - 1] = NULL;
9955
	}
9956

9957
	kvm_page_track_free_memslot(slot);
9958 9959
}

9960 9961
static int kvm_alloc_memslot_metadata(struct kvm_memory_slot *slot,
				      unsigned long npages)
9962 9963 9964
{
	int i;

9965 9966 9967 9968 9969 9970 9971
	/*
	 * Clear out the previous array pointers for the KVM_MR_MOVE case.  The
	 * old arrays will be freed by __kvm_set_memory_region() if installing
	 * the new memslot is successful.
	 */
	memset(&slot->arch, 0, sizeof(slot->arch));

9972
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
9973
		struct kvm_lpage_info *linfo;
9974 9975
		unsigned long ugfn;
		int lpages;
9976
		int level = i + 1;
9977 9978 9979 9980

		lpages = gfn_to_index(slot->base_gfn + npages - 1,
				      slot->base_gfn, level) + 1;

9981
		slot->arch.rmap[i] =
K
Kees Cook 已提交
9982
			kvcalloc(lpages, sizeof(*slot->arch.rmap[i]),
9983
				 GFP_KERNEL_ACCOUNT);
9984
		if (!slot->arch.rmap[i])
9985
			goto out_free;
9986 9987
		if (i == 0)
			continue;
9988

9989
		linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
9990
		if (!linfo)
9991 9992
			goto out_free;

9993 9994
		slot->arch.lpage_info[i - 1] = linfo;

9995
		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
9996
			linfo[0].disallow_lpage = 1;
9997
		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
9998
			linfo[lpages - 1].disallow_lpage = 1;
9999 10000 10001
		ugfn = slot->userspace_addr >> PAGE_SHIFT;
		/*
		 * If the gfn and userspace address are not aligned wrt each
10002
		 * other, disable large page support for this slot.
10003
		 */
10004
		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) {
10005 10006 10007
			unsigned long j;

			for (j = 0; j < lpages; ++j)
10008
				linfo[j].disallow_lpage = 1;
10009 10010 10011
		}
	}

10012 10013 10014
	if (kvm_page_track_create_memslot(slot, npages))
		goto out_free;

10015 10016 10017
	return 0;

out_free:
10018
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
T
Thomas Huth 已提交
10019
		kvfree(slot->arch.rmap[i]);
10020 10021 10022 10023
		slot->arch.rmap[i] = NULL;
		if (i == 0)
			continue;

T
Thomas Huth 已提交
10024
		kvfree(slot->arch.lpage_info[i - 1]);
10025
		slot->arch.lpage_info[i - 1] = NULL;
10026 10027 10028 10029
	}
	return -ENOMEM;
}

10030
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
10031
{
10032 10033 10034
	struct kvm_vcpu *vcpu;
	int i;

10035 10036 10037 10038
	/*
	 * memslots->generation has been incremented.
	 * mmio generation may have reached its maximum value.
	 */
10039
	kvm_mmu_invalidate_mmio_sptes(kvm, gen);
10040 10041 10042 10043

	/* Force re-initialization of steal_time cache */
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_vcpu_kick(vcpu);
10044 10045
}

10046 10047
int kvm_arch_prepare_memory_region(struct kvm *kvm,
				struct kvm_memory_slot *memslot,
10048
				const struct kvm_userspace_memory_region *mem,
10049
				enum kvm_mr_change change)
10050
{
10051 10052 10053
	if (change == KVM_MR_CREATE || change == KVM_MR_MOVE)
		return kvm_alloc_memslot_metadata(memslot,
						  mem->memory_size >> PAGE_SHIFT);
10054 10055 10056
	return 0;
}

10057 10058 10059 10060 10061
static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
				     struct kvm_memory_slot *new)
{
	/* Still write protect RO slot */
	if (new->flags & KVM_MEM_READONLY) {
10062
		kvm_mmu_slot_remove_write_access(kvm, new, PT_PAGE_TABLE_LEVEL);
10063 10064 10065 10066 10067 10068
		return;
	}

	/*
	 * Call kvm_x86_ops dirty logging hooks when they are valid.
	 *
10069
	 * kvm_x86_ops.slot_disable_log_dirty is called when:
10070 10071 10072 10073 10074 10075
	 *
	 *  - KVM_MR_CREATE with dirty logging is disabled
	 *  - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag
	 *
	 * The reason is, in case of PML, we need to set D-bit for any slots
	 * with dirty logging disabled in order to eliminate unnecessary GPA
M
Miaohe Lin 已提交
10076
	 * logging in PML buffer (and potential PML buffer full VMEXIT). This
10077
	 * guarantees leaving PML enabled during guest's lifetime won't have
W
Wei Yang 已提交
10078
	 * any additional overhead from PML when guest is running with dirty
10079 10080
	 * logging disabled for memory slots.
	 *
10081
	 * kvm_x86_ops.slot_enable_log_dirty is called when switching new slot
10082 10083 10084 10085 10086 10087 10088 10089 10090 10091 10092 10093 10094 10095 10096
	 * to dirty logging mode.
	 *
	 * If kvm_x86_ops dirty logging hooks are invalid, use write protect.
	 *
	 * In case of write protect:
	 *
	 * Write protect all pages for dirty logging.
	 *
	 * All the sptes including the large sptes which point to this
	 * slot are set to readonly. We can not create any new large
	 * spte on this slot until the end of the logging.
	 *
	 * See the comments in fast_page_fault().
	 */
	if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
10097 10098
		if (kvm_x86_ops.slot_enable_log_dirty) {
			kvm_x86_ops.slot_enable_log_dirty(kvm, new);
10099 10100 10101 10102 10103 10104 10105 10106 10107 10108 10109 10110 10111 10112 10113
		} else {
			int level =
				kvm_dirty_log_manual_protect_and_init_set(kvm) ?
				PT_DIRECTORY_LEVEL : PT_PAGE_TABLE_LEVEL;

			/*
			 * If we're with initial-all-set, we don't need
			 * to write protect any small page because
			 * they're reported as dirty already.  However
			 * we still need to write-protect huge pages
			 * so that the page split can happen lazily on
			 * the first write to the huge page.
			 */
			kvm_mmu_slot_remove_write_access(kvm, new, level);
		}
10114
	} else {
10115 10116
		if (kvm_x86_ops.slot_disable_log_dirty)
			kvm_x86_ops.slot_disable_log_dirty(kvm, new);
10117 10118 10119
	}
}

10120
void kvm_arch_commit_memory_region(struct kvm *kvm,
10121
				const struct kvm_userspace_memory_region *mem,
10122
				struct kvm_memory_slot *old,
10123
				const struct kvm_memory_slot *new,
10124
				enum kvm_mr_change change)
10125
{
10126
	if (!kvm->arch.n_requested_mmu_pages)
10127 10128
		kvm_mmu_change_mmu_pages(kvm,
				kvm_mmu_calculate_default_mmu_pages(kvm));
10129

10130 10131 10132 10133 10134 10135 10136 10137 10138 10139 10140
	/*
	 * Dirty logging tracks sptes in 4k granularity, meaning that large
	 * sptes have to be split.  If live migration is successful, the guest
	 * in the source machine will be destroyed and large sptes will be
	 * created in the destination. However, if the guest continues to run
	 * in the source machine (for example if live migration fails), small
	 * sptes will remain around and cause bad performance.
	 *
	 * Scan sptes if dirty logging has been stopped, dropping those
	 * which can be collapsed into a single large-page spte.  Later
	 * page faults will create the large-page sptes.
10141 10142 10143 10144 10145
	 *
	 * There is no need to do this in any of the following cases:
	 * CREATE:	No dirty mappings will already exist.
	 * MOVE/DELETE:	The old mappings will already have been cleaned up by
	 *		kvm_arch_flush_shadow_memslot()
10146
	 */
10147
	if (change == KVM_MR_FLAGS_ONLY &&
10148 10149 10150 10151
		(old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
		!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
		kvm_mmu_zap_collapsible_sptes(kvm, new);

10152
	/*
10153
	 * Set up write protection and/or dirty logging for the new slot.
10154
	 *
10155 10156 10157 10158
	 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have
	 * been zapped so no dirty logging staff is needed for old slot. For
	 * KVM_MR_FLAGS_ONLY, the old slot is essentially the same one as the
	 * new and it's also covered when dealing with the new slot.
10159 10160
	 *
	 * FIXME: const-ify all uses of struct kvm_memory_slot.
10161
	 */
10162
	if (change != KVM_MR_DELETE)
10163
		kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new);
10164 10165 10166

	/* Free the arrays associated with the old memslot. */
	if (change == KVM_MR_MOVE)
10167
		kvm_arch_free_memslot(kvm, old);
10168
}
10169

10170
void kvm_arch_flush_shadow_all(struct kvm *kvm)
10171
{
10172
	kvm_mmu_zap_all(kvm);
10173 10174
}

10175 10176 10177
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
10178
	kvm_page_track_flush_slot(kvm, slot);
10179 10180
}

10181 10182 10183
static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
{
	return (is_guest_mode(vcpu) &&
10184 10185
			kvm_x86_ops.guest_apic_has_interrupt &&
			kvm_x86_ops.guest_apic_has_interrupt(vcpu));
10186 10187
}

10188 10189 10190 10191 10192 10193 10194 10195 10196 10197 10198
static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
{
	if (!list_empty_careful(&vcpu->async_pf.done))
		return true;

	if (kvm_apic_has_events(vcpu))
		return true;

	if (vcpu->arch.pv.pv_unhalted)
		return true;

10199 10200 10201
	if (vcpu->arch.exception.pending)
		return true;

10202 10203
	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
	    (vcpu->arch.nmi_pending &&
10204
	     kvm_x86_ops.nmi_allowed(vcpu)))
10205 10206
		return true;

10207
	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
10208 10209
	    (vcpu->arch.smi_pending &&
	     kvm_x86_ops.smi_allowed(vcpu)))
P
Paolo Bonzini 已提交
10210 10211
		return true;

10212
	if (kvm_arch_interrupt_allowed(vcpu) &&
10213 10214
	    (kvm_cpu_has_interrupt(vcpu) ||
	    kvm_guest_apic_has_interrupt(vcpu)))
10215 10216
		return true;

A
Andrey Smetanin 已提交
10217 10218 10219
	if (kvm_hv_has_stimer_pending(vcpu))
		return true;

10220 10221 10222 10223 10224
	if (is_guest_mode(vcpu) &&
	    kvm_x86_ops.nested_ops->hv_timer_pending &&
	    kvm_x86_ops.nested_ops->hv_timer_pending(vcpu))
		return true;

10225 10226 10227
	return false;
}

10228 10229
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
10230
	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
10231
}
10232

10233 10234 10235 10236 10237 10238 10239 10240 10241 10242
bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
{
	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
		return true;

	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
		kvm_test_request(KVM_REQ_SMI, vcpu) ||
		 kvm_test_request(KVM_REQ_EVENT, vcpu))
		return true;

10243
	if (vcpu->arch.apicv_active && kvm_x86_ops.dy_apicv_has_pending_interrupt(vcpu))
10244 10245 10246 10247 10248
		return true;

	return false;
}

10249 10250
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
{
10251
	return vcpu->arch.preempted_in_kernel;
10252 10253
}

10254
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
10255
{
10256
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
10257
}
10258 10259 10260

int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
{
10261
	return kvm_x86_ops.interrupt_allowed(vcpu);
10262
}
10263

10264
unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
J
Jan Kiszka 已提交
10265
{
10266 10267 10268 10269 10270 10271
	if (is_64_bit_mode(vcpu))
		return kvm_rip_read(vcpu);
	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
		     kvm_rip_read(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
J
Jan Kiszka 已提交
10272

10273 10274 10275
bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
{
	return kvm_get_linear_rip(vcpu) == linear_rip;
J
Jan Kiszka 已提交
10276 10277 10278
}
EXPORT_SYMBOL_GPL(kvm_is_linear_rip);

10279 10280 10281 10282
unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
{
	unsigned long rflags;

10283
	rflags = kvm_x86_ops.get_rflags(vcpu);
10284
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
10285
		rflags &= ~X86_EFLAGS_TF;
10286 10287 10288 10289
	return rflags;
}
EXPORT_SYMBOL_GPL(kvm_get_rflags);

10290
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
10291 10292
{
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
J
Jan Kiszka 已提交
10293
	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
10294
		rflags |= X86_EFLAGS_TF;
10295
	kvm_x86_ops.set_rflags(vcpu, rflags);
10296 10297 10298 10299 10300
}

void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
	__kvm_set_rflags(vcpu, rflags);
10301
	kvm_make_request(KVM_REQ_EVENT, vcpu);
10302 10303 10304
}
EXPORT_SYMBOL_GPL(kvm_set_rflags);

G
Gleb Natapov 已提交
10305 10306 10307 10308
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
{
	int r;

10309
	if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||
10310
	      work->wakeup_all)
G
Gleb Natapov 已提交
10311 10312 10313 10314 10315 10316
		return;

	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		return;

10317
	if (!vcpu->arch.mmu->direct_map &&
10318
	      work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu))
X
Xiao Guangrong 已提交
10319 10320
		return;

10321
	kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true);
G
Gleb Natapov 已提交
10322 10323
}

10324 10325 10326 10327 10328 10329 10330 10331 10332 10333 10334 10335 10336 10337 10338 10339 10340 10341 10342 10343 10344 10345 10346 10347 10348 10349
static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
{
	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
}

static inline u32 kvm_async_pf_next_probe(u32 key)
{
	return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
}

static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 key = kvm_async_pf_hash_fn(gfn);

	while (vcpu->arch.apf.gfns[key] != ~0)
		key = kvm_async_pf_next_probe(key);

	vcpu->arch.apf.gfns[key] = gfn;
}

static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	int i;
	u32 key = kvm_async_pf_hash_fn(gfn);

	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
10350 10351
		     (vcpu->arch.apf.gfns[key] != gfn &&
		      vcpu->arch.apf.gfns[key] != ~0); i++)
10352 10353 10354 10355 10356 10357 10358 10359 10360 10361 10362 10363 10364 10365 10366 10367 10368 10369 10370 10371 10372 10373 10374 10375 10376 10377 10378 10379 10380 10381 10382 10383 10384
		key = kvm_async_pf_next_probe(key);

	return key;
}

bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
}

static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 i, j, k;

	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
	while (true) {
		vcpu->arch.apf.gfns[i] = ~0;
		do {
			j = kvm_async_pf_next_probe(j);
			if (vcpu->arch.apf.gfns[j] == ~0)
				return;
			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
			/*
			 * k lies cyclically in ]i,j]
			 * |    i.k.j |
			 * |....j i.k.| or  |.k..j i...|
			 */
		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
		i = j;
	}
}

10385 10386
static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
{
10387 10388 10389

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
				      sizeof(val));
10390 10391
}

10392 10393 10394 10395 10396 10397 10398
static int apf_get_user(struct kvm_vcpu *vcpu, u32 *val)
{

	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, val,
				      sizeof(u32));
}

10399 10400 10401 10402 10403 10404 10405
static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
{
	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
		return false;

	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
	    (vcpu->arch.apf.send_user_only &&
10406
	     kvm_x86_ops.get_cpl(vcpu) == 0))
10407 10408 10409 10410 10411 10412 10413 10414 10415 10416 10417 10418 10419 10420 10421 10422 10423 10424 10425
		return false;

	return true;
}

bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
{
	if (unlikely(!lapic_in_kernel(vcpu) ||
		     kvm_event_needs_reinjection(vcpu) ||
		     vcpu->arch.exception.pending))
		return false;

	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
		return false;

	/*
	 * If interrupts are off we cannot even use an artificial
	 * halt state.
	 */
10426
	return kvm_x86_ops.interrupt_allowed(vcpu);
10427 10428
}

10429 10430 10431
void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
				     struct kvm_async_pf *work)
{
10432 10433
	struct x86_exception fault;

10434
	trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa);
10435
	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
10436

10437 10438
	if (kvm_can_deliver_async_pf(vcpu) &&
	    !apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
10439 10440 10441 10442 10443
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
10444
		fault.async_page_fault = true;
10445
		kvm_inject_page_fault(vcpu, &fault);
10446 10447 10448 10449 10450 10451 10452 10453 10454 10455
	} else {
		/*
		 * It is not possible to deliver a paravirtualized asynchronous
		 * page fault, but putting the guest in an artificial halt state
		 * can be beneficial nevertheless: if an interrupt arrives, we
		 * can deliver it timely and perhaps the guest will schedule
		 * another process.  When the instruction that triggered a page
		 * fault is retried, hopefully the page will be ready in the host.
		 */
		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
10456
	}
10457 10458 10459 10460 10461
}

void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
				 struct kvm_async_pf *work)
{
10462
	struct x86_exception fault;
10463
	u32 val;
10464

10465
	if (work->wakeup_all)
10466 10467 10468
		work->arch.token = ~0; /* broadcast wakeup */
	else
		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
10469
	trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);
10470

10471 10472 10473 10474 10475 10476 10477 10478 10479 10480 10481
	if (vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED &&
	    !apf_get_user(vcpu, &val)) {
		if (val == KVM_PV_REASON_PAGE_NOT_PRESENT &&
		    vcpu->arch.exception.pending &&
		    vcpu->arch.exception.nr == PF_VECTOR &&
		    !apf_put_user(vcpu, 0)) {
			vcpu->arch.exception.injected = false;
			vcpu->arch.exception.pending = false;
			vcpu->arch.exception.nr = 0;
			vcpu->arch.exception.has_error_code = false;
			vcpu->arch.exception.error_code = 0;
10482 10483
			vcpu->arch.exception.has_payload = false;
			vcpu->arch.exception.payload = 0;
10484 10485 10486 10487 10488 10489 10490 10491 10492
		} else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
			fault.vector = PF_VECTOR;
			fault.error_code_valid = true;
			fault.error_code = 0;
			fault.nested_page_fault = false;
			fault.address = work->arch.token;
			fault.async_page_fault = true;
			kvm_inject_page_fault(vcpu, &fault);
		}
10493
	}
10494
	vcpu->arch.apf.halted = false;
10495
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10496 10497 10498 10499 10500 10501 10502
}

bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
		return true;
	else
10503
		return kvm_can_do_async_pf(vcpu);
10504 10505
}

10506 10507 10508 10509 10510 10511 10512 10513 10514 10515 10516 10517 10518 10519 10520 10521 10522 10523
void kvm_arch_start_assignment(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);

void kvm_arch_end_assignment(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);

bool kvm_arch_has_assigned_device(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);

10524 10525 10526 10527 10528 10529 10530 10531 10532 10533 10534 10535 10536 10537 10538 10539 10540 10541
void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);

void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);

bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);

10542 10543
bool kvm_arch_has_irq_bypass(void)
{
10544
	return true;
10545 10546
}

F
Feng Wu 已提交
10547 10548 10549 10550 10551 10552
int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

10553
	irqfd->producer = prod;
F
Feng Wu 已提交
10554

10555
	return kvm_x86_ops.update_pi_irte(irqfd->kvm,
10556
					   prod->irq, irqfd->gsi, 1);
F
Feng Wu 已提交
10557 10558 10559 10560 10561 10562 10563 10564 10565 10566 10567 10568 10569 10570 10571
}

void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	int ret;
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

	WARN_ON(irqfd->producer != prod);
	irqfd->producer = NULL;

	/*
	 * When producer of consumer is unregistered, we change back to
	 * remapped mode, so we can re-use the current implementation
A
Andrea Gelmini 已提交
10572
	 * when the irq is masked/disabled or the consumer side (KVM
F
Feng Wu 已提交
10573 10574
	 * int this case doesn't want to receive the interrupts.
	*/
10575
	ret = kvm_x86_ops.update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0);
F
Feng Wu 已提交
10576 10577 10578 10579 10580 10581 10582 10583
	if (ret)
		printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
		       " fails: %d\n", irqfd->consumer.token, ret);
}

int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
				   uint32_t guest_irq, bool set)
{
10584
	return kvm_x86_ops.update_pi_irte(kvm, host_irq, guest_irq, set);
F
Feng Wu 已提交
10585 10586
}

10587 10588 10589 10590 10591
bool kvm_vector_hashing_enabled(void)
{
	return vector_hashing;
}

10592 10593 10594 10595 10596 10597
bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
}
EXPORT_SYMBOL_GPL(kvm_arch_no_poll);

10598 10599 10600 10601 10602 10603 10604 10605 10606 10607 10608 10609 10610 10611 10612 10613 10614 10615 10616 10617 10618 10619
u64 kvm_spec_ctrl_valid_bits(struct kvm_vcpu *vcpu)
{
	uint64_t bits = SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD;

	/* The STIBP bit doesn't fault even if it's not advertised */
	if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS))
		bits &= ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP);
	if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL) &&
	    !boot_cpu_has(X86_FEATURE_AMD_IBRS))
		bits &= ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP);

	if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL_SSBD) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
		bits &= ~SPEC_CTRL_SSBD;
	if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
	    !boot_cpu_has(X86_FEATURE_AMD_SSBD))
		bits &= ~SPEC_CTRL_SSBD;

	return bits;
}
EXPORT_SYMBOL_GPL(kvm_spec_ctrl_valid_bits);
10620

10621
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
J
Jason Wang 已提交
10622
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
10623 10624 10625 10626
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
10627
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
10628
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
10629
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
10630
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
10631
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
10632
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
10633
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
10634
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
10635
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
P
Peter Xu 已提交
10636
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
K
Kai Huang 已提交
10637
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
10638
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
10639 10640
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);
10641
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log);
10642
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_update_request);