x86.c 271.2 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
B
Ben-Ami Yassour 已提交
8 9
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
N
Nicolas Kaiser 已提交
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11 12 13 14
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
B
Ben-Ami Yassour 已提交
15 16
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
17 18
 */

19
#include <linux/kvm_host.h>
20
#include "irq.h"
21
#include "mmu.h"
S
Sheng Yang 已提交
22
#include "i8254.h"
23
#include "tss.h"
24
#include "kvm_cache_regs.h"
25
#include "x86.h"
A
Avi Kivity 已提交
26
#include "cpuid.h"
27
#include "pmu.h"
28
#include "hyperv.h"
29

30
#include <linux/clocksource.h>
B
Ben-Ami Yassour 已提交
31
#include <linux/interrupt.h>
32 33 34
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
35 36
#include <linux/export.h>
#include <linux/moduleparam.h>
37
#include <linux/mman.h>
38
#include <linux/highmem.h>
J
Joerg Roedel 已提交
39
#include <linux/iommu.h>
B
Ben-Ami Yassour 已提交
40
#include <linux/intel-iommu.h>
41
#include <linux/cpufreq.h>
A
Avi Kivity 已提交
42
#include <linux/user-return-notifier.h>
43
#include <linux/srcu.h>
44
#include <linux/slab.h>
45
#include <linux/perf_event.h>
46
#include <linux/uaccess.h>
47
#include <linux/hash.h>
48
#include <linux/pci.h>
49 50
#include <linux/timekeeper_internal.h>
#include <linux/pvclock_gtod.h>
F
Feng Wu 已提交
51 52
#include <linux/kvm_irqfd.h>
#include <linux/irqbypass.h>
53
#include <linux/sched/stat.h>
54
#include <linux/sched/isolation.h>
55
#include <linux/mem_encrypt.h>
56

A
Avi Kivity 已提交
57
#include <trace/events/kvm.h>
X
Xiao Guangrong 已提交
58

59
#include <asm/debugreg.h>
60
#include <asm/msr.h>
61
#include <asm/desc.h>
H
Huang Ying 已提交
62
#include <asm/mce.h>
63
#include <linux/kernel_stat.h>
64
#include <asm/fpu/internal.h> /* Ugh! */
65
#include <asm/pvclock.h>
66
#include <asm/div64.h>
67
#include <asm/irq_remapping.h>
68
#include <asm/mshyperv.h>
69
#include <asm/hypervisor.h>
70
#include <asm/intel_pt.h>
71
#include <asm/emulate_prefix.h>
72
#include <clocksource/hyperv_timer.h>
73

74 75 76
#define CREATE_TRACE_POINTS
#include "trace.h"

77
#define MAX_IO_MSRS 256
H
Huang Ying 已提交
78
#define KVM_MAX_MCE_BANKS 32
79 80
u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P;
EXPORT_SYMBOL_GPL(kvm_mce_cap_supported);
H
Huang Ying 已提交
81

82 83 84
#define emul_to_vcpu(ctxt) \
	container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)

85 86 87 88 89
/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
90 91
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
92
#else
93
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
94
#endif
95

96 97
static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;

98 99
#define VM_STAT(x, ...) offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__
#define VCPU_STAT(x, ...) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__
100

101 102
#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
                                    KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
103

104
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
A
Avi Kivity 已提交
105
static void process_nmi(struct kvm_vcpu *vcpu);
106
static void enter_smm(struct kvm_vcpu *vcpu);
107
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
K
Ken Hofsass 已提交
108 109
static void store_regs(struct kvm_vcpu *vcpu);
static int sync_regs(struct kvm_vcpu *vcpu);
110

111
struct kvm_x86_ops *kvm_x86_ops __read_mostly;
112
EXPORT_SYMBOL_GPL(kvm_x86_ops);
113

114
static bool __read_mostly ignore_msrs = 0;
115
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
116

117 118 119
static bool __read_mostly report_ignored_msrs = true;
module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);

120
unsigned int min_timer_period_us = 200;
121 122
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

123 124 125
static bool __read_mostly kvmclock_periodic_sync = true;
module_param(kvmclock_periodic_sync, bool, S_IRUGO);

126
bool __read_mostly kvm_has_tsc_control;
127
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
128
u32  __read_mostly kvm_max_guest_tsc_khz;
129
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
130 131 132 133
u8   __read_mostly kvm_tsc_scaling_ratio_frac_bits;
EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits);
u64  __read_mostly kvm_max_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
134 135
u64 __read_mostly kvm_default_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
136

137
/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
138
static u32 __read_mostly tsc_tolerance_ppm = 250;
139 140
module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);

141 142 143 144 145 146 147
/*
 * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
 * adaptive tuning starting from default advancment of 1000ns.  '0' disables
 * advancement entirely.  Any other value is used as-is and disables adaptive
 * tuning, i.e. allows priveleged userspace to set an exact advancement time.
 */
static int __read_mostly lapic_timer_advance_ns = -1;
148
module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
149

150 151 152
static bool __read_mostly vector_hashing = true;
module_param(vector_hashing, bool, S_IRUGO);

153 154 155 156
bool __read_mostly enable_vmware_backdoor = false;
module_param(enable_vmware_backdoor, bool, S_IRUGO);
EXPORT_SYMBOL_GPL(enable_vmware_backdoor);

157 158 159
static bool __read_mostly force_emulation_prefix = false;
module_param(force_emulation_prefix, bool, S_IRUGO);

160 161 162
int __read_mostly pi_inject_timer = -1;
module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR);

A
Avi Kivity 已提交
163 164 165 166
#define KVM_NR_SHARED_MSRS 16

struct kvm_shared_msrs_global {
	int nr;
167
	u32 msrs[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
168 169 170 171 172
};

struct kvm_shared_msrs {
	struct user_return_notifier urn;
	bool registered;
173 174 175 176
	struct kvm_shared_msr_values {
		u64 host;
		u64 curr;
	} values[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
177 178 179
};

static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
180
static struct kvm_shared_msrs __percpu *shared_msrs;
A
Avi Kivity 已提交
181

182 183
static u64 __read_mostly host_xss;

184
struct kvm_stats_debugfs_item debugfs_entries[] = {
185 186 187 188 189 190 191 192 193
	{ "pf_fixed", VCPU_STAT(pf_fixed) },
	{ "pf_guest", VCPU_STAT(pf_guest) },
	{ "tlb_flush", VCPU_STAT(tlb_flush) },
	{ "invlpg", VCPU_STAT(invlpg) },
	{ "exits", VCPU_STAT(exits) },
	{ "io_exits", VCPU_STAT(io_exits) },
	{ "mmio_exits", VCPU_STAT(mmio_exits) },
	{ "signal_exits", VCPU_STAT(signal_exits) },
	{ "irq_window", VCPU_STAT(irq_window_exits) },
194
	{ "nmi_window", VCPU_STAT(nmi_window_exits) },
195
	{ "halt_exits", VCPU_STAT(halt_exits) },
196
	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
197
	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
198
	{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
199
	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
A
Amit Shah 已提交
200
	{ "hypercalls", VCPU_STAT(hypercalls) },
201 202 203 204 205 206
	{ "request_irq", VCPU_STAT(request_irq_exits) },
	{ "irq_exits", VCPU_STAT(irq_exits) },
	{ "host_state_reload", VCPU_STAT(host_state_reload) },
	{ "fpu_reload", VCPU_STAT(fpu_reload) },
	{ "insn_emulation", VCPU_STAT(insn_emulation) },
	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
207
	{ "irq_injections", VCPU_STAT(irq_injections) },
208
	{ "nmi_injections", VCPU_STAT(nmi_injections) },
209
	{ "req_event", VCPU_STAT(req_event) },
P
Paolo Bonzini 已提交
210
	{ "l1d_flush", VCPU_STAT(l1d_flush) },
A
Avi Kivity 已提交
211 212 213 214 215 216
	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
	{ "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
	{ "mmu_flooded", VM_STAT(mmu_flooded) },
	{ "mmu_recycled", VM_STAT(mmu_recycled) },
A
Avi Kivity 已提交
217
	{ "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
218
	{ "mmu_unsync", VM_STAT(mmu_unsync) },
219
	{ "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
220
	{ "largepages", VM_STAT(lpages, .mode = 0444) },
P
Paolo Bonzini 已提交
221
	{ "nx_largepages_splitted", VM_STAT(nx_lpage_splits, .mode = 0444) },
222 223
	{ "max_mmu_page_hash_collisions",
		VM_STAT(max_mmu_page_hash_collisions) },
224 225 226
	{ NULL }
};

227 228
u64 __read_mostly host_xcr0;

229 230 231
struct kmem_cache *x86_fpu_cache;
EXPORT_SYMBOL_GPL(x86_fpu_cache);

232
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
233

234 235 236 237 238 239 240
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
	int i;
	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
		vcpu->arch.apf.gfns[i] = ~0;
}

A
Avi Kivity 已提交
241 242 243 244 245
static void kvm_on_user_return(struct user_return_notifier *urn)
{
	unsigned slot;
	struct kvm_shared_msrs *locals
		= container_of(urn, struct kvm_shared_msrs, urn);
246
	struct kvm_shared_msr_values *values;
247 248 249 250 251 252 253 254 255 256 257 258
	unsigned long flags;

	/*
	 * Disabling irqs at this point since the following code could be
	 * interrupted and executed through kvm_arch_hardware_disable()
	 */
	local_irq_save(flags);
	if (locals->registered) {
		locals->registered = false;
		user_return_notifier_unregister(urn);
	}
	local_irq_restore(flags);
A
Avi Kivity 已提交
259
	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
260 261 262 263
		values = &locals->values[slot];
		if (values->host != values->curr) {
			wrmsrl(shared_msrs_global.msrs[slot], values->host);
			values->curr = values->host;
A
Avi Kivity 已提交
264 265 266 267
		}
	}
}

268 269
void kvm_define_shared_msr(unsigned slot, u32 msr)
{
270
	BUG_ON(slot >= KVM_NR_SHARED_MSRS);
271
	shared_msrs_global.msrs[slot] = msr;
A
Avi Kivity 已提交
272 273 274 275 276 277 278
	if (slot >= shared_msrs_global.nr)
		shared_msrs_global.nr = slot + 1;
}
EXPORT_SYMBOL_GPL(kvm_define_shared_msr);

static void kvm_shared_msr_cpu_online(void)
{
279 280 281 282
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
	u64 value;
	int i;
A
Avi Kivity 已提交
283

284 285 286 287 288
	for (i = 0; i < shared_msrs_global.nr; ++i) {
		rdmsrl_safe(shared_msrs_global.msrs[i], &value);
		smsr->values[i].host = value;
		smsr->values[i].curr = value;
	}
A
Avi Kivity 已提交
289 290
}

291
int kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
A
Avi Kivity 已提交
292
{
293 294
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
295
	int err;
A
Avi Kivity 已提交
296

297 298
	value = (value & mask) | (smsr->values[slot].host & ~mask);
	if (value == smsr->values[slot].curr)
299 300 301 302 303
		return 0;
	err = wrmsrl_safe(shared_msrs_global.msrs[slot], value);
	if (err)
		return 1;

304
	smsr->values[slot].curr = value;
A
Avi Kivity 已提交
305 306 307 308 309
	if (!smsr->registered) {
		smsr->urn.on_user_return = kvm_on_user_return;
		user_return_notifier_register(&smsr->urn);
		smsr->registered = true;
	}
310
	return 0;
A
Avi Kivity 已提交
311 312 313
}
EXPORT_SYMBOL_GPL(kvm_set_shared_msr);

314
static void drop_user_return_notifiers(void)
315
{
316 317
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
318 319 320 321 322

	if (smsr->registered)
		kvm_on_user_return(&smsr->urn);
}

323 324
u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
325
	return vcpu->arch.apic_base;
326 327 328
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

329 330 331 332 333 334
enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
{
	return kvm_apic_mode(kvm_get_apic_base(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_apic_mode);

335 336
int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
337 338
	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
339 340
	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |
		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
341

342
	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
343
		return 1;
344 345 346 347 348 349
	if (!msr_info->host_initiated) {
		if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
			return 1;
		if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
			return 1;
	}
350 351 352

	kvm_lapic_set_base(vcpu, msr_info->data);
	return 0;
353 354 355
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

356
asmlinkage __visible void kvm_spurious_fault(void)
357 358
{
	/* Fault while not rebooting.  We want the trace. */
359
	BUG_ON(!kvm_rebooting);
360 361 362
}
EXPORT_SYMBOL_GPL(kvm_spurious_fault);

363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
#define EXCPT_BENIGN		0
#define EXCPT_CONTRIBUTORY	1
#define EXCPT_PF		2

static int exception_class(int vector)
{
	switch (vector) {
	case PF_VECTOR:
		return EXCPT_PF;
	case DE_VECTOR:
	case TS_VECTOR:
	case NP_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
		return EXCPT_CONTRIBUTORY;
	default:
		break;
	}
	return EXCPT_BENIGN;
}

384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
#define EXCPT_FAULT		0
#define EXCPT_TRAP		1
#define EXCPT_ABORT		2
#define EXCPT_INTERRUPT		3

static int exception_type(int vector)
{
	unsigned int mask;

	if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
		return EXCPT_INTERRUPT;

	mask = 1 << vector;

	/* #DB is trap, as instruction watchpoints are handled elsewhere */
	if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR)))
		return EXCPT_TRAP;

	if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
		return EXCPT_ABORT;

	/* Reserved exceptions will result in fault */
	return EXCPT_FAULT;
}

409 410 411 412 413 414 415 416 417 418
void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
{
	unsigned nr = vcpu->arch.exception.nr;
	bool has_payload = vcpu->arch.exception.has_payload;
	unsigned long payload = vcpu->arch.exception.payload;

	if (!has_payload)
		return;

	switch (nr) {
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
	case DB_VECTOR:
		/*
		 * "Certain debug exceptions may clear bit 0-3.  The
		 * remaining contents of the DR6 register are never
		 * cleared by the processor".
		 */
		vcpu->arch.dr6 &= ~DR_TRAP_BITS;
		/*
		 * DR6.RTM is set by all #DB exceptions that don't clear it.
		 */
		vcpu->arch.dr6 |= DR6_RTM;
		vcpu->arch.dr6 |= payload;
		/*
		 * Bit 16 should be set in the payload whenever the #DB
		 * exception should clear DR6.RTM. This makes the payload
		 * compatible with the pending debug exceptions under VMX.
		 * Though not currently documented in the SDM, this also
		 * makes the payload compatible with the exit qualification
		 * for #DB exceptions under VMX.
		 */
		vcpu->arch.dr6 ^= payload & DR6_RTM;
		break;
441 442 443 444 445 446 447 448 449 450
	case PF_VECTOR:
		vcpu->arch.cr2 = payload;
		break;
	}

	vcpu->arch.exception.has_payload = false;
	vcpu->arch.exception.payload = 0;
}
EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);

451
static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
452
		unsigned nr, bool has_error, u32 error_code,
453
	        bool has_payload, unsigned long payload, bool reinject)
454 455 456 457
{
	u32 prev_nr;
	int class1, class2;

458 459
	kvm_make_request(KVM_REQ_EVENT, vcpu);

460
	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
461
	queue:
462 463
		if (has_error && !is_protmode(vcpu))
			has_error = false;
464 465 466 467 468 469 470 471 472 473 474
		if (reinject) {
			/*
			 * On vmentry, vcpu->arch.exception.pending is only
			 * true if an event injection was blocked by
			 * nested_run_pending.  In that case, however,
			 * vcpu_enter_guest requests an immediate exit,
			 * and the guest shouldn't proceed far enough to
			 * need reinjection.
			 */
			WARN_ON_ONCE(vcpu->arch.exception.pending);
			vcpu->arch.exception.injected = true;
475 476 477 478 479 480 481 482
			if (WARN_ON_ONCE(has_payload)) {
				/*
				 * A reinjected event has already
				 * delivered its payload.
				 */
				has_payload = false;
				payload = 0;
			}
483 484 485 486
		} else {
			vcpu->arch.exception.pending = true;
			vcpu->arch.exception.injected = false;
		}
487 488 489
		vcpu->arch.exception.has_error_code = has_error;
		vcpu->arch.exception.nr = nr;
		vcpu->arch.exception.error_code = error_code;
490 491
		vcpu->arch.exception.has_payload = has_payload;
		vcpu->arch.exception.payload = payload;
492 493 494
		/*
		 * In guest mode, payload delivery should be deferred,
		 * so that the L1 hypervisor can intercept #PF before
495 496 497 498 499 500 501
		 * CR2 is modified (or intercept #DB before DR6 is
		 * modified under nVMX).  However, for ABI
		 * compatibility with KVM_GET_VCPU_EVENTS and
		 * KVM_SET_VCPU_EVENTS, we can't delay payload
		 * delivery unless userspace has enabled this
		 * functionality via the per-VM capability,
		 * KVM_CAP_EXCEPTION_PAYLOAD.
502 503 504 505
		 */
		if (!vcpu->kvm->arch.exception_payload_enabled ||
		    !is_guest_mode(vcpu))
			kvm_deliver_exception_payload(vcpu);
506 507 508 509 510 511 512
		return;
	}

	/* to check exception */
	prev_nr = vcpu->arch.exception.nr;
	if (prev_nr == DF_VECTOR) {
		/* triple fault -> shutdown */
513
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
514 515 516 517 518 519
		return;
	}
	class1 = exception_class(prev_nr);
	class2 = exception_class(nr);
	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
520 521 522 523 524
		/*
		 * Generate double fault per SDM Table 5-5.  Set
		 * exception.pending = true so that the double fault
		 * can trigger a nested vmexit.
		 */
525
		vcpu->arch.exception.pending = true;
526
		vcpu->arch.exception.injected = false;
527 528 529
		vcpu->arch.exception.has_error_code = true;
		vcpu->arch.exception.nr = DF_VECTOR;
		vcpu->arch.exception.error_code = 0;
530 531
		vcpu->arch.exception.has_payload = false;
		vcpu->arch.exception.payload = 0;
532 533 534 535 536 537 538
	} else
		/* replace previous exception with a new one in a hope
		   that instruction re-execution will regenerate lost
		   exception */
		goto queue;
}

539 540
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
541
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
542 543 544
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

545 546
void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
547
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
548 549 550
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

551 552 553 554 555 556
static void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
				  unsigned long payload)
{
	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
}

557 558 559 560 561 562 563
static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
				    u32 error_code, unsigned long payload)
{
	kvm_multiple_exception(vcpu, nr, true, error_code,
			       true, payload, false);
}

564
int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
565
{
566 567 568
	if (err)
		kvm_inject_gp(vcpu, 0);
	else
569 570 571
		return kvm_skip_emulated_instruction(vcpu);

	return 1;
572 573
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
574

575
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
576 577
{
	++vcpu->stat.pf_guest;
578 579
	vcpu->arch.exception.nested_apf =
		is_guest_mode(vcpu) && fault->async_page_fault;
580
	if (vcpu->arch.exception.nested_apf) {
581
		vcpu->arch.apf.nested_apf_token = fault->address;
582 583 584 585 586
		kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
	} else {
		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
					fault->address);
	}
587
}
N
Nadav Har'El 已提交
588
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
589

590
static bool kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
591
{
592 593
	if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
		vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
594
	else
595
		vcpu->arch.mmu->inject_page_fault(vcpu, fault);
596 597

	return fault->nested_page_fault;
598 599
}

600 601
void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
602 603
	atomic_inc(&vcpu->arch.nmi_queued);
	kvm_make_request(KVM_REQ_NMI, vcpu);
604 605 606
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

607 608
void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
609
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
610 611 612
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

613 614
void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
615
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
616 617 618
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

619 620 621 622 623
/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
624
{
625 626 627 628
	if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
		return true;
	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
	return false;
629
}
630
EXPORT_SYMBOL_GPL(kvm_require_cpl);
631

632 633 634 635 636 637 638 639 640 641
bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
{
	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
		return true;

	kvm_queue_exception(vcpu, UD_VECTOR);
	return false;
}
EXPORT_SYMBOL_GPL(kvm_require_dr);

642 643
/*
 * This function will be used to read from the physical memory of the currently
644
 * running guest. The difference to kvm_vcpu_read_guest_page is that this function
645 646 647 648 649 650
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gfn_t ngfn, void *data, int offset, int len,
			    u32 access)
{
651
	struct x86_exception exception;
652 653 654 655
	gfn_t real_gfn;
	gpa_t ngpa;

	ngpa     = gfn_to_gpa(ngfn);
656
	real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);
657 658 659 660 661
	if (real_gfn == UNMAPPED_GVA)
		return -EFAULT;

	real_gfn = gpa_to_gfn(real_gfn);

662
	return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len);
663 664 665
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

666
static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
667 668 669 670 671 672
			       void *data, int offset, int len, u32 access)
{
	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
				       data, offset, len, access);
}

673 674 675 676 677 678
static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
{
	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) | rsvd_bits(5, 8) |
	       rsvd_bits(1, 2);
}

679
/*
680
 * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
681
 */
682
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
683 684 685 686 687
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
	int i;
	int ret;
688
	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
689

690 691 692
	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
				      offset * sizeof(u64), sizeof(pdpte),
				      PFERR_USER_MASK|PFERR_WRITE_MASK);
693 694 695 696 697
	if (ret < 0) {
		ret = 0;
		goto out;
	}
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
B
Bandan Das 已提交
698
		if ((pdpte[i] & PT_PRESENT_MASK) &&
699
		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
700 701 702 703 704 705
			ret = 0;
			goto out;
		}
	}
	ret = 1;

706
	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
707 708
	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);

709 710 711 712
out:

	return ret;
}
713
EXPORT_SYMBOL_GPL(load_pdptrs);
714

715
bool pdptrs_changed(struct kvm_vcpu *vcpu)
716
{
717
	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
718 719
	int offset;
	gfn_t gfn;
720 721
	int r;

722
	if (!is_pae_paging(vcpu))
723 724
		return false;

725
	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
A
Avi Kivity 已提交
726 727
		return true;

728 729
	gfn = (kvm_read_cr3(vcpu) & 0xffffffe0ul) >> PAGE_SHIFT;
	offset = (kvm_read_cr3(vcpu) & 0xffffffe0ul) & (PAGE_SIZE - 1);
730 731
	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
				       PFERR_USER_MASK | PFERR_WRITE_MASK);
732
	if (r < 0)
733
		return true;
734

735
	return memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
736
}
737
EXPORT_SYMBOL_GPL(pdptrs_changed);
738

739
int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
740
{
741
	unsigned long old_cr0 = kvm_read_cr0(vcpu);
742
	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP;
743

744 745
	cr0 |= X86_CR0_ET;

746
#ifdef CONFIG_X86_64
747 748
	if (cr0 & 0xffffffff00000000UL)
		return 1;
749 750 751
#endif

	cr0 &= ~CR0_RESERVED_BITS;
752

753 754
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
		return 1;
755

756 757
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
		return 1;
758 759 760

	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
#ifdef CONFIG_X86_64
761
		if ((vcpu->arch.efer & EFER_LME)) {
762 763
			int cs_db, cs_l;

764 765
			if (!is_pae(vcpu))
				return 1;
766
			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
767 768
			if (cs_l)
				return 1;
769 770
		} else
#endif
771
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
772
						 kvm_read_cr3(vcpu)))
773
			return 1;
774 775
	}

776 777 778
	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
		return 1;

779 780
	kvm_x86_ops->set_cr0(vcpu, cr0);

781
	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
782
		kvm_clear_async_pf_completion_queue(vcpu);
783 784
		kvm_async_pf_hash_reset(vcpu);
	}
785

786 787
	if ((cr0 ^ old_cr0) & update_bits)
		kvm_mmu_reset_context(vcpu);
788

789 790 791
	if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
792 793
		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);

794 795
	return 0;
}
796
EXPORT_SYMBOL_GPL(kvm_set_cr0);
797

798
void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
799
{
800
	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
801
}
802
EXPORT_SYMBOL_GPL(kvm_lmsw);
803

804
void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
805
{
806 807 808 809 810 811 812 813 814
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {

		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);

		if (vcpu->arch.xsaves_enabled &&
		    vcpu->arch.ia32_xss != host_xss)
			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
	}
815
}
816
EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state);
817

818
void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
819
{
820 821 822 823 824 825 826 827 828 829
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {

		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);

		if (vcpu->arch.xsaves_enabled &&
		    vcpu->arch.ia32_xss != host_xss)
			wrmsrl(MSR_IA32_XSS, host_xss);
	}

830
}
831
EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state);
832

833
static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
834
{
835 836
	u64 xcr0 = xcr;
	u64 old_xcr0 = vcpu->arch.xcr0;
837
	u64 valid_bits;
838 839 840 841

	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
	if (index != XCR_XFEATURE_ENABLED_MASK)
		return 1;
D
Dave Hansen 已提交
842
	if (!(xcr0 & XFEATURE_MASK_FP))
843
		return 1;
D
Dave Hansen 已提交
844
	if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
845
		return 1;
846 847 848 849 850 851

	/*
	 * Do not allow the guest to set bits that we do not support
	 * saving.  However, xcr0 bit 0 is always set, even if the
	 * emulated CPU does not support XSAVE (see fx_init).
	 */
D
Dave Hansen 已提交
852
	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
853
	if (xcr0 & ~valid_bits)
854
		return 1;
855

D
Dave Hansen 已提交
856 857
	if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
	    (!(xcr0 & XFEATURE_MASK_BNDCSR)))
858 859
		return 1;

D
Dave Hansen 已提交
860 861
	if (xcr0 & XFEATURE_MASK_AVX512) {
		if (!(xcr0 & XFEATURE_MASK_YMM))
862
			return 1;
D
Dave Hansen 已提交
863
		if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
864 865
			return 1;
	}
866
	vcpu->arch.xcr0 = xcr0;
867

D
Dave Hansen 已提交
868
	if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
869
		kvm_update_cpuid(vcpu);
870 871 872 873 874
	return 0;
}

int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
875 876
	if (kvm_x86_ops->get_cpl(vcpu) != 0 ||
	    __kvm_set_xcr(vcpu, index, xcr)) {
877 878 879 880 881 882 883
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_xcr);

884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902
#define __cr4_reserved_bits(__cpu_has, __c)		\
({							\
	u64 __reserved_bits = CR4_RESERVED_BITS;	\
							\
	if (!__cpu_has(__c, X86_FEATURE_XSAVE))		\
		__reserved_bits |= X86_CR4_OSXSAVE;	\
	if (!__cpu_has(__c, X86_FEATURE_SMEP))		\
		__reserved_bits |= X86_CR4_SMEP;	\
	if (!__cpu_has(__c, X86_FEATURE_SMAP))		\
		__reserved_bits |= X86_CR4_SMAP;	\
	if (!__cpu_has(__c, X86_FEATURE_FSGSBASE))	\
		__reserved_bits |= X86_CR4_FSGSBASE;	\
	if (!__cpu_has(__c, X86_FEATURE_PKU))		\
		__reserved_bits |= X86_CR4_PKE;		\
	if (!__cpu_has(__c, X86_FEATURE_LA57))		\
		__reserved_bits |= X86_CR4_LA57;	\
	__reserved_bits;				\
})

903 904
static u64 kvm_host_cr4_reserved_bits(struct cpuinfo_x86 *c)
{
905
	u64 reserved_bits = __cr4_reserved_bits(cpu_has, c);
906

907
	if (cpuid_ecx(0x7) & feature_bit(LA57))
908
		reserved_bits &= ~X86_CR4_LA57;
909

910 911
	if (kvm_x86_ops->umip_emulated())
		reserved_bits &= ~X86_CR4_UMIP;
912 913 914 915

	return reserved_bits;
}

916
static int kvm_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
917
{
918
	if (cr4 & cr4_reserved_bits)
919
		return -EINVAL;
920

921
	if (cr4 & __cr4_reserved_bits(guest_cpuid_has, vcpu))
922 923 924 925 926 927 928 929 930 931 932 933
		return -EINVAL;

	return 0;
}

int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
	unsigned long old_cr4 = kvm_read_cr4(vcpu);
	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE |
				   X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE;

	if (kvm_valid_cr4(vcpu, cr4))
P
Paolo Bonzini 已提交
934 935
		return 1;

936
	if (is_long_mode(vcpu)) {
937 938
		if (!(cr4 & X86_CR4_PAE))
			return 1;
939 940
	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
		   && ((cr4 ^ old_cr4) & pdptr_bits)
941 942
		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
				   kvm_read_cr3(vcpu)))
943 944
		return 1;

945
	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
946
		if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
947 948 949 950 951 952 953
			return 1;

		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
			return 1;
	}

954
	if (kvm_x86_ops->set_cr4(vcpu, cr4))
955
		return 1;
956

957 958
	if (((cr4 ^ old_cr4) & pdptr_bits) ||
	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
959
		kvm_mmu_reset_context(vcpu);
960

961
	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
A
Avi Kivity 已提交
962
		kvm_update_cpuid(vcpu);
963

964 965
	return 0;
}
966
EXPORT_SYMBOL_GPL(kvm_set_cr4);
967

968
int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
969
{
970
	bool skip_tlb_flush = false;
971
#ifdef CONFIG_X86_64
972 973
	bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);

974
	if (pcid_enabled) {
975 976
		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
		cr3 &= ~X86_CR3_PCID_NOFLUSH;
977
	}
978
#endif
N
Nadav Amit 已提交
979

980
	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
981 982
		if (!skip_tlb_flush) {
			kvm_mmu_sync_roots(vcpu);
983
			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
984
		}
985
		return 0;
986 987
	}

988
	if (is_long_mode(vcpu) &&
989
	    (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63)))
990
		return 1;
991 992
	else if (is_pae_paging(vcpu) &&
		 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
N
Nadav Amit 已提交
993
		return 1;
994

995
	kvm_mmu_new_cr3(vcpu, cr3, skip_tlb_flush);
996
	vcpu->arch.cr3 = cr3;
997
	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
998

999 1000
	return 0;
}
1001
EXPORT_SYMBOL_GPL(kvm_set_cr3);
1002

A
Andre Przywara 已提交
1003
int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
1004
{
1005 1006
	if (cr8 & CR8_RESERVED_BITS)
		return 1;
1007
	if (lapic_in_kernel(vcpu))
1008 1009
		kvm_lapic_set_tpr(vcpu, cr8);
	else
1010
		vcpu->arch.cr8 = cr8;
1011 1012
	return 0;
}
1013
EXPORT_SYMBOL_GPL(kvm_set_cr8);
1014

1015
unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1016
{
1017
	if (lapic_in_kernel(vcpu))
1018 1019
		return kvm_lapic_get_cr8(vcpu);
	else
1020
		return vcpu->arch.cr8;
1021
}
1022
EXPORT_SYMBOL_GPL(kvm_get_cr8);
1023

1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
{
	int i;

	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
	}
}

J
Jan Kiszka 已提交
1035 1036 1037 1038 1039 1040
static void kvm_update_dr6(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
		kvm_x86_ops->set_dr6(vcpu, vcpu->arch.dr6);
}

1041 1042 1043 1044 1045 1046 1047 1048 1049
static void kvm_update_dr7(struct kvm_vcpu *vcpu)
{
	unsigned long dr7;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		dr7 = vcpu->arch.guest_debug_dr7;
	else
		dr7 = vcpu->arch.dr7;
	kvm_x86_ops->set_dr7(vcpu, dr7);
1050 1051 1052
	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
	if (dr7 & DR7_BP_EN_MASK)
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1053 1054
}

1055 1056 1057 1058
static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
{
	u64 fixed = DR6_FIXED_1;

1059
	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1060 1061 1062 1063
		fixed |= DR6_RTM;
	return fixed;
}

1064
static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
{
	switch (dr) {
	case 0 ... 3:
		vcpu->arch.db[dr] = val;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = val;
		break;
	case 4:
		/* fall through */
	case 6:
1075 1076
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
1077
		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
J
Jan Kiszka 已提交
1078
		kvm_update_dr6(vcpu);
1079 1080 1081 1082
		break;
	case 5:
		/* fall through */
	default: /* 7 */
1083 1084
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
1085
		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1086
		kvm_update_dr7(vcpu);
1087 1088 1089 1090 1091
		break;
	}

	return 0;
}
1092 1093 1094

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
1095
	if (__kvm_set_dr(vcpu, dr, val)) {
1096
		kvm_inject_gp(vcpu, 0);
1097 1098 1099
		return 1;
	}
	return 0;
1100
}
1101 1102
EXPORT_SYMBOL_GPL(kvm_set_dr);

1103
int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1104 1105 1106 1107 1108 1109 1110 1111
{
	switch (dr) {
	case 0 ... 3:
		*val = vcpu->arch.db[dr];
		break;
	case 4:
		/* fall through */
	case 6:
J
Jan Kiszka 已提交
1112 1113 1114 1115
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
			*val = vcpu->arch.dr6;
		else
			*val = kvm_x86_ops->get_dr6(vcpu);
1116 1117 1118 1119 1120 1121 1122
		break;
	case 5:
		/* fall through */
	default: /* 7 */
		*val = vcpu->arch.dr7;
		break;
	}
1123 1124
	return 0;
}
1125 1126
EXPORT_SYMBOL_GPL(kvm_get_dr);

A
Avi Kivity 已提交
1127 1128
bool kvm_rdpmc(struct kvm_vcpu *vcpu)
{
1129
	u32 ecx = kvm_rcx_read(vcpu);
A
Avi Kivity 已提交
1130 1131 1132
	u64 data;
	int err;

1133
	err = kvm_pmu_rdpmc(vcpu, ecx, &data);
A
Avi Kivity 已提交
1134 1135
	if (err)
		return err;
1136 1137
	kvm_rax_write(vcpu, (u32)data);
	kvm_rdx_write(vcpu, data >> 32);
A
Avi Kivity 已提交
1138 1139 1140 1141
	return err;
}
EXPORT_SYMBOL_GPL(kvm_rdpmc);

1142 1143 1144 1145
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
1146 1147 1148
 * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features)
 * extract the supported MSRs from the related const lists.
 * msrs_to_save is selected from the msrs_to_save_all to reflect the
1149
 * capabilities of the host cpu. This capabilities test skips MSRs that are
1150
 * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs
1151
 * may depend on host virtualization features rather than host cpu features.
1152
 */
1153

1154
static const u32 msrs_to_save_all[] = {
1155
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
B
Brian Gerst 已提交
1156
	MSR_STAR,
1157 1158 1159
#ifdef CONFIG_X86_64
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
1160
	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
1161
	MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
1162
	MSR_IA32_SPEC_CTRL,
1163 1164 1165 1166 1167 1168
	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
1169 1170
	MSR_IA32_UMWAIT_CONTROL,

1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
	MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
	MSR_ARCH_PERFMON_FIXED_CTR0 + 2, MSR_ARCH_PERFMON_FIXED_CTR0 + 3,
	MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
	MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
	MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
	MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
	MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
	MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
	MSR_ARCH_PERFMON_PERFCTR0 + 8, MSR_ARCH_PERFMON_PERFCTR0 + 9,
	MSR_ARCH_PERFMON_PERFCTR0 + 10, MSR_ARCH_PERFMON_PERFCTR0 + 11,
	MSR_ARCH_PERFMON_PERFCTR0 + 12, MSR_ARCH_PERFMON_PERFCTR0 + 13,
	MSR_ARCH_PERFMON_PERFCTR0 + 14, MSR_ARCH_PERFMON_PERFCTR0 + 15,
	MSR_ARCH_PERFMON_PERFCTR0 + 16, MSR_ARCH_PERFMON_PERFCTR0 + 17,
	MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
	MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
	MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
	MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
	MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
	MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
	MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
	MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
	MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
1193 1194
};

1195
static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_all)];
1196 1197
static unsigned num_msrs_to_save;

1198
static const u32 emulated_msrs_all[] = {
1199 1200 1201 1202
	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
1203
	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
1204 1205
	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
1206
	HV_X64_MSR_RESET,
1207
	HV_X64_MSR_VP_INDEX,
1208
	HV_X64_MSR_VP_RUNTIME,
1209
	HV_X64_MSR_SCONTROL,
A
Andrey Smetanin 已提交
1210
	HV_X64_MSR_STIMER0_CONFIG,
1211
	HV_X64_MSR_VP_ASSIST_PAGE,
1212 1213 1214 1215
	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
	HV_X64_MSR_TSC_EMULATION_STATUS,

	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
1216 1217
	MSR_KVM_PV_EOI_EN,

W
Will Auld 已提交
1218
	MSR_IA32_TSC_ADJUST,
1219
	MSR_IA32_TSCDEADLINE,
1220
	MSR_IA32_ARCH_CAPABILITIES,
1221
	MSR_IA32_MISC_ENABLE,
1222 1223
	MSR_IA32_MCG_STATUS,
	MSR_IA32_MCG_CTL,
1224
	MSR_IA32_MCG_EXT_CTL,
P
Paolo Bonzini 已提交
1225
	MSR_IA32_SMBASE,
1226
	MSR_SMI_COUNT,
K
Kyle Huey 已提交
1227 1228
	MSR_PLATFORM_INFO,
	MSR_MISC_FEATURES_ENABLES,
1229
	MSR_AMD64_VIRT_SPEC_CTRL,
1230
	MSR_IA32_POWER_CTL,
1231
	MSR_IA32_UCODE_REV,
1232

1233 1234 1235 1236 1237
	/*
	 * The following list leaves out MSRs whose values are determined
	 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
	 * We always support the "true" VMX control MSRs, even if the host
	 * processor does not, so I am putting these registers here rather
1238
	 * than in msrs_to_save_all.
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252
	 */
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1253
	MSR_K7_HWCR,
1254
	MSR_KVM_POLL_CONTROL,
1255 1256
};

1257
static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
1258 1259
static unsigned num_emulated_msrs;

1260 1261 1262 1263
/*
 * List of msr numbers which are used to expose MSR-based features that
 * can be used by a hypervisor to validate requested CPU features.
 */
1264
static const u32 msr_based_features_all[] = {
1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR0_FIXED1,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_CR4_FIXED1,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1284
	MSR_F10H_DECFG,
1285
	MSR_IA32_UCODE_REV,
1286
	MSR_IA32_ARCH_CAPABILITIES,
1287 1288
};

1289
static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)];
1290 1291
static unsigned int num_msr_based_features;

1292
static u64 kvm_get_arch_capabilities(void)
1293
{
1294
	u64 data = 0;
1295

1296 1297
	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);
1298

P
Paolo Bonzini 已提交
1299 1300 1301 1302 1303 1304 1305 1306
	/*
	 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
	 * the nested hypervisor runs with NX huge pages.  If it is not,
	 * L1 is anyway vulnerable to ITLB_MULTIHIT explots from other
	 * L1 guests, so it need not worry about its own (L2) guests.
	 */
	data |= ARCH_CAP_PSCHANGE_MC_NO;

1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
	/*
	 * If we're doing cache flushes (either "always" or "cond")
	 * we will do one whenever the guest does a vmlaunch/vmresume.
	 * If an outer hypervisor is doing the cache flush for us
	 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
	 * capability to the guest too, and if EPT is disabled we're not
	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
	 * require a nested hypervisor to do a flush of its own.
	 */
	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;

1319 1320 1321 1322 1323 1324 1325
	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
		data |= ARCH_CAP_RDCL_NO;
	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
		data |= ARCH_CAP_SSB_NO;
	if (!boot_cpu_has_bug(X86_BUG_MDS))
		data |= ARCH_CAP_MDS_NO;

1326
	/*
1327 1328 1329 1330
	 * On TAA affected systems:
	 *      - nothing to do if TSX is disabled on the host.
	 *      - we emulate TSX_CTRL if present on the host.
	 *	  This lets the guest use VERW to clear CPU buffers.
1331
	 */
1332
	if (!boot_cpu_has(X86_FEATURE_RTM))
1333
		data &= ~(ARCH_CAP_TAA_NO | ARCH_CAP_TSX_CTRL_MSR);
1334 1335
	else if (!boot_cpu_has_bug(X86_BUG_TAA))
		data |= ARCH_CAP_TAA_NO;
1336

1337 1338 1339
	return data;
}

1340 1341 1342
static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
{
	switch (msr->index) {
1343
	case MSR_IA32_ARCH_CAPABILITIES:
1344 1345 1346
		msr->data = kvm_get_arch_capabilities();
		break;
	case MSR_IA32_UCODE_REV:
1347
		rdmsrl_safe(msr->index, &msr->data);
1348
		break;
1349 1350 1351 1352 1353 1354 1355
	default:
		if (kvm_x86_ops->get_msr_feature(msr))
			return 1;
	}
	return 0;
}

1356 1357 1358
static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	struct kvm_msr_entry msr;
1359
	int r;
1360 1361

	msr.index = index;
1362 1363 1364
	r = kvm_get_msr_feature(&msr);
	if (r)
		return r;
1365 1366 1367 1368 1369 1370

	*data = msr.data;

	return 0;
}

1371
static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1372
{
1373
	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1374
		return false;
A
Alexander Graf 已提交
1375

1376
	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1377
		return false;
1378

1379 1380 1381 1382 1383 1384
	if (efer & (EFER_LME | EFER_LMA) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))
		return false;

	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
		return false;
1385

1386
	return true;
1387 1388 1389 1390 1391 1392 1393 1394

}
bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	if (efer & efer_reserved_bits)
		return false;

	return __kvm_valid_efer(vcpu, efer);
1395 1396 1397
}
EXPORT_SYMBOL_GPL(kvm_valid_efer);

1398
static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1399 1400
{
	u64 old_efer = vcpu->arch.efer;
1401
	u64 efer = msr_info->data;
1402

1403
	if (efer & efer_reserved_bits)
1404
		return 1;
1405

1406 1407 1408 1409 1410 1411 1412 1413
	if (!msr_info->host_initiated) {
		if (!__kvm_valid_efer(vcpu, efer))
			return 1;

		if (is_paging(vcpu) &&
		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
			return 1;
	}
1414

1415
	efer &= ~EFER_LMA;
1416
	efer |= vcpu->arch.efer & EFER_LMA;
1417

1418 1419
	kvm_x86_ops->set_efer(vcpu, efer);

1420 1421 1422 1423
	/* Update reserved bits */
	if ((efer ^ old_efer) & EFER_NX)
		kvm_mmu_reset_context(vcpu);

1424
	return 0;
1425 1426
}

1427 1428 1429 1430 1431 1432
void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);

1433
/*
1434 1435
 * Write @data into the MSR specified by @index.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
1436 1437 1438
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1439 1440
static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
			 bool host_initiated)
1441
{
1442 1443 1444
	struct msr_data msr;

	switch (index) {
1445 1446 1447 1448 1449
	case MSR_FS_BASE:
	case MSR_GS_BASE:
	case MSR_KERNEL_GS_BASE:
	case MSR_CSTAR:
	case MSR_LSTAR:
1450
		if (is_noncanonical_address(data, vcpu))
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
			return 1;
		break;
	case MSR_IA32_SYSENTER_EIP:
	case MSR_IA32_SYSENTER_ESP:
		/*
		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
		 * non-canonical address is written on Intel but not on
		 * AMD (which ignores the top 32-bits, because it does
		 * not implement 64-bit SYSENTER).
		 *
		 * 64-bit code should hence be able to write a non-canonical
		 * value on AMD.  Making the address canonical ensures that
		 * vmentry does not fail on Intel after writing a non-canonical
		 * value, and that something deterministic happens if the guest
		 * invokes 64-bit SYSENTER.
		 */
1467
		data = get_canonical(data, vcpu_virt_addr_bits(vcpu));
1468
	}
1469 1470 1471 1472 1473 1474

	msr.data = data;
	msr.index = index;
	msr.host_initiated = host_initiated;

	return kvm_x86_ops->set_msr(vcpu, &msr);
1475 1476
}

1477
/*
1478 1479 1480 1481
 * Read the MSR specified by @index into @data.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
1482
 */
1483 1484
int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
		  bool host_initiated)
1485 1486
{
	struct msr_data msr;
1487
	int ret;
1488 1489

	msr.index = index;
1490
	msr.host_initiated = host_initiated;
1491

1492 1493 1494 1495
	ret = kvm_x86_ops->get_msr(vcpu, &msr);
	if (!ret)
		*data = msr.data;
	return ret;
1496 1497
}

1498
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1499
{
1500 1501 1502
	return __kvm_get_msr(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_get_msr);
1503

1504 1505 1506 1507 1508 1509
int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
{
	return __kvm_set_msr(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_set_msr);

1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data;

	if (kvm_get_msr(vcpu, ecx, &data)) {
		trace_kvm_msr_read_ex(ecx);
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

	trace_kvm_msr_read(ecx, data);

	kvm_rax_write(vcpu, data & -1u);
	kvm_rdx_write(vcpu, (data >> 32) & -1u);
	return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);

int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data = kvm_read_edx_eax(vcpu);

	if (kvm_set_msr(vcpu, ecx, data)) {
		trace_kvm_msr_write_ex(ecx, data);
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

	trace_kvm_msr_write(ecx, data);
	return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);

1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587
/*
 * The fast path for frequent and performance sensitive wrmsr emulation,
 * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces
 * the latency of virtual IPI by avoiding the expensive bits of transitioning
 * from guest to host, e.g. reacquiring KVM's SRCU lock. In contrast to the
 * other cases which must be called after interrupts are enabled on the host.
 */
static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
{
	if (lapic_in_kernel(vcpu) && apic_x2apic_mode(vcpu->arch.apic) &&
		((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) &&
		((data & APIC_MODE_MASK) == APIC_DM_FIXED)) {

		kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR2, (u32)(data >> 32));
		return kvm_lapic_reg_write(vcpu->arch.apic, APIC_ICR, (u32)data);
	}

	return 1;
}

enum exit_fastpath_completion handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
{
	u32 msr = kvm_rcx_read(vcpu);
	u64 data = kvm_read_edx_eax(vcpu);
	int ret = 0;

	switch (msr) {
	case APIC_BASE_MSR + (APIC_ICR >> 4):
		ret = handle_fastpath_set_x2apic_icr_irqoff(vcpu, data);
		break;
	default:
		return EXIT_FASTPATH_NONE;
	}

	if (!ret) {
		trace_kvm_msr_write(msr, data);
		return EXIT_FASTPATH_SKIP_EMUL_INS;
	}

	return EXIT_FASTPATH_NONE;
}
EXPORT_SYMBOL_GPL(handle_fastpath_set_msr_irqoff);

1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598
/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return __kvm_get_msr(vcpu, index, data, true);
}

static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return __kvm_set_msr(vcpu, index, *data, true);
1599 1600
}

1601
#ifdef CONFIG_X86_64
1602 1603 1604 1605 1606 1607 1608 1609
struct pvclock_clock {
	int vclock_mode;
	u64 cycle_last;
	u64 mask;
	u32 mult;
	u32 shift;
};

1610 1611 1612
struct pvclock_gtod_data {
	seqcount_t	seq;

1613 1614
	struct pvclock_clock clock; /* extract of a clocksource struct */
	struct pvclock_clock raw_clock; /* extract of a clocksource struct */
1615

1616
	u64		boot_ns_raw;
1617 1618
	u64		boot_ns;
	u64		nsec_base;
1619
	u64		wall_time_sec;
1620
	u64		monotonic_raw_nsec;
1621 1622 1623 1624 1625 1626 1627
};

static struct pvclock_gtod_data pvclock_gtod_data;

static void update_pvclock_gtod(struct timekeeper *tk)
{
	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
1628
	u64 boot_ns, boot_ns_raw;
1629

1630
	boot_ns = ktime_to_ns(ktime_add(tk->tkr_mono.base, tk->offs_boot));
1631
	boot_ns_raw = ktime_to_ns(ktime_add(tk->tkr_raw.base, tk->offs_boot));
1632 1633 1634 1635

	write_seqcount_begin(&vdata->seq);

	/* copy pvclock gtod data */
1636 1637 1638 1639 1640
	vdata->clock.vclock_mode	= tk->tkr_mono.clock->archdata.vclock_mode;
	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
	vdata->clock.mask		= tk->tkr_mono.mask;
	vdata->clock.mult		= tk->tkr_mono.mult;
	vdata->clock.shift		= tk->tkr_mono.shift;
1641

1642 1643 1644 1645 1646 1647
	vdata->raw_clock.vclock_mode	= tk->tkr_raw.clock->archdata.vclock_mode;
	vdata->raw_clock.cycle_last	= tk->tkr_raw.cycle_last;
	vdata->raw_clock.mask		= tk->tkr_raw.mask;
	vdata->raw_clock.mult		= tk->tkr_raw.mult;
	vdata->raw_clock.shift		= tk->tkr_raw.shift;

1648
	vdata->boot_ns			= boot_ns;
1649
	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
1650

1651 1652
	vdata->wall_time_sec            = tk->xtime_sec;

1653 1654 1655
	vdata->boot_ns_raw		= boot_ns_raw;
	vdata->monotonic_raw_nsec	= tk->tkr_raw.xtime_nsec;

1656 1657 1658 1659
	write_seqcount_end(&vdata->seq);
}
#endif

1660 1661 1662
void kvm_set_pending_timer(struct kvm_vcpu *vcpu)
{
	kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
1663
	kvm_vcpu_kick(vcpu);
1664
}
1665

1666 1667
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
{
1668 1669
	int version;
	int r;
1670
	struct pvclock_wall_clock wc;
A
Arnd Bergmann 已提交
1671
	struct timespec64 boot;
1672 1673 1674 1675

	if (!wall_clock)
		return;

1676 1677 1678 1679 1680 1681 1682 1683
	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
	if (r)
		return;

	if (version & 1)
		++version;  /* first time write, random junk */

	++version;
1684

1685 1686
	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
		return;
1687

1688 1689
	/*
	 * The guest calculates current wall clock time by adding
Z
Zachary Amsden 已提交
1690
	 * system time (updated by kvm_guest_time_update below) to the
1691 1692 1693
	 * wall clock specified here.  guest system time equals host
	 * system time for us, thus we must fill in host boot time here.
	 */
A
Arnd Bergmann 已提交
1694
	getboottime64(&boot);
1695

1696
	if (kvm->arch.kvmclock_offset) {
A
Arnd Bergmann 已提交
1697 1698
		struct timespec64 ts = ns_to_timespec64(kvm->arch.kvmclock_offset);
		boot = timespec64_sub(boot, ts);
1699
	}
A
Arnd Bergmann 已提交
1700
	wc.sec = (u32)boot.tv_sec; /* overflow in 2106 guest time */
1701 1702
	wc.nsec = boot.tv_nsec;
	wc.version = version;
1703 1704 1705 1706 1707 1708 1709

	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));

	version++;
	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
}

1710 1711
static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
1712 1713
	do_shl32_div32(dividend, divisor);
	return dividend;
1714 1715
}

1716
static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
1717
			       s8 *pshift, u32 *pmultiplier)
1718
{
1719
	uint64_t scaled64;
1720 1721 1722 1723
	int32_t  shift = 0;
	uint64_t tps64;
	uint32_t tps32;

1724 1725
	tps64 = base_hz;
	scaled64 = scaled_hz;
1726
	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
1727 1728 1729 1730 1731
		tps64 >>= 1;
		shift--;
	}

	tps32 = (uint32_t)tps64;
1732 1733
	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
1734 1735 1736
			scaled64 >>= 1;
		else
			tps32 <<= 1;
1737 1738 1739
		shift++;
	}

1740 1741
	*pshift = shift;
	*pmultiplier = div_frac(scaled64, tps32);
1742 1743
}

1744
#ifdef CONFIG_X86_64
1745
static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
1746
#endif
1747

1748
static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
1749
static unsigned long max_tsc_khz;
1750

1751
static u32 adjust_tsc_khz(u32 khz, s32 ppm)
1752
{
1753 1754 1755
	u64 v = (u64)khz * (1000000 + ppm);
	do_div(v, 1000000);
	return v;
1756 1757
}

1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
{
	u64 ratio;

	/* Guest TSC same frequency as host TSC? */
	if (!scale) {
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
		return 0;
	}

	/* TSC scaling supported? */
	if (!kvm_has_tsc_control) {
		if (user_tsc_khz > tsc_khz) {
			vcpu->arch.tsc_catchup = 1;
			vcpu->arch.tsc_always_catchup = 1;
			return 0;
		} else {
1775
			pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
1776 1777 1778 1779 1780 1781 1782 1783 1784
			return -1;
		}
	}

	/* TSC scaling required  - calculate ratio */
	ratio = mul_u64_u32_div(1ULL << kvm_tsc_scaling_ratio_frac_bits,
				user_tsc_khz, tsc_khz);

	if (ratio == 0 || ratio >= kvm_max_tsc_scaling_ratio) {
1785 1786
		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
			            user_tsc_khz);
1787 1788 1789 1790 1791 1792 1793
		return -1;
	}

	vcpu->arch.tsc_scaling_ratio = ratio;
	return 0;
}

1794
static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
1795
{
1796 1797
	u32 thresh_lo, thresh_hi;
	int use_scaling = 0;
1798

1799
	/* tsc_khz can be zero if TSC calibration fails */
1800
	if (user_tsc_khz == 0) {
1801 1802
		/* set tsc_scaling_ratio to a safe value */
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
1803
		return -1;
1804
	}
1805

Z
Zachary Amsden 已提交
1806
	/* Compute a scale to convert nanoseconds in TSC cycles */
1807
	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
1808 1809
			   &vcpu->arch.virtual_tsc_shift,
			   &vcpu->arch.virtual_tsc_mult);
1810
	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
1811 1812 1813 1814 1815 1816 1817 1818 1819

	/*
	 * Compute the variation in TSC rate which is acceptable
	 * within the range of tolerance and decide if the
	 * rate being applied is within that bounds of the hardware
	 * rate.  If so, no scaling or compensation need be done.
	 */
	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
1820 1821
	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", user_tsc_khz, thresh_lo, thresh_hi);
1822 1823
		use_scaling = 1;
	}
1824
	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
Z
Zachary Amsden 已提交
1825 1826 1827 1828
}

static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
1829
	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
1830 1831
				      vcpu->arch.virtual_tsc_mult,
				      vcpu->arch.virtual_tsc_shift);
1832
	tsc += vcpu->arch.this_tsc_write;
Z
Zachary Amsden 已提交
1833 1834 1835
	return tsc;
}

1836 1837 1838 1839 1840
static inline int gtod_is_based_on_tsc(int mode)
{
	return mode == VCLOCK_TSC || mode == VCLOCK_HVCLOCK;
}

1841
static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
1842 1843 1844 1845 1846 1847 1848 1849 1850
{
#ifdef CONFIG_X86_64
	bool vcpus_matched;
	struct kvm_arch *ka = &vcpu->kvm->arch;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			 atomic_read(&vcpu->kvm->online_vcpus));

1851 1852 1853 1854 1855 1856 1857 1858 1859
	/*
	 * Once the masterclock is enabled, always perform request in
	 * order to update it.
	 *
	 * In order to enable masterclock, the host clocksource must be TSC
	 * and the vcpus need to have matched TSCs.  When that happens,
	 * perform request to enable masterclock.
	 */
	if (ka->use_master_clock ||
1860
	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
1861 1862 1863 1864 1865 1866 1867 1868
		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);

	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
			    atomic_read(&vcpu->kvm->online_vcpus),
		            ka->use_master_clock, gtod->clock.vclock_mode);
#endif
}

W
Will Auld 已提交
1869 1870
static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
{
1871
	u64 curr_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
W
Will Auld 已提交
1872 1873 1874
	vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
}

1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901
/*
 * Multiply tsc by a fixed point number represented by ratio.
 *
 * The most significant 64-N bits (mult) of ratio represent the
 * integral part of the fixed point number; the remaining N bits
 * (frac) represent the fractional part, ie. ratio represents a fixed
 * point number (mult + frac * 2^(-N)).
 *
 * N equals to kvm_tsc_scaling_ratio_frac_bits.
 */
static inline u64 __scale_tsc(u64 ratio, u64 tsc)
{
	return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits);
}

u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc)
{
	u64 _tsc = tsc;
	u64 ratio = vcpu->arch.tsc_scaling_ratio;

	if (ratio != kvm_default_tsc_scaling_ratio)
		_tsc = __scale_tsc(ratio, tsc);

	return _tsc;
}
EXPORT_SYMBOL_GPL(kvm_scale_tsc);

1902 1903 1904 1905 1906 1907 1908 1909 1910
static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
{
	u64 tsc;

	tsc = kvm_scale_tsc(vcpu, rdtsc());

	return target_tsc - tsc;
}

1911 1912
u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
{
1913 1914 1915
	u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);

	return tsc_offset + kvm_scale_tsc(vcpu, host_tsc);
1916 1917 1918
}
EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);

1919 1920
static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
{
1921
	vcpu->arch.tsc_offset = kvm_x86_ops->write_l1_tsc_offset(vcpu, offset);
1922 1923
}

1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936
static inline bool kvm_check_tsc_unstable(void)
{
#ifdef CONFIG_X86_64
	/*
	 * TSC is marked unstable when we're running on Hyper-V,
	 * 'TSC page' clocksource is good.
	 */
	if (pvclock_gtod_data.clock.vclock_mode == VCLOCK_HVCLOCK)
		return false;
#endif
	return check_tsc_unstable();
}

1937
void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
1938 1939
{
	struct kvm *kvm = vcpu->kvm;
Z
Zachary Amsden 已提交
1940
	u64 offset, ns, elapsed;
1941
	unsigned long flags;
1942
	bool matched;
T
Tomasz Grabiec 已提交
1943
	bool already_matched;
1944
	u64 data = msr->data;
1945
	bool synchronizing = false;
1946

1947
	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
1948
	offset = kvm_compute_tsc_offset(vcpu, data);
1949
	ns = ktime_get_boottime_ns();
Z
Zachary Amsden 已提交
1950
	elapsed = ns - kvm->arch.last_tsc_nsec;
1951

1952
	if (vcpu->arch.virtual_tsc_khz) {
1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
		if (data == 0 && msr->host_initiated) {
			/*
			 * detection of vcpu initialization -- need to sync
			 * with other vCPUs. This particularly helps to keep
			 * kvm_clock stable after CPU hotplug
			 */
			synchronizing = true;
		} else {
			u64 tsc_exp = kvm->arch.last_tsc_write +
						nsec_to_cycles(vcpu, elapsed);
			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
			/*
			 * Special case: TSC write with a small delta (1 second)
			 * of virtual cycle time against real time is
			 * interpreted as an attempt to synchronize the CPU.
			 */
			synchronizing = data < tsc_exp + tsc_hz &&
					data + tsc_hz > tsc_exp;
		}
1972
	}
Z
Zachary Amsden 已提交
1973 1974

	/*
1975 1976 1977 1978 1979
	 * For a reliable TSC, we can match TSC offsets, and for an unstable
	 * TSC, we add elapsed time in this computation.  We could let the
	 * compensation code attempt to catch up if we fall behind, but
	 * it's better to try to match offsets from the beginning.
         */
1980
	if (synchronizing &&
1981
	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
1982
		if (!kvm_check_tsc_unstable()) {
1983
			offset = kvm->arch.cur_tsc_offset;
Z
Zachary Amsden 已提交
1984
		} else {
1985
			u64 delta = nsec_to_cycles(vcpu, elapsed);
1986
			data += delta;
1987
			offset = kvm_compute_tsc_offset(vcpu, data);
Z
Zachary Amsden 已提交
1988
		}
1989
		matched = true;
T
Tomasz Grabiec 已提交
1990
		already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
1991 1992 1993 1994 1995 1996
	} else {
		/*
		 * We split periods of matched TSC writes into generations.
		 * For each generation, we track the original measured
		 * nanosecond time, offset, and write, so if TSCs are in
		 * sync, we can match exact offset, and if not, we can match
G
Guo Chao 已提交
1997
		 * exact software computation in compute_guest_tsc()
1998 1999 2000 2001 2002 2003 2004
		 *
		 * These values are tracked in kvm->arch.cur_xxx variables.
		 */
		kvm->arch.cur_tsc_generation++;
		kvm->arch.cur_tsc_nsec = ns;
		kvm->arch.cur_tsc_write = data;
		kvm->arch.cur_tsc_offset = offset;
2005
		matched = false;
Z
Zachary Amsden 已提交
2006
	}
2007 2008 2009 2010 2011

	/*
	 * We also track th most recent recorded KHZ, write and time to
	 * allow the matching interval to be extended at each write.
	 */
Z
Zachary Amsden 已提交
2012 2013
	kvm->arch.last_tsc_nsec = ns;
	kvm->arch.last_tsc_write = data;
2014
	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
2015

2016
	vcpu->arch.last_guest_tsc = data;
2017 2018 2019 2020 2021 2022

	/* Keep track of which generation this VCPU has synchronized to */
	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;

2023
	if (!msr->host_initiated && guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST))
W
Will Auld 已提交
2024
		update_ia32_tsc_adjust_msr(vcpu, offset);
2025

2026
	kvm_vcpu_write_tsc_offset(vcpu, offset);
2027
	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
2028 2029

	spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
T
Tomasz Grabiec 已提交
2030
	if (!matched) {
2031
		kvm->arch.nr_vcpus_matched_tsc = 0;
T
Tomasz Grabiec 已提交
2032 2033 2034
	} else if (!already_matched) {
		kvm->arch.nr_vcpus_matched_tsc++;
	}
2035 2036 2037

	kvm_track_tsc_matching(vcpu);
	spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
2038
}
2039

2040 2041
EXPORT_SYMBOL_GPL(kvm_write_tsc);

2042 2043 2044
static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
					   s64 adjustment)
{
2045 2046
	u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2047 2048 2049 2050 2051 2052 2053
}

static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
{
	if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
		WARN_ON(adjustment < 0);
	adjustment = kvm_scale_tsc(vcpu, (u64) adjustment);
2054
	adjust_tsc_offset_guest(vcpu, adjustment);
2055 2056
}

2057 2058
#ifdef CONFIG_X86_64

2059
static u64 read_tsc(void)
2060
{
2061
	u64 ret = (u64)rdtsc_ordered();
2062
	u64 last = pvclock_gtod_data.clock.cycle_last;
2063 2064 2065 2066 2067 2068

	if (likely(ret >= last))
		return ret;

	/*
	 * GCC likes to generate cmov here, but this branch is extremely
2069
	 * predictable (it's just a function of time and the likely is
2070 2071 2072 2073 2074 2075 2076 2077 2078
	 * very likely) and there's a data dependence, so force GCC
	 * to generate a branch instead.  I don't barrier() because
	 * we don't actually need a barrier, and if this function
	 * ever gets inlined it will generate worse code.
	 */
	asm volatile ("");
	return last;
}

2079 2080
static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
			  int *mode)
2081 2082
{
	long v;
2083 2084
	u64 tsc_pg_val;

2085
	switch (clock->vclock_mode) {
2086 2087 2088 2089 2090 2091
	case VCLOCK_HVCLOCK:
		tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
						  tsc_timestamp);
		if (tsc_pg_val != U64_MAX) {
			/* TSC page valid */
			*mode = VCLOCK_HVCLOCK;
2092 2093
			v = (tsc_pg_val - clock->cycle_last) &
				clock->mask;
2094 2095 2096 2097 2098 2099 2100 2101
		} else {
			/* TSC page invalid */
			*mode = VCLOCK_NONE;
		}
		break;
	case VCLOCK_TSC:
		*mode = VCLOCK_TSC;
		*tsc_timestamp = read_tsc();
2102 2103
		v = (*tsc_timestamp - clock->cycle_last) &
			clock->mask;
2104 2105 2106 2107
		break;
	default:
		*mode = VCLOCK_NONE;
	}
2108

2109 2110
	if (*mode == VCLOCK_NONE)
		*tsc_timestamp = v = 0;
2111

2112
	return v * clock->mult;
2113 2114
}

2115
static int do_monotonic_raw(s64 *t, u64 *tsc_timestamp)
2116
{
2117
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2118 2119
	unsigned long seq;
	int mode;
2120
	u64 ns;
2121 2122 2123

	do {
		seq = read_seqcount_begin(&gtod->seq);
2124 2125
		ns = gtod->monotonic_raw_nsec;
		ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
2126
		ns >>= gtod->clock.shift;
2127
		ns += gtod->boot_ns_raw;
2128
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2129
	*t = ns;
2130 2131 2132 2133

	return mode;
}

2134
static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	unsigned long seq;
	int mode;
	u64 ns;

	do {
		seq = read_seqcount_begin(&gtod->seq);
		ts->tv_sec = gtod->wall_time_sec;
		ns = gtod->nsec_base;
2145
		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
2146 2147 2148 2149 2150 2151 2152 2153 2154
		ns >>= gtod->clock.shift;
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));

	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return mode;
}

2155 2156
/* returns true if host is using TSC based clocksource */
static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
2157 2158
{
	/* checked again under seqlock below */
2159
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2160 2161
		return false;

2162
	return gtod_is_based_on_tsc(do_monotonic_raw(kernel_ns,
2163
						      tsc_timestamp));
2164
}
2165

2166
/* returns true if host is using TSC based clocksource */
2167
static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
2168
					   u64 *tsc_timestamp)
2169 2170
{
	/* checked again under seqlock below */
2171
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2172 2173
		return false;

2174
	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
2175
}
2176 2177 2178 2179
#endif

/*
 *
2180 2181 2182
 * Assuming a stable TSC across physical CPUS, and a stable TSC
 * across virtual CPUs, the following condition is possible.
 * Each numbered line represents an event visible to both
2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214
 * CPUs at the next numbered event.
 *
 * "timespecX" represents host monotonic time. "tscX" represents
 * RDTSC value.
 *
 * 		VCPU0 on CPU0		|	VCPU1 on CPU1
 *
 * 1.  read timespec0,tsc0
 * 2.					| timespec1 = timespec0 + N
 * 					| tsc1 = tsc0 + M
 * 3. transition to guest		| transition to guest
 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
 * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
 * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
 *
 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
 *
 * 	- ret0 < ret1
 *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
 *		...
 *	- 0 < N - M => M < N
 *
 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
 * always the case (the difference between two distinct xtime instances
 * might be smaller then the difference between corresponding TSC reads,
 * when updating guest vcpus pvclock areas).
 *
 * To avoid that problem, do not allow visibility of distinct
 * system_timestamp/tsc_timestamp values simultaneously: use a master
 * copy of host monotonic time values. Update that master copy
 * in lockstep.
 *
2215
 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
2216 2217 2218 2219 2220 2221 2222 2223
 *
 */

static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	struct kvm_arch *ka = &kvm->arch;
	int vclock_mode;
2224 2225 2226 2227
	bool host_tsc_clocksource, vcpus_matched;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			atomic_read(&kvm->online_vcpus));
2228 2229 2230 2231 2232

	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
2233
	host_tsc_clocksource = kvm_get_time_and_clockread(
2234 2235 2236
					&ka->master_kernel_ns,
					&ka->master_cycle_now);

2237
	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
2238
				&& !ka->backwards_tsc_observed
2239
				&& !ka->boot_vcpu_runs_old_kvmclock;
2240

2241 2242 2243 2244
	if (ka->use_master_clock)
		atomic_set(&kvm_guest_has_master_clock, 1);

	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
2245 2246
	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
					vcpus_matched);
2247 2248 2249
#endif
}

2250 2251 2252 2253 2254
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
}

2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267
static void kvm_gen_update_masterclock(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	int i;
	struct kvm_vcpu *vcpu;
	struct kvm_arch *ka = &kvm->arch;

	spin_lock(&ka->pvclock_gtod_sync_lock);
	kvm_make_mclock_inprogress_request(kvm);
	/* no guest entries from this point */
	pvclock_update_vm_gtod_copy(kvm);

	kvm_for_each_vcpu(i, vcpu, kvm)
2268
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2269 2270 2271

	/* guest entries allowed */
	kvm_for_each_vcpu(i, vcpu, kvm)
2272
		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
2273 2274 2275 2276 2277

	spin_unlock(&ka->pvclock_gtod_sync_lock);
#endif
}

2278
u64 get_kvmclock_ns(struct kvm *kvm)
2279 2280
{
	struct kvm_arch *ka = &kvm->arch;
2281
	struct pvclock_vcpu_time_info hv_clock;
2282
	u64 ret;
2283

2284 2285 2286
	spin_lock(&ka->pvclock_gtod_sync_lock);
	if (!ka->use_master_clock) {
		spin_unlock(&ka->pvclock_gtod_sync_lock);
2287
		return ktime_get_boottime_ns() + ka->kvmclock_offset;
2288 2289
	}

2290 2291 2292 2293
	hv_clock.tsc_timestamp = ka->master_cycle_now;
	hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
	spin_unlock(&ka->pvclock_gtod_sync_lock);

2294 2295 2296
	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
	get_cpu();

2297 2298 2299 2300 2301 2302
	if (__this_cpu_read(cpu_tsc_khz)) {
		kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL,
				   &hv_clock.tsc_shift,
				   &hv_clock.tsc_to_system_mul);
		ret = __pvclock_read_cycles(&hv_clock, rdtsc());
	} else
2303
		ret = ktime_get_boottime_ns() + ka->kvmclock_offset;
2304 2305 2306 2307

	put_cpu();

	return ret;
2308 2309
}

2310 2311 2312 2313 2314
static void kvm_setup_pvclock_page(struct kvm_vcpu *v)
{
	struct kvm_vcpu_arch *vcpu = &v->arch;
	struct pvclock_vcpu_time_info guest_hv_clock;

2315
	if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334
		&guest_hv_clock, sizeof(guest_hv_clock))))
		return;

	/* This VCPU is paused, but it's legal for a guest to read another
	 * VCPU's kvmclock, so we really have to follow the specification where
	 * it says that version is odd if data is being modified, and even after
	 * it is consistent.
	 *
	 * Version field updates must be kept separate.  This is because
	 * kvm_write_guest_cached might use a "rep movs" instruction, and
	 * writes within a string instruction are weakly ordered.  So there
	 * are three writes overall.
	 *
	 * As a small optimization, only write the version field in the first
	 * and third write.  The vcpu->pv_time cache is still valid, because the
	 * version field is the first in the struct.
	 */
	BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);

2335 2336 2337
	if (guest_hv_clock.version & 1)
		++guest_hv_clock.version;  /* first time write, random junk */

2338
	vcpu->hv_clock.version = guest_hv_clock.version + 1;
2339 2340 2341
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354

	smp_wmb();

	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
	vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);

	if (vcpu->pvclock_set_guest_stopped_request) {
		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
		vcpu->pvclock_set_guest_stopped_request = false;
	}

	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);

2355 2356 2357
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock));
2358 2359 2360 2361

	smp_wmb();

	vcpu->hv_clock.version++;
2362 2363 2364
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2365 2366
}

Z
Zachary Amsden 已提交
2367
static int kvm_guest_time_update(struct kvm_vcpu *v)
2368
{
2369
	unsigned long flags, tgt_tsc_khz;
2370
	struct kvm_vcpu_arch *vcpu = &v->arch;
2371
	struct kvm_arch *ka = &v->kvm->arch;
2372
	s64 kernel_ns;
2373
	u64 tsc_timestamp, host_tsc;
2374
	u8 pvclock_flags;
2375 2376 2377 2378
	bool use_master_clock;

	kernel_ns = 0;
	host_tsc = 0;
2379

2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390
	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
	spin_lock(&ka->pvclock_gtod_sync_lock);
	use_master_clock = ka->use_master_clock;
	if (use_master_clock) {
		host_tsc = ka->master_cycle_now;
		kernel_ns = ka->master_kernel_ns;
	}
	spin_unlock(&ka->pvclock_gtod_sync_lock);
2391 2392 2393

	/* Keep irq disabled to prevent changes to the clock */
	local_irq_save(flags);
2394 2395
	tgt_tsc_khz = __this_cpu_read(cpu_tsc_khz);
	if (unlikely(tgt_tsc_khz == 0)) {
2396 2397 2398 2399
		local_irq_restore(flags);
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
		return 1;
	}
2400
	if (!use_master_clock) {
2401
		host_tsc = rdtsc();
2402
		kernel_ns = ktime_get_boottime_ns();
2403 2404
	}

2405
	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
2406

Z
Zachary Amsden 已提交
2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419
	/*
	 * We may have to catch up the TSC to match elapsed wall clock
	 * time for two reasons, even if kvmclock is used.
	 *   1) CPU could have been running below the maximum TSC rate
	 *   2) Broken TSC compensation resets the base at each VCPU
	 *      entry to avoid unknown leaps of TSC even when running
	 *      again on the same CPU.  This may cause apparent elapsed
	 *      time to disappear, and the guest to stand still or run
	 *	very slowly.
	 */
	if (vcpu->tsc_catchup) {
		u64 tsc = compute_guest_tsc(v, kernel_ns);
		if (tsc > tsc_timestamp) {
2420
			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
Z
Zachary Amsden 已提交
2421 2422
			tsc_timestamp = tsc;
		}
2423 2424
	}

2425 2426
	local_irq_restore(flags);

2427
	/* With all the info we got, fill in the values */
2428

2429 2430 2431 2432
	if (kvm_has_tsc_control)
		tgt_tsc_khz = kvm_scale_tsc(v, tgt_tsc_khz);

	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
2433
		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
2434 2435
				   &vcpu->hv_clock.tsc_shift,
				   &vcpu->hv_clock.tsc_to_system_mul);
2436
		vcpu->hw_tsc_khz = tgt_tsc_khz;
2437 2438
	}

2439
	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
2440
	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
Z
Zachary Amsden 已提交
2441
	vcpu->last_guest_tsc = tsc_timestamp;
2442

2443
	/* If the host uses TSC clocksource, then it is stable */
2444
	pvclock_flags = 0;
2445 2446 2447
	if (use_master_clock)
		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;

2448 2449
	vcpu->hv_clock.flags = pvclock_flags;

P
Paolo Bonzini 已提交
2450 2451 2452 2453
	if (vcpu->pv_time_enabled)
		kvm_setup_pvclock_page(v);
	if (v == kvm_get_vcpu(v->kvm, 0))
		kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
2454
	return 0;
2455 2456
}

2457 2458 2459 2460 2461 2462 2463 2464
/*
 * kvmclock updates which are isolated to a given vcpu, such as
 * vcpu->cpu migration, should not allow system_timestamp from
 * the rest of the vcpus to remain static. Otherwise ntp frequency
 * correction applies to one vcpu's system_timestamp but not
 * the others.
 *
 * So in those cases, request a kvmclock update for all vcpus.
2465 2466 2467 2468
 * We need to rate-limit these requests though, as they can
 * considerably slow guests that have a large number of vcpus.
 * The time for a remote vcpu to update its kvmclock is bound
 * by the delay we use to rate-limit the updates.
2469 2470
 */

2471 2472 2473
#define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)

static void kvmclock_update_fn(struct work_struct *work)
2474 2475
{
	int i;
2476 2477 2478 2479
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_update_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);
2480 2481 2482
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm) {
2483
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2484 2485 2486 2487
		kvm_vcpu_kick(vcpu);
	}
}

2488 2489 2490 2491
static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
{
	struct kvm *kvm = v->kvm;

2492
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
2493 2494 2495 2496
	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
					KVMCLOCK_UPDATE_DELAY);
}

2497 2498 2499 2500 2501 2502 2503 2504 2505
#define KVMCLOCK_SYNC_PERIOD (300 * HZ)

static void kvmclock_sync_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_sync_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);

2506 2507 2508
	if (!kvmclock_periodic_sync)
		return;

2509 2510 2511 2512 2513
	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
}

2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525
/*
 * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
 */
static bool can_set_mci_status(struct kvm_vcpu *vcpu)
{
	/* McStatusWrEn enabled? */
	if (guest_cpuid_is_amd(vcpu))
		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));

	return false;
}

2526
static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2527
{
H
Huang Ying 已提交
2528 2529
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2530 2531
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
H
Huang Ying 已提交
2532

2533 2534
	switch (msr) {
	case MSR_IA32_MCG_STATUS:
H
Huang Ying 已提交
2535
		vcpu->arch.mcg_status = data;
2536
		break;
2537
	case MSR_IA32_MCG_CTL:
2538 2539
		if (!(mcg_cap & MCG_CTL_P) &&
		    (data || !msr_info->host_initiated))
H
Huang Ying 已提交
2540 2541
			return 1;
		if (data != 0 && data != ~(u64)0)
2542
			return 1;
H
Huang Ying 已提交
2543 2544 2545 2546
		vcpu->arch.mcg_ctl = data;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2547
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
H
Huang Ying 已提交
2548
			u32 offset = msr - MSR_IA32_MC0_CTL;
2549 2550 2551 2552 2553
			/* only 0 or all 1s can be written to IA32_MCi_CTL
			 * some Linux kernels though clear bit 10 in bank 4 to
			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
			 * this to avoid an uncatched #GP in the guest
			 */
H
Huang Ying 已提交
2554
			if ((offset & 0x3) == 0 &&
2555
			    data != 0 && (data | (1 << 10)) != ~(u64)0)
H
Huang Ying 已提交
2556
				return -1;
2557 2558

			/* MCi_STATUS */
2559
			if (!msr_info->host_initiated &&
2560 2561 2562 2563 2564
			    (offset & 0x3) == 1 && data != 0) {
				if (!can_set_mci_status(vcpu))
					return -1;
			}

H
Huang Ying 已提交
2565 2566 2567 2568 2569 2570 2571 2572
			vcpu->arch.mce_banks[offset] = data;
			break;
		}
		return 1;
	}
	return 0;
}

E
Ed Swierk 已提交
2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589
static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
	int lm = is_long_mode(vcpu);
	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
		: kvm->arch.xen_hvm_config.blob_size_32;
	u32 page_num = data & ~PAGE_MASK;
	u64 page_addr = data & PAGE_MASK;
	u8 *page;
	int r;

	r = -E2BIG;
	if (page_num >= blob_size)
		goto out;
	r = -ENOMEM;
2590 2591 2592
	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
	if (IS_ERR(page)) {
		r = PTR_ERR(page);
E
Ed Swierk 已提交
2593
		goto out;
2594
	}
2595
	if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE))
E
Ed Swierk 已提交
2596 2597 2598 2599 2600 2601 2602 2603
		goto out_free;
	r = 0;
out_free:
	kfree(page);
out:
	return r;
}

2604 2605 2606 2607
static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
{
	gpa_t gpa = data & ~0x3f;

2608 2609
	/* Bits 3:5 are reserved, Should be zero */
	if (data & 0x38)
2610 2611 2612 2613 2614 2615 2616 2617 2618 2619
		return 1;

	vcpu->arch.apf.msr_val = data;

	if (!(data & KVM_ASYNC_PF_ENABLED)) {
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_async_pf_hash_reset(vcpu);
		return 0;
	}

2620
	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
2621
					sizeof(u32)))
2622 2623
		return 1;

2624
	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
2625
	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
2626 2627 2628 2629
	kvm_async_pf_wakeup_all(vcpu);
	return 0;
}

2630 2631
static void kvmclock_reset(struct kvm_vcpu *vcpu)
{
2632
	vcpu->arch.pv_time_enabled = false;
P
Paolo Bonzini 已提交
2633
	vcpu->arch.time = 0;
2634 2635
}

2636 2637 2638 2639 2640 2641
static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
{
	++vcpu->stat.tlb_flush;
	kvm_x86_ops->tlb_flush(vcpu, invalidate_gpa);
}

G
Glauber Costa 已提交
2642 2643 2644 2645 2646
static void record_steal_time(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

2647
	if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
G
Glauber Costa 已提交
2648 2649 2650
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
		return;

2651 2652 2653 2654
	/*
	 * Doing a TLB flush here, on the guest's behalf, can avoid
	 * expensive IPIs.
	 */
2655 2656
	trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
		vcpu->arch.st.steal.preempted & KVM_VCPU_FLUSH_TLB);
2657 2658
	if (xchg(&vcpu->arch.st.steal.preempted, 0) & KVM_VCPU_FLUSH_TLB)
		kvm_vcpu_flush_tlb(vcpu, false);
2659

W
Wanpeng Li 已提交
2660 2661 2662 2663 2664
	if (vcpu->arch.st.steal.version & 1)
		vcpu->arch.st.steal.version += 1;  /* first time write, random junk */

	vcpu->arch.st.steal.version += 1;

2665
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
W
Wanpeng Li 已提交
2666 2667 2668 2669
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));

	smp_wmb();

2670 2671 2672
	vcpu->arch.st.steal.steal += current->sched_info.run_delay -
		vcpu->arch.st.last_steal;
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
W
Wanpeng Li 已提交
2673

2674
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
W
Wanpeng Li 已提交
2675 2676 2677 2678 2679
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));

	smp_wmb();

	vcpu->arch.st.steal.version += 1;
G
Glauber Costa 已提交
2680

2681
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
G
Glauber Costa 已提交
2682 2683 2684
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
}

2685
int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2686
{
2687
	bool pr = false;
2688 2689
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
2690

2691
	switch (msr) {
2692 2693 2694 2695 2696
	case MSR_AMD64_NB_CFG:
	case MSR_IA32_UCODE_WRITE:
	case MSR_VM_HSAVE_PA:
	case MSR_AMD64_PATCH_LOADER:
	case MSR_AMD64_BU_CFG2:
2697
	case MSR_AMD64_DC_CFG:
2698
	case MSR_F15H_EX_CFG:
2699 2700
		break;

2701 2702 2703 2704
	case MSR_IA32_UCODE_REV:
		if (msr_info->host_initiated)
			vcpu->arch.microcode_version = data;
		break;
2705 2706 2707 2708 2709
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.arch_capabilities = data;
		break;
2710
	case MSR_EFER:
2711
		return set_efer(vcpu, msr_info);
2712 2713
	case MSR_K7_HWCR:
		data &= ~(u64)0x40;	/* ignore flush filter disable */
2714
		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
2715
		data &= ~(u64)0x8;	/* ignore TLB cache disable */
2716 2717 2718 2719 2720

		/* Handle McStatusWrEn */
		if (data == BIT_ULL(18)) {
			vcpu->arch.msr_hwcr = data;
		} else if (data != 0) {
2721 2722
			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
				    data);
2723 2724
			return 1;
		}
2725
		break;
2726 2727
	case MSR_FAM10H_MMIO_CONF_BASE:
		if (data != 0) {
2728 2729
			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
				    "0x%llx\n", data);
2730 2731
			return 1;
		}
2732
		break;
2733 2734 2735 2736 2737 2738 2739 2740 2741
	case MSR_IA32_DEBUGCTLMSR:
		if (!data) {
			/* We support the non-activated case already */
			break;
		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
			/* Values other than LBR and BTF are vendor-specific,
			   thus reserved and should throw a #GP */
			return 1;
		}
2742 2743
		vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
			    __func__, data);
2744
		break;
A
Avi Kivity 已提交
2745
	case 0x200 ... 0x2ff:
2746
		return kvm_mtrr_set_msr(vcpu, msr, data);
2747
	case MSR_IA32_APICBASE:
2748
		return kvm_set_apic_base(vcpu, msr_info);
G
Gleb Natapov 已提交
2749 2750
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_write(vcpu, msr, data);
2751 2752 2753
	case MSR_IA32_TSCDEADLINE:
		kvm_set_lapic_tscdeadline_msr(vcpu, data);
		break;
W
Will Auld 已提交
2754
	case MSR_IA32_TSC_ADJUST:
2755
		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
W
Will Auld 已提交
2756
			if (!msr_info->host_initiated) {
2757
				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
2758
				adjust_tsc_offset_guest(vcpu, adj);
W
Will Auld 已提交
2759 2760 2761 2762
			}
			vcpu->arch.ia32_tsc_adjust_msr = data;
		}
		break;
2763
	case MSR_IA32_MISC_ENABLE:
2764 2765 2766 2767 2768 2769 2770 2771 2772
		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
		    ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
				return 1;
			vcpu->arch.ia32_misc_enable_msr = data;
			kvm_update_cpuid(vcpu);
		} else {
			vcpu->arch.ia32_misc_enable_msr = data;
		}
2773
		break;
P
Paolo Bonzini 已提交
2774 2775 2776 2777 2778
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smbase = data;
		break;
2779 2780 2781
	case MSR_IA32_POWER_CTL:
		vcpu->arch.msr_ia32_power_ctl = data;
		break;
2782 2783 2784
	case MSR_IA32_TSC:
		kvm_write_tsc(vcpu, msr_info);
		break;
2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798
	case MSR_IA32_XSS:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
			return 1;
		/*
		 * We do support PT if kvm_x86_ops->pt_supported(), but we do
		 * not support IA32_XSS[bit 8]. Guests will have to use
		 * RDMSR/WRMSR rather than XSAVES/XRSTORS to save/restore PT
		 * MSRs.
		 */
		if (data != 0)
			return 1;
		vcpu->arch.ia32_xss = data;
		break;
2799 2800 2801 2802 2803
	case MSR_SMI_COUNT:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smi_count = data;
		break;
2804
	case MSR_KVM_WALL_CLOCK_NEW:
2805 2806 2807 2808
	case MSR_KVM_WALL_CLOCK:
		vcpu->kvm->arch.wall_clock = data;
		kvm_write_wall_clock(vcpu->kvm, data);
		break;
2809
	case MSR_KVM_SYSTEM_TIME_NEW:
2810
	case MSR_KVM_SYSTEM_TIME: {
2811 2812 2813 2814 2815 2816
		struct kvm_arch *ka = &vcpu->kvm->arch;

		if (vcpu->vcpu_id == 0 && !msr_info->host_initiated) {
			bool tmp = (msr == MSR_KVM_SYSTEM_TIME);

			if (ka->boot_vcpu_runs_old_kvmclock != tmp)
2817
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2818 2819 2820 2821

			ka->boot_vcpu_runs_old_kvmclock = tmp;
		}

2822
		vcpu->arch.time = data;
2823
		kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2824 2825

		/* we verify if the enable bit is set... */
P
Paolo Bonzini 已提交
2826
		vcpu->arch.pv_time_enabled = false;
2827 2828 2829
		if (!(data & 1))
			break;

P
Paolo Bonzini 已提交
2830
		if (!kvm_gfn_to_hva_cache_init(vcpu->kvm,
2831 2832
		     &vcpu->arch.pv_time, data & ~1ULL,
		     sizeof(struct pvclock_vcpu_time_info)))
2833
			vcpu->arch.pv_time_enabled = true;
2834

2835 2836
		break;
	}
2837 2838 2839 2840
	case MSR_KVM_ASYNC_PF_EN:
		if (kvm_pv_enable_async_pf(vcpu, data))
			return 1;
		break;
G
Glauber Costa 已提交
2841 2842 2843 2844 2845 2846 2847 2848
	case MSR_KVM_STEAL_TIME:

		if (unlikely(!sched_info_on()))
			return 1;

		if (data & KVM_STEAL_RESERVED_MASK)
			return 1;

2849
		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
2850 2851
						data & KVM_STEAL_VALID_BITS,
						sizeof(struct kvm_steal_time)))
G
Glauber Costa 已提交
2852 2853 2854 2855 2856 2857 2858 2859 2860 2861
			return 1;

		vcpu->arch.st.msr_val = data;

		if (!(data & KVM_MSR_ENABLED))
			break;

		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);

		break;
2862
	case MSR_KVM_PV_EOI_EN:
2863
		if (kvm_lapic_enable_pv_eoi(vcpu, data, sizeof(u8)))
2864 2865
			return 1;
		break;
G
Glauber Costa 已提交
2866

2867 2868 2869 2870 2871 2872 2873 2874
	case MSR_KVM_POLL_CONTROL:
		/* only enable bit supported */
		if (data & (-1ULL << 1))
			return 1;

		vcpu->arch.msr_kvm_poll_control = data;
		break;

H
Huang Ying 已提交
2875 2876
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
2877
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
2878
		return set_msr_mce(vcpu, msr_info);
2879

2880 2881 2882 2883 2884
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
		pr = true; /* fall through */
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
2885
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2886
			return kvm_pmu_set_msr(vcpu, msr_info);
2887 2888

		if (pr || data != 0)
2889 2890
			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
				    "0x%x data 0x%llx\n", msr, data);
2891
		break;
2892 2893 2894 2895 2896
	case MSR_K7_CLK_CTL:
		/*
		 * Ignore all writes to this no longer documented MSR.
		 * Writes are only relevant for old K7 processors,
		 * all pre-dating SVM, but a recommended workaround from
G
Guo Chao 已提交
2897
		 * AMD for these chips. It is possible to specify the
2898 2899 2900 2901
		 * affected processor models on the command line, hence
		 * the need to ignore the workaround.
		 */
		break;
2902
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
2903 2904
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
A
Andrey Smetanin 已提交
2905
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
2906 2907 2908
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
2909 2910
		return kvm_hv_set_msr_common(vcpu, msr, data,
					     msr_info->host_initiated);
2911 2912 2913 2914
	case MSR_IA32_BBL_CR_CTL3:
		/* Drop writes to this legacy MSR -- see rdmsr
		 * counterpart for further detail.
		 */
2915 2916 2917
		if (report_ignored_msrs)
			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
				msr, data);
2918
		break;
2919
	case MSR_AMD64_OSVW_ID_LENGTH:
2920
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2921 2922 2923 2924
			return 1;
		vcpu->arch.osvw.length = data;
		break;
	case MSR_AMD64_OSVW_STATUS:
2925
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2926 2927 2928
			return 1;
		vcpu->arch.osvw.status = data;
		break;
K
Kyle Huey 已提交
2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942
	case MSR_PLATFORM_INFO:
		if (!msr_info->host_initiated ||
		    (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
		     cpuid_fault_enabled(vcpu)))
			return 1;
		vcpu->arch.msr_platform_info = data;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
		     !supports_cpuid_fault(vcpu)))
			return 1;
		vcpu->arch.msr_misc_features_enables = data;
		break;
2943
	default:
E
Ed Swierk 已提交
2944 2945
		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
			return xen_hvm_config(vcpu, data);
2946
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2947
			return kvm_pmu_set_msr(vcpu, msr_info);
2948
		if (!ignore_msrs) {
2949
			vcpu_debug_ratelimited(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n",
2950
				    msr, data);
2951 2952
			return 1;
		} else {
2953 2954 2955 2956
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu,
					"ignored wrmsr: 0x%x data 0x%llx\n",
					msr, data);
2957 2958
			break;
		}
2959 2960 2961 2962 2963
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);

2964
static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
2965 2966
{
	u64 data;
H
Huang Ying 已提交
2967 2968
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2969 2970 2971 2972

	switch (msr) {
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
H
Huang Ying 已提交
2973 2974
		data = 0;
		break;
2975
	case MSR_IA32_MCG_CAP:
H
Huang Ying 已提交
2976 2977
		data = vcpu->arch.mcg_cap;
		break;
2978
	case MSR_IA32_MCG_CTL:
2979
		if (!(mcg_cap & MCG_CTL_P) && !host)
H
Huang Ying 已提交
2980 2981 2982 2983 2984 2985 2986 2987
			return 1;
		data = vcpu->arch.mcg_ctl;
		break;
	case MSR_IA32_MCG_STATUS:
		data = vcpu->arch.mcg_status;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2988
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
H
Huang Ying 已提交
2989 2990 2991 2992 2993 2994 2995 2996 2997 2998
			u32 offset = msr - MSR_IA32_MC0_CTL;
			data = vcpu->arch.mce_banks[offset];
			break;
		}
		return 1;
	}
	*pdata = data;
	return 0;
}

2999
int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
H
Huang Ying 已提交
3000
{
3001
	switch (msr_info->index) {
H
Huang Ying 已提交
3002
	case MSR_IA32_PLATFORM_ID:
3003
	case MSR_IA32_EBL_CR_POWERON:
3004 3005 3006 3007 3008
	case MSR_IA32_DEBUGCTLMSR:
	case MSR_IA32_LASTBRANCHFROMIP:
	case MSR_IA32_LASTBRANCHTOIP:
	case MSR_IA32_LASTINTFROMIP:
	case MSR_IA32_LASTINTTOIP:
3009
	case MSR_K8_SYSCFG:
3010 3011
	case MSR_K8_TSEG_ADDR:
	case MSR_K8_TSEG_MASK:
3012
	case MSR_VM_HSAVE_PA:
3013
	case MSR_K8_INT_PENDING_MSG:
3014
	case MSR_AMD64_NB_CFG:
3015
	case MSR_FAM10H_MMIO_CONF_BASE:
3016
	case MSR_AMD64_BU_CFG2:
D
Dmitry Bilunov 已提交
3017
	case MSR_IA32_PERF_CTL:
3018
	case MSR_AMD64_DC_CFG:
3019
	case MSR_F15H_EX_CFG:
3020
		msr_info->data = 0;
3021
		break;
3022
	case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
3023 3024 3025 3026
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3027
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3028 3029
			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
		msr_info->data = 0;
3030
		break;
3031
	case MSR_IA32_UCODE_REV:
3032
		msr_info->data = vcpu->arch.microcode_version;
3033
		break;
3034 3035 3036 3037 3038 3039
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
			return 1;
		msr_info->data = vcpu->arch.arch_capabilities;
		break;
3040 3041 3042
	case MSR_IA32_POWER_CTL:
		msr_info->data = vcpu->arch.msr_ia32_power_ctl;
		break;
3043 3044 3045
	case MSR_IA32_TSC:
		msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + vcpu->arch.tsc_offset;
		break;
A
Avi Kivity 已提交
3046 3047
	case MSR_MTRRcap:
	case 0x200 ... 0x2ff:
3048
		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
3049
	case 0xcd: /* fsb frequency */
3050
		msr_info->data = 3;
3051
		break;
3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063
		/*
		 * MSR_EBC_FREQUENCY_ID
		 * Conservative value valid for even the basic CPU models.
		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
		 * and 266MHz for model 3, or 4. Set Core Clock
		 * Frequency to System Bus Frequency Ratio to 1 (bits
		 * 31:24) even though these are only valid for CPU
		 * models > 2, however guests may end up dividing or
		 * multiplying by zero otherwise.
		 */
	case MSR_EBC_FREQUENCY_ID:
3064
		msr_info->data = 1 << 24;
3065
		break;
3066
	case MSR_IA32_APICBASE:
3067
		msr_info->data = kvm_get_apic_base(vcpu);
3068
		break;
G
Gleb Natapov 已提交
3069
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
3070
		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
G
Gleb Natapov 已提交
3071
		break;
3072
	case MSR_IA32_TSCDEADLINE:
3073
		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
3074
		break;
W
Will Auld 已提交
3075
	case MSR_IA32_TSC_ADJUST:
3076
		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
W
Will Auld 已提交
3077
		break;
3078
	case MSR_IA32_MISC_ENABLE:
3079
		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
3080
		break;
P
Paolo Bonzini 已提交
3081 3082 3083 3084
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		msr_info->data = vcpu->arch.smbase;
3085
		break;
3086 3087 3088
	case MSR_SMI_COUNT:
		msr_info->data = vcpu->arch.smi_count;
		break;
3089 3090
	case MSR_IA32_PERF_STATUS:
		/* TSC increment by tick */
3091
		msr_info->data = 1000ULL;
3092
		/* CPU multiplier */
3093
		msr_info->data |= (((uint64_t)4ULL) << 40);
3094
		break;
3095
	case MSR_EFER:
3096
		msr_info->data = vcpu->arch.efer;
3097
		break;
3098
	case MSR_KVM_WALL_CLOCK:
3099
	case MSR_KVM_WALL_CLOCK_NEW:
3100
		msr_info->data = vcpu->kvm->arch.wall_clock;
3101 3102
		break;
	case MSR_KVM_SYSTEM_TIME:
3103
	case MSR_KVM_SYSTEM_TIME_NEW:
3104
		msr_info->data = vcpu->arch.time;
3105
		break;
3106
	case MSR_KVM_ASYNC_PF_EN:
3107
		msr_info->data = vcpu->arch.apf.msr_val;
3108
		break;
G
Glauber Costa 已提交
3109
	case MSR_KVM_STEAL_TIME:
3110
		msr_info->data = vcpu->arch.st.msr_val;
G
Glauber Costa 已提交
3111
		break;
3112
	case MSR_KVM_PV_EOI_EN:
3113
		msr_info->data = vcpu->arch.pv_eoi.msr_val;
3114
		break;
3115 3116 3117
	case MSR_KVM_POLL_CONTROL:
		msr_info->data = vcpu->arch.msr_kvm_poll_control;
		break;
H
Huang Ying 已提交
3118 3119 3120 3121 3122
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
3123
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3124 3125
		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
				   msr_info->host_initiated);
3126 3127 3128 3129 3130 3131
	case MSR_IA32_XSS:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
			return 1;
		msr_info->data = vcpu->arch.ia32_xss;
		break;
3132 3133 3134 3135 3136 3137 3138 3139 3140 3141
	case MSR_K7_CLK_CTL:
		/*
		 * Provide expected ramp-up count for K7. All other
		 * are set to zero, indicating minimum divisors for
		 * every field.
		 *
		 * This prevents guest kernels on AMD host with CPU
		 * type 6, model 8 and higher from exploding due to
		 * the rdmsr failing.
		 */
3142
		msr_info->data = 0x20000000;
3143
		break;
3144
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
3145 3146
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
A
Andrey Smetanin 已提交
3147
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
3148 3149 3150
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
3151
		return kvm_hv_get_msr_common(vcpu,
3152 3153
					     msr_info->index, &msr_info->data,
					     msr_info->host_initiated);
3154
		break;
3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165
	case MSR_IA32_BBL_CR_CTL3:
		/* This legacy MSR exists but isn't fully documented in current
		 * silicon.  It is however accessed by winxp in very narrow
		 * scenarios where it sets bit #19, itself documented as
		 * a "reserved" bit.  Best effort attempt to source coherent
		 * read data here should the balance of the register be
		 * interpreted by the guest:
		 *
		 * L2 cache control register 3: 64GB range, 256KB size,
		 * enabled, latency 0x1, configured
		 */
3166
		msr_info->data = 0xbe702111;
3167
		break;
3168
	case MSR_AMD64_OSVW_ID_LENGTH:
3169
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3170
			return 1;
3171
		msr_info->data = vcpu->arch.osvw.length;
3172 3173
		break;
	case MSR_AMD64_OSVW_STATUS:
3174
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3175
			return 1;
3176
		msr_info->data = vcpu->arch.osvw.status;
3177
		break;
K
Kyle Huey 已提交
3178
	case MSR_PLATFORM_INFO:
3179 3180 3181
		if (!msr_info->host_initiated &&
		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
			return 1;
K
Kyle Huey 已提交
3182 3183 3184 3185 3186
		msr_info->data = vcpu->arch.msr_platform_info;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		msr_info->data = vcpu->arch.msr_misc_features_enables;
		break;
3187 3188 3189
	case MSR_K7_HWCR:
		msr_info->data = vcpu->arch.msr_hwcr;
		break;
3190
	default:
3191
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3192
			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
3193
		if (!ignore_msrs) {
3194 3195
			vcpu_debug_ratelimited(vcpu, "unhandled rdmsr: 0x%x\n",
					       msr_info->index);
3196 3197
			return 1;
		} else {
3198 3199 3200
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n",
					msr_info->index);
3201
			msr_info->data = 0;
3202 3203
		}
		break;
3204 3205 3206 3207 3208
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

3209 3210 3211 3212 3213 3214 3215 3216 3217 3218
/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
3219
	int i;
3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243

	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
3244
	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
3245 3246 3247 3248 3249 3250 3251
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
3252 3253 3254
	entries = memdup_user(user_msrs->entries, size);
	if (IS_ERR(entries)) {
		r = PTR_ERR(entries);
3255
		goto out;
3256
	}
3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268

	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
3269
	kfree(entries);
3270 3271 3272 3273
out:
	return r;
}

3274 3275 3276
static inline bool kvm_can_mwait_in_guest(void)
{
	return boot_cpu_has(X86_FEATURE_MWAIT) &&
3277 3278
		!boot_cpu_has_bug(X86_BUG_MONITOR) &&
		boot_cpu_has(X86_FEATURE_ARAT);
3279 3280
}

3281
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
3282
{
3283
	int r = 0;
3284 3285 3286 3287 3288 3289

	switch (ext) {
	case KVM_CAP_IRQCHIP:
	case KVM_CAP_HLT:
	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
	case KVM_CAP_SET_TSS_ADDR:
3290
	case KVM_CAP_EXT_CPUID:
B
Borislav Petkov 已提交
3291
	case KVM_CAP_EXT_EMUL_CPUID:
3292
	case KVM_CAP_CLOCKSOURCE:
S
Sheng Yang 已提交
3293
	case KVM_CAP_PIT:
3294
	case KVM_CAP_NOP_IO_DELAY:
3295
	case KVM_CAP_MP_STATE:
3296
	case KVM_CAP_SYNC_MMU:
3297
	case KVM_CAP_USER_NMI:
3298
	case KVM_CAP_REINJECT_CONTROL:
3299
	case KVM_CAP_IRQ_INJECT_STATUS:
G
Gregory Haskins 已提交
3300
	case KVM_CAP_IOEVENTFD:
3301
	case KVM_CAP_IOEVENTFD_NO_LENGTH:
3302
	case KVM_CAP_PIT2:
B
Beth Kon 已提交
3303
	case KVM_CAP_PIT_STATE2:
3304
	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
E
Ed Swierk 已提交
3305
	case KVM_CAP_XEN_HVM:
J
Jan Kiszka 已提交
3306
	case KVM_CAP_VCPU_EVENTS:
3307
	case KVM_CAP_HYPERV:
G
Gleb Natapov 已提交
3308
	case KVM_CAP_HYPERV_VAPIC:
3309
	case KVM_CAP_HYPERV_SPIN:
3310
	case KVM_CAP_HYPERV_SYNIC:
3311
	case KVM_CAP_HYPERV_SYNIC2:
3312
	case KVM_CAP_HYPERV_VP_INDEX:
3313
	case KVM_CAP_HYPERV_EVENTFD:
3314
	case KVM_CAP_HYPERV_TLBFLUSH:
3315
	case KVM_CAP_HYPERV_SEND_IPI:
3316
	case KVM_CAP_HYPERV_CPUID:
3317
	case KVM_CAP_PCI_SEGMENT:
3318
	case KVM_CAP_DEBUGREGS:
3319
	case KVM_CAP_X86_ROBUST_SINGLESTEP:
3320
	case KVM_CAP_XSAVE:
3321
	case KVM_CAP_ASYNC_PF:
3322
	case KVM_CAP_GET_TSC_KHZ:
3323
	case KVM_CAP_KVMCLOCK_CTRL:
X
Xiao Guangrong 已提交
3324
	case KVM_CAP_READONLY_MEM:
3325
	case KVM_CAP_HYPERV_TIME:
3326
	case KVM_CAP_IOAPIC_POLARITY_IGNORED:
3327
	case KVM_CAP_TSC_DEADLINE_TIMER:
3328
	case KVM_CAP_DISABLE_QUIRKS:
3329
	case KVM_CAP_SET_BOOT_CPU_ID:
3330
 	case KVM_CAP_SPLIT_IRQCHIP:
3331
	case KVM_CAP_IMMEDIATE_EXIT:
E
Eric Hankland 已提交
3332
	case KVM_CAP_PMU_EVENT_FILTER:
3333
	case KVM_CAP_GET_MSR_FEATURES:
3334
	case KVM_CAP_MSR_PLATFORM_INFO:
3335
	case KVM_CAP_EXCEPTION_PAYLOAD:
3336 3337
		r = 1;
		break;
K
Ken Hofsass 已提交
3338 3339 3340
	case KVM_CAP_SYNC_REGS:
		r = KVM_SYNC_X86_VALID_FIELDS;
		break;
3341 3342 3343
	case KVM_CAP_ADJUST_CLOCK:
		r = KVM_CLOCK_TSC_STABLE;
		break;
3344
	case KVM_CAP_X86_DISABLE_EXITS:
3345 3346
		r |=  KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE |
		      KVM_X86_DISABLE_EXITS_CSTATE;
3347 3348
		if(kvm_can_mwait_in_guest())
			r |= KVM_X86_DISABLE_EXITS_MWAIT;
3349
		break;
3350 3351 3352 3353 3354 3355 3356 3357 3358
	case KVM_CAP_X86_SMM:
		/* SMBASE is usually relocated above 1M on modern chipsets,
		 * and SMM handlers might indeed rely on 4G segment limits,
		 * so do not report SMM to be available if real mode is
		 * emulated via vm86 mode.  Still, do not go to great lengths
		 * to avoid userspace's usage of the feature, because it is a
		 * fringe case that is not enabled except via specific settings
		 * of the module parameters.
		 */
3359
		r = kvm_x86_ops->has_emulated_msr(MSR_IA32_SMBASE);
3360
		break;
3361 3362 3363
	case KVM_CAP_VAPIC:
		r = !kvm_x86_ops->cpu_has_accelerated_tpr();
		break;
3364
	case KVM_CAP_NR_VCPUS:
3365 3366 3367
		r = KVM_SOFT_MAX_VCPUS;
		break;
	case KVM_CAP_MAX_VCPUS:
3368 3369
		r = KVM_MAX_VCPUS;
		break;
3370 3371 3372
	case KVM_CAP_MAX_VCPU_ID:
		r = KVM_MAX_VCPU_ID;
		break;
3373 3374
	case KVM_CAP_PV_MMU:	/* obsolete */
		r = 0;
3375
		break;
H
Huang Ying 已提交
3376 3377 3378
	case KVM_CAP_MCE:
		r = KVM_MAX_MCE_BANKS;
		break;
3379
	case KVM_CAP_XCRS:
3380
		r = boot_cpu_has(X86_FEATURE_XSAVE);
3381
		break;
3382 3383 3384
	case KVM_CAP_TSC_CONTROL:
		r = kvm_has_tsc_control;
		break;
3385 3386 3387
	case KVM_CAP_X2APIC_API:
		r = KVM_X2APIC_API_VALID_FLAGS;
		break;
3388 3389
	case KVM_CAP_NESTED_STATE:
		r = kvm_x86_ops->get_nested_state ?
3390
			kvm_x86_ops->get_nested_state(NULL, NULL, 0) : 0;
3391
		break;
3392
	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
3393 3394 3395 3396
		r = kvm_x86_ops->enable_direct_tlbflush != NULL;
		break;
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
		r = kvm_x86_ops->nested_enable_evmcs != NULL;
3397
		break;
3398 3399 3400 3401 3402 3403 3404
	default:
		break;
	}
	return r;

}

3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417
long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {
	case KVM_GET_MSR_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
3418
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
3419 3420
			goto out;
		n = msr_list.nmsrs;
3421
		msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
3422
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
3423 3424
			goto out;
		r = -E2BIG;
J
Jan Kiszka 已提交
3425
		if (n < msr_list.nmsrs)
3426 3427 3428 3429 3430
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
				 num_msrs_to_save * sizeof(u32)))
			goto out;
J
Jan Kiszka 已提交
3431
		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
3432
				 &emulated_msrs,
3433
				 num_emulated_msrs * sizeof(u32)))
3434 3435 3436 3437
			goto out;
		r = 0;
		break;
	}
B
Borislav Petkov 已提交
3438 3439
	case KVM_GET_SUPPORTED_CPUID:
	case KVM_GET_EMULATED_CPUID: {
3440 3441 3442 3443
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
3444
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
3445
			goto out;
B
Borislav Petkov 已提交
3446 3447 3448

		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
					    ioctl);
3449 3450 3451 3452
		if (r)
			goto out;

		r = -EFAULT;
3453
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
3454 3455 3456 3457
			goto out;
		r = 0;
		break;
	}
H
Huang Ying 已提交
3458 3459
	case KVM_X86_GET_MCE_CAP_SUPPORTED: {
		r = -EFAULT;
3460 3461
		if (copy_to_user(argp, &kvm_mce_cap_supported,
				 sizeof(kvm_mce_cap_supported)))
H
Huang Ying 已提交
3462 3463 3464
			goto out;
		r = 0;
		break;
3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489
	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned int n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
			goto out;
		n = msr_list.nmsrs;
		msr_list.nmsrs = num_msr_based_features;
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
			goto out;
		r = -E2BIG;
		if (n < msr_list.nmsrs)
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msr_based_features,
				 num_msr_based_features * sizeof(u32)))
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_MSRS:
		r = msr_io(NULL, argp, do_get_msr_feature, 1);
		break;
H
Huang Ying 已提交
3490
	}
3491 3492 3493 3494 3495 3496 3497
	default:
		r = -EINVAL;
	}
out:
	return r;
}

3498 3499 3500 3501 3502 3503 3504
static void wbinvd_ipi(void *garbage)
{
	wbinvd();
}

static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
3505
	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
3506 3507
}

3508 3509
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
3510 3511 3512 3513 3514 3515 3516 3517 3518
	/* Address WBINVD may be executed by guest */
	if (need_emulate_wbinvd(vcpu)) {
		if (kvm_x86_ops->has_wbinvd_exit())
			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
			smp_call_function_single(vcpu->cpu,
					wbinvd_ipi, NULL, 1);
	}

3519
	kvm_x86_ops->vcpu_load(vcpu, cpu);
3520

3521 3522 3523 3524
	fpregs_assert_state_consistent();
	if (test_thread_flag(TIF_NEED_FPU_LOAD))
		switch_fpu_return();

3525 3526 3527 3528
	/* Apply any externally detected TSC adjustments (due to suspend) */
	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
		vcpu->arch.tsc_offset_adjustment = 0;
3529
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3530
	}
3531

3532
	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
3533
		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
3534
				rdtsc() - vcpu->arch.last_host_tsc;
Z
Zachary Amsden 已提交
3535 3536
		if (tsc_delta < 0)
			mark_tsc_unstable("KVM discovered backwards TSC");
3537

3538
		if (kvm_check_tsc_unstable()) {
3539
			u64 offset = kvm_compute_tsc_offset(vcpu,
3540
						vcpu->arch.last_guest_tsc);
3541
			kvm_vcpu_write_tsc_offset(vcpu, offset);
Z
Zachary Amsden 已提交
3542 3543
			vcpu->arch.tsc_catchup = 1;
		}
3544 3545 3546 3547

		if (kvm_lapic_hv_timer_in_use(vcpu))
			kvm_lapic_restart_hv_timer(vcpu);

3548 3549 3550 3551 3552
		/*
		 * On a host with synchronized TSC, there is no need to update
		 * kvmclock on vcpu->cpu migration
		 */
		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
3553
			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
Z
Zachary Amsden 已提交
3554
		if (vcpu->cpu != cpu)
3555
			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
Z
Zachary Amsden 已提交
3556
		vcpu->cpu = cpu;
Z
Zachary Amsden 已提交
3557
	}
G
Glauber Costa 已提交
3558 3559

	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3560 3561
}

3562 3563 3564 3565 3566
static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

W
Wanpeng Li 已提交
3567
	vcpu->arch.st.steal.preempted = KVM_VCPU_PREEMPTED;
3568

3569
	kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.st.stime,
3570 3571 3572 3573 3574
			&vcpu->arch.st.steal.preempted,
			offsetof(struct kvm_steal_time, preempted),
			sizeof(vcpu->arch.st.steal.preempted));
}

3575 3576
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
3577
	int idx;
3578 3579 3580 3581

	if (vcpu->preempted)
		vcpu->arch.preempted_in_kernel = !kvm_x86_ops->get_cpl(vcpu);

3582 3583 3584 3585 3586 3587 3588 3589 3590
	/*
	 * Disable page faults because we're in atomic context here.
	 * kvm_write_guest_offset_cached() would call might_fault()
	 * that relies on pagefault_disable() to tell if there's a
	 * bug. NOTE: the write to guest memory may not go through if
	 * during postcopy live migration or if there's heavy guest
	 * paging.
	 */
	pagefault_disable();
3591 3592 3593 3594 3595
	/*
	 * kvm_memslots() will be called by
	 * kvm_write_guest_offset_cached() so take the srcu lock.
	 */
	idx = srcu_read_lock(&vcpu->kvm->srcu);
3596
	kvm_steal_time_set_preempted(vcpu);
3597
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
3598
	pagefault_enable();
3599
	kvm_x86_ops->vcpu_put(vcpu);
3600
	vcpu->arch.last_host_tsc = rdtsc();
3601
	/*
3602 3603 3604
	 * If userspace has set any breakpoints or watchpoints, dr6 is restored
	 * on every vmexit, but if not, we might have a stale dr6 from the
	 * guest. do_debug expects dr6 to be cleared after it runs, do the same.
3605
	 */
3606
	set_debugreg(0, 6);
3607 3608 3609 3610 3611
}

static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3612
	if (vcpu->arch.apicv_active)
3613 3614
		kvm_x86_ops->sync_pir_to_irr(vcpu);

3615
	return kvm_apic_get_state(vcpu, s);
3616 3617 3618 3619 3620
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3621 3622 3623 3624 3625
	int r;

	r = kvm_apic_set_state(vcpu, s);
	if (r)
		return r;
3626
	update_cr8_intercept(vcpu);
3627 3628 3629 3630

	return 0;
}

3631 3632 3633 3634 3635 3636
static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
{
	return (!lapic_in_kernel(vcpu) ||
		kvm_apic_accept_pic_intr(vcpu));
}

3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650
/*
 * if userspace requested an interrupt window, check that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
{
	return kvm_arch_interrupt_allowed(vcpu) &&
		!kvm_cpu_has_interrupt(vcpu) &&
		!kvm_event_needs_reinjection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);
}

3651 3652 3653
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
{
3654
	if (irq->irq >= KVM_NR_INTERRUPTS)
3655
		return -EINVAL;
3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667

	if (!irqchip_in_kernel(vcpu->kvm)) {
		kvm_queue_interrupt(vcpu, irq->irq, false);
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		return 0;
	}

	/*
	 * With in-kernel LAPIC, we only use this to inject EXTINT, so
	 * fail for in-kernel 8259.
	 */
	if (pic_in_kernel(vcpu->kvm))
3668 3669
		return -ENXIO;

3670 3671
	if (vcpu->arch.pending_external_vector != -1)
		return -EEXIST;
3672

3673
	vcpu->arch.pending_external_vector = irq->irq;
3674
	kvm_make_request(KVM_REQ_EVENT, vcpu);
3675 3676 3677
	return 0;
}

3678 3679 3680 3681 3682 3683 3684
static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
{
	kvm_inject_nmi(vcpu);

	return 0;
}

3685 3686
static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
{
P
Paolo Bonzini 已提交
3687 3688
	kvm_make_request(KVM_REQ_SMI, vcpu);

3689 3690 3691
	return 0;
}

3692 3693 3694 3695 3696 3697 3698 3699 3700
static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
					   struct kvm_tpr_access_ctl *tac)
{
	if (tac->flags)
		return -EINVAL;
	vcpu->arch.tpr_access_reporting = !!tac->enabled;
	return 0;
}

H
Huang Ying 已提交
3701 3702 3703 3704 3705 3706 3707
static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
					u64 mcg_cap)
{
	int r;
	unsigned bank_num = mcg_cap & 0xff, bank;

	r = -EINVAL;
3708
	if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
H
Huang Ying 已提交
3709
		goto out;
3710
	if (mcg_cap & ~(kvm_mce_cap_supported | 0xff | 0xff0000))
H
Huang Ying 已提交
3711 3712 3713 3714 3715 3716 3717 3718 3719
		goto out;
	r = 0;
	vcpu->arch.mcg_cap = mcg_cap;
	/* Init IA32_MCG_CTL to all 1s */
	if (mcg_cap & MCG_CTL_P)
		vcpu->arch.mcg_ctl = ~(u64)0;
	/* Init IA32_MCi_CTL to all 1s */
	for (bank = 0; bank < bank_num; bank++)
		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
3720

3721
	kvm_x86_ops->setup_mce(vcpu);
H
Huang Ying 已提交
3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750
out:
	return r;
}

static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
				      struct kvm_x86_mce *mce)
{
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
	u64 *banks = vcpu->arch.mce_banks;

	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
		return -EINVAL;
	/*
	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
	 * reporting is disabled
	 */
	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
	    vcpu->arch.mcg_ctl != ~(u64)0)
		return 0;
	banks += 4 * mce->bank;
	/*
	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
	 * reporting is disabled for the bank
	 */
	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
		return 0;
	if (mce->status & MCI_STATUS_UC) {
		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
3751
		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
3752
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
H
Huang Ying 已提交
3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773
			return 0;
		}
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		vcpu->arch.mcg_status = mce->mcg_status;
		banks[1] = mce->status;
		kvm_queue_exception(vcpu, MC_VECTOR);
	} else if (!(banks[1] & MCI_STATUS_VAL)
		   || !(banks[1] & MCI_STATUS_UC)) {
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		banks[1] = mce->status;
	} else
		banks[1] |= MCI_STATUS_OVER;
	return 0;
}

J
Jan Kiszka 已提交
3774 3775 3776
static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
					       struct kvm_vcpu_events *events)
{
A
Avi Kivity 已提交
3777
	process_nmi(vcpu);
3778

3779
	/*
3780 3781 3782 3783
	 * The API doesn't provide the instruction length for software
	 * exceptions, so don't report them. As long as the guest RIP
	 * isn't advanced, we should expect to encounter the exception
	 * again.
3784
	 */
3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799
	if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {
		events->exception.injected = 0;
		events->exception.pending = 0;
	} else {
		events->exception.injected = vcpu->arch.exception.injected;
		events->exception.pending = vcpu->arch.exception.pending;
		/*
		 * For ABI compatibility, deliberately conflate
		 * pending and injected exceptions when
		 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
		 */
		if (!vcpu->kvm->arch.exception_payload_enabled)
			events->exception.injected |=
				vcpu->arch.exception.pending;
	}
J
Jan Kiszka 已提交
3800 3801 3802
	events->exception.nr = vcpu->arch.exception.nr;
	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
	events->exception.error_code = vcpu->arch.exception.error_code;
3803 3804
	events->exception_has_payload = vcpu->arch.exception.has_payload;
	events->exception_payload = vcpu->arch.exception.payload;
J
Jan Kiszka 已提交
3805

3806
	events->interrupt.injected =
3807
		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
J
Jan Kiszka 已提交
3808
	events->interrupt.nr = vcpu->arch.interrupt.nr;
3809
	events->interrupt.soft = 0;
3810
	events->interrupt.shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
J
Jan Kiszka 已提交
3811 3812

	events->nmi.injected = vcpu->arch.nmi_injected;
A
Avi Kivity 已提交
3813
	events->nmi.pending = vcpu->arch.nmi_pending != 0;
J
Jan Kiszka 已提交
3814
	events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
3815
	events->nmi.pad = 0;
J
Jan Kiszka 已提交
3816

3817
	events->sipi_vector = 0; /* never valid when reporting to user space */
J
Jan Kiszka 已提交
3818

3819 3820 3821 3822 3823 3824
	events->smi.smm = is_smm(vcpu);
	events->smi.pending = vcpu->arch.smi_pending;
	events->smi.smm_inside_nmi =
		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
	events->smi.latched_init = kvm_lapic_latched_init(vcpu);

3825
	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
3826 3827
			 | KVM_VCPUEVENT_VALID_SHADOW
			 | KVM_VCPUEVENT_VALID_SMM);
3828 3829 3830
	if (vcpu->kvm->arch.exception_payload_enabled)
		events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;

3831
	memset(&events->reserved, 0, sizeof(events->reserved));
J
Jan Kiszka 已提交
3832 3833
}

3834
static void kvm_smm_changed(struct kvm_vcpu *vcpu);
3835

J
Jan Kiszka 已提交
3836 3837 3838
static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
					      struct kvm_vcpu_events *events)
{
3839
	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
3840
			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
3841
			      | KVM_VCPUEVENT_VALID_SHADOW
3842 3843
			      | KVM_VCPUEVENT_VALID_SMM
			      | KVM_VCPUEVENT_VALID_PAYLOAD))
J
Jan Kiszka 已提交
3844 3845
		return -EINVAL;

3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859
	if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
		if (!vcpu->kvm->arch.exception_payload_enabled)
			return -EINVAL;
		if (events->exception.pending)
			events->exception.injected = 0;
		else
			events->exception_has_payload = 0;
	} else {
		events->exception.pending = 0;
		events->exception_has_payload = 0;
	}

	if ((events->exception.injected || events->exception.pending) &&
	    (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
3860 3861
		return -EINVAL;

3862 3863 3864 3865 3866 3867
	/* INITs are latched while in SMM */
	if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
	    (events->smi.smm || events->smi.pending) &&
	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
		return -EINVAL;

A
Avi Kivity 已提交
3868
	process_nmi(vcpu);
3869 3870
	vcpu->arch.exception.injected = events->exception.injected;
	vcpu->arch.exception.pending = events->exception.pending;
J
Jan Kiszka 已提交
3871 3872 3873
	vcpu->arch.exception.nr = events->exception.nr;
	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
	vcpu->arch.exception.error_code = events->exception.error_code;
3874 3875
	vcpu->arch.exception.has_payload = events->exception_has_payload;
	vcpu->arch.exception.payload = events->exception_payload;
J
Jan Kiszka 已提交
3876

3877
	vcpu->arch.interrupt.injected = events->interrupt.injected;
J
Jan Kiszka 已提交
3878 3879
	vcpu->arch.interrupt.nr = events->interrupt.nr;
	vcpu->arch.interrupt.soft = events->interrupt.soft;
3880 3881 3882
	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
		kvm_x86_ops->set_interrupt_shadow(vcpu,
						  events->interrupt.shadow);
J
Jan Kiszka 已提交
3883 3884

	vcpu->arch.nmi_injected = events->nmi.injected;
3885 3886
	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
		vcpu->arch.nmi_pending = events->nmi.pending;
J
Jan Kiszka 已提交
3887 3888
	kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);

3889
	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
3890
	    lapic_in_kernel(vcpu))
3891
		vcpu->arch.apic->sipi_vector = events->sipi_vector;
J
Jan Kiszka 已提交
3892

3893
	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
3894 3895 3896 3897 3898 3899 3900
		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
			if (events->smi.smm)
				vcpu->arch.hflags |= HF_SMM_MASK;
			else
				vcpu->arch.hflags &= ~HF_SMM_MASK;
			kvm_smm_changed(vcpu);
		}
3901

3902
		vcpu->arch.smi_pending = events->smi.pending;
3903 3904 3905 3906

		if (events->smi.smm) {
			if (events->smi.smm_inside_nmi)
				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
3907
			else
3908
				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
3909 3910 3911 3912 3913 3914 3915
		}

		if (lapic_in_kernel(vcpu)) {
			if (events->smi.latched_init)
				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
			else
				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
3916 3917 3918
		}
	}

3919 3920
	kvm_make_request(KVM_REQ_EVENT, vcpu);

J
Jan Kiszka 已提交
3921 3922 3923
	return 0;
}

3924 3925 3926
static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
					     struct kvm_debugregs *dbgregs)
{
J
Jan Kiszka 已提交
3927 3928
	unsigned long val;

3929
	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
3930
	kvm_get_dr(vcpu, 6, &val);
J
Jan Kiszka 已提交
3931
	dbgregs->dr6 = val;
3932 3933
	dbgregs->dr7 = vcpu->arch.dr7;
	dbgregs->flags = 0;
3934
	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
3935 3936 3937 3938 3939 3940 3941 3942
}

static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
					    struct kvm_debugregs *dbgregs)
{
	if (dbgregs->flags)
		return -EINVAL;

3943 3944 3945 3946 3947
	if (dbgregs->dr6 & ~0xffffffffull)
		return -EINVAL;
	if (dbgregs->dr7 & ~0xffffffffull)
		return -EINVAL;

3948
	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
3949
	kvm_update_dr0123(vcpu);
3950
	vcpu->arch.dr6 = dbgregs->dr6;
J
Jan Kiszka 已提交
3951
	kvm_update_dr6(vcpu);
3952
	vcpu->arch.dr7 = dbgregs->dr7;
3953
	kvm_update_dr7(vcpu);
3954 3955 3956 3957

	return 0;
}

3958 3959 3960 3961
#define XSTATE_COMPACTION_ENABLED (1ULL << 63)

static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu)
{
3962
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
3963
	u64 xstate_bv = xsave->header.xfeatures;
3964 3965 3966 3967 3968 3969 3970 3971 3972
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(dest, xsave, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV */
3973
	xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;
3974 3975 3976 3977 3978 3979
	*(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv;

	/*
	 * Copy each region from the possibly compacted offset to the
	 * non-compacted offset.
	 */
D
Dave Hansen 已提交
3980
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
3981
	while (valid) {
3982 3983 3984
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *src = get_xsave_addr(xsave, xfeature_nr);
3985 3986 3987

		if (src) {
			u32 size, offset, ecx, edx;
3988
			cpuid_count(XSTATE_CPUID, xfeature_nr,
3989
				    &size, &offset, &ecx, &edx);
3990
			if (xfeature_nr == XFEATURE_PKRU)
3991 3992 3993 3994 3995
				memcpy(dest + offset, &vcpu->arch.pkru,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest + offset, src, size);

3996 3997
		}

3998
		valid -= xfeature_mask;
3999 4000 4001 4002 4003
	}
}

static void load_xsave(struct kvm_vcpu *vcpu, u8 *src)
{
4004
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
4005 4006 4007 4008 4009 4010 4011 4012 4013 4014
	u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET);
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(xsave, src, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV and possibly XCOMP_BV.  */
4015
	xsave->header.xfeatures = xstate_bv;
4016
	if (boot_cpu_has(X86_FEATURE_XSAVES))
4017
		xsave->header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED;
4018 4019 4020 4021 4022

	/*
	 * Copy each region from the non-compacted offset to the
	 * possibly compacted offset.
	 */
D
Dave Hansen 已提交
4023
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
4024
	while (valid) {
4025 4026 4027
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *dest = get_xsave_addr(xsave, xfeature_nr);
4028 4029 4030

		if (dest) {
			u32 size, offset, ecx, edx;
4031
			cpuid_count(XSTATE_CPUID, xfeature_nr,
4032
				    &size, &offset, &ecx, &edx);
4033
			if (xfeature_nr == XFEATURE_PKRU)
4034 4035 4036 4037
				memcpy(&vcpu->arch.pkru, src + offset,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest, src + offset, size);
4038
		}
4039

4040
		valid -= xfeature_mask;
4041 4042 4043
	}
}

4044 4045 4046
static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
					 struct kvm_xsave *guest_xsave)
{
4047
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
4048 4049
		memset(guest_xsave, 0, sizeof(struct kvm_xsave));
		fill_xsave((u8 *) guest_xsave->region, vcpu);
4050
	} else {
4051
		memcpy(guest_xsave->region,
4052
			&vcpu->arch.guest_fpu->state.fxsave,
4053
			sizeof(struct fxregs_state));
4054
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
D
Dave Hansen 已提交
4055
			XFEATURE_MASK_FPSSE;
4056 4057 4058
	}
}

4059 4060
#define XSAVE_MXCSR_OFFSET 24

4061 4062 4063 4064 4065
static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
					struct kvm_xsave *guest_xsave)
{
	u64 xstate_bv =
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
4066
	u32 mxcsr = *(u32 *)&guest_xsave->region[XSAVE_MXCSR_OFFSET / sizeof(u32)];
4067

4068
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
4069 4070 4071 4072 4073
		/*
		 * Here we allow setting states that are not present in
		 * CPUID leaf 0xD, index 0, EDX:EAX.  This is for compatibility
		 * with old userspace.
		 */
4074 4075
		if (xstate_bv & ~kvm_supported_xcr0() ||
			mxcsr & ~mxcsr_feature_mask)
4076
			return -EINVAL;
4077
		load_xsave(vcpu, (u8 *)guest_xsave->region);
4078
	} else {
4079 4080
		if (xstate_bv & ~XFEATURE_MASK_FPSSE ||
			mxcsr & ~mxcsr_feature_mask)
4081
			return -EINVAL;
4082
		memcpy(&vcpu->arch.guest_fpu->state.fxsave,
4083
			guest_xsave->region, sizeof(struct fxregs_state));
4084 4085 4086 4087 4088 4089 4090
	}
	return 0;
}

static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
					struct kvm_xcrs *guest_xcrs)
{
4091
	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106
		guest_xcrs->nr_xcrs = 0;
		return;
	}

	guest_xcrs->nr_xcrs = 1;
	guest_xcrs->flags = 0;
	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
}

static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
				       struct kvm_xcrs *guest_xcrs)
{
	int i, r = 0;

4107
	if (!boot_cpu_has(X86_FEATURE_XSAVE))
4108 4109 4110 4111 4112 4113 4114
		return -EINVAL;

	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
		return -EINVAL;

	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
		/* Only support XCR0 currently */
P
Paolo Bonzini 已提交
4115
		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
4116
			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
P
Paolo Bonzini 已提交
4117
				guest_xcrs->xcrs[i].value);
4118 4119 4120 4121 4122 4123 4124
			break;
		}
	if (r)
		r = -EINVAL;
	return r;
}

4125 4126 4127 4128 4129 4130 4131 4132
/*
 * kvm_set_guest_paused() indicates to the guest kernel that it has been
 * stopped by the hypervisor.  This function will be called from the host only.
 * EINVAL is returned when the host attempts to set the flag for a guest that
 * does not support pv clocks.
 */
static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
{
4133
	if (!vcpu->arch.pv_time_enabled)
4134
		return -EINVAL;
4135
	vcpu->arch.pvclock_set_guest_stopped_request = true;
4136 4137 4138 4139
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	return 0;
}

4140 4141 4142
static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
				     struct kvm_enable_cap *cap)
{
4143 4144 4145 4146
	int r;
	uint16_t vmcs_version;
	void __user *user_ptr;

4147 4148 4149 4150
	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
4151 4152 4153
	case KVM_CAP_HYPERV_SYNIC2:
		if (cap->args[0])
			return -EINVAL;
4154 4155
		/* fall through */

4156
	case KVM_CAP_HYPERV_SYNIC:
4157 4158
		if (!irqchip_in_kernel(vcpu->kvm))
			return -EINVAL;
4159 4160
		return kvm_hv_activate_synic(vcpu, cap->cap ==
					     KVM_CAP_HYPERV_SYNIC2);
4161
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
4162 4163
		if (!kvm_x86_ops->nested_enable_evmcs)
			return -ENOTTY;
4164 4165 4166 4167 4168 4169 4170 4171
		r = kvm_x86_ops->nested_enable_evmcs(vcpu, &vmcs_version);
		if (!r) {
			user_ptr = (void __user *)(uintptr_t)cap->args[0];
			if (copy_to_user(user_ptr, &vmcs_version,
					 sizeof(vmcs_version)))
				r = -EFAULT;
		}
		return r;
4172 4173 4174 4175 4176
	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
		if (!kvm_x86_ops->enable_direct_tlbflush)
			return -ENOTTY;

		return kvm_x86_ops->enable_direct_tlbflush(vcpu);
4177

4178 4179 4180 4181 4182
	default:
		return -EINVAL;
	}
}

4183 4184 4185 4186 4187 4188
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r;
4189 4190 4191 4192 4193 4194 4195
	union {
		struct kvm_lapic_state *lapic;
		struct kvm_xsave *xsave;
		struct kvm_xcrs *xcrs;
		void *buffer;
	} u;

4196 4197
	vcpu_load(vcpu);

4198
	u.buffer = NULL;
4199 4200
	switch (ioctl) {
	case KVM_GET_LAPIC: {
4201
		r = -EINVAL;
4202
		if (!lapic_in_kernel(vcpu))
4203
			goto out;
4204 4205
		u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
				GFP_KERNEL_ACCOUNT);
4206

4207
		r = -ENOMEM;
4208
		if (!u.lapic)
4209
			goto out;
4210
		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
4211 4212 4213
		if (r)
			goto out;
		r = -EFAULT;
4214
		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
4215 4216 4217 4218 4219
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
4220
		r = -EINVAL;
4221
		if (!lapic_in_kernel(vcpu))
4222
			goto out;
4223
		u.lapic = memdup_user(argp, sizeof(*u.lapic));
4224 4225 4226 4227
		if (IS_ERR(u.lapic)) {
			r = PTR_ERR(u.lapic);
			goto out_nofree;
		}
4228

4229
		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
4230 4231
		break;
	}
4232 4233 4234 4235
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
4236
		if (copy_from_user(&irq, argp, sizeof(irq)))
4237 4238 4239 4240
			goto out;
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
		break;
	}
4241 4242 4243 4244
	case KVM_NMI: {
		r = kvm_vcpu_ioctl_nmi(vcpu);
		break;
	}
4245 4246 4247 4248
	case KVM_SMI: {
		r = kvm_vcpu_ioctl_smi(vcpu);
		break;
	}
4249 4250 4251 4252 4253
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
4254
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4255 4256 4257 4258
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		break;
	}
4259 4260 4261 4262 4263
	case KVM_SET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4264
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4265 4266
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
4267
					      cpuid_arg->entries);
4268 4269 4270 4271 4272 4273 4274
		break;
	}
	case KVM_GET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4275
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4276 4277
			goto out;
		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
4278
					      cpuid_arg->entries);
4279 4280 4281
		if (r)
			goto out;
		r = -EFAULT;
4282
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4283 4284 4285 4286
			goto out;
		r = 0;
		break;
	}
4287 4288
	case KVM_GET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4289
		r = msr_io(vcpu, argp, do_get_msr, 1);
4290
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4291
		break;
4292 4293 4294
	}
	case KVM_SET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4295
		r = msr_io(vcpu, argp, do_set_msr, 0);
4296
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4297
		break;
4298
	}
4299 4300 4301 4302
	case KVM_TPR_ACCESS_REPORTING: {
		struct kvm_tpr_access_ctl tac;

		r = -EFAULT;
4303
		if (copy_from_user(&tac, argp, sizeof(tac)))
4304 4305 4306 4307 4308
			goto out;
		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
		if (r)
			goto out;
		r = -EFAULT;
4309
		if (copy_to_user(argp, &tac, sizeof(tac)))
4310 4311 4312 4313
			goto out;
		r = 0;
		break;
	};
A
Avi Kivity 已提交
4314 4315
	case KVM_SET_VAPIC_ADDR: {
		struct kvm_vapic_addr va;
4316
		int idx;
A
Avi Kivity 已提交
4317 4318

		r = -EINVAL;
4319
		if (!lapic_in_kernel(vcpu))
A
Avi Kivity 已提交
4320 4321
			goto out;
		r = -EFAULT;
4322
		if (copy_from_user(&va, argp, sizeof(va)))
A
Avi Kivity 已提交
4323
			goto out;
4324
		idx = srcu_read_lock(&vcpu->kvm->srcu);
4325
		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
4326
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
A
Avi Kivity 已提交
4327 4328
		break;
	}
H
Huang Ying 已提交
4329 4330 4331 4332
	case KVM_X86_SETUP_MCE: {
		u64 mcg_cap;

		r = -EFAULT;
4333
		if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
H
Huang Ying 已提交
4334 4335 4336 4337 4338 4339 4340 4341
			goto out;
		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
		break;
	}
	case KVM_X86_SET_MCE: {
		struct kvm_x86_mce mce;

		r = -EFAULT;
4342
		if (copy_from_user(&mce, argp, sizeof(mce)))
H
Huang Ying 已提交
4343 4344 4345 4346
			goto out;
		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
		break;
	}
J
Jan Kiszka 已提交
4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);

		r = -EFAULT;
		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		r = -EFAULT;
		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
			break;

		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
		break;
	}
4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390
	case KVM_GET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);

		r = -EFAULT;
		if (copy_to_user(argp, &dbgregs,
				 sizeof(struct kvm_debugregs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		r = -EFAULT;
		if (copy_from_user(&dbgregs, argp,
				   sizeof(struct kvm_debugregs)))
			break;

		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
		break;
	}
4391
	case KVM_GET_XSAVE: {
4392
		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
4393
		r = -ENOMEM;
4394
		if (!u.xsave)
4395 4396
			break;

4397
		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
4398 4399

		r = -EFAULT;
4400
		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
4401 4402 4403 4404 4405
			break;
		r = 0;
		break;
	}
	case KVM_SET_XSAVE: {
4406
		u.xsave = memdup_user(argp, sizeof(*u.xsave));
4407 4408 4409 4410
		if (IS_ERR(u.xsave)) {
			r = PTR_ERR(u.xsave);
			goto out_nofree;
		}
4411

4412
		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
4413 4414 4415
		break;
	}
	case KVM_GET_XCRS: {
4416
		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
4417
		r = -ENOMEM;
4418
		if (!u.xcrs)
4419 4420
			break;

4421
		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
4422 4423

		r = -EFAULT;
4424
		if (copy_to_user(argp, u.xcrs,
4425 4426 4427 4428 4429 4430
				 sizeof(struct kvm_xcrs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XCRS: {
4431
		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
4432 4433 4434 4435
		if (IS_ERR(u.xcrs)) {
			r = PTR_ERR(u.xcrs);
			goto out_nofree;
		}
4436

4437
		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
4438 4439
		break;
	}
4440 4441 4442 4443 4444 4445 4446 4447 4448
	case KVM_SET_TSC_KHZ: {
		u32 user_tsc_khz;

		r = -EINVAL;
		user_tsc_khz = (u32)arg;

		if (user_tsc_khz >= kvm_max_guest_tsc_khz)
			goto out;

4449 4450 4451
		if (user_tsc_khz == 0)
			user_tsc_khz = tsc_khz;

4452 4453
		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
			r = 0;
4454 4455 4456 4457

		goto out;
	}
	case KVM_GET_TSC_KHZ: {
4458
		r = vcpu->arch.virtual_tsc_khz;
4459 4460
		goto out;
	}
4461 4462 4463 4464
	case KVM_KVMCLOCK_CTRL: {
		r = kvm_set_guest_paused(vcpu);
		goto out;
	}
4465 4466 4467 4468 4469 4470 4471 4472 4473
	case KVM_ENABLE_CAP: {
		struct kvm_enable_cap cap;

		r = -EFAULT;
		if (copy_from_user(&cap, argp, sizeof(cap)))
			goto out;
		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
		break;
	}
4474 4475 4476 4477 4478 4479 4480 4481 4482
	case KVM_GET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		u32 user_data_size;

		r = -EINVAL;
		if (!kvm_x86_ops->get_nested_state)
			break;

		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
4483
		r = -EFAULT;
4484
		if (get_user(user_data_size, &user_kvm_nested_state->size))
4485
			break;
4486 4487 4488 4489

		r = kvm_x86_ops->get_nested_state(vcpu, user_kvm_nested_state,
						  user_data_size);
		if (r < 0)
4490
			break;
4491 4492 4493

		if (r > user_data_size) {
			if (put_user(r, &user_kvm_nested_state->size))
4494 4495 4496 4497
				r = -EFAULT;
			else
				r = -E2BIG;
			break;
4498
		}
4499

4500 4501 4502 4503 4504 4505
		r = 0;
		break;
	}
	case KVM_SET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		struct kvm_nested_state kvm_state;
4506
		int idx;
4507 4508 4509 4510 4511

		r = -EINVAL;
		if (!kvm_x86_ops->set_nested_state)
			break;

4512
		r = -EFAULT;
4513
		if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
4514
			break;
4515

4516
		r = -EINVAL;
4517
		if (kvm_state.size < sizeof(kvm_state))
4518
			break;
4519 4520

		if (kvm_state.flags &
4521 4522
		    ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
		      | KVM_STATE_NESTED_EVMCS))
4523
			break;
4524 4525

		/* nested_run_pending implies guest_mode.  */
4526 4527
		if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
		    && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
4528
			break;
4529

4530
		idx = srcu_read_lock(&vcpu->kvm->srcu);
4531
		r = kvm_x86_ops->set_nested_state(vcpu, user_kvm_nested_state, &kvm_state);
4532
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4533 4534
		break;
	}
4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553
	case KVM_GET_SUPPORTED_HV_CPUID: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
			goto out;

		r = kvm_vcpu_ioctl_get_hv_cpuid(vcpu, &cpuid,
						cpuid_arg->entries);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
			goto out;
		r = 0;
		break;
	}
4554 4555 4556 4557
	default:
		r = -EINVAL;
	}
out:
4558
	kfree(u.buffer);
4559 4560
out_nofree:
	vcpu_put(vcpu);
4561 4562 4563
	return r;
}

4564
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4565 4566 4567 4568
{
	return VM_FAULT_SIGBUS;
}

4569 4570 4571 4572 4573
static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
{
	int ret;

	if (addr > (unsigned int)(-3 * PAGE_SIZE))
4574
		return -EINVAL;
4575 4576 4577 4578
	ret = kvm_x86_ops->set_tss_addr(kvm, addr);
	return ret;
}

4579 4580 4581
static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
					      u64 ident_addr)
{
4582
	return kvm_x86_ops->set_identity_map_addr(kvm, ident_addr);
4583 4584
}

4585
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
4586
					 unsigned long kvm_nr_mmu_pages)
4587 4588 4589 4590
{
	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
		return -EINVAL;

4591
	mutex_lock(&kvm->slots_lock);
4592 4593

	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
4594
	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
4595

4596
	mutex_unlock(&kvm->slots_lock);
4597 4598 4599
	return 0;
}

4600
static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
4601
{
4602
	return kvm->arch.n_max_mmu_pages;
4603 4604 4605 4606
}

static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4607
	struct kvm_pic *pic = kvm->arch.vpic;
4608 4609 4610 4611 4612
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4613
		memcpy(&chip->chip.pic, &pic->pics[0],
4614 4615 4616
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4617
		memcpy(&chip->chip.pic, &pic->pics[1],
4618 4619 4620
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_IOAPIC:
4621
		kvm_get_ioapic(kvm, &chip->chip.ioapic);
4622 4623 4624 4625 4626 4627 4628 4629 4630 4631
		break;
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4632
	struct kvm_pic *pic = kvm->arch.vpic;
4633 4634 4635 4636 4637
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4638 4639
		spin_lock(&pic->lock);
		memcpy(&pic->pics[0], &chip->chip.pic,
4640
			sizeof(struct kvm_pic_state));
4641
		spin_unlock(&pic->lock);
4642 4643
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4644 4645
		spin_lock(&pic->lock);
		memcpy(&pic->pics[1], &chip->chip.pic,
4646
			sizeof(struct kvm_pic_state));
4647
		spin_unlock(&pic->lock);
4648 4649
		break;
	case KVM_IRQCHIP_IOAPIC:
4650
		kvm_set_ioapic(kvm, &chip->chip.ioapic);
4651 4652 4653 4654 4655
		break;
	default:
		r = -EINVAL;
		break;
	}
4656
	kvm_pic_update_irq(pic);
4657 4658 4659
	return r;
}

4660 4661
static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4662 4663 4664 4665 4666 4667 4668
	struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;

	BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));

	mutex_lock(&kps->lock);
	memcpy(ps, &kps->channels, sizeof(*ps));
	mutex_unlock(&kps->lock);
4669
	return 0;
4670 4671 4672 4673
}

static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4674
	int i;
4675 4676 4677
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
4678
	memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
4679
	for (i = 0; i < 3; i++)
4680 4681
		kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
	mutex_unlock(&pit->pit_state.lock);
4682
	return 0;
B
Beth Kon 已提交
4683 4684 4685 4686 4687 4688 4689 4690 4691
}

static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
		sizeof(ps->channels));
	ps->flags = kvm->arch.vpit->pit_state.flags;
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
4692
	memset(&ps->reserved, 0, sizeof(ps->reserved));
4693
	return 0;
B
Beth Kon 已提交
4694 4695 4696 4697
}

static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
4698
	int start = 0;
4699
	int i;
B
Beth Kon 已提交
4700
	u32 prev_legacy, cur_legacy;
4701 4702 4703 4704
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
	prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
B
Beth Kon 已提交
4705 4706 4707
	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
	if (!prev_legacy && cur_legacy)
		start = 1;
4708 4709 4710
	memcpy(&pit->pit_state.channels, &ps->channels,
	       sizeof(pit->pit_state.channels));
	pit->pit_state.flags = ps->flags;
4711
	for (i = 0; i < 3; i++)
4712
		kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
4713
				   start && i == 0);
4714
	mutex_unlock(&pit->pit_state.lock);
4715
	return 0;
4716 4717
}

4718 4719 4720
static int kvm_vm_ioctl_reinject(struct kvm *kvm,
				 struct kvm_reinject_control *control)
{
4721 4722 4723 4724 4725 4726 4727 4728 4729
	struct kvm_pit *pit = kvm->arch.vpit;

	/* pit->pit_state.lock was overloaded to prevent userspace from getting
	 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
	 * ioctls in parallel.  Use a separate lock if that ioctl isn't rare.
	 */
	mutex_lock(&pit->pit_state.lock);
	kvm_pit_set_reinject(pit, control->pit_reinject);
	mutex_unlock(&pit->pit_state.lock);
4730

4731 4732 4733
	return 0;
}

4734
/**
4735 4736 4737
 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
 * @kvm: kvm instance
 * @log: slot id and address to which we copy the log
4738
 *
4739 4740 4741 4742 4743 4744 4745 4746
 * Steps 1-4 below provide general overview of dirty page logging. See
 * kvm_get_dirty_log_protect() function description for additional details.
 *
 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
 * always flush the TLB (step 4) even if previous step failed  and the dirty
 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
 * does not preclude user space subsequent dirty log read. Flushing TLB ensures
 * writes will be marked dirty for next log read.
4747
 *
4748 4749
 *   1. Take a snapshot of the bit and clear it if needed.
 *   2. Write protect the corresponding page.
4750 4751
 *   3. Copy the snapshot to the userspace.
 *   4. Flush TLB's if needed.
4752
 */
4753
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
4754
{
4755
	bool flush = false;
4756
	int r;
4757

4758
	mutex_lock(&kvm->slots_lock);
4759

4760 4761 4762 4763 4764 4765
	/*
	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
	 */
	if (kvm_x86_ops->flush_log_dirty)
		kvm_x86_ops->flush_log_dirty(kvm);

4766
	r = kvm_get_dirty_log_protect(kvm, log, &flush);
4767 4768 4769 4770 4771

	/*
	 * All the TLBs can be flushed out of mmu lock, see the comments in
	 * kvm_mmu_slot_remove_write_access().
	 */
4772
	lockdep_assert_held(&kvm->slots_lock);
4773
	if (flush)
4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800
		kvm_flush_remote_tlbs(kvm);

	mutex_unlock(&kvm->slots_lock);
	return r;
}

int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log)
{
	bool flush = false;
	int r;

	mutex_lock(&kvm->slots_lock);

	/*
	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
	 */
	if (kvm_x86_ops->flush_log_dirty)
		kvm_x86_ops->flush_log_dirty(kvm);

	r = kvm_clear_dirty_log_protect(kvm, log, &flush);

	/*
	 * All the TLBs can be flushed out of mmu lock, see the comments in
	 * kvm_mmu_slot_remove_write_access().
	 */
	lockdep_assert_held(&kvm->slots_lock);
	if (flush)
4801 4802
		kvm_flush_remote_tlbs(kvm);

4803
	mutex_unlock(&kvm->slots_lock);
4804 4805 4806
	return r;
}

4807 4808
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
			bool line_status)
4809 4810 4811 4812 4813
{
	if (!irqchip_in_kernel(kvm))
		return -ENXIO;

	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
4814 4815
					irq_event->irq, irq_event->level,
					line_status);
4816 4817 4818
	return 0;
}

4819 4820
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
			    struct kvm_enable_cap *cap)
4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831
{
	int r;

	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
	case KVM_CAP_DISABLE_QUIRKS:
		kvm->arch.disabled_quirks = cap->args[0];
		r = 0;
		break;
4832 4833
	case KVM_CAP_SPLIT_IRQCHIP: {
		mutex_lock(&kvm->lock);
4834 4835 4836
		r = -EINVAL;
		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
			goto split_irqchip_unlock;
4837 4838 4839
		r = -EEXIST;
		if (irqchip_in_kernel(kvm))
			goto split_irqchip_unlock;
P
Paolo Bonzini 已提交
4840
		if (kvm->created_vcpus)
4841 4842
			goto split_irqchip_unlock;
		r = kvm_setup_empty_irq_routing(kvm);
4843
		if (r)
4844 4845 4846
			goto split_irqchip_unlock;
		/* Pairs with irqchip_in_kernel. */
		smp_wmb();
4847
		kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
4848
		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
4849 4850 4851 4852 4853
		r = 0;
split_irqchip_unlock:
		mutex_unlock(&kvm->lock);
		break;
	}
4854 4855 4856 4857 4858 4859 4860
	case KVM_CAP_X2APIC_API:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
			break;

		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
			kvm->arch.x2apic_format = true;
4861 4862
		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
			kvm->arch.x2apic_broadcast_quirk_disabled = true;
4863 4864 4865

		r = 0;
		break;
4866 4867 4868 4869 4870 4871 4872 4873
	case KVM_CAP_X86_DISABLE_EXITS:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
			break;

		if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
			kvm_can_mwait_in_guest())
			kvm->arch.mwait_in_guest = true;
M
Michael S. Tsirkin 已提交
4874
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
4875
			kvm->arch.hlt_in_guest = true;
4876 4877
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
			kvm->arch.pause_in_guest = true;
4878 4879
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
			kvm->arch.cstate_in_guest = true;
4880 4881
		r = 0;
		break;
4882 4883 4884
	case KVM_CAP_MSR_PLATFORM_INFO:
		kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
		r = 0;
4885 4886 4887 4888
		break;
	case KVM_CAP_EXCEPTION_PAYLOAD:
		kvm->arch.exception_payload_enabled = cap->args[0];
		r = 0;
4889
		break;
4890 4891 4892 4893 4894 4895 4896
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

4897 4898 4899 4900 4901
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
4902
	int r = -ENOTTY;
4903 4904 4905 4906 4907 4908 4909
	/*
	 * This union makes it completely explicit to gcc-3.x
	 * that these two variables' stack usage should be
	 * combined, not added together.
	 */
	union {
		struct kvm_pit_state ps;
B
Beth Kon 已提交
4910
		struct kvm_pit_state2 ps2;
4911
		struct kvm_pit_config pit_config;
4912
	} u;
4913 4914 4915 4916 4917

	switch (ioctl) {
	case KVM_SET_TSS_ADDR:
		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
		break;
4918 4919 4920
	case KVM_SET_IDENTITY_MAP_ADDR: {
		u64 ident_addr;

4921 4922 4923 4924
		mutex_lock(&kvm->lock);
		r = -EINVAL;
		if (kvm->created_vcpus)
			goto set_identity_unlock;
4925
		r = -EFAULT;
4926
		if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
4927
			goto set_identity_unlock;
4928
		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
4929 4930
set_identity_unlock:
		mutex_unlock(&kvm->lock);
4931 4932
		break;
	}
4933 4934 4935 4936 4937 4938
	case KVM_SET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
		break;
	case KVM_GET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
		break;
4939 4940
	case KVM_CREATE_IRQCHIP: {
		mutex_lock(&kvm->lock);
4941

4942
		r = -EEXIST;
4943
		if (irqchip_in_kernel(kvm))
4944
			goto create_irqchip_unlock;
4945

4946
		r = -EINVAL;
P
Paolo Bonzini 已提交
4947
		if (kvm->created_vcpus)
4948
			goto create_irqchip_unlock;
4949 4950 4951

		r = kvm_pic_init(kvm);
		if (r)
4952
			goto create_irqchip_unlock;
4953 4954 4955 4956

		r = kvm_ioapic_init(kvm);
		if (r) {
			kvm_pic_destroy(kvm);
4957
			goto create_irqchip_unlock;
4958 4959
		}

4960 4961
		r = kvm_setup_default_irq_routing(kvm);
		if (r) {
4962
			kvm_ioapic_destroy(kvm);
4963
			kvm_pic_destroy(kvm);
4964
			goto create_irqchip_unlock;
4965
		}
4966
		/* Write kvm->irq_routing before enabling irqchip_in_kernel. */
4967
		smp_wmb();
4968
		kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
4969 4970
	create_irqchip_unlock:
		mutex_unlock(&kvm->lock);
4971
		break;
4972
	}
S
Sheng Yang 已提交
4973
	case KVM_CREATE_PIT:
4974 4975 4976 4977 4978 4979 4980 4981
		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
		goto create_pit;
	case KVM_CREATE_PIT2:
		r = -EFAULT;
		if (copy_from_user(&u.pit_config, argp,
				   sizeof(struct kvm_pit_config)))
			goto out;
	create_pit:
4982
		mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
4983 4984 4985
		r = -EEXIST;
		if (kvm->arch.vpit)
			goto create_pit_unlock;
S
Sheng Yang 已提交
4986
		r = -ENOMEM;
4987
		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
S
Sheng Yang 已提交
4988 4989
		if (kvm->arch.vpit)
			r = 0;
A
Avi Kivity 已提交
4990
	create_pit_unlock:
4991
		mutex_unlock(&kvm->lock);
S
Sheng Yang 已提交
4992
		break;
4993 4994
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
4995
		struct kvm_irqchip *chip;
4996

4997 4998 4999
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
5000
			goto out;
5001 5002
		}

5003
		r = -ENXIO;
5004
		if (!irqchip_kernel(kvm))
5005 5006
			goto get_irqchip_out;
		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
5007
		if (r)
5008
			goto get_irqchip_out;
5009
		r = -EFAULT;
5010
		if (copy_to_user(argp, chip, sizeof(*chip)))
5011
			goto get_irqchip_out;
5012
		r = 0;
5013 5014
	get_irqchip_out:
		kfree(chip);
5015 5016 5017 5018
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
5019
		struct kvm_irqchip *chip;
5020

5021 5022 5023
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
5024
			goto out;
5025 5026
		}

5027
		r = -ENXIO;
5028
		if (!irqchip_kernel(kvm))
5029 5030 5031 5032
			goto set_irqchip_out;
		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
	set_irqchip_out:
		kfree(chip);
5033 5034
		break;
	}
5035 5036
	case KVM_GET_PIT: {
		r = -EFAULT;
5037
		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
5038 5039 5040 5041
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
5042
		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
5043 5044 5045
		if (r)
			goto out;
		r = -EFAULT;
5046
		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
5047 5048 5049 5050 5051 5052
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT: {
		r = -EFAULT;
5053
		if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
5054 5055 5056 5057
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
5058
		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
5059 5060
		break;
	}
B
Beth Kon 已提交
5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083
	case KVM_GET_PIT2: {
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT2: {
		r = -EFAULT;
		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
		break;
	}
5084 5085 5086 5087 5088
	case KVM_REINJECT_CONTROL: {
		struct kvm_reinject_control control;
		r =  -EFAULT;
		if (copy_from_user(&control, argp, sizeof(control)))
			goto out;
5089 5090 5091
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
5092 5093 5094
		r = kvm_vm_ioctl_reinject(kvm, &control);
		break;
	}
5095 5096 5097
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
		mutex_lock(&kvm->lock);
P
Paolo Bonzini 已提交
5098
		if (kvm->created_vcpus)
5099 5100 5101 5102 5103
			r = -EBUSY;
		else
			kvm->arch.bsp_vcpu_id = arg;
		mutex_unlock(&kvm->lock);
		break;
E
Ed Swierk 已提交
5104
	case KVM_XEN_HVM_CONFIG: {
5105
		struct kvm_xen_hvm_config xhc;
E
Ed Swierk 已提交
5106
		r = -EFAULT;
5107
		if (copy_from_user(&xhc, argp, sizeof(xhc)))
E
Ed Swierk 已提交
5108 5109
			goto out;
		r = -EINVAL;
5110
		if (xhc.flags)
E
Ed Swierk 已提交
5111
			goto out;
5112
		memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));
E
Ed Swierk 已提交
5113 5114 5115
		r = 0;
		break;
	}
5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128
	case KVM_SET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

		r = -EFAULT;
		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
			goto out;

		r = -EINVAL;
		if (user_ns.flags)
			goto out;

		r = 0;
5129 5130 5131 5132 5133 5134
		/*
		 * TODO: userspace has to take care of races with VCPU_RUN, so
		 * kvm_gen_update_masterclock() can be cut down to locked
		 * pvclock_update_vm_gtod_copy().
		 */
		kvm_gen_update_masterclock(kvm);
5135
		now_ns = get_kvmclock_ns(kvm);
5136
		kvm->arch.kvmclock_offset += user_ns.clock - now_ns;
5137
		kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE);
5138 5139 5140 5141 5142 5143
		break;
	}
	case KVM_GET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

5144
		now_ns = get_kvmclock_ns(kvm);
5145
		user_ns.clock = now_ns;
5146
		user_ns.flags = kvm->arch.use_master_clock ? KVM_CLOCK_TSC_STABLE : 0;
5147
		memset(&user_ns.pad, 0, sizeof(user_ns.pad));
5148 5149 5150 5151 5152 5153 5154

		r = -EFAULT;
		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
			goto out;
		r = 0;
		break;
	}
5155 5156 5157 5158 5159 5160
	case KVM_MEMORY_ENCRYPT_OP: {
		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_op)
			r = kvm_x86_ops->mem_enc_op(kvm, argp);
		break;
	}
5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184
	case KVM_MEMORY_ENCRYPT_REG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_reg_region)
			r = kvm_x86_ops->mem_enc_reg_region(kvm, &region);
		break;
	}
	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_unreg_region)
			r = kvm_x86_ops->mem_enc_unreg_region(kvm, &region);
		break;
	}
5185 5186 5187 5188 5189 5190 5191 5192 5193
	case KVM_HYPERV_EVENTFD: {
		struct kvm_hyperv_eventfd hvevfd;

		r = -EFAULT;
		if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
			goto out;
		r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
		break;
	}
E
Eric Hankland 已提交
5194 5195 5196
	case KVM_SET_PMU_EVENT_FILTER:
		r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
		break;
5197
	default:
5198
		r = -ENOTTY;
5199 5200 5201 5202 5203
	}
out:
	return r;
}

5204
static void kvm_init_msr_list(void)
5205
{
5206
	struct x86_pmu_capability x86_pmu;
5207
	u32 dummy[2];
5208
	unsigned i;
5209

5210
	BUILD_BUG_ON_MSG(INTEL_PMC_MAX_FIXED != 4,
5211
			 "Please update the fixed PMCs in msrs_to_saved_all[]");
5212 5213

	perf_get_x86_pmu_capability(&x86_pmu);
5214

5215 5216 5217 5218
	num_msrs_to_save = 0;
	num_emulated_msrs = 0;
	num_msr_based_features = 0;

5219 5220
	for (i = 0; i < ARRAY_SIZE(msrs_to_save_all); i++) {
		if (rdmsr_safe(msrs_to_save_all[i], &dummy[0], &dummy[1]) < 0)
5221
			continue;
5222 5223 5224

		/*
		 * Even MSRs that are valid in the host may not be exposed
5225
		 * to the guests in some cases.
5226
		 */
5227
		switch (msrs_to_save_all[i]) {
5228
		case MSR_IA32_BNDCFGS:
5229
			if (!kvm_mpx_supported())
5230 5231
				continue;
			break;
5232 5233 5234 5235
		case MSR_TSC_AUX:
			if (!kvm_x86_ops->rdtscp_supported())
				continue;
			break;
5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254
		case MSR_IA32_RTIT_CTL:
		case MSR_IA32_RTIT_STATUS:
			if (!kvm_x86_ops->pt_supported())
				continue;
			break;
		case MSR_IA32_RTIT_CR3_MATCH:
			if (!kvm_x86_ops->pt_supported() ||
			    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
				continue;
			break;
		case MSR_IA32_RTIT_OUTPUT_BASE:
		case MSR_IA32_RTIT_OUTPUT_MASK:
			if (!kvm_x86_ops->pt_supported() ||
				(!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
				 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
				continue;
			break;
		case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: {
			if (!kvm_x86_ops->pt_supported() ||
5255
				msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >=
5256 5257 5258
				intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
				continue;
			break;
5259
		case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17:
5260
			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
5261 5262 5263
			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
				continue;
			break;
5264
		case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 17:
5265
			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >=
5266 5267
			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
				continue;
5268
		}
5269 5270 5271 5272
		default:
			break;
		}

5273
		msrs_to_save[num_msrs_to_save++] = msrs_to_save_all[i];
5274
	}
5275

5276 5277
	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
		if (!kvm_x86_ops->has_emulated_msr(emulated_msrs_all[i]))
5278
			continue;
5279

5280
		emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
5281
	}
5282

5283
	for (i = 0; i < ARRAY_SIZE(msr_based_features_all); i++) {
5284 5285
		struct kvm_msr_entry msr;

5286
		msr.index = msr_based_features_all[i];
5287
		if (kvm_get_msr_feature(&msr))
5288 5289
			continue;

5290
		msr_based_features[num_msr_based_features++] = msr_based_features_all[i];
5291
	}
5292 5293
}

5294 5295
static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
			   const void *v)
5296
{
5297 5298 5299 5300 5301
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5302
		if (!(lapic_in_kernel(vcpu) &&
5303 5304
		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
5305 5306 5307 5308 5309 5310
			break;
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5311

5312
	return handled;
5313 5314
}

5315
static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
5316
{
5317 5318 5319 5320 5321
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5322
		if (!(lapic_in_kernel(vcpu) &&
5323 5324 5325
		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
					 addr, n, v))
		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
5326
			break;
5327
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
5328 5329 5330 5331 5332
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5333

5334
	return handled;
5335 5336
}

5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348
static void kvm_set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
	kvm_x86_ops->set_segment(vcpu, var, seg);
}

void kvm_get_segment(struct kvm_vcpu *vcpu,
		     struct kvm_segment *var, int seg)
{
	kvm_x86_ops->get_segment(vcpu, var, seg);
}

5349 5350
gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
			   struct x86_exception *exception)
5351 5352 5353 5354 5355 5356 5357
{
	gpa_t t_gpa;

	BUG_ON(!mmu_is_nested(vcpu));

	/* NPT walks are always user-walks */
	access |= PFERR_USER_MASK;
5358
	t_gpa  = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception);
5359 5360 5361 5362

	return t_gpa;
}

5363 5364
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
			      struct x86_exception *exception)
5365 5366
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5367
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5368 5369
}

5370 5371
 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5372 5373 5374
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_FETCH_MASK;
5375
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5376 5377
}

5378 5379
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
			       struct x86_exception *exception)
5380 5381 5382
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_WRITE_MASK;
5383
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5384 5385 5386
}

/* uses this to access any guest's mapped memory without checking CPL */
5387 5388
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5389
{
5390
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
5391 5392 5393 5394
}

static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
5395
				      struct x86_exception *exception)
5396 5397
{
	void *data = val;
5398
	int r = X86EMUL_CONTINUE;
5399 5400

	while (bytes) {
5401
		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
5402
							    exception);
5403
		unsigned offset = addr & (PAGE_SIZE-1);
5404
		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
5405 5406
		int ret;

5407
		if (gpa == UNMAPPED_GVA)
5408
			return X86EMUL_PROPAGATE_FAULT;
5409 5410
		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
					       offset, toread);
5411
		if (ret < 0) {
5412
			r = X86EMUL_IO_NEEDED;
5413 5414
			goto out;
		}
5415

5416 5417 5418
		bytes -= toread;
		data += toread;
		addr += toread;
5419
	}
5420 5421
out:
	return r;
5422
}
5423

5424
/* used for instruction fetching */
5425 5426
static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
				gva_t addr, void *val, unsigned int bytes,
5427
				struct x86_exception *exception)
5428
{
5429
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5430
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5431 5432
	unsigned offset;
	int ret;
5433

5434 5435 5436 5437 5438 5439 5440 5441 5442
	/* Inline kvm_read_guest_virt_helper for speed.  */
	gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,
						    exception);
	if (unlikely(gpa == UNMAPPED_GVA))
		return X86EMUL_PROPAGATE_FAULT;

	offset = addr & (PAGE_SIZE-1);
	if (WARN_ON(offset + bytes > PAGE_SIZE))
		bytes = (unsigned)PAGE_SIZE - offset;
5443 5444
	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
				       offset, bytes);
5445 5446 5447 5448
	if (unlikely(ret < 0))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
5449 5450
}

5451
int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
5452
			       gva_t addr, void *val, unsigned int bytes,
5453
			       struct x86_exception *exception)
5454 5455
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5456

5457 5458 5459 5460 5461 5462 5463
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5464
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
5465
					  exception);
5466
}
5467
EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
5468

5469 5470
static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
			     gva_t addr, void *val, unsigned int bytes,
5471
			     struct x86_exception *exception, bool system)
5472
{
5473
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5474 5475 5476 5477 5478 5479
	u32 access = 0;

	if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
		access |= PFERR_USER_MASK;

	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
5480 5481
}

5482 5483 5484 5485 5486 5487 5488 5489 5490
static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
		unsigned long addr, void *val, unsigned int bytes)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);

	return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
}

5491 5492 5493
static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
				      struct x86_exception *exception)
5494 5495 5496 5497 5498
{
	void *data = val;
	int r = X86EMUL_CONTINUE;

	while (bytes) {
5499
		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
5500
							     access,
5501
							     exception);
5502 5503 5504 5505
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
		int ret;

5506
		if (gpa == UNMAPPED_GVA)
5507
			return X86EMUL_PROPAGATE_FAULT;
5508
		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
5509
		if (ret < 0) {
5510
			r = X86EMUL_IO_NEEDED;
5511 5512 5513 5514 5515 5516 5517 5518 5519 5520
			goto out;
		}

		bytes -= towrite;
		data += towrite;
		addr += towrite;
	}
out:
	return r;
}
5521 5522

static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
5523 5524
			      unsigned int bytes, struct x86_exception *exception,
			      bool system)
5525 5526
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5527 5528 5529 5530
	u32 access = PFERR_WRITE_MASK;

	if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
		access |= PFERR_USER_MASK;
5531 5532

	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
5533
					   access, exception);
5534 5535 5536 5537 5538
}

int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
				unsigned int bytes, struct x86_exception *exception)
{
P
Paolo Bonzini 已提交
5539 5540 5541
	/* kvm_write_guest_virt_system can pull in tons of pages. */
	vcpu->arch.l1tf_flush_l1d = true;

5542 5543 5544 5545 5546 5547 5548
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5549 5550 5551
	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
					   PFERR_WRITE_MASK, exception);
}
N
Nadav Har'El 已提交
5552
EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
5553

W
Wanpeng Li 已提交
5554 5555
int handle_ud(struct kvm_vcpu *vcpu)
{
5556
	static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX };
5557 5558 5559 5560 5561
	int emul_type = EMULTYPE_TRAP_UD;
	char sig[5]; /* ud2; .ascii "kvm" */
	struct x86_exception e;

	if (force_emulation_prefix &&
5562 5563
	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
				sig, sizeof(sig), &e) == 0 &&
5564
	    memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) {
5565
		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
5566
		emul_type = EMULTYPE_TRAP_UD_FORCED;
5567
	}
W
Wanpeng Li 已提交
5568

5569
	return kvm_emulate_instruction(vcpu, emul_type);
W
Wanpeng Li 已提交
5570 5571 5572
}
EXPORT_SYMBOL_GPL(handle_ud);

5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587
static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
			    gpa_t gpa, bool write)
{
	/* For APIC access vmexit */
	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		return 1;

	if (vcpu_match_mmio_gpa(vcpu, gpa)) {
		trace_vcpu_match_mmio(gva, gpa, write, true);
		return 1;
	}

	return 0;
}

5588 5589 5590 5591
static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
				gpa_t *gpa, struct x86_exception *exception,
				bool write)
{
5592 5593
	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
		| (write ? PFERR_WRITE_MASK : 0);
5594

5595 5596 5597 5598 5599
	/*
	 * currently PKRU is only applied to ept enabled guest so
	 * there is no pkey in EPT page table for L1 guest or EPT
	 * shadow page table for L2 guest.
	 */
5600
	if (vcpu_match_mmio_gva(vcpu, gva)
F
Feng Wu 已提交
5601
	    && !permission_fault(vcpu, vcpu->arch.walk_mmu,
5602
				 vcpu->arch.mmio_access, 0, access)) {
5603 5604
		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
					(gva & (PAGE_SIZE - 1));
X
Xiao Guangrong 已提交
5605
		trace_vcpu_match_mmio(gva, *gpa, write, false);
5606 5607 5608
		return 1;
	}

5609 5610 5611 5612 5613
	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);

	if (*gpa == UNMAPPED_GVA)
		return -1;

5614
	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
5615 5616
}

5617
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
5618
			const void *val, int bytes)
5619 5620 5621
{
	int ret;

5622
	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
5623
	if (ret < 0)
5624
		return 0;
5625
	kvm_page_track_write(vcpu, gpa, val, bytes);
5626 5627 5628
	return 1;
}

5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644
struct read_write_emulator_ops {
	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
				  int bytes);
	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
				  void *val, int bytes);
	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
			       int bytes, void *val);
	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
				    void *val, int bytes);
	bool write;
};

static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
{
	if (vcpu->mmio_read_completed) {
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
5645
			       vcpu->mmio_fragments[0].gpa, val);
5646 5647 5648 5649 5650 5651 5652 5653 5654 5655
		vcpu->mmio_read_completed = 0;
		return 1;
	}

	return 0;
}

static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			void *val, int bytes)
{
5656
	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
5657 5658 5659 5660 5661 5662 5663 5664 5665 5666
}

static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			 void *val, int bytes)
{
	return emulator_write_phys(vcpu, gpa, val, bytes);
}

static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
{
5667
	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
5668 5669 5670 5671 5672 5673
	return vcpu_mmio_write(vcpu, gpa, bytes, val);
}

static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			  void *val, int bytes)
{
5674
	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
5675 5676 5677 5678 5679 5680
	return X86EMUL_IO_NEEDED;
}

static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			   void *val, int bytes)
{
A
Avi Kivity 已提交
5681 5682
	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];

5683
	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
5684 5685 5686
	return X86EMUL_CONTINUE;
}

5687
static const struct read_write_emulator_ops read_emultor = {
5688 5689 5690 5691 5692 5693
	.read_write_prepare = read_prepare,
	.read_write_emulate = read_emulate,
	.read_write_mmio = vcpu_mmio_read,
	.read_write_exit_mmio = read_exit_mmio,
};

5694
static const struct read_write_emulator_ops write_emultor = {
5695 5696 5697 5698 5699 5700
	.read_write_emulate = write_emulate,
	.read_write_mmio = write_mmio,
	.read_write_exit_mmio = write_exit_mmio,
	.write = true,
};

5701 5702 5703 5704
static int emulator_read_write_onepage(unsigned long addr, void *val,
				       unsigned int bytes,
				       struct x86_exception *exception,
				       struct kvm_vcpu *vcpu,
5705
				       const struct read_write_emulator_ops *ops)
5706
{
5707 5708
	gpa_t gpa;
	int handled, ret;
5709
	bool write = ops->write;
A
Avi Kivity 已提交
5710
	struct kvm_mmio_fragment *frag;
5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;

	/*
	 * If the exit was due to a NPF we may already have a GPA.
	 * If the GPA is present, use it to avoid the GVA to GPA table walk.
	 * Note, this cannot be used on string operations since string
	 * operation using rep will only have the initial GPA from the NPF
	 * occurred.
	 */
	if (vcpu->arch.gpa_available &&
	    emulator_can_use_gpa(ctxt) &&
5722 5723 5724 5725 5726 5727 5728
	    (addr & ~PAGE_MASK) == (vcpu->arch.gpa_val & ~PAGE_MASK)) {
		gpa = vcpu->arch.gpa_val;
		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
	} else {
		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
		if (ret < 0)
			return X86EMUL_PROPAGATE_FAULT;
5729
	}
5730

5731
	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
5732 5733 5734 5735 5736
		return X86EMUL_CONTINUE;

	/*
	 * Is this MMIO handled locally?
	 */
5737
	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
5738
	if (handled == bytes)
5739 5740
		return X86EMUL_CONTINUE;

5741 5742 5743 5744
	gpa += handled;
	bytes -= handled;
	val += handled;

5745 5746 5747 5748 5749
	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
	frag->gpa = gpa;
	frag->data = val;
	frag->len = bytes;
A
Avi Kivity 已提交
5750
	return X86EMUL_CONTINUE;
5751 5752
}

5753 5754
static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
			unsigned long addr,
5755 5756
			void *val, unsigned int bytes,
			struct x86_exception *exception,
5757
			const struct read_write_emulator_ops *ops)
5758
{
5759
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
A
Avi Kivity 已提交
5760 5761 5762 5763 5764 5765 5766 5767
	gpa_t gpa;
	int rc;

	if (ops->read_write_prepare &&
		  ops->read_write_prepare(vcpu, val, bytes))
		return X86EMUL_CONTINUE;

	vcpu->mmio_nr_fragments = 0;
5768

5769 5770
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
A
Avi Kivity 已提交
5771
		int now;
5772 5773

		now = -addr & ~PAGE_MASK;
5774 5775 5776
		rc = emulator_read_write_onepage(addr, val, now, exception,
						 vcpu, ops);

5777 5778 5779
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
5780 5781
		if (ctxt->mode != X86EMUL_MODE_PROT64)
			addr = (u32)addr;
5782 5783 5784
		val += now;
		bytes -= now;
	}
5785

A
Avi Kivity 已提交
5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798
	rc = emulator_read_write_onepage(addr, val, bytes, exception,
					 vcpu, ops);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	if (!vcpu->mmio_nr_fragments)
		return rc;

	gpa = vcpu->mmio_fragments[0].gpa;

	vcpu->mmio_needed = 1;
	vcpu->mmio_cur_fragment = 0;

5799
	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
A
Avi Kivity 已提交
5800 5801 5802 5803 5804
	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
	vcpu->run->exit_reason = KVM_EXIT_MMIO;
	vcpu->run->mmio.phys_addr = gpa;

	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816
}

static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
				  unsigned long addr,
				  void *val,
				  unsigned int bytes,
				  struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, val, bytes,
				   exception, &read_emultor);
}

5817
static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
5818 5819 5820 5821 5822 5823 5824
			    unsigned long addr,
			    const void *val,
			    unsigned int bytes,
			    struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, (void *)val, bytes,
				   exception, &write_emultor);
5825 5826
}

5827 5828 5829 5830 5831 5832 5833
#define CMPXCHG_TYPE(t, ptr, old, new) \
	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))

#ifdef CONFIG_X86_64
#  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
#else
#  define CMPXCHG64(ptr, old, new) \
5834
	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
5835 5836
#endif

5837 5838
static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
				     unsigned long addr,
5839 5840 5841
				     const void *old,
				     const void *new,
				     unsigned int bytes,
5842
				     struct x86_exception *exception)
5843
{
5844
	struct kvm_host_map map;
5845
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5846 5847 5848
	gpa_t gpa;
	char *kaddr;
	bool exchanged;
5849

5850 5851 5852
	/* guests cmpxchg8b have to be emulated atomically */
	if (bytes > 8 || (bytes & (bytes - 1)))
		goto emul_write;
5853

5854
	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
5855

5856 5857 5858
	if (gpa == UNMAPPED_GVA ||
	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		goto emul_write;
5859

5860 5861
	if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
		goto emul_write;
5862

5863
	if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))
5864
		goto emul_write;
5865

5866 5867
	kaddr = map.hva + offset_in_page(gpa);

5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882
	switch (bytes) {
	case 1:
		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
		break;
	case 2:
		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
		break;
	case 4:
		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
		break;
	case 8:
		exchanged = CMPXCHG64(kaddr, old, new);
		break;
	default:
		BUG();
5883
	}
5884 5885

	kvm_vcpu_unmap(vcpu, &map, true);
5886 5887 5888 5889

	if (!exchanged)
		return X86EMUL_CMPXCHG_FAILED;

5890
	kvm_page_track_write(vcpu, gpa, new, bytes);
5891 5892

	return X86EMUL_CONTINUE;
5893

5894
emul_write:
5895
	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
5896

5897
	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
5898 5899
}

5900 5901
static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
{
5902
	int r = 0, i;
5903

5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915
	for (i = 0; i < vcpu->arch.pio.count; i++) {
		if (vcpu->arch.pio.in)
			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
					    vcpu->arch.pio.size, pd);
		else
			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
					     vcpu->arch.pio.port, vcpu->arch.pio.size,
					     pd);
		if (r)
			break;
		pd += vcpu->arch.pio.size;
	}
5916 5917 5918
	return r;
}

5919 5920 5921
static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
			       unsigned short port, void *val,
			       unsigned int count, bool in)
5922 5923
{
	vcpu->arch.pio.port = port;
5924
	vcpu->arch.pio.in = in;
5925
	vcpu->arch.pio.count  = count;
5926 5927 5928
	vcpu->arch.pio.size = size;

	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
5929
		vcpu->arch.pio.count = 0;
5930 5931 5932 5933
		return 1;
	}

	vcpu->run->exit_reason = KVM_EXIT_IO;
5934
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
5935 5936 5937 5938 5939 5940 5941 5942
	vcpu->run->io.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = count;
	vcpu->run->io.port = port;

	return 0;
}

5943 5944 5945
static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
				    int size, unsigned short port, void *val,
				    unsigned int count)
5946
{
5947
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5948
	int ret;
5949

5950 5951
	if (vcpu->arch.pio.count)
		goto data_avail;
5952

5953 5954
	memset(vcpu->arch.pio_data, 0, size * count);

5955 5956 5957 5958
	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
	if (ret) {
data_avail:
		memcpy(val, vcpu->arch.pio_data, size * count);
5959
		trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
5960
		vcpu->arch.pio.count = 0;
5961 5962 5963 5964 5965 5966
		return 1;
	}

	return 0;
}

5967 5968 5969 5970 5971 5972 5973
static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
				     int size, unsigned short port,
				     const void *val, unsigned int count)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	memcpy(vcpu->arch.pio_data, val, size * count);
5974
	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
5975 5976 5977
	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
}

5978 5979 5980 5981 5982
static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	return kvm_x86_ops->get_segment_base(vcpu, seg);
}

5983
static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
5984
{
5985
	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
5986 5987
}

5988
static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
5989 5990 5991 5992 5993
{
	if (!need_emulate_wbinvd(vcpu))
		return X86EMUL_CONTINUE;

	if (kvm_x86_ops->has_wbinvd_exit()) {
5994 5995 5996
		int cpu = get_cpu();

		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
5997 5998
		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
				wbinvd_ipi, NULL, 1);
5999
		put_cpu();
6000
		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
6001 6002
	} else
		wbinvd();
6003 6004
	return X86EMUL_CONTINUE;
}
6005 6006 6007

int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
6008 6009
	kvm_emulate_wbinvd_noskip(vcpu);
	return kvm_skip_emulated_instruction(vcpu);
6010
}
6011 6012
EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);

6013 6014


6015 6016
static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
{
6017
	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
6018 6019
}

6020 6021
static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long *dest)
6022
{
6023
	return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
6024 6025
}

6026 6027
static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long value)
6028
{
6029

6030
	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
6031 6032
}

6033
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
6034
{
6035
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
6036 6037
}

6038
static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
6039
{
6040
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6041 6042 6043 6044 6045 6046 6047 6048 6049 6050
	unsigned long value;

	switch (cr) {
	case 0:
		value = kvm_read_cr0(vcpu);
		break;
	case 2:
		value = vcpu->arch.cr2;
		break;
	case 3:
6051
		value = kvm_read_cr3(vcpu);
6052 6053 6054 6055 6056 6057 6058 6059
		break;
	case 4:
		value = kvm_read_cr4(vcpu);
		break;
	case 8:
		value = kvm_get_cr8(vcpu);
		break;
	default:
6060
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
6061 6062 6063 6064 6065 6066
		return 0;
	}

	return value;
}

6067
static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
6068
{
6069
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6070 6071
	int res = 0;

6072 6073
	switch (cr) {
	case 0:
6074
		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
6075 6076 6077 6078 6079
		break;
	case 2:
		vcpu->arch.cr2 = val;
		break;
	case 3:
6080
		res = kvm_set_cr3(vcpu, val);
6081 6082
		break;
	case 4:
6083
		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
6084 6085
		break;
	case 8:
A
Andre Przywara 已提交
6086
		res = kvm_set_cr8(vcpu, val);
6087 6088
		break;
	default:
6089
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
6090
		res = -1;
6091
	}
6092 6093

	return res;
6094 6095
}

6096
static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
6097
{
6098
	return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
6099 6100
}

6101
static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6102
{
6103
	kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
6104 6105
}

6106
static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6107
{
6108
	kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
6109 6110
}

6111 6112 6113 6114 6115 6116 6117 6118 6119 6120
static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
}

static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
}

6121 6122
static unsigned long emulator_get_cached_segment_base(
	struct x86_emulate_ctxt *ctxt, int seg)
6123
{
6124
	return get_segment_base(emul_to_vcpu(ctxt), seg);
6125 6126
}

6127 6128 6129
static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
				 struct desc_struct *desc, u32 *base3,
				 int seg)
6130 6131 6132
{
	struct kvm_segment var;

6133
	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
6134
	*selector = var.selector;
6135

6136 6137
	if (var.unusable) {
		memset(desc, 0, sizeof(*desc));
6138 6139
		if (base3)
			*base3 = 0;
6140
		return false;
6141
	}
6142 6143 6144 6145 6146

	if (var.g)
		var.limit >>= 12;
	set_desc_limit(desc, var.limit);
	set_desc_base(desc, (unsigned long)var.base);
6147 6148 6149 6150
#ifdef CONFIG_X86_64
	if (base3)
		*base3 = var.base >> 32;
#endif
6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162
	desc->type = var.type;
	desc->s = var.s;
	desc->dpl = var.dpl;
	desc->p = var.present;
	desc->avl = var.avl;
	desc->l = var.l;
	desc->d = var.db;
	desc->g = var.g;

	return true;
}

6163 6164 6165
static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
				 struct desc_struct *desc, u32 base3,
				 int seg)
6166
{
6167
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6168 6169
	struct kvm_segment var;

6170
	var.selector = selector;
6171
	var.base = get_desc_base(desc);
6172 6173 6174
#ifdef CONFIG_X86_64
	var.base |= ((u64)base3) << 32;
#endif
6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192
	var.limit = get_desc_limit(desc);
	if (desc->g)
		var.limit = (var.limit << 12) | 0xfff;
	var.type = desc->type;
	var.dpl = desc->dpl;
	var.db = desc->d;
	var.s = desc->s;
	var.l = desc->l;
	var.g = desc->g;
	var.avl = desc->avl;
	var.present = desc->p;
	var.unusable = !var.present;
	var.padding = 0;

	kvm_set_segment(vcpu, &var, seg);
	return;
}

6193 6194 6195
static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 *pdata)
{
6196
	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
6197 6198 6199 6200 6201
}

static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 data)
{
6202
	return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
6203 6204
}

P
Paolo Bonzini 已提交
6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218
static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	return vcpu->arch.smbase;
}

static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	vcpu->arch.smbase = smbase;
}

6219 6220 6221
static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
			      u32 pmc)
{
6222
	return kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc);
6223 6224
}

6225 6226 6227
static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
			     u32 pmc, u64 *pdata)
{
6228
	return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
6229 6230
}

6231 6232 6233 6234 6235
static void emulator_halt(struct x86_emulate_ctxt *ctxt)
{
	emul_to_vcpu(ctxt)->arch.halt_request = 1;
}

6236
static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
6237
			      struct x86_instruction_info *info,
6238 6239
			      enum x86_intercept_stage stage)
{
6240
	return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
6241 6242
}

6243 6244
static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
			u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, bool check_limit)
6245
{
6246
	return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, check_limit);
6247 6248
}

6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263
static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_LM);
}

static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE);
}

static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR);
}

6264 6265 6266 6267 6268 6269 6270 6271 6272 6273
static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
{
	return kvm_register_read(emul_to_vcpu(ctxt), reg);
}

static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
{
	kvm_register_write(emul_to_vcpu(ctxt), reg, val);
}

6274 6275 6276 6277 6278
static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
{
	kvm_x86_ops->set_nmi_mask(emul_to_vcpu(ctxt), masked);
}

6279 6280 6281 6282 6283 6284 6285
static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
{
	return emul_to_vcpu(ctxt)->arch.hflags;
}

static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_flags)
{
6286
	emul_to_vcpu(ctxt)->arch.hflags = emul_flags;
6287 6288
}

6289 6290
static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt,
				  const char *smstate)
6291
{
6292
	return kvm_x86_ops->pre_leave_smm(emul_to_vcpu(ctxt), smstate);
6293 6294
}

6295 6296 6297 6298 6299
static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt)
{
	kvm_smm_changed(emul_to_vcpu(ctxt));
}

6300 6301 6302 6303 6304
static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
{
	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
}

6305
static const struct x86_emulate_ops emulate_ops = {
6306 6307
	.read_gpr            = emulator_read_gpr,
	.write_gpr           = emulator_write_gpr,
6308 6309
	.read_std            = emulator_read_std,
	.write_std           = emulator_write_std,
6310
	.read_phys           = kvm_read_guest_phys_system,
6311
	.fetch               = kvm_fetch_guest_virt,
6312 6313 6314
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
6315
	.invlpg              = emulator_invlpg,
6316 6317
	.pio_in_emulated     = emulator_pio_in_emulated,
	.pio_out_emulated    = emulator_pio_out_emulated,
6318 6319
	.get_segment         = emulator_get_segment,
	.set_segment         = emulator_set_segment,
6320
	.get_cached_segment_base = emulator_get_cached_segment_base,
6321
	.get_gdt             = emulator_get_gdt,
6322
	.get_idt	     = emulator_get_idt,
6323 6324
	.set_gdt             = emulator_set_gdt,
	.set_idt	     = emulator_set_idt,
6325 6326
	.get_cr              = emulator_get_cr,
	.set_cr              = emulator_set_cr,
6327
	.cpl                 = emulator_get_cpl,
6328 6329
	.get_dr              = emulator_get_dr,
	.set_dr              = emulator_set_dr,
P
Paolo Bonzini 已提交
6330 6331
	.get_smbase          = emulator_get_smbase,
	.set_smbase          = emulator_set_smbase,
6332 6333
	.set_msr             = emulator_set_msr,
	.get_msr             = emulator_get_msr,
6334
	.check_pmc	     = emulator_check_pmc,
6335
	.read_pmc            = emulator_read_pmc,
6336
	.halt                = emulator_halt,
6337
	.wbinvd              = emulator_wbinvd,
6338
	.fix_hypercall       = emulator_fix_hypercall,
6339
	.intercept           = emulator_intercept,
6340
	.get_cpuid           = emulator_get_cpuid,
6341 6342 6343
	.guest_has_long_mode = emulator_guest_has_long_mode,
	.guest_has_movbe     = emulator_guest_has_movbe,
	.guest_has_fxsr      = emulator_guest_has_fxsr,
6344
	.set_nmi_mask        = emulator_set_nmi_mask,
6345 6346
	.get_hflags          = emulator_get_hflags,
	.set_hflags          = emulator_set_hflags,
6347
	.pre_leave_smm       = emulator_pre_leave_smm,
6348
	.post_leave_smm      = emulator_post_leave_smm,
6349
	.set_xcr             = emulator_set_xcr,
6350 6351
};

6352 6353
static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
{
6354
	u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
6355 6356 6357 6358 6359 6360 6361
	/*
	 * an sti; sti; sequence only disable interrupts for the first
	 * instruction. So, if the last instruction, be it emulated or
	 * not, left the system with the INT_STI flag enabled, it
	 * means that the last instruction is an sti. We should not
	 * leave the flag on in this case. The same goes for mov ss
	 */
6362 6363
	if (int_shadow & mask)
		mask = 0;
6364
	if (unlikely(int_shadow || mask)) {
6365
		kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
6366 6367 6368
		if (!mask)
			kvm_make_request(KVM_REQ_EVENT, vcpu);
	}
6369 6370
}

6371
static bool inject_emulated_exception(struct kvm_vcpu *vcpu)
6372 6373
{
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6374
	if (ctxt->exception.vector == PF_VECTOR)
6375 6376 6377
		return kvm_propagate_fault(vcpu, &ctxt->exception);

	if (ctxt->exception.error_code_valid)
6378 6379
		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
				      ctxt->exception.error_code);
6380
	else
6381
		kvm_queue_exception(vcpu, ctxt->exception.vector);
6382
	return false;
6383 6384
}

6385 6386
static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
{
6387
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6388 6389 6390 6391
	int cs_db, cs_l;

	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);

6392
	ctxt->eflags = kvm_get_rflags(vcpu);
6393 6394
	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;

6395 6396 6397
	ctxt->eip = kvm_rip_read(vcpu);
	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
6398
		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :
6399 6400
		     cs_db				? X86EMUL_MODE_PROT32 :
							  X86EMUL_MODE_PROT16;
6401
	BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK);
P
Paolo Bonzini 已提交
6402 6403
	BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK);
	BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK);
6404

6405
	init_decode_cache(ctxt);
6406
	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6407 6408
}

6409
void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
6410
{
6411
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6412 6413 6414 6415
	int ret;

	init_emulate_ctxt(vcpu);

6416 6417 6418
	ctxt->op_bytes = 2;
	ctxt->ad_bytes = 2;
	ctxt->_eip = ctxt->eip + inc_eip;
6419
	ret = emulate_int_real(ctxt, irq);
6420

6421 6422 6423 6424 6425 6426 6427
	if (ret != X86EMUL_CONTINUE) {
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
	} else {
		ctxt->eip = ctxt->_eip;
		kvm_rip_write(vcpu, ctxt->eip);
		kvm_set_rflags(vcpu, ctxt->eflags);
	}
6428 6429 6430
}
EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);

6431
static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
6432 6433 6434
{
	++vcpu->stat.insn_emulation_fail;
	trace_kvm_emulate_insn_failed(vcpu);
6435

6436 6437
	if (emulation_type & EMULTYPE_VMWARE_GP) {
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6438
		return 1;
6439
	}
6440

6441 6442 6443 6444
	if (emulation_type & EMULTYPE_SKIP) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6445
		return 0;
6446 6447
	}

6448 6449
	kvm_queue_exception(vcpu, UD_VECTOR);

6450
	if (!is_guest_mode(vcpu) && kvm_x86_ops->get_cpl(vcpu) == 0) {
6451 6452 6453
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6454
		return 0;
6455
	}
6456

6457
	return 1;
6458 6459
}

6460
static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
6461 6462
				  bool write_fault_to_shadow_pgtable,
				  int emulation_type)
6463
{
6464
	gpa_t gpa = cr2_or_gpa;
D
Dan Williams 已提交
6465
	kvm_pfn_t pfn;
6466

6467
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
6468 6469
		return false;

6470 6471 6472
	if (WARN_ON_ONCE(is_guest_mode(vcpu)))
		return false;

6473
	if (!vcpu->arch.mmu->direct_map) {
6474 6475 6476 6477
		/*
		 * Write permission should be allowed since only
		 * write access need to be emulated.
		 */
6478
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
6479

6480 6481 6482 6483 6484 6485 6486
		/*
		 * If the mapping is invalid in guest, let cpu retry
		 * it to generate fault.
		 */
		if (gpa == UNMAPPED_GVA)
			return true;
	}
6487

6488 6489 6490 6491 6492 6493 6494
	/*
	 * Do not retry the unhandleable instruction if it faults on the
	 * readonly host memory, otherwise it will goto a infinite loop:
	 * retry instruction -> write #PF -> emulation fail -> retry
	 * instruction -> ...
	 */
	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505

	/*
	 * If the instruction failed on the error pfn, it can not be fixed,
	 * report the error to userspace.
	 */
	if (is_error_noslot_pfn(pfn))
		return false;

	kvm_release_pfn_clean(pfn);

	/* The instructions are well-emulated on direct mmu. */
6506
	if (vcpu->arch.mmu->direct_map) {
6507 6508 6509 6510 6511 6512 6513 6514 6515
		unsigned int indirect_shadow_pages;

		spin_lock(&vcpu->kvm->mmu_lock);
		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
		spin_unlock(&vcpu->kvm->mmu_lock);

		if (indirect_shadow_pages)
			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));

6516
		return true;
6517
	}
6518

6519 6520 6521 6522 6523 6524
	/*
	 * if emulation was due to access to shadowed page table
	 * and it failed try to unshadow page and re-enter the
	 * guest to let CPU execute the instruction.
	 */
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6525 6526 6527 6528 6529 6530 6531

	/*
	 * If the access faults on its page table, it can not
	 * be fixed by unprotecting shadow page and it should
	 * be reported to userspace.
	 */
	return !write_fault_to_shadow_pgtable;
6532 6533
}

6534
static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
6535
			      gpa_t cr2_or_gpa,  int emulation_type)
6536 6537
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6538
	unsigned long last_retry_eip, last_retry_addr, gpa = cr2_or_gpa;
6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557

	last_retry_eip = vcpu->arch.last_retry_eip;
	last_retry_addr = vcpu->arch.last_retry_addr;

	/*
	 * If the emulation is caused by #PF and it is non-page_table
	 * writing instruction, it means the VM-EXIT is caused by shadow
	 * page protected, we can zap the shadow page and retry this
	 * instruction directly.
	 *
	 * Note: if the guest uses a non-page-table modifying instruction
	 * on the PDE that points to the instruction, then we will unmap
	 * the instruction and go to an infinite loop. So, we cache the
	 * last retried eip and the last fault address, if we meet the eip
	 * and the address again, we can break out of the potential infinite
	 * loop.
	 */
	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;

6558
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
6559 6560
		return false;

6561 6562 6563
	if (WARN_ON_ONCE(is_guest_mode(vcpu)))
		return false;

6564 6565 6566
	if (x86_page_table_writing_insn(ctxt))
		return false;

6567
	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa)
6568 6569 6570
		return false;

	vcpu->arch.last_retry_eip = ctxt->eip;
6571
	vcpu->arch.last_retry_addr = cr2_or_gpa;
6572

6573
	if (!vcpu->arch.mmu->direct_map)
6574
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
6575

6576
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6577 6578 6579 6580

	return true;
}

6581 6582 6583
static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
static int complete_emulated_pio(struct kvm_vcpu *vcpu);

P
Paolo Bonzini 已提交
6584
static void kvm_smm_changed(struct kvm_vcpu *vcpu)
6585
{
P
Paolo Bonzini 已提交
6586
	if (!(vcpu->arch.hflags & HF_SMM_MASK)) {
6587 6588 6589
		/* This is a good place to trace that we are exiting SMM.  */
		trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false);

6590 6591
		/* Process a latched INIT or SMI, if any.  */
		kvm_make_request(KVM_REQ_EVENT, vcpu);
P
Paolo Bonzini 已提交
6592
	}
6593 6594

	kvm_mmu_reset_context(vcpu);
P
Paolo Bonzini 已提交
6595 6596
}

6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611
static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
				unsigned long *db)
{
	u32 dr6 = 0;
	int i;
	u32 enable, rwlen;

	enable = dr7;
	rwlen = dr7 >> 16;
	for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
		if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
			dr6 |= (1 << i);
	return dr6;
}

6612
static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
6613 6614 6615
{
	struct kvm_run *kvm_run = vcpu->run;

6616 6617 6618 6619 6620
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
		kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1 | DR6_RTM;
		kvm_run->debug.arch.pc = vcpu->arch.singlestep_rip;
		kvm_run->debug.arch.exception = DB_VECTOR;
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
6621
		return 0;
6622
	}
6623
	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
6624
	return 1;
6625 6626
}

6627 6628 6629
int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
	unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6630
	int r;
6631

6632
	r = kvm_x86_ops->skip_emulated_instruction(vcpu);
6633
	if (unlikely(!r))
6634
		return 0;
6635 6636 6637 6638 6639 6640 6641 6642 6643 6644

	/*
	 * rflags is the old, "raw" value of the flags.  The new value has
	 * not been saved yet.
	 *
	 * This is correct even for TF set by the guest, because "the
	 * processor will not generate this exception after the instruction
	 * that sets the TF flag".
	 */
	if (unlikely(rflags & X86_EFLAGS_TF))
6645
		r = kvm_vcpu_do_singlestep(vcpu);
6646
	return r;
6647 6648 6649
}
EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);

6650 6651 6652 6653
static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
{
	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
6654 6655 6656
		struct kvm_run *kvm_run = vcpu->run;
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6657 6658 6659 6660
					   vcpu->arch.guest_debug_dr7,
					   vcpu->arch.eff_db);

		if (dr6 != 0) {
6661
			kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
6662
			kvm_run->debug.arch.pc = eip;
6663 6664
			kvm_run->debug.arch.exception = DB_VECTOR;
			kvm_run->exit_reason = KVM_EXIT_DEBUG;
6665
			*r = 0;
6666 6667 6668 6669
			return true;
		}
	}

6670 6671
	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
	    !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {
6672 6673
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6674 6675 6676 6677
					   vcpu->arch.dr7,
					   vcpu->arch.db);

		if (dr6 != 0) {
6678
			vcpu->arch.dr6 &= ~DR_TRAP_BITS;
6679
			vcpu->arch.dr6 |= dr6 | DR6_RTM;
6680
			kvm_queue_exception(vcpu, DB_VECTOR);
6681
			*r = 1;
6682 6683 6684 6685 6686 6687 6688
			return true;
		}
	}

	return false;
}

6689 6690
static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
{
6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714
	switch (ctxt->opcode_len) {
	case 1:
		switch (ctxt->b) {
		case 0xe4:	/* IN */
		case 0xe5:
		case 0xec:
		case 0xed:
		case 0xe6:	/* OUT */
		case 0xe7:
		case 0xee:
		case 0xef:
		case 0x6c:	/* INS */
		case 0x6d:
		case 0x6e:	/* OUTS */
		case 0x6f:
			return true;
		}
		break;
	case 2:
		switch (ctxt->b) {
		case 0x33:	/* RDPMC */
			return true;
		}
		break;
6715 6716 6717 6718 6719
	}

	return false;
}

6720 6721
int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
			    int emulation_type, void *insn, int insn_len)
6722
{
6723
	int r;
6724
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6725
	bool writeback = true;
6726
	bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
6727

P
Paolo Bonzini 已提交
6728 6729
	vcpu->arch.l1tf_flush_l1d = true;

6730 6731 6732 6733 6734
	/*
	 * Clear write_fault_to_shadow_pgtable here to ensure it is
	 * never reused.
	 */
	vcpu->arch.write_fault_to_shadow_pgtable = false;
6735
	kvm_clear_exception_queue(vcpu);
G
Gleb Natapov 已提交
6736

6737
	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
6738
		init_emulate_ctxt(vcpu);
6739 6740 6741 6742 6743 6744 6745

		/*
		 * We will reenter on the same instruction since
		 * we do not set complete_userspace_io.  This does not
		 * handle watchpoints yet, those would be handled in
		 * the emulate_ops.
		 */
6746 6747
		if (!(emulation_type & EMULTYPE_SKIP) &&
		    kvm_vcpu_check_breakpoint(vcpu, &r))
6748 6749
			return r;

6750 6751
		ctxt->interruptibility = 0;
		ctxt->have_exception = false;
6752
		ctxt->exception.vector = -1;
6753
		ctxt->perm_ok = false;
6754

6755
		ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
6756

6757
		r = x86_decode_insn(ctxt, insn, insn_len);
6758

A
Avi Kivity 已提交
6759
		trace_kvm_emulate_insn_start(vcpu);
6760
		++vcpu->stat.insn_emulation;
6761
		if (r != EMULATION_OK)  {
6762
			if ((emulation_type & EMULTYPE_TRAP_UD) ||
6763 6764
			    (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
				kvm_queue_exception(vcpu, UD_VECTOR);
6765
				return 1;
6766
			}
6767 6768 6769
			if (reexecute_instruction(vcpu, cr2_or_gpa,
						  write_fault_to_spt,
						  emulation_type))
6770
				return 1;
6771
			if (ctxt->have_exception) {
6772 6773 6774 6775 6776 6777
				/*
				 * #UD should result in just EMULATION_FAILED, and trap-like
				 * exception should not be encountered during decode.
				 */
				WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
					     exception_type(ctxt->exception.vector) == EXCPT_TRAP);
6778
				inject_emulated_exception(vcpu);
6779
				return 1;
6780
			}
6781
			return handle_emulation_failure(vcpu, emulation_type);
6782 6783 6784
		}
	}

6785 6786 6787
	if ((emulation_type & EMULTYPE_VMWARE_GP) &&
	    !is_vmware_backdoor_opcode(ctxt)) {
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6788
		return 1;
6789
	}
6790

6791 6792 6793 6794 6795
	/*
	 * Note, EMULTYPE_SKIP is intended for use *only* by vendor callbacks
	 * for kvm_skip_emulated_instruction().  The caller is responsible for
	 * updating interruptibility state and injecting single-step #DBs.
	 */
6796
	if (emulation_type & EMULTYPE_SKIP) {
6797
		kvm_rip_write(vcpu, ctxt->_eip);
6798 6799
		if (ctxt->eflags & X86_EFLAGS_RF)
			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
6800
		return 1;
6801 6802
	}

6803
	if (retry_instruction(ctxt, cr2_or_gpa, emulation_type))
6804
		return 1;
6805

6806
	/* this is needed for vmware backdoor interface to work since it
6807
	   changes registers values  during IO operation */
6808 6809
	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6810
		emulator_invalidate_register_cache(ctxt);
6811
	}
6812

6813
restart:
6814
	/* Save the faulting GPA (cr2) in the address field */
6815
	ctxt->exception.address = cr2_or_gpa;
6816

6817
	r = x86_emulate_insn(ctxt);
6818

6819
	if (r == EMULATION_INTERCEPTED)
6820
		return 1;
6821

6822
	if (r == EMULATION_FAILED) {
6823
		if (reexecute_instruction(vcpu, cr2_or_gpa, write_fault_to_spt,
6824
					emulation_type))
6825
			return 1;
6826

6827
		return handle_emulation_failure(vcpu, emulation_type);
6828 6829
	}

6830
	if (ctxt->have_exception) {
6831
		r = 1;
6832 6833
		if (inject_emulated_exception(vcpu))
			return r;
6834
	} else if (vcpu->arch.pio.count) {
6835 6836
		if (!vcpu->arch.pio.in) {
			/* FIXME: return into emulator if single-stepping.  */
6837
			vcpu->arch.pio.count = 0;
6838
		} else {
6839
			writeback = false;
6840 6841
			vcpu->arch.complete_userspace_io = complete_emulated_pio;
		}
6842
		r = 0;
6843
	} else if (vcpu->mmio_needed) {
6844 6845
		++vcpu->stat.mmio_exits;

6846 6847
		if (!vcpu->mmio_is_write)
			writeback = false;
6848
		r = 0;
6849
		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
6850
	} else if (r == EMULATION_RESTART)
6851
		goto restart;
6852
	else
6853
		r = 1;
6854

6855
	if (writeback) {
6856
		unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6857
		toggle_interruptibility(vcpu, ctxt->interruptibility);
6858
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
6859
		if (!ctxt->have_exception ||
6860 6861
		    exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
			kvm_rip_write(vcpu, ctxt->eip);
6862
			if (r && ctxt->tf)
6863
				r = kvm_vcpu_do_singlestep(vcpu);
6864
			__kvm_set_rflags(vcpu, ctxt->eflags);
6865
		}
6866 6867 6868 6869 6870 6871 6872 6873 6874

		/*
		 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
		 * do nothing, and it will be requested again as soon as
		 * the shadow expires.  But we still need to check here,
		 * because POPF has no interrupt shadow.
		 */
		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
			kvm_make_request(KVM_REQ_EVENT, vcpu);
6875 6876
	} else
		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
6877 6878

	return r;
6879
}
6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892

int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
{
	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction);

int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
					void *insn, int insn_len)
{
	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
6893

6894 6895 6896 6897 6898 6899
static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;
	return 1;
}

6900 6901 6902 6903 6904 6905 6906 6907 6908 6909
static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;

	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
		return 1;

	return kvm_skip_emulated_instruction(vcpu);
}

6910 6911
static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
			    unsigned short port)
6912
{
6913
	unsigned long val = kvm_rax_read(vcpu);
6914 6915
	int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
					    size, port, &val, 1);
6916 6917
	if (ret)
		return ret;
6918

6919 6920 6921 6922 6923 6924 6925 6926 6927 6928
	/*
	 * Workaround userspace that relies on old KVM behavior of %rip being
	 * incremented prior to exiting to userspace to handle "OUT 0x7e".
	 */
	if (port == 0x7e &&
	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
		vcpu->arch.complete_userspace_io =
			complete_fast_pio_out_port_0x7e;
		kvm_skip_emulated_instruction(vcpu);
	} else {
6929 6930 6931
		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
		vcpu->arch.complete_userspace_io = complete_fast_pio_out;
	}
6932
	return 0;
6933 6934
}

6935 6936 6937 6938 6939 6940 6941
static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
{
	unsigned long val;

	/* We should only ever be called with arch.pio.count equal to 1 */
	BUG_ON(vcpu->arch.pio.count != 1);

6942 6943 6944 6945 6946
	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
		vcpu->arch.pio.count = 0;
		return 1;
	}

6947
	/* For size less than 4 we merge, else we zero extend */
6948
	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
6949 6950 6951 6952 6953 6954 6955

	/*
	 * Since vcpu->arch.pio.count == 1 let emulator_pio_in_emulated perform
	 * the copy and tracing
	 */
	emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, vcpu->arch.pio.size,
				 vcpu->arch.pio.port, &val, 1);
6956
	kvm_rax_write(vcpu, val);
6957

6958
	return kvm_skip_emulated_instruction(vcpu);
6959 6960
}

6961 6962
static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
			   unsigned short port)
6963 6964 6965 6966 6967
{
	unsigned long val;
	int ret;

	/* For size less than 4 we merge, else we zero extend */
6968
	val = (size < 4) ? kvm_rax_read(vcpu) : 0;
6969 6970 6971 6972

	ret = emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, size, port,
				       &val, 1);
	if (ret) {
6973
		kvm_rax_write(vcpu, val);
6974 6975 6976
		return ret;
	}

6977
	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
6978 6979 6980 6981
	vcpu->arch.complete_userspace_io = complete_fast_pio_in;

	return 0;
}
6982 6983 6984

int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
{
6985
	int ret;
6986 6987

	if (in)
6988
		ret = kvm_fast_pio_in(vcpu, size, port);
6989
	else
6990 6991
		ret = kvm_fast_pio_out(vcpu, size, port);
	return ret && kvm_skip_emulated_instruction(vcpu);
6992 6993
}
EXPORT_SYMBOL_GPL(kvm_fast_pio);
6994

6995
static int kvmclock_cpu_down_prep(unsigned int cpu)
6996
{
T
Tejun Heo 已提交
6997
	__this_cpu_write(cpu_tsc_khz, 0);
6998
	return 0;
6999 7000 7001
}

static void tsc_khz_changed(void *data)
7002
{
7003 7004 7005 7006 7007 7008 7009 7010 7011
	struct cpufreq_freqs *freq = data;
	unsigned long khz = 0;

	if (data)
		khz = freq->new;
	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		khz = cpufreq_quick_get(raw_smp_processor_id());
	if (!khz)
		khz = tsc_khz;
T
Tejun Heo 已提交
7012
	__this_cpu_write(cpu_tsc_khz, khz);
7013 7014
}

7015
#ifdef CONFIG_X86_64
7016 7017 7018 7019 7020 7021
static void kvm_hyperv_tsc_notifier(void)
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int cpu;

J
Junaid Shahid 已提交
7022
	mutex_lock(&kvm_lock);
7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_make_mclock_inprogress_request(kvm);

	hyperv_stop_tsc_emulation();

	/* TSC frequency always matches when on Hyper-V */
	for_each_present_cpu(cpu)
		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
	kvm_max_guest_tsc_khz = tsc_khz;

	list_for_each_entry(kvm, &vm_list, vm_list) {
		struct kvm_arch *ka = &kvm->arch;

		spin_lock(&ka->pvclock_gtod_sync_lock);

		pvclock_update_vm_gtod_copy(kvm);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);

		spin_unlock(&ka->pvclock_gtod_sync_lock);
	}
J
Junaid Shahid 已提交
7048
	mutex_unlock(&kvm_lock);
7049
}
7050
#endif
7051

7052
static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
7053 7054 7055 7056 7057
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i, send_ipi = 0;

7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096
	/*
	 * We allow guests to temporarily run on slowing clocks,
	 * provided we notify them after, or to run on accelerating
	 * clocks, provided we notify them before.  Thus time never
	 * goes backwards.
	 *
	 * However, we have a problem.  We can't atomically update
	 * the frequency of a given CPU from this function; it is
	 * merely a notifier, which can be called from any CPU.
	 * Changing the TSC frequency at arbitrary points in time
	 * requires a recomputation of local variables related to
	 * the TSC for each VCPU.  We must flag these local variables
	 * to be updated and be sure the update takes place with the
	 * new frequency before any guests proceed.
	 *
	 * Unfortunately, the combination of hotplug CPU and frequency
	 * change creates an intractable locking scenario; the order
	 * of when these callouts happen is undefined with respect to
	 * CPU hotplug, and they can race with each other.  As such,
	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
	 * undefined; you can actually have a CPU frequency change take
	 * place in between the computation of X and the setting of the
	 * variable.  To protect against this problem, all updates of
	 * the per_cpu tsc_khz variable are done in an interrupt
	 * protected IPI, and all callers wishing to update the value
	 * must wait for a synchronous IPI to complete (which is trivial
	 * if the caller is on the CPU already).  This establishes the
	 * necessary total order on variable updates.
	 *
	 * Note that because a guest time update may take place
	 * anytime after the setting of the VCPU's request bit, the
	 * correct TSC value must be set before the request.  However,
	 * to ensure the update actually makes it to any guest which
	 * starts running in hardware virtualization between the set
	 * and the acquisition of the spinlock, we must also ping the
	 * CPU after setting the request bit.
	 *
	 */

7097
	smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
7098

J
Junaid Shahid 已提交
7099
	mutex_lock(&kvm_lock);
7100
	list_for_each_entry(kvm, &vm_list, vm_list) {
7101
		kvm_for_each_vcpu(i, vcpu, kvm) {
7102
			if (vcpu->cpu != cpu)
7103
				continue;
Z
Zachary Amsden 已提交
7104
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
J
Junaid Shahid 已提交
7105
			if (vcpu->cpu != raw_smp_processor_id())
7106
				send_ipi = 1;
7107 7108
		}
	}
J
Junaid Shahid 已提交
7109
	mutex_unlock(&kvm_lock);
7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123

	if (freq->old < freq->new && send_ipi) {
		/*
		 * We upscale the frequency.  Must make the guest
		 * doesn't see old kvmclock values while running with
		 * the new frequency, otherwise we risk the guest sees
		 * time go backwards.
		 *
		 * In case we update the frequency for another cpu
		 * (which might be in guest context) send an interrupt
		 * to kick the cpu out of guest context.  Next time
		 * guest context is entered kvmclock will be updated,
		 * so the guest will not see stale values.
		 */
7124
		smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
7125
	}
7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141
}

static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
				     void *data)
{
	struct cpufreq_freqs *freq = data;
	int cpu;

	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
		return 0;
	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
		return 0;

	for_each_cpu(cpu, freq->policy->cpus)
		__kvmclock_cpufreq_notifier(freq, cpu);

7142 7143 7144 7145
	return 0;
}

static struct notifier_block kvmclock_cpufreq_notifier_block = {
7146 7147 7148
	.notifier_call  = kvmclock_cpufreq_notifier
};

7149
static int kvmclock_cpu_online(unsigned int cpu)
7150
{
7151 7152
	tsc_khz_changed(NULL);
	return 0;
7153 7154
}

7155 7156
static void kvm_timer_init(void)
{
Z
Zachary Amsden 已提交
7157
	max_tsc_khz = tsc_khz;
7158

7159
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
Z
Zachary Amsden 已提交
7160 7161
#ifdef CONFIG_CPU_FREQ
		struct cpufreq_policy policy;
7162 7163
		int cpu;

Z
Zachary Amsden 已提交
7164
		memset(&policy, 0, sizeof(policy));
7165 7166
		cpu = get_cpu();
		cpufreq_get_policy(&policy, cpu);
Z
Zachary Amsden 已提交
7167 7168
		if (policy.cpuinfo.max_freq)
			max_tsc_khz = policy.cpuinfo.max_freq;
7169
		put_cpu();
Z
Zachary Amsden 已提交
7170
#endif
7171 7172 7173
		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
					  CPUFREQ_TRANSITION_NOTIFIER);
	}
7174

T
Thomas Gleixner 已提交
7175
	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
7176
			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
7177 7178
}

7179 7180
DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu);
7181

7182
int kvm_is_in_guest(void)
7183
{
7184
	return __this_cpu_read(current_vcpu) != NULL;
7185 7186 7187 7188 7189
}

static int kvm_is_user_mode(void)
{
	int user_mode = 3;
7190

7191 7192
	if (__this_cpu_read(current_vcpu))
		user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
7193

7194 7195 7196 7197 7198 7199
	return user_mode != 0;
}

static unsigned long kvm_get_guest_ip(void)
{
	unsigned long ip = 0;
7200

7201 7202
	if (__this_cpu_read(current_vcpu))
		ip = kvm_rip_read(__this_cpu_read(current_vcpu));
7203

7204 7205 7206
	return ip;
}

L
Luwei Kang 已提交
7207 7208 7209 7210 7211 7212 7213 7214 7215
static void kvm_handle_intel_pt_intr(void)
{
	struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);

	kvm_make_request(KVM_REQ_PMI, vcpu);
	__set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
			(unsigned long *)&vcpu->arch.pmu.global_status);
}

7216 7217 7218 7219
static struct perf_guest_info_callbacks kvm_guest_cbs = {
	.is_in_guest		= kvm_is_in_guest,
	.is_user_mode		= kvm_is_user_mode,
	.get_guest_ip		= kvm_get_guest_ip,
L
Luwei Kang 已提交
7220
	.handle_intel_pt_intr	= kvm_handle_intel_pt_intr,
7221 7222
};

7223 7224 7225
#ifdef CONFIG_X86_64
static void pvclock_gtod_update_fn(struct work_struct *work)
{
7226 7227 7228 7229 7230
	struct kvm *kvm;

	struct kvm_vcpu *vcpu;
	int i;

J
Junaid Shahid 已提交
7231
	mutex_lock(&kvm_lock);
7232 7233
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_for_each_vcpu(i, vcpu, kvm)
7234
			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
7235
	atomic_set(&kvm_guest_has_master_clock, 0);
J
Junaid Shahid 已提交
7236
	mutex_unlock(&kvm_lock);
7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252
}

static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);

/*
 * Notification about pvclock gtod data update.
 */
static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
			       void *priv)
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	struct timekeeper *tk = priv;

	update_pvclock_gtod(tk);

	/* disable master clock if host does not trust, or does not
7253
	 * use, TSC based clocksource.
7254
	 */
7255
	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266
	    atomic_read(&kvm_guest_has_master_clock) != 0)
		queue_work(system_long_wq, &pvclock_gtod_work);

	return 0;
}

static struct notifier_block pvclock_gtod_notifier = {
	.notifier_call = pvclock_gtod_notify,
};
#endif

7267
int kvm_arch_init(void *opaque)
7268
{
7269
	int r;
M
Mathias Krause 已提交
7270
	struct kvm_x86_ops *ops = opaque;
7271 7272 7273

	if (kvm_x86_ops) {
		printk(KERN_ERR "kvm: already loaded the other module\n");
7274 7275
		r = -EEXIST;
		goto out;
7276 7277 7278 7279
	}

	if (!ops->cpu_has_kvm_support()) {
		printk(KERN_ERR "kvm: no hardware support\n");
7280 7281
		r = -EOPNOTSUPP;
		goto out;
7282 7283 7284
	}
	if (ops->disabled_by_bios()) {
		printk(KERN_ERR "kvm: disabled by bios\n");
7285 7286
		r = -EOPNOTSUPP;
		goto out;
7287 7288
	}

7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299
	/*
	 * KVM explicitly assumes that the guest has an FPU and
	 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
	 * vCPU's FPU state as a fxregs_state struct.
	 */
	if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
		printk(KERN_ERR "kvm: inadequate fpu\n");
		r = -EOPNOTSUPP;
		goto out;
	}

7300
	r = -ENOMEM;
7301
	x86_fpu_cache = kmem_cache_create("x86_fpu", sizeof(struct fpu),
7302 7303 7304 7305 7306 7307 7308
					  __alignof__(struct fpu), SLAB_ACCOUNT,
					  NULL);
	if (!x86_fpu_cache) {
		printk(KERN_ERR "kvm: failed to allocate cache for x86 fpu\n");
		goto out;
	}

7309 7310 7311
	shared_msrs = alloc_percpu(struct kvm_shared_msrs);
	if (!shared_msrs) {
		printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
7312
		goto out_free_x86_fpu_cache;
7313 7314
	}

7315 7316
	r = kvm_mmu_module_init();
	if (r)
7317
		goto out_free_percpu;
7318

7319
	kvm_x86_ops = ops;
P
Paolo Bonzini 已提交
7320

S
Sheng Yang 已提交
7321
	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
7322
			PT_DIRTY_MASK, PT64_NX_MASK, 0,
7323
			PT_PRESENT_MASK, 0, sme_me_mask);
7324
	kvm_timer_init();
7325

7326 7327
	perf_register_guest_info_callbacks(&kvm_guest_cbs);

7328
	if (boot_cpu_has(X86_FEATURE_XSAVE))
7329 7330
		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);

7331
	kvm_lapic_init();
7332 7333
	if (pi_inject_timer == -1)
		pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);
7334 7335
#ifdef CONFIG_X86_64
	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
7336

7337
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7338
		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
7339 7340
#endif

7341
	return 0;
7342

7343 7344
out_free_percpu:
	free_percpu(shared_msrs);
7345 7346
out_free_x86_fpu_cache:
	kmem_cache_destroy(x86_fpu_cache);
7347 7348
out:
	return r;
7349
}
7350

7351 7352
void kvm_arch_exit(void)
{
7353
#ifdef CONFIG_X86_64
7354
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7355 7356
		clear_hv_tscchange_cb();
#endif
7357
	kvm_lapic_exit();
7358 7359
	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);

7360 7361 7362
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
					    CPUFREQ_TRANSITION_NOTIFIER);
7363
	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
7364 7365 7366
#ifdef CONFIG_X86_64
	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
#endif
7367
	kvm_x86_ops = NULL;
7368
	kvm_mmu_module_exit();
7369
	free_percpu(shared_msrs);
7370
	kmem_cache_destroy(x86_fpu_cache);
7371
}
7372

7373
int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
7374 7375
{
	++vcpu->stat.halt_exits;
7376
	if (lapic_in_kernel(vcpu)) {
7377
		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
7378 7379 7380 7381 7382 7383
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
}
7384 7385 7386 7387
EXPORT_SYMBOL_GPL(kvm_vcpu_halt);

int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
7388 7389 7390 7391 7392 7393
	int ret = kvm_skip_emulated_instruction(vcpu);
	/*
	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
	 * KVM_EXIT_DEBUG here.
	 */
	return kvm_vcpu_halt(vcpu) && ret;
7394
}
7395 7396
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

7397
#ifdef CONFIG_X86_64
7398 7399 7400 7401
static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
			        unsigned long clock_type)
{
	struct kvm_clock_pairing clock_pairing;
7402
	struct timespec64 ts;
P
Paolo Bonzini 已提交
7403
	u64 cycle;
7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415
	int ret;

	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
		return -KVM_EOPNOTSUPP;

	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)
		return -KVM_EOPNOTSUPP;

	clock_pairing.sec = ts.tv_sec;
	clock_pairing.nsec = ts.tv_nsec;
	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
	clock_pairing.flags = 0;
7416
	memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
7417 7418 7419 7420 7421 7422 7423 7424

	ret = 0;
	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
			    sizeof(struct kvm_clock_pairing)))
		ret = -KVM_EFAULT;

	return ret;
}
7425
#endif
7426

7427 7428 7429 7430 7431 7432 7433
/*
 * kvm_pv_kick_cpu_op:  Kick a vcpu.
 *
 * @apicid - apicid of vcpu to be kicked.
 */
static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
{
7434
	struct kvm_lapic_irq lapic_irq;
7435

7436
	lapic_irq.shorthand = APIC_DEST_NOSHORT;
7437
	lapic_irq.dest_mode = APIC_DEST_PHYSICAL;
7438
	lapic_irq.level = 0;
7439
	lapic_irq.dest_id = apicid;
7440
	lapic_irq.msi_redir_hint = false;
7441

7442
	lapic_irq.delivery_mode = APIC_DM_REMRD;
7443
	kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
7444 7445
}

7446 7447
void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu)
{
7448 7449 7450 7451 7452 7453 7454
	if (!lapic_in_kernel(vcpu)) {
		WARN_ON_ONCE(vcpu->arch.apicv_active);
		return;
	}
	if (!vcpu->arch.apicv_active)
		return;

7455 7456 7457 7458
	vcpu->arch.apicv_active = false;
	kvm_x86_ops->refresh_apicv_exec_ctrl(vcpu);
}

7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471
static void kvm_sched_yield(struct kvm *kvm, unsigned long dest_id)
{
	struct kvm_vcpu *target = NULL;
	struct kvm_apic_map *map;

	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);

	if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])
		target = map->phys_map[dest_id]->vcpu;

	rcu_read_unlock();

7472
	if (target && READ_ONCE(target->ready))
7473 7474 7475
		kvm_vcpu_yield_to(target);
}

7476 7477 7478
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
{
	unsigned long nr, a0, a1, a2, a3, ret;
7479
	int op_64_bit;
7480

7481 7482
	if (kvm_hv_hypercall_enabled(vcpu->kvm))
		return kvm_hv_hypercall(vcpu);
7483

7484 7485 7486 7487 7488
	nr = kvm_rax_read(vcpu);
	a0 = kvm_rbx_read(vcpu);
	a1 = kvm_rcx_read(vcpu);
	a2 = kvm_rdx_read(vcpu);
	a3 = kvm_rsi_read(vcpu);
7489

7490
	trace_kvm_hypercall(nr, a0, a1, a2, a3);
F
Feng (Eric) Liu 已提交
7491

7492 7493
	op_64_bit = is_64_bit_mode(vcpu);
	if (!op_64_bit) {
7494 7495 7496 7497 7498 7499 7500
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
	}

7501 7502
	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
		ret = -KVM_EPERM;
7503
		goto out;
7504 7505
	}

7506
	switch (nr) {
A
Avi Kivity 已提交
7507 7508 7509
	case KVM_HC_VAPIC_POLL_IRQ:
		ret = 0;
		break;
7510 7511
	case KVM_HC_KICK_CPU:
		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
7512
		kvm_sched_yield(vcpu->kvm, a1);
7513 7514
		ret = 0;
		break;
7515
#ifdef CONFIG_X86_64
7516 7517 7518
	case KVM_HC_CLOCK_PAIRING:
		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
		break;
7519
#endif
7520 7521 7522
	case KVM_HC_SEND_IPI:
		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
		break;
7523 7524 7525 7526
	case KVM_HC_SCHED_YIELD:
		kvm_sched_yield(vcpu->kvm, a0);
		ret = 0;
		break;
7527 7528 7529 7530
	default:
		ret = -KVM_ENOSYS;
		break;
	}
7531
out:
7532 7533
	if (!op_64_bit)
		ret = (u32)ret;
7534
	kvm_rax_write(vcpu, ret);
7535

A
Amit Shah 已提交
7536
	++vcpu->stat.hypercalls;
7537
	return kvm_skip_emulated_instruction(vcpu);
7538 7539 7540
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

7541
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
7542
{
7543
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7544
	char instruction[3];
7545
	unsigned long rip = kvm_rip_read(vcpu);
7546 7547 7548

	kvm_x86_ops->patch_hypercall(vcpu, instruction);

7549 7550
	return emulator_write_emulated(ctxt, rip, instruction, 3,
		&ctxt->exception);
7551 7552
}

A
Avi Kivity 已提交
7553
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
7554
{
7555 7556
	return vcpu->run->request_interrupt_window &&
		likely(!pic_in_kernel(vcpu->kvm));
7557 7558
}

A
Avi Kivity 已提交
7559
static void post_kvm_run_save(struct kvm_vcpu *vcpu)
7560
{
A
Avi Kivity 已提交
7561 7562
	struct kvm_run *kvm_run = vcpu->run;

7563
	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
7564
	kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;
7565
	kvm_run->cr8 = kvm_get_cr8(vcpu);
7566
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
7567 7568
	kvm_run->ready_for_interrupt_injection =
		pic_in_kernel(vcpu->kvm) ||
7569
		kvm_vcpu_ready_for_interrupt_injection(vcpu);
7570 7571
}

7572 7573 7574 7575 7576 7577 7578
static void update_cr8_intercept(struct kvm_vcpu *vcpu)
{
	int max_irr, tpr;

	if (!kvm_x86_ops->update_cr8_intercept)
		return;

7579
	if (!lapic_in_kernel(vcpu))
7580 7581
		return;

7582 7583 7584
	if (vcpu->arch.apicv_active)
		return;

7585 7586 7587 7588
	if (!vcpu->arch.apic->vapic_addr)
		max_irr = kvm_lapic_find_highest_irr(vcpu);
	else
		max_irr = -1;
7589 7590 7591 7592 7593 7594 7595 7596 7597

	if (max_irr != -1)
		max_irr >>= 4;

	tpr = kvm_lapic_get_cr8(vcpu);

	kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
}

7598
static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win)
7599
{
7600 7601
	int r;

7602
	/* try to reinject previous events if any */
7603

7604 7605
	if (vcpu->arch.exception.injected)
		kvm_x86_ops->queue_exception(vcpu);
7606
	/*
7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618
	 * Do not inject an NMI or interrupt if there is a pending
	 * exception.  Exceptions and interrupts are recognized at
	 * instruction boundaries, i.e. the start of an instruction.
	 * Trap-like exceptions, e.g. #DB, have higher priority than
	 * NMIs and interrupts, i.e. traps are recognized before an
	 * NMI/interrupt that's pending on the same instruction.
	 * Fault-like exceptions, e.g. #GP and #PF, are the lowest
	 * priority, but are only generated (pended) during instruction
	 * execution, i.e. a pending fault-like exception means the
	 * fault occurred on the *previous* instruction and must be
	 * serviced prior to recognizing any new events in order to
	 * fully complete the previous instruction.
7619
	 */
7620 7621
	else if (!vcpu->arch.exception.pending) {
		if (vcpu->arch.nmi_injected)
7622
			kvm_x86_ops->set_nmi(vcpu);
7623
		else if (vcpu->arch.interrupt.injected)
7624 7625 7626
			kvm_x86_ops->set_irq(vcpu);
	}

7627 7628 7629 7630 7631 7632
	/*
	 * Call check_nested_events() even if we reinjected a previous event
	 * in order for caller to determine if it should require immediate-exit
	 * from L2 to L1 due to pending L1 events which require exit
	 * from L2 to L1.
	 */
7633 7634 7635 7636 7637 7638 7639
	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
		r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
		if (r != 0)
			return r;
	}

	/* try to inject new event if pending */
7640
	if (vcpu->arch.exception.pending) {
A
Avi Kivity 已提交
7641 7642 7643
		trace_kvm_inj_exception(vcpu->arch.exception.nr,
					vcpu->arch.exception.has_error_code,
					vcpu->arch.exception.error_code);
7644

7645
		WARN_ON_ONCE(vcpu->arch.exception.injected);
7646 7647 7648
		vcpu->arch.exception.pending = false;
		vcpu->arch.exception.injected = true;

7649 7650 7651 7652
		if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
					     X86_EFLAGS_RF);

7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668
		if (vcpu->arch.exception.nr == DB_VECTOR) {
			/*
			 * This code assumes that nSVM doesn't use
			 * check_nested_events(). If it does, the
			 * DR6/DR7 changes should happen before L1
			 * gets a #VMEXIT for an intercepted #DB in
			 * L2.  (Under VMX, on the other hand, the
			 * DR6/DR7 changes should not happen in the
			 * event of a VM-exit to L1 for an intercepted
			 * #DB in L2.)
			 */
			kvm_deliver_exception_payload(vcpu);
			if (vcpu->arch.dr7 & DR7_GD) {
				vcpu->arch.dr7 &= ~DR7_GD;
				kvm_update_dr7(vcpu);
			}
7669 7670
		}

7671
		kvm_x86_ops->queue_exception(vcpu);
7672 7673 7674 7675 7676 7677 7678 7679
	}

	/* Don't consider new event if we re-injected an event */
	if (kvm_event_needs_reinjection(vcpu))
		return 0;

	if (vcpu->arch.smi_pending && !is_smm(vcpu) &&
	    kvm_x86_ops->smi_allowed(vcpu)) {
7680
		vcpu->arch.smi_pending = false;
7681
		++vcpu->arch.smi_count;
7682
		enter_smm(vcpu);
7683
	} else if (vcpu->arch.nmi_pending && kvm_x86_ops->nmi_allowed(vcpu)) {
7684 7685 7686
		--vcpu->arch.nmi_pending;
		vcpu->arch.nmi_injected = true;
		kvm_x86_ops->set_nmi(vcpu);
7687
	} else if (kvm_cpu_has_injectable_intr(vcpu)) {
7688 7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699
		/*
		 * Because interrupts can be injected asynchronously, we are
		 * calling check_nested_events again here to avoid a race condition.
		 * See https://lkml.org/lkml/2014/7/2/60 for discussion about this
		 * proposal and current concerns.  Perhaps we should be setting
		 * KVM_REQ_EVENT only on certain events and not unconditionally?
		 */
		if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
			r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
			if (r != 0)
				return r;
		}
7700
		if (kvm_x86_ops->interrupt_allowed(vcpu)) {
7701 7702 7703
			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
					    false);
			kvm_x86_ops->set_irq(vcpu);
7704 7705
		}
	}
7706

7707
	return 0;
7708 7709
}

A
Avi Kivity 已提交
7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723 7724 7725 7726
static void process_nmi(struct kvm_vcpu *vcpu)
{
	unsigned limit = 2;

	/*
	 * x86 is limited to one NMI running, and one NMI pending after it.
	 * If an NMI is already in progress, limit further NMIs to just one.
	 * Otherwise, allow two (and we'll inject the first one immediately).
	 */
	if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
		limit = 1;

	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

7727
static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738 7739 7740
{
	u32 flags = 0;
	flags |= seg->g       << 23;
	flags |= seg->db      << 22;
	flags |= seg->l       << 21;
	flags |= seg->avl     << 20;
	flags |= seg->present << 15;
	flags |= seg->dpl     << 13;
	flags |= seg->s       << 12;
	flags |= seg->type    << 8;
	return flags;
}

7741
static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
7742 7743 7744 7745 7746 7747 7748 7749 7750 7751 7752 7753 7754 7755
{
	struct kvm_segment seg;
	int offset;

	kvm_get_segment(vcpu, &seg, n);
	put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector);

	if (n < 3)
		offset = 0x7f84 + n * 12;
	else
		offset = 0x7f2c + (n - 3) * 12;

	put_smstate(u32, buf, offset + 8, seg.base);
	put_smstate(u32, buf, offset + 4, seg.limit);
7756
	put_smstate(u32, buf, offset, enter_smm_get_segment_flags(&seg));
7757 7758
}

7759
#ifdef CONFIG_X86_64
7760
static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
7761 7762 7763 7764 7765 7766 7767 7768
{
	struct kvm_segment seg;
	int offset;
	u16 flags;

	kvm_get_segment(vcpu, &seg, n);
	offset = 0x7e00 + n * 16;

7769
	flags = enter_smm_get_segment_flags(&seg) >> 8;
7770 7771 7772 7773 7774
	put_smstate(u16, buf, offset, seg.selector);
	put_smstate(u16, buf, offset + 2, flags);
	put_smstate(u32, buf, offset + 4, seg.limit);
	put_smstate(u64, buf, offset + 8, seg.base);
}
7775
#endif
7776

7777
static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
	put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
	put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
	put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));

	for (i = 0; i < 8; i++)
		put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read(vcpu, i));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u32, buf, 0x7fcc, (u32)val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u32, buf, 0x7fc8, (u32)val);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u32, buf, 0x7fc4, seg.selector);
	put_smstate(u32, buf, 0x7f64, seg.base);
	put_smstate(u32, buf, 0x7f60, seg.limit);
7801
	put_smstate(u32, buf, 0x7f5c, enter_smm_get_segment_flags(&seg));
7802 7803 7804 7805 7806

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u32, buf, 0x7fc0, seg.selector);
	put_smstate(u32, buf, 0x7f80, seg.base);
	put_smstate(u32, buf, 0x7f7c, seg.limit);
7807
	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
7808 7809 7810 7811 7812 7813 7814 7815 7816 7817

	kvm_x86_ops->get_gdt(vcpu, &dt);
	put_smstate(u32, buf, 0x7f74, dt.address);
	put_smstate(u32, buf, 0x7f70, dt.size);

	kvm_x86_ops->get_idt(vcpu, &dt);
	put_smstate(u32, buf, 0x7f58, dt.address);
	put_smstate(u32, buf, 0x7f54, dt.size);

	for (i = 0; i < 6; i++)
7818
		enter_smm_save_seg_32(vcpu, buf, i);
7819 7820 7821 7822 7823 7824 7825 7826

	put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020000);
	put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
}

7827
#ifdef CONFIG_X86_64
7828
static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	for (i = 0; i < 16; i++)
		put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read(vcpu, i));

	put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
	put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u64, buf, 0x7f68, val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u64, buf, 0x7f60, val);

	put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
	put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
	put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));

	put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020064);

	put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u16, buf, 0x7e90, seg.selector);
7859
	put_smstate(u16, buf, 0x7e92, enter_smm_get_segment_flags(&seg) >> 8);
7860 7861 7862 7863 7864 7865 7866 7867 7868
	put_smstate(u32, buf, 0x7e94, seg.limit);
	put_smstate(u64, buf, 0x7e98, seg.base);

	kvm_x86_ops->get_idt(vcpu, &dt);
	put_smstate(u32, buf, 0x7e84, dt.size);
	put_smstate(u64, buf, 0x7e88, dt.address);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u16, buf, 0x7e70, seg.selector);
7869
	put_smstate(u16, buf, 0x7e72, enter_smm_get_segment_flags(&seg) >> 8);
7870 7871 7872 7873 7874 7875 7876 7877
	put_smstate(u32, buf, 0x7e74, seg.limit);
	put_smstate(u64, buf, 0x7e78, seg.base);

	kvm_x86_ops->get_gdt(vcpu, &dt);
	put_smstate(u32, buf, 0x7e64, dt.size);
	put_smstate(u64, buf, 0x7e68, dt.address);

	for (i = 0; i < 6; i++)
7878
		enter_smm_save_seg_64(vcpu, buf, i);
7879
}
7880
#endif
7881

7882
static void enter_smm(struct kvm_vcpu *vcpu)
P
Paolo Bonzini 已提交
7883
{
7884
	struct kvm_segment cs, ds;
7885
	struct desc_ptr dt;
7886 7887 7888 7889 7890
	char buf[512];
	u32 cr0;

	trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true);
	memset(buf, 0, 512);
7891
#ifdef CONFIG_X86_64
7892
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7893
		enter_smm_save_state_64(vcpu, buf);
7894
	else
7895
#endif
7896
		enter_smm_save_state_32(vcpu, buf);
7897

7898 7899 7900 7901 7902 7903 7904 7905
	/*
	 * Give pre_enter_smm() a chance to make ISA-specific changes to the
	 * vCPU state (e.g. leave guest mode) after we've saved the state into
	 * the SMM state-save area.
	 */
	kvm_x86_ops->pre_enter_smm(vcpu, buf);

	vcpu->arch.hflags |= HF_SMM_MASK;
7906
	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
7907 7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921

	if (kvm_x86_ops->get_nmi_mask(vcpu))
		vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
	else
		kvm_x86_ops->set_nmi_mask(vcpu, true);

	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
	kvm_rip_write(vcpu, 0x8000);

	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
	kvm_x86_ops->set_cr0(vcpu, cr0);
	vcpu->arch.cr0 = cr0;

	kvm_x86_ops->set_cr4(vcpu, 0);

7922 7923 7924 7925
	/* Undocumented: IDT limit is set to zero on entry to SMM.  */
	dt.address = dt.size = 0;
	kvm_x86_ops->set_idt(vcpu, &dt);

7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952
	__kvm_set_dr(vcpu, 7, DR7_FIXED_1);

	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
	cs.base = vcpu->arch.smbase;

	ds.selector = 0;
	ds.base = 0;

	cs.limit    = ds.limit = 0xffffffff;
	cs.type     = ds.type = 0x3;
	cs.dpl      = ds.dpl = 0;
	cs.db       = ds.db = 0;
	cs.s        = ds.s = 1;
	cs.l        = ds.l = 0;
	cs.g        = ds.g = 1;
	cs.avl      = ds.avl = 0;
	cs.present  = ds.present = 1;
	cs.unusable = ds.unusable = 0;
	cs.padding  = ds.padding = 0;

	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);

7953
#ifdef CONFIG_X86_64
7954
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7955
		kvm_x86_ops->set_efer(vcpu, 0);
7956
#endif
7957 7958 7959

	kvm_update_cpuid(vcpu);
	kvm_mmu_reset_context(vcpu);
P
Paolo Bonzini 已提交
7960 7961
}

7962
static void process_smi(struct kvm_vcpu *vcpu)
7963 7964 7965 7966 7967
{
	vcpu->arch.smi_pending = true;
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

7968 7969 7970 7971 7972 7973 7974
void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
				       unsigned long *vcpu_bitmap)
{
	cpumask_var_t cpus;

	zalloc_cpumask_var(&cpus, GFP_ATOMIC);

7975 7976
	kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC,
				    vcpu_bitmap, cpus);
7977 7978 7979 7980

	free_cpumask_var(cpus);
}

7981 7982 7983 7984 7985
void kvm_make_scan_ioapic_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
}

7986
static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
7987
{
7988
	if (!kvm_apic_present(vcpu))
7989
		return;
7990

7991
	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
7992

7993
	if (irqchip_split(vcpu->kvm))
7994
		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
7995
	else {
7996
		if (vcpu->arch.apicv_active)
7997
			kvm_x86_ops->sync_pir_to_irr(vcpu);
7998 7999
		if (ioapic_in_kernel(vcpu->kvm))
			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
8000
	}
8001 8002 8003 8004 8005 8006 8007 8008 8009 8010 8011 8012 8013 8014

	if (is_guest_mode(vcpu))
		vcpu->arch.load_eoi_exitmap_pending = true;
	else
		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
}

static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
{
	u64 eoi_exit_bitmap[4];

	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
		return;

8015 8016 8017
	bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
		  vcpu_to_synic(vcpu)->vec_bitmap, 256);
	kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
8018 8019
}

8020 8021 8022
int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
		unsigned long start, unsigned long end,
		bool blockable)
8023 8024 8025 8026 8027 8028 8029 8030 8031 8032
{
	unsigned long apic_address;

	/*
	 * The physical address of apic access page is stored in the VMCS.
	 * Update it when it becomes invalid.
	 */
	apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
	if (start <= apic_address && apic_address < end)
		kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
8033 8034

	return 0;
8035 8036
}

8037 8038
void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
{
8039 8040
	struct page *page = NULL;

8041
	if (!lapic_in_kernel(vcpu))
8042 8043
		return;

8044 8045 8046
	if (!kvm_x86_ops->set_apic_access_page_addr)
		return;

8047
	page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
8048 8049
	if (is_error_page(page))
		return;
8050 8051 8052 8053 8054 8055 8056
	kvm_x86_ops->set_apic_access_page_addr(vcpu, page_to_phys(page));

	/*
	 * Do not pin apic access page in memory, the MMU notifier
	 * will call us again if it is migrated or swapped out.
	 */
	put_page(page);
8057 8058
}

8059 8060 8061 8062 8063 8064
void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
{
	smp_send_reschedule(vcpu->cpu);
}
EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);

8065
/*
8066
 * Returns 1 to let vcpu_run() continue the guest execution loop without
8067 8068 8069
 * exiting to the userspace.  Otherwise, the value will be returned to the
 * userspace.
 */
A
Avi Kivity 已提交
8070
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
8071 8072
{
	int r;
8073 8074 8075
	bool req_int_win =
		dm_request_for_irq_injection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);
8076
	enum exit_fastpath_completion exit_fastpath = EXIT_FASTPATH_NONE;
8077

8078
	bool req_immediate_exit = false;
8079

R
Radim Krčmář 已提交
8080
	if (kvm_request_pending(vcpu)) {
8081 8082 8083 8084 8085 8086
		if (kvm_check_request(KVM_REQ_GET_VMCS12_PAGES, vcpu)) {
			if (unlikely(!kvm_x86_ops->get_vmcs12_pages(vcpu))) {
				r = 0;
				goto out;
			}
		}
8087
		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
8088
			kvm_mmu_unload(vcpu);
8089
		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
M
Marcelo Tosatti 已提交
8090
			__kvm_migrate_timers(vcpu);
8091 8092
		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
			kvm_gen_update_masterclock(vcpu->kvm);
8093 8094
		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
			kvm_gen_kvmclock_update(vcpu);
Z
Zachary Amsden 已提交
8095 8096
		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
			r = kvm_guest_time_update(vcpu);
8097 8098 8099
			if (unlikely(r))
				goto out;
		}
8100
		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
8101
			kvm_mmu_sync_roots(vcpu);
8102 8103
		if (kvm_check_request(KVM_REQ_LOAD_CR3, vcpu))
			kvm_mmu_load_cr3(vcpu);
8104
		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
8105
			kvm_vcpu_flush_tlb(vcpu, true);
8106
		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
A
Avi Kivity 已提交
8107
			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
A
Avi Kivity 已提交
8108 8109 8110
			r = 0;
			goto out;
		}
8111
		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
A
Avi Kivity 已提交
8112
			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
8113
			vcpu->mmio_needed = 0;
J
Joerg Roedel 已提交
8114 8115 8116
			r = 0;
			goto out;
		}
8117 8118 8119 8120 8121 8122
		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
			/* Page is swapped out. Do synthetic halt */
			vcpu->arch.apf.halted = true;
			r = 1;
			goto out;
		}
G
Glauber Costa 已提交
8123 8124
		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
			record_steal_time(vcpu);
P
Paolo Bonzini 已提交
8125 8126
		if (kvm_check_request(KVM_REQ_SMI, vcpu))
			process_smi(vcpu);
A
Avi Kivity 已提交
8127 8128
		if (kvm_check_request(KVM_REQ_NMI, vcpu))
			process_nmi(vcpu);
8129
		if (kvm_check_request(KVM_REQ_PMU, vcpu))
8130
			kvm_pmu_handle_event(vcpu);
8131
		if (kvm_check_request(KVM_REQ_PMI, vcpu))
8132
			kvm_pmu_deliver_pmi(vcpu);
8133 8134 8135
		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
			if (test_bit(vcpu->arch.pending_ioapic_eoi,
8136
				     vcpu->arch.ioapic_handled_vectors)) {
8137 8138 8139 8140 8141 8142 8143
				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
				vcpu->run->eoi.vector =
						vcpu->arch.pending_ioapic_eoi;
				r = 0;
				goto out;
			}
		}
8144 8145
		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
			vcpu_scan_ioapic(vcpu);
8146 8147
		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
			vcpu_load_eoi_exitmap(vcpu);
8148 8149
		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
			kvm_vcpu_reload_apic_access_page(vcpu);
8150 8151 8152 8153 8154 8155
		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
			r = 0;
			goto out;
		}
8156 8157 8158 8159 8160 8161
		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
			r = 0;
			goto out;
		}
A
Andrey Smetanin 已提交
8162 8163 8164 8165 8166 8167
		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_HYPERV;
			vcpu->run->hyperv = vcpu->arch.hyperv.exit;
			r = 0;
			goto out;
		}
8168 8169 8170 8171 8172 8173

		/*
		 * KVM_REQ_HV_STIMER has to be processed after
		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
		 * depend on the guest clock being up-to-date
		 */
A
Andrey Smetanin 已提交
8174 8175
		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
			kvm_hv_process_stimers(vcpu);
8176
	}
A
Avi Kivity 已提交
8177

A
Avi Kivity 已提交
8178
	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
8179
		++vcpu->stat.req_event;
8180 8181 8182 8183 8184 8185
		kvm_apic_accept_events(vcpu);
		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
			r = 1;
			goto out;
		}

8186 8187
		if (inject_pending_event(vcpu, req_int_win) != 0)
			req_immediate_exit = true;
8188
		else {
8189
			/* Enable SMI/NMI/IRQ window open exits if needed.
8190
			 *
8191 8192 8193 8194 8195 8196 8197 8198 8199 8200 8201
			 * SMIs have three cases:
			 * 1) They can be nested, and then there is nothing to
			 *    do here because RSM will cause a vmexit anyway.
			 * 2) There is an ISA-specific reason why SMI cannot be
			 *    injected, and the moment when this changes can be
			 *    intercepted.
			 * 3) Or the SMI can be pending because
			 *    inject_pending_event has completed the injection
			 *    of an IRQ or NMI from the previous vmexit, and
			 *    then we request an immediate exit to inject the
			 *    SMI.
8202 8203
			 */
			if (vcpu->arch.smi_pending && !is_smm(vcpu))
8204 8205
				if (!kvm_x86_ops->enable_smi_window(vcpu))
					req_immediate_exit = true;
8206 8207 8208 8209
			if (vcpu->arch.nmi_pending)
				kvm_x86_ops->enable_nmi_window(vcpu);
			if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
				kvm_x86_ops->enable_irq_window(vcpu);
8210
			WARN_ON(vcpu->arch.exception.pending);
8211
		}
A
Avi Kivity 已提交
8212 8213 8214 8215 8216 8217 8218

		if (kvm_lapic_enabled(vcpu)) {
			update_cr8_intercept(vcpu);
			kvm_lapic_sync_to_vapic(vcpu);
		}
	}

8219 8220
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r)) {
8221
		goto cancel_injection;
8222 8223
	}

8224 8225 8226
	preempt_disable();

	kvm_x86_ops->prepare_guest_switch(vcpu);
8227 8228 8229 8230 8231 8232 8233

	/*
	 * Disable IRQs before setting IN_GUEST_MODE.  Posted interrupt
	 * IPI are then delayed after guest entry, which ensures that they
	 * result in virtual interrupt delivery.
	 */
	local_irq_disable();
8234 8235
	vcpu->mode = IN_GUEST_MODE;

8236 8237
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);

8238
	/*
8239
	 * 1) We should set ->mode before checking ->requests.  Please see
8240
	 * the comment in kvm_vcpu_exiting_guest_mode().
8241
	 *
8242
	 * 2) For APICv, we should set ->mode before checking PID.ON. This
8243 8244 8245 8246 8247 8248
	 * pairs with the memory barrier implicit in pi_test_and_set_on
	 * (see vmx_deliver_posted_interrupt).
	 *
	 * 3) This also orders the write to mode from any reads to the page
	 * tables done while the VCPU is running.  Please see the comment
	 * in kvm_flush_remote_tlbs.
8249
	 */
8250
	smp_mb__after_srcu_read_unlock();
8251

8252 8253 8254 8255
	/*
	 * This handles the case where a posted interrupt was
	 * notified with kvm_vcpu_kick.
	 */
8256 8257
	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
		kvm_x86_ops->sync_pir_to_irr(vcpu);
8258

R
Radim Krčmář 已提交
8259
	if (vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu)
A
Avi Kivity 已提交
8260
	    || need_resched() || signal_pending(current)) {
8261
		vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
8262
		smp_wmb();
8263 8264
		local_irq_enable();
		preempt_enable();
8265
		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8266
		r = 1;
8267
		goto cancel_injection;
8268 8269
	}

8270 8271
	if (req_immediate_exit) {
		kvm_make_request(KVM_REQ_EVENT, vcpu);
8272
		kvm_x86_ops->request_immediate_exit(vcpu);
8273
	}
8274

8275
	trace_kvm_entry(vcpu->vcpu_id);
8276
	guest_enter_irqoff();
8277

8278 8279
	/* The preempt notifier should have taken care of the FPU already.  */
	WARN_ON_ONCE(test_thread_flag(TIF_NEED_FPU_LOAD));
8280

8281 8282 8283 8284 8285 8286
	if (unlikely(vcpu->arch.switch_db_regs)) {
		set_debugreg(0, 7);
		set_debugreg(vcpu->arch.eff_db[0], 0);
		set_debugreg(vcpu->arch.eff_db[1], 1);
		set_debugreg(vcpu->arch.eff_db[2], 2);
		set_debugreg(vcpu->arch.eff_db[3], 3);
8287
		set_debugreg(vcpu->arch.dr6, 6);
8288
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8289
	}
8290

A
Avi Kivity 已提交
8291
	kvm_x86_ops->run(vcpu);
8292

8293 8294 8295 8296 8297 8298 8299 8300 8301
	/*
	 * Do this here before restoring debug registers on the host.  And
	 * since we do this before handling the vmexit, a DR access vmexit
	 * can (a) read the correct value of the debug registers, (b) set
	 * KVM_DEBUGREG_WONT_EXIT again.
	 */
	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
		kvm_x86_ops->sync_dirty_debug_regs(vcpu);
8302 8303 8304 8305
		kvm_update_dr0123(vcpu);
		kvm_update_dr6(vcpu);
		kvm_update_dr7(vcpu);
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8306 8307
	}

8308 8309 8310 8311 8312 8313 8314
	/*
	 * If the guest has used debug registers, at least dr7
	 * will be disabled while returning to the host.
	 * If we don't have active breakpoints in the host, we don't
	 * care about the messed up debug address registers. But if
	 * we have some of them active, restore the old state.
	 */
8315
	if (hw_breakpoint_active())
8316
		hw_breakpoint_restore();
8317

8318
	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
8319

8320
	vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
8321
	smp_wmb();
8322

8323
	kvm_x86_ops->handle_exit_irqoff(vcpu, &exit_fastpath);
8324

8325 8326 8327 8328 8329 8330 8331 8332 8333
	/*
	 * Consume any pending interrupts, including the possible source of
	 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
	 * An instruction is required after local_irq_enable() to fully unblock
	 * interrupts on processors that implement an interrupt shadow, the
	 * stat.exits increment will do nicely.
	 */
	kvm_before_interrupt(vcpu);
	local_irq_enable();
8334
	++vcpu->stat.exits;
8335 8336
	local_irq_disable();
	kvm_after_interrupt(vcpu);
8337

P
Paolo Bonzini 已提交
8338
	guest_exit_irqoff();
8339 8340 8341 8342 8343 8344 8345
	if (lapic_in_kernel(vcpu)) {
		s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;
		if (delta != S64_MIN) {
			trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta);
			vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;
		}
	}
8346

P
Paolo Bonzini 已提交
8347
	local_irq_enable();
8348 8349
	preempt_enable();

8350
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8351

8352 8353 8354 8355
	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
8356 8357
		unsigned long rip = kvm_rip_read(vcpu);
		profile_hit(KVM_PROFILING, (void *)rip);
8358 8359
	}

8360 8361
	if (unlikely(vcpu->arch.tsc_always_catchup))
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
8362

8363 8364
	if (vcpu->arch.apic_attention)
		kvm_lapic_sync_from_vapic(vcpu);
A
Avi Kivity 已提交
8365

8366
	vcpu->arch.gpa_available = false;
8367
	r = kvm_x86_ops->handle_exit(vcpu, exit_fastpath);
8368 8369 8370 8371
	return r;

cancel_injection:
	kvm_x86_ops->cancel_injection(vcpu);
8372 8373
	if (unlikely(vcpu->arch.apic_attention))
		kvm_lapic_sync_from_vapic(vcpu);
8374 8375 8376
out:
	return r;
}
8377

8378 8379
static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
{
8380 8381
	if (!kvm_arch_vcpu_runnable(vcpu) &&
	    (!kvm_x86_ops->pre_block || kvm_x86_ops->pre_block(vcpu) == 0)) {
8382 8383 8384
		srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
		kvm_vcpu_block(vcpu);
		vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8385 8386 8387 8388

		if (kvm_x86_ops->post_block)
			kvm_x86_ops->post_block(vcpu);

8389 8390 8391
		if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
			return 1;
	}
8392 8393 8394 8395 8396 8397 8398

	kvm_apic_accept_events(vcpu);
	switch(vcpu->arch.mp_state) {
	case KVM_MP_STATE_HALTED:
		vcpu->arch.pv.pv_unhalted = false;
		vcpu->arch.mp_state =
			KVM_MP_STATE_RUNNABLE;
8399
		/* fall through */
8400 8401 8402 8403 8404 8405 8406 8407 8408 8409 8410
	case KVM_MP_STATE_RUNNABLE:
		vcpu->arch.apf.halted = false;
		break;
	case KVM_MP_STATE_INIT_RECEIVED:
		break;
	default:
		return -EINTR;
		break;
	}
	return 1;
}
8411

8412 8413
static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
{
8414 8415 8416
	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events)
		kvm_x86_ops->check_nested_events(vcpu, false);

8417 8418 8419 8420
	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		!vcpu->arch.apf.halted);
}

8421
static int vcpu_run(struct kvm_vcpu *vcpu)
8422 8423
{
	int r;
8424
	struct kvm *kvm = vcpu->kvm;
8425

8426
	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
P
Paolo Bonzini 已提交
8427
	vcpu->arch.l1tf_flush_l1d = true;
8428

8429
	for (;;) {
8430
		if (kvm_vcpu_running(vcpu)) {
A
Avi Kivity 已提交
8431
			r = vcpu_enter_guest(vcpu);
8432
		} else {
8433
			r = vcpu_block(kvm, vcpu);
8434 8435
		}

8436 8437 8438
		if (r <= 0)
			break;

8439
		kvm_clear_request(KVM_REQ_PENDING_TIMER, vcpu);
8440 8441 8442
		if (kvm_cpu_has_pending_timer(vcpu))
			kvm_inject_pending_timer_irqs(vcpu);

8443 8444
		if (dm_request_for_irq_injection(vcpu) &&
			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
8445 8446
			r = 0;
			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
8447
			++vcpu->stat.request_irq_exits;
8448
			break;
8449
		}
8450 8451 8452

		kvm_check_async_pf_completion(vcpu);

8453 8454
		if (signal_pending(current)) {
			r = -EINTR;
A
Avi Kivity 已提交
8455
			vcpu->run->exit_reason = KVM_EXIT_INTR;
8456
			++vcpu->stat.signal_exits;
8457
			break;
8458 8459
		}
		if (need_resched()) {
8460
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8461
			cond_resched();
8462
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8463
		}
8464 8465
	}

8466
	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8467 8468 8469 8470

	return r;
}

8471 8472 8473
static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
{
	int r;
8474

8475
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8476
	r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
8477
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
8478
	return r;
8479 8480 8481 8482 8483 8484 8485 8486 8487
}

static int complete_emulated_pio(struct kvm_vcpu *vcpu)
{
	BUG_ON(!vcpu->arch.pio.count);

	return complete_emulated_io(vcpu);
}

A
Avi Kivity 已提交
8488 8489 8490 8491 8492
/*
 * Implements the following, as a state machine:
 *
 * read:
 *   for each fragment
8493 8494 8495 8496
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       exit
 *       copy data
A
Avi Kivity 已提交
8497 8498 8499 8500
 *   execute insn
 *
 * write:
 *   for each fragment
8501 8502 8503 8504
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       copy data
 *       exit
A
Avi Kivity 已提交
8505
 */
8506
static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
8507 8508
{
	struct kvm_run *run = vcpu->run;
A
Avi Kivity 已提交
8509
	struct kvm_mmio_fragment *frag;
8510
	unsigned len;
8511

8512
	BUG_ON(!vcpu->mmio_needed);
8513

8514
	/* Complete previous fragment */
8515 8516
	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
	len = min(8u, frag->len);
8517
	if (!vcpu->mmio_is_write)
8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530
		memcpy(frag->data, run->mmio.data, len);

	if (frag->len <= 8) {
		/* Switch to the next fragment. */
		frag++;
		vcpu->mmio_cur_fragment++;
	} else {
		/* Go forward to the next mmio piece. */
		frag->data += len;
		frag->gpa += len;
		frag->len -= len;
	}

8531
	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
8532
		vcpu->mmio_needed = 0;
8533 8534

		/* FIXME: return into emulator if single-stepping.  */
A
Avi Kivity 已提交
8535
		if (vcpu->mmio_is_write)
8536 8537 8538 8539
			return 1;
		vcpu->mmio_read_completed = 1;
		return complete_emulated_io(vcpu);
	}
8540

8541 8542 8543
	run->exit_reason = KVM_EXIT_MMIO;
	run->mmio.phys_addr = frag->gpa;
	if (vcpu->mmio_is_write)
8544 8545
		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
	run->mmio.len = min(8u, frag->len);
8546 8547 8548
	run->mmio.is_write = vcpu->mmio_is_write;
	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
	return 0;
8549 8550
}

8551 8552 8553
/* Swap (qemu) user FPU context for the guest FPU context. */
static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
8554 8555
	fpregs_lock();

8556 8557 8558 8559 8560 8561 8562
	/*
	 * Reloading userspace's FPU is handled by kvm_arch_vcpu_load(), both
	 * for direct calls from userspace (via vcpu_load()) and if this task
	 * is preempted (via kvm_sched_in()) between vcpu_load() and now.
	 */
	WARN_ON_ONCE(test_thread_flag(TIF_NEED_FPU_LOAD));

8563
	copy_fpregs_to_fpstate(vcpu->arch.user_fpu);
8564
	/* PKRU is separately restored in kvm_x86_ops->run.  */
8565
	__copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,
8566
				~XFEATURE_MASK_PKRU);
8567 8568 8569 8570

	fpregs_mark_activate();
	fpregs_unlock();

8571 8572 8573 8574 8575 8576
	trace_kvm_fpu(1);
}

/* When vcpu_run ends, restore user space FPU context. */
static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
8577 8578
	fpregs_lock();

8579
	copy_fpregs_to_fpstate(vcpu->arch.guest_fpu);
8580
	copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);
8581 8582 8583 8584

	fpregs_mark_activate();
	fpregs_unlock();

8585 8586 8587 8588
	++vcpu->stat.fpu_reload;
	trace_kvm_fpu(0);
}

8589 8590 8591 8592
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
{
	int r;

8593
	vcpu_load(vcpu);
8594
	kvm_sigset_activate(vcpu);
8595 8596
	kvm_load_guest_fpu(vcpu);

8597
	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
8598 8599 8600 8601
		if (kvm_run->immediate_exit) {
			r = -EINTR;
			goto out;
		}
8602
		kvm_vcpu_block(vcpu);
8603
		kvm_apic_accept_events(vcpu);
8604
		kvm_clear_request(KVM_REQ_UNHALT, vcpu);
8605
		r = -EAGAIN;
8606 8607 8608 8609 8610
		if (signal_pending(current)) {
			r = -EINTR;
			vcpu->run->exit_reason = KVM_EXIT_INTR;
			++vcpu->stat.signal_exits;
		}
8611
		goto out;
8612 8613
	}

K
Ken Hofsass 已提交
8614 8615 8616 8617 8618 8619 8620 8621 8622 8623 8624
	if (vcpu->run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) {
		r = -EINVAL;
		goto out;
	}

	if (vcpu->run->kvm_dirty_regs) {
		r = sync_regs(vcpu);
		if (r != 0)
			goto out;
	}

8625
	/* re-sync apic's tpr */
8626
	if (!lapic_in_kernel(vcpu)) {
A
Andre Przywara 已提交
8627 8628 8629 8630 8631
		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
			r = -EINVAL;
			goto out;
		}
	}
8632

8633 8634 8635 8636 8637
	if (unlikely(vcpu->arch.complete_userspace_io)) {
		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
		vcpu->arch.complete_userspace_io = NULL;
		r = cui(vcpu);
		if (r <= 0)
8638
			goto out;
8639 8640
	} else
		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
8641

8642 8643 8644 8645
	if (kvm_run->immediate_exit)
		r = -EINTR;
	else
		r = vcpu_run(vcpu);
8646 8647

out:
8648
	kvm_put_guest_fpu(vcpu);
K
Ken Hofsass 已提交
8649 8650
	if (vcpu->run->kvm_valid_regs)
		store_regs(vcpu);
8651
	post_kvm_run_save(vcpu);
8652
	kvm_sigset_deactivate(vcpu);
8653

8654
	vcpu_put(vcpu);
8655 8656 8657
	return r;
}

K
Ken Hofsass 已提交
8658
static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8659
{
8660 8661 8662 8663
	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
		/*
		 * We are here if userspace calls get_regs() in the middle of
		 * instruction emulation. Registers state needs to be copied
G
Guo Chao 已提交
8664
		 * back from emulation context to vcpu. Userspace shouldn't do
8665 8666 8667
		 * that usually, but some bad designed PV devices (vmware
		 * backdoor interface) need this to work
		 */
8668
		emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
8669 8670
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
	}
8671 8672 8673 8674 8675 8676
	regs->rax = kvm_rax_read(vcpu);
	regs->rbx = kvm_rbx_read(vcpu);
	regs->rcx = kvm_rcx_read(vcpu);
	regs->rdx = kvm_rdx_read(vcpu);
	regs->rsi = kvm_rsi_read(vcpu);
	regs->rdi = kvm_rdi_read(vcpu);
8677
	regs->rsp = kvm_rsp_read(vcpu);
8678
	regs->rbp = kvm_rbp_read(vcpu);
8679
#ifdef CONFIG_X86_64
8680 8681 8682 8683 8684 8685 8686 8687
	regs->r8 = kvm_r8_read(vcpu);
	regs->r9 = kvm_r9_read(vcpu);
	regs->r10 = kvm_r10_read(vcpu);
	regs->r11 = kvm_r11_read(vcpu);
	regs->r12 = kvm_r12_read(vcpu);
	regs->r13 = kvm_r13_read(vcpu);
	regs->r14 = kvm_r14_read(vcpu);
	regs->r15 = kvm_r15_read(vcpu);
8688 8689
#endif

8690
	regs->rip = kvm_rip_read(vcpu);
8691
	regs->rflags = kvm_get_rflags(vcpu);
K
Ken Hofsass 已提交
8692
}
8693

K
Ken Hofsass 已提交
8694 8695 8696 8697
int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__get_regs(vcpu, regs);
8698
	vcpu_put(vcpu);
8699 8700 8701
	return 0;
}

K
Ken Hofsass 已提交
8702
static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8703
{
8704 8705 8706
	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;

8707 8708 8709 8710 8711 8712
	kvm_rax_write(vcpu, regs->rax);
	kvm_rbx_write(vcpu, regs->rbx);
	kvm_rcx_write(vcpu, regs->rcx);
	kvm_rdx_write(vcpu, regs->rdx);
	kvm_rsi_write(vcpu, regs->rsi);
	kvm_rdi_write(vcpu, regs->rdi);
8713
	kvm_rsp_write(vcpu, regs->rsp);
8714
	kvm_rbp_write(vcpu, regs->rbp);
8715
#ifdef CONFIG_X86_64
8716 8717 8718 8719 8720 8721 8722 8723
	kvm_r8_write(vcpu, regs->r8);
	kvm_r9_write(vcpu, regs->r9);
	kvm_r10_write(vcpu, regs->r10);
	kvm_r11_write(vcpu, regs->r11);
	kvm_r12_write(vcpu, regs->r12);
	kvm_r13_write(vcpu, regs->r13);
	kvm_r14_write(vcpu, regs->r14);
	kvm_r15_write(vcpu, regs->r15);
8724 8725
#endif

8726
	kvm_rip_write(vcpu, regs->rip);
8727
	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
8728

8729 8730
	vcpu->arch.exception.pending = false;

8731
	kvm_make_request(KVM_REQ_EVENT, vcpu);
K
Ken Hofsass 已提交
8732
}
8733

K
Ken Hofsass 已提交
8734 8735 8736 8737
int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__set_regs(vcpu, regs);
8738
	vcpu_put(vcpu);
8739 8740 8741 8742 8743 8744 8745
	return 0;
}

void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

8746
	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
8747 8748 8749 8750 8751
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

K
Ken Hofsass 已提交
8752
static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8753
{
8754
	struct desc_ptr dt;
8755

8756 8757 8758 8759 8760 8761
	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8762

8763 8764
	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
8765 8766

	kvm_x86_ops->get_idt(vcpu, &dt);
8767 8768
	sregs->idt.limit = dt.size;
	sregs->idt.base = dt.address;
8769
	kvm_x86_ops->get_gdt(vcpu, &dt);
8770 8771
	sregs->gdt.limit = dt.size;
	sregs->gdt.base = dt.address;
8772

8773
	sregs->cr0 = kvm_read_cr0(vcpu);
8774
	sregs->cr2 = vcpu->arch.cr2;
8775
	sregs->cr3 = kvm_read_cr3(vcpu);
8776
	sregs->cr4 = kvm_read_cr4(vcpu);
8777
	sregs->cr8 = kvm_get_cr8(vcpu);
8778
	sregs->efer = vcpu->arch.efer;
8779 8780
	sregs->apic_base = kvm_get_apic_base(vcpu);

8781
	memset(sregs->interrupt_bitmap, 0, sizeof(sregs->interrupt_bitmap));
8782

8783
	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
8784 8785
		set_bit(vcpu->arch.interrupt.nr,
			(unsigned long *)sregs->interrupt_bitmap);
K
Ken Hofsass 已提交
8786
}
8787

K
Ken Hofsass 已提交
8788 8789 8790 8791 8792
int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	vcpu_load(vcpu);
	__get_sregs(vcpu, sregs);
8793
	vcpu_put(vcpu);
8794 8795 8796
	return 0;
}

8797 8798 8799
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
8800
	vcpu_load(vcpu);
8801 8802
	if (kvm_mpx_supported())
		kvm_load_guest_fpu(vcpu);
8803

8804
	kvm_apic_accept_events(vcpu);
8805 8806 8807 8808 8809 8810
	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
					vcpu->arch.pv.pv_unhalted)
		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
	else
		mp_state->mp_state = vcpu->arch.mp_state;

8811 8812
	if (kvm_mpx_supported())
		kvm_put_guest_fpu(vcpu);
8813
	vcpu_put(vcpu);
8814 8815 8816 8817 8818 8819
	return 0;
}

int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
8820 8821 8822 8823
	int ret = -EINVAL;

	vcpu_load(vcpu);

8824
	if (!lapic_in_kernel(vcpu) &&
8825
	    mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
8826
		goto out;
8827

8828 8829 8830 8831 8832 8833
	/*
	 * KVM_MP_STATE_INIT_RECEIVED means the processor is in
	 * INIT state; latched init should be reported using
	 * KVM_SET_VCPU_EVENTS, so reject it here.
	 */
	if ((kvm_vcpu_latch_init(vcpu) || vcpu->arch.smi_pending) &&
8834 8835
	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
8836
		goto out;
8837

8838 8839 8840 8841 8842
	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
	} else
		vcpu->arch.mp_state = mp_state->mp_state;
8843
	kvm_make_request(KVM_REQ_EVENT, vcpu);
8844 8845 8846 8847 8848

	ret = 0;
out:
	vcpu_put(vcpu);
	return ret;
8849 8850
}

8851 8852
int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
		    int reason, bool has_error_code, u32 error_code)
8853
{
8854
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
8855
	int ret;
8856

8857
	init_emulate_ctxt(vcpu);
8858

8859
	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
8860
				   has_error_code, error_code);
8861 8862 8863 8864
	if (ret) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
8865
		return 0;
8866
	}
8867

8868 8869
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
8870
	kvm_make_request(KVM_REQ_EVENT, vcpu);
8871
	return 1;
8872 8873 8874
}
EXPORT_SYMBOL_GPL(kvm_task_switch);

P
Peng Hao 已提交
8875
static int kvm_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8876
{
8877
	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
8878 8879 8880 8881 8882
		/*
		 * When EFER.LME and CR0.PG are set, the processor is in
		 * 64-bit mode (though maybe in a 32-bit code segment).
		 * CR4.PAE and EFER.LMA must be set.
		 */
8883
		if (!(sregs->cr4 & X86_CR4_PAE)
8884 8885 8886 8887 8888 8889 8890 8891 8892 8893 8894
		    || !(sregs->efer & EFER_LMA))
			return -EINVAL;
	} else {
		/*
		 * Not in 64-bit mode: EFER.LMA is clear and the code
		 * segment cannot be 64-bit.
		 */
		if (sregs->efer & EFER_LMA || sregs->cs.l)
			return -EINVAL;
	}

8895
	return kvm_valid_cr4(vcpu, sregs->cr4);
8896 8897
}

K
Ken Hofsass 已提交
8898
static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8899
{
8900
	struct msr_data apic_base_msr;
8901
	int mmu_reset_needed = 0;
8902
	int cpuid_update_needed = 0;
8903
	int pending_vec, max_bits, idx;
8904
	struct desc_ptr dt;
8905 8906
	int ret = -EINVAL;

8907
	if (kvm_valid_sregs(vcpu, sregs))
8908
		goto out;
8909

8910 8911 8912
	apic_base_msr.data = sregs->apic_base;
	apic_base_msr.host_initiated = true;
	if (kvm_set_apic_base(vcpu, &apic_base_msr))
8913
		goto out;
8914

8915 8916
	dt.size = sregs->idt.limit;
	dt.address = sregs->idt.base;
8917
	kvm_x86_ops->set_idt(vcpu, &dt);
8918 8919
	dt.size = sregs->gdt.limit;
	dt.address = sregs->gdt.base;
8920 8921
	kvm_x86_ops->set_gdt(vcpu, &dt);

8922
	vcpu->arch.cr2 = sregs->cr2;
8923
	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
8924
	vcpu->arch.cr3 = sregs->cr3;
8925
	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
8926

8927
	kvm_set_cr8(vcpu, sregs->cr8);
8928

8929
	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
8930 8931
	kvm_x86_ops->set_efer(vcpu, sregs->efer);

8932
	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
8933
	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
8934
	vcpu->arch.cr0 = sregs->cr0;
8935

8936
	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
8937 8938
	cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) &
				(X86_CR4_OSXSAVE | X86_CR4_PKE));
8939
	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
8940
	if (cpuid_update_needed)
A
Avi Kivity 已提交
8941
		kvm_update_cpuid(vcpu);
8942 8943

	idx = srcu_read_lock(&vcpu->kvm->srcu);
8944
	if (is_pae_paging(vcpu)) {
8945
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
8946 8947
		mmu_reset_needed = 1;
	}
8948
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
8949 8950 8951 8952

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

8953
	max_bits = KVM_NR_INTERRUPTS;
G
Gleb Natapov 已提交
8954 8955 8956
	pending_vec = find_first_bit(
		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
	if (pending_vec < max_bits) {
8957
		kvm_queue_interrupt(vcpu, pending_vec, false);
G
Gleb Natapov 已提交
8958
		pr_debug("Set back pending irq %d\n", pending_vec);
8959 8960
	}

8961 8962 8963 8964 8965 8966
	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8967

8968 8969
	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
8970

8971 8972
	update_cr8_intercept(vcpu);

M
Marcelo Tosatti 已提交
8973
	/* Older userspace won't unhalt the vcpu on reset. */
8974
	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
M
Marcelo Tosatti 已提交
8975
	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
8976
	    !is_protmode(vcpu))
M
Marcelo Tosatti 已提交
8977 8978
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;

8979 8980
	kvm_make_request(KVM_REQ_EVENT, vcpu);

8981 8982
	ret = 0;
out:
K
Ken Hofsass 已提交
8983 8984 8985 8986 8987 8988 8989 8990 8991 8992
	return ret;
}

int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	int ret;

	vcpu_load(vcpu);
	ret = __set_sregs(vcpu, sregs);
8993 8994
	vcpu_put(vcpu);
	return ret;
8995 8996
}

J
Jan Kiszka 已提交
8997 8998
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg)
8999
{
9000
	unsigned long rflags;
9001
	int i, r;
9002

9003 9004
	vcpu_load(vcpu);

9005 9006 9007
	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
		r = -EBUSY;
		if (vcpu->arch.exception.pending)
9008
			goto out;
9009 9010 9011 9012 9013 9014
		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
			kvm_queue_exception(vcpu, DB_VECTOR);
		else
			kvm_queue_exception(vcpu, BP_VECTOR);
	}

9015 9016 9017 9018 9019
	/*
	 * Read rflags as long as potentially injected trace flags are still
	 * filtered out.
	 */
	rflags = kvm_get_rflags(vcpu);
9020 9021 9022 9023 9024 9025

	vcpu->guest_debug = dbg->control;
	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
		vcpu->guest_debug = 0;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
9026 9027
		for (i = 0; i < KVM_NR_DB_REGS; ++i)
			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
9028
		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
9029 9030 9031 9032
	} else {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
	}
9033
	kvm_update_dr7(vcpu);
9034

J
Jan Kiszka 已提交
9035 9036 9037
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
			get_segment_base(vcpu, VCPU_SREG_CS);
9038

9039 9040 9041 9042 9043
	/*
	 * Trigger an rflags update that will inject or remove the trace
	 * flags.
	 */
	kvm_set_rflags(vcpu, rflags);
9044

9045
	kvm_x86_ops->update_bp_intercept(vcpu);
9046

9047
	r = 0;
J
Jan Kiszka 已提交
9048

9049
out:
9050
	vcpu_put(vcpu);
9051 9052 9053
	return r;
}

9054 9055 9056 9057 9058 9059 9060 9061
/*
 * Translate a guest virtual address to a guest physical address.
 */
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;
9062
	int idx;
9063

9064 9065
	vcpu_load(vcpu);

9066
	idx = srcu_read_lock(&vcpu->kvm->srcu);
9067
	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
9068
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9069 9070 9071 9072 9073
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;

9074
	vcpu_put(vcpu);
9075 9076 9077
	return 0;
}

9078 9079
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
9080
	struct fxregs_state *fxsave;
9081

9082
	vcpu_load(vcpu);
9083

9084
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
9085 9086 9087 9088 9089 9090 9091
	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
9092
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
9093

9094
	vcpu_put(vcpu);
9095 9096 9097 9098 9099
	return 0;
}

int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
9100 9101 9102 9103
	struct fxregs_state *fxsave;

	vcpu_load(vcpu);

9104
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
9105 9106 9107 9108 9109 9110 9111 9112

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
9113
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
9114

9115
	vcpu_put(vcpu);
9116 9117 9118
	return 0;
}

K
Ken Hofsass 已提交
9119 9120 9121 9122 9123 9124 9125 9126 9127 9128 9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149 9150 9151 9152 9153 9154 9155 9156 9157
static void store_regs(struct kvm_vcpu *vcpu)
{
	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
		__get_regs(vcpu, &vcpu->run->s.regs.regs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
		kvm_vcpu_ioctl_x86_get_vcpu_events(
				vcpu, &vcpu->run->s.regs.events);
}

static int sync_regs(struct kvm_vcpu *vcpu)
{
	if (vcpu->run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)
		return -EINVAL;

	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
		__set_regs(vcpu, &vcpu->run->s.regs.regs);
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
		if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
		if (kvm_vcpu_ioctl_x86_set_vcpu_events(
				vcpu, &vcpu->run->s.regs.events))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
	}

	return 0;
}

I
Ingo Molnar 已提交
9158
static void fx_init(struct kvm_vcpu *vcpu)
9159
{
9160
	fpstate_init(&vcpu->arch.guest_fpu->state);
9161
	if (boot_cpu_has(X86_FEATURE_XSAVES))
9162
		vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =
9163
			host_xcr0 | XSTATE_COMPACTION_ENABLED;
9164

9165 9166 9167
	/*
	 * Ensure guest xcr0 is valid for loading
	 */
D
Dave Hansen 已提交
9168
	vcpu->arch.xcr0 = XFEATURE_MASK_FP;
9169

9170
	vcpu->arch.cr0 |= X86_CR0_ET;
9171 9172
}

9173 9174
void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
{
9175 9176
	void *wbinvd_dirty_mask = vcpu->arch.wbinvd_dirty_mask;

9177
	kvmclock_reset(vcpu);
9178

9179
	kvm_x86_ops->vcpu_free(vcpu);
9180
	free_cpumask_var(wbinvd_dirty_mask);
9181 9182 9183 9184 9185
}

struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
						unsigned int id)
{
9186 9187
	struct kvm_vcpu *vcpu;

9188
	if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
Z
Zachary Amsden 已提交
9189 9190 9191
		printk_once(KERN_WARNING
		"kvm: SMP vm created on host with unstable TSC; "
		"guest TSC will not be reliable\n");
9192 9193 9194 9195

	vcpu = kvm_x86_ops->vcpu_create(kvm, id);

	return vcpu;
9196
}
9197

9198 9199
int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
{
9200
	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
9201
	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
X
Xiao Guangrong 已提交
9202
	kvm_vcpu_mtrr_init(vcpu);
9203
	vcpu_load(vcpu);
9204
	kvm_vcpu_reset(vcpu, false);
9205
	kvm_init_mmu(vcpu, false);
9206
	vcpu_put(vcpu);
9207
	return 0;
9208 9209
}

9210
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
9211
{
9212
	struct msr_data msr;
9213
	struct kvm *kvm = vcpu->kvm;
9214

9215 9216
	kvm_hv_vcpu_postcreate(vcpu);

9217
	if (mutex_lock_killable(&vcpu->mutex))
9218
		return;
9219
	vcpu_load(vcpu);
9220 9221 9222 9223
	msr.data = 0x0;
	msr.index = MSR_IA32_TSC;
	msr.host_initiated = true;
	kvm_write_tsc(vcpu, &msr);
9224
	vcpu_put(vcpu);
9225 9226 9227 9228

	/* poll control enabled by default */
	vcpu->arch.msr_kvm_poll_control = 1;

9229
	mutex_unlock(&vcpu->mutex);
9230

9231 9232 9233
	if (!kvmclock_periodic_sync)
		return;

9234 9235
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
9236 9237
}

9238
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
9239
{
9240 9241
	vcpu->arch.apf.msr_val = 0;

9242
	vcpu_load(vcpu);
9243 9244 9245
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);

9246
	kvm_arch_vcpu_free(vcpu);
9247 9248
}

9249
void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
9250
{
9251 9252
	kvm_lapic_reset(vcpu, init_event);

9253 9254
	vcpu->arch.hflags = 0;

9255
	vcpu->arch.smi_pending = 0;
9256
	vcpu->arch.smi_count = 0;
A
Avi Kivity 已提交
9257 9258
	atomic_set(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = 0;
9259
	vcpu->arch.nmi_injected = false;
9260 9261
	kvm_clear_interrupt_queue(vcpu);
	kvm_clear_exception_queue(vcpu);
9262
	vcpu->arch.exception.pending = false;
9263

9264
	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
9265
	kvm_update_dr0123(vcpu);
9266
	vcpu->arch.dr6 = DR6_INIT;
J
Jan Kiszka 已提交
9267
	kvm_update_dr6(vcpu);
9268
	vcpu->arch.dr7 = DR7_FIXED_1;
9269
	kvm_update_dr7(vcpu);
9270

N
Nadav Amit 已提交
9271 9272
	vcpu->arch.cr2 = 0;

9273
	kvm_make_request(KVM_REQ_EVENT, vcpu);
9274
	vcpu->arch.apf.msr_val = 0;
G
Glauber Costa 已提交
9275
	vcpu->arch.st.msr_val = 0;
9276

9277 9278
	kvmclock_reset(vcpu);

9279 9280 9281
	kvm_clear_async_pf_completion_queue(vcpu);
	kvm_async_pf_hash_reset(vcpu);
	vcpu->arch.apf.halted = false;
9282

9283 9284 9285 9286 9287 9288 9289
	if (kvm_mpx_supported()) {
		void *mpx_state_buffer;

		/*
		 * To avoid have the INIT path from kvm_apic_has_events() that be
		 * called with loaded FPU and does not let userspace fix the state.
		 */
9290 9291
		if (init_event)
			kvm_put_guest_fpu(vcpu);
9292
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9293
					XFEATURE_BNDREGS);
9294 9295
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndreg_state));
9296
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9297
					XFEATURE_BNDCSR);
9298 9299
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndcsr));
9300 9301
		if (init_event)
			kvm_load_guest_fpu(vcpu);
9302 9303
	}

P
Paolo Bonzini 已提交
9304
	if (!init_event) {
9305
		kvm_pmu_reset(vcpu);
P
Paolo Bonzini 已提交
9306
		vcpu->arch.smbase = 0x30000;
K
Kyle Huey 已提交
9307 9308

		vcpu->arch.msr_misc_features_enables = 0;
9309 9310

		vcpu->arch.xcr0 = XFEATURE_MASK_FP;
P
Paolo Bonzini 已提交
9311
	}
9312

9313 9314 9315 9316
	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;

9317 9318
	vcpu->arch.ia32_xss = 0;

9319
	kvm_x86_ops->vcpu_reset(vcpu, init_event);
9320 9321
}

9322
void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
9323 9324 9325 9326 9327 9328 9329 9330
{
	struct kvm_segment cs;

	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs.selector = vector << 8;
	cs.base = vector << 12;
	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_rip_write(vcpu, 0);
9331 9332
}

9333
int kvm_arch_hardware_enable(void)
9334
{
9335 9336 9337
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;
9338 9339 9340 9341
	int ret;
	u64 local_tsc;
	u64 max_tsc = 0;
	bool stable, backwards_tsc = false;
A
Avi Kivity 已提交
9342 9343

	kvm_shared_msr_cpu_online();
9344
	ret = kvm_x86_ops->hardware_enable();
9345 9346 9347
	if (ret != 0)
		return ret;

9348
	local_tsc = rdtsc();
9349
	stable = !kvm_check_tsc_unstable();
9350 9351 9352
	list_for_each_entry(kvm, &vm_list, vm_list) {
		kvm_for_each_vcpu(i, vcpu, kvm) {
			if (!stable && vcpu->cpu == smp_processor_id())
9353
				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
9354 9355 9356 9357 9358 9359 9360 9361 9362 9363 9364 9365 9366 9367 9368 9369
			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
				backwards_tsc = true;
				if (vcpu->arch.last_host_tsc > max_tsc)
					max_tsc = vcpu->arch.last_host_tsc;
			}
		}
	}

	/*
	 * Sometimes, even reliable TSCs go backwards.  This happens on
	 * platforms that reset TSC during suspend or hibernate actions, but
	 * maintain synchronization.  We must compensate.  Fortunately, we can
	 * detect that condition here, which happens early in CPU bringup,
	 * before any KVM threads can be running.  Unfortunately, we can't
	 * bring the TSCs fully up to date with real time, as we aren't yet far
	 * enough into CPU bringup that we know how much real time has actually
9370
	 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot
9371 9372 9373 9374 9375 9376 9377 9378 9379 9380 9381 9382 9383 9384 9385 9386 9387 9388 9389 9390 9391 9392 9393 9394
	 * variables that haven't been updated yet.
	 *
	 * So we simply find the maximum observed TSC above, then record the
	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
	 * the adjustment will be applied.  Note that we accumulate
	 * adjustments, in case multiple suspend cycles happen before some VCPU
	 * gets a chance to run again.  In the event that no KVM threads get a
	 * chance to run, we will miss the entire elapsed period, as we'll have
	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
	 * loose cycle time.  This isn't too big a deal, since the loss will be
	 * uniform across all VCPUs (not to mention the scenario is extremely
	 * unlikely). It is possible that a second hibernate recovery happens
	 * much faster than a first, causing the observed TSC here to be
	 * smaller; this would require additional padding adjustment, which is
	 * why we set last_host_tsc to the local tsc observed here.
	 *
	 * N.B. - this code below runs only on platforms with reliable TSC,
	 * as that is the only way backwards_tsc is set above.  Also note
	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
	 * have the same delta_cyc adjustment applied if backwards_tsc
	 * is detected.  Note further, this adjustment is only done once,
	 * as we reset last_host_tsc on all VCPUs to stop this from being
	 * called multiple times (one for each physical CPU bringup).
	 *
G
Guo Chao 已提交
9395
	 * Platforms with unreliable TSCs don't have to deal with this, they
9396 9397 9398 9399 9400 9401 9402
	 * will be compensated by the logic in vcpu_load, which sets the TSC to
	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
	 * guarantee that they stay in perfect synchronization.
	 */
	if (backwards_tsc) {
		u64 delta_cyc = max_tsc - local_tsc;
		list_for_each_entry(kvm, &vm_list, vm_list) {
9403
			kvm->arch.backwards_tsc_observed = true;
9404 9405 9406
			kvm_for_each_vcpu(i, vcpu, kvm) {
				vcpu->arch.tsc_offset_adjustment += delta_cyc;
				vcpu->arch.last_host_tsc = local_tsc;
9407
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
9408 9409 9410 9411 9412 9413 9414 9415 9416 9417 9418 9419 9420 9421
			}

			/*
			 * We have to disable TSC offset matching.. if you were
			 * booting a VM while issuing an S4 host suspend....
			 * you may have some problem.  Solving this issue is
			 * left as an exercise to the reader.
			 */
			kvm->arch.last_tsc_nsec = 0;
			kvm->arch.last_tsc_write = 0;
		}

	}
	return 0;
9422 9423
}

9424
void kvm_arch_hardware_disable(void)
9425
{
9426 9427
	kvm_x86_ops->hardware_disable();
	drop_user_return_notifiers();
9428 9429 9430 9431
}

int kvm_arch_hardware_setup(void)
{
9432 9433 9434 9435 9436 9437
	int r;

	r = kvm_x86_ops->hardware_setup();
	if (r != 0)
		return r;

9438 9439
	cr4_reserved_bits = kvm_host_cr4_reserved_bits(&boot_cpu_data);

9440 9441 9442 9443
	if (kvm_has_tsc_control) {
		/*
		 * Make sure the user can only configure tsc_khz values that
		 * fit into a signed integer.
9444
		 * A min value is not calculated because it will always
9445 9446 9447 9448 9449 9450
		 * be 1 on all machines.
		 */
		u64 max = min(0x7fffffffULL,
			      __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz));
		kvm_max_guest_tsc_khz = max;

9451
		kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits;
9452
	}
9453

9454 9455 9456
	if (boot_cpu_has(X86_FEATURE_XSAVES))
		rdmsrl(MSR_IA32_XSS, host_xss);

9457 9458
	kvm_init_msr_list();
	return 0;
9459 9460 9461 9462 9463 9464 9465
}

void kvm_arch_hardware_unsetup(void)
{
	kvm_x86_ops->hardware_unsetup();
}

9466
int kvm_arch_check_processor_compat(void)
9467
{
9468 9469 9470 9471 9472 9473 9474
	struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());

	WARN_ON(!irqs_disabled());

	if (kvm_host_cr4_reserved_bits(c) != cr4_reserved_bits)
		return -EIO;

9475
	return kvm_x86_ops->check_processor_compatibility();
9476 9477 9478 9479 9480 9481 9482 9483 9484 9485 9486
}

bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
{
	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);

bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
9487 9488
}

9489
struct static_key kvm_no_apic_vcpu __read_mostly;
9490
EXPORT_SYMBOL_GPL(kvm_no_apic_vcpu);
9491

9492 9493 9494 9495 9496
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
	struct page *page;
	int r;

9497
	vcpu->arch.emulate_ctxt.ops = &emulate_ops;
9498
	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
9499
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
9500
	else
9501
		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
9502 9503 9504 9505 9506 9507

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
9508
	vcpu->arch.pio_data = page_address(page);
9509

9510
	kvm_set_tsc_khz(vcpu, max_tsc_khz);
Z
Zachary Amsden 已提交
9511

9512 9513 9514 9515
	r = kvm_mmu_create(vcpu);
	if (r < 0)
		goto fail_free_pio_data;

9516
	if (irqchip_in_kernel(vcpu->kvm)) {
9517
		vcpu->arch.apicv_active = kvm_x86_ops->get_enable_apicv(vcpu->kvm);
9518
		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
9519 9520
		if (r < 0)
			goto fail_mmu_destroy;
9521 9522
	} else
		static_key_slow_inc(&kvm_no_apic_vcpu);
9523

H
Huang Ying 已提交
9524
	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
9525
				       GFP_KERNEL_ACCOUNT);
H
Huang Ying 已提交
9526 9527
	if (!vcpu->arch.mce_banks) {
		r = -ENOMEM;
9528
		goto fail_free_lapic;
H
Huang Ying 已提交
9529 9530 9531
	}
	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;

9532 9533
	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
				GFP_KERNEL_ACCOUNT)) {
9534
		r = -ENOMEM;
9535
		goto fail_free_mce_banks;
9536
	}
9537

I
Ingo Molnar 已提交
9538
	fx_init(vcpu);
9539

9540
	vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
9541

9542 9543
	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);

9544 9545
	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;

9546
	kvm_async_pf_hash_reset(vcpu);
9547
	kvm_pmu_init(vcpu);
9548

9549
	vcpu->arch.pending_external_vector = -1;
9550
	vcpu->arch.preempted_in_kernel = false;
9551

9552 9553
	kvm_hv_vcpu_init(vcpu);

9554
	return 0;
I
Ingo Molnar 已提交
9555

9556 9557
fail_free_mce_banks:
	kfree(vcpu->arch.mce_banks);
9558 9559
fail_free_lapic:
	kvm_free_lapic(vcpu);
9560 9561 9562
fail_mmu_destroy:
	kvm_mmu_destroy(vcpu);
fail_free_pio_data:
9563
	free_page((unsigned long)vcpu->arch.pio_data);
9564 9565 9566 9567 9568 9569
fail:
	return r;
}

void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
{
9570 9571
	int idx;

A
Andrey Smetanin 已提交
9572
	kvm_hv_vcpu_uninit(vcpu);
9573
	kvm_pmu_destroy(vcpu);
9574
	kfree(vcpu->arch.mce_banks);
9575
	kvm_free_lapic(vcpu);
9576
	idx = srcu_read_lock(&vcpu->kvm->srcu);
9577
	kvm_mmu_destroy(vcpu);
9578
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9579
	free_page((unsigned long)vcpu->arch.pio_data);
9580
	if (!lapic_in_kernel(vcpu))
9581
		static_key_slow_dec(&kvm_no_apic_vcpu);
9582
}
9583

R
Radim Krčmář 已提交
9584 9585
void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
{
9586 9587
	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);

P
Paolo Bonzini 已提交
9588
	vcpu->arch.l1tf_flush_l1d = true;
9589 9590 9591 9592
	if (pmu->version && unlikely(pmu->event_count)) {
		pmu->need_cleanup = true;
		kvm_make_request(KVM_REQ_PMU, vcpu);
	}
9593
	kvm_x86_ops->sched_in(vcpu, cpu);
R
Radim Krčmář 已提交
9594 9595
}

9596
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
9597
{
9598 9599 9600
	if (type)
		return -EINVAL;

9601
	INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
9602
	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
9603
	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
9604
	INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages);
B
Ben-Ami Yassour 已提交
9605
	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
9606
	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
9607

9608 9609
	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
9610 9611 9612
	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
		&kvm->arch.irq_sources_bitmap);
9613

9614
	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
9615
	mutex_init(&kvm->arch.apic_map_lock);
9616 9617
	spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);

9618
	kvm->arch.kvmclock_offset = -ktime_get_boottime_ns();
9619
	pvclock_update_vm_gtod_copy(kvm);
9620

9621 9622
	kvm->arch.guest_can_read_msr_platform_info = true;

9623
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
9624
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
9625

9626
	kvm_hv_init_vm(kvm);
9627
	kvm_page_track_init(kvm);
9628
	kvm_mmu_init_vm(kvm);
9629

9630
	return kvm_x86_ops->vm_init(kvm);
9631 9632
}

9633 9634 9635 9636 9637
int kvm_arch_post_init_vm(struct kvm *kvm)
{
	return kvm_mmu_post_init_vm(kvm);
}

9638 9639
static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
9640
	vcpu_load(vcpu);
9641 9642 9643 9644 9645 9646 9647
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;
9648
	struct kvm_vcpu *vcpu;
9649 9650 9651 9652

	/*
	 * Unpin any mmu pages first.
	 */
9653 9654
	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_clear_async_pf_completion_queue(vcpu);
9655
		kvm_unload_vcpu_mmu(vcpu);
9656
	}
9657 9658 9659 9660 9661 9662
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_arch_vcpu_free(vcpu);

	mutex_lock(&kvm->lock);
	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
		kvm->vcpus[i] = NULL;
9663

9664 9665
	atomic_set(&kvm->online_vcpus, 0);
	mutex_unlock(&kvm->lock);
9666 9667
}

9668 9669
void kvm_arch_sync_events(struct kvm *kvm)
{
9670
	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
9671
	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
9672
	kvm_free_pit(kvm);
9673 9674
}

9675
int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9676 9677
{
	int i, r;
9678
	unsigned long hva;
9679 9680
	struct kvm_memslots *slots = kvm_memslots(kvm);
	struct kvm_memory_slot *slot, old;
9681 9682

	/* Called with kvm->slots_lock held.  */
9683 9684
	if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
		return -EINVAL;
9685

9686 9687
	slot = id_to_memslot(slots, id);
	if (size) {
9688
		if (slot->npages)
9689 9690 9691 9692 9693 9694 9695 9696 9697 9698 9699 9700 9701 9702 9703 9704 9705 9706
			return -EEXIST;

		/*
		 * MAP_SHARED to prevent internal slot pages from being moved
		 * by fork()/COW.
		 */
		hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
			      MAP_SHARED | MAP_ANONYMOUS, 0);
		if (IS_ERR((void *)hva))
			return PTR_ERR((void *)hva);
	} else {
		if (!slot->npages)
			return 0;

		hva = 0;
	}

	old = *slot;
9707
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
9708
		struct kvm_userspace_memory_region m;
9709

9710 9711 9712
		m.slot = id | (i << 16);
		m.flags = 0;
		m.guest_phys_addr = gpa;
9713
		m.userspace_addr = hva;
9714
		m.memory_size = size;
9715 9716 9717 9718 9719
		r = __kvm_set_memory_region(kvm, &m);
		if (r < 0)
			return r;
	}

9720 9721
	if (!size)
		vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE);
9722

9723 9724 9725 9726
	return 0;
}
EXPORT_SYMBOL_GPL(__x86_set_memory_region);

9727
int x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9728 9729 9730 9731
{
	int r;

	mutex_lock(&kvm->slots_lock);
9732
	r = __x86_set_memory_region(kvm, id, gpa, size);
9733 9734 9735 9736 9737 9738
	mutex_unlock(&kvm->slots_lock);

	return r;
}
EXPORT_SYMBOL_GPL(x86_set_memory_region);

9739 9740 9741 9742 9743
void kvm_arch_pre_destroy_vm(struct kvm *kvm)
{
	kvm_mmu_pre_destroy_vm(kvm);
}

9744 9745
void kvm_arch_destroy_vm(struct kvm *kvm)
{
9746 9747 9748 9749 9750 9751
	if (current->mm == kvm->mm) {
		/*
		 * Free memory regions allocated on behalf of userspace,
		 * unless the the memory map has changed due to process exit
		 * or fd copying.
		 */
9752 9753 9754
		x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0);
		x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 0, 0);
		x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
9755
	}
9756 9757
	if (kvm_x86_ops->vm_destroy)
		kvm_x86_ops->vm_destroy(kvm);
9758 9759
	kvm_pic_destroy(kvm);
	kvm_ioapic_destroy(kvm);
9760
	kvm_free_vcpus(kvm);
9761
	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
E
Eric Hankland 已提交
9762
	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
9763
	kvm_mmu_uninit_vm(kvm);
9764
	kvm_page_track_cleanup(kvm);
9765
	kvm_hv_destroy_vm(kvm);
9766
}
9767

9768
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
9769 9770 9771 9772
			   struct kvm_memory_slot *dont)
{
	int i;

9773 9774
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
		if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
T
Thomas Huth 已提交
9775
			kvfree(free->arch.rmap[i]);
9776
			free->arch.rmap[i] = NULL;
9777
		}
9778 9779 9780 9781 9782
		if (i == 0)
			continue;

		if (!dont || free->arch.lpage_info[i - 1] !=
			     dont->arch.lpage_info[i - 1]) {
T
Thomas Huth 已提交
9783
			kvfree(free->arch.lpage_info[i - 1]);
9784
			free->arch.lpage_info[i - 1] = NULL;
9785 9786
		}
	}
9787 9788

	kvm_page_track_free_memslot(free, dont);
9789 9790
}

9791 9792
int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
			    unsigned long npages)
9793 9794 9795
{
	int i;

9796
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
9797
		struct kvm_lpage_info *linfo;
9798 9799
		unsigned long ugfn;
		int lpages;
9800
		int level = i + 1;
9801 9802 9803 9804

		lpages = gfn_to_index(slot->base_gfn + npages - 1,
				      slot->base_gfn, level) + 1;

9805
		slot->arch.rmap[i] =
K
Kees Cook 已提交
9806
			kvcalloc(lpages, sizeof(*slot->arch.rmap[i]),
9807
				 GFP_KERNEL_ACCOUNT);
9808
		if (!slot->arch.rmap[i])
9809
			goto out_free;
9810 9811
		if (i == 0)
			continue;
9812

9813
		linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
9814
		if (!linfo)
9815 9816
			goto out_free;

9817 9818
		slot->arch.lpage_info[i - 1] = linfo;

9819
		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
9820
			linfo[0].disallow_lpage = 1;
9821
		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
9822
			linfo[lpages - 1].disallow_lpage = 1;
9823 9824 9825 9826 9827 9828 9829 9830 9831 9832 9833
		ugfn = slot->userspace_addr >> PAGE_SHIFT;
		/*
		 * If the gfn and userspace address are not aligned wrt each
		 * other, or if explicitly asked to, disable large page
		 * support for this slot
		 */
		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
		    !kvm_largepages_enabled()) {
			unsigned long j;

			for (j = 0; j < lpages; ++j)
9834
				linfo[j].disallow_lpage = 1;
9835 9836 9837
		}
	}

9838 9839 9840
	if (kvm_page_track_create_memslot(slot, npages))
		goto out_free;

9841 9842 9843
	return 0;

out_free:
9844
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
T
Thomas Huth 已提交
9845
		kvfree(slot->arch.rmap[i]);
9846 9847 9848 9849
		slot->arch.rmap[i] = NULL;
		if (i == 0)
			continue;

T
Thomas Huth 已提交
9850
		kvfree(slot->arch.lpage_info[i - 1]);
9851
		slot->arch.lpage_info[i - 1] = NULL;
9852 9853 9854 9855
	}
	return -ENOMEM;
}

9856
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
9857
{
9858 9859 9860 9861
	/*
	 * memslots->generation has been incremented.
	 * mmio generation may have reached its maximum value.
	 */
9862
	kvm_mmu_invalidate_mmio_sptes(kvm, gen);
9863 9864
}

9865 9866
int kvm_arch_prepare_memory_region(struct kvm *kvm,
				struct kvm_memory_slot *memslot,
9867
				const struct kvm_userspace_memory_region *mem,
9868
				enum kvm_mr_change change)
9869
{
9870 9871 9872
	return 0;
}

9873 9874 9875 9876 9877 9878 9879 9880 9881 9882 9883 9884 9885 9886 9887 9888 9889 9890 9891
static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
				     struct kvm_memory_slot *new)
{
	/* Still write protect RO slot */
	if (new->flags & KVM_MEM_READONLY) {
		kvm_mmu_slot_remove_write_access(kvm, new);
		return;
	}

	/*
	 * Call kvm_x86_ops dirty logging hooks when they are valid.
	 *
	 * kvm_x86_ops->slot_disable_log_dirty is called when:
	 *
	 *  - KVM_MR_CREATE with dirty logging is disabled
	 *  - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag
	 *
	 * The reason is, in case of PML, we need to set D-bit for any slots
	 * with dirty logging disabled in order to eliminate unnecessary GPA
M
Miaohe Lin 已提交
9892
	 * logging in PML buffer (and potential PML buffer full VMEXIT). This
9893
	 * guarantees leaving PML enabled during guest's lifetime won't have
W
Wei Yang 已提交
9894
	 * any additional overhead from PML when guest is running with dirty
9895 9896 9897 9898 9899 9900 9901 9902 9903 9904 9905 9906 9907 9908 9909 9910 9911 9912 9913 9914 9915 9916 9917 9918 9919 9920 9921 9922
	 * logging disabled for memory slots.
	 *
	 * kvm_x86_ops->slot_enable_log_dirty is called when switching new slot
	 * to dirty logging mode.
	 *
	 * If kvm_x86_ops dirty logging hooks are invalid, use write protect.
	 *
	 * In case of write protect:
	 *
	 * Write protect all pages for dirty logging.
	 *
	 * All the sptes including the large sptes which point to this
	 * slot are set to readonly. We can not create any new large
	 * spte on this slot until the end of the logging.
	 *
	 * See the comments in fast_page_fault().
	 */
	if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
		if (kvm_x86_ops->slot_enable_log_dirty)
			kvm_x86_ops->slot_enable_log_dirty(kvm, new);
		else
			kvm_mmu_slot_remove_write_access(kvm, new);
	} else {
		if (kvm_x86_ops->slot_disable_log_dirty)
			kvm_x86_ops->slot_disable_log_dirty(kvm, new);
	}
}

9923
void kvm_arch_commit_memory_region(struct kvm *kvm,
9924
				const struct kvm_userspace_memory_region *mem,
9925
				const struct kvm_memory_slot *old,
9926
				const struct kvm_memory_slot *new,
9927
				enum kvm_mr_change change)
9928
{
9929
	if (!kvm->arch.n_requested_mmu_pages)
9930 9931
		kvm_mmu_change_mmu_pages(kvm,
				kvm_mmu_calculate_default_mmu_pages(kvm));
9932

9933 9934 9935 9936 9937 9938 9939 9940 9941 9942 9943
	/*
	 * Dirty logging tracks sptes in 4k granularity, meaning that large
	 * sptes have to be split.  If live migration is successful, the guest
	 * in the source machine will be destroyed and large sptes will be
	 * created in the destination. However, if the guest continues to run
	 * in the source machine (for example if live migration fails), small
	 * sptes will remain around and cause bad performance.
	 *
	 * Scan sptes if dirty logging has been stopped, dropping those
	 * which can be collapsed into a single large-page spte.  Later
	 * page faults will create the large-page sptes.
9944 9945 9946 9947 9948
	 *
	 * There is no need to do this in any of the following cases:
	 * CREATE:	No dirty mappings will already exist.
	 * MOVE/DELETE:	The old mappings will already have been cleaned up by
	 *		kvm_arch_flush_shadow_memslot()
9949
	 */
9950
	if (change == KVM_MR_FLAGS_ONLY &&
9951 9952 9953 9954
		(old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
		!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
		kvm_mmu_zap_collapsible_sptes(kvm, new);

9955
	/*
9956
	 * Set up write protection and/or dirty logging for the new slot.
9957
	 *
9958 9959 9960 9961
	 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have
	 * been zapped so no dirty logging staff is needed for old slot. For
	 * KVM_MR_FLAGS_ONLY, the old slot is essentially the same one as the
	 * new and it's also covered when dealing with the new slot.
9962 9963
	 *
	 * FIXME: const-ify all uses of struct kvm_memory_slot.
9964
	 */
9965
	if (change != KVM_MR_DELETE)
9966
		kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new);
9967
}
9968

9969
void kvm_arch_flush_shadow_all(struct kvm *kvm)
9970
{
9971
	kvm_mmu_zap_all(kvm);
9972 9973
}

9974 9975 9976
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
9977
	kvm_page_track_flush_slot(kvm, slot);
9978 9979
}

9980 9981 9982 9983 9984 9985 9986
static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
{
	return (is_guest_mode(vcpu) &&
			kvm_x86_ops->guest_apic_has_interrupt &&
			kvm_x86_ops->guest_apic_has_interrupt(vcpu));
}

9987 9988 9989 9990 9991 9992 9993 9994 9995 9996 9997
static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
{
	if (!list_empty_careful(&vcpu->async_pf.done))
		return true;

	if (kvm_apic_has_events(vcpu))
		return true;

	if (vcpu->arch.pv.pv_unhalted)
		return true;

9998 9999 10000
	if (vcpu->arch.exception.pending)
		return true;

10001 10002 10003
	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
	    (vcpu->arch.nmi_pending &&
	     kvm_x86_ops->nmi_allowed(vcpu)))
10004 10005
		return true;

10006 10007
	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
	    (vcpu->arch.smi_pending && !is_smm(vcpu)))
P
Paolo Bonzini 已提交
10008 10009
		return true;

10010
	if (kvm_arch_interrupt_allowed(vcpu) &&
10011 10012
	    (kvm_cpu_has_interrupt(vcpu) ||
	    kvm_guest_apic_has_interrupt(vcpu)))
10013 10014
		return true;

A
Andrey Smetanin 已提交
10015 10016 10017
	if (kvm_hv_has_stimer_pending(vcpu))
		return true;

10018 10019 10020
	return false;
}

10021 10022
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
10023
	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
10024
}
10025

10026 10027 10028 10029 10030 10031 10032 10033 10034 10035 10036 10037 10038 10039 10040 10041
bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
{
	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
		return true;

	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
		kvm_test_request(KVM_REQ_SMI, vcpu) ||
		 kvm_test_request(KVM_REQ_EVENT, vcpu))
		return true;

	if (vcpu->arch.apicv_active && kvm_x86_ops->dy_apicv_has_pending_interrupt(vcpu))
		return true;

	return false;
}

10042 10043
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
{
10044
	return vcpu->arch.preempted_in_kernel;
10045 10046
}

10047
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
10048
{
10049
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
10050
}
10051 10052 10053 10054 10055

int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	return kvm_x86_ops->interrupt_allowed(vcpu);
}
10056

10057
unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
J
Jan Kiszka 已提交
10058
{
10059 10060 10061 10062 10063 10064
	if (is_64_bit_mode(vcpu))
		return kvm_rip_read(vcpu);
	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
		     kvm_rip_read(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
J
Jan Kiszka 已提交
10065

10066 10067 10068
bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
{
	return kvm_get_linear_rip(vcpu) == linear_rip;
J
Jan Kiszka 已提交
10069 10070 10071
}
EXPORT_SYMBOL_GPL(kvm_is_linear_rip);

10072 10073 10074 10075 10076 10077
unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
{
	unsigned long rflags;

	rflags = kvm_x86_ops->get_rflags(vcpu);
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
10078
		rflags &= ~X86_EFLAGS_TF;
10079 10080 10081 10082
	return rflags;
}
EXPORT_SYMBOL_GPL(kvm_get_rflags);

10083
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
10084 10085
{
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
J
Jan Kiszka 已提交
10086
	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
10087
		rflags |= X86_EFLAGS_TF;
10088
	kvm_x86_ops->set_rflags(vcpu, rflags);
10089 10090 10091 10092 10093
}

void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
	__kvm_set_rflags(vcpu, rflags);
10094
	kvm_make_request(KVM_REQ_EVENT, vcpu);
10095 10096 10097
}
EXPORT_SYMBOL_GPL(kvm_set_rflags);

G
Gleb Natapov 已提交
10098 10099 10100 10101
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
{
	int r;

10102
	if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||
10103
	      work->wakeup_all)
G
Gleb Natapov 已提交
10104 10105 10106 10107 10108 10109
		return;

	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		return;

10110 10111
	if (!vcpu->arch.mmu->direct_map &&
	      work->arch.cr3 != vcpu->arch.mmu->get_cr3(vcpu))
X
Xiao Guangrong 已提交
10112 10113
		return;

10114
	vcpu->arch.mmu->page_fault(vcpu, work->cr2_or_gpa, 0, true);
G
Gleb Natapov 已提交
10115 10116
}

10117 10118 10119 10120 10121 10122 10123 10124 10125 10126 10127 10128 10129 10130 10131 10132 10133 10134 10135 10136 10137 10138 10139 10140 10141 10142
static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
{
	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
}

static inline u32 kvm_async_pf_next_probe(u32 key)
{
	return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
}

static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 key = kvm_async_pf_hash_fn(gfn);

	while (vcpu->arch.apf.gfns[key] != ~0)
		key = kvm_async_pf_next_probe(key);

	vcpu->arch.apf.gfns[key] = gfn;
}

static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	int i;
	u32 key = kvm_async_pf_hash_fn(gfn);

	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
10143 10144
		     (vcpu->arch.apf.gfns[key] != gfn &&
		      vcpu->arch.apf.gfns[key] != ~0); i++)
10145 10146 10147 10148 10149 10150 10151 10152 10153 10154 10155 10156 10157 10158 10159 10160 10161 10162 10163 10164 10165 10166 10167 10168 10169 10170 10171 10172 10173 10174 10175 10176 10177
		key = kvm_async_pf_next_probe(key);

	return key;
}

bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
}

static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 i, j, k;

	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
	while (true) {
		vcpu->arch.apf.gfns[i] = ~0;
		do {
			j = kvm_async_pf_next_probe(j);
			if (vcpu->arch.apf.gfns[j] == ~0)
				return;
			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
			/*
			 * k lies cyclically in ]i,j]
			 * |    i.k.j |
			 * |....j i.k.| or  |.k..j i...|
			 */
		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
		i = j;
	}
}

10178 10179
static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
{
10180 10181 10182

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
				      sizeof(val));
10183 10184
}

10185 10186 10187 10188 10189 10190 10191
static int apf_get_user(struct kvm_vcpu *vcpu, u32 *val)
{

	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, val,
				      sizeof(u32));
}

10192 10193 10194 10195 10196 10197 10198 10199 10200 10201 10202 10203 10204 10205 10206 10207 10208 10209 10210 10211 10212 10213 10214 10215 10216 10217 10218 10219 10220 10221
static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
{
	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
		return false;

	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
	    (vcpu->arch.apf.send_user_only &&
	     kvm_x86_ops->get_cpl(vcpu) == 0))
		return false;

	return true;
}

bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
{
	if (unlikely(!lapic_in_kernel(vcpu) ||
		     kvm_event_needs_reinjection(vcpu) ||
		     vcpu->arch.exception.pending))
		return false;

	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
		return false;

	/*
	 * If interrupts are off we cannot even use an artificial
	 * halt state.
	 */
	return kvm_x86_ops->interrupt_allowed(vcpu);
}

10222 10223 10224
void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
				     struct kvm_async_pf *work)
{
10225 10226
	struct x86_exception fault;

10227
	trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa);
10228
	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
10229

10230 10231
	if (kvm_can_deliver_async_pf(vcpu) &&
	    !apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
10232 10233 10234 10235 10236
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
10237
		fault.async_page_fault = true;
10238
		kvm_inject_page_fault(vcpu, &fault);
10239 10240 10241 10242 10243 10244 10245 10246 10247 10248
	} else {
		/*
		 * It is not possible to deliver a paravirtualized asynchronous
		 * page fault, but putting the guest in an artificial halt state
		 * can be beneficial nevertheless: if an interrupt arrives, we
		 * can deliver it timely and perhaps the guest will schedule
		 * another process.  When the instruction that triggered a page
		 * fault is retried, hopefully the page will be ready in the host.
		 */
		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
10249
	}
10250 10251 10252 10253 10254
}

void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
				 struct kvm_async_pf *work)
{
10255
	struct x86_exception fault;
10256
	u32 val;
10257

10258
	if (work->wakeup_all)
10259 10260 10261
		work->arch.token = ~0; /* broadcast wakeup */
	else
		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
10262
	trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);
10263

10264 10265 10266 10267 10268 10269 10270 10271 10272 10273 10274
	if (vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED &&
	    !apf_get_user(vcpu, &val)) {
		if (val == KVM_PV_REASON_PAGE_NOT_PRESENT &&
		    vcpu->arch.exception.pending &&
		    vcpu->arch.exception.nr == PF_VECTOR &&
		    !apf_put_user(vcpu, 0)) {
			vcpu->arch.exception.injected = false;
			vcpu->arch.exception.pending = false;
			vcpu->arch.exception.nr = 0;
			vcpu->arch.exception.has_error_code = false;
			vcpu->arch.exception.error_code = 0;
10275 10276
			vcpu->arch.exception.has_payload = false;
			vcpu->arch.exception.payload = 0;
10277 10278 10279 10280 10281 10282 10283 10284 10285
		} else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
			fault.vector = PF_VECTOR;
			fault.error_code_valid = true;
			fault.error_code = 0;
			fault.nested_page_fault = false;
			fault.address = work->arch.token;
			fault.async_page_fault = true;
			kvm_inject_page_fault(vcpu, &fault);
		}
10286
	}
10287
	vcpu->arch.apf.halted = false;
10288
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10289 10290 10291 10292 10293 10294 10295
}

bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
		return true;
	else
10296
		return kvm_can_do_async_pf(vcpu);
10297 10298
}

10299 10300 10301 10302 10303 10304 10305 10306 10307 10308 10309 10310 10311 10312 10313 10314 10315 10316
void kvm_arch_start_assignment(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);

void kvm_arch_end_assignment(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);

bool kvm_arch_has_assigned_device(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);

10317 10318 10319 10320 10321 10322 10323 10324 10325 10326 10327 10328 10329 10330 10331 10332 10333 10334
void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);

void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);

bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);

10335 10336
bool kvm_arch_has_irq_bypass(void)
{
10337
	return true;
10338 10339
}

F
Feng Wu 已提交
10340 10341 10342 10343 10344 10345
int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

10346
	irqfd->producer = prod;
F
Feng Wu 已提交
10347

10348 10349
	return kvm_x86_ops->update_pi_irte(irqfd->kvm,
					   prod->irq, irqfd->gsi, 1);
F
Feng Wu 已提交
10350 10351 10352 10353 10354 10355 10356 10357 10358 10359 10360 10361 10362 10363 10364
}

void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	int ret;
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

	WARN_ON(irqfd->producer != prod);
	irqfd->producer = NULL;

	/*
	 * When producer of consumer is unregistered, we change back to
	 * remapped mode, so we can re-use the current implementation
A
Andrea Gelmini 已提交
10365
	 * when the irq is masked/disabled or the consumer side (KVM
F
Feng Wu 已提交
10366 10367 10368 10369 10370 10371 10372 10373 10374 10375 10376 10377 10378 10379
	 * int this case doesn't want to receive the interrupts.
	*/
	ret = kvm_x86_ops->update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0);
	if (ret)
		printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
		       " fails: %d\n", irqfd->consumer.token, ret);
}

int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
				   uint32_t guest_irq, bool set)
{
	return kvm_x86_ops->update_pi_irte(kvm, host_irq, guest_irq, set);
}

10380 10381 10382 10383 10384 10385
bool kvm_vector_hashing_enabled(void)
{
	return vector_hashing;
}
EXPORT_SYMBOL_GPL(kvm_vector_hashing_enabled);

10386 10387 10388 10389 10390 10391
bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
}
EXPORT_SYMBOL_GPL(kvm_arch_no_poll);

10392 10393 10394 10395 10396 10397 10398 10399 10400 10401 10402 10403 10404 10405 10406 10407 10408 10409 10410 10411 10412 10413
u64 kvm_spec_ctrl_valid_bits(struct kvm_vcpu *vcpu)
{
	uint64_t bits = SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD;

	/* The STIBP bit doesn't fault even if it's not advertised */
	if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS))
		bits &= ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP);
	if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL) &&
	    !boot_cpu_has(X86_FEATURE_AMD_IBRS))
		bits &= ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP);

	if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL_SSBD) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
		bits &= ~SPEC_CTRL_SSBD;
	if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
	    !boot_cpu_has(X86_FEATURE_AMD_SSBD))
		bits &= ~SPEC_CTRL_SSBD;

	return bits;
}
EXPORT_SYMBOL_GPL(kvm_spec_ctrl_valid_bits);
10414

10415
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
J
Jason Wang 已提交
10416
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
10417 10418 10419 10420
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
10421
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
10422
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
10423
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
10424
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
10425
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
10426
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
10427
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
10428
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
10429
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
P
Peter Xu 已提交
10430
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
K
Kai Huang 已提交
10431
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
10432
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
10433 10434
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);