x86.c 264.9 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
B
Ben-Ami Yassour 已提交
8 9
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
N
Nicolas Kaiser 已提交
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11 12 13 14
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
B
Ben-Ami Yassour 已提交
15 16
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
17 18
 */

19
#include <linux/kvm_host.h>
20
#include "irq.h"
21
#include "mmu.h"
S
Sheng Yang 已提交
22
#include "i8254.h"
23
#include "tss.h"
24
#include "kvm_cache_regs.h"
25
#include "x86.h"
A
Avi Kivity 已提交
26
#include "cpuid.h"
27
#include "pmu.h"
28
#include "hyperv.h"
29

30
#include <linux/clocksource.h>
B
Ben-Ami Yassour 已提交
31
#include <linux/interrupt.h>
32 33 34
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
35 36
#include <linux/export.h>
#include <linux/moduleparam.h>
37
#include <linux/mman.h>
38
#include <linux/highmem.h>
J
Joerg Roedel 已提交
39
#include <linux/iommu.h>
B
Ben-Ami Yassour 已提交
40
#include <linux/intel-iommu.h>
41
#include <linux/cpufreq.h>
A
Avi Kivity 已提交
42
#include <linux/user-return-notifier.h>
43
#include <linux/srcu.h>
44
#include <linux/slab.h>
45
#include <linux/perf_event.h>
46
#include <linux/uaccess.h>
47
#include <linux/hash.h>
48
#include <linux/pci.h>
49 50
#include <linux/timekeeper_internal.h>
#include <linux/pvclock_gtod.h>
F
Feng Wu 已提交
51 52
#include <linux/kvm_irqfd.h>
#include <linux/irqbypass.h>
53
#include <linux/sched/stat.h>
54
#include <linux/sched/isolation.h>
55
#include <linux/mem_encrypt.h>
56

A
Avi Kivity 已提交
57
#include <trace/events/kvm.h>
X
Xiao Guangrong 已提交
58

59
#include <asm/debugreg.h>
60
#include <asm/msr.h>
61
#include <asm/desc.h>
H
Huang Ying 已提交
62
#include <asm/mce.h>
63
#include <linux/kernel_stat.h>
64
#include <asm/fpu/internal.h> /* Ugh! */
65
#include <asm/pvclock.h>
66
#include <asm/div64.h>
67
#include <asm/irq_remapping.h>
68
#include <asm/mshyperv.h>
69
#include <asm/hypervisor.h>
70
#include <asm/intel_pt.h>
71
#include <clocksource/hyperv_timer.h>
72

73 74 75
#define CREATE_TRACE_POINTS
#include "trace.h"

76
#define MAX_IO_MSRS 256
H
Huang Ying 已提交
77
#define KVM_MAX_MCE_BANKS 32
78 79
u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P;
EXPORT_SYMBOL_GPL(kvm_mce_cap_supported);
H
Huang Ying 已提交
80

81 82 83
#define emul_to_vcpu(ctxt) \
	container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)

84 85 86 87 88
/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
89 90
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
91
#else
92
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
93
#endif
94

95 96
#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
97

98 99
#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
                                    KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
100

101
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
A
Avi Kivity 已提交
102
static void process_nmi(struct kvm_vcpu *vcpu);
103
static void enter_smm(struct kvm_vcpu *vcpu);
104
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
K
Ken Hofsass 已提交
105 106
static void store_regs(struct kvm_vcpu *vcpu);
static int sync_regs(struct kvm_vcpu *vcpu);
107

108
struct kvm_x86_ops *kvm_x86_ops __read_mostly;
109
EXPORT_SYMBOL_GPL(kvm_x86_ops);
110

111
static bool __read_mostly ignore_msrs = 0;
112
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
113

114 115 116
static bool __read_mostly report_ignored_msrs = true;
module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);

117
unsigned int min_timer_period_us = 200;
118 119
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

120 121 122
static bool __read_mostly kvmclock_periodic_sync = true;
module_param(kvmclock_periodic_sync, bool, S_IRUGO);

123
bool __read_mostly kvm_has_tsc_control;
124
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
125
u32  __read_mostly kvm_max_guest_tsc_khz;
126
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
127 128 129 130
u8   __read_mostly kvm_tsc_scaling_ratio_frac_bits;
EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits);
u64  __read_mostly kvm_max_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
131 132
u64 __read_mostly kvm_default_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
133

134
/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
135
static u32 __read_mostly tsc_tolerance_ppm = 250;
136 137
module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);

138 139 140 141 142 143 144
/*
 * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
 * adaptive tuning starting from default advancment of 1000ns.  '0' disables
 * advancement entirely.  Any other value is used as-is and disables adaptive
 * tuning, i.e. allows priveleged userspace to set an exact advancement time.
 */
static int __read_mostly lapic_timer_advance_ns = -1;
145
module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
146

147 148 149
static bool __read_mostly vector_hashing = true;
module_param(vector_hashing, bool, S_IRUGO);

150 151 152 153
bool __read_mostly enable_vmware_backdoor = false;
module_param(enable_vmware_backdoor, bool, S_IRUGO);
EXPORT_SYMBOL_GPL(enable_vmware_backdoor);

154 155 156
static bool __read_mostly force_emulation_prefix = false;
module_param(force_emulation_prefix, bool, S_IRUGO);

157 158 159
int __read_mostly pi_inject_timer = -1;
module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR);

A
Avi Kivity 已提交
160 161 162 163
#define KVM_NR_SHARED_MSRS 16

struct kvm_shared_msrs_global {
	int nr;
164
	u32 msrs[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
165 166 167 168 169
};

struct kvm_shared_msrs {
	struct user_return_notifier urn;
	bool registered;
170 171 172 173
	struct kvm_shared_msr_values {
		u64 host;
		u64 curr;
	} values[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
174 175 176
};

static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
177
static struct kvm_shared_msrs __percpu *shared_msrs;
A
Avi Kivity 已提交
178

179
struct kvm_stats_debugfs_item debugfs_entries[] = {
180 181 182 183 184 185 186 187 188
	{ "pf_fixed", VCPU_STAT(pf_fixed) },
	{ "pf_guest", VCPU_STAT(pf_guest) },
	{ "tlb_flush", VCPU_STAT(tlb_flush) },
	{ "invlpg", VCPU_STAT(invlpg) },
	{ "exits", VCPU_STAT(exits) },
	{ "io_exits", VCPU_STAT(io_exits) },
	{ "mmio_exits", VCPU_STAT(mmio_exits) },
	{ "signal_exits", VCPU_STAT(signal_exits) },
	{ "irq_window", VCPU_STAT(irq_window_exits) },
189
	{ "nmi_window", VCPU_STAT(nmi_window_exits) },
190
	{ "halt_exits", VCPU_STAT(halt_exits) },
191
	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
192
	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
193
	{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
194
	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
A
Amit Shah 已提交
195
	{ "hypercalls", VCPU_STAT(hypercalls) },
196 197 198 199 200 201
	{ "request_irq", VCPU_STAT(request_irq_exits) },
	{ "irq_exits", VCPU_STAT(irq_exits) },
	{ "host_state_reload", VCPU_STAT(host_state_reload) },
	{ "fpu_reload", VCPU_STAT(fpu_reload) },
	{ "insn_emulation", VCPU_STAT(insn_emulation) },
	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
202
	{ "irq_injections", VCPU_STAT(irq_injections) },
203
	{ "nmi_injections", VCPU_STAT(nmi_injections) },
204
	{ "req_event", VCPU_STAT(req_event) },
P
Paolo Bonzini 已提交
205
	{ "l1d_flush", VCPU_STAT(l1d_flush) },
A
Avi Kivity 已提交
206 207 208 209 210 211
	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
	{ "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
	{ "mmu_flooded", VM_STAT(mmu_flooded) },
	{ "mmu_recycled", VM_STAT(mmu_recycled) },
A
Avi Kivity 已提交
212
	{ "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
213
	{ "mmu_unsync", VM_STAT(mmu_unsync) },
214
	{ "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
M
Marcelo Tosatti 已提交
215
	{ "largepages", VM_STAT(lpages) },
216 217
	{ "max_mmu_page_hash_collisions",
		VM_STAT(max_mmu_page_hash_collisions) },
218 219 220
	{ NULL }
};

221 222
u64 __read_mostly host_xcr0;

223 224 225
struct kmem_cache *x86_fpu_cache;
EXPORT_SYMBOL_GPL(x86_fpu_cache);

226
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
227

228 229 230 231 232 233 234
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
	int i;
	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
		vcpu->arch.apf.gfns[i] = ~0;
}

A
Avi Kivity 已提交
235 236 237 238 239
static void kvm_on_user_return(struct user_return_notifier *urn)
{
	unsigned slot;
	struct kvm_shared_msrs *locals
		= container_of(urn, struct kvm_shared_msrs, urn);
240
	struct kvm_shared_msr_values *values;
241 242 243 244 245 246 247 248 249 250 251 252
	unsigned long flags;

	/*
	 * Disabling irqs at this point since the following code could be
	 * interrupted and executed through kvm_arch_hardware_disable()
	 */
	local_irq_save(flags);
	if (locals->registered) {
		locals->registered = false;
		user_return_notifier_unregister(urn);
	}
	local_irq_restore(flags);
A
Avi Kivity 已提交
253
	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
254 255 256 257
		values = &locals->values[slot];
		if (values->host != values->curr) {
			wrmsrl(shared_msrs_global.msrs[slot], values->host);
			values->curr = values->host;
A
Avi Kivity 已提交
258 259 260 261
		}
	}
}

262
static void shared_msr_update(unsigned slot, u32 msr)
A
Avi Kivity 已提交
263 264
{
	u64 value;
265 266
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
A
Avi Kivity 已提交
267

268 269 270 271 272 273 274 275 276 277 278 279 280
	/* only read, and nobody should modify it at this time,
	 * so don't need lock */
	if (slot >= shared_msrs_global.nr) {
		printk(KERN_ERR "kvm: invalid MSR slot!");
		return;
	}
	rdmsrl_safe(msr, &value);
	smsr->values[slot].host = value;
	smsr->values[slot].curr = value;
}

void kvm_define_shared_msr(unsigned slot, u32 msr)
{
281
	BUG_ON(slot >= KVM_NR_SHARED_MSRS);
282
	shared_msrs_global.msrs[slot] = msr;
A
Avi Kivity 已提交
283 284 285 286 287 288 289 290 291 292
	if (slot >= shared_msrs_global.nr)
		shared_msrs_global.nr = slot + 1;
}
EXPORT_SYMBOL_GPL(kvm_define_shared_msr);

static void kvm_shared_msr_cpu_online(void)
{
	unsigned i;

	for (i = 0; i < shared_msrs_global.nr; ++i)
293
		shared_msr_update(i, shared_msrs_global.msrs[i]);
A
Avi Kivity 已提交
294 295
}

296
int kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
A
Avi Kivity 已提交
297
{
298 299
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
300
	int err;
A
Avi Kivity 已提交
301

302
	if (((value ^ smsr->values[slot].curr) & mask) == 0)
303
		return 0;
304
	smsr->values[slot].curr = value;
305 306 307 308
	err = wrmsrl_safe(shared_msrs_global.msrs[slot], value);
	if (err)
		return 1;

A
Avi Kivity 已提交
309 310 311 312 313
	if (!smsr->registered) {
		smsr->urn.on_user_return = kvm_on_user_return;
		user_return_notifier_register(&smsr->urn);
		smsr->registered = true;
	}
314
	return 0;
A
Avi Kivity 已提交
315 316 317
}
EXPORT_SYMBOL_GPL(kvm_set_shared_msr);

318
static void drop_user_return_notifiers(void)
319
{
320 321
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
322 323 324 325 326

	if (smsr->registered)
		kvm_on_user_return(&smsr->urn);
}

327 328
u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
329
	return vcpu->arch.apic_base;
330 331 332
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

333 334 335 336 337 338
enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
{
	return kvm_apic_mode(kvm_get_apic_base(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_apic_mode);

339 340
int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
341 342
	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
343 344
	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |
		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
345

346
	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
347
		return 1;
348 349 350 351 352 353
	if (!msr_info->host_initiated) {
		if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
			return 1;
		if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
			return 1;
	}
354 355 356

	kvm_lapic_set_base(vcpu, msr_info->data);
	return 0;
357 358 359
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

360
asmlinkage __visible void kvm_spurious_fault(void)
361 362
{
	/* Fault while not rebooting.  We want the trace. */
363 364
	if (!kvm_rebooting)
		BUG();
365 366 367
}
EXPORT_SYMBOL_GPL(kvm_spurious_fault);

368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
#define EXCPT_BENIGN		0
#define EXCPT_CONTRIBUTORY	1
#define EXCPT_PF		2

static int exception_class(int vector)
{
	switch (vector) {
	case PF_VECTOR:
		return EXCPT_PF;
	case DE_VECTOR:
	case TS_VECTOR:
	case NP_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
		return EXCPT_CONTRIBUTORY;
	default:
		break;
	}
	return EXCPT_BENIGN;
}

389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
#define EXCPT_FAULT		0
#define EXCPT_TRAP		1
#define EXCPT_ABORT		2
#define EXCPT_INTERRUPT		3

static int exception_type(int vector)
{
	unsigned int mask;

	if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
		return EXCPT_INTERRUPT;

	mask = 1 << vector;

	/* #DB is trap, as instruction watchpoints are handled elsewhere */
	if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR)))
		return EXCPT_TRAP;

	if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
		return EXCPT_ABORT;

	/* Reserved exceptions will result in fault */
	return EXCPT_FAULT;
}

414 415 416 417 418 419 420 421 422 423
void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
{
	unsigned nr = vcpu->arch.exception.nr;
	bool has_payload = vcpu->arch.exception.has_payload;
	unsigned long payload = vcpu->arch.exception.payload;

	if (!has_payload)
		return;

	switch (nr) {
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
	case DB_VECTOR:
		/*
		 * "Certain debug exceptions may clear bit 0-3.  The
		 * remaining contents of the DR6 register are never
		 * cleared by the processor".
		 */
		vcpu->arch.dr6 &= ~DR_TRAP_BITS;
		/*
		 * DR6.RTM is set by all #DB exceptions that don't clear it.
		 */
		vcpu->arch.dr6 |= DR6_RTM;
		vcpu->arch.dr6 |= payload;
		/*
		 * Bit 16 should be set in the payload whenever the #DB
		 * exception should clear DR6.RTM. This makes the payload
		 * compatible with the pending debug exceptions under VMX.
		 * Though not currently documented in the SDM, this also
		 * makes the payload compatible with the exit qualification
		 * for #DB exceptions under VMX.
		 */
		vcpu->arch.dr6 ^= payload & DR6_RTM;
		break;
446 447 448 449 450 451 452 453 454 455
	case PF_VECTOR:
		vcpu->arch.cr2 = payload;
		break;
	}

	vcpu->arch.exception.has_payload = false;
	vcpu->arch.exception.payload = 0;
}
EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);

456
static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
457
		unsigned nr, bool has_error, u32 error_code,
458
	        bool has_payload, unsigned long payload, bool reinject)
459 460 461 462
{
	u32 prev_nr;
	int class1, class2;

463 464
	kvm_make_request(KVM_REQ_EVENT, vcpu);

465
	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
466
	queue:
467 468
		if (has_error && !is_protmode(vcpu))
			has_error = false;
469 470 471 472 473 474 475 476 477 478 479
		if (reinject) {
			/*
			 * On vmentry, vcpu->arch.exception.pending is only
			 * true if an event injection was blocked by
			 * nested_run_pending.  In that case, however,
			 * vcpu_enter_guest requests an immediate exit,
			 * and the guest shouldn't proceed far enough to
			 * need reinjection.
			 */
			WARN_ON_ONCE(vcpu->arch.exception.pending);
			vcpu->arch.exception.injected = true;
480 481 482 483 484 485 486 487
			if (WARN_ON_ONCE(has_payload)) {
				/*
				 * A reinjected event has already
				 * delivered its payload.
				 */
				has_payload = false;
				payload = 0;
			}
488 489 490 491
		} else {
			vcpu->arch.exception.pending = true;
			vcpu->arch.exception.injected = false;
		}
492 493 494
		vcpu->arch.exception.has_error_code = has_error;
		vcpu->arch.exception.nr = nr;
		vcpu->arch.exception.error_code = error_code;
495 496
		vcpu->arch.exception.has_payload = has_payload;
		vcpu->arch.exception.payload = payload;
497 498 499
		/*
		 * In guest mode, payload delivery should be deferred,
		 * so that the L1 hypervisor can intercept #PF before
500 501 502 503 504 505 506
		 * CR2 is modified (or intercept #DB before DR6 is
		 * modified under nVMX).  However, for ABI
		 * compatibility with KVM_GET_VCPU_EVENTS and
		 * KVM_SET_VCPU_EVENTS, we can't delay payload
		 * delivery unless userspace has enabled this
		 * functionality via the per-VM capability,
		 * KVM_CAP_EXCEPTION_PAYLOAD.
507 508 509 510
		 */
		if (!vcpu->kvm->arch.exception_payload_enabled ||
		    !is_guest_mode(vcpu))
			kvm_deliver_exception_payload(vcpu);
511 512 513 514 515 516 517
		return;
	}

	/* to check exception */
	prev_nr = vcpu->arch.exception.nr;
	if (prev_nr == DF_VECTOR) {
		/* triple fault -> shutdown */
518
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
519 520 521 522 523 524
		return;
	}
	class1 = exception_class(prev_nr);
	class2 = exception_class(nr);
	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
525 526 527 528 529
		/*
		 * Generate double fault per SDM Table 5-5.  Set
		 * exception.pending = true so that the double fault
		 * can trigger a nested vmexit.
		 */
530
		vcpu->arch.exception.pending = true;
531
		vcpu->arch.exception.injected = false;
532 533 534
		vcpu->arch.exception.has_error_code = true;
		vcpu->arch.exception.nr = DF_VECTOR;
		vcpu->arch.exception.error_code = 0;
535 536
		vcpu->arch.exception.has_payload = false;
		vcpu->arch.exception.payload = 0;
537 538 539 540 541 542 543
	} else
		/* replace previous exception with a new one in a hope
		   that instruction re-execution will regenerate lost
		   exception */
		goto queue;
}

544 545
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
546
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
547 548 549
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

550 551
void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
552
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
553 554 555
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

556 557 558 559 560 561
static void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
				  unsigned long payload)
{
	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
}

562 563 564 565 566 567 568
static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
				    u32 error_code, unsigned long payload)
{
	kvm_multiple_exception(vcpu, nr, true, error_code,
			       true, payload, false);
}

569
int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
570
{
571 572 573
	if (err)
		kvm_inject_gp(vcpu, 0);
	else
574 575 576
		return kvm_skip_emulated_instruction(vcpu);

	return 1;
577 578
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
579

580
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
581 582
{
	++vcpu->stat.pf_guest;
583 584
	vcpu->arch.exception.nested_apf =
		is_guest_mode(vcpu) && fault->async_page_fault;
585
	if (vcpu->arch.exception.nested_apf) {
586
		vcpu->arch.apf.nested_apf_token = fault->address;
587 588 589 590 591
		kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
	} else {
		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
					fault->address);
	}
592
}
N
Nadav Har'El 已提交
593
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
594

595
static bool kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
596
{
597 598
	if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
		vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
599
	else
600
		vcpu->arch.mmu->inject_page_fault(vcpu, fault);
601 602

	return fault->nested_page_fault;
603 604
}

605 606
void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
607 608
	atomic_inc(&vcpu->arch.nmi_queued);
	kvm_make_request(KVM_REQ_NMI, vcpu);
609 610 611
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

612 613
void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
614
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
615 616 617
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

618 619
void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
620
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
621 622 623
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

624 625 626 627 628
/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
629
{
630 631 632 633
	if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
		return true;
	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
	return false;
634
}
635
EXPORT_SYMBOL_GPL(kvm_require_cpl);
636

637 638 639 640 641 642 643 644 645 646
bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
{
	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
		return true;

	kvm_queue_exception(vcpu, UD_VECTOR);
	return false;
}
EXPORT_SYMBOL_GPL(kvm_require_dr);

647 648
/*
 * This function will be used to read from the physical memory of the currently
649
 * running guest. The difference to kvm_vcpu_read_guest_page is that this function
650 651 652 653 654 655
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gfn_t ngfn, void *data, int offset, int len,
			    u32 access)
{
656
	struct x86_exception exception;
657 658 659 660
	gfn_t real_gfn;
	gpa_t ngpa;

	ngpa     = gfn_to_gpa(ngfn);
661
	real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);
662 663 664 665 666
	if (real_gfn == UNMAPPED_GVA)
		return -EFAULT;

	real_gfn = gpa_to_gfn(real_gfn);

667
	return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len);
668 669 670
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

671
static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
672 673 674 675 676 677
			       void *data, int offset, int len, u32 access)
{
	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
				       data, offset, len, access);
}

678 679 680 681 682 683
static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
{
	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) | rsvd_bits(5, 8) |
	       rsvd_bits(1, 2);
}

684
/*
685
 * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
686
 */
687
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
688 689 690 691 692
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
	int i;
	int ret;
693
	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
694

695 696 697
	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
				      offset * sizeof(u64), sizeof(pdpte),
				      PFERR_USER_MASK|PFERR_WRITE_MASK);
698 699 700 701 702
	if (ret < 0) {
		ret = 0;
		goto out;
	}
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
B
Bandan Das 已提交
703
		if ((pdpte[i] & PT_PRESENT_MASK) &&
704
		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
705 706 707 708 709 710
			ret = 0;
			goto out;
		}
	}
	ret = 1;

711
	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
A
Avi Kivity 已提交
712 713 714 715
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
716 717 718 719
out:

	return ret;
}
720
EXPORT_SYMBOL_GPL(load_pdptrs);
721

722
bool pdptrs_changed(struct kvm_vcpu *vcpu)
723
{
724
	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
725
	bool changed = true;
726 727
	int offset;
	gfn_t gfn;
728 729
	int r;

730
	if (!is_pae_paging(vcpu))
731 732
		return false;

A
Avi Kivity 已提交
733 734 735 736
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_avail))
		return true;

737 738
	gfn = (kvm_read_cr3(vcpu) & 0xffffffe0ul) >> PAGE_SHIFT;
	offset = (kvm_read_cr3(vcpu) & 0xffffffe0ul) & (PAGE_SIZE - 1);
739 740
	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
				       PFERR_USER_MASK | PFERR_WRITE_MASK);
741 742
	if (r < 0)
		goto out;
743
	changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
744 745 746 747
out:

	return changed;
}
748
EXPORT_SYMBOL_GPL(pdptrs_changed);
749

750
int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
751
{
752
	unsigned long old_cr0 = kvm_read_cr0(vcpu);
753
	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP;
754

755 756
	cr0 |= X86_CR0_ET;

757
#ifdef CONFIG_X86_64
758 759
	if (cr0 & 0xffffffff00000000UL)
		return 1;
760 761 762
#endif

	cr0 &= ~CR0_RESERVED_BITS;
763

764 765
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
		return 1;
766

767 768
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
		return 1;
769 770 771

	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
#ifdef CONFIG_X86_64
772
		if ((vcpu->arch.efer & EFER_LME)) {
773 774
			int cs_db, cs_l;

775 776
			if (!is_pae(vcpu))
				return 1;
777
			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
778 779
			if (cs_l)
				return 1;
780 781
		} else
#endif
782
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
783
						 kvm_read_cr3(vcpu)))
784
			return 1;
785 786
	}

787 788 789
	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
		return 1;

790 791
	kvm_x86_ops->set_cr0(vcpu, cr0);

792
	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
793
		kvm_clear_async_pf_completion_queue(vcpu);
794 795
		kvm_async_pf_hash_reset(vcpu);
	}
796

797 798
	if ((cr0 ^ old_cr0) & update_bits)
		kvm_mmu_reset_context(vcpu);
799

800 801 802
	if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
803 804
		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);

805 806
	return 0;
}
807
EXPORT_SYMBOL_GPL(kvm_set_cr0);
808

809
void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
810
{
811
	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
812
}
813
EXPORT_SYMBOL_GPL(kvm_lmsw);
814

815
void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
816 817 818 819
{
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
			!vcpu->guest_xcr0_loaded) {
		/* kvm_set_xcr() also depends on this */
820 821
		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
822 823 824
		vcpu->guest_xcr0_loaded = 1;
	}
}
825
EXPORT_SYMBOL_GPL(kvm_load_guest_xcr0);
826

827
void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
828 829 830 831 832 833 834
{
	if (vcpu->guest_xcr0_loaded) {
		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
		vcpu->guest_xcr0_loaded = 0;
	}
}
835
EXPORT_SYMBOL_GPL(kvm_put_guest_xcr0);
836

837
static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
838
{
839 840
	u64 xcr0 = xcr;
	u64 old_xcr0 = vcpu->arch.xcr0;
841
	u64 valid_bits;
842 843 844 845

	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
	if (index != XCR_XFEATURE_ENABLED_MASK)
		return 1;
D
Dave Hansen 已提交
846
	if (!(xcr0 & XFEATURE_MASK_FP))
847
		return 1;
D
Dave Hansen 已提交
848
	if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
849
		return 1;
850 851 852 853 854 855

	/*
	 * Do not allow the guest to set bits that we do not support
	 * saving.  However, xcr0 bit 0 is always set, even if the
	 * emulated CPU does not support XSAVE (see fx_init).
	 */
D
Dave Hansen 已提交
856
	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
857
	if (xcr0 & ~valid_bits)
858
		return 1;
859

D
Dave Hansen 已提交
860 861
	if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
	    (!(xcr0 & XFEATURE_MASK_BNDCSR)))
862 863
		return 1;

D
Dave Hansen 已提交
864 865
	if (xcr0 & XFEATURE_MASK_AVX512) {
		if (!(xcr0 & XFEATURE_MASK_YMM))
866
			return 1;
D
Dave Hansen 已提交
867
		if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
868 869
			return 1;
	}
870
	vcpu->arch.xcr0 = xcr0;
871

D
Dave Hansen 已提交
872
	if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
873
		kvm_update_cpuid(vcpu);
874 875 876 877 878
	return 0;
}

int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
879 880
	if (kvm_x86_ops->get_cpl(vcpu) != 0 ||
	    __kvm_set_xcr(vcpu, index, xcr)) {
881 882 883 884 885 886 887
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_xcr);

888
static int kvm_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
889
{
890
	if (cr4 & CR4_RESERVED_BITS)
891
		return -EINVAL;
892

893
	if (!guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && (cr4 & X86_CR4_OSXSAVE))
894
		return -EINVAL;
895

896
	if (!guest_cpuid_has(vcpu, X86_FEATURE_SMEP) && (cr4 & X86_CR4_SMEP))
897
		return -EINVAL;
898

899
	if (!guest_cpuid_has(vcpu, X86_FEATURE_SMAP) && (cr4 & X86_CR4_SMAP))
900
		return -EINVAL;
901

902
	if (!guest_cpuid_has(vcpu, X86_FEATURE_FSGSBASE) && (cr4 & X86_CR4_FSGSBASE))
903
		return -EINVAL;
F
Feng Wu 已提交
904

905
	if (!guest_cpuid_has(vcpu, X86_FEATURE_PKU) && (cr4 & X86_CR4_PKE))
906
		return -EINVAL;
907

908
	if (!guest_cpuid_has(vcpu, X86_FEATURE_LA57) && (cr4 & X86_CR4_LA57))
909
		return -EINVAL;
910

P
Paolo Bonzini 已提交
911
	if (!guest_cpuid_has(vcpu, X86_FEATURE_UMIP) && (cr4 & X86_CR4_UMIP))
912 913 914 915 916 917 918 919 920 921 922 923
		return -EINVAL;

	return 0;
}

int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
	unsigned long old_cr4 = kvm_read_cr4(vcpu);
	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE |
				   X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE;

	if (kvm_valid_cr4(vcpu, cr4))
P
Paolo Bonzini 已提交
924 925
		return 1;

926
	if (is_long_mode(vcpu)) {
927 928
		if (!(cr4 & X86_CR4_PAE))
			return 1;
929 930
	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
		   && ((cr4 ^ old_cr4) & pdptr_bits)
931 932
		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
				   kvm_read_cr3(vcpu)))
933 934
		return 1;

935
	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
936
		if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
937 938 939 940 941 942 943
			return 1;

		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
			return 1;
	}

944
	if (kvm_x86_ops->set_cr4(vcpu, cr4))
945
		return 1;
946

947 948
	if (((cr4 ^ old_cr4) & pdptr_bits) ||
	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
949
		kvm_mmu_reset_context(vcpu);
950

951
	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
A
Avi Kivity 已提交
952
		kvm_update_cpuid(vcpu);
953

954 955
	return 0;
}
956
EXPORT_SYMBOL_GPL(kvm_set_cr4);
957

958
int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
959
{
960
	bool skip_tlb_flush = false;
961
#ifdef CONFIG_X86_64
962 963
	bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);

964
	if (pcid_enabled) {
965 966
		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
		cr3 &= ~X86_CR3_PCID_NOFLUSH;
967
	}
968
#endif
N
Nadav Amit 已提交
969

970
	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
971 972
		if (!skip_tlb_flush) {
			kvm_mmu_sync_roots(vcpu);
973
			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
974
		}
975
		return 0;
976 977
	}

978
	if (is_long_mode(vcpu) &&
979
	    (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63)))
980
		return 1;
981 982
	else if (is_pae_paging(vcpu) &&
		 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
N
Nadav Amit 已提交
983
		return 1;
984

985
	kvm_mmu_new_cr3(vcpu, cr3, skip_tlb_flush);
986
	vcpu->arch.cr3 = cr3;
987
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
988

989 990
	return 0;
}
991
EXPORT_SYMBOL_GPL(kvm_set_cr3);
992

A
Andre Przywara 已提交
993
int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
994
{
995 996
	if (cr8 & CR8_RESERVED_BITS)
		return 1;
997
	if (lapic_in_kernel(vcpu))
998 999
		kvm_lapic_set_tpr(vcpu, cr8);
	else
1000
		vcpu->arch.cr8 = cr8;
1001 1002
	return 0;
}
1003
EXPORT_SYMBOL_GPL(kvm_set_cr8);
1004

1005
unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1006
{
1007
	if (lapic_in_kernel(vcpu))
1008 1009
		return kvm_lapic_get_cr8(vcpu);
	else
1010
		return vcpu->arch.cr8;
1011
}
1012
EXPORT_SYMBOL_GPL(kvm_get_cr8);
1013

1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
{
	int i;

	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
	}
}

J
Jan Kiszka 已提交
1025 1026 1027 1028 1029 1030
static void kvm_update_dr6(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
		kvm_x86_ops->set_dr6(vcpu, vcpu->arch.dr6);
}

1031 1032 1033 1034 1035 1036 1037 1038 1039
static void kvm_update_dr7(struct kvm_vcpu *vcpu)
{
	unsigned long dr7;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		dr7 = vcpu->arch.guest_debug_dr7;
	else
		dr7 = vcpu->arch.dr7;
	kvm_x86_ops->set_dr7(vcpu, dr7);
1040 1041 1042
	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
	if (dr7 & DR7_BP_EN_MASK)
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1043 1044
}

1045 1046 1047 1048
static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
{
	u64 fixed = DR6_FIXED_1;

1049
	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1050 1051 1052 1053
		fixed |= DR6_RTM;
	return fixed;
}

1054
static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
{
	switch (dr) {
	case 0 ... 3:
		vcpu->arch.db[dr] = val;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = val;
		break;
	case 4:
		/* fall through */
	case 6:
1065 1066
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
1067
		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
J
Jan Kiszka 已提交
1068
		kvm_update_dr6(vcpu);
1069 1070 1071 1072
		break;
	case 5:
		/* fall through */
	default: /* 7 */
1073 1074
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
1075
		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1076
		kvm_update_dr7(vcpu);
1077 1078 1079 1080 1081
		break;
	}

	return 0;
}
1082 1083 1084

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
1085
	if (__kvm_set_dr(vcpu, dr, val)) {
1086
		kvm_inject_gp(vcpu, 0);
1087 1088 1089
		return 1;
	}
	return 0;
1090
}
1091 1092
EXPORT_SYMBOL_GPL(kvm_set_dr);

1093
int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1094 1095 1096 1097 1098 1099 1100 1101
{
	switch (dr) {
	case 0 ... 3:
		*val = vcpu->arch.db[dr];
		break;
	case 4:
		/* fall through */
	case 6:
J
Jan Kiszka 已提交
1102 1103 1104 1105
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
			*val = vcpu->arch.dr6;
		else
			*val = kvm_x86_ops->get_dr6(vcpu);
1106 1107 1108 1109 1110 1111 1112
		break;
	case 5:
		/* fall through */
	default: /* 7 */
		*val = vcpu->arch.dr7;
		break;
	}
1113 1114
	return 0;
}
1115 1116
EXPORT_SYMBOL_GPL(kvm_get_dr);

A
Avi Kivity 已提交
1117 1118
bool kvm_rdpmc(struct kvm_vcpu *vcpu)
{
1119
	u32 ecx = kvm_rcx_read(vcpu);
A
Avi Kivity 已提交
1120 1121 1122
	u64 data;
	int err;

1123
	err = kvm_pmu_rdpmc(vcpu, ecx, &data);
A
Avi Kivity 已提交
1124 1125
	if (err)
		return err;
1126 1127
	kvm_rax_write(vcpu, (u32)data);
	kvm_rdx_write(vcpu, data >> 32);
A
Avi Kivity 已提交
1128 1129 1130 1131
	return err;
}
EXPORT_SYMBOL_GPL(kvm_rdpmc);

1132 1133 1134 1135 1136
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
 * This list is modified at module load time to reflect the
1137
 * capabilities of the host cpu. This capabilities test skips MSRs that are
1138 1139
 * kvm-specific. Those are put in emulated_msrs; filtering of emulated_msrs
 * may depend on host virtualization features rather than host cpu features.
1140
 */
1141

1142 1143
static u32 msrs_to_save[] = {
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
B
Brian Gerst 已提交
1144
	MSR_STAR,
1145 1146 1147
#ifdef CONFIG_X86_64
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
1148
	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
1149
	MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
1150
	MSR_IA32_SPEC_CTRL,
1151 1152 1153 1154 1155 1156
	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
1157 1158
	MSR_IA32_UMWAIT_CONTROL,

1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
	MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
	MSR_ARCH_PERFMON_FIXED_CTR0 + 2, MSR_ARCH_PERFMON_FIXED_CTR0 + 3,
	MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
	MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
	MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
	MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
	MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
	MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
	MSR_ARCH_PERFMON_PERFCTR0 + 8, MSR_ARCH_PERFMON_PERFCTR0 + 9,
	MSR_ARCH_PERFMON_PERFCTR0 + 10, MSR_ARCH_PERFMON_PERFCTR0 + 11,
	MSR_ARCH_PERFMON_PERFCTR0 + 12, MSR_ARCH_PERFMON_PERFCTR0 + 13,
	MSR_ARCH_PERFMON_PERFCTR0 + 14, MSR_ARCH_PERFMON_PERFCTR0 + 15,
	MSR_ARCH_PERFMON_PERFCTR0 + 16, MSR_ARCH_PERFMON_PERFCTR0 + 17,
	MSR_ARCH_PERFMON_PERFCTR0 + 18, MSR_ARCH_PERFMON_PERFCTR0 + 19,
	MSR_ARCH_PERFMON_PERFCTR0 + 20, MSR_ARCH_PERFMON_PERFCTR0 + 21,
	MSR_ARCH_PERFMON_PERFCTR0 + 22, MSR_ARCH_PERFMON_PERFCTR0 + 23,
	MSR_ARCH_PERFMON_PERFCTR0 + 24, MSR_ARCH_PERFMON_PERFCTR0 + 25,
	MSR_ARCH_PERFMON_PERFCTR0 + 26, MSR_ARCH_PERFMON_PERFCTR0 + 27,
	MSR_ARCH_PERFMON_PERFCTR0 + 28, MSR_ARCH_PERFMON_PERFCTR0 + 29,
	MSR_ARCH_PERFMON_PERFCTR0 + 30, MSR_ARCH_PERFMON_PERFCTR0 + 31,
	MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
	MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
	MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
	MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
	MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
	MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
	MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
	MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
	MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
	MSR_ARCH_PERFMON_EVENTSEL0 + 18, MSR_ARCH_PERFMON_EVENTSEL0 + 19,
	MSR_ARCH_PERFMON_EVENTSEL0 + 20, MSR_ARCH_PERFMON_EVENTSEL0 + 21,
	MSR_ARCH_PERFMON_EVENTSEL0 + 22, MSR_ARCH_PERFMON_EVENTSEL0 + 23,
	MSR_ARCH_PERFMON_EVENTSEL0 + 24, MSR_ARCH_PERFMON_EVENTSEL0 + 25,
	MSR_ARCH_PERFMON_EVENTSEL0 + 26, MSR_ARCH_PERFMON_EVENTSEL0 + 27,
	MSR_ARCH_PERFMON_EVENTSEL0 + 28, MSR_ARCH_PERFMON_EVENTSEL0 + 29,
	MSR_ARCH_PERFMON_EVENTSEL0 + 30, MSR_ARCH_PERFMON_EVENTSEL0 + 31,
1195 1196 1197 1198
};

static unsigned num_msrs_to_save;

1199 1200 1201 1202 1203
static u32 emulated_msrs[] = {
	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
1204
	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
1205 1206
	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
1207
	HV_X64_MSR_RESET,
1208
	HV_X64_MSR_VP_INDEX,
1209
	HV_X64_MSR_VP_RUNTIME,
1210
	HV_X64_MSR_SCONTROL,
A
Andrey Smetanin 已提交
1211
	HV_X64_MSR_STIMER0_CONFIG,
1212
	HV_X64_MSR_VP_ASSIST_PAGE,
1213 1214 1215 1216
	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
	HV_X64_MSR_TSC_EMULATION_STATUS,

	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
1217 1218
	MSR_KVM_PV_EOI_EN,

W
Will Auld 已提交
1219
	MSR_IA32_TSC_ADJUST,
1220
	MSR_IA32_TSCDEADLINE,
1221
	MSR_IA32_ARCH_CAPABILITIES,
1222
	MSR_IA32_MISC_ENABLE,
1223 1224
	MSR_IA32_MCG_STATUS,
	MSR_IA32_MCG_CTL,
1225
	MSR_IA32_MCG_EXT_CTL,
P
Paolo Bonzini 已提交
1226
	MSR_IA32_SMBASE,
1227
	MSR_SMI_COUNT,
K
Kyle Huey 已提交
1228 1229
	MSR_PLATFORM_INFO,
	MSR_MISC_FEATURES_ENABLES,
1230
	MSR_AMD64_VIRT_SPEC_CTRL,
1231
	MSR_IA32_POWER_CTL,
1232

1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252
	/*
	 * The following list leaves out MSRs whose values are determined
	 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
	 * We always support the "true" VMX control MSRs, even if the host
	 * processor does not, so I am putting these registers here rather
	 * than in msrs_to_save.
	 */
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1253
	MSR_K7_HWCR,
1254
	MSR_KVM_POLL_CONTROL,
1255 1256
};

1257 1258
static unsigned num_emulated_msrs;

1259 1260 1261 1262 1263
/*
 * List of msr numbers which are used to expose MSR-based features that
 * can be used by a hypervisor to validate requested CPU features.
 */
static u32 msr_based_features[] = {
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR0_FIXED1,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_CR4_FIXED1,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1283
	MSR_F10H_DECFG,
1284
	MSR_IA32_UCODE_REV,
1285
	MSR_IA32_ARCH_CAPABILITIES,
1286 1287 1288 1289
};

static unsigned int num_msr_based_features;

1290
static u64 kvm_get_arch_capabilities(void)
1291
{
1292
	u64 data = 0;
1293

1294 1295
	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308

	/*
	 * If we're doing cache flushes (either "always" or "cond")
	 * we will do one whenever the guest does a vmlaunch/vmresume.
	 * If an outer hypervisor is doing the cache flush for us
	 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
	 * capability to the guest too, and if EPT is disabled we're not
	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
	 * require a nested hypervisor to do a flush of its own.
	 */
	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;

1309 1310 1311 1312 1313 1314 1315
	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
		data |= ARCH_CAP_RDCL_NO;
	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
		data |= ARCH_CAP_SSB_NO;
	if (!boot_cpu_has_bug(X86_BUG_MDS))
		data |= ARCH_CAP_MDS_NO;

1316 1317 1318
	return data;
}

1319 1320 1321
static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
{
	switch (msr->index) {
1322
	case MSR_IA32_ARCH_CAPABILITIES:
1323 1324 1325
		msr->data = kvm_get_arch_capabilities();
		break;
	case MSR_IA32_UCODE_REV:
1326
		rdmsrl_safe(msr->index, &msr->data);
1327
		break;
1328 1329 1330 1331 1332 1333 1334
	default:
		if (kvm_x86_ops->get_msr_feature(msr))
			return 1;
	}
	return 0;
}

1335 1336 1337
static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	struct kvm_msr_entry msr;
1338
	int r;
1339 1340

	msr.index = index;
1341 1342 1343
	r = kvm_get_msr_feature(&msr);
	if (r)
		return r;
1344 1345 1346 1347 1348 1349

	*data = msr.data;

	return 0;
}

1350
static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1351
{
1352
	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1353
		return false;
A
Alexander Graf 已提交
1354

1355
	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1356
		return false;
1357

1358 1359 1360 1361 1362 1363
	if (efer & (EFER_LME | EFER_LMA) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))
		return false;

	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
		return false;
1364

1365
	return true;
1366 1367 1368 1369 1370 1371 1372 1373

}
bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	if (efer & efer_reserved_bits)
		return false;

	return __kvm_valid_efer(vcpu, efer);
1374 1375 1376
}
EXPORT_SYMBOL_GPL(kvm_valid_efer);

1377
static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1378 1379
{
	u64 old_efer = vcpu->arch.efer;
1380
	u64 efer = msr_info->data;
1381

1382
	if (efer & efer_reserved_bits)
1383
		return 1;
1384

1385 1386 1387 1388 1389 1390 1391 1392
	if (!msr_info->host_initiated) {
		if (!__kvm_valid_efer(vcpu, efer))
			return 1;

		if (is_paging(vcpu) &&
		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
			return 1;
	}
1393

1394
	efer &= ~EFER_LMA;
1395
	efer |= vcpu->arch.efer & EFER_LMA;
1396

1397 1398
	kvm_x86_ops->set_efer(vcpu, efer);

1399 1400 1401 1402
	/* Update reserved bits */
	if ((efer ^ old_efer) & EFER_NX)
		kvm_mmu_reset_context(vcpu);

1403
	return 0;
1404 1405
}

1406 1407 1408 1409 1410 1411
void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);

1412
/*
1413 1414
 * Write @data into the MSR specified by @index.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
1415 1416 1417
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1418 1419
static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
			 bool host_initiated)
1420
{
1421 1422 1423
	struct msr_data msr;

	switch (index) {
1424 1425 1426 1427 1428
	case MSR_FS_BASE:
	case MSR_GS_BASE:
	case MSR_KERNEL_GS_BASE:
	case MSR_CSTAR:
	case MSR_LSTAR:
1429
		if (is_noncanonical_address(data, vcpu))
1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
			return 1;
		break;
	case MSR_IA32_SYSENTER_EIP:
	case MSR_IA32_SYSENTER_ESP:
		/*
		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
		 * non-canonical address is written on Intel but not on
		 * AMD (which ignores the top 32-bits, because it does
		 * not implement 64-bit SYSENTER).
		 *
		 * 64-bit code should hence be able to write a non-canonical
		 * value on AMD.  Making the address canonical ensures that
		 * vmentry does not fail on Intel after writing a non-canonical
		 * value, and that something deterministic happens if the guest
		 * invokes 64-bit SYSENTER.
		 */
1446
		data = get_canonical(data, vcpu_virt_addr_bits(vcpu));
1447
	}
1448 1449 1450 1451 1452 1453

	msr.data = data;
	msr.index = index;
	msr.host_initiated = host_initiated;

	return kvm_x86_ops->set_msr(vcpu, &msr);
1454 1455
}

1456
/*
1457 1458 1459 1460
 * Read the MSR specified by @index into @data.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
1461
 */
1462 1463
static int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
			 bool host_initiated)
1464 1465
{
	struct msr_data msr;
1466
	int ret;
1467 1468

	msr.index = index;
1469
	msr.host_initiated = host_initiated;
1470

1471 1472 1473 1474
	ret = kvm_x86_ops->get_msr(vcpu, &msr);
	if (!ret)
		*data = msr.data;
	return ret;
1475 1476
}

1477
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1478
{
1479 1480 1481
	return __kvm_get_msr(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_get_msr);
1482

1483 1484 1485 1486 1487 1488
int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
{
	return __kvm_set_msr(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_set_msr);

1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data;

	if (kvm_get_msr(vcpu, ecx, &data)) {
		trace_kvm_msr_read_ex(ecx);
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

	trace_kvm_msr_read(ecx, data);

	kvm_rax_write(vcpu, data & -1u);
	kvm_rdx_write(vcpu, (data >> 32) & -1u);
	return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);

int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data = kvm_read_edx_eax(vcpu);

	if (kvm_set_msr(vcpu, ecx, data)) {
		trace_kvm_msr_write_ex(ecx, data);
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

	trace_kvm_msr_write(ecx, data);
	return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);

1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534
/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return __kvm_get_msr(vcpu, index, data, true);
}

static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return __kvm_set_msr(vcpu, index, *data, true);
1535 1536
}

1537 1538 1539 1540 1541 1542
#ifdef CONFIG_X86_64
struct pvclock_gtod_data {
	seqcount_t	seq;

	struct { /* extract of a clocksource struct */
		int vclock_mode;
1543 1544
		u64	cycle_last;
		u64	mask;
1545 1546 1547 1548
		u32	mult;
		u32	shift;
	} clock;

1549 1550
	u64		boot_ns;
	u64		nsec_base;
1551
	u64		wall_time_sec;
1552 1553 1554 1555 1556 1557 1558
};

static struct pvclock_gtod_data pvclock_gtod_data;

static void update_pvclock_gtod(struct timekeeper *tk)
{
	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
1559 1560
	u64 boot_ns;

1561
	boot_ns = ktime_to_ns(ktime_add(tk->tkr_mono.base, tk->offs_boot));
1562 1563 1564 1565

	write_seqcount_begin(&vdata->seq);

	/* copy pvclock gtod data */
1566 1567 1568 1569 1570
	vdata->clock.vclock_mode	= tk->tkr_mono.clock->archdata.vclock_mode;
	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
	vdata->clock.mask		= tk->tkr_mono.mask;
	vdata->clock.mult		= tk->tkr_mono.mult;
	vdata->clock.shift		= tk->tkr_mono.shift;
1571

1572
	vdata->boot_ns			= boot_ns;
1573
	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
1574

1575 1576
	vdata->wall_time_sec            = tk->xtime_sec;

1577 1578 1579 1580
	write_seqcount_end(&vdata->seq);
}
#endif

1581 1582 1583
void kvm_set_pending_timer(struct kvm_vcpu *vcpu)
{
	kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
1584
	kvm_vcpu_kick(vcpu);
1585
}
1586

1587 1588
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
{
1589 1590
	int version;
	int r;
1591
	struct pvclock_wall_clock wc;
A
Arnd Bergmann 已提交
1592
	struct timespec64 boot;
1593 1594 1595 1596

	if (!wall_clock)
		return;

1597 1598 1599 1600 1601 1602 1603 1604
	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
	if (r)
		return;

	if (version & 1)
		++version;  /* first time write, random junk */

	++version;
1605

1606 1607
	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
		return;
1608

1609 1610
	/*
	 * The guest calculates current wall clock time by adding
Z
Zachary Amsden 已提交
1611
	 * system time (updated by kvm_guest_time_update below) to the
1612 1613 1614
	 * wall clock specified here.  guest system time equals host
	 * system time for us, thus we must fill in host boot time here.
	 */
A
Arnd Bergmann 已提交
1615
	getboottime64(&boot);
1616

1617
	if (kvm->arch.kvmclock_offset) {
A
Arnd Bergmann 已提交
1618 1619
		struct timespec64 ts = ns_to_timespec64(kvm->arch.kvmclock_offset);
		boot = timespec64_sub(boot, ts);
1620
	}
A
Arnd Bergmann 已提交
1621
	wc.sec = (u32)boot.tv_sec; /* overflow in 2106 guest time */
1622 1623
	wc.nsec = boot.tv_nsec;
	wc.version = version;
1624 1625 1626 1627 1628 1629 1630

	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));

	version++;
	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
}

1631 1632
static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
1633 1634
	do_shl32_div32(dividend, divisor);
	return dividend;
1635 1636
}

1637
static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
1638
			       s8 *pshift, u32 *pmultiplier)
1639
{
1640
	uint64_t scaled64;
1641 1642 1643 1644
	int32_t  shift = 0;
	uint64_t tps64;
	uint32_t tps32;

1645 1646
	tps64 = base_hz;
	scaled64 = scaled_hz;
1647
	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
1648 1649 1650 1651 1652
		tps64 >>= 1;
		shift--;
	}

	tps32 = (uint32_t)tps64;
1653 1654
	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
1655 1656 1657
			scaled64 >>= 1;
		else
			tps32 <<= 1;
1658 1659 1660
		shift++;
	}

1661 1662
	*pshift = shift;
	*pmultiplier = div_frac(scaled64, tps32);
1663 1664
}

1665
#ifdef CONFIG_X86_64
1666
static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
1667
#endif
1668

1669
static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
1670
static unsigned long max_tsc_khz;
1671

1672
static u32 adjust_tsc_khz(u32 khz, s32 ppm)
1673
{
1674 1675 1676
	u64 v = (u64)khz * (1000000 + ppm);
	do_div(v, 1000000);
	return v;
1677 1678
}

1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
{
	u64 ratio;

	/* Guest TSC same frequency as host TSC? */
	if (!scale) {
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
		return 0;
	}

	/* TSC scaling supported? */
	if (!kvm_has_tsc_control) {
		if (user_tsc_khz > tsc_khz) {
			vcpu->arch.tsc_catchup = 1;
			vcpu->arch.tsc_always_catchup = 1;
			return 0;
		} else {
1696
			pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
1697 1698 1699 1700 1701 1702 1703 1704 1705
			return -1;
		}
	}

	/* TSC scaling required  - calculate ratio */
	ratio = mul_u64_u32_div(1ULL << kvm_tsc_scaling_ratio_frac_bits,
				user_tsc_khz, tsc_khz);

	if (ratio == 0 || ratio >= kvm_max_tsc_scaling_ratio) {
1706 1707
		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
			            user_tsc_khz);
1708 1709 1710 1711 1712 1713 1714
		return -1;
	}

	vcpu->arch.tsc_scaling_ratio = ratio;
	return 0;
}

1715
static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
1716
{
1717 1718
	u32 thresh_lo, thresh_hi;
	int use_scaling = 0;
1719

1720
	/* tsc_khz can be zero if TSC calibration fails */
1721
	if (user_tsc_khz == 0) {
1722 1723
		/* set tsc_scaling_ratio to a safe value */
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
1724
		return -1;
1725
	}
1726

Z
Zachary Amsden 已提交
1727
	/* Compute a scale to convert nanoseconds in TSC cycles */
1728
	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
1729 1730
			   &vcpu->arch.virtual_tsc_shift,
			   &vcpu->arch.virtual_tsc_mult);
1731
	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
1732 1733 1734 1735 1736 1737 1738 1739 1740

	/*
	 * Compute the variation in TSC rate which is acceptable
	 * within the range of tolerance and decide if the
	 * rate being applied is within that bounds of the hardware
	 * rate.  If so, no scaling or compensation need be done.
	 */
	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
1741 1742
	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", user_tsc_khz, thresh_lo, thresh_hi);
1743 1744
		use_scaling = 1;
	}
1745
	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
Z
Zachary Amsden 已提交
1746 1747 1748 1749
}

static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
1750
	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
1751 1752
				      vcpu->arch.virtual_tsc_mult,
				      vcpu->arch.virtual_tsc_shift);
1753
	tsc += vcpu->arch.this_tsc_write;
Z
Zachary Amsden 已提交
1754 1755 1756
	return tsc;
}

1757 1758 1759 1760 1761
static inline int gtod_is_based_on_tsc(int mode)
{
	return mode == VCLOCK_TSC || mode == VCLOCK_HVCLOCK;
}

1762
static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
1763 1764 1765 1766 1767 1768 1769 1770 1771
{
#ifdef CONFIG_X86_64
	bool vcpus_matched;
	struct kvm_arch *ka = &vcpu->kvm->arch;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			 atomic_read(&vcpu->kvm->online_vcpus));

1772 1773 1774 1775 1776 1777 1778 1779 1780
	/*
	 * Once the masterclock is enabled, always perform request in
	 * order to update it.
	 *
	 * In order to enable masterclock, the host clocksource must be TSC
	 * and the vcpus need to have matched TSCs.  When that happens,
	 * perform request to enable masterclock.
	 */
	if (ka->use_master_clock ||
1781
	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
1782 1783 1784 1785 1786 1787 1788 1789
		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);

	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
			    atomic_read(&vcpu->kvm->online_vcpus),
		            ka->use_master_clock, gtod->clock.vclock_mode);
#endif
}

W
Will Auld 已提交
1790 1791
static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
{
1792
	u64 curr_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
W
Will Auld 已提交
1793 1794 1795
	vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
}

1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822
/*
 * Multiply tsc by a fixed point number represented by ratio.
 *
 * The most significant 64-N bits (mult) of ratio represent the
 * integral part of the fixed point number; the remaining N bits
 * (frac) represent the fractional part, ie. ratio represents a fixed
 * point number (mult + frac * 2^(-N)).
 *
 * N equals to kvm_tsc_scaling_ratio_frac_bits.
 */
static inline u64 __scale_tsc(u64 ratio, u64 tsc)
{
	return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits);
}

u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc)
{
	u64 _tsc = tsc;
	u64 ratio = vcpu->arch.tsc_scaling_ratio;

	if (ratio != kvm_default_tsc_scaling_ratio)
		_tsc = __scale_tsc(ratio, tsc);

	return _tsc;
}
EXPORT_SYMBOL_GPL(kvm_scale_tsc);

1823 1824 1825 1826 1827 1828 1829 1830 1831
static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
{
	u64 tsc;

	tsc = kvm_scale_tsc(vcpu, rdtsc());

	return target_tsc - tsc;
}

1832 1833
u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
{
1834 1835 1836
	u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);

	return tsc_offset + kvm_scale_tsc(vcpu, host_tsc);
1837 1838 1839
}
EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);

1840 1841
static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
{
1842
	vcpu->arch.tsc_offset = kvm_x86_ops->write_l1_tsc_offset(vcpu, offset);
1843 1844
}

1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857
static inline bool kvm_check_tsc_unstable(void)
{
#ifdef CONFIG_X86_64
	/*
	 * TSC is marked unstable when we're running on Hyper-V,
	 * 'TSC page' clocksource is good.
	 */
	if (pvclock_gtod_data.clock.vclock_mode == VCLOCK_HVCLOCK)
		return false;
#endif
	return check_tsc_unstable();
}

1858
void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
1859 1860
{
	struct kvm *kvm = vcpu->kvm;
Z
Zachary Amsden 已提交
1861
	u64 offset, ns, elapsed;
1862
	unsigned long flags;
1863
	bool matched;
T
Tomasz Grabiec 已提交
1864
	bool already_matched;
1865
	u64 data = msr->data;
1866
	bool synchronizing = false;
1867

1868
	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
1869
	offset = kvm_compute_tsc_offset(vcpu, data);
1870
	ns = ktime_get_boottime_ns();
Z
Zachary Amsden 已提交
1871
	elapsed = ns - kvm->arch.last_tsc_nsec;
1872

1873
	if (vcpu->arch.virtual_tsc_khz) {
1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892
		if (data == 0 && msr->host_initiated) {
			/*
			 * detection of vcpu initialization -- need to sync
			 * with other vCPUs. This particularly helps to keep
			 * kvm_clock stable after CPU hotplug
			 */
			synchronizing = true;
		} else {
			u64 tsc_exp = kvm->arch.last_tsc_write +
						nsec_to_cycles(vcpu, elapsed);
			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
			/*
			 * Special case: TSC write with a small delta (1 second)
			 * of virtual cycle time against real time is
			 * interpreted as an attempt to synchronize the CPU.
			 */
			synchronizing = data < tsc_exp + tsc_hz &&
					data + tsc_hz > tsc_exp;
		}
1893
	}
Z
Zachary Amsden 已提交
1894 1895

	/*
1896 1897 1898 1899 1900
	 * For a reliable TSC, we can match TSC offsets, and for an unstable
	 * TSC, we add elapsed time in this computation.  We could let the
	 * compensation code attempt to catch up if we fall behind, but
	 * it's better to try to match offsets from the beginning.
         */
1901
	if (synchronizing &&
1902
	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
1903
		if (!kvm_check_tsc_unstable()) {
1904
			offset = kvm->arch.cur_tsc_offset;
Z
Zachary Amsden 已提交
1905
		} else {
1906
			u64 delta = nsec_to_cycles(vcpu, elapsed);
1907
			data += delta;
1908
			offset = kvm_compute_tsc_offset(vcpu, data);
Z
Zachary Amsden 已提交
1909
		}
1910
		matched = true;
T
Tomasz Grabiec 已提交
1911
		already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
1912 1913 1914 1915 1916 1917
	} else {
		/*
		 * We split periods of matched TSC writes into generations.
		 * For each generation, we track the original measured
		 * nanosecond time, offset, and write, so if TSCs are in
		 * sync, we can match exact offset, and if not, we can match
G
Guo Chao 已提交
1918
		 * exact software computation in compute_guest_tsc()
1919 1920 1921 1922 1923 1924 1925
		 *
		 * These values are tracked in kvm->arch.cur_xxx variables.
		 */
		kvm->arch.cur_tsc_generation++;
		kvm->arch.cur_tsc_nsec = ns;
		kvm->arch.cur_tsc_write = data;
		kvm->arch.cur_tsc_offset = offset;
1926
		matched = false;
Z
Zachary Amsden 已提交
1927
	}
1928 1929 1930 1931 1932

	/*
	 * We also track th most recent recorded KHZ, write and time to
	 * allow the matching interval to be extended at each write.
	 */
Z
Zachary Amsden 已提交
1933 1934
	kvm->arch.last_tsc_nsec = ns;
	kvm->arch.last_tsc_write = data;
1935
	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
1936

1937
	vcpu->arch.last_guest_tsc = data;
1938 1939 1940 1941 1942 1943

	/* Keep track of which generation this VCPU has synchronized to */
	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;

1944
	if (!msr->host_initiated && guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST))
W
Will Auld 已提交
1945
		update_ia32_tsc_adjust_msr(vcpu, offset);
1946

1947
	kvm_vcpu_write_tsc_offset(vcpu, offset);
1948
	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
1949 1950

	spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
T
Tomasz Grabiec 已提交
1951
	if (!matched) {
1952
		kvm->arch.nr_vcpus_matched_tsc = 0;
T
Tomasz Grabiec 已提交
1953 1954 1955
	} else if (!already_matched) {
		kvm->arch.nr_vcpus_matched_tsc++;
	}
1956 1957 1958

	kvm_track_tsc_matching(vcpu);
	spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
1959
}
1960

1961 1962
EXPORT_SYMBOL_GPL(kvm_write_tsc);

1963 1964 1965
static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
					   s64 adjustment)
{
1966 1967
	u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
1968 1969 1970 1971 1972 1973 1974
}

static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
{
	if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
		WARN_ON(adjustment < 0);
	adjustment = kvm_scale_tsc(vcpu, (u64) adjustment);
1975
	adjust_tsc_offset_guest(vcpu, adjustment);
1976 1977
}

1978 1979
#ifdef CONFIG_X86_64

1980
static u64 read_tsc(void)
1981
{
1982
	u64 ret = (u64)rdtsc_ordered();
1983
	u64 last = pvclock_gtod_data.clock.cycle_last;
1984 1985 1986 1987 1988 1989

	if (likely(ret >= last))
		return ret;

	/*
	 * GCC likes to generate cmov here, but this branch is extremely
1990
	 * predictable (it's just a function of time and the likely is
1991 1992 1993 1994 1995 1996 1997 1998 1999
	 * very likely) and there's a data dependence, so force GCC
	 * to generate a branch instead.  I don't barrier() because
	 * we don't actually need a barrier, and if this function
	 * ever gets inlined it will generate worse code.
	 */
	asm volatile ("");
	return last;
}

2000
static inline u64 vgettsc(u64 *tsc_timestamp, int *mode)
2001 2002 2003
{
	long v;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028
	u64 tsc_pg_val;

	switch (gtod->clock.vclock_mode) {
	case VCLOCK_HVCLOCK:
		tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
						  tsc_timestamp);
		if (tsc_pg_val != U64_MAX) {
			/* TSC page valid */
			*mode = VCLOCK_HVCLOCK;
			v = (tsc_pg_val - gtod->clock.cycle_last) &
				gtod->clock.mask;
		} else {
			/* TSC page invalid */
			*mode = VCLOCK_NONE;
		}
		break;
	case VCLOCK_TSC:
		*mode = VCLOCK_TSC;
		*tsc_timestamp = read_tsc();
		v = (*tsc_timestamp - gtod->clock.cycle_last) &
			gtod->clock.mask;
		break;
	default:
		*mode = VCLOCK_NONE;
	}
2029

2030 2031
	if (*mode == VCLOCK_NONE)
		*tsc_timestamp = v = 0;
2032 2033 2034 2035

	return v * gtod->clock.mult;
}

2036
static int do_monotonic_boot(s64 *t, u64 *tsc_timestamp)
2037
{
2038
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2039 2040
	unsigned long seq;
	int mode;
2041
	u64 ns;
2042 2043 2044

	do {
		seq = read_seqcount_begin(&gtod->seq);
2045
		ns = gtod->nsec_base;
2046
		ns += vgettsc(tsc_timestamp, &mode);
2047
		ns >>= gtod->clock.shift;
2048
		ns += gtod->boot_ns;
2049
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2050
	*t = ns;
2051 2052 2053 2054

	return mode;
}

2055
static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	unsigned long seq;
	int mode;
	u64 ns;

	do {
		seq = read_seqcount_begin(&gtod->seq);
		ts->tv_sec = gtod->wall_time_sec;
		ns = gtod->nsec_base;
2066
		ns += vgettsc(tsc_timestamp, &mode);
2067 2068 2069 2070 2071 2072 2073 2074 2075
		ns >>= gtod->clock.shift;
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));

	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return mode;
}

2076 2077
/* returns true if host is using TSC based clocksource */
static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
2078 2079
{
	/* checked again under seqlock below */
2080
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2081 2082
		return false;

2083 2084
	return gtod_is_based_on_tsc(do_monotonic_boot(kernel_ns,
						      tsc_timestamp));
2085
}
2086

2087
/* returns true if host is using TSC based clocksource */
2088
static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
2089
					   u64 *tsc_timestamp)
2090 2091
{
	/* checked again under seqlock below */
2092
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2093 2094
		return false;

2095
	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
2096
}
2097 2098 2099 2100
#endif

/*
 *
2101 2102 2103
 * Assuming a stable TSC across physical CPUS, and a stable TSC
 * across virtual CPUs, the following condition is possible.
 * Each numbered line represents an event visible to both
2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135
 * CPUs at the next numbered event.
 *
 * "timespecX" represents host monotonic time. "tscX" represents
 * RDTSC value.
 *
 * 		VCPU0 on CPU0		|	VCPU1 on CPU1
 *
 * 1.  read timespec0,tsc0
 * 2.					| timespec1 = timespec0 + N
 * 					| tsc1 = tsc0 + M
 * 3. transition to guest		| transition to guest
 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
 * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
 * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
 *
 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
 *
 * 	- ret0 < ret1
 *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
 *		...
 *	- 0 < N - M => M < N
 *
 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
 * always the case (the difference between two distinct xtime instances
 * might be smaller then the difference between corresponding TSC reads,
 * when updating guest vcpus pvclock areas).
 *
 * To avoid that problem, do not allow visibility of distinct
 * system_timestamp/tsc_timestamp values simultaneously: use a master
 * copy of host monotonic time values. Update that master copy
 * in lockstep.
 *
2136
 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
2137 2138 2139 2140 2141 2142 2143 2144
 *
 */

static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	struct kvm_arch *ka = &kvm->arch;
	int vclock_mode;
2145 2146 2147 2148
	bool host_tsc_clocksource, vcpus_matched;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			atomic_read(&kvm->online_vcpus));
2149 2150 2151 2152 2153

	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
2154
	host_tsc_clocksource = kvm_get_time_and_clockread(
2155 2156 2157
					&ka->master_kernel_ns,
					&ka->master_cycle_now);

2158
	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
2159
				&& !ka->backwards_tsc_observed
2160
				&& !ka->boot_vcpu_runs_old_kvmclock;
2161

2162 2163 2164 2165
	if (ka->use_master_clock)
		atomic_set(&kvm_guest_has_master_clock, 1);

	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
2166 2167
	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
					vcpus_matched);
2168 2169 2170
#endif
}

2171 2172 2173 2174 2175
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
}

2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188
static void kvm_gen_update_masterclock(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	int i;
	struct kvm_vcpu *vcpu;
	struct kvm_arch *ka = &kvm->arch;

	spin_lock(&ka->pvclock_gtod_sync_lock);
	kvm_make_mclock_inprogress_request(kvm);
	/* no guest entries from this point */
	pvclock_update_vm_gtod_copy(kvm);

	kvm_for_each_vcpu(i, vcpu, kvm)
2189
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2190 2191 2192

	/* guest entries allowed */
	kvm_for_each_vcpu(i, vcpu, kvm)
2193
		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
2194 2195 2196 2197 2198

	spin_unlock(&ka->pvclock_gtod_sync_lock);
#endif
}

2199
u64 get_kvmclock_ns(struct kvm *kvm)
2200 2201
{
	struct kvm_arch *ka = &kvm->arch;
2202
	struct pvclock_vcpu_time_info hv_clock;
2203
	u64 ret;
2204

2205 2206 2207
	spin_lock(&ka->pvclock_gtod_sync_lock);
	if (!ka->use_master_clock) {
		spin_unlock(&ka->pvclock_gtod_sync_lock);
2208
		return ktime_get_boottime_ns() + ka->kvmclock_offset;
2209 2210
	}

2211 2212 2213 2214
	hv_clock.tsc_timestamp = ka->master_cycle_now;
	hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
	spin_unlock(&ka->pvclock_gtod_sync_lock);

2215 2216 2217
	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
	get_cpu();

2218 2219 2220 2221 2222 2223
	if (__this_cpu_read(cpu_tsc_khz)) {
		kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL,
				   &hv_clock.tsc_shift,
				   &hv_clock.tsc_to_system_mul);
		ret = __pvclock_read_cycles(&hv_clock, rdtsc());
	} else
2224
		ret = ktime_get_boottime_ns() + ka->kvmclock_offset;
2225 2226 2227 2228

	put_cpu();

	return ret;
2229 2230
}

2231 2232 2233 2234 2235
static void kvm_setup_pvclock_page(struct kvm_vcpu *v)
{
	struct kvm_vcpu_arch *vcpu = &v->arch;
	struct pvclock_vcpu_time_info guest_hv_clock;

2236
	if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255
		&guest_hv_clock, sizeof(guest_hv_clock))))
		return;

	/* This VCPU is paused, but it's legal for a guest to read another
	 * VCPU's kvmclock, so we really have to follow the specification where
	 * it says that version is odd if data is being modified, and even after
	 * it is consistent.
	 *
	 * Version field updates must be kept separate.  This is because
	 * kvm_write_guest_cached might use a "rep movs" instruction, and
	 * writes within a string instruction are weakly ordered.  So there
	 * are three writes overall.
	 *
	 * As a small optimization, only write the version field in the first
	 * and third write.  The vcpu->pv_time cache is still valid, because the
	 * version field is the first in the struct.
	 */
	BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);

2256 2257 2258
	if (guest_hv_clock.version & 1)
		++guest_hv_clock.version;  /* first time write, random junk */

2259
	vcpu->hv_clock.version = guest_hv_clock.version + 1;
2260 2261 2262
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275

	smp_wmb();

	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
	vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);

	if (vcpu->pvclock_set_guest_stopped_request) {
		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
		vcpu->pvclock_set_guest_stopped_request = false;
	}

	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);

2276 2277 2278
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock));
2279 2280 2281 2282

	smp_wmb();

	vcpu->hv_clock.version++;
2283 2284 2285
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2286 2287
}

Z
Zachary Amsden 已提交
2288
static int kvm_guest_time_update(struct kvm_vcpu *v)
2289
{
2290
	unsigned long flags, tgt_tsc_khz;
2291
	struct kvm_vcpu_arch *vcpu = &v->arch;
2292
	struct kvm_arch *ka = &v->kvm->arch;
2293
	s64 kernel_ns;
2294
	u64 tsc_timestamp, host_tsc;
2295
	u8 pvclock_flags;
2296 2297 2298 2299
	bool use_master_clock;

	kernel_ns = 0;
	host_tsc = 0;
2300

2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311
	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
	spin_lock(&ka->pvclock_gtod_sync_lock);
	use_master_clock = ka->use_master_clock;
	if (use_master_clock) {
		host_tsc = ka->master_cycle_now;
		kernel_ns = ka->master_kernel_ns;
	}
	spin_unlock(&ka->pvclock_gtod_sync_lock);
2312 2313 2314

	/* Keep irq disabled to prevent changes to the clock */
	local_irq_save(flags);
2315 2316
	tgt_tsc_khz = __this_cpu_read(cpu_tsc_khz);
	if (unlikely(tgt_tsc_khz == 0)) {
2317 2318 2319 2320
		local_irq_restore(flags);
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
		return 1;
	}
2321
	if (!use_master_clock) {
2322
		host_tsc = rdtsc();
2323
		kernel_ns = ktime_get_boottime_ns();
2324 2325
	}

2326
	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
2327

Z
Zachary Amsden 已提交
2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
	/*
	 * We may have to catch up the TSC to match elapsed wall clock
	 * time for two reasons, even if kvmclock is used.
	 *   1) CPU could have been running below the maximum TSC rate
	 *   2) Broken TSC compensation resets the base at each VCPU
	 *      entry to avoid unknown leaps of TSC even when running
	 *      again on the same CPU.  This may cause apparent elapsed
	 *      time to disappear, and the guest to stand still or run
	 *	very slowly.
	 */
	if (vcpu->tsc_catchup) {
		u64 tsc = compute_guest_tsc(v, kernel_ns);
		if (tsc > tsc_timestamp) {
2341
			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
Z
Zachary Amsden 已提交
2342 2343
			tsc_timestamp = tsc;
		}
2344 2345
	}

2346 2347
	local_irq_restore(flags);

2348
	/* With all the info we got, fill in the values */
2349

2350 2351 2352 2353
	if (kvm_has_tsc_control)
		tgt_tsc_khz = kvm_scale_tsc(v, tgt_tsc_khz);

	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
2354
		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
2355 2356
				   &vcpu->hv_clock.tsc_shift,
				   &vcpu->hv_clock.tsc_to_system_mul);
2357
		vcpu->hw_tsc_khz = tgt_tsc_khz;
2358 2359
	}

2360
	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
2361
	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
Z
Zachary Amsden 已提交
2362
	vcpu->last_guest_tsc = tsc_timestamp;
2363

2364
	/* If the host uses TSC clocksource, then it is stable */
2365
	pvclock_flags = 0;
2366 2367 2368
	if (use_master_clock)
		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;

2369 2370
	vcpu->hv_clock.flags = pvclock_flags;

P
Paolo Bonzini 已提交
2371 2372 2373 2374
	if (vcpu->pv_time_enabled)
		kvm_setup_pvclock_page(v);
	if (v == kvm_get_vcpu(v->kvm, 0))
		kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
2375
	return 0;
2376 2377
}

2378 2379 2380 2381 2382 2383 2384 2385
/*
 * kvmclock updates which are isolated to a given vcpu, such as
 * vcpu->cpu migration, should not allow system_timestamp from
 * the rest of the vcpus to remain static. Otherwise ntp frequency
 * correction applies to one vcpu's system_timestamp but not
 * the others.
 *
 * So in those cases, request a kvmclock update for all vcpus.
2386 2387 2388 2389
 * We need to rate-limit these requests though, as they can
 * considerably slow guests that have a large number of vcpus.
 * The time for a remote vcpu to update its kvmclock is bound
 * by the delay we use to rate-limit the updates.
2390 2391
 */

2392 2393 2394
#define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)

static void kvmclock_update_fn(struct work_struct *work)
2395 2396
{
	int i;
2397 2398 2399 2400
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_update_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);
2401 2402 2403
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm) {
2404
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2405 2406 2407 2408
		kvm_vcpu_kick(vcpu);
	}
}

2409 2410 2411 2412
static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
{
	struct kvm *kvm = v->kvm;

2413
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
2414 2415 2416 2417
	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
					KVMCLOCK_UPDATE_DELAY);
}

2418 2419 2420 2421 2422 2423 2424 2425 2426
#define KVMCLOCK_SYNC_PERIOD (300 * HZ)

static void kvmclock_sync_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_sync_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);

2427 2428 2429
	if (!kvmclock_periodic_sync)
		return;

2430 2431 2432 2433 2434
	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
}

2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446
/*
 * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
 */
static bool can_set_mci_status(struct kvm_vcpu *vcpu)
{
	/* McStatusWrEn enabled? */
	if (guest_cpuid_is_amd(vcpu))
		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));

	return false;
}

2447
static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2448
{
H
Huang Ying 已提交
2449 2450
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2451 2452
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
H
Huang Ying 已提交
2453

2454 2455
	switch (msr) {
	case MSR_IA32_MCG_STATUS:
H
Huang Ying 已提交
2456
		vcpu->arch.mcg_status = data;
2457
		break;
2458
	case MSR_IA32_MCG_CTL:
2459 2460
		if (!(mcg_cap & MCG_CTL_P) &&
		    (data || !msr_info->host_initiated))
H
Huang Ying 已提交
2461 2462
			return 1;
		if (data != 0 && data != ~(u64)0)
2463
			return 1;
H
Huang Ying 已提交
2464 2465 2466 2467
		vcpu->arch.mcg_ctl = data;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2468
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
H
Huang Ying 已提交
2469
			u32 offset = msr - MSR_IA32_MC0_CTL;
2470 2471 2472 2473 2474
			/* only 0 or all 1s can be written to IA32_MCi_CTL
			 * some Linux kernels though clear bit 10 in bank 4 to
			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
			 * this to avoid an uncatched #GP in the guest
			 */
H
Huang Ying 已提交
2475
			if ((offset & 0x3) == 0 &&
2476
			    data != 0 && (data | (1 << 10)) != ~(u64)0)
H
Huang Ying 已提交
2477
				return -1;
2478 2479

			/* MCi_STATUS */
2480
			if (!msr_info->host_initiated &&
2481 2482 2483 2484 2485
			    (offset & 0x3) == 1 && data != 0) {
				if (!can_set_mci_status(vcpu))
					return -1;
			}

H
Huang Ying 已提交
2486 2487 2488 2489 2490 2491 2492 2493
			vcpu->arch.mce_banks[offset] = data;
			break;
		}
		return 1;
	}
	return 0;
}

E
Ed Swierk 已提交
2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510
static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
	int lm = is_long_mode(vcpu);
	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
		: kvm->arch.xen_hvm_config.blob_size_32;
	u32 page_num = data & ~PAGE_MASK;
	u64 page_addr = data & PAGE_MASK;
	u8 *page;
	int r;

	r = -E2BIG;
	if (page_num >= blob_size)
		goto out;
	r = -ENOMEM;
2511 2512 2513
	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
	if (IS_ERR(page)) {
		r = PTR_ERR(page);
E
Ed Swierk 已提交
2514
		goto out;
2515
	}
2516
	if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE))
E
Ed Swierk 已提交
2517 2518 2519 2520 2521 2522 2523 2524
		goto out_free;
	r = 0;
out_free:
	kfree(page);
out:
	return r;
}

2525 2526 2527 2528
static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
{
	gpa_t gpa = data & ~0x3f;

2529 2530
	/* Bits 3:5 are reserved, Should be zero */
	if (data & 0x38)
2531 2532 2533 2534 2535 2536 2537 2538 2539 2540
		return 1;

	vcpu->arch.apf.msr_val = data;

	if (!(data & KVM_ASYNC_PF_ENABLED)) {
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_async_pf_hash_reset(vcpu);
		return 0;
	}

2541
	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
2542
					sizeof(u32)))
2543 2544
		return 1;

2545
	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
2546
	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
2547 2548 2549 2550
	kvm_async_pf_wakeup_all(vcpu);
	return 0;
}

2551 2552
static void kvmclock_reset(struct kvm_vcpu *vcpu)
{
2553
	vcpu->arch.pv_time_enabled = false;
2554 2555
}

2556 2557 2558 2559 2560 2561
static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
{
	++vcpu->stat.tlb_flush;
	kvm_x86_ops->tlb_flush(vcpu, invalidate_gpa);
}

G
Glauber Costa 已提交
2562 2563 2564 2565 2566
static void record_steal_time(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

2567
	if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
G
Glauber Costa 已提交
2568 2569 2570
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
		return;

2571 2572 2573 2574
	/*
	 * Doing a TLB flush here, on the guest's behalf, can avoid
	 * expensive IPIs.
	 */
2575 2576
	trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
		vcpu->arch.st.steal.preempted & KVM_VCPU_FLUSH_TLB);
2577 2578
	if (xchg(&vcpu->arch.st.steal.preempted, 0) & KVM_VCPU_FLUSH_TLB)
		kvm_vcpu_flush_tlb(vcpu, false);
2579

W
Wanpeng Li 已提交
2580 2581 2582 2583 2584
	if (vcpu->arch.st.steal.version & 1)
		vcpu->arch.st.steal.version += 1;  /* first time write, random junk */

	vcpu->arch.st.steal.version += 1;

2585
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
W
Wanpeng Li 已提交
2586 2587 2588 2589
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));

	smp_wmb();

2590 2591 2592
	vcpu->arch.st.steal.steal += current->sched_info.run_delay -
		vcpu->arch.st.last_steal;
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
W
Wanpeng Li 已提交
2593

2594
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
W
Wanpeng Li 已提交
2595 2596 2597 2598 2599
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));

	smp_wmb();

	vcpu->arch.st.steal.version += 1;
G
Glauber Costa 已提交
2600

2601
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
G
Glauber Costa 已提交
2602 2603 2604
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
}

2605
int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2606
{
2607
	bool pr = false;
2608 2609
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
2610

2611
	switch (msr) {
2612 2613 2614 2615 2616
	case MSR_AMD64_NB_CFG:
	case MSR_IA32_UCODE_WRITE:
	case MSR_VM_HSAVE_PA:
	case MSR_AMD64_PATCH_LOADER:
	case MSR_AMD64_BU_CFG2:
2617
	case MSR_AMD64_DC_CFG:
2618
	case MSR_F15H_EX_CFG:
2619 2620
		break;

2621 2622 2623 2624
	case MSR_IA32_UCODE_REV:
		if (msr_info->host_initiated)
			vcpu->arch.microcode_version = data;
		break;
2625 2626 2627 2628 2629
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.arch_capabilities = data;
		break;
2630
	case MSR_EFER:
2631
		return set_efer(vcpu, msr_info);
2632 2633
	case MSR_K7_HWCR:
		data &= ~(u64)0x40;	/* ignore flush filter disable */
2634
		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
2635
		data &= ~(u64)0x8;	/* ignore TLB cache disable */
2636 2637 2638 2639 2640

		/* Handle McStatusWrEn */
		if (data == BIT_ULL(18)) {
			vcpu->arch.msr_hwcr = data;
		} else if (data != 0) {
2641 2642
			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
				    data);
2643 2644
			return 1;
		}
2645
		break;
2646 2647
	case MSR_FAM10H_MMIO_CONF_BASE:
		if (data != 0) {
2648 2649
			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
				    "0x%llx\n", data);
2650 2651
			return 1;
		}
2652
		break;
2653 2654 2655 2656 2657 2658 2659 2660 2661
	case MSR_IA32_DEBUGCTLMSR:
		if (!data) {
			/* We support the non-activated case already */
			break;
		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
			/* Values other than LBR and BTF are vendor-specific,
			   thus reserved and should throw a #GP */
			return 1;
		}
2662 2663
		vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
			    __func__, data);
2664
		break;
A
Avi Kivity 已提交
2665
	case 0x200 ... 0x2ff:
2666
		return kvm_mtrr_set_msr(vcpu, msr, data);
2667
	case MSR_IA32_APICBASE:
2668
		return kvm_set_apic_base(vcpu, msr_info);
G
Gleb Natapov 已提交
2669 2670
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_write(vcpu, msr, data);
2671 2672 2673
	case MSR_IA32_TSCDEADLINE:
		kvm_set_lapic_tscdeadline_msr(vcpu, data);
		break;
W
Will Auld 已提交
2674
	case MSR_IA32_TSC_ADJUST:
2675
		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
W
Will Auld 已提交
2676
			if (!msr_info->host_initiated) {
2677
				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
2678
				adjust_tsc_offset_guest(vcpu, adj);
W
Will Auld 已提交
2679 2680 2681 2682
			}
			vcpu->arch.ia32_tsc_adjust_msr = data;
		}
		break;
2683
	case MSR_IA32_MISC_ENABLE:
2684 2685 2686 2687 2688 2689 2690 2691 2692
		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
		    ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
				return 1;
			vcpu->arch.ia32_misc_enable_msr = data;
			kvm_update_cpuid(vcpu);
		} else {
			vcpu->arch.ia32_misc_enable_msr = data;
		}
2693
		break;
P
Paolo Bonzini 已提交
2694 2695 2696 2697 2698
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smbase = data;
		break;
2699 2700 2701
	case MSR_IA32_POWER_CTL:
		vcpu->arch.msr_ia32_power_ctl = data;
		break;
2702 2703 2704
	case MSR_IA32_TSC:
		kvm_write_tsc(vcpu, msr_info);
		break;
2705 2706 2707 2708 2709
	case MSR_SMI_COUNT:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smi_count = data;
		break;
2710
	case MSR_KVM_WALL_CLOCK_NEW:
2711 2712 2713 2714
	case MSR_KVM_WALL_CLOCK:
		vcpu->kvm->arch.wall_clock = data;
		kvm_write_wall_clock(vcpu->kvm, data);
		break;
2715
	case MSR_KVM_SYSTEM_TIME_NEW:
2716
	case MSR_KVM_SYSTEM_TIME: {
2717 2718
		struct kvm_arch *ka = &vcpu->kvm->arch;

2719
		kvmclock_reset(vcpu);
2720

2721 2722 2723 2724
		if (vcpu->vcpu_id == 0 && !msr_info->host_initiated) {
			bool tmp = (msr == MSR_KVM_SYSTEM_TIME);

			if (ka->boot_vcpu_runs_old_kvmclock != tmp)
2725
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2726 2727 2728 2729

			ka->boot_vcpu_runs_old_kvmclock = tmp;
		}

2730
		vcpu->arch.time = data;
2731
		kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2732 2733 2734 2735 2736

		/* we verify if the enable bit is set... */
		if (!(data & 1))
			break;

2737
		if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2738 2739
		     &vcpu->arch.pv_time, data & ~1ULL,
		     sizeof(struct pvclock_vcpu_time_info)))
2740 2741 2742
			vcpu->arch.pv_time_enabled = false;
		else
			vcpu->arch.pv_time_enabled = true;
2743

2744 2745
		break;
	}
2746 2747 2748 2749
	case MSR_KVM_ASYNC_PF_EN:
		if (kvm_pv_enable_async_pf(vcpu, data))
			return 1;
		break;
G
Glauber Costa 已提交
2750 2751 2752 2753 2754 2755 2756 2757
	case MSR_KVM_STEAL_TIME:

		if (unlikely(!sched_info_on()))
			return 1;

		if (data & KVM_STEAL_RESERVED_MASK)
			return 1;

2758
		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
2759 2760
						data & KVM_STEAL_VALID_BITS,
						sizeof(struct kvm_steal_time)))
G
Glauber Costa 已提交
2761 2762 2763 2764 2765 2766 2767 2768 2769 2770
			return 1;

		vcpu->arch.st.msr_val = data;

		if (!(data & KVM_MSR_ENABLED))
			break;

		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);

		break;
2771
	case MSR_KVM_PV_EOI_EN:
2772
		if (kvm_lapic_enable_pv_eoi(vcpu, data, sizeof(u8)))
2773 2774
			return 1;
		break;
G
Glauber Costa 已提交
2775

2776 2777 2778 2779 2780 2781 2782 2783
	case MSR_KVM_POLL_CONTROL:
		/* only enable bit supported */
		if (data & (-1ULL << 1))
			return 1;

		vcpu->arch.msr_kvm_poll_control = data;
		break;

H
Huang Ying 已提交
2784 2785
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
2786
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
2787
		return set_msr_mce(vcpu, msr_info);
2788

2789 2790 2791 2792 2793
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
		pr = true; /* fall through */
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
2794
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2795
			return kvm_pmu_set_msr(vcpu, msr_info);
2796 2797

		if (pr || data != 0)
2798 2799
			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
				    "0x%x data 0x%llx\n", msr, data);
2800
		break;
2801 2802 2803 2804 2805
	case MSR_K7_CLK_CTL:
		/*
		 * Ignore all writes to this no longer documented MSR.
		 * Writes are only relevant for old K7 processors,
		 * all pre-dating SVM, but a recommended workaround from
G
Guo Chao 已提交
2806
		 * AMD for these chips. It is possible to specify the
2807 2808 2809 2810
		 * affected processor models on the command line, hence
		 * the need to ignore the workaround.
		 */
		break;
2811
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
2812 2813
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
A
Andrey Smetanin 已提交
2814
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
2815 2816 2817
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
2818 2819
		return kvm_hv_set_msr_common(vcpu, msr, data,
					     msr_info->host_initiated);
2820 2821 2822 2823
	case MSR_IA32_BBL_CR_CTL3:
		/* Drop writes to this legacy MSR -- see rdmsr
		 * counterpart for further detail.
		 */
2824 2825 2826
		if (report_ignored_msrs)
			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
				msr, data);
2827
		break;
2828
	case MSR_AMD64_OSVW_ID_LENGTH:
2829
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2830 2831 2832 2833
			return 1;
		vcpu->arch.osvw.length = data;
		break;
	case MSR_AMD64_OSVW_STATUS:
2834
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2835 2836 2837
			return 1;
		vcpu->arch.osvw.status = data;
		break;
K
Kyle Huey 已提交
2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851
	case MSR_PLATFORM_INFO:
		if (!msr_info->host_initiated ||
		    (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
		     cpuid_fault_enabled(vcpu)))
			return 1;
		vcpu->arch.msr_platform_info = data;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
		     !supports_cpuid_fault(vcpu)))
			return 1;
		vcpu->arch.msr_misc_features_enables = data;
		break;
2852
	default:
E
Ed Swierk 已提交
2853 2854
		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
			return xen_hvm_config(vcpu, data);
2855
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2856
			return kvm_pmu_set_msr(vcpu, msr_info);
2857
		if (!ignore_msrs) {
2858
			vcpu_debug_ratelimited(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n",
2859
				    msr, data);
2860 2861
			return 1;
		} else {
2862 2863 2864 2865
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu,
					"ignored wrmsr: 0x%x data 0x%llx\n",
					msr, data);
2866 2867
			break;
		}
2868 2869 2870 2871 2872
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);

2873
static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
2874 2875
{
	u64 data;
H
Huang Ying 已提交
2876 2877
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2878 2879 2880 2881

	switch (msr) {
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
H
Huang Ying 已提交
2882 2883
		data = 0;
		break;
2884
	case MSR_IA32_MCG_CAP:
H
Huang Ying 已提交
2885 2886
		data = vcpu->arch.mcg_cap;
		break;
2887
	case MSR_IA32_MCG_CTL:
2888
		if (!(mcg_cap & MCG_CTL_P) && !host)
H
Huang Ying 已提交
2889 2890 2891 2892 2893 2894 2895 2896
			return 1;
		data = vcpu->arch.mcg_ctl;
		break;
	case MSR_IA32_MCG_STATUS:
		data = vcpu->arch.mcg_status;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2897
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
H
Huang Ying 已提交
2898 2899 2900 2901 2902 2903 2904 2905 2906 2907
			u32 offset = msr - MSR_IA32_MC0_CTL;
			data = vcpu->arch.mce_banks[offset];
			break;
		}
		return 1;
	}
	*pdata = data;
	return 0;
}

2908
int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
H
Huang Ying 已提交
2909
{
2910
	switch (msr_info->index) {
H
Huang Ying 已提交
2911
	case MSR_IA32_PLATFORM_ID:
2912
	case MSR_IA32_EBL_CR_POWERON:
2913 2914 2915 2916 2917
	case MSR_IA32_DEBUGCTLMSR:
	case MSR_IA32_LASTBRANCHFROMIP:
	case MSR_IA32_LASTBRANCHTOIP:
	case MSR_IA32_LASTINTFROMIP:
	case MSR_IA32_LASTINTTOIP:
2918
	case MSR_K8_SYSCFG:
2919 2920
	case MSR_K8_TSEG_ADDR:
	case MSR_K8_TSEG_MASK:
2921
	case MSR_VM_HSAVE_PA:
2922
	case MSR_K8_INT_PENDING_MSG:
2923
	case MSR_AMD64_NB_CFG:
2924
	case MSR_FAM10H_MMIO_CONF_BASE:
2925
	case MSR_AMD64_BU_CFG2:
D
Dmitry Bilunov 已提交
2926
	case MSR_IA32_PERF_CTL:
2927
	case MSR_AMD64_DC_CFG:
2928
	case MSR_F15H_EX_CFG:
2929
		msr_info->data = 0;
2930
		break;
2931
	case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
2932 2933 2934 2935
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
2936
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
2937 2938
			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
		msr_info->data = 0;
2939
		break;
2940
	case MSR_IA32_UCODE_REV:
2941
		msr_info->data = vcpu->arch.microcode_version;
2942
		break;
2943 2944 2945 2946 2947 2948
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
			return 1;
		msr_info->data = vcpu->arch.arch_capabilities;
		break;
2949 2950 2951
	case MSR_IA32_POWER_CTL:
		msr_info->data = vcpu->arch.msr_ia32_power_ctl;
		break;
2952 2953 2954
	case MSR_IA32_TSC:
		msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + vcpu->arch.tsc_offset;
		break;
A
Avi Kivity 已提交
2955 2956
	case MSR_MTRRcap:
	case 0x200 ... 0x2ff:
2957
		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
2958
	case 0xcd: /* fsb frequency */
2959
		msr_info->data = 3;
2960
		break;
2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972
		/*
		 * MSR_EBC_FREQUENCY_ID
		 * Conservative value valid for even the basic CPU models.
		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
		 * and 266MHz for model 3, or 4. Set Core Clock
		 * Frequency to System Bus Frequency Ratio to 1 (bits
		 * 31:24) even though these are only valid for CPU
		 * models > 2, however guests may end up dividing or
		 * multiplying by zero otherwise.
		 */
	case MSR_EBC_FREQUENCY_ID:
2973
		msr_info->data = 1 << 24;
2974
		break;
2975
	case MSR_IA32_APICBASE:
2976
		msr_info->data = kvm_get_apic_base(vcpu);
2977
		break;
G
Gleb Natapov 已提交
2978
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
2979
		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
G
Gleb Natapov 已提交
2980
		break;
2981
	case MSR_IA32_TSCDEADLINE:
2982
		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
2983
		break;
W
Will Auld 已提交
2984
	case MSR_IA32_TSC_ADJUST:
2985
		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
W
Will Auld 已提交
2986
		break;
2987
	case MSR_IA32_MISC_ENABLE:
2988
		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
2989
		break;
P
Paolo Bonzini 已提交
2990 2991 2992 2993
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		msr_info->data = vcpu->arch.smbase;
2994
		break;
2995 2996 2997
	case MSR_SMI_COUNT:
		msr_info->data = vcpu->arch.smi_count;
		break;
2998 2999
	case MSR_IA32_PERF_STATUS:
		/* TSC increment by tick */
3000
		msr_info->data = 1000ULL;
3001
		/* CPU multiplier */
3002
		msr_info->data |= (((uint64_t)4ULL) << 40);
3003
		break;
3004
	case MSR_EFER:
3005
		msr_info->data = vcpu->arch.efer;
3006
		break;
3007
	case MSR_KVM_WALL_CLOCK:
3008
	case MSR_KVM_WALL_CLOCK_NEW:
3009
		msr_info->data = vcpu->kvm->arch.wall_clock;
3010 3011
		break;
	case MSR_KVM_SYSTEM_TIME:
3012
	case MSR_KVM_SYSTEM_TIME_NEW:
3013
		msr_info->data = vcpu->arch.time;
3014
		break;
3015
	case MSR_KVM_ASYNC_PF_EN:
3016
		msr_info->data = vcpu->arch.apf.msr_val;
3017
		break;
G
Glauber Costa 已提交
3018
	case MSR_KVM_STEAL_TIME:
3019
		msr_info->data = vcpu->arch.st.msr_val;
G
Glauber Costa 已提交
3020
		break;
3021
	case MSR_KVM_PV_EOI_EN:
3022
		msr_info->data = vcpu->arch.pv_eoi.msr_val;
3023
		break;
3024 3025 3026
	case MSR_KVM_POLL_CONTROL:
		msr_info->data = vcpu->arch.msr_kvm_poll_control;
		break;
H
Huang Ying 已提交
3027 3028 3029 3030 3031
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
3032
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3033 3034
		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
				   msr_info->host_initiated);
3035 3036 3037 3038 3039 3040 3041 3042 3043 3044
	case MSR_K7_CLK_CTL:
		/*
		 * Provide expected ramp-up count for K7. All other
		 * are set to zero, indicating minimum divisors for
		 * every field.
		 *
		 * This prevents guest kernels on AMD host with CPU
		 * type 6, model 8 and higher from exploding due to
		 * the rdmsr failing.
		 */
3045
		msr_info->data = 0x20000000;
3046
		break;
3047
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
3048 3049
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
A
Andrey Smetanin 已提交
3050
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
3051 3052 3053
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
3054
		return kvm_hv_get_msr_common(vcpu,
3055 3056
					     msr_info->index, &msr_info->data,
					     msr_info->host_initiated);
3057
		break;
3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068
	case MSR_IA32_BBL_CR_CTL3:
		/* This legacy MSR exists but isn't fully documented in current
		 * silicon.  It is however accessed by winxp in very narrow
		 * scenarios where it sets bit #19, itself documented as
		 * a "reserved" bit.  Best effort attempt to source coherent
		 * read data here should the balance of the register be
		 * interpreted by the guest:
		 *
		 * L2 cache control register 3: 64GB range, 256KB size,
		 * enabled, latency 0x1, configured
		 */
3069
		msr_info->data = 0xbe702111;
3070
		break;
3071
	case MSR_AMD64_OSVW_ID_LENGTH:
3072
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3073
			return 1;
3074
		msr_info->data = vcpu->arch.osvw.length;
3075 3076
		break;
	case MSR_AMD64_OSVW_STATUS:
3077
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3078
			return 1;
3079
		msr_info->data = vcpu->arch.osvw.status;
3080
		break;
K
Kyle Huey 已提交
3081
	case MSR_PLATFORM_INFO:
3082 3083 3084
		if (!msr_info->host_initiated &&
		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
			return 1;
K
Kyle Huey 已提交
3085 3086 3087 3088 3089
		msr_info->data = vcpu->arch.msr_platform_info;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		msr_info->data = vcpu->arch.msr_misc_features_enables;
		break;
3090 3091 3092
	case MSR_K7_HWCR:
		msr_info->data = vcpu->arch.msr_hwcr;
		break;
3093
	default:
3094
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3095
			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
3096
		if (!ignore_msrs) {
3097 3098
			vcpu_debug_ratelimited(vcpu, "unhandled rdmsr: 0x%x\n",
					       msr_info->index);
3099 3100
			return 1;
		} else {
3101 3102 3103
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n",
					msr_info->index);
3104
			msr_info->data = 0;
3105 3106
		}
		break;
3107 3108 3109 3110 3111
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

3112 3113 3114 3115 3116 3117 3118 3119 3120 3121
/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
3122
	int i;
3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146

	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
3147
	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
3148 3149 3150 3151 3152 3153 3154
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
3155 3156 3157
	entries = memdup_user(user_msrs->entries, size);
	if (IS_ERR(entries)) {
		r = PTR_ERR(entries);
3158
		goto out;
3159
	}
3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171

	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
3172
	kfree(entries);
3173 3174 3175 3176
out:
	return r;
}

3177 3178 3179
static inline bool kvm_can_mwait_in_guest(void)
{
	return boot_cpu_has(X86_FEATURE_MWAIT) &&
3180 3181
		!boot_cpu_has_bug(X86_BUG_MONITOR) &&
		boot_cpu_has(X86_FEATURE_ARAT);
3182 3183
}

3184
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
3185
{
3186
	int r = 0;
3187 3188 3189 3190 3191 3192

	switch (ext) {
	case KVM_CAP_IRQCHIP:
	case KVM_CAP_HLT:
	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
	case KVM_CAP_SET_TSS_ADDR:
3193
	case KVM_CAP_EXT_CPUID:
B
Borislav Petkov 已提交
3194
	case KVM_CAP_EXT_EMUL_CPUID:
3195
	case KVM_CAP_CLOCKSOURCE:
S
Sheng Yang 已提交
3196
	case KVM_CAP_PIT:
3197
	case KVM_CAP_NOP_IO_DELAY:
3198
	case KVM_CAP_MP_STATE:
3199
	case KVM_CAP_SYNC_MMU:
3200
	case KVM_CAP_USER_NMI:
3201
	case KVM_CAP_REINJECT_CONTROL:
3202
	case KVM_CAP_IRQ_INJECT_STATUS:
G
Gregory Haskins 已提交
3203
	case KVM_CAP_IOEVENTFD:
3204
	case KVM_CAP_IOEVENTFD_NO_LENGTH:
3205
	case KVM_CAP_PIT2:
B
Beth Kon 已提交
3206
	case KVM_CAP_PIT_STATE2:
3207
	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
E
Ed Swierk 已提交
3208
	case KVM_CAP_XEN_HVM:
J
Jan Kiszka 已提交
3209
	case KVM_CAP_VCPU_EVENTS:
3210
	case KVM_CAP_HYPERV:
G
Gleb Natapov 已提交
3211
	case KVM_CAP_HYPERV_VAPIC:
3212
	case KVM_CAP_HYPERV_SPIN:
3213
	case KVM_CAP_HYPERV_SYNIC:
3214
	case KVM_CAP_HYPERV_SYNIC2:
3215
	case KVM_CAP_HYPERV_VP_INDEX:
3216
	case KVM_CAP_HYPERV_EVENTFD:
3217
	case KVM_CAP_HYPERV_TLBFLUSH:
3218
	case KVM_CAP_HYPERV_SEND_IPI:
3219
	case KVM_CAP_HYPERV_CPUID:
3220
	case KVM_CAP_PCI_SEGMENT:
3221
	case KVM_CAP_DEBUGREGS:
3222
	case KVM_CAP_X86_ROBUST_SINGLESTEP:
3223
	case KVM_CAP_XSAVE:
3224
	case KVM_CAP_ASYNC_PF:
3225
	case KVM_CAP_GET_TSC_KHZ:
3226
	case KVM_CAP_KVMCLOCK_CTRL:
X
Xiao Guangrong 已提交
3227
	case KVM_CAP_READONLY_MEM:
3228
	case KVM_CAP_HYPERV_TIME:
3229
	case KVM_CAP_IOAPIC_POLARITY_IGNORED:
3230
	case KVM_CAP_TSC_DEADLINE_TIMER:
3231
	case KVM_CAP_DISABLE_QUIRKS:
3232
	case KVM_CAP_SET_BOOT_CPU_ID:
3233
 	case KVM_CAP_SPLIT_IRQCHIP:
3234
	case KVM_CAP_IMMEDIATE_EXIT:
E
Eric Hankland 已提交
3235
	case KVM_CAP_PMU_EVENT_FILTER:
3236
	case KVM_CAP_GET_MSR_FEATURES:
3237
	case KVM_CAP_MSR_PLATFORM_INFO:
3238
	case KVM_CAP_EXCEPTION_PAYLOAD:
3239 3240
		r = 1;
		break;
K
Ken Hofsass 已提交
3241 3242 3243
	case KVM_CAP_SYNC_REGS:
		r = KVM_SYNC_X86_VALID_FIELDS;
		break;
3244 3245 3246
	case KVM_CAP_ADJUST_CLOCK:
		r = KVM_CLOCK_TSC_STABLE;
		break;
3247
	case KVM_CAP_X86_DISABLE_EXITS:
3248 3249
		r |=  KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE |
		      KVM_X86_DISABLE_EXITS_CSTATE;
3250 3251
		if(kvm_can_mwait_in_guest())
			r |= KVM_X86_DISABLE_EXITS_MWAIT;
3252
		break;
3253 3254 3255 3256 3257 3258 3259 3260 3261
	case KVM_CAP_X86_SMM:
		/* SMBASE is usually relocated above 1M on modern chipsets,
		 * and SMM handlers might indeed rely on 4G segment limits,
		 * so do not report SMM to be available if real mode is
		 * emulated via vm86 mode.  Still, do not go to great lengths
		 * to avoid userspace's usage of the feature, because it is a
		 * fringe case that is not enabled except via specific settings
		 * of the module parameters.
		 */
3262
		r = kvm_x86_ops->has_emulated_msr(MSR_IA32_SMBASE);
3263
		break;
3264 3265 3266
	case KVM_CAP_VAPIC:
		r = !kvm_x86_ops->cpu_has_accelerated_tpr();
		break;
3267
	case KVM_CAP_NR_VCPUS:
3268 3269 3270
		r = KVM_SOFT_MAX_VCPUS;
		break;
	case KVM_CAP_MAX_VCPUS:
3271 3272
		r = KVM_MAX_VCPUS;
		break;
3273 3274 3275
	case KVM_CAP_MAX_VCPU_ID:
		r = KVM_MAX_VCPU_ID;
		break;
3276 3277
	case KVM_CAP_PV_MMU:	/* obsolete */
		r = 0;
3278
		break;
H
Huang Ying 已提交
3279 3280 3281
	case KVM_CAP_MCE:
		r = KVM_MAX_MCE_BANKS;
		break;
3282
	case KVM_CAP_XCRS:
3283
		r = boot_cpu_has(X86_FEATURE_XSAVE);
3284
		break;
3285 3286 3287
	case KVM_CAP_TSC_CONTROL:
		r = kvm_has_tsc_control;
		break;
3288 3289 3290
	case KVM_CAP_X2APIC_API:
		r = KVM_X2APIC_API_VALID_FLAGS;
		break;
3291 3292
	case KVM_CAP_NESTED_STATE:
		r = kvm_x86_ops->get_nested_state ?
3293
			kvm_x86_ops->get_nested_state(NULL, NULL, 0) : 0;
3294
		break;
3295
	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
3296 3297 3298 3299
		r = kvm_x86_ops->enable_direct_tlbflush != NULL;
		break;
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
		r = kvm_x86_ops->nested_enable_evmcs != NULL;
3300
		break;
3301 3302 3303 3304 3305 3306 3307
	default:
		break;
	}
	return r;

}

3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320
long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {
	case KVM_GET_MSR_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
3321
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
3322 3323
			goto out;
		n = msr_list.nmsrs;
3324
		msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
3325
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
3326 3327
			goto out;
		r = -E2BIG;
J
Jan Kiszka 已提交
3328
		if (n < msr_list.nmsrs)
3329 3330 3331 3332 3333
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
				 num_msrs_to_save * sizeof(u32)))
			goto out;
J
Jan Kiszka 已提交
3334
		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
3335
				 &emulated_msrs,
3336
				 num_emulated_msrs * sizeof(u32)))
3337 3338 3339 3340
			goto out;
		r = 0;
		break;
	}
B
Borislav Petkov 已提交
3341 3342
	case KVM_GET_SUPPORTED_CPUID:
	case KVM_GET_EMULATED_CPUID: {
3343 3344 3345 3346
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
3347
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
3348
			goto out;
B
Borislav Petkov 已提交
3349 3350 3351

		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
					    ioctl);
3352 3353 3354 3355
		if (r)
			goto out;

		r = -EFAULT;
3356
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
3357 3358 3359 3360
			goto out;
		r = 0;
		break;
	}
H
Huang Ying 已提交
3361 3362
	case KVM_X86_GET_MCE_CAP_SUPPORTED: {
		r = -EFAULT;
3363 3364
		if (copy_to_user(argp, &kvm_mce_cap_supported,
				 sizeof(kvm_mce_cap_supported)))
H
Huang Ying 已提交
3365 3366 3367
			goto out;
		r = 0;
		break;
3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392
	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned int n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
			goto out;
		n = msr_list.nmsrs;
		msr_list.nmsrs = num_msr_based_features;
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
			goto out;
		r = -E2BIG;
		if (n < msr_list.nmsrs)
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msr_based_features,
				 num_msr_based_features * sizeof(u32)))
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_MSRS:
		r = msr_io(NULL, argp, do_get_msr_feature, 1);
		break;
H
Huang Ying 已提交
3393
	}
3394 3395 3396 3397 3398 3399 3400
	default:
		r = -EINVAL;
	}
out:
	return r;
}

3401 3402 3403 3404 3405 3406 3407
static void wbinvd_ipi(void *garbage)
{
	wbinvd();
}

static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
3408
	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
3409 3410
}

3411 3412
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
3413 3414 3415 3416 3417 3418 3419 3420 3421
	/* Address WBINVD may be executed by guest */
	if (need_emulate_wbinvd(vcpu)) {
		if (kvm_x86_ops->has_wbinvd_exit())
			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
			smp_call_function_single(vcpu->cpu,
					wbinvd_ipi, NULL, 1);
	}

3422
	kvm_x86_ops->vcpu_load(vcpu, cpu);
3423

3424 3425 3426 3427
	fpregs_assert_state_consistent();
	if (test_thread_flag(TIF_NEED_FPU_LOAD))
		switch_fpu_return();

3428 3429 3430 3431
	/* Apply any externally detected TSC adjustments (due to suspend) */
	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
		vcpu->arch.tsc_offset_adjustment = 0;
3432
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3433
	}
3434

3435
	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
3436
		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
3437
				rdtsc() - vcpu->arch.last_host_tsc;
Z
Zachary Amsden 已提交
3438 3439
		if (tsc_delta < 0)
			mark_tsc_unstable("KVM discovered backwards TSC");
3440

3441
		if (kvm_check_tsc_unstable()) {
3442
			u64 offset = kvm_compute_tsc_offset(vcpu,
3443
						vcpu->arch.last_guest_tsc);
3444
			kvm_vcpu_write_tsc_offset(vcpu, offset);
Z
Zachary Amsden 已提交
3445 3446
			vcpu->arch.tsc_catchup = 1;
		}
3447 3448 3449 3450

		if (kvm_lapic_hv_timer_in_use(vcpu))
			kvm_lapic_restart_hv_timer(vcpu);

3451 3452 3453 3454 3455
		/*
		 * On a host with synchronized TSC, there is no need to update
		 * kvmclock on vcpu->cpu migration
		 */
		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
3456
			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
Z
Zachary Amsden 已提交
3457
		if (vcpu->cpu != cpu)
3458
			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
Z
Zachary Amsden 已提交
3459
		vcpu->cpu = cpu;
Z
Zachary Amsden 已提交
3460
	}
G
Glauber Costa 已提交
3461 3462

	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3463 3464
}

3465 3466 3467 3468 3469
static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

W
Wanpeng Li 已提交
3470
	vcpu->arch.st.steal.preempted = KVM_VCPU_PREEMPTED;
3471

3472
	kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.st.stime,
3473 3474 3475 3476 3477
			&vcpu->arch.st.steal.preempted,
			offsetof(struct kvm_steal_time, preempted),
			sizeof(vcpu->arch.st.steal.preempted));
}

3478 3479
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
3480
	int idx;
3481 3482 3483 3484

	if (vcpu->preempted)
		vcpu->arch.preempted_in_kernel = !kvm_x86_ops->get_cpl(vcpu);

3485 3486 3487 3488 3489 3490 3491 3492 3493
	/*
	 * Disable page faults because we're in atomic context here.
	 * kvm_write_guest_offset_cached() would call might_fault()
	 * that relies on pagefault_disable() to tell if there's a
	 * bug. NOTE: the write to guest memory may not go through if
	 * during postcopy live migration or if there's heavy guest
	 * paging.
	 */
	pagefault_disable();
3494 3495 3496 3497 3498
	/*
	 * kvm_memslots() will be called by
	 * kvm_write_guest_offset_cached() so take the srcu lock.
	 */
	idx = srcu_read_lock(&vcpu->kvm->srcu);
3499
	kvm_steal_time_set_preempted(vcpu);
3500
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
3501
	pagefault_enable();
3502
	kvm_x86_ops->vcpu_put(vcpu);
3503
	vcpu->arch.last_host_tsc = rdtsc();
3504
	/*
3505 3506 3507
	 * If userspace has set any breakpoints or watchpoints, dr6 is restored
	 * on every vmexit, but if not, we might have a stale dr6 from the
	 * guest. do_debug expects dr6 to be cleared after it runs, do the same.
3508
	 */
3509
	set_debugreg(0, 6);
3510 3511 3512 3513 3514
}

static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3515
	if (vcpu->arch.apicv_active)
3516 3517
		kvm_x86_ops->sync_pir_to_irr(vcpu);

3518
	return kvm_apic_get_state(vcpu, s);
3519 3520 3521 3522 3523
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3524 3525 3526 3527 3528
	int r;

	r = kvm_apic_set_state(vcpu, s);
	if (r)
		return r;
3529
	update_cr8_intercept(vcpu);
3530 3531 3532 3533

	return 0;
}

3534 3535 3536 3537 3538 3539
static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
{
	return (!lapic_in_kernel(vcpu) ||
		kvm_apic_accept_pic_intr(vcpu));
}

3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553
/*
 * if userspace requested an interrupt window, check that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
{
	return kvm_arch_interrupt_allowed(vcpu) &&
		!kvm_cpu_has_interrupt(vcpu) &&
		!kvm_event_needs_reinjection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);
}

3554 3555 3556
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
{
3557
	if (irq->irq >= KVM_NR_INTERRUPTS)
3558
		return -EINVAL;
3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570

	if (!irqchip_in_kernel(vcpu->kvm)) {
		kvm_queue_interrupt(vcpu, irq->irq, false);
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		return 0;
	}

	/*
	 * With in-kernel LAPIC, we only use this to inject EXTINT, so
	 * fail for in-kernel 8259.
	 */
	if (pic_in_kernel(vcpu->kvm))
3571 3572
		return -ENXIO;

3573 3574
	if (vcpu->arch.pending_external_vector != -1)
		return -EEXIST;
3575

3576
	vcpu->arch.pending_external_vector = irq->irq;
3577
	kvm_make_request(KVM_REQ_EVENT, vcpu);
3578 3579 3580
	return 0;
}

3581 3582 3583 3584 3585 3586 3587
static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
{
	kvm_inject_nmi(vcpu);

	return 0;
}

3588 3589
static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
{
P
Paolo Bonzini 已提交
3590 3591
	kvm_make_request(KVM_REQ_SMI, vcpu);

3592 3593 3594
	return 0;
}

3595 3596 3597 3598 3599 3600 3601 3602 3603
static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
					   struct kvm_tpr_access_ctl *tac)
{
	if (tac->flags)
		return -EINVAL;
	vcpu->arch.tpr_access_reporting = !!tac->enabled;
	return 0;
}

H
Huang Ying 已提交
3604 3605 3606 3607 3608 3609 3610
static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
					u64 mcg_cap)
{
	int r;
	unsigned bank_num = mcg_cap & 0xff, bank;

	r = -EINVAL;
3611
	if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
H
Huang Ying 已提交
3612
		goto out;
3613
	if (mcg_cap & ~(kvm_mce_cap_supported | 0xff | 0xff0000))
H
Huang Ying 已提交
3614 3615 3616 3617 3618 3619 3620 3621 3622
		goto out;
	r = 0;
	vcpu->arch.mcg_cap = mcg_cap;
	/* Init IA32_MCG_CTL to all 1s */
	if (mcg_cap & MCG_CTL_P)
		vcpu->arch.mcg_ctl = ~(u64)0;
	/* Init IA32_MCi_CTL to all 1s */
	for (bank = 0; bank < bank_num; bank++)
		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
3623

3624
	kvm_x86_ops->setup_mce(vcpu);
H
Huang Ying 已提交
3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653
out:
	return r;
}

static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
				      struct kvm_x86_mce *mce)
{
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
	u64 *banks = vcpu->arch.mce_banks;

	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
		return -EINVAL;
	/*
	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
	 * reporting is disabled
	 */
	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
	    vcpu->arch.mcg_ctl != ~(u64)0)
		return 0;
	banks += 4 * mce->bank;
	/*
	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
	 * reporting is disabled for the bank
	 */
	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
		return 0;
	if (mce->status & MCI_STATUS_UC) {
		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
3654
		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
3655
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
H
Huang Ying 已提交
3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676
			return 0;
		}
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		vcpu->arch.mcg_status = mce->mcg_status;
		banks[1] = mce->status;
		kvm_queue_exception(vcpu, MC_VECTOR);
	} else if (!(banks[1] & MCI_STATUS_VAL)
		   || !(banks[1] & MCI_STATUS_UC)) {
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		banks[1] = mce->status;
	} else
		banks[1] |= MCI_STATUS_OVER;
	return 0;
}

J
Jan Kiszka 已提交
3677 3678 3679
static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
					       struct kvm_vcpu_events *events)
{
A
Avi Kivity 已提交
3680
	process_nmi(vcpu);
3681

3682
	/*
3683 3684 3685 3686
	 * The API doesn't provide the instruction length for software
	 * exceptions, so don't report them. As long as the guest RIP
	 * isn't advanced, we should expect to encounter the exception
	 * again.
3687
	 */
3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702
	if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {
		events->exception.injected = 0;
		events->exception.pending = 0;
	} else {
		events->exception.injected = vcpu->arch.exception.injected;
		events->exception.pending = vcpu->arch.exception.pending;
		/*
		 * For ABI compatibility, deliberately conflate
		 * pending and injected exceptions when
		 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
		 */
		if (!vcpu->kvm->arch.exception_payload_enabled)
			events->exception.injected |=
				vcpu->arch.exception.pending;
	}
J
Jan Kiszka 已提交
3703 3704 3705
	events->exception.nr = vcpu->arch.exception.nr;
	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
	events->exception.error_code = vcpu->arch.exception.error_code;
3706 3707
	events->exception_has_payload = vcpu->arch.exception.has_payload;
	events->exception_payload = vcpu->arch.exception.payload;
J
Jan Kiszka 已提交
3708

3709
	events->interrupt.injected =
3710
		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
J
Jan Kiszka 已提交
3711
	events->interrupt.nr = vcpu->arch.interrupt.nr;
3712
	events->interrupt.soft = 0;
3713
	events->interrupt.shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
J
Jan Kiszka 已提交
3714 3715

	events->nmi.injected = vcpu->arch.nmi_injected;
A
Avi Kivity 已提交
3716
	events->nmi.pending = vcpu->arch.nmi_pending != 0;
J
Jan Kiszka 已提交
3717
	events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
3718
	events->nmi.pad = 0;
J
Jan Kiszka 已提交
3719

3720
	events->sipi_vector = 0; /* never valid when reporting to user space */
J
Jan Kiszka 已提交
3721

3722 3723 3724 3725 3726 3727
	events->smi.smm = is_smm(vcpu);
	events->smi.pending = vcpu->arch.smi_pending;
	events->smi.smm_inside_nmi =
		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
	events->smi.latched_init = kvm_lapic_latched_init(vcpu);

3728
	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
3729 3730
			 | KVM_VCPUEVENT_VALID_SHADOW
			 | KVM_VCPUEVENT_VALID_SMM);
3731 3732 3733
	if (vcpu->kvm->arch.exception_payload_enabled)
		events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;

3734
	memset(&events->reserved, 0, sizeof(events->reserved));
J
Jan Kiszka 已提交
3735 3736
}

3737
static void kvm_smm_changed(struct kvm_vcpu *vcpu);
3738

J
Jan Kiszka 已提交
3739 3740 3741
static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
					      struct kvm_vcpu_events *events)
{
3742
	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
3743
			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
3744
			      | KVM_VCPUEVENT_VALID_SHADOW
3745 3746
			      | KVM_VCPUEVENT_VALID_SMM
			      | KVM_VCPUEVENT_VALID_PAYLOAD))
J
Jan Kiszka 已提交
3747 3748
		return -EINVAL;

3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762
	if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
		if (!vcpu->kvm->arch.exception_payload_enabled)
			return -EINVAL;
		if (events->exception.pending)
			events->exception.injected = 0;
		else
			events->exception_has_payload = 0;
	} else {
		events->exception.pending = 0;
		events->exception_has_payload = 0;
	}

	if ((events->exception.injected || events->exception.pending) &&
	    (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
3763 3764
		return -EINVAL;

3765 3766 3767 3768 3769 3770
	/* INITs are latched while in SMM */
	if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
	    (events->smi.smm || events->smi.pending) &&
	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
		return -EINVAL;

A
Avi Kivity 已提交
3771
	process_nmi(vcpu);
3772 3773
	vcpu->arch.exception.injected = events->exception.injected;
	vcpu->arch.exception.pending = events->exception.pending;
J
Jan Kiszka 已提交
3774 3775 3776
	vcpu->arch.exception.nr = events->exception.nr;
	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
	vcpu->arch.exception.error_code = events->exception.error_code;
3777 3778
	vcpu->arch.exception.has_payload = events->exception_has_payload;
	vcpu->arch.exception.payload = events->exception_payload;
J
Jan Kiszka 已提交
3779

3780
	vcpu->arch.interrupt.injected = events->interrupt.injected;
J
Jan Kiszka 已提交
3781 3782
	vcpu->arch.interrupt.nr = events->interrupt.nr;
	vcpu->arch.interrupt.soft = events->interrupt.soft;
3783 3784 3785
	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
		kvm_x86_ops->set_interrupt_shadow(vcpu,
						  events->interrupt.shadow);
J
Jan Kiszka 已提交
3786 3787

	vcpu->arch.nmi_injected = events->nmi.injected;
3788 3789
	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
		vcpu->arch.nmi_pending = events->nmi.pending;
J
Jan Kiszka 已提交
3790 3791
	kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);

3792
	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
3793
	    lapic_in_kernel(vcpu))
3794
		vcpu->arch.apic->sipi_vector = events->sipi_vector;
J
Jan Kiszka 已提交
3795

3796
	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
3797 3798 3799 3800 3801 3802 3803
		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
			if (events->smi.smm)
				vcpu->arch.hflags |= HF_SMM_MASK;
			else
				vcpu->arch.hflags &= ~HF_SMM_MASK;
			kvm_smm_changed(vcpu);
		}
3804

3805
		vcpu->arch.smi_pending = events->smi.pending;
3806 3807 3808 3809

		if (events->smi.smm) {
			if (events->smi.smm_inside_nmi)
				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
3810
			else
3811 3812 3813 3814 3815 3816 3817
				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
			if (lapic_in_kernel(vcpu)) {
				if (events->smi.latched_init)
					set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
				else
					clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
			}
3818 3819 3820
		}
	}

3821 3822
	kvm_make_request(KVM_REQ_EVENT, vcpu);

J
Jan Kiszka 已提交
3823 3824 3825
	return 0;
}

3826 3827 3828
static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
					     struct kvm_debugregs *dbgregs)
{
J
Jan Kiszka 已提交
3829 3830
	unsigned long val;

3831
	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
3832
	kvm_get_dr(vcpu, 6, &val);
J
Jan Kiszka 已提交
3833
	dbgregs->dr6 = val;
3834 3835
	dbgregs->dr7 = vcpu->arch.dr7;
	dbgregs->flags = 0;
3836
	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
3837 3838 3839 3840 3841 3842 3843 3844
}

static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
					    struct kvm_debugregs *dbgregs)
{
	if (dbgregs->flags)
		return -EINVAL;

3845 3846 3847 3848 3849
	if (dbgregs->dr6 & ~0xffffffffull)
		return -EINVAL;
	if (dbgregs->dr7 & ~0xffffffffull)
		return -EINVAL;

3850
	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
3851
	kvm_update_dr0123(vcpu);
3852
	vcpu->arch.dr6 = dbgregs->dr6;
J
Jan Kiszka 已提交
3853
	kvm_update_dr6(vcpu);
3854
	vcpu->arch.dr7 = dbgregs->dr7;
3855
	kvm_update_dr7(vcpu);
3856 3857 3858 3859

	return 0;
}

3860 3861 3862 3863
#define XSTATE_COMPACTION_ENABLED (1ULL << 63)

static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu)
{
3864
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
3865
	u64 xstate_bv = xsave->header.xfeatures;
3866 3867 3868 3869 3870 3871 3872 3873 3874
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(dest, xsave, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV */
3875
	xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;
3876 3877 3878 3879 3880 3881
	*(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv;

	/*
	 * Copy each region from the possibly compacted offset to the
	 * non-compacted offset.
	 */
D
Dave Hansen 已提交
3882
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
3883
	while (valid) {
3884 3885 3886
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *src = get_xsave_addr(xsave, xfeature_nr);
3887 3888 3889

		if (src) {
			u32 size, offset, ecx, edx;
3890
			cpuid_count(XSTATE_CPUID, xfeature_nr,
3891
				    &size, &offset, &ecx, &edx);
3892
			if (xfeature_nr == XFEATURE_PKRU)
3893 3894 3895 3896 3897
				memcpy(dest + offset, &vcpu->arch.pkru,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest + offset, src, size);

3898 3899
		}

3900
		valid -= xfeature_mask;
3901 3902 3903 3904 3905
	}
}

static void load_xsave(struct kvm_vcpu *vcpu, u8 *src)
{
3906
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
3907 3908 3909 3910 3911 3912 3913 3914 3915 3916
	u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET);
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(xsave, src, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV and possibly XCOMP_BV.  */
3917
	xsave->header.xfeatures = xstate_bv;
3918
	if (boot_cpu_has(X86_FEATURE_XSAVES))
3919
		xsave->header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED;
3920 3921 3922 3923 3924

	/*
	 * Copy each region from the non-compacted offset to the
	 * possibly compacted offset.
	 */
D
Dave Hansen 已提交
3925
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
3926
	while (valid) {
3927 3928 3929
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *dest = get_xsave_addr(xsave, xfeature_nr);
3930 3931 3932

		if (dest) {
			u32 size, offset, ecx, edx;
3933
			cpuid_count(XSTATE_CPUID, xfeature_nr,
3934
				    &size, &offset, &ecx, &edx);
3935
			if (xfeature_nr == XFEATURE_PKRU)
3936 3937 3938 3939
				memcpy(&vcpu->arch.pkru, src + offset,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest, src + offset, size);
3940
		}
3941

3942
		valid -= xfeature_mask;
3943 3944 3945
	}
}

3946 3947 3948
static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
					 struct kvm_xsave *guest_xsave)
{
3949
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
3950 3951
		memset(guest_xsave, 0, sizeof(struct kvm_xsave));
		fill_xsave((u8 *) guest_xsave->region, vcpu);
3952
	} else {
3953
		memcpy(guest_xsave->region,
3954
			&vcpu->arch.guest_fpu->state.fxsave,
3955
			sizeof(struct fxregs_state));
3956
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
D
Dave Hansen 已提交
3957
			XFEATURE_MASK_FPSSE;
3958 3959 3960
	}
}

3961 3962
#define XSAVE_MXCSR_OFFSET 24

3963 3964 3965 3966 3967
static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
					struct kvm_xsave *guest_xsave)
{
	u64 xstate_bv =
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
3968
	u32 mxcsr = *(u32 *)&guest_xsave->region[XSAVE_MXCSR_OFFSET / sizeof(u32)];
3969

3970
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
3971 3972 3973 3974 3975
		/*
		 * Here we allow setting states that are not present in
		 * CPUID leaf 0xD, index 0, EDX:EAX.  This is for compatibility
		 * with old userspace.
		 */
3976 3977
		if (xstate_bv & ~kvm_supported_xcr0() ||
			mxcsr & ~mxcsr_feature_mask)
3978
			return -EINVAL;
3979
		load_xsave(vcpu, (u8 *)guest_xsave->region);
3980
	} else {
3981 3982
		if (xstate_bv & ~XFEATURE_MASK_FPSSE ||
			mxcsr & ~mxcsr_feature_mask)
3983
			return -EINVAL;
3984
		memcpy(&vcpu->arch.guest_fpu->state.fxsave,
3985
			guest_xsave->region, sizeof(struct fxregs_state));
3986 3987 3988 3989 3990 3991 3992
	}
	return 0;
}

static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
					struct kvm_xcrs *guest_xcrs)
{
3993
	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008
		guest_xcrs->nr_xcrs = 0;
		return;
	}

	guest_xcrs->nr_xcrs = 1;
	guest_xcrs->flags = 0;
	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
}

static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
				       struct kvm_xcrs *guest_xcrs)
{
	int i, r = 0;

4009
	if (!boot_cpu_has(X86_FEATURE_XSAVE))
4010 4011 4012 4013 4014 4015 4016
		return -EINVAL;

	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
		return -EINVAL;

	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
		/* Only support XCR0 currently */
P
Paolo Bonzini 已提交
4017
		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
4018
			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
P
Paolo Bonzini 已提交
4019
				guest_xcrs->xcrs[i].value);
4020 4021 4022 4023 4024 4025 4026
			break;
		}
	if (r)
		r = -EINVAL;
	return r;
}

4027 4028 4029 4030 4031 4032 4033 4034
/*
 * kvm_set_guest_paused() indicates to the guest kernel that it has been
 * stopped by the hypervisor.  This function will be called from the host only.
 * EINVAL is returned when the host attempts to set the flag for a guest that
 * does not support pv clocks.
 */
static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
{
4035
	if (!vcpu->arch.pv_time_enabled)
4036
		return -EINVAL;
4037
	vcpu->arch.pvclock_set_guest_stopped_request = true;
4038 4039 4040 4041
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	return 0;
}

4042 4043 4044
static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
				     struct kvm_enable_cap *cap)
{
4045 4046 4047 4048
	int r;
	uint16_t vmcs_version;
	void __user *user_ptr;

4049 4050 4051 4052
	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
4053 4054 4055
	case KVM_CAP_HYPERV_SYNIC2:
		if (cap->args[0])
			return -EINVAL;
4056 4057
		/* fall through */

4058
	case KVM_CAP_HYPERV_SYNIC:
4059 4060
		if (!irqchip_in_kernel(vcpu->kvm))
			return -EINVAL;
4061 4062
		return kvm_hv_activate_synic(vcpu, cap->cap ==
					     KVM_CAP_HYPERV_SYNIC2);
4063
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
4064 4065
		if (!kvm_x86_ops->nested_enable_evmcs)
			return -ENOTTY;
4066 4067 4068 4069 4070 4071 4072 4073
		r = kvm_x86_ops->nested_enable_evmcs(vcpu, &vmcs_version);
		if (!r) {
			user_ptr = (void __user *)(uintptr_t)cap->args[0];
			if (copy_to_user(user_ptr, &vmcs_version,
					 sizeof(vmcs_version)))
				r = -EFAULT;
		}
		return r;
4074 4075 4076 4077 4078
	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
		if (!kvm_x86_ops->enable_direct_tlbflush)
			return -ENOTTY;

		return kvm_x86_ops->enable_direct_tlbflush(vcpu);
4079

4080 4081 4082 4083 4084
	default:
		return -EINVAL;
	}
}

4085 4086 4087 4088 4089 4090
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r;
4091 4092 4093 4094 4095 4096 4097
	union {
		struct kvm_lapic_state *lapic;
		struct kvm_xsave *xsave;
		struct kvm_xcrs *xcrs;
		void *buffer;
	} u;

4098 4099
	vcpu_load(vcpu);

4100
	u.buffer = NULL;
4101 4102
	switch (ioctl) {
	case KVM_GET_LAPIC: {
4103
		r = -EINVAL;
4104
		if (!lapic_in_kernel(vcpu))
4105
			goto out;
4106 4107
		u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
				GFP_KERNEL_ACCOUNT);
4108

4109
		r = -ENOMEM;
4110
		if (!u.lapic)
4111
			goto out;
4112
		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
4113 4114 4115
		if (r)
			goto out;
		r = -EFAULT;
4116
		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
4117 4118 4119 4120 4121
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
4122
		r = -EINVAL;
4123
		if (!lapic_in_kernel(vcpu))
4124
			goto out;
4125
		u.lapic = memdup_user(argp, sizeof(*u.lapic));
4126 4127 4128 4129
		if (IS_ERR(u.lapic)) {
			r = PTR_ERR(u.lapic);
			goto out_nofree;
		}
4130

4131
		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
4132 4133
		break;
	}
4134 4135 4136 4137
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
4138
		if (copy_from_user(&irq, argp, sizeof(irq)))
4139 4140 4141 4142
			goto out;
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
		break;
	}
4143 4144 4145 4146
	case KVM_NMI: {
		r = kvm_vcpu_ioctl_nmi(vcpu);
		break;
	}
4147 4148 4149 4150
	case KVM_SMI: {
		r = kvm_vcpu_ioctl_smi(vcpu);
		break;
	}
4151 4152 4153 4154 4155
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
4156
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4157 4158 4159 4160
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		break;
	}
4161 4162 4163 4164 4165
	case KVM_SET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4166
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4167 4168
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
4169
					      cpuid_arg->entries);
4170 4171 4172 4173 4174 4175 4176
		break;
	}
	case KVM_GET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4177
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4178 4179
			goto out;
		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
4180
					      cpuid_arg->entries);
4181 4182 4183
		if (r)
			goto out;
		r = -EFAULT;
4184
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4185 4186 4187 4188
			goto out;
		r = 0;
		break;
	}
4189 4190
	case KVM_GET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4191
		r = msr_io(vcpu, argp, do_get_msr, 1);
4192
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4193
		break;
4194 4195 4196
	}
	case KVM_SET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4197
		r = msr_io(vcpu, argp, do_set_msr, 0);
4198
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4199
		break;
4200
	}
4201 4202 4203 4204
	case KVM_TPR_ACCESS_REPORTING: {
		struct kvm_tpr_access_ctl tac;

		r = -EFAULT;
4205
		if (copy_from_user(&tac, argp, sizeof(tac)))
4206 4207 4208 4209 4210
			goto out;
		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
		if (r)
			goto out;
		r = -EFAULT;
4211
		if (copy_to_user(argp, &tac, sizeof(tac)))
4212 4213 4214 4215
			goto out;
		r = 0;
		break;
	};
A
Avi Kivity 已提交
4216 4217
	case KVM_SET_VAPIC_ADDR: {
		struct kvm_vapic_addr va;
4218
		int idx;
A
Avi Kivity 已提交
4219 4220

		r = -EINVAL;
4221
		if (!lapic_in_kernel(vcpu))
A
Avi Kivity 已提交
4222 4223
			goto out;
		r = -EFAULT;
4224
		if (copy_from_user(&va, argp, sizeof(va)))
A
Avi Kivity 已提交
4225
			goto out;
4226
		idx = srcu_read_lock(&vcpu->kvm->srcu);
4227
		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
4228
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
A
Avi Kivity 已提交
4229 4230
		break;
	}
H
Huang Ying 已提交
4231 4232 4233 4234
	case KVM_X86_SETUP_MCE: {
		u64 mcg_cap;

		r = -EFAULT;
4235
		if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
H
Huang Ying 已提交
4236 4237 4238 4239 4240 4241 4242 4243
			goto out;
		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
		break;
	}
	case KVM_X86_SET_MCE: {
		struct kvm_x86_mce mce;

		r = -EFAULT;
4244
		if (copy_from_user(&mce, argp, sizeof(mce)))
H
Huang Ying 已提交
4245 4246 4247 4248
			goto out;
		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
		break;
	}
J
Jan Kiszka 已提交
4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);

		r = -EFAULT;
		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		r = -EFAULT;
		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
			break;

		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
		break;
	}
4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292
	case KVM_GET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);

		r = -EFAULT;
		if (copy_to_user(argp, &dbgregs,
				 sizeof(struct kvm_debugregs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		r = -EFAULT;
		if (copy_from_user(&dbgregs, argp,
				   sizeof(struct kvm_debugregs)))
			break;

		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
		break;
	}
4293
	case KVM_GET_XSAVE: {
4294
		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
4295
		r = -ENOMEM;
4296
		if (!u.xsave)
4297 4298
			break;

4299
		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
4300 4301

		r = -EFAULT;
4302
		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
4303 4304 4305 4306 4307
			break;
		r = 0;
		break;
	}
	case KVM_SET_XSAVE: {
4308
		u.xsave = memdup_user(argp, sizeof(*u.xsave));
4309 4310 4311 4312
		if (IS_ERR(u.xsave)) {
			r = PTR_ERR(u.xsave);
			goto out_nofree;
		}
4313

4314
		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
4315 4316 4317
		break;
	}
	case KVM_GET_XCRS: {
4318
		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
4319
		r = -ENOMEM;
4320
		if (!u.xcrs)
4321 4322
			break;

4323
		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
4324 4325

		r = -EFAULT;
4326
		if (copy_to_user(argp, u.xcrs,
4327 4328 4329 4330 4331 4332
				 sizeof(struct kvm_xcrs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XCRS: {
4333
		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
4334 4335 4336 4337
		if (IS_ERR(u.xcrs)) {
			r = PTR_ERR(u.xcrs);
			goto out_nofree;
		}
4338

4339
		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
4340 4341
		break;
	}
4342 4343 4344 4345 4346 4347 4348 4349 4350
	case KVM_SET_TSC_KHZ: {
		u32 user_tsc_khz;

		r = -EINVAL;
		user_tsc_khz = (u32)arg;

		if (user_tsc_khz >= kvm_max_guest_tsc_khz)
			goto out;

4351 4352 4353
		if (user_tsc_khz == 0)
			user_tsc_khz = tsc_khz;

4354 4355
		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
			r = 0;
4356 4357 4358 4359

		goto out;
	}
	case KVM_GET_TSC_KHZ: {
4360
		r = vcpu->arch.virtual_tsc_khz;
4361 4362
		goto out;
	}
4363 4364 4365 4366
	case KVM_KVMCLOCK_CTRL: {
		r = kvm_set_guest_paused(vcpu);
		goto out;
	}
4367 4368 4369 4370 4371 4372 4373 4374 4375
	case KVM_ENABLE_CAP: {
		struct kvm_enable_cap cap;

		r = -EFAULT;
		if (copy_from_user(&cap, argp, sizeof(cap)))
			goto out;
		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
		break;
	}
4376 4377 4378 4379 4380 4381 4382 4383 4384
	case KVM_GET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		u32 user_data_size;

		r = -EINVAL;
		if (!kvm_x86_ops->get_nested_state)
			break;

		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
4385
		r = -EFAULT;
4386
		if (get_user(user_data_size, &user_kvm_nested_state->size))
4387
			break;
4388 4389 4390 4391

		r = kvm_x86_ops->get_nested_state(vcpu, user_kvm_nested_state,
						  user_data_size);
		if (r < 0)
4392
			break;
4393 4394 4395

		if (r > user_data_size) {
			if (put_user(r, &user_kvm_nested_state->size))
4396 4397 4398 4399
				r = -EFAULT;
			else
				r = -E2BIG;
			break;
4400
		}
4401

4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412
		r = 0;
		break;
	}
	case KVM_SET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		struct kvm_nested_state kvm_state;

		r = -EINVAL;
		if (!kvm_x86_ops->set_nested_state)
			break;

4413
		r = -EFAULT;
4414
		if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
4415
			break;
4416

4417
		r = -EINVAL;
4418
		if (kvm_state.size < sizeof(kvm_state))
4419
			break;
4420 4421

		if (kvm_state.flags &
4422 4423
		    ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
		      | KVM_STATE_NESTED_EVMCS))
4424
			break;
4425 4426

		/* nested_run_pending implies guest_mode.  */
4427 4428
		if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
		    && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
4429
			break;
4430 4431 4432 4433

		r = kvm_x86_ops->set_nested_state(vcpu, user_kvm_nested_state, &kvm_state);
		break;
	}
4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452
	case KVM_GET_SUPPORTED_HV_CPUID: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
			goto out;

		r = kvm_vcpu_ioctl_get_hv_cpuid(vcpu, &cpuid,
						cpuid_arg->entries);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
			goto out;
		r = 0;
		break;
	}
4453 4454 4455 4456
	default:
		r = -EINVAL;
	}
out:
4457
	kfree(u.buffer);
4458 4459
out_nofree:
	vcpu_put(vcpu);
4460 4461 4462
	return r;
}

4463
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4464 4465 4466 4467
{
	return VM_FAULT_SIGBUS;
}

4468 4469 4470 4471 4472
static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
{
	int ret;

	if (addr > (unsigned int)(-3 * PAGE_SIZE))
4473
		return -EINVAL;
4474 4475 4476 4477
	ret = kvm_x86_ops->set_tss_addr(kvm, addr);
	return ret;
}

4478 4479 4480
static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
					      u64 ident_addr)
{
4481
	return kvm_x86_ops->set_identity_map_addr(kvm, ident_addr);
4482 4483
}

4484
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
4485
					 unsigned long kvm_nr_mmu_pages)
4486 4487 4488 4489
{
	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
		return -EINVAL;

4490
	mutex_lock(&kvm->slots_lock);
4491 4492

	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
4493
	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
4494

4495
	mutex_unlock(&kvm->slots_lock);
4496 4497 4498
	return 0;
}

4499
static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
4500
{
4501
	return kvm->arch.n_max_mmu_pages;
4502 4503 4504 4505
}

static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4506
	struct kvm_pic *pic = kvm->arch.vpic;
4507 4508 4509 4510 4511
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4512
		memcpy(&chip->chip.pic, &pic->pics[0],
4513 4514 4515
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4516
		memcpy(&chip->chip.pic, &pic->pics[1],
4517 4518 4519
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_IOAPIC:
4520
		kvm_get_ioapic(kvm, &chip->chip.ioapic);
4521 4522 4523 4524 4525 4526 4527 4528 4529 4530
		break;
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4531
	struct kvm_pic *pic = kvm->arch.vpic;
4532 4533 4534 4535 4536
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4537 4538
		spin_lock(&pic->lock);
		memcpy(&pic->pics[0], &chip->chip.pic,
4539
			sizeof(struct kvm_pic_state));
4540
		spin_unlock(&pic->lock);
4541 4542
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4543 4544
		spin_lock(&pic->lock);
		memcpy(&pic->pics[1], &chip->chip.pic,
4545
			sizeof(struct kvm_pic_state));
4546
		spin_unlock(&pic->lock);
4547 4548
		break;
	case KVM_IRQCHIP_IOAPIC:
4549
		kvm_set_ioapic(kvm, &chip->chip.ioapic);
4550 4551 4552 4553 4554
		break;
	default:
		r = -EINVAL;
		break;
	}
4555
	kvm_pic_update_irq(pic);
4556 4557 4558
	return r;
}

4559 4560
static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4561 4562 4563 4564 4565 4566 4567
	struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;

	BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));

	mutex_lock(&kps->lock);
	memcpy(ps, &kps->channels, sizeof(*ps));
	mutex_unlock(&kps->lock);
4568
	return 0;
4569 4570 4571 4572
}

static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4573
	int i;
4574 4575 4576
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
4577
	memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
4578
	for (i = 0; i < 3; i++)
4579 4580
		kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
	mutex_unlock(&pit->pit_state.lock);
4581
	return 0;
B
Beth Kon 已提交
4582 4583 4584 4585 4586 4587 4588 4589 4590
}

static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
		sizeof(ps->channels));
	ps->flags = kvm->arch.vpit->pit_state.flags;
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
4591
	memset(&ps->reserved, 0, sizeof(ps->reserved));
4592
	return 0;
B
Beth Kon 已提交
4593 4594 4595 4596
}

static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
4597
	int start = 0;
4598
	int i;
B
Beth Kon 已提交
4599
	u32 prev_legacy, cur_legacy;
4600 4601 4602 4603
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
	prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
B
Beth Kon 已提交
4604 4605 4606
	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
	if (!prev_legacy && cur_legacy)
		start = 1;
4607 4608 4609
	memcpy(&pit->pit_state.channels, &ps->channels,
	       sizeof(pit->pit_state.channels));
	pit->pit_state.flags = ps->flags;
4610
	for (i = 0; i < 3; i++)
4611
		kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
4612
				   start && i == 0);
4613
	mutex_unlock(&pit->pit_state.lock);
4614
	return 0;
4615 4616
}

4617 4618 4619
static int kvm_vm_ioctl_reinject(struct kvm *kvm,
				 struct kvm_reinject_control *control)
{
4620 4621 4622
	struct kvm_pit *pit = kvm->arch.vpit;

	if (!pit)
4623
		return -ENXIO;
4624

4625 4626 4627 4628 4629 4630 4631
	/* pit->pit_state.lock was overloaded to prevent userspace from getting
	 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
	 * ioctls in parallel.  Use a separate lock if that ioctl isn't rare.
	 */
	mutex_lock(&pit->pit_state.lock);
	kvm_pit_set_reinject(pit, control->pit_reinject);
	mutex_unlock(&pit->pit_state.lock);
4632

4633 4634 4635
	return 0;
}

4636
/**
4637 4638 4639
 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
 * @kvm: kvm instance
 * @log: slot id and address to which we copy the log
4640
 *
4641 4642 4643 4644 4645 4646 4647 4648
 * Steps 1-4 below provide general overview of dirty page logging. See
 * kvm_get_dirty_log_protect() function description for additional details.
 *
 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
 * always flush the TLB (step 4) even if previous step failed  and the dirty
 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
 * does not preclude user space subsequent dirty log read. Flushing TLB ensures
 * writes will be marked dirty for next log read.
4649
 *
4650 4651
 *   1. Take a snapshot of the bit and clear it if needed.
 *   2. Write protect the corresponding page.
4652 4653
 *   3. Copy the snapshot to the userspace.
 *   4. Flush TLB's if needed.
4654
 */
4655
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
4656
{
4657
	bool flush = false;
4658
	int r;
4659

4660
	mutex_lock(&kvm->slots_lock);
4661

4662 4663 4664 4665 4666 4667
	/*
	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
	 */
	if (kvm_x86_ops->flush_log_dirty)
		kvm_x86_ops->flush_log_dirty(kvm);

4668
	r = kvm_get_dirty_log_protect(kvm, log, &flush);
4669 4670 4671 4672 4673

	/*
	 * All the TLBs can be flushed out of mmu lock, see the comments in
	 * kvm_mmu_slot_remove_write_access().
	 */
4674
	lockdep_assert_held(&kvm->slots_lock);
4675
	if (flush)
4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702
		kvm_flush_remote_tlbs(kvm);

	mutex_unlock(&kvm->slots_lock);
	return r;
}

int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log)
{
	bool flush = false;
	int r;

	mutex_lock(&kvm->slots_lock);

	/*
	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
	 */
	if (kvm_x86_ops->flush_log_dirty)
		kvm_x86_ops->flush_log_dirty(kvm);

	r = kvm_clear_dirty_log_protect(kvm, log, &flush);

	/*
	 * All the TLBs can be flushed out of mmu lock, see the comments in
	 * kvm_mmu_slot_remove_write_access().
	 */
	lockdep_assert_held(&kvm->slots_lock);
	if (flush)
4703 4704
		kvm_flush_remote_tlbs(kvm);

4705
	mutex_unlock(&kvm->slots_lock);
4706 4707 4708
	return r;
}

4709 4710
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
			bool line_status)
4711 4712 4713 4714 4715
{
	if (!irqchip_in_kernel(kvm))
		return -ENXIO;

	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
4716 4717
					irq_event->irq, irq_event->level,
					line_status);
4718 4719 4720
	return 0;
}

4721 4722
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
			    struct kvm_enable_cap *cap)
4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733
{
	int r;

	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
	case KVM_CAP_DISABLE_QUIRKS:
		kvm->arch.disabled_quirks = cap->args[0];
		r = 0;
		break;
4734 4735
	case KVM_CAP_SPLIT_IRQCHIP: {
		mutex_lock(&kvm->lock);
4736 4737 4738
		r = -EINVAL;
		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
			goto split_irqchip_unlock;
4739 4740 4741
		r = -EEXIST;
		if (irqchip_in_kernel(kvm))
			goto split_irqchip_unlock;
P
Paolo Bonzini 已提交
4742
		if (kvm->created_vcpus)
4743 4744
			goto split_irqchip_unlock;
		r = kvm_setup_empty_irq_routing(kvm);
4745
		if (r)
4746 4747 4748
			goto split_irqchip_unlock;
		/* Pairs with irqchip_in_kernel. */
		smp_wmb();
4749
		kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
4750
		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
4751 4752 4753 4754 4755
		r = 0;
split_irqchip_unlock:
		mutex_unlock(&kvm->lock);
		break;
	}
4756 4757 4758 4759 4760 4761 4762
	case KVM_CAP_X2APIC_API:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
			break;

		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
			kvm->arch.x2apic_format = true;
4763 4764
		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
			kvm->arch.x2apic_broadcast_quirk_disabled = true;
4765 4766 4767

		r = 0;
		break;
4768 4769 4770 4771 4772 4773 4774 4775
	case KVM_CAP_X86_DISABLE_EXITS:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
			break;

		if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
			kvm_can_mwait_in_guest())
			kvm->arch.mwait_in_guest = true;
M
Michael S. Tsirkin 已提交
4776
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
4777
			kvm->arch.hlt_in_guest = true;
4778 4779
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
			kvm->arch.pause_in_guest = true;
4780 4781
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
			kvm->arch.cstate_in_guest = true;
4782 4783
		r = 0;
		break;
4784 4785 4786
	case KVM_CAP_MSR_PLATFORM_INFO:
		kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
		r = 0;
4787 4788 4789 4790
		break;
	case KVM_CAP_EXCEPTION_PAYLOAD:
		kvm->arch.exception_payload_enabled = cap->args[0];
		r = 0;
4791
		break;
4792 4793 4794 4795 4796 4797 4798
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

4799 4800 4801 4802 4803
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
4804
	int r = -ENOTTY;
4805 4806 4807 4808 4809 4810 4811
	/*
	 * This union makes it completely explicit to gcc-3.x
	 * that these two variables' stack usage should be
	 * combined, not added together.
	 */
	union {
		struct kvm_pit_state ps;
B
Beth Kon 已提交
4812
		struct kvm_pit_state2 ps2;
4813
		struct kvm_pit_config pit_config;
4814
	} u;
4815 4816 4817 4818 4819

	switch (ioctl) {
	case KVM_SET_TSS_ADDR:
		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
		break;
4820 4821 4822
	case KVM_SET_IDENTITY_MAP_ADDR: {
		u64 ident_addr;

4823 4824 4825 4826
		mutex_lock(&kvm->lock);
		r = -EINVAL;
		if (kvm->created_vcpus)
			goto set_identity_unlock;
4827
		r = -EFAULT;
4828
		if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
4829
			goto set_identity_unlock;
4830
		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
4831 4832
set_identity_unlock:
		mutex_unlock(&kvm->lock);
4833 4834
		break;
	}
4835 4836 4837 4838 4839 4840
	case KVM_SET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
		break;
	case KVM_GET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
		break;
4841 4842
	case KVM_CREATE_IRQCHIP: {
		mutex_lock(&kvm->lock);
4843

4844
		r = -EEXIST;
4845
		if (irqchip_in_kernel(kvm))
4846
			goto create_irqchip_unlock;
4847

4848
		r = -EINVAL;
P
Paolo Bonzini 已提交
4849
		if (kvm->created_vcpus)
4850
			goto create_irqchip_unlock;
4851 4852 4853

		r = kvm_pic_init(kvm);
		if (r)
4854
			goto create_irqchip_unlock;
4855 4856 4857 4858

		r = kvm_ioapic_init(kvm);
		if (r) {
			kvm_pic_destroy(kvm);
4859
			goto create_irqchip_unlock;
4860 4861
		}

4862 4863
		r = kvm_setup_default_irq_routing(kvm);
		if (r) {
4864
			kvm_ioapic_destroy(kvm);
4865
			kvm_pic_destroy(kvm);
4866
			goto create_irqchip_unlock;
4867
		}
4868
		/* Write kvm->irq_routing before enabling irqchip_in_kernel. */
4869
		smp_wmb();
4870
		kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
4871 4872
	create_irqchip_unlock:
		mutex_unlock(&kvm->lock);
4873
		break;
4874
	}
S
Sheng Yang 已提交
4875
	case KVM_CREATE_PIT:
4876 4877 4878 4879 4880 4881 4882 4883
		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
		goto create_pit;
	case KVM_CREATE_PIT2:
		r = -EFAULT;
		if (copy_from_user(&u.pit_config, argp,
				   sizeof(struct kvm_pit_config)))
			goto out;
	create_pit:
4884
		mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
4885 4886 4887
		r = -EEXIST;
		if (kvm->arch.vpit)
			goto create_pit_unlock;
S
Sheng Yang 已提交
4888
		r = -ENOMEM;
4889
		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
S
Sheng Yang 已提交
4890 4891
		if (kvm->arch.vpit)
			r = 0;
A
Avi Kivity 已提交
4892
	create_pit_unlock:
4893
		mutex_unlock(&kvm->lock);
S
Sheng Yang 已提交
4894
		break;
4895 4896
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
4897
		struct kvm_irqchip *chip;
4898

4899 4900 4901
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
4902
			goto out;
4903 4904
		}

4905
		r = -ENXIO;
4906
		if (!irqchip_kernel(kvm))
4907 4908
			goto get_irqchip_out;
		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
4909
		if (r)
4910
			goto get_irqchip_out;
4911
		r = -EFAULT;
4912
		if (copy_to_user(argp, chip, sizeof(*chip)))
4913
			goto get_irqchip_out;
4914
		r = 0;
4915 4916
	get_irqchip_out:
		kfree(chip);
4917 4918 4919 4920
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
4921
		struct kvm_irqchip *chip;
4922

4923 4924 4925
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
4926
			goto out;
4927 4928
		}

4929
		r = -ENXIO;
4930
		if (!irqchip_kernel(kvm))
4931 4932
			goto set_irqchip_out;
		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
4933
		if (r)
4934
			goto set_irqchip_out;
4935
		r = 0;
4936 4937
	set_irqchip_out:
		kfree(chip);
4938 4939
		break;
	}
4940 4941
	case KVM_GET_PIT: {
		r = -EFAULT;
4942
		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
4943 4944 4945 4946
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
4947
		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
4948 4949 4950
		if (r)
			goto out;
		r = -EFAULT;
4951
		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
4952 4953 4954 4955 4956 4957
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT: {
		r = -EFAULT;
4958
		if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
4959 4960 4961 4962
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
4963
		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
4964 4965
		break;
	}
B
Beth Kon 已提交
4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988
	case KVM_GET_PIT2: {
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT2: {
		r = -EFAULT;
		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
		break;
	}
4989 4990 4991 4992 4993 4994 4995 4996
	case KVM_REINJECT_CONTROL: {
		struct kvm_reinject_control control;
		r =  -EFAULT;
		if (copy_from_user(&control, argp, sizeof(control)))
			goto out;
		r = kvm_vm_ioctl_reinject(kvm, &control);
		break;
	}
4997 4998 4999
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
		mutex_lock(&kvm->lock);
P
Paolo Bonzini 已提交
5000
		if (kvm->created_vcpus)
5001 5002 5003 5004 5005
			r = -EBUSY;
		else
			kvm->arch.bsp_vcpu_id = arg;
		mutex_unlock(&kvm->lock);
		break;
E
Ed Swierk 已提交
5006
	case KVM_XEN_HVM_CONFIG: {
5007
		struct kvm_xen_hvm_config xhc;
E
Ed Swierk 已提交
5008
		r = -EFAULT;
5009
		if (copy_from_user(&xhc, argp, sizeof(xhc)))
E
Ed Swierk 已提交
5010 5011
			goto out;
		r = -EINVAL;
5012
		if (xhc.flags)
E
Ed Swierk 已提交
5013
			goto out;
5014
		memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));
E
Ed Swierk 已提交
5015 5016 5017
		r = 0;
		break;
	}
5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030
	case KVM_SET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

		r = -EFAULT;
		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
			goto out;

		r = -EINVAL;
		if (user_ns.flags)
			goto out;

		r = 0;
5031 5032 5033 5034 5035 5036
		/*
		 * TODO: userspace has to take care of races with VCPU_RUN, so
		 * kvm_gen_update_masterclock() can be cut down to locked
		 * pvclock_update_vm_gtod_copy().
		 */
		kvm_gen_update_masterclock(kvm);
5037
		now_ns = get_kvmclock_ns(kvm);
5038
		kvm->arch.kvmclock_offset += user_ns.clock - now_ns;
5039
		kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE);
5040 5041 5042 5043 5044 5045
		break;
	}
	case KVM_GET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

5046
		now_ns = get_kvmclock_ns(kvm);
5047
		user_ns.clock = now_ns;
5048
		user_ns.flags = kvm->arch.use_master_clock ? KVM_CLOCK_TSC_STABLE : 0;
5049
		memset(&user_ns.pad, 0, sizeof(user_ns.pad));
5050 5051 5052 5053 5054 5055 5056

		r = -EFAULT;
		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
			goto out;
		r = 0;
		break;
	}
5057 5058 5059 5060 5061 5062
	case KVM_MEMORY_ENCRYPT_OP: {
		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_op)
			r = kvm_x86_ops->mem_enc_op(kvm, argp);
		break;
	}
5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086
	case KVM_MEMORY_ENCRYPT_REG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_reg_region)
			r = kvm_x86_ops->mem_enc_reg_region(kvm, &region);
		break;
	}
	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_unreg_region)
			r = kvm_x86_ops->mem_enc_unreg_region(kvm, &region);
		break;
	}
5087 5088 5089 5090 5091 5092 5093 5094 5095
	case KVM_HYPERV_EVENTFD: {
		struct kvm_hyperv_eventfd hvevfd;

		r = -EFAULT;
		if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
			goto out;
		r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
		break;
	}
E
Eric Hankland 已提交
5096 5097 5098
	case KVM_SET_PMU_EVENT_FILTER:
		r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
		break;
5099
	default:
5100
		r = -ENOTTY;
5101 5102 5103 5104 5105
	}
out:
	return r;
}

5106
static void kvm_init_msr_list(void)
5107 5108 5109 5110
{
	u32 dummy[2];
	unsigned i, j;

5111 5112 5113 5114 5115
	BUILD_BUG_ON_MSG(INTEL_PMC_MAX_FIXED != 4,
			 "Please update the fixed PMCs in msrs_to_save[]");
	BUILD_BUG_ON_MSG(INTEL_PMC_MAX_GENERIC != 32,
			 "Please update the generic perfctr/eventsel MSRs in msrs_to_save[]");

5116
	for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
5117 5118
		if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
			continue;
5119 5120 5121

		/*
		 * Even MSRs that are valid in the host may not be exposed
5122
		 * to the guests in some cases.
5123 5124 5125
		 */
		switch (msrs_to_save[i]) {
		case MSR_IA32_BNDCFGS:
5126
			if (!kvm_mpx_supported())
5127 5128
				continue;
			break;
5129 5130 5131 5132
		case MSR_TSC_AUX:
			if (!kvm_x86_ops->rdtscp_supported())
				continue;
			break;
5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156
		case MSR_IA32_RTIT_CTL:
		case MSR_IA32_RTIT_STATUS:
			if (!kvm_x86_ops->pt_supported())
				continue;
			break;
		case MSR_IA32_RTIT_CR3_MATCH:
			if (!kvm_x86_ops->pt_supported() ||
			    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
				continue;
			break;
		case MSR_IA32_RTIT_OUTPUT_BASE:
		case MSR_IA32_RTIT_OUTPUT_MASK:
			if (!kvm_x86_ops->pt_supported() ||
				(!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
				 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
				continue;
			break;
		case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: {
			if (!kvm_x86_ops->pt_supported() ||
				msrs_to_save[i] - MSR_IA32_RTIT_ADDR0_A >=
				intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
				continue;
			break;
		}
5157 5158 5159 5160
		default:
			break;
		}

5161 5162 5163 5164 5165
		if (j < i)
			msrs_to_save[j] = msrs_to_save[i];
		j++;
	}
	num_msrs_to_save = j;
5166 5167

	for (i = j = 0; i < ARRAY_SIZE(emulated_msrs); i++) {
5168 5169
		if (!kvm_x86_ops->has_emulated_msr(emulated_msrs[i]))
			continue;
5170 5171 5172 5173 5174 5175

		if (j < i)
			emulated_msrs[j] = emulated_msrs[i];
		j++;
	}
	num_emulated_msrs = j;
5176 5177 5178 5179 5180

	for (i = j = 0; i < ARRAY_SIZE(msr_based_features); i++) {
		struct kvm_msr_entry msr;

		msr.index = msr_based_features[i];
5181
		if (kvm_get_msr_feature(&msr))
5182 5183 5184 5185 5186 5187 5188
			continue;

		if (j < i)
			msr_based_features[j] = msr_based_features[i];
		j++;
	}
	num_msr_based_features = j;
5189 5190
}

5191 5192
static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
			   const void *v)
5193
{
5194 5195 5196 5197 5198
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5199
		if (!(lapic_in_kernel(vcpu) &&
5200 5201
		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
5202 5203 5204 5205 5206 5207
			break;
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5208

5209
	return handled;
5210 5211
}

5212
static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
5213
{
5214 5215 5216 5217 5218
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5219
		if (!(lapic_in_kernel(vcpu) &&
5220 5221 5222
		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
					 addr, n, v))
		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
5223
			break;
5224
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
5225 5226 5227 5228 5229
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5230

5231
	return handled;
5232 5233
}

5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245
static void kvm_set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
	kvm_x86_ops->set_segment(vcpu, var, seg);
}

void kvm_get_segment(struct kvm_vcpu *vcpu,
		     struct kvm_segment *var, int seg)
{
	kvm_x86_ops->get_segment(vcpu, var, seg);
}

5246 5247
gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
			   struct x86_exception *exception)
5248 5249 5250 5251 5252 5253 5254
{
	gpa_t t_gpa;

	BUG_ON(!mmu_is_nested(vcpu));

	/* NPT walks are always user-walks */
	access |= PFERR_USER_MASK;
5255
	t_gpa  = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception);
5256 5257 5258 5259

	return t_gpa;
}

5260 5261
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
			      struct x86_exception *exception)
5262 5263
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5264
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5265 5266
}

5267 5268
 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5269 5270 5271
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_FETCH_MASK;
5272
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5273 5274
}

5275 5276
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
			       struct x86_exception *exception)
5277 5278 5279
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_WRITE_MASK;
5280
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5281 5282 5283
}

/* uses this to access any guest's mapped memory without checking CPL */
5284 5285
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5286
{
5287
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
5288 5289 5290 5291
}

static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
5292
				      struct x86_exception *exception)
5293 5294
{
	void *data = val;
5295
	int r = X86EMUL_CONTINUE;
5296 5297

	while (bytes) {
5298
		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
5299
							    exception);
5300
		unsigned offset = addr & (PAGE_SIZE-1);
5301
		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
5302 5303
		int ret;

5304
		if (gpa == UNMAPPED_GVA)
5305
			return X86EMUL_PROPAGATE_FAULT;
5306 5307
		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
					       offset, toread);
5308
		if (ret < 0) {
5309
			r = X86EMUL_IO_NEEDED;
5310 5311
			goto out;
		}
5312

5313 5314 5315
		bytes -= toread;
		data += toread;
		addr += toread;
5316
	}
5317 5318
out:
	return r;
5319
}
5320

5321
/* used for instruction fetching */
5322 5323
static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
				gva_t addr, void *val, unsigned int bytes,
5324
				struct x86_exception *exception)
5325
{
5326
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5327
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5328 5329
	unsigned offset;
	int ret;
5330

5331 5332 5333 5334 5335 5336 5337 5338 5339
	/* Inline kvm_read_guest_virt_helper for speed.  */
	gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,
						    exception);
	if (unlikely(gpa == UNMAPPED_GVA))
		return X86EMUL_PROPAGATE_FAULT;

	offset = addr & (PAGE_SIZE-1);
	if (WARN_ON(offset + bytes > PAGE_SIZE))
		bytes = (unsigned)PAGE_SIZE - offset;
5340 5341
	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
				       offset, bytes);
5342 5343 5344 5345
	if (unlikely(ret < 0))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
5346 5347
}

5348
int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
5349
			       gva_t addr, void *val, unsigned int bytes,
5350
			       struct x86_exception *exception)
5351 5352
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5353

5354 5355 5356 5357 5358 5359 5360
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5361
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
5362
					  exception);
5363
}
5364
EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
5365

5366 5367
static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
			     gva_t addr, void *val, unsigned int bytes,
5368
			     struct x86_exception *exception, bool system)
5369
{
5370
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5371 5372 5373 5374 5375 5376
	u32 access = 0;

	if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
		access |= PFERR_USER_MASK;

	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
5377 5378
}

5379 5380 5381 5382 5383 5384 5385 5386 5387
static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
		unsigned long addr, void *val, unsigned int bytes)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);

	return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
}

5388 5389 5390
static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
				      struct x86_exception *exception)
5391 5392 5393 5394 5395
{
	void *data = val;
	int r = X86EMUL_CONTINUE;

	while (bytes) {
5396
		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
5397
							     access,
5398
							     exception);
5399 5400 5401 5402
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
		int ret;

5403
		if (gpa == UNMAPPED_GVA)
5404
			return X86EMUL_PROPAGATE_FAULT;
5405
		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
5406
		if (ret < 0) {
5407
			r = X86EMUL_IO_NEEDED;
5408 5409 5410 5411 5412 5413 5414 5415 5416 5417
			goto out;
		}

		bytes -= towrite;
		data += towrite;
		addr += towrite;
	}
out:
	return r;
}
5418 5419

static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
5420 5421
			      unsigned int bytes, struct x86_exception *exception,
			      bool system)
5422 5423
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5424 5425 5426 5427
	u32 access = PFERR_WRITE_MASK;

	if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
		access |= PFERR_USER_MASK;
5428 5429

	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
5430
					   access, exception);
5431 5432 5433 5434 5435
}

int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
				unsigned int bytes, struct x86_exception *exception)
{
P
Paolo Bonzini 已提交
5436 5437 5438
	/* kvm_write_guest_virt_system can pull in tons of pages. */
	vcpu->arch.l1tf_flush_l1d = true;

5439 5440 5441 5442 5443 5444 5445
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5446 5447 5448
	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
					   PFERR_WRITE_MASK, exception);
}
N
Nadav Har'El 已提交
5449
EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
5450

W
Wanpeng Li 已提交
5451 5452
int handle_ud(struct kvm_vcpu *vcpu)
{
5453 5454 5455 5456 5457
	int emul_type = EMULTYPE_TRAP_UD;
	char sig[5]; /* ud2; .ascii "kvm" */
	struct x86_exception e;

	if (force_emulation_prefix &&
5458 5459
	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
				sig, sizeof(sig), &e) == 0 &&
5460 5461
	    memcmp(sig, "\xf\xbkvm", sizeof(sig)) == 0) {
		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
5462
		emul_type = EMULTYPE_TRAP_UD_FORCED;
5463
	}
W
Wanpeng Li 已提交
5464

5465
	return kvm_emulate_instruction(vcpu, emul_type);
W
Wanpeng Li 已提交
5466 5467 5468
}
EXPORT_SYMBOL_GPL(handle_ud);

5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483
static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
			    gpa_t gpa, bool write)
{
	/* For APIC access vmexit */
	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		return 1;

	if (vcpu_match_mmio_gpa(vcpu, gpa)) {
		trace_vcpu_match_mmio(gva, gpa, write, true);
		return 1;
	}

	return 0;
}

5484 5485 5486 5487
static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
				gpa_t *gpa, struct x86_exception *exception,
				bool write)
{
5488 5489
	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
		| (write ? PFERR_WRITE_MASK : 0);
5490

5491 5492 5493 5494 5495
	/*
	 * currently PKRU is only applied to ept enabled guest so
	 * there is no pkey in EPT page table for L1 guest or EPT
	 * shadow page table for L2 guest.
	 */
5496
	if (vcpu_match_mmio_gva(vcpu, gva)
F
Feng Wu 已提交
5497
	    && !permission_fault(vcpu, vcpu->arch.walk_mmu,
5498
				 vcpu->arch.mmio_access, 0, access)) {
5499 5500
		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
					(gva & (PAGE_SIZE - 1));
X
Xiao Guangrong 已提交
5501
		trace_vcpu_match_mmio(gva, *gpa, write, false);
5502 5503 5504
		return 1;
	}

5505 5506 5507 5508 5509
	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);

	if (*gpa == UNMAPPED_GVA)
		return -1;

5510
	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
5511 5512
}

5513
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
5514
			const void *val, int bytes)
5515 5516 5517
{
	int ret;

5518
	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
5519
	if (ret < 0)
5520
		return 0;
5521
	kvm_page_track_write(vcpu, gpa, val, bytes);
5522 5523 5524
	return 1;
}

5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540
struct read_write_emulator_ops {
	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
				  int bytes);
	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
				  void *val, int bytes);
	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
			       int bytes, void *val);
	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
				    void *val, int bytes);
	bool write;
};

static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
{
	if (vcpu->mmio_read_completed) {
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
5541
			       vcpu->mmio_fragments[0].gpa, val);
5542 5543 5544 5545 5546 5547 5548 5549 5550 5551
		vcpu->mmio_read_completed = 0;
		return 1;
	}

	return 0;
}

static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			void *val, int bytes)
{
5552
	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
5553 5554 5555 5556 5557 5558 5559 5560 5561 5562
}

static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			 void *val, int bytes)
{
	return emulator_write_phys(vcpu, gpa, val, bytes);
}

static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
{
5563
	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
5564 5565 5566 5567 5568 5569
	return vcpu_mmio_write(vcpu, gpa, bytes, val);
}

static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			  void *val, int bytes)
{
5570
	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
5571 5572 5573 5574 5575 5576
	return X86EMUL_IO_NEEDED;
}

static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			   void *val, int bytes)
{
A
Avi Kivity 已提交
5577 5578
	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];

5579
	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
5580 5581 5582
	return X86EMUL_CONTINUE;
}

5583
static const struct read_write_emulator_ops read_emultor = {
5584 5585 5586 5587 5588 5589
	.read_write_prepare = read_prepare,
	.read_write_emulate = read_emulate,
	.read_write_mmio = vcpu_mmio_read,
	.read_write_exit_mmio = read_exit_mmio,
};

5590
static const struct read_write_emulator_ops write_emultor = {
5591 5592 5593 5594 5595 5596
	.read_write_emulate = write_emulate,
	.read_write_mmio = write_mmio,
	.read_write_exit_mmio = write_exit_mmio,
	.write = true,
};

5597 5598 5599 5600
static int emulator_read_write_onepage(unsigned long addr, void *val,
				       unsigned int bytes,
				       struct x86_exception *exception,
				       struct kvm_vcpu *vcpu,
5601
				       const struct read_write_emulator_ops *ops)
5602
{
5603 5604
	gpa_t gpa;
	int handled, ret;
5605
	bool write = ops->write;
A
Avi Kivity 已提交
5606
	struct kvm_mmio_fragment *frag;
5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;

	/*
	 * If the exit was due to a NPF we may already have a GPA.
	 * If the GPA is present, use it to avoid the GVA to GPA table walk.
	 * Note, this cannot be used on string operations since string
	 * operation using rep will only have the initial GPA from the NPF
	 * occurred.
	 */
	if (vcpu->arch.gpa_available &&
	    emulator_can_use_gpa(ctxt) &&
5618 5619 5620 5621 5622 5623 5624
	    (addr & ~PAGE_MASK) == (vcpu->arch.gpa_val & ~PAGE_MASK)) {
		gpa = vcpu->arch.gpa_val;
		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
	} else {
		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
		if (ret < 0)
			return X86EMUL_PROPAGATE_FAULT;
5625
	}
5626

5627
	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
5628 5629 5630 5631 5632
		return X86EMUL_CONTINUE;

	/*
	 * Is this MMIO handled locally?
	 */
5633
	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
5634
	if (handled == bytes)
5635 5636
		return X86EMUL_CONTINUE;

5637 5638 5639 5640
	gpa += handled;
	bytes -= handled;
	val += handled;

5641 5642 5643 5644 5645
	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
	frag->gpa = gpa;
	frag->data = val;
	frag->len = bytes;
A
Avi Kivity 已提交
5646
	return X86EMUL_CONTINUE;
5647 5648
}

5649 5650
static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
			unsigned long addr,
5651 5652
			void *val, unsigned int bytes,
			struct x86_exception *exception,
5653
			const struct read_write_emulator_ops *ops)
5654
{
5655
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
A
Avi Kivity 已提交
5656 5657 5658 5659 5660 5661 5662 5663
	gpa_t gpa;
	int rc;

	if (ops->read_write_prepare &&
		  ops->read_write_prepare(vcpu, val, bytes))
		return X86EMUL_CONTINUE;

	vcpu->mmio_nr_fragments = 0;
5664

5665 5666
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
A
Avi Kivity 已提交
5667
		int now;
5668 5669

		now = -addr & ~PAGE_MASK;
5670 5671 5672
		rc = emulator_read_write_onepage(addr, val, now, exception,
						 vcpu, ops);

5673 5674 5675
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
5676 5677
		if (ctxt->mode != X86EMUL_MODE_PROT64)
			addr = (u32)addr;
5678 5679 5680
		val += now;
		bytes -= now;
	}
5681

A
Avi Kivity 已提交
5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694
	rc = emulator_read_write_onepage(addr, val, bytes, exception,
					 vcpu, ops);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	if (!vcpu->mmio_nr_fragments)
		return rc;

	gpa = vcpu->mmio_fragments[0].gpa;

	vcpu->mmio_needed = 1;
	vcpu->mmio_cur_fragment = 0;

5695
	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
A
Avi Kivity 已提交
5696 5697 5698 5699 5700
	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
	vcpu->run->exit_reason = KVM_EXIT_MMIO;
	vcpu->run->mmio.phys_addr = gpa;

	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712
}

static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
				  unsigned long addr,
				  void *val,
				  unsigned int bytes,
				  struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, val, bytes,
				   exception, &read_emultor);
}

5713
static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
5714 5715 5716 5717 5718 5719 5720
			    unsigned long addr,
			    const void *val,
			    unsigned int bytes,
			    struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, (void *)val, bytes,
				   exception, &write_emultor);
5721 5722
}

5723 5724 5725 5726 5727 5728 5729
#define CMPXCHG_TYPE(t, ptr, old, new) \
	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))

#ifdef CONFIG_X86_64
#  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
#else
#  define CMPXCHG64(ptr, old, new) \
5730
	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
5731 5732
#endif

5733 5734
static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
				     unsigned long addr,
5735 5736 5737
				     const void *old,
				     const void *new,
				     unsigned int bytes,
5738
				     struct x86_exception *exception)
5739
{
5740
	struct kvm_host_map map;
5741
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5742 5743 5744
	gpa_t gpa;
	char *kaddr;
	bool exchanged;
5745

5746 5747 5748
	/* guests cmpxchg8b have to be emulated atomically */
	if (bytes > 8 || (bytes & (bytes - 1)))
		goto emul_write;
5749

5750
	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
5751

5752 5753 5754
	if (gpa == UNMAPPED_GVA ||
	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		goto emul_write;
5755

5756 5757
	if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
		goto emul_write;
5758

5759
	if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))
5760
		goto emul_write;
5761

5762 5763
	kaddr = map.hva + offset_in_page(gpa);

5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778
	switch (bytes) {
	case 1:
		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
		break;
	case 2:
		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
		break;
	case 4:
		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
		break;
	case 8:
		exchanged = CMPXCHG64(kaddr, old, new);
		break;
	default:
		BUG();
5779
	}
5780 5781

	kvm_vcpu_unmap(vcpu, &map, true);
5782 5783 5784 5785

	if (!exchanged)
		return X86EMUL_CMPXCHG_FAILED;

5786
	kvm_page_track_write(vcpu, gpa, new, bytes);
5787 5788

	return X86EMUL_CONTINUE;
5789

5790
emul_write:
5791
	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
5792

5793
	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
5794 5795
}

5796 5797
static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
{
5798
	int r = 0, i;
5799

5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811
	for (i = 0; i < vcpu->arch.pio.count; i++) {
		if (vcpu->arch.pio.in)
			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
					    vcpu->arch.pio.size, pd);
		else
			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
					     vcpu->arch.pio.port, vcpu->arch.pio.size,
					     pd);
		if (r)
			break;
		pd += vcpu->arch.pio.size;
	}
5812 5813 5814
	return r;
}

5815 5816 5817
static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
			       unsigned short port, void *val,
			       unsigned int count, bool in)
5818 5819
{
	vcpu->arch.pio.port = port;
5820
	vcpu->arch.pio.in = in;
5821
	vcpu->arch.pio.count  = count;
5822 5823 5824
	vcpu->arch.pio.size = size;

	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
5825
		vcpu->arch.pio.count = 0;
5826 5827 5828 5829
		return 1;
	}

	vcpu->run->exit_reason = KVM_EXIT_IO;
5830
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
5831 5832 5833 5834 5835 5836 5837 5838
	vcpu->run->io.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = count;
	vcpu->run->io.port = port;

	return 0;
}

5839 5840 5841
static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
				    int size, unsigned short port, void *val,
				    unsigned int count)
5842
{
5843
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5844
	int ret;
5845

5846 5847
	if (vcpu->arch.pio.count)
		goto data_avail;
5848

5849 5850
	memset(vcpu->arch.pio_data, 0, size * count);

5851 5852 5853 5854
	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
	if (ret) {
data_avail:
		memcpy(val, vcpu->arch.pio_data, size * count);
5855
		trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
5856
		vcpu->arch.pio.count = 0;
5857 5858 5859 5860 5861 5862
		return 1;
	}

	return 0;
}

5863 5864 5865 5866 5867 5868 5869
static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
				     int size, unsigned short port,
				     const void *val, unsigned int count)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	memcpy(vcpu->arch.pio_data, val, size * count);
5870
	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
5871 5872 5873
	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
}

5874 5875 5876 5877 5878
static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	return kvm_x86_ops->get_segment_base(vcpu, seg);
}

5879
static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
5880
{
5881
	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
5882 5883
}

5884
static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
5885 5886 5887 5888 5889
{
	if (!need_emulate_wbinvd(vcpu))
		return X86EMUL_CONTINUE;

	if (kvm_x86_ops->has_wbinvd_exit()) {
5890 5891 5892
		int cpu = get_cpu();

		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
5893 5894
		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
				wbinvd_ipi, NULL, 1);
5895
		put_cpu();
5896
		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
5897 5898
	} else
		wbinvd();
5899 5900
	return X86EMUL_CONTINUE;
}
5901 5902 5903

int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
5904 5905
	kvm_emulate_wbinvd_noskip(vcpu);
	return kvm_skip_emulated_instruction(vcpu);
5906
}
5907 5908
EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);

5909 5910


5911 5912
static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
{
5913
	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
5914 5915
}

5916 5917
static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long *dest)
5918
{
5919
	return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
5920 5921
}

5922 5923
static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long value)
5924
{
5925

5926
	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
5927 5928
}

5929
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
5930
{
5931
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
5932 5933
}

5934
static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
5935
{
5936
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5937 5938 5939 5940 5941 5942 5943 5944 5945 5946
	unsigned long value;

	switch (cr) {
	case 0:
		value = kvm_read_cr0(vcpu);
		break;
	case 2:
		value = vcpu->arch.cr2;
		break;
	case 3:
5947
		value = kvm_read_cr3(vcpu);
5948 5949 5950 5951 5952 5953 5954 5955
		break;
	case 4:
		value = kvm_read_cr4(vcpu);
		break;
	case 8:
		value = kvm_get_cr8(vcpu);
		break;
	default:
5956
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
5957 5958 5959 5960 5961 5962
		return 0;
	}

	return value;
}

5963
static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
5964
{
5965
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5966 5967
	int res = 0;

5968 5969
	switch (cr) {
	case 0:
5970
		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
5971 5972 5973 5974 5975
		break;
	case 2:
		vcpu->arch.cr2 = val;
		break;
	case 3:
5976
		res = kvm_set_cr3(vcpu, val);
5977 5978
		break;
	case 4:
5979
		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
5980 5981
		break;
	case 8:
A
Andre Przywara 已提交
5982
		res = kvm_set_cr8(vcpu, val);
5983 5984
		break;
	default:
5985
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
5986
		res = -1;
5987
	}
5988 5989

	return res;
5990 5991
}

5992
static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
5993
{
5994
	return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
5995 5996
}

5997
static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
5998
{
5999
	kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
6000 6001
}

6002
static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6003
{
6004
	kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
6005 6006
}

6007 6008 6009 6010 6011 6012 6013 6014 6015 6016
static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
}

static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
}

6017 6018
static unsigned long emulator_get_cached_segment_base(
	struct x86_emulate_ctxt *ctxt, int seg)
6019
{
6020
	return get_segment_base(emul_to_vcpu(ctxt), seg);
6021 6022
}

6023 6024 6025
static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
				 struct desc_struct *desc, u32 *base3,
				 int seg)
6026 6027 6028
{
	struct kvm_segment var;

6029
	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
6030
	*selector = var.selector;
6031

6032 6033
	if (var.unusable) {
		memset(desc, 0, sizeof(*desc));
6034 6035
		if (base3)
			*base3 = 0;
6036
		return false;
6037
	}
6038 6039 6040 6041 6042

	if (var.g)
		var.limit >>= 12;
	set_desc_limit(desc, var.limit);
	set_desc_base(desc, (unsigned long)var.base);
6043 6044 6045 6046
#ifdef CONFIG_X86_64
	if (base3)
		*base3 = var.base >> 32;
#endif
6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058
	desc->type = var.type;
	desc->s = var.s;
	desc->dpl = var.dpl;
	desc->p = var.present;
	desc->avl = var.avl;
	desc->l = var.l;
	desc->d = var.db;
	desc->g = var.g;

	return true;
}

6059 6060 6061
static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
				 struct desc_struct *desc, u32 base3,
				 int seg)
6062
{
6063
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6064 6065
	struct kvm_segment var;

6066
	var.selector = selector;
6067
	var.base = get_desc_base(desc);
6068 6069 6070
#ifdef CONFIG_X86_64
	var.base |= ((u64)base3) << 32;
#endif
6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088
	var.limit = get_desc_limit(desc);
	if (desc->g)
		var.limit = (var.limit << 12) | 0xfff;
	var.type = desc->type;
	var.dpl = desc->dpl;
	var.db = desc->d;
	var.s = desc->s;
	var.l = desc->l;
	var.g = desc->g;
	var.avl = desc->avl;
	var.present = desc->p;
	var.unusable = !var.present;
	var.padding = 0;

	kvm_set_segment(vcpu, &var, seg);
	return;
}

6089 6090 6091
static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 *pdata)
{
6092
	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
6093 6094 6095 6096 6097
}

static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 data)
{
6098
	return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
6099 6100
}

P
Paolo Bonzini 已提交
6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114
static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	return vcpu->arch.smbase;
}

static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	vcpu->arch.smbase = smbase;
}

6115 6116 6117
static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
			      u32 pmc)
{
6118
	return kvm_pmu_is_valid_msr_idx(emul_to_vcpu(ctxt), pmc);
6119 6120
}

6121 6122 6123
static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
			     u32 pmc, u64 *pdata)
{
6124
	return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
6125 6126
}

6127 6128 6129 6130 6131
static void emulator_halt(struct x86_emulate_ctxt *ctxt)
{
	emul_to_vcpu(ctxt)->arch.halt_request = 1;
}

6132
static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
6133
			      struct x86_instruction_info *info,
6134 6135
			      enum x86_intercept_stage stage)
{
6136
	return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
6137 6138
}

6139 6140
static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
			u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, bool check_limit)
6141
{
6142
	return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, check_limit);
6143 6144
}

6145 6146 6147 6148 6149 6150 6151 6152 6153 6154
static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
{
	return kvm_register_read(emul_to_vcpu(ctxt), reg);
}

static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
{
	kvm_register_write(emul_to_vcpu(ctxt), reg, val);
}

6155 6156 6157 6158 6159
static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
{
	kvm_x86_ops->set_nmi_mask(emul_to_vcpu(ctxt), masked);
}

6160 6161 6162 6163 6164 6165 6166
static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
{
	return emul_to_vcpu(ctxt)->arch.hflags;
}

static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_flags)
{
6167
	emul_to_vcpu(ctxt)->arch.hflags = emul_flags;
6168 6169
}

6170 6171
static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt,
				  const char *smstate)
6172
{
6173
	return kvm_x86_ops->pre_leave_smm(emul_to_vcpu(ctxt), smstate);
6174 6175
}

6176 6177 6178 6179 6180
static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt)
{
	kvm_smm_changed(emul_to_vcpu(ctxt));
}

6181 6182 6183 6184 6185
static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
{
	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
}

6186
static const struct x86_emulate_ops emulate_ops = {
6187 6188
	.read_gpr            = emulator_read_gpr,
	.write_gpr           = emulator_write_gpr,
6189 6190
	.read_std            = emulator_read_std,
	.write_std           = emulator_write_std,
6191
	.read_phys           = kvm_read_guest_phys_system,
6192
	.fetch               = kvm_fetch_guest_virt,
6193 6194 6195
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
6196
	.invlpg              = emulator_invlpg,
6197 6198
	.pio_in_emulated     = emulator_pio_in_emulated,
	.pio_out_emulated    = emulator_pio_out_emulated,
6199 6200
	.get_segment         = emulator_get_segment,
	.set_segment         = emulator_set_segment,
6201
	.get_cached_segment_base = emulator_get_cached_segment_base,
6202
	.get_gdt             = emulator_get_gdt,
6203
	.get_idt	     = emulator_get_idt,
6204 6205
	.set_gdt             = emulator_set_gdt,
	.set_idt	     = emulator_set_idt,
6206 6207
	.get_cr              = emulator_get_cr,
	.set_cr              = emulator_set_cr,
6208
	.cpl                 = emulator_get_cpl,
6209 6210
	.get_dr              = emulator_get_dr,
	.set_dr              = emulator_set_dr,
P
Paolo Bonzini 已提交
6211 6212
	.get_smbase          = emulator_get_smbase,
	.set_smbase          = emulator_set_smbase,
6213 6214
	.set_msr             = emulator_set_msr,
	.get_msr             = emulator_get_msr,
6215
	.check_pmc	     = emulator_check_pmc,
6216
	.read_pmc            = emulator_read_pmc,
6217
	.halt                = emulator_halt,
6218
	.wbinvd              = emulator_wbinvd,
6219
	.fix_hypercall       = emulator_fix_hypercall,
6220
	.intercept           = emulator_intercept,
6221
	.get_cpuid           = emulator_get_cpuid,
6222
	.set_nmi_mask        = emulator_set_nmi_mask,
6223 6224
	.get_hflags          = emulator_get_hflags,
	.set_hflags          = emulator_set_hflags,
6225
	.pre_leave_smm       = emulator_pre_leave_smm,
6226
	.post_leave_smm      = emulator_post_leave_smm,
6227
	.set_xcr             = emulator_set_xcr,
6228 6229
};

6230 6231
static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
{
6232
	u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
6233 6234 6235 6236 6237 6238 6239
	/*
	 * an sti; sti; sequence only disable interrupts for the first
	 * instruction. So, if the last instruction, be it emulated or
	 * not, left the system with the INT_STI flag enabled, it
	 * means that the last instruction is an sti. We should not
	 * leave the flag on in this case. The same goes for mov ss
	 */
6240 6241
	if (int_shadow & mask)
		mask = 0;
6242
	if (unlikely(int_shadow || mask)) {
6243
		kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
6244 6245 6246
		if (!mask)
			kvm_make_request(KVM_REQ_EVENT, vcpu);
	}
6247 6248
}

6249
static bool inject_emulated_exception(struct kvm_vcpu *vcpu)
6250 6251
{
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6252
	if (ctxt->exception.vector == PF_VECTOR)
6253 6254 6255
		return kvm_propagate_fault(vcpu, &ctxt->exception);

	if (ctxt->exception.error_code_valid)
6256 6257
		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
				      ctxt->exception.error_code);
6258
	else
6259
		kvm_queue_exception(vcpu, ctxt->exception.vector);
6260
	return false;
6261 6262
}

6263 6264
static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
{
6265
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6266 6267 6268 6269
	int cs_db, cs_l;

	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);

6270
	ctxt->eflags = kvm_get_rflags(vcpu);
6271 6272
	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;

6273 6274 6275
	ctxt->eip = kvm_rip_read(vcpu);
	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
6276
		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :
6277 6278
		     cs_db				? X86EMUL_MODE_PROT32 :
							  X86EMUL_MODE_PROT16;
6279
	BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK);
P
Paolo Bonzini 已提交
6280 6281
	BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK);
	BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK);
6282

6283
	init_decode_cache(ctxt);
6284
	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6285 6286
}

6287
void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
6288
{
6289
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6290 6291 6292 6293
	int ret;

	init_emulate_ctxt(vcpu);

6294 6295 6296
	ctxt->op_bytes = 2;
	ctxt->ad_bytes = 2;
	ctxt->_eip = ctxt->eip + inc_eip;
6297
	ret = emulate_int_real(ctxt, irq);
6298

6299 6300 6301 6302 6303 6304 6305
	if (ret != X86EMUL_CONTINUE) {
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
	} else {
		ctxt->eip = ctxt->_eip;
		kvm_rip_write(vcpu, ctxt->eip);
		kvm_set_rflags(vcpu, ctxt->eflags);
	}
6306 6307 6308
}
EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);

6309
static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
6310 6311 6312
{
	++vcpu->stat.insn_emulation_fail;
	trace_kvm_emulate_insn_failed(vcpu);
6313

6314 6315
	if (emulation_type & EMULTYPE_VMWARE_GP) {
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6316
		return 1;
6317
	}
6318

6319 6320 6321 6322
	if (emulation_type & EMULTYPE_SKIP) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6323
		return 0;
6324 6325
	}

6326 6327
	kvm_queue_exception(vcpu, UD_VECTOR);

6328
	if (!is_guest_mode(vcpu) && kvm_x86_ops->get_cpl(vcpu) == 0) {
6329 6330 6331
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6332
		return 0;
6333
	}
6334

6335
	return 1;
6336 6337
}

6338
static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,
6339 6340
				  bool write_fault_to_shadow_pgtable,
				  int emulation_type)
6341
{
6342
	gpa_t gpa = cr2;
D
Dan Williams 已提交
6343
	kvm_pfn_t pfn;
6344

6345
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
6346 6347
		return false;

6348 6349 6350
	if (WARN_ON_ONCE(is_guest_mode(vcpu)))
		return false;

6351
	if (!vcpu->arch.mmu->direct_map) {
6352 6353 6354 6355 6356
		/*
		 * Write permission should be allowed since only
		 * write access need to be emulated.
		 */
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
6357

6358 6359 6360 6361 6362 6363 6364
		/*
		 * If the mapping is invalid in guest, let cpu retry
		 * it to generate fault.
		 */
		if (gpa == UNMAPPED_GVA)
			return true;
	}
6365

6366 6367 6368 6369 6370 6371 6372
	/*
	 * Do not retry the unhandleable instruction if it faults on the
	 * readonly host memory, otherwise it will goto a infinite loop:
	 * retry instruction -> write #PF -> emulation fail -> retry
	 * instruction -> ...
	 */
	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383

	/*
	 * If the instruction failed on the error pfn, it can not be fixed,
	 * report the error to userspace.
	 */
	if (is_error_noslot_pfn(pfn))
		return false;

	kvm_release_pfn_clean(pfn);

	/* The instructions are well-emulated on direct mmu. */
6384
	if (vcpu->arch.mmu->direct_map) {
6385 6386 6387 6388 6389 6390 6391 6392 6393
		unsigned int indirect_shadow_pages;

		spin_lock(&vcpu->kvm->mmu_lock);
		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
		spin_unlock(&vcpu->kvm->mmu_lock);

		if (indirect_shadow_pages)
			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));

6394
		return true;
6395
	}
6396

6397 6398 6399 6400 6401 6402
	/*
	 * if emulation was due to access to shadowed page table
	 * and it failed try to unshadow page and re-enter the
	 * guest to let CPU execute the instruction.
	 */
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6403 6404 6405 6406 6407 6408 6409

	/*
	 * If the access faults on its page table, it can not
	 * be fixed by unprotecting shadow page and it should
	 * be reported to userspace.
	 */
	return !write_fault_to_shadow_pgtable;
6410 6411
}

6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435
static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
			      unsigned long cr2,  int emulation_type)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	unsigned long last_retry_eip, last_retry_addr, gpa = cr2;

	last_retry_eip = vcpu->arch.last_retry_eip;
	last_retry_addr = vcpu->arch.last_retry_addr;

	/*
	 * If the emulation is caused by #PF and it is non-page_table
	 * writing instruction, it means the VM-EXIT is caused by shadow
	 * page protected, we can zap the shadow page and retry this
	 * instruction directly.
	 *
	 * Note: if the guest uses a non-page-table modifying instruction
	 * on the PDE that points to the instruction, then we will unmap
	 * the instruction and go to an infinite loop. So, we cache the
	 * last retried eip and the last fault address, if we meet the eip
	 * and the address again, we can break out of the potential infinite
	 * loop.
	 */
	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;

6436
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
6437 6438
		return false;

6439 6440 6441
	if (WARN_ON_ONCE(is_guest_mode(vcpu)))
		return false;

6442 6443 6444 6445 6446 6447 6448 6449 6450
	if (x86_page_table_writing_insn(ctxt))
		return false;

	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2)
		return false;

	vcpu->arch.last_retry_eip = ctxt->eip;
	vcpu->arch.last_retry_addr = cr2;

6451
	if (!vcpu->arch.mmu->direct_map)
6452 6453
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);

6454
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6455 6456 6457 6458

	return true;
}

6459 6460 6461
static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
static int complete_emulated_pio(struct kvm_vcpu *vcpu);

P
Paolo Bonzini 已提交
6462
static void kvm_smm_changed(struct kvm_vcpu *vcpu)
6463
{
P
Paolo Bonzini 已提交
6464
	if (!(vcpu->arch.hflags & HF_SMM_MASK)) {
6465 6466 6467
		/* This is a good place to trace that we are exiting SMM.  */
		trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false);

6468 6469
		/* Process a latched INIT or SMI, if any.  */
		kvm_make_request(KVM_REQ_EVENT, vcpu);
P
Paolo Bonzini 已提交
6470
	}
6471 6472

	kvm_mmu_reset_context(vcpu);
P
Paolo Bonzini 已提交
6473 6474
}

6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489
static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
				unsigned long *db)
{
	u32 dr6 = 0;
	int i;
	u32 enable, rwlen;

	enable = dr7;
	rwlen = dr7 >> 16;
	for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
		if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
			dr6 |= (1 << i);
	return dr6;
}

6490
static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
6491 6492 6493
{
	struct kvm_run *kvm_run = vcpu->run;

6494 6495 6496 6497 6498
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
		kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1 | DR6_RTM;
		kvm_run->debug.arch.pc = vcpu->arch.singlestep_rip;
		kvm_run->debug.arch.exception = DB_VECTOR;
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
6499
		return 0;
6500
	}
6501
	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
6502
	return 1;
6503 6504
}

6505 6506 6507
int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
	unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6508
	int r;
6509

6510
	r = kvm_x86_ops->skip_emulated_instruction(vcpu);
6511
	if (unlikely(!r))
6512
		return 0;
6513 6514 6515 6516 6517 6518 6519 6520 6521 6522

	/*
	 * rflags is the old, "raw" value of the flags.  The new value has
	 * not been saved yet.
	 *
	 * This is correct even for TF set by the guest, because "the
	 * processor will not generate this exception after the instruction
	 * that sets the TF flag".
	 */
	if (unlikely(rflags & X86_EFLAGS_TF))
6523
		r = kvm_vcpu_do_singlestep(vcpu);
6524
	return r;
6525 6526 6527
}
EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);

6528 6529 6530 6531
static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
{
	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
6532 6533 6534
		struct kvm_run *kvm_run = vcpu->run;
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6535 6536 6537 6538
					   vcpu->arch.guest_debug_dr7,
					   vcpu->arch.eff_db);

		if (dr6 != 0) {
6539
			kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
6540
			kvm_run->debug.arch.pc = eip;
6541 6542
			kvm_run->debug.arch.exception = DB_VECTOR;
			kvm_run->exit_reason = KVM_EXIT_DEBUG;
6543
			*r = 0;
6544 6545 6546 6547
			return true;
		}
	}

6548 6549
	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
	    !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {
6550 6551
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6552 6553 6554 6555
					   vcpu->arch.dr7,
					   vcpu->arch.db);

		if (dr6 != 0) {
6556
			vcpu->arch.dr6 &= ~DR_TRAP_BITS;
6557
			vcpu->arch.dr6 |= dr6 | DR6_RTM;
6558
			kvm_queue_exception(vcpu, DB_VECTOR);
6559
			*r = 1;
6560 6561 6562 6563 6564 6565 6566
			return true;
		}
	}

	return false;
}

6567 6568
static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
{
6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592
	switch (ctxt->opcode_len) {
	case 1:
		switch (ctxt->b) {
		case 0xe4:	/* IN */
		case 0xe5:
		case 0xec:
		case 0xed:
		case 0xe6:	/* OUT */
		case 0xe7:
		case 0xee:
		case 0xef:
		case 0x6c:	/* INS */
		case 0x6d:
		case 0x6e:	/* OUTS */
		case 0x6f:
			return true;
		}
		break;
	case 2:
		switch (ctxt->b) {
		case 0x33:	/* RDPMC */
			return true;
		}
		break;
6593 6594 6595 6596 6597
	}

	return false;
}

6598 6599
int x86_emulate_instruction(struct kvm_vcpu *vcpu,
			    unsigned long cr2,
6600 6601 6602
			    int emulation_type,
			    void *insn,
			    int insn_len)
6603
{
6604
	int r;
6605
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6606
	bool writeback = true;
6607
	bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
6608

P
Paolo Bonzini 已提交
6609 6610
	vcpu->arch.l1tf_flush_l1d = true;

6611 6612 6613 6614 6615
	/*
	 * Clear write_fault_to_shadow_pgtable here to ensure it is
	 * never reused.
	 */
	vcpu->arch.write_fault_to_shadow_pgtable = false;
6616
	kvm_clear_exception_queue(vcpu);
G
Gleb Natapov 已提交
6617

6618
	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
6619
		init_emulate_ctxt(vcpu);
6620 6621 6622 6623 6624 6625 6626

		/*
		 * We will reenter on the same instruction since
		 * we do not set complete_userspace_io.  This does not
		 * handle watchpoints yet, those would be handled in
		 * the emulate_ops.
		 */
6627 6628
		if (!(emulation_type & EMULTYPE_SKIP) &&
		    kvm_vcpu_check_breakpoint(vcpu, &r))
6629 6630
			return r;

6631 6632
		ctxt->interruptibility = 0;
		ctxt->have_exception = false;
6633
		ctxt->exception.vector = -1;
6634
		ctxt->perm_ok = false;
6635

6636
		ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
6637

6638
		r = x86_decode_insn(ctxt, insn, insn_len);
6639

A
Avi Kivity 已提交
6640
		trace_kvm_emulate_insn_start(vcpu);
6641
		++vcpu->stat.insn_emulation;
6642
		if (r != EMULATION_OK)  {
6643
			if ((emulation_type & EMULTYPE_TRAP_UD) ||
6644 6645
			    (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
				kvm_queue_exception(vcpu, UD_VECTOR);
6646
				return 1;
6647
			}
6648 6649
			if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
						emulation_type))
6650
				return 1;
6651
			if (ctxt->have_exception) {
6652 6653 6654 6655 6656 6657
				/*
				 * #UD should result in just EMULATION_FAILED, and trap-like
				 * exception should not be encountered during decode.
				 */
				WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
					     exception_type(ctxt->exception.vector) == EXCPT_TRAP);
6658
				inject_emulated_exception(vcpu);
6659
				return 1;
6660
			}
6661
			return handle_emulation_failure(vcpu, emulation_type);
6662 6663 6664
		}
	}

6665 6666 6667
	if ((emulation_type & EMULTYPE_VMWARE_GP) &&
	    !is_vmware_backdoor_opcode(ctxt)) {
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6668
		return 1;
6669
	}
6670

6671 6672 6673 6674 6675
	/*
	 * Note, EMULTYPE_SKIP is intended for use *only* by vendor callbacks
	 * for kvm_skip_emulated_instruction().  The caller is responsible for
	 * updating interruptibility state and injecting single-step #DBs.
	 */
6676
	if (emulation_type & EMULTYPE_SKIP) {
6677
		kvm_rip_write(vcpu, ctxt->_eip);
6678 6679
		if (ctxt->eflags & X86_EFLAGS_RF)
			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
6680
		return 1;
6681 6682
	}

6683
	if (retry_instruction(ctxt, cr2, emulation_type))
6684
		return 1;
6685

6686
	/* this is needed for vmware backdoor interface to work since it
6687
	   changes registers values  during IO operation */
6688 6689
	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6690
		emulator_invalidate_register_cache(ctxt);
6691
	}
6692

6693
restart:
6694 6695 6696
	/* Save the faulting GPA (cr2) in the address field */
	ctxt->exception.address = cr2;

6697
	r = x86_emulate_insn(ctxt);
6698

6699
	if (r == EMULATION_INTERCEPTED)
6700
		return 1;
6701

6702
	if (r == EMULATION_FAILED) {
6703 6704
		if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
					emulation_type))
6705
			return 1;
6706

6707
		return handle_emulation_failure(vcpu, emulation_type);
6708 6709
	}

6710
	if (ctxt->have_exception) {
6711
		r = 1;
6712 6713
		if (inject_emulated_exception(vcpu))
			return r;
6714
	} else if (vcpu->arch.pio.count) {
6715 6716
		if (!vcpu->arch.pio.in) {
			/* FIXME: return into emulator if single-stepping.  */
6717
			vcpu->arch.pio.count = 0;
6718
		} else {
6719
			writeback = false;
6720 6721
			vcpu->arch.complete_userspace_io = complete_emulated_pio;
		}
6722
		r = 0;
6723
	} else if (vcpu->mmio_needed) {
6724 6725
		++vcpu->stat.mmio_exits;

6726 6727
		if (!vcpu->mmio_is_write)
			writeback = false;
6728
		r = 0;
6729
		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
6730
	} else if (r == EMULATION_RESTART)
6731
		goto restart;
6732
	else
6733
		r = 1;
6734

6735
	if (writeback) {
6736
		unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6737
		toggle_interruptibility(vcpu, ctxt->interruptibility);
6738
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
6739
		if (!ctxt->have_exception ||
6740 6741
		    exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
			kvm_rip_write(vcpu, ctxt->eip);
6742
			if (r && ctxt->tf)
6743
				r = kvm_vcpu_do_singlestep(vcpu);
6744
			__kvm_set_rflags(vcpu, ctxt->eflags);
6745
		}
6746 6747 6748 6749 6750 6751 6752 6753 6754

		/*
		 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
		 * do nothing, and it will be requested again as soon as
		 * the shadow expires.  But we still need to check here,
		 * because POPF has no interrupt shadow.
		 */
		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
			kvm_make_request(KVM_REQ_EVENT, vcpu);
6755 6756
	} else
		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
6757 6758

	return r;
6759
}
6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772

int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
{
	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction);

int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
					void *insn, int insn_len)
{
	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
6773

6774 6775 6776 6777 6778 6779
static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;
	return 1;
}

6780 6781 6782 6783 6784 6785 6786 6787 6788 6789
static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;

	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
		return 1;

	return kvm_skip_emulated_instruction(vcpu);
}

6790 6791
static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
			    unsigned short port)
6792
{
6793
	unsigned long val = kvm_rax_read(vcpu);
6794 6795
	int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
					    size, port, &val, 1);
6796 6797
	if (ret)
		return ret;
6798

6799 6800 6801 6802 6803 6804 6805 6806 6807 6808
	/*
	 * Workaround userspace that relies on old KVM behavior of %rip being
	 * incremented prior to exiting to userspace to handle "OUT 0x7e".
	 */
	if (port == 0x7e &&
	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
		vcpu->arch.complete_userspace_io =
			complete_fast_pio_out_port_0x7e;
		kvm_skip_emulated_instruction(vcpu);
	} else {
6809 6810 6811
		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
		vcpu->arch.complete_userspace_io = complete_fast_pio_out;
	}
6812
	return 0;
6813 6814
}

6815 6816 6817 6818 6819 6820 6821
static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
{
	unsigned long val;

	/* We should only ever be called with arch.pio.count equal to 1 */
	BUG_ON(vcpu->arch.pio.count != 1);

6822 6823 6824 6825 6826
	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
		vcpu->arch.pio.count = 0;
		return 1;
	}

6827
	/* For size less than 4 we merge, else we zero extend */
6828
	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
6829 6830 6831 6832 6833 6834 6835

	/*
	 * Since vcpu->arch.pio.count == 1 let emulator_pio_in_emulated perform
	 * the copy and tracing
	 */
	emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, vcpu->arch.pio.size,
				 vcpu->arch.pio.port, &val, 1);
6836
	kvm_rax_write(vcpu, val);
6837

6838
	return kvm_skip_emulated_instruction(vcpu);
6839 6840
}

6841 6842
static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
			   unsigned short port)
6843 6844 6845 6846 6847
{
	unsigned long val;
	int ret;

	/* For size less than 4 we merge, else we zero extend */
6848
	val = (size < 4) ? kvm_rax_read(vcpu) : 0;
6849 6850 6851 6852

	ret = emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, size, port,
				       &val, 1);
	if (ret) {
6853
		kvm_rax_write(vcpu, val);
6854 6855 6856
		return ret;
	}

6857
	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
6858 6859 6860 6861
	vcpu->arch.complete_userspace_io = complete_fast_pio_in;

	return 0;
}
6862 6863 6864

int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
{
6865
	int ret;
6866 6867

	if (in)
6868
		ret = kvm_fast_pio_in(vcpu, size, port);
6869
	else
6870 6871
		ret = kvm_fast_pio_out(vcpu, size, port);
	return ret && kvm_skip_emulated_instruction(vcpu);
6872 6873
}
EXPORT_SYMBOL_GPL(kvm_fast_pio);
6874

6875
static int kvmclock_cpu_down_prep(unsigned int cpu)
6876
{
T
Tejun Heo 已提交
6877
	__this_cpu_write(cpu_tsc_khz, 0);
6878
	return 0;
6879 6880 6881
}

static void tsc_khz_changed(void *data)
6882
{
6883 6884 6885 6886 6887 6888 6889 6890 6891
	struct cpufreq_freqs *freq = data;
	unsigned long khz = 0;

	if (data)
		khz = freq->new;
	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		khz = cpufreq_quick_get(raw_smp_processor_id());
	if (!khz)
		khz = tsc_khz;
T
Tejun Heo 已提交
6892
	__this_cpu_write(cpu_tsc_khz, khz);
6893 6894
}

6895
#ifdef CONFIG_X86_64
6896 6897 6898 6899 6900 6901
static void kvm_hyperv_tsc_notifier(void)
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int cpu;

J
Junaid Shahid 已提交
6902
	mutex_lock(&kvm_lock);
6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_make_mclock_inprogress_request(kvm);

	hyperv_stop_tsc_emulation();

	/* TSC frequency always matches when on Hyper-V */
	for_each_present_cpu(cpu)
		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
	kvm_max_guest_tsc_khz = tsc_khz;

	list_for_each_entry(kvm, &vm_list, vm_list) {
		struct kvm_arch *ka = &kvm->arch;

		spin_lock(&ka->pvclock_gtod_sync_lock);

		pvclock_update_vm_gtod_copy(kvm);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);

		spin_unlock(&ka->pvclock_gtod_sync_lock);
	}
J
Junaid Shahid 已提交
6928
	mutex_unlock(&kvm_lock);
6929
}
6930
#endif
6931

6932
static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
6933 6934 6935 6936 6937
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i, send_ipi = 0;

6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976
	/*
	 * We allow guests to temporarily run on slowing clocks,
	 * provided we notify them after, or to run on accelerating
	 * clocks, provided we notify them before.  Thus time never
	 * goes backwards.
	 *
	 * However, we have a problem.  We can't atomically update
	 * the frequency of a given CPU from this function; it is
	 * merely a notifier, which can be called from any CPU.
	 * Changing the TSC frequency at arbitrary points in time
	 * requires a recomputation of local variables related to
	 * the TSC for each VCPU.  We must flag these local variables
	 * to be updated and be sure the update takes place with the
	 * new frequency before any guests proceed.
	 *
	 * Unfortunately, the combination of hotplug CPU and frequency
	 * change creates an intractable locking scenario; the order
	 * of when these callouts happen is undefined with respect to
	 * CPU hotplug, and they can race with each other.  As such,
	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
	 * undefined; you can actually have a CPU frequency change take
	 * place in between the computation of X and the setting of the
	 * variable.  To protect against this problem, all updates of
	 * the per_cpu tsc_khz variable are done in an interrupt
	 * protected IPI, and all callers wishing to update the value
	 * must wait for a synchronous IPI to complete (which is trivial
	 * if the caller is on the CPU already).  This establishes the
	 * necessary total order on variable updates.
	 *
	 * Note that because a guest time update may take place
	 * anytime after the setting of the VCPU's request bit, the
	 * correct TSC value must be set before the request.  However,
	 * to ensure the update actually makes it to any guest which
	 * starts running in hardware virtualization between the set
	 * and the acquisition of the spinlock, we must also ping the
	 * CPU after setting the request bit.
	 *
	 */

6977
	smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
6978

J
Junaid Shahid 已提交
6979
	mutex_lock(&kvm_lock);
6980
	list_for_each_entry(kvm, &vm_list, vm_list) {
6981
		kvm_for_each_vcpu(i, vcpu, kvm) {
6982
			if (vcpu->cpu != cpu)
6983
				continue;
Z
Zachary Amsden 已提交
6984
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
J
Junaid Shahid 已提交
6985
			if (vcpu->cpu != raw_smp_processor_id())
6986
				send_ipi = 1;
6987 6988
		}
	}
J
Junaid Shahid 已提交
6989
	mutex_unlock(&kvm_lock);
6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003

	if (freq->old < freq->new && send_ipi) {
		/*
		 * We upscale the frequency.  Must make the guest
		 * doesn't see old kvmclock values while running with
		 * the new frequency, otherwise we risk the guest sees
		 * time go backwards.
		 *
		 * In case we update the frequency for another cpu
		 * (which might be in guest context) send an interrupt
		 * to kick the cpu out of guest context.  Next time
		 * guest context is entered kvmclock will be updated,
		 * so the guest will not see stale values.
		 */
7004
		smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
7005
	}
7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7017 7018 7019 7020 7021
}

static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
				     void *data)
{
	struct cpufreq_freqs *freq = data;
	int cpu;

	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
		return 0;
	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
		return 0;

	for_each_cpu(cpu, freq->policy->cpus)
		__kvmclock_cpufreq_notifier(freq, cpu);

7022 7023 7024 7025
	return 0;
}

static struct notifier_block kvmclock_cpufreq_notifier_block = {
7026 7027 7028
	.notifier_call  = kvmclock_cpufreq_notifier
};

7029
static int kvmclock_cpu_online(unsigned int cpu)
7030
{
7031 7032
	tsc_khz_changed(NULL);
	return 0;
7033 7034
}

7035 7036
static void kvm_timer_init(void)
{
Z
Zachary Amsden 已提交
7037
	max_tsc_khz = tsc_khz;
7038

7039
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
Z
Zachary Amsden 已提交
7040 7041
#ifdef CONFIG_CPU_FREQ
		struct cpufreq_policy policy;
7042 7043
		int cpu;

Z
Zachary Amsden 已提交
7044
		memset(&policy, 0, sizeof(policy));
7045 7046
		cpu = get_cpu();
		cpufreq_get_policy(&policy, cpu);
Z
Zachary Amsden 已提交
7047 7048
		if (policy.cpuinfo.max_freq)
			max_tsc_khz = policy.cpuinfo.max_freq;
7049
		put_cpu();
Z
Zachary Amsden 已提交
7050
#endif
7051 7052 7053
		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
					  CPUFREQ_TRANSITION_NOTIFIER);
	}
7054

T
Thomas Gleixner 已提交
7055
	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
7056
			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
7057 7058
}

7059 7060
DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu);
7061

7062
int kvm_is_in_guest(void)
7063
{
7064
	return __this_cpu_read(current_vcpu) != NULL;
7065 7066 7067 7068 7069
}

static int kvm_is_user_mode(void)
{
	int user_mode = 3;
7070

7071 7072
	if (__this_cpu_read(current_vcpu))
		user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
7073

7074 7075 7076 7077 7078 7079
	return user_mode != 0;
}

static unsigned long kvm_get_guest_ip(void)
{
	unsigned long ip = 0;
7080

7081 7082
	if (__this_cpu_read(current_vcpu))
		ip = kvm_rip_read(__this_cpu_read(current_vcpu));
7083

7084 7085 7086
	return ip;
}

L
Luwei Kang 已提交
7087 7088 7089 7090 7091 7092 7093 7094 7095
static void kvm_handle_intel_pt_intr(void)
{
	struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);

	kvm_make_request(KVM_REQ_PMI, vcpu);
	__set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
			(unsigned long *)&vcpu->arch.pmu.global_status);
}

7096 7097 7098 7099
static struct perf_guest_info_callbacks kvm_guest_cbs = {
	.is_in_guest		= kvm_is_in_guest,
	.is_user_mode		= kvm_is_user_mode,
	.get_guest_ip		= kvm_get_guest_ip,
L
Luwei Kang 已提交
7100
	.handle_intel_pt_intr	= kvm_handle_intel_pt_intr,
7101 7102
};

7103 7104 7105
#ifdef CONFIG_X86_64
static void pvclock_gtod_update_fn(struct work_struct *work)
{
7106 7107 7108 7109 7110
	struct kvm *kvm;

	struct kvm_vcpu *vcpu;
	int i;

J
Junaid Shahid 已提交
7111
	mutex_lock(&kvm_lock);
7112 7113
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_for_each_vcpu(i, vcpu, kvm)
7114
			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
7115
	atomic_set(&kvm_guest_has_master_clock, 0);
J
Junaid Shahid 已提交
7116
	mutex_unlock(&kvm_lock);
7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132
}

static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);

/*
 * Notification about pvclock gtod data update.
 */
static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
			       void *priv)
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	struct timekeeper *tk = priv;

	update_pvclock_gtod(tk);

	/* disable master clock if host does not trust, or does not
7133
	 * use, TSC based clocksource.
7134
	 */
7135
	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146
	    atomic_read(&kvm_guest_has_master_clock) != 0)
		queue_work(system_long_wq, &pvclock_gtod_work);

	return 0;
}

static struct notifier_block pvclock_gtod_notifier = {
	.notifier_call = pvclock_gtod_notify,
};
#endif

7147
int kvm_arch_init(void *opaque)
7148
{
7149
	int r;
M
Mathias Krause 已提交
7150
	struct kvm_x86_ops *ops = opaque;
7151 7152 7153

	if (kvm_x86_ops) {
		printk(KERN_ERR "kvm: already loaded the other module\n");
7154 7155
		r = -EEXIST;
		goto out;
7156 7157 7158 7159
	}

	if (!ops->cpu_has_kvm_support()) {
		printk(KERN_ERR "kvm: no hardware support\n");
7160 7161
		r = -EOPNOTSUPP;
		goto out;
7162 7163 7164
	}
	if (ops->disabled_by_bios()) {
		printk(KERN_ERR "kvm: disabled by bios\n");
7165 7166
		r = -EOPNOTSUPP;
		goto out;
7167 7168
	}

7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179
	/*
	 * KVM explicitly assumes that the guest has an FPU and
	 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
	 * vCPU's FPU state as a fxregs_state struct.
	 */
	if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
		printk(KERN_ERR "kvm: inadequate fpu\n");
		r = -EOPNOTSUPP;
		goto out;
	}

7180
	r = -ENOMEM;
7181
	x86_fpu_cache = kmem_cache_create("x86_fpu", sizeof(struct fpu),
7182 7183 7184 7185 7186 7187 7188
					  __alignof__(struct fpu), SLAB_ACCOUNT,
					  NULL);
	if (!x86_fpu_cache) {
		printk(KERN_ERR "kvm: failed to allocate cache for x86 fpu\n");
		goto out;
	}

7189 7190 7191
	shared_msrs = alloc_percpu(struct kvm_shared_msrs);
	if (!shared_msrs) {
		printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
7192
		goto out_free_x86_fpu_cache;
7193 7194
	}

7195 7196
	r = kvm_mmu_module_init();
	if (r)
7197
		goto out_free_percpu;
7198

7199
	kvm_x86_ops = ops;
P
Paolo Bonzini 已提交
7200

S
Sheng Yang 已提交
7201
	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
7202
			PT_DIRTY_MASK, PT64_NX_MASK, 0,
7203
			PT_PRESENT_MASK, 0, sme_me_mask);
7204
	kvm_timer_init();
7205

7206 7207
	perf_register_guest_info_callbacks(&kvm_guest_cbs);

7208
	if (boot_cpu_has(X86_FEATURE_XSAVE))
7209 7210
		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);

7211
	kvm_lapic_init();
7212 7213
	if (pi_inject_timer == -1)
		pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);
7214 7215
#ifdef CONFIG_X86_64
	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
7216

7217
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7218
		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
7219 7220
#endif

7221
	return 0;
7222

7223 7224
out_free_percpu:
	free_percpu(shared_msrs);
7225 7226
out_free_x86_fpu_cache:
	kmem_cache_destroy(x86_fpu_cache);
7227 7228
out:
	return r;
7229
}
7230

7231 7232
void kvm_arch_exit(void)
{
7233
#ifdef CONFIG_X86_64
7234
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7235 7236
		clear_hv_tscchange_cb();
#endif
7237
	kvm_lapic_exit();
7238 7239
	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);

7240 7241 7242
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
					    CPUFREQ_TRANSITION_NOTIFIER);
7243
	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
7244 7245 7246
#ifdef CONFIG_X86_64
	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
#endif
7247
	kvm_x86_ops = NULL;
7248
	kvm_mmu_module_exit();
7249
	free_percpu(shared_msrs);
7250
	kmem_cache_destroy(x86_fpu_cache);
7251
}
7252

7253
int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
7254 7255
{
	++vcpu->stat.halt_exits;
7256
	if (lapic_in_kernel(vcpu)) {
7257
		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
7258 7259 7260 7261 7262 7263
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
}
7264 7265 7266 7267
EXPORT_SYMBOL_GPL(kvm_vcpu_halt);

int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
7268 7269 7270 7271 7272 7273
	int ret = kvm_skip_emulated_instruction(vcpu);
	/*
	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
	 * KVM_EXIT_DEBUG here.
	 */
	return kvm_vcpu_halt(vcpu) && ret;
7274
}
7275 7276
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

7277
#ifdef CONFIG_X86_64
7278 7279 7280 7281
static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
			        unsigned long clock_type)
{
	struct kvm_clock_pairing clock_pairing;
7282
	struct timespec64 ts;
P
Paolo Bonzini 已提交
7283
	u64 cycle;
7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295
	int ret;

	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
		return -KVM_EOPNOTSUPP;

	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)
		return -KVM_EOPNOTSUPP;

	clock_pairing.sec = ts.tv_sec;
	clock_pairing.nsec = ts.tv_nsec;
	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
	clock_pairing.flags = 0;
7296
	memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
7297 7298 7299 7300 7301 7302 7303 7304

	ret = 0;
	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
			    sizeof(struct kvm_clock_pairing)))
		ret = -KVM_EFAULT;

	return ret;
}
7305
#endif
7306

7307 7308 7309 7310 7311 7312 7313
/*
 * kvm_pv_kick_cpu_op:  Kick a vcpu.
 *
 * @apicid - apicid of vcpu to be kicked.
 */
static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
{
7314
	struct kvm_lapic_irq lapic_irq;
7315

7316 7317
	lapic_irq.shorthand = 0;
	lapic_irq.dest_mode = 0;
7318
	lapic_irq.level = 0;
7319
	lapic_irq.dest_id = apicid;
7320
	lapic_irq.msi_redir_hint = false;
7321

7322
	lapic_irq.delivery_mode = APIC_DM_REMRD;
7323
	kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
7324 7325
}

7326 7327
void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu)
{
7328 7329 7330 7331 7332 7333 7334
	if (!lapic_in_kernel(vcpu)) {
		WARN_ON_ONCE(vcpu->arch.apicv_active);
		return;
	}
	if (!vcpu->arch.apicv_active)
		return;

7335 7336 7337 7338
	vcpu->arch.apicv_active = false;
	kvm_x86_ops->refresh_apicv_exec_ctrl(vcpu);
}

7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351
static void kvm_sched_yield(struct kvm *kvm, unsigned long dest_id)
{
	struct kvm_vcpu *target = NULL;
	struct kvm_apic_map *map;

	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);

	if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])
		target = map->phys_map[dest_id]->vcpu;

	rcu_read_unlock();

7352
	if (target && READ_ONCE(target->ready))
7353 7354 7355
		kvm_vcpu_yield_to(target);
}

7356 7357 7358
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
{
	unsigned long nr, a0, a1, a2, a3, ret;
7359
	int op_64_bit;
7360

7361 7362
	if (kvm_hv_hypercall_enabled(vcpu->kvm))
		return kvm_hv_hypercall(vcpu);
7363

7364 7365 7366 7367 7368
	nr = kvm_rax_read(vcpu);
	a0 = kvm_rbx_read(vcpu);
	a1 = kvm_rcx_read(vcpu);
	a2 = kvm_rdx_read(vcpu);
	a3 = kvm_rsi_read(vcpu);
7369

7370
	trace_kvm_hypercall(nr, a0, a1, a2, a3);
F
Feng (Eric) Liu 已提交
7371

7372 7373
	op_64_bit = is_64_bit_mode(vcpu);
	if (!op_64_bit) {
7374 7375 7376 7377 7378 7379 7380
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
	}

7381 7382
	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
		ret = -KVM_EPERM;
7383
		goto out;
7384 7385
	}

7386
	switch (nr) {
A
Avi Kivity 已提交
7387 7388 7389
	case KVM_HC_VAPIC_POLL_IRQ:
		ret = 0;
		break;
7390 7391
	case KVM_HC_KICK_CPU:
		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
7392
		kvm_sched_yield(vcpu->kvm, a1);
7393 7394
		ret = 0;
		break;
7395
#ifdef CONFIG_X86_64
7396 7397 7398
	case KVM_HC_CLOCK_PAIRING:
		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
		break;
7399
#endif
7400 7401 7402
	case KVM_HC_SEND_IPI:
		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
		break;
7403 7404 7405 7406
	case KVM_HC_SCHED_YIELD:
		kvm_sched_yield(vcpu->kvm, a0);
		ret = 0;
		break;
7407 7408 7409 7410
	default:
		ret = -KVM_ENOSYS;
		break;
	}
7411
out:
7412 7413
	if (!op_64_bit)
		ret = (u32)ret;
7414
	kvm_rax_write(vcpu, ret);
7415

A
Amit Shah 已提交
7416
	++vcpu->stat.hypercalls;
7417
	return kvm_skip_emulated_instruction(vcpu);
7418 7419 7420
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

7421
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
7422
{
7423
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7424
	char instruction[3];
7425
	unsigned long rip = kvm_rip_read(vcpu);
7426 7427 7428

	kvm_x86_ops->patch_hypercall(vcpu, instruction);

7429 7430
	return emulator_write_emulated(ctxt, rip, instruction, 3,
		&ctxt->exception);
7431 7432
}

A
Avi Kivity 已提交
7433
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
7434
{
7435 7436
	return vcpu->run->request_interrupt_window &&
		likely(!pic_in_kernel(vcpu->kvm));
7437 7438
}

A
Avi Kivity 已提交
7439
static void post_kvm_run_save(struct kvm_vcpu *vcpu)
7440
{
A
Avi Kivity 已提交
7441 7442
	struct kvm_run *kvm_run = vcpu->run;

7443
	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
7444
	kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;
7445
	kvm_run->cr8 = kvm_get_cr8(vcpu);
7446
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
7447 7448
	kvm_run->ready_for_interrupt_injection =
		pic_in_kernel(vcpu->kvm) ||
7449
		kvm_vcpu_ready_for_interrupt_injection(vcpu);
7450 7451
}

7452 7453 7454 7455 7456 7457 7458
static void update_cr8_intercept(struct kvm_vcpu *vcpu)
{
	int max_irr, tpr;

	if (!kvm_x86_ops->update_cr8_intercept)
		return;

7459
	if (!lapic_in_kernel(vcpu))
7460 7461
		return;

7462 7463 7464
	if (vcpu->arch.apicv_active)
		return;

7465 7466 7467 7468
	if (!vcpu->arch.apic->vapic_addr)
		max_irr = kvm_lapic_find_highest_irr(vcpu);
	else
		max_irr = -1;
7469 7470 7471 7472 7473 7474 7475 7476 7477

	if (max_irr != -1)
		max_irr >>= 4;

	tpr = kvm_lapic_get_cr8(vcpu);

	kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
}

7478
static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win)
7479
{
7480 7481
	int r;

7482
	/* try to reinject previous events if any */
7483

7484 7485
	if (vcpu->arch.exception.injected)
		kvm_x86_ops->queue_exception(vcpu);
7486
	/*
7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498
	 * Do not inject an NMI or interrupt if there is a pending
	 * exception.  Exceptions and interrupts are recognized at
	 * instruction boundaries, i.e. the start of an instruction.
	 * Trap-like exceptions, e.g. #DB, have higher priority than
	 * NMIs and interrupts, i.e. traps are recognized before an
	 * NMI/interrupt that's pending on the same instruction.
	 * Fault-like exceptions, e.g. #GP and #PF, are the lowest
	 * priority, but are only generated (pended) during instruction
	 * execution, i.e. a pending fault-like exception means the
	 * fault occurred on the *previous* instruction and must be
	 * serviced prior to recognizing any new events in order to
	 * fully complete the previous instruction.
7499
	 */
7500 7501
	else if (!vcpu->arch.exception.pending) {
		if (vcpu->arch.nmi_injected)
7502
			kvm_x86_ops->set_nmi(vcpu);
7503
		else if (vcpu->arch.interrupt.injected)
7504 7505 7506
			kvm_x86_ops->set_irq(vcpu);
	}

7507 7508 7509 7510 7511 7512
	/*
	 * Call check_nested_events() even if we reinjected a previous event
	 * in order for caller to determine if it should require immediate-exit
	 * from L2 to L1 due to pending L1 events which require exit
	 * from L2 to L1.
	 */
7513 7514 7515 7516 7517 7518 7519
	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
		r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
		if (r != 0)
			return r;
	}

	/* try to inject new event if pending */
7520
	if (vcpu->arch.exception.pending) {
A
Avi Kivity 已提交
7521 7522 7523
		trace_kvm_inj_exception(vcpu->arch.exception.nr,
					vcpu->arch.exception.has_error_code,
					vcpu->arch.exception.error_code);
7524

7525
		WARN_ON_ONCE(vcpu->arch.exception.injected);
7526 7527 7528
		vcpu->arch.exception.pending = false;
		vcpu->arch.exception.injected = true;

7529 7530 7531 7532
		if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
					     X86_EFLAGS_RF);

7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548
		if (vcpu->arch.exception.nr == DB_VECTOR) {
			/*
			 * This code assumes that nSVM doesn't use
			 * check_nested_events(). If it does, the
			 * DR6/DR7 changes should happen before L1
			 * gets a #VMEXIT for an intercepted #DB in
			 * L2.  (Under VMX, on the other hand, the
			 * DR6/DR7 changes should not happen in the
			 * event of a VM-exit to L1 for an intercepted
			 * #DB in L2.)
			 */
			kvm_deliver_exception_payload(vcpu);
			if (vcpu->arch.dr7 & DR7_GD) {
				vcpu->arch.dr7 &= ~DR7_GD;
				kvm_update_dr7(vcpu);
			}
7549 7550
		}

7551
		kvm_x86_ops->queue_exception(vcpu);
7552 7553 7554 7555 7556 7557 7558 7559
	}

	/* Don't consider new event if we re-injected an event */
	if (kvm_event_needs_reinjection(vcpu))
		return 0;

	if (vcpu->arch.smi_pending && !is_smm(vcpu) &&
	    kvm_x86_ops->smi_allowed(vcpu)) {
7560
		vcpu->arch.smi_pending = false;
7561
		++vcpu->arch.smi_count;
7562
		enter_smm(vcpu);
7563
	} else if (vcpu->arch.nmi_pending && kvm_x86_ops->nmi_allowed(vcpu)) {
7564 7565 7566
		--vcpu->arch.nmi_pending;
		vcpu->arch.nmi_injected = true;
		kvm_x86_ops->set_nmi(vcpu);
7567
	} else if (kvm_cpu_has_injectable_intr(vcpu)) {
7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579
		/*
		 * Because interrupts can be injected asynchronously, we are
		 * calling check_nested_events again here to avoid a race condition.
		 * See https://lkml.org/lkml/2014/7/2/60 for discussion about this
		 * proposal and current concerns.  Perhaps we should be setting
		 * KVM_REQ_EVENT only on certain events and not unconditionally?
		 */
		if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
			r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
			if (r != 0)
				return r;
		}
7580
		if (kvm_x86_ops->interrupt_allowed(vcpu)) {
7581 7582 7583
			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
					    false);
			kvm_x86_ops->set_irq(vcpu);
7584 7585
		}
	}
7586

7587
	return 0;
7588 7589
}

A
Avi Kivity 已提交
7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606
static void process_nmi(struct kvm_vcpu *vcpu)
{
	unsigned limit = 2;

	/*
	 * x86 is limited to one NMI running, and one NMI pending after it.
	 * If an NMI is already in progress, limit further NMIs to just one.
	 * Otherwise, allow two (and we'll inject the first one immediately).
	 */
	if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
		limit = 1;

	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

7607
static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620
{
	u32 flags = 0;
	flags |= seg->g       << 23;
	flags |= seg->db      << 22;
	flags |= seg->l       << 21;
	flags |= seg->avl     << 20;
	flags |= seg->present << 15;
	flags |= seg->dpl     << 13;
	flags |= seg->s       << 12;
	flags |= seg->type    << 8;
	return flags;
}

7621
static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635
{
	struct kvm_segment seg;
	int offset;

	kvm_get_segment(vcpu, &seg, n);
	put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector);

	if (n < 3)
		offset = 0x7f84 + n * 12;
	else
		offset = 0x7f2c + (n - 3) * 12;

	put_smstate(u32, buf, offset + 8, seg.base);
	put_smstate(u32, buf, offset + 4, seg.limit);
7636
	put_smstate(u32, buf, offset, enter_smm_get_segment_flags(&seg));
7637 7638
}

7639
#ifdef CONFIG_X86_64
7640
static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
7641 7642 7643 7644 7645 7646 7647 7648
{
	struct kvm_segment seg;
	int offset;
	u16 flags;

	kvm_get_segment(vcpu, &seg, n);
	offset = 0x7e00 + n * 16;

7649
	flags = enter_smm_get_segment_flags(&seg) >> 8;
7650 7651 7652 7653 7654
	put_smstate(u16, buf, offset, seg.selector);
	put_smstate(u16, buf, offset + 2, flags);
	put_smstate(u32, buf, offset + 4, seg.limit);
	put_smstate(u64, buf, offset + 8, seg.base);
}
7655
#endif
7656

7657
static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678 7679 7680
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
	put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
	put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
	put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));

	for (i = 0; i < 8; i++)
		put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read(vcpu, i));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u32, buf, 0x7fcc, (u32)val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u32, buf, 0x7fc8, (u32)val);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u32, buf, 0x7fc4, seg.selector);
	put_smstate(u32, buf, 0x7f64, seg.base);
	put_smstate(u32, buf, 0x7f60, seg.limit);
7681
	put_smstate(u32, buf, 0x7f5c, enter_smm_get_segment_flags(&seg));
7682 7683 7684 7685 7686

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u32, buf, 0x7fc0, seg.selector);
	put_smstate(u32, buf, 0x7f80, seg.base);
	put_smstate(u32, buf, 0x7f7c, seg.limit);
7687
	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
7688 7689 7690 7691 7692 7693 7694 7695 7696 7697

	kvm_x86_ops->get_gdt(vcpu, &dt);
	put_smstate(u32, buf, 0x7f74, dt.address);
	put_smstate(u32, buf, 0x7f70, dt.size);

	kvm_x86_ops->get_idt(vcpu, &dt);
	put_smstate(u32, buf, 0x7f58, dt.address);
	put_smstate(u32, buf, 0x7f54, dt.size);

	for (i = 0; i < 6; i++)
7698
		enter_smm_save_seg_32(vcpu, buf, i);
7699 7700 7701 7702 7703 7704 7705 7706

	put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020000);
	put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
}

7707
#ifdef CONFIG_X86_64
7708
static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	for (i = 0; i < 16; i++)
		put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read(vcpu, i));

	put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
	put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u64, buf, 0x7f68, val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u64, buf, 0x7f60, val);

	put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
	put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
	put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));

	put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020064);

	put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u16, buf, 0x7e90, seg.selector);
7739
	put_smstate(u16, buf, 0x7e92, enter_smm_get_segment_flags(&seg) >> 8);
7740 7741 7742 7743 7744 7745 7746 7747 7748
	put_smstate(u32, buf, 0x7e94, seg.limit);
	put_smstate(u64, buf, 0x7e98, seg.base);

	kvm_x86_ops->get_idt(vcpu, &dt);
	put_smstate(u32, buf, 0x7e84, dt.size);
	put_smstate(u64, buf, 0x7e88, dt.address);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u16, buf, 0x7e70, seg.selector);
7749
	put_smstate(u16, buf, 0x7e72, enter_smm_get_segment_flags(&seg) >> 8);
7750 7751 7752 7753 7754 7755 7756 7757
	put_smstate(u32, buf, 0x7e74, seg.limit);
	put_smstate(u64, buf, 0x7e78, seg.base);

	kvm_x86_ops->get_gdt(vcpu, &dt);
	put_smstate(u32, buf, 0x7e64, dt.size);
	put_smstate(u64, buf, 0x7e68, dt.address);

	for (i = 0; i < 6; i++)
7758
		enter_smm_save_seg_64(vcpu, buf, i);
7759
}
7760
#endif
7761

7762
static void enter_smm(struct kvm_vcpu *vcpu)
P
Paolo Bonzini 已提交
7763
{
7764
	struct kvm_segment cs, ds;
7765
	struct desc_ptr dt;
7766 7767 7768 7769 7770
	char buf[512];
	u32 cr0;

	trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true);
	memset(buf, 0, 512);
7771
#ifdef CONFIG_X86_64
7772
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7773
		enter_smm_save_state_64(vcpu, buf);
7774
	else
7775
#endif
7776
		enter_smm_save_state_32(vcpu, buf);
7777

7778 7779 7780 7781 7782 7783 7784 7785
	/*
	 * Give pre_enter_smm() a chance to make ISA-specific changes to the
	 * vCPU state (e.g. leave guest mode) after we've saved the state into
	 * the SMM state-save area.
	 */
	kvm_x86_ops->pre_enter_smm(vcpu, buf);

	vcpu->arch.hflags |= HF_SMM_MASK;
7786
	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801

	if (kvm_x86_ops->get_nmi_mask(vcpu))
		vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
	else
		kvm_x86_ops->set_nmi_mask(vcpu, true);

	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
	kvm_rip_write(vcpu, 0x8000);

	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
	kvm_x86_ops->set_cr0(vcpu, cr0);
	vcpu->arch.cr0 = cr0;

	kvm_x86_ops->set_cr4(vcpu, 0);

7802 7803 7804 7805
	/* Undocumented: IDT limit is set to zero on entry to SMM.  */
	dt.address = dt.size = 0;
	kvm_x86_ops->set_idt(vcpu, &dt);

7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832
	__kvm_set_dr(vcpu, 7, DR7_FIXED_1);

	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
	cs.base = vcpu->arch.smbase;

	ds.selector = 0;
	ds.base = 0;

	cs.limit    = ds.limit = 0xffffffff;
	cs.type     = ds.type = 0x3;
	cs.dpl      = ds.dpl = 0;
	cs.db       = ds.db = 0;
	cs.s        = ds.s = 1;
	cs.l        = ds.l = 0;
	cs.g        = ds.g = 1;
	cs.avl      = ds.avl = 0;
	cs.present  = ds.present = 1;
	cs.unusable = ds.unusable = 0;
	cs.padding  = ds.padding = 0;

	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);

7833
#ifdef CONFIG_X86_64
7834
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7835
		kvm_x86_ops->set_efer(vcpu, 0);
7836
#endif
7837 7838 7839

	kvm_update_cpuid(vcpu);
	kvm_mmu_reset_context(vcpu);
P
Paolo Bonzini 已提交
7840 7841
}

7842
static void process_smi(struct kvm_vcpu *vcpu)
7843 7844 7845 7846 7847
{
	vcpu->arch.smi_pending = true;
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

7848 7849 7850 7851 7852
void kvm_make_scan_ioapic_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
}

7853
static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
7854
{
7855
	if (!kvm_apic_present(vcpu))
7856
		return;
7857

7858
	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
7859

7860
	if (irqchip_split(vcpu->kvm))
7861
		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
7862
	else {
7863
		if (vcpu->arch.apicv_active)
7864
			kvm_x86_ops->sync_pir_to_irr(vcpu);
7865 7866
		if (ioapic_in_kernel(vcpu->kvm))
			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
7867
	}
7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881

	if (is_guest_mode(vcpu))
		vcpu->arch.load_eoi_exitmap_pending = true;
	else
		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
}

static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
{
	u64 eoi_exit_bitmap[4];

	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
		return;

7882 7883 7884
	bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
		  vcpu_to_synic(vcpu)->vec_bitmap, 256);
	kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
7885 7886
}

7887 7888 7889
int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
		unsigned long start, unsigned long end,
		bool blockable)
7890 7891 7892 7893 7894 7895 7896 7897 7898 7899
{
	unsigned long apic_address;

	/*
	 * The physical address of apic access page is stored in the VMCS.
	 * Update it when it becomes invalid.
	 */
	apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
	if (start <= apic_address && apic_address < end)
		kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
7900 7901

	return 0;
7902 7903
}

7904 7905
void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
{
7906 7907
	struct page *page = NULL;

7908
	if (!lapic_in_kernel(vcpu))
7909 7910
		return;

7911 7912 7913
	if (!kvm_x86_ops->set_apic_access_page_addr)
		return;

7914
	page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
7915 7916
	if (is_error_page(page))
		return;
7917 7918 7919 7920 7921 7922 7923
	kvm_x86_ops->set_apic_access_page_addr(vcpu, page_to_phys(page));

	/*
	 * Do not pin apic access page in memory, the MMU notifier
	 * will call us again if it is migrated or swapped out.
	 */
	put_page(page);
7924 7925 7926
}
EXPORT_SYMBOL_GPL(kvm_vcpu_reload_apic_access_page);

7927 7928 7929 7930 7931 7932
void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
{
	smp_send_reschedule(vcpu->cpu);
}
EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);

7933
/*
7934
 * Returns 1 to let vcpu_run() continue the guest execution loop without
7935 7936 7937
 * exiting to the userspace.  Otherwise, the value will be returned to the
 * userspace.
 */
A
Avi Kivity 已提交
7938
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
7939 7940
{
	int r;
7941 7942 7943 7944
	bool req_int_win =
		dm_request_for_irq_injection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);

7945
	bool req_immediate_exit = false;
7946

R
Radim Krčmář 已提交
7947
	if (kvm_request_pending(vcpu)) {
7948 7949
		if (kvm_check_request(KVM_REQ_GET_VMCS12_PAGES, vcpu))
			kvm_x86_ops->get_vmcs12_pages(vcpu);
7950
		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
7951
			kvm_mmu_unload(vcpu);
7952
		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
M
Marcelo Tosatti 已提交
7953
			__kvm_migrate_timers(vcpu);
7954 7955
		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
			kvm_gen_update_masterclock(vcpu->kvm);
7956 7957
		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
			kvm_gen_kvmclock_update(vcpu);
Z
Zachary Amsden 已提交
7958 7959
		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
			r = kvm_guest_time_update(vcpu);
7960 7961 7962
			if (unlikely(r))
				goto out;
		}
7963
		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
7964
			kvm_mmu_sync_roots(vcpu);
7965 7966
		if (kvm_check_request(KVM_REQ_LOAD_CR3, vcpu))
			kvm_mmu_load_cr3(vcpu);
7967
		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
7968
			kvm_vcpu_flush_tlb(vcpu, true);
7969
		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
A
Avi Kivity 已提交
7970
			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
A
Avi Kivity 已提交
7971 7972 7973
			r = 0;
			goto out;
		}
7974
		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
A
Avi Kivity 已提交
7975
			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
7976
			vcpu->mmio_needed = 0;
J
Joerg Roedel 已提交
7977 7978 7979
			r = 0;
			goto out;
		}
7980 7981 7982 7983 7984 7985
		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
			/* Page is swapped out. Do synthetic halt */
			vcpu->arch.apf.halted = true;
			r = 1;
			goto out;
		}
G
Glauber Costa 已提交
7986 7987
		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
			record_steal_time(vcpu);
P
Paolo Bonzini 已提交
7988 7989
		if (kvm_check_request(KVM_REQ_SMI, vcpu))
			process_smi(vcpu);
A
Avi Kivity 已提交
7990 7991
		if (kvm_check_request(KVM_REQ_NMI, vcpu))
			process_nmi(vcpu);
7992
		if (kvm_check_request(KVM_REQ_PMU, vcpu))
7993
			kvm_pmu_handle_event(vcpu);
7994
		if (kvm_check_request(KVM_REQ_PMI, vcpu))
7995
			kvm_pmu_deliver_pmi(vcpu);
7996 7997 7998
		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
			if (test_bit(vcpu->arch.pending_ioapic_eoi,
7999
				     vcpu->arch.ioapic_handled_vectors)) {
8000 8001 8002 8003 8004 8005 8006
				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
				vcpu->run->eoi.vector =
						vcpu->arch.pending_ioapic_eoi;
				r = 0;
				goto out;
			}
		}
8007 8008
		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
			vcpu_scan_ioapic(vcpu);
8009 8010
		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
			vcpu_load_eoi_exitmap(vcpu);
8011 8012
		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
			kvm_vcpu_reload_apic_access_page(vcpu);
8013 8014 8015 8016 8017 8018
		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
			r = 0;
			goto out;
		}
8019 8020 8021 8022 8023 8024
		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
			r = 0;
			goto out;
		}
A
Andrey Smetanin 已提交
8025 8026 8027 8028 8029 8030
		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_HYPERV;
			vcpu->run->hyperv = vcpu->arch.hyperv.exit;
			r = 0;
			goto out;
		}
8031 8032 8033 8034 8035 8036

		/*
		 * KVM_REQ_HV_STIMER has to be processed after
		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
		 * depend on the guest clock being up-to-date
		 */
A
Andrey Smetanin 已提交
8037 8038
		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
			kvm_hv_process_stimers(vcpu);
8039
	}
A
Avi Kivity 已提交
8040

A
Avi Kivity 已提交
8041
	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
8042
		++vcpu->stat.req_event;
8043 8044 8045 8046 8047 8048
		kvm_apic_accept_events(vcpu);
		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
			r = 1;
			goto out;
		}

8049 8050
		if (inject_pending_event(vcpu, req_int_win) != 0)
			req_immediate_exit = true;
8051
		else {
8052
			/* Enable SMI/NMI/IRQ window open exits if needed.
8053
			 *
8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064
			 * SMIs have three cases:
			 * 1) They can be nested, and then there is nothing to
			 *    do here because RSM will cause a vmexit anyway.
			 * 2) There is an ISA-specific reason why SMI cannot be
			 *    injected, and the moment when this changes can be
			 *    intercepted.
			 * 3) Or the SMI can be pending because
			 *    inject_pending_event has completed the injection
			 *    of an IRQ or NMI from the previous vmexit, and
			 *    then we request an immediate exit to inject the
			 *    SMI.
8065 8066
			 */
			if (vcpu->arch.smi_pending && !is_smm(vcpu))
8067 8068
				if (!kvm_x86_ops->enable_smi_window(vcpu))
					req_immediate_exit = true;
8069 8070 8071 8072
			if (vcpu->arch.nmi_pending)
				kvm_x86_ops->enable_nmi_window(vcpu);
			if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
				kvm_x86_ops->enable_irq_window(vcpu);
8073
			WARN_ON(vcpu->arch.exception.pending);
8074
		}
A
Avi Kivity 已提交
8075 8076 8077 8078 8079 8080 8081

		if (kvm_lapic_enabled(vcpu)) {
			update_cr8_intercept(vcpu);
			kvm_lapic_sync_to_vapic(vcpu);
		}
	}

8082 8083
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r)) {
8084
		goto cancel_injection;
8085 8086
	}

8087 8088 8089
	preempt_disable();

	kvm_x86_ops->prepare_guest_switch(vcpu);
8090 8091 8092 8093 8094 8095 8096

	/*
	 * Disable IRQs before setting IN_GUEST_MODE.  Posted interrupt
	 * IPI are then delayed after guest entry, which ensures that they
	 * result in virtual interrupt delivery.
	 */
	local_irq_disable();
8097 8098
	vcpu->mode = IN_GUEST_MODE;

8099 8100
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);

8101
	/*
8102
	 * 1) We should set ->mode before checking ->requests.  Please see
8103
	 * the comment in kvm_vcpu_exiting_guest_mode().
8104
	 *
8105
	 * 2) For APICv, we should set ->mode before checking PID.ON. This
8106 8107 8108 8109 8110 8111
	 * pairs with the memory barrier implicit in pi_test_and_set_on
	 * (see vmx_deliver_posted_interrupt).
	 *
	 * 3) This also orders the write to mode from any reads to the page
	 * tables done while the VCPU is running.  Please see the comment
	 * in kvm_flush_remote_tlbs.
8112
	 */
8113
	smp_mb__after_srcu_read_unlock();
8114

8115 8116 8117 8118
	/*
	 * This handles the case where a posted interrupt was
	 * notified with kvm_vcpu_kick.
	 */
8119 8120
	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
		kvm_x86_ops->sync_pir_to_irr(vcpu);
8121

R
Radim Krčmář 已提交
8122
	if (vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu)
A
Avi Kivity 已提交
8123
	    || need_resched() || signal_pending(current)) {
8124
		vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
8125
		smp_wmb();
8126 8127
		local_irq_enable();
		preempt_enable();
8128
		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8129
		r = 1;
8130
		goto cancel_injection;
8131 8132
	}

8133 8134
	if (req_immediate_exit) {
		kvm_make_request(KVM_REQ_EVENT, vcpu);
8135
		kvm_x86_ops->request_immediate_exit(vcpu);
8136
	}
8137

8138
	trace_kvm_entry(vcpu->vcpu_id);
8139
	guest_enter_irqoff();
8140

8141 8142
	/* The preempt notifier should have taken care of the FPU already.  */
	WARN_ON_ONCE(test_thread_flag(TIF_NEED_FPU_LOAD));
8143

8144 8145 8146 8147 8148 8149
	if (unlikely(vcpu->arch.switch_db_regs)) {
		set_debugreg(0, 7);
		set_debugreg(vcpu->arch.eff_db[0], 0);
		set_debugreg(vcpu->arch.eff_db[1], 1);
		set_debugreg(vcpu->arch.eff_db[2], 2);
		set_debugreg(vcpu->arch.eff_db[3], 3);
8150
		set_debugreg(vcpu->arch.dr6, 6);
8151
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8152
	}
8153

A
Avi Kivity 已提交
8154
	kvm_x86_ops->run(vcpu);
8155

8156 8157 8158 8159 8160 8161 8162 8163 8164
	/*
	 * Do this here before restoring debug registers on the host.  And
	 * since we do this before handling the vmexit, a DR access vmexit
	 * can (a) read the correct value of the debug registers, (b) set
	 * KVM_DEBUGREG_WONT_EXIT again.
	 */
	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
		kvm_x86_ops->sync_dirty_debug_regs(vcpu);
8165 8166 8167 8168
		kvm_update_dr0123(vcpu);
		kvm_update_dr6(vcpu);
		kvm_update_dr7(vcpu);
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8169 8170
	}

8171 8172 8173 8174 8175 8176 8177
	/*
	 * If the guest has used debug registers, at least dr7
	 * will be disabled while returning to the host.
	 * If we don't have active breakpoints in the host, we don't
	 * care about the messed up debug address registers. But if
	 * we have some of them active, restore the old state.
	 */
8178
	if (hw_breakpoint_active())
8179
		hw_breakpoint_restore();
8180

8181
	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
8182

8183
	vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
8184
	smp_wmb();
8185

8186
	kvm_x86_ops->handle_exit_irqoff(vcpu);
8187

8188 8189 8190 8191 8192 8193 8194 8195 8196
	/*
	 * Consume any pending interrupts, including the possible source of
	 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
	 * An instruction is required after local_irq_enable() to fully unblock
	 * interrupts on processors that implement an interrupt shadow, the
	 * stat.exits increment will do nicely.
	 */
	kvm_before_interrupt(vcpu);
	local_irq_enable();
8197
	++vcpu->stat.exits;
8198 8199
	local_irq_disable();
	kvm_after_interrupt(vcpu);
8200

P
Paolo Bonzini 已提交
8201
	guest_exit_irqoff();
8202 8203 8204 8205 8206 8207 8208
	if (lapic_in_kernel(vcpu)) {
		s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;
		if (delta != S64_MIN) {
			trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta);
			vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;
		}
	}
8209

P
Paolo Bonzini 已提交
8210
	local_irq_enable();
8211 8212
	preempt_enable();

8213
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8214

8215 8216 8217 8218
	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
8219 8220
		unsigned long rip = kvm_rip_read(vcpu);
		profile_hit(KVM_PROFILING, (void *)rip);
8221 8222
	}

8223 8224
	if (unlikely(vcpu->arch.tsc_always_catchup))
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
8225

8226 8227
	if (vcpu->arch.apic_attention)
		kvm_lapic_sync_from_vapic(vcpu);
A
Avi Kivity 已提交
8228

8229
	vcpu->arch.gpa_available = false;
A
Avi Kivity 已提交
8230
	r = kvm_x86_ops->handle_exit(vcpu);
8231 8232 8233 8234
	return r;

cancel_injection:
	kvm_x86_ops->cancel_injection(vcpu);
8235 8236
	if (unlikely(vcpu->arch.apic_attention))
		kvm_lapic_sync_from_vapic(vcpu);
8237 8238 8239
out:
	return r;
}
8240

8241 8242
static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
{
8243 8244
	if (!kvm_arch_vcpu_runnable(vcpu) &&
	    (!kvm_x86_ops->pre_block || kvm_x86_ops->pre_block(vcpu) == 0)) {
8245 8246 8247
		srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
		kvm_vcpu_block(vcpu);
		vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8248 8249 8250 8251

		if (kvm_x86_ops->post_block)
			kvm_x86_ops->post_block(vcpu);

8252 8253 8254
		if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
			return 1;
	}
8255 8256 8257 8258 8259 8260 8261

	kvm_apic_accept_events(vcpu);
	switch(vcpu->arch.mp_state) {
	case KVM_MP_STATE_HALTED:
		vcpu->arch.pv.pv_unhalted = false;
		vcpu->arch.mp_state =
			KVM_MP_STATE_RUNNABLE;
8262
		/* fall through */
8263 8264 8265 8266 8267 8268 8269 8270 8271 8272 8273
	case KVM_MP_STATE_RUNNABLE:
		vcpu->arch.apf.halted = false;
		break;
	case KVM_MP_STATE_INIT_RECEIVED:
		break;
	default:
		return -EINTR;
		break;
	}
	return 1;
}
8274

8275 8276
static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
{
8277 8278 8279
	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events)
		kvm_x86_ops->check_nested_events(vcpu, false);

8280 8281 8282 8283
	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		!vcpu->arch.apf.halted);
}

8284
static int vcpu_run(struct kvm_vcpu *vcpu)
8285 8286
{
	int r;
8287
	struct kvm *kvm = vcpu->kvm;
8288

8289
	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
P
Paolo Bonzini 已提交
8290
	vcpu->arch.l1tf_flush_l1d = true;
8291

8292
	for (;;) {
8293
		if (kvm_vcpu_running(vcpu)) {
A
Avi Kivity 已提交
8294
			r = vcpu_enter_guest(vcpu);
8295
		} else {
8296
			r = vcpu_block(kvm, vcpu);
8297 8298
		}

8299 8300 8301
		if (r <= 0)
			break;

8302
		kvm_clear_request(KVM_REQ_PENDING_TIMER, vcpu);
8303 8304 8305
		if (kvm_cpu_has_pending_timer(vcpu))
			kvm_inject_pending_timer_irqs(vcpu);

8306 8307
		if (dm_request_for_irq_injection(vcpu) &&
			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
8308 8309
			r = 0;
			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
8310
			++vcpu->stat.request_irq_exits;
8311
			break;
8312
		}
8313 8314 8315

		kvm_check_async_pf_completion(vcpu);

8316 8317
		if (signal_pending(current)) {
			r = -EINTR;
A
Avi Kivity 已提交
8318
			vcpu->run->exit_reason = KVM_EXIT_INTR;
8319
			++vcpu->stat.signal_exits;
8320
			break;
8321 8322
		}
		if (need_resched()) {
8323
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8324
			cond_resched();
8325
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8326
		}
8327 8328
	}

8329
	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8330 8331 8332 8333

	return r;
}

8334 8335 8336
static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
{
	int r;
8337

8338
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8339
	r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
8340
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
8341
	return r;
8342 8343 8344 8345 8346 8347 8348 8349 8350
}

static int complete_emulated_pio(struct kvm_vcpu *vcpu)
{
	BUG_ON(!vcpu->arch.pio.count);

	return complete_emulated_io(vcpu);
}

A
Avi Kivity 已提交
8351 8352 8353 8354 8355
/*
 * Implements the following, as a state machine:
 *
 * read:
 *   for each fragment
8356 8357 8358 8359
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       exit
 *       copy data
A
Avi Kivity 已提交
8360 8361 8362 8363
 *   execute insn
 *
 * write:
 *   for each fragment
8364 8365 8366 8367
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       copy data
 *       exit
A
Avi Kivity 已提交
8368
 */
8369
static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
8370 8371
{
	struct kvm_run *run = vcpu->run;
A
Avi Kivity 已提交
8372
	struct kvm_mmio_fragment *frag;
8373
	unsigned len;
8374

8375
	BUG_ON(!vcpu->mmio_needed);
8376

8377
	/* Complete previous fragment */
8378 8379
	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
	len = min(8u, frag->len);
8380
	if (!vcpu->mmio_is_write)
8381 8382 8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393
		memcpy(frag->data, run->mmio.data, len);

	if (frag->len <= 8) {
		/* Switch to the next fragment. */
		frag++;
		vcpu->mmio_cur_fragment++;
	} else {
		/* Go forward to the next mmio piece. */
		frag->data += len;
		frag->gpa += len;
		frag->len -= len;
	}

8394
	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
8395
		vcpu->mmio_needed = 0;
8396 8397

		/* FIXME: return into emulator if single-stepping.  */
A
Avi Kivity 已提交
8398
		if (vcpu->mmio_is_write)
8399 8400 8401 8402
			return 1;
		vcpu->mmio_read_completed = 1;
		return complete_emulated_io(vcpu);
	}
8403

8404 8405 8406
	run->exit_reason = KVM_EXIT_MMIO;
	run->mmio.phys_addr = frag->gpa;
	if (vcpu->mmio_is_write)
8407 8408
		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
	run->mmio.len = min(8u, frag->len);
8409 8410 8411
	run->mmio.is_write = vcpu->mmio_is_write;
	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
	return 0;
8412 8413
}

8414 8415 8416
/* Swap (qemu) user FPU context for the guest FPU context. */
static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
8417 8418
	fpregs_lock();

8419
	copy_fpregs_to_fpstate(vcpu->arch.user_fpu);
8420
	/* PKRU is separately restored in kvm_x86_ops->run.  */
8421
	__copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,
8422
				~XFEATURE_MASK_PKRU);
8423 8424 8425 8426

	fpregs_mark_activate();
	fpregs_unlock();

8427 8428 8429 8430 8431 8432
	trace_kvm_fpu(1);
}

/* When vcpu_run ends, restore user space FPU context. */
static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
8433 8434
	fpregs_lock();

8435
	copy_fpregs_to_fpstate(vcpu->arch.guest_fpu);
8436
	copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);
8437 8438 8439 8440

	fpregs_mark_activate();
	fpregs_unlock();

8441 8442 8443 8444
	++vcpu->stat.fpu_reload;
	trace_kvm_fpu(0);
}

8445 8446 8447 8448
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
{
	int r;

8449
	vcpu_load(vcpu);
8450
	kvm_sigset_activate(vcpu);
8451 8452
	kvm_load_guest_fpu(vcpu);

8453
	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
8454 8455 8456 8457
		if (kvm_run->immediate_exit) {
			r = -EINTR;
			goto out;
		}
8458
		kvm_vcpu_block(vcpu);
8459
		kvm_apic_accept_events(vcpu);
8460
		kvm_clear_request(KVM_REQ_UNHALT, vcpu);
8461
		r = -EAGAIN;
8462 8463 8464 8465 8466
		if (signal_pending(current)) {
			r = -EINTR;
			vcpu->run->exit_reason = KVM_EXIT_INTR;
			++vcpu->stat.signal_exits;
		}
8467
		goto out;
8468 8469
	}

K
Ken Hofsass 已提交
8470 8471 8472 8473 8474 8475 8476 8477 8478 8479 8480
	if (vcpu->run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) {
		r = -EINVAL;
		goto out;
	}

	if (vcpu->run->kvm_dirty_regs) {
		r = sync_regs(vcpu);
		if (r != 0)
			goto out;
	}

8481
	/* re-sync apic's tpr */
8482
	if (!lapic_in_kernel(vcpu)) {
A
Andre Przywara 已提交
8483 8484 8485 8486 8487
		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
			r = -EINVAL;
			goto out;
		}
	}
8488

8489 8490 8491 8492 8493
	if (unlikely(vcpu->arch.complete_userspace_io)) {
		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
		vcpu->arch.complete_userspace_io = NULL;
		r = cui(vcpu);
		if (r <= 0)
8494
			goto out;
8495 8496
	} else
		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
8497

8498 8499 8500 8501
	if (kvm_run->immediate_exit)
		r = -EINTR;
	else
		r = vcpu_run(vcpu);
8502 8503

out:
8504
	kvm_put_guest_fpu(vcpu);
K
Ken Hofsass 已提交
8505 8506
	if (vcpu->run->kvm_valid_regs)
		store_regs(vcpu);
8507
	post_kvm_run_save(vcpu);
8508
	kvm_sigset_deactivate(vcpu);
8509

8510
	vcpu_put(vcpu);
8511 8512 8513
	return r;
}

K
Ken Hofsass 已提交
8514
static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8515
{
8516 8517 8518 8519
	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
		/*
		 * We are here if userspace calls get_regs() in the middle of
		 * instruction emulation. Registers state needs to be copied
G
Guo Chao 已提交
8520
		 * back from emulation context to vcpu. Userspace shouldn't do
8521 8522 8523
		 * that usually, but some bad designed PV devices (vmware
		 * backdoor interface) need this to work
		 */
8524
		emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
8525 8526
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
	}
8527 8528 8529 8530 8531 8532
	regs->rax = kvm_rax_read(vcpu);
	regs->rbx = kvm_rbx_read(vcpu);
	regs->rcx = kvm_rcx_read(vcpu);
	regs->rdx = kvm_rdx_read(vcpu);
	regs->rsi = kvm_rsi_read(vcpu);
	regs->rdi = kvm_rdi_read(vcpu);
8533
	regs->rsp = kvm_rsp_read(vcpu);
8534
	regs->rbp = kvm_rbp_read(vcpu);
8535
#ifdef CONFIG_X86_64
8536 8537 8538 8539 8540 8541 8542 8543
	regs->r8 = kvm_r8_read(vcpu);
	regs->r9 = kvm_r9_read(vcpu);
	regs->r10 = kvm_r10_read(vcpu);
	regs->r11 = kvm_r11_read(vcpu);
	regs->r12 = kvm_r12_read(vcpu);
	regs->r13 = kvm_r13_read(vcpu);
	regs->r14 = kvm_r14_read(vcpu);
	regs->r15 = kvm_r15_read(vcpu);
8544 8545
#endif

8546
	regs->rip = kvm_rip_read(vcpu);
8547
	regs->rflags = kvm_get_rflags(vcpu);
K
Ken Hofsass 已提交
8548
}
8549

K
Ken Hofsass 已提交
8550 8551 8552 8553
int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__get_regs(vcpu, regs);
8554
	vcpu_put(vcpu);
8555 8556 8557
	return 0;
}

K
Ken Hofsass 已提交
8558
static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8559
{
8560 8561 8562
	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;

8563 8564 8565 8566 8567 8568
	kvm_rax_write(vcpu, regs->rax);
	kvm_rbx_write(vcpu, regs->rbx);
	kvm_rcx_write(vcpu, regs->rcx);
	kvm_rdx_write(vcpu, regs->rdx);
	kvm_rsi_write(vcpu, regs->rsi);
	kvm_rdi_write(vcpu, regs->rdi);
8569
	kvm_rsp_write(vcpu, regs->rsp);
8570
	kvm_rbp_write(vcpu, regs->rbp);
8571
#ifdef CONFIG_X86_64
8572 8573 8574 8575 8576 8577 8578 8579
	kvm_r8_write(vcpu, regs->r8);
	kvm_r9_write(vcpu, regs->r9);
	kvm_r10_write(vcpu, regs->r10);
	kvm_r11_write(vcpu, regs->r11);
	kvm_r12_write(vcpu, regs->r12);
	kvm_r13_write(vcpu, regs->r13);
	kvm_r14_write(vcpu, regs->r14);
	kvm_r15_write(vcpu, regs->r15);
8580 8581
#endif

8582
	kvm_rip_write(vcpu, regs->rip);
8583
	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
8584

8585 8586
	vcpu->arch.exception.pending = false;

8587
	kvm_make_request(KVM_REQ_EVENT, vcpu);
K
Ken Hofsass 已提交
8588
}
8589

K
Ken Hofsass 已提交
8590 8591 8592 8593
int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__set_regs(vcpu, regs);
8594
	vcpu_put(vcpu);
8595 8596 8597 8598 8599 8600 8601
	return 0;
}

void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

8602
	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
8603 8604 8605 8606 8607
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

K
Ken Hofsass 已提交
8608
static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8609
{
8610
	struct desc_ptr dt;
8611

8612 8613 8614 8615 8616 8617
	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8618

8619 8620
	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
8621 8622

	kvm_x86_ops->get_idt(vcpu, &dt);
8623 8624
	sregs->idt.limit = dt.size;
	sregs->idt.base = dt.address;
8625
	kvm_x86_ops->get_gdt(vcpu, &dt);
8626 8627
	sregs->gdt.limit = dt.size;
	sregs->gdt.base = dt.address;
8628

8629
	sregs->cr0 = kvm_read_cr0(vcpu);
8630
	sregs->cr2 = vcpu->arch.cr2;
8631
	sregs->cr3 = kvm_read_cr3(vcpu);
8632
	sregs->cr4 = kvm_read_cr4(vcpu);
8633
	sregs->cr8 = kvm_get_cr8(vcpu);
8634
	sregs->efer = vcpu->arch.efer;
8635 8636
	sregs->apic_base = kvm_get_apic_base(vcpu);

8637
	memset(sregs->interrupt_bitmap, 0, sizeof(sregs->interrupt_bitmap));
8638

8639
	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
8640 8641
		set_bit(vcpu->arch.interrupt.nr,
			(unsigned long *)sregs->interrupt_bitmap);
K
Ken Hofsass 已提交
8642
}
8643

K
Ken Hofsass 已提交
8644 8645 8646 8647 8648
int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	vcpu_load(vcpu);
	__get_sregs(vcpu, sregs);
8649
	vcpu_put(vcpu);
8650 8651 8652
	return 0;
}

8653 8654 8655
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
8656 8657
	vcpu_load(vcpu);

8658
	kvm_apic_accept_events(vcpu);
8659 8660 8661 8662 8663 8664
	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
					vcpu->arch.pv.pv_unhalted)
		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
	else
		mp_state->mp_state = vcpu->arch.mp_state;

8665
	vcpu_put(vcpu);
8666 8667 8668 8669 8670 8671
	return 0;
}

int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
8672 8673 8674 8675
	int ret = -EINVAL;

	vcpu_load(vcpu);

8676
	if (!lapic_in_kernel(vcpu) &&
8677
	    mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
8678
		goto out;
8679

8680 8681 8682 8683
	/* INITs are latched while in SMM */
	if ((is_smm(vcpu) || vcpu->arch.smi_pending) &&
	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
8684
		goto out;
8685

8686 8687 8688 8689 8690
	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
	} else
		vcpu->arch.mp_state = mp_state->mp_state;
8691
	kvm_make_request(KVM_REQ_EVENT, vcpu);
8692 8693 8694 8695 8696

	ret = 0;
out:
	vcpu_put(vcpu);
	return ret;
8697 8698
}

8699 8700
int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
		    int reason, bool has_error_code, u32 error_code)
8701
{
8702
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
8703
	int ret;
8704

8705
	init_emulate_ctxt(vcpu);
8706

8707
	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
8708
				   has_error_code, error_code);
8709 8710 8711 8712
	if (ret) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
8713
		return 0;
8714
	}
8715

8716 8717
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
8718
	kvm_make_request(KVM_REQ_EVENT, vcpu);
8719
	return 1;
8720 8721 8722
}
EXPORT_SYMBOL_GPL(kvm_task_switch);

P
Peng Hao 已提交
8723
static int kvm_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8724
{
8725
	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
8726 8727 8728 8729 8730
		/*
		 * When EFER.LME and CR0.PG are set, the processor is in
		 * 64-bit mode (though maybe in a 32-bit code segment).
		 * CR4.PAE and EFER.LMA must be set.
		 */
8731
		if (!(sregs->cr4 & X86_CR4_PAE)
8732 8733 8734 8735 8736 8737 8738 8739 8740 8741 8742
		    || !(sregs->efer & EFER_LMA))
			return -EINVAL;
	} else {
		/*
		 * Not in 64-bit mode: EFER.LMA is clear and the code
		 * segment cannot be 64-bit.
		 */
		if (sregs->efer & EFER_LMA || sregs->cs.l)
			return -EINVAL;
	}

8743
	return kvm_valid_cr4(vcpu, sregs->cr4);
8744 8745
}

K
Ken Hofsass 已提交
8746
static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8747
{
8748
	struct msr_data apic_base_msr;
8749
	int mmu_reset_needed = 0;
8750
	int cpuid_update_needed = 0;
8751
	int pending_vec, max_bits, idx;
8752
	struct desc_ptr dt;
8753 8754
	int ret = -EINVAL;

8755
	if (kvm_valid_sregs(vcpu, sregs))
8756
		goto out;
8757

8758 8759 8760
	apic_base_msr.data = sregs->apic_base;
	apic_base_msr.host_initiated = true;
	if (kvm_set_apic_base(vcpu, &apic_base_msr))
8761
		goto out;
8762

8763 8764
	dt.size = sregs->idt.limit;
	dt.address = sregs->idt.base;
8765
	kvm_x86_ops->set_idt(vcpu, &dt);
8766 8767
	dt.size = sregs->gdt.limit;
	dt.address = sregs->gdt.base;
8768 8769
	kvm_x86_ops->set_gdt(vcpu, &dt);

8770
	vcpu->arch.cr2 = sregs->cr2;
8771
	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
8772
	vcpu->arch.cr3 = sregs->cr3;
8773
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
8774

8775
	kvm_set_cr8(vcpu, sregs->cr8);
8776

8777
	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
8778 8779
	kvm_x86_ops->set_efer(vcpu, sregs->efer);

8780
	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
8781
	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
8782
	vcpu->arch.cr0 = sregs->cr0;
8783

8784
	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
8785 8786
	cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) &
				(X86_CR4_OSXSAVE | X86_CR4_PKE));
8787
	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
8788
	if (cpuid_update_needed)
A
Avi Kivity 已提交
8789
		kvm_update_cpuid(vcpu);
8790 8791

	idx = srcu_read_lock(&vcpu->kvm->srcu);
8792
	if (is_pae_paging(vcpu)) {
8793
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
8794 8795
		mmu_reset_needed = 1;
	}
8796
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
8797 8798 8799 8800

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

8801
	max_bits = KVM_NR_INTERRUPTS;
G
Gleb Natapov 已提交
8802 8803 8804
	pending_vec = find_first_bit(
		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
	if (pending_vec < max_bits) {
8805
		kvm_queue_interrupt(vcpu, pending_vec, false);
G
Gleb Natapov 已提交
8806
		pr_debug("Set back pending irq %d\n", pending_vec);
8807 8808
	}

8809 8810 8811 8812 8813 8814
	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8815

8816 8817
	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
8818

8819 8820
	update_cr8_intercept(vcpu);

M
Marcelo Tosatti 已提交
8821
	/* Older userspace won't unhalt the vcpu on reset. */
8822
	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
M
Marcelo Tosatti 已提交
8823
	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
8824
	    !is_protmode(vcpu))
M
Marcelo Tosatti 已提交
8825 8826
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;

8827 8828
	kvm_make_request(KVM_REQ_EVENT, vcpu);

8829 8830
	ret = 0;
out:
K
Ken Hofsass 已提交
8831 8832 8833 8834 8835 8836 8837 8838 8839 8840
	return ret;
}

int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	int ret;

	vcpu_load(vcpu);
	ret = __set_sregs(vcpu, sregs);
8841 8842
	vcpu_put(vcpu);
	return ret;
8843 8844
}

J
Jan Kiszka 已提交
8845 8846
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg)
8847
{
8848
	unsigned long rflags;
8849
	int i, r;
8850

8851 8852
	vcpu_load(vcpu);

8853 8854 8855
	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
		r = -EBUSY;
		if (vcpu->arch.exception.pending)
8856
			goto out;
8857 8858 8859 8860 8861 8862
		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
			kvm_queue_exception(vcpu, DB_VECTOR);
		else
			kvm_queue_exception(vcpu, BP_VECTOR);
	}

8863 8864 8865 8866 8867
	/*
	 * Read rflags as long as potentially injected trace flags are still
	 * filtered out.
	 */
	rflags = kvm_get_rflags(vcpu);
8868 8869 8870 8871 8872 8873

	vcpu->guest_debug = dbg->control;
	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
		vcpu->guest_debug = 0;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
8874 8875
		for (i = 0; i < KVM_NR_DB_REGS; ++i)
			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
8876
		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
8877 8878 8879 8880
	} else {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
	}
8881
	kvm_update_dr7(vcpu);
8882

J
Jan Kiszka 已提交
8883 8884 8885
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
			get_segment_base(vcpu, VCPU_SREG_CS);
8886

8887 8888 8889 8890 8891
	/*
	 * Trigger an rflags update that will inject or remove the trace
	 * flags.
	 */
	kvm_set_rflags(vcpu, rflags);
8892

8893
	kvm_x86_ops->update_bp_intercept(vcpu);
8894

8895
	r = 0;
J
Jan Kiszka 已提交
8896

8897
out:
8898
	vcpu_put(vcpu);
8899 8900 8901
	return r;
}

8902 8903 8904 8905 8906 8907 8908 8909
/*
 * Translate a guest virtual address to a guest physical address.
 */
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;
8910
	int idx;
8911

8912 8913
	vcpu_load(vcpu);

8914
	idx = srcu_read_lock(&vcpu->kvm->srcu);
8915
	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
8916
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
8917 8918 8919 8920 8921
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;

8922
	vcpu_put(vcpu);
8923 8924 8925
	return 0;
}

8926 8927
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
8928
	struct fxregs_state *fxsave;
8929

8930
	vcpu_load(vcpu);
8931

8932
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
8933 8934 8935 8936 8937 8938 8939
	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
8940
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
8941

8942
	vcpu_put(vcpu);
8943 8944 8945 8946 8947
	return 0;
}

int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
8948 8949 8950 8951
	struct fxregs_state *fxsave;

	vcpu_load(vcpu);

8952
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
8953 8954 8955 8956 8957 8958 8959 8960

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
8961
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
8962

8963
	vcpu_put(vcpu);
8964 8965 8966
	return 0;
}

K
Ken Hofsass 已提交
8967 8968 8969 8970 8971 8972 8973 8974 8975 8976 8977 8978 8979 8980 8981 8982 8983 8984 8985 8986 8987 8988 8989 8990 8991 8992 8993 8994 8995 8996 8997 8998 8999 9000 9001 9002 9003 9004 9005
static void store_regs(struct kvm_vcpu *vcpu)
{
	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
		__get_regs(vcpu, &vcpu->run->s.regs.regs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
		kvm_vcpu_ioctl_x86_get_vcpu_events(
				vcpu, &vcpu->run->s.regs.events);
}

static int sync_regs(struct kvm_vcpu *vcpu)
{
	if (vcpu->run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)
		return -EINVAL;

	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
		__set_regs(vcpu, &vcpu->run->s.regs.regs);
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
		if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
		if (kvm_vcpu_ioctl_x86_set_vcpu_events(
				vcpu, &vcpu->run->s.regs.events))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
	}

	return 0;
}

I
Ingo Molnar 已提交
9006
static void fx_init(struct kvm_vcpu *vcpu)
9007
{
9008
	fpstate_init(&vcpu->arch.guest_fpu->state);
9009
	if (boot_cpu_has(X86_FEATURE_XSAVES))
9010
		vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =
9011
			host_xcr0 | XSTATE_COMPACTION_ENABLED;
9012

9013 9014 9015
	/*
	 * Ensure guest xcr0 is valid for loading
	 */
D
Dave Hansen 已提交
9016
	vcpu->arch.xcr0 = XFEATURE_MASK_FP;
9017

9018
	vcpu->arch.cr0 |= X86_CR0_ET;
9019 9020
}

9021 9022
void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
{
9023 9024
	void *wbinvd_dirty_mask = vcpu->arch.wbinvd_dirty_mask;

9025
	kvmclock_reset(vcpu);
9026

9027
	kvm_x86_ops->vcpu_free(vcpu);
9028
	free_cpumask_var(wbinvd_dirty_mask);
9029 9030 9031 9032 9033
}

struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
						unsigned int id)
{
9034 9035
	struct kvm_vcpu *vcpu;

9036
	if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
Z
Zachary Amsden 已提交
9037 9038 9039
		printk_once(KERN_WARNING
		"kvm: SMP vm created on host with unstable TSC; "
		"guest TSC will not be reliable\n");
9040 9041 9042 9043

	vcpu = kvm_x86_ops->vcpu_create(kvm, id);

	return vcpu;
9044
}
9045

9046 9047
int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
{
9048
	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
9049
	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
X
Xiao Guangrong 已提交
9050
	kvm_vcpu_mtrr_init(vcpu);
9051
	vcpu_load(vcpu);
9052
	kvm_vcpu_reset(vcpu, false);
9053
	kvm_init_mmu(vcpu, false);
9054
	vcpu_put(vcpu);
9055
	return 0;
9056 9057
}

9058
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
9059
{
9060
	struct msr_data msr;
9061
	struct kvm *kvm = vcpu->kvm;
9062

9063 9064
	kvm_hv_vcpu_postcreate(vcpu);

9065
	if (mutex_lock_killable(&vcpu->mutex))
9066
		return;
9067
	vcpu_load(vcpu);
9068 9069 9070 9071
	msr.data = 0x0;
	msr.index = MSR_IA32_TSC;
	msr.host_initiated = true;
	kvm_write_tsc(vcpu, &msr);
9072
	vcpu_put(vcpu);
9073 9074 9075 9076

	/* poll control enabled by default */
	vcpu->arch.msr_kvm_poll_control = 1;

9077
	mutex_unlock(&vcpu->mutex);
9078

9079 9080 9081
	if (!kvmclock_periodic_sync)
		return;

9082 9083
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
9084 9085
}

9086
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
9087
{
9088 9089
	vcpu->arch.apf.msr_val = 0;

9090
	vcpu_load(vcpu);
9091 9092 9093 9094 9095 9096
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);

	kvm_x86_ops->vcpu_free(vcpu);
}

9097
void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
9098
{
9099 9100
	kvm_lapic_reset(vcpu, init_event);

9101 9102
	vcpu->arch.hflags = 0;

9103
	vcpu->arch.smi_pending = 0;
9104
	vcpu->arch.smi_count = 0;
A
Avi Kivity 已提交
9105 9106
	atomic_set(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = 0;
9107
	vcpu->arch.nmi_injected = false;
9108 9109
	kvm_clear_interrupt_queue(vcpu);
	kvm_clear_exception_queue(vcpu);
9110
	vcpu->arch.exception.pending = false;
9111

9112
	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
9113
	kvm_update_dr0123(vcpu);
9114
	vcpu->arch.dr6 = DR6_INIT;
J
Jan Kiszka 已提交
9115
	kvm_update_dr6(vcpu);
9116
	vcpu->arch.dr7 = DR7_FIXED_1;
9117
	kvm_update_dr7(vcpu);
9118

N
Nadav Amit 已提交
9119 9120
	vcpu->arch.cr2 = 0;

9121
	kvm_make_request(KVM_REQ_EVENT, vcpu);
9122
	vcpu->arch.apf.msr_val = 0;
G
Glauber Costa 已提交
9123
	vcpu->arch.st.msr_val = 0;
9124

9125 9126
	kvmclock_reset(vcpu);

9127 9128 9129
	kvm_clear_async_pf_completion_queue(vcpu);
	kvm_async_pf_hash_reset(vcpu);
	vcpu->arch.apf.halted = false;
9130

9131 9132 9133 9134 9135 9136 9137
	if (kvm_mpx_supported()) {
		void *mpx_state_buffer;

		/*
		 * To avoid have the INIT path from kvm_apic_has_events() that be
		 * called with loaded FPU and does not let userspace fix the state.
		 */
9138 9139
		if (init_event)
			kvm_put_guest_fpu(vcpu);
9140
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9141
					XFEATURE_BNDREGS);
9142 9143
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndreg_state));
9144
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9145
					XFEATURE_BNDCSR);
9146 9147
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndcsr));
9148 9149
		if (init_event)
			kvm_load_guest_fpu(vcpu);
9150 9151
	}

P
Paolo Bonzini 已提交
9152
	if (!init_event) {
9153
		kvm_pmu_reset(vcpu);
P
Paolo Bonzini 已提交
9154
		vcpu->arch.smbase = 0x30000;
K
Kyle Huey 已提交
9155 9156

		vcpu->arch.msr_misc_features_enables = 0;
9157 9158

		vcpu->arch.xcr0 = XFEATURE_MASK_FP;
P
Paolo Bonzini 已提交
9159
	}
9160

9161 9162 9163 9164
	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;

9165 9166
	vcpu->arch.ia32_xss = 0;

9167
	kvm_x86_ops->vcpu_reset(vcpu, init_event);
9168 9169
}

9170
void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
9171 9172 9173 9174 9175 9176 9177 9178
{
	struct kvm_segment cs;

	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs.selector = vector << 8;
	cs.base = vector << 12;
	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_rip_write(vcpu, 0);
9179 9180
}

9181
int kvm_arch_hardware_enable(void)
9182
{
9183 9184 9185
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;
9186 9187 9188 9189
	int ret;
	u64 local_tsc;
	u64 max_tsc = 0;
	bool stable, backwards_tsc = false;
A
Avi Kivity 已提交
9190 9191

	kvm_shared_msr_cpu_online();
9192
	ret = kvm_x86_ops->hardware_enable();
9193 9194 9195
	if (ret != 0)
		return ret;

9196
	local_tsc = rdtsc();
9197
	stable = !kvm_check_tsc_unstable();
9198 9199 9200
	list_for_each_entry(kvm, &vm_list, vm_list) {
		kvm_for_each_vcpu(i, vcpu, kvm) {
			if (!stable && vcpu->cpu == smp_processor_id())
9201
				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
9202 9203 9204 9205 9206 9207 9208 9209 9210 9211 9212 9213 9214 9215 9216 9217
			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
				backwards_tsc = true;
				if (vcpu->arch.last_host_tsc > max_tsc)
					max_tsc = vcpu->arch.last_host_tsc;
			}
		}
	}

	/*
	 * Sometimes, even reliable TSCs go backwards.  This happens on
	 * platforms that reset TSC during suspend or hibernate actions, but
	 * maintain synchronization.  We must compensate.  Fortunately, we can
	 * detect that condition here, which happens early in CPU bringup,
	 * before any KVM threads can be running.  Unfortunately, we can't
	 * bring the TSCs fully up to date with real time, as we aren't yet far
	 * enough into CPU bringup that we know how much real time has actually
9218
	 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot
9219 9220 9221 9222 9223 9224 9225 9226 9227 9228 9229 9230 9231 9232 9233 9234 9235 9236 9237 9238 9239 9240 9241 9242
	 * variables that haven't been updated yet.
	 *
	 * So we simply find the maximum observed TSC above, then record the
	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
	 * the adjustment will be applied.  Note that we accumulate
	 * adjustments, in case multiple suspend cycles happen before some VCPU
	 * gets a chance to run again.  In the event that no KVM threads get a
	 * chance to run, we will miss the entire elapsed period, as we'll have
	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
	 * loose cycle time.  This isn't too big a deal, since the loss will be
	 * uniform across all VCPUs (not to mention the scenario is extremely
	 * unlikely). It is possible that a second hibernate recovery happens
	 * much faster than a first, causing the observed TSC here to be
	 * smaller; this would require additional padding adjustment, which is
	 * why we set last_host_tsc to the local tsc observed here.
	 *
	 * N.B. - this code below runs only on platforms with reliable TSC,
	 * as that is the only way backwards_tsc is set above.  Also note
	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
	 * have the same delta_cyc adjustment applied if backwards_tsc
	 * is detected.  Note further, this adjustment is only done once,
	 * as we reset last_host_tsc on all VCPUs to stop this from being
	 * called multiple times (one for each physical CPU bringup).
	 *
G
Guo Chao 已提交
9243
	 * Platforms with unreliable TSCs don't have to deal with this, they
9244 9245 9246 9247 9248 9249 9250
	 * will be compensated by the logic in vcpu_load, which sets the TSC to
	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
	 * guarantee that they stay in perfect synchronization.
	 */
	if (backwards_tsc) {
		u64 delta_cyc = max_tsc - local_tsc;
		list_for_each_entry(kvm, &vm_list, vm_list) {
9251
			kvm->arch.backwards_tsc_observed = true;
9252 9253 9254
			kvm_for_each_vcpu(i, vcpu, kvm) {
				vcpu->arch.tsc_offset_adjustment += delta_cyc;
				vcpu->arch.last_host_tsc = local_tsc;
9255
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
9256 9257 9258 9259 9260 9261 9262 9263 9264 9265 9266 9267 9268 9269
			}

			/*
			 * We have to disable TSC offset matching.. if you were
			 * booting a VM while issuing an S4 host suspend....
			 * you may have some problem.  Solving this issue is
			 * left as an exercise to the reader.
			 */
			kvm->arch.last_tsc_nsec = 0;
			kvm->arch.last_tsc_write = 0;
		}

	}
	return 0;
9270 9271
}

9272
void kvm_arch_hardware_disable(void)
9273
{
9274 9275
	kvm_x86_ops->hardware_disable();
	drop_user_return_notifiers();
9276 9277 9278 9279
}

int kvm_arch_hardware_setup(void)
{
9280 9281 9282 9283 9284 9285
	int r;

	r = kvm_x86_ops->hardware_setup();
	if (r != 0)
		return r;

9286 9287 9288 9289
	if (kvm_has_tsc_control) {
		/*
		 * Make sure the user can only configure tsc_khz values that
		 * fit into a signed integer.
9290
		 * A min value is not calculated because it will always
9291 9292 9293 9294 9295 9296
		 * be 1 on all machines.
		 */
		u64 max = min(0x7fffffffULL,
			      __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz));
		kvm_max_guest_tsc_khz = max;

9297
		kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits;
9298
	}
9299

9300 9301
	kvm_init_msr_list();
	return 0;
9302 9303 9304 9305 9306 9307 9308
}

void kvm_arch_hardware_unsetup(void)
{
	kvm_x86_ops->hardware_unsetup();
}

9309
int kvm_arch_check_processor_compat(void)
9310
{
9311
	return kvm_x86_ops->check_processor_compatibility();
9312 9313 9314 9315 9316 9317 9318 9319 9320 9321 9322
}

bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
{
	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);

bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
9323 9324
}

9325
struct static_key kvm_no_apic_vcpu __read_mostly;
9326
EXPORT_SYMBOL_GPL(kvm_no_apic_vcpu);
9327

9328 9329 9330 9331 9332
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
	struct page *page;
	int r;

9333
	vcpu->arch.emulate_ctxt.ops = &emulate_ops;
9334
	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
9335
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
9336
	else
9337
		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
9338 9339 9340 9341 9342 9343

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
9344
	vcpu->arch.pio_data = page_address(page);
9345

9346
	kvm_set_tsc_khz(vcpu, max_tsc_khz);
Z
Zachary Amsden 已提交
9347

9348 9349 9350 9351
	r = kvm_mmu_create(vcpu);
	if (r < 0)
		goto fail_free_pio_data;

9352
	if (irqchip_in_kernel(vcpu->kvm)) {
9353
		vcpu->arch.apicv_active = kvm_x86_ops->get_enable_apicv(vcpu);
9354
		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
9355 9356
		if (r < 0)
			goto fail_mmu_destroy;
9357 9358
	} else
		static_key_slow_inc(&kvm_no_apic_vcpu);
9359

H
Huang Ying 已提交
9360
	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
9361
				       GFP_KERNEL_ACCOUNT);
H
Huang Ying 已提交
9362 9363
	if (!vcpu->arch.mce_banks) {
		r = -ENOMEM;
9364
		goto fail_free_lapic;
H
Huang Ying 已提交
9365 9366 9367
	}
	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;

9368 9369
	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
				GFP_KERNEL_ACCOUNT)) {
9370
		r = -ENOMEM;
9371
		goto fail_free_mce_banks;
9372
	}
9373

I
Ingo Molnar 已提交
9374
	fx_init(vcpu);
9375

9376
	vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
9377

9378 9379
	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);

9380 9381
	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;

9382
	kvm_async_pf_hash_reset(vcpu);
9383
	kvm_pmu_init(vcpu);
9384

9385
	vcpu->arch.pending_external_vector = -1;
9386
	vcpu->arch.preempted_in_kernel = false;
9387

9388 9389
	kvm_hv_vcpu_init(vcpu);

9390
	return 0;
I
Ingo Molnar 已提交
9391

9392 9393
fail_free_mce_banks:
	kfree(vcpu->arch.mce_banks);
9394 9395
fail_free_lapic:
	kvm_free_lapic(vcpu);
9396 9397 9398
fail_mmu_destroy:
	kvm_mmu_destroy(vcpu);
fail_free_pio_data:
9399
	free_page((unsigned long)vcpu->arch.pio_data);
9400 9401 9402 9403 9404 9405
fail:
	return r;
}

void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
{
9406 9407
	int idx;

A
Andrey Smetanin 已提交
9408
	kvm_hv_vcpu_uninit(vcpu);
9409
	kvm_pmu_destroy(vcpu);
9410
	kfree(vcpu->arch.mce_banks);
9411
	kvm_free_lapic(vcpu);
9412
	idx = srcu_read_lock(&vcpu->kvm->srcu);
9413
	kvm_mmu_destroy(vcpu);
9414
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9415
	free_page((unsigned long)vcpu->arch.pio_data);
9416
	if (!lapic_in_kernel(vcpu))
9417
		static_key_slow_dec(&kvm_no_apic_vcpu);
9418
}
9419

R
Radim Krčmář 已提交
9420 9421
void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
{
P
Paolo Bonzini 已提交
9422
	vcpu->arch.l1tf_flush_l1d = true;
9423
	kvm_x86_ops->sched_in(vcpu, cpu);
R
Radim Krčmář 已提交
9424 9425
}

9426
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
9427
{
9428 9429 9430
	if (type)
		return -EINVAL;

9431
	INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
9432
	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
9433
	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
B
Ben-Ami Yassour 已提交
9434
	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
9435
	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
9436

9437 9438
	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
9439 9440 9441
	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
		&kvm->arch.irq_sources_bitmap);
9442

9443
	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
9444
	mutex_init(&kvm->arch.apic_map_lock);
9445 9446
	spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);

9447
	kvm->arch.kvmclock_offset = -ktime_get_boottime_ns();
9448
	pvclock_update_vm_gtod_copy(kvm);
9449

9450 9451
	kvm->arch.guest_can_read_msr_platform_info = true;

9452
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
9453
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
9454

9455
	kvm_hv_init_vm(kvm);
9456
	kvm_page_track_init(kvm);
9457
	kvm_mmu_init_vm(kvm);
9458

9459
	return kvm_x86_ops->vm_init(kvm);
9460 9461 9462 9463
}

static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
9464
	vcpu_load(vcpu);
9465 9466 9467 9468 9469 9470 9471
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;
9472
	struct kvm_vcpu *vcpu;
9473 9474 9475 9476

	/*
	 * Unpin any mmu pages first.
	 */
9477 9478
	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_clear_async_pf_completion_queue(vcpu);
9479
		kvm_unload_vcpu_mmu(vcpu);
9480
	}
9481 9482 9483 9484 9485 9486
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_arch_vcpu_free(vcpu);

	mutex_lock(&kvm->lock);
	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
		kvm->vcpus[i] = NULL;
9487

9488 9489
	atomic_set(&kvm->online_vcpus, 0);
	mutex_unlock(&kvm->lock);
9490 9491
}

9492 9493
void kvm_arch_sync_events(struct kvm *kvm)
{
9494
	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
9495
	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
9496
	kvm_free_pit(kvm);
9497 9498
}

9499
int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9500 9501
{
	int i, r;
9502
	unsigned long hva;
9503 9504
	struct kvm_memslots *slots = kvm_memslots(kvm);
	struct kvm_memory_slot *slot, old;
9505 9506

	/* Called with kvm->slots_lock held.  */
9507 9508
	if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
		return -EINVAL;
9509

9510 9511
	slot = id_to_memslot(slots, id);
	if (size) {
9512
		if (slot->npages)
9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528 9529 9530
			return -EEXIST;

		/*
		 * MAP_SHARED to prevent internal slot pages from being moved
		 * by fork()/COW.
		 */
		hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
			      MAP_SHARED | MAP_ANONYMOUS, 0);
		if (IS_ERR((void *)hva))
			return PTR_ERR((void *)hva);
	} else {
		if (!slot->npages)
			return 0;

		hva = 0;
	}

	old = *slot;
9531
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
9532
		struct kvm_userspace_memory_region m;
9533

9534 9535 9536
		m.slot = id | (i << 16);
		m.flags = 0;
		m.guest_phys_addr = gpa;
9537
		m.userspace_addr = hva;
9538
		m.memory_size = size;
9539 9540 9541 9542 9543
		r = __kvm_set_memory_region(kvm, &m);
		if (r < 0)
			return r;
	}

9544 9545
	if (!size)
		vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE);
9546

9547 9548 9549 9550
	return 0;
}
EXPORT_SYMBOL_GPL(__x86_set_memory_region);

9551
int x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9552 9553 9554 9555
{
	int r;

	mutex_lock(&kvm->slots_lock);
9556
	r = __x86_set_memory_region(kvm, id, gpa, size);
9557 9558 9559 9560 9561 9562
	mutex_unlock(&kvm->slots_lock);

	return r;
}
EXPORT_SYMBOL_GPL(x86_set_memory_region);

9563 9564
void kvm_arch_destroy_vm(struct kvm *kvm)
{
9565 9566 9567 9568 9569 9570
	if (current->mm == kvm->mm) {
		/*
		 * Free memory regions allocated on behalf of userspace,
		 * unless the the memory map has changed due to process exit
		 * or fd copying.
		 */
9571 9572 9573
		x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0);
		x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 0, 0);
		x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
9574
	}
9575 9576
	if (kvm_x86_ops->vm_destroy)
		kvm_x86_ops->vm_destroy(kvm);
9577 9578
	kvm_pic_destroy(kvm);
	kvm_ioapic_destroy(kvm);
9579
	kvm_free_vcpus(kvm);
9580
	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
E
Eric Hankland 已提交
9581
	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
9582
	kvm_mmu_uninit_vm(kvm);
9583
	kvm_page_track_cleanup(kvm);
9584
	kvm_hv_destroy_vm(kvm);
9585
}
9586

9587
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
9588 9589 9590 9591
			   struct kvm_memory_slot *dont)
{
	int i;

9592 9593
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
		if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
T
Thomas Huth 已提交
9594
			kvfree(free->arch.rmap[i]);
9595
			free->arch.rmap[i] = NULL;
9596
		}
9597 9598 9599 9600 9601
		if (i == 0)
			continue;

		if (!dont || free->arch.lpage_info[i - 1] !=
			     dont->arch.lpage_info[i - 1]) {
T
Thomas Huth 已提交
9602
			kvfree(free->arch.lpage_info[i - 1]);
9603
			free->arch.lpage_info[i - 1] = NULL;
9604 9605
		}
	}
9606 9607

	kvm_page_track_free_memslot(free, dont);
9608 9609
}

9610 9611
int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
			    unsigned long npages)
9612 9613 9614
{
	int i;

9615
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
9616
		struct kvm_lpage_info *linfo;
9617 9618
		unsigned long ugfn;
		int lpages;
9619
		int level = i + 1;
9620 9621 9622 9623

		lpages = gfn_to_index(slot->base_gfn + npages - 1,
				      slot->base_gfn, level) + 1;

9624
		slot->arch.rmap[i] =
K
Kees Cook 已提交
9625
			kvcalloc(lpages, sizeof(*slot->arch.rmap[i]),
9626
				 GFP_KERNEL_ACCOUNT);
9627
		if (!slot->arch.rmap[i])
9628
			goto out_free;
9629 9630
		if (i == 0)
			continue;
9631

9632
		linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
9633
		if (!linfo)
9634 9635
			goto out_free;

9636 9637
		slot->arch.lpage_info[i - 1] = linfo;

9638
		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
9639
			linfo[0].disallow_lpage = 1;
9640
		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
9641
			linfo[lpages - 1].disallow_lpage = 1;
9642 9643 9644 9645 9646 9647 9648 9649 9650 9651 9652
		ugfn = slot->userspace_addr >> PAGE_SHIFT;
		/*
		 * If the gfn and userspace address are not aligned wrt each
		 * other, or if explicitly asked to, disable large page
		 * support for this slot
		 */
		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
		    !kvm_largepages_enabled()) {
			unsigned long j;

			for (j = 0; j < lpages; ++j)
9653
				linfo[j].disallow_lpage = 1;
9654 9655 9656
		}
	}

9657 9658 9659
	if (kvm_page_track_create_memslot(slot, npages))
		goto out_free;

9660 9661 9662
	return 0;

out_free:
9663
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
T
Thomas Huth 已提交
9664
		kvfree(slot->arch.rmap[i]);
9665 9666 9667 9668
		slot->arch.rmap[i] = NULL;
		if (i == 0)
			continue;

T
Thomas Huth 已提交
9669
		kvfree(slot->arch.lpage_info[i - 1]);
9670
		slot->arch.lpage_info[i - 1] = NULL;
9671 9672 9673 9674
	}
	return -ENOMEM;
}

9675
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
9676
{
9677 9678 9679 9680
	/*
	 * memslots->generation has been incremented.
	 * mmio generation may have reached its maximum value.
	 */
9681
	kvm_mmu_invalidate_mmio_sptes(kvm, gen);
9682 9683
}

9684 9685
int kvm_arch_prepare_memory_region(struct kvm *kvm,
				struct kvm_memory_slot *memslot,
9686
				const struct kvm_userspace_memory_region *mem,
9687
				enum kvm_mr_change change)
9688
{
9689 9690 9691
	return 0;
}

9692 9693 9694 9695 9696 9697 9698 9699 9700 9701 9702 9703 9704 9705 9706 9707 9708 9709 9710 9711 9712
static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
				     struct kvm_memory_slot *new)
{
	/* Still write protect RO slot */
	if (new->flags & KVM_MEM_READONLY) {
		kvm_mmu_slot_remove_write_access(kvm, new);
		return;
	}

	/*
	 * Call kvm_x86_ops dirty logging hooks when they are valid.
	 *
	 * kvm_x86_ops->slot_disable_log_dirty is called when:
	 *
	 *  - KVM_MR_CREATE with dirty logging is disabled
	 *  - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag
	 *
	 * The reason is, in case of PML, we need to set D-bit for any slots
	 * with dirty logging disabled in order to eliminate unnecessary GPA
	 * logging in PML buffer (and potential PML buffer full VMEXT). This
	 * guarantees leaving PML enabled during guest's lifetime won't have
W
Wei Yang 已提交
9713
	 * any additional overhead from PML when guest is running with dirty
9714 9715 9716 9717 9718 9719 9720 9721 9722 9723 9724 9725 9726 9727 9728 9729 9730 9731 9732 9733 9734 9735 9736 9737 9738 9739 9740 9741
	 * logging disabled for memory slots.
	 *
	 * kvm_x86_ops->slot_enable_log_dirty is called when switching new slot
	 * to dirty logging mode.
	 *
	 * If kvm_x86_ops dirty logging hooks are invalid, use write protect.
	 *
	 * In case of write protect:
	 *
	 * Write protect all pages for dirty logging.
	 *
	 * All the sptes including the large sptes which point to this
	 * slot are set to readonly. We can not create any new large
	 * spte on this slot until the end of the logging.
	 *
	 * See the comments in fast_page_fault().
	 */
	if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
		if (kvm_x86_ops->slot_enable_log_dirty)
			kvm_x86_ops->slot_enable_log_dirty(kvm, new);
		else
			kvm_mmu_slot_remove_write_access(kvm, new);
	} else {
		if (kvm_x86_ops->slot_disable_log_dirty)
			kvm_x86_ops->slot_disable_log_dirty(kvm, new);
	}
}

9742
void kvm_arch_commit_memory_region(struct kvm *kvm,
9743
				const struct kvm_userspace_memory_region *mem,
9744
				const struct kvm_memory_slot *old,
9745
				const struct kvm_memory_slot *new,
9746
				enum kvm_mr_change change)
9747
{
9748
	if (!kvm->arch.n_requested_mmu_pages)
9749 9750
		kvm_mmu_change_mmu_pages(kvm,
				kvm_mmu_calculate_default_mmu_pages(kvm));
9751

9752 9753 9754 9755 9756 9757 9758 9759 9760 9761 9762
	/*
	 * Dirty logging tracks sptes in 4k granularity, meaning that large
	 * sptes have to be split.  If live migration is successful, the guest
	 * in the source machine will be destroyed and large sptes will be
	 * created in the destination. However, if the guest continues to run
	 * in the source machine (for example if live migration fails), small
	 * sptes will remain around and cause bad performance.
	 *
	 * Scan sptes if dirty logging has been stopped, dropping those
	 * which can be collapsed into a single large-page spte.  Later
	 * page faults will create the large-page sptes.
9763 9764 9765 9766 9767
	 *
	 * There is no need to do this in any of the following cases:
	 * CREATE:	No dirty mappings will already exist.
	 * MOVE/DELETE:	The old mappings will already have been cleaned up by
	 *		kvm_arch_flush_shadow_memslot()
9768
	 */
9769
	if (change == KVM_MR_FLAGS_ONLY &&
9770 9771 9772 9773
		(old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
		!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
		kvm_mmu_zap_collapsible_sptes(kvm, new);

9774
	/*
9775
	 * Set up write protection and/or dirty logging for the new slot.
9776
	 *
9777 9778 9779 9780
	 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have
	 * been zapped so no dirty logging staff is needed for old slot. For
	 * KVM_MR_FLAGS_ONLY, the old slot is essentially the same one as the
	 * new and it's also covered when dealing with the new slot.
9781 9782
	 *
	 * FIXME: const-ify all uses of struct kvm_memory_slot.
9783
	 */
9784
	if (change != KVM_MR_DELETE)
9785
		kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new);
9786
}
9787

9788
void kvm_arch_flush_shadow_all(struct kvm *kvm)
9789
{
9790
	kvm_mmu_zap_all(kvm);
9791 9792
}

9793 9794 9795
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
9796
	kvm_page_track_flush_slot(kvm, slot);
9797 9798
}

9799 9800 9801 9802 9803 9804 9805
static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
{
	return (is_guest_mode(vcpu) &&
			kvm_x86_ops->guest_apic_has_interrupt &&
			kvm_x86_ops->guest_apic_has_interrupt(vcpu));
}

9806 9807 9808 9809 9810 9811 9812 9813 9814 9815 9816
static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
{
	if (!list_empty_careful(&vcpu->async_pf.done))
		return true;

	if (kvm_apic_has_events(vcpu))
		return true;

	if (vcpu->arch.pv.pv_unhalted)
		return true;

9817 9818 9819
	if (vcpu->arch.exception.pending)
		return true;

9820 9821 9822
	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
	    (vcpu->arch.nmi_pending &&
	     kvm_x86_ops->nmi_allowed(vcpu)))
9823 9824
		return true;

9825 9826
	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
	    (vcpu->arch.smi_pending && !is_smm(vcpu)))
P
Paolo Bonzini 已提交
9827 9828
		return true;

9829
	if (kvm_arch_interrupt_allowed(vcpu) &&
9830 9831
	    (kvm_cpu_has_interrupt(vcpu) ||
	    kvm_guest_apic_has_interrupt(vcpu)))
9832 9833
		return true;

A
Andrey Smetanin 已提交
9834 9835 9836
	if (kvm_hv_has_stimer_pending(vcpu))
		return true;

9837 9838 9839
	return false;
}

9840 9841
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
9842
	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
9843
}
9844

9845 9846 9847 9848 9849 9850 9851 9852 9853 9854 9855 9856 9857 9858 9859 9860
bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
{
	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
		return true;

	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
		kvm_test_request(KVM_REQ_SMI, vcpu) ||
		 kvm_test_request(KVM_REQ_EVENT, vcpu))
		return true;

	if (vcpu->arch.apicv_active && kvm_x86_ops->dy_apicv_has_pending_interrupt(vcpu))
		return true;

	return false;
}

9861 9862
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
{
9863
	return vcpu->arch.preempted_in_kernel;
9864 9865
}

9866
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
9867
{
9868
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
9869
}
9870 9871 9872 9873 9874

int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	return kvm_x86_ops->interrupt_allowed(vcpu);
}
9875

9876
unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
J
Jan Kiszka 已提交
9877
{
9878 9879 9880 9881 9882 9883
	if (is_64_bit_mode(vcpu))
		return kvm_rip_read(vcpu);
	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
		     kvm_rip_read(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
J
Jan Kiszka 已提交
9884

9885 9886 9887
bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
{
	return kvm_get_linear_rip(vcpu) == linear_rip;
J
Jan Kiszka 已提交
9888 9889 9890
}
EXPORT_SYMBOL_GPL(kvm_is_linear_rip);

9891 9892 9893 9894 9895 9896
unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
{
	unsigned long rflags;

	rflags = kvm_x86_ops->get_rflags(vcpu);
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
9897
		rflags &= ~X86_EFLAGS_TF;
9898 9899 9900 9901
	return rflags;
}
EXPORT_SYMBOL_GPL(kvm_get_rflags);

9902
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
9903 9904
{
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
J
Jan Kiszka 已提交
9905
	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
9906
		rflags |= X86_EFLAGS_TF;
9907
	kvm_x86_ops->set_rflags(vcpu, rflags);
9908 9909 9910 9911 9912
}

void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
	__kvm_set_rflags(vcpu, rflags);
9913
	kvm_make_request(KVM_REQ_EVENT, vcpu);
9914 9915 9916
}
EXPORT_SYMBOL_GPL(kvm_set_rflags);

G
Gleb Natapov 已提交
9917 9918 9919 9920
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
{
	int r;

9921
	if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||
9922
	      work->wakeup_all)
G
Gleb Natapov 已提交
9923 9924 9925 9926 9927 9928
		return;

	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		return;

9929 9930
	if (!vcpu->arch.mmu->direct_map &&
	      work->arch.cr3 != vcpu->arch.mmu->get_cr3(vcpu))
X
Xiao Guangrong 已提交
9931 9932
		return;

9933
	vcpu->arch.mmu->page_fault(vcpu, work->gva, 0, true);
G
Gleb Natapov 已提交
9934 9935
}

9936 9937 9938 9939 9940 9941 9942 9943 9944 9945 9946 9947 9948 9949 9950 9951 9952 9953 9954 9955 9956 9957 9958 9959 9960 9961
static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
{
	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
}

static inline u32 kvm_async_pf_next_probe(u32 key)
{
	return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
}

static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 key = kvm_async_pf_hash_fn(gfn);

	while (vcpu->arch.apf.gfns[key] != ~0)
		key = kvm_async_pf_next_probe(key);

	vcpu->arch.apf.gfns[key] = gfn;
}

static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	int i;
	u32 key = kvm_async_pf_hash_fn(gfn);

	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
9962 9963
		     (vcpu->arch.apf.gfns[key] != gfn &&
		      vcpu->arch.apf.gfns[key] != ~0); i++)
9964 9965 9966 9967 9968 9969 9970 9971 9972 9973 9974 9975 9976 9977 9978 9979 9980 9981 9982 9983 9984 9985 9986 9987 9988 9989 9990 9991 9992 9993 9994 9995 9996
		key = kvm_async_pf_next_probe(key);

	return key;
}

bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
}

static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 i, j, k;

	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
	while (true) {
		vcpu->arch.apf.gfns[i] = ~0;
		do {
			j = kvm_async_pf_next_probe(j);
			if (vcpu->arch.apf.gfns[j] == ~0)
				return;
			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
			/*
			 * k lies cyclically in ]i,j]
			 * |    i.k.j |
			 * |....j i.k.| or  |.k..j i...|
			 */
		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
		i = j;
	}
}

9997 9998
static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
{
9999 10000 10001

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
				      sizeof(val));
10002 10003
}

10004 10005 10006 10007 10008 10009 10010
static int apf_get_user(struct kvm_vcpu *vcpu, u32 *val)
{

	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, val,
				      sizeof(u32));
}

10011 10012 10013 10014 10015 10016 10017 10018 10019 10020 10021 10022 10023 10024 10025 10026 10027 10028 10029 10030 10031 10032 10033 10034 10035 10036 10037 10038 10039 10040
static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
{
	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
		return false;

	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
	    (vcpu->arch.apf.send_user_only &&
	     kvm_x86_ops->get_cpl(vcpu) == 0))
		return false;

	return true;
}

bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
{
	if (unlikely(!lapic_in_kernel(vcpu) ||
		     kvm_event_needs_reinjection(vcpu) ||
		     vcpu->arch.exception.pending))
		return false;

	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
		return false;

	/*
	 * If interrupts are off we cannot even use an artificial
	 * halt state.
	 */
	return kvm_x86_ops->interrupt_allowed(vcpu);
}

10041 10042 10043
void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
				     struct kvm_async_pf *work)
{
10044 10045
	struct x86_exception fault;

10046
	trace_kvm_async_pf_not_present(work->arch.token, work->gva);
10047
	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
10048

10049 10050
	if (kvm_can_deliver_async_pf(vcpu) &&
	    !apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
10051 10052 10053 10054 10055
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
10056
		fault.async_page_fault = true;
10057
		kvm_inject_page_fault(vcpu, &fault);
10058 10059 10060 10061 10062 10063 10064 10065 10066 10067
	} else {
		/*
		 * It is not possible to deliver a paravirtualized asynchronous
		 * page fault, but putting the guest in an artificial halt state
		 * can be beneficial nevertheless: if an interrupt arrives, we
		 * can deliver it timely and perhaps the guest will schedule
		 * another process.  When the instruction that triggered a page
		 * fault is retried, hopefully the page will be ready in the host.
		 */
		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
10068
	}
10069 10070 10071 10072 10073
}

void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
				 struct kvm_async_pf *work)
{
10074
	struct x86_exception fault;
10075
	u32 val;
10076

10077
	if (work->wakeup_all)
10078 10079 10080
		work->arch.token = ~0; /* broadcast wakeup */
	else
		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
10081
	trace_kvm_async_pf_ready(work->arch.token, work->gva);
10082

10083 10084 10085 10086 10087 10088 10089 10090 10091 10092 10093
	if (vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED &&
	    !apf_get_user(vcpu, &val)) {
		if (val == KVM_PV_REASON_PAGE_NOT_PRESENT &&
		    vcpu->arch.exception.pending &&
		    vcpu->arch.exception.nr == PF_VECTOR &&
		    !apf_put_user(vcpu, 0)) {
			vcpu->arch.exception.injected = false;
			vcpu->arch.exception.pending = false;
			vcpu->arch.exception.nr = 0;
			vcpu->arch.exception.has_error_code = false;
			vcpu->arch.exception.error_code = 0;
10094 10095
			vcpu->arch.exception.has_payload = false;
			vcpu->arch.exception.payload = 0;
10096 10097 10098 10099 10100 10101 10102 10103 10104
		} else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
			fault.vector = PF_VECTOR;
			fault.error_code_valid = true;
			fault.error_code = 0;
			fault.nested_page_fault = false;
			fault.address = work->arch.token;
			fault.async_page_fault = true;
			kvm_inject_page_fault(vcpu, &fault);
		}
10105
	}
10106
	vcpu->arch.apf.halted = false;
10107
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10108 10109 10110 10111 10112 10113 10114
}

bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
		return true;
	else
10115
		return kvm_can_do_async_pf(vcpu);
10116 10117
}

10118 10119 10120 10121 10122 10123 10124 10125 10126 10127 10128 10129 10130 10131 10132 10133 10134 10135
void kvm_arch_start_assignment(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);

void kvm_arch_end_assignment(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);

bool kvm_arch_has_assigned_device(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);

10136 10137 10138 10139 10140 10141 10142 10143 10144 10145 10146 10147 10148 10149 10150 10151 10152 10153
void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);

void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);

bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);

10154 10155
bool kvm_arch_has_irq_bypass(void)
{
10156
	return true;
10157 10158
}

F
Feng Wu 已提交
10159 10160 10161 10162 10163 10164
int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

10165
	irqfd->producer = prod;
F
Feng Wu 已提交
10166

10167 10168
	return kvm_x86_ops->update_pi_irte(irqfd->kvm,
					   prod->irq, irqfd->gsi, 1);
F
Feng Wu 已提交
10169 10170 10171 10172 10173 10174 10175 10176 10177 10178 10179 10180 10181 10182 10183
}

void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	int ret;
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

	WARN_ON(irqfd->producer != prod);
	irqfd->producer = NULL;

	/*
	 * When producer of consumer is unregistered, we change back to
	 * remapped mode, so we can re-use the current implementation
A
Andrea Gelmini 已提交
10184
	 * when the irq is masked/disabled or the consumer side (KVM
F
Feng Wu 已提交
10185 10186 10187 10188 10189 10190 10191 10192 10193 10194 10195 10196 10197 10198
	 * int this case doesn't want to receive the interrupts.
	*/
	ret = kvm_x86_ops->update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0);
	if (ret)
		printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
		       " fails: %d\n", irqfd->consumer.token, ret);
}

int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
				   uint32_t guest_irq, bool set)
{
	return kvm_x86_ops->update_pi_irte(kvm, host_irq, guest_irq, set);
}

10199 10200 10201 10202 10203 10204
bool kvm_vector_hashing_enabled(void)
{
	return vector_hashing;
}
EXPORT_SYMBOL_GPL(kvm_vector_hashing_enabled);

10205 10206 10207 10208 10209 10210 10211
bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
}
EXPORT_SYMBOL_GPL(kvm_arch_no_poll);


10212
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
J
Jason Wang 已提交
10213
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
10214 10215 10216 10217
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
10218
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
10219
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
10220
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
10221
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
10222
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
10223
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
10224
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
10225
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
10226
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
P
Peter Xu 已提交
10227
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
K
Kai Huang 已提交
10228
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
10229
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
10230 10231
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);