x86.c 256.8 KB
Newer Older
1 2 3 4 5 6
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
B
Ben-Ami Yassour 已提交
7 8
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
N
Nicolas Kaiser 已提交
9
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
10 11 12 13
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
B
Ben-Ami Yassour 已提交
14 15
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
16 17 18 19 20 21
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

22
#include <linux/kvm_host.h>
23
#include "irq.h"
24
#include "mmu.h"
S
Sheng Yang 已提交
25
#include "i8254.h"
26
#include "tss.h"
27
#include "kvm_cache_regs.h"
28
#include "x86.h"
A
Avi Kivity 已提交
29
#include "cpuid.h"
30
#include "pmu.h"
31
#include "hyperv.h"
32

33
#include <linux/clocksource.h>
B
Ben-Ami Yassour 已提交
34
#include <linux/interrupt.h>
35 36 37
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
38 39
#include <linux/export.h>
#include <linux/moduleparam.h>
40
#include <linux/mman.h>
41
#include <linux/highmem.h>
J
Joerg Roedel 已提交
42
#include <linux/iommu.h>
B
Ben-Ami Yassour 已提交
43
#include <linux/intel-iommu.h>
44
#include <linux/cpufreq.h>
A
Avi Kivity 已提交
45
#include <linux/user-return-notifier.h>
46
#include <linux/srcu.h>
47
#include <linux/slab.h>
48
#include <linux/perf_event.h>
49
#include <linux/uaccess.h>
50
#include <linux/hash.h>
51
#include <linux/pci.h>
52 53
#include <linux/timekeeper_internal.h>
#include <linux/pvclock_gtod.h>
F
Feng Wu 已提交
54 55
#include <linux/kvm_irqfd.h>
#include <linux/irqbypass.h>
56
#include <linux/sched/stat.h>
57
#include <linux/mem_encrypt.h>
58

A
Avi Kivity 已提交
59
#include <trace/events/kvm.h>
X
Xiao Guangrong 已提交
60

61
#include <asm/debugreg.h>
62
#include <asm/msr.h>
63
#include <asm/desc.h>
H
Huang Ying 已提交
64
#include <asm/mce.h>
65
#include <linux/kernel_stat.h>
66
#include <asm/fpu/internal.h> /* Ugh! */
67
#include <asm/pvclock.h>
68
#include <asm/div64.h>
69
#include <asm/irq_remapping.h>
70
#include <asm/mshyperv.h>
71
#include <asm/hypervisor.h>
72
#include <asm/intel_pt.h>
73

74 75 76
#define CREATE_TRACE_POINTS
#include "trace.h"

77
#define MAX_IO_MSRS 256
H
Huang Ying 已提交
78
#define KVM_MAX_MCE_BANKS 32
79 80
u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P;
EXPORT_SYMBOL_GPL(kvm_mce_cap_supported);
H
Huang Ying 已提交
81

82 83 84
#define emul_to_vcpu(ctxt) \
	container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)

85 86 87 88 89
/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
90 91
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
92
#else
93
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
94
#endif
95

96 97
#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
98

99 100
#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
                                    KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
101

102
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
A
Avi Kivity 已提交
103
static void process_nmi(struct kvm_vcpu *vcpu);
104
static void enter_smm(struct kvm_vcpu *vcpu);
105
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
K
Ken Hofsass 已提交
106 107
static void store_regs(struct kvm_vcpu *vcpu);
static int sync_regs(struct kvm_vcpu *vcpu);
108

109
struct kvm_x86_ops *kvm_x86_ops __read_mostly;
110
EXPORT_SYMBOL_GPL(kvm_x86_ops);
111

112
static bool __read_mostly ignore_msrs = 0;
113
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
114

115 116 117
static bool __read_mostly report_ignored_msrs = true;
module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);

118
unsigned int min_timer_period_us = 200;
119 120
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

121 122 123
static bool __read_mostly kvmclock_periodic_sync = true;
module_param(kvmclock_periodic_sync, bool, S_IRUGO);

124
bool __read_mostly kvm_has_tsc_control;
125
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
126
u32  __read_mostly kvm_max_guest_tsc_khz;
127
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
128 129 130 131
u8   __read_mostly kvm_tsc_scaling_ratio_frac_bits;
EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits);
u64  __read_mostly kvm_max_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
132 133
u64 __read_mostly kvm_default_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
134

135
/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
136
static u32 __read_mostly tsc_tolerance_ppm = 250;
137 138
module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);

139 140 141 142 143 144 145
/*
 * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
 * adaptive tuning starting from default advancment of 1000ns.  '0' disables
 * advancement entirely.  Any other value is used as-is and disables adaptive
 * tuning, i.e. allows priveleged userspace to set an exact advancement time.
 */
static int __read_mostly lapic_timer_advance_ns = -1;
146
module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
147

148 149 150
static bool __read_mostly vector_hashing = true;
module_param(vector_hashing, bool, S_IRUGO);

151 152 153 154
bool __read_mostly enable_vmware_backdoor = false;
module_param(enable_vmware_backdoor, bool, S_IRUGO);
EXPORT_SYMBOL_GPL(enable_vmware_backdoor);

155 156 157
static bool __read_mostly force_emulation_prefix = false;
module_param(force_emulation_prefix, bool, S_IRUGO);

A
Avi Kivity 已提交
158 159 160 161
#define KVM_NR_SHARED_MSRS 16

struct kvm_shared_msrs_global {
	int nr;
162
	u32 msrs[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
163 164 165 166 167
};

struct kvm_shared_msrs {
	struct user_return_notifier urn;
	bool registered;
168 169 170 171
	struct kvm_shared_msr_values {
		u64 host;
		u64 curr;
	} values[KVM_NR_SHARED_MSRS];
A
Avi Kivity 已提交
172 173 174
};

static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
175
static struct kvm_shared_msrs __percpu *shared_msrs;
A
Avi Kivity 已提交
176

177
struct kvm_stats_debugfs_item debugfs_entries[] = {
178 179 180 181 182 183 184 185 186
	{ "pf_fixed", VCPU_STAT(pf_fixed) },
	{ "pf_guest", VCPU_STAT(pf_guest) },
	{ "tlb_flush", VCPU_STAT(tlb_flush) },
	{ "invlpg", VCPU_STAT(invlpg) },
	{ "exits", VCPU_STAT(exits) },
	{ "io_exits", VCPU_STAT(io_exits) },
	{ "mmio_exits", VCPU_STAT(mmio_exits) },
	{ "signal_exits", VCPU_STAT(signal_exits) },
	{ "irq_window", VCPU_STAT(irq_window_exits) },
187
	{ "nmi_window", VCPU_STAT(nmi_window_exits) },
188
	{ "halt_exits", VCPU_STAT(halt_exits) },
189
	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
190
	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
191
	{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
192
	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
A
Amit Shah 已提交
193
	{ "hypercalls", VCPU_STAT(hypercalls) },
194 195 196 197 198 199
	{ "request_irq", VCPU_STAT(request_irq_exits) },
	{ "irq_exits", VCPU_STAT(irq_exits) },
	{ "host_state_reload", VCPU_STAT(host_state_reload) },
	{ "fpu_reload", VCPU_STAT(fpu_reload) },
	{ "insn_emulation", VCPU_STAT(insn_emulation) },
	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
200
	{ "irq_injections", VCPU_STAT(irq_injections) },
201
	{ "nmi_injections", VCPU_STAT(nmi_injections) },
202
	{ "req_event", VCPU_STAT(req_event) },
P
Paolo Bonzini 已提交
203
	{ "l1d_flush", VCPU_STAT(l1d_flush) },
A
Avi Kivity 已提交
204 205 206 207 208 209
	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
	{ "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
	{ "mmu_flooded", VM_STAT(mmu_flooded) },
	{ "mmu_recycled", VM_STAT(mmu_recycled) },
A
Avi Kivity 已提交
210
	{ "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
211
	{ "mmu_unsync", VM_STAT(mmu_unsync) },
212
	{ "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
M
Marcelo Tosatti 已提交
213
	{ "largepages", VM_STAT(lpages) },
214 215
	{ "max_mmu_page_hash_collisions",
		VM_STAT(max_mmu_page_hash_collisions) },
216 217 218
	{ NULL }
};

219 220
u64 __read_mostly host_xcr0;

221 222 223
struct kmem_cache *x86_fpu_cache;
EXPORT_SYMBOL_GPL(x86_fpu_cache);

224
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
225

226 227 228 229 230 231 232
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
	int i;
	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
		vcpu->arch.apf.gfns[i] = ~0;
}

A
Avi Kivity 已提交
233 234 235 236 237
static void kvm_on_user_return(struct user_return_notifier *urn)
{
	unsigned slot;
	struct kvm_shared_msrs *locals
		= container_of(urn, struct kvm_shared_msrs, urn);
238
	struct kvm_shared_msr_values *values;
239 240 241 242 243 244 245 246 247 248 249 250
	unsigned long flags;

	/*
	 * Disabling irqs at this point since the following code could be
	 * interrupted and executed through kvm_arch_hardware_disable()
	 */
	local_irq_save(flags);
	if (locals->registered) {
		locals->registered = false;
		user_return_notifier_unregister(urn);
	}
	local_irq_restore(flags);
A
Avi Kivity 已提交
251
	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
252 253 254 255
		values = &locals->values[slot];
		if (values->host != values->curr) {
			wrmsrl(shared_msrs_global.msrs[slot], values->host);
			values->curr = values->host;
A
Avi Kivity 已提交
256 257 258 259
		}
	}
}

260
static void shared_msr_update(unsigned slot, u32 msr)
A
Avi Kivity 已提交
261 262
{
	u64 value;
263 264
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
A
Avi Kivity 已提交
265

266 267 268 269 270 271 272 273 274 275 276 277 278
	/* only read, and nobody should modify it at this time,
	 * so don't need lock */
	if (slot >= shared_msrs_global.nr) {
		printk(KERN_ERR "kvm: invalid MSR slot!");
		return;
	}
	rdmsrl_safe(msr, &value);
	smsr->values[slot].host = value;
	smsr->values[slot].curr = value;
}

void kvm_define_shared_msr(unsigned slot, u32 msr)
{
279
	BUG_ON(slot >= KVM_NR_SHARED_MSRS);
280
	shared_msrs_global.msrs[slot] = msr;
A
Avi Kivity 已提交
281 282 283 284 285 286 287 288 289 290
	if (slot >= shared_msrs_global.nr)
		shared_msrs_global.nr = slot + 1;
}
EXPORT_SYMBOL_GPL(kvm_define_shared_msr);

static void kvm_shared_msr_cpu_online(void)
{
	unsigned i;

	for (i = 0; i < shared_msrs_global.nr; ++i)
291
		shared_msr_update(i, shared_msrs_global.msrs[i]);
A
Avi Kivity 已提交
292 293
}

294
int kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
A
Avi Kivity 已提交
295
{
296 297
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
298
	int err;
A
Avi Kivity 已提交
299

300
	if (((value ^ smsr->values[slot].curr) & mask) == 0)
301
		return 0;
302
	smsr->values[slot].curr = value;
303 304 305 306
	err = wrmsrl_safe(shared_msrs_global.msrs[slot], value);
	if (err)
		return 1;

A
Avi Kivity 已提交
307 308 309 310 311
	if (!smsr->registered) {
		smsr->urn.on_user_return = kvm_on_user_return;
		user_return_notifier_register(&smsr->urn);
		smsr->registered = true;
	}
312
	return 0;
A
Avi Kivity 已提交
313 314 315
}
EXPORT_SYMBOL_GPL(kvm_set_shared_msr);

316
static void drop_user_return_notifiers(void)
317
{
318 319
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
320 321 322 323 324

	if (smsr->registered)
		kvm_on_user_return(&smsr->urn);
}

325 326
u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
327
	return vcpu->arch.apic_base;
328 329 330
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

331 332 333 334 335 336
enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
{
	return kvm_apic_mode(kvm_get_apic_base(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_apic_mode);

337 338
int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
339 340
	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
341 342
	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |
		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
343

344
	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
345
		return 1;
346 347 348 349 350 351
	if (!msr_info->host_initiated) {
		if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
			return 1;
		if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
			return 1;
	}
352 353 354

	kvm_lapic_set_base(vcpu, msr_info->data);
	return 0;
355 356 357
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

358
asmlinkage __visible void kvm_spurious_fault(void)
359 360 361 362 363 364
{
	/* Fault while not rebooting.  We want the trace. */
	BUG();
}
EXPORT_SYMBOL_GPL(kvm_spurious_fault);

365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
#define EXCPT_BENIGN		0
#define EXCPT_CONTRIBUTORY	1
#define EXCPT_PF		2

static int exception_class(int vector)
{
	switch (vector) {
	case PF_VECTOR:
		return EXCPT_PF;
	case DE_VECTOR:
	case TS_VECTOR:
	case NP_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
		return EXCPT_CONTRIBUTORY;
	default:
		break;
	}
	return EXCPT_BENIGN;
}

386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
#define EXCPT_FAULT		0
#define EXCPT_TRAP		1
#define EXCPT_ABORT		2
#define EXCPT_INTERRUPT		3

static int exception_type(int vector)
{
	unsigned int mask;

	if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
		return EXCPT_INTERRUPT;

	mask = 1 << vector;

	/* #DB is trap, as instruction watchpoints are handled elsewhere */
	if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR)))
		return EXCPT_TRAP;

	if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
		return EXCPT_ABORT;

	/* Reserved exceptions will result in fault */
	return EXCPT_FAULT;
}

411 412 413 414 415 416 417 418 419 420
void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
{
	unsigned nr = vcpu->arch.exception.nr;
	bool has_payload = vcpu->arch.exception.has_payload;
	unsigned long payload = vcpu->arch.exception.payload;

	if (!has_payload)
		return;

	switch (nr) {
421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
	case DB_VECTOR:
		/*
		 * "Certain debug exceptions may clear bit 0-3.  The
		 * remaining contents of the DR6 register are never
		 * cleared by the processor".
		 */
		vcpu->arch.dr6 &= ~DR_TRAP_BITS;
		/*
		 * DR6.RTM is set by all #DB exceptions that don't clear it.
		 */
		vcpu->arch.dr6 |= DR6_RTM;
		vcpu->arch.dr6 |= payload;
		/*
		 * Bit 16 should be set in the payload whenever the #DB
		 * exception should clear DR6.RTM. This makes the payload
		 * compatible with the pending debug exceptions under VMX.
		 * Though not currently documented in the SDM, this also
		 * makes the payload compatible with the exit qualification
		 * for #DB exceptions under VMX.
		 */
		vcpu->arch.dr6 ^= payload & DR6_RTM;
		break;
443 444 445 446 447 448 449 450 451 452
	case PF_VECTOR:
		vcpu->arch.cr2 = payload;
		break;
	}

	vcpu->arch.exception.has_payload = false;
	vcpu->arch.exception.payload = 0;
}
EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);

453
static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
454
		unsigned nr, bool has_error, u32 error_code,
455
	        bool has_payload, unsigned long payload, bool reinject)
456 457 458 459
{
	u32 prev_nr;
	int class1, class2;

460 461
	kvm_make_request(KVM_REQ_EVENT, vcpu);

462
	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
463
	queue:
464 465
		if (has_error && !is_protmode(vcpu))
			has_error = false;
466 467 468 469 470 471 472 473 474 475 476
		if (reinject) {
			/*
			 * On vmentry, vcpu->arch.exception.pending is only
			 * true if an event injection was blocked by
			 * nested_run_pending.  In that case, however,
			 * vcpu_enter_guest requests an immediate exit,
			 * and the guest shouldn't proceed far enough to
			 * need reinjection.
			 */
			WARN_ON_ONCE(vcpu->arch.exception.pending);
			vcpu->arch.exception.injected = true;
477 478 479 480 481 482 483 484
			if (WARN_ON_ONCE(has_payload)) {
				/*
				 * A reinjected event has already
				 * delivered its payload.
				 */
				has_payload = false;
				payload = 0;
			}
485 486 487 488
		} else {
			vcpu->arch.exception.pending = true;
			vcpu->arch.exception.injected = false;
		}
489 490 491
		vcpu->arch.exception.has_error_code = has_error;
		vcpu->arch.exception.nr = nr;
		vcpu->arch.exception.error_code = error_code;
492 493
		vcpu->arch.exception.has_payload = has_payload;
		vcpu->arch.exception.payload = payload;
494 495 496
		/*
		 * In guest mode, payload delivery should be deferred,
		 * so that the L1 hypervisor can intercept #PF before
497 498 499 500 501 502 503
		 * CR2 is modified (or intercept #DB before DR6 is
		 * modified under nVMX).  However, for ABI
		 * compatibility with KVM_GET_VCPU_EVENTS and
		 * KVM_SET_VCPU_EVENTS, we can't delay payload
		 * delivery unless userspace has enabled this
		 * functionality via the per-VM capability,
		 * KVM_CAP_EXCEPTION_PAYLOAD.
504 505 506 507
		 */
		if (!vcpu->kvm->arch.exception_payload_enabled ||
		    !is_guest_mode(vcpu))
			kvm_deliver_exception_payload(vcpu);
508 509 510 511 512 513 514
		return;
	}

	/* to check exception */
	prev_nr = vcpu->arch.exception.nr;
	if (prev_nr == DF_VECTOR) {
		/* triple fault -> shutdown */
515
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
516 517 518 519 520 521
		return;
	}
	class1 = exception_class(prev_nr);
	class2 = exception_class(nr);
	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
522 523 524 525 526
		/*
		 * Generate double fault per SDM Table 5-5.  Set
		 * exception.pending = true so that the double fault
		 * can trigger a nested vmexit.
		 */
527
		vcpu->arch.exception.pending = true;
528
		vcpu->arch.exception.injected = false;
529 530 531
		vcpu->arch.exception.has_error_code = true;
		vcpu->arch.exception.nr = DF_VECTOR;
		vcpu->arch.exception.error_code = 0;
532 533
		vcpu->arch.exception.has_payload = false;
		vcpu->arch.exception.payload = 0;
534 535 536 537 538 539 540
	} else
		/* replace previous exception with a new one in a hope
		   that instruction re-execution will regenerate lost
		   exception */
		goto queue;
}

541 542
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
543
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
544 545 546
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

547 548
void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
549
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
550 551 552
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

553 554 555 556 557 558
static void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
				  unsigned long payload)
{
	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
}

559 560 561 562 563 564 565
static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
				    u32 error_code, unsigned long payload)
{
	kvm_multiple_exception(vcpu, nr, true, error_code,
			       true, payload, false);
}

566
int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
567
{
568 569 570
	if (err)
		kvm_inject_gp(vcpu, 0);
	else
571 572 573
		return kvm_skip_emulated_instruction(vcpu);

	return 1;
574 575
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
576

577
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
578 579
{
	++vcpu->stat.pf_guest;
580 581
	vcpu->arch.exception.nested_apf =
		is_guest_mode(vcpu) && fault->async_page_fault;
582
	if (vcpu->arch.exception.nested_apf) {
583
		vcpu->arch.apf.nested_apf_token = fault->address;
584 585 586 587 588
		kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
	} else {
		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
					fault->address);
	}
589
}
N
Nadav Har'El 已提交
590
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
591

592
static bool kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
593
{
594 595
	if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
		vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
596
	else
597
		vcpu->arch.mmu->inject_page_fault(vcpu, fault);
598 599

	return fault->nested_page_fault;
600 601
}

602 603
void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
604 605
	atomic_inc(&vcpu->arch.nmi_queued);
	kvm_make_request(KVM_REQ_NMI, vcpu);
606 607 608
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

609 610
void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
611
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
612 613 614
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

615 616
void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
617
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
618 619 620
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

621 622 623 624 625
/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
626
{
627 628 629 630
	if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
		return true;
	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
	return false;
631
}
632
EXPORT_SYMBOL_GPL(kvm_require_cpl);
633

634 635 636 637 638 639 640 641 642 643
bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
{
	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
		return true;

	kvm_queue_exception(vcpu, UD_VECTOR);
	return false;
}
EXPORT_SYMBOL_GPL(kvm_require_dr);

644 645
/*
 * This function will be used to read from the physical memory of the currently
646
 * running guest. The difference to kvm_vcpu_read_guest_page is that this function
647 648 649 650 651 652
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gfn_t ngfn, void *data, int offset, int len,
			    u32 access)
{
653
	struct x86_exception exception;
654 655 656 657
	gfn_t real_gfn;
	gpa_t ngpa;

	ngpa     = gfn_to_gpa(ngfn);
658
	real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);
659 660 661 662 663
	if (real_gfn == UNMAPPED_GVA)
		return -EFAULT;

	real_gfn = gpa_to_gfn(real_gfn);

664
	return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len);
665 666 667
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

668
static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
669 670 671 672 673 674
			       void *data, int offset, int len, u32 access)
{
	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
				       data, offset, len, access);
}

675 676 677
/*
 * Load the pae pdptrs.  Return true is they are all valid.
 */
678
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
679 680 681 682 683
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
	int i;
	int ret;
684
	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
685

686 687 688
	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
				      offset * sizeof(u64), sizeof(pdpte),
				      PFERR_USER_MASK|PFERR_WRITE_MASK);
689 690 691 692 693
	if (ret < 0) {
		ret = 0;
		goto out;
	}
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
B
Bandan Das 已提交
694
		if ((pdpte[i] & PT_PRESENT_MASK) &&
695
		    (pdpte[i] &
696
		     vcpu->arch.mmu->guest_rsvd_check.rsvd_bits_mask[0][2])) {
697 698 699 700 701 702
			ret = 0;
			goto out;
		}
	}
	ret = 1;

703
	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
A
Avi Kivity 已提交
704 705 706 707
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
708 709 710 711
out:

	return ret;
}
712
EXPORT_SYMBOL_GPL(load_pdptrs);
713

714
bool pdptrs_changed(struct kvm_vcpu *vcpu)
715
{
716
	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
717
	bool changed = true;
718 719
	int offset;
	gfn_t gfn;
720 721
	int r;

722
	if (is_long_mode(vcpu) || !is_pae(vcpu) || !is_paging(vcpu))
723 724
		return false;

A
Avi Kivity 已提交
725 726 727 728
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_avail))
		return true;

729 730
	gfn = (kvm_read_cr3(vcpu) & 0xffffffe0ul) >> PAGE_SHIFT;
	offset = (kvm_read_cr3(vcpu) & 0xffffffe0ul) & (PAGE_SIZE - 1);
731 732
	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
				       PFERR_USER_MASK | PFERR_WRITE_MASK);
733 734
	if (r < 0)
		goto out;
735
	changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
736 737 738 739
out:

	return changed;
}
740
EXPORT_SYMBOL_GPL(pdptrs_changed);
741

742
int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
743
{
744
	unsigned long old_cr0 = kvm_read_cr0(vcpu);
745
	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP;
746

747 748
	cr0 |= X86_CR0_ET;

749
#ifdef CONFIG_X86_64
750 751
	if (cr0 & 0xffffffff00000000UL)
		return 1;
752 753 754
#endif

	cr0 &= ~CR0_RESERVED_BITS;
755

756 757
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
		return 1;
758

759 760
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
		return 1;
761 762 763

	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
#ifdef CONFIG_X86_64
764
		if ((vcpu->arch.efer & EFER_LME)) {
765 766
			int cs_db, cs_l;

767 768
			if (!is_pae(vcpu))
				return 1;
769
			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
770 771
			if (cs_l)
				return 1;
772 773
		} else
#endif
774
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
775
						 kvm_read_cr3(vcpu)))
776
			return 1;
777 778
	}

779 780 781
	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
		return 1;

782 783
	kvm_x86_ops->set_cr0(vcpu, cr0);

784
	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
785
		kvm_clear_async_pf_completion_queue(vcpu);
786 787
		kvm_async_pf_hash_reset(vcpu);
	}
788

789 790
	if ((cr0 ^ old_cr0) & update_bits)
		kvm_mmu_reset_context(vcpu);
791

792 793 794
	if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
795 796
		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);

797 798
	return 0;
}
799
EXPORT_SYMBOL_GPL(kvm_set_cr0);
800

801
void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
802
{
803
	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
804
}
805
EXPORT_SYMBOL_GPL(kvm_lmsw);
806

807
void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
808 809 810 811
{
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
			!vcpu->guest_xcr0_loaded) {
		/* kvm_set_xcr() also depends on this */
812 813
		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
814 815 816
		vcpu->guest_xcr0_loaded = 1;
	}
}
817
EXPORT_SYMBOL_GPL(kvm_load_guest_xcr0);
818

819
void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
820 821 822 823 824 825 826
{
	if (vcpu->guest_xcr0_loaded) {
		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
		vcpu->guest_xcr0_loaded = 0;
	}
}
827
EXPORT_SYMBOL_GPL(kvm_put_guest_xcr0);
828

829
static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
830
{
831 832
	u64 xcr0 = xcr;
	u64 old_xcr0 = vcpu->arch.xcr0;
833
	u64 valid_bits;
834 835 836 837

	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
	if (index != XCR_XFEATURE_ENABLED_MASK)
		return 1;
D
Dave Hansen 已提交
838
	if (!(xcr0 & XFEATURE_MASK_FP))
839
		return 1;
D
Dave Hansen 已提交
840
	if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
841
		return 1;
842 843 844 845 846 847

	/*
	 * Do not allow the guest to set bits that we do not support
	 * saving.  However, xcr0 bit 0 is always set, even if the
	 * emulated CPU does not support XSAVE (see fx_init).
	 */
D
Dave Hansen 已提交
848
	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
849
	if (xcr0 & ~valid_bits)
850
		return 1;
851

D
Dave Hansen 已提交
852 853
	if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
	    (!(xcr0 & XFEATURE_MASK_BNDCSR)))
854 855
		return 1;

D
Dave Hansen 已提交
856 857
	if (xcr0 & XFEATURE_MASK_AVX512) {
		if (!(xcr0 & XFEATURE_MASK_YMM))
858
			return 1;
D
Dave Hansen 已提交
859
		if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
860 861
			return 1;
	}
862
	vcpu->arch.xcr0 = xcr0;
863

D
Dave Hansen 已提交
864
	if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
865
		kvm_update_cpuid(vcpu);
866 867 868 869 870
	return 0;
}

int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
871 872
	if (kvm_x86_ops->get_cpl(vcpu) != 0 ||
	    __kvm_set_xcr(vcpu, index, xcr)) {
873 874 875 876 877 878 879
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_xcr);

880
int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
881
{
882
	unsigned long old_cr4 = kvm_read_cr4(vcpu);
883
	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE |
884
				   X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE;
885

886 887
	if (cr4 & CR4_RESERVED_BITS)
		return 1;
888

889
	if (!guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && (cr4 & X86_CR4_OSXSAVE))
890 891
		return 1;

892
	if (!guest_cpuid_has(vcpu, X86_FEATURE_SMEP) && (cr4 & X86_CR4_SMEP))
893 894
		return 1;

895
	if (!guest_cpuid_has(vcpu, X86_FEATURE_SMAP) && (cr4 & X86_CR4_SMAP))
896 897
		return 1;

898
	if (!guest_cpuid_has(vcpu, X86_FEATURE_FSGSBASE) && (cr4 & X86_CR4_FSGSBASE))
F
Feng Wu 已提交
899 900
		return 1;

901
	if (!guest_cpuid_has(vcpu, X86_FEATURE_PKU) && (cr4 & X86_CR4_PKE))
902 903
		return 1;

904
	if (!guest_cpuid_has(vcpu, X86_FEATURE_LA57) && (cr4 & X86_CR4_LA57))
905 906
		return 1;

P
Paolo Bonzini 已提交
907 908 909
	if (!guest_cpuid_has(vcpu, X86_FEATURE_UMIP) && (cr4 & X86_CR4_UMIP))
		return 1;

910
	if (is_long_mode(vcpu)) {
911 912
		if (!(cr4 & X86_CR4_PAE))
			return 1;
913 914
	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
		   && ((cr4 ^ old_cr4) & pdptr_bits)
915 916
		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
				   kvm_read_cr3(vcpu)))
917 918
		return 1;

919
	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
920
		if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
921 922 923 924 925 926 927
			return 1;

		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
			return 1;
	}

928
	if (kvm_x86_ops->set_cr4(vcpu, cr4))
929
		return 1;
930

931 932
	if (((cr4 ^ old_cr4) & pdptr_bits) ||
	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
933
		kvm_mmu_reset_context(vcpu);
934

935
	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
A
Avi Kivity 已提交
936
		kvm_update_cpuid(vcpu);
937

938 939
	return 0;
}
940
EXPORT_SYMBOL_GPL(kvm_set_cr4);
941

942
int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
943
{
944
	bool skip_tlb_flush = false;
945
#ifdef CONFIG_X86_64
946 947
	bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);

948
	if (pcid_enabled) {
949 950
		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
		cr3 &= ~X86_CR3_PCID_NOFLUSH;
951
	}
952
#endif
N
Nadav Amit 已提交
953

954
	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
955 956
		if (!skip_tlb_flush) {
			kvm_mmu_sync_roots(vcpu);
957
			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
958
		}
959
		return 0;
960 961
	}

962
	if (is_long_mode(vcpu) &&
963
	    (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63)))
964 965
		return 1;
	else if (is_pae(vcpu) && is_paging(vcpu) &&
966
		   !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
N
Nadav Amit 已提交
967
		return 1;
968

969
	kvm_mmu_new_cr3(vcpu, cr3, skip_tlb_flush);
970
	vcpu->arch.cr3 = cr3;
971
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
972

973 974
	return 0;
}
975
EXPORT_SYMBOL_GPL(kvm_set_cr3);
976

A
Andre Przywara 已提交
977
int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
978
{
979 980
	if (cr8 & CR8_RESERVED_BITS)
		return 1;
981
	if (lapic_in_kernel(vcpu))
982 983
		kvm_lapic_set_tpr(vcpu, cr8);
	else
984
		vcpu->arch.cr8 = cr8;
985 986
	return 0;
}
987
EXPORT_SYMBOL_GPL(kvm_set_cr8);
988

989
unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
990
{
991
	if (lapic_in_kernel(vcpu))
992 993
		return kvm_lapic_get_cr8(vcpu);
	else
994
		return vcpu->arch.cr8;
995
}
996
EXPORT_SYMBOL_GPL(kvm_get_cr8);
997

998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
{
	int i;

	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
	}
}

J
Jan Kiszka 已提交
1009 1010 1011 1012 1013 1014
static void kvm_update_dr6(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
		kvm_x86_ops->set_dr6(vcpu, vcpu->arch.dr6);
}

1015 1016 1017 1018 1019 1020 1021 1022 1023
static void kvm_update_dr7(struct kvm_vcpu *vcpu)
{
	unsigned long dr7;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		dr7 = vcpu->arch.guest_debug_dr7;
	else
		dr7 = vcpu->arch.dr7;
	kvm_x86_ops->set_dr7(vcpu, dr7);
1024 1025 1026
	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
	if (dr7 & DR7_BP_EN_MASK)
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1027 1028
}

1029 1030 1031 1032
static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
{
	u64 fixed = DR6_FIXED_1;

1033
	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1034 1035 1036 1037
		fixed |= DR6_RTM;
	return fixed;
}

1038
static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
{
	switch (dr) {
	case 0 ... 3:
		vcpu->arch.db[dr] = val;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = val;
		break;
	case 4:
		/* fall through */
	case 6:
1049 1050
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
1051
		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
J
Jan Kiszka 已提交
1052
		kvm_update_dr6(vcpu);
1053 1054 1055 1056
		break;
	case 5:
		/* fall through */
	default: /* 7 */
1057 1058
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
1059
		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1060
		kvm_update_dr7(vcpu);
1061 1062 1063 1064 1065
		break;
	}

	return 0;
}
1066 1067 1068

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
1069
	if (__kvm_set_dr(vcpu, dr, val)) {
1070
		kvm_inject_gp(vcpu, 0);
1071 1072 1073
		return 1;
	}
	return 0;
1074
}
1075 1076
EXPORT_SYMBOL_GPL(kvm_set_dr);

1077
int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1078 1079 1080 1081 1082 1083 1084 1085
{
	switch (dr) {
	case 0 ... 3:
		*val = vcpu->arch.db[dr];
		break;
	case 4:
		/* fall through */
	case 6:
J
Jan Kiszka 已提交
1086 1087 1088 1089
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
			*val = vcpu->arch.dr6;
		else
			*val = kvm_x86_ops->get_dr6(vcpu);
1090 1091 1092 1093 1094 1095 1096
		break;
	case 5:
		/* fall through */
	default: /* 7 */
		*val = vcpu->arch.dr7;
		break;
	}
1097 1098
	return 0;
}
1099 1100
EXPORT_SYMBOL_GPL(kvm_get_dr);

A
Avi Kivity 已提交
1101 1102
bool kvm_rdpmc(struct kvm_vcpu *vcpu)
{
1103
	u32 ecx = kvm_rcx_read(vcpu);
A
Avi Kivity 已提交
1104 1105 1106
	u64 data;
	int err;

1107
	err = kvm_pmu_rdpmc(vcpu, ecx, &data);
A
Avi Kivity 已提交
1108 1109
	if (err)
		return err;
1110 1111
	kvm_rax_write(vcpu, (u32)data);
	kvm_rdx_write(vcpu, data >> 32);
A
Avi Kivity 已提交
1112 1113 1114 1115
	return err;
}
EXPORT_SYMBOL_GPL(kvm_rdpmc);

1116 1117 1118 1119 1120
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
 * This list is modified at module load time to reflect the
1121
 * capabilities of the host cpu. This capabilities test skips MSRs that are
1122 1123
 * kvm-specific. Those are put in emulated_msrs; filtering of emulated_msrs
 * may depend on host virtualization features rather than host cpu features.
1124
 */
1125

1126 1127
static u32 msrs_to_save[] = {
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
B
Brian Gerst 已提交
1128
	MSR_STAR,
1129 1130 1131
#ifdef CONFIG_X86_64
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
1132
	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
1133
	MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
1134
	MSR_IA32_SPEC_CTRL,
1135 1136 1137 1138 1139 1140
	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
1141 1142 1143 1144
};

static unsigned num_msrs_to_save;

1145 1146 1147 1148 1149
static u32 emulated_msrs[] = {
	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
1150
	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
1151 1152
	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
1153
	HV_X64_MSR_RESET,
1154
	HV_X64_MSR_VP_INDEX,
1155
	HV_X64_MSR_VP_RUNTIME,
1156
	HV_X64_MSR_SCONTROL,
A
Andrey Smetanin 已提交
1157
	HV_X64_MSR_STIMER0_CONFIG,
1158
	HV_X64_MSR_VP_ASSIST_PAGE,
1159 1160 1161 1162
	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
	HV_X64_MSR_TSC_EMULATION_STATUS,

	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
1163 1164
	MSR_KVM_PV_EOI_EN,

W
Will Auld 已提交
1165
	MSR_IA32_TSC_ADJUST,
1166
	MSR_IA32_TSCDEADLINE,
1167
	MSR_IA32_ARCH_CAPABILITIES,
1168
	MSR_IA32_MISC_ENABLE,
1169 1170
	MSR_IA32_MCG_STATUS,
	MSR_IA32_MCG_CTL,
1171
	MSR_IA32_MCG_EXT_CTL,
P
Paolo Bonzini 已提交
1172
	MSR_IA32_SMBASE,
1173
	MSR_SMI_COUNT,
K
Kyle Huey 已提交
1174 1175
	MSR_PLATFORM_INFO,
	MSR_MISC_FEATURES_ENABLES,
1176
	MSR_AMD64_VIRT_SPEC_CTRL,
1177
	MSR_IA32_POWER_CTL,
1178 1179

	MSR_K7_HWCR,
1180 1181
};

1182 1183
static unsigned num_emulated_msrs;

1184 1185 1186 1187 1188
/*
 * List of msr numbers which are used to expose MSR-based features that
 * can be used by a hypervisor to validate requested CPU features.
 */
static u32 msr_based_features[] = {
1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR0_FIXED1,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_CR4_FIXED1,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1208
	MSR_F10H_DECFG,
1209
	MSR_IA32_UCODE_REV,
1210
	MSR_IA32_ARCH_CAPABILITIES,
1211 1212 1213 1214
};

static unsigned int num_msr_based_features;

1215
static u64 kvm_get_arch_capabilities(void)
1216
{
1217
	u64 data = 0;
1218

1219 1220
	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236

	/*
	 * If we're doing cache flushes (either "always" or "cond")
	 * we will do one whenever the guest does a vmlaunch/vmresume.
	 * If an outer hypervisor is doing the cache flush for us
	 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
	 * capability to the guest too, and if EPT is disabled we're not
	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
	 * require a nested hypervisor to do a flush of its own.
	 */
	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;

	return data;
}

1237 1238 1239
static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
{
	switch (msr->index) {
1240
	case MSR_IA32_ARCH_CAPABILITIES:
1241 1242 1243
		msr->data = kvm_get_arch_capabilities();
		break;
	case MSR_IA32_UCODE_REV:
1244
		rdmsrl_safe(msr->index, &msr->data);
1245
		break;
1246 1247 1248 1249 1250 1251 1252
	default:
		if (kvm_x86_ops->get_msr_feature(msr))
			return 1;
	}
	return 0;
}

1253 1254 1255
static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	struct kvm_msr_entry msr;
1256
	int r;
1257 1258

	msr.index = index;
1259 1260 1261
	r = kvm_get_msr_feature(&msr);
	if (r)
		return r;
1262 1263 1264 1265 1266 1267

	*data = msr.data;

	return 0;
}

1268
static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1269
{
1270
	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1271
		return false;
A
Alexander Graf 已提交
1272

1273
	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1274
		return false;
1275

1276 1277 1278 1279 1280 1281
	if (efer & (EFER_LME | EFER_LMA) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))
		return false;

	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
		return false;
1282

1283
	return true;
1284 1285 1286 1287 1288 1289 1290 1291

}
bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	if (efer & efer_reserved_bits)
		return false;

	return __kvm_valid_efer(vcpu, efer);
1292 1293 1294
}
EXPORT_SYMBOL_GPL(kvm_valid_efer);

1295
static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1296 1297
{
	u64 old_efer = vcpu->arch.efer;
1298
	u64 efer = msr_info->data;
1299

1300
	if (efer & efer_reserved_bits)
1301
		return 1;
1302

1303 1304 1305 1306 1307 1308 1309 1310
	if (!msr_info->host_initiated) {
		if (!__kvm_valid_efer(vcpu, efer))
			return 1;

		if (is_paging(vcpu) &&
		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
			return 1;
	}
1311

1312
	efer &= ~EFER_LMA;
1313
	efer |= vcpu->arch.efer & EFER_LMA;
1314

1315 1316
	kvm_x86_ops->set_efer(vcpu, efer);

1317 1318 1319 1320
	/* Update reserved bits */
	if ((efer ^ old_efer) & EFER_NX)
		kvm_mmu_reset_context(vcpu);

1321
	return 0;
1322 1323
}

1324 1325 1326 1327 1328 1329
void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);

1330 1331 1332 1333 1334
/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1335
int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
1336
{
1337 1338 1339 1340 1341 1342
	switch (msr->index) {
	case MSR_FS_BASE:
	case MSR_GS_BASE:
	case MSR_KERNEL_GS_BASE:
	case MSR_CSTAR:
	case MSR_LSTAR:
1343
		if (is_noncanonical_address(msr->data, vcpu))
1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
			return 1;
		break;
	case MSR_IA32_SYSENTER_EIP:
	case MSR_IA32_SYSENTER_ESP:
		/*
		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
		 * non-canonical address is written on Intel but not on
		 * AMD (which ignores the top 32-bits, because it does
		 * not implement 64-bit SYSENTER).
		 *
		 * 64-bit code should hence be able to write a non-canonical
		 * value on AMD.  Making the address canonical ensures that
		 * vmentry does not fail on Intel after writing a non-canonical
		 * value, and that something deterministic happens if the guest
		 * invokes 64-bit SYSENTER.
		 */
1360
		msr->data = get_canonical(msr->data, vcpu_virt_addr_bits(vcpu));
1361
	}
1362
	return kvm_x86_ops->set_msr(vcpu, msr);
1363
}
1364
EXPORT_SYMBOL_GPL(kvm_set_msr);
1365

1366 1367 1368
/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383
static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	struct msr_data msr;
	int r;

	msr.index = index;
	msr.host_initiated = true;
	r = kvm_get_msr(vcpu, &msr);
	if (r)
		return r;

	*data = msr.data;
	return 0;
}

1384 1385
static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
1386 1387 1388 1389 1390 1391
	struct msr_data msr;

	msr.data = *data;
	msr.index = index;
	msr.host_initiated = true;
	return kvm_set_msr(vcpu, &msr);
1392 1393
}

1394 1395 1396 1397 1398 1399
#ifdef CONFIG_X86_64
struct pvclock_gtod_data {
	seqcount_t	seq;

	struct { /* extract of a clocksource struct */
		int vclock_mode;
1400 1401
		u64	cycle_last;
		u64	mask;
1402 1403 1404 1405
		u32	mult;
		u32	shift;
	} clock;

1406 1407
	u64		boot_ns;
	u64		nsec_base;
1408
	u64		wall_time_sec;
1409 1410 1411 1412 1413 1414 1415
};

static struct pvclock_gtod_data pvclock_gtod_data;

static void update_pvclock_gtod(struct timekeeper *tk)
{
	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
1416 1417
	u64 boot_ns;

1418
	boot_ns = ktime_to_ns(ktime_add(tk->tkr_mono.base, tk->offs_boot));
1419 1420 1421 1422

	write_seqcount_begin(&vdata->seq);

	/* copy pvclock gtod data */
1423 1424 1425 1426 1427
	vdata->clock.vclock_mode	= tk->tkr_mono.clock->archdata.vclock_mode;
	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
	vdata->clock.mask		= tk->tkr_mono.mask;
	vdata->clock.mult		= tk->tkr_mono.mult;
	vdata->clock.shift		= tk->tkr_mono.shift;
1428

1429
	vdata->boot_ns			= boot_ns;
1430
	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
1431

1432 1433
	vdata->wall_time_sec            = tk->xtime_sec;

1434 1435 1436 1437
	write_seqcount_end(&vdata->seq);
}
#endif

1438 1439 1440 1441 1442 1443 1444 1445 1446
void kvm_set_pending_timer(struct kvm_vcpu *vcpu)
{
	/*
	 * Note: KVM_REQ_PENDING_TIMER is implicitly checked in
	 * vcpu_enter_guest.  This function is only called from
	 * the physical CPU that is running vcpu.
	 */
	kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
}
1447

1448 1449
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
{
1450 1451
	int version;
	int r;
1452
	struct pvclock_wall_clock wc;
A
Arnd Bergmann 已提交
1453
	struct timespec64 boot;
1454 1455 1456 1457

	if (!wall_clock)
		return;

1458 1459 1460 1461 1462 1463 1464 1465
	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
	if (r)
		return;

	if (version & 1)
		++version;  /* first time write, random junk */

	++version;
1466

1467 1468
	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
		return;
1469

1470 1471
	/*
	 * The guest calculates current wall clock time by adding
Z
Zachary Amsden 已提交
1472
	 * system time (updated by kvm_guest_time_update below) to the
1473 1474 1475
	 * wall clock specified here.  guest system time equals host
	 * system time for us, thus we must fill in host boot time here.
	 */
A
Arnd Bergmann 已提交
1476
	getboottime64(&boot);
1477

1478
	if (kvm->arch.kvmclock_offset) {
A
Arnd Bergmann 已提交
1479 1480
		struct timespec64 ts = ns_to_timespec64(kvm->arch.kvmclock_offset);
		boot = timespec64_sub(boot, ts);
1481
	}
A
Arnd Bergmann 已提交
1482
	wc.sec = (u32)boot.tv_sec; /* overflow in 2106 guest time */
1483 1484
	wc.nsec = boot.tv_nsec;
	wc.version = version;
1485 1486 1487 1488 1489 1490 1491

	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));

	version++;
	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
}

1492 1493
static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
1494 1495
	do_shl32_div32(dividend, divisor);
	return dividend;
1496 1497
}

1498
static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
1499
			       s8 *pshift, u32 *pmultiplier)
1500
{
1501
	uint64_t scaled64;
1502 1503 1504 1505
	int32_t  shift = 0;
	uint64_t tps64;
	uint32_t tps32;

1506 1507
	tps64 = base_hz;
	scaled64 = scaled_hz;
1508
	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
1509 1510 1511 1512 1513
		tps64 >>= 1;
		shift--;
	}

	tps32 = (uint32_t)tps64;
1514 1515
	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
1516 1517 1518
			scaled64 >>= 1;
		else
			tps32 <<= 1;
1519 1520 1521
		shift++;
	}

1522 1523
	*pshift = shift;
	*pmultiplier = div_frac(scaled64, tps32);
1524

1525 1526
	pr_debug("%s: base_hz %llu => %llu, shift %d, mul %u\n",
		 __func__, base_hz, scaled_hz, shift, *pmultiplier);
1527 1528
}

1529
#ifdef CONFIG_X86_64
1530
static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
1531
#endif
1532

1533
static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
1534
static unsigned long max_tsc_khz;
1535

1536
static u32 adjust_tsc_khz(u32 khz, s32 ppm)
1537
{
1538 1539 1540
	u64 v = (u64)khz * (1000000 + ppm);
	do_div(v, 1000000);
	return v;
1541 1542
}

1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578
static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
{
	u64 ratio;

	/* Guest TSC same frequency as host TSC? */
	if (!scale) {
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
		return 0;
	}

	/* TSC scaling supported? */
	if (!kvm_has_tsc_control) {
		if (user_tsc_khz > tsc_khz) {
			vcpu->arch.tsc_catchup = 1;
			vcpu->arch.tsc_always_catchup = 1;
			return 0;
		} else {
			WARN(1, "user requested TSC rate below hardware speed\n");
			return -1;
		}
	}

	/* TSC scaling required  - calculate ratio */
	ratio = mul_u64_u32_div(1ULL << kvm_tsc_scaling_ratio_frac_bits,
				user_tsc_khz, tsc_khz);

	if (ratio == 0 || ratio >= kvm_max_tsc_scaling_ratio) {
		WARN_ONCE(1, "Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
			  user_tsc_khz);
		return -1;
	}

	vcpu->arch.tsc_scaling_ratio = ratio;
	return 0;
}

1579
static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
1580
{
1581 1582
	u32 thresh_lo, thresh_hi;
	int use_scaling = 0;
1583

1584
	/* tsc_khz can be zero if TSC calibration fails */
1585
	if (user_tsc_khz == 0) {
1586 1587
		/* set tsc_scaling_ratio to a safe value */
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
1588
		return -1;
1589
	}
1590

Z
Zachary Amsden 已提交
1591
	/* Compute a scale to convert nanoseconds in TSC cycles */
1592
	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
1593 1594
			   &vcpu->arch.virtual_tsc_shift,
			   &vcpu->arch.virtual_tsc_mult);
1595
	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
1596 1597 1598 1599 1600 1601 1602 1603 1604

	/*
	 * Compute the variation in TSC rate which is acceptable
	 * within the range of tolerance and decide if the
	 * rate being applied is within that bounds of the hardware
	 * rate.  If so, no scaling or compensation need be done.
	 */
	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
1605 1606
	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", user_tsc_khz, thresh_lo, thresh_hi);
1607 1608
		use_scaling = 1;
	}
1609
	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
Z
Zachary Amsden 已提交
1610 1611 1612 1613
}

static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
1614
	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
1615 1616
				      vcpu->arch.virtual_tsc_mult,
				      vcpu->arch.virtual_tsc_shift);
1617
	tsc += vcpu->arch.this_tsc_write;
Z
Zachary Amsden 已提交
1618 1619 1620
	return tsc;
}

1621 1622 1623 1624 1625
static inline int gtod_is_based_on_tsc(int mode)
{
	return mode == VCLOCK_TSC || mode == VCLOCK_HVCLOCK;
}

1626
static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
1627 1628 1629 1630 1631 1632 1633 1634 1635
{
#ifdef CONFIG_X86_64
	bool vcpus_matched;
	struct kvm_arch *ka = &vcpu->kvm->arch;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			 atomic_read(&vcpu->kvm->online_vcpus));

1636 1637 1638 1639 1640 1641 1642 1643 1644
	/*
	 * Once the masterclock is enabled, always perform request in
	 * order to update it.
	 *
	 * In order to enable masterclock, the host clocksource must be TSC
	 * and the vcpus need to have matched TSCs.  When that happens,
	 * perform request to enable masterclock.
	 */
	if (ka->use_master_clock ||
1645
	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
1646 1647 1648 1649 1650 1651 1652 1653
		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);

	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
			    atomic_read(&vcpu->kvm->online_vcpus),
		            ka->use_master_clock, gtod->clock.vclock_mode);
#endif
}

W
Will Auld 已提交
1654 1655
static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
{
1656
	u64 curr_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
W
Will Auld 已提交
1657 1658 1659
	vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
}

1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686
/*
 * Multiply tsc by a fixed point number represented by ratio.
 *
 * The most significant 64-N bits (mult) of ratio represent the
 * integral part of the fixed point number; the remaining N bits
 * (frac) represent the fractional part, ie. ratio represents a fixed
 * point number (mult + frac * 2^(-N)).
 *
 * N equals to kvm_tsc_scaling_ratio_frac_bits.
 */
static inline u64 __scale_tsc(u64 ratio, u64 tsc)
{
	return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits);
}

u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc)
{
	u64 _tsc = tsc;
	u64 ratio = vcpu->arch.tsc_scaling_ratio;

	if (ratio != kvm_default_tsc_scaling_ratio)
		_tsc = __scale_tsc(ratio, tsc);

	return _tsc;
}
EXPORT_SYMBOL_GPL(kvm_scale_tsc);

1687 1688 1689 1690 1691 1692 1693 1694 1695
static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
{
	u64 tsc;

	tsc = kvm_scale_tsc(vcpu, rdtsc());

	return target_tsc - tsc;
}

1696 1697
u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
{
1698 1699 1700
	u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);

	return tsc_offset + kvm_scale_tsc(vcpu, host_tsc);
1701 1702 1703
}
EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);

1704 1705
static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
{
1706
	vcpu->arch.tsc_offset = kvm_x86_ops->write_l1_tsc_offset(vcpu, offset);
1707 1708
}

1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721
static inline bool kvm_check_tsc_unstable(void)
{
#ifdef CONFIG_X86_64
	/*
	 * TSC is marked unstable when we're running on Hyper-V,
	 * 'TSC page' clocksource is good.
	 */
	if (pvclock_gtod_data.clock.vclock_mode == VCLOCK_HVCLOCK)
		return false;
#endif
	return check_tsc_unstable();
}

1722
void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
1723 1724
{
	struct kvm *kvm = vcpu->kvm;
Z
Zachary Amsden 已提交
1725
	u64 offset, ns, elapsed;
1726
	unsigned long flags;
1727
	bool matched;
T
Tomasz Grabiec 已提交
1728
	bool already_matched;
1729
	u64 data = msr->data;
1730
	bool synchronizing = false;
1731

1732
	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
1733
	offset = kvm_compute_tsc_offset(vcpu, data);
1734
	ns = ktime_get_boot_ns();
Z
Zachary Amsden 已提交
1735
	elapsed = ns - kvm->arch.last_tsc_nsec;
1736

1737
	if (vcpu->arch.virtual_tsc_khz) {
1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
		if (data == 0 && msr->host_initiated) {
			/*
			 * detection of vcpu initialization -- need to sync
			 * with other vCPUs. This particularly helps to keep
			 * kvm_clock stable after CPU hotplug
			 */
			synchronizing = true;
		} else {
			u64 tsc_exp = kvm->arch.last_tsc_write +
						nsec_to_cycles(vcpu, elapsed);
			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
			/*
			 * Special case: TSC write with a small delta (1 second)
			 * of virtual cycle time against real time is
			 * interpreted as an attempt to synchronize the CPU.
			 */
			synchronizing = data < tsc_exp + tsc_hz &&
					data + tsc_hz > tsc_exp;
		}
1757
	}
Z
Zachary Amsden 已提交
1758 1759

	/*
1760 1761 1762 1763 1764
	 * For a reliable TSC, we can match TSC offsets, and for an unstable
	 * TSC, we add elapsed time in this computation.  We could let the
	 * compensation code attempt to catch up if we fall behind, but
	 * it's better to try to match offsets from the beginning.
         */
1765
	if (synchronizing &&
1766
	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
1767
		if (!kvm_check_tsc_unstable()) {
1768
			offset = kvm->arch.cur_tsc_offset;
Z
Zachary Amsden 已提交
1769 1770
			pr_debug("kvm: matched tsc offset for %llu\n", data);
		} else {
1771
			u64 delta = nsec_to_cycles(vcpu, elapsed);
1772
			data += delta;
1773
			offset = kvm_compute_tsc_offset(vcpu, data);
1774
			pr_debug("kvm: adjusted tsc offset by %llu\n", delta);
Z
Zachary Amsden 已提交
1775
		}
1776
		matched = true;
T
Tomasz Grabiec 已提交
1777
		already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
1778 1779 1780 1781 1782 1783
	} else {
		/*
		 * We split periods of matched TSC writes into generations.
		 * For each generation, we track the original measured
		 * nanosecond time, offset, and write, so if TSCs are in
		 * sync, we can match exact offset, and if not, we can match
G
Guo Chao 已提交
1784
		 * exact software computation in compute_guest_tsc()
1785 1786 1787 1788 1789 1790 1791
		 *
		 * These values are tracked in kvm->arch.cur_xxx variables.
		 */
		kvm->arch.cur_tsc_generation++;
		kvm->arch.cur_tsc_nsec = ns;
		kvm->arch.cur_tsc_write = data;
		kvm->arch.cur_tsc_offset = offset;
1792
		matched = false;
T
Tomasz Grabiec 已提交
1793
		pr_debug("kvm: new tsc generation %llu, clock %llu\n",
1794
			 kvm->arch.cur_tsc_generation, data);
Z
Zachary Amsden 已提交
1795
	}
1796 1797 1798 1799 1800

	/*
	 * We also track th most recent recorded KHZ, write and time to
	 * allow the matching interval to be extended at each write.
	 */
Z
Zachary Amsden 已提交
1801 1802
	kvm->arch.last_tsc_nsec = ns;
	kvm->arch.last_tsc_write = data;
1803
	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
1804

1805
	vcpu->arch.last_guest_tsc = data;
1806 1807 1808 1809 1810 1811

	/* Keep track of which generation this VCPU has synchronized to */
	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;

1812
	if (!msr->host_initiated && guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST))
W
Will Auld 已提交
1813
		update_ia32_tsc_adjust_msr(vcpu, offset);
1814

1815
	kvm_vcpu_write_tsc_offset(vcpu, offset);
1816
	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
1817 1818

	spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
T
Tomasz Grabiec 已提交
1819
	if (!matched) {
1820
		kvm->arch.nr_vcpus_matched_tsc = 0;
T
Tomasz Grabiec 已提交
1821 1822 1823
	} else if (!already_matched) {
		kvm->arch.nr_vcpus_matched_tsc++;
	}
1824 1825 1826

	kvm_track_tsc_matching(vcpu);
	spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
1827
}
1828

1829 1830
EXPORT_SYMBOL_GPL(kvm_write_tsc);

1831 1832 1833
static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
					   s64 adjustment)
{
1834 1835
	u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
1836 1837 1838 1839 1840 1841 1842
}

static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
{
	if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
		WARN_ON(adjustment < 0);
	adjustment = kvm_scale_tsc(vcpu, (u64) adjustment);
1843
	adjust_tsc_offset_guest(vcpu, adjustment);
1844 1845
}

1846 1847
#ifdef CONFIG_X86_64

1848
static u64 read_tsc(void)
1849
{
1850
	u64 ret = (u64)rdtsc_ordered();
1851
	u64 last = pvclock_gtod_data.clock.cycle_last;
1852 1853 1854 1855 1856 1857

	if (likely(ret >= last))
		return ret;

	/*
	 * GCC likes to generate cmov here, but this branch is extremely
1858
	 * predictable (it's just a function of time and the likely is
1859 1860 1861 1862 1863 1864 1865 1866 1867
	 * very likely) and there's a data dependence, so force GCC
	 * to generate a branch instead.  I don't barrier() because
	 * we don't actually need a barrier, and if this function
	 * ever gets inlined it will generate worse code.
	 */
	asm volatile ("");
	return last;
}

1868
static inline u64 vgettsc(u64 *tsc_timestamp, int *mode)
1869 1870 1871
{
	long v;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896
	u64 tsc_pg_val;

	switch (gtod->clock.vclock_mode) {
	case VCLOCK_HVCLOCK:
		tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
						  tsc_timestamp);
		if (tsc_pg_val != U64_MAX) {
			/* TSC page valid */
			*mode = VCLOCK_HVCLOCK;
			v = (tsc_pg_val - gtod->clock.cycle_last) &
				gtod->clock.mask;
		} else {
			/* TSC page invalid */
			*mode = VCLOCK_NONE;
		}
		break;
	case VCLOCK_TSC:
		*mode = VCLOCK_TSC;
		*tsc_timestamp = read_tsc();
		v = (*tsc_timestamp - gtod->clock.cycle_last) &
			gtod->clock.mask;
		break;
	default:
		*mode = VCLOCK_NONE;
	}
1897

1898 1899
	if (*mode == VCLOCK_NONE)
		*tsc_timestamp = v = 0;
1900 1901 1902 1903

	return v * gtod->clock.mult;
}

1904
static int do_monotonic_boot(s64 *t, u64 *tsc_timestamp)
1905
{
1906
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
1907 1908
	unsigned long seq;
	int mode;
1909
	u64 ns;
1910 1911 1912

	do {
		seq = read_seqcount_begin(&gtod->seq);
1913
		ns = gtod->nsec_base;
1914
		ns += vgettsc(tsc_timestamp, &mode);
1915
		ns >>= gtod->clock.shift;
1916
		ns += gtod->boot_ns;
1917
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
1918
	*t = ns;
1919 1920 1921 1922

	return mode;
}

1923
static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
1924 1925 1926 1927 1928 1929 1930 1931 1932 1933
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	unsigned long seq;
	int mode;
	u64 ns;

	do {
		seq = read_seqcount_begin(&gtod->seq);
		ts->tv_sec = gtod->wall_time_sec;
		ns = gtod->nsec_base;
1934
		ns += vgettsc(tsc_timestamp, &mode);
1935 1936 1937 1938 1939 1940 1941 1942 1943
		ns >>= gtod->clock.shift;
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));

	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return mode;
}

1944 1945
/* returns true if host is using TSC based clocksource */
static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
1946 1947
{
	/* checked again under seqlock below */
1948
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
1949 1950
		return false;

1951 1952
	return gtod_is_based_on_tsc(do_monotonic_boot(kernel_ns,
						      tsc_timestamp));
1953
}
1954

1955
/* returns true if host is using TSC based clocksource */
1956
static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
1957
					   u64 *tsc_timestamp)
1958 1959
{
	/* checked again under seqlock below */
1960
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
1961 1962
		return false;

1963
	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
1964
}
1965 1966 1967 1968
#endif

/*
 *
1969 1970 1971
 * Assuming a stable TSC across physical CPUS, and a stable TSC
 * across virtual CPUs, the following condition is possible.
 * Each numbered line represents an event visible to both
1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003
 * CPUs at the next numbered event.
 *
 * "timespecX" represents host monotonic time. "tscX" represents
 * RDTSC value.
 *
 * 		VCPU0 on CPU0		|	VCPU1 on CPU1
 *
 * 1.  read timespec0,tsc0
 * 2.					| timespec1 = timespec0 + N
 * 					| tsc1 = tsc0 + M
 * 3. transition to guest		| transition to guest
 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
 * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
 * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
 *
 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
 *
 * 	- ret0 < ret1
 *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
 *		...
 *	- 0 < N - M => M < N
 *
 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
 * always the case (the difference between two distinct xtime instances
 * might be smaller then the difference between corresponding TSC reads,
 * when updating guest vcpus pvclock areas).
 *
 * To avoid that problem, do not allow visibility of distinct
 * system_timestamp/tsc_timestamp values simultaneously: use a master
 * copy of host monotonic time values. Update that master copy
 * in lockstep.
 *
2004
 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
2005 2006 2007 2008 2009 2010 2011 2012
 *
 */

static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	struct kvm_arch *ka = &kvm->arch;
	int vclock_mode;
2013 2014 2015 2016
	bool host_tsc_clocksource, vcpus_matched;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			atomic_read(&kvm->online_vcpus));
2017 2018 2019 2020 2021

	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
2022
	host_tsc_clocksource = kvm_get_time_and_clockread(
2023 2024 2025
					&ka->master_kernel_ns,
					&ka->master_cycle_now);

2026
	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
2027
				&& !ka->backwards_tsc_observed
2028
				&& !ka->boot_vcpu_runs_old_kvmclock;
2029

2030 2031 2032 2033
	if (ka->use_master_clock)
		atomic_set(&kvm_guest_has_master_clock, 1);

	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
2034 2035
	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
					vcpus_matched);
2036 2037 2038
#endif
}

2039 2040 2041 2042 2043
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
}

2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056
static void kvm_gen_update_masterclock(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	int i;
	struct kvm_vcpu *vcpu;
	struct kvm_arch *ka = &kvm->arch;

	spin_lock(&ka->pvclock_gtod_sync_lock);
	kvm_make_mclock_inprogress_request(kvm);
	/* no guest entries from this point */
	pvclock_update_vm_gtod_copy(kvm);

	kvm_for_each_vcpu(i, vcpu, kvm)
2057
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2058 2059 2060

	/* guest entries allowed */
	kvm_for_each_vcpu(i, vcpu, kvm)
2061
		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
2062 2063 2064 2065 2066

	spin_unlock(&ka->pvclock_gtod_sync_lock);
#endif
}

2067
u64 get_kvmclock_ns(struct kvm *kvm)
2068 2069
{
	struct kvm_arch *ka = &kvm->arch;
2070
	struct pvclock_vcpu_time_info hv_clock;
2071
	u64 ret;
2072

2073 2074 2075 2076
	spin_lock(&ka->pvclock_gtod_sync_lock);
	if (!ka->use_master_clock) {
		spin_unlock(&ka->pvclock_gtod_sync_lock);
		return ktime_get_boot_ns() + ka->kvmclock_offset;
2077 2078
	}

2079 2080 2081 2082
	hv_clock.tsc_timestamp = ka->master_cycle_now;
	hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
	spin_unlock(&ka->pvclock_gtod_sync_lock);

2083 2084 2085
	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
	get_cpu();

2086 2087 2088 2089 2090 2091 2092
	if (__this_cpu_read(cpu_tsc_khz)) {
		kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL,
				   &hv_clock.tsc_shift,
				   &hv_clock.tsc_to_system_mul);
		ret = __pvclock_read_cycles(&hv_clock, rdtsc());
	} else
		ret = ktime_get_boot_ns() + ka->kvmclock_offset;
2093 2094 2095 2096

	put_cpu();

	return ret;
2097 2098
}

2099 2100 2101 2102 2103
static void kvm_setup_pvclock_page(struct kvm_vcpu *v)
{
	struct kvm_vcpu_arch *vcpu = &v->arch;
	struct pvclock_vcpu_time_info guest_hv_clock;

2104
	if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123
		&guest_hv_clock, sizeof(guest_hv_clock))))
		return;

	/* This VCPU is paused, but it's legal for a guest to read another
	 * VCPU's kvmclock, so we really have to follow the specification where
	 * it says that version is odd if data is being modified, and even after
	 * it is consistent.
	 *
	 * Version field updates must be kept separate.  This is because
	 * kvm_write_guest_cached might use a "rep movs" instruction, and
	 * writes within a string instruction are weakly ordered.  So there
	 * are three writes overall.
	 *
	 * As a small optimization, only write the version field in the first
	 * and third write.  The vcpu->pv_time cache is still valid, because the
	 * version field is the first in the struct.
	 */
	BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);

2124 2125 2126
	if (guest_hv_clock.version & 1)
		++guest_hv_clock.version;  /* first time write, random junk */

2127
	vcpu->hv_clock.version = guest_hv_clock.version + 1;
2128 2129 2130
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143

	smp_wmb();

	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
	vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);

	if (vcpu->pvclock_set_guest_stopped_request) {
		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
		vcpu->pvclock_set_guest_stopped_request = false;
	}

	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);

2144 2145 2146
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock));
2147 2148 2149 2150

	smp_wmb();

	vcpu->hv_clock.version++;
2151 2152 2153
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2154 2155
}

Z
Zachary Amsden 已提交
2156
static int kvm_guest_time_update(struct kvm_vcpu *v)
2157
{
2158
	unsigned long flags, tgt_tsc_khz;
2159
	struct kvm_vcpu_arch *vcpu = &v->arch;
2160
	struct kvm_arch *ka = &v->kvm->arch;
2161
	s64 kernel_ns;
2162
	u64 tsc_timestamp, host_tsc;
2163
	u8 pvclock_flags;
2164 2165 2166 2167
	bool use_master_clock;

	kernel_ns = 0;
	host_tsc = 0;
2168

2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179
	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
	spin_lock(&ka->pvclock_gtod_sync_lock);
	use_master_clock = ka->use_master_clock;
	if (use_master_clock) {
		host_tsc = ka->master_cycle_now;
		kernel_ns = ka->master_kernel_ns;
	}
	spin_unlock(&ka->pvclock_gtod_sync_lock);
2180 2181 2182

	/* Keep irq disabled to prevent changes to the clock */
	local_irq_save(flags);
2183 2184
	tgt_tsc_khz = __this_cpu_read(cpu_tsc_khz);
	if (unlikely(tgt_tsc_khz == 0)) {
2185 2186 2187 2188
		local_irq_restore(flags);
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
		return 1;
	}
2189
	if (!use_master_clock) {
2190
		host_tsc = rdtsc();
2191
		kernel_ns = ktime_get_boot_ns();
2192 2193
	}

2194
	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
2195

Z
Zachary Amsden 已提交
2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208
	/*
	 * We may have to catch up the TSC to match elapsed wall clock
	 * time for two reasons, even if kvmclock is used.
	 *   1) CPU could have been running below the maximum TSC rate
	 *   2) Broken TSC compensation resets the base at each VCPU
	 *      entry to avoid unknown leaps of TSC even when running
	 *      again on the same CPU.  This may cause apparent elapsed
	 *      time to disappear, and the guest to stand still or run
	 *	very slowly.
	 */
	if (vcpu->tsc_catchup) {
		u64 tsc = compute_guest_tsc(v, kernel_ns);
		if (tsc > tsc_timestamp) {
2209
			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
Z
Zachary Amsden 已提交
2210 2211
			tsc_timestamp = tsc;
		}
2212 2213
	}

2214 2215
	local_irq_restore(flags);

2216
	/* With all the info we got, fill in the values */
2217

2218 2219 2220 2221
	if (kvm_has_tsc_control)
		tgt_tsc_khz = kvm_scale_tsc(v, tgt_tsc_khz);

	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
2222
		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
2223 2224
				   &vcpu->hv_clock.tsc_shift,
				   &vcpu->hv_clock.tsc_to_system_mul);
2225
		vcpu->hw_tsc_khz = tgt_tsc_khz;
2226 2227
	}

2228
	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
2229
	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
Z
Zachary Amsden 已提交
2230
	vcpu->last_guest_tsc = tsc_timestamp;
2231

2232
	/* If the host uses TSC clocksource, then it is stable */
2233
	pvclock_flags = 0;
2234 2235 2236
	if (use_master_clock)
		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;

2237 2238
	vcpu->hv_clock.flags = pvclock_flags;

P
Paolo Bonzini 已提交
2239 2240 2241 2242
	if (vcpu->pv_time_enabled)
		kvm_setup_pvclock_page(v);
	if (v == kvm_get_vcpu(v->kvm, 0))
		kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
2243
	return 0;
2244 2245
}

2246 2247 2248 2249 2250 2251 2252 2253
/*
 * kvmclock updates which are isolated to a given vcpu, such as
 * vcpu->cpu migration, should not allow system_timestamp from
 * the rest of the vcpus to remain static. Otherwise ntp frequency
 * correction applies to one vcpu's system_timestamp but not
 * the others.
 *
 * So in those cases, request a kvmclock update for all vcpus.
2254 2255 2256 2257
 * We need to rate-limit these requests though, as they can
 * considerably slow guests that have a large number of vcpus.
 * The time for a remote vcpu to update its kvmclock is bound
 * by the delay we use to rate-limit the updates.
2258 2259
 */

2260 2261 2262
#define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)

static void kvmclock_update_fn(struct work_struct *work)
2263 2264
{
	int i;
2265 2266 2267 2268
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_update_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);
2269 2270 2271
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm) {
2272
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2273 2274 2275 2276
		kvm_vcpu_kick(vcpu);
	}
}

2277 2278 2279 2280
static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
{
	struct kvm *kvm = v->kvm;

2281
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
2282 2283 2284 2285
	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
					KVMCLOCK_UPDATE_DELAY);
}

2286 2287 2288 2289 2290 2291 2292 2293 2294
#define KVMCLOCK_SYNC_PERIOD (300 * HZ)

static void kvmclock_sync_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_sync_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);

2295 2296 2297
	if (!kvmclock_periodic_sync)
		return;

2298 2299 2300 2301 2302
	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
}

2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
/*
 * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
 */
static bool can_set_mci_status(struct kvm_vcpu *vcpu)
{
	/* McStatusWrEn enabled? */
	if (guest_cpuid_is_amd(vcpu))
		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));

	return false;
}

2315
static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2316
{
H
Huang Ying 已提交
2317 2318
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2319 2320
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
H
Huang Ying 已提交
2321

2322 2323
	switch (msr) {
	case MSR_IA32_MCG_STATUS:
H
Huang Ying 已提交
2324
		vcpu->arch.mcg_status = data;
2325
		break;
2326
	case MSR_IA32_MCG_CTL:
2327 2328
		if (!(mcg_cap & MCG_CTL_P) &&
		    (data || !msr_info->host_initiated))
H
Huang Ying 已提交
2329 2330
			return 1;
		if (data != 0 && data != ~(u64)0)
2331
			return 1;
H
Huang Ying 已提交
2332 2333 2334 2335
		vcpu->arch.mcg_ctl = data;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2336
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
H
Huang Ying 已提交
2337
			u32 offset = msr - MSR_IA32_MC0_CTL;
2338 2339 2340 2341 2342
			/* only 0 or all 1s can be written to IA32_MCi_CTL
			 * some Linux kernels though clear bit 10 in bank 4 to
			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
			 * this to avoid an uncatched #GP in the guest
			 */
H
Huang Ying 已提交
2343
			if ((offset & 0x3) == 0 &&
2344
			    data != 0 && (data | (1 << 10)) != ~(u64)0)
H
Huang Ying 已提交
2345
				return -1;
2346 2347

			/* MCi_STATUS */
2348
			if (!msr_info->host_initiated &&
2349 2350 2351 2352 2353
			    (offset & 0x3) == 1 && data != 0) {
				if (!can_set_mci_status(vcpu))
					return -1;
			}

H
Huang Ying 已提交
2354 2355 2356 2357 2358 2359 2360 2361
			vcpu->arch.mce_banks[offset] = data;
			break;
		}
		return 1;
	}
	return 0;
}

E
Ed Swierk 已提交
2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378
static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
	int lm = is_long_mode(vcpu);
	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
		: kvm->arch.xen_hvm_config.blob_size_32;
	u32 page_num = data & ~PAGE_MASK;
	u64 page_addr = data & PAGE_MASK;
	u8 *page;
	int r;

	r = -E2BIG;
	if (page_num >= blob_size)
		goto out;
	r = -ENOMEM;
2379 2380 2381
	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
	if (IS_ERR(page)) {
		r = PTR_ERR(page);
E
Ed Swierk 已提交
2382
		goto out;
2383
	}
2384
	if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE))
E
Ed Swierk 已提交
2385 2386 2387 2388 2389 2390 2391 2392
		goto out_free;
	r = 0;
out_free:
	kfree(page);
out:
	return r;
}

2393 2394 2395 2396
static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
{
	gpa_t gpa = data & ~0x3f;

2397 2398
	/* Bits 3:5 are reserved, Should be zero */
	if (data & 0x38)
2399 2400 2401 2402 2403 2404 2405 2406 2407 2408
		return 1;

	vcpu->arch.apf.msr_val = data;

	if (!(data & KVM_ASYNC_PF_ENABLED)) {
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_async_pf_hash_reset(vcpu);
		return 0;
	}

2409
	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
2410
					sizeof(u32)))
2411 2412
		return 1;

2413
	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
2414
	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
2415 2416 2417 2418
	kvm_async_pf_wakeup_all(vcpu);
	return 0;
}

2419 2420
static void kvmclock_reset(struct kvm_vcpu *vcpu)
{
2421
	vcpu->arch.pv_time_enabled = false;
2422 2423
}

2424 2425 2426 2427 2428 2429
static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
{
	++vcpu->stat.tlb_flush;
	kvm_x86_ops->tlb_flush(vcpu, invalidate_gpa);
}

G
Glauber Costa 已提交
2430 2431 2432 2433 2434
static void record_steal_time(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

2435
	if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
G
Glauber Costa 已提交
2436 2437 2438
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
		return;

2439 2440 2441 2442 2443 2444
	/*
	 * Doing a TLB flush here, on the guest's behalf, can avoid
	 * expensive IPIs.
	 */
	if (xchg(&vcpu->arch.st.steal.preempted, 0) & KVM_VCPU_FLUSH_TLB)
		kvm_vcpu_flush_tlb(vcpu, false);
2445

W
Wanpeng Li 已提交
2446 2447 2448 2449 2450
	if (vcpu->arch.st.steal.version & 1)
		vcpu->arch.st.steal.version += 1;  /* first time write, random junk */

	vcpu->arch.st.steal.version += 1;

2451
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
W
Wanpeng Li 已提交
2452 2453 2454 2455
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));

	smp_wmb();

2456 2457 2458
	vcpu->arch.st.steal.steal += current->sched_info.run_delay -
		vcpu->arch.st.last_steal;
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
W
Wanpeng Li 已提交
2459

2460
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
W
Wanpeng Li 已提交
2461 2462 2463 2464 2465
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));

	smp_wmb();

	vcpu->arch.st.steal.version += 1;
G
Glauber Costa 已提交
2466

2467
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
G
Glauber Costa 已提交
2468 2469 2470
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
}

2471
int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2472
{
2473
	bool pr = false;
2474 2475
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
2476

2477
	switch (msr) {
2478 2479 2480 2481 2482
	case MSR_AMD64_NB_CFG:
	case MSR_IA32_UCODE_WRITE:
	case MSR_VM_HSAVE_PA:
	case MSR_AMD64_PATCH_LOADER:
	case MSR_AMD64_BU_CFG2:
2483
	case MSR_AMD64_DC_CFG:
2484
	case MSR_F15H_EX_CFG:
2485 2486
		break;

2487 2488 2489 2490
	case MSR_IA32_UCODE_REV:
		if (msr_info->host_initiated)
			vcpu->arch.microcode_version = data;
		break;
2491 2492 2493 2494 2495
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.arch_capabilities = data;
		break;
2496
	case MSR_EFER:
2497
		return set_efer(vcpu, msr_info);
2498 2499
	case MSR_K7_HWCR:
		data &= ~(u64)0x40;	/* ignore flush filter disable */
2500
		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
2501
		data &= ~(u64)0x8;	/* ignore TLB cache disable */
2502 2503 2504 2505 2506

		/* Handle McStatusWrEn */
		if (data == BIT_ULL(18)) {
			vcpu->arch.msr_hwcr = data;
		} else if (data != 0) {
2507 2508
			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
				    data);
2509 2510
			return 1;
		}
2511
		break;
2512 2513
	case MSR_FAM10H_MMIO_CONF_BASE:
		if (data != 0) {
2514 2515
			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
				    "0x%llx\n", data);
2516 2517
			return 1;
		}
2518
		break;
2519 2520 2521 2522 2523 2524 2525 2526 2527
	case MSR_IA32_DEBUGCTLMSR:
		if (!data) {
			/* We support the non-activated case already */
			break;
		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
			/* Values other than LBR and BTF are vendor-specific,
			   thus reserved and should throw a #GP */
			return 1;
		}
2528 2529
		vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
			    __func__, data);
2530
		break;
A
Avi Kivity 已提交
2531
	case 0x200 ... 0x2ff:
2532
		return kvm_mtrr_set_msr(vcpu, msr, data);
2533
	case MSR_IA32_APICBASE:
2534
		return kvm_set_apic_base(vcpu, msr_info);
G
Gleb Natapov 已提交
2535 2536
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_write(vcpu, msr, data);
2537 2538 2539
	case MSR_IA32_TSCDEADLINE:
		kvm_set_lapic_tscdeadline_msr(vcpu, data);
		break;
W
Will Auld 已提交
2540
	case MSR_IA32_TSC_ADJUST:
2541
		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
W
Will Auld 已提交
2542
			if (!msr_info->host_initiated) {
2543
				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
2544
				adjust_tsc_offset_guest(vcpu, adj);
W
Will Auld 已提交
2545 2546 2547 2548
			}
			vcpu->arch.ia32_tsc_adjust_msr = data;
		}
		break;
2549
	case MSR_IA32_MISC_ENABLE:
2550 2551 2552 2553 2554 2555 2556 2557 2558
		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
		    ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
				return 1;
			vcpu->arch.ia32_misc_enable_msr = data;
			kvm_update_cpuid(vcpu);
		} else {
			vcpu->arch.ia32_misc_enable_msr = data;
		}
2559
		break;
P
Paolo Bonzini 已提交
2560 2561 2562 2563 2564
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smbase = data;
		break;
2565 2566 2567
	case MSR_IA32_TSC:
		kvm_write_tsc(vcpu, msr_info);
		break;
2568 2569 2570 2571 2572
	case MSR_SMI_COUNT:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smi_count = data;
		break;
2573
	case MSR_KVM_WALL_CLOCK_NEW:
2574 2575 2576 2577
	case MSR_KVM_WALL_CLOCK:
		vcpu->kvm->arch.wall_clock = data;
		kvm_write_wall_clock(vcpu->kvm, data);
		break;
2578
	case MSR_KVM_SYSTEM_TIME_NEW:
2579
	case MSR_KVM_SYSTEM_TIME: {
2580 2581
		struct kvm_arch *ka = &vcpu->kvm->arch;

2582
		kvmclock_reset(vcpu);
2583

2584 2585 2586 2587
		if (vcpu->vcpu_id == 0 && !msr_info->host_initiated) {
			bool tmp = (msr == MSR_KVM_SYSTEM_TIME);

			if (ka->boot_vcpu_runs_old_kvmclock != tmp)
2588
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2589 2590 2591 2592

			ka->boot_vcpu_runs_old_kvmclock = tmp;
		}

2593
		vcpu->arch.time = data;
2594
		kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2595 2596 2597 2598 2599

		/* we verify if the enable bit is set... */
		if (!(data & 1))
			break;

2600
		if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2601 2602
		     &vcpu->arch.pv_time, data & ~1ULL,
		     sizeof(struct pvclock_vcpu_time_info)))
2603 2604 2605
			vcpu->arch.pv_time_enabled = false;
		else
			vcpu->arch.pv_time_enabled = true;
2606

2607 2608
		break;
	}
2609 2610 2611 2612
	case MSR_KVM_ASYNC_PF_EN:
		if (kvm_pv_enable_async_pf(vcpu, data))
			return 1;
		break;
G
Glauber Costa 已提交
2613 2614 2615 2616 2617 2618 2619 2620
	case MSR_KVM_STEAL_TIME:

		if (unlikely(!sched_info_on()))
			return 1;

		if (data & KVM_STEAL_RESERVED_MASK)
			return 1;

2621
		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
2622 2623
						data & KVM_STEAL_VALID_BITS,
						sizeof(struct kvm_steal_time)))
G
Glauber Costa 已提交
2624 2625 2626 2627 2628 2629 2630 2631 2632 2633
			return 1;

		vcpu->arch.st.msr_val = data;

		if (!(data & KVM_MSR_ENABLED))
			break;

		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);

		break;
2634
	case MSR_KVM_PV_EOI_EN:
2635
		if (kvm_lapic_enable_pv_eoi(vcpu, data, sizeof(u8)))
2636 2637
			return 1;
		break;
G
Glauber Costa 已提交
2638

H
Huang Ying 已提交
2639 2640
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
2641
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
2642
		return set_msr_mce(vcpu, msr_info);
2643

2644 2645 2646 2647 2648
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
		pr = true; /* fall through */
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
2649
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2650
			return kvm_pmu_set_msr(vcpu, msr_info);
2651 2652

		if (pr || data != 0)
2653 2654
			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
				    "0x%x data 0x%llx\n", msr, data);
2655
		break;
2656 2657 2658 2659 2660
	case MSR_K7_CLK_CTL:
		/*
		 * Ignore all writes to this no longer documented MSR.
		 * Writes are only relevant for old K7 processors,
		 * all pre-dating SVM, but a recommended workaround from
G
Guo Chao 已提交
2661
		 * AMD for these chips. It is possible to specify the
2662 2663 2664 2665
		 * affected processor models on the command line, hence
		 * the need to ignore the workaround.
		 */
		break;
2666
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
2667 2668
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
A
Andrey Smetanin 已提交
2669
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
2670 2671 2672
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
2673 2674
		return kvm_hv_set_msr_common(vcpu, msr, data,
					     msr_info->host_initiated);
2675 2676 2677 2678
	case MSR_IA32_BBL_CR_CTL3:
		/* Drop writes to this legacy MSR -- see rdmsr
		 * counterpart for further detail.
		 */
2679 2680 2681
		if (report_ignored_msrs)
			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
				msr, data);
2682
		break;
2683
	case MSR_AMD64_OSVW_ID_LENGTH:
2684
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2685 2686 2687 2688
			return 1;
		vcpu->arch.osvw.length = data;
		break;
	case MSR_AMD64_OSVW_STATUS:
2689
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2690 2691 2692
			return 1;
		vcpu->arch.osvw.status = data;
		break;
K
Kyle Huey 已提交
2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706
	case MSR_PLATFORM_INFO:
		if (!msr_info->host_initiated ||
		    (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
		     cpuid_fault_enabled(vcpu)))
			return 1;
		vcpu->arch.msr_platform_info = data;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
		     !supports_cpuid_fault(vcpu)))
			return 1;
		vcpu->arch.msr_misc_features_enables = data;
		break;
2707
	default:
E
Ed Swierk 已提交
2708 2709
		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
			return xen_hvm_config(vcpu, data);
2710
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2711
			return kvm_pmu_set_msr(vcpu, msr_info);
2712
		if (!ignore_msrs) {
2713
			vcpu_debug_ratelimited(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n",
2714
				    msr, data);
2715 2716
			return 1;
		} else {
2717 2718 2719 2720
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu,
					"ignored wrmsr: 0x%x data 0x%llx\n",
					msr, data);
2721 2722
			break;
		}
2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);


/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
2734
int kvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
2735
{
2736
	return kvm_x86_ops->get_msr(vcpu, msr);
2737
}
2738
EXPORT_SYMBOL_GPL(kvm_get_msr);
2739

2740
static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
2741 2742
{
	u64 data;
H
Huang Ying 已提交
2743 2744
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2745 2746 2747 2748

	switch (msr) {
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
H
Huang Ying 已提交
2749 2750
		data = 0;
		break;
2751
	case MSR_IA32_MCG_CAP:
H
Huang Ying 已提交
2752 2753
		data = vcpu->arch.mcg_cap;
		break;
2754
	case MSR_IA32_MCG_CTL:
2755
		if (!(mcg_cap & MCG_CTL_P) && !host)
H
Huang Ying 已提交
2756 2757 2758 2759 2760 2761 2762 2763
			return 1;
		data = vcpu->arch.mcg_ctl;
		break;
	case MSR_IA32_MCG_STATUS:
		data = vcpu->arch.mcg_status;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2764
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
H
Huang Ying 已提交
2765 2766 2767 2768 2769 2770 2771 2772 2773 2774
			u32 offset = msr - MSR_IA32_MC0_CTL;
			data = vcpu->arch.mce_banks[offset];
			break;
		}
		return 1;
	}
	*pdata = data;
	return 0;
}

2775
int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
H
Huang Ying 已提交
2776
{
2777
	switch (msr_info->index) {
H
Huang Ying 已提交
2778
	case MSR_IA32_PLATFORM_ID:
2779
	case MSR_IA32_EBL_CR_POWERON:
2780 2781 2782 2783 2784
	case MSR_IA32_DEBUGCTLMSR:
	case MSR_IA32_LASTBRANCHFROMIP:
	case MSR_IA32_LASTBRANCHTOIP:
	case MSR_IA32_LASTINTFROMIP:
	case MSR_IA32_LASTINTTOIP:
2785
	case MSR_K8_SYSCFG:
2786 2787
	case MSR_K8_TSEG_ADDR:
	case MSR_K8_TSEG_MASK:
2788
	case MSR_VM_HSAVE_PA:
2789
	case MSR_K8_INT_PENDING_MSG:
2790
	case MSR_AMD64_NB_CFG:
2791
	case MSR_FAM10H_MMIO_CONF_BASE:
2792
	case MSR_AMD64_BU_CFG2:
D
Dmitry Bilunov 已提交
2793
	case MSR_IA32_PERF_CTL:
2794
	case MSR_AMD64_DC_CFG:
2795
	case MSR_F15H_EX_CFG:
2796
		msr_info->data = 0;
2797
		break;
2798
	case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
2799 2800 2801 2802
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
2803
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
2804 2805
			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
		msr_info->data = 0;
2806
		break;
2807
	case MSR_IA32_UCODE_REV:
2808
		msr_info->data = vcpu->arch.microcode_version;
2809
		break;
2810 2811 2812 2813 2814 2815
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
			return 1;
		msr_info->data = vcpu->arch.arch_capabilities;
		break;
2816 2817 2818
	case MSR_IA32_TSC:
		msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + vcpu->arch.tsc_offset;
		break;
A
Avi Kivity 已提交
2819 2820
	case MSR_MTRRcap:
	case 0x200 ... 0x2ff:
2821
		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
2822
	case 0xcd: /* fsb frequency */
2823
		msr_info->data = 3;
2824
		break;
2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836
		/*
		 * MSR_EBC_FREQUENCY_ID
		 * Conservative value valid for even the basic CPU models.
		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
		 * and 266MHz for model 3, or 4. Set Core Clock
		 * Frequency to System Bus Frequency Ratio to 1 (bits
		 * 31:24) even though these are only valid for CPU
		 * models > 2, however guests may end up dividing or
		 * multiplying by zero otherwise.
		 */
	case MSR_EBC_FREQUENCY_ID:
2837
		msr_info->data = 1 << 24;
2838
		break;
2839
	case MSR_IA32_APICBASE:
2840
		msr_info->data = kvm_get_apic_base(vcpu);
2841
		break;
G
Gleb Natapov 已提交
2842
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
2843
		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
G
Gleb Natapov 已提交
2844
		break;
2845
	case MSR_IA32_TSCDEADLINE:
2846
		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
2847
		break;
W
Will Auld 已提交
2848
	case MSR_IA32_TSC_ADJUST:
2849
		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
W
Will Auld 已提交
2850
		break;
2851
	case MSR_IA32_MISC_ENABLE:
2852
		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
2853
		break;
P
Paolo Bonzini 已提交
2854 2855 2856 2857
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		msr_info->data = vcpu->arch.smbase;
2858
		break;
2859 2860 2861
	case MSR_SMI_COUNT:
		msr_info->data = vcpu->arch.smi_count;
		break;
2862 2863
	case MSR_IA32_PERF_STATUS:
		/* TSC increment by tick */
2864
		msr_info->data = 1000ULL;
2865
		/* CPU multiplier */
2866
		msr_info->data |= (((uint64_t)4ULL) << 40);
2867
		break;
2868
	case MSR_EFER:
2869
		msr_info->data = vcpu->arch.efer;
2870
		break;
2871
	case MSR_KVM_WALL_CLOCK:
2872
	case MSR_KVM_WALL_CLOCK_NEW:
2873
		msr_info->data = vcpu->kvm->arch.wall_clock;
2874 2875
		break;
	case MSR_KVM_SYSTEM_TIME:
2876
	case MSR_KVM_SYSTEM_TIME_NEW:
2877
		msr_info->data = vcpu->arch.time;
2878
		break;
2879
	case MSR_KVM_ASYNC_PF_EN:
2880
		msr_info->data = vcpu->arch.apf.msr_val;
2881
		break;
G
Glauber Costa 已提交
2882
	case MSR_KVM_STEAL_TIME:
2883
		msr_info->data = vcpu->arch.st.msr_val;
G
Glauber Costa 已提交
2884
		break;
2885
	case MSR_KVM_PV_EOI_EN:
2886
		msr_info->data = vcpu->arch.pv_eoi.msr_val;
2887
		break;
H
Huang Ying 已提交
2888 2889 2890 2891 2892
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
2893
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
2894 2895
		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
				   msr_info->host_initiated);
2896 2897 2898 2899 2900 2901 2902 2903 2904 2905
	case MSR_K7_CLK_CTL:
		/*
		 * Provide expected ramp-up count for K7. All other
		 * are set to zero, indicating minimum divisors for
		 * every field.
		 *
		 * This prevents guest kernels on AMD host with CPU
		 * type 6, model 8 and higher from exploding due to
		 * the rdmsr failing.
		 */
2906
		msr_info->data = 0x20000000;
2907
		break;
2908
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
2909 2910
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
A
Andrey Smetanin 已提交
2911
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
2912 2913 2914
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
2915
		return kvm_hv_get_msr_common(vcpu,
2916 2917
					     msr_info->index, &msr_info->data,
					     msr_info->host_initiated);
2918
		break;
2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929
	case MSR_IA32_BBL_CR_CTL3:
		/* This legacy MSR exists but isn't fully documented in current
		 * silicon.  It is however accessed by winxp in very narrow
		 * scenarios where it sets bit #19, itself documented as
		 * a "reserved" bit.  Best effort attempt to source coherent
		 * read data here should the balance of the register be
		 * interpreted by the guest:
		 *
		 * L2 cache control register 3: 64GB range, 256KB size,
		 * enabled, latency 0x1, configured
		 */
2930
		msr_info->data = 0xbe702111;
2931
		break;
2932
	case MSR_AMD64_OSVW_ID_LENGTH:
2933
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2934
			return 1;
2935
		msr_info->data = vcpu->arch.osvw.length;
2936 2937
		break;
	case MSR_AMD64_OSVW_STATUS:
2938
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2939
			return 1;
2940
		msr_info->data = vcpu->arch.osvw.status;
2941
		break;
K
Kyle Huey 已提交
2942
	case MSR_PLATFORM_INFO:
2943 2944 2945
		if (!msr_info->host_initiated &&
		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
			return 1;
K
Kyle Huey 已提交
2946 2947 2948 2949 2950
		msr_info->data = vcpu->arch.msr_platform_info;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		msr_info->data = vcpu->arch.msr_misc_features_enables;
		break;
2951 2952 2953
	case MSR_K7_HWCR:
		msr_info->data = vcpu->arch.msr_hwcr;
		break;
2954
	default:
2955
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
2956
			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
2957
		if (!ignore_msrs) {
2958 2959
			vcpu_debug_ratelimited(vcpu, "unhandled rdmsr: 0x%x\n",
					       msr_info->index);
2960 2961
			return 1;
		} else {
2962 2963 2964
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n",
					msr_info->index);
2965
			msr_info->data = 0;
2966 2967
		}
		break;
2968 2969 2970 2971 2972
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

2973 2974 2975 2976 2977 2978 2979 2980 2981 2982
/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
2983
	int i;
2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007

	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
3008
	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
3009 3010 3011 3012 3013 3014 3015
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
3016 3017 3018
	entries = memdup_user(user_msrs->entries, size);
	if (IS_ERR(entries)) {
		r = PTR_ERR(entries);
3019
		goto out;
3020
	}
3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032

	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
3033
	kfree(entries);
3034 3035 3036 3037
out:
	return r;
}

3038 3039 3040
static inline bool kvm_can_mwait_in_guest(void)
{
	return boot_cpu_has(X86_FEATURE_MWAIT) &&
3041 3042
		!boot_cpu_has_bug(X86_BUG_MONITOR) &&
		boot_cpu_has(X86_FEATURE_ARAT);
3043 3044
}

3045
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
3046
{
3047
	int r = 0;
3048 3049 3050 3051 3052 3053

	switch (ext) {
	case KVM_CAP_IRQCHIP:
	case KVM_CAP_HLT:
	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
	case KVM_CAP_SET_TSS_ADDR:
3054
	case KVM_CAP_EXT_CPUID:
B
Borislav Petkov 已提交
3055
	case KVM_CAP_EXT_EMUL_CPUID:
3056
	case KVM_CAP_CLOCKSOURCE:
S
Sheng Yang 已提交
3057
	case KVM_CAP_PIT:
3058
	case KVM_CAP_NOP_IO_DELAY:
3059
	case KVM_CAP_MP_STATE:
3060
	case KVM_CAP_SYNC_MMU:
3061
	case KVM_CAP_USER_NMI:
3062
	case KVM_CAP_REINJECT_CONTROL:
3063
	case KVM_CAP_IRQ_INJECT_STATUS:
G
Gregory Haskins 已提交
3064
	case KVM_CAP_IOEVENTFD:
3065
	case KVM_CAP_IOEVENTFD_NO_LENGTH:
3066
	case KVM_CAP_PIT2:
B
Beth Kon 已提交
3067
	case KVM_CAP_PIT_STATE2:
3068
	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
E
Ed Swierk 已提交
3069
	case KVM_CAP_XEN_HVM:
J
Jan Kiszka 已提交
3070
	case KVM_CAP_VCPU_EVENTS:
3071
	case KVM_CAP_HYPERV:
G
Gleb Natapov 已提交
3072
	case KVM_CAP_HYPERV_VAPIC:
3073
	case KVM_CAP_HYPERV_SPIN:
3074
	case KVM_CAP_HYPERV_SYNIC:
3075
	case KVM_CAP_HYPERV_SYNIC2:
3076
	case KVM_CAP_HYPERV_VP_INDEX:
3077
	case KVM_CAP_HYPERV_EVENTFD:
3078
	case KVM_CAP_HYPERV_TLBFLUSH:
3079
	case KVM_CAP_HYPERV_SEND_IPI:
3080
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
3081
	case KVM_CAP_HYPERV_CPUID:
3082
	case KVM_CAP_PCI_SEGMENT:
3083
	case KVM_CAP_DEBUGREGS:
3084
	case KVM_CAP_X86_ROBUST_SINGLESTEP:
3085
	case KVM_CAP_XSAVE:
3086
	case KVM_CAP_ASYNC_PF:
3087
	case KVM_CAP_GET_TSC_KHZ:
3088
	case KVM_CAP_KVMCLOCK_CTRL:
X
Xiao Guangrong 已提交
3089
	case KVM_CAP_READONLY_MEM:
3090
	case KVM_CAP_HYPERV_TIME:
3091
	case KVM_CAP_IOAPIC_POLARITY_IGNORED:
3092
	case KVM_CAP_TSC_DEADLINE_TIMER:
3093
	case KVM_CAP_DISABLE_QUIRKS:
3094
	case KVM_CAP_SET_BOOT_CPU_ID:
3095
 	case KVM_CAP_SPLIT_IRQCHIP:
3096
	case KVM_CAP_IMMEDIATE_EXIT:
3097
	case KVM_CAP_GET_MSR_FEATURES:
3098
	case KVM_CAP_MSR_PLATFORM_INFO:
3099
	case KVM_CAP_EXCEPTION_PAYLOAD:
3100 3101
		r = 1;
		break;
K
Ken Hofsass 已提交
3102 3103 3104
	case KVM_CAP_SYNC_REGS:
		r = KVM_SYNC_X86_VALID_FIELDS;
		break;
3105 3106 3107
	case KVM_CAP_ADJUST_CLOCK:
		r = KVM_CLOCK_TSC_STABLE;
		break;
3108
	case KVM_CAP_X86_DISABLE_EXITS:
3109 3110
		r |=  KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE |
		      KVM_X86_DISABLE_EXITS_CSTATE;
3111 3112
		if(kvm_can_mwait_in_guest())
			r |= KVM_X86_DISABLE_EXITS_MWAIT;
3113
		break;
3114 3115 3116 3117 3118 3119 3120 3121 3122
	case KVM_CAP_X86_SMM:
		/* SMBASE is usually relocated above 1M on modern chipsets,
		 * and SMM handlers might indeed rely on 4G segment limits,
		 * so do not report SMM to be available if real mode is
		 * emulated via vm86 mode.  Still, do not go to great lengths
		 * to avoid userspace's usage of the feature, because it is a
		 * fringe case that is not enabled except via specific settings
		 * of the module parameters.
		 */
3123
		r = kvm_x86_ops->has_emulated_msr(MSR_IA32_SMBASE);
3124
		break;
3125 3126 3127
	case KVM_CAP_VAPIC:
		r = !kvm_x86_ops->cpu_has_accelerated_tpr();
		break;
3128
	case KVM_CAP_NR_VCPUS:
3129 3130 3131
		r = KVM_SOFT_MAX_VCPUS;
		break;
	case KVM_CAP_MAX_VCPUS:
3132 3133
		r = KVM_MAX_VCPUS;
		break;
3134 3135 3136
	case KVM_CAP_MAX_VCPU_ID:
		r = KVM_MAX_VCPU_ID;
		break;
3137 3138
	case KVM_CAP_PV_MMU:	/* obsolete */
		r = 0;
3139
		break;
H
Huang Ying 已提交
3140 3141 3142
	case KVM_CAP_MCE:
		r = KVM_MAX_MCE_BANKS;
		break;
3143
	case KVM_CAP_XCRS:
3144
		r = boot_cpu_has(X86_FEATURE_XSAVE);
3145
		break;
3146 3147 3148
	case KVM_CAP_TSC_CONTROL:
		r = kvm_has_tsc_control;
		break;
3149 3150 3151
	case KVM_CAP_X2APIC_API:
		r = KVM_X2APIC_API_VALID_FLAGS;
		break;
3152 3153
	case KVM_CAP_NESTED_STATE:
		r = kvm_x86_ops->get_nested_state ?
3154
			kvm_x86_ops->get_nested_state(NULL, NULL, 0) : 0;
3155
		break;
3156 3157 3158 3159 3160 3161 3162
	default:
		break;
	}
	return r;

}

3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175
long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {
	case KVM_GET_MSR_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
3176
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
3177 3178
			goto out;
		n = msr_list.nmsrs;
3179
		msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
3180
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
3181 3182
			goto out;
		r = -E2BIG;
J
Jan Kiszka 已提交
3183
		if (n < msr_list.nmsrs)
3184 3185 3186 3187 3188
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
				 num_msrs_to_save * sizeof(u32)))
			goto out;
J
Jan Kiszka 已提交
3189
		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
3190
				 &emulated_msrs,
3191
				 num_emulated_msrs * sizeof(u32)))
3192 3193 3194 3195
			goto out;
		r = 0;
		break;
	}
B
Borislav Petkov 已提交
3196 3197
	case KVM_GET_SUPPORTED_CPUID:
	case KVM_GET_EMULATED_CPUID: {
3198 3199 3200 3201
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
3202
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
3203
			goto out;
B
Borislav Petkov 已提交
3204 3205 3206

		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
					    ioctl);
3207 3208 3209 3210
		if (r)
			goto out;

		r = -EFAULT;
3211
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
3212 3213 3214 3215
			goto out;
		r = 0;
		break;
	}
H
Huang Ying 已提交
3216 3217
	case KVM_X86_GET_MCE_CAP_SUPPORTED: {
		r = -EFAULT;
3218 3219
		if (copy_to_user(argp, &kvm_mce_cap_supported,
				 sizeof(kvm_mce_cap_supported)))
H
Huang Ying 已提交
3220 3221 3222
			goto out;
		r = 0;
		break;
3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247
	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned int n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
			goto out;
		n = msr_list.nmsrs;
		msr_list.nmsrs = num_msr_based_features;
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
			goto out;
		r = -E2BIG;
		if (n < msr_list.nmsrs)
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msr_based_features,
				 num_msr_based_features * sizeof(u32)))
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_MSRS:
		r = msr_io(NULL, argp, do_get_msr_feature, 1);
		break;
H
Huang Ying 已提交
3248
	}
3249 3250 3251 3252 3253 3254 3255
	default:
		r = -EINVAL;
	}
out:
	return r;
}

3256 3257 3258 3259 3260 3261 3262
static void wbinvd_ipi(void *garbage)
{
	wbinvd();
}

static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
3263
	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
3264 3265
}

3266 3267
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
3268 3269 3270 3271 3272 3273 3274 3275 3276
	/* Address WBINVD may be executed by guest */
	if (need_emulate_wbinvd(vcpu)) {
		if (kvm_x86_ops->has_wbinvd_exit())
			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
			smp_call_function_single(vcpu->cpu,
					wbinvd_ipi, NULL, 1);
	}

3277
	kvm_x86_ops->vcpu_load(vcpu, cpu);
3278

3279 3280 3281 3282
	/* Apply any externally detected TSC adjustments (due to suspend) */
	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
		vcpu->arch.tsc_offset_adjustment = 0;
3283
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3284
	}
3285

3286
	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
3287
		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
3288
				rdtsc() - vcpu->arch.last_host_tsc;
Z
Zachary Amsden 已提交
3289 3290
		if (tsc_delta < 0)
			mark_tsc_unstable("KVM discovered backwards TSC");
3291

3292
		if (kvm_check_tsc_unstable()) {
3293
			u64 offset = kvm_compute_tsc_offset(vcpu,
3294
						vcpu->arch.last_guest_tsc);
3295
			kvm_vcpu_write_tsc_offset(vcpu, offset);
Z
Zachary Amsden 已提交
3296 3297
			vcpu->arch.tsc_catchup = 1;
		}
3298 3299 3300 3301

		if (kvm_lapic_hv_timer_in_use(vcpu))
			kvm_lapic_restart_hv_timer(vcpu);

3302 3303 3304 3305 3306
		/*
		 * On a host with synchronized TSC, there is no need to update
		 * kvmclock on vcpu->cpu migration
		 */
		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
3307
			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
Z
Zachary Amsden 已提交
3308
		if (vcpu->cpu != cpu)
3309
			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
Z
Zachary Amsden 已提交
3310
		vcpu->cpu = cpu;
Z
Zachary Amsden 已提交
3311
	}
G
Glauber Costa 已提交
3312 3313

	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3314 3315
}

3316 3317 3318 3319 3320
static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

W
Wanpeng Li 已提交
3321
	vcpu->arch.st.steal.preempted = KVM_VCPU_PREEMPTED;
3322

3323
	kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.st.stime,
3324 3325 3326 3327 3328
			&vcpu->arch.st.steal.preempted,
			offsetof(struct kvm_steal_time, preempted),
			sizeof(vcpu->arch.st.steal.preempted));
}

3329 3330
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
3331
	int idx;
3332 3333 3334 3335

	if (vcpu->preempted)
		vcpu->arch.preempted_in_kernel = !kvm_x86_ops->get_cpl(vcpu);

3336 3337 3338 3339 3340 3341 3342 3343 3344
	/*
	 * Disable page faults because we're in atomic context here.
	 * kvm_write_guest_offset_cached() would call might_fault()
	 * that relies on pagefault_disable() to tell if there's a
	 * bug. NOTE: the write to guest memory may not go through if
	 * during postcopy live migration or if there's heavy guest
	 * paging.
	 */
	pagefault_disable();
3345 3346 3347 3348 3349
	/*
	 * kvm_memslots() will be called by
	 * kvm_write_guest_offset_cached() so take the srcu lock.
	 */
	idx = srcu_read_lock(&vcpu->kvm->srcu);
3350
	kvm_steal_time_set_preempted(vcpu);
3351
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
3352
	pagefault_enable();
3353
	kvm_x86_ops->vcpu_put(vcpu);
3354
	vcpu->arch.last_host_tsc = rdtsc();
3355
	/*
3356 3357 3358
	 * If userspace has set any breakpoints or watchpoints, dr6 is restored
	 * on every vmexit, but if not, we might have a stale dr6 from the
	 * guest. do_debug expects dr6 to be cleared after it runs, do the same.
3359
	 */
3360
	set_debugreg(0, 6);
3361 3362 3363 3364 3365
}

static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3366
	if (vcpu->arch.apicv_active)
3367 3368
		kvm_x86_ops->sync_pir_to_irr(vcpu);

3369
	return kvm_apic_get_state(vcpu, s);
3370 3371 3372 3373 3374
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3375 3376 3377 3378 3379
	int r;

	r = kvm_apic_set_state(vcpu, s);
	if (r)
		return r;
3380
	update_cr8_intercept(vcpu);
3381 3382 3383 3384

	return 0;
}

3385 3386 3387 3388 3389 3390
static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
{
	return (!lapic_in_kernel(vcpu) ||
		kvm_apic_accept_pic_intr(vcpu));
}

3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404
/*
 * if userspace requested an interrupt window, check that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
{
	return kvm_arch_interrupt_allowed(vcpu) &&
		!kvm_cpu_has_interrupt(vcpu) &&
		!kvm_event_needs_reinjection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);
}

3405 3406 3407
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
{
3408
	if (irq->irq >= KVM_NR_INTERRUPTS)
3409
		return -EINVAL;
3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421

	if (!irqchip_in_kernel(vcpu->kvm)) {
		kvm_queue_interrupt(vcpu, irq->irq, false);
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		return 0;
	}

	/*
	 * With in-kernel LAPIC, we only use this to inject EXTINT, so
	 * fail for in-kernel 8259.
	 */
	if (pic_in_kernel(vcpu->kvm))
3422 3423
		return -ENXIO;

3424 3425
	if (vcpu->arch.pending_external_vector != -1)
		return -EEXIST;
3426

3427
	vcpu->arch.pending_external_vector = irq->irq;
3428
	kvm_make_request(KVM_REQ_EVENT, vcpu);
3429 3430 3431
	return 0;
}

3432 3433 3434 3435 3436 3437 3438
static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
{
	kvm_inject_nmi(vcpu);

	return 0;
}

3439 3440
static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
{
P
Paolo Bonzini 已提交
3441 3442
	kvm_make_request(KVM_REQ_SMI, vcpu);

3443 3444 3445
	return 0;
}

3446 3447 3448 3449 3450 3451 3452 3453 3454
static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
					   struct kvm_tpr_access_ctl *tac)
{
	if (tac->flags)
		return -EINVAL;
	vcpu->arch.tpr_access_reporting = !!tac->enabled;
	return 0;
}

H
Huang Ying 已提交
3455 3456 3457 3458 3459 3460 3461
static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
					u64 mcg_cap)
{
	int r;
	unsigned bank_num = mcg_cap & 0xff, bank;

	r = -EINVAL;
3462
	if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
H
Huang Ying 已提交
3463
		goto out;
3464
	if (mcg_cap & ~(kvm_mce_cap_supported | 0xff | 0xff0000))
H
Huang Ying 已提交
3465 3466 3467 3468 3469 3470 3471 3472 3473
		goto out;
	r = 0;
	vcpu->arch.mcg_cap = mcg_cap;
	/* Init IA32_MCG_CTL to all 1s */
	if (mcg_cap & MCG_CTL_P)
		vcpu->arch.mcg_ctl = ~(u64)0;
	/* Init IA32_MCi_CTL to all 1s */
	for (bank = 0; bank < bank_num; bank++)
		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
3474 3475 3476

	if (kvm_x86_ops->setup_mce)
		kvm_x86_ops->setup_mce(vcpu);
H
Huang Ying 已提交
3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505
out:
	return r;
}

static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
				      struct kvm_x86_mce *mce)
{
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
	u64 *banks = vcpu->arch.mce_banks;

	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
		return -EINVAL;
	/*
	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
	 * reporting is disabled
	 */
	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
	    vcpu->arch.mcg_ctl != ~(u64)0)
		return 0;
	banks += 4 * mce->bank;
	/*
	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
	 * reporting is disabled for the bank
	 */
	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
		return 0;
	if (mce->status & MCI_STATUS_UC) {
		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
3506
		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
3507
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
H
Huang Ying 已提交
3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528
			return 0;
		}
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		vcpu->arch.mcg_status = mce->mcg_status;
		banks[1] = mce->status;
		kvm_queue_exception(vcpu, MC_VECTOR);
	} else if (!(banks[1] & MCI_STATUS_VAL)
		   || !(banks[1] & MCI_STATUS_UC)) {
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		banks[1] = mce->status;
	} else
		banks[1] |= MCI_STATUS_OVER;
	return 0;
}

J
Jan Kiszka 已提交
3529 3530 3531
static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
					       struct kvm_vcpu_events *events)
{
A
Avi Kivity 已提交
3532
	process_nmi(vcpu);
3533

3534
	/*
3535 3536 3537 3538
	 * The API doesn't provide the instruction length for software
	 * exceptions, so don't report them. As long as the guest RIP
	 * isn't advanced, we should expect to encounter the exception
	 * again.
3539
	 */
3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554
	if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {
		events->exception.injected = 0;
		events->exception.pending = 0;
	} else {
		events->exception.injected = vcpu->arch.exception.injected;
		events->exception.pending = vcpu->arch.exception.pending;
		/*
		 * For ABI compatibility, deliberately conflate
		 * pending and injected exceptions when
		 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
		 */
		if (!vcpu->kvm->arch.exception_payload_enabled)
			events->exception.injected |=
				vcpu->arch.exception.pending;
	}
J
Jan Kiszka 已提交
3555 3556 3557
	events->exception.nr = vcpu->arch.exception.nr;
	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
	events->exception.error_code = vcpu->arch.exception.error_code;
3558 3559
	events->exception_has_payload = vcpu->arch.exception.has_payload;
	events->exception_payload = vcpu->arch.exception.payload;
J
Jan Kiszka 已提交
3560

3561
	events->interrupt.injected =
3562
		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
J
Jan Kiszka 已提交
3563
	events->interrupt.nr = vcpu->arch.interrupt.nr;
3564
	events->interrupt.soft = 0;
3565
	events->interrupt.shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
J
Jan Kiszka 已提交
3566 3567

	events->nmi.injected = vcpu->arch.nmi_injected;
A
Avi Kivity 已提交
3568
	events->nmi.pending = vcpu->arch.nmi_pending != 0;
J
Jan Kiszka 已提交
3569
	events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
3570
	events->nmi.pad = 0;
J
Jan Kiszka 已提交
3571

3572
	events->sipi_vector = 0; /* never valid when reporting to user space */
J
Jan Kiszka 已提交
3573

3574 3575 3576 3577 3578 3579
	events->smi.smm = is_smm(vcpu);
	events->smi.pending = vcpu->arch.smi_pending;
	events->smi.smm_inside_nmi =
		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
	events->smi.latched_init = kvm_lapic_latched_init(vcpu);

3580
	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
3581 3582
			 | KVM_VCPUEVENT_VALID_SHADOW
			 | KVM_VCPUEVENT_VALID_SMM);
3583 3584 3585
	if (vcpu->kvm->arch.exception_payload_enabled)
		events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;

3586
	memset(&events->reserved, 0, sizeof(events->reserved));
J
Jan Kiszka 已提交
3587 3588
}

3589
static void kvm_smm_changed(struct kvm_vcpu *vcpu);
3590

J
Jan Kiszka 已提交
3591 3592 3593
static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
					      struct kvm_vcpu_events *events)
{
3594
	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
3595
			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
3596
			      | KVM_VCPUEVENT_VALID_SHADOW
3597 3598
			      | KVM_VCPUEVENT_VALID_SMM
			      | KVM_VCPUEVENT_VALID_PAYLOAD))
J
Jan Kiszka 已提交
3599 3600
		return -EINVAL;

3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614
	if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
		if (!vcpu->kvm->arch.exception_payload_enabled)
			return -EINVAL;
		if (events->exception.pending)
			events->exception.injected = 0;
		else
			events->exception_has_payload = 0;
	} else {
		events->exception.pending = 0;
		events->exception_has_payload = 0;
	}

	if ((events->exception.injected || events->exception.pending) &&
	    (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
3615 3616
		return -EINVAL;

3617 3618 3619 3620 3621 3622
	/* INITs are latched while in SMM */
	if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
	    (events->smi.smm || events->smi.pending) &&
	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
		return -EINVAL;

A
Avi Kivity 已提交
3623
	process_nmi(vcpu);
3624 3625
	vcpu->arch.exception.injected = events->exception.injected;
	vcpu->arch.exception.pending = events->exception.pending;
J
Jan Kiszka 已提交
3626 3627 3628
	vcpu->arch.exception.nr = events->exception.nr;
	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
	vcpu->arch.exception.error_code = events->exception.error_code;
3629 3630
	vcpu->arch.exception.has_payload = events->exception_has_payload;
	vcpu->arch.exception.payload = events->exception_payload;
J
Jan Kiszka 已提交
3631

3632
	vcpu->arch.interrupt.injected = events->interrupt.injected;
J
Jan Kiszka 已提交
3633 3634
	vcpu->arch.interrupt.nr = events->interrupt.nr;
	vcpu->arch.interrupt.soft = events->interrupt.soft;
3635 3636 3637
	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
		kvm_x86_ops->set_interrupt_shadow(vcpu,
						  events->interrupt.shadow);
J
Jan Kiszka 已提交
3638 3639

	vcpu->arch.nmi_injected = events->nmi.injected;
3640 3641
	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
		vcpu->arch.nmi_pending = events->nmi.pending;
J
Jan Kiszka 已提交
3642 3643
	kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);

3644
	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
3645
	    lapic_in_kernel(vcpu))
3646
		vcpu->arch.apic->sipi_vector = events->sipi_vector;
J
Jan Kiszka 已提交
3647

3648
	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
3649 3650 3651 3652 3653 3654 3655
		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
			if (events->smi.smm)
				vcpu->arch.hflags |= HF_SMM_MASK;
			else
				vcpu->arch.hflags &= ~HF_SMM_MASK;
			kvm_smm_changed(vcpu);
		}
3656

3657
		vcpu->arch.smi_pending = events->smi.pending;
3658 3659 3660 3661

		if (events->smi.smm) {
			if (events->smi.smm_inside_nmi)
				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
3662
			else
3663 3664 3665 3666 3667 3668 3669
				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
			if (lapic_in_kernel(vcpu)) {
				if (events->smi.latched_init)
					set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
				else
					clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
			}
3670 3671 3672
		}
	}

3673 3674
	kvm_make_request(KVM_REQ_EVENT, vcpu);

J
Jan Kiszka 已提交
3675 3676 3677
	return 0;
}

3678 3679 3680
static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
					     struct kvm_debugregs *dbgregs)
{
J
Jan Kiszka 已提交
3681 3682
	unsigned long val;

3683
	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
3684
	kvm_get_dr(vcpu, 6, &val);
J
Jan Kiszka 已提交
3685
	dbgregs->dr6 = val;
3686 3687
	dbgregs->dr7 = vcpu->arch.dr7;
	dbgregs->flags = 0;
3688
	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
3689 3690 3691 3692 3693 3694 3695 3696
}

static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
					    struct kvm_debugregs *dbgregs)
{
	if (dbgregs->flags)
		return -EINVAL;

3697 3698 3699 3700 3701
	if (dbgregs->dr6 & ~0xffffffffull)
		return -EINVAL;
	if (dbgregs->dr7 & ~0xffffffffull)
		return -EINVAL;

3702
	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
3703
	kvm_update_dr0123(vcpu);
3704
	vcpu->arch.dr6 = dbgregs->dr6;
J
Jan Kiszka 已提交
3705
	kvm_update_dr6(vcpu);
3706
	vcpu->arch.dr7 = dbgregs->dr7;
3707
	kvm_update_dr7(vcpu);
3708 3709 3710 3711

	return 0;
}

3712 3713 3714 3715
#define XSTATE_COMPACTION_ENABLED (1ULL << 63)

static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu)
{
3716
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
3717
	u64 xstate_bv = xsave->header.xfeatures;
3718 3719 3720 3721 3722 3723 3724 3725 3726
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(dest, xsave, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV */
3727
	xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;
3728 3729 3730 3731 3732 3733
	*(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv;

	/*
	 * Copy each region from the possibly compacted offset to the
	 * non-compacted offset.
	 */
D
Dave Hansen 已提交
3734
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
3735
	while (valid) {
3736 3737 3738
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *src = get_xsave_addr(xsave, xfeature_nr);
3739 3740 3741

		if (src) {
			u32 size, offset, ecx, edx;
3742
			cpuid_count(XSTATE_CPUID, xfeature_nr,
3743
				    &size, &offset, &ecx, &edx);
3744
			if (xfeature_nr == XFEATURE_PKRU)
3745 3746 3747 3748 3749
				memcpy(dest + offset, &vcpu->arch.pkru,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest + offset, src, size);

3750 3751
		}

3752
		valid -= xfeature_mask;
3753 3754 3755 3756 3757
	}
}

static void load_xsave(struct kvm_vcpu *vcpu, u8 *src)
{
3758
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
3759 3760 3761 3762 3763 3764 3765 3766 3767 3768
	u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET);
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(xsave, src, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV and possibly XCOMP_BV.  */
3769
	xsave->header.xfeatures = xstate_bv;
3770
	if (boot_cpu_has(X86_FEATURE_XSAVES))
3771
		xsave->header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED;
3772 3773 3774 3775 3776

	/*
	 * Copy each region from the non-compacted offset to the
	 * possibly compacted offset.
	 */
D
Dave Hansen 已提交
3777
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
3778
	while (valid) {
3779 3780 3781
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *dest = get_xsave_addr(xsave, xfeature_nr);
3782 3783 3784

		if (dest) {
			u32 size, offset, ecx, edx;
3785
			cpuid_count(XSTATE_CPUID, xfeature_nr,
3786
				    &size, &offset, &ecx, &edx);
3787
			if (xfeature_nr == XFEATURE_PKRU)
3788 3789 3790 3791
				memcpy(&vcpu->arch.pkru, src + offset,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest, src + offset, size);
3792
		}
3793

3794
		valid -= xfeature_mask;
3795 3796 3797
	}
}

3798 3799 3800
static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
					 struct kvm_xsave *guest_xsave)
{
3801
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
3802 3803
		memset(guest_xsave, 0, sizeof(struct kvm_xsave));
		fill_xsave((u8 *) guest_xsave->region, vcpu);
3804
	} else {
3805
		memcpy(guest_xsave->region,
3806
			&vcpu->arch.guest_fpu->state.fxsave,
3807
			sizeof(struct fxregs_state));
3808
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
D
Dave Hansen 已提交
3809
			XFEATURE_MASK_FPSSE;
3810 3811 3812
	}
}

3813 3814
#define XSAVE_MXCSR_OFFSET 24

3815 3816 3817 3818 3819
static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
					struct kvm_xsave *guest_xsave)
{
	u64 xstate_bv =
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
3820
	u32 mxcsr = *(u32 *)&guest_xsave->region[XSAVE_MXCSR_OFFSET / sizeof(u32)];
3821

3822
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
3823 3824 3825 3826 3827
		/*
		 * Here we allow setting states that are not present in
		 * CPUID leaf 0xD, index 0, EDX:EAX.  This is for compatibility
		 * with old userspace.
		 */
3828 3829
		if (xstate_bv & ~kvm_supported_xcr0() ||
			mxcsr & ~mxcsr_feature_mask)
3830
			return -EINVAL;
3831
		load_xsave(vcpu, (u8 *)guest_xsave->region);
3832
	} else {
3833 3834
		if (xstate_bv & ~XFEATURE_MASK_FPSSE ||
			mxcsr & ~mxcsr_feature_mask)
3835
			return -EINVAL;
3836
		memcpy(&vcpu->arch.guest_fpu->state.fxsave,
3837
			guest_xsave->region, sizeof(struct fxregs_state));
3838 3839 3840 3841 3842 3843 3844
	}
	return 0;
}

static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
					struct kvm_xcrs *guest_xcrs)
{
3845
	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860
		guest_xcrs->nr_xcrs = 0;
		return;
	}

	guest_xcrs->nr_xcrs = 1;
	guest_xcrs->flags = 0;
	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
}

static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
				       struct kvm_xcrs *guest_xcrs)
{
	int i, r = 0;

3861
	if (!boot_cpu_has(X86_FEATURE_XSAVE))
3862 3863 3864 3865 3866 3867 3868
		return -EINVAL;

	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
		return -EINVAL;

	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
		/* Only support XCR0 currently */
P
Paolo Bonzini 已提交
3869
		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
3870
			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
P
Paolo Bonzini 已提交
3871
				guest_xcrs->xcrs[i].value);
3872 3873 3874 3875 3876 3877 3878
			break;
		}
	if (r)
		r = -EINVAL;
	return r;
}

3879 3880 3881 3882 3883 3884 3885 3886
/*
 * kvm_set_guest_paused() indicates to the guest kernel that it has been
 * stopped by the hypervisor.  This function will be called from the host only.
 * EINVAL is returned when the host attempts to set the flag for a guest that
 * does not support pv clocks.
 */
static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
{
3887
	if (!vcpu->arch.pv_time_enabled)
3888
		return -EINVAL;
3889
	vcpu->arch.pvclock_set_guest_stopped_request = true;
3890 3891 3892 3893
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	return 0;
}

3894 3895 3896
static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
				     struct kvm_enable_cap *cap)
{
3897 3898 3899 3900
	int r;
	uint16_t vmcs_version;
	void __user *user_ptr;

3901 3902 3903 3904
	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
3905 3906 3907
	case KVM_CAP_HYPERV_SYNIC2:
		if (cap->args[0])
			return -EINVAL;
3908 3909
		/* fall through */

3910
	case KVM_CAP_HYPERV_SYNIC:
3911 3912
		if (!irqchip_in_kernel(vcpu->kvm))
			return -EINVAL;
3913 3914
		return kvm_hv_activate_synic(vcpu, cap->cap ==
					     KVM_CAP_HYPERV_SYNIC2);
3915
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
3916 3917
		if (!kvm_x86_ops->nested_enable_evmcs)
			return -ENOTTY;
3918 3919 3920 3921 3922 3923 3924 3925 3926
		r = kvm_x86_ops->nested_enable_evmcs(vcpu, &vmcs_version);
		if (!r) {
			user_ptr = (void __user *)(uintptr_t)cap->args[0];
			if (copy_to_user(user_ptr, &vmcs_version,
					 sizeof(vmcs_version)))
				r = -EFAULT;
		}
		return r;

3927 3928 3929 3930 3931
	default:
		return -EINVAL;
	}
}

3932 3933 3934 3935 3936 3937
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r;
3938 3939 3940 3941 3942 3943 3944
	union {
		struct kvm_lapic_state *lapic;
		struct kvm_xsave *xsave;
		struct kvm_xcrs *xcrs;
		void *buffer;
	} u;

3945 3946
	vcpu_load(vcpu);

3947
	u.buffer = NULL;
3948 3949
	switch (ioctl) {
	case KVM_GET_LAPIC: {
3950
		r = -EINVAL;
3951
		if (!lapic_in_kernel(vcpu))
3952
			goto out;
3953 3954
		u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
				GFP_KERNEL_ACCOUNT);
3955

3956
		r = -ENOMEM;
3957
		if (!u.lapic)
3958
			goto out;
3959
		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
3960 3961 3962
		if (r)
			goto out;
		r = -EFAULT;
3963
		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
3964 3965 3966 3967 3968
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
3969
		r = -EINVAL;
3970
		if (!lapic_in_kernel(vcpu))
3971
			goto out;
3972
		u.lapic = memdup_user(argp, sizeof(*u.lapic));
3973 3974 3975 3976
		if (IS_ERR(u.lapic)) {
			r = PTR_ERR(u.lapic);
			goto out_nofree;
		}
3977

3978
		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
3979 3980
		break;
	}
3981 3982 3983 3984
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
3985
		if (copy_from_user(&irq, argp, sizeof(irq)))
3986 3987 3988 3989
			goto out;
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
		break;
	}
3990 3991 3992 3993
	case KVM_NMI: {
		r = kvm_vcpu_ioctl_nmi(vcpu);
		break;
	}
3994 3995 3996 3997
	case KVM_SMI: {
		r = kvm_vcpu_ioctl_smi(vcpu);
		break;
	}
3998 3999 4000 4001 4002
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
4003
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4004 4005 4006 4007
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		break;
	}
4008 4009 4010 4011 4012
	case KVM_SET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4013
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4014 4015
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
4016
					      cpuid_arg->entries);
4017 4018 4019 4020 4021 4022 4023
		break;
	}
	case KVM_GET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4024
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4025 4026
			goto out;
		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
4027
					      cpuid_arg->entries);
4028 4029 4030
		if (r)
			goto out;
		r = -EFAULT;
4031
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4032 4033 4034 4035
			goto out;
		r = 0;
		break;
	}
4036 4037
	case KVM_GET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4038
		r = msr_io(vcpu, argp, do_get_msr, 1);
4039
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4040
		break;
4041 4042 4043
	}
	case KVM_SET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4044
		r = msr_io(vcpu, argp, do_set_msr, 0);
4045
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4046
		break;
4047
	}
4048 4049 4050 4051
	case KVM_TPR_ACCESS_REPORTING: {
		struct kvm_tpr_access_ctl tac;

		r = -EFAULT;
4052
		if (copy_from_user(&tac, argp, sizeof(tac)))
4053 4054 4055 4056 4057
			goto out;
		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
		if (r)
			goto out;
		r = -EFAULT;
4058
		if (copy_to_user(argp, &tac, sizeof(tac)))
4059 4060 4061 4062
			goto out;
		r = 0;
		break;
	};
A
Avi Kivity 已提交
4063 4064
	case KVM_SET_VAPIC_ADDR: {
		struct kvm_vapic_addr va;
4065
		int idx;
A
Avi Kivity 已提交
4066 4067

		r = -EINVAL;
4068
		if (!lapic_in_kernel(vcpu))
A
Avi Kivity 已提交
4069 4070
			goto out;
		r = -EFAULT;
4071
		if (copy_from_user(&va, argp, sizeof(va)))
A
Avi Kivity 已提交
4072
			goto out;
4073
		idx = srcu_read_lock(&vcpu->kvm->srcu);
4074
		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
4075
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
A
Avi Kivity 已提交
4076 4077
		break;
	}
H
Huang Ying 已提交
4078 4079 4080 4081
	case KVM_X86_SETUP_MCE: {
		u64 mcg_cap;

		r = -EFAULT;
4082
		if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
H
Huang Ying 已提交
4083 4084 4085 4086 4087 4088 4089 4090
			goto out;
		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
		break;
	}
	case KVM_X86_SET_MCE: {
		struct kvm_x86_mce mce;

		r = -EFAULT;
4091
		if (copy_from_user(&mce, argp, sizeof(mce)))
H
Huang Ying 已提交
4092 4093 4094 4095
			goto out;
		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
		break;
	}
J
Jan Kiszka 已提交
4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);

		r = -EFAULT;
		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		r = -EFAULT;
		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
			break;

		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
		break;
	}
4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139
	case KVM_GET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);

		r = -EFAULT;
		if (copy_to_user(argp, &dbgregs,
				 sizeof(struct kvm_debugregs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		r = -EFAULT;
		if (copy_from_user(&dbgregs, argp,
				   sizeof(struct kvm_debugregs)))
			break;

		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
		break;
	}
4140
	case KVM_GET_XSAVE: {
4141
		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
4142
		r = -ENOMEM;
4143
		if (!u.xsave)
4144 4145
			break;

4146
		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
4147 4148

		r = -EFAULT;
4149
		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
4150 4151 4152 4153 4154
			break;
		r = 0;
		break;
	}
	case KVM_SET_XSAVE: {
4155
		u.xsave = memdup_user(argp, sizeof(*u.xsave));
4156 4157 4158 4159
		if (IS_ERR(u.xsave)) {
			r = PTR_ERR(u.xsave);
			goto out_nofree;
		}
4160

4161
		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
4162 4163 4164
		break;
	}
	case KVM_GET_XCRS: {
4165
		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
4166
		r = -ENOMEM;
4167
		if (!u.xcrs)
4168 4169
			break;

4170
		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
4171 4172

		r = -EFAULT;
4173
		if (copy_to_user(argp, u.xcrs,
4174 4175 4176 4177 4178 4179
				 sizeof(struct kvm_xcrs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XCRS: {
4180
		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
4181 4182 4183 4184
		if (IS_ERR(u.xcrs)) {
			r = PTR_ERR(u.xcrs);
			goto out_nofree;
		}
4185

4186
		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
4187 4188
		break;
	}
4189 4190 4191 4192 4193 4194 4195 4196 4197
	case KVM_SET_TSC_KHZ: {
		u32 user_tsc_khz;

		r = -EINVAL;
		user_tsc_khz = (u32)arg;

		if (user_tsc_khz >= kvm_max_guest_tsc_khz)
			goto out;

4198 4199 4200
		if (user_tsc_khz == 0)
			user_tsc_khz = tsc_khz;

4201 4202
		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
			r = 0;
4203 4204 4205 4206

		goto out;
	}
	case KVM_GET_TSC_KHZ: {
4207
		r = vcpu->arch.virtual_tsc_khz;
4208 4209
		goto out;
	}
4210 4211 4212 4213
	case KVM_KVMCLOCK_CTRL: {
		r = kvm_set_guest_paused(vcpu);
		goto out;
	}
4214 4215 4216 4217 4218 4219 4220 4221 4222
	case KVM_ENABLE_CAP: {
		struct kvm_enable_cap cap;

		r = -EFAULT;
		if (copy_from_user(&cap, argp, sizeof(cap)))
			goto out;
		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
		break;
	}
4223 4224 4225 4226 4227 4228 4229 4230 4231
	case KVM_GET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		u32 user_data_size;

		r = -EINVAL;
		if (!kvm_x86_ops->get_nested_state)
			break;

		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
4232
		r = -EFAULT;
4233
		if (get_user(user_data_size, &user_kvm_nested_state->size))
4234
			break;
4235 4236 4237 4238

		r = kvm_x86_ops->get_nested_state(vcpu, user_kvm_nested_state,
						  user_data_size);
		if (r < 0)
4239
			break;
4240 4241 4242

		if (r > user_data_size) {
			if (put_user(r, &user_kvm_nested_state->size))
4243 4244 4245 4246
				r = -EFAULT;
			else
				r = -E2BIG;
			break;
4247
		}
4248

4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259
		r = 0;
		break;
	}
	case KVM_SET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		struct kvm_nested_state kvm_state;

		r = -EINVAL;
		if (!kvm_x86_ops->set_nested_state)
			break;

4260
		r = -EFAULT;
4261
		if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
4262
			break;
4263

4264
		r = -EINVAL;
4265
		if (kvm_state.size < sizeof(kvm_state))
4266
			break;
4267 4268

		if (kvm_state.flags &
4269 4270
		    ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
		      | KVM_STATE_NESTED_EVMCS))
4271
			break;
4272 4273

		/* nested_run_pending implies guest_mode.  */
4274 4275
		if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
		    && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
4276
			break;
4277 4278 4279 4280

		r = kvm_x86_ops->set_nested_state(vcpu, user_kvm_nested_state, &kvm_state);
		break;
	}
4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299
	case KVM_GET_SUPPORTED_HV_CPUID: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
			goto out;

		r = kvm_vcpu_ioctl_get_hv_cpuid(vcpu, &cpuid,
						cpuid_arg->entries);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
			goto out;
		r = 0;
		break;
	}
4300 4301 4302 4303
	default:
		r = -EINVAL;
	}
out:
4304
	kfree(u.buffer);
4305 4306
out_nofree:
	vcpu_put(vcpu);
4307 4308 4309
	return r;
}

4310
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4311 4312 4313 4314
{
	return VM_FAULT_SIGBUS;
}

4315 4316 4317 4318 4319
static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
{
	int ret;

	if (addr > (unsigned int)(-3 * PAGE_SIZE))
4320
		return -EINVAL;
4321 4322 4323 4324
	ret = kvm_x86_ops->set_tss_addr(kvm, addr);
	return ret;
}

4325 4326 4327
static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
					      u64 ident_addr)
{
4328
	return kvm_x86_ops->set_identity_map_addr(kvm, ident_addr);
4329 4330
}

4331
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
4332
					 unsigned long kvm_nr_mmu_pages)
4333 4334 4335 4336
{
	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
		return -EINVAL;

4337
	mutex_lock(&kvm->slots_lock);
4338 4339

	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
4340
	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
4341

4342
	mutex_unlock(&kvm->slots_lock);
4343 4344 4345
	return 0;
}

4346
static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
4347
{
4348
	return kvm->arch.n_max_mmu_pages;
4349 4350 4351 4352
}

static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4353
	struct kvm_pic *pic = kvm->arch.vpic;
4354 4355 4356 4357 4358
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4359
		memcpy(&chip->chip.pic, &pic->pics[0],
4360 4361 4362
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4363
		memcpy(&chip->chip.pic, &pic->pics[1],
4364 4365 4366
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_IOAPIC:
4367
		kvm_get_ioapic(kvm, &chip->chip.ioapic);
4368 4369 4370 4371 4372 4373 4374 4375 4376 4377
		break;
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4378
	struct kvm_pic *pic = kvm->arch.vpic;
4379 4380 4381 4382 4383
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4384 4385
		spin_lock(&pic->lock);
		memcpy(&pic->pics[0], &chip->chip.pic,
4386
			sizeof(struct kvm_pic_state));
4387
		spin_unlock(&pic->lock);
4388 4389
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4390 4391
		spin_lock(&pic->lock);
		memcpy(&pic->pics[1], &chip->chip.pic,
4392
			sizeof(struct kvm_pic_state));
4393
		spin_unlock(&pic->lock);
4394 4395
		break;
	case KVM_IRQCHIP_IOAPIC:
4396
		kvm_set_ioapic(kvm, &chip->chip.ioapic);
4397 4398 4399 4400 4401
		break;
	default:
		r = -EINVAL;
		break;
	}
4402
	kvm_pic_update_irq(pic);
4403 4404 4405
	return r;
}

4406 4407
static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4408 4409 4410 4411 4412 4413 4414
	struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;

	BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));

	mutex_lock(&kps->lock);
	memcpy(ps, &kps->channels, sizeof(*ps));
	mutex_unlock(&kps->lock);
4415
	return 0;
4416 4417 4418 4419
}

static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4420
	int i;
4421 4422 4423
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
4424
	memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
4425
	for (i = 0; i < 3; i++)
4426 4427
		kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
	mutex_unlock(&pit->pit_state.lock);
4428
	return 0;
B
Beth Kon 已提交
4429 4430 4431 4432 4433 4434 4435 4436 4437
}

static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
		sizeof(ps->channels));
	ps->flags = kvm->arch.vpit->pit_state.flags;
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
4438
	memset(&ps->reserved, 0, sizeof(ps->reserved));
4439
	return 0;
B
Beth Kon 已提交
4440 4441 4442 4443
}

static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
4444
	int start = 0;
4445
	int i;
B
Beth Kon 已提交
4446
	u32 prev_legacy, cur_legacy;
4447 4448 4449 4450
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
	prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
B
Beth Kon 已提交
4451 4452 4453
	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
	if (!prev_legacy && cur_legacy)
		start = 1;
4454 4455 4456
	memcpy(&pit->pit_state.channels, &ps->channels,
	       sizeof(pit->pit_state.channels));
	pit->pit_state.flags = ps->flags;
4457
	for (i = 0; i < 3; i++)
4458
		kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
4459
				   start && i == 0);
4460
	mutex_unlock(&pit->pit_state.lock);
4461
	return 0;
4462 4463
}

4464 4465 4466
static int kvm_vm_ioctl_reinject(struct kvm *kvm,
				 struct kvm_reinject_control *control)
{
4467 4468 4469
	struct kvm_pit *pit = kvm->arch.vpit;

	if (!pit)
4470
		return -ENXIO;
4471

4472 4473 4474 4475 4476 4477 4478
	/* pit->pit_state.lock was overloaded to prevent userspace from getting
	 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
	 * ioctls in parallel.  Use a separate lock if that ioctl isn't rare.
	 */
	mutex_lock(&pit->pit_state.lock);
	kvm_pit_set_reinject(pit, control->pit_reinject);
	mutex_unlock(&pit->pit_state.lock);
4479

4480 4481 4482
	return 0;
}

4483
/**
4484 4485 4486
 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
 * @kvm: kvm instance
 * @log: slot id and address to which we copy the log
4487
 *
4488 4489 4490 4491 4492 4493 4494 4495
 * Steps 1-4 below provide general overview of dirty page logging. See
 * kvm_get_dirty_log_protect() function description for additional details.
 *
 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
 * always flush the TLB (step 4) even if previous step failed  and the dirty
 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
 * does not preclude user space subsequent dirty log read. Flushing TLB ensures
 * writes will be marked dirty for next log read.
4496
 *
4497 4498
 *   1. Take a snapshot of the bit and clear it if needed.
 *   2. Write protect the corresponding page.
4499 4500
 *   3. Copy the snapshot to the userspace.
 *   4. Flush TLB's if needed.
4501
 */
4502
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
4503
{
4504
	bool flush = false;
4505
	int r;
4506

4507
	mutex_lock(&kvm->slots_lock);
4508

4509 4510 4511 4512 4513 4514
	/*
	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
	 */
	if (kvm_x86_ops->flush_log_dirty)
		kvm_x86_ops->flush_log_dirty(kvm);

4515
	r = kvm_get_dirty_log_protect(kvm, log, &flush);
4516 4517 4518 4519 4520

	/*
	 * All the TLBs can be flushed out of mmu lock, see the comments in
	 * kvm_mmu_slot_remove_write_access().
	 */
4521
	lockdep_assert_held(&kvm->slots_lock);
4522
	if (flush)
4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549
		kvm_flush_remote_tlbs(kvm);

	mutex_unlock(&kvm->slots_lock);
	return r;
}

int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log)
{
	bool flush = false;
	int r;

	mutex_lock(&kvm->slots_lock);

	/*
	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
	 */
	if (kvm_x86_ops->flush_log_dirty)
		kvm_x86_ops->flush_log_dirty(kvm);

	r = kvm_clear_dirty_log_protect(kvm, log, &flush);

	/*
	 * All the TLBs can be flushed out of mmu lock, see the comments in
	 * kvm_mmu_slot_remove_write_access().
	 */
	lockdep_assert_held(&kvm->slots_lock);
	if (flush)
4550 4551
		kvm_flush_remote_tlbs(kvm);

4552
	mutex_unlock(&kvm->slots_lock);
4553 4554 4555
	return r;
}

4556 4557
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
			bool line_status)
4558 4559 4560 4561 4562
{
	if (!irqchip_in_kernel(kvm))
		return -ENXIO;

	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
4563 4564
					irq_event->irq, irq_event->level,
					line_status);
4565 4566 4567
	return 0;
}

4568 4569
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
			    struct kvm_enable_cap *cap)
4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580
{
	int r;

	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
	case KVM_CAP_DISABLE_QUIRKS:
		kvm->arch.disabled_quirks = cap->args[0];
		r = 0;
		break;
4581 4582
	case KVM_CAP_SPLIT_IRQCHIP: {
		mutex_lock(&kvm->lock);
4583 4584 4585
		r = -EINVAL;
		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
			goto split_irqchip_unlock;
4586 4587 4588
		r = -EEXIST;
		if (irqchip_in_kernel(kvm))
			goto split_irqchip_unlock;
P
Paolo Bonzini 已提交
4589
		if (kvm->created_vcpus)
4590 4591
			goto split_irqchip_unlock;
		r = kvm_setup_empty_irq_routing(kvm);
4592
		if (r)
4593 4594 4595
			goto split_irqchip_unlock;
		/* Pairs with irqchip_in_kernel. */
		smp_wmb();
4596
		kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
4597
		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
4598 4599 4600 4601 4602
		r = 0;
split_irqchip_unlock:
		mutex_unlock(&kvm->lock);
		break;
	}
4603 4604 4605 4606 4607 4608 4609
	case KVM_CAP_X2APIC_API:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
			break;

		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
			kvm->arch.x2apic_format = true;
4610 4611
		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
			kvm->arch.x2apic_broadcast_quirk_disabled = true;
4612 4613 4614

		r = 0;
		break;
4615 4616 4617 4618 4619 4620 4621 4622
	case KVM_CAP_X86_DISABLE_EXITS:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
			break;

		if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
			kvm_can_mwait_in_guest())
			kvm->arch.mwait_in_guest = true;
M
Michael S. Tsirkin 已提交
4623
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
4624
			kvm->arch.hlt_in_guest = true;
4625 4626
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
			kvm->arch.pause_in_guest = true;
4627 4628
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
			kvm->arch.cstate_in_guest = true;
4629 4630
		r = 0;
		break;
4631 4632 4633
	case KVM_CAP_MSR_PLATFORM_INFO:
		kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
		r = 0;
4634 4635 4636 4637
		break;
	case KVM_CAP_EXCEPTION_PAYLOAD:
		kvm->arch.exception_payload_enabled = cap->args[0];
		r = 0;
4638
		break;
4639 4640 4641 4642 4643 4644 4645
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

4646 4647 4648 4649 4650
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
4651
	int r = -ENOTTY;
4652 4653 4654 4655 4656 4657 4658
	/*
	 * This union makes it completely explicit to gcc-3.x
	 * that these two variables' stack usage should be
	 * combined, not added together.
	 */
	union {
		struct kvm_pit_state ps;
B
Beth Kon 已提交
4659
		struct kvm_pit_state2 ps2;
4660
		struct kvm_pit_config pit_config;
4661
	} u;
4662 4663 4664 4665 4666

	switch (ioctl) {
	case KVM_SET_TSS_ADDR:
		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
		break;
4667 4668 4669
	case KVM_SET_IDENTITY_MAP_ADDR: {
		u64 ident_addr;

4670 4671 4672 4673
		mutex_lock(&kvm->lock);
		r = -EINVAL;
		if (kvm->created_vcpus)
			goto set_identity_unlock;
4674
		r = -EFAULT;
4675
		if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
4676
			goto set_identity_unlock;
4677
		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
4678 4679
set_identity_unlock:
		mutex_unlock(&kvm->lock);
4680 4681
		break;
	}
4682 4683 4684 4685 4686 4687
	case KVM_SET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
		break;
	case KVM_GET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
		break;
4688 4689
	case KVM_CREATE_IRQCHIP: {
		mutex_lock(&kvm->lock);
4690

4691
		r = -EEXIST;
4692
		if (irqchip_in_kernel(kvm))
4693
			goto create_irqchip_unlock;
4694

4695
		r = -EINVAL;
P
Paolo Bonzini 已提交
4696
		if (kvm->created_vcpus)
4697
			goto create_irqchip_unlock;
4698 4699 4700

		r = kvm_pic_init(kvm);
		if (r)
4701
			goto create_irqchip_unlock;
4702 4703 4704 4705

		r = kvm_ioapic_init(kvm);
		if (r) {
			kvm_pic_destroy(kvm);
4706
			goto create_irqchip_unlock;
4707 4708
		}

4709 4710
		r = kvm_setup_default_irq_routing(kvm);
		if (r) {
4711
			kvm_ioapic_destroy(kvm);
4712
			kvm_pic_destroy(kvm);
4713
			goto create_irqchip_unlock;
4714
		}
4715
		/* Write kvm->irq_routing before enabling irqchip_in_kernel. */
4716
		smp_wmb();
4717
		kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
4718 4719
	create_irqchip_unlock:
		mutex_unlock(&kvm->lock);
4720
		break;
4721
	}
S
Sheng Yang 已提交
4722
	case KVM_CREATE_PIT:
4723 4724 4725 4726 4727 4728 4729 4730
		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
		goto create_pit;
	case KVM_CREATE_PIT2:
		r = -EFAULT;
		if (copy_from_user(&u.pit_config, argp,
				   sizeof(struct kvm_pit_config)))
			goto out;
	create_pit:
4731
		mutex_lock(&kvm->lock);
A
Avi Kivity 已提交
4732 4733 4734
		r = -EEXIST;
		if (kvm->arch.vpit)
			goto create_pit_unlock;
S
Sheng Yang 已提交
4735
		r = -ENOMEM;
4736
		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
S
Sheng Yang 已提交
4737 4738
		if (kvm->arch.vpit)
			r = 0;
A
Avi Kivity 已提交
4739
	create_pit_unlock:
4740
		mutex_unlock(&kvm->lock);
S
Sheng Yang 已提交
4741
		break;
4742 4743
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
4744
		struct kvm_irqchip *chip;
4745

4746 4747 4748
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
4749
			goto out;
4750 4751
		}

4752
		r = -ENXIO;
4753
		if (!irqchip_kernel(kvm))
4754 4755
			goto get_irqchip_out;
		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
4756
		if (r)
4757
			goto get_irqchip_out;
4758
		r = -EFAULT;
4759
		if (copy_to_user(argp, chip, sizeof(*chip)))
4760
			goto get_irqchip_out;
4761
		r = 0;
4762 4763
	get_irqchip_out:
		kfree(chip);
4764 4765 4766 4767
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
4768
		struct kvm_irqchip *chip;
4769

4770 4771 4772
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
4773
			goto out;
4774 4775
		}

4776
		r = -ENXIO;
4777
		if (!irqchip_kernel(kvm))
4778 4779
			goto set_irqchip_out;
		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
4780
		if (r)
4781
			goto set_irqchip_out;
4782
		r = 0;
4783 4784
	set_irqchip_out:
		kfree(chip);
4785 4786
		break;
	}
4787 4788
	case KVM_GET_PIT: {
		r = -EFAULT;
4789
		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
4790 4791 4792 4793
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
4794
		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
4795 4796 4797
		if (r)
			goto out;
		r = -EFAULT;
4798
		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
4799 4800 4801 4802 4803 4804
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT: {
		r = -EFAULT;
4805
		if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
4806 4807 4808 4809
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
4810
		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
4811 4812
		break;
	}
B
Beth Kon 已提交
4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835
	case KVM_GET_PIT2: {
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT2: {
		r = -EFAULT;
		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
		break;
	}
4836 4837 4838 4839 4840 4841 4842 4843
	case KVM_REINJECT_CONTROL: {
		struct kvm_reinject_control control;
		r =  -EFAULT;
		if (copy_from_user(&control, argp, sizeof(control)))
			goto out;
		r = kvm_vm_ioctl_reinject(kvm, &control);
		break;
	}
4844 4845 4846
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
		mutex_lock(&kvm->lock);
P
Paolo Bonzini 已提交
4847
		if (kvm->created_vcpus)
4848 4849 4850 4851 4852
			r = -EBUSY;
		else
			kvm->arch.bsp_vcpu_id = arg;
		mutex_unlock(&kvm->lock);
		break;
E
Ed Swierk 已提交
4853
	case KVM_XEN_HVM_CONFIG: {
4854
		struct kvm_xen_hvm_config xhc;
E
Ed Swierk 已提交
4855
		r = -EFAULT;
4856
		if (copy_from_user(&xhc, argp, sizeof(xhc)))
E
Ed Swierk 已提交
4857 4858
			goto out;
		r = -EINVAL;
4859
		if (xhc.flags)
E
Ed Swierk 已提交
4860
			goto out;
4861
		memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));
E
Ed Swierk 已提交
4862 4863 4864
		r = 0;
		break;
	}
4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877
	case KVM_SET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

		r = -EFAULT;
		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
			goto out;

		r = -EINVAL;
		if (user_ns.flags)
			goto out;

		r = 0;
4878 4879 4880 4881 4882 4883
		/*
		 * TODO: userspace has to take care of races with VCPU_RUN, so
		 * kvm_gen_update_masterclock() can be cut down to locked
		 * pvclock_update_vm_gtod_copy().
		 */
		kvm_gen_update_masterclock(kvm);
4884
		now_ns = get_kvmclock_ns(kvm);
4885
		kvm->arch.kvmclock_offset += user_ns.clock - now_ns;
4886
		kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE);
4887 4888 4889 4890 4891 4892
		break;
	}
	case KVM_GET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

4893
		now_ns = get_kvmclock_ns(kvm);
4894
		user_ns.clock = now_ns;
4895
		user_ns.flags = kvm->arch.use_master_clock ? KVM_CLOCK_TSC_STABLE : 0;
4896
		memset(&user_ns.pad, 0, sizeof(user_ns.pad));
4897 4898 4899 4900 4901 4902 4903

		r = -EFAULT;
		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
			goto out;
		r = 0;
		break;
	}
4904 4905 4906 4907 4908 4909
	case KVM_MEMORY_ENCRYPT_OP: {
		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_op)
			r = kvm_x86_ops->mem_enc_op(kvm, argp);
		break;
	}
4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933
	case KVM_MEMORY_ENCRYPT_REG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_reg_region)
			r = kvm_x86_ops->mem_enc_reg_region(kvm, &region);
		break;
	}
	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_unreg_region)
			r = kvm_x86_ops->mem_enc_unreg_region(kvm, &region);
		break;
	}
4934 4935 4936 4937 4938 4939 4940 4941 4942
	case KVM_HYPERV_EVENTFD: {
		struct kvm_hyperv_eventfd hvevfd;

		r = -EFAULT;
		if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
			goto out;
		r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
		break;
	}
4943
	default:
4944
		r = -ENOTTY;
4945 4946 4947 4948 4949
	}
out:
	return r;
}

4950
static void kvm_init_msr_list(void)
4951 4952 4953 4954
{
	u32 dummy[2];
	unsigned i, j;

4955
	for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
4956 4957
		if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
			continue;
4958 4959 4960

		/*
		 * Even MSRs that are valid in the host may not be exposed
4961
		 * to the guests in some cases.
4962 4963 4964
		 */
		switch (msrs_to_save[i]) {
		case MSR_IA32_BNDCFGS:
4965
			if (!kvm_mpx_supported())
4966 4967
				continue;
			break;
4968 4969 4970 4971
		case MSR_TSC_AUX:
			if (!kvm_x86_ops->rdtscp_supported())
				continue;
			break;
4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995
		case MSR_IA32_RTIT_CTL:
		case MSR_IA32_RTIT_STATUS:
			if (!kvm_x86_ops->pt_supported())
				continue;
			break;
		case MSR_IA32_RTIT_CR3_MATCH:
			if (!kvm_x86_ops->pt_supported() ||
			    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
				continue;
			break;
		case MSR_IA32_RTIT_OUTPUT_BASE:
		case MSR_IA32_RTIT_OUTPUT_MASK:
			if (!kvm_x86_ops->pt_supported() ||
				(!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
				 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
				continue;
			break;
		case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: {
			if (!kvm_x86_ops->pt_supported() ||
				msrs_to_save[i] - MSR_IA32_RTIT_ADDR0_A >=
				intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
				continue;
			break;
		}
4996 4997 4998 4999
		default:
			break;
		}

5000 5001 5002 5003 5004
		if (j < i)
			msrs_to_save[j] = msrs_to_save[i];
		j++;
	}
	num_msrs_to_save = j;
5005 5006

	for (i = j = 0; i < ARRAY_SIZE(emulated_msrs); i++) {
5007 5008
		if (!kvm_x86_ops->has_emulated_msr(emulated_msrs[i]))
			continue;
5009 5010 5011 5012 5013 5014

		if (j < i)
			emulated_msrs[j] = emulated_msrs[i];
		j++;
	}
	num_emulated_msrs = j;
5015 5016 5017 5018 5019

	for (i = j = 0; i < ARRAY_SIZE(msr_based_features); i++) {
		struct kvm_msr_entry msr;

		msr.index = msr_based_features[i];
5020
		if (kvm_get_msr_feature(&msr))
5021 5022 5023 5024 5025 5026 5027
			continue;

		if (j < i)
			msr_based_features[j] = msr_based_features[i];
		j++;
	}
	num_msr_based_features = j;
5028 5029
}

5030 5031
static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
			   const void *v)
5032
{
5033 5034 5035 5036 5037
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5038
		if (!(lapic_in_kernel(vcpu) &&
5039 5040
		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
5041 5042 5043 5044 5045 5046
			break;
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5047

5048
	return handled;
5049 5050
}

5051
static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
5052
{
5053 5054 5055 5056 5057
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5058
		if (!(lapic_in_kernel(vcpu) &&
5059 5060 5061
		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
					 addr, n, v))
		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
5062
			break;
5063
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
5064 5065 5066 5067 5068
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5069

5070
	return handled;
5071 5072
}

5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084
static void kvm_set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
	kvm_x86_ops->set_segment(vcpu, var, seg);
}

void kvm_get_segment(struct kvm_vcpu *vcpu,
		     struct kvm_segment *var, int seg)
{
	kvm_x86_ops->get_segment(vcpu, var, seg);
}

5085 5086
gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
			   struct x86_exception *exception)
5087 5088 5089 5090 5091 5092 5093
{
	gpa_t t_gpa;

	BUG_ON(!mmu_is_nested(vcpu));

	/* NPT walks are always user-walks */
	access |= PFERR_USER_MASK;
5094
	t_gpa  = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception);
5095 5096 5097 5098

	return t_gpa;
}

5099 5100
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
			      struct x86_exception *exception)
5101 5102
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5103
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5104 5105
}

5106 5107
 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5108 5109 5110
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_FETCH_MASK;
5111
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5112 5113
}

5114 5115
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
			       struct x86_exception *exception)
5116 5117 5118
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_WRITE_MASK;
5119
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5120 5121 5122
}

/* uses this to access any guest's mapped memory without checking CPL */
5123 5124
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5125
{
5126
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
5127 5128 5129 5130
}

static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
5131
				      struct x86_exception *exception)
5132 5133
{
	void *data = val;
5134
	int r = X86EMUL_CONTINUE;
5135 5136

	while (bytes) {
5137
		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
5138
							    exception);
5139
		unsigned offset = addr & (PAGE_SIZE-1);
5140
		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
5141 5142
		int ret;

5143
		if (gpa == UNMAPPED_GVA)
5144
			return X86EMUL_PROPAGATE_FAULT;
5145 5146
		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
					       offset, toread);
5147
		if (ret < 0) {
5148
			r = X86EMUL_IO_NEEDED;
5149 5150
			goto out;
		}
5151

5152 5153 5154
		bytes -= toread;
		data += toread;
		addr += toread;
5155
	}
5156 5157
out:
	return r;
5158
}
5159

5160
/* used for instruction fetching */
5161 5162
static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
				gva_t addr, void *val, unsigned int bytes,
5163
				struct x86_exception *exception)
5164
{
5165
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5166
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5167 5168
	unsigned offset;
	int ret;
5169

5170 5171 5172 5173 5174 5175 5176 5177 5178
	/* Inline kvm_read_guest_virt_helper for speed.  */
	gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,
						    exception);
	if (unlikely(gpa == UNMAPPED_GVA))
		return X86EMUL_PROPAGATE_FAULT;

	offset = addr & (PAGE_SIZE-1);
	if (WARN_ON(offset + bytes > PAGE_SIZE))
		bytes = (unsigned)PAGE_SIZE - offset;
5179 5180
	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
				       offset, bytes);
5181 5182 5183 5184
	if (unlikely(ret < 0))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
5185 5186
}

5187
int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
5188
			       gva_t addr, void *val, unsigned int bytes,
5189
			       struct x86_exception *exception)
5190 5191
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5192

5193 5194 5195 5196 5197 5198 5199
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5200
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
5201
					  exception);
5202
}
5203
EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
5204

5205 5206
static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
			     gva_t addr, void *val, unsigned int bytes,
5207
			     struct x86_exception *exception, bool system)
5208
{
5209
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5210 5211 5212 5213 5214 5215
	u32 access = 0;

	if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
		access |= PFERR_USER_MASK;

	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
5216 5217
}

5218 5219 5220 5221 5222 5223 5224 5225 5226
static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
		unsigned long addr, void *val, unsigned int bytes)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);

	return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
}

5227 5228 5229
static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
				      struct x86_exception *exception)
5230 5231 5232 5233 5234
{
	void *data = val;
	int r = X86EMUL_CONTINUE;

	while (bytes) {
5235
		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
5236
							     access,
5237
							     exception);
5238 5239 5240 5241
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
		int ret;

5242
		if (gpa == UNMAPPED_GVA)
5243
			return X86EMUL_PROPAGATE_FAULT;
5244
		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
5245
		if (ret < 0) {
5246
			r = X86EMUL_IO_NEEDED;
5247 5248 5249 5250 5251 5252 5253 5254 5255 5256
			goto out;
		}

		bytes -= towrite;
		data += towrite;
		addr += towrite;
	}
out:
	return r;
}
5257 5258

static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
5259 5260
			      unsigned int bytes, struct x86_exception *exception,
			      bool system)
5261 5262
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5263 5264 5265 5266
	u32 access = PFERR_WRITE_MASK;

	if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
		access |= PFERR_USER_MASK;
5267 5268

	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
5269
					   access, exception);
5270 5271 5272 5273 5274
}

int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
				unsigned int bytes, struct x86_exception *exception)
{
P
Paolo Bonzini 已提交
5275 5276 5277
	/* kvm_write_guest_virt_system can pull in tons of pages. */
	vcpu->arch.l1tf_flush_l1d = true;

5278 5279 5280
	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
					   PFERR_WRITE_MASK, exception);
}
N
Nadav Har'El 已提交
5281
EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
5282

W
Wanpeng Li 已提交
5283 5284
int handle_ud(struct kvm_vcpu *vcpu)
{
5285
	int emul_type = EMULTYPE_TRAP_UD;
W
Wanpeng Li 已提交
5286
	enum emulation_result er;
5287 5288 5289 5290
	char sig[5]; /* ud2; .ascii "kvm" */
	struct x86_exception e;

	if (force_emulation_prefix &&
5291 5292
	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
				sig, sizeof(sig), &e) == 0 &&
5293 5294 5295 5296
	    memcmp(sig, "\xf\xbkvm", sizeof(sig)) == 0) {
		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
		emul_type = 0;
	}
W
Wanpeng Li 已提交
5297

5298
	er = kvm_emulate_instruction(vcpu, emul_type);
W
Wanpeng Li 已提交
5299 5300 5301 5302 5303 5304 5305 5306
	if (er == EMULATE_USER_EXIT)
		return 0;
	if (er != EMULATE_DONE)
		kvm_queue_exception(vcpu, UD_VECTOR);
	return 1;
}
EXPORT_SYMBOL_GPL(handle_ud);

5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321
static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
			    gpa_t gpa, bool write)
{
	/* For APIC access vmexit */
	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		return 1;

	if (vcpu_match_mmio_gpa(vcpu, gpa)) {
		trace_vcpu_match_mmio(gva, gpa, write, true);
		return 1;
	}

	return 0;
}

5322 5323 5324 5325
static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
				gpa_t *gpa, struct x86_exception *exception,
				bool write)
{
5326 5327
	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
		| (write ? PFERR_WRITE_MASK : 0);
5328

5329 5330 5331 5332 5333
	/*
	 * currently PKRU is only applied to ept enabled guest so
	 * there is no pkey in EPT page table for L1 guest or EPT
	 * shadow page table for L2 guest.
	 */
5334
	if (vcpu_match_mmio_gva(vcpu, gva)
F
Feng Wu 已提交
5335
	    && !permission_fault(vcpu, vcpu->arch.walk_mmu,
5336
				 vcpu->arch.access, 0, access)) {
5337 5338
		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
					(gva & (PAGE_SIZE - 1));
X
Xiao Guangrong 已提交
5339
		trace_vcpu_match_mmio(gva, *gpa, write, false);
5340 5341 5342
		return 1;
	}

5343 5344 5345 5346 5347
	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);

	if (*gpa == UNMAPPED_GVA)
		return -1;

5348
	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
5349 5350
}

5351
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
5352
			const void *val, int bytes)
5353 5354 5355
{
	int ret;

5356
	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
5357
	if (ret < 0)
5358
		return 0;
5359
	kvm_page_track_write(vcpu, gpa, val, bytes);
5360 5361 5362
	return 1;
}

5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378
struct read_write_emulator_ops {
	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
				  int bytes);
	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
				  void *val, int bytes);
	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
			       int bytes, void *val);
	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
				    void *val, int bytes);
	bool write;
};

static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
{
	if (vcpu->mmio_read_completed) {
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
5379
			       vcpu->mmio_fragments[0].gpa, val);
5380 5381 5382 5383 5384 5385 5386 5387 5388 5389
		vcpu->mmio_read_completed = 0;
		return 1;
	}

	return 0;
}

static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			void *val, int bytes)
{
5390
	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
5391 5392 5393 5394 5395 5396 5397 5398 5399 5400
}

static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			 void *val, int bytes)
{
	return emulator_write_phys(vcpu, gpa, val, bytes);
}

static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
{
5401
	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
5402 5403 5404 5405 5406 5407
	return vcpu_mmio_write(vcpu, gpa, bytes, val);
}

static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			  void *val, int bytes)
{
5408
	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
5409 5410 5411 5412 5413 5414
	return X86EMUL_IO_NEEDED;
}

static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			   void *val, int bytes)
{
A
Avi Kivity 已提交
5415 5416
	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];

5417
	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
5418 5419 5420
	return X86EMUL_CONTINUE;
}

5421
static const struct read_write_emulator_ops read_emultor = {
5422 5423 5424 5425 5426 5427
	.read_write_prepare = read_prepare,
	.read_write_emulate = read_emulate,
	.read_write_mmio = vcpu_mmio_read,
	.read_write_exit_mmio = read_exit_mmio,
};

5428
static const struct read_write_emulator_ops write_emultor = {
5429 5430 5431 5432 5433 5434
	.read_write_emulate = write_emulate,
	.read_write_mmio = write_mmio,
	.read_write_exit_mmio = write_exit_mmio,
	.write = true,
};

5435 5436 5437 5438
static int emulator_read_write_onepage(unsigned long addr, void *val,
				       unsigned int bytes,
				       struct x86_exception *exception,
				       struct kvm_vcpu *vcpu,
5439
				       const struct read_write_emulator_ops *ops)
5440
{
5441 5442
	gpa_t gpa;
	int handled, ret;
5443
	bool write = ops->write;
A
Avi Kivity 已提交
5444
	struct kvm_mmio_fragment *frag;
5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;

	/*
	 * If the exit was due to a NPF we may already have a GPA.
	 * If the GPA is present, use it to avoid the GVA to GPA table walk.
	 * Note, this cannot be used on string operations since string
	 * operation using rep will only have the initial GPA from the NPF
	 * occurred.
	 */
	if (vcpu->arch.gpa_available &&
	    emulator_can_use_gpa(ctxt) &&
5456 5457 5458 5459 5460 5461 5462
	    (addr & ~PAGE_MASK) == (vcpu->arch.gpa_val & ~PAGE_MASK)) {
		gpa = vcpu->arch.gpa_val;
		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
	} else {
		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
		if (ret < 0)
			return X86EMUL_PROPAGATE_FAULT;
5463
	}
5464

5465
	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
5466 5467 5468 5469 5470
		return X86EMUL_CONTINUE;

	/*
	 * Is this MMIO handled locally?
	 */
5471
	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
5472
	if (handled == bytes)
5473 5474
		return X86EMUL_CONTINUE;

5475 5476 5477 5478
	gpa += handled;
	bytes -= handled;
	val += handled;

5479 5480 5481 5482 5483
	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
	frag->gpa = gpa;
	frag->data = val;
	frag->len = bytes;
A
Avi Kivity 已提交
5484
	return X86EMUL_CONTINUE;
5485 5486
}

5487 5488
static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
			unsigned long addr,
5489 5490
			void *val, unsigned int bytes,
			struct x86_exception *exception,
5491
			const struct read_write_emulator_ops *ops)
5492
{
5493
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
A
Avi Kivity 已提交
5494 5495 5496 5497 5498 5499 5500 5501
	gpa_t gpa;
	int rc;

	if (ops->read_write_prepare &&
		  ops->read_write_prepare(vcpu, val, bytes))
		return X86EMUL_CONTINUE;

	vcpu->mmio_nr_fragments = 0;
5502

5503 5504
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
A
Avi Kivity 已提交
5505
		int now;
5506 5507

		now = -addr & ~PAGE_MASK;
5508 5509 5510
		rc = emulator_read_write_onepage(addr, val, now, exception,
						 vcpu, ops);

5511 5512 5513
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
5514 5515
		if (ctxt->mode != X86EMUL_MODE_PROT64)
			addr = (u32)addr;
5516 5517 5518
		val += now;
		bytes -= now;
	}
5519

A
Avi Kivity 已提交
5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532
	rc = emulator_read_write_onepage(addr, val, bytes, exception,
					 vcpu, ops);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	if (!vcpu->mmio_nr_fragments)
		return rc;

	gpa = vcpu->mmio_fragments[0].gpa;

	vcpu->mmio_needed = 1;
	vcpu->mmio_cur_fragment = 0;

5533
	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
A
Avi Kivity 已提交
5534 5535 5536 5537 5538
	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
	vcpu->run->exit_reason = KVM_EXIT_MMIO;
	vcpu->run->mmio.phys_addr = gpa;

	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550
}

static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
				  unsigned long addr,
				  void *val,
				  unsigned int bytes,
				  struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, val, bytes,
				   exception, &read_emultor);
}

5551
static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
5552 5553 5554 5555 5556 5557 5558
			    unsigned long addr,
			    const void *val,
			    unsigned int bytes,
			    struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, (void *)val, bytes,
				   exception, &write_emultor);
5559 5560
}

5561 5562 5563 5564 5565 5566 5567
#define CMPXCHG_TYPE(t, ptr, old, new) \
	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))

#ifdef CONFIG_X86_64
#  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
#else
#  define CMPXCHG64(ptr, old, new) \
5568
	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
5569 5570
#endif

5571 5572
static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
				     unsigned long addr,
5573 5574 5575
				     const void *old,
				     const void *new,
				     unsigned int bytes,
5576
				     struct x86_exception *exception)
5577
{
5578
	struct kvm_host_map map;
5579
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5580 5581 5582
	gpa_t gpa;
	char *kaddr;
	bool exchanged;
5583

5584 5585 5586
	/* guests cmpxchg8b have to be emulated atomically */
	if (bytes > 8 || (bytes & (bytes - 1)))
		goto emul_write;
5587

5588
	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
5589

5590 5591 5592
	if (gpa == UNMAPPED_GVA ||
	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		goto emul_write;
5593

5594 5595
	if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
		goto emul_write;
5596

5597
	if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))
5598
		goto emul_write;
5599

5600 5601
	kaddr = map.hva + offset_in_page(gpa);

5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616
	switch (bytes) {
	case 1:
		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
		break;
	case 2:
		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
		break;
	case 4:
		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
		break;
	case 8:
		exchanged = CMPXCHG64(kaddr, old, new);
		break;
	default:
		BUG();
5617
	}
5618 5619

	kvm_vcpu_unmap(vcpu, &map, true);
5620 5621 5622 5623

	if (!exchanged)
		return X86EMUL_CMPXCHG_FAILED;

5624
	kvm_page_track_write(vcpu, gpa, new, bytes);
5625 5626

	return X86EMUL_CONTINUE;
5627

5628
emul_write:
5629
	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
5630

5631
	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
5632 5633
}

5634 5635
static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
{
5636
	int r = 0, i;
5637

5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649
	for (i = 0; i < vcpu->arch.pio.count; i++) {
		if (vcpu->arch.pio.in)
			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
					    vcpu->arch.pio.size, pd);
		else
			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
					     vcpu->arch.pio.port, vcpu->arch.pio.size,
					     pd);
		if (r)
			break;
		pd += vcpu->arch.pio.size;
	}
5650 5651 5652
	return r;
}

5653 5654 5655
static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
			       unsigned short port, void *val,
			       unsigned int count, bool in)
5656 5657
{
	vcpu->arch.pio.port = port;
5658
	vcpu->arch.pio.in = in;
5659
	vcpu->arch.pio.count  = count;
5660 5661 5662
	vcpu->arch.pio.size = size;

	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
5663
		vcpu->arch.pio.count = 0;
5664 5665 5666 5667
		return 1;
	}

	vcpu->run->exit_reason = KVM_EXIT_IO;
5668
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
5669 5670 5671 5672 5673 5674 5675 5676
	vcpu->run->io.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = count;
	vcpu->run->io.port = port;

	return 0;
}

5677 5678 5679
static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
				    int size, unsigned short port, void *val,
				    unsigned int count)
5680
{
5681
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5682
	int ret;
5683

5684 5685
	if (vcpu->arch.pio.count)
		goto data_avail;
5686

5687 5688
	memset(vcpu->arch.pio_data, 0, size * count);

5689 5690 5691 5692
	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
	if (ret) {
data_avail:
		memcpy(val, vcpu->arch.pio_data, size * count);
5693
		trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
5694
		vcpu->arch.pio.count = 0;
5695 5696 5697 5698 5699 5700
		return 1;
	}

	return 0;
}

5701 5702 5703 5704 5705 5706 5707
static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
				     int size, unsigned short port,
				     const void *val, unsigned int count)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	memcpy(vcpu->arch.pio_data, val, size * count);
5708
	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
5709 5710 5711
	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
}

5712 5713 5714 5715 5716
static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	return kvm_x86_ops->get_segment_base(vcpu, seg);
}

5717
static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
5718
{
5719
	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
5720 5721
}

5722
static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
5723 5724 5725 5726 5727
{
	if (!need_emulate_wbinvd(vcpu))
		return X86EMUL_CONTINUE;

	if (kvm_x86_ops->has_wbinvd_exit()) {
5728 5729 5730
		int cpu = get_cpu();

		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
5731 5732
		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
				wbinvd_ipi, NULL, 1);
5733
		put_cpu();
5734
		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
5735 5736
	} else
		wbinvd();
5737 5738
	return X86EMUL_CONTINUE;
}
5739 5740 5741

int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
5742 5743
	kvm_emulate_wbinvd_noskip(vcpu);
	return kvm_skip_emulated_instruction(vcpu);
5744
}
5745 5746
EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);

5747 5748


5749 5750
static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
{
5751
	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
5752 5753
}

5754 5755
static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long *dest)
5756
{
5757
	return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
5758 5759
}

5760 5761
static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long value)
5762
{
5763

5764
	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
5765 5766
}

5767
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
5768
{
5769
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
5770 5771
}

5772
static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
5773
{
5774
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5775 5776 5777 5778 5779 5780 5781 5782 5783 5784
	unsigned long value;

	switch (cr) {
	case 0:
		value = kvm_read_cr0(vcpu);
		break;
	case 2:
		value = vcpu->arch.cr2;
		break;
	case 3:
5785
		value = kvm_read_cr3(vcpu);
5786 5787 5788 5789 5790 5791 5792 5793
		break;
	case 4:
		value = kvm_read_cr4(vcpu);
		break;
	case 8:
		value = kvm_get_cr8(vcpu);
		break;
	default:
5794
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
5795 5796 5797 5798 5799 5800
		return 0;
	}

	return value;
}

5801
static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
5802
{
5803
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5804 5805
	int res = 0;

5806 5807
	switch (cr) {
	case 0:
5808
		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
5809 5810 5811 5812 5813
		break;
	case 2:
		vcpu->arch.cr2 = val;
		break;
	case 3:
5814
		res = kvm_set_cr3(vcpu, val);
5815 5816
		break;
	case 4:
5817
		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
5818 5819
		break;
	case 8:
A
Andre Przywara 已提交
5820
		res = kvm_set_cr8(vcpu, val);
5821 5822
		break;
	default:
5823
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
5824
		res = -1;
5825
	}
5826 5827

	return res;
5828 5829
}

5830
static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
5831
{
5832
	return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
5833 5834
}

5835
static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
5836
{
5837
	kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
5838 5839
}

5840
static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
5841
{
5842
	kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
5843 5844
}

5845 5846 5847 5848 5849 5850 5851 5852 5853 5854
static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
}

static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
}

5855 5856
static unsigned long emulator_get_cached_segment_base(
	struct x86_emulate_ctxt *ctxt, int seg)
5857
{
5858
	return get_segment_base(emul_to_vcpu(ctxt), seg);
5859 5860
}

5861 5862 5863
static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
				 struct desc_struct *desc, u32 *base3,
				 int seg)
5864 5865 5866
{
	struct kvm_segment var;

5867
	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
5868
	*selector = var.selector;
5869

5870 5871
	if (var.unusable) {
		memset(desc, 0, sizeof(*desc));
5872 5873
		if (base3)
			*base3 = 0;
5874
		return false;
5875
	}
5876 5877 5878 5879 5880

	if (var.g)
		var.limit >>= 12;
	set_desc_limit(desc, var.limit);
	set_desc_base(desc, (unsigned long)var.base);
5881 5882 5883 5884
#ifdef CONFIG_X86_64
	if (base3)
		*base3 = var.base >> 32;
#endif
5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896
	desc->type = var.type;
	desc->s = var.s;
	desc->dpl = var.dpl;
	desc->p = var.present;
	desc->avl = var.avl;
	desc->l = var.l;
	desc->d = var.db;
	desc->g = var.g;

	return true;
}

5897 5898 5899
static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
				 struct desc_struct *desc, u32 base3,
				 int seg)
5900
{
5901
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5902 5903
	struct kvm_segment var;

5904
	var.selector = selector;
5905
	var.base = get_desc_base(desc);
5906 5907 5908
#ifdef CONFIG_X86_64
	var.base |= ((u64)base3) << 32;
#endif
5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926
	var.limit = get_desc_limit(desc);
	if (desc->g)
		var.limit = (var.limit << 12) | 0xfff;
	var.type = desc->type;
	var.dpl = desc->dpl;
	var.db = desc->d;
	var.s = desc->s;
	var.l = desc->l;
	var.g = desc->g;
	var.avl = desc->avl;
	var.present = desc->p;
	var.unusable = !var.present;
	var.padding = 0;

	kvm_set_segment(vcpu, &var, seg);
	return;
}

5927 5928 5929
static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 *pdata)
{
5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940
	struct msr_data msr;
	int r;

	msr.index = msr_index;
	msr.host_initiated = false;
	r = kvm_get_msr(emul_to_vcpu(ctxt), &msr);
	if (r)
		return r;

	*pdata = msr.data;
	return 0;
5941 5942 5943 5944 5945
}

static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 data)
{
5946 5947 5948 5949 5950 5951
	struct msr_data msr;

	msr.data = data;
	msr.index = msr_index;
	msr.host_initiated = false;
	return kvm_set_msr(emul_to_vcpu(ctxt), &msr);
5952 5953
}

P
Paolo Bonzini 已提交
5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967
static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	return vcpu->arch.smbase;
}

static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	vcpu->arch.smbase = smbase;
}

5968 5969 5970
static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
			      u32 pmc)
{
5971
	return kvm_pmu_is_valid_msr_idx(emul_to_vcpu(ctxt), pmc);
5972 5973
}

5974 5975 5976
static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
			     u32 pmc, u64 *pdata)
{
5977
	return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
5978 5979
}

5980 5981 5982 5983 5984
static void emulator_halt(struct x86_emulate_ctxt *ctxt)
{
	emul_to_vcpu(ctxt)->arch.halt_request = 1;
}

5985
static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
5986
			      struct x86_instruction_info *info,
5987 5988
			      enum x86_intercept_stage stage)
{
5989
	return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
5990 5991
}

5992 5993
static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
			u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, bool check_limit)
5994
{
5995
	return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, check_limit);
5996 5997
}

5998 5999 6000 6001 6002 6003 6004 6005 6006 6007
static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
{
	return kvm_register_read(emul_to_vcpu(ctxt), reg);
}

static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
{
	kvm_register_write(emul_to_vcpu(ctxt), reg, val);
}

6008 6009 6010 6011 6012
static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
{
	kvm_x86_ops->set_nmi_mask(emul_to_vcpu(ctxt), masked);
}

6013 6014 6015 6016 6017 6018 6019
static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
{
	return emul_to_vcpu(ctxt)->arch.hflags;
}

static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_flags)
{
6020
	emul_to_vcpu(ctxt)->arch.hflags = emul_flags;
6021 6022
}

6023 6024
static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt,
				  const char *smstate)
6025
{
6026
	return kvm_x86_ops->pre_leave_smm(emul_to_vcpu(ctxt), smstate);
6027 6028
}

6029 6030 6031 6032 6033
static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt)
{
	kvm_smm_changed(emul_to_vcpu(ctxt));
}

6034
static const struct x86_emulate_ops emulate_ops = {
6035 6036
	.read_gpr            = emulator_read_gpr,
	.write_gpr           = emulator_write_gpr,
6037 6038
	.read_std            = emulator_read_std,
	.write_std           = emulator_write_std,
6039
	.read_phys           = kvm_read_guest_phys_system,
6040
	.fetch               = kvm_fetch_guest_virt,
6041 6042 6043
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
6044
	.invlpg              = emulator_invlpg,
6045 6046
	.pio_in_emulated     = emulator_pio_in_emulated,
	.pio_out_emulated    = emulator_pio_out_emulated,
6047 6048
	.get_segment         = emulator_get_segment,
	.set_segment         = emulator_set_segment,
6049
	.get_cached_segment_base = emulator_get_cached_segment_base,
6050
	.get_gdt             = emulator_get_gdt,
6051
	.get_idt	     = emulator_get_idt,
6052 6053
	.set_gdt             = emulator_set_gdt,
	.set_idt	     = emulator_set_idt,
6054 6055
	.get_cr              = emulator_get_cr,
	.set_cr              = emulator_set_cr,
6056
	.cpl                 = emulator_get_cpl,
6057 6058
	.get_dr              = emulator_get_dr,
	.set_dr              = emulator_set_dr,
P
Paolo Bonzini 已提交
6059 6060
	.get_smbase          = emulator_get_smbase,
	.set_smbase          = emulator_set_smbase,
6061 6062
	.set_msr             = emulator_set_msr,
	.get_msr             = emulator_get_msr,
6063
	.check_pmc	     = emulator_check_pmc,
6064
	.read_pmc            = emulator_read_pmc,
6065
	.halt                = emulator_halt,
6066
	.wbinvd              = emulator_wbinvd,
6067
	.fix_hypercall       = emulator_fix_hypercall,
6068
	.intercept           = emulator_intercept,
6069
	.get_cpuid           = emulator_get_cpuid,
6070
	.set_nmi_mask        = emulator_set_nmi_mask,
6071 6072
	.get_hflags          = emulator_get_hflags,
	.set_hflags          = emulator_set_hflags,
6073
	.pre_leave_smm       = emulator_pre_leave_smm,
6074
	.post_leave_smm      = emulator_post_leave_smm,
6075 6076
};

6077 6078
static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
{
6079
	u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
6080 6081 6082 6083 6084 6085 6086
	/*
	 * an sti; sti; sequence only disable interrupts for the first
	 * instruction. So, if the last instruction, be it emulated or
	 * not, left the system with the INT_STI flag enabled, it
	 * means that the last instruction is an sti. We should not
	 * leave the flag on in this case. The same goes for mov ss
	 */
6087 6088
	if (int_shadow & mask)
		mask = 0;
6089
	if (unlikely(int_shadow || mask)) {
6090
		kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
6091 6092 6093
		if (!mask)
			kvm_make_request(KVM_REQ_EVENT, vcpu);
	}
6094 6095
}

6096
static bool inject_emulated_exception(struct kvm_vcpu *vcpu)
6097 6098
{
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6099
	if (ctxt->exception.vector == PF_VECTOR)
6100 6101 6102
		return kvm_propagate_fault(vcpu, &ctxt->exception);

	if (ctxt->exception.error_code_valid)
6103 6104
		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
				      ctxt->exception.error_code);
6105
	else
6106
		kvm_queue_exception(vcpu, ctxt->exception.vector);
6107
	return false;
6108 6109
}

6110 6111
static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
{
6112
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6113 6114 6115 6116
	int cs_db, cs_l;

	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);

6117
	ctxt->eflags = kvm_get_rflags(vcpu);
6118 6119
	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;

6120 6121 6122
	ctxt->eip = kvm_rip_read(vcpu);
	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
6123
		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :
6124 6125
		     cs_db				? X86EMUL_MODE_PROT32 :
							  X86EMUL_MODE_PROT16;
6126
	BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK);
P
Paolo Bonzini 已提交
6127 6128
	BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK);
	BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK);
6129

6130
	init_decode_cache(ctxt);
6131
	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6132 6133
}

6134
int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
6135
{
6136
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6137 6138 6139 6140
	int ret;

	init_emulate_ctxt(vcpu);

6141 6142 6143
	ctxt->op_bytes = 2;
	ctxt->ad_bytes = 2;
	ctxt->_eip = ctxt->eip + inc_eip;
6144
	ret = emulate_int_real(ctxt, irq);
6145 6146 6147 6148

	if (ret != X86EMUL_CONTINUE)
		return EMULATE_FAIL;

6149
	ctxt->eip = ctxt->_eip;
6150 6151
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
6152 6153 6154 6155 6156

	return EMULATE_DONE;
}
EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);

6157
static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
6158
{
6159 6160
	int r = EMULATE_DONE;

6161 6162
	++vcpu->stat.insn_emulation_fail;
	trace_kvm_emulate_insn_failed(vcpu);
6163 6164 6165 6166

	if (emulation_type & EMULTYPE_NO_UD_ON_FAIL)
		return EMULATE_FAIL;

6167
	if (!is_guest_mode(vcpu) && kvm_x86_ops->get_cpl(vcpu) == 0) {
6168 6169 6170
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6171
		r = EMULATE_USER_EXIT;
6172
	}
6173

6174
	kvm_queue_exception(vcpu, UD_VECTOR);
6175 6176

	return r;
6177 6178
}

6179
static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,
6180 6181
				  bool write_fault_to_shadow_pgtable,
				  int emulation_type)
6182
{
6183
	gpa_t gpa = cr2;
D
Dan Williams 已提交
6184
	kvm_pfn_t pfn;
6185

6186
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
6187 6188
		return false;

6189 6190 6191
	if (WARN_ON_ONCE(is_guest_mode(vcpu)))
		return false;

6192
	if (!vcpu->arch.mmu->direct_map) {
6193 6194 6195 6196 6197
		/*
		 * Write permission should be allowed since only
		 * write access need to be emulated.
		 */
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
6198

6199 6200 6201 6202 6203 6204 6205
		/*
		 * If the mapping is invalid in guest, let cpu retry
		 * it to generate fault.
		 */
		if (gpa == UNMAPPED_GVA)
			return true;
	}
6206

6207 6208 6209 6210 6211 6212 6213
	/*
	 * Do not retry the unhandleable instruction if it faults on the
	 * readonly host memory, otherwise it will goto a infinite loop:
	 * retry instruction -> write #PF -> emulation fail -> retry
	 * instruction -> ...
	 */
	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224

	/*
	 * If the instruction failed on the error pfn, it can not be fixed,
	 * report the error to userspace.
	 */
	if (is_error_noslot_pfn(pfn))
		return false;

	kvm_release_pfn_clean(pfn);

	/* The instructions are well-emulated on direct mmu. */
6225
	if (vcpu->arch.mmu->direct_map) {
6226 6227 6228 6229 6230 6231 6232 6233 6234
		unsigned int indirect_shadow_pages;

		spin_lock(&vcpu->kvm->mmu_lock);
		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
		spin_unlock(&vcpu->kvm->mmu_lock);

		if (indirect_shadow_pages)
			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));

6235
		return true;
6236
	}
6237

6238 6239 6240 6241 6242 6243
	/*
	 * if emulation was due to access to shadowed page table
	 * and it failed try to unshadow page and re-enter the
	 * guest to let CPU execute the instruction.
	 */
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6244 6245 6246 6247 6248 6249 6250

	/*
	 * If the access faults on its page table, it can not
	 * be fixed by unprotecting shadow page and it should
	 * be reported to userspace.
	 */
	return !write_fault_to_shadow_pgtable;
6251 6252
}

6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276
static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
			      unsigned long cr2,  int emulation_type)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	unsigned long last_retry_eip, last_retry_addr, gpa = cr2;

	last_retry_eip = vcpu->arch.last_retry_eip;
	last_retry_addr = vcpu->arch.last_retry_addr;

	/*
	 * If the emulation is caused by #PF and it is non-page_table
	 * writing instruction, it means the VM-EXIT is caused by shadow
	 * page protected, we can zap the shadow page and retry this
	 * instruction directly.
	 *
	 * Note: if the guest uses a non-page-table modifying instruction
	 * on the PDE that points to the instruction, then we will unmap
	 * the instruction and go to an infinite loop. So, we cache the
	 * last retried eip and the last fault address, if we meet the eip
	 * and the address again, we can break out of the potential infinite
	 * loop.
	 */
	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;

6277
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
6278 6279
		return false;

6280 6281 6282
	if (WARN_ON_ONCE(is_guest_mode(vcpu)))
		return false;

6283 6284 6285 6286 6287 6288 6289 6290 6291
	if (x86_page_table_writing_insn(ctxt))
		return false;

	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2)
		return false;

	vcpu->arch.last_retry_eip = ctxt->eip;
	vcpu->arch.last_retry_addr = cr2;

6292
	if (!vcpu->arch.mmu->direct_map)
6293 6294
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);

6295
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6296 6297 6298 6299

	return true;
}

6300 6301 6302
static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
static int complete_emulated_pio(struct kvm_vcpu *vcpu);

P
Paolo Bonzini 已提交
6303
static void kvm_smm_changed(struct kvm_vcpu *vcpu)
6304
{
P
Paolo Bonzini 已提交
6305
	if (!(vcpu->arch.hflags & HF_SMM_MASK)) {
6306 6307 6308
		/* This is a good place to trace that we are exiting SMM.  */
		trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false);

6309 6310
		/* Process a latched INIT or SMI, if any.  */
		kvm_make_request(KVM_REQ_EVENT, vcpu);
P
Paolo Bonzini 已提交
6311
	}
6312 6313

	kvm_mmu_reset_context(vcpu);
P
Paolo Bonzini 已提交
6314 6315
}

6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330
static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
				unsigned long *db)
{
	u32 dr6 = 0;
	int i;
	u32 enable, rwlen;

	enable = dr7;
	rwlen = dr7 >> 16;
	for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
		if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
			dr6 |= (1 << i);
	return dr6;
}

6331
static void kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu, int *r)
6332 6333 6334
{
	struct kvm_run *kvm_run = vcpu->run;

6335 6336 6337 6338 6339 6340 6341
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
		kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1 | DR6_RTM;
		kvm_run->debug.arch.pc = vcpu->arch.singlestep_rip;
		kvm_run->debug.arch.exception = DB_VECTOR;
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
		*r = EMULATE_USER_EXIT;
	} else {
6342
		kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
6343 6344 6345
	}
}

6346 6347 6348 6349 6350 6351
int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
	unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
	int r = EMULATE_DONE;

	kvm_x86_ops->skip_emulated_instruction(vcpu);
6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362

	/*
	 * rflags is the old, "raw" value of the flags.  The new value has
	 * not been saved yet.
	 *
	 * This is correct even for TF set by the guest, because "the
	 * processor will not generate this exception after the instruction
	 * that sets the TF flag".
	 */
	if (unlikely(rflags & X86_EFLAGS_TF))
		kvm_vcpu_do_singlestep(vcpu, &r);
6363 6364 6365 6366
	return r == EMULATE_DONE;
}
EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);

6367 6368 6369 6370
static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
{
	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
6371 6372 6373
		struct kvm_run *kvm_run = vcpu->run;
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6374 6375 6376 6377
					   vcpu->arch.guest_debug_dr7,
					   vcpu->arch.eff_db);

		if (dr6 != 0) {
6378
			kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
6379
			kvm_run->debug.arch.pc = eip;
6380 6381 6382 6383 6384 6385 6386
			kvm_run->debug.arch.exception = DB_VECTOR;
			kvm_run->exit_reason = KVM_EXIT_DEBUG;
			*r = EMULATE_USER_EXIT;
			return true;
		}
	}

6387 6388
	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
	    !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {
6389 6390
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6391 6392 6393 6394 6395
					   vcpu->arch.dr7,
					   vcpu->arch.db);

		if (dr6 != 0) {
			vcpu->arch.dr6 &= ~15;
6396
			vcpu->arch.dr6 |= dr6 | DR6_RTM;
6397 6398 6399 6400 6401 6402 6403 6404 6405
			kvm_queue_exception(vcpu, DB_VECTOR);
			*r = EMULATE_DONE;
			return true;
		}
	}

	return false;
}

6406 6407
static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
{
6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431
	switch (ctxt->opcode_len) {
	case 1:
		switch (ctxt->b) {
		case 0xe4:	/* IN */
		case 0xe5:
		case 0xec:
		case 0xed:
		case 0xe6:	/* OUT */
		case 0xe7:
		case 0xee:
		case 0xef:
		case 0x6c:	/* INS */
		case 0x6d:
		case 0x6e:	/* OUTS */
		case 0x6f:
			return true;
		}
		break;
	case 2:
		switch (ctxt->b) {
		case 0x33:	/* RDPMC */
			return true;
		}
		break;
6432 6433 6434 6435 6436
	}

	return false;
}

6437 6438
int x86_emulate_instruction(struct kvm_vcpu *vcpu,
			    unsigned long cr2,
6439 6440 6441
			    int emulation_type,
			    void *insn,
			    int insn_len)
6442
{
6443
	int r;
6444
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6445
	bool writeback = true;
6446
	bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
6447

P
Paolo Bonzini 已提交
6448 6449
	vcpu->arch.l1tf_flush_l1d = true;

6450 6451 6452 6453 6454
	/*
	 * Clear write_fault_to_shadow_pgtable here to ensure it is
	 * never reused.
	 */
	vcpu->arch.write_fault_to_shadow_pgtable = false;
6455
	kvm_clear_exception_queue(vcpu);
G
Gleb Natapov 已提交
6456

6457
	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
6458
		init_emulate_ctxt(vcpu);
6459 6460 6461 6462 6463 6464 6465

		/*
		 * We will reenter on the same instruction since
		 * we do not set complete_userspace_io.  This does not
		 * handle watchpoints yet, those would be handled in
		 * the emulate_ops.
		 */
6466 6467
		if (!(emulation_type & EMULTYPE_SKIP) &&
		    kvm_vcpu_check_breakpoint(vcpu, &r))
6468 6469
			return r;

6470 6471
		ctxt->interruptibility = 0;
		ctxt->have_exception = false;
6472
		ctxt->exception.vector = -1;
6473
		ctxt->perm_ok = false;
6474

6475
		ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
6476

6477
		r = x86_decode_insn(ctxt, insn, insn_len);
6478

A
Avi Kivity 已提交
6479
		trace_kvm_emulate_insn_start(vcpu);
6480
		++vcpu->stat.insn_emulation;
6481
		if (r != EMULATION_OK)  {
6482 6483
			if (emulation_type & EMULTYPE_TRAP_UD)
				return EMULATE_FAIL;
6484 6485
			if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
						emulation_type))
6486
				return EMULATE_DONE;
6487 6488
			if (ctxt->have_exception && inject_emulated_exception(vcpu))
				return EMULATE_DONE;
6489 6490
			if (emulation_type & EMULTYPE_SKIP)
				return EMULATE_FAIL;
6491
			return handle_emulation_failure(vcpu, emulation_type);
6492 6493 6494
		}
	}

6495 6496 6497 6498
	if ((emulation_type & EMULTYPE_VMWARE) &&
	    !is_vmware_backdoor_opcode(ctxt))
		return EMULATE_FAIL;

6499
	if (emulation_type & EMULTYPE_SKIP) {
6500
		kvm_rip_write(vcpu, ctxt->_eip);
6501 6502
		if (ctxt->eflags & X86_EFLAGS_RF)
			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
6503 6504 6505
		return EMULATE_DONE;
	}

6506 6507 6508
	if (retry_instruction(ctxt, cr2, emulation_type))
		return EMULATE_DONE;

6509
	/* this is needed for vmware backdoor interface to work since it
6510
	   changes registers values  during IO operation */
6511 6512
	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6513
		emulator_invalidate_register_cache(ctxt);
6514
	}
6515

6516
restart:
6517 6518 6519
	/* Save the faulting GPA (cr2) in the address field */
	ctxt->exception.address = cr2;

6520
	r = x86_emulate_insn(ctxt);
6521

6522 6523 6524
	if (r == EMULATION_INTERCEPTED)
		return EMULATE_DONE;

6525
	if (r == EMULATION_FAILED) {
6526 6527
		if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
					emulation_type))
6528 6529
			return EMULATE_DONE;

6530
		return handle_emulation_failure(vcpu, emulation_type);
6531 6532
	}

6533
	if (ctxt->have_exception) {
6534
		r = EMULATE_DONE;
6535 6536
		if (inject_emulated_exception(vcpu))
			return r;
6537
	} else if (vcpu->arch.pio.count) {
6538 6539
		if (!vcpu->arch.pio.in) {
			/* FIXME: return into emulator if single-stepping.  */
6540
			vcpu->arch.pio.count = 0;
6541
		} else {
6542
			writeback = false;
6543 6544
			vcpu->arch.complete_userspace_io = complete_emulated_pio;
		}
P
Paolo Bonzini 已提交
6545
		r = EMULATE_USER_EXIT;
6546 6547 6548
	} else if (vcpu->mmio_needed) {
		if (!vcpu->mmio_is_write)
			writeback = false;
P
Paolo Bonzini 已提交
6549
		r = EMULATE_USER_EXIT;
6550
		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
6551
	} else if (r == EMULATION_RESTART)
6552
		goto restart;
6553 6554
	else
		r = EMULATE_DONE;
6555

6556
	if (writeback) {
6557
		unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6558
		toggle_interruptibility(vcpu, ctxt->interruptibility);
6559
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
6560
		kvm_rip_write(vcpu, ctxt->eip);
6561
		if (r == EMULATE_DONE && ctxt->tf)
6562
			kvm_vcpu_do_singlestep(vcpu, &r);
6563 6564 6565
		if (!ctxt->have_exception ||
		    exception_type(ctxt->exception.vector) == EXCPT_TRAP)
			__kvm_set_rflags(vcpu, ctxt->eflags);
6566 6567 6568 6569 6570 6571 6572 6573 6574

		/*
		 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
		 * do nothing, and it will be requested again as soon as
		 * the shadow expires.  But we still need to check here,
		 * because POPF has no interrupt shadow.
		 */
		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
			kvm_make_request(KVM_REQ_EVENT, vcpu);
6575 6576
	} else
		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
6577 6578

	return r;
6579
}
6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592

int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
{
	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction);

int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
					void *insn, int insn_len)
{
	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
6593

6594 6595 6596 6597 6598 6599
static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;
	return 1;
}

6600 6601 6602 6603 6604 6605 6606 6607 6608 6609
static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;

	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
		return 1;

	return kvm_skip_emulated_instruction(vcpu);
}

6610 6611
static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
			    unsigned short port)
6612
{
6613
	unsigned long val = kvm_rax_read(vcpu);
6614 6615
	int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
					    size, port, &val, 1);
6616 6617
	if (ret)
		return ret;
6618

6619 6620 6621 6622 6623 6624 6625 6626 6627 6628
	/*
	 * Workaround userspace that relies on old KVM behavior of %rip being
	 * incremented prior to exiting to userspace to handle "OUT 0x7e".
	 */
	if (port == 0x7e &&
	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
		vcpu->arch.complete_userspace_io =
			complete_fast_pio_out_port_0x7e;
		kvm_skip_emulated_instruction(vcpu);
	} else {
6629 6630 6631
		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
		vcpu->arch.complete_userspace_io = complete_fast_pio_out;
	}
6632
	return 0;
6633 6634
}

6635 6636 6637 6638 6639 6640 6641
static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
{
	unsigned long val;

	/* We should only ever be called with arch.pio.count equal to 1 */
	BUG_ON(vcpu->arch.pio.count != 1);

6642 6643 6644 6645 6646
	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
		vcpu->arch.pio.count = 0;
		return 1;
	}

6647
	/* For size less than 4 we merge, else we zero extend */
6648
	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
6649 6650 6651 6652 6653 6654 6655

	/*
	 * Since vcpu->arch.pio.count == 1 let emulator_pio_in_emulated perform
	 * the copy and tracing
	 */
	emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, vcpu->arch.pio.size,
				 vcpu->arch.pio.port, &val, 1);
6656
	kvm_rax_write(vcpu, val);
6657

6658
	return kvm_skip_emulated_instruction(vcpu);
6659 6660
}

6661 6662
static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
			   unsigned short port)
6663 6664 6665 6666 6667
{
	unsigned long val;
	int ret;

	/* For size less than 4 we merge, else we zero extend */
6668
	val = (size < 4) ? kvm_rax_read(vcpu) : 0;
6669 6670 6671 6672

	ret = emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, size, port,
				       &val, 1);
	if (ret) {
6673
		kvm_rax_write(vcpu, val);
6674 6675 6676
		return ret;
	}

6677
	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
6678 6679 6680 6681
	vcpu->arch.complete_userspace_io = complete_fast_pio_in;

	return 0;
}
6682 6683 6684

int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
{
6685
	int ret;
6686 6687

	if (in)
6688
		ret = kvm_fast_pio_in(vcpu, size, port);
6689
	else
6690 6691
		ret = kvm_fast_pio_out(vcpu, size, port);
	return ret && kvm_skip_emulated_instruction(vcpu);
6692 6693
}
EXPORT_SYMBOL_GPL(kvm_fast_pio);
6694

6695
static int kvmclock_cpu_down_prep(unsigned int cpu)
6696
{
T
Tejun Heo 已提交
6697
	__this_cpu_write(cpu_tsc_khz, 0);
6698
	return 0;
6699 6700 6701
}

static void tsc_khz_changed(void *data)
6702
{
6703 6704 6705 6706 6707 6708 6709 6710 6711
	struct cpufreq_freqs *freq = data;
	unsigned long khz = 0;

	if (data)
		khz = freq->new;
	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		khz = cpufreq_quick_get(raw_smp_processor_id());
	if (!khz)
		khz = tsc_khz;
T
Tejun Heo 已提交
6712
	__this_cpu_write(cpu_tsc_khz, khz);
6713 6714
}

6715
#ifdef CONFIG_X86_64
6716 6717 6718 6719 6720 6721
static void kvm_hyperv_tsc_notifier(void)
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int cpu;

J
Junaid Shahid 已提交
6722
	mutex_lock(&kvm_lock);
6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_make_mclock_inprogress_request(kvm);

	hyperv_stop_tsc_emulation();

	/* TSC frequency always matches when on Hyper-V */
	for_each_present_cpu(cpu)
		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
	kvm_max_guest_tsc_khz = tsc_khz;

	list_for_each_entry(kvm, &vm_list, vm_list) {
		struct kvm_arch *ka = &kvm->arch;

		spin_lock(&ka->pvclock_gtod_sync_lock);

		pvclock_update_vm_gtod_copy(kvm);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);

		spin_unlock(&ka->pvclock_gtod_sync_lock);
	}
J
Junaid Shahid 已提交
6748
	mutex_unlock(&kvm_lock);
6749
}
6750
#endif
6751

6752
static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
6753 6754 6755 6756 6757
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i, send_ipi = 0;

6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796
	/*
	 * We allow guests to temporarily run on slowing clocks,
	 * provided we notify them after, or to run on accelerating
	 * clocks, provided we notify them before.  Thus time never
	 * goes backwards.
	 *
	 * However, we have a problem.  We can't atomically update
	 * the frequency of a given CPU from this function; it is
	 * merely a notifier, which can be called from any CPU.
	 * Changing the TSC frequency at arbitrary points in time
	 * requires a recomputation of local variables related to
	 * the TSC for each VCPU.  We must flag these local variables
	 * to be updated and be sure the update takes place with the
	 * new frequency before any guests proceed.
	 *
	 * Unfortunately, the combination of hotplug CPU and frequency
	 * change creates an intractable locking scenario; the order
	 * of when these callouts happen is undefined with respect to
	 * CPU hotplug, and they can race with each other.  As such,
	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
	 * undefined; you can actually have a CPU frequency change take
	 * place in between the computation of X and the setting of the
	 * variable.  To protect against this problem, all updates of
	 * the per_cpu tsc_khz variable are done in an interrupt
	 * protected IPI, and all callers wishing to update the value
	 * must wait for a synchronous IPI to complete (which is trivial
	 * if the caller is on the CPU already).  This establishes the
	 * necessary total order on variable updates.
	 *
	 * Note that because a guest time update may take place
	 * anytime after the setting of the VCPU's request bit, the
	 * correct TSC value must be set before the request.  However,
	 * to ensure the update actually makes it to any guest which
	 * starts running in hardware virtualization between the set
	 * and the acquisition of the spinlock, we must also ping the
	 * CPU after setting the request bit.
	 *
	 */

6797
	smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
6798

J
Junaid Shahid 已提交
6799
	mutex_lock(&kvm_lock);
6800
	list_for_each_entry(kvm, &vm_list, vm_list) {
6801
		kvm_for_each_vcpu(i, vcpu, kvm) {
6802
			if (vcpu->cpu != cpu)
6803
				continue;
Z
Zachary Amsden 已提交
6804
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
J
Junaid Shahid 已提交
6805
			if (vcpu->cpu != raw_smp_processor_id())
6806
				send_ipi = 1;
6807 6808
		}
	}
J
Junaid Shahid 已提交
6809
	mutex_unlock(&kvm_lock);
6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823

	if (freq->old < freq->new && send_ipi) {
		/*
		 * We upscale the frequency.  Must make the guest
		 * doesn't see old kvmclock values while running with
		 * the new frequency, otherwise we risk the guest sees
		 * time go backwards.
		 *
		 * In case we update the frequency for another cpu
		 * (which might be in guest context) send an interrupt
		 * to kick the cpu out of guest context.  Next time
		 * guest context is entered kvmclock will be updated,
		 * so the guest will not see stale values.
		 */
6824
		smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
6825
	}
6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841
}

static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
				     void *data)
{
	struct cpufreq_freqs *freq = data;
	int cpu;

	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
		return 0;
	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
		return 0;

	for_each_cpu(cpu, freq->policy->cpus)
		__kvmclock_cpufreq_notifier(freq, cpu);

6842 6843 6844 6845
	return 0;
}

static struct notifier_block kvmclock_cpufreq_notifier_block = {
6846 6847 6848
	.notifier_call  = kvmclock_cpufreq_notifier
};

6849
static int kvmclock_cpu_online(unsigned int cpu)
6850
{
6851 6852
	tsc_khz_changed(NULL);
	return 0;
6853 6854
}

6855 6856
static void kvm_timer_init(void)
{
Z
Zachary Amsden 已提交
6857
	max_tsc_khz = tsc_khz;
6858

6859
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
Z
Zachary Amsden 已提交
6860 6861
#ifdef CONFIG_CPU_FREQ
		struct cpufreq_policy policy;
6862 6863
		int cpu;

Z
Zachary Amsden 已提交
6864
		memset(&policy, 0, sizeof(policy));
6865 6866
		cpu = get_cpu();
		cpufreq_get_policy(&policy, cpu);
Z
Zachary Amsden 已提交
6867 6868
		if (policy.cpuinfo.max_freq)
			max_tsc_khz = policy.cpuinfo.max_freq;
6869
		put_cpu();
Z
Zachary Amsden 已提交
6870
#endif
6871 6872 6873
		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
					  CPUFREQ_TRANSITION_NOTIFIER);
	}
Z
Zachary Amsden 已提交
6874
	pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz);
6875

T
Thomas Gleixner 已提交
6876
	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
6877
			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
6878 6879
}

6880 6881
DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu);
6882

6883
int kvm_is_in_guest(void)
6884
{
6885
	return __this_cpu_read(current_vcpu) != NULL;
6886 6887 6888 6889 6890
}

static int kvm_is_user_mode(void)
{
	int user_mode = 3;
6891

6892 6893
	if (__this_cpu_read(current_vcpu))
		user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
6894

6895 6896 6897 6898 6899 6900
	return user_mode != 0;
}

static unsigned long kvm_get_guest_ip(void)
{
	unsigned long ip = 0;
6901

6902 6903
	if (__this_cpu_read(current_vcpu))
		ip = kvm_rip_read(__this_cpu_read(current_vcpu));
6904

6905 6906 6907
	return ip;
}

L
Luwei Kang 已提交
6908 6909 6910 6911 6912 6913 6914 6915 6916
static void kvm_handle_intel_pt_intr(void)
{
	struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);

	kvm_make_request(KVM_REQ_PMI, vcpu);
	__set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
			(unsigned long *)&vcpu->arch.pmu.global_status);
}

6917 6918 6919 6920
static struct perf_guest_info_callbacks kvm_guest_cbs = {
	.is_in_guest		= kvm_is_in_guest,
	.is_user_mode		= kvm_is_user_mode,
	.get_guest_ip		= kvm_get_guest_ip,
L
Luwei Kang 已提交
6921
	.handle_intel_pt_intr	= kvm_handle_intel_pt_intr,
6922 6923
};

6924 6925 6926
#ifdef CONFIG_X86_64
static void pvclock_gtod_update_fn(struct work_struct *work)
{
6927 6928 6929 6930 6931
	struct kvm *kvm;

	struct kvm_vcpu *vcpu;
	int i;

J
Junaid Shahid 已提交
6932
	mutex_lock(&kvm_lock);
6933 6934
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_for_each_vcpu(i, vcpu, kvm)
6935
			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
6936
	atomic_set(&kvm_guest_has_master_clock, 0);
J
Junaid Shahid 已提交
6937
	mutex_unlock(&kvm_lock);
6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953
}

static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);

/*
 * Notification about pvclock gtod data update.
 */
static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
			       void *priv)
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	struct timekeeper *tk = priv;

	update_pvclock_gtod(tk);

	/* disable master clock if host does not trust, or does not
6954
	 * use, TSC based clocksource.
6955
	 */
6956
	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967
	    atomic_read(&kvm_guest_has_master_clock) != 0)
		queue_work(system_long_wq, &pvclock_gtod_work);

	return 0;
}

static struct notifier_block pvclock_gtod_notifier = {
	.notifier_call = pvclock_gtod_notify,
};
#endif

6968
int kvm_arch_init(void *opaque)
6969
{
6970
	int r;
M
Mathias Krause 已提交
6971
	struct kvm_x86_ops *ops = opaque;
6972 6973 6974

	if (kvm_x86_ops) {
		printk(KERN_ERR "kvm: already loaded the other module\n");
6975 6976
		r = -EEXIST;
		goto out;
6977 6978 6979 6980
	}

	if (!ops->cpu_has_kvm_support()) {
		printk(KERN_ERR "kvm: no hardware support\n");
6981 6982
		r = -EOPNOTSUPP;
		goto out;
6983 6984 6985
	}
	if (ops->disabled_by_bios()) {
		printk(KERN_ERR "kvm: disabled by bios\n");
6986 6987
		r = -EOPNOTSUPP;
		goto out;
6988 6989
	}

6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000
	/*
	 * KVM explicitly assumes that the guest has an FPU and
	 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
	 * vCPU's FPU state as a fxregs_state struct.
	 */
	if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
		printk(KERN_ERR "kvm: inadequate fpu\n");
		r = -EOPNOTSUPP;
		goto out;
	}

7001
	r = -ENOMEM;
7002
	x86_fpu_cache = kmem_cache_create("x86_fpu", sizeof(struct fpu),
7003 7004 7005 7006 7007 7008 7009
					  __alignof__(struct fpu), SLAB_ACCOUNT,
					  NULL);
	if (!x86_fpu_cache) {
		printk(KERN_ERR "kvm: failed to allocate cache for x86 fpu\n");
		goto out;
	}

7010 7011 7012
	shared_msrs = alloc_percpu(struct kvm_shared_msrs);
	if (!shared_msrs) {
		printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
7013
		goto out_free_x86_fpu_cache;
7014 7015
	}

7016 7017
	r = kvm_mmu_module_init();
	if (r)
7018
		goto out_free_percpu;
7019

7020
	kvm_x86_ops = ops;
P
Paolo Bonzini 已提交
7021

S
Sheng Yang 已提交
7022
	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
7023
			PT_DIRTY_MASK, PT64_NX_MASK, 0,
7024
			PT_PRESENT_MASK, 0, sme_me_mask);
7025
	kvm_timer_init();
7026

7027 7028
	perf_register_guest_info_callbacks(&kvm_guest_cbs);

7029
	if (boot_cpu_has(X86_FEATURE_XSAVE))
7030 7031
		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);

7032
	kvm_lapic_init();
7033 7034
#ifdef CONFIG_X86_64
	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
7035

7036
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7037
		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
7038 7039
#endif

7040
	return 0;
7041

7042 7043
out_free_percpu:
	free_percpu(shared_msrs);
7044 7045
out_free_x86_fpu_cache:
	kmem_cache_destroy(x86_fpu_cache);
7046 7047
out:
	return r;
7048
}
7049

7050 7051
void kvm_arch_exit(void)
{
7052
#ifdef CONFIG_X86_64
7053
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7054 7055
		clear_hv_tscchange_cb();
#endif
7056
	kvm_lapic_exit();
7057 7058
	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);

7059 7060 7061
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
					    CPUFREQ_TRANSITION_NOTIFIER);
7062
	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
7063 7064 7065
#ifdef CONFIG_X86_64
	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
#endif
7066
	kvm_x86_ops = NULL;
7067
	kvm_mmu_module_exit();
7068
	free_percpu(shared_msrs);
7069
	kmem_cache_destroy(x86_fpu_cache);
7070
}
7071

7072
int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
7073 7074
{
	++vcpu->stat.halt_exits;
7075
	if (lapic_in_kernel(vcpu)) {
7076
		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
7077 7078 7079 7080 7081 7082
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
}
7083 7084 7085 7086
EXPORT_SYMBOL_GPL(kvm_vcpu_halt);

int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
7087 7088 7089 7090 7091 7092
	int ret = kvm_skip_emulated_instruction(vcpu);
	/*
	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
	 * KVM_EXIT_DEBUG here.
	 */
	return kvm_vcpu_halt(vcpu) && ret;
7093
}
7094 7095
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

7096
#ifdef CONFIG_X86_64
7097 7098 7099 7100
static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
			        unsigned long clock_type)
{
	struct kvm_clock_pairing clock_pairing;
7101
	struct timespec64 ts;
P
Paolo Bonzini 已提交
7102
	u64 cycle;
7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114
	int ret;

	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
		return -KVM_EOPNOTSUPP;

	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)
		return -KVM_EOPNOTSUPP;

	clock_pairing.sec = ts.tv_sec;
	clock_pairing.nsec = ts.tv_nsec;
	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
	clock_pairing.flags = 0;
7115
	memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
7116 7117 7118 7119 7120 7121 7122 7123

	ret = 0;
	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
			    sizeof(struct kvm_clock_pairing)))
		ret = -KVM_EFAULT;

	return ret;
}
7124
#endif
7125

7126 7127 7128 7129 7130 7131 7132
/*
 * kvm_pv_kick_cpu_op:  Kick a vcpu.
 *
 * @apicid - apicid of vcpu to be kicked.
 */
static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
{
7133
	struct kvm_lapic_irq lapic_irq;
7134

7135 7136
	lapic_irq.shorthand = 0;
	lapic_irq.dest_mode = 0;
7137
	lapic_irq.level = 0;
7138
	lapic_irq.dest_id = apicid;
7139
	lapic_irq.msi_redir_hint = false;
7140

7141
	lapic_irq.delivery_mode = APIC_DM_REMRD;
7142
	kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
7143 7144
}

7145 7146
void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu)
{
7147 7148 7149 7150 7151 7152 7153
	if (!lapic_in_kernel(vcpu)) {
		WARN_ON_ONCE(vcpu->arch.apicv_active);
		return;
	}
	if (!vcpu->arch.apicv_active)
		return;

7154 7155 7156 7157
	vcpu->arch.apicv_active = false;
	kvm_x86_ops->refresh_apicv_exec_ctrl(vcpu);
}

7158 7159 7160
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
{
	unsigned long nr, a0, a1, a2, a3, ret;
7161
	int op_64_bit;
7162

7163 7164
	if (kvm_hv_hypercall_enabled(vcpu->kvm))
		return kvm_hv_hypercall(vcpu);
7165

7166 7167 7168 7169 7170
	nr = kvm_rax_read(vcpu);
	a0 = kvm_rbx_read(vcpu);
	a1 = kvm_rcx_read(vcpu);
	a2 = kvm_rdx_read(vcpu);
	a3 = kvm_rsi_read(vcpu);
7171

7172
	trace_kvm_hypercall(nr, a0, a1, a2, a3);
F
Feng (Eric) Liu 已提交
7173

7174 7175
	op_64_bit = is_64_bit_mode(vcpu);
	if (!op_64_bit) {
7176 7177 7178 7179 7180 7181 7182
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
	}

7183 7184
	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
		ret = -KVM_EPERM;
7185
		goto out;
7186 7187
	}

7188
	switch (nr) {
A
Avi Kivity 已提交
7189 7190 7191
	case KVM_HC_VAPIC_POLL_IRQ:
		ret = 0;
		break;
7192 7193 7194 7195
	case KVM_HC_KICK_CPU:
		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
		ret = 0;
		break;
7196
#ifdef CONFIG_X86_64
7197 7198 7199
	case KVM_HC_CLOCK_PAIRING:
		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
		break;
7200
#endif
7201 7202 7203
	case KVM_HC_SEND_IPI:
		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
		break;
7204 7205 7206 7207
	default:
		ret = -KVM_ENOSYS;
		break;
	}
7208
out:
7209 7210
	if (!op_64_bit)
		ret = (u32)ret;
7211
	kvm_rax_write(vcpu, ret);
7212

A
Amit Shah 已提交
7213
	++vcpu->stat.hypercalls;
7214
	return kvm_skip_emulated_instruction(vcpu);
7215 7216 7217
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

7218
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
7219
{
7220
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7221
	char instruction[3];
7222
	unsigned long rip = kvm_rip_read(vcpu);
7223 7224 7225

	kvm_x86_ops->patch_hypercall(vcpu, instruction);

7226 7227
	return emulator_write_emulated(ctxt, rip, instruction, 3,
		&ctxt->exception);
7228 7229
}

A
Avi Kivity 已提交
7230
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
7231
{
7232 7233
	return vcpu->run->request_interrupt_window &&
		likely(!pic_in_kernel(vcpu->kvm));
7234 7235
}

A
Avi Kivity 已提交
7236
static void post_kvm_run_save(struct kvm_vcpu *vcpu)
7237
{
A
Avi Kivity 已提交
7238 7239
	struct kvm_run *kvm_run = vcpu->run;

7240
	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
7241
	kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;
7242
	kvm_run->cr8 = kvm_get_cr8(vcpu);
7243
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
7244 7245
	kvm_run->ready_for_interrupt_injection =
		pic_in_kernel(vcpu->kvm) ||
7246
		kvm_vcpu_ready_for_interrupt_injection(vcpu);
7247 7248
}

7249 7250 7251 7252 7253 7254 7255
static void update_cr8_intercept(struct kvm_vcpu *vcpu)
{
	int max_irr, tpr;

	if (!kvm_x86_ops->update_cr8_intercept)
		return;

7256
	if (!lapic_in_kernel(vcpu))
7257 7258
		return;

7259 7260 7261
	if (vcpu->arch.apicv_active)
		return;

7262 7263 7264 7265
	if (!vcpu->arch.apic->vapic_addr)
		max_irr = kvm_lapic_find_highest_irr(vcpu);
	else
		max_irr = -1;
7266 7267 7268 7269 7270 7271 7272 7273 7274

	if (max_irr != -1)
		max_irr >>= 4;

	tpr = kvm_lapic_get_cr8(vcpu);

	kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
}

7275
static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win)
7276
{
7277 7278
	int r;

7279
	/* try to reinject previous events if any */
7280

7281 7282
	if (vcpu->arch.exception.injected)
		kvm_x86_ops->queue_exception(vcpu);
7283
	/*
7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295
	 * Do not inject an NMI or interrupt if there is a pending
	 * exception.  Exceptions and interrupts are recognized at
	 * instruction boundaries, i.e. the start of an instruction.
	 * Trap-like exceptions, e.g. #DB, have higher priority than
	 * NMIs and interrupts, i.e. traps are recognized before an
	 * NMI/interrupt that's pending on the same instruction.
	 * Fault-like exceptions, e.g. #GP and #PF, are the lowest
	 * priority, but are only generated (pended) during instruction
	 * execution, i.e. a pending fault-like exception means the
	 * fault occurred on the *previous* instruction and must be
	 * serviced prior to recognizing any new events in order to
	 * fully complete the previous instruction.
7296
	 */
7297 7298
	else if (!vcpu->arch.exception.pending) {
		if (vcpu->arch.nmi_injected)
7299
			kvm_x86_ops->set_nmi(vcpu);
7300
		else if (vcpu->arch.interrupt.injected)
7301 7302 7303
			kvm_x86_ops->set_irq(vcpu);
	}

7304 7305 7306 7307 7308 7309
	/*
	 * Call check_nested_events() even if we reinjected a previous event
	 * in order for caller to determine if it should require immediate-exit
	 * from L2 to L1 due to pending L1 events which require exit
	 * from L2 to L1.
	 */
7310 7311 7312 7313 7314 7315 7316
	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
		r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
		if (r != 0)
			return r;
	}

	/* try to inject new event if pending */
7317
	if (vcpu->arch.exception.pending) {
A
Avi Kivity 已提交
7318 7319 7320
		trace_kvm_inj_exception(vcpu->arch.exception.nr,
					vcpu->arch.exception.has_error_code,
					vcpu->arch.exception.error_code);
7321

7322
		WARN_ON_ONCE(vcpu->arch.exception.injected);
7323 7324 7325
		vcpu->arch.exception.pending = false;
		vcpu->arch.exception.injected = true;

7326 7327 7328 7329
		if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
					     X86_EFLAGS_RF);

7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345
		if (vcpu->arch.exception.nr == DB_VECTOR) {
			/*
			 * This code assumes that nSVM doesn't use
			 * check_nested_events(). If it does, the
			 * DR6/DR7 changes should happen before L1
			 * gets a #VMEXIT for an intercepted #DB in
			 * L2.  (Under VMX, on the other hand, the
			 * DR6/DR7 changes should not happen in the
			 * event of a VM-exit to L1 for an intercepted
			 * #DB in L2.)
			 */
			kvm_deliver_exception_payload(vcpu);
			if (vcpu->arch.dr7 & DR7_GD) {
				vcpu->arch.dr7 &= ~DR7_GD;
				kvm_update_dr7(vcpu);
			}
7346 7347
		}

7348
		kvm_x86_ops->queue_exception(vcpu);
7349 7350 7351 7352 7353 7354 7355 7356
	}

	/* Don't consider new event if we re-injected an event */
	if (kvm_event_needs_reinjection(vcpu))
		return 0;

	if (vcpu->arch.smi_pending && !is_smm(vcpu) &&
	    kvm_x86_ops->smi_allowed(vcpu)) {
7357
		vcpu->arch.smi_pending = false;
7358
		++vcpu->arch.smi_count;
7359
		enter_smm(vcpu);
7360
	} else if (vcpu->arch.nmi_pending && kvm_x86_ops->nmi_allowed(vcpu)) {
7361 7362 7363
		--vcpu->arch.nmi_pending;
		vcpu->arch.nmi_injected = true;
		kvm_x86_ops->set_nmi(vcpu);
7364
	} else if (kvm_cpu_has_injectable_intr(vcpu)) {
7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376
		/*
		 * Because interrupts can be injected asynchronously, we are
		 * calling check_nested_events again here to avoid a race condition.
		 * See https://lkml.org/lkml/2014/7/2/60 for discussion about this
		 * proposal and current concerns.  Perhaps we should be setting
		 * KVM_REQ_EVENT only on certain events and not unconditionally?
		 */
		if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
			r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
			if (r != 0)
				return r;
		}
7377
		if (kvm_x86_ops->interrupt_allowed(vcpu)) {
7378 7379 7380
			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
					    false);
			kvm_x86_ops->set_irq(vcpu);
7381 7382
		}
	}
7383

7384
	return 0;
7385 7386
}

A
Avi Kivity 已提交
7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403
static void process_nmi(struct kvm_vcpu *vcpu)
{
	unsigned limit = 2;

	/*
	 * x86 is limited to one NMI running, and one NMI pending after it.
	 * If an NMI is already in progress, limit further NMIs to just one.
	 * Otherwise, allow two (and we'll inject the first one immediately).
	 */
	if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
		limit = 1;

	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

7404
static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417
{
	u32 flags = 0;
	flags |= seg->g       << 23;
	flags |= seg->db      << 22;
	flags |= seg->l       << 21;
	flags |= seg->avl     << 20;
	flags |= seg->present << 15;
	flags |= seg->dpl     << 13;
	flags |= seg->s       << 12;
	flags |= seg->type    << 8;
	return flags;
}

7418
static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432
{
	struct kvm_segment seg;
	int offset;

	kvm_get_segment(vcpu, &seg, n);
	put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector);

	if (n < 3)
		offset = 0x7f84 + n * 12;
	else
		offset = 0x7f2c + (n - 3) * 12;

	put_smstate(u32, buf, offset + 8, seg.base);
	put_smstate(u32, buf, offset + 4, seg.limit);
7433
	put_smstate(u32, buf, offset, enter_smm_get_segment_flags(&seg));
7434 7435
}

7436
#ifdef CONFIG_X86_64
7437
static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
7438 7439 7440 7441 7442 7443 7444 7445
{
	struct kvm_segment seg;
	int offset;
	u16 flags;

	kvm_get_segment(vcpu, &seg, n);
	offset = 0x7e00 + n * 16;

7446
	flags = enter_smm_get_segment_flags(&seg) >> 8;
7447 7448 7449 7450 7451
	put_smstate(u16, buf, offset, seg.selector);
	put_smstate(u16, buf, offset + 2, flags);
	put_smstate(u32, buf, offset + 4, seg.limit);
	put_smstate(u64, buf, offset + 8, seg.base);
}
7452
#endif
7453

7454
static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
	put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
	put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
	put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));

	for (i = 0; i < 8; i++)
		put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read(vcpu, i));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u32, buf, 0x7fcc, (u32)val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u32, buf, 0x7fc8, (u32)val);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u32, buf, 0x7fc4, seg.selector);
	put_smstate(u32, buf, 0x7f64, seg.base);
	put_smstate(u32, buf, 0x7f60, seg.limit);
7478
	put_smstate(u32, buf, 0x7f5c, enter_smm_get_segment_flags(&seg));
7479 7480 7481 7482 7483

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u32, buf, 0x7fc0, seg.selector);
	put_smstate(u32, buf, 0x7f80, seg.base);
	put_smstate(u32, buf, 0x7f7c, seg.limit);
7484
	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
7485 7486 7487 7488 7489 7490 7491 7492 7493 7494

	kvm_x86_ops->get_gdt(vcpu, &dt);
	put_smstate(u32, buf, 0x7f74, dt.address);
	put_smstate(u32, buf, 0x7f70, dt.size);

	kvm_x86_ops->get_idt(vcpu, &dt);
	put_smstate(u32, buf, 0x7f58, dt.address);
	put_smstate(u32, buf, 0x7f54, dt.size);

	for (i = 0; i < 6; i++)
7495
		enter_smm_save_seg_32(vcpu, buf, i);
7496 7497 7498 7499 7500 7501 7502 7503

	put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020000);
	put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
}

7504
#ifdef CONFIG_X86_64
7505
static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	for (i = 0; i < 16; i++)
		put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read(vcpu, i));

	put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
	put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u64, buf, 0x7f68, val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u64, buf, 0x7f60, val);

	put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
	put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
	put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));

	put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020064);

	put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u16, buf, 0x7e90, seg.selector);
7536
	put_smstate(u16, buf, 0x7e92, enter_smm_get_segment_flags(&seg) >> 8);
7537 7538 7539 7540 7541 7542 7543 7544 7545
	put_smstate(u32, buf, 0x7e94, seg.limit);
	put_smstate(u64, buf, 0x7e98, seg.base);

	kvm_x86_ops->get_idt(vcpu, &dt);
	put_smstate(u32, buf, 0x7e84, dt.size);
	put_smstate(u64, buf, 0x7e88, dt.address);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u16, buf, 0x7e70, seg.selector);
7546
	put_smstate(u16, buf, 0x7e72, enter_smm_get_segment_flags(&seg) >> 8);
7547 7548 7549 7550 7551 7552 7553 7554
	put_smstate(u32, buf, 0x7e74, seg.limit);
	put_smstate(u64, buf, 0x7e78, seg.base);

	kvm_x86_ops->get_gdt(vcpu, &dt);
	put_smstate(u32, buf, 0x7e64, dt.size);
	put_smstate(u64, buf, 0x7e68, dt.address);

	for (i = 0; i < 6; i++)
7555
		enter_smm_save_seg_64(vcpu, buf, i);
7556
}
7557
#endif
7558

7559
static void enter_smm(struct kvm_vcpu *vcpu)
P
Paolo Bonzini 已提交
7560
{
7561
	struct kvm_segment cs, ds;
7562
	struct desc_ptr dt;
7563 7564 7565 7566 7567
	char buf[512];
	u32 cr0;

	trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true);
	memset(buf, 0, 512);
7568
#ifdef CONFIG_X86_64
7569
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7570
		enter_smm_save_state_64(vcpu, buf);
7571
	else
7572
#endif
7573
		enter_smm_save_state_32(vcpu, buf);
7574

7575 7576 7577 7578 7579 7580 7581 7582
	/*
	 * Give pre_enter_smm() a chance to make ISA-specific changes to the
	 * vCPU state (e.g. leave guest mode) after we've saved the state into
	 * the SMM state-save area.
	 */
	kvm_x86_ops->pre_enter_smm(vcpu, buf);

	vcpu->arch.hflags |= HF_SMM_MASK;
7583
	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598

	if (kvm_x86_ops->get_nmi_mask(vcpu))
		vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
	else
		kvm_x86_ops->set_nmi_mask(vcpu, true);

	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
	kvm_rip_write(vcpu, 0x8000);

	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
	kvm_x86_ops->set_cr0(vcpu, cr0);
	vcpu->arch.cr0 = cr0;

	kvm_x86_ops->set_cr4(vcpu, 0);

7599 7600 7601 7602
	/* Undocumented: IDT limit is set to zero on entry to SMM.  */
	dt.address = dt.size = 0;
	kvm_x86_ops->set_idt(vcpu, &dt);

7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629
	__kvm_set_dr(vcpu, 7, DR7_FIXED_1);

	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
	cs.base = vcpu->arch.smbase;

	ds.selector = 0;
	ds.base = 0;

	cs.limit    = ds.limit = 0xffffffff;
	cs.type     = ds.type = 0x3;
	cs.dpl      = ds.dpl = 0;
	cs.db       = ds.db = 0;
	cs.s        = ds.s = 1;
	cs.l        = ds.l = 0;
	cs.g        = ds.g = 1;
	cs.avl      = ds.avl = 0;
	cs.present  = ds.present = 1;
	cs.unusable = ds.unusable = 0;
	cs.padding  = ds.padding = 0;

	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);

7630
#ifdef CONFIG_X86_64
7631
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7632
		kvm_x86_ops->set_efer(vcpu, 0);
7633
#endif
7634 7635 7636

	kvm_update_cpuid(vcpu);
	kvm_mmu_reset_context(vcpu);
P
Paolo Bonzini 已提交
7637 7638
}

7639
static void process_smi(struct kvm_vcpu *vcpu)
7640 7641 7642 7643 7644
{
	vcpu->arch.smi_pending = true;
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

7645 7646 7647 7648 7649
void kvm_make_scan_ioapic_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
}

7650
static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
7651
{
7652
	if (!kvm_apic_present(vcpu))
7653
		return;
7654

7655
	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
7656

7657
	if (irqchip_split(vcpu->kvm))
7658
		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
7659
	else {
7660
		if (vcpu->arch.apicv_active)
7661
			kvm_x86_ops->sync_pir_to_irr(vcpu);
7662 7663
		if (ioapic_in_kernel(vcpu->kvm))
			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
7664
	}
7665 7666 7667 7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678

	if (is_guest_mode(vcpu))
		vcpu->arch.load_eoi_exitmap_pending = true;
	else
		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
}

static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
{
	u64 eoi_exit_bitmap[4];

	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
		return;

7679 7680 7681
	bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
		  vcpu_to_synic(vcpu)->vec_bitmap, 256);
	kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
7682 7683
}

7684 7685 7686
int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
		unsigned long start, unsigned long end,
		bool blockable)
7687 7688 7689 7690 7691 7692 7693 7694 7695 7696
{
	unsigned long apic_address;

	/*
	 * The physical address of apic access page is stored in the VMCS.
	 * Update it when it becomes invalid.
	 */
	apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
	if (start <= apic_address && apic_address < end)
		kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
7697 7698

	return 0;
7699 7700
}

7701 7702
void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
{
7703 7704
	struct page *page = NULL;

7705
	if (!lapic_in_kernel(vcpu))
7706 7707
		return;

7708 7709 7710
	if (!kvm_x86_ops->set_apic_access_page_addr)
		return;

7711
	page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
7712 7713
	if (is_error_page(page))
		return;
7714 7715 7716 7717 7718 7719 7720
	kvm_x86_ops->set_apic_access_page_addr(vcpu, page_to_phys(page));

	/*
	 * Do not pin apic access page in memory, the MMU notifier
	 * will call us again if it is migrated or swapped out.
	 */
	put_page(page);
7721 7722 7723
}
EXPORT_SYMBOL_GPL(kvm_vcpu_reload_apic_access_page);

7724 7725 7726 7727 7728 7729
void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
{
	smp_send_reschedule(vcpu->cpu);
}
EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);

7730
/*
7731
 * Returns 1 to let vcpu_run() continue the guest execution loop without
7732 7733 7734
 * exiting to the userspace.  Otherwise, the value will be returned to the
 * userspace.
 */
A
Avi Kivity 已提交
7735
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
7736 7737
{
	int r;
7738 7739 7740 7741
	bool req_int_win =
		dm_request_for_irq_injection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);

7742
	bool req_immediate_exit = false;
7743

R
Radim Krčmář 已提交
7744
	if (kvm_request_pending(vcpu)) {
7745 7746
		if (kvm_check_request(KVM_REQ_GET_VMCS12_PAGES, vcpu))
			kvm_x86_ops->get_vmcs12_pages(vcpu);
7747
		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
7748
			kvm_mmu_unload(vcpu);
7749
		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
M
Marcelo Tosatti 已提交
7750
			__kvm_migrate_timers(vcpu);
7751 7752
		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
			kvm_gen_update_masterclock(vcpu->kvm);
7753 7754
		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
			kvm_gen_kvmclock_update(vcpu);
Z
Zachary Amsden 已提交
7755 7756
		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
			r = kvm_guest_time_update(vcpu);
7757 7758 7759
			if (unlikely(r))
				goto out;
		}
7760
		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
7761
			kvm_mmu_sync_roots(vcpu);
7762 7763
		if (kvm_check_request(KVM_REQ_LOAD_CR3, vcpu))
			kvm_mmu_load_cr3(vcpu);
7764
		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
7765
			kvm_vcpu_flush_tlb(vcpu, true);
7766
		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
A
Avi Kivity 已提交
7767
			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
A
Avi Kivity 已提交
7768 7769 7770
			r = 0;
			goto out;
		}
7771
		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
A
Avi Kivity 已提交
7772
			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
7773
			vcpu->mmio_needed = 0;
J
Joerg Roedel 已提交
7774 7775 7776
			r = 0;
			goto out;
		}
7777 7778 7779 7780 7781 7782
		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
			/* Page is swapped out. Do synthetic halt */
			vcpu->arch.apf.halted = true;
			r = 1;
			goto out;
		}
G
Glauber Costa 已提交
7783 7784
		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
			record_steal_time(vcpu);
P
Paolo Bonzini 已提交
7785 7786
		if (kvm_check_request(KVM_REQ_SMI, vcpu))
			process_smi(vcpu);
A
Avi Kivity 已提交
7787 7788
		if (kvm_check_request(KVM_REQ_NMI, vcpu))
			process_nmi(vcpu);
7789
		if (kvm_check_request(KVM_REQ_PMU, vcpu))
7790
			kvm_pmu_handle_event(vcpu);
7791
		if (kvm_check_request(KVM_REQ_PMI, vcpu))
7792
			kvm_pmu_deliver_pmi(vcpu);
7793 7794 7795
		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
			if (test_bit(vcpu->arch.pending_ioapic_eoi,
7796
				     vcpu->arch.ioapic_handled_vectors)) {
7797 7798 7799 7800 7801 7802 7803
				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
				vcpu->run->eoi.vector =
						vcpu->arch.pending_ioapic_eoi;
				r = 0;
				goto out;
			}
		}
7804 7805
		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
			vcpu_scan_ioapic(vcpu);
7806 7807
		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
			vcpu_load_eoi_exitmap(vcpu);
7808 7809
		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
			kvm_vcpu_reload_apic_access_page(vcpu);
7810 7811 7812 7813 7814 7815
		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
			r = 0;
			goto out;
		}
7816 7817 7818 7819 7820 7821
		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
			r = 0;
			goto out;
		}
A
Andrey Smetanin 已提交
7822 7823 7824 7825 7826 7827
		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_HYPERV;
			vcpu->run->hyperv = vcpu->arch.hyperv.exit;
			r = 0;
			goto out;
		}
7828 7829 7830 7831 7832 7833

		/*
		 * KVM_REQ_HV_STIMER has to be processed after
		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
		 * depend on the guest clock being up-to-date
		 */
A
Andrey Smetanin 已提交
7834 7835
		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
			kvm_hv_process_stimers(vcpu);
7836
	}
A
Avi Kivity 已提交
7837

A
Avi Kivity 已提交
7838
	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
7839
		++vcpu->stat.req_event;
7840 7841 7842 7843 7844 7845
		kvm_apic_accept_events(vcpu);
		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
			r = 1;
			goto out;
		}

7846 7847
		if (inject_pending_event(vcpu, req_int_win) != 0)
			req_immediate_exit = true;
7848
		else {
7849
			/* Enable SMI/NMI/IRQ window open exits if needed.
7850
			 *
7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861
			 * SMIs have three cases:
			 * 1) They can be nested, and then there is nothing to
			 *    do here because RSM will cause a vmexit anyway.
			 * 2) There is an ISA-specific reason why SMI cannot be
			 *    injected, and the moment when this changes can be
			 *    intercepted.
			 * 3) Or the SMI can be pending because
			 *    inject_pending_event has completed the injection
			 *    of an IRQ or NMI from the previous vmexit, and
			 *    then we request an immediate exit to inject the
			 *    SMI.
7862 7863
			 */
			if (vcpu->arch.smi_pending && !is_smm(vcpu))
7864 7865
				if (!kvm_x86_ops->enable_smi_window(vcpu))
					req_immediate_exit = true;
7866 7867 7868 7869
			if (vcpu->arch.nmi_pending)
				kvm_x86_ops->enable_nmi_window(vcpu);
			if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
				kvm_x86_ops->enable_irq_window(vcpu);
7870
			WARN_ON(vcpu->arch.exception.pending);
7871
		}
A
Avi Kivity 已提交
7872 7873 7874 7875 7876 7877 7878

		if (kvm_lapic_enabled(vcpu)) {
			update_cr8_intercept(vcpu);
			kvm_lapic_sync_to_vapic(vcpu);
		}
	}

7879 7880
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r)) {
7881
		goto cancel_injection;
7882 7883
	}

7884 7885 7886
	preempt_disable();

	kvm_x86_ops->prepare_guest_switch(vcpu);
7887 7888 7889 7890 7891 7892 7893

	/*
	 * Disable IRQs before setting IN_GUEST_MODE.  Posted interrupt
	 * IPI are then delayed after guest entry, which ensures that they
	 * result in virtual interrupt delivery.
	 */
	local_irq_disable();
7894 7895
	vcpu->mode = IN_GUEST_MODE;

7896 7897
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);

7898
	/*
7899
	 * 1) We should set ->mode before checking ->requests.  Please see
7900
	 * the comment in kvm_vcpu_exiting_guest_mode().
7901
	 *
7902
	 * 2) For APICv, we should set ->mode before checking PID.ON. This
7903 7904 7905 7906 7907 7908
	 * pairs with the memory barrier implicit in pi_test_and_set_on
	 * (see vmx_deliver_posted_interrupt).
	 *
	 * 3) This also orders the write to mode from any reads to the page
	 * tables done while the VCPU is running.  Please see the comment
	 * in kvm_flush_remote_tlbs.
7909
	 */
7910
	smp_mb__after_srcu_read_unlock();
7911

7912 7913 7914 7915
	/*
	 * This handles the case where a posted interrupt was
	 * notified with kvm_vcpu_kick.
	 */
7916 7917
	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
		kvm_x86_ops->sync_pir_to_irr(vcpu);
7918

R
Radim Krčmář 已提交
7919
	if (vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu)
A
Avi Kivity 已提交
7920
	    || need_resched() || signal_pending(current)) {
7921
		vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
7922
		smp_wmb();
7923 7924
		local_irq_enable();
		preempt_enable();
7925
		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
7926
		r = 1;
7927
		goto cancel_injection;
7928 7929
	}

7930 7931
	if (req_immediate_exit) {
		kvm_make_request(KVM_REQ_EVENT, vcpu);
7932
		kvm_x86_ops->request_immediate_exit(vcpu);
7933
	}
7934

7935
	trace_kvm_entry(vcpu->vcpu_id);
7936
	guest_enter_irqoff();
7937

7938 7939 7940 7941
	fpregs_assert_state_consistent();
	if (test_thread_flag(TIF_NEED_FPU_LOAD))
		switch_fpu_return();

7942 7943 7944 7945 7946 7947
	if (unlikely(vcpu->arch.switch_db_regs)) {
		set_debugreg(0, 7);
		set_debugreg(vcpu->arch.eff_db[0], 0);
		set_debugreg(vcpu->arch.eff_db[1], 1);
		set_debugreg(vcpu->arch.eff_db[2], 2);
		set_debugreg(vcpu->arch.eff_db[3], 3);
7948
		set_debugreg(vcpu->arch.dr6, 6);
7949
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
7950
	}
7951

A
Avi Kivity 已提交
7952
	kvm_x86_ops->run(vcpu);
7953

7954 7955 7956 7957 7958 7959 7960 7961 7962
	/*
	 * Do this here before restoring debug registers on the host.  And
	 * since we do this before handling the vmexit, a DR access vmexit
	 * can (a) read the correct value of the debug registers, (b) set
	 * KVM_DEBUGREG_WONT_EXIT again.
	 */
	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
		kvm_x86_ops->sync_dirty_debug_regs(vcpu);
7963 7964 7965 7966
		kvm_update_dr0123(vcpu);
		kvm_update_dr6(vcpu);
		kvm_update_dr7(vcpu);
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
7967 7968
	}

7969 7970 7971 7972 7973 7974 7975
	/*
	 * If the guest has used debug registers, at least dr7
	 * will be disabled while returning to the host.
	 * If we don't have active breakpoints in the host, we don't
	 * care about the messed up debug address registers. But if
	 * we have some of them active, restore the old state.
	 */
7976
	if (hw_breakpoint_active())
7977
		hw_breakpoint_restore();
7978

7979
	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
7980

7981
	vcpu->mode = OUTSIDE_GUEST_MODE;
A
Avi Kivity 已提交
7982
	smp_wmb();
7983

7984
	kvm_before_interrupt(vcpu);
7985
	kvm_x86_ops->handle_external_intr(vcpu);
7986
	kvm_after_interrupt(vcpu);
7987 7988 7989

	++vcpu->stat.exits;

P
Paolo Bonzini 已提交
7990
	guest_exit_irqoff();
7991 7992 7993 7994 7995 7996 7997
	if (lapic_in_kernel(vcpu)) {
		s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;
		if (delta != S64_MIN) {
			trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta);
			vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;
		}
	}
7998

P
Paolo Bonzini 已提交
7999
	local_irq_enable();
8000 8001
	preempt_enable();

8002
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8003

8004 8005 8006 8007
	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
8008 8009
		unsigned long rip = kvm_rip_read(vcpu);
		profile_hit(KVM_PROFILING, (void *)rip);
8010 8011
	}

8012 8013
	if (unlikely(vcpu->arch.tsc_always_catchup))
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
8014

8015 8016
	if (vcpu->arch.apic_attention)
		kvm_lapic_sync_from_vapic(vcpu);
A
Avi Kivity 已提交
8017

8018
	vcpu->arch.gpa_available = false;
A
Avi Kivity 已提交
8019
	r = kvm_x86_ops->handle_exit(vcpu);
8020 8021 8022 8023
	return r;

cancel_injection:
	kvm_x86_ops->cancel_injection(vcpu);
8024 8025
	if (unlikely(vcpu->arch.apic_attention))
		kvm_lapic_sync_from_vapic(vcpu);
8026 8027 8028
out:
	return r;
}
8029

8030 8031
static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
{
8032 8033
	if (!kvm_arch_vcpu_runnable(vcpu) &&
	    (!kvm_x86_ops->pre_block || kvm_x86_ops->pre_block(vcpu) == 0)) {
8034 8035 8036
		srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
		kvm_vcpu_block(vcpu);
		vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8037 8038 8039 8040

		if (kvm_x86_ops->post_block)
			kvm_x86_ops->post_block(vcpu);

8041 8042 8043
		if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
			return 1;
	}
8044 8045 8046 8047 8048 8049 8050

	kvm_apic_accept_events(vcpu);
	switch(vcpu->arch.mp_state) {
	case KVM_MP_STATE_HALTED:
		vcpu->arch.pv.pv_unhalted = false;
		vcpu->arch.mp_state =
			KVM_MP_STATE_RUNNABLE;
8051
		/* fall through */
8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062
	case KVM_MP_STATE_RUNNABLE:
		vcpu->arch.apf.halted = false;
		break;
	case KVM_MP_STATE_INIT_RECEIVED:
		break;
	default:
		return -EINTR;
		break;
	}
	return 1;
}
8063

8064 8065
static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
{
8066 8067 8068
	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events)
		kvm_x86_ops->check_nested_events(vcpu, false);

8069 8070 8071 8072
	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		!vcpu->arch.apf.halted);
}

8073
static int vcpu_run(struct kvm_vcpu *vcpu)
8074 8075
{
	int r;
8076
	struct kvm *kvm = vcpu->kvm;
8077

8078
	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
P
Paolo Bonzini 已提交
8079
	vcpu->arch.l1tf_flush_l1d = true;
8080

8081
	for (;;) {
8082
		if (kvm_vcpu_running(vcpu)) {
A
Avi Kivity 已提交
8083
			r = vcpu_enter_guest(vcpu);
8084
		} else {
8085
			r = vcpu_block(kvm, vcpu);
8086 8087
		}

8088 8089 8090
		if (r <= 0)
			break;

8091
		kvm_clear_request(KVM_REQ_PENDING_TIMER, vcpu);
8092 8093 8094
		if (kvm_cpu_has_pending_timer(vcpu))
			kvm_inject_pending_timer_irqs(vcpu);

8095 8096
		if (dm_request_for_irq_injection(vcpu) &&
			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
8097 8098
			r = 0;
			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
8099
			++vcpu->stat.request_irq_exits;
8100
			break;
8101
		}
8102 8103 8104

		kvm_check_async_pf_completion(vcpu);

8105 8106
		if (signal_pending(current)) {
			r = -EINTR;
A
Avi Kivity 已提交
8107
			vcpu->run->exit_reason = KVM_EXIT_INTR;
8108
			++vcpu->stat.signal_exits;
8109
			break;
8110 8111
		}
		if (need_resched()) {
8112
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8113
			cond_resched();
8114
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8115
		}
8116 8117
	}

8118
	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8119 8120 8121 8122

	return r;
}

8123 8124 8125 8126
static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
{
	int r;
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8127
	r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
	if (r != EMULATE_DONE)
		return 0;
	return 1;
}

static int complete_emulated_pio(struct kvm_vcpu *vcpu)
{
	BUG_ON(!vcpu->arch.pio.count);

	return complete_emulated_io(vcpu);
}

A
Avi Kivity 已提交
8141 8142 8143 8144 8145
/*
 * Implements the following, as a state machine:
 *
 * read:
 *   for each fragment
8146 8147 8148 8149
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       exit
 *       copy data
A
Avi Kivity 已提交
8150 8151 8152 8153
 *   execute insn
 *
 * write:
 *   for each fragment
8154 8155 8156 8157
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       copy data
 *       exit
A
Avi Kivity 已提交
8158
 */
8159
static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
8160 8161
{
	struct kvm_run *run = vcpu->run;
A
Avi Kivity 已提交
8162
	struct kvm_mmio_fragment *frag;
8163
	unsigned len;
8164

8165
	BUG_ON(!vcpu->mmio_needed);
8166

8167
	/* Complete previous fragment */
8168 8169
	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
	len = min(8u, frag->len);
8170
	if (!vcpu->mmio_is_write)
8171 8172 8173 8174 8175 8176 8177 8178 8179 8180 8181 8182 8183
		memcpy(frag->data, run->mmio.data, len);

	if (frag->len <= 8) {
		/* Switch to the next fragment. */
		frag++;
		vcpu->mmio_cur_fragment++;
	} else {
		/* Go forward to the next mmio piece. */
		frag->data += len;
		frag->gpa += len;
		frag->len -= len;
	}

8184
	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
8185
		vcpu->mmio_needed = 0;
8186 8187

		/* FIXME: return into emulator if single-stepping.  */
A
Avi Kivity 已提交
8188
		if (vcpu->mmio_is_write)
8189 8190 8191 8192
			return 1;
		vcpu->mmio_read_completed = 1;
		return complete_emulated_io(vcpu);
	}
8193

8194 8195 8196
	run->exit_reason = KVM_EXIT_MMIO;
	run->mmio.phys_addr = frag->gpa;
	if (vcpu->mmio_is_write)
8197 8198
		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
	run->mmio.len = min(8u, frag->len);
8199 8200 8201
	run->mmio.is_write = vcpu->mmio_is_write;
	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
	return 0;
8202 8203
}

8204 8205 8206
/* Swap (qemu) user FPU context for the guest FPU context. */
static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
8207 8208
	fpregs_lock();

8209
	copy_fpregs_to_fpstate(&current->thread.fpu);
8210
	/* PKRU is separately restored in kvm_x86_ops->run.  */
8211
	__copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,
8212
				~XFEATURE_MASK_PKRU);
8213 8214 8215 8216

	fpregs_mark_activate();
	fpregs_unlock();

8217 8218 8219 8220 8221 8222
	trace_kvm_fpu(1);
}

/* When vcpu_run ends, restore user space FPU context. */
static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
8223 8224
	fpregs_lock();

8225
	copy_fpregs_to_fpstate(vcpu->arch.guest_fpu);
8226
	copy_kernel_to_fpregs(&current->thread.fpu.state);
8227 8228 8229 8230

	fpregs_mark_activate();
	fpregs_unlock();

8231 8232 8233 8234
	++vcpu->stat.fpu_reload;
	trace_kvm_fpu(0);
}

8235 8236 8237 8238
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
{
	int r;

8239
	vcpu_load(vcpu);
8240
	kvm_sigset_activate(vcpu);
8241 8242
	kvm_load_guest_fpu(vcpu);

8243
	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
8244 8245 8246 8247
		if (kvm_run->immediate_exit) {
			r = -EINTR;
			goto out;
		}
8248
		kvm_vcpu_block(vcpu);
8249
		kvm_apic_accept_events(vcpu);
8250
		kvm_clear_request(KVM_REQ_UNHALT, vcpu);
8251
		r = -EAGAIN;
8252 8253 8254 8255 8256
		if (signal_pending(current)) {
			r = -EINTR;
			vcpu->run->exit_reason = KVM_EXIT_INTR;
			++vcpu->stat.signal_exits;
		}
8257
		goto out;
8258 8259
	}

K
Ken Hofsass 已提交
8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270
	if (vcpu->run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) {
		r = -EINVAL;
		goto out;
	}

	if (vcpu->run->kvm_dirty_regs) {
		r = sync_regs(vcpu);
		if (r != 0)
			goto out;
	}

8271
	/* re-sync apic's tpr */
8272
	if (!lapic_in_kernel(vcpu)) {
A
Andre Przywara 已提交
8273 8274 8275 8276 8277
		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
			r = -EINVAL;
			goto out;
		}
	}
8278

8279 8280 8281 8282 8283
	if (unlikely(vcpu->arch.complete_userspace_io)) {
		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
		vcpu->arch.complete_userspace_io = NULL;
		r = cui(vcpu);
		if (r <= 0)
8284
			goto out;
8285 8286
	} else
		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
8287

8288 8289 8290 8291
	if (kvm_run->immediate_exit)
		r = -EINTR;
	else
		r = vcpu_run(vcpu);
8292 8293

out:
8294
	kvm_put_guest_fpu(vcpu);
K
Ken Hofsass 已提交
8295 8296
	if (vcpu->run->kvm_valid_regs)
		store_regs(vcpu);
8297
	post_kvm_run_save(vcpu);
8298
	kvm_sigset_deactivate(vcpu);
8299

8300
	vcpu_put(vcpu);
8301 8302 8303
	return r;
}

K
Ken Hofsass 已提交
8304
static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8305
{
8306 8307 8308 8309
	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
		/*
		 * We are here if userspace calls get_regs() in the middle of
		 * instruction emulation. Registers state needs to be copied
G
Guo Chao 已提交
8310
		 * back from emulation context to vcpu. Userspace shouldn't do
8311 8312 8313
		 * that usually, but some bad designed PV devices (vmware
		 * backdoor interface) need this to work
		 */
8314
		emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
8315 8316
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
	}
8317 8318 8319 8320 8321 8322
	regs->rax = kvm_rax_read(vcpu);
	regs->rbx = kvm_rbx_read(vcpu);
	regs->rcx = kvm_rcx_read(vcpu);
	regs->rdx = kvm_rdx_read(vcpu);
	regs->rsi = kvm_rsi_read(vcpu);
	regs->rdi = kvm_rdi_read(vcpu);
8323
	regs->rsp = kvm_rsp_read(vcpu);
8324
	regs->rbp = kvm_rbp_read(vcpu);
8325
#ifdef CONFIG_X86_64
8326 8327 8328 8329 8330 8331 8332 8333
	regs->r8 = kvm_r8_read(vcpu);
	regs->r9 = kvm_r9_read(vcpu);
	regs->r10 = kvm_r10_read(vcpu);
	regs->r11 = kvm_r11_read(vcpu);
	regs->r12 = kvm_r12_read(vcpu);
	regs->r13 = kvm_r13_read(vcpu);
	regs->r14 = kvm_r14_read(vcpu);
	regs->r15 = kvm_r15_read(vcpu);
8334 8335
#endif

8336
	regs->rip = kvm_rip_read(vcpu);
8337
	regs->rflags = kvm_get_rflags(vcpu);
K
Ken Hofsass 已提交
8338
}
8339

K
Ken Hofsass 已提交
8340 8341 8342 8343
int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__get_regs(vcpu, regs);
8344
	vcpu_put(vcpu);
8345 8346 8347
	return 0;
}

K
Ken Hofsass 已提交
8348
static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8349
{
8350 8351 8352
	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;

8353 8354 8355 8356 8357 8358
	kvm_rax_write(vcpu, regs->rax);
	kvm_rbx_write(vcpu, regs->rbx);
	kvm_rcx_write(vcpu, regs->rcx);
	kvm_rdx_write(vcpu, regs->rdx);
	kvm_rsi_write(vcpu, regs->rsi);
	kvm_rdi_write(vcpu, regs->rdi);
8359
	kvm_rsp_write(vcpu, regs->rsp);
8360
	kvm_rbp_write(vcpu, regs->rbp);
8361
#ifdef CONFIG_X86_64
8362 8363 8364 8365 8366 8367 8368 8369
	kvm_r8_write(vcpu, regs->r8);
	kvm_r9_write(vcpu, regs->r9);
	kvm_r10_write(vcpu, regs->r10);
	kvm_r11_write(vcpu, regs->r11);
	kvm_r12_write(vcpu, regs->r12);
	kvm_r13_write(vcpu, regs->r13);
	kvm_r14_write(vcpu, regs->r14);
	kvm_r15_write(vcpu, regs->r15);
8370 8371
#endif

8372
	kvm_rip_write(vcpu, regs->rip);
8373
	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
8374

8375 8376
	vcpu->arch.exception.pending = false;

8377
	kvm_make_request(KVM_REQ_EVENT, vcpu);
K
Ken Hofsass 已提交
8378
}
8379

K
Ken Hofsass 已提交
8380 8381 8382 8383
int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__set_regs(vcpu, regs);
8384
	vcpu_put(vcpu);
8385 8386 8387 8388 8389 8390 8391
	return 0;
}

void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

8392
	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
8393 8394 8395 8396 8397
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

K
Ken Hofsass 已提交
8398
static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8399
{
8400
	struct desc_ptr dt;
8401

8402 8403 8404 8405 8406 8407
	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8408

8409 8410
	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
8411 8412

	kvm_x86_ops->get_idt(vcpu, &dt);
8413 8414
	sregs->idt.limit = dt.size;
	sregs->idt.base = dt.address;
8415
	kvm_x86_ops->get_gdt(vcpu, &dt);
8416 8417
	sregs->gdt.limit = dt.size;
	sregs->gdt.base = dt.address;
8418

8419
	sregs->cr0 = kvm_read_cr0(vcpu);
8420
	sregs->cr2 = vcpu->arch.cr2;
8421
	sregs->cr3 = kvm_read_cr3(vcpu);
8422
	sregs->cr4 = kvm_read_cr4(vcpu);
8423
	sregs->cr8 = kvm_get_cr8(vcpu);
8424
	sregs->efer = vcpu->arch.efer;
8425 8426
	sregs->apic_base = kvm_get_apic_base(vcpu);

8427
	memset(sregs->interrupt_bitmap, 0, sizeof(sregs->interrupt_bitmap));
8428

8429
	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
8430 8431
		set_bit(vcpu->arch.interrupt.nr,
			(unsigned long *)sregs->interrupt_bitmap);
K
Ken Hofsass 已提交
8432
}
8433

K
Ken Hofsass 已提交
8434 8435 8436 8437 8438
int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	vcpu_load(vcpu);
	__get_sregs(vcpu, sregs);
8439
	vcpu_put(vcpu);
8440 8441 8442
	return 0;
}

8443 8444 8445
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
8446 8447
	vcpu_load(vcpu);

8448
	kvm_apic_accept_events(vcpu);
8449 8450 8451 8452 8453 8454
	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
					vcpu->arch.pv.pv_unhalted)
		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
	else
		mp_state->mp_state = vcpu->arch.mp_state;

8455
	vcpu_put(vcpu);
8456 8457 8458 8459 8460 8461
	return 0;
}

int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
8462 8463 8464 8465
	int ret = -EINVAL;

	vcpu_load(vcpu);

8466
	if (!lapic_in_kernel(vcpu) &&
8467
	    mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
8468
		goto out;
8469

8470 8471 8472 8473
	/* INITs are latched while in SMM */
	if ((is_smm(vcpu) || vcpu->arch.smi_pending) &&
	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
8474
		goto out;
8475

8476 8477 8478 8479 8480
	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
	} else
		vcpu->arch.mp_state = mp_state->mp_state;
8481
	kvm_make_request(KVM_REQ_EVENT, vcpu);
8482 8483 8484 8485 8486

	ret = 0;
out:
	vcpu_put(vcpu);
	return ret;
8487 8488
}

8489 8490
int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
		    int reason, bool has_error_code, u32 error_code)
8491
{
8492
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
8493
	int ret;
8494

8495
	init_emulate_ctxt(vcpu);
8496

8497
	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
8498
				   has_error_code, error_code);
8499 8500

	if (ret)
8501
		return EMULATE_FAIL;
8502

8503 8504
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
8505
	kvm_make_request(KVM_REQ_EVENT, vcpu);
8506
	return EMULATE_DONE;
8507 8508 8509
}
EXPORT_SYMBOL_GPL(kvm_task_switch);

P
Peng Hao 已提交
8510
static int kvm_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8511
{
8512 8513 8514 8515
	if (!guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
			(sregs->cr4 & X86_CR4_OSXSAVE))
		return  -EINVAL;

8516
	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
8517 8518 8519 8520 8521
		/*
		 * When EFER.LME and CR0.PG are set, the processor is in
		 * 64-bit mode (though maybe in a 32-bit code segment).
		 * CR4.PAE and EFER.LMA must be set.
		 */
8522
		if (!(sregs->cr4 & X86_CR4_PAE)
8523 8524 8525 8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536
		    || !(sregs->efer & EFER_LMA))
			return -EINVAL;
	} else {
		/*
		 * Not in 64-bit mode: EFER.LMA is clear and the code
		 * segment cannot be 64-bit.
		 */
		if (sregs->efer & EFER_LMA || sregs->cs.l)
			return -EINVAL;
	}

	return 0;
}

K
Ken Hofsass 已提交
8537
static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8538
{
8539
	struct msr_data apic_base_msr;
8540
	int mmu_reset_needed = 0;
8541
	int cpuid_update_needed = 0;
8542
	int pending_vec, max_bits, idx;
8543
	struct desc_ptr dt;
8544 8545
	int ret = -EINVAL;

8546
	if (kvm_valid_sregs(vcpu, sregs))
8547
		goto out;
8548

8549 8550 8551
	apic_base_msr.data = sregs->apic_base;
	apic_base_msr.host_initiated = true;
	if (kvm_set_apic_base(vcpu, &apic_base_msr))
8552
		goto out;
8553

8554 8555
	dt.size = sregs->idt.limit;
	dt.address = sregs->idt.base;
8556
	kvm_x86_ops->set_idt(vcpu, &dt);
8557 8558
	dt.size = sregs->gdt.limit;
	dt.address = sregs->gdt.base;
8559 8560
	kvm_x86_ops->set_gdt(vcpu, &dt);

8561
	vcpu->arch.cr2 = sregs->cr2;
8562
	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
8563
	vcpu->arch.cr3 = sregs->cr3;
8564
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
8565

8566
	kvm_set_cr8(vcpu, sregs->cr8);
8567

8568
	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
8569 8570
	kvm_x86_ops->set_efer(vcpu, sregs->efer);

8571
	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
8572
	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
8573
	vcpu->arch.cr0 = sregs->cr0;
8574

8575
	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
8576 8577
	cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) &
				(X86_CR4_OSXSAVE | X86_CR4_PKE));
8578
	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
8579
	if (cpuid_update_needed)
A
Avi Kivity 已提交
8580
		kvm_update_cpuid(vcpu);
8581 8582

	idx = srcu_read_lock(&vcpu->kvm->srcu);
8583
	if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu)) {
8584
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
8585 8586
		mmu_reset_needed = 1;
	}
8587
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
8588 8589 8590 8591

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

8592
	max_bits = KVM_NR_INTERRUPTS;
G
Gleb Natapov 已提交
8593 8594 8595
	pending_vec = find_first_bit(
		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
	if (pending_vec < max_bits) {
8596
		kvm_queue_interrupt(vcpu, pending_vec, false);
G
Gleb Natapov 已提交
8597
		pr_debug("Set back pending irq %d\n", pending_vec);
8598 8599
	}

8600 8601 8602 8603 8604 8605
	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8606

8607 8608
	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
8609

8610 8611
	update_cr8_intercept(vcpu);

M
Marcelo Tosatti 已提交
8612
	/* Older userspace won't unhalt the vcpu on reset. */
8613
	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
M
Marcelo Tosatti 已提交
8614
	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
8615
	    !is_protmode(vcpu))
M
Marcelo Tosatti 已提交
8616 8617
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;

8618 8619
	kvm_make_request(KVM_REQ_EVENT, vcpu);

8620 8621
	ret = 0;
out:
K
Ken Hofsass 已提交
8622 8623 8624 8625 8626 8627 8628 8629 8630 8631
	return ret;
}

int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	int ret;

	vcpu_load(vcpu);
	ret = __set_sregs(vcpu, sregs);
8632 8633
	vcpu_put(vcpu);
	return ret;
8634 8635
}

J
Jan Kiszka 已提交
8636 8637
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg)
8638
{
8639
	unsigned long rflags;
8640
	int i, r;
8641

8642 8643
	vcpu_load(vcpu);

8644 8645 8646
	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
		r = -EBUSY;
		if (vcpu->arch.exception.pending)
8647
			goto out;
8648 8649 8650 8651 8652 8653
		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
			kvm_queue_exception(vcpu, DB_VECTOR);
		else
			kvm_queue_exception(vcpu, BP_VECTOR);
	}

8654 8655 8656 8657 8658
	/*
	 * Read rflags as long as potentially injected trace flags are still
	 * filtered out.
	 */
	rflags = kvm_get_rflags(vcpu);
8659 8660 8661 8662 8663 8664

	vcpu->guest_debug = dbg->control;
	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
		vcpu->guest_debug = 0;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
8665 8666
		for (i = 0; i < KVM_NR_DB_REGS; ++i)
			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
8667
		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
8668 8669 8670 8671
	} else {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
	}
8672
	kvm_update_dr7(vcpu);
8673

J
Jan Kiszka 已提交
8674 8675 8676
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
			get_segment_base(vcpu, VCPU_SREG_CS);
8677

8678 8679 8680 8681 8682
	/*
	 * Trigger an rflags update that will inject or remove the trace
	 * flags.
	 */
	kvm_set_rflags(vcpu, rflags);
8683

8684
	kvm_x86_ops->update_bp_intercept(vcpu);
8685

8686
	r = 0;
J
Jan Kiszka 已提交
8687

8688
out:
8689
	vcpu_put(vcpu);
8690 8691 8692
	return r;
}

8693 8694 8695 8696 8697 8698 8699 8700
/*
 * Translate a guest virtual address to a guest physical address.
 */
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;
8701
	int idx;
8702

8703 8704
	vcpu_load(vcpu);

8705
	idx = srcu_read_lock(&vcpu->kvm->srcu);
8706
	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
8707
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
8708 8709 8710 8711 8712
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;

8713
	vcpu_put(vcpu);
8714 8715 8716
	return 0;
}

8717 8718
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
8719
	struct fxregs_state *fxsave;
8720

8721
	vcpu_load(vcpu);
8722

8723
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
8724 8725 8726 8727 8728 8729 8730
	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
8731
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
8732

8733
	vcpu_put(vcpu);
8734 8735 8736 8737 8738
	return 0;
}

int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
8739 8740 8741 8742
	struct fxregs_state *fxsave;

	vcpu_load(vcpu);

8743
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
8744 8745 8746 8747 8748 8749 8750 8751

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
8752
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
8753

8754
	vcpu_put(vcpu);
8755 8756 8757
	return 0;
}

K
Ken Hofsass 已提交
8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781 8782 8783 8784 8785 8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796
static void store_regs(struct kvm_vcpu *vcpu)
{
	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
		__get_regs(vcpu, &vcpu->run->s.regs.regs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
		kvm_vcpu_ioctl_x86_get_vcpu_events(
				vcpu, &vcpu->run->s.regs.events);
}

static int sync_regs(struct kvm_vcpu *vcpu)
{
	if (vcpu->run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)
		return -EINVAL;

	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
		__set_regs(vcpu, &vcpu->run->s.regs.regs);
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
		if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
		if (kvm_vcpu_ioctl_x86_set_vcpu_events(
				vcpu, &vcpu->run->s.regs.events))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
	}

	return 0;
}

I
Ingo Molnar 已提交
8797
static void fx_init(struct kvm_vcpu *vcpu)
8798
{
8799
	fpstate_init(&vcpu->arch.guest_fpu->state);
8800
	if (boot_cpu_has(X86_FEATURE_XSAVES))
8801
		vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =
8802
			host_xcr0 | XSTATE_COMPACTION_ENABLED;
8803

8804 8805 8806
	/*
	 * Ensure guest xcr0 is valid for loading
	 */
D
Dave Hansen 已提交
8807
	vcpu->arch.xcr0 = XFEATURE_MASK_FP;
8808

8809
	vcpu->arch.cr0 |= X86_CR0_ET;
8810 8811
}

8812 8813
void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
{
8814 8815
	void *wbinvd_dirty_mask = vcpu->arch.wbinvd_dirty_mask;

8816
	kvmclock_reset(vcpu);
8817

8818
	kvm_x86_ops->vcpu_free(vcpu);
8819
	free_cpumask_var(wbinvd_dirty_mask);
8820 8821 8822 8823 8824
}

struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
						unsigned int id)
{
8825 8826
	struct kvm_vcpu *vcpu;

8827
	if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
Z
Zachary Amsden 已提交
8828 8829 8830
		printk_once(KERN_WARNING
		"kvm: SMP vm created on host with unstable TSC; "
		"guest TSC will not be reliable\n");
8831 8832 8833 8834

	vcpu = kvm_x86_ops->vcpu_create(kvm, id);

	return vcpu;
8835
}
8836

8837 8838
int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
{
8839
	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
8840
	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
X
Xiao Guangrong 已提交
8841
	kvm_vcpu_mtrr_init(vcpu);
8842
	vcpu_load(vcpu);
8843
	kvm_vcpu_reset(vcpu, false);
8844
	kvm_init_mmu(vcpu, false);
8845
	vcpu_put(vcpu);
8846
	return 0;
8847 8848
}

8849
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
8850
{
8851
	struct msr_data msr;
8852
	struct kvm *kvm = vcpu->kvm;
8853

8854 8855
	kvm_hv_vcpu_postcreate(vcpu);

8856
	if (mutex_lock_killable(&vcpu->mutex))
8857
		return;
8858
	vcpu_load(vcpu);
8859 8860 8861 8862
	msr.data = 0x0;
	msr.index = MSR_IA32_TSC;
	msr.host_initiated = true;
	kvm_write_tsc(vcpu, &msr);
8863
	vcpu_put(vcpu);
8864
	mutex_unlock(&vcpu->mutex);
8865

8866 8867 8868
	if (!kvmclock_periodic_sync)
		return;

8869 8870
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
8871 8872
}

8873
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
8874
{
8875 8876
	vcpu->arch.apf.msr_val = 0;

8877
	vcpu_load(vcpu);
8878 8879 8880 8881 8882 8883
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);

	kvm_x86_ops->vcpu_free(vcpu);
}

8884
void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
8885
{
8886 8887
	kvm_lapic_reset(vcpu, init_event);

8888 8889
	vcpu->arch.hflags = 0;

8890
	vcpu->arch.smi_pending = 0;
8891
	vcpu->arch.smi_count = 0;
A
Avi Kivity 已提交
8892 8893
	atomic_set(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = 0;
8894
	vcpu->arch.nmi_injected = false;
8895 8896
	kvm_clear_interrupt_queue(vcpu);
	kvm_clear_exception_queue(vcpu);
8897
	vcpu->arch.exception.pending = false;
8898

8899
	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
8900
	kvm_update_dr0123(vcpu);
8901
	vcpu->arch.dr6 = DR6_INIT;
J
Jan Kiszka 已提交
8902
	kvm_update_dr6(vcpu);
8903
	vcpu->arch.dr7 = DR7_FIXED_1;
8904
	kvm_update_dr7(vcpu);
8905

N
Nadav Amit 已提交
8906 8907
	vcpu->arch.cr2 = 0;

8908
	kvm_make_request(KVM_REQ_EVENT, vcpu);
8909
	vcpu->arch.apf.msr_val = 0;
G
Glauber Costa 已提交
8910
	vcpu->arch.st.msr_val = 0;
8911

8912 8913
	kvmclock_reset(vcpu);

8914 8915 8916
	kvm_clear_async_pf_completion_queue(vcpu);
	kvm_async_pf_hash_reset(vcpu);
	vcpu->arch.apf.halted = false;
8917

8918 8919 8920 8921 8922 8923 8924
	if (kvm_mpx_supported()) {
		void *mpx_state_buffer;

		/*
		 * To avoid have the INIT path from kvm_apic_has_events() that be
		 * called with loaded FPU and does not let userspace fix the state.
		 */
8925 8926
		if (init_event)
			kvm_put_guest_fpu(vcpu);
8927
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
8928
					XFEATURE_BNDREGS);
8929 8930
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndreg_state));
8931
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
8932
					XFEATURE_BNDCSR);
8933 8934
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndcsr));
8935 8936
		if (init_event)
			kvm_load_guest_fpu(vcpu);
8937 8938
	}

P
Paolo Bonzini 已提交
8939
	if (!init_event) {
8940
		kvm_pmu_reset(vcpu);
P
Paolo Bonzini 已提交
8941
		vcpu->arch.smbase = 0x30000;
K
Kyle Huey 已提交
8942 8943

		vcpu->arch.msr_misc_features_enables = 0;
8944 8945

		vcpu->arch.xcr0 = XFEATURE_MASK_FP;
P
Paolo Bonzini 已提交
8946
	}
8947

8948 8949 8950 8951
	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;

8952 8953
	vcpu->arch.ia32_xss = 0;

8954
	kvm_x86_ops->vcpu_reset(vcpu, init_event);
8955 8956
}

8957
void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
8958 8959 8960 8961 8962 8963 8964 8965
{
	struct kvm_segment cs;

	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs.selector = vector << 8;
	cs.base = vector << 12;
	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_rip_write(vcpu, 0);
8966 8967
}

8968
int kvm_arch_hardware_enable(void)
8969
{
8970 8971 8972
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;
8973 8974 8975 8976
	int ret;
	u64 local_tsc;
	u64 max_tsc = 0;
	bool stable, backwards_tsc = false;
A
Avi Kivity 已提交
8977 8978

	kvm_shared_msr_cpu_online();
8979
	ret = kvm_x86_ops->hardware_enable();
8980 8981 8982
	if (ret != 0)
		return ret;

8983
	local_tsc = rdtsc();
8984
	stable = !kvm_check_tsc_unstable();
8985 8986 8987
	list_for_each_entry(kvm, &vm_list, vm_list) {
		kvm_for_each_vcpu(i, vcpu, kvm) {
			if (!stable && vcpu->cpu == smp_processor_id())
8988
				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
8989 8990 8991 8992 8993 8994 8995 8996 8997 8998 8999 9000 9001 9002 9003 9004
			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
				backwards_tsc = true;
				if (vcpu->arch.last_host_tsc > max_tsc)
					max_tsc = vcpu->arch.last_host_tsc;
			}
		}
	}

	/*
	 * Sometimes, even reliable TSCs go backwards.  This happens on
	 * platforms that reset TSC during suspend or hibernate actions, but
	 * maintain synchronization.  We must compensate.  Fortunately, we can
	 * detect that condition here, which happens early in CPU bringup,
	 * before any KVM threads can be running.  Unfortunately, we can't
	 * bring the TSCs fully up to date with real time, as we aren't yet far
	 * enough into CPU bringup that we know how much real time has actually
9005
	 * elapsed; our helper function, ktime_get_boot_ns() will be using boot
9006 9007 9008 9009 9010 9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021 9022 9023 9024 9025 9026 9027 9028 9029
	 * variables that haven't been updated yet.
	 *
	 * So we simply find the maximum observed TSC above, then record the
	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
	 * the adjustment will be applied.  Note that we accumulate
	 * adjustments, in case multiple suspend cycles happen before some VCPU
	 * gets a chance to run again.  In the event that no KVM threads get a
	 * chance to run, we will miss the entire elapsed period, as we'll have
	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
	 * loose cycle time.  This isn't too big a deal, since the loss will be
	 * uniform across all VCPUs (not to mention the scenario is extremely
	 * unlikely). It is possible that a second hibernate recovery happens
	 * much faster than a first, causing the observed TSC here to be
	 * smaller; this would require additional padding adjustment, which is
	 * why we set last_host_tsc to the local tsc observed here.
	 *
	 * N.B. - this code below runs only on platforms with reliable TSC,
	 * as that is the only way backwards_tsc is set above.  Also note
	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
	 * have the same delta_cyc adjustment applied if backwards_tsc
	 * is detected.  Note further, this adjustment is only done once,
	 * as we reset last_host_tsc on all VCPUs to stop this from being
	 * called multiple times (one for each physical CPU bringup).
	 *
G
Guo Chao 已提交
9030
	 * Platforms with unreliable TSCs don't have to deal with this, they
9031 9032 9033 9034 9035 9036 9037
	 * will be compensated by the logic in vcpu_load, which sets the TSC to
	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
	 * guarantee that they stay in perfect synchronization.
	 */
	if (backwards_tsc) {
		u64 delta_cyc = max_tsc - local_tsc;
		list_for_each_entry(kvm, &vm_list, vm_list) {
9038
			kvm->arch.backwards_tsc_observed = true;
9039 9040 9041
			kvm_for_each_vcpu(i, vcpu, kvm) {
				vcpu->arch.tsc_offset_adjustment += delta_cyc;
				vcpu->arch.last_host_tsc = local_tsc;
9042
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
9043 9044 9045 9046 9047 9048 9049 9050 9051 9052 9053 9054 9055 9056
			}

			/*
			 * We have to disable TSC offset matching.. if you were
			 * booting a VM while issuing an S4 host suspend....
			 * you may have some problem.  Solving this issue is
			 * left as an exercise to the reader.
			 */
			kvm->arch.last_tsc_nsec = 0;
			kvm->arch.last_tsc_write = 0;
		}

	}
	return 0;
9057 9058
}

9059
void kvm_arch_hardware_disable(void)
9060
{
9061 9062
	kvm_x86_ops->hardware_disable();
	drop_user_return_notifiers();
9063 9064 9065 9066
}

int kvm_arch_hardware_setup(void)
{
9067 9068 9069 9070 9071 9072
	int r;

	r = kvm_x86_ops->hardware_setup();
	if (r != 0)
		return r;

9073 9074 9075 9076
	if (kvm_has_tsc_control) {
		/*
		 * Make sure the user can only configure tsc_khz values that
		 * fit into a signed integer.
9077
		 * A min value is not calculated because it will always
9078 9079 9080 9081 9082 9083
		 * be 1 on all machines.
		 */
		u64 max = min(0x7fffffffULL,
			      __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz));
		kvm_max_guest_tsc_khz = max;

9084
		kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits;
9085
	}
9086

9087 9088
	kvm_init_msr_list();
	return 0;
9089 9090 9091 9092 9093 9094 9095
}

void kvm_arch_hardware_unsetup(void)
{
	kvm_x86_ops->hardware_unsetup();
}

9096
int kvm_arch_check_processor_compat(void)
9097
{
9098
	return kvm_x86_ops->check_processor_compatibility();
9099 9100 9101 9102 9103 9104 9105 9106 9107 9108 9109
}

bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
{
	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);

bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
9110 9111
}

9112
struct static_key kvm_no_apic_vcpu __read_mostly;
9113
EXPORT_SYMBOL_GPL(kvm_no_apic_vcpu);
9114

9115 9116 9117 9118 9119
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
	struct page *page;
	int r;

9120
	vcpu->arch.emulate_ctxt.ops = &emulate_ops;
9121
	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
9122
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
9123
	else
9124
		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
9125 9126 9127 9128 9129 9130

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
9131
	vcpu->arch.pio_data = page_address(page);
9132

9133
	kvm_set_tsc_khz(vcpu, max_tsc_khz);
Z
Zachary Amsden 已提交
9134

9135 9136 9137 9138
	r = kvm_mmu_create(vcpu);
	if (r < 0)
		goto fail_free_pio_data;

9139
	if (irqchip_in_kernel(vcpu->kvm)) {
9140
		vcpu->arch.apicv_active = kvm_x86_ops->get_enable_apicv(vcpu);
9141
		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
9142 9143
		if (r < 0)
			goto fail_mmu_destroy;
9144 9145
	} else
		static_key_slow_inc(&kvm_no_apic_vcpu);
9146

H
Huang Ying 已提交
9147
	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
9148
				       GFP_KERNEL_ACCOUNT);
H
Huang Ying 已提交
9149 9150
	if (!vcpu->arch.mce_banks) {
		r = -ENOMEM;
9151
		goto fail_free_lapic;
H
Huang Ying 已提交
9152 9153 9154
	}
	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;

9155 9156
	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
				GFP_KERNEL_ACCOUNT)) {
9157
		r = -ENOMEM;
9158
		goto fail_free_mce_banks;
9159
	}
9160

I
Ingo Molnar 已提交
9161
	fx_init(vcpu);
9162

9163
	vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
9164

9165 9166
	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);

9167 9168
	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;

9169
	kvm_async_pf_hash_reset(vcpu);
9170
	kvm_pmu_init(vcpu);
9171

9172
	vcpu->arch.pending_external_vector = -1;
9173
	vcpu->arch.preempted_in_kernel = false;
9174

9175 9176
	kvm_hv_vcpu_init(vcpu);

9177
	return 0;
I
Ingo Molnar 已提交
9178

9179 9180
fail_free_mce_banks:
	kfree(vcpu->arch.mce_banks);
9181 9182
fail_free_lapic:
	kvm_free_lapic(vcpu);
9183 9184 9185
fail_mmu_destroy:
	kvm_mmu_destroy(vcpu);
fail_free_pio_data:
9186
	free_page((unsigned long)vcpu->arch.pio_data);
9187 9188 9189 9190 9191 9192
fail:
	return r;
}

void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
{
9193 9194
	int idx;

A
Andrey Smetanin 已提交
9195
	kvm_hv_vcpu_uninit(vcpu);
9196
	kvm_pmu_destroy(vcpu);
9197
	kfree(vcpu->arch.mce_banks);
9198
	kvm_free_lapic(vcpu);
9199
	idx = srcu_read_lock(&vcpu->kvm->srcu);
9200
	kvm_mmu_destroy(vcpu);
9201
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9202
	free_page((unsigned long)vcpu->arch.pio_data);
9203
	if (!lapic_in_kernel(vcpu))
9204
		static_key_slow_dec(&kvm_no_apic_vcpu);
9205
}
9206

R
Radim Krčmář 已提交
9207 9208
void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
{
P
Paolo Bonzini 已提交
9209
	vcpu->arch.l1tf_flush_l1d = true;
9210
	kvm_x86_ops->sched_in(vcpu, cpu);
R
Radim Krčmář 已提交
9211 9212
}

9213
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
9214
{
9215 9216 9217
	if (type)
		return -EINVAL;

9218
	INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
9219
	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
B
Ben-Ami Yassour 已提交
9220
	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
9221
	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
9222

9223 9224
	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
9225 9226 9227
	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
		&kvm->arch.irq_sources_bitmap);
9228

9229
	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
9230
	mutex_init(&kvm->arch.apic_map_lock);
9231 9232
	spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);

9233
	kvm->arch.kvmclock_offset = -ktime_get_boot_ns();
9234
	pvclock_update_vm_gtod_copy(kvm);
9235

9236 9237
	kvm->arch.guest_can_read_msr_platform_info = true;

9238
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
9239
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
9240

9241
	kvm_hv_init_vm(kvm);
9242
	kvm_page_track_init(kvm);
9243
	kvm_mmu_init_vm(kvm);
9244

9245 9246 9247
	if (kvm_x86_ops->vm_init)
		return kvm_x86_ops->vm_init(kvm);

9248
	return 0;
9249 9250 9251 9252
}

static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
9253
	vcpu_load(vcpu);
9254 9255 9256 9257 9258 9259 9260
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;
9261
	struct kvm_vcpu *vcpu;
9262 9263 9264 9265

	/*
	 * Unpin any mmu pages first.
	 */
9266 9267
	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_clear_async_pf_completion_queue(vcpu);
9268
		kvm_unload_vcpu_mmu(vcpu);
9269
	}
9270 9271 9272 9273 9274 9275
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_arch_vcpu_free(vcpu);

	mutex_lock(&kvm->lock);
	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
		kvm->vcpus[i] = NULL;
9276

9277 9278
	atomic_set(&kvm->online_vcpus, 0);
	mutex_unlock(&kvm->lock);
9279 9280
}

9281 9282
void kvm_arch_sync_events(struct kvm *kvm)
{
9283
	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
9284
	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
9285
	kvm_free_pit(kvm);
9286 9287
}

9288
int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9289 9290
{
	int i, r;
9291
	unsigned long hva;
9292 9293
	struct kvm_memslots *slots = kvm_memslots(kvm);
	struct kvm_memory_slot *slot, old;
9294 9295

	/* Called with kvm->slots_lock held.  */
9296 9297
	if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
		return -EINVAL;
9298

9299 9300
	slot = id_to_memslot(slots, id);
	if (size) {
9301
		if (slot->npages)
9302 9303 9304 9305 9306 9307 9308 9309 9310 9311 9312 9313 9314 9315 9316 9317 9318 9319
			return -EEXIST;

		/*
		 * MAP_SHARED to prevent internal slot pages from being moved
		 * by fork()/COW.
		 */
		hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
			      MAP_SHARED | MAP_ANONYMOUS, 0);
		if (IS_ERR((void *)hva))
			return PTR_ERR((void *)hva);
	} else {
		if (!slot->npages)
			return 0;

		hva = 0;
	}

	old = *slot;
9320
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
9321
		struct kvm_userspace_memory_region m;
9322

9323 9324 9325
		m.slot = id | (i << 16);
		m.flags = 0;
		m.guest_phys_addr = gpa;
9326
		m.userspace_addr = hva;
9327
		m.memory_size = size;
9328 9329 9330 9331 9332
		r = __kvm_set_memory_region(kvm, &m);
		if (r < 0)
			return r;
	}

9333 9334
	if (!size)
		vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE);
9335

9336 9337 9338 9339
	return 0;
}
EXPORT_SYMBOL_GPL(__x86_set_memory_region);

9340
int x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9341 9342 9343 9344
{
	int r;

	mutex_lock(&kvm->slots_lock);
9345
	r = __x86_set_memory_region(kvm, id, gpa, size);
9346 9347 9348 9349 9350 9351
	mutex_unlock(&kvm->slots_lock);

	return r;
}
EXPORT_SYMBOL_GPL(x86_set_memory_region);

9352 9353
void kvm_arch_destroy_vm(struct kvm *kvm)
{
9354 9355 9356 9357 9358 9359
	if (current->mm == kvm->mm) {
		/*
		 * Free memory regions allocated on behalf of userspace,
		 * unless the the memory map has changed due to process exit
		 * or fd copying.
		 */
9360 9361 9362
		x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0);
		x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 0, 0);
		x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
9363
	}
9364 9365
	if (kvm_x86_ops->vm_destroy)
		kvm_x86_ops->vm_destroy(kvm);
9366 9367
	kvm_pic_destroy(kvm);
	kvm_ioapic_destroy(kvm);
9368
	kvm_free_vcpus(kvm);
9369
	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
9370
	kvm_mmu_uninit_vm(kvm);
9371
	kvm_page_track_cleanup(kvm);
9372
	kvm_hv_destroy_vm(kvm);
9373
}
9374

9375
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
9376 9377 9378 9379
			   struct kvm_memory_slot *dont)
{
	int i;

9380 9381
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
		if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
T
Thomas Huth 已提交
9382
			kvfree(free->arch.rmap[i]);
9383
			free->arch.rmap[i] = NULL;
9384
		}
9385 9386 9387 9388 9389
		if (i == 0)
			continue;

		if (!dont || free->arch.lpage_info[i - 1] !=
			     dont->arch.lpage_info[i - 1]) {
T
Thomas Huth 已提交
9390
			kvfree(free->arch.lpage_info[i - 1]);
9391
			free->arch.lpage_info[i - 1] = NULL;
9392 9393
		}
	}
9394 9395

	kvm_page_track_free_memslot(free, dont);
9396 9397
}

9398 9399
int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
			    unsigned long npages)
9400 9401 9402
{
	int i;

9403
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
9404
		struct kvm_lpage_info *linfo;
9405 9406
		unsigned long ugfn;
		int lpages;
9407
		int level = i + 1;
9408 9409 9410 9411

		lpages = gfn_to_index(slot->base_gfn + npages - 1,
				      slot->base_gfn, level) + 1;

9412
		slot->arch.rmap[i] =
K
Kees Cook 已提交
9413
			kvcalloc(lpages, sizeof(*slot->arch.rmap[i]),
9414
				 GFP_KERNEL_ACCOUNT);
9415
		if (!slot->arch.rmap[i])
9416
			goto out_free;
9417 9418
		if (i == 0)
			continue;
9419

9420
		linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
9421
		if (!linfo)
9422 9423
			goto out_free;

9424 9425
		slot->arch.lpage_info[i - 1] = linfo;

9426
		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
9427
			linfo[0].disallow_lpage = 1;
9428
		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
9429
			linfo[lpages - 1].disallow_lpage = 1;
9430 9431 9432 9433 9434 9435 9436 9437 9438 9439 9440
		ugfn = slot->userspace_addr >> PAGE_SHIFT;
		/*
		 * If the gfn and userspace address are not aligned wrt each
		 * other, or if explicitly asked to, disable large page
		 * support for this slot
		 */
		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
		    !kvm_largepages_enabled()) {
			unsigned long j;

			for (j = 0; j < lpages; ++j)
9441
				linfo[j].disallow_lpage = 1;
9442 9443 9444
		}
	}

9445 9446 9447
	if (kvm_page_track_create_memslot(slot, npages))
		goto out_free;

9448 9449 9450
	return 0;

out_free:
9451
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
T
Thomas Huth 已提交
9452
		kvfree(slot->arch.rmap[i]);
9453 9454 9455 9456
		slot->arch.rmap[i] = NULL;
		if (i == 0)
			continue;

T
Thomas Huth 已提交
9457
		kvfree(slot->arch.lpage_info[i - 1]);
9458
		slot->arch.lpage_info[i - 1] = NULL;
9459 9460 9461 9462
	}
	return -ENOMEM;
}

9463
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
9464
{
9465 9466 9467 9468
	/*
	 * memslots->generation has been incremented.
	 * mmio generation may have reached its maximum value.
	 */
9469
	kvm_mmu_invalidate_mmio_sptes(kvm, gen);
9470 9471
}

9472 9473
int kvm_arch_prepare_memory_region(struct kvm *kvm,
				struct kvm_memory_slot *memslot,
9474
				const struct kvm_userspace_memory_region *mem,
9475
				enum kvm_mr_change change)
9476
{
9477 9478 9479
	return 0;
}

9480 9481 9482 9483 9484 9485 9486 9487 9488 9489 9490 9491 9492 9493 9494 9495 9496 9497 9498 9499 9500
static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
				     struct kvm_memory_slot *new)
{
	/* Still write protect RO slot */
	if (new->flags & KVM_MEM_READONLY) {
		kvm_mmu_slot_remove_write_access(kvm, new);
		return;
	}

	/*
	 * Call kvm_x86_ops dirty logging hooks when they are valid.
	 *
	 * kvm_x86_ops->slot_disable_log_dirty is called when:
	 *
	 *  - KVM_MR_CREATE with dirty logging is disabled
	 *  - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag
	 *
	 * The reason is, in case of PML, we need to set D-bit for any slots
	 * with dirty logging disabled in order to eliminate unnecessary GPA
	 * logging in PML buffer (and potential PML buffer full VMEXT). This
	 * guarantees leaving PML enabled during guest's lifetime won't have
W
Wei Yang 已提交
9501
	 * any additional overhead from PML when guest is running with dirty
9502 9503 9504 9505 9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528 9529
	 * logging disabled for memory slots.
	 *
	 * kvm_x86_ops->slot_enable_log_dirty is called when switching new slot
	 * to dirty logging mode.
	 *
	 * If kvm_x86_ops dirty logging hooks are invalid, use write protect.
	 *
	 * In case of write protect:
	 *
	 * Write protect all pages for dirty logging.
	 *
	 * All the sptes including the large sptes which point to this
	 * slot are set to readonly. We can not create any new large
	 * spte on this slot until the end of the logging.
	 *
	 * See the comments in fast_page_fault().
	 */
	if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
		if (kvm_x86_ops->slot_enable_log_dirty)
			kvm_x86_ops->slot_enable_log_dirty(kvm, new);
		else
			kvm_mmu_slot_remove_write_access(kvm, new);
	} else {
		if (kvm_x86_ops->slot_disable_log_dirty)
			kvm_x86_ops->slot_disable_log_dirty(kvm, new);
	}
}

9530
void kvm_arch_commit_memory_region(struct kvm *kvm,
9531
				const struct kvm_userspace_memory_region *mem,
9532
				const struct kvm_memory_slot *old,
9533
				const struct kvm_memory_slot *new,
9534
				enum kvm_mr_change change)
9535
{
9536
	if (!kvm->arch.n_requested_mmu_pages)
9537 9538
		kvm_mmu_change_mmu_pages(kvm,
				kvm_mmu_calculate_default_mmu_pages(kvm));
9539

9540 9541 9542 9543 9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556
	/*
	 * Dirty logging tracks sptes in 4k granularity, meaning that large
	 * sptes have to be split.  If live migration is successful, the guest
	 * in the source machine will be destroyed and large sptes will be
	 * created in the destination. However, if the guest continues to run
	 * in the source machine (for example if live migration fails), small
	 * sptes will remain around and cause bad performance.
	 *
	 * Scan sptes if dirty logging has been stopped, dropping those
	 * which can be collapsed into a single large-page spte.  Later
	 * page faults will create the large-page sptes.
	 */
	if ((change != KVM_MR_DELETE) &&
		(old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
		!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
		kvm_mmu_zap_collapsible_sptes(kvm, new);

9557
	/*
9558
	 * Set up write protection and/or dirty logging for the new slot.
9559
	 *
9560 9561 9562 9563
	 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have
	 * been zapped so no dirty logging staff is needed for old slot. For
	 * KVM_MR_FLAGS_ONLY, the old slot is essentially the same one as the
	 * new and it's also covered when dealing with the new slot.
9564 9565
	 *
	 * FIXME: const-ify all uses of struct kvm_memory_slot.
9566
	 */
9567
	if (change != KVM_MR_DELETE)
9568
		kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new);
9569
}
9570

9571
void kvm_arch_flush_shadow_all(struct kvm *kvm)
9572
{
9573
	kvm_mmu_zap_all(kvm);
9574 9575
}

9576 9577 9578
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
9579
	kvm_page_track_flush_slot(kvm, slot);
9580 9581
}

9582 9583 9584 9585 9586 9587 9588
static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
{
	return (is_guest_mode(vcpu) &&
			kvm_x86_ops->guest_apic_has_interrupt &&
			kvm_x86_ops->guest_apic_has_interrupt(vcpu));
}

9589 9590 9591 9592 9593 9594 9595 9596 9597 9598 9599
static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
{
	if (!list_empty_careful(&vcpu->async_pf.done))
		return true;

	if (kvm_apic_has_events(vcpu))
		return true;

	if (vcpu->arch.pv.pv_unhalted)
		return true;

9600 9601 9602
	if (vcpu->arch.exception.pending)
		return true;

9603 9604 9605
	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
	    (vcpu->arch.nmi_pending &&
	     kvm_x86_ops->nmi_allowed(vcpu)))
9606 9607
		return true;

9608 9609
	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
	    (vcpu->arch.smi_pending && !is_smm(vcpu)))
P
Paolo Bonzini 已提交
9610 9611
		return true;

9612
	if (kvm_arch_interrupt_allowed(vcpu) &&
9613 9614
	    (kvm_cpu_has_interrupt(vcpu) ||
	    kvm_guest_apic_has_interrupt(vcpu)))
9615 9616
		return true;

A
Andrey Smetanin 已提交
9617 9618 9619
	if (kvm_hv_has_stimer_pending(vcpu))
		return true;

9620 9621 9622
	return false;
}

9623 9624
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
9625
	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
9626
}
9627

9628 9629
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
{
9630
	return vcpu->arch.preempted_in_kernel;
9631 9632
}

9633
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
9634
{
9635
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
9636
}
9637 9638 9639 9640 9641

int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	return kvm_x86_ops->interrupt_allowed(vcpu);
}
9642

9643
unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
J
Jan Kiszka 已提交
9644
{
9645 9646 9647 9648 9649 9650
	if (is_64_bit_mode(vcpu))
		return kvm_rip_read(vcpu);
	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
		     kvm_rip_read(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
J
Jan Kiszka 已提交
9651

9652 9653 9654
bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
{
	return kvm_get_linear_rip(vcpu) == linear_rip;
J
Jan Kiszka 已提交
9655 9656 9657
}
EXPORT_SYMBOL_GPL(kvm_is_linear_rip);

9658 9659 9660 9661 9662 9663
unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
{
	unsigned long rflags;

	rflags = kvm_x86_ops->get_rflags(vcpu);
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
9664
		rflags &= ~X86_EFLAGS_TF;
9665 9666 9667 9668
	return rflags;
}
EXPORT_SYMBOL_GPL(kvm_get_rflags);

9669
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
9670 9671
{
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
J
Jan Kiszka 已提交
9672
	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
9673
		rflags |= X86_EFLAGS_TF;
9674
	kvm_x86_ops->set_rflags(vcpu, rflags);
9675 9676 9677 9678 9679
}

void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
	__kvm_set_rflags(vcpu, rflags);
9680
	kvm_make_request(KVM_REQ_EVENT, vcpu);
9681 9682 9683
}
EXPORT_SYMBOL_GPL(kvm_set_rflags);

G
Gleb Natapov 已提交
9684 9685 9686 9687
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
{
	int r;

9688
	if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||
9689
	      work->wakeup_all)
G
Gleb Natapov 已提交
9690 9691 9692 9693 9694 9695
		return;

	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		return;

9696 9697
	if (!vcpu->arch.mmu->direct_map &&
	      work->arch.cr3 != vcpu->arch.mmu->get_cr3(vcpu))
X
Xiao Guangrong 已提交
9698 9699
		return;

9700
	vcpu->arch.mmu->page_fault(vcpu, work->gva, 0, true);
G
Gleb Natapov 已提交
9701 9702
}

9703 9704 9705 9706 9707 9708 9709 9710 9711 9712 9713 9714 9715 9716 9717 9718 9719 9720 9721 9722 9723 9724 9725 9726 9727 9728
static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
{
	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
}

static inline u32 kvm_async_pf_next_probe(u32 key)
{
	return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
}

static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 key = kvm_async_pf_hash_fn(gfn);

	while (vcpu->arch.apf.gfns[key] != ~0)
		key = kvm_async_pf_next_probe(key);

	vcpu->arch.apf.gfns[key] = gfn;
}

static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	int i;
	u32 key = kvm_async_pf_hash_fn(gfn);

	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
9729 9730
		     (vcpu->arch.apf.gfns[key] != gfn &&
		      vcpu->arch.apf.gfns[key] != ~0); i++)
9731 9732 9733 9734 9735 9736 9737 9738 9739 9740 9741 9742 9743 9744 9745 9746 9747 9748 9749 9750 9751 9752 9753 9754 9755 9756 9757 9758 9759 9760 9761 9762 9763
		key = kvm_async_pf_next_probe(key);

	return key;
}

bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
}

static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 i, j, k;

	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
	while (true) {
		vcpu->arch.apf.gfns[i] = ~0;
		do {
			j = kvm_async_pf_next_probe(j);
			if (vcpu->arch.apf.gfns[j] == ~0)
				return;
			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
			/*
			 * k lies cyclically in ]i,j]
			 * |    i.k.j |
			 * |....j i.k.| or  |.k..j i...|
			 */
		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
		i = j;
	}
}

9764 9765
static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
{
9766 9767 9768

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
				      sizeof(val));
9769 9770
}

9771 9772 9773 9774 9775 9776 9777
static int apf_get_user(struct kvm_vcpu *vcpu, u32 *val)
{

	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, val,
				      sizeof(u32));
}

9778 9779 9780 9781 9782 9783 9784 9785 9786 9787 9788 9789 9790 9791 9792 9793 9794 9795 9796 9797 9798 9799 9800 9801 9802 9803 9804 9805 9806 9807
static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
{
	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
		return false;

	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
	    (vcpu->arch.apf.send_user_only &&
	     kvm_x86_ops->get_cpl(vcpu) == 0))
		return false;

	return true;
}

bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
{
	if (unlikely(!lapic_in_kernel(vcpu) ||
		     kvm_event_needs_reinjection(vcpu) ||
		     vcpu->arch.exception.pending))
		return false;

	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
		return false;

	/*
	 * If interrupts are off we cannot even use an artificial
	 * halt state.
	 */
	return kvm_x86_ops->interrupt_allowed(vcpu);
}

9808 9809 9810
void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
				     struct kvm_async_pf *work)
{
9811 9812
	struct x86_exception fault;

9813
	trace_kvm_async_pf_not_present(work->arch.token, work->gva);
9814
	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
9815

9816 9817
	if (kvm_can_deliver_async_pf(vcpu) &&
	    !apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
9818 9819 9820 9821 9822
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
9823
		fault.async_page_fault = true;
9824
		kvm_inject_page_fault(vcpu, &fault);
9825 9826 9827 9828 9829 9830 9831 9832 9833 9834
	} else {
		/*
		 * It is not possible to deliver a paravirtualized asynchronous
		 * page fault, but putting the guest in an artificial halt state
		 * can be beneficial nevertheless: if an interrupt arrives, we
		 * can deliver it timely and perhaps the guest will schedule
		 * another process.  When the instruction that triggered a page
		 * fault is retried, hopefully the page will be ready in the host.
		 */
		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
9835
	}
9836 9837 9838 9839 9840
}

void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
				 struct kvm_async_pf *work)
{
9841
	struct x86_exception fault;
9842
	u32 val;
9843

9844
	if (work->wakeup_all)
9845 9846 9847
		work->arch.token = ~0; /* broadcast wakeup */
	else
		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
9848
	trace_kvm_async_pf_ready(work->arch.token, work->gva);
9849

9850 9851 9852 9853 9854 9855 9856 9857 9858 9859 9860
	if (vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED &&
	    !apf_get_user(vcpu, &val)) {
		if (val == KVM_PV_REASON_PAGE_NOT_PRESENT &&
		    vcpu->arch.exception.pending &&
		    vcpu->arch.exception.nr == PF_VECTOR &&
		    !apf_put_user(vcpu, 0)) {
			vcpu->arch.exception.injected = false;
			vcpu->arch.exception.pending = false;
			vcpu->arch.exception.nr = 0;
			vcpu->arch.exception.has_error_code = false;
			vcpu->arch.exception.error_code = 0;
9861 9862
			vcpu->arch.exception.has_payload = false;
			vcpu->arch.exception.payload = 0;
9863 9864 9865 9866 9867 9868 9869 9870 9871
		} else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
			fault.vector = PF_VECTOR;
			fault.error_code_valid = true;
			fault.error_code = 0;
			fault.nested_page_fault = false;
			fault.address = work->arch.token;
			fault.async_page_fault = true;
			kvm_inject_page_fault(vcpu, &fault);
		}
9872
	}
9873
	vcpu->arch.apf.halted = false;
9874
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
9875 9876 9877 9878 9879 9880 9881
}

bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
		return true;
	else
9882
		return kvm_can_do_async_pf(vcpu);
9883 9884
}

9885 9886 9887 9888 9889 9890 9891 9892 9893 9894 9895 9896 9897 9898 9899 9900 9901 9902
void kvm_arch_start_assignment(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);

void kvm_arch_end_assignment(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);

bool kvm_arch_has_assigned_device(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);

9903 9904 9905 9906 9907 9908 9909 9910 9911 9912 9913 9914 9915 9916 9917 9918 9919 9920
void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);

void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);

bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);

9921 9922 9923 9924 9925
bool kvm_arch_has_irq_bypass(void)
{
	return kvm_x86_ops->update_pi_irte != NULL;
}

F
Feng Wu 已提交
9926 9927 9928 9929 9930 9931
int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

9932
	irqfd->producer = prod;
F
Feng Wu 已提交
9933

9934 9935
	return kvm_x86_ops->update_pi_irte(irqfd->kvm,
					   prod->irq, irqfd->gsi, 1);
F
Feng Wu 已提交
9936 9937 9938 9939 9940 9941 9942 9943 9944 9945 9946 9947 9948 9949 9950
}

void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	int ret;
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

	WARN_ON(irqfd->producer != prod);
	irqfd->producer = NULL;

	/*
	 * When producer of consumer is unregistered, we change back to
	 * remapped mode, so we can re-use the current implementation
A
Andrea Gelmini 已提交
9951
	 * when the irq is masked/disabled or the consumer side (KVM
F
Feng Wu 已提交
9952 9953 9954 9955 9956 9957 9958 9959 9960 9961 9962 9963 9964 9965 9966 9967 9968
	 * int this case doesn't want to receive the interrupts.
	*/
	ret = kvm_x86_ops->update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0);
	if (ret)
		printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
		       " fails: %d\n", irqfd->consumer.token, ret);
}

int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
				   uint32_t guest_irq, bool set)
{
	if (!kvm_x86_ops->update_pi_irte)
		return -EINVAL;

	return kvm_x86_ops->update_pi_irte(kvm, host_irq, guest_irq, set);
}

9969 9970 9971 9972 9973 9974
bool kvm_vector_hashing_enabled(void)
{
	return vector_hashing;
}
EXPORT_SYMBOL_GPL(kvm_vector_hashing_enabled);

9975
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
J
Jason Wang 已提交
9976
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
9977 9978 9979 9980
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
9981
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
9982
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
9983
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
9984
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
9985
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
9986
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
9987
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
9988
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
9989
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window);
K
Kai Huang 已提交
9990
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
9991
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
9992 9993
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);