vmx.c 116.2 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "irq.h"
20
#include "mmu.h"
A
Avi Kivity 已提交
21

22
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
23
#include <linux/module.h>
24
#include <linux/kernel.h>
A
Avi Kivity 已提交
25 26
#include <linux/mm.h>
#include <linux/highmem.h>
A
Alexey Dobriyan 已提交
27
#include <linux/sched.h>
28
#include <linux/moduleparam.h>
29
#include <linux/ftrace_event.h>
30
#include <linux/slab.h>
31
#include <linux/tboot.h>
32
#include "kvm_cache_regs.h"
33
#include "x86.h"
A
Avi Kivity 已提交
34

A
Avi Kivity 已提交
35
#include <asm/io.h>
A
Anthony Liguori 已提交
36
#include <asm/desc.h>
37
#include <asm/vmx.h>
38
#include <asm/virtext.h>
A
Andi Kleen 已提交
39
#include <asm/mce.h>
40 41
#include <asm/i387.h>
#include <asm/xcr.h>
A
Avi Kivity 已提交
42

43 44
#include "trace.h"

45 46
#define __ex(x) __kvm_handle_fault_on_reboot(x)

A
Avi Kivity 已提交
47 48 49
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

50
static int __read_mostly bypass_guest_pf = 1;
51
module_param(bypass_guest_pf, bool, S_IRUGO);
52

53
static int __read_mostly enable_vpid = 1;
54
module_param_named(vpid, enable_vpid, bool, 0444);
55

56
static int __read_mostly flexpriority_enabled = 1;
57
module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
58

59
static int __read_mostly enable_ept = 1;
60
module_param_named(ept, enable_ept, bool, S_IRUGO);
S
Sheng Yang 已提交
61

62 63 64 65
static int __read_mostly enable_unrestricted_guest = 1;
module_param_named(unrestricted_guest,
			enable_unrestricted_guest, bool, S_IRUGO);

66
static int __read_mostly emulate_invalid_guest_state = 0;
67
module_param(emulate_invalid_guest_state, bool, S_IRUGO);
68

69 70 71
static int __read_mostly vmm_exclusive = 1;
module_param(vmm_exclusive, bool, S_IRUGO);

72 73 74
static int __read_mostly yield_on_hlt = 1;
module_param(yield_on_hlt, bool, S_IRUGO);

75 76 77 78 79
#define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST				\
	(X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD)
#define KVM_GUEST_CR0_MASK						\
	(KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST				\
80
	(X86_CR0_WP | X86_CR0_NE)
81 82
#define KVM_VM_CR0_ALWAYS_ON						\
	(KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
83 84 85 86
#define KVM_CR4_GUEST_OWNED_BITS				      \
	(X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
	 | X86_CR4_OSXMMEXCPT)

87 88 89
#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)

90 91
#define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))

92 93 94 95
/*
 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 * ple_gap:    upper bound on the amount of time between two successive
 *             executions of PAUSE in a loop. Also indicate if ple enabled.
96
 *             According to test, this time is usually smaller than 128 cycles.
97 98 99 100 101 102
 * ple_window: upper bound on the amount of time a guest is allowed to execute
 *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 *             less than 2^12 cycles
 * Time is measured based on a counter that runs at the same rate as the TSC,
 * refer SDM volume 3b section 21.6.13 & 22.1.3.
 */
103
#define KVM_VMX_DEFAULT_PLE_GAP    128
104 105 106 107 108 109 110
#define KVM_VMX_DEFAULT_PLE_WINDOW 4096
static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
module_param(ple_gap, int, S_IRUGO);

static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
module_param(ple_window, int, S_IRUGO);

111 112
#define NR_AUTOLOAD_MSRS 1

113 114 115 116 117 118
struct vmcs {
	u32 revision_id;
	u32 abort;
	char data[0];
};

119 120 121
struct shared_msr_entry {
	unsigned index;
	u64 data;
122
	u64 mask;
123 124
};

125
struct vcpu_vmx {
R
Rusty Russell 已提交
126
	struct kvm_vcpu       vcpu;
127
	struct list_head      local_vcpus_link;
128
	unsigned long         host_rsp;
129
	int                   launched;
130
	u8                    fail;
131
	u32                   exit_intr_info;
132
	u32                   idt_vectoring_info;
133
	struct shared_msr_entry *guest_msrs;
134 135 136
	int                   nmsrs;
	int                   save_nmsrs;
#ifdef CONFIG_X86_64
137 138
	u64 		      msr_host_kernel_gs_base;
	u64 		      msr_guest_kernel_gs_base;
139 140
#endif
	struct vmcs          *vmcs;
141 142 143 144 145
	struct msr_autoload {
		unsigned nr;
		struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS];
		struct vmx_msr_entry host[NR_AUTOLOAD_MSRS];
	} msr_autoload;
146 147 148
	struct {
		int           loaded;
		u16           fs_sel, gs_sel, ldt_sel;
149 150
		int           gs_ldt_reload_needed;
		int           fs_reload_needed;
M
Mike Day 已提交
151
	} host_state;
152
	struct {
153
		int vm86_active;
154
		ulong save_rflags;
155 156 157 158 159 160
		struct kvm_save_segment {
			u16 selector;
			unsigned long base;
			u32 limit;
			u32 ar;
		} tr, es, ds, fs, gs;
161
	} rmode;
162
	int vpid;
163
	bool emulation_required;
164 165 166 167 168

	/* Support for vnmi-less CPUs */
	int soft_vnmi_blocked;
	ktime_t entry_time;
	s64 vnmi_blocked_time;
A
Andi Kleen 已提交
169
	u32 exit_reason;
170 171

	bool rdtscp_enabled;
172 173 174 175
};

static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
176
	return container_of(vcpu, struct vcpu_vmx, vcpu);
177 178
}

179
static int init_rmode(struct kvm *kvm);
180
static u64 construct_eptp(unsigned long root_hpa);
181 182
static void kvm_cpu_vmxon(u64 addr);
static void kvm_cpu_vmxoff(void);
183
static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
184

A
Avi Kivity 已提交
185 186
static DEFINE_PER_CPU(struct vmcs *, vmxarea);
static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
187
static DEFINE_PER_CPU(struct list_head, vcpus_on_cpu);
188
static DEFINE_PER_CPU(struct desc_ptr, host_gdt);
A
Avi Kivity 已提交
189

190 191
static unsigned long *vmx_io_bitmap_a;
static unsigned long *vmx_io_bitmap_b;
192 193
static unsigned long *vmx_msr_bitmap_legacy;
static unsigned long *vmx_msr_bitmap_longmode;
194

A
Avi Kivity 已提交
195 196
static bool cpu_has_load_ia32_efer;

197 198 199
static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
static DEFINE_SPINLOCK(vmx_vpid_lock);

200
static struct vmcs_config {
A
Avi Kivity 已提交
201 202 203
	int size;
	int order;
	u32 revision_id;
204 205
	u32 pin_based_exec_ctrl;
	u32 cpu_based_exec_ctrl;
206
	u32 cpu_based_2nd_exec_ctrl;
207 208 209
	u32 vmexit_ctrl;
	u32 vmentry_ctrl;
} vmcs_config;
A
Avi Kivity 已提交
210

H
Hannes Eder 已提交
211
static struct vmx_capability {
S
Sheng Yang 已提交
212 213 214 215
	u32 ept;
	u32 vpid;
} vmx_capability;

A
Avi Kivity 已提交
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
#define VMX_SEGMENT_FIELD(seg)					\
	[VCPU_SREG_##seg] = {                                   \
		.selector = GUEST_##seg##_SELECTOR,		\
		.base = GUEST_##seg##_BASE,		   	\
		.limit = GUEST_##seg##_LIMIT,		   	\
		.ar_bytes = GUEST_##seg##_AR_BYTES,	   	\
	}

static struct kvm_vmx_segment_field {
	unsigned selector;
	unsigned base;
	unsigned limit;
	unsigned ar_bytes;
} kvm_vmx_segment_fields[] = {
	VMX_SEGMENT_FIELD(CS),
	VMX_SEGMENT_FIELD(DS),
	VMX_SEGMENT_FIELD(ES),
	VMX_SEGMENT_FIELD(FS),
	VMX_SEGMENT_FIELD(GS),
	VMX_SEGMENT_FIELD(SS),
	VMX_SEGMENT_FIELD(TR),
	VMX_SEGMENT_FIELD(LDTR),
};

240 241
static u64 host_efer;

A
Avi Kivity 已提交
242 243
static void ept_save_pdptrs(struct kvm_vcpu *vcpu);

244
/*
B
Brian Gerst 已提交
245
 * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
246 247
 * away by decrementing the array size.
 */
A
Avi Kivity 已提交
248
static const u32 vmx_msr_index[] = {
249
#ifdef CONFIG_X86_64
250
	MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
A
Avi Kivity 已提交
251
#endif
B
Brian Gerst 已提交
252
	MSR_EFER, MSR_TSC_AUX, MSR_STAR,
A
Avi Kivity 已提交
253
};
254
#define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
A
Avi Kivity 已提交
255

256
static inline bool is_page_fault(u32 intr_info)
A
Avi Kivity 已提交
257 258 259
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
260
		(INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
A
Avi Kivity 已提交
261 262
}

263
static inline bool is_no_device(u32 intr_info)
264 265 266
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
267
		(INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
268 269
}

270
static inline bool is_invalid_opcode(u32 intr_info)
271 272 273
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
274
		(INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK);
275 276
}

277
static inline bool is_external_interrupt(u32 intr_info)
A
Avi Kivity 已提交
278 279 280 281 282
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
		== (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
}

283
static inline bool is_machine_check(u32 intr_info)
A
Andi Kleen 已提交
284 285 286 287 288 289
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
		(INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
}

290
static inline bool cpu_has_vmx_msr_bitmap(void)
S
Sheng Yang 已提交
291
{
292
	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
S
Sheng Yang 已提交
293 294
}

295
static inline bool cpu_has_vmx_tpr_shadow(void)
296
{
297
	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
298 299
}

300
static inline bool vm_need_tpr_shadow(struct kvm *kvm)
301
{
302
	return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
303 304
}

305
static inline bool cpu_has_secondary_exec_ctrls(void)
306
{
307 308
	return vmcs_config.cpu_based_exec_ctrl &
		CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
309 310
}

311
static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
312
{
313 314 315 316 317 318 319 320
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
}

static inline bool cpu_has_vmx_flexpriority(void)
{
	return cpu_has_vmx_tpr_shadow() &&
		cpu_has_vmx_virtualize_apic_accesses();
321 322
}

323 324
static inline bool cpu_has_vmx_ept_execute_only(void)
{
325
	return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
326 327 328 329
}

static inline bool cpu_has_vmx_eptp_uncacheable(void)
{
330
	return vmx_capability.ept & VMX_EPTP_UC_BIT;
331 332 333 334
}

static inline bool cpu_has_vmx_eptp_writeback(void)
{
335
	return vmx_capability.ept & VMX_EPTP_WB_BIT;
336 337 338 339
}

static inline bool cpu_has_vmx_ept_2m_page(void)
{
340
	return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
341 342
}

343 344
static inline bool cpu_has_vmx_ept_1g_page(void)
{
345
	return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
346 347
}

348 349 350 351 352
static inline bool cpu_has_vmx_ept_4levels(void)
{
	return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
}

353
static inline bool cpu_has_vmx_invept_individual_addr(void)
S
Sheng Yang 已提交
354
{
355
	return vmx_capability.ept & VMX_EPT_EXTENT_INDIVIDUAL_BIT;
S
Sheng Yang 已提交
356 357
}

358
static inline bool cpu_has_vmx_invept_context(void)
S
Sheng Yang 已提交
359
{
360
	return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
S
Sheng Yang 已提交
361 362
}

363
static inline bool cpu_has_vmx_invept_global(void)
S
Sheng Yang 已提交
364
{
365
	return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
S
Sheng Yang 已提交
366 367
}

368 369 370 371 372
static inline bool cpu_has_vmx_invvpid_single(void)
{
	return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
}

373 374 375 376 377
static inline bool cpu_has_vmx_invvpid_global(void)
{
	return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
}

378
static inline bool cpu_has_vmx_ept(void)
S
Sheng Yang 已提交
379
{
380 381
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_EPT;
S
Sheng Yang 已提交
382 383
}

384
static inline bool cpu_has_vmx_unrestricted_guest(void)
385 386 387 388 389
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_UNRESTRICTED_GUEST;
}

390
static inline bool cpu_has_vmx_ple(void)
391 392 393 394 395
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_PAUSE_LOOP_EXITING;
}

396
static inline bool vm_need_virtualize_apic_accesses(struct kvm *kvm)
397
{
398
	return flexpriority_enabled && irqchip_in_kernel(kvm);
399 400
}

401
static inline bool cpu_has_vmx_vpid(void)
402
{
403 404
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_VPID;
405 406
}

407
static inline bool cpu_has_vmx_rdtscp(void)
408 409 410 411 412
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_RDTSCP;
}

413
static inline bool cpu_has_virtual_nmis(void)
414 415 416 417
{
	return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
}

418 419 420 421 422 423
static inline bool cpu_has_vmx_wbinvd_exit(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_WBINVD_EXITING;
}

424 425 426 427 428
static inline bool report_flexpriority(void)
{
	return flexpriority_enabled;
}

R
Rusty Russell 已提交
429
static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
430 431 432
{
	int i;

433
	for (i = 0; i < vmx->nmsrs; ++i)
434
		if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
435 436 437 438
			return i;
	return -1;
}

439 440 441 442 443 444 445 446
static inline void __invvpid(int ext, u16 vpid, gva_t gva)
{
    struct {
	u64 vpid : 16;
	u64 rsvd : 48;
	u64 gva;
    } operand = { vpid, 0, gva };

447
    asm volatile (__ex(ASM_VMX_INVVPID)
448 449 450 451 452
		  /* CF==1 or ZF==1 --> rc = -1 */
		  "; ja 1f ; ud2 ; 1:"
		  : : "a"(&operand), "c"(ext) : "cc", "memory");
}

453 454 455 456 457 458
static inline void __invept(int ext, u64 eptp, gpa_t gpa)
{
	struct {
		u64 eptp, gpa;
	} operand = {eptp, gpa};

459
	asm volatile (__ex(ASM_VMX_INVEPT)
460 461 462 463 464
			/* CF==1 or ZF==1 --> rc = -1 */
			"; ja 1f ; ud2 ; 1:\n"
			: : "a" (&operand), "c" (ext) : "cc", "memory");
}

465
static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
466 467 468
{
	int i;

R
Rusty Russell 已提交
469
	i = __find_msr_index(vmx, msr);
470
	if (i >= 0)
471
		return &vmx->guest_msrs[i];
A
Al Viro 已提交
472
	return NULL;
473 474
}

A
Avi Kivity 已提交
475 476 477 478 479
static void vmcs_clear(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(vmcs);
	u8 error;

480
	asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
481
		      : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
A
Avi Kivity 已提交
482 483 484 485 486 487
		      : "cc", "memory");
	if (error)
		printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
		       vmcs, phys_addr);
}

488 489 490 491 492 493
static void vmcs_load(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(vmcs);
	u8 error;

	asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
494
			: "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
495 496 497 498 499 500
			: "cc", "memory");
	if (error)
		printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
		       vmcs, phys_addr);
}

A
Avi Kivity 已提交
501 502
static void __vcpu_clear(void *arg)
{
R
Rusty Russell 已提交
503
	struct vcpu_vmx *vmx = arg;
504
	int cpu = raw_smp_processor_id();
A
Avi Kivity 已提交
505

R
Rusty Russell 已提交
506
	if (vmx->vcpu.cpu == cpu)
507 508
		vmcs_clear(vmx->vmcs);
	if (per_cpu(current_vmcs, cpu) == vmx->vmcs)
A
Avi Kivity 已提交
509
		per_cpu(current_vmcs, cpu) = NULL;
510 511 512
	list_del(&vmx->local_vcpus_link);
	vmx->vcpu.cpu = -1;
	vmx->launched = 0;
A
Avi Kivity 已提交
513 514
}

R
Rusty Russell 已提交
515
static void vcpu_clear(struct vcpu_vmx *vmx)
A
Avi Kivity 已提交
516
{
517 518
	if (vmx->vcpu.cpu == -1)
		return;
519
	smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 1);
A
Avi Kivity 已提交
520 521
}

522
static inline void vpid_sync_vcpu_single(struct vcpu_vmx *vmx)
523 524 525 526
{
	if (vmx->vpid == 0)
		return;

527 528
	if (cpu_has_vmx_invvpid_single())
		__invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vmx->vpid, 0);
529 530
}

531 532 533 534 535 536 537 538 539
static inline void vpid_sync_vcpu_global(void)
{
	if (cpu_has_vmx_invvpid_global())
		__invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
}

static inline void vpid_sync_context(struct vcpu_vmx *vmx)
{
	if (cpu_has_vmx_invvpid_single())
540
		vpid_sync_vcpu_single(vmx);
541 542 543 544
	else
		vpid_sync_vcpu_global();
}

545 546 547 548 549 550 551 552
static inline void ept_sync_global(void)
{
	if (cpu_has_vmx_invept_global())
		__invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
}

static inline void ept_sync_context(u64 eptp)
{
553
	if (enable_ept) {
554 555 556 557 558 559 560 561 562
		if (cpu_has_vmx_invept_context())
			__invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
		else
			ept_sync_global();
	}
}

static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa)
{
563
	if (enable_ept) {
564 565 566 567 568 569 570 571
		if (cpu_has_vmx_invept_individual_addr())
			__invept(VMX_EPT_EXTENT_INDIVIDUAL_ADDR,
					eptp, gpa);
		else
			ept_sync_context(eptp);
	}
}

A
Avi Kivity 已提交
572 573
static unsigned long vmcs_readl(unsigned long field)
{
574
	unsigned long value = 0;
A
Avi Kivity 已提交
575

576
	asm volatile (__ex(ASM_VMX_VMREAD_RDX_RAX)
577
		      : "+a"(value) : "d"(field) : "cc");
A
Avi Kivity 已提交
578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
	return value;
}

static u16 vmcs_read16(unsigned long field)
{
	return vmcs_readl(field);
}

static u32 vmcs_read32(unsigned long field)
{
	return vmcs_readl(field);
}

static u64 vmcs_read64(unsigned long field)
{
593
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
594 595 596 597 598 599
	return vmcs_readl(field);
#else
	return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
#endif
}

600 601 602 603 604 605 606
static noinline void vmwrite_error(unsigned long field, unsigned long value)
{
	printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
	       field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
	dump_stack();
}

A
Avi Kivity 已提交
607 608 609 610
static void vmcs_writel(unsigned long field, unsigned long value)
{
	u8 error;

611
	asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
M
Mike Day 已提交
612
		       : "=q"(error) : "a"(value), "d"(field) : "cc");
613 614
	if (unlikely(error))
		vmwrite_error(field, value);
A
Avi Kivity 已提交
615 616 617 618 619 620 621 622 623 624 625 626 627 628 629
}

static void vmcs_write16(unsigned long field, u16 value)
{
	vmcs_writel(field, value);
}

static void vmcs_write32(unsigned long field, u32 value)
{
	vmcs_writel(field, value);
}

static void vmcs_write64(unsigned long field, u64 value)
{
	vmcs_writel(field, value);
630
#ifndef CONFIG_X86_64
A
Avi Kivity 已提交
631 632 633 634 635
	asm volatile ("");
	vmcs_writel(field+1, value >> 32);
#endif
}

636 637 638 639 640 641 642 643 644 645
static void vmcs_clear_bits(unsigned long field, u32 mask)
{
	vmcs_writel(field, vmcs_readl(field) & ~mask);
}

static void vmcs_set_bits(unsigned long field, u32 mask)
{
	vmcs_writel(field, vmcs_readl(field) | mask);
}

646 647 648 649
static void update_exception_bitmap(struct kvm_vcpu *vcpu)
{
	u32 eb;

J
Jan Kiszka 已提交
650 651 652 653 654 655
	eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
	     (1u << NM_VECTOR) | (1u << DB_VECTOR);
	if ((vcpu->guest_debug &
	     (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
	    (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
		eb |= 1u << BP_VECTOR;
656
	if (to_vmx(vcpu)->rmode.vm86_active)
657
		eb = ~0;
658
	if (enable_ept)
659
		eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
660 661
	if (vcpu->fpu_active)
		eb &= ~(1u << NM_VECTOR);
662 663 664
	vmcs_write32(EXCEPTION_BITMAP, eb);
}

665 666 667 668 669
static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
{
	unsigned i;
	struct msr_autoload *m = &vmx->msr_autoload;

A
Avi Kivity 已提交
670 671 672 673 674 675
	if (msr == MSR_EFER && cpu_has_load_ia32_efer) {
		vmcs_clear_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER);
		vmcs_clear_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER);
		return;
	}

676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
	for (i = 0; i < m->nr; ++i)
		if (m->guest[i].index == msr)
			break;

	if (i == m->nr)
		return;
	--m->nr;
	m->guest[i] = m->guest[m->nr];
	m->host[i] = m->host[m->nr];
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
}

static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
				  u64 guest_val, u64 host_val)
{
	unsigned i;
	struct msr_autoload *m = &vmx->msr_autoload;

A
Avi Kivity 已提交
695 696 697 698 699 700 701 702
	if (msr == MSR_EFER && cpu_has_load_ia32_efer) {
		vmcs_write64(GUEST_IA32_EFER, guest_val);
		vmcs_write64(HOST_IA32_EFER, host_val);
		vmcs_set_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER);
		vmcs_set_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER);
		return;
	}

703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
	for (i = 0; i < m->nr; ++i)
		if (m->guest[i].index == msr)
			break;

	if (i == m->nr) {
		++m->nr;
		vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
		vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
	}

	m->guest[i].index = msr;
	m->guest[i].value = guest_val;
	m->host[i].index = msr;
	m->host[i].value = host_val;
}

719 720 721 722 723
static void reload_tss(void)
{
	/*
	 * VT restores TR but not its size.  Useless.
	 */
724
	struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
725
	struct desc_struct *descs;
726

727
	descs = (void *)gdt->address;
728 729 730 731
	descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
	load_TR_desc();
}

A
Avi Kivity 已提交
732
static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
733
{
R
Roel Kluin 已提交
734
	u64 guest_efer;
735 736
	u64 ignore_bits;

737
	guest_efer = vmx->vcpu.arch.efer;
R
Roel Kluin 已提交
738

739 740 741 742 743 744 745 746 747 748 749 750 751
	/*
	 * NX is emulated; LMA and LME handled by hardware; SCE meaninless
	 * outside long mode
	 */
	ignore_bits = EFER_NX | EFER_SCE;
#ifdef CONFIG_X86_64
	ignore_bits |= EFER_LMA | EFER_LME;
	/* SCE is meaningful only in long mode on Intel */
	if (guest_efer & EFER_LMA)
		ignore_bits &= ~(u64)EFER_SCE;
#endif
	guest_efer &= ~ignore_bits;
	guest_efer |= host_efer & ignore_bits;
752
	vmx->guest_msrs[efer_offset].data = guest_efer;
753
	vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
754 755 756 757 758 759 760 761 762 763 764

	clear_atomic_switch_msr(vmx, MSR_EFER);
	/* On ept, can't emulate nx, and must switch nx atomically */
	if (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX)) {
		guest_efer = vmx->vcpu.arch.efer;
		if (!(guest_efer & EFER_LMA))
			guest_efer &= ~EFER_LME;
		add_atomic_switch_msr(vmx, MSR_EFER, guest_efer, host_efer);
		return false;
	}

765
	return true;
766 767
}

768 769
static unsigned long segment_base(u16 selector)
{
770
	struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
771 772 773 774 775 776 777
	struct desc_struct *d;
	unsigned long table_base;
	unsigned long v;

	if (!(selector & ~3))
		return 0;

778
	table_base = gdt->address;
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803

	if (selector & 4) {           /* from ldt */
		u16 ldt_selector = kvm_read_ldt();

		if (!(ldt_selector & ~3))
			return 0;

		table_base = segment_base(ldt_selector);
	}
	d = (struct desc_struct *)(table_base + (selector & ~7));
	v = get_desc_base(d);
#ifdef CONFIG_X86_64
       if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
               v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
#endif
	return v;
}

static inline unsigned long kvm_read_tr_base(void)
{
	u16 tr;
	asm("str %0" : "=g"(tr));
	return segment_base(tr);
}

804
static void vmx_save_host_state(struct kvm_vcpu *vcpu)
805
{
806
	struct vcpu_vmx *vmx = to_vmx(vcpu);
807
	int i;
808

809
	if (vmx->host_state.loaded)
810 811
		return;

812
	vmx->host_state.loaded = 1;
813 814 815 816
	/*
	 * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
	 * allow segment selectors with cpl > 0 or ti == 1.
	 */
817
	vmx->host_state.ldt_sel = kvm_read_ldt();
818
	vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
819
	savesegment(fs, vmx->host_state.fs_sel);
820
	if (!(vmx->host_state.fs_sel & 7)) {
821
		vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
822 823
		vmx->host_state.fs_reload_needed = 0;
	} else {
824
		vmcs_write16(HOST_FS_SELECTOR, 0);
825
		vmx->host_state.fs_reload_needed = 1;
826
	}
827
	savesegment(gs, vmx->host_state.gs_sel);
828 829
	if (!(vmx->host_state.gs_sel & 7))
		vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
830 831
	else {
		vmcs_write16(HOST_GS_SELECTOR, 0);
832
		vmx->host_state.gs_ldt_reload_needed = 1;
833 834 835 836 837 838
	}

#ifdef CONFIG_X86_64
	vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
	vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
#else
839 840
	vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
	vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
841
#endif
842 843

#ifdef CONFIG_X86_64
844 845
	rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
	if (is_long_mode(&vmx->vcpu))
846
		wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
847
#endif
848 849
	for (i = 0; i < vmx->save_nmsrs; ++i)
		kvm_set_shared_msr(vmx->guest_msrs[i].index,
850 851
				   vmx->guest_msrs[i].data,
				   vmx->guest_msrs[i].mask);
852 853
}

854
static void __vmx_load_host_state(struct vcpu_vmx *vmx)
855
{
856
	if (!vmx->host_state.loaded)
857 858
		return;

859
	++vmx->vcpu.stat.host_state_reload;
860
	vmx->host_state.loaded = 0;
861 862 863 864
#ifdef CONFIG_X86_64
	if (is_long_mode(&vmx->vcpu))
		rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
#endif
865
	if (vmx->host_state.gs_ldt_reload_needed) {
866
		kvm_load_ldt(vmx->host_state.ldt_sel);
867
#ifdef CONFIG_X86_64
868 869 870
		load_gs_index(vmx->host_state.gs_sel);
#else
		loadsegment(gs, vmx->host_state.gs_sel);
871 872
#endif
	}
873 874
	if (vmx->host_state.fs_reload_needed)
		loadsegment(fs, vmx->host_state.fs_sel);
875
	reload_tss();
876
#ifdef CONFIG_X86_64
877
	wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
878
#endif
879 880
	if (current_thread_info()->status & TS_USEDFPU)
		clts();
881
	load_gdt(&__get_cpu_var(host_gdt));
882 883
}

884 885 886 887 888 889 890
static void vmx_load_host_state(struct vcpu_vmx *vmx)
{
	preempt_disable();
	__vmx_load_host_state(vmx);
	preempt_enable();
}

A
Avi Kivity 已提交
891 892 893 894
/*
 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
 * vcpu mutex is already taken.
 */
895
static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
A
Avi Kivity 已提交
896
{
897
	struct vcpu_vmx *vmx = to_vmx(vcpu);
898
	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
A
Avi Kivity 已提交
899

900 901 902
	if (!vmm_exclusive)
		kvm_cpu_vmxon(phys_addr);
	else if (vcpu->cpu != cpu)
R
Rusty Russell 已提交
903
		vcpu_clear(vmx);
A
Avi Kivity 已提交
904

905 906
	if (per_cpu(current_vmcs, cpu) != vmx->vmcs) {
		per_cpu(current_vmcs, cpu) = vmx->vmcs;
907
		vmcs_load(vmx->vmcs);
A
Avi Kivity 已提交
908 909 910
	}

	if (vcpu->cpu != cpu) {
911
		struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
A
Avi Kivity 已提交
912 913
		unsigned long sysenter_esp;

914
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
915 916 917 918 919
		local_irq_disable();
		list_add(&vmx->local_vcpus_link,
			 &per_cpu(vcpus_on_cpu, cpu));
		local_irq_enable();

A
Avi Kivity 已提交
920 921 922 923
		/*
		 * Linux uses per-cpu TSS and GDT, so set these when switching
		 * processors.
		 */
924
		vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
925
		vmcs_writel(HOST_GDTR_BASE, gdt->address);   /* 22.2.4 */
A
Avi Kivity 已提交
926 927 928 929 930 931 932 933

		rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
		vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
	}
}

static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
{
934
	__vmx_load_host_state(to_vmx(vcpu));
935
	if (!vmm_exclusive) {
936
		__vcpu_clear(to_vmx(vcpu));
937 938
		kvm_cpu_vmxoff();
	}
A
Avi Kivity 已提交
939 940
}

941 942
static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
{
943 944
	ulong cr0;

945 946 947
	if (vcpu->fpu_active)
		return;
	vcpu->fpu_active = 1;
948 949 950 951
	cr0 = vmcs_readl(GUEST_CR0);
	cr0 &= ~(X86_CR0_TS | X86_CR0_MP);
	cr0 |= kvm_read_cr0_bits(vcpu, X86_CR0_TS | X86_CR0_MP);
	vmcs_writel(GUEST_CR0, cr0);
952
	update_exception_bitmap(vcpu);
953 954
	vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
955 956
}

957 958
static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);

959 960
static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
{
961
	vmx_decache_cr0_guest_bits(vcpu);
962
	vmcs_set_bits(GUEST_CR0, X86_CR0_TS | X86_CR0_MP);
963
	update_exception_bitmap(vcpu);
964 965 966
	vcpu->arch.cr0_guest_owned_bits = 0;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
	vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
967 968
}

A
Avi Kivity 已提交
969 970
static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
{
971
	unsigned long rflags, save_rflags;
972 973

	rflags = vmcs_readl(GUEST_RFLAGS);
974 975 976 977 978
	if (to_vmx(vcpu)->rmode.vm86_active) {
		rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
		save_rflags = to_vmx(vcpu)->rmode.save_rflags;
		rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
	}
979
	return rflags;
A
Avi Kivity 已提交
980 981 982 983
}

static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
984 985
	if (to_vmx(vcpu)->rmode.vm86_active) {
		to_vmx(vcpu)->rmode.save_rflags = rflags;
986
		rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
987
	}
A
Avi Kivity 已提交
988 989 990
	vmcs_writel(GUEST_RFLAGS, rflags);
}

991 992 993 994 995 996
static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	int ret = 0;

	if (interruptibility & GUEST_INTR_STATE_STI)
997
		ret |= KVM_X86_SHADOW_INT_STI;
998
	if (interruptibility & GUEST_INTR_STATE_MOV_SS)
999
		ret |= KVM_X86_SHADOW_INT_MOV_SS;
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010

	return ret & mask;
}

static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	u32 interruptibility = interruptibility_old;

	interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);

1011
	if (mask & KVM_X86_SHADOW_INT_MOV_SS)
1012
		interruptibility |= GUEST_INTR_STATE_MOV_SS;
1013
	else if (mask & KVM_X86_SHADOW_INT_STI)
1014 1015 1016 1017 1018 1019
		interruptibility |= GUEST_INTR_STATE_STI;

	if ((interruptibility != interruptibility_old))
		vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
}

A
Avi Kivity 已提交
1020 1021 1022 1023
static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
	unsigned long rip;

1024
	rip = kvm_rip_read(vcpu);
A
Avi Kivity 已提交
1025
	rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
1026
	kvm_rip_write(vcpu, rip);
A
Avi Kivity 已提交
1027

1028 1029
	/* skipping an emulated instruction also counts */
	vmx_set_interrupt_shadow(vcpu, 0);
A
Avi Kivity 已提交
1030 1031
}

1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
{
	/* Ensure that we clear the HLT state in the VMCS.  We don't need to
	 * explicitly skip the instruction because if the HLT state is set, then
	 * the instruction is already executing and RIP has already been
	 * advanced. */
	if (!yield_on_hlt &&
	    vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
		vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
}

1043
static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
1044 1045
				bool has_error_code, u32 error_code,
				bool reinject)
1046
{
1047
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1048
	u32 intr_info = nr | INTR_INFO_VALID_MASK;
1049

1050
	if (has_error_code) {
1051
		vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
1052 1053
		intr_info |= INTR_INFO_DELIVER_CODE_MASK;
	}
1054

1055
	if (vmx->rmode.vm86_active) {
1056 1057
		if (kvm_inject_realmode_interrupt(vcpu, nr) != EMULATE_DONE)
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1058 1059 1060
		return;
	}

1061 1062 1063
	if (kvm_exception_is_soft(nr)) {
		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
			     vmx->vcpu.arch.event_exit_inst_len);
1064 1065 1066 1067 1068
		intr_info |= INTR_TYPE_SOFT_EXCEPTION;
	} else
		intr_info |= INTR_TYPE_HARD_EXCEPTION;

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
1069
	vmx_clear_hlt(vcpu);
1070 1071
}

1072 1073 1074 1075 1076
static bool vmx_rdtscp_supported(void)
{
	return cpu_has_vmx_rdtscp();
}

1077 1078 1079
/*
 * Swap MSR entry in host/guest MSR entry array.
 */
R
Rusty Russell 已提交
1080
static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
1081
{
1082
	struct shared_msr_entry tmp;
1083 1084 1085 1086

	tmp = vmx->guest_msrs[to];
	vmx->guest_msrs[to] = vmx->guest_msrs[from];
	vmx->guest_msrs[from] = tmp;
1087 1088
}

1089 1090 1091 1092 1093
/*
 * Set up the vmcs to automatically save and restore system
 * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
 * mode, as fiddling with msrs is very expensive.
 */
R
Rusty Russell 已提交
1094
static void setup_msrs(struct vcpu_vmx *vmx)
1095
{
1096
	int save_nmsrs, index;
1097
	unsigned long *msr_bitmap;
1098

1099
	vmx_load_host_state(vmx);
1100 1101
	save_nmsrs = 0;
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
1102 1103
	if (is_long_mode(&vmx->vcpu)) {
		index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
1104
		if (index >= 0)
R
Rusty Russell 已提交
1105 1106
			move_msr_up(vmx, index, save_nmsrs++);
		index = __find_msr_index(vmx, MSR_LSTAR);
1107
		if (index >= 0)
R
Rusty Russell 已提交
1108 1109
			move_msr_up(vmx, index, save_nmsrs++);
		index = __find_msr_index(vmx, MSR_CSTAR);
1110
		if (index >= 0)
R
Rusty Russell 已提交
1111
			move_msr_up(vmx, index, save_nmsrs++);
1112 1113 1114
		index = __find_msr_index(vmx, MSR_TSC_AUX);
		if (index >= 0 && vmx->rdtscp_enabled)
			move_msr_up(vmx, index, save_nmsrs++);
1115
		/*
B
Brian Gerst 已提交
1116
		 * MSR_STAR is only needed on long mode guests, and only
1117 1118
		 * if efer.sce is enabled.
		 */
B
Brian Gerst 已提交
1119
		index = __find_msr_index(vmx, MSR_STAR);
1120
		if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
R
Rusty Russell 已提交
1121
			move_msr_up(vmx, index, save_nmsrs++);
1122 1123
	}
#endif
A
Avi Kivity 已提交
1124 1125
	index = __find_msr_index(vmx, MSR_EFER);
	if (index >= 0 && update_transition_efer(vmx, index))
1126
		move_msr_up(vmx, index, save_nmsrs++);
1127

1128
	vmx->save_nmsrs = save_nmsrs;
1129 1130 1131 1132 1133 1134 1135 1136 1137

	if (cpu_has_vmx_msr_bitmap()) {
		if (is_long_mode(&vmx->vcpu))
			msr_bitmap = vmx_msr_bitmap_longmode;
		else
			msr_bitmap = vmx_msr_bitmap_legacy;

		vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
	}
1138 1139
}

A
Avi Kivity 已提交
1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153
/*
 * reads and returns guest's timestamp counter "register"
 * guest_tsc = host_tsc + tsc_offset    -- 21.3
 */
static u64 guest_read_tsc(void)
{
	u64 host_tsc, tsc_offset;

	rdtscll(host_tsc);
	tsc_offset = vmcs_read64(TSC_OFFSET);
	return host_tsc + tsc_offset;
}

/*
1154
 * writes 'offset' into guest's timestamp counter offset register
A
Avi Kivity 已提交
1155
 */
1156
static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
A
Avi Kivity 已提交
1157
{
1158
	vmcs_write64(TSC_OFFSET, offset);
A
Avi Kivity 已提交
1159 1160
}

Z
Zachary Amsden 已提交
1161 1162 1163 1164 1165 1166
static void vmx_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment)
{
	u64 offset = vmcs_read64(TSC_OFFSET);
	vmcs_write64(TSC_OFFSET, offset + adjustment);
}

A
Avi Kivity 已提交
1167 1168 1169 1170 1171 1172 1173 1174
/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
	u64 data;
1175
	struct shared_msr_entry *msr;
A
Avi Kivity 已提交
1176 1177 1178 1179 1180 1181 1182

	if (!pdata) {
		printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
		return -EINVAL;
	}

	switch (msr_index) {
1183
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1184 1185 1186 1187 1188 1189
	case MSR_FS_BASE:
		data = vmcs_readl(GUEST_FS_BASE);
		break;
	case MSR_GS_BASE:
		data = vmcs_readl(GUEST_GS_BASE);
		break;
1190 1191 1192 1193
	case MSR_KERNEL_GS_BASE:
		vmx_load_host_state(to_vmx(vcpu));
		data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
		break;
1194
#endif
A
Avi Kivity 已提交
1195
	case MSR_EFER:
1196
		return kvm_get_msr_common(vcpu, msr_index, pdata);
1197
	case MSR_IA32_TSC:
A
Avi Kivity 已提交
1198 1199 1200 1201 1202 1203
		data = guest_read_tsc();
		break;
	case MSR_IA32_SYSENTER_CS:
		data = vmcs_read32(GUEST_SYSENTER_CS);
		break;
	case MSR_IA32_SYSENTER_EIP:
A
Avi Kivity 已提交
1204
		data = vmcs_readl(GUEST_SYSENTER_EIP);
A
Avi Kivity 已提交
1205 1206
		break;
	case MSR_IA32_SYSENTER_ESP:
A
Avi Kivity 已提交
1207
		data = vmcs_readl(GUEST_SYSENTER_ESP);
A
Avi Kivity 已提交
1208
		break;
1209 1210 1211 1212
	case MSR_TSC_AUX:
		if (!to_vmx(vcpu)->rdtscp_enabled)
			return 1;
		/* Otherwise falls through */
A
Avi Kivity 已提交
1213
	default:
1214
		vmx_load_host_state(to_vmx(vcpu));
R
Rusty Russell 已提交
1215
		msr = find_msr_entry(to_vmx(vcpu), msr_index);
1216
		if (msr) {
1217
			vmx_load_host_state(to_vmx(vcpu));
1218 1219
			data = msr->data;
			break;
A
Avi Kivity 已提交
1220
		}
1221
		return kvm_get_msr_common(vcpu, msr_index, pdata);
A
Avi Kivity 已提交
1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
	}

	*pdata = data;
	return 0;
}

/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
{
1235
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1236
	struct shared_msr_entry *msr;
1237 1238
	int ret = 0;

A
Avi Kivity 已提交
1239
	switch (msr_index) {
1240
	case MSR_EFER:
1241
		vmx_load_host_state(vmx);
1242 1243
		ret = kvm_set_msr_common(vcpu, msr_index, data);
		break;
1244
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1245 1246 1247 1248 1249 1250
	case MSR_FS_BASE:
		vmcs_writel(GUEST_FS_BASE, data);
		break;
	case MSR_GS_BASE:
		vmcs_writel(GUEST_GS_BASE, data);
		break;
1251 1252 1253 1254
	case MSR_KERNEL_GS_BASE:
		vmx_load_host_state(vmx);
		vmx->msr_guest_kernel_gs_base = data;
		break;
A
Avi Kivity 已提交
1255 1256 1257 1258 1259
#endif
	case MSR_IA32_SYSENTER_CS:
		vmcs_write32(GUEST_SYSENTER_CS, data);
		break;
	case MSR_IA32_SYSENTER_EIP:
A
Avi Kivity 已提交
1260
		vmcs_writel(GUEST_SYSENTER_EIP, data);
A
Avi Kivity 已提交
1261 1262
		break;
	case MSR_IA32_SYSENTER_ESP:
A
Avi Kivity 已提交
1263
		vmcs_writel(GUEST_SYSENTER_ESP, data);
A
Avi Kivity 已提交
1264
		break;
1265
	case MSR_IA32_TSC:
1266
		kvm_write_tsc(vcpu, data);
A
Avi Kivity 已提交
1267
		break;
S
Sheng Yang 已提交
1268 1269 1270 1271 1272 1273
	case MSR_IA32_CR_PAT:
		if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
			vmcs_write64(GUEST_IA32_PAT, data);
			vcpu->arch.pat = data;
			break;
		}
1274 1275 1276 1277 1278 1279 1280 1281 1282
		ret = kvm_set_msr_common(vcpu, msr_index, data);
		break;
	case MSR_TSC_AUX:
		if (!vmx->rdtscp_enabled)
			return 1;
		/* Check reserved bit, higher 32 bits should be zero */
		if ((data >> 32) != 0)
			return 1;
		/* Otherwise falls through */
A
Avi Kivity 已提交
1283
	default:
R
Rusty Russell 已提交
1284
		msr = find_msr_entry(vmx, msr_index);
1285
		if (msr) {
1286
			vmx_load_host_state(vmx);
1287 1288
			msr->data = data;
			break;
A
Avi Kivity 已提交
1289
		}
1290
		ret = kvm_set_msr_common(vcpu, msr_index, data);
A
Avi Kivity 已提交
1291 1292
	}

1293
	return ret;
A
Avi Kivity 已提交
1294 1295
}

1296
static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
A
Avi Kivity 已提交
1297
{
1298 1299 1300 1301 1302 1303 1304 1305
	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
	switch (reg) {
	case VCPU_REGS_RSP:
		vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
		break;
	case VCPU_REGS_RIP:
		vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
		break;
A
Avi Kivity 已提交
1306 1307 1308 1309
	case VCPU_EXREG_PDPTR:
		if (enable_ept)
			ept_save_pdptrs(vcpu);
		break;
1310 1311 1312
	default:
		break;
	}
A
Avi Kivity 已提交
1313 1314
}

1315
static void set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
A
Avi Kivity 已提交
1316
{
1317 1318 1319 1320 1321
	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]);
	else
		vmcs_writel(GUEST_DR7, vcpu->arch.dr7);

1322
	update_exception_bitmap(vcpu);
A
Avi Kivity 已提交
1323 1324 1325 1326
}

static __init int cpu_has_kvm_support(void)
{
1327
	return cpu_has_vmx();
A
Avi Kivity 已提交
1328 1329 1330 1331 1332 1333 1334
}

static __init int vmx_disabled_by_bios(void)
{
	u64 msr;

	rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
1335
	if (msr & FEATURE_CONTROL_LOCKED) {
1336
		/* launched w/ TXT and VMX disabled */
1337 1338 1339
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
			&& tboot_enabled())
			return 1;
1340
		/* launched w/o TXT and VMX only enabled w/ TXT */
1341
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
1342
			&& (msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
1343 1344
			&& !tboot_enabled()) {
			printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
1345
				"activate TXT before enabling KVM\n");
1346
			return 1;
1347
		}
1348 1349 1350 1351
		/* launched w/o TXT and VMX disabled */
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
			&& !tboot_enabled())
			return 1;
1352 1353 1354
	}

	return 0;
A
Avi Kivity 已提交
1355 1356
}

1357 1358 1359 1360 1361 1362 1363
static void kvm_cpu_vmxon(u64 addr)
{
	asm volatile (ASM_VMX_VMXON_RAX
			: : "a"(&addr), "m"(addr)
			: "memory", "cc");
}

1364
static int hardware_enable(void *garbage)
A
Avi Kivity 已提交
1365 1366 1367
{
	int cpu = raw_smp_processor_id();
	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
1368
	u64 old, test_bits;
A
Avi Kivity 已提交
1369

1370 1371 1372
	if (read_cr4() & X86_CR4_VMXE)
		return -EBUSY;

1373
	INIT_LIST_HEAD(&per_cpu(vcpus_on_cpu, cpu));
A
Avi Kivity 已提交
1374
	rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
1375 1376 1377 1378 1379 1380 1381

	test_bits = FEATURE_CONTROL_LOCKED;
	test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
	if (tboot_enabled())
		test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;

	if ((old & test_bits) != test_bits) {
A
Avi Kivity 已提交
1382
		/* enable and lock */
1383 1384
		wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
	}
1385
	write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
1386

1387 1388 1389 1390
	if (vmm_exclusive) {
		kvm_cpu_vmxon(phys_addr);
		ept_sync_global();
	}
1391

1392 1393
	store_gdt(&__get_cpu_var(host_gdt));

1394
	return 0;
A
Avi Kivity 已提交
1395 1396
}

1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
static void vmclear_local_vcpus(void)
{
	int cpu = raw_smp_processor_id();
	struct vcpu_vmx *vmx, *n;

	list_for_each_entry_safe(vmx, n, &per_cpu(vcpus_on_cpu, cpu),
				 local_vcpus_link)
		__vcpu_clear(vmx);
}

1407 1408 1409 1410 1411

/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
 * tricks.
 */
static void kvm_cpu_vmxoff(void)
A
Avi Kivity 已提交
1412
{
1413
	asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
A
Avi Kivity 已提交
1414 1415
}

1416 1417
static void hardware_disable(void *garbage)
{
1418 1419 1420 1421
	if (vmm_exclusive) {
		vmclear_local_vcpus();
		kvm_cpu_vmxoff();
	}
1422
	write_cr4(read_cr4() & ~X86_CR4_VMXE);
1423 1424
}

1425
static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
M
Mike Day 已提交
1426
				      u32 msr, u32 *result)
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
{
	u32 vmx_msr_low, vmx_msr_high;
	u32 ctl = ctl_min | ctl_opt;

	rdmsr(msr, vmx_msr_low, vmx_msr_high);

	ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
	ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */

	/* Ensure minimum (required) set of control bits are supported. */
	if (ctl_min & ~ctl)
Y
Yang, Sheng 已提交
1438
		return -EIO;
1439 1440 1441 1442 1443

	*result = ctl;
	return 0;
}

A
Avi Kivity 已提交
1444 1445 1446 1447 1448 1449 1450 1451
static __init bool allow_1_setting(u32 msr, u32 ctl)
{
	u32 vmx_msr_low, vmx_msr_high;

	rdmsr(msr, vmx_msr_low, vmx_msr_high);
	return vmx_msr_high & ctl;
}

Y
Yang, Sheng 已提交
1452
static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
A
Avi Kivity 已提交
1453 1454
{
	u32 vmx_msr_low, vmx_msr_high;
S
Sheng Yang 已提交
1455
	u32 min, opt, min2, opt2;
1456 1457
	u32 _pin_based_exec_control = 0;
	u32 _cpu_based_exec_control = 0;
1458
	u32 _cpu_based_2nd_exec_control = 0;
1459 1460 1461 1462
	u32 _vmexit_control = 0;
	u32 _vmentry_control = 0;

	min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
1463
	opt = PIN_BASED_VIRTUAL_NMIS;
1464 1465
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
				&_pin_based_exec_control) < 0)
Y
Yang, Sheng 已提交
1466
		return -EIO;
1467

1468
	min =
1469 1470 1471 1472
#ifdef CONFIG_X86_64
	      CPU_BASED_CR8_LOAD_EXITING |
	      CPU_BASED_CR8_STORE_EXITING |
#endif
S
Sheng Yang 已提交
1473 1474
	      CPU_BASED_CR3_LOAD_EXITING |
	      CPU_BASED_CR3_STORE_EXITING |
1475 1476
	      CPU_BASED_USE_IO_BITMAPS |
	      CPU_BASED_MOV_DR_EXITING |
M
Marcelo Tosatti 已提交
1477
	      CPU_BASED_USE_TSC_OFFSETING |
1478 1479
	      CPU_BASED_MWAIT_EXITING |
	      CPU_BASED_MONITOR_EXITING |
M
Marcelo Tosatti 已提交
1480
	      CPU_BASED_INVLPG_EXITING;
1481 1482 1483 1484

	if (yield_on_hlt)
		min |= CPU_BASED_HLT_EXITING;

1485
	opt = CPU_BASED_TPR_SHADOW |
S
Sheng Yang 已提交
1486
	      CPU_BASED_USE_MSR_BITMAPS |
1487
	      CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1488 1489
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
				&_cpu_based_exec_control) < 0)
Y
Yang, Sheng 已提交
1490
		return -EIO;
1491 1492 1493 1494 1495
#ifdef CONFIG_X86_64
	if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
		_cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
					   ~CPU_BASED_CR8_STORE_EXITING;
#endif
1496
	if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
S
Sheng Yang 已提交
1497 1498
		min2 = 0;
		opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
1499
			SECONDARY_EXEC_WBINVD_EXITING |
S
Sheng Yang 已提交
1500
			SECONDARY_EXEC_ENABLE_VPID |
1501
			SECONDARY_EXEC_ENABLE_EPT |
1502
			SECONDARY_EXEC_UNRESTRICTED_GUEST |
1503 1504
			SECONDARY_EXEC_PAUSE_LOOP_EXITING |
			SECONDARY_EXEC_RDTSCP;
S
Sheng Yang 已提交
1505 1506
		if (adjust_vmx_controls(min2, opt2,
					MSR_IA32_VMX_PROCBASED_CTLS2,
1507 1508 1509 1510 1511 1512 1513 1514
					&_cpu_based_2nd_exec_control) < 0)
			return -EIO;
	}
#ifndef CONFIG_X86_64
	if (!(_cpu_based_2nd_exec_control &
				SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
		_cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
#endif
S
Sheng Yang 已提交
1515
	if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
M
Marcelo Tosatti 已提交
1516 1517
		/* CR3 accesses and invlpg don't need to cause VM Exits when EPT
		   enabled */
1518 1519 1520
		_cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
					     CPU_BASED_CR3_STORE_EXITING |
					     CPU_BASED_INVLPG_EXITING);
S
Sheng Yang 已提交
1521 1522 1523
		rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
		      vmx_capability.ept, vmx_capability.vpid);
	}
1524 1525 1526 1527 1528

	min = 0;
#ifdef CONFIG_X86_64
	min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
#endif
S
Sheng Yang 已提交
1529
	opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT;
1530 1531
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
				&_vmexit_control) < 0)
Y
Yang, Sheng 已提交
1532
		return -EIO;
1533

S
Sheng Yang 已提交
1534 1535
	min = 0;
	opt = VM_ENTRY_LOAD_IA32_PAT;
1536 1537
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
				&_vmentry_control) < 0)
Y
Yang, Sheng 已提交
1538
		return -EIO;
A
Avi Kivity 已提交
1539

N
Nguyen Anh Quynh 已提交
1540
	rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
1541 1542 1543

	/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
	if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
Y
Yang, Sheng 已提交
1544
		return -EIO;
1545 1546 1547 1548

#ifdef CONFIG_X86_64
	/* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
	if (vmx_msr_high & (1u<<16))
Y
Yang, Sheng 已提交
1549
		return -EIO;
1550 1551 1552 1553
#endif

	/* Require Write-Back (WB) memory type for VMCS accesses. */
	if (((vmx_msr_high >> 18) & 15) != 6)
Y
Yang, Sheng 已提交
1554
		return -EIO;
1555

Y
Yang, Sheng 已提交
1556 1557 1558
	vmcs_conf->size = vmx_msr_high & 0x1fff;
	vmcs_conf->order = get_order(vmcs_config.size);
	vmcs_conf->revision_id = vmx_msr_low;
1559

Y
Yang, Sheng 已提交
1560 1561
	vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
	vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
1562
	vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
Y
Yang, Sheng 已提交
1563 1564
	vmcs_conf->vmexit_ctrl         = _vmexit_control;
	vmcs_conf->vmentry_ctrl        = _vmentry_control;
1565

A
Avi Kivity 已提交
1566 1567 1568 1569 1570 1571
	cpu_has_load_ia32_efer =
		allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
				VM_ENTRY_LOAD_IA32_EFER)
		&& allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
				   VM_EXIT_LOAD_IA32_EFER);

1572
	return 0;
N
Nguyen Anh Quynh 已提交
1573
}
A
Avi Kivity 已提交
1574 1575 1576 1577 1578 1579 1580

static struct vmcs *alloc_vmcs_cpu(int cpu)
{
	int node = cpu_to_node(cpu);
	struct page *pages;
	struct vmcs *vmcs;

1581
	pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order);
A
Avi Kivity 已提交
1582 1583 1584
	if (!pages)
		return NULL;
	vmcs = page_address(pages);
1585 1586
	memset(vmcs, 0, vmcs_config.size);
	vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
A
Avi Kivity 已提交
1587 1588 1589 1590 1591
	return vmcs;
}

static struct vmcs *alloc_vmcs(void)
{
1592
	return alloc_vmcs_cpu(raw_smp_processor_id());
A
Avi Kivity 已提交
1593 1594 1595 1596
}

static void free_vmcs(struct vmcs *vmcs)
{
1597
	free_pages((unsigned long)vmcs, vmcs_config.order);
A
Avi Kivity 已提交
1598 1599
}

1600
static void free_kvm_area(void)
A
Avi Kivity 已提交
1601 1602 1603
{
	int cpu;

Z
Zachary Amsden 已提交
1604
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
1605
		free_vmcs(per_cpu(vmxarea, cpu));
Z
Zachary Amsden 已提交
1606 1607
		per_cpu(vmxarea, cpu) = NULL;
	}
A
Avi Kivity 已提交
1608 1609 1610 1611 1612 1613
}

static __init int alloc_kvm_area(void)
{
	int cpu;

Z
Zachary Amsden 已提交
1614
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629
		struct vmcs *vmcs;

		vmcs = alloc_vmcs_cpu(cpu);
		if (!vmcs) {
			free_kvm_area();
			return -ENOMEM;
		}

		per_cpu(vmxarea, cpu) = vmcs;
	}
	return 0;
}

static __init int hardware_setup(void)
{
Y
Yang, Sheng 已提交
1630 1631
	if (setup_vmcs_config(&vmcs_config) < 0)
		return -EIO;
1632 1633 1634 1635

	if (boot_cpu_has(X86_FEATURE_NX))
		kvm_enable_efer_bits(EFER_NX);

S
Sheng Yang 已提交
1636 1637 1638
	if (!cpu_has_vmx_vpid())
		enable_vpid = 0;

1639 1640
	if (!cpu_has_vmx_ept() ||
	    !cpu_has_vmx_ept_4levels()) {
S
Sheng Yang 已提交
1641
		enable_ept = 0;
1642 1643 1644 1645 1646
		enable_unrestricted_guest = 0;
	}

	if (!cpu_has_vmx_unrestricted_guest())
		enable_unrestricted_guest = 0;
S
Sheng Yang 已提交
1647 1648 1649 1650

	if (!cpu_has_vmx_flexpriority())
		flexpriority_enabled = 0;

1651 1652 1653
	if (!cpu_has_vmx_tpr_shadow())
		kvm_x86_ops->update_cr8_intercept = NULL;

1654 1655 1656
	if (enable_ept && !cpu_has_vmx_ept_2m_page())
		kvm_disable_largepages();

1657 1658 1659
	if (!cpu_has_vmx_ple())
		ple_gap = 0;

A
Avi Kivity 已提交
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671
	return alloc_kvm_area();
}

static __exit void hardware_unsetup(void)
{
	free_kvm_area();
}

static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
{
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];

1672
	if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
A
Avi Kivity 已提交
1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686
		vmcs_write16(sf->selector, save->selector);
		vmcs_writel(sf->base, save->base);
		vmcs_write32(sf->limit, save->limit);
		vmcs_write32(sf->ar_bytes, save->ar);
	} else {
		u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
			<< AR_DPL_SHIFT;
		vmcs_write32(sf->ar_bytes, 0x93 | dpl);
	}
}

static void enter_pmode(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
1687
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
1688

1689
	vmx->emulation_required = 1;
1690
	vmx->rmode.vm86_active = 0;
A
Avi Kivity 已提交
1691

1692
	vmcs_write16(GUEST_TR_SELECTOR, vmx->rmode.tr.selector);
1693 1694 1695
	vmcs_writel(GUEST_TR_BASE, vmx->rmode.tr.base);
	vmcs_write32(GUEST_TR_LIMIT, vmx->rmode.tr.limit);
	vmcs_write32(GUEST_TR_AR_BYTES, vmx->rmode.tr.ar);
A
Avi Kivity 已提交
1696 1697

	flags = vmcs_readl(GUEST_RFLAGS);
1698 1699
	flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
	flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
A
Avi Kivity 已提交
1700 1701
	vmcs_writel(GUEST_RFLAGS, flags);

1702 1703
	vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
			(vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
A
Avi Kivity 已提交
1704 1705 1706

	update_exception_bitmap(vcpu);

1707 1708 1709
	if (emulate_invalid_guest_state)
		return;

1710 1711 1712 1713
	fix_pmode_dataseg(VCPU_SREG_ES, &vmx->rmode.es);
	fix_pmode_dataseg(VCPU_SREG_DS, &vmx->rmode.ds);
	fix_pmode_dataseg(VCPU_SREG_GS, &vmx->rmode.gs);
	fix_pmode_dataseg(VCPU_SREG_FS, &vmx->rmode.fs);
A
Avi Kivity 已提交
1714 1715 1716 1717 1718 1719 1720 1721 1722

	vmcs_write16(GUEST_SS_SELECTOR, 0);
	vmcs_write32(GUEST_SS_AR_BYTES, 0x93);

	vmcs_write16(GUEST_CS_SELECTOR,
		     vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
	vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
}

M
Mike Day 已提交
1723
static gva_t rmode_tss_base(struct kvm *kvm)
A
Avi Kivity 已提交
1724
{
1725
	if (!kvm->arch.tss_addr) {
1726 1727 1728
		struct kvm_memslots *slots;
		gfn_t base_gfn;

1729
		slots = kvm_memslots(kvm);
1730
		base_gfn = slots->memslots[0].base_gfn +
1731
				 kvm->memslots->memslots[0].npages - 3;
1732 1733
		return base_gfn << PAGE_SHIFT;
	}
1734
	return kvm->arch.tss_addr;
A
Avi Kivity 已提交
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
}

static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
{
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];

	save->selector = vmcs_read16(sf->selector);
	save->base = vmcs_readl(sf->base);
	save->limit = vmcs_read32(sf->limit);
	save->ar = vmcs_read32(sf->ar_bytes);
1745
	vmcs_write16(sf->selector, save->base >> 4);
1746
	vmcs_write32(sf->base, save->base & 0xffff0);
A
Avi Kivity 已提交
1747 1748
	vmcs_write32(sf->limit, 0xffff);
	vmcs_write32(sf->ar_bytes, 0xf3);
1749 1750 1751 1752
	if (save->base & 0xf)
		printk_once(KERN_WARNING "kvm: segment base is not paragraph"
			    " aligned when entering protected mode (seg=%d)",
			    seg);
A
Avi Kivity 已提交
1753 1754 1755 1756 1757
}

static void enter_rmode(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
1758
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
1759

1760 1761 1762
	if (enable_unrestricted_guest)
		return;

1763
	vmx->emulation_required = 1;
1764
	vmx->rmode.vm86_active = 1;
A
Avi Kivity 已提交
1765

1766
	vmx->rmode.tr.selector = vmcs_read16(GUEST_TR_SELECTOR);
1767
	vmx->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
A
Avi Kivity 已提交
1768 1769
	vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));

1770
	vmx->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
A
Avi Kivity 已提交
1771 1772
	vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);

1773
	vmx->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
A
Avi Kivity 已提交
1774 1775 1776
	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);

	flags = vmcs_readl(GUEST_RFLAGS);
1777
	vmx->rmode.save_rflags = flags;
A
Avi Kivity 已提交
1778

1779
	flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
A
Avi Kivity 已提交
1780 1781

	vmcs_writel(GUEST_RFLAGS, flags);
1782
	vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
A
Avi Kivity 已提交
1783 1784
	update_exception_bitmap(vcpu);

1785 1786 1787
	if (emulate_invalid_guest_state)
		goto continue_rmode;

A
Avi Kivity 已提交
1788 1789 1790 1791 1792
	vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4);
	vmcs_write32(GUEST_SS_LIMIT, 0xffff);
	vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);

	vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
1793
	vmcs_write32(GUEST_CS_LIMIT, 0xffff);
1794 1795
	if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
		vmcs_writel(GUEST_CS_BASE, 0xf0000);
A
Avi Kivity 已提交
1796 1797
	vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);

1798 1799 1800 1801
	fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.es);
	fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.ds);
	fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.gs);
	fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.fs);
1802

1803
continue_rmode:
1804
	kvm_mmu_reset_context(vcpu);
1805
	init_rmode(vcpu->kvm);
A
Avi Kivity 已提交
1806 1807
}

1808 1809 1810
static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1811 1812 1813 1814
	struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);

	if (!msr)
		return;
1815

1816 1817 1818 1819 1820
	/*
	 * Force kernel_gs_base reloading before EFER changes, as control
	 * of this msr depends on is_long_mode().
	 */
	vmx_load_host_state(to_vmx(vcpu));
1821
	vcpu->arch.efer = efer;
1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836
	if (efer & EFER_LMA) {
		vmcs_write32(VM_ENTRY_CONTROLS,
			     vmcs_read32(VM_ENTRY_CONTROLS) |
			     VM_ENTRY_IA32E_MODE);
		msr->data = efer;
	} else {
		vmcs_write32(VM_ENTRY_CONTROLS,
			     vmcs_read32(VM_ENTRY_CONTROLS) &
			     ~VM_ENTRY_IA32E_MODE);

		msr->data = efer & ~EFER_LME;
	}
	setup_msrs(vmx);
}

1837
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1838 1839 1840 1841 1842 1843 1844 1845

static void enter_lmode(struct kvm_vcpu *vcpu)
{
	u32 guest_tr_ar;

	guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
	if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
		printk(KERN_DEBUG "%s: tss fixup for long mode. \n",
1846
		       __func__);
A
Avi Kivity 已提交
1847 1848 1849 1850
		vmcs_write32(GUEST_TR_AR_BYTES,
			     (guest_tr_ar & ~AR_TYPE_MASK)
			     | AR_TYPE_BUSY_64_TSS);
	}
1851
	vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
A
Avi Kivity 已提交
1852 1853 1854 1855 1856 1857
}

static void exit_lmode(struct kvm_vcpu *vcpu)
{
	vmcs_write32(VM_ENTRY_CONTROLS,
		     vmcs_read32(VM_ENTRY_CONTROLS)
1858
		     & ~VM_ENTRY_IA32E_MODE);
1859
	vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
A
Avi Kivity 已提交
1860 1861 1862 1863
}

#endif

1864 1865
static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
{
1866
	vpid_sync_context(to_vmx(vcpu));
1867 1868 1869
	if (enable_ept) {
		if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
			return;
1870
		ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
1871
	}
1872 1873
}

1874 1875 1876 1877 1878 1879 1880 1881
static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
{
	ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;

	vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
	vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
}

1882 1883 1884 1885 1886 1887 1888
static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
{
	if (enable_ept && is_paging(vcpu))
		vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
}

1889
static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1890
{
1891 1892 1893 1894
	ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;

	vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
	vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
1895 1896
}

1897 1898
static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
1899 1900 1901 1902
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_dirty))
		return;

1903
	if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1904 1905 1906 1907
		vmcs_write64(GUEST_PDPTR0, vcpu->arch.mmu.pdptrs[0]);
		vmcs_write64(GUEST_PDPTR1, vcpu->arch.mmu.pdptrs[1]);
		vmcs_write64(GUEST_PDPTR2, vcpu->arch.mmu.pdptrs[2]);
		vmcs_write64(GUEST_PDPTR3, vcpu->arch.mmu.pdptrs[3]);
1908 1909 1910
	}
}

1911 1912 1913
static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
{
	if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1914 1915 1916 1917
		vcpu->arch.mmu.pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
		vcpu->arch.mmu.pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
		vcpu->arch.mmu.pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
		vcpu->arch.mmu.pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
1918
	}
A
Avi Kivity 已提交
1919 1920 1921 1922 1923

	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
1924 1925
}

1926 1927 1928 1929 1930 1931
static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);

static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
					unsigned long cr0,
					struct kvm_vcpu *vcpu)
{
1932
	vmx_decache_cr3(vcpu);
1933 1934 1935
	if (!(cr0 & X86_CR0_PG)) {
		/* From paging/starting to nonpaging */
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
1936
			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
1937 1938 1939
			     (CPU_BASED_CR3_LOAD_EXITING |
			      CPU_BASED_CR3_STORE_EXITING));
		vcpu->arch.cr0 = cr0;
1940
		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
1941 1942 1943
	} else if (!is_paging(vcpu)) {
		/* From nonpaging to paging */
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
1944
			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
1945 1946 1947
			     ~(CPU_BASED_CR3_LOAD_EXITING |
			       CPU_BASED_CR3_STORE_EXITING));
		vcpu->arch.cr0 = cr0;
1948
		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
1949
	}
1950 1951 1952

	if (!(cr0 & X86_CR0_WP))
		*hw_cr0 &= ~X86_CR0_WP;
1953 1954
}

A
Avi Kivity 已提交
1955 1956
static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
1957
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1958 1959 1960 1961 1962 1963 1964
	unsigned long hw_cr0;

	if (enable_unrestricted_guest)
		hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST)
			| KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
	else
		hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON;
1965

1966
	if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
A
Avi Kivity 已提交
1967 1968
		enter_pmode(vcpu);

1969
	if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
A
Avi Kivity 已提交
1970 1971
		enter_rmode(vcpu);

1972
#ifdef CONFIG_X86_64
1973
	if (vcpu->arch.efer & EFER_LME) {
1974
		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
A
Avi Kivity 已提交
1975
			enter_lmode(vcpu);
1976
		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
A
Avi Kivity 已提交
1977 1978 1979 1980
			exit_lmode(vcpu);
	}
#endif

1981
	if (enable_ept)
1982 1983
		ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);

1984
	if (!vcpu->fpu_active)
1985
		hw_cr0 |= X86_CR0_TS | X86_CR0_MP;
1986

A
Avi Kivity 已提交
1987
	vmcs_writel(CR0_READ_SHADOW, cr0);
1988
	vmcs_writel(GUEST_CR0, hw_cr0);
1989
	vcpu->arch.cr0 = cr0;
A
Avi Kivity 已提交
1990 1991
}

1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003
static u64 construct_eptp(unsigned long root_hpa)
{
	u64 eptp;

	/* TODO write the value reading from MSR */
	eptp = VMX_EPT_DEFAULT_MT |
		VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
	eptp |= (root_hpa & PAGE_MASK);

	return eptp;
}

A
Avi Kivity 已提交
2004 2005
static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
{
2006 2007 2008 2009
	unsigned long guest_cr3;
	u64 eptp;

	guest_cr3 = cr3;
2010
	if (enable_ept) {
2011 2012
		eptp = construct_eptp(cr3);
		vmcs_write64(EPT_POINTER, eptp);
2013
		guest_cr3 = is_paging(vcpu) ? kvm_read_cr3(vcpu) :
2014
			vcpu->kvm->arch.ept_identity_map_addr;
2015
		ept_load_pdptrs(vcpu);
2016 2017
	}

2018
	vmx_flush_tlb(vcpu);
2019
	vmcs_writel(GUEST_CR3, guest_cr3);
A
Avi Kivity 已提交
2020 2021 2022 2023
}

static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
2024
	unsigned long hw_cr4 = cr4 | (to_vmx(vcpu)->rmode.vm86_active ?
2025 2026
		    KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);

2027
	vcpu->arch.cr4 = cr4;
2028 2029 2030 2031 2032 2033 2034 2035
	if (enable_ept) {
		if (!is_paging(vcpu)) {
			hw_cr4 &= ~X86_CR4_PAE;
			hw_cr4 |= X86_CR4_PSE;
		} else if (!(cr4 & X86_CR4_PAE)) {
			hw_cr4 &= ~X86_CR4_PAE;
		}
	}
2036 2037 2038

	vmcs_writel(CR4_READ_SHADOW, cr4);
	vmcs_writel(GUEST_CR4, hw_cr4);
A
Avi Kivity 已提交
2039 2040 2041 2042 2043
}

static void vmx_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
2044
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
2045
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2046
	struct kvm_save_segment *save;
A
Avi Kivity 已提交
2047 2048
	u32 ar;

2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069
	if (vmx->rmode.vm86_active
	    && (seg == VCPU_SREG_TR || seg == VCPU_SREG_ES
		|| seg == VCPU_SREG_DS || seg == VCPU_SREG_FS
		|| seg == VCPU_SREG_GS)
	    && !emulate_invalid_guest_state) {
		switch (seg) {
		case VCPU_SREG_TR: save = &vmx->rmode.tr; break;
		case VCPU_SREG_ES: save = &vmx->rmode.es; break;
		case VCPU_SREG_DS: save = &vmx->rmode.ds; break;
		case VCPU_SREG_FS: save = &vmx->rmode.fs; break;
		case VCPU_SREG_GS: save = &vmx->rmode.gs; break;
		default: BUG();
		}
		var->selector = save->selector;
		var->base = save->base;
		var->limit = save->limit;
		ar = save->ar;
		if (seg == VCPU_SREG_TR
		    || var->selector == vmcs_read16(sf->selector))
			goto use_saved_rmode_seg;
	}
A
Avi Kivity 已提交
2070 2071 2072 2073
	var->base = vmcs_readl(sf->base);
	var->limit = vmcs_read32(sf->limit);
	var->selector = vmcs_read16(sf->selector);
	ar = vmcs_read32(sf->ar_bytes);
2074
use_saved_rmode_seg:
2075
	if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state)
A
Avi Kivity 已提交
2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087
		ar = 0;
	var->type = ar & 15;
	var->s = (ar >> 4) & 1;
	var->dpl = (ar >> 5) & 3;
	var->present = (ar >> 7) & 1;
	var->avl = (ar >> 12) & 1;
	var->l = (ar >> 13) & 1;
	var->db = (ar >> 14) & 1;
	var->g = (ar >> 15) & 1;
	var->unusable = (ar >> 16) & 1;
}

2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099
static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
	struct kvm_segment s;

	if (to_vmx(vcpu)->rmode.vm86_active) {
		vmx_get_segment(vcpu, &s, seg);
		return s.base;
	}
	return vmcs_readl(sf->base);
}

2100 2101
static int vmx_get_cpl(struct kvm_vcpu *vcpu)
{
2102
	if (!is_protmode(vcpu))
2103 2104 2105 2106 2107
		return 0;

	if (vmx_get_rflags(vcpu) & X86_EFLAGS_VM) /* if virtual 8086 */
		return 3;

A
Avi Kivity 已提交
2108
	return vmcs_read16(GUEST_CS_SELECTOR) & 3;
2109 2110
}

2111
static u32 vmx_segment_access_rights(struct kvm_segment *var)
A
Avi Kivity 已提交
2112 2113 2114
{
	u32 ar;

2115
	if (var->unusable)
A
Avi Kivity 已提交
2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126
		ar = 1 << 16;
	else {
		ar = var->type & 15;
		ar |= (var->s & 1) << 4;
		ar |= (var->dpl & 3) << 5;
		ar |= (var->present & 1) << 7;
		ar |= (var->avl & 1) << 12;
		ar |= (var->l & 1) << 13;
		ar |= (var->db & 1) << 14;
		ar |= (var->g & 1) << 15;
	}
2127 2128
	if (ar == 0) /* a 0 value means unusable */
		ar = AR_UNUSABLE_MASK;
2129 2130 2131 2132 2133 2134 2135

	return ar;
}

static void vmx_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
2136
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2137 2138 2139
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
	u32 ar;

2140 2141 2142 2143 2144
	if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) {
		vmx->rmode.tr.selector = var->selector;
		vmx->rmode.tr.base = var->base;
		vmx->rmode.tr.limit = var->limit;
		vmx->rmode.tr.ar = vmx_segment_access_rights(var);
2145 2146 2147 2148 2149
		return;
	}
	vmcs_writel(sf->base, var->base);
	vmcs_write32(sf->limit, var->limit);
	vmcs_write16(sf->selector, var->selector);
2150
	if (vmx->rmode.vm86_active && var->s) {
2151 2152 2153 2154 2155 2156 2157 2158
		/*
		 * Hack real-mode segments into vm86 compatibility.
		 */
		if (var->base == 0xffff0000 && var->selector == 0xf000)
			vmcs_writel(sf->base, 0xf0000);
		ar = 0xf3;
	} else
		ar = vmx_segment_access_rights(var);
2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173

	/*
	 *   Fix the "Accessed" bit in AR field of segment registers for older
	 * qemu binaries.
	 *   IA32 arch specifies that at the time of processor reset the
	 * "Accessed" bit in the AR field of segment registers is 1. And qemu
	 * is setting it to 0 in the usedland code. This causes invalid guest
	 * state vmexit when "unrestricted guest" mode is turned on.
	 *    Fix for this setup issue in cpu_reset is being pushed in the qemu
	 * tree. Newer qemu binaries with that qemu fix would not need this
	 * kvm hack.
	 */
	if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
		ar |= 0x1; /* Accessed */

A
Avi Kivity 已提交
2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184
	vmcs_write32(sf->ar_bytes, ar);
}

static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	u32 ar = vmcs_read32(GUEST_CS_AR_BYTES);

	*db = (ar >> 14) & 1;
	*l = (ar >> 13) & 1;
}

2185
static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2186
{
2187 2188
	dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
	dt->address = vmcs_readl(GUEST_IDTR_BASE);
A
Avi Kivity 已提交
2189 2190
}

2191
static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2192
{
2193 2194
	vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
	vmcs_writel(GUEST_IDTR_BASE, dt->address);
A
Avi Kivity 已提交
2195 2196
}

2197
static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2198
{
2199 2200
	dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
	dt->address = vmcs_readl(GUEST_GDTR_BASE);
A
Avi Kivity 已提交
2201 2202
}

2203
static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2204
{
2205 2206
	vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
	vmcs_writel(GUEST_GDTR_BASE, dt->address);
A
Avi Kivity 已提交
2207 2208
}

2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234
static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment var;
	u32 ar;

	vmx_get_segment(vcpu, &var, seg);
	ar = vmx_segment_access_rights(&var);

	if (var.base != (var.selector << 4))
		return false;
	if (var.limit != 0xffff)
		return false;
	if (ar != 0xf3)
		return false;

	return true;
}

static bool code_segment_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	unsigned int cs_rpl;

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs_rpl = cs.selector & SELECTOR_RPL_MASK;

2235 2236
	if (cs.unusable)
		return false;
2237 2238 2239 2240
	if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK))
		return false;
	if (!cs.s)
		return false;
2241
	if (cs.type & AR_TYPE_WRITEABLE_MASK) {
2242 2243
		if (cs.dpl > cs_rpl)
			return false;
2244
	} else {
2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262
		if (cs.dpl != cs_rpl)
			return false;
	}
	if (!cs.present)
		return false;

	/* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
	return true;
}

static bool stack_segment_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment ss;
	unsigned int ss_rpl;

	vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
	ss_rpl = ss.selector & SELECTOR_RPL_MASK;

2263 2264 2265
	if (ss.unusable)
		return true;
	if (ss.type != 3 && ss.type != 7)
2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284
		return false;
	if (!ss.s)
		return false;
	if (ss.dpl != ss_rpl) /* DPL != RPL */
		return false;
	if (!ss.present)
		return false;

	return true;
}

static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment var;
	unsigned int rpl;

	vmx_get_segment(vcpu, &var, seg);
	rpl = var.selector & SELECTOR_RPL_MASK;

2285 2286
	if (var.unusable)
		return true;
2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307
	if (!var.s)
		return false;
	if (!var.present)
		return false;
	if (~var.type & (AR_TYPE_CODE_MASK|AR_TYPE_WRITEABLE_MASK)) {
		if (var.dpl < rpl) /* DPL < RPL */
			return false;
	}

	/* TODO: Add other members to kvm_segment_field to allow checking for other access
	 * rights flags
	 */
	return true;
}

static bool tr_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment tr;

	vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);

2308 2309
	if (tr.unusable)
		return false;
2310 2311
	if (tr.selector & SELECTOR_TI_MASK)	/* TI = 1 */
		return false;
2312
	if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325
		return false;
	if (!tr.present)
		return false;

	return true;
}

static bool ldtr_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment ldtr;

	vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);

2326 2327
	if (ldtr.unusable)
		return true;
2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356
	if (ldtr.selector & SELECTOR_TI_MASK)	/* TI = 1 */
		return false;
	if (ldtr.type != 2)
		return false;
	if (!ldtr.present)
		return false;

	return true;
}

static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs, ss;

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);

	return ((cs.selector & SELECTOR_RPL_MASK) ==
		 (ss.selector & SELECTOR_RPL_MASK));
}

/*
 * Check if guest state is valid. Returns true if valid, false if
 * not.
 * We assume that registers are always usable
 */
static bool guest_state_valid(struct kvm_vcpu *vcpu)
{
	/* real mode guest state checks */
2357
	if (!is_protmode(vcpu)) {
2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398
		if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
			return false;
	} else {
	/* protected mode guest state checks */
		if (!cs_ss_rpl_check(vcpu))
			return false;
		if (!code_segment_valid(vcpu))
			return false;
		if (!stack_segment_valid(vcpu))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_DS))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_ES))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_FS))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_GS))
			return false;
		if (!tr_valid(vcpu))
			return false;
		if (!ldtr_valid(vcpu))
			return false;
	}
	/* TODO:
	 * - Add checks on RIP
	 * - Add checks on RFLAGS
	 */

	return true;
}

M
Mike Day 已提交
2399
static int init_rmode_tss(struct kvm *kvm)
A
Avi Kivity 已提交
2400 2401
{
	gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
2402
	u16 data = 0;
2403
	int ret = 0;
2404
	int r;
A
Avi Kivity 已提交
2405

2406 2407
	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
	if (r < 0)
2408
		goto out;
2409
	data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
2410 2411
	r = kvm_write_guest_page(kvm, fn++, &data,
			TSS_IOPB_BASE_OFFSET, sizeof(u16));
2412
	if (r < 0)
2413
		goto out;
2414 2415
	r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
	if (r < 0)
2416
		goto out;
2417 2418
	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
	if (r < 0)
2419
		goto out;
2420
	data = ~0;
2421 2422 2423
	r = kvm_write_guest_page(kvm, fn, &data,
				 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
				 sizeof(u8));
2424
	if (r < 0)
2425 2426 2427 2428 2429
		goto out;

	ret = 1;
out:
	return ret;
A
Avi Kivity 已提交
2430 2431
}

2432 2433 2434 2435 2436 2437
static int init_rmode_identity_map(struct kvm *kvm)
{
	int i, r, ret;
	pfn_t identity_map_pfn;
	u32 tmp;

2438
	if (!enable_ept)
2439 2440 2441 2442 2443 2444 2445 2446 2447
		return 1;
	if (unlikely(!kvm->arch.ept_identity_pagetable)) {
		printk(KERN_ERR "EPT: identity-mapping pagetable "
			"haven't been allocated!\n");
		return 0;
	}
	if (likely(kvm->arch.ept_identity_pagetable_done))
		return 1;
	ret = 0;
2448
	identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466
	r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
	if (r < 0)
		goto out;
	/* Set up identity-mapping pagetable for EPT in real mode */
	for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
		tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
			_PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
		r = kvm_write_guest_page(kvm, identity_map_pfn,
				&tmp, i * sizeof(tmp), sizeof(tmp));
		if (r < 0)
			goto out;
	}
	kvm->arch.ept_identity_pagetable_done = true;
	ret = 1;
out:
	return ret;
}

A
Avi Kivity 已提交
2467 2468 2469
static void seg_setup(int seg)
{
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2470
	unsigned int ar;
A
Avi Kivity 已提交
2471 2472 2473 2474

	vmcs_write16(sf->selector, 0);
	vmcs_writel(sf->base, 0);
	vmcs_write32(sf->limit, 0xffff);
2475 2476 2477 2478 2479 2480 2481 2482
	if (enable_unrestricted_guest) {
		ar = 0x93;
		if (seg == VCPU_SREG_CS)
			ar |= 0x08; /* code segment */
	} else
		ar = 0xf3;

	vmcs_write32(sf->ar_bytes, ar);
A
Avi Kivity 已提交
2483 2484
}

2485 2486 2487 2488 2489
static int alloc_apic_access_page(struct kvm *kvm)
{
	struct kvm_userspace_memory_region kvm_userspace_mem;
	int r = 0;

2490
	mutex_lock(&kvm->slots_lock);
2491
	if (kvm->arch.apic_access_page)
2492 2493 2494 2495 2496 2497 2498 2499
		goto out;
	kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
	kvm_userspace_mem.flags = 0;
	kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL;
	kvm_userspace_mem.memory_size = PAGE_SIZE;
	r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
	if (r)
		goto out;
2500

2501
	kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00);
2502
out:
2503
	mutex_unlock(&kvm->slots_lock);
2504 2505 2506
	return r;
}

2507 2508 2509 2510 2511
static int alloc_identity_pagetable(struct kvm *kvm)
{
	struct kvm_userspace_memory_region kvm_userspace_mem;
	int r = 0;

2512
	mutex_lock(&kvm->slots_lock);
2513 2514 2515 2516
	if (kvm->arch.ept_identity_pagetable)
		goto out;
	kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
	kvm_userspace_mem.flags = 0;
2517 2518
	kvm_userspace_mem.guest_phys_addr =
		kvm->arch.ept_identity_map_addr;
2519 2520 2521 2522 2523 2524
	kvm_userspace_mem.memory_size = PAGE_SIZE;
	r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
	if (r)
		goto out;

	kvm->arch.ept_identity_pagetable = gfn_to_page(kvm,
2525
			kvm->arch.ept_identity_map_addr >> PAGE_SHIFT);
2526
out:
2527
	mutex_unlock(&kvm->slots_lock);
2528 2529 2530
	return r;
}

2531 2532 2533 2534 2535
static void allocate_vpid(struct vcpu_vmx *vmx)
{
	int vpid;

	vmx->vpid = 0;
2536
	if (!enable_vpid)
2537 2538 2539 2540 2541 2542 2543 2544 2545 2546
		return;
	spin_lock(&vmx_vpid_lock);
	vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
	if (vpid < VMX_NR_VPIDS) {
		vmx->vpid = vpid;
		__set_bit(vpid, vmx_vpid_bitmap);
	}
	spin_unlock(&vmx_vpid_lock);
}

2547 2548 2549 2550 2551 2552 2553 2554 2555 2556
static void free_vpid(struct vcpu_vmx *vmx)
{
	if (!enable_vpid)
		return;
	spin_lock(&vmx_vpid_lock);
	if (vmx->vpid != 0)
		__clear_bit(vmx->vpid, vmx_vpid_bitmap);
	spin_unlock(&vmx_vpid_lock);
}

2557
static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, u32 msr)
S
Sheng Yang 已提交
2558
{
2559
	int f = sizeof(unsigned long);
S
Sheng Yang 已提交
2560 2561 2562 2563 2564 2565 2566 2567 2568 2569

	if (!cpu_has_vmx_msr_bitmap())
		return;

	/*
	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
	 * have the write-low and read-high bitmap offsets the wrong way round.
	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
	 */
	if (msr <= 0x1fff) {
2570 2571
		__clear_bit(msr, msr_bitmap + 0x000 / f); /* read-low */
		__clear_bit(msr, msr_bitmap + 0x800 / f); /* write-low */
S
Sheng Yang 已提交
2572 2573
	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
		msr &= 0x1fff;
2574 2575
		__clear_bit(msr, msr_bitmap + 0x400 / f); /* read-high */
		__clear_bit(msr, msr_bitmap + 0xc00 / f); /* write-high */
S
Sheng Yang 已提交
2576 2577 2578
	}
}

2579 2580 2581 2582 2583 2584 2585
static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
{
	if (!longmode_only)
		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy, msr);
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode, msr);
}

A
Avi Kivity 已提交
2586 2587 2588
/*
 * Sets up the vmcs for emulated real mode.
 */
R
Rusty Russell 已提交
2589
static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
A
Avi Kivity 已提交
2590
{
S
Sheng Yang 已提交
2591
	u32 host_sysenter_cs, msr_low, msr_high;
A
Avi Kivity 已提交
2592
	u32 junk;
2593
	u64 host_pat;
A
Avi Kivity 已提交
2594
	unsigned long a;
2595
	struct desc_ptr dt;
A
Avi Kivity 已提交
2596
	int i;
2597
	unsigned long kvm_vmx_return;
2598
	u32 exec_control;
A
Avi Kivity 已提交
2599 2600

	/* I/O */
2601 2602
	vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
	vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
A
Avi Kivity 已提交
2603

S
Sheng Yang 已提交
2604
	if (cpu_has_vmx_msr_bitmap())
2605
		vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
S
Sheng Yang 已提交
2606

A
Avi Kivity 已提交
2607 2608 2609
	vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */

	/* Control */
2610 2611
	vmcs_write32(PIN_BASED_VM_EXEC_CONTROL,
		vmcs_config.pin_based_exec_ctrl);
2612 2613 2614 2615 2616 2617 2618 2619 2620

	exec_control = vmcs_config.cpu_based_exec_ctrl;
	if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
		exec_control &= ~CPU_BASED_TPR_SHADOW;
#ifdef CONFIG_X86_64
		exec_control |= CPU_BASED_CR8_STORE_EXITING |
				CPU_BASED_CR8_LOAD_EXITING;
#endif
	}
2621
	if (!enable_ept)
S
Sheng Yang 已提交
2622
		exec_control |= CPU_BASED_CR3_STORE_EXITING |
2623 2624
				CPU_BASED_CR3_LOAD_EXITING  |
				CPU_BASED_INVLPG_EXITING;
2625
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
A
Avi Kivity 已提交
2626

2627 2628 2629 2630 2631
	if (cpu_has_secondary_exec_ctrls()) {
		exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
		if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
			exec_control &=
				~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
2632 2633
		if (vmx->vpid == 0)
			exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
2634
		if (!enable_ept) {
S
Sheng Yang 已提交
2635
			exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
2636 2637
			enable_unrestricted_guest = 0;
		}
2638 2639
		if (!enable_unrestricted_guest)
			exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
2640 2641
		if (!ple_gap)
			exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
2642 2643
		vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
	}
2644

2645 2646 2647 2648 2649
	if (ple_gap) {
		vmcs_write32(PLE_GAP, ple_gap);
		vmcs_write32(PLE_WINDOW, ple_window);
	}

2650 2651
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf);
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf);
A
Avi Kivity 已提交
2652 2653
	vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */

2654
	vmcs_writel(HOST_CR0, read_cr0() | X86_CR0_TS);  /* 22.2.3 */
A
Avi Kivity 已提交
2655 2656 2657 2658 2659 2660
	vmcs_writel(HOST_CR4, read_cr4());  /* 22.2.3, 22.2.5 */
	vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */

	vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
	vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
	vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
2661 2662
	vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
	vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
A
Avi Kivity 已提交
2663
	vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
2664
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675
	rdmsrl(MSR_FS_BASE, a);
	vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
	rdmsrl(MSR_GS_BASE, a);
	vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
#else
	vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
	vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
#endif

	vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */

2676
	native_store_idt(&dt);
2677
	vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */
A
Avi Kivity 已提交
2678

M
Mike Day 已提交
2679
	asm("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return));
2680
	vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */
2681 2682
	vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
2683
	vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
2684
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
2685
	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
A
Avi Kivity 已提交
2686 2687 2688 2689 2690 2691 2692 2693

	rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk);
	vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs);
	rdmsrl(MSR_IA32_SYSENTER_ESP, a);
	vmcs_writel(HOST_IA32_SYSENTER_ESP, a);   /* 22.2.3 */
	rdmsrl(MSR_IA32_SYSENTER_EIP, a);
	vmcs_writel(HOST_IA32_SYSENTER_EIP, a);   /* 22.2.3 */

S
Sheng Yang 已提交
2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707
	if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
		rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
		host_pat = msr_low | ((u64) msr_high << 32);
		vmcs_write64(HOST_IA32_PAT, host_pat);
	}
	if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
		rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
		host_pat = msr_low | ((u64) msr_high << 32);
		/* Write the default value follow host pat */
		vmcs_write64(GUEST_IA32_PAT, host_pat);
		/* Keep arch.pat sync with GUEST_IA32_PAT */
		vmx->vcpu.arch.pat = host_pat;
	}

A
Avi Kivity 已提交
2708 2709 2710
	for (i = 0; i < NR_VMX_MSR; ++i) {
		u32 index = vmx_msr_index[i];
		u32 data_low, data_high;
2711
		int j = vmx->nmsrs;
A
Avi Kivity 已提交
2712 2713 2714

		if (rdmsr_safe(index, &data_low, &data_high) < 0)
			continue;
2715 2716
		if (wrmsr_safe(index, data_low, data_high) < 0)
			continue;
2717 2718
		vmx->guest_msrs[j].index = i;
		vmx->guest_msrs[j].data = 0;
2719
		vmx->guest_msrs[j].mask = -1ull;
2720
		++vmx->nmsrs;
A
Avi Kivity 已提交
2721 2722
	}

2723
	vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
A
Avi Kivity 已提交
2724 2725

	/* 22.2.1, 20.8.1 */
2726 2727
	vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl);

2728
	vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
2729
	vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
2730 2731
	if (enable_ept)
		vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
2732
	vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
2733

2734
	kvm_write_tsc(&vmx->vcpu, 0);
2735

2736 2737 2738
	return 0;
}

2739 2740
static int init_rmode(struct kvm *kvm)
{
2741 2742 2743
	int idx, ret = 0;

	idx = srcu_read_lock(&kvm->srcu);
2744
	if (!init_rmode_tss(kvm))
2745
		goto exit;
2746
	if (!init_rmode_identity_map(kvm))
2747 2748 2749 2750 2751 2752
		goto exit;

	ret = 1;
exit:
	srcu_read_unlock(&kvm->srcu, idx);
	return ret;
2753 2754
}

2755 2756 2757 2758
static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u64 msr;
2759
	int ret;
2760

2761
	vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP));
2762
	if (!init_rmode(vmx->vcpu.kvm)) {
2763 2764 2765 2766
		ret = -ENOMEM;
		goto out;
	}

2767
	vmx->rmode.vm86_active = 0;
2768

2769 2770
	vmx->soft_vnmi_blocked = 0;

2771
	vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
2772
	kvm_set_cr8(&vmx->vcpu, 0);
2773
	msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
2774
	if (kvm_vcpu_is_bsp(&vmx->vcpu))
2775 2776 2777
		msr |= MSR_IA32_APICBASE_BSP;
	kvm_set_apic_base(&vmx->vcpu, msr);

2778 2779 2780
	ret = fx_init(&vmx->vcpu);
	if (ret != 0)
		goto out;
2781

2782
	seg_setup(VCPU_SREG_CS);
2783 2784 2785 2786
	/*
	 * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
	 * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4.  Sigh.
	 */
2787
	if (kvm_vcpu_is_bsp(&vmx->vcpu)) {
2788 2789 2790
		vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
		vmcs_writel(GUEST_CS_BASE, 0x000f0000);
	} else {
2791 2792
		vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8);
		vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12);
2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815
	}

	seg_setup(VCPU_SREG_DS);
	seg_setup(VCPU_SREG_ES);
	seg_setup(VCPU_SREG_FS);
	seg_setup(VCPU_SREG_GS);
	seg_setup(VCPU_SREG_SS);

	vmcs_write16(GUEST_TR_SELECTOR, 0);
	vmcs_writel(GUEST_TR_BASE, 0);
	vmcs_write32(GUEST_TR_LIMIT, 0xffff);
	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);

	vmcs_write16(GUEST_LDTR_SELECTOR, 0);
	vmcs_writel(GUEST_LDTR_BASE, 0);
	vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
	vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);

	vmcs_write32(GUEST_SYSENTER_CS, 0);
	vmcs_writel(GUEST_SYSENTER_ESP, 0);
	vmcs_writel(GUEST_SYSENTER_EIP, 0);

	vmcs_writel(GUEST_RFLAGS, 0x02);
2816
	if (kvm_vcpu_is_bsp(&vmx->vcpu))
2817
		kvm_rip_write(vcpu, 0xfff0);
2818
	else
2819 2820
		kvm_rip_write(vcpu, 0);
	kvm_register_write(vcpu, VCPU_REGS_RSP, 0);
2821 2822 2823 2824 2825 2826 2827 2828 2829

	vmcs_writel(GUEST_DR7, 0x400);

	vmcs_writel(GUEST_GDTR_BASE, 0);
	vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);

	vmcs_writel(GUEST_IDTR_BASE, 0);
	vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);

2830
	vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
2831 2832 2833 2834 2835 2836 2837 2838
	vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
	vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);

	/* Special registers */
	vmcs_write64(GUEST_IA32_DEBUGCTL, 0);

	setup_msrs(vmx);

A
Avi Kivity 已提交
2839 2840
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */

2841 2842 2843 2844
	if (cpu_has_vmx_tpr_shadow()) {
		vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
		if (vm_need_tpr_shadow(vmx->vcpu.kvm))
			vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
2845
				page_to_phys(vmx->vcpu.arch.apic->regs_page));
2846 2847 2848 2849 2850
		vmcs_write32(TPR_THRESHOLD, 0);
	}

	if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
		vmcs_write64(APIC_ACCESS_ADDR,
2851
			     page_to_phys(vmx->vcpu.kvm->arch.apic_access_page));
A
Avi Kivity 已提交
2852

2853 2854 2855
	if (vmx->vpid != 0)
		vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);

2856
	vmx->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
2857
	vmx_set_cr0(&vmx->vcpu, kvm_read_cr0(vcpu)); /* enter rmode */
R
Rusty Russell 已提交
2858 2859 2860 2861
	vmx_set_cr4(&vmx->vcpu, 0);
	vmx_set_efer(&vmx->vcpu, 0);
	vmx_fpu_activate(&vmx->vcpu);
	update_exception_bitmap(&vmx->vcpu);
A
Avi Kivity 已提交
2862

2863
	vpid_sync_context(vmx);
2864

2865
	ret = 0;
A
Avi Kivity 已提交
2866

2867 2868 2869
	/* HACK: Don't enable emulation on guest boot/reset */
	vmx->emulation_required = 0;

A
Avi Kivity 已提交
2870 2871 2872 2873
out:
	return ret;
}

2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891
static void enable_irq_window(struct kvm_vcpu *vcpu)
{
	u32 cpu_based_vm_exec_control;

	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

static void enable_nmi_window(struct kvm_vcpu *vcpu)
{
	u32 cpu_based_vm_exec_control;

	if (!cpu_has_virtual_nmis()) {
		enable_irq_window(vcpu);
		return;
	}

2892 2893 2894 2895
	if (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
		enable_irq_window(vcpu);
		return;
	}
2896 2897 2898 2899 2900
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

2901
static void vmx_inject_irq(struct kvm_vcpu *vcpu)
2902
{
2903
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2904 2905
	uint32_t intr;
	int irq = vcpu->arch.interrupt.nr;
2906

2907
	trace_kvm_inj_virq(irq);
F
Feng (Eric) Liu 已提交
2908

2909
	++vcpu->stat.irq_injections;
2910
	if (vmx->rmode.vm86_active) {
2911 2912
		if (kvm_inject_realmode_interrupt(vcpu, irq) != EMULATE_DONE)
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2913 2914
		return;
	}
2915 2916 2917 2918 2919 2920 2921 2922
	intr = irq | INTR_INFO_VALID_MASK;
	if (vcpu->arch.interrupt.soft) {
		intr |= INTR_TYPE_SOFT_INTR;
		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
			     vmx->vcpu.arch.event_exit_inst_len);
	} else
		intr |= INTR_TYPE_EXT_INTR;
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
2923
	vmx_clear_hlt(vcpu);
2924 2925
}

2926 2927
static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
{
J
Jan Kiszka 已提交
2928 2929
	struct vcpu_vmx *vmx = to_vmx(vcpu);

2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942
	if (!cpu_has_virtual_nmis()) {
		/*
		 * Tracking the NMI-blocked state in software is built upon
		 * finding the next open IRQ window. This, in turn, depends on
		 * well-behaving guests: They have to keep IRQs disabled at
		 * least as long as the NMI handler runs. Otherwise we may
		 * cause NMI nesting, maybe breaking the guest. But as this is
		 * highly unlikely, we can live with the residual risk.
		 */
		vmx->soft_vnmi_blocked = 1;
		vmx->vnmi_blocked_time = 0;
	}

2943
	++vcpu->stat.nmi_injections;
2944
	if (vmx->rmode.vm86_active) {
2945 2946
		if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR) != EMULATE_DONE)
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
J
Jan Kiszka 已提交
2947 2948
		return;
	}
2949 2950
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
			INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
2951
	vmx_clear_hlt(vcpu);
2952 2953
}

2954
static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
2955
{
2956
	if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
2957
		return 0;
2958

2959
	return	!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2960 2961
		  (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
		   | GUEST_INTR_STATE_NMI));
2962 2963
}

J
Jan Kiszka 已提交
2964 2965 2966 2967
static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
{
	if (!cpu_has_virtual_nmis())
		return to_vmx(vcpu)->soft_vnmi_blocked;
2968
	return vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)	& GUEST_INTR_STATE_NMI;
J
Jan Kiszka 已提交
2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989
}

static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (!cpu_has_virtual_nmis()) {
		if (vmx->soft_vnmi_blocked != masked) {
			vmx->soft_vnmi_blocked = masked;
			vmx->vnmi_blocked_time = 0;
		}
	} else {
		if (masked)
			vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				      GUEST_INTR_STATE_NMI);
		else
			vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
					GUEST_INTR_STATE_NMI);
	}
}

2990 2991
static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
{
2992 2993 2994
	return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
		!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
			(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
2995 2996
}

2997 2998 2999 3000
static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
{
	int ret;
	struct kvm_userspace_memory_region tss_mem = {
3001
		.slot = TSS_PRIVATE_MEMSLOT,
3002 3003 3004 3005 3006 3007 3008 3009
		.guest_phys_addr = addr,
		.memory_size = PAGE_SIZE * 3,
		.flags = 0,
	};

	ret = kvm_set_memory_region(kvm, &tss_mem, 0);
	if (ret)
		return ret;
3010
	kvm->arch.tss_addr = addr;
3011 3012 3013
	return 0;
}

A
Avi Kivity 已提交
3014 3015 3016
static int handle_rmode_exception(struct kvm_vcpu *vcpu,
				  int vec, u32 err_code)
{
3017 3018 3019 3020 3021
	/*
	 * Instruction with address size override prefix opcode 0x67
	 * Cause the #SS fault with 0 error code in VM86 mode.
	 */
	if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
3022
		if (emulate_instruction(vcpu, 0) == EMULATE_DONE)
A
Avi Kivity 已提交
3023
			return 1;
3024 3025 3026 3027 3028 3029 3030
	/*
	 * Forward all other exceptions that are valid in real mode.
	 * FIXME: Breaks guest debugging in real mode, needs to be fixed with
	 *        the required debugging infrastructure rework.
	 */
	switch (vec) {
	case DB_VECTOR:
J
Jan Kiszka 已提交
3031 3032 3033 3034 3035
		if (vcpu->guest_debug &
		    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
			return 0;
		kvm_queue_exception(vcpu, vec);
		return 1;
3036
	case BP_VECTOR:
3037 3038 3039 3040 3041 3042
		/*
		 * Update instruction length as we may reinject the exception
		 * from user space while in guest debugging mode.
		 */
		to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
			vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
J
Jan Kiszka 已提交
3043 3044 3045 3046
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
			return 0;
		/* fall through */
	case DE_VECTOR:
3047 3048 3049 3050 3051 3052 3053 3054 3055 3056
	case OF_VECTOR:
	case BR_VECTOR:
	case UD_VECTOR:
	case DF_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
	case MF_VECTOR:
		kvm_queue_exception(vcpu, vec);
		return 1;
	}
A
Avi Kivity 已提交
3057 3058 3059
	return 0;
}

A
Andi Kleen 已提交
3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078
/*
 * Trigger machine check on the host. We assume all the MSRs are already set up
 * by the CPU and that we still run on the same CPU as the MCE occurred on.
 * We pass a fake environment to the machine check handler because we want
 * the guest to be always treated like user space, no matter what context
 * it used internally.
 */
static void kvm_machine_check(void)
{
#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
	struct pt_regs regs = {
		.cs = 3, /* Fake ring 3 no matter what the guest ran on */
		.flags = X86_EFLAGS_IF,
	};

	do_machine_check(&regs, 0);
#endif
}

A
Avi Kivity 已提交
3079
static int handle_machine_check(struct kvm_vcpu *vcpu)
A
Andi Kleen 已提交
3080 3081 3082 3083 3084
{
	/* already handled by vcpu_run */
	return 1;
}

A
Avi Kivity 已提交
3085
static int handle_exception(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3086
{
3087
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
3088
	struct kvm_run *kvm_run = vcpu->run;
J
Jan Kiszka 已提交
3089
	u32 intr_info, ex_no, error_code;
3090
	unsigned long cr2, rip, dr6;
A
Avi Kivity 已提交
3091 3092 3093
	u32 vect_info;
	enum emulation_result er;

3094
	vect_info = vmx->idt_vectoring_info;
A
Avi Kivity 已提交
3095 3096
	intr_info = vmcs_read32(VM_EXIT_INTR_INFO);

A
Andi Kleen 已提交
3097
	if (is_machine_check(intr_info))
A
Avi Kivity 已提交
3098
		return handle_machine_check(vcpu);
A
Andi Kleen 已提交
3099

A
Avi Kivity 已提交
3100
	if ((vect_info & VECTORING_INFO_VALID_MASK) &&
3101 3102 3103 3104 3105 3106 3107 3108
	    !is_page_fault(intr_info)) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
		vcpu->run->internal.ndata = 2;
		vcpu->run->internal.data[0] = vect_info;
		vcpu->run->internal.data[1] = intr_info;
		return 0;
	}
A
Avi Kivity 已提交
3109

3110
	if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR)
3111
		return 1;  /* already handled by vmx_vcpu_run() */
3112 3113

	if (is_no_device(intr_info)) {
3114
		vmx_fpu_activate(vcpu);
3115 3116 3117
		return 1;
	}

3118
	if (is_invalid_opcode(intr_info)) {
3119
		er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
3120
		if (er != EMULATE_DONE)
3121
			kvm_queue_exception(vcpu, UD_VECTOR);
3122 3123 3124
		return 1;
	}

A
Avi Kivity 已提交
3125
	error_code = 0;
3126
	rip = kvm_rip_read(vcpu);
3127
	if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
A
Avi Kivity 已提交
3128 3129
		error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
	if (is_page_fault(intr_info)) {
3130
		/* EPT won't cause page fault directly */
3131
		if (enable_ept)
3132
			BUG();
A
Avi Kivity 已提交
3133
		cr2 = vmcs_readl(EXIT_QUALIFICATION);
3134 3135
		trace_kvm_page_fault(cr2, error_code);

3136
		if (kvm_event_needs_reinjection(vcpu))
3137
			kvm_mmu_unprotect_page_virt(vcpu, cr2);
3138
		return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0);
A
Avi Kivity 已提交
3139 3140
	}

3141
	if (vmx->rmode.vm86_active &&
A
Avi Kivity 已提交
3142
	    handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
3143
								error_code)) {
3144 3145
		if (vcpu->arch.halt_request) {
			vcpu->arch.halt_request = 0;
3146 3147
			return kvm_emulate_halt(vcpu);
		}
A
Avi Kivity 已提交
3148
		return 1;
3149
	}
A
Avi Kivity 已提交
3150

J
Jan Kiszka 已提交
3151
	ex_no = intr_info & INTR_INFO_VECTOR_MASK;
3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164
	switch (ex_no) {
	case DB_VECTOR:
		dr6 = vmcs_readl(EXIT_QUALIFICATION);
		if (!(vcpu->guest_debug &
		      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
			vcpu->arch.dr6 = dr6 | DR6_FIXED_1;
			kvm_queue_exception(vcpu, DB_VECTOR);
			return 1;
		}
		kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
		kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
		/* fall through */
	case BP_VECTOR:
3165 3166 3167 3168 3169 3170 3171
		/*
		 * Update instruction length as we may reinject #BP from
		 * user space while in guest debugging mode. Reading it for
		 * #DB as well causes no harm, it is not used in that case.
		 */
		vmx->vcpu.arch.event_exit_inst_len =
			vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
A
Avi Kivity 已提交
3172
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
J
Jan Kiszka 已提交
3173 3174
		kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
		kvm_run->debug.arch.exception = ex_no;
3175 3176
		break;
	default:
J
Jan Kiszka 已提交
3177 3178 3179
		kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
		kvm_run->ex.exception = ex_no;
		kvm_run->ex.error_code = error_code;
3180
		break;
A
Avi Kivity 已提交
3181 3182 3183 3184
	}
	return 0;
}

A
Avi Kivity 已提交
3185
static int handle_external_interrupt(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3186
{
A
Avi Kivity 已提交
3187
	++vcpu->stat.irq_exits;
A
Avi Kivity 已提交
3188 3189 3190
	return 1;
}

A
Avi Kivity 已提交
3191
static int handle_triple_fault(struct kvm_vcpu *vcpu)
3192
{
A
Avi Kivity 已提交
3193
	vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
3194 3195
	return 0;
}
A
Avi Kivity 已提交
3196

A
Avi Kivity 已提交
3197
static int handle_io(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3198
{
3199
	unsigned long exit_qualification;
3200
	int size, in, string;
3201
	unsigned port;
A
Avi Kivity 已提交
3202

3203
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3204
	string = (exit_qualification & 16) != 0;
3205
	in = (exit_qualification & 8) != 0;
3206

3207
	++vcpu->stat.io_exits;
3208

3209
	if (string || in)
3210
		return emulate_instruction(vcpu, 0) == EMULATE_DONE;
3211

3212 3213
	port = exit_qualification >> 16;
	size = (exit_qualification & 7) + 1;
3214
	skip_emulated_instruction(vcpu);
3215 3216

	return kvm_fast_pio_out(vcpu, size, port);
A
Avi Kivity 已提交
3217 3218
}

I
Ingo Molnar 已提交
3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229
static void
vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
{
	/*
	 * Patch in the VMCALL instruction:
	 */
	hypercall[0] = 0x0f;
	hypercall[1] = 0x01;
	hypercall[2] = 0xc1;
}

A
Avi Kivity 已提交
3230
static int handle_cr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3231
{
3232
	unsigned long exit_qualification, val;
A
Avi Kivity 已提交
3233 3234
	int cr;
	int reg;
3235
	int err;
A
Avi Kivity 已提交
3236

3237
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
A
Avi Kivity 已提交
3238 3239 3240 3241
	cr = exit_qualification & 15;
	reg = (exit_qualification >> 8) & 15;
	switch ((exit_qualification >> 4) & 3) {
	case 0: /* mov to cr */
3242 3243
		val = kvm_register_read(vcpu, reg);
		trace_kvm_cr_write(cr, val);
A
Avi Kivity 已提交
3244 3245
		switch (cr) {
		case 0:
3246
			err = kvm_set_cr0(vcpu, val);
3247
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
3248 3249
			return 1;
		case 3:
3250
			err = kvm_set_cr3(vcpu, val);
3251
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
3252 3253
			return 1;
		case 4:
3254
			err = kvm_set_cr4(vcpu, val);
3255
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
3256
			return 1;
3257 3258 3259
		case 8: {
				u8 cr8_prev = kvm_get_cr8(vcpu);
				u8 cr8 = kvm_register_read(vcpu, reg);
A
Andre Przywara 已提交
3260
				err = kvm_set_cr8(vcpu, cr8);
3261
				kvm_complete_insn_gp(vcpu, err);
3262 3263 3264 3265
				if (irqchip_in_kernel(vcpu->kvm))
					return 1;
				if (cr8_prev <= cr8)
					return 1;
A
Avi Kivity 已提交
3266
				vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
3267 3268
				return 0;
			}
A
Avi Kivity 已提交
3269 3270
		};
		break;
3271
	case 2: /* clts */
3272
		vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
3273
		trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
3274
		skip_emulated_instruction(vcpu);
A
Avi Kivity 已提交
3275
		vmx_fpu_activate(vcpu);
3276
		return 1;
A
Avi Kivity 已提交
3277 3278 3279
	case 1: /*mov from cr*/
		switch (cr) {
		case 3:
3280 3281 3282
			val = kvm_read_cr3(vcpu);
			kvm_register_write(vcpu, reg, val);
			trace_kvm_cr_read(cr, val);
A
Avi Kivity 已提交
3283 3284 3285
			skip_emulated_instruction(vcpu);
			return 1;
		case 8:
3286 3287 3288
			val = kvm_get_cr8(vcpu);
			kvm_register_write(vcpu, reg, val);
			trace_kvm_cr_read(cr, val);
A
Avi Kivity 已提交
3289 3290 3291 3292 3293
			skip_emulated_instruction(vcpu);
			return 1;
		}
		break;
	case 3: /* lmsw */
3294
		val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
3295
		trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
3296
		kvm_lmsw(vcpu, val);
A
Avi Kivity 已提交
3297 3298 3299 3300 3301 3302

		skip_emulated_instruction(vcpu);
		return 1;
	default:
		break;
	}
A
Avi Kivity 已提交
3303
	vcpu->run->exit_reason = 0;
3304
	pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
A
Avi Kivity 已提交
3305 3306 3307 3308
	       (int)(exit_qualification >> 4) & 3, cr);
	return 0;
}

A
Avi Kivity 已提交
3309
static int handle_dr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3310
{
3311
	unsigned long exit_qualification;
A
Avi Kivity 已提交
3312 3313
	int dr, reg;

3314
	/* Do not handle if the CPL > 0, will trigger GP on re-entry */
3315 3316
	if (!kvm_require_cpl(vcpu, 0))
		return 1;
3317 3318 3319 3320 3321 3322 3323 3324
	dr = vmcs_readl(GUEST_DR7);
	if (dr & DR7_GD) {
		/*
		 * As the vm-exit takes precedence over the debug trap, we
		 * need to emulate the latter, either for the host or the
		 * guest debugging itself.
		 */
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
A
Avi Kivity 已提交
3325 3326 3327
			vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
			vcpu->run->debug.arch.dr7 = dr;
			vcpu->run->debug.arch.pc =
3328 3329
				vmcs_readl(GUEST_CS_BASE) +
				vmcs_readl(GUEST_RIP);
A
Avi Kivity 已提交
3330 3331
			vcpu->run->debug.arch.exception = DB_VECTOR;
			vcpu->run->exit_reason = KVM_EXIT_DEBUG;
3332 3333 3334 3335 3336 3337 3338 3339 3340 3341
			return 0;
		} else {
			vcpu->arch.dr7 &= ~DR7_GD;
			vcpu->arch.dr6 |= DR6_BD;
			vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
			kvm_queue_exception(vcpu, DB_VECTOR);
			return 1;
		}
	}

3342
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3343 3344 3345
	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
	reg = DEBUG_REG_ACCESS_REG(exit_qualification);
	if (exit_qualification & TYPE_MOV_FROM_DR) {
3346 3347 3348 3349 3350
		unsigned long val;
		if (!kvm_get_dr(vcpu, dr, &val))
			kvm_register_write(vcpu, reg, val);
	} else
		kvm_set_dr(vcpu, dr, vcpu->arch.regs[reg]);
A
Avi Kivity 已提交
3351 3352 3353 3354
	skip_emulated_instruction(vcpu);
	return 1;
}

3355 3356 3357 3358 3359
static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
{
	vmcs_writel(GUEST_DR7, val);
}

A
Avi Kivity 已提交
3360
static int handle_cpuid(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3361
{
3362 3363
	kvm_emulate_cpuid(vcpu);
	return 1;
A
Avi Kivity 已提交
3364 3365
}

A
Avi Kivity 已提交
3366
static int handle_rdmsr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3367
{
3368
	u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
A
Avi Kivity 已提交
3369 3370 3371
	u64 data;

	if (vmx_get_msr(vcpu, ecx, &data)) {
3372
		trace_kvm_msr_read_ex(ecx);
3373
		kvm_inject_gp(vcpu, 0);
A
Avi Kivity 已提交
3374 3375 3376
		return 1;
	}

3377
	trace_kvm_msr_read(ecx, data);
F
Feng (Eric) Liu 已提交
3378

A
Avi Kivity 已提交
3379
	/* FIXME: handling of bits 32:63 of rax, rdx */
3380 3381
	vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u;
	vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
A
Avi Kivity 已提交
3382 3383 3384 3385
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
3386
static int handle_wrmsr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3387
{
3388 3389 3390
	u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
	u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
		| ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
A
Avi Kivity 已提交
3391 3392

	if (vmx_set_msr(vcpu, ecx, data) != 0) {
3393
		trace_kvm_msr_write_ex(ecx, data);
3394
		kvm_inject_gp(vcpu, 0);
A
Avi Kivity 已提交
3395 3396 3397
		return 1;
	}

3398
	trace_kvm_msr_write(ecx, data);
A
Avi Kivity 已提交
3399 3400 3401 3402
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
3403
static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
3404
{
3405
	kvm_make_request(KVM_REQ_EVENT, vcpu);
3406 3407 3408
	return 1;
}

A
Avi Kivity 已提交
3409
static int handle_interrupt_window(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3410
{
3411 3412 3413 3414 3415 3416
	u32 cpu_based_vm_exec_control;

	/* clear pending irq */
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
F
Feng (Eric) Liu 已提交
3417

3418 3419
	kvm_make_request(KVM_REQ_EVENT, vcpu);

3420
	++vcpu->stat.irq_window_exits;
F
Feng (Eric) Liu 已提交
3421

3422 3423 3424 3425
	/*
	 * If the user space waits to inject interrupts, exit as soon as
	 * possible
	 */
3426
	if (!irqchip_in_kernel(vcpu->kvm) &&
A
Avi Kivity 已提交
3427
	    vcpu->run->request_interrupt_window &&
3428
	    !kvm_cpu_has_interrupt(vcpu)) {
A
Avi Kivity 已提交
3429
		vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
3430 3431
		return 0;
	}
A
Avi Kivity 已提交
3432 3433 3434
	return 1;
}

A
Avi Kivity 已提交
3435
static int handle_halt(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3436 3437
{
	skip_emulated_instruction(vcpu);
3438
	return kvm_emulate_halt(vcpu);
A
Avi Kivity 已提交
3439 3440
}

A
Avi Kivity 已提交
3441
static int handle_vmcall(struct kvm_vcpu *vcpu)
3442
{
3443
	skip_emulated_instruction(vcpu);
3444 3445
	kvm_emulate_hypercall(vcpu);
	return 1;
3446 3447
}

A
Avi Kivity 已提交
3448
static int handle_vmx_insn(struct kvm_vcpu *vcpu)
3449 3450 3451 3452 3453
{
	kvm_queue_exception(vcpu, UD_VECTOR);
	return 1;
}

3454 3455
static int handle_invd(struct kvm_vcpu *vcpu)
{
3456
	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
3457 3458
}

A
Avi Kivity 已提交
3459
static int handle_invlpg(struct kvm_vcpu *vcpu)
M
Marcelo Tosatti 已提交
3460
{
3461
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
M
Marcelo Tosatti 已提交
3462 3463 3464 3465 3466 3467

	kvm_mmu_invlpg(vcpu, exit_qualification);
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
3468
static int handle_wbinvd(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
3469 3470
{
	skip_emulated_instruction(vcpu);
3471
	kvm_emulate_wbinvd(vcpu);
E
Eddie Dong 已提交
3472 3473 3474
	return 1;
}

3475 3476 3477 3478 3479 3480 3481 3482 3483 3484
static int handle_xsetbv(struct kvm_vcpu *vcpu)
{
	u64 new_bv = kvm_read_edx_eax(vcpu);
	u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);

	if (kvm_set_xcr(vcpu, index, new_bv) == 0)
		skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
3485
static int handle_apic_access(struct kvm_vcpu *vcpu)
3486
{
3487
	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
3488 3489
}

A
Avi Kivity 已提交
3490
static int handle_task_switch(struct kvm_vcpu *vcpu)
3491
{
J
Jan Kiszka 已提交
3492
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3493
	unsigned long exit_qualification;
3494 3495
	bool has_error_code = false;
	u32 error_code = 0;
3496
	u16 tss_selector;
3497 3498 3499 3500
	int reason, type, idt_v;

	idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
	type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
3501 3502 3503 3504

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	reason = (u32)exit_qualification >> 30;
3505 3506 3507 3508 3509 3510 3511 3512 3513
	if (reason == TASK_SWITCH_GATE && idt_v) {
		switch (type) {
		case INTR_TYPE_NMI_INTR:
			vcpu->arch.nmi_injected = false;
			if (cpu_has_virtual_nmis())
				vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
					      GUEST_INTR_STATE_NMI);
			break;
		case INTR_TYPE_EXT_INTR:
3514
		case INTR_TYPE_SOFT_INTR:
3515 3516 3517
			kvm_clear_interrupt_queue(vcpu);
			break;
		case INTR_TYPE_HARD_EXCEPTION:
3518 3519 3520 3521 3522 3523 3524
			if (vmx->idt_vectoring_info &
			    VECTORING_INFO_DELIVER_CODE_MASK) {
				has_error_code = true;
				error_code =
					vmcs_read32(IDT_VECTORING_ERROR_CODE);
			}
			/* fall through */
3525 3526 3527 3528 3529 3530
		case INTR_TYPE_SOFT_EXCEPTION:
			kvm_clear_exception_queue(vcpu);
			break;
		default:
			break;
		}
J
Jan Kiszka 已提交
3531
	}
3532 3533
	tss_selector = exit_qualification;

3534 3535 3536 3537 3538
	if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
		       type != INTR_TYPE_EXT_INTR &&
		       type != INTR_TYPE_NMI_INTR))
		skip_emulated_instruction(vcpu);

3539 3540 3541 3542 3543
	if (kvm_task_switch(vcpu, tss_selector, reason,
				has_error_code, error_code) == EMULATE_FAIL) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
3544
		return 0;
3545
	}
3546 3547 3548 3549 3550 3551 3552 3553 3554 3555

	/* clear all local breakpoint enable flags */
	vmcs_writel(GUEST_DR7, vmcs_readl(GUEST_DR7) & ~55);

	/*
	 * TODO: What about debug traps on tss switch?
	 *       Are we supposed to inject them and update dr6?
	 */

	return 1;
3556 3557
}

A
Avi Kivity 已提交
3558
static int handle_ept_violation(struct kvm_vcpu *vcpu)
3559
{
3560
	unsigned long exit_qualification;
3561 3562 3563
	gpa_t gpa;
	int gla_validity;

3564
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3565 3566 3567

	if (exit_qualification & (1 << 6)) {
		printk(KERN_ERR "EPT: GPA exceeds GAW!\n");
3568
		return -EINVAL;
3569 3570 3571 3572 3573 3574 3575
	}

	gla_validity = (exit_qualification >> 7) & 0x3;
	if (gla_validity != 0x3 && gla_validity != 0x1 && gla_validity != 0) {
		printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
		printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
			(long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
3576
			vmcs_readl(GUEST_LINEAR_ADDRESS));
3577 3578
		printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
			(long unsigned int)exit_qualification);
A
Avi Kivity 已提交
3579 3580
		vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
		vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
3581
		return 0;
3582 3583 3584
	}

	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
3585
	trace_kvm_page_fault(gpa, exit_qualification);
3586
	return kvm_mmu_page_fault(vcpu, gpa, exit_qualification & 0x3, NULL, 0);
3587 3588
}

3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649
static u64 ept_rsvd_mask(u64 spte, int level)
{
	int i;
	u64 mask = 0;

	for (i = 51; i > boot_cpu_data.x86_phys_bits; i--)
		mask |= (1ULL << i);

	if (level > 2)
		/* bits 7:3 reserved */
		mask |= 0xf8;
	else if (level == 2) {
		if (spte & (1ULL << 7))
			/* 2MB ref, bits 20:12 reserved */
			mask |= 0x1ff000;
		else
			/* bits 6:3 reserved */
			mask |= 0x78;
	}

	return mask;
}

static void ept_misconfig_inspect_spte(struct kvm_vcpu *vcpu, u64 spte,
				       int level)
{
	printk(KERN_ERR "%s: spte 0x%llx level %d\n", __func__, spte, level);

	/* 010b (write-only) */
	WARN_ON((spte & 0x7) == 0x2);

	/* 110b (write/execute) */
	WARN_ON((spte & 0x7) == 0x6);

	/* 100b (execute-only) and value not supported by logical processor */
	if (!cpu_has_vmx_ept_execute_only())
		WARN_ON((spte & 0x7) == 0x4);

	/* not 000b */
	if ((spte & 0x7)) {
		u64 rsvd_bits = spte & ept_rsvd_mask(spte, level);

		if (rsvd_bits != 0) {
			printk(KERN_ERR "%s: rsvd_bits = 0x%llx\n",
					 __func__, rsvd_bits);
			WARN_ON(1);
		}

		if (level == 1 || (level == 2 && (spte & (1ULL << 7)))) {
			u64 ept_mem_type = (spte & 0x38) >> 3;

			if (ept_mem_type == 2 || ept_mem_type == 3 ||
			    ept_mem_type == 7) {
				printk(KERN_ERR "%s: ept_mem_type=0x%llx\n",
						__func__, ept_mem_type);
				WARN_ON(1);
			}
		}
	}
}

A
Avi Kivity 已提交
3650
static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665
{
	u64 sptes[4];
	int nr_sptes, i;
	gpa_t gpa;

	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);

	printk(KERN_ERR "EPT: Misconfiguration.\n");
	printk(KERN_ERR "EPT: GPA: 0x%llx\n", gpa);

	nr_sptes = kvm_mmu_get_spte_hierarchy(vcpu, gpa, sptes);

	for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i)
		ept_misconfig_inspect_spte(vcpu, sptes[i-1], i);

A
Avi Kivity 已提交
3666 3667
	vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
	vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
3668 3669 3670 3671

	return 0;
}

A
Avi Kivity 已提交
3672
static int handle_nmi_window(struct kvm_vcpu *vcpu)
3673 3674 3675 3676 3677 3678 3679 3680
{
	u32 cpu_based_vm_exec_control;

	/* clear pending NMI */
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
	++vcpu->stat.nmi_window_exits;
3681
	kvm_make_request(KVM_REQ_EVENT, vcpu);
3682 3683 3684 3685

	return 1;
}

3686
static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
3687
{
3688 3689
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	enum emulation_result err = EMULATE_DONE;
3690
	int ret = 1;
3691 3692 3693 3694 3695
	u32 cpu_exec_ctrl;
	bool intr_window_requested;

	cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
3696 3697

	while (!guest_state_valid(vcpu)) {
3698 3699 3700 3701
		if (intr_window_requested
		    && (kvm_get_rflags(&vmx->vcpu) & X86_EFLAGS_IF))
			return handle_interrupt_window(&vmx->vcpu);

3702
		err = emulate_instruction(vcpu, 0);
3703

3704 3705 3706 3707
		if (err == EMULATE_DO_MMIO) {
			ret = 0;
			goto out;
		}
3708

3709 3710
		if (err != EMULATE_DONE)
			return 0;
3711 3712

		if (signal_pending(current))
3713
			goto out;
3714 3715 3716 3717
		if (need_resched())
			schedule();
	}

3718 3719 3720
	vmx->emulation_required = 0;
out:
	return ret;
3721 3722
}

3723 3724 3725 3726
/*
 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
 * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
 */
3727
static int handle_pause(struct kvm_vcpu *vcpu)
3728 3729 3730 3731 3732 3733 3734
{
	skip_emulated_instruction(vcpu);
	kvm_vcpu_on_spin(vcpu);

	return 1;
}

3735 3736 3737 3738 3739 3740
static int handle_invalid_op(struct kvm_vcpu *vcpu)
{
	kvm_queue_exception(vcpu, UD_VECTOR);
	return 1;
}

A
Avi Kivity 已提交
3741 3742 3743 3744 3745
/*
 * The exit handlers return 1 if the exit was handled fully and guest execution
 * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
 * to be done to userspace and return 0.
 */
A
Avi Kivity 已提交
3746
static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
A
Avi Kivity 已提交
3747 3748
	[EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
	[EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
3749
	[EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
3750
	[EXIT_REASON_NMI_WINDOW]	      = handle_nmi_window,
A
Avi Kivity 已提交
3751 3752 3753 3754 3755 3756 3757 3758
	[EXIT_REASON_IO_INSTRUCTION]          = handle_io,
	[EXIT_REASON_CR_ACCESS]               = handle_cr,
	[EXIT_REASON_DR_ACCESS]               = handle_dr,
	[EXIT_REASON_CPUID]                   = handle_cpuid,
	[EXIT_REASON_MSR_READ]                = handle_rdmsr,
	[EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
	[EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
	[EXIT_REASON_HLT]                     = handle_halt,
3759
	[EXIT_REASON_INVD]		      = handle_invd,
M
Marcelo Tosatti 已提交
3760
	[EXIT_REASON_INVLPG]		      = handle_invlpg,
3761
	[EXIT_REASON_VMCALL]                  = handle_vmcall,
3762 3763 3764 3765 3766 3767 3768 3769 3770
	[EXIT_REASON_VMCLEAR]	              = handle_vmx_insn,
	[EXIT_REASON_VMLAUNCH]                = handle_vmx_insn,
	[EXIT_REASON_VMPTRLD]                 = handle_vmx_insn,
	[EXIT_REASON_VMPTRST]                 = handle_vmx_insn,
	[EXIT_REASON_VMREAD]                  = handle_vmx_insn,
	[EXIT_REASON_VMRESUME]                = handle_vmx_insn,
	[EXIT_REASON_VMWRITE]                 = handle_vmx_insn,
	[EXIT_REASON_VMOFF]                   = handle_vmx_insn,
	[EXIT_REASON_VMON]                    = handle_vmx_insn,
3771 3772
	[EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
	[EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
E
Eddie Dong 已提交
3773
	[EXIT_REASON_WBINVD]                  = handle_wbinvd,
3774
	[EXIT_REASON_XSETBV]                  = handle_xsetbv,
3775
	[EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
A
Andi Kleen 已提交
3776
	[EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
3777 3778
	[EXIT_REASON_EPT_VIOLATION]	      = handle_ept_violation,
	[EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
3779
	[EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
3780 3781
	[EXIT_REASON_MWAIT_INSTRUCTION]	      = handle_invalid_op,
	[EXIT_REASON_MONITOR_INSTRUCTION]     = handle_invalid_op,
A
Avi Kivity 已提交
3782 3783 3784
};

static const int kvm_vmx_max_exit_handlers =
3785
	ARRAY_SIZE(kvm_vmx_exit_handlers);
A
Avi Kivity 已提交
3786

3787 3788 3789 3790 3791 3792
static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
{
	*info1 = vmcs_readl(EXIT_QUALIFICATION);
	*info2 = vmcs_read32(VM_EXIT_INTR_INFO);
}

A
Avi Kivity 已提交
3793 3794 3795 3796
/*
 * The guest has exited.  See if we can fix it or if we need userspace
 * assistance.
 */
A
Avi Kivity 已提交
3797
static int vmx_handle_exit(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3798
{
3799
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Andi Kleen 已提交
3800
	u32 exit_reason = vmx->exit_reason;
3801
	u32 vectoring_info = vmx->idt_vectoring_info;
3802

3803
	trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);
F
Feng (Eric) Liu 已提交
3804

3805 3806 3807
	/* If guest state is invalid, start emulating */
	if (vmx->emulation_required && emulate_invalid_guest_state)
		return handle_invalid_guest_state(vcpu);
3808

3809 3810 3811 3812 3813 3814 3815
	if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		vcpu->run->fail_entry.hardware_entry_failure_reason
			= exit_reason;
		return 0;
	}

3816
	if (unlikely(vmx->fail)) {
A
Avi Kivity 已提交
3817 3818
		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		vcpu->run->fail_entry.hardware_entry_failure_reason
3819 3820 3821
			= vmcs_read32(VM_INSTRUCTION_ERROR);
		return 0;
	}
A
Avi Kivity 已提交
3822

M
Mike Day 已提交
3823
	if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
3824
			(exit_reason != EXIT_REASON_EXCEPTION_NMI &&
J
Jan Kiszka 已提交
3825 3826 3827 3828 3829
			exit_reason != EXIT_REASON_EPT_VIOLATION &&
			exit_reason != EXIT_REASON_TASK_SWITCH))
		printk(KERN_WARNING "%s: unexpected, valid vectoring info "
		       "(0x%x) and exit reason is 0x%x\n",
		       __func__, vectoring_info, exit_reason);
3830 3831

	if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked)) {
3832
		if (vmx_interrupt_allowed(vcpu)) {
3833 3834
			vmx->soft_vnmi_blocked = 0;
		} else if (vmx->vnmi_blocked_time > 1000000000LL &&
3835
			   vcpu->arch.nmi_pending) {
3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848
			/*
			 * This CPU don't support us in finding the end of an
			 * NMI-blocked window if the guest runs with IRQs
			 * disabled. So we pull the trigger after 1 s of
			 * futile waiting, but inform the user about this.
			 */
			printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
			       "state on VCPU %d after 1 s timeout\n",
			       __func__, vcpu->vcpu_id);
			vmx->soft_vnmi_blocked = 0;
		}
	}

A
Avi Kivity 已提交
3849 3850
	if (exit_reason < kvm_vmx_max_exit_handlers
	    && kvm_vmx_exit_handlers[exit_reason])
A
Avi Kivity 已提交
3851
		return kvm_vmx_exit_handlers[exit_reason](vcpu);
A
Avi Kivity 已提交
3852
	else {
A
Avi Kivity 已提交
3853 3854
		vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
		vcpu->run->hw.hardware_exit_reason = exit_reason;
A
Avi Kivity 已提交
3855 3856 3857 3858
	}
	return 0;
}

3859
static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3860
{
3861
	if (irr == -1 || tpr < irr) {
3862 3863 3864 3865
		vmcs_write32(TPR_THRESHOLD, 0);
		return;
	}

3866
	vmcs_write32(TPR_THRESHOLD, irr);
3867 3868
}

3869
static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
3870
{
3871
	u32 exit_intr_info = vmx->exit_intr_info;
A
Andi Kleen 已提交
3872 3873 3874 3875 3876 3877 3878

	/* Handle machine checks before interrupts are enabled */
	if ((vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)
	    || (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI
		&& is_machine_check(exit_intr_info)))
		kvm_machine_check();

3879 3880
	/* We need to handle NMIs before interrupts are enabled */
	if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
3881 3882
	    (exit_intr_info & INTR_INFO_VALID_MASK)) {
		kvm_before_handle_nmi(&vmx->vcpu);
3883
		asm("int $2");
3884 3885
		kvm_after_handle_nmi(&vmx->vcpu);
	}
3886
}
3887

3888 3889 3890 3891 3892 3893 3894 3895
static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
{
	u32 exit_intr_info = vmx->exit_intr_info;
	bool unblock_nmi;
	u8 vector;
	bool idtv_info_valid;

	idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
3896

3897 3898 3899 3900
	if (cpu_has_virtual_nmis()) {
		unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
		vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
		/*
3901
		 * SDM 3: 27.7.1.2 (September 2008)
3902 3903
		 * Re-set bit "block by NMI" before VM entry if vmexit caused by
		 * a guest IRET fault.
3904 3905 3906 3907 3908
		 * SDM 3: 23.2.2 (September 2008)
		 * Bit 12 is undefined in any of the following cases:
		 *  If the VM exit sets the valid bit in the IDT-vectoring
		 *   information field.
		 *  If the VM exit is due to a double fault.
3909
		 */
3910 3911
		if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
		    vector != DF_VECTOR && !idtv_info_valid)
3912 3913
			vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				      GUEST_INTR_STATE_NMI);
3914 3915 3916
	} else if (unlikely(vmx->soft_vnmi_blocked))
		vmx->vnmi_blocked_time +=
			ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
3917 3918
}

3919 3920 3921 3922
static void __vmx_complete_interrupts(struct vcpu_vmx *vmx,
				      u32 idt_vectoring_info,
				      int instr_len_field,
				      int error_code_field)
3923 3924 3925 3926 3927 3928
{
	u8 vector;
	int type;
	bool idtv_info_valid;

	idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
3929

3930 3931 3932 3933 3934 3935 3936
	vmx->vcpu.arch.nmi_injected = false;
	kvm_clear_exception_queue(&vmx->vcpu);
	kvm_clear_interrupt_queue(&vmx->vcpu);

	if (!idtv_info_valid)
		return;

3937 3938
	kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);

3939 3940
	vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
	type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
3941

3942
	switch (type) {
3943 3944
	case INTR_TYPE_NMI_INTR:
		vmx->vcpu.arch.nmi_injected = true;
3945
		/*
3946
		 * SDM 3: 27.7.1.2 (September 2008)
3947 3948
		 * Clear bit "block by NMI" before VM entry if a NMI
		 * delivery faulted.
3949
		 */
3950 3951 3952 3953
		vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
				GUEST_INTR_STATE_NMI);
		break;
	case INTR_TYPE_SOFT_EXCEPTION:
3954
		vmx->vcpu.arch.event_exit_inst_len =
3955
			vmcs_read32(instr_len_field);
3956 3957
		/* fall through */
	case INTR_TYPE_HARD_EXCEPTION:
3958
		if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
3959
			u32 err = vmcs_read32(error_code_field);
3960
			kvm_queue_exception_e(&vmx->vcpu, vector, err);
3961 3962
		} else
			kvm_queue_exception(&vmx->vcpu, vector);
3963
		break;
3964 3965
	case INTR_TYPE_SOFT_INTR:
		vmx->vcpu.arch.event_exit_inst_len =
3966
			vmcs_read32(instr_len_field);
3967
		/* fall through */
3968
	case INTR_TYPE_EXT_INTR:
3969 3970
		kvm_queue_interrupt(&vmx->vcpu, vector,
			type == INTR_TYPE_SOFT_INTR);
3971 3972 3973
		break;
	default:
		break;
3974
	}
3975 3976
}

3977 3978 3979 3980 3981 3982 3983
static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
{
	__vmx_complete_interrupts(vmx, vmx->idt_vectoring_info,
				  VM_EXIT_INSTRUCTION_LEN,
				  IDT_VECTORING_ERROR_CODE);
}

A
Avi Kivity 已提交
3984 3985 3986 3987 3988 3989 3990 3991 3992 3993
static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
{
	__vmx_complete_interrupts(to_vmx(vcpu),
				  vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
				  VM_ENTRY_INSTRUCTION_LEN,
				  VM_ENTRY_EXCEPTION_ERROR_CODE);

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
}

3994 3995 3996 3997 3998 3999 4000 4001
#ifdef CONFIG_X86_64
#define R "r"
#define Q "q"
#else
#define R "e"
#define Q "l"
#endif

4002
static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4003
{
4004
	struct vcpu_vmx *vmx = to_vmx(vcpu);
4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028

	/* Record the guest's net vcpu time for enforced NMI injections. */
	if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
		vmx->entry_time = ktime_get();

	/* Don't enter VMX if guest state is invalid, let the exit handler
	   start emulation until we arrive back to a valid state */
	if (vmx->emulation_required && emulate_invalid_guest_state)
		return;

	if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
		vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
	if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
		vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);

	/* When single-stepping over STI and MOV SS, we must clear the
	 * corresponding interruptibility bits in the guest state. Otherwise
	 * vmentry fails as it then expects bit 14 (BS) in pending debug
	 * exceptions being set, but that's not correct for the guest debugging
	 * case. */
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vmx_set_interrupt_shadow(vcpu, 0);

	asm(
A
Avi Kivity 已提交
4029
		/* Store host registers */
4030
		"push %%"R"dx; push %%"R"bp;"
4031
		"push %%"R"cx \n\t" /* placeholder for guest rcx */
4032
		"push %%"R"cx \n\t"
4033 4034 4035
		"cmp %%"R"sp, %c[host_rsp](%0) \n\t"
		"je 1f \n\t"
		"mov %%"R"sp, %c[host_rsp](%0) \n\t"
4036
		__ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
4037
		"1: \n\t"
4038 4039 4040 4041 4042 4043 4044
		/* Reload cr2 if changed */
		"mov %c[cr2](%0), %%"R"ax \n\t"
		"mov %%cr2, %%"R"dx \n\t"
		"cmp %%"R"ax, %%"R"dx \n\t"
		"je 2f \n\t"
		"mov %%"R"ax, %%cr2 \n\t"
		"2: \n\t"
A
Avi Kivity 已提交
4045
		/* Check if vmlaunch of vmresume is needed */
4046
		"cmpl $0, %c[launched](%0) \n\t"
A
Avi Kivity 已提交
4047
		/* Load guest registers.  Don't clobber flags. */
4048 4049 4050 4051 4052 4053
		"mov %c[rax](%0), %%"R"ax \n\t"
		"mov %c[rbx](%0), %%"R"bx \n\t"
		"mov %c[rdx](%0), %%"R"dx \n\t"
		"mov %c[rsi](%0), %%"R"si \n\t"
		"mov %c[rdi](%0), %%"R"di \n\t"
		"mov %c[rbp](%0), %%"R"bp \n\t"
4054
#ifdef CONFIG_X86_64
4055 4056 4057 4058 4059 4060 4061 4062
		"mov %c[r8](%0),  %%r8  \n\t"
		"mov %c[r9](%0),  %%r9  \n\t"
		"mov %c[r10](%0), %%r10 \n\t"
		"mov %c[r11](%0), %%r11 \n\t"
		"mov %c[r12](%0), %%r12 \n\t"
		"mov %c[r13](%0), %%r13 \n\t"
		"mov %c[r14](%0), %%r14 \n\t"
		"mov %c[r15](%0), %%r15 \n\t"
A
Avi Kivity 已提交
4063
#endif
4064 4065
		"mov %c[rcx](%0), %%"R"cx \n\t" /* kills %0 (ecx) */

A
Avi Kivity 已提交
4066
		/* Enter guest mode */
4067
		"jne .Llaunched \n\t"
4068
		__ex(ASM_VMX_VMLAUNCH) "\n\t"
4069
		"jmp .Lkvm_vmx_return \n\t"
4070
		".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t"
4071
		".Lkvm_vmx_return: "
A
Avi Kivity 已提交
4072
		/* Save guest registers, load host registers, keep flags */
4073 4074
		"mov %0, %c[wordsize](%%"R"sp) \n\t"
		"pop %0 \n\t"
4075 4076
		"mov %%"R"ax, %c[rax](%0) \n\t"
		"mov %%"R"bx, %c[rbx](%0) \n\t"
4077
		"pop"Q" %c[rcx](%0) \n\t"
4078 4079 4080 4081
		"mov %%"R"dx, %c[rdx](%0) \n\t"
		"mov %%"R"si, %c[rsi](%0) \n\t"
		"mov %%"R"di, %c[rdi](%0) \n\t"
		"mov %%"R"bp, %c[rbp](%0) \n\t"
4082
#ifdef CONFIG_X86_64
4083 4084 4085 4086 4087 4088 4089 4090
		"mov %%r8,  %c[r8](%0) \n\t"
		"mov %%r9,  %c[r9](%0) \n\t"
		"mov %%r10, %c[r10](%0) \n\t"
		"mov %%r11, %c[r11](%0) \n\t"
		"mov %%r12, %c[r12](%0) \n\t"
		"mov %%r13, %c[r13](%0) \n\t"
		"mov %%r14, %c[r14](%0) \n\t"
		"mov %%r15, %c[r15](%0) \n\t"
A
Avi Kivity 已提交
4091
#endif
4092 4093 4094
		"mov %%cr2, %%"R"ax   \n\t"
		"mov %%"R"ax, %c[cr2](%0) \n\t"

4095
		"pop  %%"R"bp; pop  %%"R"dx \n\t"
4096 4097 4098 4099
		"setbe %c[fail](%0) \n\t"
	      : : "c"(vmx), "d"((unsigned long)HOST_RSP),
		[launched]"i"(offsetof(struct vcpu_vmx, launched)),
		[fail]"i"(offsetof(struct vcpu_vmx, fail)),
4100
		[host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
4101 4102 4103 4104 4105 4106 4107
		[rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
		[rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
		[rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
		[rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
		[rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
		[rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
		[rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
4108
#ifdef CONFIG_X86_64
4109 4110 4111 4112 4113 4114 4115 4116
		[r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
		[r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
		[r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
		[r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
		[r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
		[r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
		[r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
		[r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
A
Avi Kivity 已提交
4117
#endif
4118 4119
		[cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
		[wordsize]"i"(sizeof(ulong))
4120
	      : "cc", "memory"
4121
		, R"ax", R"bx", R"di", R"si"
4122 4123 4124 4125
#ifdef CONFIG_X86_64
		, "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
#endif
	      );
A
Avi Kivity 已提交
4126

A
Avi Kivity 已提交
4127
	vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
4128 4129
				  | (1 << VCPU_EXREG_PDPTR)
				  | (1 << VCPU_EXREG_CR3));
4130 4131
	vcpu->arch.regs_dirty = 0;

4132 4133
	vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);

M
Mike Day 已提交
4134
	asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
4135
	vmx->launched = 1;
4136

4137 4138 4139 4140 4141
	vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
	vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);

	vmx_complete_atomic_exit(vmx);
	vmx_recover_nmi_blocking(vmx);
4142
	vmx_complete_interrupts(vmx);
A
Avi Kivity 已提交
4143 4144
}

4145 4146 4147
#undef R
#undef Q

A
Avi Kivity 已提交
4148 4149
static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
{
4150 4151 4152
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (vmx->vmcs) {
4153
		vcpu_clear(vmx);
4154 4155
		free_vmcs(vmx->vmcs);
		vmx->vmcs = NULL;
A
Avi Kivity 已提交
4156 4157 4158 4159 4160
	}
}

static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
4161 4162
	struct vcpu_vmx *vmx = to_vmx(vcpu);

4163
	free_vpid(vmx);
A
Avi Kivity 已提交
4164
	vmx_free_vmcs(vcpu);
R
Rusty Russell 已提交
4165 4166
	kfree(vmx->guest_msrs);
	kvm_vcpu_uninit(vcpu);
4167
	kmem_cache_free(kvm_vcpu_cache, vmx);
A
Avi Kivity 已提交
4168 4169
}

4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182
static inline void vmcs_init(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(per_cpu(vmxarea, raw_smp_processor_id()));

	if (!vmm_exclusive)
		kvm_cpu_vmxon(phys_addr);

	vmcs_clear(vmcs);

	if (!vmm_exclusive)
		kvm_cpu_vmxoff();
}

R
Rusty Russell 已提交
4183
static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
A
Avi Kivity 已提交
4184
{
R
Rusty Russell 已提交
4185
	int err;
4186
	struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
4187
	int cpu;
A
Avi Kivity 已提交
4188

4189
	if (!vmx)
R
Rusty Russell 已提交
4190 4191
		return ERR_PTR(-ENOMEM);

4192 4193
	allocate_vpid(vmx);

R
Rusty Russell 已提交
4194 4195 4196
	err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
	if (err)
		goto free_vcpu;
4197

4198
	vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
R
Rusty Russell 已提交
4199 4200 4201 4202
	if (!vmx->guest_msrs) {
		err = -ENOMEM;
		goto uninit_vcpu;
	}
4203

4204 4205
	vmx->vmcs = alloc_vmcs();
	if (!vmx->vmcs)
R
Rusty Russell 已提交
4206
		goto free_msrs;
4207

4208
	vmcs_init(vmx->vmcs);
4209

4210 4211
	cpu = get_cpu();
	vmx_vcpu_load(&vmx->vcpu, cpu);
Z
Zachary Amsden 已提交
4212
	vmx->vcpu.cpu = cpu;
R
Rusty Russell 已提交
4213
	err = vmx_vcpu_setup(vmx);
R
Rusty Russell 已提交
4214
	vmx_vcpu_put(&vmx->vcpu);
4215
	put_cpu();
R
Rusty Russell 已提交
4216 4217
	if (err)
		goto free_vmcs;
4218 4219 4220
	if (vm_need_virtualize_apic_accesses(kvm))
		if (alloc_apic_access_page(kvm) != 0)
			goto free_vmcs;
R
Rusty Russell 已提交
4221

4222 4223 4224 4225
	if (enable_ept) {
		if (!kvm->arch.ept_identity_map_addr)
			kvm->arch.ept_identity_map_addr =
				VMX_EPT_IDENTITY_PAGETABLE_ADDR;
4226 4227
		if (alloc_identity_pagetable(kvm) != 0)
			goto free_vmcs;
4228
	}
4229

R
Rusty Russell 已提交
4230 4231 4232 4233 4234 4235 4236 4237 4238
	return &vmx->vcpu;

free_vmcs:
	free_vmcs(vmx->vmcs);
free_msrs:
	kfree(vmx->guest_msrs);
uninit_vcpu:
	kvm_vcpu_uninit(&vmx->vcpu);
free_vcpu:
4239
	free_vpid(vmx);
4240
	kmem_cache_free(kvm_vcpu_cache, vmx);
R
Rusty Russell 已提交
4241
	return ERR_PTR(err);
A
Avi Kivity 已提交
4242 4243
}

Y
Yang, Sheng 已提交
4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257
static void __init vmx_check_processor_compat(void *rtn)
{
	struct vmcs_config vmcs_conf;

	*(int *)rtn = 0;
	if (setup_vmcs_config(&vmcs_conf) < 0)
		*(int *)rtn = -EIO;
	if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
		printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
				smp_processor_id());
		*(int *)rtn = -EIO;
	}
}

4258 4259 4260 4261 4262
static int get_ept_level(void)
{
	return VMX_EPT_DEFAULT_GAW + 1;
}

4263
static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
S
Sheng Yang 已提交
4264
{
4265 4266
	u64 ret;

4267 4268 4269 4270 4271 4272 4273 4274
	/* For VT-d and EPT combination
	 * 1. MMIO: always map as UC
	 * 2. EPT with VT-d:
	 *   a. VT-d without snooping control feature: can't guarantee the
	 *	result, try to trust guest.
	 *   b. VT-d with snooping control feature: snooping control feature of
	 *	VT-d engine can guarantee the cache correctness. Just set it
	 *	to WB to keep consistent with host. So the same as item 3.
4275
	 * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
4276 4277
	 *    consistent with host MTRR
	 */
4278 4279
	if (is_mmio)
		ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
4280 4281 4282 4283
	else if (vcpu->kvm->arch.iommu_domain &&
		!(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY))
		ret = kvm_get_guest_memory_type(vcpu, gfn) <<
		      VMX_EPT_MT_EPTE_SHIFT;
4284
	else
4285
		ret = (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT)
4286
			| VMX_EPT_IPAT_BIT;
4287 4288

	return ret;
S
Sheng Yang 已提交
4289 4290
}

4291 4292
#define _ER(x) { EXIT_REASON_##x, #x }

4293
static const struct trace_print_flags vmx_exit_reasons_str[] = {
4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328
	_ER(EXCEPTION_NMI),
	_ER(EXTERNAL_INTERRUPT),
	_ER(TRIPLE_FAULT),
	_ER(PENDING_INTERRUPT),
	_ER(NMI_WINDOW),
	_ER(TASK_SWITCH),
	_ER(CPUID),
	_ER(HLT),
	_ER(INVLPG),
	_ER(RDPMC),
	_ER(RDTSC),
	_ER(VMCALL),
	_ER(VMCLEAR),
	_ER(VMLAUNCH),
	_ER(VMPTRLD),
	_ER(VMPTRST),
	_ER(VMREAD),
	_ER(VMRESUME),
	_ER(VMWRITE),
	_ER(VMOFF),
	_ER(VMON),
	_ER(CR_ACCESS),
	_ER(DR_ACCESS),
	_ER(IO_INSTRUCTION),
	_ER(MSR_READ),
	_ER(MSR_WRITE),
	_ER(MWAIT_INSTRUCTION),
	_ER(MONITOR_INSTRUCTION),
	_ER(PAUSE_INSTRUCTION),
	_ER(MCE_DURING_VMENTRY),
	_ER(TPR_BELOW_THRESHOLD),
	_ER(APIC_ACCESS),
	_ER(EPT_VIOLATION),
	_ER(EPT_MISCONFIG),
	_ER(WBINVD),
4329 4330 4331
	{ -1, NULL }
};

4332 4333
#undef _ER

4334
static int vmx_get_lpage_level(void)
4335
{
4336 4337 4338 4339 4340
	if (enable_ept && !cpu_has_vmx_ept_1g_page())
		return PT_DIRECTORY_LEVEL;
	else
		/* For shadow and EPT supported 1GB page */
		return PT_PDPE_LEVEL;
4341 4342
}

4343 4344
static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
{
4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362
	struct kvm_cpuid_entry2 *best;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u32 exec_control;

	vmx->rdtscp_enabled = false;
	if (vmx_rdtscp_supported()) {
		exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
		if (exec_control & SECONDARY_EXEC_RDTSCP) {
			best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
			if (best && (best->edx & bit(X86_FEATURE_RDTSCP)))
				vmx->rdtscp_enabled = true;
			else {
				exec_control &= ~SECONDARY_EXEC_RDTSCP;
				vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
						exec_control);
			}
		}
	}
4363 4364
}

4365 4366 4367 4368
static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
{
}

4369
static struct kvm_x86_ops vmx_x86_ops = {
A
Avi Kivity 已提交
4370 4371 4372 4373
	.cpu_has_kvm_support = cpu_has_kvm_support,
	.disabled_by_bios = vmx_disabled_by_bios,
	.hardware_setup = hardware_setup,
	.hardware_unsetup = hardware_unsetup,
Y
Yang, Sheng 已提交
4374
	.check_processor_compatibility = vmx_check_processor_compat,
A
Avi Kivity 已提交
4375 4376
	.hardware_enable = hardware_enable,
	.hardware_disable = hardware_disable,
4377
	.cpu_has_accelerated_tpr = report_flexpriority,
A
Avi Kivity 已提交
4378 4379 4380

	.vcpu_create = vmx_create_vcpu,
	.vcpu_free = vmx_free_vcpu,
4381
	.vcpu_reset = vmx_vcpu_reset,
A
Avi Kivity 已提交
4382

4383
	.prepare_guest_switch = vmx_save_host_state,
A
Avi Kivity 已提交
4384 4385 4386 4387 4388 4389 4390 4391 4392
	.vcpu_load = vmx_vcpu_load,
	.vcpu_put = vmx_vcpu_put,

	.set_guest_debug = set_guest_debug,
	.get_msr = vmx_get_msr,
	.set_msr = vmx_set_msr,
	.get_segment_base = vmx_get_segment_base,
	.get_segment = vmx_get_segment,
	.set_segment = vmx_set_segment,
4393
	.get_cpl = vmx_get_cpl,
A
Avi Kivity 已提交
4394
	.get_cs_db_l_bits = vmx_get_cs_db_l_bits,
4395
	.decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
4396
	.decache_cr3 = vmx_decache_cr3,
4397
	.decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
A
Avi Kivity 已提交
4398 4399 4400 4401 4402 4403 4404 4405
	.set_cr0 = vmx_set_cr0,
	.set_cr3 = vmx_set_cr3,
	.set_cr4 = vmx_set_cr4,
	.set_efer = vmx_set_efer,
	.get_idt = vmx_get_idt,
	.set_idt = vmx_set_idt,
	.get_gdt = vmx_get_gdt,
	.set_gdt = vmx_set_gdt,
4406
	.set_dr7 = vmx_set_dr7,
4407
	.cache_reg = vmx_cache_reg,
A
Avi Kivity 已提交
4408 4409
	.get_rflags = vmx_get_rflags,
	.set_rflags = vmx_set_rflags,
4410
	.fpu_activate = vmx_fpu_activate,
4411
	.fpu_deactivate = vmx_fpu_deactivate,
A
Avi Kivity 已提交
4412 4413 4414 4415

	.tlb_flush = vmx_flush_tlb,

	.run = vmx_vcpu_run,
4416
	.handle_exit = vmx_handle_exit,
A
Avi Kivity 已提交
4417
	.skip_emulated_instruction = skip_emulated_instruction,
4418 4419
	.set_interrupt_shadow = vmx_set_interrupt_shadow,
	.get_interrupt_shadow = vmx_get_interrupt_shadow,
I
Ingo Molnar 已提交
4420
	.patch_hypercall = vmx_patch_hypercall,
E
Eddie Dong 已提交
4421
	.set_irq = vmx_inject_irq,
4422
	.set_nmi = vmx_inject_nmi,
4423
	.queue_exception = vmx_queue_exception,
A
Avi Kivity 已提交
4424
	.cancel_injection = vmx_cancel_injection,
4425
	.interrupt_allowed = vmx_interrupt_allowed,
4426
	.nmi_allowed = vmx_nmi_allowed,
J
Jan Kiszka 已提交
4427 4428
	.get_nmi_mask = vmx_get_nmi_mask,
	.set_nmi_mask = vmx_set_nmi_mask,
4429 4430 4431 4432
	.enable_nmi_window = enable_nmi_window,
	.enable_irq_window = enable_irq_window,
	.update_cr8_intercept = update_cr8_intercept,

4433
	.set_tss_addr = vmx_set_tss_addr,
4434
	.get_tdp_level = get_ept_level,
4435
	.get_mt_mask = vmx_get_mt_mask,
4436

4437
	.get_exit_info = vmx_get_exit_info,
4438
	.exit_reasons_str = vmx_exit_reasons_str,
4439

4440
	.get_lpage_level = vmx_get_lpage_level,
4441 4442

	.cpuid_update = vmx_cpuid_update,
4443 4444

	.rdtscp_supported = vmx_rdtscp_supported,
4445 4446

	.set_supported_cpuid = vmx_set_supported_cpuid,
4447 4448

	.has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
4449 4450

	.write_tsc_offset = vmx_write_tsc_offset,
Z
Zachary Amsden 已提交
4451
	.adjust_tsc_offset = vmx_adjust_tsc_offset,
4452 4453

	.set_tdp_cr3 = vmx_set_cr3,
A
Avi Kivity 已提交
4454 4455 4456 4457
};

static int __init vmx_init(void)
{
4458 4459 4460 4461 4462 4463
	int r, i;

	rdmsrl_safe(MSR_EFER, &host_efer);

	for (i = 0; i < NR_VMX_MSR; ++i)
		kvm_define_shared_msr(i, vmx_msr_index[i]);
4464

4465
	vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
4466 4467 4468
	if (!vmx_io_bitmap_a)
		return -ENOMEM;

4469
	vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
4470 4471 4472 4473 4474
	if (!vmx_io_bitmap_b) {
		r = -ENOMEM;
		goto out;
	}

4475 4476
	vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_legacy) {
S
Sheng Yang 已提交
4477 4478 4479 4480
		r = -ENOMEM;
		goto out1;
	}

4481 4482 4483 4484 4485 4486
	vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_longmode) {
		r = -ENOMEM;
		goto out2;
	}

4487 4488 4489 4490
	/*
	 * Allow direct access to the PC debug port (it is often used for I/O
	 * delays, but the vmexits simply slow things down).
	 */
4491 4492
	memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
	clear_bit(0x80, vmx_io_bitmap_a);
4493

4494
	memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
4495

4496 4497
	memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
	memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
S
Sheng Yang 已提交
4498

4499 4500
	set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */

4501 4502
	r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
		     __alignof__(struct vcpu_vmx), THIS_MODULE);
4503
	if (r)
4504
		goto out3;
S
Sheng Yang 已提交
4505

4506 4507 4508 4509 4510 4511
	vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
	vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
	vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
4512

4513
	if (enable_ept) {
4514
		bypass_guest_pf = 0;
4515
		kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull,
4516
				VMX_EPT_EXECUTABLE_MASK);
4517 4518 4519
		kvm_enable_tdp();
	} else
		kvm_disable_tdp();
4520

4521 4522 4523
	if (bypass_guest_pf)
		kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull);

4524 4525
	return 0;

4526 4527
out3:
	free_page((unsigned long)vmx_msr_bitmap_longmode);
S
Sheng Yang 已提交
4528
out2:
4529
	free_page((unsigned long)vmx_msr_bitmap_legacy);
4530
out1:
4531
	free_page((unsigned long)vmx_io_bitmap_b);
4532
out:
4533
	free_page((unsigned long)vmx_io_bitmap_a);
4534
	return r;
A
Avi Kivity 已提交
4535 4536 4537 4538
}

static void __exit vmx_exit(void)
{
4539 4540
	free_page((unsigned long)vmx_msr_bitmap_legacy);
	free_page((unsigned long)vmx_msr_bitmap_longmode);
4541 4542
	free_page((unsigned long)vmx_io_bitmap_b);
	free_page((unsigned long)vmx_io_bitmap_a);
4543

4544
	kvm_exit();
A
Avi Kivity 已提交
4545 4546 4547 4548
}

module_init(vmx_init)
module_exit(vmx_exit)