vmx.c 115.9 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "irq.h"
20
#include "mmu.h"
A
Avi Kivity 已提交
21

22
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
23
#include <linux/module.h>
24
#include <linux/kernel.h>
A
Avi Kivity 已提交
25 26
#include <linux/mm.h>
#include <linux/highmem.h>
A
Alexey Dobriyan 已提交
27
#include <linux/sched.h>
28
#include <linux/moduleparam.h>
29
#include <linux/ftrace_event.h>
30
#include <linux/slab.h>
31
#include <linux/tboot.h>
32
#include "kvm_cache_regs.h"
33
#include "x86.h"
A
Avi Kivity 已提交
34

A
Avi Kivity 已提交
35
#include <asm/io.h>
A
Anthony Liguori 已提交
36
#include <asm/desc.h>
37
#include <asm/vmx.h>
38
#include <asm/virtext.h>
A
Andi Kleen 已提交
39
#include <asm/mce.h>
40 41
#include <asm/i387.h>
#include <asm/xcr.h>
A
Avi Kivity 已提交
42

43 44
#include "trace.h"

45 46
#define __ex(x) __kvm_handle_fault_on_reboot(x)

A
Avi Kivity 已提交
47 48 49
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

50
static int __read_mostly bypass_guest_pf = 1;
51
module_param(bypass_guest_pf, bool, S_IRUGO);
52

53
static int __read_mostly enable_vpid = 1;
54
module_param_named(vpid, enable_vpid, bool, 0444);
55

56
static int __read_mostly flexpriority_enabled = 1;
57
module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
58

59
static int __read_mostly enable_ept = 1;
60
module_param_named(ept, enable_ept, bool, S_IRUGO);
S
Sheng Yang 已提交
61

62 63 64 65
static int __read_mostly enable_unrestricted_guest = 1;
module_param_named(unrestricted_guest,
			enable_unrestricted_guest, bool, S_IRUGO);

66
static int __read_mostly emulate_invalid_guest_state = 0;
67
module_param(emulate_invalid_guest_state, bool, S_IRUGO);
68

69 70 71
static int __read_mostly vmm_exclusive = 1;
module_param(vmm_exclusive, bool, S_IRUGO);

72 73 74
static int __read_mostly yield_on_hlt = 1;
module_param(yield_on_hlt, bool, S_IRUGO);

75 76 77 78 79
#define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST				\
	(X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD)
#define KVM_GUEST_CR0_MASK						\
	(KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST				\
80
	(X86_CR0_WP | X86_CR0_NE)
81 82
#define KVM_VM_CR0_ALWAYS_ON						\
	(KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
83 84 85 86
#define KVM_CR4_GUEST_OWNED_BITS				      \
	(X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
	 | X86_CR4_OSXMMEXCPT)

87 88 89
#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)

90 91
#define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))

92 93 94 95
/*
 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 * ple_gap:    upper bound on the amount of time between two successive
 *             executions of PAUSE in a loop. Also indicate if ple enabled.
96
 *             According to test, this time is usually smaller than 128 cycles.
97 98 99 100 101 102
 * ple_window: upper bound on the amount of time a guest is allowed to execute
 *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 *             less than 2^12 cycles
 * Time is measured based on a counter that runs at the same rate as the TSC,
 * refer SDM volume 3b section 21.6.13 & 22.1.3.
 */
103
#define KVM_VMX_DEFAULT_PLE_GAP    128
104 105 106 107 108 109 110
#define KVM_VMX_DEFAULT_PLE_WINDOW 4096
static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
module_param(ple_gap, int, S_IRUGO);

static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
module_param(ple_window, int, S_IRUGO);

111 112
#define NR_AUTOLOAD_MSRS 1

113 114 115 116 117 118
struct vmcs {
	u32 revision_id;
	u32 abort;
	char data[0];
};

119 120 121
struct shared_msr_entry {
	unsigned index;
	u64 data;
122
	u64 mask;
123 124
};

125
struct vcpu_vmx {
R
Rusty Russell 已提交
126
	struct kvm_vcpu       vcpu;
127
	struct list_head      local_vcpus_link;
128
	unsigned long         host_rsp;
129
	int                   launched;
130
	u8                    fail;
131
	u32                   exit_intr_info;
132
	u32                   idt_vectoring_info;
133
	struct shared_msr_entry *guest_msrs;
134 135 136
	int                   nmsrs;
	int                   save_nmsrs;
#ifdef CONFIG_X86_64
137 138
	u64 		      msr_host_kernel_gs_base;
	u64 		      msr_guest_kernel_gs_base;
139 140
#endif
	struct vmcs          *vmcs;
141 142 143 144 145
	struct msr_autoload {
		unsigned nr;
		struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS];
		struct vmx_msr_entry host[NR_AUTOLOAD_MSRS];
	} msr_autoload;
146 147 148
	struct {
		int           loaded;
		u16           fs_sel, gs_sel, ldt_sel;
149 150
		int           gs_ldt_reload_needed;
		int           fs_reload_needed;
M
Mike Day 已提交
151
	} host_state;
152
	struct {
153
		int vm86_active;
154
		ulong save_rflags;
155 156 157 158 159 160
		struct kvm_save_segment {
			u16 selector;
			unsigned long base;
			u32 limit;
			u32 ar;
		} tr, es, ds, fs, gs;
161
	} rmode;
162
	int vpid;
163
	bool emulation_required;
164 165 166 167 168

	/* Support for vnmi-less CPUs */
	int soft_vnmi_blocked;
	ktime_t entry_time;
	s64 vnmi_blocked_time;
A
Andi Kleen 已提交
169
	u32 exit_reason;
170 171

	bool rdtscp_enabled;
172 173 174 175
};

static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
176
	return container_of(vcpu, struct vcpu_vmx, vcpu);
177 178
}

179
static int init_rmode(struct kvm *kvm);
180
static u64 construct_eptp(unsigned long root_hpa);
181 182
static void kvm_cpu_vmxon(u64 addr);
static void kvm_cpu_vmxoff(void);
183
static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
184

A
Avi Kivity 已提交
185 186
static DEFINE_PER_CPU(struct vmcs *, vmxarea);
static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
187
static DEFINE_PER_CPU(struct list_head, vcpus_on_cpu);
188
static DEFINE_PER_CPU(struct desc_ptr, host_gdt);
A
Avi Kivity 已提交
189

190 191
static unsigned long *vmx_io_bitmap_a;
static unsigned long *vmx_io_bitmap_b;
192 193
static unsigned long *vmx_msr_bitmap_legacy;
static unsigned long *vmx_msr_bitmap_longmode;
194

A
Avi Kivity 已提交
195 196
static bool cpu_has_load_ia32_efer;

197 198 199
static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
static DEFINE_SPINLOCK(vmx_vpid_lock);

200
static struct vmcs_config {
A
Avi Kivity 已提交
201 202 203
	int size;
	int order;
	u32 revision_id;
204 205
	u32 pin_based_exec_ctrl;
	u32 cpu_based_exec_ctrl;
206
	u32 cpu_based_2nd_exec_ctrl;
207 208 209
	u32 vmexit_ctrl;
	u32 vmentry_ctrl;
} vmcs_config;
A
Avi Kivity 已提交
210

H
Hannes Eder 已提交
211
static struct vmx_capability {
S
Sheng Yang 已提交
212 213 214 215
	u32 ept;
	u32 vpid;
} vmx_capability;

A
Avi Kivity 已提交
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
#define VMX_SEGMENT_FIELD(seg)					\
	[VCPU_SREG_##seg] = {                                   \
		.selector = GUEST_##seg##_SELECTOR,		\
		.base = GUEST_##seg##_BASE,		   	\
		.limit = GUEST_##seg##_LIMIT,		   	\
		.ar_bytes = GUEST_##seg##_AR_BYTES,	   	\
	}

static struct kvm_vmx_segment_field {
	unsigned selector;
	unsigned base;
	unsigned limit;
	unsigned ar_bytes;
} kvm_vmx_segment_fields[] = {
	VMX_SEGMENT_FIELD(CS),
	VMX_SEGMENT_FIELD(DS),
	VMX_SEGMENT_FIELD(ES),
	VMX_SEGMENT_FIELD(FS),
	VMX_SEGMENT_FIELD(GS),
	VMX_SEGMENT_FIELD(SS),
	VMX_SEGMENT_FIELD(TR),
	VMX_SEGMENT_FIELD(LDTR),
};

240 241
static u64 host_efer;

A
Avi Kivity 已提交
242 243
static void ept_save_pdptrs(struct kvm_vcpu *vcpu);

244
/*
B
Brian Gerst 已提交
245
 * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
246 247
 * away by decrementing the array size.
 */
A
Avi Kivity 已提交
248
static const u32 vmx_msr_index[] = {
249
#ifdef CONFIG_X86_64
250
	MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
A
Avi Kivity 已提交
251
#endif
B
Brian Gerst 已提交
252
	MSR_EFER, MSR_TSC_AUX, MSR_STAR,
A
Avi Kivity 已提交
253
};
254
#define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
A
Avi Kivity 已提交
255

256
static inline bool is_page_fault(u32 intr_info)
A
Avi Kivity 已提交
257 258 259
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
260
		(INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
A
Avi Kivity 已提交
261 262
}

263
static inline bool is_no_device(u32 intr_info)
264 265 266
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
267
		(INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
268 269
}

270
static inline bool is_invalid_opcode(u32 intr_info)
271 272 273
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
274
		(INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK);
275 276
}

277
static inline bool is_external_interrupt(u32 intr_info)
A
Avi Kivity 已提交
278 279 280 281 282
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
		== (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
}

283
static inline bool is_machine_check(u32 intr_info)
A
Andi Kleen 已提交
284 285 286 287 288 289
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
		(INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
}

290
static inline bool cpu_has_vmx_msr_bitmap(void)
S
Sheng Yang 已提交
291
{
292
	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
S
Sheng Yang 已提交
293 294
}

295
static inline bool cpu_has_vmx_tpr_shadow(void)
296
{
297
	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
298 299
}

300
static inline bool vm_need_tpr_shadow(struct kvm *kvm)
301
{
302
	return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
303 304
}

305
static inline bool cpu_has_secondary_exec_ctrls(void)
306
{
307 308
	return vmcs_config.cpu_based_exec_ctrl &
		CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
309 310
}

311
static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
312
{
313 314 315 316 317 318 319 320
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
}

static inline bool cpu_has_vmx_flexpriority(void)
{
	return cpu_has_vmx_tpr_shadow() &&
		cpu_has_vmx_virtualize_apic_accesses();
321 322
}

323 324
static inline bool cpu_has_vmx_ept_execute_only(void)
{
325
	return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
326 327 328 329
}

static inline bool cpu_has_vmx_eptp_uncacheable(void)
{
330
	return vmx_capability.ept & VMX_EPTP_UC_BIT;
331 332 333 334
}

static inline bool cpu_has_vmx_eptp_writeback(void)
{
335
	return vmx_capability.ept & VMX_EPTP_WB_BIT;
336 337 338 339
}

static inline bool cpu_has_vmx_ept_2m_page(void)
{
340
	return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
341 342
}

343 344
static inline bool cpu_has_vmx_ept_1g_page(void)
{
345
	return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
346 347
}

348 349 350 351 352
static inline bool cpu_has_vmx_ept_4levels(void)
{
	return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
}

353
static inline bool cpu_has_vmx_invept_individual_addr(void)
S
Sheng Yang 已提交
354
{
355
	return vmx_capability.ept & VMX_EPT_EXTENT_INDIVIDUAL_BIT;
S
Sheng Yang 已提交
356 357
}

358
static inline bool cpu_has_vmx_invept_context(void)
S
Sheng Yang 已提交
359
{
360
	return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
S
Sheng Yang 已提交
361 362
}

363
static inline bool cpu_has_vmx_invept_global(void)
S
Sheng Yang 已提交
364
{
365
	return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
S
Sheng Yang 已提交
366 367
}

368 369 370 371 372
static inline bool cpu_has_vmx_invvpid_single(void)
{
	return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
}

373 374 375 376 377
static inline bool cpu_has_vmx_invvpid_global(void)
{
	return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
}

378
static inline bool cpu_has_vmx_ept(void)
S
Sheng Yang 已提交
379
{
380 381
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_EPT;
S
Sheng Yang 已提交
382 383
}

384
static inline bool cpu_has_vmx_unrestricted_guest(void)
385 386 387 388 389
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_UNRESTRICTED_GUEST;
}

390
static inline bool cpu_has_vmx_ple(void)
391 392 393 394 395
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_PAUSE_LOOP_EXITING;
}

396
static inline bool vm_need_virtualize_apic_accesses(struct kvm *kvm)
397
{
398
	return flexpriority_enabled && irqchip_in_kernel(kvm);
399 400
}

401
static inline bool cpu_has_vmx_vpid(void)
402
{
403 404
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_VPID;
405 406
}

407
static inline bool cpu_has_vmx_rdtscp(void)
408 409 410 411 412
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_RDTSCP;
}

413
static inline bool cpu_has_virtual_nmis(void)
414 415 416 417
{
	return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
}

418 419 420 421 422 423
static inline bool cpu_has_vmx_wbinvd_exit(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_WBINVD_EXITING;
}

424 425 426 427 428
static inline bool report_flexpriority(void)
{
	return flexpriority_enabled;
}

R
Rusty Russell 已提交
429
static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
430 431 432
{
	int i;

433
	for (i = 0; i < vmx->nmsrs; ++i)
434
		if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
435 436 437 438
			return i;
	return -1;
}

439 440 441 442 443 444 445 446
static inline void __invvpid(int ext, u16 vpid, gva_t gva)
{
    struct {
	u64 vpid : 16;
	u64 rsvd : 48;
	u64 gva;
    } operand = { vpid, 0, gva };

447
    asm volatile (__ex(ASM_VMX_INVVPID)
448 449 450 451 452
		  /* CF==1 or ZF==1 --> rc = -1 */
		  "; ja 1f ; ud2 ; 1:"
		  : : "a"(&operand), "c"(ext) : "cc", "memory");
}

453 454 455 456 457 458
static inline void __invept(int ext, u64 eptp, gpa_t gpa)
{
	struct {
		u64 eptp, gpa;
	} operand = {eptp, gpa};

459
	asm volatile (__ex(ASM_VMX_INVEPT)
460 461 462 463 464
			/* CF==1 or ZF==1 --> rc = -1 */
			"; ja 1f ; ud2 ; 1:\n"
			: : "a" (&operand), "c" (ext) : "cc", "memory");
}

465
static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
466 467 468
{
	int i;

R
Rusty Russell 已提交
469
	i = __find_msr_index(vmx, msr);
470
	if (i >= 0)
471
		return &vmx->guest_msrs[i];
A
Al Viro 已提交
472
	return NULL;
473 474
}

A
Avi Kivity 已提交
475 476 477 478 479
static void vmcs_clear(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(vmcs);
	u8 error;

480
	asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
481
		      : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
A
Avi Kivity 已提交
482 483 484 485 486 487
		      : "cc", "memory");
	if (error)
		printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
		       vmcs, phys_addr);
}

488 489 490 491 492 493
static void vmcs_load(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(vmcs);
	u8 error;

	asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
494
			: "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
495 496 497 498 499 500
			: "cc", "memory");
	if (error)
		printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
		       vmcs, phys_addr);
}

A
Avi Kivity 已提交
501 502
static void __vcpu_clear(void *arg)
{
R
Rusty Russell 已提交
503
	struct vcpu_vmx *vmx = arg;
504
	int cpu = raw_smp_processor_id();
A
Avi Kivity 已提交
505

R
Rusty Russell 已提交
506
	if (vmx->vcpu.cpu == cpu)
507 508
		vmcs_clear(vmx->vmcs);
	if (per_cpu(current_vmcs, cpu) == vmx->vmcs)
A
Avi Kivity 已提交
509
		per_cpu(current_vmcs, cpu) = NULL;
510 511 512
	list_del(&vmx->local_vcpus_link);
	vmx->vcpu.cpu = -1;
	vmx->launched = 0;
A
Avi Kivity 已提交
513 514
}

R
Rusty Russell 已提交
515
static void vcpu_clear(struct vcpu_vmx *vmx)
A
Avi Kivity 已提交
516
{
517 518
	if (vmx->vcpu.cpu == -1)
		return;
519
	smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 1);
A
Avi Kivity 已提交
520 521
}

522
static inline void vpid_sync_vcpu_single(struct vcpu_vmx *vmx)
523 524 525 526
{
	if (vmx->vpid == 0)
		return;

527 528
	if (cpu_has_vmx_invvpid_single())
		__invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vmx->vpid, 0);
529 530
}

531 532 533 534 535 536 537 538 539
static inline void vpid_sync_vcpu_global(void)
{
	if (cpu_has_vmx_invvpid_global())
		__invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
}

static inline void vpid_sync_context(struct vcpu_vmx *vmx)
{
	if (cpu_has_vmx_invvpid_single())
540
		vpid_sync_vcpu_single(vmx);
541 542 543 544
	else
		vpid_sync_vcpu_global();
}

545 546 547 548 549 550 551 552
static inline void ept_sync_global(void)
{
	if (cpu_has_vmx_invept_global())
		__invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
}

static inline void ept_sync_context(u64 eptp)
{
553
	if (enable_ept) {
554 555 556 557 558 559 560 561 562
		if (cpu_has_vmx_invept_context())
			__invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
		else
			ept_sync_global();
	}
}

static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa)
{
563
	if (enable_ept) {
564 565 566 567 568 569 570 571
		if (cpu_has_vmx_invept_individual_addr())
			__invept(VMX_EPT_EXTENT_INDIVIDUAL_ADDR,
					eptp, gpa);
		else
			ept_sync_context(eptp);
	}
}

A
Avi Kivity 已提交
572 573
static unsigned long vmcs_readl(unsigned long field)
{
574
	unsigned long value = 0;
A
Avi Kivity 已提交
575

576
	asm volatile (__ex(ASM_VMX_VMREAD_RDX_RAX)
577
		      : "+a"(value) : "d"(field) : "cc");
A
Avi Kivity 已提交
578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
	return value;
}

static u16 vmcs_read16(unsigned long field)
{
	return vmcs_readl(field);
}

static u32 vmcs_read32(unsigned long field)
{
	return vmcs_readl(field);
}

static u64 vmcs_read64(unsigned long field)
{
593
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
594 595 596 597 598 599
	return vmcs_readl(field);
#else
	return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
#endif
}

600 601 602 603 604 605 606
static noinline void vmwrite_error(unsigned long field, unsigned long value)
{
	printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
	       field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
	dump_stack();
}

A
Avi Kivity 已提交
607 608 609 610
static void vmcs_writel(unsigned long field, unsigned long value)
{
	u8 error;

611
	asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
M
Mike Day 已提交
612
		       : "=q"(error) : "a"(value), "d"(field) : "cc");
613 614
	if (unlikely(error))
		vmwrite_error(field, value);
A
Avi Kivity 已提交
615 616 617 618 619 620 621 622 623 624 625 626 627 628 629
}

static void vmcs_write16(unsigned long field, u16 value)
{
	vmcs_writel(field, value);
}

static void vmcs_write32(unsigned long field, u32 value)
{
	vmcs_writel(field, value);
}

static void vmcs_write64(unsigned long field, u64 value)
{
	vmcs_writel(field, value);
630
#ifndef CONFIG_X86_64
A
Avi Kivity 已提交
631 632 633 634 635
	asm volatile ("");
	vmcs_writel(field+1, value >> 32);
#endif
}

636 637 638 639 640 641 642 643 644 645
static void vmcs_clear_bits(unsigned long field, u32 mask)
{
	vmcs_writel(field, vmcs_readl(field) & ~mask);
}

static void vmcs_set_bits(unsigned long field, u32 mask)
{
	vmcs_writel(field, vmcs_readl(field) | mask);
}

646 647 648 649
static void update_exception_bitmap(struct kvm_vcpu *vcpu)
{
	u32 eb;

J
Jan Kiszka 已提交
650 651 652 653 654 655
	eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
	     (1u << NM_VECTOR) | (1u << DB_VECTOR);
	if ((vcpu->guest_debug &
	     (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
	    (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
		eb |= 1u << BP_VECTOR;
656
	if (to_vmx(vcpu)->rmode.vm86_active)
657
		eb = ~0;
658
	if (enable_ept)
659
		eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
660 661
	if (vcpu->fpu_active)
		eb &= ~(1u << NM_VECTOR);
662 663 664
	vmcs_write32(EXCEPTION_BITMAP, eb);
}

665 666 667 668 669
static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
{
	unsigned i;
	struct msr_autoload *m = &vmx->msr_autoload;

A
Avi Kivity 已提交
670 671 672 673 674 675
	if (msr == MSR_EFER && cpu_has_load_ia32_efer) {
		vmcs_clear_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER);
		vmcs_clear_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER);
		return;
	}

676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
	for (i = 0; i < m->nr; ++i)
		if (m->guest[i].index == msr)
			break;

	if (i == m->nr)
		return;
	--m->nr;
	m->guest[i] = m->guest[m->nr];
	m->host[i] = m->host[m->nr];
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
}

static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
				  u64 guest_val, u64 host_val)
{
	unsigned i;
	struct msr_autoload *m = &vmx->msr_autoload;

A
Avi Kivity 已提交
695 696 697 698 699 700 701 702
	if (msr == MSR_EFER && cpu_has_load_ia32_efer) {
		vmcs_write64(GUEST_IA32_EFER, guest_val);
		vmcs_write64(HOST_IA32_EFER, host_val);
		vmcs_set_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER);
		vmcs_set_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER);
		return;
	}

703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
	for (i = 0; i < m->nr; ++i)
		if (m->guest[i].index == msr)
			break;

	if (i == m->nr) {
		++m->nr;
		vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
		vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
	}

	m->guest[i].index = msr;
	m->guest[i].value = guest_val;
	m->host[i].index = msr;
	m->host[i].value = host_val;
}

719 720 721 722 723
static void reload_tss(void)
{
	/*
	 * VT restores TR but not its size.  Useless.
	 */
724
	struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
725
	struct desc_struct *descs;
726

727
	descs = (void *)gdt->address;
728 729 730 731
	descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
	load_TR_desc();
}

A
Avi Kivity 已提交
732
static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
733
{
R
Roel Kluin 已提交
734
	u64 guest_efer;
735 736
	u64 ignore_bits;

737
	guest_efer = vmx->vcpu.arch.efer;
R
Roel Kluin 已提交
738

739 740 741 742 743 744 745 746 747 748 749 750 751
	/*
	 * NX is emulated; LMA and LME handled by hardware; SCE meaninless
	 * outside long mode
	 */
	ignore_bits = EFER_NX | EFER_SCE;
#ifdef CONFIG_X86_64
	ignore_bits |= EFER_LMA | EFER_LME;
	/* SCE is meaningful only in long mode on Intel */
	if (guest_efer & EFER_LMA)
		ignore_bits &= ~(u64)EFER_SCE;
#endif
	guest_efer &= ~ignore_bits;
	guest_efer |= host_efer & ignore_bits;
752
	vmx->guest_msrs[efer_offset].data = guest_efer;
753
	vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
754 755 756 757 758 759 760 761 762 763 764

	clear_atomic_switch_msr(vmx, MSR_EFER);
	/* On ept, can't emulate nx, and must switch nx atomically */
	if (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX)) {
		guest_efer = vmx->vcpu.arch.efer;
		if (!(guest_efer & EFER_LMA))
			guest_efer &= ~EFER_LME;
		add_atomic_switch_msr(vmx, MSR_EFER, guest_efer, host_efer);
		return false;
	}

765
	return true;
766 767
}

768 769
static unsigned long segment_base(u16 selector)
{
770
	struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
771 772 773 774 775 776 777
	struct desc_struct *d;
	unsigned long table_base;
	unsigned long v;

	if (!(selector & ~3))
		return 0;

778
	table_base = gdt->address;
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803

	if (selector & 4) {           /* from ldt */
		u16 ldt_selector = kvm_read_ldt();

		if (!(ldt_selector & ~3))
			return 0;

		table_base = segment_base(ldt_selector);
	}
	d = (struct desc_struct *)(table_base + (selector & ~7));
	v = get_desc_base(d);
#ifdef CONFIG_X86_64
       if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
               v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
#endif
	return v;
}

static inline unsigned long kvm_read_tr_base(void)
{
	u16 tr;
	asm("str %0" : "=g"(tr));
	return segment_base(tr);
}

804
static void vmx_save_host_state(struct kvm_vcpu *vcpu)
805
{
806
	struct vcpu_vmx *vmx = to_vmx(vcpu);
807
	int i;
808

809
	if (vmx->host_state.loaded)
810 811
		return;

812
	vmx->host_state.loaded = 1;
813 814 815 816
	/*
	 * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
	 * allow segment selectors with cpl > 0 or ti == 1.
	 */
817
	vmx->host_state.ldt_sel = kvm_read_ldt();
818
	vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
819
	savesegment(fs, vmx->host_state.fs_sel);
820
	if (!(vmx->host_state.fs_sel & 7)) {
821
		vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
822 823
		vmx->host_state.fs_reload_needed = 0;
	} else {
824
		vmcs_write16(HOST_FS_SELECTOR, 0);
825
		vmx->host_state.fs_reload_needed = 1;
826
	}
827
	savesegment(gs, vmx->host_state.gs_sel);
828 829
	if (!(vmx->host_state.gs_sel & 7))
		vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
830 831
	else {
		vmcs_write16(HOST_GS_SELECTOR, 0);
832
		vmx->host_state.gs_ldt_reload_needed = 1;
833 834 835 836 837 838
	}

#ifdef CONFIG_X86_64
	vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
	vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
#else
839 840
	vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
	vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
841
#endif
842 843

#ifdef CONFIG_X86_64
844 845
	rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
	if (is_long_mode(&vmx->vcpu))
846
		wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
847
#endif
848 849
	for (i = 0; i < vmx->save_nmsrs; ++i)
		kvm_set_shared_msr(vmx->guest_msrs[i].index,
850 851
				   vmx->guest_msrs[i].data,
				   vmx->guest_msrs[i].mask);
852 853
}

854
static void __vmx_load_host_state(struct vcpu_vmx *vmx)
855
{
856
	if (!vmx->host_state.loaded)
857 858
		return;

859
	++vmx->vcpu.stat.host_state_reload;
860
	vmx->host_state.loaded = 0;
861 862 863 864
#ifdef CONFIG_X86_64
	if (is_long_mode(&vmx->vcpu))
		rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
#endif
865
	if (vmx->host_state.gs_ldt_reload_needed) {
866
		kvm_load_ldt(vmx->host_state.ldt_sel);
867
#ifdef CONFIG_X86_64
868 869 870
		load_gs_index(vmx->host_state.gs_sel);
#else
		loadsegment(gs, vmx->host_state.gs_sel);
871 872
#endif
	}
873 874
	if (vmx->host_state.fs_reload_needed)
		loadsegment(fs, vmx->host_state.fs_sel);
875
	reload_tss();
876
#ifdef CONFIG_X86_64
877
	wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
878
#endif
879 880
	if (current_thread_info()->status & TS_USEDFPU)
		clts();
881
	load_gdt(&__get_cpu_var(host_gdt));
882 883
}

884 885 886 887 888 889 890
static void vmx_load_host_state(struct vcpu_vmx *vmx)
{
	preempt_disable();
	__vmx_load_host_state(vmx);
	preempt_enable();
}

A
Avi Kivity 已提交
891 892 893 894
/*
 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
 * vcpu mutex is already taken.
 */
895
static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
A
Avi Kivity 已提交
896
{
897
	struct vcpu_vmx *vmx = to_vmx(vcpu);
898
	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
A
Avi Kivity 已提交
899

900 901 902
	if (!vmm_exclusive)
		kvm_cpu_vmxon(phys_addr);
	else if (vcpu->cpu != cpu)
R
Rusty Russell 已提交
903
		vcpu_clear(vmx);
A
Avi Kivity 已提交
904

905 906
	if (per_cpu(current_vmcs, cpu) != vmx->vmcs) {
		per_cpu(current_vmcs, cpu) = vmx->vmcs;
907
		vmcs_load(vmx->vmcs);
A
Avi Kivity 已提交
908 909 910
	}

	if (vcpu->cpu != cpu) {
911
		struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
A
Avi Kivity 已提交
912 913
		unsigned long sysenter_esp;

914
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
915 916 917 918 919
		local_irq_disable();
		list_add(&vmx->local_vcpus_link,
			 &per_cpu(vcpus_on_cpu, cpu));
		local_irq_enable();

A
Avi Kivity 已提交
920 921 922 923
		/*
		 * Linux uses per-cpu TSS and GDT, so set these when switching
		 * processors.
		 */
924
		vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
925
		vmcs_writel(HOST_GDTR_BASE, gdt->address);   /* 22.2.4 */
A
Avi Kivity 已提交
926 927 928 929 930 931 932 933

		rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
		vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
	}
}

static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
{
934
	__vmx_load_host_state(to_vmx(vcpu));
935
	if (!vmm_exclusive) {
936
		__vcpu_clear(to_vmx(vcpu));
937 938
		kvm_cpu_vmxoff();
	}
A
Avi Kivity 已提交
939 940
}

941 942
static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
{
943 944
	ulong cr0;

945 946 947
	if (vcpu->fpu_active)
		return;
	vcpu->fpu_active = 1;
948 949 950 951
	cr0 = vmcs_readl(GUEST_CR0);
	cr0 &= ~(X86_CR0_TS | X86_CR0_MP);
	cr0 |= kvm_read_cr0_bits(vcpu, X86_CR0_TS | X86_CR0_MP);
	vmcs_writel(GUEST_CR0, cr0);
952
	update_exception_bitmap(vcpu);
953 954
	vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
955 956
}

957 958
static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);

959 960
static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
{
961
	vmx_decache_cr0_guest_bits(vcpu);
962
	vmcs_set_bits(GUEST_CR0, X86_CR0_TS | X86_CR0_MP);
963
	update_exception_bitmap(vcpu);
964 965 966
	vcpu->arch.cr0_guest_owned_bits = 0;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
	vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
967 968
}

A
Avi Kivity 已提交
969 970
static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
{
971
	unsigned long rflags, save_rflags;
972 973

	rflags = vmcs_readl(GUEST_RFLAGS);
974 975 976 977 978
	if (to_vmx(vcpu)->rmode.vm86_active) {
		rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
		save_rflags = to_vmx(vcpu)->rmode.save_rflags;
		rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
	}
979
	return rflags;
A
Avi Kivity 已提交
980 981 982 983
}

static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
984 985
	if (to_vmx(vcpu)->rmode.vm86_active) {
		to_vmx(vcpu)->rmode.save_rflags = rflags;
986
		rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
987
	}
A
Avi Kivity 已提交
988 989 990
	vmcs_writel(GUEST_RFLAGS, rflags);
}

991 992 993 994 995 996
static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	int ret = 0;

	if (interruptibility & GUEST_INTR_STATE_STI)
997
		ret |= KVM_X86_SHADOW_INT_STI;
998
	if (interruptibility & GUEST_INTR_STATE_MOV_SS)
999
		ret |= KVM_X86_SHADOW_INT_MOV_SS;
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010

	return ret & mask;
}

static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	u32 interruptibility = interruptibility_old;

	interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);

1011
	if (mask & KVM_X86_SHADOW_INT_MOV_SS)
1012
		interruptibility |= GUEST_INTR_STATE_MOV_SS;
1013
	else if (mask & KVM_X86_SHADOW_INT_STI)
1014 1015 1016 1017 1018 1019
		interruptibility |= GUEST_INTR_STATE_STI;

	if ((interruptibility != interruptibility_old))
		vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
}

A
Avi Kivity 已提交
1020 1021 1022 1023
static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
	unsigned long rip;

1024
	rip = kvm_rip_read(vcpu);
A
Avi Kivity 已提交
1025
	rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
1026
	kvm_rip_write(vcpu, rip);
A
Avi Kivity 已提交
1027

1028 1029
	/* skipping an emulated instruction also counts */
	vmx_set_interrupt_shadow(vcpu, 0);
A
Avi Kivity 已提交
1030 1031
}

1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
{
	/* Ensure that we clear the HLT state in the VMCS.  We don't need to
	 * explicitly skip the instruction because if the HLT state is set, then
	 * the instruction is already executing and RIP has already been
	 * advanced. */
	if (!yield_on_hlt &&
	    vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
		vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
}

1043
static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
1044 1045
				bool has_error_code, u32 error_code,
				bool reinject)
1046
{
1047
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1048
	u32 intr_info = nr | INTR_INFO_VALID_MASK;
1049

1050
	if (has_error_code) {
1051
		vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
1052 1053
		intr_info |= INTR_INFO_DELIVER_CODE_MASK;
	}
1054

1055
	if (vmx->rmode.vm86_active) {
1056 1057
		if (kvm_inject_realmode_interrupt(vcpu, nr) != EMULATE_DONE)
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1058 1059 1060
		return;
	}

1061 1062 1063
	if (kvm_exception_is_soft(nr)) {
		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
			     vmx->vcpu.arch.event_exit_inst_len);
1064 1065 1066 1067 1068
		intr_info |= INTR_TYPE_SOFT_EXCEPTION;
	} else
		intr_info |= INTR_TYPE_HARD_EXCEPTION;

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
1069
	vmx_clear_hlt(vcpu);
1070 1071
}

1072 1073 1074 1075 1076
static bool vmx_rdtscp_supported(void)
{
	return cpu_has_vmx_rdtscp();
}

1077 1078 1079
/*
 * Swap MSR entry in host/guest MSR entry array.
 */
R
Rusty Russell 已提交
1080
static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
1081
{
1082
	struct shared_msr_entry tmp;
1083 1084 1085 1086

	tmp = vmx->guest_msrs[to];
	vmx->guest_msrs[to] = vmx->guest_msrs[from];
	vmx->guest_msrs[from] = tmp;
1087 1088
}

1089 1090 1091 1092 1093
/*
 * Set up the vmcs to automatically save and restore system
 * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
 * mode, as fiddling with msrs is very expensive.
 */
R
Rusty Russell 已提交
1094
static void setup_msrs(struct vcpu_vmx *vmx)
1095
{
1096
	int save_nmsrs, index;
1097
	unsigned long *msr_bitmap;
1098

1099
	vmx_load_host_state(vmx);
1100 1101
	save_nmsrs = 0;
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
1102 1103
	if (is_long_mode(&vmx->vcpu)) {
		index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
1104
		if (index >= 0)
R
Rusty Russell 已提交
1105 1106
			move_msr_up(vmx, index, save_nmsrs++);
		index = __find_msr_index(vmx, MSR_LSTAR);
1107
		if (index >= 0)
R
Rusty Russell 已提交
1108 1109
			move_msr_up(vmx, index, save_nmsrs++);
		index = __find_msr_index(vmx, MSR_CSTAR);
1110
		if (index >= 0)
R
Rusty Russell 已提交
1111
			move_msr_up(vmx, index, save_nmsrs++);
1112 1113 1114
		index = __find_msr_index(vmx, MSR_TSC_AUX);
		if (index >= 0 && vmx->rdtscp_enabled)
			move_msr_up(vmx, index, save_nmsrs++);
1115
		/*
B
Brian Gerst 已提交
1116
		 * MSR_STAR is only needed on long mode guests, and only
1117 1118
		 * if efer.sce is enabled.
		 */
B
Brian Gerst 已提交
1119
		index = __find_msr_index(vmx, MSR_STAR);
1120
		if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
R
Rusty Russell 已提交
1121
			move_msr_up(vmx, index, save_nmsrs++);
1122 1123
	}
#endif
A
Avi Kivity 已提交
1124 1125
	index = __find_msr_index(vmx, MSR_EFER);
	if (index >= 0 && update_transition_efer(vmx, index))
1126
		move_msr_up(vmx, index, save_nmsrs++);
1127

1128
	vmx->save_nmsrs = save_nmsrs;
1129 1130 1131 1132 1133 1134 1135 1136 1137

	if (cpu_has_vmx_msr_bitmap()) {
		if (is_long_mode(&vmx->vcpu))
			msr_bitmap = vmx_msr_bitmap_longmode;
		else
			msr_bitmap = vmx_msr_bitmap_legacy;

		vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
	}
1138 1139
}

A
Avi Kivity 已提交
1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153
/*
 * reads and returns guest's timestamp counter "register"
 * guest_tsc = host_tsc + tsc_offset    -- 21.3
 */
static u64 guest_read_tsc(void)
{
	u64 host_tsc, tsc_offset;

	rdtscll(host_tsc);
	tsc_offset = vmcs_read64(TSC_OFFSET);
	return host_tsc + tsc_offset;
}

/*
1154
 * writes 'offset' into guest's timestamp counter offset register
A
Avi Kivity 已提交
1155
 */
1156
static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
A
Avi Kivity 已提交
1157
{
1158
	vmcs_write64(TSC_OFFSET, offset);
A
Avi Kivity 已提交
1159 1160
}

Z
Zachary Amsden 已提交
1161 1162 1163 1164 1165 1166
static void vmx_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment)
{
	u64 offset = vmcs_read64(TSC_OFFSET);
	vmcs_write64(TSC_OFFSET, offset + adjustment);
}

A
Avi Kivity 已提交
1167 1168 1169 1170 1171 1172 1173 1174
/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
	u64 data;
1175
	struct shared_msr_entry *msr;
A
Avi Kivity 已提交
1176 1177 1178 1179 1180 1181 1182

	if (!pdata) {
		printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
		return -EINVAL;
	}

	switch (msr_index) {
1183
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1184 1185 1186 1187 1188 1189
	case MSR_FS_BASE:
		data = vmcs_readl(GUEST_FS_BASE);
		break;
	case MSR_GS_BASE:
		data = vmcs_readl(GUEST_GS_BASE);
		break;
1190 1191 1192 1193
	case MSR_KERNEL_GS_BASE:
		vmx_load_host_state(to_vmx(vcpu));
		data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
		break;
1194
#endif
A
Avi Kivity 已提交
1195
	case MSR_EFER:
1196
		return kvm_get_msr_common(vcpu, msr_index, pdata);
1197
	case MSR_IA32_TSC:
A
Avi Kivity 已提交
1198 1199 1200 1201 1202 1203
		data = guest_read_tsc();
		break;
	case MSR_IA32_SYSENTER_CS:
		data = vmcs_read32(GUEST_SYSENTER_CS);
		break;
	case MSR_IA32_SYSENTER_EIP:
A
Avi Kivity 已提交
1204
		data = vmcs_readl(GUEST_SYSENTER_EIP);
A
Avi Kivity 已提交
1205 1206
		break;
	case MSR_IA32_SYSENTER_ESP:
A
Avi Kivity 已提交
1207
		data = vmcs_readl(GUEST_SYSENTER_ESP);
A
Avi Kivity 已提交
1208
		break;
1209 1210 1211 1212
	case MSR_TSC_AUX:
		if (!to_vmx(vcpu)->rdtscp_enabled)
			return 1;
		/* Otherwise falls through */
A
Avi Kivity 已提交
1213
	default:
1214
		vmx_load_host_state(to_vmx(vcpu));
R
Rusty Russell 已提交
1215
		msr = find_msr_entry(to_vmx(vcpu), msr_index);
1216
		if (msr) {
1217
			vmx_load_host_state(to_vmx(vcpu));
1218 1219
			data = msr->data;
			break;
A
Avi Kivity 已提交
1220
		}
1221
		return kvm_get_msr_common(vcpu, msr_index, pdata);
A
Avi Kivity 已提交
1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
	}

	*pdata = data;
	return 0;
}

/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
{
1235
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1236
	struct shared_msr_entry *msr;
1237 1238
	int ret = 0;

A
Avi Kivity 已提交
1239
	switch (msr_index) {
1240
	case MSR_EFER:
1241
		vmx_load_host_state(vmx);
1242 1243
		ret = kvm_set_msr_common(vcpu, msr_index, data);
		break;
1244
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1245 1246 1247 1248 1249 1250
	case MSR_FS_BASE:
		vmcs_writel(GUEST_FS_BASE, data);
		break;
	case MSR_GS_BASE:
		vmcs_writel(GUEST_GS_BASE, data);
		break;
1251 1252 1253 1254
	case MSR_KERNEL_GS_BASE:
		vmx_load_host_state(vmx);
		vmx->msr_guest_kernel_gs_base = data;
		break;
A
Avi Kivity 已提交
1255 1256 1257 1258 1259
#endif
	case MSR_IA32_SYSENTER_CS:
		vmcs_write32(GUEST_SYSENTER_CS, data);
		break;
	case MSR_IA32_SYSENTER_EIP:
A
Avi Kivity 已提交
1260
		vmcs_writel(GUEST_SYSENTER_EIP, data);
A
Avi Kivity 已提交
1261 1262
		break;
	case MSR_IA32_SYSENTER_ESP:
A
Avi Kivity 已提交
1263
		vmcs_writel(GUEST_SYSENTER_ESP, data);
A
Avi Kivity 已提交
1264
		break;
1265
	case MSR_IA32_TSC:
1266
		kvm_write_tsc(vcpu, data);
A
Avi Kivity 已提交
1267
		break;
S
Sheng Yang 已提交
1268 1269 1270 1271 1272 1273
	case MSR_IA32_CR_PAT:
		if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
			vmcs_write64(GUEST_IA32_PAT, data);
			vcpu->arch.pat = data;
			break;
		}
1274 1275 1276 1277 1278 1279 1280 1281 1282
		ret = kvm_set_msr_common(vcpu, msr_index, data);
		break;
	case MSR_TSC_AUX:
		if (!vmx->rdtscp_enabled)
			return 1;
		/* Check reserved bit, higher 32 bits should be zero */
		if ((data >> 32) != 0)
			return 1;
		/* Otherwise falls through */
A
Avi Kivity 已提交
1283
	default:
R
Rusty Russell 已提交
1284
		msr = find_msr_entry(vmx, msr_index);
1285
		if (msr) {
1286
			vmx_load_host_state(vmx);
1287 1288
			msr->data = data;
			break;
A
Avi Kivity 已提交
1289
		}
1290
		ret = kvm_set_msr_common(vcpu, msr_index, data);
A
Avi Kivity 已提交
1291 1292
	}

1293
	return ret;
A
Avi Kivity 已提交
1294 1295
}

1296
static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
A
Avi Kivity 已提交
1297
{
1298 1299 1300 1301 1302 1303 1304 1305
	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
	switch (reg) {
	case VCPU_REGS_RSP:
		vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
		break;
	case VCPU_REGS_RIP:
		vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
		break;
A
Avi Kivity 已提交
1306 1307 1308 1309
	case VCPU_EXREG_PDPTR:
		if (enable_ept)
			ept_save_pdptrs(vcpu);
		break;
1310 1311 1312
	default:
		break;
	}
A
Avi Kivity 已提交
1313 1314
}

1315
static void set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
A
Avi Kivity 已提交
1316
{
1317 1318 1319 1320 1321
	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]);
	else
		vmcs_writel(GUEST_DR7, vcpu->arch.dr7);

1322
	update_exception_bitmap(vcpu);
A
Avi Kivity 已提交
1323 1324 1325 1326
}

static __init int cpu_has_kvm_support(void)
{
1327
	return cpu_has_vmx();
A
Avi Kivity 已提交
1328 1329 1330 1331 1332 1333 1334
}

static __init int vmx_disabled_by_bios(void)
{
	u64 msr;

	rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
1335 1336 1337 1338 1339
	if (msr & FEATURE_CONTROL_LOCKED) {
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
			&& tboot_enabled())
			return 1;
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
1340 1341 1342
			&& !tboot_enabled()) {
			printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
				" activate TXT before enabling KVM\n");
1343
			return 1;
1344
		}
1345 1346 1347
	}

	return 0;
1348
	/* locked but not enabled */
A
Avi Kivity 已提交
1349 1350
}

1351 1352 1353 1354 1355 1356 1357
static void kvm_cpu_vmxon(u64 addr)
{
	asm volatile (ASM_VMX_VMXON_RAX
			: : "a"(&addr), "m"(addr)
			: "memory", "cc");
}

1358
static int hardware_enable(void *garbage)
A
Avi Kivity 已提交
1359 1360 1361
{
	int cpu = raw_smp_processor_id();
	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
1362
	u64 old, test_bits;
A
Avi Kivity 已提交
1363

1364 1365 1366
	if (read_cr4() & X86_CR4_VMXE)
		return -EBUSY;

1367
	INIT_LIST_HEAD(&per_cpu(vcpus_on_cpu, cpu));
A
Avi Kivity 已提交
1368
	rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
1369 1370 1371 1372 1373 1374 1375

	test_bits = FEATURE_CONTROL_LOCKED;
	test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
	if (tboot_enabled())
		test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;

	if ((old & test_bits) != test_bits) {
A
Avi Kivity 已提交
1376
		/* enable and lock */
1377 1378
		wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
	}
1379
	write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
1380

1381 1382 1383 1384
	if (vmm_exclusive) {
		kvm_cpu_vmxon(phys_addr);
		ept_sync_global();
	}
1385

1386 1387
	store_gdt(&__get_cpu_var(host_gdt));

1388
	return 0;
A
Avi Kivity 已提交
1389 1390
}

1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
static void vmclear_local_vcpus(void)
{
	int cpu = raw_smp_processor_id();
	struct vcpu_vmx *vmx, *n;

	list_for_each_entry_safe(vmx, n, &per_cpu(vcpus_on_cpu, cpu),
				 local_vcpus_link)
		__vcpu_clear(vmx);
}

1401 1402 1403 1404 1405

/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
 * tricks.
 */
static void kvm_cpu_vmxoff(void)
A
Avi Kivity 已提交
1406
{
1407
	asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
A
Avi Kivity 已提交
1408 1409
}

1410 1411
static void hardware_disable(void *garbage)
{
1412 1413 1414 1415
	if (vmm_exclusive) {
		vmclear_local_vcpus();
		kvm_cpu_vmxoff();
	}
1416
	write_cr4(read_cr4() & ~X86_CR4_VMXE);
1417 1418
}

1419
static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
M
Mike Day 已提交
1420
				      u32 msr, u32 *result)
1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
{
	u32 vmx_msr_low, vmx_msr_high;
	u32 ctl = ctl_min | ctl_opt;

	rdmsr(msr, vmx_msr_low, vmx_msr_high);

	ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
	ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */

	/* Ensure minimum (required) set of control bits are supported. */
	if (ctl_min & ~ctl)
Y
Yang, Sheng 已提交
1432
		return -EIO;
1433 1434 1435 1436 1437

	*result = ctl;
	return 0;
}

A
Avi Kivity 已提交
1438 1439 1440 1441 1442 1443 1444 1445
static __init bool allow_1_setting(u32 msr, u32 ctl)
{
	u32 vmx_msr_low, vmx_msr_high;

	rdmsr(msr, vmx_msr_low, vmx_msr_high);
	return vmx_msr_high & ctl;
}

Y
Yang, Sheng 已提交
1446
static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
A
Avi Kivity 已提交
1447 1448
{
	u32 vmx_msr_low, vmx_msr_high;
S
Sheng Yang 已提交
1449
	u32 min, opt, min2, opt2;
1450 1451
	u32 _pin_based_exec_control = 0;
	u32 _cpu_based_exec_control = 0;
1452
	u32 _cpu_based_2nd_exec_control = 0;
1453 1454 1455 1456
	u32 _vmexit_control = 0;
	u32 _vmentry_control = 0;

	min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
1457
	opt = PIN_BASED_VIRTUAL_NMIS;
1458 1459
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
				&_pin_based_exec_control) < 0)
Y
Yang, Sheng 已提交
1460
		return -EIO;
1461

1462
	min =
1463 1464 1465 1466
#ifdef CONFIG_X86_64
	      CPU_BASED_CR8_LOAD_EXITING |
	      CPU_BASED_CR8_STORE_EXITING |
#endif
S
Sheng Yang 已提交
1467 1468
	      CPU_BASED_CR3_LOAD_EXITING |
	      CPU_BASED_CR3_STORE_EXITING |
1469 1470
	      CPU_BASED_USE_IO_BITMAPS |
	      CPU_BASED_MOV_DR_EXITING |
M
Marcelo Tosatti 已提交
1471
	      CPU_BASED_USE_TSC_OFFSETING |
1472 1473
	      CPU_BASED_MWAIT_EXITING |
	      CPU_BASED_MONITOR_EXITING |
M
Marcelo Tosatti 已提交
1474
	      CPU_BASED_INVLPG_EXITING;
1475 1476 1477 1478

	if (yield_on_hlt)
		min |= CPU_BASED_HLT_EXITING;

1479
	opt = CPU_BASED_TPR_SHADOW |
S
Sheng Yang 已提交
1480
	      CPU_BASED_USE_MSR_BITMAPS |
1481
	      CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1482 1483
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
				&_cpu_based_exec_control) < 0)
Y
Yang, Sheng 已提交
1484
		return -EIO;
1485 1486 1487 1488 1489
#ifdef CONFIG_X86_64
	if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
		_cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
					   ~CPU_BASED_CR8_STORE_EXITING;
#endif
1490
	if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
S
Sheng Yang 已提交
1491 1492
		min2 = 0;
		opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
1493
			SECONDARY_EXEC_WBINVD_EXITING |
S
Sheng Yang 已提交
1494
			SECONDARY_EXEC_ENABLE_VPID |
1495
			SECONDARY_EXEC_ENABLE_EPT |
1496
			SECONDARY_EXEC_UNRESTRICTED_GUEST |
1497 1498
			SECONDARY_EXEC_PAUSE_LOOP_EXITING |
			SECONDARY_EXEC_RDTSCP;
S
Sheng Yang 已提交
1499 1500
		if (adjust_vmx_controls(min2, opt2,
					MSR_IA32_VMX_PROCBASED_CTLS2,
1501 1502 1503 1504 1505 1506 1507 1508
					&_cpu_based_2nd_exec_control) < 0)
			return -EIO;
	}
#ifndef CONFIG_X86_64
	if (!(_cpu_based_2nd_exec_control &
				SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
		_cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
#endif
S
Sheng Yang 已提交
1509
	if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
M
Marcelo Tosatti 已提交
1510 1511
		/* CR3 accesses and invlpg don't need to cause VM Exits when EPT
		   enabled */
1512 1513 1514
		_cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
					     CPU_BASED_CR3_STORE_EXITING |
					     CPU_BASED_INVLPG_EXITING);
S
Sheng Yang 已提交
1515 1516 1517
		rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
		      vmx_capability.ept, vmx_capability.vpid);
	}
1518 1519 1520 1521 1522

	min = 0;
#ifdef CONFIG_X86_64
	min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
#endif
S
Sheng Yang 已提交
1523
	opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT;
1524 1525
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
				&_vmexit_control) < 0)
Y
Yang, Sheng 已提交
1526
		return -EIO;
1527

S
Sheng Yang 已提交
1528 1529
	min = 0;
	opt = VM_ENTRY_LOAD_IA32_PAT;
1530 1531
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
				&_vmentry_control) < 0)
Y
Yang, Sheng 已提交
1532
		return -EIO;
A
Avi Kivity 已提交
1533

N
Nguyen Anh Quynh 已提交
1534
	rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
1535 1536 1537

	/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
	if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
Y
Yang, Sheng 已提交
1538
		return -EIO;
1539 1540 1541 1542

#ifdef CONFIG_X86_64
	/* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
	if (vmx_msr_high & (1u<<16))
Y
Yang, Sheng 已提交
1543
		return -EIO;
1544 1545 1546 1547
#endif

	/* Require Write-Back (WB) memory type for VMCS accesses. */
	if (((vmx_msr_high >> 18) & 15) != 6)
Y
Yang, Sheng 已提交
1548
		return -EIO;
1549

Y
Yang, Sheng 已提交
1550 1551 1552
	vmcs_conf->size = vmx_msr_high & 0x1fff;
	vmcs_conf->order = get_order(vmcs_config.size);
	vmcs_conf->revision_id = vmx_msr_low;
1553

Y
Yang, Sheng 已提交
1554 1555
	vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
	vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
1556
	vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
Y
Yang, Sheng 已提交
1557 1558
	vmcs_conf->vmexit_ctrl         = _vmexit_control;
	vmcs_conf->vmentry_ctrl        = _vmentry_control;
1559

A
Avi Kivity 已提交
1560 1561 1562 1563 1564 1565
	cpu_has_load_ia32_efer =
		allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
				VM_ENTRY_LOAD_IA32_EFER)
		&& allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
				   VM_EXIT_LOAD_IA32_EFER);

1566
	return 0;
N
Nguyen Anh Quynh 已提交
1567
}
A
Avi Kivity 已提交
1568 1569 1570 1571 1572 1573 1574

static struct vmcs *alloc_vmcs_cpu(int cpu)
{
	int node = cpu_to_node(cpu);
	struct page *pages;
	struct vmcs *vmcs;

1575
	pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order);
A
Avi Kivity 已提交
1576 1577 1578
	if (!pages)
		return NULL;
	vmcs = page_address(pages);
1579 1580
	memset(vmcs, 0, vmcs_config.size);
	vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
A
Avi Kivity 已提交
1581 1582 1583 1584 1585
	return vmcs;
}

static struct vmcs *alloc_vmcs(void)
{
1586
	return alloc_vmcs_cpu(raw_smp_processor_id());
A
Avi Kivity 已提交
1587 1588 1589 1590
}

static void free_vmcs(struct vmcs *vmcs)
{
1591
	free_pages((unsigned long)vmcs, vmcs_config.order);
A
Avi Kivity 已提交
1592 1593
}

1594
static void free_kvm_area(void)
A
Avi Kivity 已提交
1595 1596 1597
{
	int cpu;

Z
Zachary Amsden 已提交
1598
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
1599
		free_vmcs(per_cpu(vmxarea, cpu));
Z
Zachary Amsden 已提交
1600 1601
		per_cpu(vmxarea, cpu) = NULL;
	}
A
Avi Kivity 已提交
1602 1603 1604 1605 1606 1607
}

static __init int alloc_kvm_area(void)
{
	int cpu;

Z
Zachary Amsden 已提交
1608
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
		struct vmcs *vmcs;

		vmcs = alloc_vmcs_cpu(cpu);
		if (!vmcs) {
			free_kvm_area();
			return -ENOMEM;
		}

		per_cpu(vmxarea, cpu) = vmcs;
	}
	return 0;
}

static __init int hardware_setup(void)
{
Y
Yang, Sheng 已提交
1624 1625
	if (setup_vmcs_config(&vmcs_config) < 0)
		return -EIO;
1626 1627 1628 1629

	if (boot_cpu_has(X86_FEATURE_NX))
		kvm_enable_efer_bits(EFER_NX);

S
Sheng Yang 已提交
1630 1631 1632
	if (!cpu_has_vmx_vpid())
		enable_vpid = 0;

1633 1634
	if (!cpu_has_vmx_ept() ||
	    !cpu_has_vmx_ept_4levels()) {
S
Sheng Yang 已提交
1635
		enable_ept = 0;
1636 1637 1638 1639 1640
		enable_unrestricted_guest = 0;
	}

	if (!cpu_has_vmx_unrestricted_guest())
		enable_unrestricted_guest = 0;
S
Sheng Yang 已提交
1641 1642 1643 1644

	if (!cpu_has_vmx_flexpriority())
		flexpriority_enabled = 0;

1645 1646 1647
	if (!cpu_has_vmx_tpr_shadow())
		kvm_x86_ops->update_cr8_intercept = NULL;

1648 1649 1650
	if (enable_ept && !cpu_has_vmx_ept_2m_page())
		kvm_disable_largepages();

1651 1652 1653
	if (!cpu_has_vmx_ple())
		ple_gap = 0;

A
Avi Kivity 已提交
1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665
	return alloc_kvm_area();
}

static __exit void hardware_unsetup(void)
{
	free_kvm_area();
}

static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
{
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];

1666
	if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
A
Avi Kivity 已提交
1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
		vmcs_write16(sf->selector, save->selector);
		vmcs_writel(sf->base, save->base);
		vmcs_write32(sf->limit, save->limit);
		vmcs_write32(sf->ar_bytes, save->ar);
	} else {
		u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
			<< AR_DPL_SHIFT;
		vmcs_write32(sf->ar_bytes, 0x93 | dpl);
	}
}

static void enter_pmode(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
1681
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
1682

1683
	vmx->emulation_required = 1;
1684
	vmx->rmode.vm86_active = 0;
A
Avi Kivity 已提交
1685

1686
	vmcs_write16(GUEST_TR_SELECTOR, vmx->rmode.tr.selector);
1687 1688 1689
	vmcs_writel(GUEST_TR_BASE, vmx->rmode.tr.base);
	vmcs_write32(GUEST_TR_LIMIT, vmx->rmode.tr.limit);
	vmcs_write32(GUEST_TR_AR_BYTES, vmx->rmode.tr.ar);
A
Avi Kivity 已提交
1690 1691

	flags = vmcs_readl(GUEST_RFLAGS);
1692 1693
	flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
	flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
A
Avi Kivity 已提交
1694 1695
	vmcs_writel(GUEST_RFLAGS, flags);

1696 1697
	vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
			(vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
A
Avi Kivity 已提交
1698 1699 1700

	update_exception_bitmap(vcpu);

1701 1702 1703
	if (emulate_invalid_guest_state)
		return;

1704 1705 1706 1707
	fix_pmode_dataseg(VCPU_SREG_ES, &vmx->rmode.es);
	fix_pmode_dataseg(VCPU_SREG_DS, &vmx->rmode.ds);
	fix_pmode_dataseg(VCPU_SREG_GS, &vmx->rmode.gs);
	fix_pmode_dataseg(VCPU_SREG_FS, &vmx->rmode.fs);
A
Avi Kivity 已提交
1708 1709 1710 1711 1712 1713 1714 1715 1716

	vmcs_write16(GUEST_SS_SELECTOR, 0);
	vmcs_write32(GUEST_SS_AR_BYTES, 0x93);

	vmcs_write16(GUEST_CS_SELECTOR,
		     vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
	vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
}

M
Mike Day 已提交
1717
static gva_t rmode_tss_base(struct kvm *kvm)
A
Avi Kivity 已提交
1718
{
1719
	if (!kvm->arch.tss_addr) {
1720 1721 1722
		struct kvm_memslots *slots;
		gfn_t base_gfn;

1723
		slots = kvm_memslots(kvm);
1724
		base_gfn = slots->memslots[0].base_gfn +
1725
				 kvm->memslots->memslots[0].npages - 3;
1726 1727
		return base_gfn << PAGE_SHIFT;
	}
1728
	return kvm->arch.tss_addr;
A
Avi Kivity 已提交
1729 1730 1731 1732 1733 1734 1735 1736 1737 1738
}

static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
{
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];

	save->selector = vmcs_read16(sf->selector);
	save->base = vmcs_readl(sf->base);
	save->limit = vmcs_read32(sf->limit);
	save->ar = vmcs_read32(sf->ar_bytes);
1739
	vmcs_write16(sf->selector, save->base >> 4);
1740
	vmcs_write32(sf->base, save->base & 0xffff0);
A
Avi Kivity 已提交
1741 1742
	vmcs_write32(sf->limit, 0xffff);
	vmcs_write32(sf->ar_bytes, 0xf3);
1743 1744 1745 1746
	if (save->base & 0xf)
		printk_once(KERN_WARNING "kvm: segment base is not paragraph"
			    " aligned when entering protected mode (seg=%d)",
			    seg);
A
Avi Kivity 已提交
1747 1748 1749 1750 1751
}

static void enter_rmode(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
1752
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
1753

1754 1755 1756
	if (enable_unrestricted_guest)
		return;

1757
	vmx->emulation_required = 1;
1758
	vmx->rmode.vm86_active = 1;
A
Avi Kivity 已提交
1759

1760
	vmx->rmode.tr.selector = vmcs_read16(GUEST_TR_SELECTOR);
1761
	vmx->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
A
Avi Kivity 已提交
1762 1763
	vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));

1764
	vmx->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
A
Avi Kivity 已提交
1765 1766
	vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);

1767
	vmx->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
A
Avi Kivity 已提交
1768 1769 1770
	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);

	flags = vmcs_readl(GUEST_RFLAGS);
1771
	vmx->rmode.save_rflags = flags;
A
Avi Kivity 已提交
1772

1773
	flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
A
Avi Kivity 已提交
1774 1775

	vmcs_writel(GUEST_RFLAGS, flags);
1776
	vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
A
Avi Kivity 已提交
1777 1778
	update_exception_bitmap(vcpu);

1779 1780 1781
	if (emulate_invalid_guest_state)
		goto continue_rmode;

A
Avi Kivity 已提交
1782 1783 1784 1785 1786
	vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4);
	vmcs_write32(GUEST_SS_LIMIT, 0xffff);
	vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);

	vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
1787
	vmcs_write32(GUEST_CS_LIMIT, 0xffff);
1788 1789
	if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
		vmcs_writel(GUEST_CS_BASE, 0xf0000);
A
Avi Kivity 已提交
1790 1791
	vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);

1792 1793 1794 1795
	fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.es);
	fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.ds);
	fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.gs);
	fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.fs);
1796

1797
continue_rmode:
1798
	kvm_mmu_reset_context(vcpu);
1799
	init_rmode(vcpu->kvm);
A
Avi Kivity 已提交
1800 1801
}

1802 1803 1804
static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1805 1806 1807 1808
	struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);

	if (!msr)
		return;
1809

1810 1811 1812 1813 1814
	/*
	 * Force kernel_gs_base reloading before EFER changes, as control
	 * of this msr depends on is_long_mode().
	 */
	vmx_load_host_state(to_vmx(vcpu));
1815
	vcpu->arch.efer = efer;
1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830
	if (efer & EFER_LMA) {
		vmcs_write32(VM_ENTRY_CONTROLS,
			     vmcs_read32(VM_ENTRY_CONTROLS) |
			     VM_ENTRY_IA32E_MODE);
		msr->data = efer;
	} else {
		vmcs_write32(VM_ENTRY_CONTROLS,
			     vmcs_read32(VM_ENTRY_CONTROLS) &
			     ~VM_ENTRY_IA32E_MODE);

		msr->data = efer & ~EFER_LME;
	}
	setup_msrs(vmx);
}

1831
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1832 1833 1834 1835 1836 1837 1838 1839

static void enter_lmode(struct kvm_vcpu *vcpu)
{
	u32 guest_tr_ar;

	guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
	if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
		printk(KERN_DEBUG "%s: tss fixup for long mode. \n",
1840
		       __func__);
A
Avi Kivity 已提交
1841 1842 1843 1844
		vmcs_write32(GUEST_TR_AR_BYTES,
			     (guest_tr_ar & ~AR_TYPE_MASK)
			     | AR_TYPE_BUSY_64_TSS);
	}
1845
	vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
A
Avi Kivity 已提交
1846 1847 1848 1849 1850 1851
}

static void exit_lmode(struct kvm_vcpu *vcpu)
{
	vmcs_write32(VM_ENTRY_CONTROLS,
		     vmcs_read32(VM_ENTRY_CONTROLS)
1852
		     & ~VM_ENTRY_IA32E_MODE);
1853
	vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
A
Avi Kivity 已提交
1854 1855 1856 1857
}

#endif

1858 1859
static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
{
1860
	vpid_sync_context(to_vmx(vcpu));
1861 1862 1863
	if (enable_ept) {
		if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
			return;
1864
		ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
1865
	}
1866 1867
}

1868 1869 1870 1871 1872 1873 1874 1875
static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
{
	ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;

	vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
	vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
}

1876 1877 1878 1879 1880 1881 1882
static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
{
	if (enable_ept && is_paging(vcpu))
		vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
}

1883
static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1884
{
1885 1886 1887 1888
	ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;

	vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
	vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
1889 1890
}

1891 1892
static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
1893 1894 1895 1896
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_dirty))
		return;

1897
	if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1898 1899 1900 1901
		vmcs_write64(GUEST_PDPTR0, vcpu->arch.mmu.pdptrs[0]);
		vmcs_write64(GUEST_PDPTR1, vcpu->arch.mmu.pdptrs[1]);
		vmcs_write64(GUEST_PDPTR2, vcpu->arch.mmu.pdptrs[2]);
		vmcs_write64(GUEST_PDPTR3, vcpu->arch.mmu.pdptrs[3]);
1902 1903 1904
	}
}

1905 1906 1907
static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
{
	if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1908 1909 1910 1911
		vcpu->arch.mmu.pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
		vcpu->arch.mmu.pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
		vcpu->arch.mmu.pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
		vcpu->arch.mmu.pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
1912
	}
A
Avi Kivity 已提交
1913 1914 1915 1916 1917

	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
1918 1919
}

1920 1921 1922 1923 1924 1925
static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);

static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
					unsigned long cr0,
					struct kvm_vcpu *vcpu)
{
1926
	vmx_decache_cr3(vcpu);
1927 1928 1929
	if (!(cr0 & X86_CR0_PG)) {
		/* From paging/starting to nonpaging */
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
1930
			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
1931 1932 1933
			     (CPU_BASED_CR3_LOAD_EXITING |
			      CPU_BASED_CR3_STORE_EXITING));
		vcpu->arch.cr0 = cr0;
1934
		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
1935 1936 1937
	} else if (!is_paging(vcpu)) {
		/* From nonpaging to paging */
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
1938
			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
1939 1940 1941
			     ~(CPU_BASED_CR3_LOAD_EXITING |
			       CPU_BASED_CR3_STORE_EXITING));
		vcpu->arch.cr0 = cr0;
1942
		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
1943
	}
1944 1945 1946

	if (!(cr0 & X86_CR0_WP))
		*hw_cr0 &= ~X86_CR0_WP;
1947 1948
}

A
Avi Kivity 已提交
1949 1950
static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
1951
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1952 1953 1954 1955 1956 1957 1958
	unsigned long hw_cr0;

	if (enable_unrestricted_guest)
		hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST)
			| KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
	else
		hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON;
1959

1960
	if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
A
Avi Kivity 已提交
1961 1962
		enter_pmode(vcpu);

1963
	if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
A
Avi Kivity 已提交
1964 1965
		enter_rmode(vcpu);

1966
#ifdef CONFIG_X86_64
1967
	if (vcpu->arch.efer & EFER_LME) {
1968
		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
A
Avi Kivity 已提交
1969
			enter_lmode(vcpu);
1970
		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
A
Avi Kivity 已提交
1971 1972 1973 1974
			exit_lmode(vcpu);
	}
#endif

1975
	if (enable_ept)
1976 1977
		ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);

1978
	if (!vcpu->fpu_active)
1979
		hw_cr0 |= X86_CR0_TS | X86_CR0_MP;
1980

A
Avi Kivity 已提交
1981
	vmcs_writel(CR0_READ_SHADOW, cr0);
1982
	vmcs_writel(GUEST_CR0, hw_cr0);
1983
	vcpu->arch.cr0 = cr0;
A
Avi Kivity 已提交
1984 1985
}

1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997
static u64 construct_eptp(unsigned long root_hpa)
{
	u64 eptp;

	/* TODO write the value reading from MSR */
	eptp = VMX_EPT_DEFAULT_MT |
		VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
	eptp |= (root_hpa & PAGE_MASK);

	return eptp;
}

A
Avi Kivity 已提交
1998 1999
static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
{
2000 2001 2002 2003
	unsigned long guest_cr3;
	u64 eptp;

	guest_cr3 = cr3;
2004
	if (enable_ept) {
2005 2006
		eptp = construct_eptp(cr3);
		vmcs_write64(EPT_POINTER, eptp);
2007
		guest_cr3 = is_paging(vcpu) ? kvm_read_cr3(vcpu) :
2008
			vcpu->kvm->arch.ept_identity_map_addr;
2009
		ept_load_pdptrs(vcpu);
2010 2011
	}

2012
	vmx_flush_tlb(vcpu);
2013
	vmcs_writel(GUEST_CR3, guest_cr3);
A
Avi Kivity 已提交
2014 2015 2016 2017
}

static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
2018
	unsigned long hw_cr4 = cr4 | (to_vmx(vcpu)->rmode.vm86_active ?
2019 2020
		    KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);

2021
	vcpu->arch.cr4 = cr4;
2022 2023 2024 2025 2026 2027 2028 2029
	if (enable_ept) {
		if (!is_paging(vcpu)) {
			hw_cr4 &= ~X86_CR4_PAE;
			hw_cr4 |= X86_CR4_PSE;
		} else if (!(cr4 & X86_CR4_PAE)) {
			hw_cr4 &= ~X86_CR4_PAE;
		}
	}
2030 2031 2032

	vmcs_writel(CR4_READ_SHADOW, cr4);
	vmcs_writel(GUEST_CR4, hw_cr4);
A
Avi Kivity 已提交
2033 2034 2035 2036 2037
}

static void vmx_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
2038
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
2039
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2040
	struct kvm_save_segment *save;
A
Avi Kivity 已提交
2041 2042
	u32 ar;

2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063
	if (vmx->rmode.vm86_active
	    && (seg == VCPU_SREG_TR || seg == VCPU_SREG_ES
		|| seg == VCPU_SREG_DS || seg == VCPU_SREG_FS
		|| seg == VCPU_SREG_GS)
	    && !emulate_invalid_guest_state) {
		switch (seg) {
		case VCPU_SREG_TR: save = &vmx->rmode.tr; break;
		case VCPU_SREG_ES: save = &vmx->rmode.es; break;
		case VCPU_SREG_DS: save = &vmx->rmode.ds; break;
		case VCPU_SREG_FS: save = &vmx->rmode.fs; break;
		case VCPU_SREG_GS: save = &vmx->rmode.gs; break;
		default: BUG();
		}
		var->selector = save->selector;
		var->base = save->base;
		var->limit = save->limit;
		ar = save->ar;
		if (seg == VCPU_SREG_TR
		    || var->selector == vmcs_read16(sf->selector))
			goto use_saved_rmode_seg;
	}
A
Avi Kivity 已提交
2064 2065 2066 2067
	var->base = vmcs_readl(sf->base);
	var->limit = vmcs_read32(sf->limit);
	var->selector = vmcs_read16(sf->selector);
	ar = vmcs_read32(sf->ar_bytes);
2068
use_saved_rmode_seg:
2069
	if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state)
A
Avi Kivity 已提交
2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081
		ar = 0;
	var->type = ar & 15;
	var->s = (ar >> 4) & 1;
	var->dpl = (ar >> 5) & 3;
	var->present = (ar >> 7) & 1;
	var->avl = (ar >> 12) & 1;
	var->l = (ar >> 13) & 1;
	var->db = (ar >> 14) & 1;
	var->g = (ar >> 15) & 1;
	var->unusable = (ar >> 16) & 1;
}

2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093
static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
	struct kvm_segment s;

	if (to_vmx(vcpu)->rmode.vm86_active) {
		vmx_get_segment(vcpu, &s, seg);
		return s.base;
	}
	return vmcs_readl(sf->base);
}

2094 2095
static int vmx_get_cpl(struct kvm_vcpu *vcpu)
{
2096
	if (!is_protmode(vcpu))
2097 2098 2099 2100 2101
		return 0;

	if (vmx_get_rflags(vcpu) & X86_EFLAGS_VM) /* if virtual 8086 */
		return 3;

A
Avi Kivity 已提交
2102
	return vmcs_read16(GUEST_CS_SELECTOR) & 3;
2103 2104
}

2105
static u32 vmx_segment_access_rights(struct kvm_segment *var)
A
Avi Kivity 已提交
2106 2107 2108
{
	u32 ar;

2109
	if (var->unusable)
A
Avi Kivity 已提交
2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120
		ar = 1 << 16;
	else {
		ar = var->type & 15;
		ar |= (var->s & 1) << 4;
		ar |= (var->dpl & 3) << 5;
		ar |= (var->present & 1) << 7;
		ar |= (var->avl & 1) << 12;
		ar |= (var->l & 1) << 13;
		ar |= (var->db & 1) << 14;
		ar |= (var->g & 1) << 15;
	}
2121 2122
	if (ar == 0) /* a 0 value means unusable */
		ar = AR_UNUSABLE_MASK;
2123 2124 2125 2126 2127 2128 2129

	return ar;
}

static void vmx_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
2130
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2131 2132 2133
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
	u32 ar;

2134 2135 2136 2137 2138
	if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) {
		vmx->rmode.tr.selector = var->selector;
		vmx->rmode.tr.base = var->base;
		vmx->rmode.tr.limit = var->limit;
		vmx->rmode.tr.ar = vmx_segment_access_rights(var);
2139 2140 2141 2142 2143
		return;
	}
	vmcs_writel(sf->base, var->base);
	vmcs_write32(sf->limit, var->limit);
	vmcs_write16(sf->selector, var->selector);
2144
	if (vmx->rmode.vm86_active && var->s) {
2145 2146 2147 2148 2149 2150 2151 2152
		/*
		 * Hack real-mode segments into vm86 compatibility.
		 */
		if (var->base == 0xffff0000 && var->selector == 0xf000)
			vmcs_writel(sf->base, 0xf0000);
		ar = 0xf3;
	} else
		ar = vmx_segment_access_rights(var);
2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167

	/*
	 *   Fix the "Accessed" bit in AR field of segment registers for older
	 * qemu binaries.
	 *   IA32 arch specifies that at the time of processor reset the
	 * "Accessed" bit in the AR field of segment registers is 1. And qemu
	 * is setting it to 0 in the usedland code. This causes invalid guest
	 * state vmexit when "unrestricted guest" mode is turned on.
	 *    Fix for this setup issue in cpu_reset is being pushed in the qemu
	 * tree. Newer qemu binaries with that qemu fix would not need this
	 * kvm hack.
	 */
	if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
		ar |= 0x1; /* Accessed */

A
Avi Kivity 已提交
2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178
	vmcs_write32(sf->ar_bytes, ar);
}

static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	u32 ar = vmcs_read32(GUEST_CS_AR_BYTES);

	*db = (ar >> 14) & 1;
	*l = (ar >> 13) & 1;
}

2179
static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2180
{
2181 2182
	dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
	dt->address = vmcs_readl(GUEST_IDTR_BASE);
A
Avi Kivity 已提交
2183 2184
}

2185
static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2186
{
2187 2188
	vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
	vmcs_writel(GUEST_IDTR_BASE, dt->address);
A
Avi Kivity 已提交
2189 2190
}

2191
static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2192
{
2193 2194
	dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
	dt->address = vmcs_readl(GUEST_GDTR_BASE);
A
Avi Kivity 已提交
2195 2196
}

2197
static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2198
{
2199 2200
	vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
	vmcs_writel(GUEST_GDTR_BASE, dt->address);
A
Avi Kivity 已提交
2201 2202
}

2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228
static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment var;
	u32 ar;

	vmx_get_segment(vcpu, &var, seg);
	ar = vmx_segment_access_rights(&var);

	if (var.base != (var.selector << 4))
		return false;
	if (var.limit != 0xffff)
		return false;
	if (ar != 0xf3)
		return false;

	return true;
}

static bool code_segment_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	unsigned int cs_rpl;

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs_rpl = cs.selector & SELECTOR_RPL_MASK;

2229 2230
	if (cs.unusable)
		return false;
2231 2232 2233 2234
	if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK))
		return false;
	if (!cs.s)
		return false;
2235
	if (cs.type & AR_TYPE_WRITEABLE_MASK) {
2236 2237
		if (cs.dpl > cs_rpl)
			return false;
2238
	} else {
2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
		if (cs.dpl != cs_rpl)
			return false;
	}
	if (!cs.present)
		return false;

	/* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
	return true;
}

static bool stack_segment_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment ss;
	unsigned int ss_rpl;

	vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
	ss_rpl = ss.selector & SELECTOR_RPL_MASK;

2257 2258 2259
	if (ss.unusable)
		return true;
	if (ss.type != 3 && ss.type != 7)
2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278
		return false;
	if (!ss.s)
		return false;
	if (ss.dpl != ss_rpl) /* DPL != RPL */
		return false;
	if (!ss.present)
		return false;

	return true;
}

static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment var;
	unsigned int rpl;

	vmx_get_segment(vcpu, &var, seg);
	rpl = var.selector & SELECTOR_RPL_MASK;

2279 2280
	if (var.unusable)
		return true;
2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301
	if (!var.s)
		return false;
	if (!var.present)
		return false;
	if (~var.type & (AR_TYPE_CODE_MASK|AR_TYPE_WRITEABLE_MASK)) {
		if (var.dpl < rpl) /* DPL < RPL */
			return false;
	}

	/* TODO: Add other members to kvm_segment_field to allow checking for other access
	 * rights flags
	 */
	return true;
}

static bool tr_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment tr;

	vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);

2302 2303
	if (tr.unusable)
		return false;
2304 2305
	if (tr.selector & SELECTOR_TI_MASK)	/* TI = 1 */
		return false;
2306
	if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319
		return false;
	if (!tr.present)
		return false;

	return true;
}

static bool ldtr_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment ldtr;

	vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);

2320 2321
	if (ldtr.unusable)
		return true;
2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350
	if (ldtr.selector & SELECTOR_TI_MASK)	/* TI = 1 */
		return false;
	if (ldtr.type != 2)
		return false;
	if (!ldtr.present)
		return false;

	return true;
}

static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs, ss;

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);

	return ((cs.selector & SELECTOR_RPL_MASK) ==
		 (ss.selector & SELECTOR_RPL_MASK));
}

/*
 * Check if guest state is valid. Returns true if valid, false if
 * not.
 * We assume that registers are always usable
 */
static bool guest_state_valid(struct kvm_vcpu *vcpu)
{
	/* real mode guest state checks */
2351
	if (!is_protmode(vcpu)) {
2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392
		if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
			return false;
	} else {
	/* protected mode guest state checks */
		if (!cs_ss_rpl_check(vcpu))
			return false;
		if (!code_segment_valid(vcpu))
			return false;
		if (!stack_segment_valid(vcpu))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_DS))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_ES))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_FS))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_GS))
			return false;
		if (!tr_valid(vcpu))
			return false;
		if (!ldtr_valid(vcpu))
			return false;
	}
	/* TODO:
	 * - Add checks on RIP
	 * - Add checks on RFLAGS
	 */

	return true;
}

M
Mike Day 已提交
2393
static int init_rmode_tss(struct kvm *kvm)
A
Avi Kivity 已提交
2394 2395
{
	gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
2396
	u16 data = 0;
2397
	int ret = 0;
2398
	int r;
A
Avi Kivity 已提交
2399

2400 2401
	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
	if (r < 0)
2402
		goto out;
2403
	data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
2404 2405
	r = kvm_write_guest_page(kvm, fn++, &data,
			TSS_IOPB_BASE_OFFSET, sizeof(u16));
2406
	if (r < 0)
2407
		goto out;
2408 2409
	r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
	if (r < 0)
2410
		goto out;
2411 2412
	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
	if (r < 0)
2413
		goto out;
2414
	data = ~0;
2415 2416 2417
	r = kvm_write_guest_page(kvm, fn, &data,
				 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
				 sizeof(u8));
2418
	if (r < 0)
2419 2420 2421 2422 2423
		goto out;

	ret = 1;
out:
	return ret;
A
Avi Kivity 已提交
2424 2425
}

2426 2427 2428 2429 2430 2431
static int init_rmode_identity_map(struct kvm *kvm)
{
	int i, r, ret;
	pfn_t identity_map_pfn;
	u32 tmp;

2432
	if (!enable_ept)
2433 2434 2435 2436 2437 2438 2439 2440 2441
		return 1;
	if (unlikely(!kvm->arch.ept_identity_pagetable)) {
		printk(KERN_ERR "EPT: identity-mapping pagetable "
			"haven't been allocated!\n");
		return 0;
	}
	if (likely(kvm->arch.ept_identity_pagetable_done))
		return 1;
	ret = 0;
2442
	identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460
	r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
	if (r < 0)
		goto out;
	/* Set up identity-mapping pagetable for EPT in real mode */
	for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
		tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
			_PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
		r = kvm_write_guest_page(kvm, identity_map_pfn,
				&tmp, i * sizeof(tmp), sizeof(tmp));
		if (r < 0)
			goto out;
	}
	kvm->arch.ept_identity_pagetable_done = true;
	ret = 1;
out:
	return ret;
}

A
Avi Kivity 已提交
2461 2462 2463
static void seg_setup(int seg)
{
	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2464
	unsigned int ar;
A
Avi Kivity 已提交
2465 2466 2467 2468

	vmcs_write16(sf->selector, 0);
	vmcs_writel(sf->base, 0);
	vmcs_write32(sf->limit, 0xffff);
2469 2470 2471 2472 2473 2474 2475 2476
	if (enable_unrestricted_guest) {
		ar = 0x93;
		if (seg == VCPU_SREG_CS)
			ar |= 0x08; /* code segment */
	} else
		ar = 0xf3;

	vmcs_write32(sf->ar_bytes, ar);
A
Avi Kivity 已提交
2477 2478
}

2479 2480 2481 2482 2483
static int alloc_apic_access_page(struct kvm *kvm)
{
	struct kvm_userspace_memory_region kvm_userspace_mem;
	int r = 0;

2484
	mutex_lock(&kvm->slots_lock);
2485
	if (kvm->arch.apic_access_page)
2486 2487 2488 2489 2490 2491 2492 2493
		goto out;
	kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
	kvm_userspace_mem.flags = 0;
	kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL;
	kvm_userspace_mem.memory_size = PAGE_SIZE;
	r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
	if (r)
		goto out;
2494

2495
	kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00);
2496
out:
2497
	mutex_unlock(&kvm->slots_lock);
2498 2499 2500
	return r;
}

2501 2502 2503 2504 2505
static int alloc_identity_pagetable(struct kvm *kvm)
{
	struct kvm_userspace_memory_region kvm_userspace_mem;
	int r = 0;

2506
	mutex_lock(&kvm->slots_lock);
2507 2508 2509 2510
	if (kvm->arch.ept_identity_pagetable)
		goto out;
	kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
	kvm_userspace_mem.flags = 0;
2511 2512
	kvm_userspace_mem.guest_phys_addr =
		kvm->arch.ept_identity_map_addr;
2513 2514 2515 2516 2517 2518
	kvm_userspace_mem.memory_size = PAGE_SIZE;
	r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
	if (r)
		goto out;

	kvm->arch.ept_identity_pagetable = gfn_to_page(kvm,
2519
			kvm->arch.ept_identity_map_addr >> PAGE_SHIFT);
2520
out:
2521
	mutex_unlock(&kvm->slots_lock);
2522 2523 2524
	return r;
}

2525 2526 2527 2528 2529
static void allocate_vpid(struct vcpu_vmx *vmx)
{
	int vpid;

	vmx->vpid = 0;
2530
	if (!enable_vpid)
2531 2532 2533 2534 2535 2536 2537 2538 2539 2540
		return;
	spin_lock(&vmx_vpid_lock);
	vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
	if (vpid < VMX_NR_VPIDS) {
		vmx->vpid = vpid;
		__set_bit(vpid, vmx_vpid_bitmap);
	}
	spin_unlock(&vmx_vpid_lock);
}

2541 2542 2543 2544 2545 2546 2547 2548 2549 2550
static void free_vpid(struct vcpu_vmx *vmx)
{
	if (!enable_vpid)
		return;
	spin_lock(&vmx_vpid_lock);
	if (vmx->vpid != 0)
		__clear_bit(vmx->vpid, vmx_vpid_bitmap);
	spin_unlock(&vmx_vpid_lock);
}

2551
static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, u32 msr)
S
Sheng Yang 已提交
2552
{
2553
	int f = sizeof(unsigned long);
S
Sheng Yang 已提交
2554 2555 2556 2557 2558 2559 2560 2561 2562 2563

	if (!cpu_has_vmx_msr_bitmap())
		return;

	/*
	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
	 * have the write-low and read-high bitmap offsets the wrong way round.
	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
	 */
	if (msr <= 0x1fff) {
2564 2565
		__clear_bit(msr, msr_bitmap + 0x000 / f); /* read-low */
		__clear_bit(msr, msr_bitmap + 0x800 / f); /* write-low */
S
Sheng Yang 已提交
2566 2567
	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
		msr &= 0x1fff;
2568 2569
		__clear_bit(msr, msr_bitmap + 0x400 / f); /* read-high */
		__clear_bit(msr, msr_bitmap + 0xc00 / f); /* write-high */
S
Sheng Yang 已提交
2570 2571 2572
	}
}

2573 2574 2575 2576 2577 2578 2579
static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
{
	if (!longmode_only)
		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy, msr);
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode, msr);
}

A
Avi Kivity 已提交
2580 2581 2582
/*
 * Sets up the vmcs for emulated real mode.
 */
R
Rusty Russell 已提交
2583
static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
A
Avi Kivity 已提交
2584
{
S
Sheng Yang 已提交
2585
	u32 host_sysenter_cs, msr_low, msr_high;
A
Avi Kivity 已提交
2586
	u32 junk;
2587
	u64 host_pat;
A
Avi Kivity 已提交
2588
	unsigned long a;
2589
	struct desc_ptr dt;
A
Avi Kivity 已提交
2590
	int i;
2591
	unsigned long kvm_vmx_return;
2592
	u32 exec_control;
A
Avi Kivity 已提交
2593 2594

	/* I/O */
2595 2596
	vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
	vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
A
Avi Kivity 已提交
2597

S
Sheng Yang 已提交
2598
	if (cpu_has_vmx_msr_bitmap())
2599
		vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
S
Sheng Yang 已提交
2600

A
Avi Kivity 已提交
2601 2602 2603
	vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */

	/* Control */
2604 2605
	vmcs_write32(PIN_BASED_VM_EXEC_CONTROL,
		vmcs_config.pin_based_exec_ctrl);
2606 2607 2608 2609 2610 2611 2612 2613 2614

	exec_control = vmcs_config.cpu_based_exec_ctrl;
	if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
		exec_control &= ~CPU_BASED_TPR_SHADOW;
#ifdef CONFIG_X86_64
		exec_control |= CPU_BASED_CR8_STORE_EXITING |
				CPU_BASED_CR8_LOAD_EXITING;
#endif
	}
2615
	if (!enable_ept)
S
Sheng Yang 已提交
2616
		exec_control |= CPU_BASED_CR3_STORE_EXITING |
2617 2618
				CPU_BASED_CR3_LOAD_EXITING  |
				CPU_BASED_INVLPG_EXITING;
2619
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
A
Avi Kivity 已提交
2620

2621 2622 2623 2624 2625
	if (cpu_has_secondary_exec_ctrls()) {
		exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
		if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
			exec_control &=
				~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
2626 2627
		if (vmx->vpid == 0)
			exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
2628
		if (!enable_ept) {
S
Sheng Yang 已提交
2629
			exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
2630 2631
			enable_unrestricted_guest = 0;
		}
2632 2633
		if (!enable_unrestricted_guest)
			exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
2634 2635
		if (!ple_gap)
			exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
2636 2637
		vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
	}
2638

2639 2640 2641 2642 2643
	if (ple_gap) {
		vmcs_write32(PLE_GAP, ple_gap);
		vmcs_write32(PLE_WINDOW, ple_window);
	}

2644 2645
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf);
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf);
A
Avi Kivity 已提交
2646 2647
	vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */

2648
	vmcs_writel(HOST_CR0, read_cr0() | X86_CR0_TS);  /* 22.2.3 */
A
Avi Kivity 已提交
2649 2650 2651 2652 2653 2654
	vmcs_writel(HOST_CR4, read_cr4());  /* 22.2.3, 22.2.5 */
	vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */

	vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
	vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
	vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
2655 2656
	vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
	vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
A
Avi Kivity 已提交
2657
	vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
2658
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669
	rdmsrl(MSR_FS_BASE, a);
	vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
	rdmsrl(MSR_GS_BASE, a);
	vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
#else
	vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
	vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
#endif

	vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */

2670
	native_store_idt(&dt);
2671
	vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */
A
Avi Kivity 已提交
2672

M
Mike Day 已提交
2673
	asm("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return));
2674
	vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */
2675 2676
	vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
2677
	vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
2678
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
2679
	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
A
Avi Kivity 已提交
2680 2681 2682 2683 2684 2685 2686 2687

	rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk);
	vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs);
	rdmsrl(MSR_IA32_SYSENTER_ESP, a);
	vmcs_writel(HOST_IA32_SYSENTER_ESP, a);   /* 22.2.3 */
	rdmsrl(MSR_IA32_SYSENTER_EIP, a);
	vmcs_writel(HOST_IA32_SYSENTER_EIP, a);   /* 22.2.3 */

S
Sheng Yang 已提交
2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701
	if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
		rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
		host_pat = msr_low | ((u64) msr_high << 32);
		vmcs_write64(HOST_IA32_PAT, host_pat);
	}
	if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
		rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
		host_pat = msr_low | ((u64) msr_high << 32);
		/* Write the default value follow host pat */
		vmcs_write64(GUEST_IA32_PAT, host_pat);
		/* Keep arch.pat sync with GUEST_IA32_PAT */
		vmx->vcpu.arch.pat = host_pat;
	}

A
Avi Kivity 已提交
2702 2703 2704
	for (i = 0; i < NR_VMX_MSR; ++i) {
		u32 index = vmx_msr_index[i];
		u32 data_low, data_high;
2705
		int j = vmx->nmsrs;
A
Avi Kivity 已提交
2706 2707 2708

		if (rdmsr_safe(index, &data_low, &data_high) < 0)
			continue;
2709 2710
		if (wrmsr_safe(index, data_low, data_high) < 0)
			continue;
2711 2712
		vmx->guest_msrs[j].index = i;
		vmx->guest_msrs[j].data = 0;
2713
		vmx->guest_msrs[j].mask = -1ull;
2714
		++vmx->nmsrs;
A
Avi Kivity 已提交
2715 2716
	}

2717
	vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
A
Avi Kivity 已提交
2718 2719

	/* 22.2.1, 20.8.1 */
2720 2721
	vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl);

2722
	vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
2723
	vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
2724 2725
	if (enable_ept)
		vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
2726
	vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
2727

2728
	kvm_write_tsc(&vmx->vcpu, 0);
2729

2730 2731 2732
	return 0;
}

2733 2734
static int init_rmode(struct kvm *kvm)
{
2735 2736 2737
	int idx, ret = 0;

	idx = srcu_read_lock(&kvm->srcu);
2738
	if (!init_rmode_tss(kvm))
2739
		goto exit;
2740
	if (!init_rmode_identity_map(kvm))
2741 2742 2743 2744 2745 2746
		goto exit;

	ret = 1;
exit:
	srcu_read_unlock(&kvm->srcu, idx);
	return ret;
2747 2748
}

2749 2750 2751 2752
static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u64 msr;
2753
	int ret;
2754

2755
	vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP));
2756
	if (!init_rmode(vmx->vcpu.kvm)) {
2757 2758 2759 2760
		ret = -ENOMEM;
		goto out;
	}

2761
	vmx->rmode.vm86_active = 0;
2762

2763 2764
	vmx->soft_vnmi_blocked = 0;

2765
	vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
2766
	kvm_set_cr8(&vmx->vcpu, 0);
2767
	msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
2768
	if (kvm_vcpu_is_bsp(&vmx->vcpu))
2769 2770 2771
		msr |= MSR_IA32_APICBASE_BSP;
	kvm_set_apic_base(&vmx->vcpu, msr);

2772 2773 2774
	ret = fx_init(&vmx->vcpu);
	if (ret != 0)
		goto out;
2775

2776
	seg_setup(VCPU_SREG_CS);
2777 2778 2779 2780
	/*
	 * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
	 * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4.  Sigh.
	 */
2781
	if (kvm_vcpu_is_bsp(&vmx->vcpu)) {
2782 2783 2784
		vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
		vmcs_writel(GUEST_CS_BASE, 0x000f0000);
	} else {
2785 2786
		vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8);
		vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12);
2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809
	}

	seg_setup(VCPU_SREG_DS);
	seg_setup(VCPU_SREG_ES);
	seg_setup(VCPU_SREG_FS);
	seg_setup(VCPU_SREG_GS);
	seg_setup(VCPU_SREG_SS);

	vmcs_write16(GUEST_TR_SELECTOR, 0);
	vmcs_writel(GUEST_TR_BASE, 0);
	vmcs_write32(GUEST_TR_LIMIT, 0xffff);
	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);

	vmcs_write16(GUEST_LDTR_SELECTOR, 0);
	vmcs_writel(GUEST_LDTR_BASE, 0);
	vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
	vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);

	vmcs_write32(GUEST_SYSENTER_CS, 0);
	vmcs_writel(GUEST_SYSENTER_ESP, 0);
	vmcs_writel(GUEST_SYSENTER_EIP, 0);

	vmcs_writel(GUEST_RFLAGS, 0x02);
2810
	if (kvm_vcpu_is_bsp(&vmx->vcpu))
2811
		kvm_rip_write(vcpu, 0xfff0);
2812
	else
2813 2814
		kvm_rip_write(vcpu, 0);
	kvm_register_write(vcpu, VCPU_REGS_RSP, 0);
2815 2816 2817 2818 2819 2820 2821 2822 2823

	vmcs_writel(GUEST_DR7, 0x400);

	vmcs_writel(GUEST_GDTR_BASE, 0);
	vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);

	vmcs_writel(GUEST_IDTR_BASE, 0);
	vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);

2824
	vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
2825 2826 2827 2828 2829 2830 2831 2832
	vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
	vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);

	/* Special registers */
	vmcs_write64(GUEST_IA32_DEBUGCTL, 0);

	setup_msrs(vmx);

A
Avi Kivity 已提交
2833 2834
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */

2835 2836 2837 2838
	if (cpu_has_vmx_tpr_shadow()) {
		vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
		if (vm_need_tpr_shadow(vmx->vcpu.kvm))
			vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
2839
				page_to_phys(vmx->vcpu.arch.apic->regs_page));
2840 2841 2842 2843 2844
		vmcs_write32(TPR_THRESHOLD, 0);
	}

	if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
		vmcs_write64(APIC_ACCESS_ADDR,
2845
			     page_to_phys(vmx->vcpu.kvm->arch.apic_access_page));
A
Avi Kivity 已提交
2846

2847 2848 2849
	if (vmx->vpid != 0)
		vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);

2850
	vmx->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
2851
	vmx_set_cr0(&vmx->vcpu, kvm_read_cr0(vcpu)); /* enter rmode */
R
Rusty Russell 已提交
2852 2853 2854 2855
	vmx_set_cr4(&vmx->vcpu, 0);
	vmx_set_efer(&vmx->vcpu, 0);
	vmx_fpu_activate(&vmx->vcpu);
	update_exception_bitmap(&vmx->vcpu);
A
Avi Kivity 已提交
2856

2857
	vpid_sync_context(vmx);
2858

2859
	ret = 0;
A
Avi Kivity 已提交
2860

2861 2862 2863
	/* HACK: Don't enable emulation on guest boot/reset */
	vmx->emulation_required = 0;

A
Avi Kivity 已提交
2864 2865 2866 2867
out:
	return ret;
}

2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885
static void enable_irq_window(struct kvm_vcpu *vcpu)
{
	u32 cpu_based_vm_exec_control;

	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

static void enable_nmi_window(struct kvm_vcpu *vcpu)
{
	u32 cpu_based_vm_exec_control;

	if (!cpu_has_virtual_nmis()) {
		enable_irq_window(vcpu);
		return;
	}

2886 2887 2888 2889
	if (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
		enable_irq_window(vcpu);
		return;
	}
2890 2891 2892 2893 2894
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

2895
static void vmx_inject_irq(struct kvm_vcpu *vcpu)
2896
{
2897
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2898 2899
	uint32_t intr;
	int irq = vcpu->arch.interrupt.nr;
2900

2901
	trace_kvm_inj_virq(irq);
F
Feng (Eric) Liu 已提交
2902

2903
	++vcpu->stat.irq_injections;
2904
	if (vmx->rmode.vm86_active) {
2905 2906
		if (kvm_inject_realmode_interrupt(vcpu, irq) != EMULATE_DONE)
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2907 2908
		return;
	}
2909 2910 2911 2912 2913 2914 2915 2916
	intr = irq | INTR_INFO_VALID_MASK;
	if (vcpu->arch.interrupt.soft) {
		intr |= INTR_TYPE_SOFT_INTR;
		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
			     vmx->vcpu.arch.event_exit_inst_len);
	} else
		intr |= INTR_TYPE_EXT_INTR;
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
2917
	vmx_clear_hlt(vcpu);
2918 2919
}

2920 2921
static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
{
J
Jan Kiszka 已提交
2922 2923
	struct vcpu_vmx *vmx = to_vmx(vcpu);

2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936
	if (!cpu_has_virtual_nmis()) {
		/*
		 * Tracking the NMI-blocked state in software is built upon
		 * finding the next open IRQ window. This, in turn, depends on
		 * well-behaving guests: They have to keep IRQs disabled at
		 * least as long as the NMI handler runs. Otherwise we may
		 * cause NMI nesting, maybe breaking the guest. But as this is
		 * highly unlikely, we can live with the residual risk.
		 */
		vmx->soft_vnmi_blocked = 1;
		vmx->vnmi_blocked_time = 0;
	}

2937
	++vcpu->stat.nmi_injections;
2938
	if (vmx->rmode.vm86_active) {
2939 2940
		if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR) != EMULATE_DONE)
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
J
Jan Kiszka 已提交
2941 2942
		return;
	}
2943 2944
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
			INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
2945
	vmx_clear_hlt(vcpu);
2946 2947
}

2948
static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
2949
{
2950
	if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
2951
		return 0;
2952

2953
	return	!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2954 2955
		  (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
		   | GUEST_INTR_STATE_NMI));
2956 2957
}

J
Jan Kiszka 已提交
2958 2959 2960 2961
static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
{
	if (!cpu_has_virtual_nmis())
		return to_vmx(vcpu)->soft_vnmi_blocked;
2962
	return vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)	& GUEST_INTR_STATE_NMI;
J
Jan Kiszka 已提交
2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983
}

static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (!cpu_has_virtual_nmis()) {
		if (vmx->soft_vnmi_blocked != masked) {
			vmx->soft_vnmi_blocked = masked;
			vmx->vnmi_blocked_time = 0;
		}
	} else {
		if (masked)
			vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				      GUEST_INTR_STATE_NMI);
		else
			vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
					GUEST_INTR_STATE_NMI);
	}
}

2984 2985
static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
{
2986 2987 2988
	return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
		!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
			(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
2989 2990
}

2991 2992 2993 2994
static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
{
	int ret;
	struct kvm_userspace_memory_region tss_mem = {
2995
		.slot = TSS_PRIVATE_MEMSLOT,
2996 2997 2998 2999 3000 3001 3002 3003
		.guest_phys_addr = addr,
		.memory_size = PAGE_SIZE * 3,
		.flags = 0,
	};

	ret = kvm_set_memory_region(kvm, &tss_mem, 0);
	if (ret)
		return ret;
3004
	kvm->arch.tss_addr = addr;
3005 3006 3007
	return 0;
}

A
Avi Kivity 已提交
3008 3009 3010
static int handle_rmode_exception(struct kvm_vcpu *vcpu,
				  int vec, u32 err_code)
{
3011 3012 3013 3014 3015
	/*
	 * Instruction with address size override prefix opcode 0x67
	 * Cause the #SS fault with 0 error code in VM86 mode.
	 */
	if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
3016
		if (emulate_instruction(vcpu, 0) == EMULATE_DONE)
A
Avi Kivity 已提交
3017
			return 1;
3018 3019 3020 3021 3022 3023 3024
	/*
	 * Forward all other exceptions that are valid in real mode.
	 * FIXME: Breaks guest debugging in real mode, needs to be fixed with
	 *        the required debugging infrastructure rework.
	 */
	switch (vec) {
	case DB_VECTOR:
J
Jan Kiszka 已提交
3025 3026 3027 3028 3029
		if (vcpu->guest_debug &
		    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
			return 0;
		kvm_queue_exception(vcpu, vec);
		return 1;
3030
	case BP_VECTOR:
3031 3032 3033 3034 3035 3036
		/*
		 * Update instruction length as we may reinject the exception
		 * from user space while in guest debugging mode.
		 */
		to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
			vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
J
Jan Kiszka 已提交
3037 3038 3039 3040
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
			return 0;
		/* fall through */
	case DE_VECTOR:
3041 3042 3043 3044 3045 3046 3047 3048 3049 3050
	case OF_VECTOR:
	case BR_VECTOR:
	case UD_VECTOR:
	case DF_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
	case MF_VECTOR:
		kvm_queue_exception(vcpu, vec);
		return 1;
	}
A
Avi Kivity 已提交
3051 3052 3053
	return 0;
}

A
Andi Kleen 已提交
3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072
/*
 * Trigger machine check on the host. We assume all the MSRs are already set up
 * by the CPU and that we still run on the same CPU as the MCE occurred on.
 * We pass a fake environment to the machine check handler because we want
 * the guest to be always treated like user space, no matter what context
 * it used internally.
 */
static void kvm_machine_check(void)
{
#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
	struct pt_regs regs = {
		.cs = 3, /* Fake ring 3 no matter what the guest ran on */
		.flags = X86_EFLAGS_IF,
	};

	do_machine_check(&regs, 0);
#endif
}

A
Avi Kivity 已提交
3073
static int handle_machine_check(struct kvm_vcpu *vcpu)
A
Andi Kleen 已提交
3074 3075 3076 3077 3078
{
	/* already handled by vcpu_run */
	return 1;
}

A
Avi Kivity 已提交
3079
static int handle_exception(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3080
{
3081
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
3082
	struct kvm_run *kvm_run = vcpu->run;
J
Jan Kiszka 已提交
3083
	u32 intr_info, ex_no, error_code;
3084
	unsigned long cr2, rip, dr6;
A
Avi Kivity 已提交
3085 3086 3087
	u32 vect_info;
	enum emulation_result er;

3088
	vect_info = vmx->idt_vectoring_info;
A
Avi Kivity 已提交
3089 3090
	intr_info = vmcs_read32(VM_EXIT_INTR_INFO);

A
Andi Kleen 已提交
3091
	if (is_machine_check(intr_info))
A
Avi Kivity 已提交
3092
		return handle_machine_check(vcpu);
A
Andi Kleen 已提交
3093

A
Avi Kivity 已提交
3094
	if ((vect_info & VECTORING_INFO_VALID_MASK) &&
3095 3096 3097 3098 3099 3100 3101 3102
	    !is_page_fault(intr_info)) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
		vcpu->run->internal.ndata = 2;
		vcpu->run->internal.data[0] = vect_info;
		vcpu->run->internal.data[1] = intr_info;
		return 0;
	}
A
Avi Kivity 已提交
3103

3104
	if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR)
3105
		return 1;  /* already handled by vmx_vcpu_run() */
3106 3107

	if (is_no_device(intr_info)) {
3108
		vmx_fpu_activate(vcpu);
3109 3110 3111
		return 1;
	}

3112
	if (is_invalid_opcode(intr_info)) {
3113
		er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
3114
		if (er != EMULATE_DONE)
3115
			kvm_queue_exception(vcpu, UD_VECTOR);
3116 3117 3118
		return 1;
	}

A
Avi Kivity 已提交
3119
	error_code = 0;
3120
	rip = kvm_rip_read(vcpu);
3121
	if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
A
Avi Kivity 已提交
3122 3123
		error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
	if (is_page_fault(intr_info)) {
3124
		/* EPT won't cause page fault directly */
3125
		if (enable_ept)
3126
			BUG();
A
Avi Kivity 已提交
3127
		cr2 = vmcs_readl(EXIT_QUALIFICATION);
3128 3129
		trace_kvm_page_fault(cr2, error_code);

3130
		if (kvm_event_needs_reinjection(vcpu))
3131
			kvm_mmu_unprotect_page_virt(vcpu, cr2);
3132
		return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0);
A
Avi Kivity 已提交
3133 3134
	}

3135
	if (vmx->rmode.vm86_active &&
A
Avi Kivity 已提交
3136
	    handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
3137
								error_code)) {
3138 3139
		if (vcpu->arch.halt_request) {
			vcpu->arch.halt_request = 0;
3140 3141
			return kvm_emulate_halt(vcpu);
		}
A
Avi Kivity 已提交
3142
		return 1;
3143
	}
A
Avi Kivity 已提交
3144

J
Jan Kiszka 已提交
3145
	ex_no = intr_info & INTR_INFO_VECTOR_MASK;
3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158
	switch (ex_no) {
	case DB_VECTOR:
		dr6 = vmcs_readl(EXIT_QUALIFICATION);
		if (!(vcpu->guest_debug &
		      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
			vcpu->arch.dr6 = dr6 | DR6_FIXED_1;
			kvm_queue_exception(vcpu, DB_VECTOR);
			return 1;
		}
		kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
		kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
		/* fall through */
	case BP_VECTOR:
3159 3160 3161 3162 3163 3164 3165
		/*
		 * Update instruction length as we may reinject #BP from
		 * user space while in guest debugging mode. Reading it for
		 * #DB as well causes no harm, it is not used in that case.
		 */
		vmx->vcpu.arch.event_exit_inst_len =
			vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
A
Avi Kivity 已提交
3166
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
J
Jan Kiszka 已提交
3167 3168
		kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
		kvm_run->debug.arch.exception = ex_no;
3169 3170
		break;
	default:
J
Jan Kiszka 已提交
3171 3172 3173
		kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
		kvm_run->ex.exception = ex_no;
		kvm_run->ex.error_code = error_code;
3174
		break;
A
Avi Kivity 已提交
3175 3176 3177 3178
	}
	return 0;
}

A
Avi Kivity 已提交
3179
static int handle_external_interrupt(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3180
{
A
Avi Kivity 已提交
3181
	++vcpu->stat.irq_exits;
A
Avi Kivity 已提交
3182 3183 3184
	return 1;
}

A
Avi Kivity 已提交
3185
static int handle_triple_fault(struct kvm_vcpu *vcpu)
3186
{
A
Avi Kivity 已提交
3187
	vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
3188 3189
	return 0;
}
A
Avi Kivity 已提交
3190

A
Avi Kivity 已提交
3191
static int handle_io(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3192
{
3193
	unsigned long exit_qualification;
3194
	int size, in, string;
3195
	unsigned port;
A
Avi Kivity 已提交
3196

3197
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3198
	string = (exit_qualification & 16) != 0;
3199
	in = (exit_qualification & 8) != 0;
3200

3201
	++vcpu->stat.io_exits;
3202

3203
	if (string || in)
3204
		return emulate_instruction(vcpu, 0) == EMULATE_DONE;
3205

3206 3207
	port = exit_qualification >> 16;
	size = (exit_qualification & 7) + 1;
3208
	skip_emulated_instruction(vcpu);
3209 3210

	return kvm_fast_pio_out(vcpu, size, port);
A
Avi Kivity 已提交
3211 3212
}

I
Ingo Molnar 已提交
3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223
static void
vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
{
	/*
	 * Patch in the VMCALL instruction:
	 */
	hypercall[0] = 0x0f;
	hypercall[1] = 0x01;
	hypercall[2] = 0xc1;
}

A
Avi Kivity 已提交
3224
static int handle_cr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3225
{
3226
	unsigned long exit_qualification, val;
A
Avi Kivity 已提交
3227 3228
	int cr;
	int reg;
3229
	int err;
A
Avi Kivity 已提交
3230

3231
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
A
Avi Kivity 已提交
3232 3233 3234 3235
	cr = exit_qualification & 15;
	reg = (exit_qualification >> 8) & 15;
	switch ((exit_qualification >> 4) & 3) {
	case 0: /* mov to cr */
3236 3237
		val = kvm_register_read(vcpu, reg);
		trace_kvm_cr_write(cr, val);
A
Avi Kivity 已提交
3238 3239
		switch (cr) {
		case 0:
3240
			err = kvm_set_cr0(vcpu, val);
3241
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
3242 3243
			return 1;
		case 3:
3244
			err = kvm_set_cr3(vcpu, val);
3245
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
3246 3247
			return 1;
		case 4:
3248
			err = kvm_set_cr4(vcpu, val);
3249
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
3250
			return 1;
3251 3252 3253
		case 8: {
				u8 cr8_prev = kvm_get_cr8(vcpu);
				u8 cr8 = kvm_register_read(vcpu, reg);
A
Andre Przywara 已提交
3254
				err = kvm_set_cr8(vcpu, cr8);
3255
				kvm_complete_insn_gp(vcpu, err);
3256 3257 3258 3259
				if (irqchip_in_kernel(vcpu->kvm))
					return 1;
				if (cr8_prev <= cr8)
					return 1;
A
Avi Kivity 已提交
3260
				vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
3261 3262
				return 0;
			}
A
Avi Kivity 已提交
3263 3264
		};
		break;
3265
	case 2: /* clts */
3266
		vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
3267
		trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
3268
		skip_emulated_instruction(vcpu);
A
Avi Kivity 已提交
3269
		vmx_fpu_activate(vcpu);
3270
		return 1;
A
Avi Kivity 已提交
3271 3272 3273
	case 1: /*mov from cr*/
		switch (cr) {
		case 3:
3274 3275 3276
			val = kvm_read_cr3(vcpu);
			kvm_register_write(vcpu, reg, val);
			trace_kvm_cr_read(cr, val);
A
Avi Kivity 已提交
3277 3278 3279
			skip_emulated_instruction(vcpu);
			return 1;
		case 8:
3280 3281 3282
			val = kvm_get_cr8(vcpu);
			kvm_register_write(vcpu, reg, val);
			trace_kvm_cr_read(cr, val);
A
Avi Kivity 已提交
3283 3284 3285 3286 3287
			skip_emulated_instruction(vcpu);
			return 1;
		}
		break;
	case 3: /* lmsw */
3288
		val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
3289
		trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
3290
		kvm_lmsw(vcpu, val);
A
Avi Kivity 已提交
3291 3292 3293 3294 3295 3296

		skip_emulated_instruction(vcpu);
		return 1;
	default:
		break;
	}
A
Avi Kivity 已提交
3297
	vcpu->run->exit_reason = 0;
3298
	pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
A
Avi Kivity 已提交
3299 3300 3301 3302
	       (int)(exit_qualification >> 4) & 3, cr);
	return 0;
}

A
Avi Kivity 已提交
3303
static int handle_dr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3304
{
3305
	unsigned long exit_qualification;
A
Avi Kivity 已提交
3306 3307
	int dr, reg;

3308
	/* Do not handle if the CPL > 0, will trigger GP on re-entry */
3309 3310
	if (!kvm_require_cpl(vcpu, 0))
		return 1;
3311 3312 3313 3314 3315 3316 3317 3318
	dr = vmcs_readl(GUEST_DR7);
	if (dr & DR7_GD) {
		/*
		 * As the vm-exit takes precedence over the debug trap, we
		 * need to emulate the latter, either for the host or the
		 * guest debugging itself.
		 */
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
A
Avi Kivity 已提交
3319 3320 3321
			vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
			vcpu->run->debug.arch.dr7 = dr;
			vcpu->run->debug.arch.pc =
3322 3323
				vmcs_readl(GUEST_CS_BASE) +
				vmcs_readl(GUEST_RIP);
A
Avi Kivity 已提交
3324 3325
			vcpu->run->debug.arch.exception = DB_VECTOR;
			vcpu->run->exit_reason = KVM_EXIT_DEBUG;
3326 3327 3328 3329 3330 3331 3332 3333 3334 3335
			return 0;
		} else {
			vcpu->arch.dr7 &= ~DR7_GD;
			vcpu->arch.dr6 |= DR6_BD;
			vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
			kvm_queue_exception(vcpu, DB_VECTOR);
			return 1;
		}
	}

3336
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3337 3338 3339
	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
	reg = DEBUG_REG_ACCESS_REG(exit_qualification);
	if (exit_qualification & TYPE_MOV_FROM_DR) {
3340 3341 3342 3343 3344
		unsigned long val;
		if (!kvm_get_dr(vcpu, dr, &val))
			kvm_register_write(vcpu, reg, val);
	} else
		kvm_set_dr(vcpu, dr, vcpu->arch.regs[reg]);
A
Avi Kivity 已提交
3345 3346 3347 3348
	skip_emulated_instruction(vcpu);
	return 1;
}

3349 3350 3351 3352 3353
static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
{
	vmcs_writel(GUEST_DR7, val);
}

A
Avi Kivity 已提交
3354
static int handle_cpuid(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3355
{
3356 3357
	kvm_emulate_cpuid(vcpu);
	return 1;
A
Avi Kivity 已提交
3358 3359
}

A
Avi Kivity 已提交
3360
static int handle_rdmsr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3361
{
3362
	u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
A
Avi Kivity 已提交
3363 3364 3365
	u64 data;

	if (vmx_get_msr(vcpu, ecx, &data)) {
3366
		trace_kvm_msr_read_ex(ecx);
3367
		kvm_inject_gp(vcpu, 0);
A
Avi Kivity 已提交
3368 3369 3370
		return 1;
	}

3371
	trace_kvm_msr_read(ecx, data);
F
Feng (Eric) Liu 已提交
3372

A
Avi Kivity 已提交
3373
	/* FIXME: handling of bits 32:63 of rax, rdx */
3374 3375
	vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u;
	vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
A
Avi Kivity 已提交
3376 3377 3378 3379
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
3380
static int handle_wrmsr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3381
{
3382 3383 3384
	u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
	u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
		| ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
A
Avi Kivity 已提交
3385 3386

	if (vmx_set_msr(vcpu, ecx, data) != 0) {
3387
		trace_kvm_msr_write_ex(ecx, data);
3388
		kvm_inject_gp(vcpu, 0);
A
Avi Kivity 已提交
3389 3390 3391
		return 1;
	}

3392
	trace_kvm_msr_write(ecx, data);
A
Avi Kivity 已提交
3393 3394 3395 3396
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
3397
static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
3398
{
3399
	kvm_make_request(KVM_REQ_EVENT, vcpu);
3400 3401 3402
	return 1;
}

A
Avi Kivity 已提交
3403
static int handle_interrupt_window(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3404
{
3405 3406 3407 3408 3409 3410
	u32 cpu_based_vm_exec_control;

	/* clear pending irq */
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
F
Feng (Eric) Liu 已提交
3411

3412 3413
	kvm_make_request(KVM_REQ_EVENT, vcpu);

3414
	++vcpu->stat.irq_window_exits;
F
Feng (Eric) Liu 已提交
3415

3416 3417 3418 3419
	/*
	 * If the user space waits to inject interrupts, exit as soon as
	 * possible
	 */
3420
	if (!irqchip_in_kernel(vcpu->kvm) &&
A
Avi Kivity 已提交
3421
	    vcpu->run->request_interrupt_window &&
3422
	    !kvm_cpu_has_interrupt(vcpu)) {
A
Avi Kivity 已提交
3423
		vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
3424 3425
		return 0;
	}
A
Avi Kivity 已提交
3426 3427 3428
	return 1;
}

A
Avi Kivity 已提交
3429
static int handle_halt(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3430 3431
{
	skip_emulated_instruction(vcpu);
3432
	return kvm_emulate_halt(vcpu);
A
Avi Kivity 已提交
3433 3434
}

A
Avi Kivity 已提交
3435
static int handle_vmcall(struct kvm_vcpu *vcpu)
3436
{
3437
	skip_emulated_instruction(vcpu);
3438 3439
	kvm_emulate_hypercall(vcpu);
	return 1;
3440 3441
}

A
Avi Kivity 已提交
3442
static int handle_vmx_insn(struct kvm_vcpu *vcpu)
3443 3444 3445 3446 3447
{
	kvm_queue_exception(vcpu, UD_VECTOR);
	return 1;
}

3448 3449
static int handle_invd(struct kvm_vcpu *vcpu)
{
3450
	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
3451 3452
}

A
Avi Kivity 已提交
3453
static int handle_invlpg(struct kvm_vcpu *vcpu)
M
Marcelo Tosatti 已提交
3454
{
3455
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
M
Marcelo Tosatti 已提交
3456 3457 3458 3459 3460 3461

	kvm_mmu_invlpg(vcpu, exit_qualification);
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
3462
static int handle_wbinvd(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
3463 3464
{
	skip_emulated_instruction(vcpu);
3465
	kvm_emulate_wbinvd(vcpu);
E
Eddie Dong 已提交
3466 3467 3468
	return 1;
}

3469 3470 3471 3472 3473 3474 3475 3476 3477 3478
static int handle_xsetbv(struct kvm_vcpu *vcpu)
{
	u64 new_bv = kvm_read_edx_eax(vcpu);
	u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);

	if (kvm_set_xcr(vcpu, index, new_bv) == 0)
		skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
3479
static int handle_apic_access(struct kvm_vcpu *vcpu)
3480
{
3481
	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
3482 3483
}

A
Avi Kivity 已提交
3484
static int handle_task_switch(struct kvm_vcpu *vcpu)
3485
{
J
Jan Kiszka 已提交
3486
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3487
	unsigned long exit_qualification;
3488 3489
	bool has_error_code = false;
	u32 error_code = 0;
3490
	u16 tss_selector;
3491 3492 3493 3494
	int reason, type, idt_v;

	idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
	type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
3495 3496 3497 3498

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	reason = (u32)exit_qualification >> 30;
3499 3500 3501 3502 3503 3504 3505 3506 3507
	if (reason == TASK_SWITCH_GATE && idt_v) {
		switch (type) {
		case INTR_TYPE_NMI_INTR:
			vcpu->arch.nmi_injected = false;
			if (cpu_has_virtual_nmis())
				vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
					      GUEST_INTR_STATE_NMI);
			break;
		case INTR_TYPE_EXT_INTR:
3508
		case INTR_TYPE_SOFT_INTR:
3509 3510 3511
			kvm_clear_interrupt_queue(vcpu);
			break;
		case INTR_TYPE_HARD_EXCEPTION:
3512 3513 3514 3515 3516 3517 3518
			if (vmx->idt_vectoring_info &
			    VECTORING_INFO_DELIVER_CODE_MASK) {
				has_error_code = true;
				error_code =
					vmcs_read32(IDT_VECTORING_ERROR_CODE);
			}
			/* fall through */
3519 3520 3521 3522 3523 3524
		case INTR_TYPE_SOFT_EXCEPTION:
			kvm_clear_exception_queue(vcpu);
			break;
		default:
			break;
		}
J
Jan Kiszka 已提交
3525
	}
3526 3527
	tss_selector = exit_qualification;

3528 3529 3530 3531 3532
	if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
		       type != INTR_TYPE_EXT_INTR &&
		       type != INTR_TYPE_NMI_INTR))
		skip_emulated_instruction(vcpu);

3533 3534 3535 3536 3537
	if (kvm_task_switch(vcpu, tss_selector, reason,
				has_error_code, error_code) == EMULATE_FAIL) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
3538
		return 0;
3539
	}
3540 3541 3542 3543 3544 3545 3546 3547 3548 3549

	/* clear all local breakpoint enable flags */
	vmcs_writel(GUEST_DR7, vmcs_readl(GUEST_DR7) & ~55);

	/*
	 * TODO: What about debug traps on tss switch?
	 *       Are we supposed to inject them and update dr6?
	 */

	return 1;
3550 3551
}

A
Avi Kivity 已提交
3552
static int handle_ept_violation(struct kvm_vcpu *vcpu)
3553
{
3554
	unsigned long exit_qualification;
3555 3556 3557
	gpa_t gpa;
	int gla_validity;

3558
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3559 3560 3561

	if (exit_qualification & (1 << 6)) {
		printk(KERN_ERR "EPT: GPA exceeds GAW!\n");
3562
		return -EINVAL;
3563 3564 3565 3566 3567 3568 3569
	}

	gla_validity = (exit_qualification >> 7) & 0x3;
	if (gla_validity != 0x3 && gla_validity != 0x1 && gla_validity != 0) {
		printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
		printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
			(long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
3570
			vmcs_readl(GUEST_LINEAR_ADDRESS));
3571 3572
		printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
			(long unsigned int)exit_qualification);
A
Avi Kivity 已提交
3573 3574
		vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
		vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
3575
		return 0;
3576 3577 3578
	}

	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
3579
	trace_kvm_page_fault(gpa, exit_qualification);
3580
	return kvm_mmu_page_fault(vcpu, gpa, exit_qualification & 0x3, NULL, 0);
3581 3582
}

3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643
static u64 ept_rsvd_mask(u64 spte, int level)
{
	int i;
	u64 mask = 0;

	for (i = 51; i > boot_cpu_data.x86_phys_bits; i--)
		mask |= (1ULL << i);

	if (level > 2)
		/* bits 7:3 reserved */
		mask |= 0xf8;
	else if (level == 2) {
		if (spte & (1ULL << 7))
			/* 2MB ref, bits 20:12 reserved */
			mask |= 0x1ff000;
		else
			/* bits 6:3 reserved */
			mask |= 0x78;
	}

	return mask;
}

static void ept_misconfig_inspect_spte(struct kvm_vcpu *vcpu, u64 spte,
				       int level)
{
	printk(KERN_ERR "%s: spte 0x%llx level %d\n", __func__, spte, level);

	/* 010b (write-only) */
	WARN_ON((spte & 0x7) == 0x2);

	/* 110b (write/execute) */
	WARN_ON((spte & 0x7) == 0x6);

	/* 100b (execute-only) and value not supported by logical processor */
	if (!cpu_has_vmx_ept_execute_only())
		WARN_ON((spte & 0x7) == 0x4);

	/* not 000b */
	if ((spte & 0x7)) {
		u64 rsvd_bits = spte & ept_rsvd_mask(spte, level);

		if (rsvd_bits != 0) {
			printk(KERN_ERR "%s: rsvd_bits = 0x%llx\n",
					 __func__, rsvd_bits);
			WARN_ON(1);
		}

		if (level == 1 || (level == 2 && (spte & (1ULL << 7)))) {
			u64 ept_mem_type = (spte & 0x38) >> 3;

			if (ept_mem_type == 2 || ept_mem_type == 3 ||
			    ept_mem_type == 7) {
				printk(KERN_ERR "%s: ept_mem_type=0x%llx\n",
						__func__, ept_mem_type);
				WARN_ON(1);
			}
		}
	}
}

A
Avi Kivity 已提交
3644
static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659
{
	u64 sptes[4];
	int nr_sptes, i;
	gpa_t gpa;

	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);

	printk(KERN_ERR "EPT: Misconfiguration.\n");
	printk(KERN_ERR "EPT: GPA: 0x%llx\n", gpa);

	nr_sptes = kvm_mmu_get_spte_hierarchy(vcpu, gpa, sptes);

	for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i)
		ept_misconfig_inspect_spte(vcpu, sptes[i-1], i);

A
Avi Kivity 已提交
3660 3661
	vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
	vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
3662 3663 3664 3665

	return 0;
}

A
Avi Kivity 已提交
3666
static int handle_nmi_window(struct kvm_vcpu *vcpu)
3667 3668 3669 3670 3671 3672 3673 3674
{
	u32 cpu_based_vm_exec_control;

	/* clear pending NMI */
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
	++vcpu->stat.nmi_window_exits;
3675
	kvm_make_request(KVM_REQ_EVENT, vcpu);
3676 3677 3678 3679

	return 1;
}

3680
static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
3681
{
3682 3683
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	enum emulation_result err = EMULATE_DONE;
3684
	int ret = 1;
3685 3686 3687 3688 3689
	u32 cpu_exec_ctrl;
	bool intr_window_requested;

	cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
3690 3691

	while (!guest_state_valid(vcpu)) {
3692 3693 3694 3695
		if (intr_window_requested
		    && (kvm_get_rflags(&vmx->vcpu) & X86_EFLAGS_IF))
			return handle_interrupt_window(&vmx->vcpu);

3696
		err = emulate_instruction(vcpu, 0);
3697

3698 3699 3700 3701
		if (err == EMULATE_DO_MMIO) {
			ret = 0;
			goto out;
		}
3702

3703 3704
		if (err != EMULATE_DONE)
			return 0;
3705 3706

		if (signal_pending(current))
3707
			goto out;
3708 3709 3710 3711
		if (need_resched())
			schedule();
	}

3712 3713 3714
	vmx->emulation_required = 0;
out:
	return ret;
3715 3716
}

3717 3718 3719 3720
/*
 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
 * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
 */
3721
static int handle_pause(struct kvm_vcpu *vcpu)
3722 3723 3724 3725 3726 3727 3728
{
	skip_emulated_instruction(vcpu);
	kvm_vcpu_on_spin(vcpu);

	return 1;
}

3729 3730 3731 3732 3733 3734
static int handle_invalid_op(struct kvm_vcpu *vcpu)
{
	kvm_queue_exception(vcpu, UD_VECTOR);
	return 1;
}

A
Avi Kivity 已提交
3735 3736 3737 3738 3739
/*
 * The exit handlers return 1 if the exit was handled fully and guest execution
 * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
 * to be done to userspace and return 0.
 */
A
Avi Kivity 已提交
3740
static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
A
Avi Kivity 已提交
3741 3742
	[EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
	[EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
3743
	[EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
3744
	[EXIT_REASON_NMI_WINDOW]	      = handle_nmi_window,
A
Avi Kivity 已提交
3745 3746 3747 3748 3749 3750 3751 3752
	[EXIT_REASON_IO_INSTRUCTION]          = handle_io,
	[EXIT_REASON_CR_ACCESS]               = handle_cr,
	[EXIT_REASON_DR_ACCESS]               = handle_dr,
	[EXIT_REASON_CPUID]                   = handle_cpuid,
	[EXIT_REASON_MSR_READ]                = handle_rdmsr,
	[EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
	[EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
	[EXIT_REASON_HLT]                     = handle_halt,
3753
	[EXIT_REASON_INVD]		      = handle_invd,
M
Marcelo Tosatti 已提交
3754
	[EXIT_REASON_INVLPG]		      = handle_invlpg,
3755
	[EXIT_REASON_VMCALL]                  = handle_vmcall,
3756 3757 3758 3759 3760 3761 3762 3763 3764
	[EXIT_REASON_VMCLEAR]	              = handle_vmx_insn,
	[EXIT_REASON_VMLAUNCH]                = handle_vmx_insn,
	[EXIT_REASON_VMPTRLD]                 = handle_vmx_insn,
	[EXIT_REASON_VMPTRST]                 = handle_vmx_insn,
	[EXIT_REASON_VMREAD]                  = handle_vmx_insn,
	[EXIT_REASON_VMRESUME]                = handle_vmx_insn,
	[EXIT_REASON_VMWRITE]                 = handle_vmx_insn,
	[EXIT_REASON_VMOFF]                   = handle_vmx_insn,
	[EXIT_REASON_VMON]                    = handle_vmx_insn,
3765 3766
	[EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
	[EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
E
Eddie Dong 已提交
3767
	[EXIT_REASON_WBINVD]                  = handle_wbinvd,
3768
	[EXIT_REASON_XSETBV]                  = handle_xsetbv,
3769
	[EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
A
Andi Kleen 已提交
3770
	[EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
3771 3772
	[EXIT_REASON_EPT_VIOLATION]	      = handle_ept_violation,
	[EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
3773
	[EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
3774 3775
	[EXIT_REASON_MWAIT_INSTRUCTION]	      = handle_invalid_op,
	[EXIT_REASON_MONITOR_INSTRUCTION]     = handle_invalid_op,
A
Avi Kivity 已提交
3776 3777 3778
};

static const int kvm_vmx_max_exit_handlers =
3779
	ARRAY_SIZE(kvm_vmx_exit_handlers);
A
Avi Kivity 已提交
3780

3781 3782 3783 3784 3785 3786
static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
{
	*info1 = vmcs_readl(EXIT_QUALIFICATION);
	*info2 = vmcs_read32(VM_EXIT_INTR_INFO);
}

A
Avi Kivity 已提交
3787 3788 3789 3790
/*
 * The guest has exited.  See if we can fix it or if we need userspace
 * assistance.
 */
A
Avi Kivity 已提交
3791
static int vmx_handle_exit(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3792
{
3793
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Andi Kleen 已提交
3794
	u32 exit_reason = vmx->exit_reason;
3795
	u32 vectoring_info = vmx->idt_vectoring_info;
3796

3797
	trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);
F
Feng (Eric) Liu 已提交
3798

3799 3800 3801
	/* If guest state is invalid, start emulating */
	if (vmx->emulation_required && emulate_invalid_guest_state)
		return handle_invalid_guest_state(vcpu);
3802

3803 3804 3805 3806 3807 3808 3809
	if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		vcpu->run->fail_entry.hardware_entry_failure_reason
			= exit_reason;
		return 0;
	}

3810
	if (unlikely(vmx->fail)) {
A
Avi Kivity 已提交
3811 3812
		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		vcpu->run->fail_entry.hardware_entry_failure_reason
3813 3814 3815
			= vmcs_read32(VM_INSTRUCTION_ERROR);
		return 0;
	}
A
Avi Kivity 已提交
3816

M
Mike Day 已提交
3817
	if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
3818
			(exit_reason != EXIT_REASON_EXCEPTION_NMI &&
J
Jan Kiszka 已提交
3819 3820 3821 3822 3823
			exit_reason != EXIT_REASON_EPT_VIOLATION &&
			exit_reason != EXIT_REASON_TASK_SWITCH))
		printk(KERN_WARNING "%s: unexpected, valid vectoring info "
		       "(0x%x) and exit reason is 0x%x\n",
		       __func__, vectoring_info, exit_reason);
3824 3825

	if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked)) {
3826
		if (vmx_interrupt_allowed(vcpu)) {
3827 3828
			vmx->soft_vnmi_blocked = 0;
		} else if (vmx->vnmi_blocked_time > 1000000000LL &&
3829
			   vcpu->arch.nmi_pending) {
3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842
			/*
			 * This CPU don't support us in finding the end of an
			 * NMI-blocked window if the guest runs with IRQs
			 * disabled. So we pull the trigger after 1 s of
			 * futile waiting, but inform the user about this.
			 */
			printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
			       "state on VCPU %d after 1 s timeout\n",
			       __func__, vcpu->vcpu_id);
			vmx->soft_vnmi_blocked = 0;
		}
	}

A
Avi Kivity 已提交
3843 3844
	if (exit_reason < kvm_vmx_max_exit_handlers
	    && kvm_vmx_exit_handlers[exit_reason])
A
Avi Kivity 已提交
3845
		return kvm_vmx_exit_handlers[exit_reason](vcpu);
A
Avi Kivity 已提交
3846
	else {
A
Avi Kivity 已提交
3847 3848
		vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
		vcpu->run->hw.hardware_exit_reason = exit_reason;
A
Avi Kivity 已提交
3849 3850 3851 3852
	}
	return 0;
}

3853
static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3854
{
3855
	if (irr == -1 || tpr < irr) {
3856 3857 3858 3859
		vmcs_write32(TPR_THRESHOLD, 0);
		return;
	}

3860
	vmcs_write32(TPR_THRESHOLD, irr);
3861 3862
}

3863
static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
3864
{
3865
	u32 exit_intr_info = vmx->exit_intr_info;
A
Andi Kleen 已提交
3866 3867 3868 3869 3870 3871 3872

	/* Handle machine checks before interrupts are enabled */
	if ((vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)
	    || (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI
		&& is_machine_check(exit_intr_info)))
		kvm_machine_check();

3873 3874
	/* We need to handle NMIs before interrupts are enabled */
	if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
3875 3876
	    (exit_intr_info & INTR_INFO_VALID_MASK)) {
		kvm_before_handle_nmi(&vmx->vcpu);
3877
		asm("int $2");
3878 3879
		kvm_after_handle_nmi(&vmx->vcpu);
	}
3880
}
3881

3882 3883 3884 3885 3886 3887 3888 3889
static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
{
	u32 exit_intr_info = vmx->exit_intr_info;
	bool unblock_nmi;
	u8 vector;
	bool idtv_info_valid;

	idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
3890

3891 3892 3893 3894
	if (cpu_has_virtual_nmis()) {
		unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
		vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
		/*
3895
		 * SDM 3: 27.7.1.2 (September 2008)
3896 3897
		 * Re-set bit "block by NMI" before VM entry if vmexit caused by
		 * a guest IRET fault.
3898 3899 3900 3901 3902
		 * SDM 3: 23.2.2 (September 2008)
		 * Bit 12 is undefined in any of the following cases:
		 *  If the VM exit sets the valid bit in the IDT-vectoring
		 *   information field.
		 *  If the VM exit is due to a double fault.
3903
		 */
3904 3905
		if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
		    vector != DF_VECTOR && !idtv_info_valid)
3906 3907
			vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				      GUEST_INTR_STATE_NMI);
3908 3909 3910
	} else if (unlikely(vmx->soft_vnmi_blocked))
		vmx->vnmi_blocked_time +=
			ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
3911 3912
}

3913 3914 3915 3916
static void __vmx_complete_interrupts(struct vcpu_vmx *vmx,
				      u32 idt_vectoring_info,
				      int instr_len_field,
				      int error_code_field)
3917 3918 3919 3920 3921 3922
{
	u8 vector;
	int type;
	bool idtv_info_valid;

	idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
3923

3924 3925 3926 3927 3928 3929 3930
	vmx->vcpu.arch.nmi_injected = false;
	kvm_clear_exception_queue(&vmx->vcpu);
	kvm_clear_interrupt_queue(&vmx->vcpu);

	if (!idtv_info_valid)
		return;

3931 3932
	kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);

3933 3934
	vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
	type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
3935

3936
	switch (type) {
3937 3938
	case INTR_TYPE_NMI_INTR:
		vmx->vcpu.arch.nmi_injected = true;
3939
		/*
3940
		 * SDM 3: 27.7.1.2 (September 2008)
3941 3942
		 * Clear bit "block by NMI" before VM entry if a NMI
		 * delivery faulted.
3943
		 */
3944 3945 3946 3947
		vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
				GUEST_INTR_STATE_NMI);
		break;
	case INTR_TYPE_SOFT_EXCEPTION:
3948
		vmx->vcpu.arch.event_exit_inst_len =
3949
			vmcs_read32(instr_len_field);
3950 3951
		/* fall through */
	case INTR_TYPE_HARD_EXCEPTION:
3952
		if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
3953
			u32 err = vmcs_read32(error_code_field);
3954
			kvm_queue_exception_e(&vmx->vcpu, vector, err);
3955 3956
		} else
			kvm_queue_exception(&vmx->vcpu, vector);
3957
		break;
3958 3959
	case INTR_TYPE_SOFT_INTR:
		vmx->vcpu.arch.event_exit_inst_len =
3960
			vmcs_read32(instr_len_field);
3961
		/* fall through */
3962
	case INTR_TYPE_EXT_INTR:
3963 3964
		kvm_queue_interrupt(&vmx->vcpu, vector,
			type == INTR_TYPE_SOFT_INTR);
3965 3966 3967
		break;
	default:
		break;
3968
	}
3969 3970
}

3971 3972 3973 3974 3975 3976 3977
static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
{
	__vmx_complete_interrupts(vmx, vmx->idt_vectoring_info,
				  VM_EXIT_INSTRUCTION_LEN,
				  IDT_VECTORING_ERROR_CODE);
}

A
Avi Kivity 已提交
3978 3979 3980 3981 3982 3983 3984 3985 3986 3987
static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
{
	__vmx_complete_interrupts(to_vmx(vcpu),
				  vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
				  VM_ENTRY_INSTRUCTION_LEN,
				  VM_ENTRY_EXCEPTION_ERROR_CODE);

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
}

3988 3989 3990 3991 3992 3993 3994 3995
#ifdef CONFIG_X86_64
#define R "r"
#define Q "q"
#else
#define R "e"
#define Q "l"
#endif

3996
static void vmx_vcpu_run(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3997
{
3998
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022

	/* Record the guest's net vcpu time for enforced NMI injections. */
	if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
		vmx->entry_time = ktime_get();

	/* Don't enter VMX if guest state is invalid, let the exit handler
	   start emulation until we arrive back to a valid state */
	if (vmx->emulation_required && emulate_invalid_guest_state)
		return;

	if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
		vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
	if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
		vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);

	/* When single-stepping over STI and MOV SS, we must clear the
	 * corresponding interruptibility bits in the guest state. Otherwise
	 * vmentry fails as it then expects bit 14 (BS) in pending debug
	 * exceptions being set, but that's not correct for the guest debugging
	 * case. */
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vmx_set_interrupt_shadow(vcpu, 0);

	asm(
A
Avi Kivity 已提交
4023
		/* Store host registers */
4024 4025
		"push %%"R"dx; push %%"R"bp;"
		"push %%"R"cx \n\t"
4026 4027 4028
		"cmp %%"R"sp, %c[host_rsp](%0) \n\t"
		"je 1f \n\t"
		"mov %%"R"sp, %c[host_rsp](%0) \n\t"
4029
		__ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
4030
		"1: \n\t"
4031 4032 4033 4034 4035 4036 4037
		/* Reload cr2 if changed */
		"mov %c[cr2](%0), %%"R"ax \n\t"
		"mov %%cr2, %%"R"dx \n\t"
		"cmp %%"R"ax, %%"R"dx \n\t"
		"je 2f \n\t"
		"mov %%"R"ax, %%cr2 \n\t"
		"2: \n\t"
A
Avi Kivity 已提交
4038
		/* Check if vmlaunch of vmresume is needed */
4039
		"cmpl $0, %c[launched](%0) \n\t"
A
Avi Kivity 已提交
4040
		/* Load guest registers.  Don't clobber flags. */
4041 4042 4043 4044 4045 4046
		"mov %c[rax](%0), %%"R"ax \n\t"
		"mov %c[rbx](%0), %%"R"bx \n\t"
		"mov %c[rdx](%0), %%"R"dx \n\t"
		"mov %c[rsi](%0), %%"R"si \n\t"
		"mov %c[rdi](%0), %%"R"di \n\t"
		"mov %c[rbp](%0), %%"R"bp \n\t"
4047
#ifdef CONFIG_X86_64
4048 4049 4050 4051 4052 4053 4054 4055
		"mov %c[r8](%0),  %%r8  \n\t"
		"mov %c[r9](%0),  %%r9  \n\t"
		"mov %c[r10](%0), %%r10 \n\t"
		"mov %c[r11](%0), %%r11 \n\t"
		"mov %c[r12](%0), %%r12 \n\t"
		"mov %c[r13](%0), %%r13 \n\t"
		"mov %c[r14](%0), %%r14 \n\t"
		"mov %c[r15](%0), %%r15 \n\t"
A
Avi Kivity 已提交
4056
#endif
4057 4058
		"mov %c[rcx](%0), %%"R"cx \n\t" /* kills %0 (ecx) */

A
Avi Kivity 已提交
4059
		/* Enter guest mode */
4060
		"jne .Llaunched \n\t"
4061
		__ex(ASM_VMX_VMLAUNCH) "\n\t"
4062
		"jmp .Lkvm_vmx_return \n\t"
4063
		".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t"
4064
		".Lkvm_vmx_return: "
A
Avi Kivity 已提交
4065
		/* Save guest registers, load host registers, keep flags */
4066 4067 4068 4069 4070 4071 4072 4073
		"xchg %0,     (%%"R"sp) \n\t"
		"mov %%"R"ax, %c[rax](%0) \n\t"
		"mov %%"R"bx, %c[rbx](%0) \n\t"
		"push"Q" (%%"R"sp); pop"Q" %c[rcx](%0) \n\t"
		"mov %%"R"dx, %c[rdx](%0) \n\t"
		"mov %%"R"si, %c[rsi](%0) \n\t"
		"mov %%"R"di, %c[rdi](%0) \n\t"
		"mov %%"R"bp, %c[rbp](%0) \n\t"
4074
#ifdef CONFIG_X86_64
4075 4076 4077 4078 4079 4080 4081 4082
		"mov %%r8,  %c[r8](%0) \n\t"
		"mov %%r9,  %c[r9](%0) \n\t"
		"mov %%r10, %c[r10](%0) \n\t"
		"mov %%r11, %c[r11](%0) \n\t"
		"mov %%r12, %c[r12](%0) \n\t"
		"mov %%r13, %c[r13](%0) \n\t"
		"mov %%r14, %c[r14](%0) \n\t"
		"mov %%r15, %c[r15](%0) \n\t"
A
Avi Kivity 已提交
4083
#endif
4084 4085 4086 4087
		"mov %%cr2, %%"R"ax   \n\t"
		"mov %%"R"ax, %c[cr2](%0) \n\t"

		"pop  %%"R"bp; pop  %%"R"bp; pop  %%"R"dx \n\t"
4088 4089 4090 4091
		"setbe %c[fail](%0) \n\t"
	      : : "c"(vmx), "d"((unsigned long)HOST_RSP),
		[launched]"i"(offsetof(struct vcpu_vmx, launched)),
		[fail]"i"(offsetof(struct vcpu_vmx, fail)),
4092
		[host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
4093 4094 4095 4096 4097 4098 4099
		[rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
		[rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
		[rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
		[rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
		[rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
		[rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
		[rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
4100
#ifdef CONFIG_X86_64
4101 4102 4103 4104 4105 4106 4107 4108
		[r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
		[r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
		[r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
		[r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
		[r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
		[r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
		[r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
		[r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
A
Avi Kivity 已提交
4109
#endif
4110
		[cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2))
4111
	      : "cc", "memory"
4112
		, R"ax", R"bx", R"di", R"si"
4113 4114 4115 4116
#ifdef CONFIG_X86_64
		, "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
#endif
	      );
A
Avi Kivity 已提交
4117

A
Avi Kivity 已提交
4118
	vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
4119 4120
				  | (1 << VCPU_EXREG_PDPTR)
				  | (1 << VCPU_EXREG_CR3));
4121 4122
	vcpu->arch.regs_dirty = 0;

4123 4124
	vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);

M
Mike Day 已提交
4125
	asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
4126
	vmx->launched = 1;
4127

4128 4129 4130 4131 4132
	vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
	vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);

	vmx_complete_atomic_exit(vmx);
	vmx_recover_nmi_blocking(vmx);
4133
	vmx_complete_interrupts(vmx);
A
Avi Kivity 已提交
4134 4135
}

4136 4137 4138
#undef R
#undef Q

A
Avi Kivity 已提交
4139 4140
static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
{
4141 4142 4143
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (vmx->vmcs) {
4144
		vcpu_clear(vmx);
4145 4146
		free_vmcs(vmx->vmcs);
		vmx->vmcs = NULL;
A
Avi Kivity 已提交
4147 4148 4149 4150 4151
	}
}

static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
4152 4153
	struct vcpu_vmx *vmx = to_vmx(vcpu);

4154
	free_vpid(vmx);
A
Avi Kivity 已提交
4155
	vmx_free_vmcs(vcpu);
R
Rusty Russell 已提交
4156 4157
	kfree(vmx->guest_msrs);
	kvm_vcpu_uninit(vcpu);
4158
	kmem_cache_free(kvm_vcpu_cache, vmx);
A
Avi Kivity 已提交
4159 4160
}

4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173
static inline void vmcs_init(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(per_cpu(vmxarea, raw_smp_processor_id()));

	if (!vmm_exclusive)
		kvm_cpu_vmxon(phys_addr);

	vmcs_clear(vmcs);

	if (!vmm_exclusive)
		kvm_cpu_vmxoff();
}

R
Rusty Russell 已提交
4174
static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
A
Avi Kivity 已提交
4175
{
R
Rusty Russell 已提交
4176
	int err;
4177
	struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
4178
	int cpu;
A
Avi Kivity 已提交
4179

4180
	if (!vmx)
R
Rusty Russell 已提交
4181 4182
		return ERR_PTR(-ENOMEM);

4183 4184
	allocate_vpid(vmx);

R
Rusty Russell 已提交
4185 4186 4187
	err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
	if (err)
		goto free_vcpu;
4188

4189
	vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
R
Rusty Russell 已提交
4190 4191 4192 4193
	if (!vmx->guest_msrs) {
		err = -ENOMEM;
		goto uninit_vcpu;
	}
4194

4195 4196
	vmx->vmcs = alloc_vmcs();
	if (!vmx->vmcs)
R
Rusty Russell 已提交
4197
		goto free_msrs;
4198

4199
	vmcs_init(vmx->vmcs);
4200

4201 4202
	cpu = get_cpu();
	vmx_vcpu_load(&vmx->vcpu, cpu);
Z
Zachary Amsden 已提交
4203
	vmx->vcpu.cpu = cpu;
R
Rusty Russell 已提交
4204
	err = vmx_vcpu_setup(vmx);
R
Rusty Russell 已提交
4205
	vmx_vcpu_put(&vmx->vcpu);
4206
	put_cpu();
R
Rusty Russell 已提交
4207 4208
	if (err)
		goto free_vmcs;
4209 4210 4211
	if (vm_need_virtualize_apic_accesses(kvm))
		if (alloc_apic_access_page(kvm) != 0)
			goto free_vmcs;
R
Rusty Russell 已提交
4212

4213 4214 4215 4216
	if (enable_ept) {
		if (!kvm->arch.ept_identity_map_addr)
			kvm->arch.ept_identity_map_addr =
				VMX_EPT_IDENTITY_PAGETABLE_ADDR;
4217 4218
		if (alloc_identity_pagetable(kvm) != 0)
			goto free_vmcs;
4219
	}
4220

R
Rusty Russell 已提交
4221 4222 4223 4224 4225 4226 4227 4228 4229
	return &vmx->vcpu;

free_vmcs:
	free_vmcs(vmx->vmcs);
free_msrs:
	kfree(vmx->guest_msrs);
uninit_vcpu:
	kvm_vcpu_uninit(&vmx->vcpu);
free_vcpu:
4230
	free_vpid(vmx);
4231
	kmem_cache_free(kvm_vcpu_cache, vmx);
R
Rusty Russell 已提交
4232
	return ERR_PTR(err);
A
Avi Kivity 已提交
4233 4234
}

Y
Yang, Sheng 已提交
4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248
static void __init vmx_check_processor_compat(void *rtn)
{
	struct vmcs_config vmcs_conf;

	*(int *)rtn = 0;
	if (setup_vmcs_config(&vmcs_conf) < 0)
		*(int *)rtn = -EIO;
	if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
		printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
				smp_processor_id());
		*(int *)rtn = -EIO;
	}
}

4249 4250 4251 4252 4253
static int get_ept_level(void)
{
	return VMX_EPT_DEFAULT_GAW + 1;
}

4254
static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
S
Sheng Yang 已提交
4255
{
4256 4257
	u64 ret;

4258 4259 4260 4261 4262 4263 4264 4265
	/* For VT-d and EPT combination
	 * 1. MMIO: always map as UC
	 * 2. EPT with VT-d:
	 *   a. VT-d without snooping control feature: can't guarantee the
	 *	result, try to trust guest.
	 *   b. VT-d with snooping control feature: snooping control feature of
	 *	VT-d engine can guarantee the cache correctness. Just set it
	 *	to WB to keep consistent with host. So the same as item 3.
4266
	 * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
4267 4268
	 *    consistent with host MTRR
	 */
4269 4270
	if (is_mmio)
		ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
4271 4272 4273 4274
	else if (vcpu->kvm->arch.iommu_domain &&
		!(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY))
		ret = kvm_get_guest_memory_type(vcpu, gfn) <<
		      VMX_EPT_MT_EPTE_SHIFT;
4275
	else
4276
		ret = (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT)
4277
			| VMX_EPT_IPAT_BIT;
4278 4279

	return ret;
S
Sheng Yang 已提交
4280 4281
}

4282 4283
#define _ER(x) { EXIT_REASON_##x, #x }

4284
static const struct trace_print_flags vmx_exit_reasons_str[] = {
4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319
	_ER(EXCEPTION_NMI),
	_ER(EXTERNAL_INTERRUPT),
	_ER(TRIPLE_FAULT),
	_ER(PENDING_INTERRUPT),
	_ER(NMI_WINDOW),
	_ER(TASK_SWITCH),
	_ER(CPUID),
	_ER(HLT),
	_ER(INVLPG),
	_ER(RDPMC),
	_ER(RDTSC),
	_ER(VMCALL),
	_ER(VMCLEAR),
	_ER(VMLAUNCH),
	_ER(VMPTRLD),
	_ER(VMPTRST),
	_ER(VMREAD),
	_ER(VMRESUME),
	_ER(VMWRITE),
	_ER(VMOFF),
	_ER(VMON),
	_ER(CR_ACCESS),
	_ER(DR_ACCESS),
	_ER(IO_INSTRUCTION),
	_ER(MSR_READ),
	_ER(MSR_WRITE),
	_ER(MWAIT_INSTRUCTION),
	_ER(MONITOR_INSTRUCTION),
	_ER(PAUSE_INSTRUCTION),
	_ER(MCE_DURING_VMENTRY),
	_ER(TPR_BELOW_THRESHOLD),
	_ER(APIC_ACCESS),
	_ER(EPT_VIOLATION),
	_ER(EPT_MISCONFIG),
	_ER(WBINVD),
4320 4321 4322
	{ -1, NULL }
};

4323 4324
#undef _ER

4325
static int vmx_get_lpage_level(void)
4326
{
4327 4328 4329 4330 4331
	if (enable_ept && !cpu_has_vmx_ept_1g_page())
		return PT_DIRECTORY_LEVEL;
	else
		/* For shadow and EPT supported 1GB page */
		return PT_PDPE_LEVEL;
4332 4333
}

4334 4335
static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
{
4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353
	struct kvm_cpuid_entry2 *best;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u32 exec_control;

	vmx->rdtscp_enabled = false;
	if (vmx_rdtscp_supported()) {
		exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
		if (exec_control & SECONDARY_EXEC_RDTSCP) {
			best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
			if (best && (best->edx & bit(X86_FEATURE_RDTSCP)))
				vmx->rdtscp_enabled = true;
			else {
				exec_control &= ~SECONDARY_EXEC_RDTSCP;
				vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
						exec_control);
			}
		}
	}
4354 4355
}

4356 4357 4358 4359
static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
{
}

4360
static struct kvm_x86_ops vmx_x86_ops = {
A
Avi Kivity 已提交
4361 4362 4363 4364
	.cpu_has_kvm_support = cpu_has_kvm_support,
	.disabled_by_bios = vmx_disabled_by_bios,
	.hardware_setup = hardware_setup,
	.hardware_unsetup = hardware_unsetup,
Y
Yang, Sheng 已提交
4365
	.check_processor_compatibility = vmx_check_processor_compat,
A
Avi Kivity 已提交
4366 4367
	.hardware_enable = hardware_enable,
	.hardware_disable = hardware_disable,
4368
	.cpu_has_accelerated_tpr = report_flexpriority,
A
Avi Kivity 已提交
4369 4370 4371

	.vcpu_create = vmx_create_vcpu,
	.vcpu_free = vmx_free_vcpu,
4372
	.vcpu_reset = vmx_vcpu_reset,
A
Avi Kivity 已提交
4373

4374
	.prepare_guest_switch = vmx_save_host_state,
A
Avi Kivity 已提交
4375 4376 4377 4378 4379 4380 4381 4382 4383
	.vcpu_load = vmx_vcpu_load,
	.vcpu_put = vmx_vcpu_put,

	.set_guest_debug = set_guest_debug,
	.get_msr = vmx_get_msr,
	.set_msr = vmx_set_msr,
	.get_segment_base = vmx_get_segment_base,
	.get_segment = vmx_get_segment,
	.set_segment = vmx_set_segment,
4384
	.get_cpl = vmx_get_cpl,
A
Avi Kivity 已提交
4385
	.get_cs_db_l_bits = vmx_get_cs_db_l_bits,
4386
	.decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
4387
	.decache_cr3 = vmx_decache_cr3,
4388
	.decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
A
Avi Kivity 已提交
4389 4390 4391 4392 4393 4394 4395 4396
	.set_cr0 = vmx_set_cr0,
	.set_cr3 = vmx_set_cr3,
	.set_cr4 = vmx_set_cr4,
	.set_efer = vmx_set_efer,
	.get_idt = vmx_get_idt,
	.set_idt = vmx_set_idt,
	.get_gdt = vmx_get_gdt,
	.set_gdt = vmx_set_gdt,
4397
	.set_dr7 = vmx_set_dr7,
4398
	.cache_reg = vmx_cache_reg,
A
Avi Kivity 已提交
4399 4400
	.get_rflags = vmx_get_rflags,
	.set_rflags = vmx_set_rflags,
4401
	.fpu_activate = vmx_fpu_activate,
4402
	.fpu_deactivate = vmx_fpu_deactivate,
A
Avi Kivity 已提交
4403 4404 4405 4406

	.tlb_flush = vmx_flush_tlb,

	.run = vmx_vcpu_run,
4407
	.handle_exit = vmx_handle_exit,
A
Avi Kivity 已提交
4408
	.skip_emulated_instruction = skip_emulated_instruction,
4409 4410
	.set_interrupt_shadow = vmx_set_interrupt_shadow,
	.get_interrupt_shadow = vmx_get_interrupt_shadow,
I
Ingo Molnar 已提交
4411
	.patch_hypercall = vmx_patch_hypercall,
E
Eddie Dong 已提交
4412
	.set_irq = vmx_inject_irq,
4413
	.set_nmi = vmx_inject_nmi,
4414
	.queue_exception = vmx_queue_exception,
A
Avi Kivity 已提交
4415
	.cancel_injection = vmx_cancel_injection,
4416
	.interrupt_allowed = vmx_interrupt_allowed,
4417
	.nmi_allowed = vmx_nmi_allowed,
J
Jan Kiszka 已提交
4418 4419
	.get_nmi_mask = vmx_get_nmi_mask,
	.set_nmi_mask = vmx_set_nmi_mask,
4420 4421 4422 4423
	.enable_nmi_window = enable_nmi_window,
	.enable_irq_window = enable_irq_window,
	.update_cr8_intercept = update_cr8_intercept,

4424
	.set_tss_addr = vmx_set_tss_addr,
4425
	.get_tdp_level = get_ept_level,
4426
	.get_mt_mask = vmx_get_mt_mask,
4427

4428
	.get_exit_info = vmx_get_exit_info,
4429
	.exit_reasons_str = vmx_exit_reasons_str,
4430

4431
	.get_lpage_level = vmx_get_lpage_level,
4432 4433

	.cpuid_update = vmx_cpuid_update,
4434 4435

	.rdtscp_supported = vmx_rdtscp_supported,
4436 4437

	.set_supported_cpuid = vmx_set_supported_cpuid,
4438 4439

	.has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
4440 4441

	.write_tsc_offset = vmx_write_tsc_offset,
Z
Zachary Amsden 已提交
4442
	.adjust_tsc_offset = vmx_adjust_tsc_offset,
4443 4444

	.set_tdp_cr3 = vmx_set_cr3,
A
Avi Kivity 已提交
4445 4446 4447 4448
};

static int __init vmx_init(void)
{
4449 4450 4451 4452 4453 4454
	int r, i;

	rdmsrl_safe(MSR_EFER, &host_efer);

	for (i = 0; i < NR_VMX_MSR; ++i)
		kvm_define_shared_msr(i, vmx_msr_index[i]);
4455

4456
	vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
4457 4458 4459
	if (!vmx_io_bitmap_a)
		return -ENOMEM;

4460
	vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
4461 4462 4463 4464 4465
	if (!vmx_io_bitmap_b) {
		r = -ENOMEM;
		goto out;
	}

4466 4467
	vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_legacy) {
S
Sheng Yang 已提交
4468 4469 4470 4471
		r = -ENOMEM;
		goto out1;
	}

4472 4473 4474 4475 4476 4477
	vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_longmode) {
		r = -ENOMEM;
		goto out2;
	}

4478 4479 4480 4481
	/*
	 * Allow direct access to the PC debug port (it is often used for I/O
	 * delays, but the vmexits simply slow things down).
	 */
4482 4483
	memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
	clear_bit(0x80, vmx_io_bitmap_a);
4484

4485
	memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
4486

4487 4488
	memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
	memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
S
Sheng Yang 已提交
4489

4490 4491
	set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */

4492 4493
	r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
		     __alignof__(struct vcpu_vmx), THIS_MODULE);
4494
	if (r)
4495
		goto out3;
S
Sheng Yang 已提交
4496

4497 4498 4499 4500 4501 4502
	vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
	vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
	vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
4503

4504
	if (enable_ept) {
4505
		bypass_guest_pf = 0;
4506
		kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull,
4507
				VMX_EPT_EXECUTABLE_MASK);
4508 4509 4510
		kvm_enable_tdp();
	} else
		kvm_disable_tdp();
4511

4512 4513 4514
	if (bypass_guest_pf)
		kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull);

4515 4516
	return 0;

4517 4518
out3:
	free_page((unsigned long)vmx_msr_bitmap_longmode);
S
Sheng Yang 已提交
4519
out2:
4520
	free_page((unsigned long)vmx_msr_bitmap_legacy);
4521
out1:
4522
	free_page((unsigned long)vmx_io_bitmap_b);
4523
out:
4524
	free_page((unsigned long)vmx_io_bitmap_a);
4525
	return r;
A
Avi Kivity 已提交
4526 4527 4528 4529
}

static void __exit vmx_exit(void)
{
4530 4531
	free_page((unsigned long)vmx_msr_bitmap_legacy);
	free_page((unsigned long)vmx_msr_bitmap_longmode);
4532 4533
	free_page((unsigned long)vmx_io_bitmap_b);
	free_page((unsigned long)vmx_io_bitmap_a);
4534

4535
	kvm_exit();
A
Avi Kivity 已提交
4536 4537 4538 4539
}

module_init(vmx_init)
module_exit(vmx_exit)