vmx.c 212.4 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "irq.h"
20
#include "mmu.h"
A
Avi Kivity 已提交
21
#include "cpuid.h"
A
Avi Kivity 已提交
22

23
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
24
#include <linux/module.h>
25
#include <linux/kernel.h>
A
Avi Kivity 已提交
26 27
#include <linux/mm.h>
#include <linux/highmem.h>
A
Alexey Dobriyan 已提交
28
#include <linux/sched.h>
29
#include <linux/moduleparam.h>
30
#include <linux/mod_devicetable.h>
31
#include <linux/ftrace_event.h>
32
#include <linux/slab.h>
33
#include <linux/tboot.h>
34
#include "kvm_cache_regs.h"
35
#include "x86.h"
A
Avi Kivity 已提交
36

A
Avi Kivity 已提交
37
#include <asm/io.h>
A
Anthony Liguori 已提交
38
#include <asm/desc.h>
39
#include <asm/vmx.h>
40
#include <asm/virtext.h>
A
Andi Kleen 已提交
41
#include <asm/mce.h>
42 43
#include <asm/i387.h>
#include <asm/xcr.h>
44
#include <asm/perf_event.h>
45
#include <asm/kexec.h>
A
Avi Kivity 已提交
46

47 48
#include "trace.h"

49
#define __ex(x) __kvm_handle_fault_on_reboot(x)
50 51
#define __ex_clear(x, reg) \
	____kvm_handle_fault_on_reboot(x, "xor " reg " , " reg)
52

A
Avi Kivity 已提交
53 54 55
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

56 57 58 59 60 61
static const struct x86_cpu_id vmx_cpu_id[] = {
	X86_FEATURE_MATCH(X86_FEATURE_VMX),
	{}
};
MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);

62
static bool __read_mostly enable_vpid = 1;
63
module_param_named(vpid, enable_vpid, bool, 0444);
64

65
static bool __read_mostly flexpriority_enabled = 1;
66
module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
67

68
static bool __read_mostly enable_ept = 1;
69
module_param_named(ept, enable_ept, bool, S_IRUGO);
S
Sheng Yang 已提交
70

71
static bool __read_mostly enable_unrestricted_guest = 1;
72 73 74
module_param_named(unrestricted_guest,
			enable_unrestricted_guest, bool, S_IRUGO);

75 76 77
static bool __read_mostly enable_ept_ad_bits = 1;
module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);

78
static bool __read_mostly emulate_invalid_guest_state = true;
79
module_param(emulate_invalid_guest_state, bool, S_IRUGO);
80

81
static bool __read_mostly vmm_exclusive = 1;
82 83
module_param(vmm_exclusive, bool, S_IRUGO);

84
static bool __read_mostly fasteoi = 1;
85 86
module_param(fasteoi, bool, S_IRUGO);

87 88 89 90 91
/*
 * If nested=1, nested virtualization is supported, i.e., guests may use
 * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 * use VMX instructions.
 */
92
static bool __read_mostly nested = 0;
93 94
module_param(nested, bool, S_IRUGO);

95 96 97 98 99
#define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST				\
	(X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD)
#define KVM_GUEST_CR0_MASK						\
	(KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST				\
100
	(X86_CR0_WP | X86_CR0_NE)
101 102
#define KVM_VM_CR0_ALWAYS_ON						\
	(KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
103 104 105 106
#define KVM_CR4_GUEST_OWNED_BITS				      \
	(X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
	 | X86_CR4_OSXMMEXCPT)

107 108 109
#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)

110 111
#define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))

112 113 114 115
/*
 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 * ple_gap:    upper bound on the amount of time between two successive
 *             executions of PAUSE in a loop. Also indicate if ple enabled.
116
 *             According to test, this time is usually smaller than 128 cycles.
117 118 119 120 121 122
 * ple_window: upper bound on the amount of time a guest is allowed to execute
 *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 *             less than 2^12 cycles
 * Time is measured based on a counter that runs at the same rate as the TSC,
 * refer SDM volume 3b section 21.6.13 & 22.1.3.
 */
123
#define KVM_VMX_DEFAULT_PLE_GAP    128
124 125 126 127 128 129 130
#define KVM_VMX_DEFAULT_PLE_WINDOW 4096
static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
module_param(ple_gap, int, S_IRUGO);

static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
module_param(ple_window, int, S_IRUGO);

A
Avi Kivity 已提交
131 132
extern const ulong vmx_return;

133
#define NR_AUTOLOAD_MSRS 8
134
#define VMCS02_POOL_SIZE 1
135

136 137 138 139 140 141
struct vmcs {
	u32 revision_id;
	u32 abort;
	char data[0];
};

142 143 144 145 146 147 148 149 150 151 152 153
/*
 * Track a VMCS that may be loaded on a certain CPU. If it is (cpu!=-1), also
 * remember whether it was VMLAUNCHed, and maintain a linked list of all VMCSs
 * loaded on this CPU (so we can clear them if the CPU goes down).
 */
struct loaded_vmcs {
	struct vmcs *vmcs;
	int cpu;
	int launched;
	struct list_head loaded_vmcss_on_cpu_link;
};

154 155 156
struct shared_msr_entry {
	unsigned index;
	u64 data;
157
	u64 mask;
158 159
};

160 161 162 163 164 165 166 167 168 169 170 171 172
/*
 * struct vmcs12 describes the state that our guest hypervisor (L1) keeps for a
 * single nested guest (L2), hence the name vmcs12. Any VMX implementation has
 * a VMCS structure, and vmcs12 is our emulated VMX's VMCS. This structure is
 * stored in guest memory specified by VMPTRLD, but is opaque to the guest,
 * which must access it using VMREAD/VMWRITE/VMCLEAR instructions.
 * More than one of these structures may exist, if L1 runs multiple L2 guests.
 * nested_vmx_run() will use the data here to build a vmcs02: a VMCS for the
 * underlying hardware which will be used to run L2.
 * This structure is packed to ensure that its layout is identical across
 * machines (necessary for live migration).
 * If there are changes in this struct, VMCS12_REVISION must be changed.
 */
173
typedef u64 natural_width;
174 175 176 177 178 179
struct __packed vmcs12 {
	/* According to the Intel spec, a VMCS region must start with the
	 * following two fields. Then follow implementation-specific data.
	 */
	u32 revision_id;
	u32 abort;
180

N
Nadav Har'El 已提交
181 182 183
	u32 launch_state; /* set to 0 by VMCLEAR, to 1 by VMLAUNCH */
	u32 padding[7]; /* room for future expansion */

184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
	u64 io_bitmap_a;
	u64 io_bitmap_b;
	u64 msr_bitmap;
	u64 vm_exit_msr_store_addr;
	u64 vm_exit_msr_load_addr;
	u64 vm_entry_msr_load_addr;
	u64 tsc_offset;
	u64 virtual_apic_page_addr;
	u64 apic_access_addr;
	u64 ept_pointer;
	u64 guest_physical_address;
	u64 vmcs_link_pointer;
	u64 guest_ia32_debugctl;
	u64 guest_ia32_pat;
	u64 guest_ia32_efer;
	u64 guest_ia32_perf_global_ctrl;
	u64 guest_pdptr0;
	u64 guest_pdptr1;
	u64 guest_pdptr2;
	u64 guest_pdptr3;
	u64 host_ia32_pat;
	u64 host_ia32_efer;
	u64 host_ia32_perf_global_ctrl;
	u64 padding64[8]; /* room for future expansion */
	/*
	 * To allow migration of L1 (complete with its L2 guests) between
	 * machines of different natural widths (32 or 64 bit), we cannot have
	 * unsigned long fields with no explict size. We use u64 (aliased
	 * natural_width) instead. Luckily, x86 is little-endian.
	 */
	natural_width cr0_guest_host_mask;
	natural_width cr4_guest_host_mask;
	natural_width cr0_read_shadow;
	natural_width cr4_read_shadow;
	natural_width cr3_target_value0;
	natural_width cr3_target_value1;
	natural_width cr3_target_value2;
	natural_width cr3_target_value3;
	natural_width exit_qualification;
	natural_width guest_linear_address;
	natural_width guest_cr0;
	natural_width guest_cr3;
	natural_width guest_cr4;
	natural_width guest_es_base;
	natural_width guest_cs_base;
	natural_width guest_ss_base;
	natural_width guest_ds_base;
	natural_width guest_fs_base;
	natural_width guest_gs_base;
	natural_width guest_ldtr_base;
	natural_width guest_tr_base;
	natural_width guest_gdtr_base;
	natural_width guest_idtr_base;
	natural_width guest_dr7;
	natural_width guest_rsp;
	natural_width guest_rip;
	natural_width guest_rflags;
	natural_width guest_pending_dbg_exceptions;
	natural_width guest_sysenter_esp;
	natural_width guest_sysenter_eip;
	natural_width host_cr0;
	natural_width host_cr3;
	natural_width host_cr4;
	natural_width host_fs_base;
	natural_width host_gs_base;
	natural_width host_tr_base;
	natural_width host_gdtr_base;
	natural_width host_idtr_base;
	natural_width host_ia32_sysenter_esp;
	natural_width host_ia32_sysenter_eip;
	natural_width host_rsp;
	natural_width host_rip;
	natural_width paddingl[8]; /* room for future expansion */
	u32 pin_based_vm_exec_control;
	u32 cpu_based_vm_exec_control;
	u32 exception_bitmap;
	u32 page_fault_error_code_mask;
	u32 page_fault_error_code_match;
	u32 cr3_target_count;
	u32 vm_exit_controls;
	u32 vm_exit_msr_store_count;
	u32 vm_exit_msr_load_count;
	u32 vm_entry_controls;
	u32 vm_entry_msr_load_count;
	u32 vm_entry_intr_info_field;
	u32 vm_entry_exception_error_code;
	u32 vm_entry_instruction_len;
	u32 tpr_threshold;
	u32 secondary_vm_exec_control;
	u32 vm_instruction_error;
	u32 vm_exit_reason;
	u32 vm_exit_intr_info;
	u32 vm_exit_intr_error_code;
	u32 idt_vectoring_info_field;
	u32 idt_vectoring_error_code;
	u32 vm_exit_instruction_len;
	u32 vmx_instruction_info;
	u32 guest_es_limit;
	u32 guest_cs_limit;
	u32 guest_ss_limit;
	u32 guest_ds_limit;
	u32 guest_fs_limit;
	u32 guest_gs_limit;
	u32 guest_ldtr_limit;
	u32 guest_tr_limit;
	u32 guest_gdtr_limit;
	u32 guest_idtr_limit;
	u32 guest_es_ar_bytes;
	u32 guest_cs_ar_bytes;
	u32 guest_ss_ar_bytes;
	u32 guest_ds_ar_bytes;
	u32 guest_fs_ar_bytes;
	u32 guest_gs_ar_bytes;
	u32 guest_ldtr_ar_bytes;
	u32 guest_tr_ar_bytes;
	u32 guest_interruptibility_info;
	u32 guest_activity_state;
	u32 guest_sysenter_cs;
	u32 host_ia32_sysenter_cs;
	u32 padding32[8]; /* room for future expansion */
	u16 virtual_processor_id;
	u16 guest_es_selector;
	u16 guest_cs_selector;
	u16 guest_ss_selector;
	u16 guest_ds_selector;
	u16 guest_fs_selector;
	u16 guest_gs_selector;
	u16 guest_ldtr_selector;
	u16 guest_tr_selector;
	u16 host_es_selector;
	u16 host_cs_selector;
	u16 host_ss_selector;
	u16 host_ds_selector;
	u16 host_fs_selector;
	u16 host_gs_selector;
	u16 host_tr_selector;
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
};

/*
 * VMCS12_REVISION is an arbitrary id that should be changed if the content or
 * layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
 * VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
 */
#define VMCS12_REVISION 0x11e57ed0

/*
 * VMCS12_SIZE is the number of bytes L1 should allocate for the VMXON region
 * and any VMCS region. Although only sizeof(struct vmcs12) are used by the
 * current implementation, 4K are reserved to avoid future complications.
 */
#define VMCS12_SIZE 0x1000

336 337 338 339 340 341 342
/* Used to remember the last vmcs02 used for some recently used vmcs12s */
struct vmcs02_list {
	struct list_head list;
	gpa_t vmptr;
	struct loaded_vmcs vmcs02;
};

343 344 345 346 347 348 349
/*
 * The nested_vmx structure is part of vcpu_vmx, and holds information we need
 * for correct emulation of VMX (i.e., nested VMX) on this vcpu.
 */
struct nested_vmx {
	/* Has the level1 guest done vmxon? */
	bool vmxon;
350 351 352 353 354 355

	/* The guest-physical address of the current VMCS L1 keeps for L2 */
	gpa_t current_vmptr;
	/* The host-usable pointer to the above */
	struct page *current_vmcs12_page;
	struct vmcs12 *current_vmcs12;
356 357 358 359

	/* vmcs02_list cache of VMCSs recently used to run L2 guests */
	struct list_head vmcs02_pool;
	int vmcs02_num;
360
	u64 vmcs01_tsc_offset;
361 362
	/* L2 must run next, and mustn't decide to exit to L1. */
	bool nested_run_pending;
363 364 365 366 367
	/*
	 * Guest pages referred to in vmcs02 with host-physical pointers, so
	 * we must keep them pinned while L2 runs.
	 */
	struct page *apic_access_page;
368 369
};

370
struct vcpu_vmx {
R
Rusty Russell 已提交
371
	struct kvm_vcpu       vcpu;
372
	unsigned long         host_rsp;
373
	u8                    fail;
A
Avi Kivity 已提交
374
	u8                    cpl;
375
	bool                  nmi_known_unmasked;
376
	u32                   exit_intr_info;
377
	u32                   idt_vectoring_info;
A
Avi Kivity 已提交
378
	ulong                 rflags;
379
	struct shared_msr_entry *guest_msrs;
380 381 382
	int                   nmsrs;
	int                   save_nmsrs;
#ifdef CONFIG_X86_64
383 384
	u64 		      msr_host_kernel_gs_base;
	u64 		      msr_guest_kernel_gs_base;
385
#endif
386 387 388 389 390 391 392 393
	/*
	 * loaded_vmcs points to the VMCS currently used in this vcpu. For a
	 * non-nested (L1) guest, it always points to vmcs01. For a nested
	 * guest (L2), it points to a different VMCS.
	 */
	struct loaded_vmcs    vmcs01;
	struct loaded_vmcs   *loaded_vmcs;
	bool                  __launched; /* temporary, used in vmx_vcpu_run */
394 395 396 397 398
	struct msr_autoload {
		unsigned nr;
		struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS];
		struct vmx_msr_entry host[NR_AUTOLOAD_MSRS];
	} msr_autoload;
399 400 401
	struct {
		int           loaded;
		u16           fs_sel, gs_sel, ldt_sel;
A
Avi Kivity 已提交
402 403 404
#ifdef CONFIG_X86_64
		u16           ds_sel, es_sel;
#endif
405 406
		int           gs_ldt_reload_needed;
		int           fs_reload_needed;
M
Mike Day 已提交
407
	} host_state;
408
	struct {
409
		int vm86_active;
410
		ulong save_rflags;
411 412 413 414
		struct kvm_segment segs[8];
	} rmode;
	struct {
		u32 bitmask; /* 4 bits per segment (1 bit per field) */
415 416 417 418 419
		struct kvm_save_segment {
			u16 selector;
			unsigned long base;
			u32 limit;
			u32 ar;
420
		} seg[8];
A
Avi Kivity 已提交
421
	} segment_cache;
422
	int vpid;
423
	bool emulation_required;
424 425 426 427 428

	/* Support for vnmi-less CPUs */
	int soft_vnmi_blocked;
	ktime_t entry_time;
	s64 vnmi_blocked_time;
A
Andi Kleen 已提交
429
	u32 exit_reason;
430 431

	bool rdtscp_enabled;
432 433 434

	/* Support for a guest hypervisor (nested VMX) */
	struct nested_vmx nested;
435 436
};

A
Avi Kivity 已提交
437 438 439 440 441 442 443 444 445
enum segment_cache_field {
	SEG_FIELD_SEL = 0,
	SEG_FIELD_BASE = 1,
	SEG_FIELD_LIMIT = 2,
	SEG_FIELD_AR = 3,

	SEG_FIELD_NR = 4
};

446 447
static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
448
	return container_of(vcpu, struct vcpu_vmx, vcpu);
449 450
}

451 452 453 454 455
#define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
#define FIELD(number, name)	[number] = VMCS12_OFFSET(name)
#define FIELD64(number, name)	[number] = VMCS12_OFFSET(name), \
				[number##_HIGH] = VMCS12_OFFSET(name)+4

456
static const unsigned short vmcs_field_to_offset_table[] = {
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
	FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
	FIELD(GUEST_ES_SELECTOR, guest_es_selector),
	FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
	FIELD(GUEST_SS_SELECTOR, guest_ss_selector),
	FIELD(GUEST_DS_SELECTOR, guest_ds_selector),
	FIELD(GUEST_FS_SELECTOR, guest_fs_selector),
	FIELD(GUEST_GS_SELECTOR, guest_gs_selector),
	FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector),
	FIELD(GUEST_TR_SELECTOR, guest_tr_selector),
	FIELD(HOST_ES_SELECTOR, host_es_selector),
	FIELD(HOST_CS_SELECTOR, host_cs_selector),
	FIELD(HOST_SS_SELECTOR, host_ss_selector),
	FIELD(HOST_DS_SELECTOR, host_ds_selector),
	FIELD(HOST_FS_SELECTOR, host_fs_selector),
	FIELD(HOST_GS_SELECTOR, host_gs_selector),
	FIELD(HOST_TR_SELECTOR, host_tr_selector),
	FIELD64(IO_BITMAP_A, io_bitmap_a),
	FIELD64(IO_BITMAP_B, io_bitmap_b),
	FIELD64(MSR_BITMAP, msr_bitmap),
	FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
	FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
	FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
	FIELD64(TSC_OFFSET, tsc_offset),
	FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
	FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
	FIELD64(EPT_POINTER, ept_pointer),
	FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
	FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
	FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
	FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
	FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
	FIELD64(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl),
	FIELD64(GUEST_PDPTR0, guest_pdptr0),
	FIELD64(GUEST_PDPTR1, guest_pdptr1),
	FIELD64(GUEST_PDPTR2, guest_pdptr2),
	FIELD64(GUEST_PDPTR3, guest_pdptr3),
	FIELD64(HOST_IA32_PAT, host_ia32_pat),
	FIELD64(HOST_IA32_EFER, host_ia32_efer),
	FIELD64(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl),
	FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control),
	FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control),
	FIELD(EXCEPTION_BITMAP, exception_bitmap),
	FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask),
	FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match),
	FIELD(CR3_TARGET_COUNT, cr3_target_count),
	FIELD(VM_EXIT_CONTROLS, vm_exit_controls),
	FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count),
	FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count),
	FIELD(VM_ENTRY_CONTROLS, vm_entry_controls),
	FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count),
	FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field),
	FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, vm_entry_exception_error_code),
	FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len),
	FIELD(TPR_THRESHOLD, tpr_threshold),
	FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control),
	FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error),
	FIELD(VM_EXIT_REASON, vm_exit_reason),
	FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info),
	FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code),
	FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field),
	FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code),
	FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len),
	FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info),
	FIELD(GUEST_ES_LIMIT, guest_es_limit),
	FIELD(GUEST_CS_LIMIT, guest_cs_limit),
	FIELD(GUEST_SS_LIMIT, guest_ss_limit),
	FIELD(GUEST_DS_LIMIT, guest_ds_limit),
	FIELD(GUEST_FS_LIMIT, guest_fs_limit),
	FIELD(GUEST_GS_LIMIT, guest_gs_limit),
	FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit),
	FIELD(GUEST_TR_LIMIT, guest_tr_limit),
	FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit),
	FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit),
	FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes),
	FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes),
	FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes),
	FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes),
	FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes),
	FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes),
	FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes),
	FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes),
	FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info),
	FIELD(GUEST_ACTIVITY_STATE, guest_activity_state),
	FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs),
	FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs),
	FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask),
	FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask),
	FIELD(CR0_READ_SHADOW, cr0_read_shadow),
	FIELD(CR4_READ_SHADOW, cr4_read_shadow),
	FIELD(CR3_TARGET_VALUE0, cr3_target_value0),
	FIELD(CR3_TARGET_VALUE1, cr3_target_value1),
	FIELD(CR3_TARGET_VALUE2, cr3_target_value2),
	FIELD(CR3_TARGET_VALUE3, cr3_target_value3),
	FIELD(EXIT_QUALIFICATION, exit_qualification),
	FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address),
	FIELD(GUEST_CR0, guest_cr0),
	FIELD(GUEST_CR3, guest_cr3),
	FIELD(GUEST_CR4, guest_cr4),
	FIELD(GUEST_ES_BASE, guest_es_base),
	FIELD(GUEST_CS_BASE, guest_cs_base),
	FIELD(GUEST_SS_BASE, guest_ss_base),
	FIELD(GUEST_DS_BASE, guest_ds_base),
	FIELD(GUEST_FS_BASE, guest_fs_base),
	FIELD(GUEST_GS_BASE, guest_gs_base),
	FIELD(GUEST_LDTR_BASE, guest_ldtr_base),
	FIELD(GUEST_TR_BASE, guest_tr_base),
	FIELD(GUEST_GDTR_BASE, guest_gdtr_base),
	FIELD(GUEST_IDTR_BASE, guest_idtr_base),
	FIELD(GUEST_DR7, guest_dr7),
	FIELD(GUEST_RSP, guest_rsp),
	FIELD(GUEST_RIP, guest_rip),
	FIELD(GUEST_RFLAGS, guest_rflags),
	FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions),
	FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp),
	FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip),
	FIELD(HOST_CR0, host_cr0),
	FIELD(HOST_CR3, host_cr3),
	FIELD(HOST_CR4, host_cr4),
	FIELD(HOST_FS_BASE, host_fs_base),
	FIELD(HOST_GS_BASE, host_gs_base),
	FIELD(HOST_TR_BASE, host_tr_base),
	FIELD(HOST_GDTR_BASE, host_gdtr_base),
	FIELD(HOST_IDTR_BASE, host_idtr_base),
	FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp),
	FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip),
	FIELD(HOST_RSP, host_rsp),
	FIELD(HOST_RIP, host_rip),
};
static const int max_vmcs_field = ARRAY_SIZE(vmcs_field_to_offset_table);

static inline short vmcs_field_to_offset(unsigned long field)
{
	if (field >= max_vmcs_field || vmcs_field_to_offset_table[field] == 0)
		return -1;
	return vmcs_field_to_offset_table[field];
}

594 595 596 597 598 599 600 601
static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
{
	return to_vmx(vcpu)->nested.current_vmcs12;
}

static struct page *nested_get_page(struct kvm_vcpu *vcpu, gpa_t addr)
{
	struct page *page = gfn_to_page(vcpu->kvm, addr >> PAGE_SHIFT);
602
	if (is_error_page(page))
603
		return NULL;
604

605 606 607 608 609 610 611 612 613 614 615 616 617
	return page;
}

static void nested_release_page(struct page *page)
{
	kvm_release_page_dirty(page);
}

static void nested_release_page_clean(struct page *page)
{
	kvm_release_page_clean(page);
}

618
static u64 construct_eptp(unsigned long root_hpa);
619 620
static void kvm_cpu_vmxon(u64 addr);
static void kvm_cpu_vmxoff(void);
621
static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
622
static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
623 624 625 626
static void vmx_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg);
static void vmx_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg);
627

A
Avi Kivity 已提交
628 629
static DEFINE_PER_CPU(struct vmcs *, vmxarea);
static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
630 631 632 633 634
/*
 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
 */
static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
635
static DEFINE_PER_CPU(struct desc_ptr, host_gdt);
A
Avi Kivity 已提交
636

637 638
static unsigned long *vmx_io_bitmap_a;
static unsigned long *vmx_io_bitmap_b;
639 640
static unsigned long *vmx_msr_bitmap_legacy;
static unsigned long *vmx_msr_bitmap_longmode;
641

A
Avi Kivity 已提交
642
static bool cpu_has_load_ia32_efer;
643
static bool cpu_has_load_perf_global_ctrl;
A
Avi Kivity 已提交
644

645 646 647
static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
static DEFINE_SPINLOCK(vmx_vpid_lock);

648
static struct vmcs_config {
A
Avi Kivity 已提交
649 650 651
	int size;
	int order;
	u32 revision_id;
652 653
	u32 pin_based_exec_ctrl;
	u32 cpu_based_exec_ctrl;
654
	u32 cpu_based_2nd_exec_ctrl;
655 656 657
	u32 vmexit_ctrl;
	u32 vmentry_ctrl;
} vmcs_config;
A
Avi Kivity 已提交
658

H
Hannes Eder 已提交
659
static struct vmx_capability {
S
Sheng Yang 已提交
660 661 662 663
	u32 ept;
	u32 vpid;
} vmx_capability;

A
Avi Kivity 已提交
664 665 666 667 668 669 670 671
#define VMX_SEGMENT_FIELD(seg)					\
	[VCPU_SREG_##seg] = {                                   \
		.selector = GUEST_##seg##_SELECTOR,		\
		.base = GUEST_##seg##_BASE,		   	\
		.limit = GUEST_##seg##_LIMIT,		   	\
		.ar_bytes = GUEST_##seg##_AR_BYTES,	   	\
	}

672
static const struct kvm_vmx_segment_field {
A
Avi Kivity 已提交
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
	unsigned selector;
	unsigned base;
	unsigned limit;
	unsigned ar_bytes;
} kvm_vmx_segment_fields[] = {
	VMX_SEGMENT_FIELD(CS),
	VMX_SEGMENT_FIELD(DS),
	VMX_SEGMENT_FIELD(ES),
	VMX_SEGMENT_FIELD(FS),
	VMX_SEGMENT_FIELD(GS),
	VMX_SEGMENT_FIELD(SS),
	VMX_SEGMENT_FIELD(TR),
	VMX_SEGMENT_FIELD(LDTR),
};

688 689
static u64 host_efer;

A
Avi Kivity 已提交
690 691
static void ept_save_pdptrs(struct kvm_vcpu *vcpu);

692
/*
B
Brian Gerst 已提交
693
 * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
694 695
 * away by decrementing the array size.
 */
A
Avi Kivity 已提交
696
static const u32 vmx_msr_index[] = {
697
#ifdef CONFIG_X86_64
698
	MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
A
Avi Kivity 已提交
699
#endif
B
Brian Gerst 已提交
700
	MSR_EFER, MSR_TSC_AUX, MSR_STAR,
A
Avi Kivity 已提交
701
};
702
#define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
A
Avi Kivity 已提交
703

704
static inline bool is_page_fault(u32 intr_info)
A
Avi Kivity 已提交
705 706 707
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
708
		(INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
A
Avi Kivity 已提交
709 710
}

711
static inline bool is_no_device(u32 intr_info)
712 713 714
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
715
		(INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
716 717
}

718
static inline bool is_invalid_opcode(u32 intr_info)
719 720 721
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
722
		(INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK);
723 724
}

725
static inline bool is_external_interrupt(u32 intr_info)
A
Avi Kivity 已提交
726 727 728 729 730
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
		== (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
}

731
static inline bool is_machine_check(u32 intr_info)
A
Andi Kleen 已提交
732 733 734 735 736 737
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
		(INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
}

738
static inline bool cpu_has_vmx_msr_bitmap(void)
S
Sheng Yang 已提交
739
{
740
	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
S
Sheng Yang 已提交
741 742
}

743
static inline bool cpu_has_vmx_tpr_shadow(void)
744
{
745
	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
746 747
}

748
static inline bool vm_need_tpr_shadow(struct kvm *kvm)
749
{
750
	return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
751 752
}

753
static inline bool cpu_has_secondary_exec_ctrls(void)
754
{
755 756
	return vmcs_config.cpu_based_exec_ctrl &
		CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
757 758
}

759
static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
760
{
761 762 763 764 765 766 767 768
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
}

static inline bool cpu_has_vmx_flexpriority(void)
{
	return cpu_has_vmx_tpr_shadow() &&
		cpu_has_vmx_virtualize_apic_accesses();
769 770
}

771 772
static inline bool cpu_has_vmx_ept_execute_only(void)
{
773
	return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
774 775 776 777
}

static inline bool cpu_has_vmx_eptp_uncacheable(void)
{
778
	return vmx_capability.ept & VMX_EPTP_UC_BIT;
779 780 781 782
}

static inline bool cpu_has_vmx_eptp_writeback(void)
{
783
	return vmx_capability.ept & VMX_EPTP_WB_BIT;
784 785 786 787
}

static inline bool cpu_has_vmx_ept_2m_page(void)
{
788
	return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
789 790
}

791 792
static inline bool cpu_has_vmx_ept_1g_page(void)
{
793
	return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
794 795
}

796 797 798 799 800
static inline bool cpu_has_vmx_ept_4levels(void)
{
	return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
}

801 802 803 804 805
static inline bool cpu_has_vmx_ept_ad_bits(void)
{
	return vmx_capability.ept & VMX_EPT_AD_BIT;
}

806
static inline bool cpu_has_vmx_invept_context(void)
S
Sheng Yang 已提交
807
{
808
	return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
S
Sheng Yang 已提交
809 810
}

811
static inline bool cpu_has_vmx_invept_global(void)
S
Sheng Yang 已提交
812
{
813
	return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
S
Sheng Yang 已提交
814 815
}

816 817 818 819 820
static inline bool cpu_has_vmx_invvpid_single(void)
{
	return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
}

821 822 823 824 825
static inline bool cpu_has_vmx_invvpid_global(void)
{
	return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
}

826
static inline bool cpu_has_vmx_ept(void)
S
Sheng Yang 已提交
827
{
828 829
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_EPT;
S
Sheng Yang 已提交
830 831
}

832
static inline bool cpu_has_vmx_unrestricted_guest(void)
833 834 835 836 837
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_UNRESTRICTED_GUEST;
}

838
static inline bool cpu_has_vmx_ple(void)
839 840 841 842 843
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_PAUSE_LOOP_EXITING;
}

844
static inline bool vm_need_virtualize_apic_accesses(struct kvm *kvm)
845
{
846
	return flexpriority_enabled && irqchip_in_kernel(kvm);
847 848
}

849
static inline bool cpu_has_vmx_vpid(void)
850
{
851 852
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_VPID;
853 854
}

855
static inline bool cpu_has_vmx_rdtscp(void)
856 857 858 859 860
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_RDTSCP;
}

861 862 863 864 865 866
static inline bool cpu_has_vmx_invpcid(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_INVPCID;
}

867
static inline bool cpu_has_virtual_nmis(void)
868 869 870 871
{
	return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
}

872 873 874 875 876 877
static inline bool cpu_has_vmx_wbinvd_exit(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_WBINVD_EXITING;
}

878 879 880 881 882
static inline bool report_flexpriority(void)
{
	return flexpriority_enabled;
}

883 884 885 886 887 888 889 890 891 892 893 894
static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit)
{
	return vmcs12->cpu_based_vm_exec_control & bit;
}

static inline bool nested_cpu_has2(struct vmcs12 *vmcs12, u32 bit)
{
	return (vmcs12->cpu_based_vm_exec_control &
			CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
		(vmcs12->secondary_vm_exec_control & bit);
}

895 896 897 898 899 900 901 902 903 904 905 906 907
static inline bool nested_cpu_has_virtual_nmis(struct vmcs12 *vmcs12,
	struct kvm_vcpu *vcpu)
{
	return vmcs12->pin_based_vm_exec_control & PIN_BASED_VIRTUAL_NMIS;
}

static inline bool is_exception(u32 intr_info)
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
		== (INTR_TYPE_HARD_EXCEPTION | INTR_INFO_VALID_MASK);
}

static void nested_vmx_vmexit(struct kvm_vcpu *vcpu);
908 909 910 911
static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
			struct vmcs12 *vmcs12,
			u32 reason, unsigned long qualification);

R
Rusty Russell 已提交
912
static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
913 914 915
{
	int i;

916
	for (i = 0; i < vmx->nmsrs; ++i)
917
		if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
918 919 920 921
			return i;
	return -1;
}

922 923 924 925 926 927 928 929
static inline void __invvpid(int ext, u16 vpid, gva_t gva)
{
    struct {
	u64 vpid : 16;
	u64 rsvd : 48;
	u64 gva;
    } operand = { vpid, 0, gva };

930
    asm volatile (__ex(ASM_VMX_INVVPID)
931 932 933 934 935
		  /* CF==1 or ZF==1 --> rc = -1 */
		  "; ja 1f ; ud2 ; 1:"
		  : : "a"(&operand), "c"(ext) : "cc", "memory");
}

936 937 938 939 940 941
static inline void __invept(int ext, u64 eptp, gpa_t gpa)
{
	struct {
		u64 eptp, gpa;
	} operand = {eptp, gpa};

942
	asm volatile (__ex(ASM_VMX_INVEPT)
943 944 945 946 947
			/* CF==1 or ZF==1 --> rc = -1 */
			"; ja 1f ; ud2 ; 1:\n"
			: : "a" (&operand), "c" (ext) : "cc", "memory");
}

948
static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
949 950 951
{
	int i;

R
Rusty Russell 已提交
952
	i = __find_msr_index(vmx, msr);
953
	if (i >= 0)
954
		return &vmx->guest_msrs[i];
A
Al Viro 已提交
955
	return NULL;
956 957
}

A
Avi Kivity 已提交
958 959 960 961 962
static void vmcs_clear(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(vmcs);
	u8 error;

963
	asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
964
		      : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
A
Avi Kivity 已提交
965 966 967 968 969 970
		      : "cc", "memory");
	if (error)
		printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
		       vmcs, phys_addr);
}

971 972 973 974 975 976 977
static inline void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs)
{
	vmcs_clear(loaded_vmcs->vmcs);
	loaded_vmcs->cpu = -1;
	loaded_vmcs->launched = 0;
}

978 979 980 981 982 983
static void vmcs_load(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(vmcs);
	u8 error;

	asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
984
			: "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
985 986
			: "cc", "memory");
	if (error)
987
		printk(KERN_ERR "kvm: vmptrld %p/%llx failed\n",
988 989 990
		       vmcs, phys_addr);
}

991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
#ifdef CONFIG_KEXEC
/*
 * This bitmap is used to indicate whether the vmclear
 * operation is enabled on all cpus. All disabled by
 * default.
 */
static cpumask_t crash_vmclear_enabled_bitmap = CPU_MASK_NONE;

static inline void crash_enable_local_vmclear(int cpu)
{
	cpumask_set_cpu(cpu, &crash_vmclear_enabled_bitmap);
}

static inline void crash_disable_local_vmclear(int cpu)
{
	cpumask_clear_cpu(cpu, &crash_vmclear_enabled_bitmap);
}

static inline int crash_local_vmclear_enabled(int cpu)
{
	return cpumask_test_cpu(cpu, &crash_vmclear_enabled_bitmap);
}

static void crash_vmclear_local_loaded_vmcss(void)
{
	int cpu = raw_smp_processor_id();
	struct loaded_vmcs *v;

	if (!crash_local_vmclear_enabled(cpu))
		return;

	list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
			    loaded_vmcss_on_cpu_link)
		vmcs_clear(v->vmcs);
}
#else
static inline void crash_enable_local_vmclear(int cpu) { }
static inline void crash_disable_local_vmclear(int cpu) { }
#endif /* CONFIG_KEXEC */

1031
static void __loaded_vmcs_clear(void *arg)
A
Avi Kivity 已提交
1032
{
1033
	struct loaded_vmcs *loaded_vmcs = arg;
1034
	int cpu = raw_smp_processor_id();
A
Avi Kivity 已提交
1035

1036 1037 1038
	if (loaded_vmcs->cpu != cpu)
		return; /* vcpu migration can race with cpu offline */
	if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
A
Avi Kivity 已提交
1039
		per_cpu(current_vmcs, cpu) = NULL;
1040
	crash_disable_local_vmclear(cpu);
1041
	list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
1042 1043 1044 1045 1046 1047 1048 1049 1050

	/*
	 * we should ensure updating loaded_vmcs->loaded_vmcss_on_cpu_link
	 * is before setting loaded_vmcs->vcpu to -1 which is done in
	 * loaded_vmcs_init. Otherwise, other cpu can see vcpu = -1 fist
	 * then adds the vmcs into percpu list before it is deleted.
	 */
	smp_wmb();

1051
	loaded_vmcs_init(loaded_vmcs);
1052
	crash_enable_local_vmclear(cpu);
A
Avi Kivity 已提交
1053 1054
}

1055
static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
A
Avi Kivity 已提交
1056
{
1057 1058 1059 1060 1061
	int cpu = loaded_vmcs->cpu;

	if (cpu != -1)
		smp_call_function_single(cpu,
			 __loaded_vmcs_clear, loaded_vmcs, 1);
A
Avi Kivity 已提交
1062 1063
}

1064
static inline void vpid_sync_vcpu_single(struct vcpu_vmx *vmx)
1065 1066 1067 1068
{
	if (vmx->vpid == 0)
		return;

1069 1070
	if (cpu_has_vmx_invvpid_single())
		__invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vmx->vpid, 0);
1071 1072
}

1073 1074 1075 1076 1077 1078 1079 1080 1081
static inline void vpid_sync_vcpu_global(void)
{
	if (cpu_has_vmx_invvpid_global())
		__invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
}

static inline void vpid_sync_context(struct vcpu_vmx *vmx)
{
	if (cpu_has_vmx_invvpid_single())
1082
		vpid_sync_vcpu_single(vmx);
1083 1084 1085 1086
	else
		vpid_sync_vcpu_global();
}

1087 1088 1089 1090 1091 1092 1093 1094
static inline void ept_sync_global(void)
{
	if (cpu_has_vmx_invept_global())
		__invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
}

static inline void ept_sync_context(u64 eptp)
{
1095
	if (enable_ept) {
1096 1097 1098 1099 1100 1101 1102
		if (cpu_has_vmx_invept_context())
			__invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
		else
			ept_sync_global();
	}
}

A
Avi Kivity 已提交
1103
static __always_inline unsigned long vmcs_readl(unsigned long field)
A
Avi Kivity 已提交
1104
{
1105
	unsigned long value;
A
Avi Kivity 已提交
1106

1107 1108
	asm volatile (__ex_clear(ASM_VMX_VMREAD_RDX_RAX, "%0")
		      : "=a"(value) : "d"(field) : "cc");
A
Avi Kivity 已提交
1109 1110 1111
	return value;
}

A
Avi Kivity 已提交
1112
static __always_inline u16 vmcs_read16(unsigned long field)
A
Avi Kivity 已提交
1113 1114 1115 1116
{
	return vmcs_readl(field);
}

A
Avi Kivity 已提交
1117
static __always_inline u32 vmcs_read32(unsigned long field)
A
Avi Kivity 已提交
1118 1119 1120 1121
{
	return vmcs_readl(field);
}

A
Avi Kivity 已提交
1122
static __always_inline u64 vmcs_read64(unsigned long field)
A
Avi Kivity 已提交
1123
{
1124
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1125 1126 1127 1128 1129 1130
	return vmcs_readl(field);
#else
	return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
#endif
}

1131 1132 1133 1134 1135 1136 1137
static noinline void vmwrite_error(unsigned long field, unsigned long value)
{
	printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
	       field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
	dump_stack();
}

A
Avi Kivity 已提交
1138 1139 1140 1141
static void vmcs_writel(unsigned long field, unsigned long value)
{
	u8 error;

1142
	asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
M
Mike Day 已提交
1143
		       : "=q"(error) : "a"(value), "d"(field) : "cc");
1144 1145
	if (unlikely(error))
		vmwrite_error(field, value);
A
Avi Kivity 已提交
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
}

static void vmcs_write16(unsigned long field, u16 value)
{
	vmcs_writel(field, value);
}

static void vmcs_write32(unsigned long field, u32 value)
{
	vmcs_writel(field, value);
}

static void vmcs_write64(unsigned long field, u64 value)
{
	vmcs_writel(field, value);
1161
#ifndef CONFIG_X86_64
A
Avi Kivity 已提交
1162 1163 1164 1165 1166
	asm volatile ("");
	vmcs_writel(field+1, value >> 32);
#endif
}

1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
static void vmcs_clear_bits(unsigned long field, u32 mask)
{
	vmcs_writel(field, vmcs_readl(field) & ~mask);
}

static void vmcs_set_bits(unsigned long field, u32 mask)
{
	vmcs_writel(field, vmcs_readl(field) | mask);
}

A
Avi Kivity 已提交
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232
static void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
{
	vmx->segment_cache.bitmask = 0;
}

static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
				       unsigned field)
{
	bool ret;
	u32 mask = 1 << (seg * SEG_FIELD_NR + field);

	if (!(vmx->vcpu.arch.regs_avail & (1 << VCPU_EXREG_SEGMENTS))) {
		vmx->vcpu.arch.regs_avail |= (1 << VCPU_EXREG_SEGMENTS);
		vmx->segment_cache.bitmask = 0;
	}
	ret = vmx->segment_cache.bitmask & mask;
	vmx->segment_cache.bitmask |= mask;
	return ret;
}

static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
{
	u16 *p = &vmx->segment_cache.seg[seg].selector;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
		*p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
	return *p;
}

static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
{
	ulong *p = &vmx->segment_cache.seg[seg].base;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
		*p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
	return *p;
}

static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
{
	u32 *p = &vmx->segment_cache.seg[seg].limit;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
		*p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
	return *p;
}

static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
{
	u32 *p = &vmx->segment_cache.seg[seg].ar;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
		*p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
	return *p;
}

1233 1234 1235 1236
static void update_exception_bitmap(struct kvm_vcpu *vcpu)
{
	u32 eb;

J
Jan Kiszka 已提交
1237 1238 1239 1240 1241 1242
	eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
	     (1u << NM_VECTOR) | (1u << DB_VECTOR);
	if ((vcpu->guest_debug &
	     (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
	    (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
		eb |= 1u << BP_VECTOR;
1243
	if (to_vmx(vcpu)->rmode.vm86_active)
1244
		eb = ~0;
1245
	if (enable_ept)
1246
		eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
1247 1248
	if (vcpu->fpu_active)
		eb &= ~(1u << NM_VECTOR);
1249 1250 1251 1252 1253 1254 1255 1256 1257

	/* When we are running a nested L2 guest and L1 specified for it a
	 * certain exception bitmap, we must trap the same exceptions and pass
	 * them to L1. When running L2, we will only handle the exceptions
	 * specified above if L1 did not want them.
	 */
	if (is_guest_mode(vcpu))
		eb |= get_vmcs12(vcpu)->exception_bitmap;

1258 1259 1260
	vmcs_write32(EXCEPTION_BITMAP, eb);
}

1261 1262 1263 1264 1265 1266 1267
static void clear_atomic_switch_msr_special(unsigned long entry,
		unsigned long exit)
{
	vmcs_clear_bits(VM_ENTRY_CONTROLS, entry);
	vmcs_clear_bits(VM_EXIT_CONTROLS, exit);
}

1268 1269 1270 1271 1272
static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
{
	unsigned i;
	struct msr_autoload *m = &vmx->msr_autoload;

1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
	switch (msr) {
	case MSR_EFER:
		if (cpu_has_load_ia32_efer) {
			clear_atomic_switch_msr_special(VM_ENTRY_LOAD_IA32_EFER,
					VM_EXIT_LOAD_IA32_EFER);
			return;
		}
		break;
	case MSR_CORE_PERF_GLOBAL_CTRL:
		if (cpu_has_load_perf_global_ctrl) {
			clear_atomic_switch_msr_special(
					VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
					VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
			return;
		}
		break;
A
Avi Kivity 已提交
1289 1290
	}

1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
	for (i = 0; i < m->nr; ++i)
		if (m->guest[i].index == msr)
			break;

	if (i == m->nr)
		return;
	--m->nr;
	m->guest[i] = m->guest[m->nr];
	m->host[i] = m->host[m->nr];
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
}

1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
static void add_atomic_switch_msr_special(unsigned long entry,
		unsigned long exit, unsigned long guest_val_vmcs,
		unsigned long host_val_vmcs, u64 guest_val, u64 host_val)
{
	vmcs_write64(guest_val_vmcs, guest_val);
	vmcs_write64(host_val_vmcs, host_val);
	vmcs_set_bits(VM_ENTRY_CONTROLS, entry);
	vmcs_set_bits(VM_EXIT_CONTROLS, exit);
}

1314 1315 1316 1317 1318 1319
static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
				  u64 guest_val, u64 host_val)
{
	unsigned i;
	struct msr_autoload *m = &vmx->msr_autoload;

1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341
	switch (msr) {
	case MSR_EFER:
		if (cpu_has_load_ia32_efer) {
			add_atomic_switch_msr_special(VM_ENTRY_LOAD_IA32_EFER,
					VM_EXIT_LOAD_IA32_EFER,
					GUEST_IA32_EFER,
					HOST_IA32_EFER,
					guest_val, host_val);
			return;
		}
		break;
	case MSR_CORE_PERF_GLOBAL_CTRL:
		if (cpu_has_load_perf_global_ctrl) {
			add_atomic_switch_msr_special(
					VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
					VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
					GUEST_IA32_PERF_GLOBAL_CTRL,
					HOST_IA32_PERF_GLOBAL_CTRL,
					guest_val, host_val);
			return;
		}
		break;
A
Avi Kivity 已提交
1342 1343
	}

1344 1345 1346 1347
	for (i = 0; i < m->nr; ++i)
		if (m->guest[i].index == msr)
			break;

1348 1349 1350 1351 1352
	if (i == NR_AUTOLOAD_MSRS) {
		printk_once(KERN_WARNING"Not enough mst switch entries. "
				"Can't add msr %x\n", msr);
		return;
	} else if (i == m->nr) {
1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
		++m->nr;
		vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
		vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
	}

	m->guest[i].index = msr;
	m->guest[i].value = guest_val;
	m->host[i].index = msr;
	m->host[i].value = host_val;
}

1364 1365 1366 1367 1368
static void reload_tss(void)
{
	/*
	 * VT restores TR but not its size.  Useless.
	 */
1369
	struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
1370
	struct desc_struct *descs;
1371

1372
	descs = (void *)gdt->address;
1373 1374 1375 1376
	descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
	load_TR_desc();
}

A
Avi Kivity 已提交
1377
static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
1378
{
R
Roel Kluin 已提交
1379
	u64 guest_efer;
1380 1381
	u64 ignore_bits;

1382
	guest_efer = vmx->vcpu.arch.efer;
R
Roel Kluin 已提交
1383

1384
	/*
G
Guo Chao 已提交
1385
	 * NX is emulated; LMA and LME handled by hardware; SCE meaningless
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396
	 * outside long mode
	 */
	ignore_bits = EFER_NX | EFER_SCE;
#ifdef CONFIG_X86_64
	ignore_bits |= EFER_LMA | EFER_LME;
	/* SCE is meaningful only in long mode on Intel */
	if (guest_efer & EFER_LMA)
		ignore_bits &= ~(u64)EFER_SCE;
#endif
	guest_efer &= ~ignore_bits;
	guest_efer |= host_efer & ignore_bits;
1397
	vmx->guest_msrs[efer_offset].data = guest_efer;
1398
	vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409

	clear_atomic_switch_msr(vmx, MSR_EFER);
	/* On ept, can't emulate nx, and must switch nx atomically */
	if (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX)) {
		guest_efer = vmx->vcpu.arch.efer;
		if (!(guest_efer & EFER_LMA))
			guest_efer &= ~EFER_LME;
		add_atomic_switch_msr(vmx, MSR_EFER, guest_efer, host_efer);
		return false;
	}

1410
	return true;
1411 1412
}

1413 1414
static unsigned long segment_base(u16 selector)
{
1415
	struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
1416 1417 1418 1419 1420 1421 1422
	struct desc_struct *d;
	unsigned long table_base;
	unsigned long v;

	if (!(selector & ~3))
		return 0;

1423
	table_base = gdt->address;
1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448

	if (selector & 4) {           /* from ldt */
		u16 ldt_selector = kvm_read_ldt();

		if (!(ldt_selector & ~3))
			return 0;

		table_base = segment_base(ldt_selector);
	}
	d = (struct desc_struct *)(table_base + (selector & ~7));
	v = get_desc_base(d);
#ifdef CONFIG_X86_64
       if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
               v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
#endif
	return v;
}

static inline unsigned long kvm_read_tr_base(void)
{
	u16 tr;
	asm("str %0" : "=g"(tr));
	return segment_base(tr);
}

1449
static void vmx_save_host_state(struct kvm_vcpu *vcpu)
1450
{
1451
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1452
	int i;
1453

1454
	if (vmx->host_state.loaded)
1455 1456
		return;

1457
	vmx->host_state.loaded = 1;
1458 1459 1460 1461
	/*
	 * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
	 * allow segment selectors with cpl > 0 or ti == 1.
	 */
1462
	vmx->host_state.ldt_sel = kvm_read_ldt();
1463
	vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
1464
	savesegment(fs, vmx->host_state.fs_sel);
1465
	if (!(vmx->host_state.fs_sel & 7)) {
1466
		vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
1467 1468
		vmx->host_state.fs_reload_needed = 0;
	} else {
1469
		vmcs_write16(HOST_FS_SELECTOR, 0);
1470
		vmx->host_state.fs_reload_needed = 1;
1471
	}
1472
	savesegment(gs, vmx->host_state.gs_sel);
1473 1474
	if (!(vmx->host_state.gs_sel & 7))
		vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
1475 1476
	else {
		vmcs_write16(HOST_GS_SELECTOR, 0);
1477
		vmx->host_state.gs_ldt_reload_needed = 1;
1478 1479
	}

A
Avi Kivity 已提交
1480 1481 1482 1483 1484
#ifdef CONFIG_X86_64
	savesegment(ds, vmx->host_state.ds_sel);
	savesegment(es, vmx->host_state.es_sel);
#endif

1485 1486 1487 1488
#ifdef CONFIG_X86_64
	vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
	vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
#else
1489 1490
	vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
	vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
1491
#endif
1492 1493

#ifdef CONFIG_X86_64
1494 1495
	rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
	if (is_long_mode(&vmx->vcpu))
1496
		wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1497
#endif
1498 1499
	for (i = 0; i < vmx->save_nmsrs; ++i)
		kvm_set_shared_msr(vmx->guest_msrs[i].index,
1500 1501
				   vmx->guest_msrs[i].data,
				   vmx->guest_msrs[i].mask);
1502 1503
}

1504
static void __vmx_load_host_state(struct vcpu_vmx *vmx)
1505
{
1506
	if (!vmx->host_state.loaded)
1507 1508
		return;

1509
	++vmx->vcpu.stat.host_state_reload;
1510
	vmx->host_state.loaded = 0;
1511 1512 1513 1514
#ifdef CONFIG_X86_64
	if (is_long_mode(&vmx->vcpu))
		rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
#endif
1515
	if (vmx->host_state.gs_ldt_reload_needed) {
1516
		kvm_load_ldt(vmx->host_state.ldt_sel);
1517
#ifdef CONFIG_X86_64
1518 1519 1520
		load_gs_index(vmx->host_state.gs_sel);
#else
		loadsegment(gs, vmx->host_state.gs_sel);
1521 1522
#endif
	}
1523 1524
	if (vmx->host_state.fs_reload_needed)
		loadsegment(fs, vmx->host_state.fs_sel);
A
Avi Kivity 已提交
1525 1526 1527 1528 1529 1530
#ifdef CONFIG_X86_64
	if (unlikely(vmx->host_state.ds_sel | vmx->host_state.es_sel)) {
		loadsegment(ds, vmx->host_state.ds_sel);
		loadsegment(es, vmx->host_state.es_sel);
	}
#endif
1531
	reload_tss();
1532
#ifdef CONFIG_X86_64
1533
	wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1534
#endif
1535 1536 1537 1538 1539 1540
	/*
	 * If the FPU is not active (through the host task or
	 * the guest vcpu), then restore the cr0.TS bit.
	 */
	if (!user_has_fpu() && !vmx->vcpu.guest_fpu_loaded)
		stts();
1541
	load_gdt(&__get_cpu_var(host_gdt));
1542 1543
}

1544 1545 1546 1547 1548 1549 1550
static void vmx_load_host_state(struct vcpu_vmx *vmx)
{
	preempt_disable();
	__vmx_load_host_state(vmx);
	preempt_enable();
}

A
Avi Kivity 已提交
1551 1552 1553 1554
/*
 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
 * vcpu mutex is already taken.
 */
1555
static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
A
Avi Kivity 已提交
1556
{
1557
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1558
	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
A
Avi Kivity 已提交
1559

1560 1561
	if (!vmm_exclusive)
		kvm_cpu_vmxon(phys_addr);
1562 1563
	else if (vmx->loaded_vmcs->cpu != cpu)
		loaded_vmcs_clear(vmx->loaded_vmcs);
A
Avi Kivity 已提交
1564

1565 1566 1567
	if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
		per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
		vmcs_load(vmx->loaded_vmcs->vmcs);
A
Avi Kivity 已提交
1568 1569
	}

1570
	if (vmx->loaded_vmcs->cpu != cpu) {
1571
		struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
A
Avi Kivity 已提交
1572 1573
		unsigned long sysenter_esp;

1574
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1575
		local_irq_disable();
1576
		crash_disable_local_vmclear(cpu);
1577 1578 1579 1580 1581 1582 1583 1584

		/*
		 * Read loaded_vmcs->cpu should be before fetching
		 * loaded_vmcs->loaded_vmcss_on_cpu_link.
		 * See the comments in __loaded_vmcs_clear().
		 */
		smp_rmb();

1585 1586
		list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
			 &per_cpu(loaded_vmcss_on_cpu, cpu));
1587
		crash_enable_local_vmclear(cpu);
1588 1589
		local_irq_enable();

A
Avi Kivity 已提交
1590 1591 1592 1593
		/*
		 * Linux uses per-cpu TSS and GDT, so set these when switching
		 * processors.
		 */
1594
		vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
1595
		vmcs_writel(HOST_GDTR_BASE, gdt->address);   /* 22.2.4 */
A
Avi Kivity 已提交
1596 1597 1598

		rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
		vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
1599
		vmx->loaded_vmcs->cpu = cpu;
A
Avi Kivity 已提交
1600 1601 1602 1603 1604
	}
}

static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
{
1605
	__vmx_load_host_state(to_vmx(vcpu));
1606
	if (!vmm_exclusive) {
1607 1608
		__loaded_vmcs_clear(to_vmx(vcpu)->loaded_vmcs);
		vcpu->cpu = -1;
1609 1610
		kvm_cpu_vmxoff();
	}
A
Avi Kivity 已提交
1611 1612
}

1613 1614
static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
{
1615 1616
	ulong cr0;

1617 1618 1619
	if (vcpu->fpu_active)
		return;
	vcpu->fpu_active = 1;
1620 1621 1622 1623
	cr0 = vmcs_readl(GUEST_CR0);
	cr0 &= ~(X86_CR0_TS | X86_CR0_MP);
	cr0 |= kvm_read_cr0_bits(vcpu, X86_CR0_TS | X86_CR0_MP);
	vmcs_writel(GUEST_CR0, cr0);
1624
	update_exception_bitmap(vcpu);
1625
	vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
1626 1627 1628
	if (is_guest_mode(vcpu))
		vcpu->arch.cr0_guest_owned_bits &=
			~get_vmcs12(vcpu)->cr0_guest_host_mask;
1629
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
1630 1631
}

1632 1633
static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);

1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649
/*
 * Return the cr0 value that a nested guest would read. This is a combination
 * of the real cr0 used to run the guest (guest_cr0), and the bits shadowed by
 * its hypervisor (cr0_read_shadow).
 */
static inline unsigned long nested_read_cr0(struct vmcs12 *fields)
{
	return (fields->guest_cr0 & ~fields->cr0_guest_host_mask) |
		(fields->cr0_read_shadow & fields->cr0_guest_host_mask);
}
static inline unsigned long nested_read_cr4(struct vmcs12 *fields)
{
	return (fields->guest_cr4 & ~fields->cr4_guest_host_mask) |
		(fields->cr4_read_shadow & fields->cr4_guest_host_mask);
}

1650 1651
static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
{
1652 1653 1654
	/* Note that there is no vcpu->fpu_active = 0 here. The caller must
	 * set this *before* calling this function.
	 */
1655
	vmx_decache_cr0_guest_bits(vcpu);
1656
	vmcs_set_bits(GUEST_CR0, X86_CR0_TS | X86_CR0_MP);
1657
	update_exception_bitmap(vcpu);
1658 1659
	vcpu->arch.cr0_guest_owned_bits = 0;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674
	if (is_guest_mode(vcpu)) {
		/*
		 * L1's specified read shadow might not contain the TS bit,
		 * so now that we turned on shadowing of this bit, we need to
		 * set this bit of the shadow. Like in nested_vmx_run we need
		 * nested_read_cr0(vmcs12), but vmcs12->guest_cr0 is not yet
		 * up-to-date here because we just decached cr0.TS (and we'll
		 * only update vmcs12->guest_cr0 on nested exit).
		 */
		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
		vmcs12->guest_cr0 = (vmcs12->guest_cr0 & ~X86_CR0_TS) |
			(vcpu->arch.cr0 & X86_CR0_TS);
		vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
	} else
		vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
1675 1676
}

A
Avi Kivity 已提交
1677 1678
static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
{
1679
	unsigned long rflags, save_rflags;
1680

A
Avi Kivity 已提交
1681 1682 1683 1684 1685 1686 1687 1688 1689
	if (!test_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail)) {
		__set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
		rflags = vmcs_readl(GUEST_RFLAGS);
		if (to_vmx(vcpu)->rmode.vm86_active) {
			rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
			save_rflags = to_vmx(vcpu)->rmode.save_rflags;
			rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
		}
		to_vmx(vcpu)->rflags = rflags;
1690
	}
A
Avi Kivity 已提交
1691
	return to_vmx(vcpu)->rflags;
A
Avi Kivity 已提交
1692 1693 1694 1695
}

static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
A
Avi Kivity 已提交
1696
	__set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
A
Avi Kivity 已提交
1697
	__clear_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
A
Avi Kivity 已提交
1698
	to_vmx(vcpu)->rflags = rflags;
1699 1700
	if (to_vmx(vcpu)->rmode.vm86_active) {
		to_vmx(vcpu)->rmode.save_rflags = rflags;
1701
		rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
1702
	}
A
Avi Kivity 已提交
1703 1704 1705
	vmcs_writel(GUEST_RFLAGS, rflags);
}

1706 1707 1708 1709 1710 1711
static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	int ret = 0;

	if (interruptibility & GUEST_INTR_STATE_STI)
1712
		ret |= KVM_X86_SHADOW_INT_STI;
1713
	if (interruptibility & GUEST_INTR_STATE_MOV_SS)
1714
		ret |= KVM_X86_SHADOW_INT_MOV_SS;
1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725

	return ret & mask;
}

static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	u32 interruptibility = interruptibility_old;

	interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);

1726
	if (mask & KVM_X86_SHADOW_INT_MOV_SS)
1727
		interruptibility |= GUEST_INTR_STATE_MOV_SS;
1728
	else if (mask & KVM_X86_SHADOW_INT_STI)
1729 1730 1731 1732 1733 1734
		interruptibility |= GUEST_INTR_STATE_STI;

	if ((interruptibility != interruptibility_old))
		vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
}

A
Avi Kivity 已提交
1735 1736 1737 1738
static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
	unsigned long rip;

1739
	rip = kvm_rip_read(vcpu);
A
Avi Kivity 已提交
1740
	rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
1741
	kvm_rip_write(vcpu, rip);
A
Avi Kivity 已提交
1742

1743 1744
	/* skipping an emulated instruction also counts */
	vmx_set_interrupt_shadow(vcpu, 0);
A
Avi Kivity 已提交
1745 1746
}

1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758
/*
 * KVM wants to inject page-faults which it got to the guest. This function
 * checks whether in a nested guest, we need to inject them to L1 or L2.
 * This function assumes it is called with the exit reason in vmcs02 being
 * a #PF exception (this is the only case in which KVM injects a #PF when L2
 * is running).
 */
static int nested_pf_handled(struct kvm_vcpu *vcpu)
{
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

	/* TODO: also check PFEC_MATCH/MASK, not just EB.PF. */
1759
	if (!(vmcs12->exception_bitmap & (1u << PF_VECTOR)))
1760 1761 1762 1763 1764 1765
		return 0;

	nested_vmx_vmexit(vcpu);
	return 1;
}

1766
static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
1767 1768
				bool has_error_code, u32 error_code,
				bool reinject)
1769
{
1770
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1771
	u32 intr_info = nr | INTR_INFO_VALID_MASK;
1772

1773 1774 1775 1776
	if (nr == PF_VECTOR && is_guest_mode(vcpu) &&
		nested_pf_handled(vcpu))
		return;

1777
	if (has_error_code) {
1778
		vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
1779 1780
		intr_info |= INTR_INFO_DELIVER_CODE_MASK;
	}
1781

1782
	if (vmx->rmode.vm86_active) {
1783 1784 1785 1786
		int inc_eip = 0;
		if (kvm_exception_is_soft(nr))
			inc_eip = vcpu->arch.event_exit_inst_len;
		if (kvm_inject_realmode_interrupt(vcpu, nr, inc_eip) != EMULATE_DONE)
1787
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1788 1789 1790
		return;
	}

1791 1792 1793
	if (kvm_exception_is_soft(nr)) {
		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
			     vmx->vcpu.arch.event_exit_inst_len);
1794 1795 1796 1797 1798
		intr_info |= INTR_TYPE_SOFT_EXCEPTION;
	} else
		intr_info |= INTR_TYPE_HARD_EXCEPTION;

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
1799 1800
}

1801 1802 1803 1804 1805
static bool vmx_rdtscp_supported(void)
{
	return cpu_has_vmx_rdtscp();
}

1806 1807 1808 1809 1810
static bool vmx_invpcid_supported(void)
{
	return cpu_has_vmx_invpcid() && enable_ept;
}

1811 1812 1813
/*
 * Swap MSR entry in host/guest MSR entry array.
 */
R
Rusty Russell 已提交
1814
static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
1815
{
1816
	struct shared_msr_entry tmp;
1817 1818 1819 1820

	tmp = vmx->guest_msrs[to];
	vmx->guest_msrs[to] = vmx->guest_msrs[from];
	vmx->guest_msrs[from] = tmp;
1821 1822
}

1823 1824 1825 1826 1827
/*
 * Set up the vmcs to automatically save and restore system
 * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
 * mode, as fiddling with msrs is very expensive.
 */
R
Rusty Russell 已提交
1828
static void setup_msrs(struct vcpu_vmx *vmx)
1829
{
1830
	int save_nmsrs, index;
1831
	unsigned long *msr_bitmap;
1832

1833 1834
	save_nmsrs = 0;
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
1835 1836
	if (is_long_mode(&vmx->vcpu)) {
		index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
1837
		if (index >= 0)
R
Rusty Russell 已提交
1838 1839
			move_msr_up(vmx, index, save_nmsrs++);
		index = __find_msr_index(vmx, MSR_LSTAR);
1840
		if (index >= 0)
R
Rusty Russell 已提交
1841 1842
			move_msr_up(vmx, index, save_nmsrs++);
		index = __find_msr_index(vmx, MSR_CSTAR);
1843
		if (index >= 0)
R
Rusty Russell 已提交
1844
			move_msr_up(vmx, index, save_nmsrs++);
1845 1846 1847
		index = __find_msr_index(vmx, MSR_TSC_AUX);
		if (index >= 0 && vmx->rdtscp_enabled)
			move_msr_up(vmx, index, save_nmsrs++);
1848
		/*
B
Brian Gerst 已提交
1849
		 * MSR_STAR is only needed on long mode guests, and only
1850 1851
		 * if efer.sce is enabled.
		 */
B
Brian Gerst 已提交
1852
		index = __find_msr_index(vmx, MSR_STAR);
1853
		if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
R
Rusty Russell 已提交
1854
			move_msr_up(vmx, index, save_nmsrs++);
1855 1856
	}
#endif
A
Avi Kivity 已提交
1857 1858
	index = __find_msr_index(vmx, MSR_EFER);
	if (index >= 0 && update_transition_efer(vmx, index))
1859
		move_msr_up(vmx, index, save_nmsrs++);
1860

1861
	vmx->save_nmsrs = save_nmsrs;
1862 1863 1864 1865 1866 1867 1868 1869 1870

	if (cpu_has_vmx_msr_bitmap()) {
		if (is_long_mode(&vmx->vcpu))
			msr_bitmap = vmx_msr_bitmap_longmode;
		else
			msr_bitmap = vmx_msr_bitmap_legacy;

		vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
	}
1871 1872
}

A
Avi Kivity 已提交
1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
/*
 * reads and returns guest's timestamp counter "register"
 * guest_tsc = host_tsc + tsc_offset    -- 21.3
 */
static u64 guest_read_tsc(void)
{
	u64 host_tsc, tsc_offset;

	rdtscll(host_tsc);
	tsc_offset = vmcs_read64(TSC_OFFSET);
	return host_tsc + tsc_offset;
}

N
Nadav Har'El 已提交
1886 1887 1888 1889
/*
 * Like guest_read_tsc, but always returns L1's notion of the timestamp
 * counter, even if a nested guest (L2) is currently running.
 */
1890
u64 vmx_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
N
Nadav Har'El 已提交
1891
{
1892
	u64 tsc_offset;
N
Nadav Har'El 已提交
1893 1894 1895 1896 1897 1898 1899

	tsc_offset = is_guest_mode(vcpu) ?
		to_vmx(vcpu)->nested.vmcs01_tsc_offset :
		vmcs_read64(TSC_OFFSET);
	return host_tsc + tsc_offset;
}

1900
/*
1901 1902
 * Engage any workarounds for mis-matched TSC rates.  Currently limited to
 * software catchup for faster rates on slower CPUs.
1903
 */
1904
static void vmx_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
1905
{
1906 1907 1908 1909 1910 1911 1912 1913
	if (!scale)
		return;

	if (user_tsc_khz > tsc_khz) {
		vcpu->arch.tsc_catchup = 1;
		vcpu->arch.tsc_always_catchup = 1;
	} else
		WARN(1, "user requested TSC rate below hardware speed\n");
1914 1915
}

W
Will Auld 已提交
1916 1917 1918 1919 1920
static u64 vmx_read_tsc_offset(struct kvm_vcpu *vcpu)
{
	return vmcs_read64(TSC_OFFSET);
}

A
Avi Kivity 已提交
1921
/*
1922
 * writes 'offset' into guest's timestamp counter offset register
A
Avi Kivity 已提交
1923
 */
1924
static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
A
Avi Kivity 已提交
1925
{
1926
	if (is_guest_mode(vcpu)) {
1927
		/*
1928 1929 1930 1931
		 * We're here if L1 chose not to trap WRMSR to TSC. According
		 * to the spec, this should set L1's TSC; The offset that L1
		 * set for L2 remains unchanged, and still needs to be added
		 * to the newly set TSC to get L2's TSC.
1932
		 */
1933 1934 1935 1936 1937 1938 1939 1940 1941 1942
		struct vmcs12 *vmcs12;
		to_vmx(vcpu)->nested.vmcs01_tsc_offset = offset;
		/* recalculate vmcs02.TSC_OFFSET: */
		vmcs12 = get_vmcs12(vcpu);
		vmcs_write64(TSC_OFFSET, offset +
			(nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETING) ?
			 vmcs12->tsc_offset : 0));
	} else {
		vmcs_write64(TSC_OFFSET, offset);
	}
A
Avi Kivity 已提交
1943 1944
}

1945
static void vmx_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment, bool host)
Z
Zachary Amsden 已提交
1946 1947 1948
{
	u64 offset = vmcs_read64(TSC_OFFSET);
	vmcs_write64(TSC_OFFSET, offset + adjustment);
1949 1950 1951 1952
	if (is_guest_mode(vcpu)) {
		/* Even when running L2, the adjustment needs to apply to L1 */
		to_vmx(vcpu)->nested.vmcs01_tsc_offset += adjustment;
	}
Z
Zachary Amsden 已提交
1953 1954
}

1955 1956 1957 1958 1959
static u64 vmx_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
{
	return target_tsc - native_read_tsc();
}

1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976
static bool guest_cpuid_has_vmx(struct kvm_vcpu *vcpu)
{
	struct kvm_cpuid_entry2 *best = kvm_find_cpuid_entry(vcpu, 1, 0);
	return best && (best->ecx & (1 << (X86_FEATURE_VMX & 31)));
}

/*
 * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
 * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
 * all guests if the "nested" module option is off, and can also be disabled
 * for a single guest by disabling its VMX cpuid bit.
 */
static inline bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
{
	return nested && guest_cpuid_has_vmx(vcpu);
}

1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022
/*
 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
 * returned for the various VMX controls MSRs when nested VMX is enabled.
 * The same values should also be used to verify that vmcs12 control fields are
 * valid during nested entry from L1 to L2.
 * Each of these control msrs has a low and high 32-bit half: A low bit is on
 * if the corresponding bit in the (32-bit) control field *must* be on, and a
 * bit in the high half is on if the corresponding bit in the control field
 * may be on. See also vmx_control_verify().
 * TODO: allow these variables to be modified (downgraded) by module options
 * or other means.
 */
static u32 nested_vmx_procbased_ctls_low, nested_vmx_procbased_ctls_high;
static u32 nested_vmx_secondary_ctls_low, nested_vmx_secondary_ctls_high;
static u32 nested_vmx_pinbased_ctls_low, nested_vmx_pinbased_ctls_high;
static u32 nested_vmx_exit_ctls_low, nested_vmx_exit_ctls_high;
static u32 nested_vmx_entry_ctls_low, nested_vmx_entry_ctls_high;
static __init void nested_vmx_setup_ctls_msrs(void)
{
	/*
	 * Note that as a general rule, the high half of the MSRs (bits in
	 * the control fields which may be 1) should be initialized by the
	 * intersection of the underlying hardware's MSR (i.e., features which
	 * can be supported) and the list of features we want to expose -
	 * because they are known to be properly supported in our code.
	 * Also, usually, the low half of the MSRs (bits which must be 1) can
	 * be set to 0, meaning that L1 may turn off any of these bits. The
	 * reason is that if one of these bits is necessary, it will appear
	 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
	 * fields of vmcs01 and vmcs02, will turn these bits off - and
	 * nested_vmx_exit_handled() will not pass related exits to L1.
	 * These rules have exceptions below.
	 */

	/* pin-based controls */
	/*
	 * According to the Intel spec, if bit 55 of VMX_BASIC is off (as it is
	 * in our case), bits 1, 2 and 4 (i.e., 0x16) must be 1 in this MSR.
	 */
	nested_vmx_pinbased_ctls_low = 0x16 ;
	nested_vmx_pinbased_ctls_high = 0x16 |
		PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING |
		PIN_BASED_VIRTUAL_NMIS;

	/* exit controls */
	nested_vmx_exit_ctls_low = 0;
2023
	/* Note that guest use of VM_EXIT_ACK_INTR_ON_EXIT is not supported. */
2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
#ifdef CONFIG_X86_64
	nested_vmx_exit_ctls_high = VM_EXIT_HOST_ADDR_SPACE_SIZE;
#else
	nested_vmx_exit_ctls_high = 0;
#endif

	/* entry controls */
	rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
		nested_vmx_entry_ctls_low, nested_vmx_entry_ctls_high);
	nested_vmx_entry_ctls_low = 0;
	nested_vmx_entry_ctls_high &=
		VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_IA32E_MODE;

	/* cpu-based controls */
	rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
		nested_vmx_procbased_ctls_low, nested_vmx_procbased_ctls_high);
	nested_vmx_procbased_ctls_low = 0;
	nested_vmx_procbased_ctls_high &=
		CPU_BASED_VIRTUAL_INTR_PENDING | CPU_BASED_USE_TSC_OFFSETING |
		CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
		CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
		CPU_BASED_CR3_STORE_EXITING |
#ifdef CONFIG_X86_64
		CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
#endif
		CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
		CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_EXITING |
2051
		CPU_BASED_RDPMC_EXITING | CPU_BASED_RDTSC_EXITING |
2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
		CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
	/*
	 * We can allow some features even when not supported by the
	 * hardware. For example, L1 can specify an MSR bitmap - and we
	 * can use it to avoid exits to L1 - even when L0 runs L2
	 * without MSR bitmaps.
	 */
	nested_vmx_procbased_ctls_high |= CPU_BASED_USE_MSR_BITMAPS;

	/* secondary cpu-based controls */
	rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
		nested_vmx_secondary_ctls_low, nested_vmx_secondary_ctls_high);
	nested_vmx_secondary_ctls_low = 0;
	nested_vmx_secondary_ctls_high &=
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
}

static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
{
	/*
	 * Bits 0 in high must be 0, and bits 1 in low must be 1.
	 */
	return ((control & high) | low) == control;
}

static inline u64 vmx_control_msr(u32 low, u32 high)
{
	return low | ((u64)high << 32);
}

/*
 * If we allow our guest to use VMX instructions (i.e., nested VMX), we should
 * also let it use VMX-specific MSRs.
 * vmx_get_vmx_msr() and vmx_set_vmx_msr() return 1 when we handled a
 * VMX-specific MSR, or 0 when we haven't (and the caller should handle it
 * like all other MSRs).
 */
static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
	if (!nested_vmx_allowed(vcpu) && msr_index >= MSR_IA32_VMX_BASIC &&
		     msr_index <= MSR_IA32_VMX_TRUE_ENTRY_CTLS) {
		/*
		 * According to the spec, processors which do not support VMX
		 * should throw a #GP(0) when VMX capability MSRs are read.
		 */
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
		return 1;
	}

	switch (msr_index) {
	case MSR_IA32_FEATURE_CONTROL:
		*pdata = 0;
		break;
	case MSR_IA32_VMX_BASIC:
		/*
		 * This MSR reports some information about VMX support. We
		 * should return information about the VMX we emulate for the
		 * guest, and the VMCS structure we give it - not about the
		 * VMX support of the underlying hardware.
		 */
		*pdata = VMCS12_REVISION |
			   ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
			   (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
		break;
	case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
	case MSR_IA32_VMX_PINBASED_CTLS:
		*pdata = vmx_control_msr(nested_vmx_pinbased_ctls_low,
					nested_vmx_pinbased_ctls_high);
		break;
	case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
	case MSR_IA32_VMX_PROCBASED_CTLS:
		*pdata = vmx_control_msr(nested_vmx_procbased_ctls_low,
					nested_vmx_procbased_ctls_high);
		break;
	case MSR_IA32_VMX_TRUE_EXIT_CTLS:
	case MSR_IA32_VMX_EXIT_CTLS:
		*pdata = vmx_control_msr(nested_vmx_exit_ctls_low,
					nested_vmx_exit_ctls_high);
		break;
	case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
	case MSR_IA32_VMX_ENTRY_CTLS:
		*pdata = vmx_control_msr(nested_vmx_entry_ctls_low,
					nested_vmx_entry_ctls_high);
		break;
	case MSR_IA32_VMX_MISC:
		*pdata = 0;
		break;
	/*
	 * These MSRs specify bits which the guest must keep fixed (on or off)
	 * while L1 is in VMXON mode (in L1's root mode, or running an L2).
	 * We picked the standard core2 setting.
	 */
#define VMXON_CR0_ALWAYSON	(X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
#define VMXON_CR4_ALWAYSON	X86_CR4_VMXE
	case MSR_IA32_VMX_CR0_FIXED0:
		*pdata = VMXON_CR0_ALWAYSON;
		break;
	case MSR_IA32_VMX_CR0_FIXED1:
		*pdata = -1ULL;
		break;
	case MSR_IA32_VMX_CR4_FIXED0:
		*pdata = VMXON_CR4_ALWAYSON;
		break;
	case MSR_IA32_VMX_CR4_FIXED1:
		*pdata = -1ULL;
		break;
	case MSR_IA32_VMX_VMCS_ENUM:
		*pdata = 0x1f;
		break;
	case MSR_IA32_VMX_PROCBASED_CTLS2:
		*pdata = vmx_control_msr(nested_vmx_secondary_ctls_low,
					nested_vmx_secondary_ctls_high);
		break;
	case MSR_IA32_VMX_EPT_VPID_CAP:
		/* Currently, no nested ept or nested vpid */
		*pdata = 0;
		break;
	default:
		return 0;
	}

	return 1;
}

static int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
{
	if (!nested_vmx_allowed(vcpu))
		return 0;

	if (msr_index == MSR_IA32_FEATURE_CONTROL)
		/* TODO: the right thing. */
		return 1;
	/*
	 * No need to treat VMX capability MSRs specially: If we don't handle
	 * them, handle_wrmsr will #GP(0), which is correct (they are readonly)
	 */
	return 0;
}

A
Avi Kivity 已提交
2191 2192 2193 2194 2195 2196 2197 2198
/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
	u64 data;
2199
	struct shared_msr_entry *msr;
A
Avi Kivity 已提交
2200 2201 2202 2203 2204 2205 2206

	if (!pdata) {
		printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
		return -EINVAL;
	}

	switch (msr_index) {
2207
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2208 2209 2210 2211 2212 2213
	case MSR_FS_BASE:
		data = vmcs_readl(GUEST_FS_BASE);
		break;
	case MSR_GS_BASE:
		data = vmcs_readl(GUEST_GS_BASE);
		break;
2214 2215 2216 2217
	case MSR_KERNEL_GS_BASE:
		vmx_load_host_state(to_vmx(vcpu));
		data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
		break;
2218
#endif
A
Avi Kivity 已提交
2219
	case MSR_EFER:
2220
		return kvm_get_msr_common(vcpu, msr_index, pdata);
2221
	case MSR_IA32_TSC:
A
Avi Kivity 已提交
2222 2223 2224 2225 2226 2227
		data = guest_read_tsc();
		break;
	case MSR_IA32_SYSENTER_CS:
		data = vmcs_read32(GUEST_SYSENTER_CS);
		break;
	case MSR_IA32_SYSENTER_EIP:
A
Avi Kivity 已提交
2228
		data = vmcs_readl(GUEST_SYSENTER_EIP);
A
Avi Kivity 已提交
2229 2230
		break;
	case MSR_IA32_SYSENTER_ESP:
A
Avi Kivity 已提交
2231
		data = vmcs_readl(GUEST_SYSENTER_ESP);
A
Avi Kivity 已提交
2232
		break;
2233 2234 2235 2236
	case MSR_TSC_AUX:
		if (!to_vmx(vcpu)->rdtscp_enabled)
			return 1;
		/* Otherwise falls through */
A
Avi Kivity 已提交
2237
	default:
2238 2239
		if (vmx_get_vmx_msr(vcpu, msr_index, pdata))
			return 0;
R
Rusty Russell 已提交
2240
		msr = find_msr_entry(to_vmx(vcpu), msr_index);
2241 2242 2243
		if (msr) {
			data = msr->data;
			break;
A
Avi Kivity 已提交
2244
		}
2245
		return kvm_get_msr_common(vcpu, msr_index, pdata);
A
Avi Kivity 已提交
2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
	}

	*pdata = data;
	return 0;
}

/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
2257
static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
A
Avi Kivity 已提交
2258
{
2259
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2260
	struct shared_msr_entry *msr;
2261
	int ret = 0;
2262 2263
	u32 msr_index = msr_info->index;
	u64 data = msr_info->data;
2264

A
Avi Kivity 已提交
2265
	switch (msr_index) {
2266
	case MSR_EFER:
2267
		ret = kvm_set_msr_common(vcpu, msr_info);
2268
		break;
2269
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2270
	case MSR_FS_BASE:
A
Avi Kivity 已提交
2271
		vmx_segment_cache_clear(vmx);
A
Avi Kivity 已提交
2272 2273 2274
		vmcs_writel(GUEST_FS_BASE, data);
		break;
	case MSR_GS_BASE:
A
Avi Kivity 已提交
2275
		vmx_segment_cache_clear(vmx);
A
Avi Kivity 已提交
2276 2277
		vmcs_writel(GUEST_GS_BASE, data);
		break;
2278 2279 2280 2281
	case MSR_KERNEL_GS_BASE:
		vmx_load_host_state(vmx);
		vmx->msr_guest_kernel_gs_base = data;
		break;
A
Avi Kivity 已提交
2282 2283 2284 2285 2286
#endif
	case MSR_IA32_SYSENTER_CS:
		vmcs_write32(GUEST_SYSENTER_CS, data);
		break;
	case MSR_IA32_SYSENTER_EIP:
A
Avi Kivity 已提交
2287
		vmcs_writel(GUEST_SYSENTER_EIP, data);
A
Avi Kivity 已提交
2288 2289
		break;
	case MSR_IA32_SYSENTER_ESP:
A
Avi Kivity 已提交
2290
		vmcs_writel(GUEST_SYSENTER_ESP, data);
A
Avi Kivity 已提交
2291
		break;
2292
	case MSR_IA32_TSC:
2293
		kvm_write_tsc(vcpu, msr_info);
A
Avi Kivity 已提交
2294
		break;
S
Sheng Yang 已提交
2295 2296 2297 2298 2299 2300
	case MSR_IA32_CR_PAT:
		if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
			vmcs_write64(GUEST_IA32_PAT, data);
			vcpu->arch.pat = data;
			break;
		}
2301
		ret = kvm_set_msr_common(vcpu, msr_info);
2302
		break;
W
Will Auld 已提交
2303 2304
	case MSR_IA32_TSC_ADJUST:
		ret = kvm_set_msr_common(vcpu, msr_info);
2305 2306 2307 2308 2309 2310 2311 2312
		break;
	case MSR_TSC_AUX:
		if (!vmx->rdtscp_enabled)
			return 1;
		/* Check reserved bit, higher 32 bits should be zero */
		if ((data >> 32) != 0)
			return 1;
		/* Otherwise falls through */
A
Avi Kivity 已提交
2313
	default:
2314 2315
		if (vmx_set_vmx_msr(vcpu, msr_index, data))
			break;
R
Rusty Russell 已提交
2316
		msr = find_msr_entry(vmx, msr_index);
2317 2318
		if (msr) {
			msr->data = data;
2319 2320
			if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
				preempt_disable();
2321 2322
				kvm_set_shared_msr(msr->index, msr->data,
						   msr->mask);
2323 2324
				preempt_enable();
			}
2325
			break;
A
Avi Kivity 已提交
2326
		}
2327
		ret = kvm_set_msr_common(vcpu, msr_info);
A
Avi Kivity 已提交
2328 2329
	}

2330
	return ret;
A
Avi Kivity 已提交
2331 2332
}

2333
static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
A
Avi Kivity 已提交
2334
{
2335 2336 2337 2338 2339 2340 2341 2342
	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
	switch (reg) {
	case VCPU_REGS_RSP:
		vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
		break;
	case VCPU_REGS_RIP:
		vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
		break;
A
Avi Kivity 已提交
2343 2344 2345 2346
	case VCPU_EXREG_PDPTR:
		if (enable_ept)
			ept_save_pdptrs(vcpu);
		break;
2347 2348 2349
	default:
		break;
	}
A
Avi Kivity 已提交
2350 2351 2352 2353
}

static __init int cpu_has_kvm_support(void)
{
2354
	return cpu_has_vmx();
A
Avi Kivity 已提交
2355 2356 2357 2358 2359 2360 2361
}

static __init int vmx_disabled_by_bios(void)
{
	u64 msr;

	rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
2362
	if (msr & FEATURE_CONTROL_LOCKED) {
2363
		/* launched w/ TXT and VMX disabled */
2364 2365 2366
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
			&& tboot_enabled())
			return 1;
2367
		/* launched w/o TXT and VMX only enabled w/ TXT */
2368
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
2369
			&& (msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
2370 2371
			&& !tboot_enabled()) {
			printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
2372
				"activate TXT before enabling KVM\n");
2373
			return 1;
2374
		}
2375 2376 2377 2378
		/* launched w/o TXT and VMX disabled */
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
			&& !tboot_enabled())
			return 1;
2379 2380 2381
	}

	return 0;
A
Avi Kivity 已提交
2382 2383
}

2384 2385 2386 2387 2388 2389 2390
static void kvm_cpu_vmxon(u64 addr)
{
	asm volatile (ASM_VMX_VMXON_RAX
			: : "a"(&addr), "m"(addr)
			: "memory", "cc");
}

2391
static int hardware_enable(void *garbage)
A
Avi Kivity 已提交
2392 2393 2394
{
	int cpu = raw_smp_processor_id();
	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
2395
	u64 old, test_bits;
A
Avi Kivity 已提交
2396

2397 2398 2399
	if (read_cr4() & X86_CR4_VMXE)
		return -EBUSY;

2400
	INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412

	/*
	 * Now we can enable the vmclear operation in kdump
	 * since the loaded_vmcss_on_cpu list on this cpu
	 * has been initialized.
	 *
	 * Though the cpu is not in VMX operation now, there
	 * is no problem to enable the vmclear operation
	 * for the loaded_vmcss_on_cpu list is empty!
	 */
	crash_enable_local_vmclear(cpu);

A
Avi Kivity 已提交
2413
	rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
2414 2415 2416 2417 2418 2419 2420

	test_bits = FEATURE_CONTROL_LOCKED;
	test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
	if (tboot_enabled())
		test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;

	if ((old & test_bits) != test_bits) {
A
Avi Kivity 已提交
2421
		/* enable and lock */
2422 2423
		wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
	}
2424
	write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
2425

2426 2427 2428 2429
	if (vmm_exclusive) {
		kvm_cpu_vmxon(phys_addr);
		ept_sync_global();
	}
2430

2431 2432
	store_gdt(&__get_cpu_var(host_gdt));

2433
	return 0;
A
Avi Kivity 已提交
2434 2435
}

2436
static void vmclear_local_loaded_vmcss(void)
2437 2438
{
	int cpu = raw_smp_processor_id();
2439
	struct loaded_vmcs *v, *n;
2440

2441 2442 2443
	list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
				 loaded_vmcss_on_cpu_link)
		__loaded_vmcs_clear(v);
2444 2445
}

2446 2447 2448 2449 2450

/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
 * tricks.
 */
static void kvm_cpu_vmxoff(void)
A
Avi Kivity 已提交
2451
{
2452
	asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
A
Avi Kivity 已提交
2453 2454
}

2455 2456
static void hardware_disable(void *garbage)
{
2457
	if (vmm_exclusive) {
2458
		vmclear_local_loaded_vmcss();
2459 2460
		kvm_cpu_vmxoff();
	}
2461
	write_cr4(read_cr4() & ~X86_CR4_VMXE);
2462 2463
}

2464
static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
M
Mike Day 已提交
2465
				      u32 msr, u32 *result)
2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476
{
	u32 vmx_msr_low, vmx_msr_high;
	u32 ctl = ctl_min | ctl_opt;

	rdmsr(msr, vmx_msr_low, vmx_msr_high);

	ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
	ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */

	/* Ensure minimum (required) set of control bits are supported. */
	if (ctl_min & ~ctl)
Y
Yang, Sheng 已提交
2477
		return -EIO;
2478 2479 2480 2481 2482

	*result = ctl;
	return 0;
}

A
Avi Kivity 已提交
2483 2484 2485 2486 2487 2488 2489 2490
static __init bool allow_1_setting(u32 msr, u32 ctl)
{
	u32 vmx_msr_low, vmx_msr_high;

	rdmsr(msr, vmx_msr_low, vmx_msr_high);
	return vmx_msr_high & ctl;
}

Y
Yang, Sheng 已提交
2491
static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
A
Avi Kivity 已提交
2492 2493
{
	u32 vmx_msr_low, vmx_msr_high;
S
Sheng Yang 已提交
2494
	u32 min, opt, min2, opt2;
2495 2496
	u32 _pin_based_exec_control = 0;
	u32 _cpu_based_exec_control = 0;
2497
	u32 _cpu_based_2nd_exec_control = 0;
2498 2499 2500 2501
	u32 _vmexit_control = 0;
	u32 _vmentry_control = 0;

	min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
2502
	opt = PIN_BASED_VIRTUAL_NMIS;
2503 2504
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
				&_pin_based_exec_control) < 0)
Y
Yang, Sheng 已提交
2505
		return -EIO;
2506

R
Raghavendra K T 已提交
2507
	min = CPU_BASED_HLT_EXITING |
2508 2509 2510 2511
#ifdef CONFIG_X86_64
	      CPU_BASED_CR8_LOAD_EXITING |
	      CPU_BASED_CR8_STORE_EXITING |
#endif
S
Sheng Yang 已提交
2512 2513
	      CPU_BASED_CR3_LOAD_EXITING |
	      CPU_BASED_CR3_STORE_EXITING |
2514 2515
	      CPU_BASED_USE_IO_BITMAPS |
	      CPU_BASED_MOV_DR_EXITING |
M
Marcelo Tosatti 已提交
2516
	      CPU_BASED_USE_TSC_OFFSETING |
2517 2518
	      CPU_BASED_MWAIT_EXITING |
	      CPU_BASED_MONITOR_EXITING |
A
Avi Kivity 已提交
2519 2520
	      CPU_BASED_INVLPG_EXITING |
	      CPU_BASED_RDPMC_EXITING;
2521

2522
	opt = CPU_BASED_TPR_SHADOW |
S
Sheng Yang 已提交
2523
	      CPU_BASED_USE_MSR_BITMAPS |
2524
	      CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
2525 2526
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
				&_cpu_based_exec_control) < 0)
Y
Yang, Sheng 已提交
2527
		return -EIO;
2528 2529 2530 2531 2532
#ifdef CONFIG_X86_64
	if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
		_cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
					   ~CPU_BASED_CR8_STORE_EXITING;
#endif
2533
	if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
S
Sheng Yang 已提交
2534 2535
		min2 = 0;
		opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2536
			SECONDARY_EXEC_WBINVD_EXITING |
S
Sheng Yang 已提交
2537
			SECONDARY_EXEC_ENABLE_VPID |
2538
			SECONDARY_EXEC_ENABLE_EPT |
2539
			SECONDARY_EXEC_UNRESTRICTED_GUEST |
2540
			SECONDARY_EXEC_PAUSE_LOOP_EXITING |
2541 2542
			SECONDARY_EXEC_RDTSCP |
			SECONDARY_EXEC_ENABLE_INVPCID;
S
Sheng Yang 已提交
2543 2544
		if (adjust_vmx_controls(min2, opt2,
					MSR_IA32_VMX_PROCBASED_CTLS2,
2545 2546 2547 2548 2549 2550 2551 2552
					&_cpu_based_2nd_exec_control) < 0)
			return -EIO;
	}
#ifndef CONFIG_X86_64
	if (!(_cpu_based_2nd_exec_control &
				SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
		_cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
#endif
S
Sheng Yang 已提交
2553
	if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
M
Marcelo Tosatti 已提交
2554 2555
		/* CR3 accesses and invlpg don't need to cause VM Exits when EPT
		   enabled */
2556 2557 2558
		_cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
					     CPU_BASED_CR3_STORE_EXITING |
					     CPU_BASED_INVLPG_EXITING);
S
Sheng Yang 已提交
2559 2560 2561
		rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
		      vmx_capability.ept, vmx_capability.vpid);
	}
2562 2563 2564 2565 2566

	min = 0;
#ifdef CONFIG_X86_64
	min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
#endif
S
Sheng Yang 已提交
2567
	opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT;
2568 2569
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
				&_vmexit_control) < 0)
Y
Yang, Sheng 已提交
2570
		return -EIO;
2571

S
Sheng Yang 已提交
2572 2573
	min = 0;
	opt = VM_ENTRY_LOAD_IA32_PAT;
2574 2575
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
				&_vmentry_control) < 0)
Y
Yang, Sheng 已提交
2576
		return -EIO;
A
Avi Kivity 已提交
2577

N
Nguyen Anh Quynh 已提交
2578
	rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
2579 2580 2581

	/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
	if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
Y
Yang, Sheng 已提交
2582
		return -EIO;
2583 2584 2585 2586

#ifdef CONFIG_X86_64
	/* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
	if (vmx_msr_high & (1u<<16))
Y
Yang, Sheng 已提交
2587
		return -EIO;
2588 2589 2590 2591
#endif

	/* Require Write-Back (WB) memory type for VMCS accesses. */
	if (((vmx_msr_high >> 18) & 15) != 6)
Y
Yang, Sheng 已提交
2592
		return -EIO;
2593

Y
Yang, Sheng 已提交
2594 2595 2596
	vmcs_conf->size = vmx_msr_high & 0x1fff;
	vmcs_conf->order = get_order(vmcs_config.size);
	vmcs_conf->revision_id = vmx_msr_low;
2597

Y
Yang, Sheng 已提交
2598 2599
	vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
	vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
2600
	vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
Y
Yang, Sheng 已提交
2601 2602
	vmcs_conf->vmexit_ctrl         = _vmexit_control;
	vmcs_conf->vmentry_ctrl        = _vmentry_control;
2603

A
Avi Kivity 已提交
2604 2605 2606 2607 2608 2609
	cpu_has_load_ia32_efer =
		allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
				VM_ENTRY_LOAD_IA32_EFER)
		&& allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
				   VM_EXIT_LOAD_IA32_EFER);

2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645
	cpu_has_load_perf_global_ctrl =
		allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
				VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
		&& allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
				   VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);

	/*
	 * Some cpus support VM_ENTRY_(LOAD|SAVE)_IA32_PERF_GLOBAL_CTRL
	 * but due to arrata below it can't be used. Workaround is to use
	 * msr load mechanism to switch IA32_PERF_GLOBAL_CTRL.
	 *
	 * VM Exit May Incorrectly Clear IA32_PERF_GLOBAL_CTRL [34:32]
	 *
	 * AAK155             (model 26)
	 * AAP115             (model 30)
	 * AAT100             (model 37)
	 * BC86,AAY89,BD102   (model 44)
	 * BA97               (model 46)
	 *
	 */
	if (cpu_has_load_perf_global_ctrl && boot_cpu_data.x86 == 0x6) {
		switch (boot_cpu_data.x86_model) {
		case 26:
		case 30:
		case 37:
		case 44:
		case 46:
			cpu_has_load_perf_global_ctrl = false;
			printk_once(KERN_WARNING"kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
					"does not work properly. Using workaround\n");
			break;
		default:
			break;
		}
	}

2646
	return 0;
N
Nguyen Anh Quynh 已提交
2647
}
A
Avi Kivity 已提交
2648 2649 2650 2651 2652 2653 2654

static struct vmcs *alloc_vmcs_cpu(int cpu)
{
	int node = cpu_to_node(cpu);
	struct page *pages;
	struct vmcs *vmcs;

2655
	pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order);
A
Avi Kivity 已提交
2656 2657 2658
	if (!pages)
		return NULL;
	vmcs = page_address(pages);
2659 2660
	memset(vmcs, 0, vmcs_config.size);
	vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
A
Avi Kivity 已提交
2661 2662 2663 2664 2665
	return vmcs;
}

static struct vmcs *alloc_vmcs(void)
{
2666
	return alloc_vmcs_cpu(raw_smp_processor_id());
A
Avi Kivity 已提交
2667 2668 2669 2670
}

static void free_vmcs(struct vmcs *vmcs)
{
2671
	free_pages((unsigned long)vmcs, vmcs_config.order);
A
Avi Kivity 已提交
2672 2673
}

2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685
/*
 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
 */
static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
{
	if (!loaded_vmcs->vmcs)
		return;
	loaded_vmcs_clear(loaded_vmcs);
	free_vmcs(loaded_vmcs->vmcs);
	loaded_vmcs->vmcs = NULL;
}

2686
static void free_kvm_area(void)
A
Avi Kivity 已提交
2687 2688 2689
{
	int cpu;

Z
Zachary Amsden 已提交
2690
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
2691
		free_vmcs(per_cpu(vmxarea, cpu));
Z
Zachary Amsden 已提交
2692 2693
		per_cpu(vmxarea, cpu) = NULL;
	}
A
Avi Kivity 已提交
2694 2695 2696 2697 2698 2699
}

static __init int alloc_kvm_area(void)
{
	int cpu;

Z
Zachary Amsden 已提交
2700
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715
		struct vmcs *vmcs;

		vmcs = alloc_vmcs_cpu(cpu);
		if (!vmcs) {
			free_kvm_area();
			return -ENOMEM;
		}

		per_cpu(vmxarea, cpu) = vmcs;
	}
	return 0;
}

static __init int hardware_setup(void)
{
Y
Yang, Sheng 已提交
2716 2717
	if (setup_vmcs_config(&vmcs_config) < 0)
		return -EIO;
2718 2719 2720 2721

	if (boot_cpu_has(X86_FEATURE_NX))
		kvm_enable_efer_bits(EFER_NX);

S
Sheng Yang 已提交
2722 2723 2724
	if (!cpu_has_vmx_vpid())
		enable_vpid = 0;

2725 2726
	if (!cpu_has_vmx_ept() ||
	    !cpu_has_vmx_ept_4levels()) {
S
Sheng Yang 已提交
2727
		enable_ept = 0;
2728
		enable_unrestricted_guest = 0;
2729
		enable_ept_ad_bits = 0;
2730 2731
	}

2732 2733 2734
	if (!cpu_has_vmx_ept_ad_bits())
		enable_ept_ad_bits = 0;

2735 2736
	if (!cpu_has_vmx_unrestricted_guest())
		enable_unrestricted_guest = 0;
S
Sheng Yang 已提交
2737 2738 2739 2740

	if (!cpu_has_vmx_flexpriority())
		flexpriority_enabled = 0;

2741 2742 2743
	if (!cpu_has_vmx_tpr_shadow())
		kvm_x86_ops->update_cr8_intercept = NULL;

2744 2745 2746
	if (enable_ept && !cpu_has_vmx_ept_2m_page())
		kvm_disable_largepages();

2747 2748 2749
	if (!cpu_has_vmx_ple())
		ple_gap = 0;

2750 2751 2752
	if (nested)
		nested_vmx_setup_ctls_msrs();

A
Avi Kivity 已提交
2753 2754 2755 2756 2757 2758 2759 2760
	return alloc_kvm_area();
}

static __exit void hardware_unsetup(void)
{
	free_kvm_area();
}

2761
static void fix_pmode_dataseg(struct kvm_vcpu *vcpu, int seg, struct kvm_segment *save)
A
Avi Kivity 已提交
2762
{
2763
	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2764
	struct kvm_segment tmp = *save;
A
Avi Kivity 已提交
2765

2766 2767 2768
	if (!(vmcs_readl(sf->base) == tmp.base && tmp.s)) {
		tmp.base = vmcs_readl(sf->base);
		tmp.selector = vmcs_read16(sf->selector);
2769
		tmp.dpl = tmp.selector & SELECTOR_RPL_MASK;
2770
		tmp.s = 1;
A
Avi Kivity 已提交
2771
	}
2772
	vmx_set_segment(vcpu, &tmp, seg);
A
Avi Kivity 已提交
2773 2774 2775 2776 2777
}

static void enter_pmode(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
2778
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
2779

2780
	vmx->emulation_required = 1;
2781
	vmx->rmode.vm86_active = 0;
A
Avi Kivity 已提交
2782

A
Avi Kivity 已提交
2783 2784
	vmx_segment_cache_clear(vmx);

2785
	vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
A
Avi Kivity 已提交
2786 2787

	flags = vmcs_readl(GUEST_RFLAGS);
2788 2789
	flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
	flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
A
Avi Kivity 已提交
2790 2791
	vmcs_writel(GUEST_RFLAGS, flags);

2792 2793
	vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
			(vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
A
Avi Kivity 已提交
2794 2795 2796

	update_exception_bitmap(vcpu);

2797 2798 2799
	if (emulate_invalid_guest_state)
		return;

2800 2801 2802 2803
	fix_pmode_dataseg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
	fix_pmode_dataseg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
	fix_pmode_dataseg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
	fix_pmode_dataseg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
A
Avi Kivity 已提交
2804

A
Avi Kivity 已提交
2805 2806
	vmx_segment_cache_clear(vmx);

A
Avi Kivity 已提交
2807 2808 2809 2810 2811 2812 2813 2814
	vmcs_write16(GUEST_SS_SELECTOR, 0);
	vmcs_write32(GUEST_SS_AR_BYTES, 0x93);

	vmcs_write16(GUEST_CS_SELECTOR,
		     vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
	vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
}

M
Mike Day 已提交
2815
static gva_t rmode_tss_base(struct kvm *kvm)
A
Avi Kivity 已提交
2816
{
2817
	if (!kvm->arch.tss_addr) {
2818
		struct kvm_memslots *slots;
2819
		struct kvm_memory_slot *slot;
2820 2821
		gfn_t base_gfn;

2822
		slots = kvm_memslots(kvm);
2823 2824 2825
		slot = id_to_memslot(slots, 0);
		base_gfn = slot->base_gfn + slot->npages - 3;

2826 2827
		return base_gfn << PAGE_SHIFT;
	}
2828
	return kvm->arch.tss_addr;
A
Avi Kivity 已提交
2829 2830
}

2831
static void fix_rmode_seg(int seg, struct kvm_segment *save)
A
Avi Kivity 已提交
2832
{
2833
	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
A
Avi Kivity 已提交
2834

2835
	vmcs_write16(sf->selector, save->base >> 4);
2836
	vmcs_write32(sf->base, save->base & 0xffff0);
A
Avi Kivity 已提交
2837 2838
	vmcs_write32(sf->limit, 0xffff);
	vmcs_write32(sf->ar_bytes, 0xf3);
2839 2840 2841 2842
	if (save->base & 0xf)
		printk_once(KERN_WARNING "kvm: segment base is not paragraph"
			    " aligned when entering protected mode (seg=%d)",
			    seg);
A
Avi Kivity 已提交
2843 2844 2845 2846 2847
}

static void enter_rmode(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
2848
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2849
	struct kvm_segment var;
A
Avi Kivity 已提交
2850

2851 2852 2853
	if (enable_unrestricted_guest)
		return;

2854 2855 2856 2857 2858 2859
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);

2860
	vmx->emulation_required = 1;
2861
	vmx->rmode.vm86_active = 1;
A
Avi Kivity 已提交
2862

2863

2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875
	/*
	 * Very old userspace does not call KVM_SET_TSS_ADDR before entering
	 * vcpu. Call it here with phys address pointing 16M below 4G.
	 */
	if (!vcpu->kvm->arch.tss_addr) {
		printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
			     "called before entering vcpu\n");
		srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
		vmx_set_tss_addr(vcpu->kvm, 0xfeffd000);
		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
	}

A
Avi Kivity 已提交
2876 2877
	vmx_segment_cache_clear(vmx);

A
Avi Kivity 已提交
2878 2879 2880 2881 2882
	vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
	vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);

	flags = vmcs_readl(GUEST_RFLAGS);
2883
	vmx->rmode.save_rflags = flags;
A
Avi Kivity 已提交
2884

2885
	flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
A
Avi Kivity 已提交
2886 2887

	vmcs_writel(GUEST_RFLAGS, flags);
2888
	vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
A
Avi Kivity 已提交
2889 2890
	update_exception_bitmap(vcpu);

2891 2892 2893
	if (emulate_invalid_guest_state)
		goto continue_rmode;

2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904
	vmx_get_segment(vcpu, &var, VCPU_SREG_SS);
	vmx_set_segment(vcpu, &var, VCPU_SREG_SS);

	vmx_get_segment(vcpu, &var, VCPU_SREG_CS);
	vmx_set_segment(vcpu, &var, VCPU_SREG_CS);

	vmx_get_segment(vcpu, &var, VCPU_SREG_ES);
	vmx_set_segment(vcpu, &var, VCPU_SREG_ES);

	vmx_get_segment(vcpu, &var, VCPU_SREG_DS);
	vmx_set_segment(vcpu, &var, VCPU_SREG_DS);
A
Avi Kivity 已提交
2905

2906 2907
	vmx_get_segment(vcpu, &var, VCPU_SREG_GS);
	vmx_set_segment(vcpu, &var, VCPU_SREG_GS);
A
Avi Kivity 已提交
2908

2909 2910
	vmx_get_segment(vcpu, &var, VCPU_SREG_FS);
	vmx_set_segment(vcpu, &var, VCPU_SREG_FS);
2911

2912
continue_rmode:
2913
	kvm_mmu_reset_context(vcpu);
A
Avi Kivity 已提交
2914 2915
}

2916 2917 2918
static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2919 2920 2921 2922
	struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);

	if (!msr)
		return;
2923

2924 2925 2926 2927 2928
	/*
	 * Force kernel_gs_base reloading before EFER changes, as control
	 * of this msr depends on is_long_mode().
	 */
	vmx_load_host_state(to_vmx(vcpu));
2929
	vcpu->arch.efer = efer;
2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944
	if (efer & EFER_LMA) {
		vmcs_write32(VM_ENTRY_CONTROLS,
			     vmcs_read32(VM_ENTRY_CONTROLS) |
			     VM_ENTRY_IA32E_MODE);
		msr->data = efer;
	} else {
		vmcs_write32(VM_ENTRY_CONTROLS,
			     vmcs_read32(VM_ENTRY_CONTROLS) &
			     ~VM_ENTRY_IA32E_MODE);

		msr->data = efer & ~EFER_LME;
	}
	setup_msrs(vmx);
}

2945
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2946 2947 2948 2949 2950

static void enter_lmode(struct kvm_vcpu *vcpu)
{
	u32 guest_tr_ar;

A
Avi Kivity 已提交
2951 2952
	vmx_segment_cache_clear(to_vmx(vcpu));

A
Avi Kivity 已提交
2953 2954
	guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
	if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
2955 2956
		pr_debug_ratelimited("%s: tss fixup for long mode. \n",
				     __func__);
A
Avi Kivity 已提交
2957 2958 2959 2960
		vmcs_write32(GUEST_TR_AR_BYTES,
			     (guest_tr_ar & ~AR_TYPE_MASK)
			     | AR_TYPE_BUSY_64_TSS);
	}
2961
	vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
A
Avi Kivity 已提交
2962 2963 2964 2965 2966 2967
}

static void exit_lmode(struct kvm_vcpu *vcpu)
{
	vmcs_write32(VM_ENTRY_CONTROLS,
		     vmcs_read32(VM_ENTRY_CONTROLS)
2968
		     & ~VM_ENTRY_IA32E_MODE);
2969
	vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
A
Avi Kivity 已提交
2970 2971 2972 2973
}

#endif

2974 2975
static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
{
2976
	vpid_sync_context(to_vmx(vcpu));
2977 2978 2979
	if (enable_ept) {
		if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
			return;
2980
		ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
2981
	}
2982 2983
}

2984 2985 2986 2987 2988 2989 2990 2991
static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
{
	ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;

	vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
	vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
}

2992 2993 2994 2995 2996 2997 2998
static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
{
	if (enable_ept && is_paging(vcpu))
		vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
}

2999
static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
3000
{
3001 3002 3003 3004
	ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;

	vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
	vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
3005 3006
}

3007 3008
static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
3009 3010 3011 3012
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_dirty))
		return;

3013
	if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
3014 3015 3016 3017
		vmcs_write64(GUEST_PDPTR0, vcpu->arch.mmu.pdptrs[0]);
		vmcs_write64(GUEST_PDPTR1, vcpu->arch.mmu.pdptrs[1]);
		vmcs_write64(GUEST_PDPTR2, vcpu->arch.mmu.pdptrs[2]);
		vmcs_write64(GUEST_PDPTR3, vcpu->arch.mmu.pdptrs[3]);
3018 3019 3020
	}
}

3021 3022 3023
static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
{
	if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
3024 3025 3026 3027
		vcpu->arch.mmu.pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
		vcpu->arch.mmu.pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
		vcpu->arch.mmu.pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
		vcpu->arch.mmu.pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
3028
	}
A
Avi Kivity 已提交
3029 3030 3031 3032 3033

	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
3034 3035
}

3036
static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
3037 3038 3039 3040 3041

static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
					unsigned long cr0,
					struct kvm_vcpu *vcpu)
{
3042 3043
	if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
		vmx_decache_cr3(vcpu);
3044 3045 3046
	if (!(cr0 & X86_CR0_PG)) {
		/* From paging/starting to nonpaging */
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
3047
			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
3048 3049 3050
			     (CPU_BASED_CR3_LOAD_EXITING |
			      CPU_BASED_CR3_STORE_EXITING));
		vcpu->arch.cr0 = cr0;
3051
		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3052 3053 3054
	} else if (!is_paging(vcpu)) {
		/* From nonpaging to paging */
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
3055
			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
3056 3057 3058
			     ~(CPU_BASED_CR3_LOAD_EXITING |
			       CPU_BASED_CR3_STORE_EXITING));
		vcpu->arch.cr0 = cr0;
3059
		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3060
	}
3061 3062 3063

	if (!(cr0 & X86_CR0_WP))
		*hw_cr0 &= ~X86_CR0_WP;
3064 3065
}

A
Avi Kivity 已提交
3066 3067
static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
3068
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3069 3070 3071 3072 3073 3074 3075
	unsigned long hw_cr0;

	if (enable_unrestricted_guest)
		hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST)
			| KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
	else
		hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON;
3076

3077
	if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
A
Avi Kivity 已提交
3078 3079
		enter_pmode(vcpu);

3080
	if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
A
Avi Kivity 已提交
3081 3082
		enter_rmode(vcpu);

3083
#ifdef CONFIG_X86_64
3084
	if (vcpu->arch.efer & EFER_LME) {
3085
		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
A
Avi Kivity 已提交
3086
			enter_lmode(vcpu);
3087
		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
A
Avi Kivity 已提交
3088 3089 3090 3091
			exit_lmode(vcpu);
	}
#endif

3092
	if (enable_ept)
3093 3094
		ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);

3095
	if (!vcpu->fpu_active)
3096
		hw_cr0 |= X86_CR0_TS | X86_CR0_MP;
3097

A
Avi Kivity 已提交
3098
	vmcs_writel(CR0_READ_SHADOW, cr0);
3099
	vmcs_writel(GUEST_CR0, hw_cr0);
3100
	vcpu->arch.cr0 = cr0;
A
Avi Kivity 已提交
3101
	__clear_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
A
Avi Kivity 已提交
3102 3103
}

3104 3105 3106 3107 3108 3109 3110
static u64 construct_eptp(unsigned long root_hpa)
{
	u64 eptp;

	/* TODO write the value reading from MSR */
	eptp = VMX_EPT_DEFAULT_MT |
		VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
3111 3112
	if (enable_ept_ad_bits)
		eptp |= VMX_EPT_AD_ENABLE_BIT;
3113 3114 3115 3116 3117
	eptp |= (root_hpa & PAGE_MASK);

	return eptp;
}

A
Avi Kivity 已提交
3118 3119
static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
{
3120 3121 3122 3123
	unsigned long guest_cr3;
	u64 eptp;

	guest_cr3 = cr3;
3124
	if (enable_ept) {
3125 3126
		eptp = construct_eptp(cr3);
		vmcs_write64(EPT_POINTER, eptp);
3127
		guest_cr3 = is_paging(vcpu) ? kvm_read_cr3(vcpu) :
3128
			vcpu->kvm->arch.ept_identity_map_addr;
3129
		ept_load_pdptrs(vcpu);
3130 3131
	}

3132
	vmx_flush_tlb(vcpu);
3133
	vmcs_writel(GUEST_CR3, guest_cr3);
A
Avi Kivity 已提交
3134 3135
}

3136
static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
A
Avi Kivity 已提交
3137
{
3138
	unsigned long hw_cr4 = cr4 | (to_vmx(vcpu)->rmode.vm86_active ?
3139 3140
		    KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);

3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152
	if (cr4 & X86_CR4_VMXE) {
		/*
		 * To use VMXON (and later other VMX instructions), a guest
		 * must first be able to turn on cr4.VMXE (see handle_vmon()).
		 * So basically the check on whether to allow nested VMX
		 * is here.
		 */
		if (!nested_vmx_allowed(vcpu))
			return 1;
	} else if (to_vmx(vcpu)->nested.vmxon)
		return 1;

3153
	vcpu->arch.cr4 = cr4;
3154 3155 3156 3157 3158 3159 3160 3161
	if (enable_ept) {
		if (!is_paging(vcpu)) {
			hw_cr4 &= ~X86_CR4_PAE;
			hw_cr4 |= X86_CR4_PSE;
		} else if (!(cr4 & X86_CR4_PAE)) {
			hw_cr4 &= ~X86_CR4_PAE;
		}
	}
3162 3163 3164

	vmcs_writel(CR4_READ_SHADOW, cr4);
	vmcs_writel(GUEST_CR4, hw_cr4);
3165
	return 0;
A
Avi Kivity 已提交
3166 3167 3168 3169 3170
}

static void vmx_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
3171
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
3172 3173
	u32 ar;

3174 3175 3176
	if (vmx->rmode.vm86_active
	    && (seg == VCPU_SREG_TR || seg == VCPU_SREG_ES
		|| seg == VCPU_SREG_DS || seg == VCPU_SREG_FS
3177
		|| seg == VCPU_SREG_GS)) {
3178
		*var = vmx->rmode.segs[seg];
3179
		if (seg == VCPU_SREG_TR
A
Avi Kivity 已提交
3180
		    || var->selector == vmx_read_guest_seg_selector(vmx, seg))
3181
			return;
3182 3183 3184
		var->base = vmx_read_guest_seg_base(vmx, seg);
		var->selector = vmx_read_guest_seg_selector(vmx, seg);
		return;
3185
	}
A
Avi Kivity 已提交
3186 3187 3188 3189
	var->base = vmx_read_guest_seg_base(vmx, seg);
	var->limit = vmx_read_guest_seg_limit(vmx, seg);
	var->selector = vmx_read_guest_seg_selector(vmx, seg);
	ar = vmx_read_guest_seg_ar(vmx, seg);
3190
	if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state)
A
Avi Kivity 已提交
3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202
		ar = 0;
	var->type = ar & 15;
	var->s = (ar >> 4) & 1;
	var->dpl = (ar >> 5) & 3;
	var->present = (ar >> 7) & 1;
	var->avl = (ar >> 12) & 1;
	var->l = (ar >> 13) & 1;
	var->db = (ar >> 14) & 1;
	var->g = (ar >> 15) & 1;
	var->unusable = (ar >> 16) & 1;
}

3203 3204 3205 3206 3207 3208 3209 3210
static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment s;

	if (to_vmx(vcpu)->rmode.vm86_active) {
		vmx_get_segment(vcpu, &s, seg);
		return s.base;
	}
A
Avi Kivity 已提交
3211
	return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
3212 3213
}

A
Avi Kivity 已提交
3214
static int __vmx_get_cpl(struct kvm_vcpu *vcpu)
3215
{
3216
	if (!is_protmode(vcpu))
3217 3218
		return 0;

A
Avi Kivity 已提交
3219 3220
	if (!is_long_mode(vcpu)
	    && (kvm_get_rflags(vcpu) & X86_EFLAGS_VM)) /* if virtual 8086 */
3221 3222
		return 3;

A
Avi Kivity 已提交
3223
	return vmx_read_guest_seg_selector(to_vmx(vcpu), VCPU_SREG_CS) & 3;
3224 3225
}

A
Avi Kivity 已提交
3226 3227
static int vmx_get_cpl(struct kvm_vcpu *vcpu)
{
3228 3229 3230 3231 3232 3233 3234 3235 3236 3237
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	/*
	 * If we enter real mode with cs.sel & 3 != 0, the normal CPL calculations
	 * fail; use the cache instead.
	 */
	if (unlikely(vmx->emulation_required && emulate_invalid_guest_state)) {
		return vmx->cpl;
	}

A
Avi Kivity 已提交
3238 3239
	if (!test_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail)) {
		__set_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
3240
		vmx->cpl = __vmx_get_cpl(vcpu);
A
Avi Kivity 已提交
3241
	}
3242 3243

	return vmx->cpl;
A
Avi Kivity 已提交
3244 3245 3246
}


3247
static u32 vmx_segment_access_rights(struct kvm_segment *var)
A
Avi Kivity 已提交
3248 3249 3250
{
	u32 ar;

3251
	if (var->unusable || !var->present)
A
Avi Kivity 已提交
3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262
		ar = 1 << 16;
	else {
		ar = var->type & 15;
		ar |= (var->s & 1) << 4;
		ar |= (var->dpl & 3) << 5;
		ar |= (var->present & 1) << 7;
		ar |= (var->avl & 1) << 12;
		ar |= (var->l & 1) << 13;
		ar |= (var->db & 1) << 14;
		ar |= (var->g & 1) << 15;
	}
3263 3264 3265 3266 3267 3268 3269

	return ar;
}

static void vmx_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
3270
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3271
	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3272 3273
	u32 ar;

A
Avi Kivity 已提交
3274 3275
	vmx_segment_cache_clear(vmx);

3276
	if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) {
3277
		vmcs_write16(sf->selector, var->selector);
3278
		vmx->rmode.segs[VCPU_SREG_TR] = *var;
3279 3280 3281 3282 3283
		return;
	}
	vmcs_writel(sf->base, var->base);
	vmcs_write32(sf->limit, var->limit);
	vmcs_write16(sf->selector, var->selector);
3284
	if (vmx->rmode.vm86_active && var->s) {
3285
		vmx->rmode.segs[seg] = *var;
3286 3287 3288 3289 3290 3291 3292 3293
		/*
		 * Hack real-mode segments into vm86 compatibility.
		 */
		if (var->base == 0xffff0000 && var->selector == 0xf000)
			vmcs_writel(sf->base, 0xf0000);
		ar = 0xf3;
	} else
		ar = vmx_segment_access_rights(var);
3294 3295 3296 3297 3298 3299

	/*
	 *   Fix the "Accessed" bit in AR field of segment registers for older
	 * qemu binaries.
	 *   IA32 arch specifies that at the time of processor reset the
	 * "Accessed" bit in the AR field of segment registers is 1. And qemu
G
Guo Chao 已提交
3300
	 * is setting it to 0 in the userland code. This causes invalid guest
3301 3302 3303 3304 3305 3306 3307 3308
	 * state vmexit when "unrestricted guest" mode is turned on.
	 *    Fix for this setup issue in cpu_reset is being pushed in the qemu
	 * tree. Newer qemu binaries with that qemu fix would not need this
	 * kvm hack.
	 */
	if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
		ar |= 0x1; /* Accessed */

A
Avi Kivity 已提交
3309
	vmcs_write32(sf->ar_bytes, ar);
A
Avi Kivity 已提交
3310
	__clear_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
3311 3312 3313 3314 3315 3316 3317 3318

	/*
	 * Fix segments for real mode guest in hosts that don't have
	 * "unrestricted_mode" or it was disabled.
	 * This is done to allow migration of the guests from hosts with
	 * unrestricted guest like Westmere to older host that don't have
	 * unrestricted guest like Nehelem.
	 */
3319
	if (vmx->rmode.vm86_active) {
3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332
		switch (seg) {
		case VCPU_SREG_CS:
			vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
			vmcs_write32(GUEST_CS_LIMIT, 0xffff);
			if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
				vmcs_writel(GUEST_CS_BASE, 0xf0000);
			vmcs_write16(GUEST_CS_SELECTOR,
				     vmcs_readl(GUEST_CS_BASE) >> 4);
			break;
		case VCPU_SREG_ES:
		case VCPU_SREG_DS:
		case VCPU_SREG_GS:
		case VCPU_SREG_FS:
3333
			fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
3334 3335 3336 3337 3338 3339 3340 3341 3342
			break;
		case VCPU_SREG_SS:
			vmcs_write16(GUEST_SS_SELECTOR,
				     vmcs_readl(GUEST_SS_BASE) >> 4);
			vmcs_write32(GUEST_SS_LIMIT, 0xffff);
			vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);
			break;
		}
	}
A
Avi Kivity 已提交
3343 3344 3345 3346
}

static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
A
Avi Kivity 已提交
3347
	u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
A
Avi Kivity 已提交
3348 3349 3350 3351 3352

	*db = (ar >> 14) & 1;
	*l = (ar >> 13) & 1;
}

3353
static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
3354
{
3355 3356
	dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
	dt->address = vmcs_readl(GUEST_IDTR_BASE);
A
Avi Kivity 已提交
3357 3358
}

3359
static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
3360
{
3361 3362
	vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
	vmcs_writel(GUEST_IDTR_BASE, dt->address);
A
Avi Kivity 已提交
3363 3364
}

3365
static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
3366
{
3367 3368
	dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
	dt->address = vmcs_readl(GUEST_GDTR_BASE);
A
Avi Kivity 已提交
3369 3370
}

3371
static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
3372
{
3373 3374
	vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
	vmcs_writel(GUEST_GDTR_BASE, dt->address);
A
Avi Kivity 已提交
3375 3376
}

3377 3378 3379 3380 3381 3382 3383 3384 3385 3386
static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment var;
	u32 ar;

	vmx_get_segment(vcpu, &var, seg);
	ar = vmx_segment_access_rights(&var);

	if (var.base != (var.selector << 4))
		return false;
3387
	if (var.limit < 0xffff)
3388
		return false;
3389
	if (((ar | (3 << AR_DPL_SHIFT)) & ~(AR_G_MASK | AR_DB_MASK)) != 0xf3)
3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402
		return false;

	return true;
}

static bool code_segment_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	unsigned int cs_rpl;

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs_rpl = cs.selector & SELECTOR_RPL_MASK;

3403 3404
	if (cs.unusable)
		return false;
3405 3406 3407 3408
	if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK))
		return false;
	if (!cs.s)
		return false;
3409
	if (cs.type & AR_TYPE_WRITEABLE_MASK) {
3410 3411
		if (cs.dpl > cs_rpl)
			return false;
3412
	} else {
3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430
		if (cs.dpl != cs_rpl)
			return false;
	}
	if (!cs.present)
		return false;

	/* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
	return true;
}

static bool stack_segment_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment ss;
	unsigned int ss_rpl;

	vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
	ss_rpl = ss.selector & SELECTOR_RPL_MASK;

3431 3432 3433
	if (ss.unusable)
		return true;
	if (ss.type != 3 && ss.type != 7)
3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452
		return false;
	if (!ss.s)
		return false;
	if (ss.dpl != ss_rpl) /* DPL != RPL */
		return false;
	if (!ss.present)
		return false;

	return true;
}

static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment var;
	unsigned int rpl;

	vmx_get_segment(vcpu, &var, seg);
	rpl = var.selector & SELECTOR_RPL_MASK;

3453 3454
	if (var.unusable)
		return true;
3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475
	if (!var.s)
		return false;
	if (!var.present)
		return false;
	if (~var.type & (AR_TYPE_CODE_MASK|AR_TYPE_WRITEABLE_MASK)) {
		if (var.dpl < rpl) /* DPL < RPL */
			return false;
	}

	/* TODO: Add other members to kvm_segment_field to allow checking for other access
	 * rights flags
	 */
	return true;
}

static bool tr_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment tr;

	vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);

3476 3477
	if (tr.unusable)
		return false;
3478 3479
	if (tr.selector & SELECTOR_TI_MASK)	/* TI = 1 */
		return false;
3480
	if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493
		return false;
	if (!tr.present)
		return false;

	return true;
}

static bool ldtr_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment ldtr;

	vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);

3494 3495
	if (ldtr.unusable)
		return true;
3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524
	if (ldtr.selector & SELECTOR_TI_MASK)	/* TI = 1 */
		return false;
	if (ldtr.type != 2)
		return false;
	if (!ldtr.present)
		return false;

	return true;
}

static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs, ss;

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);

	return ((cs.selector & SELECTOR_RPL_MASK) ==
		 (ss.selector & SELECTOR_RPL_MASK));
}

/*
 * Check if guest state is valid. Returns true if valid, false if
 * not.
 * We assume that registers are always usable
 */
static bool guest_state_valid(struct kvm_vcpu *vcpu)
{
	/* real mode guest state checks */
3525
	if (!is_protmode(vcpu)) {
3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566
		if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
			return false;
	} else {
	/* protected mode guest state checks */
		if (!cs_ss_rpl_check(vcpu))
			return false;
		if (!code_segment_valid(vcpu))
			return false;
		if (!stack_segment_valid(vcpu))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_DS))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_ES))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_FS))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_GS))
			return false;
		if (!tr_valid(vcpu))
			return false;
		if (!ldtr_valid(vcpu))
			return false;
	}
	/* TODO:
	 * - Add checks on RIP
	 * - Add checks on RFLAGS
	 */

	return true;
}

M
Mike Day 已提交
3567
static int init_rmode_tss(struct kvm *kvm)
A
Avi Kivity 已提交
3568
{
3569
	gfn_t fn;
3570
	u16 data = 0;
3571
	int r, idx, ret = 0;
A
Avi Kivity 已提交
3572

3573 3574
	idx = srcu_read_lock(&kvm->srcu);
	fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
3575 3576
	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
	if (r < 0)
3577
		goto out;
3578
	data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
3579 3580
	r = kvm_write_guest_page(kvm, fn++, &data,
			TSS_IOPB_BASE_OFFSET, sizeof(u16));
3581
	if (r < 0)
3582
		goto out;
3583 3584
	r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
	if (r < 0)
3585
		goto out;
3586 3587
	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
	if (r < 0)
3588
		goto out;
3589
	data = ~0;
3590 3591 3592
	r = kvm_write_guest_page(kvm, fn, &data,
				 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
				 sizeof(u8));
3593
	if (r < 0)
3594 3595 3596 3597
		goto out;

	ret = 1;
out:
3598
	srcu_read_unlock(&kvm->srcu, idx);
3599
	return ret;
A
Avi Kivity 已提交
3600 3601
}

3602 3603
static int init_rmode_identity_map(struct kvm *kvm)
{
3604
	int i, idx, r, ret;
3605 3606 3607
	pfn_t identity_map_pfn;
	u32 tmp;

3608
	if (!enable_ept)
3609 3610 3611 3612 3613 3614 3615 3616 3617
		return 1;
	if (unlikely(!kvm->arch.ept_identity_pagetable)) {
		printk(KERN_ERR "EPT: identity-mapping pagetable "
			"haven't been allocated!\n");
		return 0;
	}
	if (likely(kvm->arch.ept_identity_pagetable_done))
		return 1;
	ret = 0;
3618
	identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
3619
	idx = srcu_read_lock(&kvm->srcu);
3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634
	r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
	if (r < 0)
		goto out;
	/* Set up identity-mapping pagetable for EPT in real mode */
	for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
		tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
			_PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
		r = kvm_write_guest_page(kvm, identity_map_pfn,
				&tmp, i * sizeof(tmp), sizeof(tmp));
		if (r < 0)
			goto out;
	}
	kvm->arch.ept_identity_pagetable_done = true;
	ret = 1;
out:
3635
	srcu_read_unlock(&kvm->srcu, idx);
3636 3637 3638
	return ret;
}

A
Avi Kivity 已提交
3639 3640
static void seg_setup(int seg)
{
3641
	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3642
	unsigned int ar;
A
Avi Kivity 已提交
3643 3644 3645 3646

	vmcs_write16(sf->selector, 0);
	vmcs_writel(sf->base, 0);
	vmcs_write32(sf->limit, 0xffff);
3647 3648 3649 3650 3651 3652 3653 3654
	if (enable_unrestricted_guest) {
		ar = 0x93;
		if (seg == VCPU_SREG_CS)
			ar |= 0x08; /* code segment */
	} else
		ar = 0xf3;

	vmcs_write32(sf->ar_bytes, ar);
A
Avi Kivity 已提交
3655 3656
}

3657 3658
static int alloc_apic_access_page(struct kvm *kvm)
{
3659
	struct page *page;
3660 3661 3662
	struct kvm_userspace_memory_region kvm_userspace_mem;
	int r = 0;

3663
	mutex_lock(&kvm->slots_lock);
3664
	if (kvm->arch.apic_access_page)
3665 3666 3667 3668 3669
		goto out;
	kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
	kvm_userspace_mem.flags = 0;
	kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL;
	kvm_userspace_mem.memory_size = PAGE_SIZE;
3670
	r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, false);
3671 3672
	if (r)
		goto out;
3673

3674 3675 3676 3677 3678 3679 3680
	page = gfn_to_page(kvm, 0xfee00);
	if (is_error_page(page)) {
		r = -EFAULT;
		goto out;
	}

	kvm->arch.apic_access_page = page;
3681
out:
3682
	mutex_unlock(&kvm->slots_lock);
3683 3684 3685
	return r;
}

3686 3687
static int alloc_identity_pagetable(struct kvm *kvm)
{
3688
	struct page *page;
3689 3690 3691
	struct kvm_userspace_memory_region kvm_userspace_mem;
	int r = 0;

3692
	mutex_lock(&kvm->slots_lock);
3693 3694 3695 3696
	if (kvm->arch.ept_identity_pagetable)
		goto out;
	kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
	kvm_userspace_mem.flags = 0;
3697 3698
	kvm_userspace_mem.guest_phys_addr =
		kvm->arch.ept_identity_map_addr;
3699
	kvm_userspace_mem.memory_size = PAGE_SIZE;
3700
	r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, false);
3701 3702 3703
	if (r)
		goto out;

3704 3705 3706 3707 3708 3709 3710
	page = gfn_to_page(kvm, kvm->arch.ept_identity_map_addr >> PAGE_SHIFT);
	if (is_error_page(page)) {
		r = -EFAULT;
		goto out;
	}

	kvm->arch.ept_identity_pagetable = page;
3711
out:
3712
	mutex_unlock(&kvm->slots_lock);
3713 3714 3715
	return r;
}

3716 3717 3718 3719 3720
static void allocate_vpid(struct vcpu_vmx *vmx)
{
	int vpid;

	vmx->vpid = 0;
3721
	if (!enable_vpid)
3722 3723 3724 3725 3726 3727 3728 3729 3730 3731
		return;
	spin_lock(&vmx_vpid_lock);
	vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
	if (vpid < VMX_NR_VPIDS) {
		vmx->vpid = vpid;
		__set_bit(vpid, vmx_vpid_bitmap);
	}
	spin_unlock(&vmx_vpid_lock);
}

3732 3733 3734 3735 3736 3737 3738 3739 3740 3741
static void free_vpid(struct vcpu_vmx *vmx)
{
	if (!enable_vpid)
		return;
	spin_lock(&vmx_vpid_lock);
	if (vmx->vpid != 0)
		__clear_bit(vmx->vpid, vmx_vpid_bitmap);
	spin_unlock(&vmx_vpid_lock);
}

3742
static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, u32 msr)
S
Sheng Yang 已提交
3743
{
3744
	int f = sizeof(unsigned long);
S
Sheng Yang 已提交
3745 3746 3747 3748 3749 3750 3751 3752 3753 3754

	if (!cpu_has_vmx_msr_bitmap())
		return;

	/*
	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
	 * have the write-low and read-high bitmap offsets the wrong way round.
	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
	 */
	if (msr <= 0x1fff) {
3755 3756
		__clear_bit(msr, msr_bitmap + 0x000 / f); /* read-low */
		__clear_bit(msr, msr_bitmap + 0x800 / f); /* write-low */
S
Sheng Yang 已提交
3757 3758
	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
		msr &= 0x1fff;
3759 3760
		__clear_bit(msr, msr_bitmap + 0x400 / f); /* read-high */
		__clear_bit(msr, msr_bitmap + 0xc00 / f); /* write-high */
S
Sheng Yang 已提交
3761 3762 3763
	}
}

3764 3765 3766 3767 3768 3769 3770
static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
{
	if (!longmode_only)
		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy, msr);
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode, msr);
}

3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782
/*
 * Set up the vmcs's constant host-state fields, i.e., host-state fields that
 * will not change in the lifetime of the guest.
 * Note that host-state that does change is set elsewhere. E.g., host-state
 * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
 */
static void vmx_set_constant_host_state(void)
{
	u32 low32, high32;
	unsigned long tmpl;
	struct desc_ptr dt;

3783
	vmcs_writel(HOST_CR0, read_cr0() & ~X86_CR0_TS);  /* 22.2.3 */
3784 3785 3786 3787
	vmcs_writel(HOST_CR4, read_cr4());  /* 22.2.3, 22.2.5 */
	vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */

	vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
A
Avi Kivity 已提交
3788 3789 3790 3791 3792 3793 3794 3795 3796
#ifdef CONFIG_X86_64
	/*
	 * Load null selectors, so we can avoid reloading them in
	 * __vmx_load_host_state(), in case userspace uses the null selectors
	 * too (the expected case).
	 */
	vmcs_write16(HOST_DS_SELECTOR, 0);
	vmcs_write16(HOST_ES_SELECTOR, 0);
#else
3797 3798
	vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
	vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
A
Avi Kivity 已提交
3799
#endif
3800 3801 3802 3803 3804 3805
	vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
	vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */

	native_store_idt(&dt);
	vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */

A
Avi Kivity 已提交
3806
	vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818

	rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
	vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
	rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
	vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */

	if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
		rdmsr(MSR_IA32_CR_PAT, low32, high32);
		vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
	}
}

3819 3820 3821 3822 3823
static void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
{
	vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
	if (enable_ept)
		vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
3824 3825 3826
	if (is_guest_mode(&vmx->vcpu))
		vmx->vcpu.arch.cr4_guest_owned_bits &=
			~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask;
3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856
	vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
}

static u32 vmx_exec_control(struct vcpu_vmx *vmx)
{
	u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
	if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
		exec_control &= ~CPU_BASED_TPR_SHADOW;
#ifdef CONFIG_X86_64
		exec_control |= CPU_BASED_CR8_STORE_EXITING |
				CPU_BASED_CR8_LOAD_EXITING;
#endif
	}
	if (!enable_ept)
		exec_control |= CPU_BASED_CR3_STORE_EXITING |
				CPU_BASED_CR3_LOAD_EXITING  |
				CPU_BASED_INVLPG_EXITING;
	return exec_control;
}

static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
{
	u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
	if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
		exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
	if (vmx->vpid == 0)
		exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
	if (!enable_ept) {
		exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
		enable_unrestricted_guest = 0;
3857 3858
		/* Enable INVPCID for non-ept guests may cause performance regression. */
		exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
3859 3860 3861 3862 3863 3864 3865 3866
	}
	if (!enable_unrestricted_guest)
		exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
	if (!ple_gap)
		exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
	return exec_control;
}

3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877
static void ept_set_mmio_spte_mask(void)
{
	/*
	 * EPT Misconfigurations can be generated if the value of bits 2:0
	 * of an EPT paging-structure entry is 110b (write/execute).
	 * Also, magic bits (0xffull << 49) is set to quickly identify mmio
	 * spte.
	 */
	kvm_mmu_set_mmio_spte_mask(0xffull << 49 | 0x6ull);
}

A
Avi Kivity 已提交
3878 3879 3880
/*
 * Sets up the vmcs for emulated real mode.
 */
R
Rusty Russell 已提交
3881
static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
A
Avi Kivity 已提交
3882
{
3883
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
3884
	unsigned long a;
3885
#endif
A
Avi Kivity 已提交
3886 3887 3888
	int i;

	/* I/O */
3889 3890
	vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
	vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
A
Avi Kivity 已提交
3891

S
Sheng Yang 已提交
3892
	if (cpu_has_vmx_msr_bitmap())
3893
		vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
S
Sheng Yang 已提交
3894

A
Avi Kivity 已提交
3895 3896 3897
	vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */

	/* Control */
3898 3899
	vmcs_write32(PIN_BASED_VM_EXEC_CONTROL,
		vmcs_config.pin_based_exec_ctrl);
3900

3901
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx_exec_control(vmx));
A
Avi Kivity 已提交
3902

3903
	if (cpu_has_secondary_exec_ctrls()) {
3904 3905
		vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
				vmx_secondary_exec_control(vmx));
3906
	}
3907

3908 3909 3910 3911 3912
	if (ple_gap) {
		vmcs_write32(PLE_GAP, ple_gap);
		vmcs_write32(PLE_WINDOW, ple_window);
	}

3913 3914
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
A
Avi Kivity 已提交
3915 3916
	vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */

3917 3918
	vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
	vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
3919
	vmx_set_constant_host_state();
3920
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
3921 3922 3923 3924 3925 3926 3927 3928 3929
	rdmsrl(MSR_FS_BASE, a);
	vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
	rdmsrl(MSR_GS_BASE, a);
	vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
#else
	vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
	vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
#endif

3930 3931
	vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
3932
	vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
3933
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
3934
	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
A
Avi Kivity 已提交
3935

S
Sheng Yang 已提交
3936
	if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
3937 3938
		u32 msr_low, msr_high;
		u64 host_pat;
S
Sheng Yang 已提交
3939 3940 3941 3942 3943 3944 3945 3946
		rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
		host_pat = msr_low | ((u64) msr_high << 32);
		/* Write the default value follow host pat */
		vmcs_write64(GUEST_IA32_PAT, host_pat);
		/* Keep arch.pat sync with GUEST_IA32_PAT */
		vmx->vcpu.arch.pat = host_pat;
	}

A
Avi Kivity 已提交
3947 3948 3949
	for (i = 0; i < NR_VMX_MSR; ++i) {
		u32 index = vmx_msr_index[i];
		u32 data_low, data_high;
3950
		int j = vmx->nmsrs;
A
Avi Kivity 已提交
3951 3952 3953

		if (rdmsr_safe(index, &data_low, &data_high) < 0)
			continue;
3954 3955
		if (wrmsr_safe(index, data_low, data_high) < 0)
			continue;
3956 3957
		vmx->guest_msrs[j].index = i;
		vmx->guest_msrs[j].data = 0;
3958
		vmx->guest_msrs[j].mask = -1ull;
3959
		++vmx->nmsrs;
A
Avi Kivity 已提交
3960 3961
	}

3962
	vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
A
Avi Kivity 已提交
3963 3964

	/* 22.2.1, 20.8.1 */
3965 3966
	vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl);

3967
	vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
3968
	set_cr4_guest_host_mask(vmx);
3969 3970 3971 3972 3973 3974 3975 3976

	return 0;
}

static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u64 msr;
3977
	int ret;
3978

3979
	vmx->rmode.vm86_active = 0;
3980

3981 3982
	vmx->soft_vnmi_blocked = 0;

3983
	vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
3984
	kvm_set_cr8(&vmx->vcpu, 0);
3985
	msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
3986
	if (kvm_vcpu_is_bsp(&vmx->vcpu))
3987 3988 3989
		msr |= MSR_IA32_APICBASE_BSP;
	kvm_set_apic_base(&vmx->vcpu, msr);

A
Avi Kivity 已提交
3990 3991
	vmx_segment_cache_clear(vmx);

3992
	seg_setup(VCPU_SREG_CS);
3993 3994 3995 3996
	/*
	 * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
	 * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4.  Sigh.
	 */
3997
	if (kvm_vcpu_is_bsp(&vmx->vcpu)) {
3998 3999 4000
		vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
		vmcs_writel(GUEST_CS_BASE, 0x000f0000);
	} else {
4001 4002
		vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8);
		vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12);
4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025
	}

	seg_setup(VCPU_SREG_DS);
	seg_setup(VCPU_SREG_ES);
	seg_setup(VCPU_SREG_FS);
	seg_setup(VCPU_SREG_GS);
	seg_setup(VCPU_SREG_SS);

	vmcs_write16(GUEST_TR_SELECTOR, 0);
	vmcs_writel(GUEST_TR_BASE, 0);
	vmcs_write32(GUEST_TR_LIMIT, 0xffff);
	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);

	vmcs_write16(GUEST_LDTR_SELECTOR, 0);
	vmcs_writel(GUEST_LDTR_BASE, 0);
	vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
	vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);

	vmcs_write32(GUEST_SYSENTER_CS, 0);
	vmcs_writel(GUEST_SYSENTER_ESP, 0);
	vmcs_writel(GUEST_SYSENTER_EIP, 0);

	vmcs_writel(GUEST_RFLAGS, 0x02);
4026
	if (kvm_vcpu_is_bsp(&vmx->vcpu))
4027
		kvm_rip_write(vcpu, 0xfff0);
4028
	else
4029
		kvm_rip_write(vcpu, 0);
4030 4031 4032 4033 4034 4035 4036

	vmcs_writel(GUEST_GDTR_BASE, 0);
	vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);

	vmcs_writel(GUEST_IDTR_BASE, 0);
	vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);

4037
	vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
4038 4039 4040 4041 4042 4043 4044 4045
	vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
	vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);

	/* Special registers */
	vmcs_write64(GUEST_IA32_DEBUGCTL, 0);

	setup_msrs(vmx);

A
Avi Kivity 已提交
4046 4047
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */

4048 4049 4050 4051
	if (cpu_has_vmx_tpr_shadow()) {
		vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
		if (vm_need_tpr_shadow(vmx->vcpu.kvm))
			vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
4052
				     __pa(vmx->vcpu.arch.apic->regs));
4053 4054 4055 4056 4057
		vmcs_write32(TPR_THRESHOLD, 0);
	}

	if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
		vmcs_write64(APIC_ACCESS_ADDR,
4058
			     page_to_phys(vmx->vcpu.kvm->arch.apic_access_page));
A
Avi Kivity 已提交
4059

4060 4061 4062
	if (vmx->vpid != 0)
		vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);

4063
	vmx->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
4064
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4065
	vmx_set_cr0(&vmx->vcpu, kvm_read_cr0(vcpu)); /* enter rmode */
4066
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
R
Rusty Russell 已提交
4067 4068 4069 4070
	vmx_set_cr4(&vmx->vcpu, 0);
	vmx_set_efer(&vmx->vcpu, 0);
	vmx_fpu_activate(&vmx->vcpu);
	update_exception_bitmap(&vmx->vcpu);
A
Avi Kivity 已提交
4071

4072
	vpid_sync_context(vmx);
4073

4074
	ret = 0;
A
Avi Kivity 已提交
4075

4076 4077 4078
	/* HACK: Don't enable emulation on guest boot/reset */
	vmx->emulation_required = 0;

A
Avi Kivity 已提交
4079 4080 4081
	return ret;
}

4082 4083 4084 4085 4086 4087 4088 4089 4090 4091
/*
 * In nested virtualization, check if L1 asked to exit on external interrupts.
 * For most existing hypervisors, this will always return true.
 */
static bool nested_exit_on_intr(struct kvm_vcpu *vcpu)
{
	return get_vmcs12(vcpu)->pin_based_vm_exec_control &
		PIN_BASED_EXT_INTR_MASK;
}

4092 4093 4094
static void enable_irq_window(struct kvm_vcpu *vcpu)
{
	u32 cpu_based_vm_exec_control;
4095 4096 4097 4098 4099
	if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) {
		/*
		 * We get here if vmx_interrupt_allowed() said we can't
		 * inject to L1 now because L2 must run. Ask L2 to exit
		 * right after entry, so we can inject to L1 more promptly.
4100
		 */
4101
		kvm_make_request(KVM_REQ_IMMEDIATE_EXIT, vcpu);
4102
		return;
4103
	}
4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118

	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

static void enable_nmi_window(struct kvm_vcpu *vcpu)
{
	u32 cpu_based_vm_exec_control;

	if (!cpu_has_virtual_nmis()) {
		enable_irq_window(vcpu);
		return;
	}

4119 4120 4121 4122
	if (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
		enable_irq_window(vcpu);
		return;
	}
4123 4124 4125 4126 4127
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

4128
static void vmx_inject_irq(struct kvm_vcpu *vcpu)
4129
{
4130
	struct vcpu_vmx *vmx = to_vmx(vcpu);
4131 4132
	uint32_t intr;
	int irq = vcpu->arch.interrupt.nr;
4133

4134
	trace_kvm_inj_virq(irq);
F
Feng (Eric) Liu 已提交
4135

4136
	++vcpu->stat.irq_injections;
4137
	if (vmx->rmode.vm86_active) {
4138 4139 4140 4141
		int inc_eip = 0;
		if (vcpu->arch.interrupt.soft)
			inc_eip = vcpu->arch.event_exit_inst_len;
		if (kvm_inject_realmode_interrupt(vcpu, irq, inc_eip) != EMULATE_DONE)
4142
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4143 4144
		return;
	}
4145 4146 4147 4148 4149 4150 4151 4152
	intr = irq | INTR_INFO_VALID_MASK;
	if (vcpu->arch.interrupt.soft) {
		intr |= INTR_TYPE_SOFT_INTR;
		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
			     vmx->vcpu.arch.event_exit_inst_len);
	} else
		intr |= INTR_TYPE_EXT_INTR;
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
4153 4154
}

4155 4156
static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
{
J
Jan Kiszka 已提交
4157 4158
	struct vcpu_vmx *vmx = to_vmx(vcpu);

4159 4160 4161
	if (is_guest_mode(vcpu))
		return;

4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174
	if (!cpu_has_virtual_nmis()) {
		/*
		 * Tracking the NMI-blocked state in software is built upon
		 * finding the next open IRQ window. This, in turn, depends on
		 * well-behaving guests: They have to keep IRQs disabled at
		 * least as long as the NMI handler runs. Otherwise we may
		 * cause NMI nesting, maybe breaking the guest. But as this is
		 * highly unlikely, we can live with the residual risk.
		 */
		vmx->soft_vnmi_blocked = 1;
		vmx->vnmi_blocked_time = 0;
	}

4175
	++vcpu->stat.nmi_injections;
4176
	vmx->nmi_known_unmasked = false;
4177
	if (vmx->rmode.vm86_active) {
4178
		if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0) != EMULATE_DONE)
4179
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
J
Jan Kiszka 已提交
4180 4181
		return;
	}
4182 4183 4184 4185
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
			INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
}

4186
static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
4187
{
4188
	if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
4189
		return 0;
4190

4191
	return	!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
4192 4193
		  (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
		   | GUEST_INTR_STATE_NMI));
4194 4195
}

J
Jan Kiszka 已提交
4196 4197 4198 4199
static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
{
	if (!cpu_has_virtual_nmis())
		return to_vmx(vcpu)->soft_vnmi_blocked;
4200 4201
	if (to_vmx(vcpu)->nmi_known_unmasked)
		return false;
4202
	return vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)	& GUEST_INTR_STATE_NMI;
J
Jan Kiszka 已提交
4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214
}

static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (!cpu_has_virtual_nmis()) {
		if (vmx->soft_vnmi_blocked != masked) {
			vmx->soft_vnmi_blocked = masked;
			vmx->vnmi_blocked_time = 0;
		}
	} else {
4215
		vmx->nmi_known_unmasked = !masked;
J
Jan Kiszka 已提交
4216 4217 4218 4219 4220 4221 4222 4223 4224
		if (masked)
			vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				      GUEST_INTR_STATE_NMI);
		else
			vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
					GUEST_INTR_STATE_NMI);
	}
}

4225 4226
static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
{
4227
	if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) {
4228 4229 4230 4231
		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
		if (to_vmx(vcpu)->nested.nested_run_pending ||
		    (vmcs12->idt_vectoring_info_field &
		     VECTORING_INFO_VALID_MASK))
4232 4233 4234 4235 4236 4237 4238
			return 0;
		nested_vmx_vmexit(vcpu);
		vmcs12->vm_exit_reason = EXIT_REASON_EXTERNAL_INTERRUPT;
		vmcs12->vm_exit_intr_info = 0;
		/* fall through to normal code, but now in L1, not L2 */
	}

4239 4240 4241
	return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
		!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
			(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
4242 4243
}

4244 4245 4246 4247
static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
{
	int ret;
	struct kvm_userspace_memory_region tss_mem = {
4248
		.slot = TSS_PRIVATE_MEMSLOT,
4249 4250 4251 4252 4253
		.guest_phys_addr = addr,
		.memory_size = PAGE_SIZE * 3,
		.flags = 0,
	};

4254
	ret = kvm_set_memory_region(kvm, &tss_mem, false);
4255 4256
	if (ret)
		return ret;
4257
	kvm->arch.tss_addr = addr;
4258 4259 4260
	if (!init_rmode_tss(kvm))
		return  -ENOMEM;

4261 4262 4263
	return 0;
}

A
Avi Kivity 已提交
4264 4265 4266
static int handle_rmode_exception(struct kvm_vcpu *vcpu,
				  int vec, u32 err_code)
{
4267 4268 4269 4270 4271
	/*
	 * Instruction with address size override prefix opcode 0x67
	 * Cause the #SS fault with 0 error code in VM86 mode.
	 */
	if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
4272
		if (emulate_instruction(vcpu, 0) == EMULATE_DONE)
A
Avi Kivity 已提交
4273
			return 1;
4274 4275 4276 4277 4278 4279 4280
	/*
	 * Forward all other exceptions that are valid in real mode.
	 * FIXME: Breaks guest debugging in real mode, needs to be fixed with
	 *        the required debugging infrastructure rework.
	 */
	switch (vec) {
	case DB_VECTOR:
J
Jan Kiszka 已提交
4281 4282 4283 4284 4285
		if (vcpu->guest_debug &
		    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
			return 0;
		kvm_queue_exception(vcpu, vec);
		return 1;
4286
	case BP_VECTOR:
4287 4288 4289 4290 4291 4292
		/*
		 * Update instruction length as we may reinject the exception
		 * from user space while in guest debugging mode.
		 */
		to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
			vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
J
Jan Kiszka 已提交
4293 4294 4295 4296
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
			return 0;
		/* fall through */
	case DE_VECTOR:
4297 4298 4299 4300 4301 4302 4303 4304 4305 4306
	case OF_VECTOR:
	case BR_VECTOR:
	case UD_VECTOR:
	case DF_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
	case MF_VECTOR:
		kvm_queue_exception(vcpu, vec);
		return 1;
	}
A
Avi Kivity 已提交
4307 4308 4309
	return 0;
}

A
Andi Kleen 已提交
4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328
/*
 * Trigger machine check on the host. We assume all the MSRs are already set up
 * by the CPU and that we still run on the same CPU as the MCE occurred on.
 * We pass a fake environment to the machine check handler because we want
 * the guest to be always treated like user space, no matter what context
 * it used internally.
 */
static void kvm_machine_check(void)
{
#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
	struct pt_regs regs = {
		.cs = 3, /* Fake ring 3 no matter what the guest ran on */
		.flags = X86_EFLAGS_IF,
	};

	do_machine_check(&regs, 0);
#endif
}

A
Avi Kivity 已提交
4329
static int handle_machine_check(struct kvm_vcpu *vcpu)
A
Andi Kleen 已提交
4330 4331 4332 4333 4334
{
	/* already handled by vcpu_run */
	return 1;
}

A
Avi Kivity 已提交
4335
static int handle_exception(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4336
{
4337
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
4338
	struct kvm_run *kvm_run = vcpu->run;
J
Jan Kiszka 已提交
4339
	u32 intr_info, ex_no, error_code;
4340
	unsigned long cr2, rip, dr6;
A
Avi Kivity 已提交
4341 4342 4343
	u32 vect_info;
	enum emulation_result er;

4344
	vect_info = vmx->idt_vectoring_info;
4345
	intr_info = vmx->exit_intr_info;
A
Avi Kivity 已提交
4346

A
Andi Kleen 已提交
4347
	if (is_machine_check(intr_info))
A
Avi Kivity 已提交
4348
		return handle_machine_check(vcpu);
A
Andi Kleen 已提交
4349

4350
	if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR)
4351
		return 1;  /* already handled by vmx_vcpu_run() */
4352 4353

	if (is_no_device(intr_info)) {
4354
		vmx_fpu_activate(vcpu);
4355 4356 4357
		return 1;
	}

4358
	if (is_invalid_opcode(intr_info)) {
4359
		er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
4360
		if (er != EMULATE_DONE)
4361
			kvm_queue_exception(vcpu, UD_VECTOR);
4362 4363 4364
		return 1;
	}

A
Avi Kivity 已提交
4365
	error_code = 0;
4366
	if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
A
Avi Kivity 已提交
4367
		error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383

	/*
	 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
	 * MMIO, it is better to report an internal error.
	 * See the comments in vmx_handle_exit.
	 */
	if ((vect_info & VECTORING_INFO_VALID_MASK) &&
	    !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
		vcpu->run->internal.ndata = 2;
		vcpu->run->internal.data[0] = vect_info;
		vcpu->run->internal.data[1] = intr_info;
		return 0;
	}

A
Avi Kivity 已提交
4384
	if (is_page_fault(intr_info)) {
4385
		/* EPT won't cause page fault directly */
J
Julia Lawall 已提交
4386
		BUG_ON(enable_ept);
A
Avi Kivity 已提交
4387
		cr2 = vmcs_readl(EXIT_QUALIFICATION);
4388 4389
		trace_kvm_page_fault(cr2, error_code);

4390
		if (kvm_event_needs_reinjection(vcpu))
4391
			kvm_mmu_unprotect_page_virt(vcpu, cr2);
4392
		return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0);
A
Avi Kivity 已提交
4393 4394
	}

4395
	if (vmx->rmode.vm86_active &&
A
Avi Kivity 已提交
4396
	    handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
4397
								error_code)) {
4398 4399
		if (vcpu->arch.halt_request) {
			vcpu->arch.halt_request = 0;
4400 4401
			return kvm_emulate_halt(vcpu);
		}
A
Avi Kivity 已提交
4402
		return 1;
4403
	}
A
Avi Kivity 已提交
4404

J
Jan Kiszka 已提交
4405
	ex_no = intr_info & INTR_INFO_VECTOR_MASK;
4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418
	switch (ex_no) {
	case DB_VECTOR:
		dr6 = vmcs_readl(EXIT_QUALIFICATION);
		if (!(vcpu->guest_debug &
		      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
			vcpu->arch.dr6 = dr6 | DR6_FIXED_1;
			kvm_queue_exception(vcpu, DB_VECTOR);
			return 1;
		}
		kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
		kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
		/* fall through */
	case BP_VECTOR:
4419 4420 4421 4422 4423 4424 4425
		/*
		 * Update instruction length as we may reinject #BP from
		 * user space while in guest debugging mode. Reading it for
		 * #DB as well causes no harm, it is not used in that case.
		 */
		vmx->vcpu.arch.event_exit_inst_len =
			vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
A
Avi Kivity 已提交
4426
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
4427
		rip = kvm_rip_read(vcpu);
J
Jan Kiszka 已提交
4428 4429
		kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
		kvm_run->debug.arch.exception = ex_no;
4430 4431
		break;
	default:
J
Jan Kiszka 已提交
4432 4433 4434
		kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
		kvm_run->ex.exception = ex_no;
		kvm_run->ex.error_code = error_code;
4435
		break;
A
Avi Kivity 已提交
4436 4437 4438 4439
	}
	return 0;
}

A
Avi Kivity 已提交
4440
static int handle_external_interrupt(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4441
{
A
Avi Kivity 已提交
4442
	++vcpu->stat.irq_exits;
A
Avi Kivity 已提交
4443 4444 4445
	return 1;
}

A
Avi Kivity 已提交
4446
static int handle_triple_fault(struct kvm_vcpu *vcpu)
4447
{
A
Avi Kivity 已提交
4448
	vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
4449 4450
	return 0;
}
A
Avi Kivity 已提交
4451

A
Avi Kivity 已提交
4452
static int handle_io(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4453
{
4454
	unsigned long exit_qualification;
4455
	int size, in, string;
4456
	unsigned port;
A
Avi Kivity 已提交
4457

4458
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4459
	string = (exit_qualification & 16) != 0;
4460
	in = (exit_qualification & 8) != 0;
4461

4462
	++vcpu->stat.io_exits;
4463

4464
	if (string || in)
4465
		return emulate_instruction(vcpu, 0) == EMULATE_DONE;
4466

4467 4468
	port = exit_qualification >> 16;
	size = (exit_qualification & 7) + 1;
4469
	skip_emulated_instruction(vcpu);
4470 4471

	return kvm_fast_pio_out(vcpu, size, port);
A
Avi Kivity 已提交
4472 4473
}

I
Ingo Molnar 已提交
4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484
static void
vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
{
	/*
	 * Patch in the VMCALL instruction:
	 */
	hypercall[0] = 0x0f;
	hypercall[1] = 0x01;
	hypercall[2] = 0xc1;
}

G
Guo Chao 已提交
4485
/* called to set cr0 as appropriate for a mov-to-cr0 exit. */
4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536
static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
{
	if (to_vmx(vcpu)->nested.vmxon &&
	    ((val & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON))
		return 1;

	if (is_guest_mode(vcpu)) {
		/*
		 * We get here when L2 changed cr0 in a way that did not change
		 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
		 * but did change L0 shadowed bits. This can currently happen
		 * with the TS bit: L0 may want to leave TS on (for lazy fpu
		 * loading) while pretending to allow the guest to change it.
		 */
		if (kvm_set_cr0(vcpu, (val & vcpu->arch.cr0_guest_owned_bits) |
			 (vcpu->arch.cr0 & ~vcpu->arch.cr0_guest_owned_bits)))
			return 1;
		vmcs_writel(CR0_READ_SHADOW, val);
		return 0;
	} else
		return kvm_set_cr0(vcpu, val);
}

static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
{
	if (is_guest_mode(vcpu)) {
		if (kvm_set_cr4(vcpu, (val & vcpu->arch.cr4_guest_owned_bits) |
			 (vcpu->arch.cr4 & ~vcpu->arch.cr4_guest_owned_bits)))
			return 1;
		vmcs_writel(CR4_READ_SHADOW, val);
		return 0;
	} else
		return kvm_set_cr4(vcpu, val);
}

/* called to set cr0 as approriate for clts instruction exit. */
static void handle_clts(struct kvm_vcpu *vcpu)
{
	if (is_guest_mode(vcpu)) {
		/*
		 * We get here when L2 did CLTS, and L1 didn't shadow CR0.TS
		 * but we did (!fpu_active). We need to keep GUEST_CR0.TS on,
		 * just pretend it's off (also in arch.cr0 for fpu_activate).
		 */
		vmcs_writel(CR0_READ_SHADOW,
			vmcs_readl(CR0_READ_SHADOW) & ~X86_CR0_TS);
		vcpu->arch.cr0 &= ~X86_CR0_TS;
	} else
		vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
}

A
Avi Kivity 已提交
4537
static int handle_cr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4538
{
4539
	unsigned long exit_qualification, val;
A
Avi Kivity 已提交
4540 4541
	int cr;
	int reg;
4542
	int err;
A
Avi Kivity 已提交
4543

4544
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
A
Avi Kivity 已提交
4545 4546 4547 4548
	cr = exit_qualification & 15;
	reg = (exit_qualification >> 8) & 15;
	switch ((exit_qualification >> 4) & 3) {
	case 0: /* mov to cr */
4549 4550
		val = kvm_register_read(vcpu, reg);
		trace_kvm_cr_write(cr, val);
A
Avi Kivity 已提交
4551 4552
		switch (cr) {
		case 0:
4553
			err = handle_set_cr0(vcpu, val);
4554
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
4555 4556
			return 1;
		case 3:
4557
			err = kvm_set_cr3(vcpu, val);
4558
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
4559 4560
			return 1;
		case 4:
4561
			err = handle_set_cr4(vcpu, val);
4562
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
4563
			return 1;
4564 4565 4566
		case 8: {
				u8 cr8_prev = kvm_get_cr8(vcpu);
				u8 cr8 = kvm_register_read(vcpu, reg);
A
Andre Przywara 已提交
4567
				err = kvm_set_cr8(vcpu, cr8);
4568
				kvm_complete_insn_gp(vcpu, err);
4569 4570 4571 4572
				if (irqchip_in_kernel(vcpu->kvm))
					return 1;
				if (cr8_prev <= cr8)
					return 1;
A
Avi Kivity 已提交
4573
				vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
4574 4575
				return 0;
			}
4576
		}
A
Avi Kivity 已提交
4577
		break;
4578
	case 2: /* clts */
4579
		handle_clts(vcpu);
4580
		trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
4581
		skip_emulated_instruction(vcpu);
A
Avi Kivity 已提交
4582
		vmx_fpu_activate(vcpu);
4583
		return 1;
A
Avi Kivity 已提交
4584 4585 4586
	case 1: /*mov from cr*/
		switch (cr) {
		case 3:
4587 4588 4589
			val = kvm_read_cr3(vcpu);
			kvm_register_write(vcpu, reg, val);
			trace_kvm_cr_read(cr, val);
A
Avi Kivity 已提交
4590 4591 4592
			skip_emulated_instruction(vcpu);
			return 1;
		case 8:
4593 4594 4595
			val = kvm_get_cr8(vcpu);
			kvm_register_write(vcpu, reg, val);
			trace_kvm_cr_read(cr, val);
A
Avi Kivity 已提交
4596 4597 4598 4599 4600
			skip_emulated_instruction(vcpu);
			return 1;
		}
		break;
	case 3: /* lmsw */
4601
		val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
4602
		trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
4603
		kvm_lmsw(vcpu, val);
A
Avi Kivity 已提交
4604 4605 4606 4607 4608 4609

		skip_emulated_instruction(vcpu);
		return 1;
	default:
		break;
	}
A
Avi Kivity 已提交
4610
	vcpu->run->exit_reason = 0;
4611
	vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
A
Avi Kivity 已提交
4612 4613 4614 4615
	       (int)(exit_qualification >> 4) & 3, cr);
	return 0;
}

A
Avi Kivity 已提交
4616
static int handle_dr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4617
{
4618
	unsigned long exit_qualification;
A
Avi Kivity 已提交
4619 4620
	int dr, reg;

4621
	/* Do not handle if the CPL > 0, will trigger GP on re-entry */
4622 4623
	if (!kvm_require_cpl(vcpu, 0))
		return 1;
4624 4625 4626 4627 4628 4629 4630 4631
	dr = vmcs_readl(GUEST_DR7);
	if (dr & DR7_GD) {
		/*
		 * As the vm-exit takes precedence over the debug trap, we
		 * need to emulate the latter, either for the host or the
		 * guest debugging itself.
		 */
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
A
Avi Kivity 已提交
4632 4633 4634
			vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
			vcpu->run->debug.arch.dr7 = dr;
			vcpu->run->debug.arch.pc =
4635 4636
				vmcs_readl(GUEST_CS_BASE) +
				vmcs_readl(GUEST_RIP);
A
Avi Kivity 已提交
4637 4638
			vcpu->run->debug.arch.exception = DB_VECTOR;
			vcpu->run->exit_reason = KVM_EXIT_DEBUG;
4639 4640 4641 4642 4643 4644 4645 4646 4647 4648
			return 0;
		} else {
			vcpu->arch.dr7 &= ~DR7_GD;
			vcpu->arch.dr6 |= DR6_BD;
			vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
			kvm_queue_exception(vcpu, DB_VECTOR);
			return 1;
		}
	}

4649
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4650 4651 4652
	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
	reg = DEBUG_REG_ACCESS_REG(exit_qualification);
	if (exit_qualification & TYPE_MOV_FROM_DR) {
4653 4654 4655 4656 4657
		unsigned long val;
		if (!kvm_get_dr(vcpu, dr, &val))
			kvm_register_write(vcpu, reg, val);
	} else
		kvm_set_dr(vcpu, dr, vcpu->arch.regs[reg]);
A
Avi Kivity 已提交
4658 4659 4660 4661
	skip_emulated_instruction(vcpu);
	return 1;
}

4662 4663 4664 4665 4666
static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
{
	vmcs_writel(GUEST_DR7, val);
}

A
Avi Kivity 已提交
4667
static int handle_cpuid(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4668
{
4669 4670
	kvm_emulate_cpuid(vcpu);
	return 1;
A
Avi Kivity 已提交
4671 4672
}

A
Avi Kivity 已提交
4673
static int handle_rdmsr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4674
{
4675
	u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
A
Avi Kivity 已提交
4676 4677 4678
	u64 data;

	if (vmx_get_msr(vcpu, ecx, &data)) {
4679
		trace_kvm_msr_read_ex(ecx);
4680
		kvm_inject_gp(vcpu, 0);
A
Avi Kivity 已提交
4681 4682 4683
		return 1;
	}

4684
	trace_kvm_msr_read(ecx, data);
F
Feng (Eric) Liu 已提交
4685

A
Avi Kivity 已提交
4686
	/* FIXME: handling of bits 32:63 of rax, rdx */
4687 4688
	vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u;
	vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
A
Avi Kivity 已提交
4689 4690 4691 4692
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
4693
static int handle_wrmsr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4694
{
4695
	struct msr_data msr;
4696 4697 4698
	u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
	u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
		| ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
A
Avi Kivity 已提交
4699

4700 4701 4702 4703
	msr.data = data;
	msr.index = ecx;
	msr.host_initiated = false;
	if (vmx_set_msr(vcpu, &msr) != 0) {
4704
		trace_kvm_msr_write_ex(ecx, data);
4705
		kvm_inject_gp(vcpu, 0);
A
Avi Kivity 已提交
4706 4707 4708
		return 1;
	}

4709
	trace_kvm_msr_write(ecx, data);
A
Avi Kivity 已提交
4710 4711 4712 4713
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
4714
static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
4715
{
4716
	kvm_make_request(KVM_REQ_EVENT, vcpu);
4717 4718 4719
	return 1;
}

A
Avi Kivity 已提交
4720
static int handle_interrupt_window(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4721
{
4722 4723 4724 4725 4726 4727
	u32 cpu_based_vm_exec_control;

	/* clear pending irq */
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
F
Feng (Eric) Liu 已提交
4728

4729 4730
	kvm_make_request(KVM_REQ_EVENT, vcpu);

4731
	++vcpu->stat.irq_window_exits;
F
Feng (Eric) Liu 已提交
4732

4733 4734 4735 4736
	/*
	 * If the user space waits to inject interrupts, exit as soon as
	 * possible
	 */
4737
	if (!irqchip_in_kernel(vcpu->kvm) &&
A
Avi Kivity 已提交
4738
	    vcpu->run->request_interrupt_window &&
4739
	    !kvm_cpu_has_interrupt(vcpu)) {
A
Avi Kivity 已提交
4740
		vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
4741 4742
		return 0;
	}
A
Avi Kivity 已提交
4743 4744 4745
	return 1;
}

A
Avi Kivity 已提交
4746
static int handle_halt(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4747 4748
{
	skip_emulated_instruction(vcpu);
4749
	return kvm_emulate_halt(vcpu);
A
Avi Kivity 已提交
4750 4751
}

A
Avi Kivity 已提交
4752
static int handle_vmcall(struct kvm_vcpu *vcpu)
4753
{
4754
	skip_emulated_instruction(vcpu);
4755 4756
	kvm_emulate_hypercall(vcpu);
	return 1;
4757 4758
}

4759 4760
static int handle_invd(struct kvm_vcpu *vcpu)
{
4761
	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
4762 4763
}

A
Avi Kivity 已提交
4764
static int handle_invlpg(struct kvm_vcpu *vcpu)
M
Marcelo Tosatti 已提交
4765
{
4766
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
M
Marcelo Tosatti 已提交
4767 4768 4769 4770 4771 4772

	kvm_mmu_invlpg(vcpu, exit_qualification);
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
4773 4774 4775 4776 4777 4778 4779 4780 4781 4782
static int handle_rdpmc(struct kvm_vcpu *vcpu)
{
	int err;

	err = kvm_rdpmc(vcpu);
	kvm_complete_insn_gp(vcpu, err);

	return 1;
}

A
Avi Kivity 已提交
4783
static int handle_wbinvd(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
4784 4785
{
	skip_emulated_instruction(vcpu);
4786
	kvm_emulate_wbinvd(vcpu);
E
Eddie Dong 已提交
4787 4788 4789
	return 1;
}

4790 4791 4792 4793 4794 4795 4796 4797 4798 4799
static int handle_xsetbv(struct kvm_vcpu *vcpu)
{
	u64 new_bv = kvm_read_edx_eax(vcpu);
	u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);

	if (kvm_set_xcr(vcpu, index, new_bv) == 0)
		skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
4800
static int handle_apic_access(struct kvm_vcpu *vcpu)
4801
{
4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819
	if (likely(fasteoi)) {
		unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
		int access_type, offset;

		access_type = exit_qualification & APIC_ACCESS_TYPE;
		offset = exit_qualification & APIC_ACCESS_OFFSET;
		/*
		 * Sane guest uses MOV to write EOI, with written value
		 * not cared. So make a short-circuit here by avoiding
		 * heavy instruction emulation.
		 */
		if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
		    (offset == APIC_EOI)) {
			kvm_lapic_set_eoi(vcpu);
			skip_emulated_instruction(vcpu);
			return 1;
		}
	}
4820
	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
4821 4822
}

A
Avi Kivity 已提交
4823
static int handle_task_switch(struct kvm_vcpu *vcpu)
4824
{
J
Jan Kiszka 已提交
4825
	struct vcpu_vmx *vmx = to_vmx(vcpu);
4826
	unsigned long exit_qualification;
4827 4828
	bool has_error_code = false;
	u32 error_code = 0;
4829
	u16 tss_selector;
4830
	int reason, type, idt_v, idt_index;
4831 4832

	idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
4833
	idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
4834
	type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
4835 4836 4837 4838

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	reason = (u32)exit_qualification >> 30;
4839 4840 4841 4842
	if (reason == TASK_SWITCH_GATE && idt_v) {
		switch (type) {
		case INTR_TYPE_NMI_INTR:
			vcpu->arch.nmi_injected = false;
4843
			vmx_set_nmi_mask(vcpu, true);
4844 4845
			break;
		case INTR_TYPE_EXT_INTR:
4846
		case INTR_TYPE_SOFT_INTR:
4847 4848 4849
			kvm_clear_interrupt_queue(vcpu);
			break;
		case INTR_TYPE_HARD_EXCEPTION:
4850 4851 4852 4853 4854 4855 4856
			if (vmx->idt_vectoring_info &
			    VECTORING_INFO_DELIVER_CODE_MASK) {
				has_error_code = true;
				error_code =
					vmcs_read32(IDT_VECTORING_ERROR_CODE);
			}
			/* fall through */
4857 4858 4859 4860 4861 4862
		case INTR_TYPE_SOFT_EXCEPTION:
			kvm_clear_exception_queue(vcpu);
			break;
		default:
			break;
		}
J
Jan Kiszka 已提交
4863
	}
4864 4865
	tss_selector = exit_qualification;

4866 4867 4868 4869 4870
	if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
		       type != INTR_TYPE_EXT_INTR &&
		       type != INTR_TYPE_NMI_INTR))
		skip_emulated_instruction(vcpu);

4871 4872 4873
	if (kvm_task_switch(vcpu, tss_selector,
			    type == INTR_TYPE_SOFT_INTR ? idt_index : -1, reason,
			    has_error_code, error_code) == EMULATE_FAIL) {
4874 4875 4876
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
4877
		return 0;
4878
	}
4879 4880 4881 4882 4883 4884 4885 4886 4887 4888

	/* clear all local breakpoint enable flags */
	vmcs_writel(GUEST_DR7, vmcs_readl(GUEST_DR7) & ~55);

	/*
	 * TODO: What about debug traps on tss switch?
	 *       Are we supposed to inject them and update dr6?
	 */

	return 1;
4889 4890
}

A
Avi Kivity 已提交
4891
static int handle_ept_violation(struct kvm_vcpu *vcpu)
4892
{
4893
	unsigned long exit_qualification;
4894
	gpa_t gpa;
4895
	u32 error_code;
4896 4897
	int gla_validity;

4898
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4899 4900 4901 4902 4903 4904

	gla_validity = (exit_qualification >> 7) & 0x3;
	if (gla_validity != 0x3 && gla_validity != 0x1 && gla_validity != 0) {
		printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
		printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
			(long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
4905
			vmcs_readl(GUEST_LINEAR_ADDRESS));
4906 4907
		printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
			(long unsigned int)exit_qualification);
A
Avi Kivity 已提交
4908 4909
		vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
		vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
4910
		return 0;
4911 4912 4913
	}

	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
4914
	trace_kvm_page_fault(gpa, exit_qualification);
4915 4916 4917 4918 4919 4920 4921

	/* It is a write fault? */
	error_code = exit_qualification & (1U << 1);
	/* ept page table is present? */
	error_code |= (exit_qualification >> 3) & 0x1;

	return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
4922 4923
}

4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984
static u64 ept_rsvd_mask(u64 spte, int level)
{
	int i;
	u64 mask = 0;

	for (i = 51; i > boot_cpu_data.x86_phys_bits; i--)
		mask |= (1ULL << i);

	if (level > 2)
		/* bits 7:3 reserved */
		mask |= 0xf8;
	else if (level == 2) {
		if (spte & (1ULL << 7))
			/* 2MB ref, bits 20:12 reserved */
			mask |= 0x1ff000;
		else
			/* bits 6:3 reserved */
			mask |= 0x78;
	}

	return mask;
}

static void ept_misconfig_inspect_spte(struct kvm_vcpu *vcpu, u64 spte,
				       int level)
{
	printk(KERN_ERR "%s: spte 0x%llx level %d\n", __func__, spte, level);

	/* 010b (write-only) */
	WARN_ON((spte & 0x7) == 0x2);

	/* 110b (write/execute) */
	WARN_ON((spte & 0x7) == 0x6);

	/* 100b (execute-only) and value not supported by logical processor */
	if (!cpu_has_vmx_ept_execute_only())
		WARN_ON((spte & 0x7) == 0x4);

	/* not 000b */
	if ((spte & 0x7)) {
		u64 rsvd_bits = spte & ept_rsvd_mask(spte, level);

		if (rsvd_bits != 0) {
			printk(KERN_ERR "%s: rsvd_bits = 0x%llx\n",
					 __func__, rsvd_bits);
			WARN_ON(1);
		}

		if (level == 1 || (level == 2 && (spte & (1ULL << 7)))) {
			u64 ept_mem_type = (spte & 0x38) >> 3;

			if (ept_mem_type == 2 || ept_mem_type == 3 ||
			    ept_mem_type == 7) {
				printk(KERN_ERR "%s: ept_mem_type=0x%llx\n",
						__func__, ept_mem_type);
				WARN_ON(1);
			}
		}
	}
}

A
Avi Kivity 已提交
4985
static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
4986 4987
{
	u64 sptes[4];
4988
	int nr_sptes, i, ret;
4989 4990 4991 4992
	gpa_t gpa;

	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);

4993 4994 4995 4996 4997 4998 4999 5000
	ret = handle_mmio_page_fault_common(vcpu, gpa, true);
	if (likely(ret == 1))
		return x86_emulate_instruction(vcpu, gpa, 0, NULL, 0) ==
					      EMULATE_DONE;
	if (unlikely(!ret))
		return 1;

	/* It is the real ept misconfig */
5001 5002 5003 5004 5005 5006 5007 5008
	printk(KERN_ERR "EPT: Misconfiguration.\n");
	printk(KERN_ERR "EPT: GPA: 0x%llx\n", gpa);

	nr_sptes = kvm_mmu_get_spte_hierarchy(vcpu, gpa, sptes);

	for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i)
		ept_misconfig_inspect_spte(vcpu, sptes[i-1], i);

A
Avi Kivity 已提交
5009 5010
	vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
	vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
5011 5012 5013 5014

	return 0;
}

A
Avi Kivity 已提交
5015
static int handle_nmi_window(struct kvm_vcpu *vcpu)
5016 5017 5018 5019 5020 5021 5022 5023
{
	u32 cpu_based_vm_exec_control;

	/* clear pending NMI */
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
	++vcpu->stat.nmi_window_exits;
5024
	kvm_make_request(KVM_REQ_EVENT, vcpu);
5025 5026 5027 5028

	return 1;
}

5029
static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
5030
{
5031 5032
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	enum emulation_result err = EMULATE_DONE;
5033
	int ret = 1;
5034 5035
	u32 cpu_exec_ctrl;
	bool intr_window_requested;
5036
	unsigned count = 130;
5037 5038 5039

	cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
5040

5041
	while (!guest_state_valid(vcpu) && count-- != 0) {
5042
		if (intr_window_requested && vmx_interrupt_allowed(vcpu))
5043 5044
			return handle_interrupt_window(&vmx->vcpu);

5045 5046 5047
		if (test_bit(KVM_REQ_EVENT, &vcpu->requests))
			return 1;

5048
		err = emulate_instruction(vcpu, 0);
5049

5050 5051 5052 5053
		if (err == EMULATE_DO_MMIO) {
			ret = 0;
			goto out;
		}
5054

5055 5056 5057 5058
		if (err != EMULATE_DONE) {
			vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
			vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
			vcpu->run->internal.ndata = 0;
5059
			return 0;
5060
		}
5061 5062

		if (signal_pending(current))
5063
			goto out;
5064 5065 5066 5067
		if (need_resched())
			schedule();
	}

5068
	vmx->emulation_required = !guest_state_valid(vcpu);
5069 5070
out:
	return ret;
5071 5072
}

5073 5074 5075 5076
/*
 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
 * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
 */
5077
static int handle_pause(struct kvm_vcpu *vcpu)
5078 5079 5080 5081 5082 5083 5084
{
	skip_emulated_instruction(vcpu);
	kvm_vcpu_on_spin(vcpu);

	return 1;
}

5085 5086 5087 5088 5089 5090
static int handle_invalid_op(struct kvm_vcpu *vcpu)
{
	kvm_queue_exception(vcpu, UD_VECTOR);
	return 1;
}

5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173
/*
 * To run an L2 guest, we need a vmcs02 based on the L1-specified vmcs12.
 * We could reuse a single VMCS for all the L2 guests, but we also want the
 * option to allocate a separate vmcs02 for each separate loaded vmcs12 - this
 * allows keeping them loaded on the processor, and in the future will allow
 * optimizations where prepare_vmcs02 doesn't need to set all the fields on
 * every entry if they never change.
 * So we keep, in vmx->nested.vmcs02_pool, a cache of size VMCS02_POOL_SIZE
 * (>=0) with a vmcs02 for each recently loaded vmcs12s, most recent first.
 *
 * The following functions allocate and free a vmcs02 in this pool.
 */

/* Get a VMCS from the pool to use as vmcs02 for the current vmcs12. */
static struct loaded_vmcs *nested_get_current_vmcs02(struct vcpu_vmx *vmx)
{
	struct vmcs02_list *item;
	list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
		if (item->vmptr == vmx->nested.current_vmptr) {
			list_move(&item->list, &vmx->nested.vmcs02_pool);
			return &item->vmcs02;
		}

	if (vmx->nested.vmcs02_num >= max(VMCS02_POOL_SIZE, 1)) {
		/* Recycle the least recently used VMCS. */
		item = list_entry(vmx->nested.vmcs02_pool.prev,
			struct vmcs02_list, list);
		item->vmptr = vmx->nested.current_vmptr;
		list_move(&item->list, &vmx->nested.vmcs02_pool);
		return &item->vmcs02;
	}

	/* Create a new VMCS */
	item = (struct vmcs02_list *)
		kmalloc(sizeof(struct vmcs02_list), GFP_KERNEL);
	if (!item)
		return NULL;
	item->vmcs02.vmcs = alloc_vmcs();
	if (!item->vmcs02.vmcs) {
		kfree(item);
		return NULL;
	}
	loaded_vmcs_init(&item->vmcs02);
	item->vmptr = vmx->nested.current_vmptr;
	list_add(&(item->list), &(vmx->nested.vmcs02_pool));
	vmx->nested.vmcs02_num++;
	return &item->vmcs02;
}

/* Free and remove from pool a vmcs02 saved for a vmcs12 (if there is one) */
static void nested_free_vmcs02(struct vcpu_vmx *vmx, gpa_t vmptr)
{
	struct vmcs02_list *item;
	list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
		if (item->vmptr == vmptr) {
			free_loaded_vmcs(&item->vmcs02);
			list_del(&item->list);
			kfree(item);
			vmx->nested.vmcs02_num--;
			return;
		}
}

/*
 * Free all VMCSs saved for this vcpu, except the one pointed by
 * vmx->loaded_vmcs. These include the VMCSs in vmcs02_pool (except the one
 * currently used, if running L2), and vmcs01 when running L2.
 */
static void nested_free_all_saved_vmcss(struct vcpu_vmx *vmx)
{
	struct vmcs02_list *item, *n;
	list_for_each_entry_safe(item, n, &vmx->nested.vmcs02_pool, list) {
		if (vmx->loaded_vmcs != &item->vmcs02)
			free_loaded_vmcs(&item->vmcs02);
		list_del(&item->list);
		kfree(item);
	}
	vmx->nested.vmcs02_num = 0;

	if (vmx->loaded_vmcs != &vmx->vmcs01)
		free_loaded_vmcs(&vmx->vmcs01);
}

5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209
/*
 * Emulate the VMXON instruction.
 * Currently, we just remember that VMX is active, and do not save or even
 * inspect the argument to VMXON (the so-called "VMXON pointer") because we
 * do not currently need to store anything in that guest-allocated memory
 * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
 * argument is different from the VMXON pointer (which the spec says they do).
 */
static int handle_vmon(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	/* The Intel VMX Instruction Reference lists a bunch of bits that
	 * are prerequisite to running VMXON, most notably cr4.VMXE must be
	 * set to 1 (see vmx_set_cr4() for when we allow the guest to set this).
	 * Otherwise, we should fail with #UD. We test these now:
	 */
	if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE) ||
	    !kvm_read_cr0_bits(vcpu, X86_CR0_PE) ||
	    (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	if (is_long_mode(vcpu) && !cs.l) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	if (vmx_get_cpl(vcpu)) {
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

5210 5211 5212
	INIT_LIST_HEAD(&(vmx->nested.vmcs02_pool));
	vmx->nested.vmcs02_num = 0;

5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257
	vmx->nested.vmxon = true;

	skip_emulated_instruction(vcpu);
	return 1;
}

/*
 * Intel's VMX Instruction Reference specifies a common set of prerequisites
 * for running VMX instructions (except VMXON, whose prerequisites are
 * slightly different). It also specifies what exception to inject otherwise.
 */
static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (!vmx->nested.vmxon) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 0;
	}

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	if ((vmx_get_rflags(vcpu) & X86_EFLAGS_VM) ||
	    (is_long_mode(vcpu) && !cs.l)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 0;
	}

	if (vmx_get_cpl(vcpu)) {
		kvm_inject_gp(vcpu, 0);
		return 0;
	}

	return 1;
}

/*
 * Free whatever needs to be freed from vmx->nested when L1 goes down, or
 * just stops using VMX.
 */
static void free_nested(struct vcpu_vmx *vmx)
{
	if (!vmx->nested.vmxon)
		return;
	vmx->nested.vmxon = false;
5258 5259 5260 5261 5262 5263
	if (vmx->nested.current_vmptr != -1ull) {
		kunmap(vmx->nested.current_vmcs12_page);
		nested_release_page(vmx->nested.current_vmcs12_page);
		vmx->nested.current_vmptr = -1ull;
		vmx->nested.current_vmcs12 = NULL;
	}
5264 5265 5266 5267 5268
	/* Unpin physical memory we referred to in current vmcs02 */
	if (vmx->nested.apic_access_page) {
		nested_release_page(vmx->nested.apic_access_page);
		vmx->nested.apic_access_page = 0;
	}
5269 5270

	nested_free_all_saved_vmcss(vmx);
5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282
}

/* Emulate the VMXOFF instruction */
static int handle_vmoff(struct kvm_vcpu *vcpu)
{
	if (!nested_vmx_check_permission(vcpu))
		return 1;
	free_nested(to_vmx(vcpu));
	skip_emulated_instruction(vcpu);
	return 1;
}

5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335
/*
 * Decode the memory-address operand of a vmx instruction, as recorded on an
 * exit caused by such an instruction (run by a guest hypervisor).
 * On success, returns 0. When the operand is invalid, returns 1 and throws
 * #UD or #GP.
 */
static int get_vmx_mem_address(struct kvm_vcpu *vcpu,
				 unsigned long exit_qualification,
				 u32 vmx_instruction_info, gva_t *ret)
{
	/*
	 * According to Vol. 3B, "Information for VM Exits Due to Instruction
	 * Execution", on an exit, vmx_instruction_info holds most of the
	 * addressing components of the operand. Only the displacement part
	 * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
	 * For how an actual address is calculated from all these components,
	 * refer to Vol. 1, "Operand Addressing".
	 */
	int  scaling = vmx_instruction_info & 3;
	int  addr_size = (vmx_instruction_info >> 7) & 7;
	bool is_reg = vmx_instruction_info & (1u << 10);
	int  seg_reg = (vmx_instruction_info >> 15) & 7;
	int  index_reg = (vmx_instruction_info >> 18) & 0xf;
	bool index_is_valid = !(vmx_instruction_info & (1u << 22));
	int  base_reg       = (vmx_instruction_info >> 23) & 0xf;
	bool base_is_valid  = !(vmx_instruction_info & (1u << 27));

	if (is_reg) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	/* Addr = segment_base + offset */
	/* offset = base + [index * scale] + displacement */
	*ret = vmx_get_segment_base(vcpu, seg_reg);
	if (base_is_valid)
		*ret += kvm_register_read(vcpu, base_reg);
	if (index_is_valid)
		*ret += kvm_register_read(vcpu, index_reg)<<scaling;
	*ret += exit_qualification; /* holds the displacement */

	if (addr_size == 1) /* 32 bit */
		*ret &= 0xffffffff;

	/*
	 * TODO: throw #GP (and return 1) in various cases that the VM*
	 * instructions require it - e.g., offset beyond segment limit,
	 * unusable or unreadable/unwritable segment, non-canonical 64-bit
	 * address, and so on. Currently these are not checked.
	 */
	return 0;
}

5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373
/*
 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
 * set the success or error code of an emulated VMX instruction, as specified
 * by Vol 2B, VMX Instruction Reference, "Conventions".
 */
static void nested_vmx_succeed(struct kvm_vcpu *vcpu)
{
	vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
			& ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
			    X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
}

static void nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
{
	vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
			& ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
			    X86_EFLAGS_SF | X86_EFLAGS_OF))
			| X86_EFLAGS_CF);
}

static void nested_vmx_failValid(struct kvm_vcpu *vcpu,
					u32 vm_instruction_error)
{
	if (to_vmx(vcpu)->nested.current_vmptr == -1ull) {
		/*
		 * failValid writes the error number to the current VMCS, which
		 * can't be done there isn't a current VMCS.
		 */
		nested_vmx_failInvalid(vcpu);
		return;
	}
	vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
			& ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
			    X86_EFLAGS_SF | X86_EFLAGS_OF))
			| X86_EFLAGS_ZF);
	get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
}

N
Nadav Har'El 已提交
5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433
/* Emulate the VMCLEAR instruction */
static int handle_vmclear(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	gva_t gva;
	gpa_t vmptr;
	struct vmcs12 *vmcs12;
	struct page *page;
	struct x86_exception e;

	if (!nested_vmx_check_permission(vcpu))
		return 1;

	if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
			vmcs_read32(VMX_INSTRUCTION_INFO), &gva))
		return 1;

	if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &vmptr,
				sizeof(vmptr), &e)) {
		kvm_inject_page_fault(vcpu, &e);
		return 1;
	}

	if (!IS_ALIGNED(vmptr, PAGE_SIZE)) {
		nested_vmx_failValid(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS);
		skip_emulated_instruction(vcpu);
		return 1;
	}

	if (vmptr == vmx->nested.current_vmptr) {
		kunmap(vmx->nested.current_vmcs12_page);
		nested_release_page(vmx->nested.current_vmcs12_page);
		vmx->nested.current_vmptr = -1ull;
		vmx->nested.current_vmcs12 = NULL;
	}

	page = nested_get_page(vcpu, vmptr);
	if (page == NULL) {
		/*
		 * For accurate processor emulation, VMCLEAR beyond available
		 * physical memory should do nothing at all. However, it is
		 * possible that a nested vmx bug, not a guest hypervisor bug,
		 * resulted in this case, so let's shut down before doing any
		 * more damage:
		 */
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
		return 1;
	}
	vmcs12 = kmap(page);
	vmcs12->launch_state = 0;
	kunmap(page);
	nested_release_page(page);

	nested_free_vmcs02(vmx, vmptr);

	skip_emulated_instruction(vcpu);
	nested_vmx_succeed(vcpu);
	return 1;
}

5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448
static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch);

/* Emulate the VMLAUNCH instruction */
static int handle_vmlaunch(struct kvm_vcpu *vcpu)
{
	return nested_vmx_run(vcpu, true);
}

/* Emulate the VMRESUME instruction */
static int handle_vmresume(struct kvm_vcpu *vcpu)
{

	return nested_vmx_run(vcpu, false);
}

5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637
enum vmcs_field_type {
	VMCS_FIELD_TYPE_U16 = 0,
	VMCS_FIELD_TYPE_U64 = 1,
	VMCS_FIELD_TYPE_U32 = 2,
	VMCS_FIELD_TYPE_NATURAL_WIDTH = 3
};

static inline int vmcs_field_type(unsigned long field)
{
	if (0x1 & field)	/* the *_HIGH fields are all 32 bit */
		return VMCS_FIELD_TYPE_U32;
	return (field >> 13) & 0x3 ;
}

static inline int vmcs_field_readonly(unsigned long field)
{
	return (((field >> 10) & 0x3) == 1);
}

/*
 * Read a vmcs12 field. Since these can have varying lengths and we return
 * one type, we chose the biggest type (u64) and zero-extend the return value
 * to that size. Note that the caller, handle_vmread, might need to use only
 * some of the bits we return here (e.g., on 32-bit guests, only 32 bits of
 * 64-bit fields are to be returned).
 */
static inline bool vmcs12_read_any(struct kvm_vcpu *vcpu,
					unsigned long field, u64 *ret)
{
	short offset = vmcs_field_to_offset(field);
	char *p;

	if (offset < 0)
		return 0;

	p = ((char *)(get_vmcs12(vcpu))) + offset;

	switch (vmcs_field_type(field)) {
	case VMCS_FIELD_TYPE_NATURAL_WIDTH:
		*ret = *((natural_width *)p);
		return 1;
	case VMCS_FIELD_TYPE_U16:
		*ret = *((u16 *)p);
		return 1;
	case VMCS_FIELD_TYPE_U32:
		*ret = *((u32 *)p);
		return 1;
	case VMCS_FIELD_TYPE_U64:
		*ret = *((u64 *)p);
		return 1;
	default:
		return 0; /* can never happen. */
	}
}

/*
 * VMX instructions which assume a current vmcs12 (i.e., that VMPTRLD was
 * used before) all generate the same failure when it is missing.
 */
static int nested_vmx_check_vmcs12(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	if (vmx->nested.current_vmptr == -1ull) {
		nested_vmx_failInvalid(vcpu);
		skip_emulated_instruction(vcpu);
		return 0;
	}
	return 1;
}

static int handle_vmread(struct kvm_vcpu *vcpu)
{
	unsigned long field;
	u64 field_value;
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
	gva_t gva = 0;

	if (!nested_vmx_check_permission(vcpu) ||
	    !nested_vmx_check_vmcs12(vcpu))
		return 1;

	/* Decode instruction info and find the field to read */
	field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
	/* Read the field, zero-extended to a u64 field_value */
	if (!vmcs12_read_any(vcpu, field, &field_value)) {
		nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
		skip_emulated_instruction(vcpu);
		return 1;
	}
	/*
	 * Now copy part of this value to register or memory, as requested.
	 * Note that the number of bits actually copied is 32 or 64 depending
	 * on the guest's mode (32 or 64 bit), not on the given field's length.
	 */
	if (vmx_instruction_info & (1u << 10)) {
		kvm_register_write(vcpu, (((vmx_instruction_info) >> 3) & 0xf),
			field_value);
	} else {
		if (get_vmx_mem_address(vcpu, exit_qualification,
				vmx_instruction_info, &gva))
			return 1;
		/* _system ok, as nested_vmx_check_permission verified cpl=0 */
		kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, gva,
			     &field_value, (is_long_mode(vcpu) ? 8 : 4), NULL);
	}

	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}


static int handle_vmwrite(struct kvm_vcpu *vcpu)
{
	unsigned long field;
	gva_t gva;
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
	char *p;
	short offset;
	/* The value to write might be 32 or 64 bits, depending on L1's long
	 * mode, and eventually we need to write that into a field of several
	 * possible lengths. The code below first zero-extends the value to 64
	 * bit (field_value), and then copies only the approriate number of
	 * bits into the vmcs12 field.
	 */
	u64 field_value = 0;
	struct x86_exception e;

	if (!nested_vmx_check_permission(vcpu) ||
	    !nested_vmx_check_vmcs12(vcpu))
		return 1;

	if (vmx_instruction_info & (1u << 10))
		field_value = kvm_register_read(vcpu,
			(((vmx_instruction_info) >> 3) & 0xf));
	else {
		if (get_vmx_mem_address(vcpu, exit_qualification,
				vmx_instruction_info, &gva))
			return 1;
		if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva,
			   &field_value, (is_long_mode(vcpu) ? 8 : 4), &e)) {
			kvm_inject_page_fault(vcpu, &e);
			return 1;
		}
	}


	field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
	if (vmcs_field_readonly(field)) {
		nested_vmx_failValid(vcpu,
			VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
		skip_emulated_instruction(vcpu);
		return 1;
	}

	offset = vmcs_field_to_offset(field);
	if (offset < 0) {
		nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
		skip_emulated_instruction(vcpu);
		return 1;
	}
	p = ((char *) get_vmcs12(vcpu)) + offset;

	switch (vmcs_field_type(field)) {
	case VMCS_FIELD_TYPE_U16:
		*(u16 *)p = field_value;
		break;
	case VMCS_FIELD_TYPE_U32:
		*(u32 *)p = field_value;
		break;
	case VMCS_FIELD_TYPE_U64:
		*(u64 *)p = field_value;
		break;
	case VMCS_FIELD_TYPE_NATURAL_WIDTH:
		*(natural_width *)p = field_value;
		break;
	default:
		nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
		skip_emulated_instruction(vcpu);
		return 1;
	}

	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}

N
Nadav Har'El 已提交
5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697
/* Emulate the VMPTRLD instruction */
static int handle_vmptrld(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	gva_t gva;
	gpa_t vmptr;
	struct x86_exception e;

	if (!nested_vmx_check_permission(vcpu))
		return 1;

	if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
			vmcs_read32(VMX_INSTRUCTION_INFO), &gva))
		return 1;

	if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &vmptr,
				sizeof(vmptr), &e)) {
		kvm_inject_page_fault(vcpu, &e);
		return 1;
	}

	if (!IS_ALIGNED(vmptr, PAGE_SIZE)) {
		nested_vmx_failValid(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS);
		skip_emulated_instruction(vcpu);
		return 1;
	}

	if (vmx->nested.current_vmptr != vmptr) {
		struct vmcs12 *new_vmcs12;
		struct page *page;
		page = nested_get_page(vcpu, vmptr);
		if (page == NULL) {
			nested_vmx_failInvalid(vcpu);
			skip_emulated_instruction(vcpu);
			return 1;
		}
		new_vmcs12 = kmap(page);
		if (new_vmcs12->revision_id != VMCS12_REVISION) {
			kunmap(page);
			nested_release_page_clean(page);
			nested_vmx_failValid(vcpu,
				VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
			skip_emulated_instruction(vcpu);
			return 1;
		}
		if (vmx->nested.current_vmptr != -1ull) {
			kunmap(vmx->nested.current_vmcs12_page);
			nested_release_page(vmx->nested.current_vmcs12_page);
		}

		vmx->nested.current_vmptr = vmptr;
		vmx->nested.current_vmcs12 = new_vmcs12;
		vmx->nested.current_vmcs12_page = page;
	}

	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}

N
Nadav Har'El 已提交
5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723
/* Emulate the VMPTRST instruction */
static int handle_vmptrst(struct kvm_vcpu *vcpu)
{
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
	gva_t vmcs_gva;
	struct x86_exception e;

	if (!nested_vmx_check_permission(vcpu))
		return 1;

	if (get_vmx_mem_address(vcpu, exit_qualification,
			vmx_instruction_info, &vmcs_gva))
		return 1;
	/* ok to use *_system, as nested_vmx_check_permission verified cpl=0 */
	if (kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, vmcs_gva,
				 (void *)&to_vmx(vcpu)->nested.current_vmptr,
				 sizeof(u64), &e)) {
		kvm_inject_page_fault(vcpu, &e);
		return 1;
	}
	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
5724 5725 5726 5727 5728
/*
 * The exit handlers return 1 if the exit was handled fully and guest execution
 * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
 * to be done to userspace and return 0.
 */
5729
static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
A
Avi Kivity 已提交
5730 5731
	[EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
	[EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
5732
	[EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
5733
	[EXIT_REASON_NMI_WINDOW]	      = handle_nmi_window,
A
Avi Kivity 已提交
5734 5735 5736 5737 5738 5739 5740 5741
	[EXIT_REASON_IO_INSTRUCTION]          = handle_io,
	[EXIT_REASON_CR_ACCESS]               = handle_cr,
	[EXIT_REASON_DR_ACCESS]               = handle_dr,
	[EXIT_REASON_CPUID]                   = handle_cpuid,
	[EXIT_REASON_MSR_READ]                = handle_rdmsr,
	[EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
	[EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
	[EXIT_REASON_HLT]                     = handle_halt,
5742
	[EXIT_REASON_INVD]		      = handle_invd,
M
Marcelo Tosatti 已提交
5743
	[EXIT_REASON_INVLPG]		      = handle_invlpg,
A
Avi Kivity 已提交
5744
	[EXIT_REASON_RDPMC]                   = handle_rdpmc,
5745
	[EXIT_REASON_VMCALL]                  = handle_vmcall,
N
Nadav Har'El 已提交
5746
	[EXIT_REASON_VMCLEAR]	              = handle_vmclear,
5747
	[EXIT_REASON_VMLAUNCH]                = handle_vmlaunch,
N
Nadav Har'El 已提交
5748
	[EXIT_REASON_VMPTRLD]                 = handle_vmptrld,
N
Nadav Har'El 已提交
5749
	[EXIT_REASON_VMPTRST]                 = handle_vmptrst,
5750
	[EXIT_REASON_VMREAD]                  = handle_vmread,
5751
	[EXIT_REASON_VMRESUME]                = handle_vmresume,
5752
	[EXIT_REASON_VMWRITE]                 = handle_vmwrite,
5753 5754
	[EXIT_REASON_VMOFF]                   = handle_vmoff,
	[EXIT_REASON_VMON]                    = handle_vmon,
5755 5756
	[EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
	[EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
E
Eddie Dong 已提交
5757
	[EXIT_REASON_WBINVD]                  = handle_wbinvd,
5758
	[EXIT_REASON_XSETBV]                  = handle_xsetbv,
5759
	[EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
A
Andi Kleen 已提交
5760
	[EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
5761 5762
	[EXIT_REASON_EPT_VIOLATION]	      = handle_ept_violation,
	[EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
5763
	[EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
5764 5765
	[EXIT_REASON_MWAIT_INSTRUCTION]	      = handle_invalid_op,
	[EXIT_REASON_MONITOR_INSTRUCTION]     = handle_invalid_op,
A
Avi Kivity 已提交
5766 5767 5768
};

static const int kvm_vmx_max_exit_handlers =
5769
	ARRAY_SIZE(kvm_vmx_exit_handlers);
A
Avi Kivity 已提交
5770

5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904
/*
 * Return 1 if we should exit from L2 to L1 to handle an MSR access access,
 * rather than handle it ourselves in L0. I.e., check whether L1 expressed
 * disinterest in the current event (read or write a specific MSR) by using an
 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
 */
static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
	struct vmcs12 *vmcs12, u32 exit_reason)
{
	u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX];
	gpa_t bitmap;

	if (!nested_cpu_has(get_vmcs12(vcpu), CPU_BASED_USE_MSR_BITMAPS))
		return 1;

	/*
	 * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
	 * for the four combinations of read/write and low/high MSR numbers.
	 * First we need to figure out which of the four to use:
	 */
	bitmap = vmcs12->msr_bitmap;
	if (exit_reason == EXIT_REASON_MSR_WRITE)
		bitmap += 2048;
	if (msr_index >= 0xc0000000) {
		msr_index -= 0xc0000000;
		bitmap += 1024;
	}

	/* Then read the msr_index'th bit from this bitmap: */
	if (msr_index < 1024*8) {
		unsigned char b;
		kvm_read_guest(vcpu->kvm, bitmap + msr_index/8, &b, 1);
		return 1 & (b >> (msr_index & 7));
	} else
		return 1; /* let L1 handle the wrong parameter */
}

/*
 * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
 * rather than handle it ourselves in L0. I.e., check if L1 wanted to
 * intercept (via guest_host_mask etc.) the current event.
 */
static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
	struct vmcs12 *vmcs12)
{
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	int cr = exit_qualification & 15;
	int reg = (exit_qualification >> 8) & 15;
	unsigned long val = kvm_register_read(vcpu, reg);

	switch ((exit_qualification >> 4) & 3) {
	case 0: /* mov to cr */
		switch (cr) {
		case 0:
			if (vmcs12->cr0_guest_host_mask &
			    (val ^ vmcs12->cr0_read_shadow))
				return 1;
			break;
		case 3:
			if ((vmcs12->cr3_target_count >= 1 &&
					vmcs12->cr3_target_value0 == val) ||
				(vmcs12->cr3_target_count >= 2 &&
					vmcs12->cr3_target_value1 == val) ||
				(vmcs12->cr3_target_count >= 3 &&
					vmcs12->cr3_target_value2 == val) ||
				(vmcs12->cr3_target_count >= 4 &&
					vmcs12->cr3_target_value3 == val))
				return 0;
			if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
				return 1;
			break;
		case 4:
			if (vmcs12->cr4_guest_host_mask &
			    (vmcs12->cr4_read_shadow ^ val))
				return 1;
			break;
		case 8:
			if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
				return 1;
			break;
		}
		break;
	case 2: /* clts */
		if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
		    (vmcs12->cr0_read_shadow & X86_CR0_TS))
			return 1;
		break;
	case 1: /* mov from cr */
		switch (cr) {
		case 3:
			if (vmcs12->cpu_based_vm_exec_control &
			    CPU_BASED_CR3_STORE_EXITING)
				return 1;
			break;
		case 8:
			if (vmcs12->cpu_based_vm_exec_control &
			    CPU_BASED_CR8_STORE_EXITING)
				return 1;
			break;
		}
		break;
	case 3: /* lmsw */
		/*
		 * lmsw can change bits 1..3 of cr0, and only set bit 0 of
		 * cr0. Other attempted changes are ignored, with no exit.
		 */
		if (vmcs12->cr0_guest_host_mask & 0xe &
		    (val ^ vmcs12->cr0_read_shadow))
			return 1;
		if ((vmcs12->cr0_guest_host_mask & 0x1) &&
		    !(vmcs12->cr0_read_shadow & 0x1) &&
		    (val & 0x1))
			return 1;
		break;
	}
	return 0;
}

/*
 * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we
 * should handle it ourselves in L0 (and then continue L2). Only call this
 * when in is_guest_mode (L2).
 */
static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
{
	u32 exit_reason = vmcs_read32(VM_EXIT_REASON);
	u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

	if (vmx->nested.nested_run_pending)
		return 0;

	if (unlikely(vmx->fail)) {
5905 5906
		pr_info_ratelimited("%s failed vm entry %x\n", __func__,
				    vmcs_read32(VM_INSTRUCTION_ERROR));
5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993
		return 1;
	}

	switch (exit_reason) {
	case EXIT_REASON_EXCEPTION_NMI:
		if (!is_exception(intr_info))
			return 0;
		else if (is_page_fault(intr_info))
			return enable_ept;
		return vmcs12->exception_bitmap &
				(1u << (intr_info & INTR_INFO_VECTOR_MASK));
	case EXIT_REASON_EXTERNAL_INTERRUPT:
		return 0;
	case EXIT_REASON_TRIPLE_FAULT:
		return 1;
	case EXIT_REASON_PENDING_INTERRUPT:
	case EXIT_REASON_NMI_WINDOW:
		/*
		 * prepare_vmcs02() set the CPU_BASED_VIRTUAL_INTR_PENDING bit
		 * (aka Interrupt Window Exiting) only when L1 turned it on,
		 * so if we got a PENDING_INTERRUPT exit, this must be for L1.
		 * Same for NMI Window Exiting.
		 */
		return 1;
	case EXIT_REASON_TASK_SWITCH:
		return 1;
	case EXIT_REASON_CPUID:
		return 1;
	case EXIT_REASON_HLT:
		return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
	case EXIT_REASON_INVD:
		return 1;
	case EXIT_REASON_INVLPG:
		return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
	case EXIT_REASON_RDPMC:
		return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
	case EXIT_REASON_RDTSC:
		return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
	case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
	case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
	case EXIT_REASON_VMPTRST: case EXIT_REASON_VMREAD:
	case EXIT_REASON_VMRESUME: case EXIT_REASON_VMWRITE:
	case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
		/*
		 * VMX instructions trap unconditionally. This allows L1 to
		 * emulate them for its L2 guest, i.e., allows 3-level nesting!
		 */
		return 1;
	case EXIT_REASON_CR_ACCESS:
		return nested_vmx_exit_handled_cr(vcpu, vmcs12);
	case EXIT_REASON_DR_ACCESS:
		return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
	case EXIT_REASON_IO_INSTRUCTION:
		/* TODO: support IO bitmaps */
		return 1;
	case EXIT_REASON_MSR_READ:
	case EXIT_REASON_MSR_WRITE:
		return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
	case EXIT_REASON_INVALID_STATE:
		return 1;
	case EXIT_REASON_MWAIT_INSTRUCTION:
		return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
	case EXIT_REASON_MONITOR_INSTRUCTION:
		return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
	case EXIT_REASON_PAUSE_INSTRUCTION:
		return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
			nested_cpu_has2(vmcs12,
				SECONDARY_EXEC_PAUSE_LOOP_EXITING);
	case EXIT_REASON_MCE_DURING_VMENTRY:
		return 0;
	case EXIT_REASON_TPR_BELOW_THRESHOLD:
		return 1;
	case EXIT_REASON_APIC_ACCESS:
		return nested_cpu_has2(vmcs12,
			SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
	case EXIT_REASON_EPT_VIOLATION:
	case EXIT_REASON_EPT_MISCONFIG:
		return 0;
	case EXIT_REASON_WBINVD:
		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
	case EXIT_REASON_XSETBV:
		return 1;
	default:
		return 1;
	}
}

5994 5995 5996 5997 5998 5999
static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
{
	*info1 = vmcs_readl(EXIT_QUALIFICATION);
	*info2 = vmcs_read32(VM_EXIT_INTR_INFO);
}

A
Avi Kivity 已提交
6000 6001 6002 6003
/*
 * The guest has exited.  See if we can fix it or if we need userspace
 * assistance.
 */
A
Avi Kivity 已提交
6004
static int vmx_handle_exit(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
6005
{
6006
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Andi Kleen 已提交
6007
	u32 exit_reason = vmx->exit_reason;
6008
	u32 vectoring_info = vmx->idt_vectoring_info;
6009

6010 6011 6012
	/* If guest state is invalid, start emulating */
	if (vmx->emulation_required && emulate_invalid_guest_state)
		return handle_invalid_guest_state(vcpu);
6013

6014 6015 6016 6017 6018 6019 6020 6021
	/*
	 * the KVM_REQ_EVENT optimization bit is only on for one entry, and if
	 * we did not inject a still-pending event to L1 now because of
	 * nested_run_pending, we need to re-enable this bit.
	 */
	if (vmx->nested.nested_run_pending)
		kvm_make_request(KVM_REQ_EVENT, vcpu);

6022 6023
	if (!is_guest_mode(vcpu) && (exit_reason == EXIT_REASON_VMLAUNCH ||
	    exit_reason == EXIT_REASON_VMRESUME))
6024 6025 6026 6027 6028 6029 6030 6031 6032
		vmx->nested.nested_run_pending = 1;
	else
		vmx->nested.nested_run_pending = 0;

	if (is_guest_mode(vcpu) && nested_vmx_exit_handled(vcpu)) {
		nested_vmx_vmexit(vcpu);
		return 1;
	}

6033 6034 6035 6036 6037 6038 6039
	if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		vcpu->run->fail_entry.hardware_entry_failure_reason
			= exit_reason;
		return 0;
	}

6040
	if (unlikely(vmx->fail)) {
A
Avi Kivity 已提交
6041 6042
		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		vcpu->run->fail_entry.hardware_entry_failure_reason
6043 6044 6045
			= vmcs_read32(VM_INSTRUCTION_ERROR);
		return 0;
	}
A
Avi Kivity 已提交
6046

6047 6048 6049 6050 6051 6052 6053
	/*
	 * Note:
	 * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
	 * delivery event since it indicates guest is accessing MMIO.
	 * The vm-exit can be triggered again after return to guest that
	 * will cause infinite loop.
	 */
M
Mike Day 已提交
6054
	if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
6055
			(exit_reason != EXIT_REASON_EXCEPTION_NMI &&
J
Jan Kiszka 已提交
6056
			exit_reason != EXIT_REASON_EPT_VIOLATION &&
6057 6058 6059 6060 6061 6062 6063 6064
			exit_reason != EXIT_REASON_TASK_SWITCH)) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
		vcpu->run->internal.ndata = 2;
		vcpu->run->internal.data[0] = vectoring_info;
		vcpu->run->internal.data[1] = exit_reason;
		return 0;
	}
6065

6066 6067 6068
	if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked &&
	    !(is_guest_mode(vcpu) && nested_cpu_has_virtual_nmis(
	                                get_vmcs12(vcpu), vcpu)))) {
6069
		if (vmx_interrupt_allowed(vcpu)) {
6070 6071
			vmx->soft_vnmi_blocked = 0;
		} else if (vmx->vnmi_blocked_time > 1000000000LL &&
6072
			   vcpu->arch.nmi_pending) {
6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085
			/*
			 * This CPU don't support us in finding the end of an
			 * NMI-blocked window if the guest runs with IRQs
			 * disabled. So we pull the trigger after 1 s of
			 * futile waiting, but inform the user about this.
			 */
			printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
			       "state on VCPU %d after 1 s timeout\n",
			       __func__, vcpu->vcpu_id);
			vmx->soft_vnmi_blocked = 0;
		}
	}

A
Avi Kivity 已提交
6086 6087
	if (exit_reason < kvm_vmx_max_exit_handlers
	    && kvm_vmx_exit_handlers[exit_reason])
A
Avi Kivity 已提交
6088
		return kvm_vmx_exit_handlers[exit_reason](vcpu);
A
Avi Kivity 已提交
6089
	else {
A
Avi Kivity 已提交
6090 6091
		vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
		vcpu->run->hw.hardware_exit_reason = exit_reason;
A
Avi Kivity 已提交
6092 6093 6094 6095
	}
	return 0;
}

6096
static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
6097
{
6098
	if (irr == -1 || tpr < irr) {
6099 6100 6101 6102
		vmcs_write32(TPR_THRESHOLD, 0);
		return;
	}

6103
	vmcs_write32(TPR_THRESHOLD, irr);
6104 6105
}

6106
static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
6107
{
6108 6109 6110 6111 6112 6113
	u32 exit_intr_info;

	if (!(vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY
	      || vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI))
		return;

6114
	vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
6115
	exit_intr_info = vmx->exit_intr_info;
A
Andi Kleen 已提交
6116 6117

	/* Handle machine checks before interrupts are enabled */
6118
	if (is_machine_check(exit_intr_info))
A
Andi Kleen 已提交
6119 6120
		kvm_machine_check();

6121
	/* We need to handle NMIs before interrupts are enabled */
6122
	if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
6123 6124
	    (exit_intr_info & INTR_INFO_VALID_MASK)) {
		kvm_before_handle_nmi(&vmx->vcpu);
6125
		asm("int $2");
6126 6127
		kvm_after_handle_nmi(&vmx->vcpu);
	}
6128
}
6129

6130 6131
static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
{
6132
	u32 exit_intr_info;
6133 6134 6135 6136 6137
	bool unblock_nmi;
	u8 vector;
	bool idtv_info_valid;

	idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
6138

6139
	if (cpu_has_virtual_nmis()) {
6140 6141
		if (vmx->nmi_known_unmasked)
			return;
6142 6143 6144 6145 6146
		/*
		 * Can't use vmx->exit_intr_info since we're not sure what
		 * the exit reason is.
		 */
		exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
6147 6148 6149
		unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
		vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
		/*
6150
		 * SDM 3: 27.7.1.2 (September 2008)
6151 6152
		 * Re-set bit "block by NMI" before VM entry if vmexit caused by
		 * a guest IRET fault.
6153 6154 6155 6156 6157
		 * SDM 3: 23.2.2 (September 2008)
		 * Bit 12 is undefined in any of the following cases:
		 *  If the VM exit sets the valid bit in the IDT-vectoring
		 *   information field.
		 *  If the VM exit is due to a double fault.
6158
		 */
6159 6160
		if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
		    vector != DF_VECTOR && !idtv_info_valid)
6161 6162
			vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				      GUEST_INTR_STATE_NMI);
6163 6164 6165 6166
		else
			vmx->nmi_known_unmasked =
				!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
				  & GUEST_INTR_STATE_NMI);
6167 6168 6169
	} else if (unlikely(vmx->soft_vnmi_blocked))
		vmx->vnmi_blocked_time +=
			ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
6170 6171
}

6172 6173 6174 6175
static void __vmx_complete_interrupts(struct vcpu_vmx *vmx,
				      u32 idt_vectoring_info,
				      int instr_len_field,
				      int error_code_field)
6176 6177 6178 6179 6180 6181
{
	u8 vector;
	int type;
	bool idtv_info_valid;

	idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
6182

6183 6184 6185 6186 6187 6188 6189
	vmx->vcpu.arch.nmi_injected = false;
	kvm_clear_exception_queue(&vmx->vcpu);
	kvm_clear_interrupt_queue(&vmx->vcpu);

	if (!idtv_info_valid)
		return;

6190 6191
	kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);

6192 6193
	vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
	type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
6194

6195
	switch (type) {
6196 6197
	case INTR_TYPE_NMI_INTR:
		vmx->vcpu.arch.nmi_injected = true;
6198
		/*
6199
		 * SDM 3: 27.7.1.2 (September 2008)
6200 6201
		 * Clear bit "block by NMI" before VM entry if a NMI
		 * delivery faulted.
6202
		 */
6203
		vmx_set_nmi_mask(&vmx->vcpu, false);
6204 6205
		break;
	case INTR_TYPE_SOFT_EXCEPTION:
6206
		vmx->vcpu.arch.event_exit_inst_len =
6207
			vmcs_read32(instr_len_field);
6208 6209
		/* fall through */
	case INTR_TYPE_HARD_EXCEPTION:
6210
		if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
6211
			u32 err = vmcs_read32(error_code_field);
6212
			kvm_queue_exception_e(&vmx->vcpu, vector, err);
6213 6214
		} else
			kvm_queue_exception(&vmx->vcpu, vector);
6215
		break;
6216 6217
	case INTR_TYPE_SOFT_INTR:
		vmx->vcpu.arch.event_exit_inst_len =
6218
			vmcs_read32(instr_len_field);
6219
		/* fall through */
6220
	case INTR_TYPE_EXT_INTR:
6221 6222
		kvm_queue_interrupt(&vmx->vcpu, vector,
			type == INTR_TYPE_SOFT_INTR);
6223 6224 6225
		break;
	default:
		break;
6226
	}
6227 6228
}

6229 6230
static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
{
6231 6232
	if (is_guest_mode(&vmx->vcpu))
		return;
6233 6234 6235 6236 6237
	__vmx_complete_interrupts(vmx, vmx->idt_vectoring_info,
				  VM_EXIT_INSTRUCTION_LEN,
				  IDT_VECTORING_ERROR_CODE);
}

A
Avi Kivity 已提交
6238 6239
static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
{
6240 6241
	if (is_guest_mode(vcpu))
		return;
A
Avi Kivity 已提交
6242 6243 6244 6245 6246 6247 6248 6249
	__vmx_complete_interrupts(to_vmx(vcpu),
				  vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
				  VM_ENTRY_INSTRUCTION_LEN,
				  VM_ENTRY_EXCEPTION_ERROR_CODE);

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
}

6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267
static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
{
	int i, nr_msrs;
	struct perf_guest_switch_msr *msrs;

	msrs = perf_guest_get_msrs(&nr_msrs);

	if (!msrs)
		return;

	for (i = 0; i < nr_msrs; i++)
		if (msrs[i].host == msrs[i].guest)
			clear_atomic_switch_msr(vmx, msrs[i].msr);
		else
			add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
					msrs[i].host);
}

6268
static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
6269
{
6270
	struct vcpu_vmx *vmx = to_vmx(vcpu);
6271
	unsigned long debugctlmsr;
6272

6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287
	if (is_guest_mode(vcpu) && !vmx->nested.nested_run_pending) {
		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
		if (vmcs12->idt_vectoring_info_field &
				VECTORING_INFO_VALID_MASK) {
			vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
				vmcs12->idt_vectoring_info_field);
			vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
				vmcs12->vm_exit_instruction_len);
			if (vmcs12->idt_vectoring_info_field &
					VECTORING_INFO_DELIVER_CODE_MASK)
				vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
					vmcs12->idt_vectoring_error_code);
		}
	}

6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309
	/* Record the guest's net vcpu time for enforced NMI injections. */
	if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
		vmx->entry_time = ktime_get();

	/* Don't enter VMX if guest state is invalid, let the exit handler
	   start emulation until we arrive back to a valid state */
	if (vmx->emulation_required && emulate_invalid_guest_state)
		return;

	if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
		vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
	if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
		vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);

	/* When single-stepping over STI and MOV SS, we must clear the
	 * corresponding interruptibility bits in the guest state. Otherwise
	 * vmentry fails as it then expects bit 14 (BS) in pending debug
	 * exceptions being set, but that's not correct for the guest debugging
	 * case. */
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vmx_set_interrupt_shadow(vcpu, 0);

6310
	atomic_switch_perf_msrs(vmx);
6311
	debugctlmsr = get_debugctlmsr();
6312

6313
	vmx->__launched = vmx->loaded_vmcs->launched;
6314
	asm(
A
Avi Kivity 已提交
6315
		/* Store host registers */
A
Avi Kivity 已提交
6316 6317 6318 6319
		"push %%" _ASM_DX "; push %%" _ASM_BP ";"
		"push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */
		"push %%" _ASM_CX " \n\t"
		"cmp %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
6320
		"je 1f \n\t"
A
Avi Kivity 已提交
6321
		"mov %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
6322
		__ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
6323
		"1: \n\t"
6324
		/* Reload cr2 if changed */
A
Avi Kivity 已提交
6325 6326 6327
		"mov %c[cr2](%0), %%" _ASM_AX " \n\t"
		"mov %%cr2, %%" _ASM_DX " \n\t"
		"cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t"
6328
		"je 2f \n\t"
A
Avi Kivity 已提交
6329
		"mov %%" _ASM_AX", %%cr2 \n\t"
6330
		"2: \n\t"
A
Avi Kivity 已提交
6331
		/* Check if vmlaunch of vmresume is needed */
6332
		"cmpl $0, %c[launched](%0) \n\t"
A
Avi Kivity 已提交
6333
		/* Load guest registers.  Don't clobber flags. */
A
Avi Kivity 已提交
6334 6335 6336 6337 6338 6339
		"mov %c[rax](%0), %%" _ASM_AX " \n\t"
		"mov %c[rbx](%0), %%" _ASM_BX " \n\t"
		"mov %c[rdx](%0), %%" _ASM_DX " \n\t"
		"mov %c[rsi](%0), %%" _ASM_SI " \n\t"
		"mov %c[rdi](%0), %%" _ASM_DI " \n\t"
		"mov %c[rbp](%0), %%" _ASM_BP " \n\t"
6340
#ifdef CONFIG_X86_64
6341 6342 6343 6344 6345 6346 6347 6348
		"mov %c[r8](%0),  %%r8  \n\t"
		"mov %c[r9](%0),  %%r9  \n\t"
		"mov %c[r10](%0), %%r10 \n\t"
		"mov %c[r11](%0), %%r11 \n\t"
		"mov %c[r12](%0), %%r12 \n\t"
		"mov %c[r13](%0), %%r13 \n\t"
		"mov %c[r14](%0), %%r14 \n\t"
		"mov %c[r15](%0), %%r15 \n\t"
A
Avi Kivity 已提交
6349
#endif
A
Avi Kivity 已提交
6350
		"mov %c[rcx](%0), %%" _ASM_CX " \n\t" /* kills %0 (ecx) */
6351

A
Avi Kivity 已提交
6352
		/* Enter guest mode */
A
Avi Kivity 已提交
6353
		"jne 1f \n\t"
6354
		__ex(ASM_VMX_VMLAUNCH) "\n\t"
A
Avi Kivity 已提交
6355 6356 6357
		"jmp 2f \n\t"
		"1: " __ex(ASM_VMX_VMRESUME) "\n\t"
		"2: "
A
Avi Kivity 已提交
6358
		/* Save guest registers, load host registers, keep flags */
A
Avi Kivity 已提交
6359
		"mov %0, %c[wordsize](%%" _ASM_SP ") \n\t"
6360
		"pop %0 \n\t"
A
Avi Kivity 已提交
6361 6362 6363 6364 6365 6366 6367
		"mov %%" _ASM_AX ", %c[rax](%0) \n\t"
		"mov %%" _ASM_BX ", %c[rbx](%0) \n\t"
		__ASM_SIZE(pop) " %c[rcx](%0) \n\t"
		"mov %%" _ASM_DX ", %c[rdx](%0) \n\t"
		"mov %%" _ASM_SI ", %c[rsi](%0) \n\t"
		"mov %%" _ASM_DI ", %c[rdi](%0) \n\t"
		"mov %%" _ASM_BP ", %c[rbp](%0) \n\t"
6368
#ifdef CONFIG_X86_64
6369 6370 6371 6372 6373 6374 6375 6376
		"mov %%r8,  %c[r8](%0) \n\t"
		"mov %%r9,  %c[r9](%0) \n\t"
		"mov %%r10, %c[r10](%0) \n\t"
		"mov %%r11, %c[r11](%0) \n\t"
		"mov %%r12, %c[r12](%0) \n\t"
		"mov %%r13, %c[r13](%0) \n\t"
		"mov %%r14, %c[r14](%0) \n\t"
		"mov %%r15, %c[r15](%0) \n\t"
A
Avi Kivity 已提交
6377
#endif
A
Avi Kivity 已提交
6378 6379
		"mov %%cr2, %%" _ASM_AX "   \n\t"
		"mov %%" _ASM_AX ", %c[cr2](%0) \n\t"
6380

A
Avi Kivity 已提交
6381
		"pop  %%" _ASM_BP "; pop  %%" _ASM_DX " \n\t"
6382
		"setbe %c[fail](%0) \n\t"
A
Avi Kivity 已提交
6383 6384 6385 6386
		".pushsection .rodata \n\t"
		".global vmx_return \n\t"
		"vmx_return: " _ASM_PTR " 2b \n\t"
		".popsection"
6387
	      : : "c"(vmx), "d"((unsigned long)HOST_RSP),
6388
		[launched]"i"(offsetof(struct vcpu_vmx, __launched)),
6389
		[fail]"i"(offsetof(struct vcpu_vmx, fail)),
6390
		[host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
6391 6392 6393 6394 6395 6396 6397
		[rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
		[rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
		[rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
		[rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
		[rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
		[rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
		[rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
6398
#ifdef CONFIG_X86_64
6399 6400 6401 6402 6403 6404 6405 6406
		[r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
		[r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
		[r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
		[r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
		[r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
		[r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
		[r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
		[r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
A
Avi Kivity 已提交
6407
#endif
6408 6409
		[cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
		[wordsize]"i"(sizeof(ulong))
6410 6411
	      : "cc", "memory"
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
6412
		, "rax", "rbx", "rdi", "rsi"
6413
		, "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
A
Avi Kivity 已提交
6414 6415
#else
		, "eax", "ebx", "edi", "esi"
6416 6417
#endif
	      );
A
Avi Kivity 已提交
6418

6419 6420 6421 6422
	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
	if (debugctlmsr)
		update_debugctlmsr(debugctlmsr);

6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435
#ifndef CONFIG_X86_64
	/*
	 * The sysexit path does not restore ds/es, so we must set them to
	 * a reasonable value ourselves.
	 *
	 * We can't defer this to vmx_load_host_state() since that function
	 * may be executed in interrupt context, which saves and restore segments
	 * around it, nullifying its effect.
	 */
	loadsegment(ds, __USER_DS);
	loadsegment(es, __USER_DS);
#endif

A
Avi Kivity 已提交
6436
	vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
A
Avi Kivity 已提交
6437
				  | (1 << VCPU_EXREG_RFLAGS)
A
Avi Kivity 已提交
6438
				  | (1 << VCPU_EXREG_CPL)
6439
				  | (1 << VCPU_EXREG_PDPTR)
A
Avi Kivity 已提交
6440
				  | (1 << VCPU_EXREG_SEGMENTS)
6441
				  | (1 << VCPU_EXREG_CR3));
6442 6443
	vcpu->arch.regs_dirty = 0;

6444 6445
	vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);

6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456
	if (is_guest_mode(vcpu)) {
		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
		vmcs12->idt_vectoring_info_field = vmx->idt_vectoring_info;
		if (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) {
			vmcs12->idt_vectoring_error_code =
				vmcs_read32(IDT_VECTORING_ERROR_CODE);
			vmcs12->vm_exit_instruction_len =
				vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
		}
	}

6457
	vmx->loaded_vmcs->launched = 1;
6458

6459
	vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
6460
	trace_kvm_exit(vmx->exit_reason, vcpu, KVM_ISA_VMX);
6461 6462 6463

	vmx_complete_atomic_exit(vmx);
	vmx_recover_nmi_blocking(vmx);
6464
	vmx_complete_interrupts(vmx);
A
Avi Kivity 已提交
6465 6466 6467 6468
}

static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
6469 6470
	struct vcpu_vmx *vmx = to_vmx(vcpu);

6471
	free_vpid(vmx);
6472
	free_nested(vmx);
6473
	free_loaded_vmcs(vmx->loaded_vmcs);
R
Rusty Russell 已提交
6474 6475
	kfree(vmx->guest_msrs);
	kvm_vcpu_uninit(vcpu);
6476
	kmem_cache_free(kvm_vcpu_cache, vmx);
A
Avi Kivity 已提交
6477 6478
}

R
Rusty Russell 已提交
6479
static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
A
Avi Kivity 已提交
6480
{
R
Rusty Russell 已提交
6481
	int err;
6482
	struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
6483
	int cpu;
A
Avi Kivity 已提交
6484

6485
	if (!vmx)
R
Rusty Russell 已提交
6486 6487
		return ERR_PTR(-ENOMEM);

6488 6489
	allocate_vpid(vmx);

R
Rusty Russell 已提交
6490 6491 6492
	err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
	if (err)
		goto free_vcpu;
6493

6494
	vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
6495
	err = -ENOMEM;
R
Rusty Russell 已提交
6496 6497 6498
	if (!vmx->guest_msrs) {
		goto uninit_vcpu;
	}
6499

6500 6501 6502
	vmx->loaded_vmcs = &vmx->vmcs01;
	vmx->loaded_vmcs->vmcs = alloc_vmcs();
	if (!vmx->loaded_vmcs->vmcs)
R
Rusty Russell 已提交
6503
		goto free_msrs;
6504 6505 6506 6507 6508
	if (!vmm_exclusive)
		kvm_cpu_vmxon(__pa(per_cpu(vmxarea, raw_smp_processor_id())));
	loaded_vmcs_init(vmx->loaded_vmcs);
	if (!vmm_exclusive)
		kvm_cpu_vmxoff();
6509

6510 6511
	cpu = get_cpu();
	vmx_vcpu_load(&vmx->vcpu, cpu);
Z
Zachary Amsden 已提交
6512
	vmx->vcpu.cpu = cpu;
R
Rusty Russell 已提交
6513
	err = vmx_vcpu_setup(vmx);
R
Rusty Russell 已提交
6514
	vmx_vcpu_put(&vmx->vcpu);
6515
	put_cpu();
R
Rusty Russell 已提交
6516 6517
	if (err)
		goto free_vmcs;
6518
	if (vm_need_virtualize_apic_accesses(kvm))
6519 6520
		err = alloc_apic_access_page(kvm);
		if (err)
6521
			goto free_vmcs;
R
Rusty Russell 已提交
6522

6523 6524 6525 6526
	if (enable_ept) {
		if (!kvm->arch.ept_identity_map_addr)
			kvm->arch.ept_identity_map_addr =
				VMX_EPT_IDENTITY_PAGETABLE_ADDR;
6527
		err = -ENOMEM;
6528 6529
		if (alloc_identity_pagetable(kvm) != 0)
			goto free_vmcs;
6530 6531
		if (!init_rmode_identity_map(kvm))
			goto free_vmcs;
6532
	}
6533

6534 6535 6536
	vmx->nested.current_vmptr = -1ull;
	vmx->nested.current_vmcs12 = NULL;

R
Rusty Russell 已提交
6537 6538 6539
	return &vmx->vcpu;

free_vmcs:
6540
	free_loaded_vmcs(vmx->loaded_vmcs);
R
Rusty Russell 已提交
6541 6542 6543 6544 6545
free_msrs:
	kfree(vmx->guest_msrs);
uninit_vcpu:
	kvm_vcpu_uninit(&vmx->vcpu);
free_vcpu:
6546
	free_vpid(vmx);
6547
	kmem_cache_free(kvm_vcpu_cache, vmx);
R
Rusty Russell 已提交
6548
	return ERR_PTR(err);
A
Avi Kivity 已提交
6549 6550
}

Y
Yang, Sheng 已提交
6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564
static void __init vmx_check_processor_compat(void *rtn)
{
	struct vmcs_config vmcs_conf;

	*(int *)rtn = 0;
	if (setup_vmcs_config(&vmcs_conf) < 0)
		*(int *)rtn = -EIO;
	if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
		printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
				smp_processor_id());
		*(int *)rtn = -EIO;
	}
}

6565 6566 6567 6568 6569
static int get_ept_level(void)
{
	return VMX_EPT_DEFAULT_GAW + 1;
}

6570
static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
S
Sheng Yang 已提交
6571
{
6572 6573
	u64 ret;

6574 6575 6576 6577 6578 6579 6580 6581
	/* For VT-d and EPT combination
	 * 1. MMIO: always map as UC
	 * 2. EPT with VT-d:
	 *   a. VT-d without snooping control feature: can't guarantee the
	 *	result, try to trust guest.
	 *   b. VT-d with snooping control feature: snooping control feature of
	 *	VT-d engine can guarantee the cache correctness. Just set it
	 *	to WB to keep consistent with host. So the same as item 3.
6582
	 * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
6583 6584
	 *    consistent with host MTRR
	 */
6585 6586
	if (is_mmio)
		ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
6587 6588 6589 6590
	else if (vcpu->kvm->arch.iommu_domain &&
		!(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY))
		ret = kvm_get_guest_memory_type(vcpu, gfn) <<
		      VMX_EPT_MT_EPTE_SHIFT;
6591
	else
6592
		ret = (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT)
6593
			| VMX_EPT_IPAT_BIT;
6594 6595

	return ret;
S
Sheng Yang 已提交
6596 6597
}

6598
static int vmx_get_lpage_level(void)
6599
{
6600 6601 6602 6603 6604
	if (enable_ept && !cpu_has_vmx_ept_1g_page())
		return PT_DIRECTORY_LEVEL;
	else
		/* For shadow and EPT supported 1GB page */
		return PT_PDPE_LEVEL;
6605 6606
}

6607 6608
static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
{
6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626
	struct kvm_cpuid_entry2 *best;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u32 exec_control;

	vmx->rdtscp_enabled = false;
	if (vmx_rdtscp_supported()) {
		exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
		if (exec_control & SECONDARY_EXEC_RDTSCP) {
			best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
			if (best && (best->edx & bit(X86_FEATURE_RDTSCP)))
				vmx->rdtscp_enabled = true;
			else {
				exec_control &= ~SECONDARY_EXEC_RDTSCP;
				vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
						exec_control);
			}
		}
	}
6627 6628 6629 6630

	/* Exposing INVPCID only when PCID is exposed */
	best = kvm_find_cpuid_entry(vcpu, 0x7, 0);
	if (vmx_invpcid_supported() &&
6631
	    best && (best->ebx & bit(X86_FEATURE_INVPCID)) &&
6632
	    guest_cpuid_has_pcid(vcpu)) {
6633
		exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
6634 6635 6636 6637
		exec_control |= SECONDARY_EXEC_ENABLE_INVPCID;
		vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
			     exec_control);
	} else {
6638 6639 6640 6641 6642 6643
		if (cpu_has_secondary_exec_ctrls()) {
			exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
			exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
			vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
				     exec_control);
		}
6644
		if (best)
6645
			best->ebx &= ~bit(X86_FEATURE_INVPCID);
6646
	}
6647 6648
}

6649 6650
static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
{
6651 6652
	if (func == 1 && nested)
		entry->ecx |= bit(X86_FEATURE_VMX);
6653 6654
}

6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847
/*
 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
 * with L0's requirements for its guest (a.k.a. vmsc01), so we can run the L2
 * guest in a way that will both be appropriate to L1's requests, and our
 * needs. In addition to modifying the active vmcs (which is vmcs02), this
 * function also has additional necessary side-effects, like setting various
 * vcpu->arch fields.
 */
static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u32 exec_control;

	vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
	vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
	vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
	vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
	vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
	vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
	vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
	vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
	vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
	vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
	vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
	vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
	vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
	vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
	vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
	vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
	vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
	vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
	vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
	vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
	vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
	vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
	vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
	vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
	vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
	vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
	vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
	vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
	vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
	vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
	vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
	vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
	vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
	vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
	vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
	vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);

	vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
		vmcs12->vm_entry_intr_info_field);
	vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
		vmcs12->vm_entry_exception_error_code);
	vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
		vmcs12->vm_entry_instruction_len);
	vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
		vmcs12->guest_interruptibility_info);
	vmcs_write32(GUEST_ACTIVITY_STATE, vmcs12->guest_activity_state);
	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
	vmcs_writel(GUEST_DR7, vmcs12->guest_dr7);
	vmcs_writel(GUEST_RFLAGS, vmcs12->guest_rflags);
	vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
		vmcs12->guest_pending_dbg_exceptions);
	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
	vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);

	vmcs_write64(VMCS_LINK_POINTER, -1ull);

	vmcs_write32(PIN_BASED_VM_EXEC_CONTROL,
		(vmcs_config.pin_based_exec_ctrl |
		 vmcs12->pin_based_vm_exec_control));

	/*
	 * Whether page-faults are trapped is determined by a combination of
	 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
	 * If enable_ept, L0 doesn't care about page faults and we should
	 * set all of these to L1's desires. However, if !enable_ept, L0 does
	 * care about (at least some) page faults, and because it is not easy
	 * (if at all possible?) to merge L0 and L1's desires, we simply ask
	 * to exit on each and every L2 page fault. This is done by setting
	 * MASK=MATCH=0 and (see below) EB.PF=1.
	 * Note that below we don't need special code to set EB.PF beyond the
	 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
	 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
	 * !enable_ept, EB.PF is 1, so the "or" will always be 1.
	 *
	 * A problem with this approach (when !enable_ept) is that L1 may be
	 * injected with more page faults than it asked for. This could have
	 * caused problems, but in practice existing hypervisors don't care.
	 * To fix this, we will need to emulate the PFEC checking (on the L1
	 * page tables), using walk_addr(), when injecting PFs to L1.
	 */
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
		enable_ept ? vmcs12->page_fault_error_code_mask : 0);
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
		enable_ept ? vmcs12->page_fault_error_code_match : 0);

	if (cpu_has_secondary_exec_ctrls()) {
		u32 exec_control = vmx_secondary_exec_control(vmx);
		if (!vmx->rdtscp_enabled)
			exec_control &= ~SECONDARY_EXEC_RDTSCP;
		/* Take the following fields only from vmcs12 */
		exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
		if (nested_cpu_has(vmcs12,
				CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
			exec_control |= vmcs12->secondary_vm_exec_control;

		if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) {
			/*
			 * Translate L1 physical address to host physical
			 * address for vmcs02. Keep the page pinned, so this
			 * physical address remains valid. We keep a reference
			 * to it so we can release it later.
			 */
			if (vmx->nested.apic_access_page) /* shouldn't happen */
				nested_release_page(vmx->nested.apic_access_page);
			vmx->nested.apic_access_page =
				nested_get_page(vcpu, vmcs12->apic_access_addr);
			/*
			 * If translation failed, no matter: This feature asks
			 * to exit when accessing the given address, and if it
			 * can never be accessed, this feature won't do
			 * anything anyway.
			 */
			if (!vmx->nested.apic_access_page)
				exec_control &=
				  ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
			else
				vmcs_write64(APIC_ACCESS_ADDR,
				  page_to_phys(vmx->nested.apic_access_page));
		}

		vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
	}


	/*
	 * Set host-state according to L0's settings (vmcs12 is irrelevant here)
	 * Some constant fields are set here by vmx_set_constant_host_state().
	 * Other fields are different per CPU, and will be set later when
	 * vmx_vcpu_load() is called, and when vmx_save_host_state() is called.
	 */
	vmx_set_constant_host_state();

	/*
	 * HOST_RSP is normally set correctly in vmx_vcpu_run() just before
	 * entry, but only if the current (host) sp changed from the value
	 * we wrote last (vmx->host_rsp). This cache is no longer relevant
	 * if we switch vmcs, and rather than hold a separate cache per vmcs,
	 * here we just force the write to happen on entry.
	 */
	vmx->host_rsp = 0;

	exec_control = vmx_exec_control(vmx); /* L0's desires */
	exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
	exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
	exec_control &= ~CPU_BASED_TPR_SHADOW;
	exec_control |= vmcs12->cpu_based_vm_exec_control;
	/*
	 * Merging of IO and MSR bitmaps not currently supported.
	 * Rather, exit every time.
	 */
	exec_control &= ~CPU_BASED_USE_MSR_BITMAPS;
	exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
	exec_control |= CPU_BASED_UNCOND_IO_EXITING;

	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);

	/* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
	 * bitwise-or of what L1 wants to trap for L2, and what we want to
	 * trap. Note that CR0.TS also needs updating - we do this later.
	 */
	update_exception_bitmap(vcpu);
	vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);

	/* Note: IA32_MODE, LOAD_IA32_EFER are modified by vmx_set_efer below */
	vmcs_write32(VM_EXIT_CONTROLS,
		vmcs12->vm_exit_controls | vmcs_config.vmexit_ctrl);
	vmcs_write32(VM_ENTRY_CONTROLS, vmcs12->vm_entry_controls |
		(vmcs_config.vmentry_ctrl & ~VM_ENTRY_IA32E_MODE));

	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)
		vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
	else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
		vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);


	set_cr4_guest_host_mask(vmx);

6848 6849 6850 6851 6852
	if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
		vmcs_write64(TSC_OFFSET,
			vmx->nested.vmcs01_tsc_offset + vmcs12->tsc_offset);
	else
		vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset);
6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894

	if (enable_vpid) {
		/*
		 * Trivially support vpid by letting L2s share their parent
		 * L1's vpid. TODO: move to a more elaborate solution, giving
		 * each L2 its own vpid and exposing the vpid feature to L1.
		 */
		vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
		vmx_flush_tlb(vcpu);
	}

	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)
		vcpu->arch.efer = vmcs12->guest_ia32_efer;
	if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
		vcpu->arch.efer |= (EFER_LMA | EFER_LME);
	else
		vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
	/* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
	vmx_set_efer(vcpu, vcpu->arch.efer);

	/*
	 * This sets GUEST_CR0 to vmcs12->guest_cr0, with possibly a modified
	 * TS bit (for lazy fpu) and bits which we consider mandatory enabled.
	 * The CR0_READ_SHADOW is what L2 should have expected to read given
	 * the specifications by L1; It's not enough to take
	 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
	 * have more bits than L1 expected.
	 */
	vmx_set_cr0(vcpu, vmcs12->guest_cr0);
	vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));

	vmx_set_cr4(vcpu, vmcs12->guest_cr4);
	vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));

	/* shadow page tables on either EPT or shadow page tables */
	kvm_set_cr3(vcpu, vmcs12->guest_cr3);
	kvm_mmu_reset_context(vcpu);

	kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp);
	kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip);
}

6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912
/*
 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
 * for running an L2 nested guest.
 */
static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
{
	struct vmcs12 *vmcs12;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int cpu;
	struct loaded_vmcs *vmcs02;

	if (!nested_vmx_check_permission(vcpu) ||
	    !nested_vmx_check_vmcs12(vcpu))
		return 1;

	skip_emulated_instruction(vcpu);
	vmcs12 = get_vmcs12(vcpu);

6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946
	/*
	 * The nested entry process starts with enforcing various prerequisites
	 * on vmcs12 as required by the Intel SDM, and act appropriately when
	 * they fail: As the SDM explains, some conditions should cause the
	 * instruction to fail, while others will cause the instruction to seem
	 * to succeed, but return an EXIT_REASON_INVALID_STATE.
	 * To speed up the normal (success) code path, we should avoid checking
	 * for misconfigurations which will anyway be caught by the processor
	 * when using the merged vmcs02.
	 */
	if (vmcs12->launch_state == launch) {
		nested_vmx_failValid(vcpu,
			launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
			       : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
		return 1;
	}

	if ((vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_MSR_BITMAPS) &&
			!IS_ALIGNED(vmcs12->msr_bitmap, PAGE_SIZE)) {
		/*TODO: Also verify bits beyond physical address width are 0*/
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
			!IS_ALIGNED(vmcs12->apic_access_addr, PAGE_SIZE)) {
		/*TODO: Also verify bits beyond physical address width are 0*/
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

	if (vmcs12->vm_entry_msr_load_count > 0 ||
	    vmcs12->vm_exit_msr_load_count > 0 ||
	    vmcs12->vm_exit_msr_store_count > 0) {
6947 6948
		pr_warn_ratelimited("%s: VMCS MSR_{LOAD,STORE} unsupported\n",
				    __func__);
6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

	if (!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
	      nested_vmx_procbased_ctls_low, nested_vmx_procbased_ctls_high) ||
	    !vmx_control_verify(vmcs12->secondary_vm_exec_control,
	      nested_vmx_secondary_ctls_low, nested_vmx_secondary_ctls_high) ||
	    !vmx_control_verify(vmcs12->pin_based_vm_exec_control,
	      nested_vmx_pinbased_ctls_low, nested_vmx_pinbased_ctls_high) ||
	    !vmx_control_verify(vmcs12->vm_exit_controls,
	      nested_vmx_exit_ctls_low, nested_vmx_exit_ctls_high) ||
	    !vmx_control_verify(vmcs12->vm_entry_controls,
	      nested_vmx_entry_ctls_low, nested_vmx_entry_ctls_high))
	{
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

	if (((vmcs12->host_cr0 & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON) ||
	    ((vmcs12->host_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) {
		nested_vmx_failValid(vcpu,
			VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
		return 1;
	}

	if (((vmcs12->guest_cr0 & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON) ||
	    ((vmcs12->guest_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) {
		nested_vmx_entry_failure(vcpu, vmcs12,
			EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
		return 1;
	}
	if (vmcs12->vmcs_link_pointer != -1ull) {
		nested_vmx_entry_failure(vcpu, vmcs12,
			EXIT_REASON_INVALID_STATE, ENTRY_FAIL_VMCS_LINK_PTR);
		return 1;
	}

	/*
	 * We're finally done with prerequisite checking, and can start with
	 * the nested entry.
	 */

6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7017 7018 7019
	vmcs02 = nested_get_current_vmcs02(vmx);
	if (!vmcs02)
		return -ENOMEM;

	enter_guest_mode(vcpu);

	vmx->nested.vmcs01_tsc_offset = vmcs_read64(TSC_OFFSET);

	cpu = get_cpu();
	vmx->loaded_vmcs = vmcs02;
	vmx_vcpu_put(vcpu);
	vmx_vcpu_load(vcpu, cpu);
	vcpu->cpu = cpu;
	put_cpu();

	vmcs12->launch_state = 1;

	prepare_vmcs02(vcpu, vmcs12);

	/*
	 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
	 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
	 * returned as far as L1 is concerned. It will only return (and set
	 * the success flag) when L2 exits (see nested_vmx_vmexit()).
	 */
	return 1;
}

N
Nadav Har'El 已提交
7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257
/*
 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
 * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK).
 * This function returns the new value we should put in vmcs12.guest_cr0.
 * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
 *  1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
 *     available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
 *     didn't trap the bit, because if L1 did, so would L0).
 *  2. Bits that L1 asked to trap (and therefore L0 also did) could not have
 *     been modified by L2, and L1 knows it. So just leave the old value of
 *     the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
 *     isn't relevant, because if L0 traps this bit it can set it to anything.
 *  3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
 *     changed these bits, and therefore they need to be updated, but L0
 *     didn't necessarily allow them to be changed in GUEST_CR0 - and rather
 *     put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
 */
static inline unsigned long
vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	return
	/*1*/	(vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
	/*2*/	(vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
	/*3*/	(vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
			vcpu->arch.cr0_guest_owned_bits));
}

static inline unsigned long
vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	return
	/*1*/	(vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
	/*2*/	(vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
	/*3*/	(vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
			vcpu->arch.cr4_guest_owned_bits));
}

/*
 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
 * and this function updates it to reflect the changes to the guest state while
 * L2 was running (and perhaps made some exits which were handled directly by L0
 * without going back to L1), and to reflect the exit reason.
 * Note that we do not have to copy here all VMCS fields, just those that
 * could have changed by the L2 guest or the exit - i.e., the guest-state and
 * exit-information fields only. Other fields are modified by L1 with VMWRITE,
 * which already writes to vmcs12 directly.
 */
void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	/* update guest state fields: */
	vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
	vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);

	kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
	vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
	vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP);
	vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);

	vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
	vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
	vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
	vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
	vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
	vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
	vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
	vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
	vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
	vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
	vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
	vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
	vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
	vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
	vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
	vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
	vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
	vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
	vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
	vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
	vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
	vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
	vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
	vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
	vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
	vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
	vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
	vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
	vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
	vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
	vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
	vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
	vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
	vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
	vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
	vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);

	vmcs12->guest_activity_state = vmcs_read32(GUEST_ACTIVITY_STATE);
	vmcs12->guest_interruptibility_info =
		vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	vmcs12->guest_pending_dbg_exceptions =
		vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);

	/* TODO: These cannot have changed unless we have MSR bitmaps and
	 * the relevant bit asks not to trap the change */
	vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
	if (vmcs12->vm_entry_controls & VM_EXIT_SAVE_IA32_PAT)
		vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT);
	vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS);
	vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP);
	vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP);

	/* update exit information fields: */

	vmcs12->vm_exit_reason  = vmcs_read32(VM_EXIT_REASON);
	vmcs12->exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	vmcs12->vm_exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
	vmcs12->vm_exit_intr_error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
	vmcs12->idt_vectoring_info_field =
		vmcs_read32(IDT_VECTORING_INFO_FIELD);
	vmcs12->idt_vectoring_error_code =
		vmcs_read32(IDT_VECTORING_ERROR_CODE);
	vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
	vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);

	/* clear vm-entry fields which are to be cleared on exit */
	if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
		vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
}

/*
 * A part of what we need to when the nested L2 guest exits and we want to
 * run its L1 parent, is to reset L1's guest state to the host state specified
 * in vmcs12.
 * This function is to be called not only on normal nested exit, but also on
 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
 * Failures During or After Loading Guest State").
 * This function should be called when the active VMCS is L1's (vmcs01).
 */
void load_vmcs12_host_state(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
		vcpu->arch.efer = vmcs12->host_ia32_efer;
	if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
		vcpu->arch.efer |= (EFER_LMA | EFER_LME);
	else
		vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
	vmx_set_efer(vcpu, vcpu->arch.efer);

	kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp);
	kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip);
	/*
	 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
	 * actually changed, because it depends on the current state of
	 * fpu_active (which may have changed).
	 * Note that vmx_set_cr0 refers to efer set above.
	 */
	kvm_set_cr0(vcpu, vmcs12->host_cr0);
	/*
	 * If we did fpu_activate()/fpu_deactivate() during L2's run, we need
	 * to apply the same changes to L1's vmcs. We just set cr0 correctly,
	 * but we also need to update cr0_guest_host_mask and exception_bitmap.
	 */
	update_exception_bitmap(vcpu);
	vcpu->arch.cr0_guest_owned_bits = (vcpu->fpu_active ? X86_CR0_TS : 0);
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);

	/*
	 * Note that CR4_GUEST_HOST_MASK is already set in the original vmcs01
	 * (KVM doesn't change it)- no reason to call set_cr4_guest_host_mask();
	 */
	vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
	kvm_set_cr4(vcpu, vmcs12->host_cr4);

	/* shadow page tables on either EPT or shadow page tables */
	kvm_set_cr3(vcpu, vmcs12->host_cr3);
	kvm_mmu_reset_context(vcpu);

	if (enable_vpid) {
		/*
		 * Trivially support vpid by letting L2s share their parent
		 * L1's vpid. TODO: move to a more elaborate solution, giving
		 * each L2 its own vpid and exposing the vpid feature to L1.
		 */
		vmx_flush_tlb(vcpu);
	}


	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
	vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
	vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
	vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);
	vmcs_writel(GUEST_TR_BASE, vmcs12->host_tr_base);
	vmcs_writel(GUEST_GS_BASE, vmcs12->host_gs_base);
	vmcs_writel(GUEST_FS_BASE, vmcs12->host_fs_base);
	vmcs_write16(GUEST_ES_SELECTOR, vmcs12->host_es_selector);
	vmcs_write16(GUEST_CS_SELECTOR, vmcs12->host_cs_selector);
	vmcs_write16(GUEST_SS_SELECTOR, vmcs12->host_ss_selector);
	vmcs_write16(GUEST_DS_SELECTOR, vmcs12->host_ds_selector);
	vmcs_write16(GUEST_FS_SELECTOR, vmcs12->host_fs_selector);
	vmcs_write16(GUEST_GS_SELECTOR, vmcs12->host_gs_selector);
	vmcs_write16(GUEST_TR_SELECTOR, vmcs12->host_tr_selector);

	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT)
		vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
		vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL,
			vmcs12->host_ia32_perf_global_ctrl);
}

/*
 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
 * and modify vmcs12 to make it see what it would expect to see there if
 * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
 */
static void nested_vmx_vmexit(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int cpu;
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

	leave_guest_mode(vcpu);
	prepare_vmcs12(vcpu, vmcs12);

	cpu = get_cpu();
	vmx->loaded_vmcs = &vmx->vmcs01;
	vmx_vcpu_put(vcpu);
	vmx_vcpu_load(vcpu, cpu);
	vcpu->cpu = cpu;
	put_cpu();

	/* if no vmcs02 cache requested, remove the one we used */
	if (VMCS02_POOL_SIZE == 0)
		nested_free_vmcs02(vmx, vmx->nested.current_vmptr);

	load_vmcs12_host_state(vcpu, vmcs12);

7258
	/* Update TSC_OFFSET if TSC was changed while L2 ran */
N
Nadav Har'El 已提交
7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281
	vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset);

	/* This is needed for same reason as it was needed in prepare_vmcs02 */
	vmx->host_rsp = 0;

	/* Unpin physical memory we referred to in vmcs02 */
	if (vmx->nested.apic_access_page) {
		nested_release_page(vmx->nested.apic_access_page);
		vmx->nested.apic_access_page = 0;
	}

	/*
	 * Exiting from L2 to L1, we're now back to L1 which thinks it just
	 * finished a VMLAUNCH or VMRESUME instruction, so we need to set the
	 * success or failure flag accordingly.
	 */
	if (unlikely(vmx->fail)) {
		vmx->fail = 0;
		nested_vmx_failValid(vcpu, vmcs_read32(VM_INSTRUCTION_ERROR));
	} else
		nested_vmx_succeed(vcpu);
}

7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298
/*
 * L1's failure to enter L2 is a subset of a normal exit, as explained in
 * 23.7 "VM-entry failures during or after loading guest state" (this also
 * lists the acceptable exit-reason and exit-qualification parameters).
 * It should only be called before L2 actually succeeded to run, and when
 * vmcs01 is current (it doesn't leave_guest_mode() or switch vmcss).
 */
static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
			struct vmcs12 *vmcs12,
			u32 reason, unsigned long qualification)
{
	load_vmcs12_host_state(vcpu, vmcs12);
	vmcs12->vm_exit_reason = reason | VMX_EXIT_REASONS_FAILED_VMENTRY;
	vmcs12->exit_qualification = qualification;
	nested_vmx_succeed(vcpu);
}

7299 7300 7301 7302 7303 7304 7305
static int vmx_check_intercept(struct kvm_vcpu *vcpu,
			       struct x86_instruction_info *info,
			       enum x86_intercept_stage stage)
{
	return X86EMUL_CONTINUE;
}

7306
static struct kvm_x86_ops vmx_x86_ops = {
A
Avi Kivity 已提交
7307 7308 7309 7310
	.cpu_has_kvm_support = cpu_has_kvm_support,
	.disabled_by_bios = vmx_disabled_by_bios,
	.hardware_setup = hardware_setup,
	.hardware_unsetup = hardware_unsetup,
Y
Yang, Sheng 已提交
7311
	.check_processor_compatibility = vmx_check_processor_compat,
A
Avi Kivity 已提交
7312 7313
	.hardware_enable = hardware_enable,
	.hardware_disable = hardware_disable,
7314
	.cpu_has_accelerated_tpr = report_flexpriority,
A
Avi Kivity 已提交
7315 7316 7317

	.vcpu_create = vmx_create_vcpu,
	.vcpu_free = vmx_free_vcpu,
7318
	.vcpu_reset = vmx_vcpu_reset,
A
Avi Kivity 已提交
7319

7320
	.prepare_guest_switch = vmx_save_host_state,
A
Avi Kivity 已提交
7321 7322 7323
	.vcpu_load = vmx_vcpu_load,
	.vcpu_put = vmx_vcpu_put,

7324
	.update_db_bp_intercept = update_exception_bitmap,
A
Avi Kivity 已提交
7325 7326 7327 7328 7329
	.get_msr = vmx_get_msr,
	.set_msr = vmx_set_msr,
	.get_segment_base = vmx_get_segment_base,
	.get_segment = vmx_get_segment,
	.set_segment = vmx_set_segment,
7330
	.get_cpl = vmx_get_cpl,
A
Avi Kivity 已提交
7331
	.get_cs_db_l_bits = vmx_get_cs_db_l_bits,
7332
	.decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
7333
	.decache_cr3 = vmx_decache_cr3,
7334
	.decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
A
Avi Kivity 已提交
7335 7336 7337 7338 7339 7340 7341 7342
	.set_cr0 = vmx_set_cr0,
	.set_cr3 = vmx_set_cr3,
	.set_cr4 = vmx_set_cr4,
	.set_efer = vmx_set_efer,
	.get_idt = vmx_get_idt,
	.set_idt = vmx_set_idt,
	.get_gdt = vmx_get_gdt,
	.set_gdt = vmx_set_gdt,
7343
	.set_dr7 = vmx_set_dr7,
7344
	.cache_reg = vmx_cache_reg,
A
Avi Kivity 已提交
7345 7346
	.get_rflags = vmx_get_rflags,
	.set_rflags = vmx_set_rflags,
7347
	.fpu_activate = vmx_fpu_activate,
7348
	.fpu_deactivate = vmx_fpu_deactivate,
A
Avi Kivity 已提交
7349 7350 7351 7352

	.tlb_flush = vmx_flush_tlb,

	.run = vmx_vcpu_run,
7353
	.handle_exit = vmx_handle_exit,
A
Avi Kivity 已提交
7354
	.skip_emulated_instruction = skip_emulated_instruction,
7355 7356
	.set_interrupt_shadow = vmx_set_interrupt_shadow,
	.get_interrupt_shadow = vmx_get_interrupt_shadow,
I
Ingo Molnar 已提交
7357
	.patch_hypercall = vmx_patch_hypercall,
E
Eddie Dong 已提交
7358
	.set_irq = vmx_inject_irq,
7359
	.set_nmi = vmx_inject_nmi,
7360
	.queue_exception = vmx_queue_exception,
A
Avi Kivity 已提交
7361
	.cancel_injection = vmx_cancel_injection,
7362
	.interrupt_allowed = vmx_interrupt_allowed,
7363
	.nmi_allowed = vmx_nmi_allowed,
J
Jan Kiszka 已提交
7364 7365
	.get_nmi_mask = vmx_get_nmi_mask,
	.set_nmi_mask = vmx_set_nmi_mask,
7366 7367 7368 7369
	.enable_nmi_window = enable_nmi_window,
	.enable_irq_window = enable_irq_window,
	.update_cr8_intercept = update_cr8_intercept,

7370
	.set_tss_addr = vmx_set_tss_addr,
7371
	.get_tdp_level = get_ept_level,
7372
	.get_mt_mask = vmx_get_mt_mask,
7373

7374 7375
	.get_exit_info = vmx_get_exit_info,

7376
	.get_lpage_level = vmx_get_lpage_level,
7377 7378

	.cpuid_update = vmx_cpuid_update,
7379 7380

	.rdtscp_supported = vmx_rdtscp_supported,
7381
	.invpcid_supported = vmx_invpcid_supported,
7382 7383

	.set_supported_cpuid = vmx_set_supported_cpuid,
7384 7385

	.has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
7386

7387
	.set_tsc_khz = vmx_set_tsc_khz,
W
Will Auld 已提交
7388
	.read_tsc_offset = vmx_read_tsc_offset,
7389
	.write_tsc_offset = vmx_write_tsc_offset,
Z
Zachary Amsden 已提交
7390
	.adjust_tsc_offset = vmx_adjust_tsc_offset,
7391
	.compute_tsc_offset = vmx_compute_tsc_offset,
N
Nadav Har'El 已提交
7392
	.read_l1_tsc = vmx_read_l1_tsc,
7393 7394

	.set_tdp_cr3 = vmx_set_cr3,
7395 7396

	.check_intercept = vmx_check_intercept,
A
Avi Kivity 已提交
7397 7398 7399 7400
};

static int __init vmx_init(void)
{
7401 7402 7403 7404 7405 7406
	int r, i;

	rdmsrl_safe(MSR_EFER, &host_efer);

	for (i = 0; i < NR_VMX_MSR; ++i)
		kvm_define_shared_msr(i, vmx_msr_index[i]);
7407

7408
	vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
7409 7410 7411
	if (!vmx_io_bitmap_a)
		return -ENOMEM;

G
Guo Chao 已提交
7412 7413
	r = -ENOMEM;

7414
	vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
G
Guo Chao 已提交
7415
	if (!vmx_io_bitmap_b)
7416 7417
		goto out;

7418
	vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
G
Guo Chao 已提交
7419
	if (!vmx_msr_bitmap_legacy)
S
Sheng Yang 已提交
7420
		goto out1;
G
Guo Chao 已提交
7421

S
Sheng Yang 已提交
7422

7423
	vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
G
Guo Chao 已提交
7424
	if (!vmx_msr_bitmap_longmode)
7425
		goto out2;
G
Guo Chao 已提交
7426

7427

7428 7429 7430 7431
	/*
	 * Allow direct access to the PC debug port (it is often used for I/O
	 * delays, but the vmexits simply slow things down).
	 */
7432 7433
	memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
	clear_bit(0x80, vmx_io_bitmap_a);
7434

7435
	memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
7436

7437 7438
	memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
	memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
S
Sheng Yang 已提交
7439

7440 7441
	set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */

7442 7443
	r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
		     __alignof__(struct vcpu_vmx), THIS_MODULE);
7444
	if (r)
7445
		goto out3;
S
Sheng Yang 已提交
7446

7447 7448 7449 7450 7451
#ifdef CONFIG_KEXEC
	rcu_assign_pointer(crash_vmclear_loaded_vmcss,
			   crash_vmclear_local_loaded_vmcss);
#endif

7452 7453 7454 7455 7456 7457
	vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
	vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
	vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
7458

7459
	if (enable_ept) {
7460 7461 7462 7463
		kvm_mmu_set_mask_ptes(0ull,
			(enable_ept_ad_bits) ? VMX_EPT_ACCESS_BIT : 0ull,
			(enable_ept_ad_bits) ? VMX_EPT_DIRTY_BIT : 0ull,
			0ull, VMX_EPT_EXECUTABLE_MASK);
7464
		ept_set_mmio_spte_mask();
7465 7466 7467
		kvm_enable_tdp();
	} else
		kvm_disable_tdp();
7468

7469 7470
	return 0;

7471 7472
out3:
	free_page((unsigned long)vmx_msr_bitmap_longmode);
S
Sheng Yang 已提交
7473
out2:
7474
	free_page((unsigned long)vmx_msr_bitmap_legacy);
7475
out1:
7476
	free_page((unsigned long)vmx_io_bitmap_b);
7477
out:
7478
	free_page((unsigned long)vmx_io_bitmap_a);
7479
	return r;
A
Avi Kivity 已提交
7480 7481 7482 7483
}

static void __exit vmx_exit(void)
{
7484 7485
	free_page((unsigned long)vmx_msr_bitmap_legacy);
	free_page((unsigned long)vmx_msr_bitmap_longmode);
7486 7487
	free_page((unsigned long)vmx_io_bitmap_b);
	free_page((unsigned long)vmx_io_bitmap_a);
7488

7489 7490 7491 7492 7493
#ifdef CONFIG_KEXEC
	rcu_assign_pointer(crash_vmclear_loaded_vmcss, NULL);
	synchronize_rcu();
#endif

7494
	kvm_exit();
A
Avi Kivity 已提交
7495 7496 7497 7498
}

module_init(vmx_init)
module_exit(vmx_exit)