vmx.c 283.0 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
8
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
9 10 11 12 13 14 15 16 17 18
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

19
#include "irq.h"
20
#include "mmu.h"
A
Avi Kivity 已提交
21
#include "cpuid.h"
A
Avi Kivity 已提交
22

23
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
24
#include <linux/module.h>
25
#include <linux/kernel.h>
A
Avi Kivity 已提交
26 27
#include <linux/mm.h>
#include <linux/highmem.h>
A
Alexey Dobriyan 已提交
28
#include <linux/sched.h>
29
#include <linux/moduleparam.h>
30
#include <linux/mod_devicetable.h>
31
#include <linux/ftrace_event.h>
32
#include <linux/slab.h>
33
#include <linux/tboot.h>
34
#include <linux/hrtimer.h>
35
#include "kvm_cache_regs.h"
36
#include "x86.h"
A
Avi Kivity 已提交
37

A
Avi Kivity 已提交
38
#include <asm/io.h>
A
Anthony Liguori 已提交
39
#include <asm/desc.h>
40
#include <asm/vmx.h>
41
#include <asm/virtext.h>
A
Andi Kleen 已提交
42
#include <asm/mce.h>
43 44
#include <asm/i387.h>
#include <asm/xcr.h>
45
#include <asm/perf_event.h>
46
#include <asm/debugreg.h>
47
#include <asm/kexec.h>
A
Avi Kivity 已提交
48

49 50
#include "trace.h"

51
#define __ex(x) __kvm_handle_fault_on_reboot(x)
52 53
#define __ex_clear(x, reg) \
	____kvm_handle_fault_on_reboot(x, "xor " reg " , " reg)
54

A
Avi Kivity 已提交
55 56 57
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

58 59 60 61 62 63
static const struct x86_cpu_id vmx_cpu_id[] = {
	X86_FEATURE_MATCH(X86_FEATURE_VMX),
	{}
};
MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);

64
static bool __read_mostly enable_vpid = 1;
65
module_param_named(vpid, enable_vpid, bool, 0444);
66

67
static bool __read_mostly flexpriority_enabled = 1;
68
module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
69

70
static bool __read_mostly enable_ept = 1;
71
module_param_named(ept, enable_ept, bool, S_IRUGO);
S
Sheng Yang 已提交
72

73
static bool __read_mostly enable_unrestricted_guest = 1;
74 75 76
module_param_named(unrestricted_guest,
			enable_unrestricted_guest, bool, S_IRUGO);

77 78 79
static bool __read_mostly enable_ept_ad_bits = 1;
module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);

80
static bool __read_mostly emulate_invalid_guest_state = true;
81
module_param(emulate_invalid_guest_state, bool, S_IRUGO);
82

83
static bool __read_mostly vmm_exclusive = 1;
84 85
module_param(vmm_exclusive, bool, S_IRUGO);

86
static bool __read_mostly fasteoi = 1;
87 88
module_param(fasteoi, bool, S_IRUGO);

89
static bool __read_mostly enable_apicv = 1;
90
module_param(enable_apicv, bool, S_IRUGO);
91

92 93
static bool __read_mostly enable_shadow_vmcs = 1;
module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
94 95 96 97 98
/*
 * If nested=1, nested virtualization is supported, i.e., guests may use
 * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 * use VMX instructions.
 */
99
static bool __read_mostly nested = 0;
100 101
module_param(nested, bool, S_IRUGO);

W
Wanpeng Li 已提交
102 103
static u64 __read_mostly host_xss;

K
Kai Huang 已提交
104 105 106
static bool __read_mostly enable_pml = 1;
module_param_named(pml, enable_pml, bool, S_IRUGO);

G
Gleb Natapov 已提交
107 108
#define KVM_GUEST_CR0_MASK (X86_CR0_NW | X86_CR0_CD)
#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST (X86_CR0_WP | X86_CR0_NE)
109 110
#define KVM_VM_CR0_ALWAYS_ON						\
	(KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
111 112
#define KVM_CR4_GUEST_OWNED_BITS				      \
	(X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
113
	 | X86_CR4_OSXMMEXCPT | X86_CR4_TSD)
114

115 116 117
#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)

118 119
#define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))

120 121
#define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5

122 123 124 125
/*
 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 * ple_gap:    upper bound on the amount of time between two successive
 *             executions of PAUSE in a loop. Also indicate if ple enabled.
126
 *             According to test, this time is usually smaller than 128 cycles.
127 128 129 130 131 132
 * ple_window: upper bound on the amount of time a guest is allowed to execute
 *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 *             less than 2^12 cycles
 * Time is measured based on a counter that runs at the same rate as the TSC,
 * refer SDM volume 3b section 21.6.13 & 22.1.3.
 */
R
Radim Krčmář 已提交
133 134 135 136 137 138 139
#define KVM_VMX_DEFAULT_PLE_GAP           128
#define KVM_VMX_DEFAULT_PLE_WINDOW        4096
#define KVM_VMX_DEFAULT_PLE_WINDOW_GROW   2
#define KVM_VMX_DEFAULT_PLE_WINDOW_SHRINK 0
#define KVM_VMX_DEFAULT_PLE_WINDOW_MAX    \
		INT_MAX / KVM_VMX_DEFAULT_PLE_WINDOW_GROW

140 141 142 143 144 145
static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
module_param(ple_gap, int, S_IRUGO);

static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
module_param(ple_window, int, S_IRUGO);

R
Radim Krčmář 已提交
146 147 148 149 150 151 152 153 154 155 156 157 158
/* Default doubles per-vcpu window every exit. */
static int ple_window_grow = KVM_VMX_DEFAULT_PLE_WINDOW_GROW;
module_param(ple_window_grow, int, S_IRUGO);

/* Default resets per-vcpu window every exit to ple_window. */
static int ple_window_shrink = KVM_VMX_DEFAULT_PLE_WINDOW_SHRINK;
module_param(ple_window_shrink, int, S_IRUGO);

/* Default is to compute the maximum so we can never overflow. */
static int ple_window_actual_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
static int ple_window_max        = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
module_param(ple_window_max, int, S_IRUGO);

A
Avi Kivity 已提交
159 160
extern const ulong vmx_return;

161
#define NR_AUTOLOAD_MSRS 8
162
#define VMCS02_POOL_SIZE 1
163

164 165 166 167 168 169
struct vmcs {
	u32 revision_id;
	u32 abort;
	char data[0];
};

170 171 172 173 174 175 176 177 178 179 180 181
/*
 * Track a VMCS that may be loaded on a certain CPU. If it is (cpu!=-1), also
 * remember whether it was VMLAUNCHed, and maintain a linked list of all VMCSs
 * loaded on this CPU (so we can clear them if the CPU goes down).
 */
struct loaded_vmcs {
	struct vmcs *vmcs;
	int cpu;
	int launched;
	struct list_head loaded_vmcss_on_cpu_link;
};

182 183 184
struct shared_msr_entry {
	unsigned index;
	u64 data;
185
	u64 mask;
186 187
};

188 189 190 191 192 193 194 195 196 197 198 199 200
/*
 * struct vmcs12 describes the state that our guest hypervisor (L1) keeps for a
 * single nested guest (L2), hence the name vmcs12. Any VMX implementation has
 * a VMCS structure, and vmcs12 is our emulated VMX's VMCS. This structure is
 * stored in guest memory specified by VMPTRLD, but is opaque to the guest,
 * which must access it using VMREAD/VMWRITE/VMCLEAR instructions.
 * More than one of these structures may exist, if L1 runs multiple L2 guests.
 * nested_vmx_run() will use the data here to build a vmcs02: a VMCS for the
 * underlying hardware which will be used to run L2.
 * This structure is packed to ensure that its layout is identical across
 * machines (necessary for live migration).
 * If there are changes in this struct, VMCS12_REVISION must be changed.
 */
201
typedef u64 natural_width;
202 203 204 205 206 207
struct __packed vmcs12 {
	/* According to the Intel spec, a VMCS region must start with the
	 * following two fields. Then follow implementation-specific data.
	 */
	u32 revision_id;
	u32 abort;
208

N
Nadav Har'El 已提交
209 210 211
	u32 launch_state; /* set to 0 by VMCLEAR, to 1 by VMLAUNCH */
	u32 padding[7]; /* room for future expansion */

212 213 214 215 216 217 218 219 220 221
	u64 io_bitmap_a;
	u64 io_bitmap_b;
	u64 msr_bitmap;
	u64 vm_exit_msr_store_addr;
	u64 vm_exit_msr_load_addr;
	u64 vm_entry_msr_load_addr;
	u64 tsc_offset;
	u64 virtual_apic_page_addr;
	u64 apic_access_addr;
	u64 ept_pointer;
222 223 224 225
	u64 eoi_exit_bitmap0;
	u64 eoi_exit_bitmap1;
	u64 eoi_exit_bitmap2;
	u64 eoi_exit_bitmap3;
226
	u64 xss_exit_bitmap;
227 228 229 230 231 232 233 234 235 236
	u64 guest_physical_address;
	u64 vmcs_link_pointer;
	u64 guest_ia32_debugctl;
	u64 guest_ia32_pat;
	u64 guest_ia32_efer;
	u64 guest_ia32_perf_global_ctrl;
	u64 guest_pdptr0;
	u64 guest_pdptr1;
	u64 guest_pdptr2;
	u64 guest_pdptr3;
237
	u64 guest_bndcfgs;
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
	u64 host_ia32_pat;
	u64 host_ia32_efer;
	u64 host_ia32_perf_global_ctrl;
	u64 padding64[8]; /* room for future expansion */
	/*
	 * To allow migration of L1 (complete with its L2 guests) between
	 * machines of different natural widths (32 or 64 bit), we cannot have
	 * unsigned long fields with no explict size. We use u64 (aliased
	 * natural_width) instead. Luckily, x86 is little-endian.
	 */
	natural_width cr0_guest_host_mask;
	natural_width cr4_guest_host_mask;
	natural_width cr0_read_shadow;
	natural_width cr4_read_shadow;
	natural_width cr3_target_value0;
	natural_width cr3_target_value1;
	natural_width cr3_target_value2;
	natural_width cr3_target_value3;
	natural_width exit_qualification;
	natural_width guest_linear_address;
	natural_width guest_cr0;
	natural_width guest_cr3;
	natural_width guest_cr4;
	natural_width guest_es_base;
	natural_width guest_cs_base;
	natural_width guest_ss_base;
	natural_width guest_ds_base;
	natural_width guest_fs_base;
	natural_width guest_gs_base;
	natural_width guest_ldtr_base;
	natural_width guest_tr_base;
	natural_width guest_gdtr_base;
	natural_width guest_idtr_base;
	natural_width guest_dr7;
	natural_width guest_rsp;
	natural_width guest_rip;
	natural_width guest_rflags;
	natural_width guest_pending_dbg_exceptions;
	natural_width guest_sysenter_esp;
	natural_width guest_sysenter_eip;
	natural_width host_cr0;
	natural_width host_cr3;
	natural_width host_cr4;
	natural_width host_fs_base;
	natural_width host_gs_base;
	natural_width host_tr_base;
	natural_width host_gdtr_base;
	natural_width host_idtr_base;
	natural_width host_ia32_sysenter_esp;
	natural_width host_ia32_sysenter_eip;
	natural_width host_rsp;
	natural_width host_rip;
	natural_width paddingl[8]; /* room for future expansion */
	u32 pin_based_vm_exec_control;
	u32 cpu_based_vm_exec_control;
	u32 exception_bitmap;
	u32 page_fault_error_code_mask;
	u32 page_fault_error_code_match;
	u32 cr3_target_count;
	u32 vm_exit_controls;
	u32 vm_exit_msr_store_count;
	u32 vm_exit_msr_load_count;
	u32 vm_entry_controls;
	u32 vm_entry_msr_load_count;
	u32 vm_entry_intr_info_field;
	u32 vm_entry_exception_error_code;
	u32 vm_entry_instruction_len;
	u32 tpr_threshold;
	u32 secondary_vm_exec_control;
	u32 vm_instruction_error;
	u32 vm_exit_reason;
	u32 vm_exit_intr_info;
	u32 vm_exit_intr_error_code;
	u32 idt_vectoring_info_field;
	u32 idt_vectoring_error_code;
	u32 vm_exit_instruction_len;
	u32 vmx_instruction_info;
	u32 guest_es_limit;
	u32 guest_cs_limit;
	u32 guest_ss_limit;
	u32 guest_ds_limit;
	u32 guest_fs_limit;
	u32 guest_gs_limit;
	u32 guest_ldtr_limit;
	u32 guest_tr_limit;
	u32 guest_gdtr_limit;
	u32 guest_idtr_limit;
	u32 guest_es_ar_bytes;
	u32 guest_cs_ar_bytes;
	u32 guest_ss_ar_bytes;
	u32 guest_ds_ar_bytes;
	u32 guest_fs_ar_bytes;
	u32 guest_gs_ar_bytes;
	u32 guest_ldtr_ar_bytes;
	u32 guest_tr_ar_bytes;
	u32 guest_interruptibility_info;
	u32 guest_activity_state;
	u32 guest_sysenter_cs;
	u32 host_ia32_sysenter_cs;
337 338
	u32 vmx_preemption_timer_value;
	u32 padding32[7]; /* room for future expansion */
339 340 341 342 343 344 345 346 347
	u16 virtual_processor_id;
	u16 guest_es_selector;
	u16 guest_cs_selector;
	u16 guest_ss_selector;
	u16 guest_ds_selector;
	u16 guest_fs_selector;
	u16 guest_gs_selector;
	u16 guest_ldtr_selector;
	u16 guest_tr_selector;
348
	u16 guest_intr_status;
349 350 351 352 353 354 355
	u16 host_es_selector;
	u16 host_cs_selector;
	u16 host_ss_selector;
	u16 host_ds_selector;
	u16 host_fs_selector;
	u16 host_gs_selector;
	u16 host_tr_selector;
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
};

/*
 * VMCS12_REVISION is an arbitrary id that should be changed if the content or
 * layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
 * VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
 */
#define VMCS12_REVISION 0x11e57ed0

/*
 * VMCS12_SIZE is the number of bytes L1 should allocate for the VMXON region
 * and any VMCS region. Although only sizeof(struct vmcs12) are used by the
 * current implementation, 4K are reserved to avoid future complications.
 */
#define VMCS12_SIZE 0x1000

372 373 374 375 376 377 378
/* Used to remember the last vmcs02 used for some recently used vmcs12s */
struct vmcs02_list {
	struct list_head list;
	gpa_t vmptr;
	struct loaded_vmcs vmcs02;
};

379 380 381 382 383 384 385
/*
 * The nested_vmx structure is part of vcpu_vmx, and holds information we need
 * for correct emulation of VMX (i.e., nested VMX) on this vcpu.
 */
struct nested_vmx {
	/* Has the level1 guest done vmxon? */
	bool vmxon;
386
	gpa_t vmxon_ptr;
387 388 389 390 391 392

	/* The guest-physical address of the current VMCS L1 keeps for L2 */
	gpa_t current_vmptr;
	/* The host-usable pointer to the above */
	struct page *current_vmcs12_page;
	struct vmcs12 *current_vmcs12;
A
Abel Gordon 已提交
393
	struct vmcs *current_shadow_vmcs;
394 395 396 397 398
	/*
	 * Indicates if the shadow vmcs must be updated with the
	 * data hold by vmcs12
	 */
	bool sync_shadow_vmcs;
399 400 401 402

	/* vmcs02_list cache of VMCSs recently used to run L2 guests */
	struct list_head vmcs02_pool;
	int vmcs02_num;
403
	u64 vmcs01_tsc_offset;
404 405
	/* L2 must run next, and mustn't decide to exit to L1. */
	bool nested_run_pending;
406 407 408 409 410
	/*
	 * Guest pages referred to in vmcs02 with host-physical pointers, so
	 * we must keep them pinned while L2 runs.
	 */
	struct page *apic_access_page;
411
	struct page *virtual_apic_page;
412
	u64 msr_ia32_feature_control;
413 414 415

	struct hrtimer preemption_timer;
	bool preemption_timer_expired;
416 417 418

	/* to migrate it to L2 if VM_ENTRY_LOAD_DEBUG_CONTROLS is off */
	u64 vmcs01_debugctl;
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435

	u32 nested_vmx_procbased_ctls_low;
	u32 nested_vmx_procbased_ctls_high;
	u32 nested_vmx_true_procbased_ctls_low;
	u32 nested_vmx_secondary_ctls_low;
	u32 nested_vmx_secondary_ctls_high;
	u32 nested_vmx_pinbased_ctls_low;
	u32 nested_vmx_pinbased_ctls_high;
	u32 nested_vmx_exit_ctls_low;
	u32 nested_vmx_exit_ctls_high;
	u32 nested_vmx_true_exit_ctls_low;
	u32 nested_vmx_entry_ctls_low;
	u32 nested_vmx_entry_ctls_high;
	u32 nested_vmx_true_entry_ctls_low;
	u32 nested_vmx_misc_low;
	u32 nested_vmx_misc_high;
	u32 nested_vmx_ept_caps;
436 437
};

438 439 440 441 442 443 444 445
#define POSTED_INTR_ON  0
/* Posted-Interrupt Descriptor */
struct pi_desc {
	u32 pir[8];     /* Posted interrupt requested */
	u32 control;	/* bit 0 of control is outstanding notification bit */
	u32 rsvd[7];
} __aligned(64);

446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462
static bool pi_test_and_set_on(struct pi_desc *pi_desc)
{
	return test_and_set_bit(POSTED_INTR_ON,
			(unsigned long *)&pi_desc->control);
}

static bool pi_test_and_clear_on(struct pi_desc *pi_desc)
{
	return test_and_clear_bit(POSTED_INTR_ON,
			(unsigned long *)&pi_desc->control);
}

static int pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
{
	return test_and_set_bit(vector, (unsigned long *)pi_desc->pir);
}

463
struct vcpu_vmx {
R
Rusty Russell 已提交
464
	struct kvm_vcpu       vcpu;
465
	unsigned long         host_rsp;
466
	u8                    fail;
467
	bool                  nmi_known_unmasked;
468
	u32                   exit_intr_info;
469
	u32                   idt_vectoring_info;
A
Avi Kivity 已提交
470
	ulong                 rflags;
471
	struct shared_msr_entry *guest_msrs;
472 473
	int                   nmsrs;
	int                   save_nmsrs;
474
	unsigned long	      host_idt_base;
475
#ifdef CONFIG_X86_64
476 477
	u64 		      msr_host_kernel_gs_base;
	u64 		      msr_guest_kernel_gs_base;
478
#endif
479 480
	u32 vm_entry_controls_shadow;
	u32 vm_exit_controls_shadow;
481 482 483 484 485 486 487 488
	/*
	 * loaded_vmcs points to the VMCS currently used in this vcpu. For a
	 * non-nested (L1) guest, it always points to vmcs01. For a nested
	 * guest (L2), it points to a different VMCS.
	 */
	struct loaded_vmcs    vmcs01;
	struct loaded_vmcs   *loaded_vmcs;
	bool                  __launched; /* temporary, used in vmx_vcpu_run */
489 490 491 492 493
	struct msr_autoload {
		unsigned nr;
		struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS];
		struct vmx_msr_entry host[NR_AUTOLOAD_MSRS];
	} msr_autoload;
494 495 496
	struct {
		int           loaded;
		u16           fs_sel, gs_sel, ldt_sel;
A
Avi Kivity 已提交
497 498 499
#ifdef CONFIG_X86_64
		u16           ds_sel, es_sel;
#endif
500 501
		int           gs_ldt_reload_needed;
		int           fs_reload_needed;
502
		u64           msr_host_bndcfgs;
503
		unsigned long vmcs_host_cr4;	/* May not match real cr4 */
M
Mike Day 已提交
504
	} host_state;
505
	struct {
506
		int vm86_active;
507
		ulong save_rflags;
508 509 510 511
		struct kvm_segment segs[8];
	} rmode;
	struct {
		u32 bitmask; /* 4 bits per segment (1 bit per field) */
512 513 514 515 516
		struct kvm_save_segment {
			u16 selector;
			unsigned long base;
			u32 limit;
			u32 ar;
517
		} seg[8];
A
Avi Kivity 已提交
518
	} segment_cache;
519
	int vpid;
520
	bool emulation_required;
521 522 523 524 525

	/* Support for vnmi-less CPUs */
	int soft_vnmi_blocked;
	ktime_t entry_time;
	s64 vnmi_blocked_time;
A
Andi Kleen 已提交
526
	u32 exit_reason;
527 528

	bool rdtscp_enabled;
529

530 531 532
	/* Posted interrupt descriptor */
	struct pi_desc pi_desc;

533 534
	/* Support for a guest hypervisor (nested VMX) */
	struct nested_vmx nested;
535 536 537 538

	/* Dynamic PLE window. */
	int ple_window;
	bool ple_window_dirty;
K
Kai Huang 已提交
539 540 541 542

	/* Support for PML */
#define PML_ENTITY_NUM		512
	struct page *pml_pg;
543 544
};

A
Avi Kivity 已提交
545 546 547 548 549 550 551 552 553
enum segment_cache_field {
	SEG_FIELD_SEL = 0,
	SEG_FIELD_BASE = 1,
	SEG_FIELD_LIMIT = 2,
	SEG_FIELD_AR = 3,

	SEG_FIELD_NR = 4
};

554 555
static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
556
	return container_of(vcpu, struct vcpu_vmx, vcpu);
557 558
}

559 560 561 562 563
#define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
#define FIELD(number, name)	[number] = VMCS12_OFFSET(name)
#define FIELD64(number, name)	[number] = VMCS12_OFFSET(name), \
				[number##_HIGH] = VMCS12_OFFSET(name)+4

564

565
static unsigned long shadow_read_only_fields[] = {
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
	/*
	 * We do NOT shadow fields that are modified when L0
	 * traps and emulates any vmx instruction (e.g. VMPTRLD,
	 * VMXON...) executed by L1.
	 * For example, VM_INSTRUCTION_ERROR is read
	 * by L1 if a vmx instruction fails (part of the error path).
	 * Note the code assumes this logic. If for some reason
	 * we start shadowing these fields then we need to
	 * force a shadow sync when L0 emulates vmx instructions
	 * (e.g. force a sync if VM_INSTRUCTION_ERROR is modified
	 * by nested_vmx_failValid)
	 */
	VM_EXIT_REASON,
	VM_EXIT_INTR_INFO,
	VM_EXIT_INSTRUCTION_LEN,
	IDT_VECTORING_INFO_FIELD,
	IDT_VECTORING_ERROR_CODE,
	VM_EXIT_INTR_ERROR_CODE,
	EXIT_QUALIFICATION,
	GUEST_LINEAR_ADDRESS,
	GUEST_PHYSICAL_ADDRESS
};
588
static int max_shadow_read_only_fields =
589 590
	ARRAY_SIZE(shadow_read_only_fields);

591
static unsigned long shadow_read_write_fields[] = {
592
	TPR_THRESHOLD,
593 594 595 596 597 598 599 600 601 602 603 604
	GUEST_RIP,
	GUEST_RSP,
	GUEST_CR0,
	GUEST_CR3,
	GUEST_CR4,
	GUEST_INTERRUPTIBILITY_INFO,
	GUEST_RFLAGS,
	GUEST_CS_SELECTOR,
	GUEST_CS_AR_BYTES,
	GUEST_CS_LIMIT,
	GUEST_CS_BASE,
	GUEST_ES_BASE,
605
	GUEST_BNDCFGS,
606 607 608 609 610 611 612 613 614 615 616 617 618 619 620
	CR0_GUEST_HOST_MASK,
	CR0_READ_SHADOW,
	CR4_READ_SHADOW,
	TSC_OFFSET,
	EXCEPTION_BITMAP,
	CPU_BASED_VM_EXEC_CONTROL,
	VM_ENTRY_EXCEPTION_ERROR_CODE,
	VM_ENTRY_INTR_INFO_FIELD,
	VM_ENTRY_INSTRUCTION_LEN,
	VM_ENTRY_EXCEPTION_ERROR_CODE,
	HOST_FS_BASE,
	HOST_GS_BASE,
	HOST_FS_SELECTOR,
	HOST_GS_SELECTOR
};
621
static int max_shadow_read_write_fields =
622 623
	ARRAY_SIZE(shadow_read_write_fields);

624
static const unsigned short vmcs_field_to_offset_table[] = {
625 626 627 628 629 630 631 632 633
	FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
	FIELD(GUEST_ES_SELECTOR, guest_es_selector),
	FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
	FIELD(GUEST_SS_SELECTOR, guest_ss_selector),
	FIELD(GUEST_DS_SELECTOR, guest_ds_selector),
	FIELD(GUEST_FS_SELECTOR, guest_fs_selector),
	FIELD(GUEST_GS_SELECTOR, guest_gs_selector),
	FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector),
	FIELD(GUEST_TR_SELECTOR, guest_tr_selector),
634
	FIELD(GUEST_INTR_STATUS, guest_intr_status),
635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
	FIELD(HOST_ES_SELECTOR, host_es_selector),
	FIELD(HOST_CS_SELECTOR, host_cs_selector),
	FIELD(HOST_SS_SELECTOR, host_ss_selector),
	FIELD(HOST_DS_SELECTOR, host_ds_selector),
	FIELD(HOST_FS_SELECTOR, host_fs_selector),
	FIELD(HOST_GS_SELECTOR, host_gs_selector),
	FIELD(HOST_TR_SELECTOR, host_tr_selector),
	FIELD64(IO_BITMAP_A, io_bitmap_a),
	FIELD64(IO_BITMAP_B, io_bitmap_b),
	FIELD64(MSR_BITMAP, msr_bitmap),
	FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
	FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
	FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
	FIELD64(TSC_OFFSET, tsc_offset),
	FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
	FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
	FIELD64(EPT_POINTER, ept_pointer),
652 653 654 655
	FIELD64(EOI_EXIT_BITMAP0, eoi_exit_bitmap0),
	FIELD64(EOI_EXIT_BITMAP1, eoi_exit_bitmap1),
	FIELD64(EOI_EXIT_BITMAP2, eoi_exit_bitmap2),
	FIELD64(EOI_EXIT_BITMAP3, eoi_exit_bitmap3),
656
	FIELD64(XSS_EXIT_BITMAP, xss_exit_bitmap),
657 658 659 660 661 662 663 664 665 666
	FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
	FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
	FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
	FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
	FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
	FIELD64(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl),
	FIELD64(GUEST_PDPTR0, guest_pdptr0),
	FIELD64(GUEST_PDPTR1, guest_pdptr1),
	FIELD64(GUEST_PDPTR2, guest_pdptr2),
	FIELD64(GUEST_PDPTR3, guest_pdptr3),
667
	FIELD64(GUEST_BNDCFGS, guest_bndcfgs),
668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716
	FIELD64(HOST_IA32_PAT, host_ia32_pat),
	FIELD64(HOST_IA32_EFER, host_ia32_efer),
	FIELD64(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl),
	FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control),
	FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control),
	FIELD(EXCEPTION_BITMAP, exception_bitmap),
	FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask),
	FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match),
	FIELD(CR3_TARGET_COUNT, cr3_target_count),
	FIELD(VM_EXIT_CONTROLS, vm_exit_controls),
	FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count),
	FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count),
	FIELD(VM_ENTRY_CONTROLS, vm_entry_controls),
	FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count),
	FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field),
	FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, vm_entry_exception_error_code),
	FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len),
	FIELD(TPR_THRESHOLD, tpr_threshold),
	FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control),
	FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error),
	FIELD(VM_EXIT_REASON, vm_exit_reason),
	FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info),
	FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code),
	FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field),
	FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code),
	FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len),
	FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info),
	FIELD(GUEST_ES_LIMIT, guest_es_limit),
	FIELD(GUEST_CS_LIMIT, guest_cs_limit),
	FIELD(GUEST_SS_LIMIT, guest_ss_limit),
	FIELD(GUEST_DS_LIMIT, guest_ds_limit),
	FIELD(GUEST_FS_LIMIT, guest_fs_limit),
	FIELD(GUEST_GS_LIMIT, guest_gs_limit),
	FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit),
	FIELD(GUEST_TR_LIMIT, guest_tr_limit),
	FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit),
	FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit),
	FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes),
	FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes),
	FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes),
	FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes),
	FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes),
	FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes),
	FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes),
	FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes),
	FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info),
	FIELD(GUEST_ACTIVITY_STATE, guest_activity_state),
	FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs),
	FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs),
717
	FIELD(VMX_PREEMPTION_TIMER_VALUE, vmx_preemption_timer_value),
718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763
	FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask),
	FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask),
	FIELD(CR0_READ_SHADOW, cr0_read_shadow),
	FIELD(CR4_READ_SHADOW, cr4_read_shadow),
	FIELD(CR3_TARGET_VALUE0, cr3_target_value0),
	FIELD(CR3_TARGET_VALUE1, cr3_target_value1),
	FIELD(CR3_TARGET_VALUE2, cr3_target_value2),
	FIELD(CR3_TARGET_VALUE3, cr3_target_value3),
	FIELD(EXIT_QUALIFICATION, exit_qualification),
	FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address),
	FIELD(GUEST_CR0, guest_cr0),
	FIELD(GUEST_CR3, guest_cr3),
	FIELD(GUEST_CR4, guest_cr4),
	FIELD(GUEST_ES_BASE, guest_es_base),
	FIELD(GUEST_CS_BASE, guest_cs_base),
	FIELD(GUEST_SS_BASE, guest_ss_base),
	FIELD(GUEST_DS_BASE, guest_ds_base),
	FIELD(GUEST_FS_BASE, guest_fs_base),
	FIELD(GUEST_GS_BASE, guest_gs_base),
	FIELD(GUEST_LDTR_BASE, guest_ldtr_base),
	FIELD(GUEST_TR_BASE, guest_tr_base),
	FIELD(GUEST_GDTR_BASE, guest_gdtr_base),
	FIELD(GUEST_IDTR_BASE, guest_idtr_base),
	FIELD(GUEST_DR7, guest_dr7),
	FIELD(GUEST_RSP, guest_rsp),
	FIELD(GUEST_RIP, guest_rip),
	FIELD(GUEST_RFLAGS, guest_rflags),
	FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions),
	FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp),
	FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip),
	FIELD(HOST_CR0, host_cr0),
	FIELD(HOST_CR3, host_cr3),
	FIELD(HOST_CR4, host_cr4),
	FIELD(HOST_FS_BASE, host_fs_base),
	FIELD(HOST_GS_BASE, host_gs_base),
	FIELD(HOST_TR_BASE, host_tr_base),
	FIELD(HOST_GDTR_BASE, host_gdtr_base),
	FIELD(HOST_IDTR_BASE, host_idtr_base),
	FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp),
	FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip),
	FIELD(HOST_RSP, host_rsp),
	FIELD(HOST_RIP, host_rip),
};

static inline short vmcs_field_to_offset(unsigned long field)
{
764 765 766 767 768 769
	BUILD_BUG_ON(ARRAY_SIZE(vmcs_field_to_offset_table) > SHRT_MAX);

	if (field >= ARRAY_SIZE(vmcs_field_to_offset_table) ||
	    vmcs_field_to_offset_table[field] == 0)
		return -ENOENT;

770 771 772
	return vmcs_field_to_offset_table[field];
}

773 774 775 776 777 778 779 780
static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
{
	return to_vmx(vcpu)->nested.current_vmcs12;
}

static struct page *nested_get_page(struct kvm_vcpu *vcpu, gpa_t addr)
{
	struct page *page = gfn_to_page(vcpu->kvm, addr >> PAGE_SHIFT);
781
	if (is_error_page(page))
782
		return NULL;
783

784 785 786 787 788 789 790 791 792 793 794 795 796
	return page;
}

static void nested_release_page(struct page *page)
{
	kvm_release_page_dirty(page);
}

static void nested_release_page_clean(struct page *page)
{
	kvm_release_page_clean(page);
}

N
Nadav Har'El 已提交
797
static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu);
798
static u64 construct_eptp(unsigned long root_hpa);
799 800
static void kvm_cpu_vmxon(u64 addr);
static void kvm_cpu_vmxoff(void);
801
static bool vmx_mpx_supported(void);
802
static bool vmx_xsaves_supported(void);
803
static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
804 805 806 807
static void vmx_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg);
static void vmx_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg);
808 809
static bool guest_state_valid(struct kvm_vcpu *vcpu);
static u32 vmx_segment_access_rights(struct kvm_segment *var);
810
static void vmx_sync_pir_to_irr_dummy(struct kvm_vcpu *vcpu);
811
static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx);
812
static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx);
813
static int alloc_identity_pagetable(struct kvm *kvm);
814

A
Avi Kivity 已提交
815 816
static DEFINE_PER_CPU(struct vmcs *, vmxarea);
static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
817 818 819 820 821
/*
 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
 */
static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
822
static DEFINE_PER_CPU(struct desc_ptr, host_gdt);
A
Avi Kivity 已提交
823

824 825
static unsigned long *vmx_io_bitmap_a;
static unsigned long *vmx_io_bitmap_b;
826 827
static unsigned long *vmx_msr_bitmap_legacy;
static unsigned long *vmx_msr_bitmap_longmode;
828 829
static unsigned long *vmx_msr_bitmap_legacy_x2apic;
static unsigned long *vmx_msr_bitmap_longmode_x2apic;
830
static unsigned long *vmx_msr_bitmap_nested;
831 832
static unsigned long *vmx_vmread_bitmap;
static unsigned long *vmx_vmwrite_bitmap;
833

A
Avi Kivity 已提交
834
static bool cpu_has_load_ia32_efer;
835
static bool cpu_has_load_perf_global_ctrl;
A
Avi Kivity 已提交
836

837 838 839
static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
static DEFINE_SPINLOCK(vmx_vpid_lock);

840
static struct vmcs_config {
A
Avi Kivity 已提交
841 842 843
	int size;
	int order;
	u32 revision_id;
844 845
	u32 pin_based_exec_ctrl;
	u32 cpu_based_exec_ctrl;
846
	u32 cpu_based_2nd_exec_ctrl;
847 848 849
	u32 vmexit_ctrl;
	u32 vmentry_ctrl;
} vmcs_config;
A
Avi Kivity 已提交
850

H
Hannes Eder 已提交
851
static struct vmx_capability {
S
Sheng Yang 已提交
852 853 854 855
	u32 ept;
	u32 vpid;
} vmx_capability;

A
Avi Kivity 已提交
856 857 858 859 860 861 862 863
#define VMX_SEGMENT_FIELD(seg)					\
	[VCPU_SREG_##seg] = {                                   \
		.selector = GUEST_##seg##_SELECTOR,		\
		.base = GUEST_##seg##_BASE,		   	\
		.limit = GUEST_##seg##_LIMIT,		   	\
		.ar_bytes = GUEST_##seg##_AR_BYTES,	   	\
	}

864
static const struct kvm_vmx_segment_field {
A
Avi Kivity 已提交
865 866 867 868 869 870 871 872 873 874 875 876 877 878 879
	unsigned selector;
	unsigned base;
	unsigned limit;
	unsigned ar_bytes;
} kvm_vmx_segment_fields[] = {
	VMX_SEGMENT_FIELD(CS),
	VMX_SEGMENT_FIELD(DS),
	VMX_SEGMENT_FIELD(ES),
	VMX_SEGMENT_FIELD(FS),
	VMX_SEGMENT_FIELD(GS),
	VMX_SEGMENT_FIELD(SS),
	VMX_SEGMENT_FIELD(TR),
	VMX_SEGMENT_FIELD(LDTR),
};

880 881
static u64 host_efer;

A
Avi Kivity 已提交
882 883
static void ept_save_pdptrs(struct kvm_vcpu *vcpu);

884
/*
B
Brian Gerst 已提交
885
 * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
886 887
 * away by decrementing the array size.
 */
A
Avi Kivity 已提交
888
static const u32 vmx_msr_index[] = {
889
#ifdef CONFIG_X86_64
890
	MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
A
Avi Kivity 已提交
891
#endif
B
Brian Gerst 已提交
892
	MSR_EFER, MSR_TSC_AUX, MSR_STAR,
A
Avi Kivity 已提交
893 894
};

895
static inline bool is_page_fault(u32 intr_info)
A
Avi Kivity 已提交
896 897 898
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
899
		(INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
A
Avi Kivity 已提交
900 901
}

902
static inline bool is_no_device(u32 intr_info)
903 904 905
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
906
		(INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
907 908
}

909
static inline bool is_invalid_opcode(u32 intr_info)
910 911 912
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
913
		(INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK);
914 915
}

916
static inline bool is_external_interrupt(u32 intr_info)
A
Avi Kivity 已提交
917 918 919 920 921
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
		== (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
}

922
static inline bool is_machine_check(u32 intr_info)
A
Andi Kleen 已提交
923 924 925 926 927 928
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
			     INTR_INFO_VALID_MASK)) ==
		(INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
}

929
static inline bool cpu_has_vmx_msr_bitmap(void)
S
Sheng Yang 已提交
930
{
931
	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
S
Sheng Yang 已提交
932 933
}

934
static inline bool cpu_has_vmx_tpr_shadow(void)
935
{
936
	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
937 938
}

939
static inline bool vm_need_tpr_shadow(struct kvm *kvm)
940
{
941
	return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
942 943
}

944
static inline bool cpu_has_secondary_exec_ctrls(void)
945
{
946 947
	return vmcs_config.cpu_based_exec_ctrl &
		CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
948 949
}

950
static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
951
{
952 953 954 955
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
}

956 957 958 959 960 961
static inline bool cpu_has_vmx_virtualize_x2apic_mode(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
}

962 963 964 965 966 967
static inline bool cpu_has_vmx_apic_register_virt(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_APIC_REGISTER_VIRT;
}

968 969 970 971 972 973
static inline bool cpu_has_vmx_virtual_intr_delivery(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
}

974 975 976 977 978 979 980 981 982 983 984 985
static inline bool cpu_has_vmx_posted_intr(void)
{
	return vmcs_config.pin_based_exec_ctrl & PIN_BASED_POSTED_INTR;
}

static inline bool cpu_has_vmx_apicv(void)
{
	return cpu_has_vmx_apic_register_virt() &&
		cpu_has_vmx_virtual_intr_delivery() &&
		cpu_has_vmx_posted_intr();
}

986 987 988 989
static inline bool cpu_has_vmx_flexpriority(void)
{
	return cpu_has_vmx_tpr_shadow() &&
		cpu_has_vmx_virtualize_apic_accesses();
990 991
}

992 993
static inline bool cpu_has_vmx_ept_execute_only(void)
{
994
	return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
995 996 997 998
}

static inline bool cpu_has_vmx_ept_2m_page(void)
{
999
	return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
1000 1001
}

1002 1003
static inline bool cpu_has_vmx_ept_1g_page(void)
{
1004
	return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
1005 1006
}

1007 1008 1009 1010 1011
static inline bool cpu_has_vmx_ept_4levels(void)
{
	return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
}

1012 1013 1014 1015 1016
static inline bool cpu_has_vmx_ept_ad_bits(void)
{
	return vmx_capability.ept & VMX_EPT_AD_BIT;
}

1017
static inline bool cpu_has_vmx_invept_context(void)
S
Sheng Yang 已提交
1018
{
1019
	return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
S
Sheng Yang 已提交
1020 1021
}

1022
static inline bool cpu_has_vmx_invept_global(void)
S
Sheng Yang 已提交
1023
{
1024
	return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
S
Sheng Yang 已提交
1025 1026
}

1027 1028 1029 1030 1031
static inline bool cpu_has_vmx_invvpid_single(void)
{
	return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
}

1032 1033 1034 1035 1036
static inline bool cpu_has_vmx_invvpid_global(void)
{
	return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
}

1037
static inline bool cpu_has_vmx_ept(void)
S
Sheng Yang 已提交
1038
{
1039 1040
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_EPT;
S
Sheng Yang 已提交
1041 1042
}

1043
static inline bool cpu_has_vmx_unrestricted_guest(void)
1044 1045 1046 1047 1048
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_UNRESTRICTED_GUEST;
}

1049
static inline bool cpu_has_vmx_ple(void)
1050 1051 1052 1053 1054
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_PAUSE_LOOP_EXITING;
}

1055
static inline bool vm_need_virtualize_apic_accesses(struct kvm *kvm)
1056
{
1057
	return flexpriority_enabled && irqchip_in_kernel(kvm);
1058 1059
}

1060
static inline bool cpu_has_vmx_vpid(void)
1061
{
1062 1063
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_VPID;
1064 1065
}

1066
static inline bool cpu_has_vmx_rdtscp(void)
1067 1068 1069 1070 1071
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_RDTSCP;
}

1072 1073 1074 1075 1076 1077
static inline bool cpu_has_vmx_invpcid(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_ENABLE_INVPCID;
}

1078
static inline bool cpu_has_virtual_nmis(void)
1079 1080 1081 1082
{
	return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
}

1083 1084 1085 1086 1087 1088
static inline bool cpu_has_vmx_wbinvd_exit(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_WBINVD_EXITING;
}

1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
static inline bool cpu_has_vmx_shadow_vmcs(void)
{
	u64 vmx_msr;
	rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
	/* check if the cpu supports writing r/o exit information fields */
	if (!(vmx_msr & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS))
		return false;

	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_SHADOW_VMCS;
}

K
Kai Huang 已提交
1101 1102 1103 1104 1105
static inline bool cpu_has_vmx_pml(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_ENABLE_PML;
}

1106 1107 1108 1109 1110
static inline bool report_flexpriority(void)
{
	return flexpriority_enabled;
}

1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit)
{
	return vmcs12->cpu_based_vm_exec_control & bit;
}

static inline bool nested_cpu_has2(struct vmcs12 *vmcs12, u32 bit)
{
	return (vmcs12->cpu_based_vm_exec_control &
			CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
		(vmcs12->secondary_vm_exec_control & bit);
}

N
Nadav Har'El 已提交
1123
static inline bool nested_cpu_has_virtual_nmis(struct vmcs12 *vmcs12)
1124 1125 1126 1127
{
	return vmcs12->pin_based_vm_exec_control & PIN_BASED_VIRTUAL_NMIS;
}

1128 1129 1130 1131 1132 1133
static inline bool nested_cpu_has_preemption_timer(struct vmcs12 *vmcs12)
{
	return vmcs12->pin_based_vm_exec_control &
		PIN_BASED_VMX_PREEMPTION_TIMER;
}

N
Nadav Har'El 已提交
1134 1135 1136 1137 1138
static inline int nested_cpu_has_ept(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_EPT);
}

1139 1140 1141 1142 1143 1144
static inline bool nested_cpu_has_xsaves(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES) &&
		vmx_xsaves_supported();
}

1145 1146 1147 1148 1149
static inline bool nested_cpu_has_virt_x2apic_mode(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
}

1150 1151 1152 1153 1154
static inline bool nested_cpu_has_apic_reg_virt(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_APIC_REGISTER_VIRT);
}

1155 1156 1157 1158 1159
static inline bool nested_cpu_has_vid(struct vmcs12 *vmcs12)
{
	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
}

1160 1161 1162 1163 1164 1165
static inline bool is_exception(u32 intr_info)
{
	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
		== (INTR_TYPE_HARD_EXCEPTION | INTR_INFO_VALID_MASK);
}

1166 1167 1168
static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
			      u32 exit_intr_info,
			      unsigned long exit_qualification);
1169 1170 1171 1172
static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
			struct vmcs12 *vmcs12,
			u32 reason, unsigned long qualification);

R
Rusty Russell 已提交
1173
static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
1174 1175 1176
{
	int i;

1177
	for (i = 0; i < vmx->nmsrs; ++i)
1178
		if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
1179 1180 1181 1182
			return i;
	return -1;
}

1183 1184 1185 1186 1187 1188 1189 1190
static inline void __invvpid(int ext, u16 vpid, gva_t gva)
{
    struct {
	u64 vpid : 16;
	u64 rsvd : 48;
	u64 gva;
    } operand = { vpid, 0, gva };

1191
    asm volatile (__ex(ASM_VMX_INVVPID)
1192 1193 1194 1195 1196
		  /* CF==1 or ZF==1 --> rc = -1 */
		  "; ja 1f ; ud2 ; 1:"
		  : : "a"(&operand), "c"(ext) : "cc", "memory");
}

1197 1198 1199 1200 1201 1202
static inline void __invept(int ext, u64 eptp, gpa_t gpa)
{
	struct {
		u64 eptp, gpa;
	} operand = {eptp, gpa};

1203
	asm volatile (__ex(ASM_VMX_INVEPT)
1204 1205 1206 1207 1208
			/* CF==1 or ZF==1 --> rc = -1 */
			"; ja 1f ; ud2 ; 1:\n"
			: : "a" (&operand), "c" (ext) : "cc", "memory");
}

1209
static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
1210 1211 1212
{
	int i;

R
Rusty Russell 已提交
1213
	i = __find_msr_index(vmx, msr);
1214
	if (i >= 0)
1215
		return &vmx->guest_msrs[i];
A
Al Viro 已提交
1216
	return NULL;
1217 1218
}

A
Avi Kivity 已提交
1219 1220 1221 1222 1223
static void vmcs_clear(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(vmcs);
	u8 error;

1224
	asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
1225
		      : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
A
Avi Kivity 已提交
1226 1227 1228 1229 1230 1231
		      : "cc", "memory");
	if (error)
		printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
		       vmcs, phys_addr);
}

1232 1233 1234 1235 1236 1237 1238
static inline void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs)
{
	vmcs_clear(loaded_vmcs->vmcs);
	loaded_vmcs->cpu = -1;
	loaded_vmcs->launched = 0;
}

1239 1240 1241 1242 1243 1244
static void vmcs_load(struct vmcs *vmcs)
{
	u64 phys_addr = __pa(vmcs);
	u8 error;

	asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
1245
			: "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
1246 1247
			: "cc", "memory");
	if (error)
1248
		printk(KERN_ERR "kvm: vmptrld %p/%llx failed\n",
1249 1250 1251
		       vmcs, phys_addr);
}

1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291
#ifdef CONFIG_KEXEC
/*
 * This bitmap is used to indicate whether the vmclear
 * operation is enabled on all cpus. All disabled by
 * default.
 */
static cpumask_t crash_vmclear_enabled_bitmap = CPU_MASK_NONE;

static inline void crash_enable_local_vmclear(int cpu)
{
	cpumask_set_cpu(cpu, &crash_vmclear_enabled_bitmap);
}

static inline void crash_disable_local_vmclear(int cpu)
{
	cpumask_clear_cpu(cpu, &crash_vmclear_enabled_bitmap);
}

static inline int crash_local_vmclear_enabled(int cpu)
{
	return cpumask_test_cpu(cpu, &crash_vmclear_enabled_bitmap);
}

static void crash_vmclear_local_loaded_vmcss(void)
{
	int cpu = raw_smp_processor_id();
	struct loaded_vmcs *v;

	if (!crash_local_vmclear_enabled(cpu))
		return;

	list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
			    loaded_vmcss_on_cpu_link)
		vmcs_clear(v->vmcs);
}
#else
static inline void crash_enable_local_vmclear(int cpu) { }
static inline void crash_disable_local_vmclear(int cpu) { }
#endif /* CONFIG_KEXEC */

1292
static void __loaded_vmcs_clear(void *arg)
A
Avi Kivity 已提交
1293
{
1294
	struct loaded_vmcs *loaded_vmcs = arg;
1295
	int cpu = raw_smp_processor_id();
A
Avi Kivity 已提交
1296

1297 1298 1299
	if (loaded_vmcs->cpu != cpu)
		return; /* vcpu migration can race with cpu offline */
	if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
A
Avi Kivity 已提交
1300
		per_cpu(current_vmcs, cpu) = NULL;
1301
	crash_disable_local_vmclear(cpu);
1302
	list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
1303 1304 1305 1306 1307 1308 1309 1310 1311

	/*
	 * we should ensure updating loaded_vmcs->loaded_vmcss_on_cpu_link
	 * is before setting loaded_vmcs->vcpu to -1 which is done in
	 * loaded_vmcs_init. Otherwise, other cpu can see vcpu = -1 fist
	 * then adds the vmcs into percpu list before it is deleted.
	 */
	smp_wmb();

1312
	loaded_vmcs_init(loaded_vmcs);
1313
	crash_enable_local_vmclear(cpu);
A
Avi Kivity 已提交
1314 1315
}

1316
static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
A
Avi Kivity 已提交
1317
{
1318 1319 1320 1321 1322
	int cpu = loaded_vmcs->cpu;

	if (cpu != -1)
		smp_call_function_single(cpu,
			 __loaded_vmcs_clear, loaded_vmcs, 1);
A
Avi Kivity 已提交
1323 1324
}

1325
static inline void vpid_sync_vcpu_single(struct vcpu_vmx *vmx)
1326 1327 1328 1329
{
	if (vmx->vpid == 0)
		return;

1330 1331
	if (cpu_has_vmx_invvpid_single())
		__invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vmx->vpid, 0);
1332 1333
}

1334 1335 1336 1337 1338 1339 1340 1341 1342
static inline void vpid_sync_vcpu_global(void)
{
	if (cpu_has_vmx_invvpid_global())
		__invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
}

static inline void vpid_sync_context(struct vcpu_vmx *vmx)
{
	if (cpu_has_vmx_invvpid_single())
1343
		vpid_sync_vcpu_single(vmx);
1344 1345 1346 1347
	else
		vpid_sync_vcpu_global();
}

1348 1349 1350 1351 1352 1353 1354 1355
static inline void ept_sync_global(void)
{
	if (cpu_has_vmx_invept_global())
		__invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
}

static inline void ept_sync_context(u64 eptp)
{
1356
	if (enable_ept) {
1357 1358 1359 1360 1361 1362 1363
		if (cpu_has_vmx_invept_context())
			__invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
		else
			ept_sync_global();
	}
}

A
Avi Kivity 已提交
1364
static __always_inline unsigned long vmcs_readl(unsigned long field)
A
Avi Kivity 已提交
1365
{
1366
	unsigned long value;
A
Avi Kivity 已提交
1367

1368 1369
	asm volatile (__ex_clear(ASM_VMX_VMREAD_RDX_RAX, "%0")
		      : "=a"(value) : "d"(field) : "cc");
A
Avi Kivity 已提交
1370 1371 1372
	return value;
}

A
Avi Kivity 已提交
1373
static __always_inline u16 vmcs_read16(unsigned long field)
A
Avi Kivity 已提交
1374 1375 1376 1377
{
	return vmcs_readl(field);
}

A
Avi Kivity 已提交
1378
static __always_inline u32 vmcs_read32(unsigned long field)
A
Avi Kivity 已提交
1379 1380 1381 1382
{
	return vmcs_readl(field);
}

A
Avi Kivity 已提交
1383
static __always_inline u64 vmcs_read64(unsigned long field)
A
Avi Kivity 已提交
1384
{
1385
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
1386 1387 1388 1389 1390 1391
	return vmcs_readl(field);
#else
	return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
#endif
}

1392 1393 1394 1395 1396 1397 1398
static noinline void vmwrite_error(unsigned long field, unsigned long value)
{
	printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
	       field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
	dump_stack();
}

A
Avi Kivity 已提交
1399 1400 1401 1402
static void vmcs_writel(unsigned long field, unsigned long value)
{
	u8 error;

1403
	asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
M
Mike Day 已提交
1404
		       : "=q"(error) : "a"(value), "d"(field) : "cc");
1405 1406
	if (unlikely(error))
		vmwrite_error(field, value);
A
Avi Kivity 已提交
1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
}

static void vmcs_write16(unsigned long field, u16 value)
{
	vmcs_writel(field, value);
}

static void vmcs_write32(unsigned long field, u32 value)
{
	vmcs_writel(field, value);
}

static void vmcs_write64(unsigned long field, u64 value)
{
	vmcs_writel(field, value);
1422
#ifndef CONFIG_X86_64
A
Avi Kivity 已提交
1423 1424 1425 1426 1427
	asm volatile ("");
	vmcs_writel(field+1, value >> 32);
#endif
}

1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
static void vmcs_clear_bits(unsigned long field, u32 mask)
{
	vmcs_writel(field, vmcs_readl(field) & ~mask);
}

static void vmcs_set_bits(unsigned long field, u32 mask)
{
	vmcs_writel(field, vmcs_readl(field) | mask);
}

1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
static inline void vm_entry_controls_init(struct vcpu_vmx *vmx, u32 val)
{
	vmcs_write32(VM_ENTRY_CONTROLS, val);
	vmx->vm_entry_controls_shadow = val;
}

static inline void vm_entry_controls_set(struct vcpu_vmx *vmx, u32 val)
{
	if (vmx->vm_entry_controls_shadow != val)
		vm_entry_controls_init(vmx, val);
}

static inline u32 vm_entry_controls_get(struct vcpu_vmx *vmx)
{
	return vmx->vm_entry_controls_shadow;
}


static inline void vm_entry_controls_setbit(struct vcpu_vmx *vmx, u32 val)
{
	vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) | val);
}

static inline void vm_entry_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
{
	vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) & ~val);
}

static inline void vm_exit_controls_init(struct vcpu_vmx *vmx, u32 val)
{
	vmcs_write32(VM_EXIT_CONTROLS, val);
	vmx->vm_exit_controls_shadow = val;
}

static inline void vm_exit_controls_set(struct vcpu_vmx *vmx, u32 val)
{
	if (vmx->vm_exit_controls_shadow != val)
		vm_exit_controls_init(vmx, val);
}

static inline u32 vm_exit_controls_get(struct vcpu_vmx *vmx)
{
	return vmx->vm_exit_controls_shadow;
}


static inline void vm_exit_controls_setbit(struct vcpu_vmx *vmx, u32 val)
{
	vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) | val);
}

static inline void vm_exit_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
{
	vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) & ~val);
}

A
Avi Kivity 已提交
1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
static void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
{
	vmx->segment_cache.bitmask = 0;
}

static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
				       unsigned field)
{
	bool ret;
	u32 mask = 1 << (seg * SEG_FIELD_NR + field);

	if (!(vmx->vcpu.arch.regs_avail & (1 << VCPU_EXREG_SEGMENTS))) {
		vmx->vcpu.arch.regs_avail |= (1 << VCPU_EXREG_SEGMENTS);
		vmx->segment_cache.bitmask = 0;
	}
	ret = vmx->segment_cache.bitmask & mask;
	vmx->segment_cache.bitmask |= mask;
	return ret;
}

static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
{
	u16 *p = &vmx->segment_cache.seg[seg].selector;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
		*p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
	return *p;
}

static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
{
	ulong *p = &vmx->segment_cache.seg[seg].base;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
		*p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
	return *p;
}

static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
{
	u32 *p = &vmx->segment_cache.seg[seg].limit;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
		*p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
	return *p;
}

static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
{
	u32 *p = &vmx->segment_cache.seg[seg].ar;

	if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
		*p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
	return *p;
}

1550 1551 1552 1553
static void update_exception_bitmap(struct kvm_vcpu *vcpu)
{
	u32 eb;

J
Jan Kiszka 已提交
1554 1555 1556 1557 1558 1559
	eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
	     (1u << NM_VECTOR) | (1u << DB_VECTOR);
	if ((vcpu->guest_debug &
	     (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
	    (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
		eb |= 1u << BP_VECTOR;
1560
	if (to_vmx(vcpu)->rmode.vm86_active)
1561
		eb = ~0;
1562
	if (enable_ept)
1563
		eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
1564 1565
	if (vcpu->fpu_active)
		eb &= ~(1u << NM_VECTOR);
1566 1567 1568 1569 1570 1571 1572 1573 1574

	/* When we are running a nested L2 guest and L1 specified for it a
	 * certain exception bitmap, we must trap the same exceptions and pass
	 * them to L1. When running L2, we will only handle the exceptions
	 * specified above if L1 did not want them.
	 */
	if (is_guest_mode(vcpu))
		eb |= get_vmcs12(vcpu)->exception_bitmap;

1575 1576 1577
	vmcs_write32(EXCEPTION_BITMAP, eb);
}

1578 1579
static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
		unsigned long entry, unsigned long exit)
1580
{
1581 1582
	vm_entry_controls_clearbit(vmx, entry);
	vm_exit_controls_clearbit(vmx, exit);
1583 1584
}

1585 1586 1587 1588 1589
static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
{
	unsigned i;
	struct msr_autoload *m = &vmx->msr_autoload;

1590 1591 1592
	switch (msr) {
	case MSR_EFER:
		if (cpu_has_load_ia32_efer) {
1593 1594
			clear_atomic_switch_msr_special(vmx,
					VM_ENTRY_LOAD_IA32_EFER,
1595 1596 1597 1598 1599 1600
					VM_EXIT_LOAD_IA32_EFER);
			return;
		}
		break;
	case MSR_CORE_PERF_GLOBAL_CTRL:
		if (cpu_has_load_perf_global_ctrl) {
1601
			clear_atomic_switch_msr_special(vmx,
1602 1603 1604 1605 1606
					VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
					VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
			return;
		}
		break;
A
Avi Kivity 已提交
1607 1608
	}

1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621
	for (i = 0; i < m->nr; ++i)
		if (m->guest[i].index == msr)
			break;

	if (i == m->nr)
		return;
	--m->nr;
	m->guest[i] = m->guest[m->nr];
	m->host[i] = m->host[m->nr];
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
}

1622 1623 1624 1625
static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
		unsigned long entry, unsigned long exit,
		unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
		u64 guest_val, u64 host_val)
1626 1627 1628
{
	vmcs_write64(guest_val_vmcs, guest_val);
	vmcs_write64(host_val_vmcs, host_val);
1629 1630
	vm_entry_controls_setbit(vmx, entry);
	vm_exit_controls_setbit(vmx, exit);
1631 1632
}

1633 1634 1635 1636 1637 1638
static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
				  u64 guest_val, u64 host_val)
{
	unsigned i;
	struct msr_autoload *m = &vmx->msr_autoload;

1639 1640 1641
	switch (msr) {
	case MSR_EFER:
		if (cpu_has_load_ia32_efer) {
1642 1643
			add_atomic_switch_msr_special(vmx,
					VM_ENTRY_LOAD_IA32_EFER,
1644 1645 1646 1647 1648 1649 1650 1651 1652
					VM_EXIT_LOAD_IA32_EFER,
					GUEST_IA32_EFER,
					HOST_IA32_EFER,
					guest_val, host_val);
			return;
		}
		break;
	case MSR_CORE_PERF_GLOBAL_CTRL:
		if (cpu_has_load_perf_global_ctrl) {
1653
			add_atomic_switch_msr_special(vmx,
1654 1655 1656 1657 1658 1659 1660 1661
					VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
					VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
					GUEST_IA32_PERF_GLOBAL_CTRL,
					HOST_IA32_PERF_GLOBAL_CTRL,
					guest_val, host_val);
			return;
		}
		break;
A
Avi Kivity 已提交
1662 1663
	}

1664 1665 1666 1667
	for (i = 0; i < m->nr; ++i)
		if (m->guest[i].index == msr)
			break;

1668
	if (i == NR_AUTOLOAD_MSRS) {
1669
		printk_once(KERN_WARNING "Not enough msr switch entries. "
1670 1671 1672
				"Can't add msr %x\n", msr);
		return;
	} else if (i == m->nr) {
1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683
		++m->nr;
		vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
		vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
	}

	m->guest[i].index = msr;
	m->guest[i].value = guest_val;
	m->host[i].index = msr;
	m->host[i].value = host_val;
}

1684 1685 1686 1687 1688
static void reload_tss(void)
{
	/*
	 * VT restores TR but not its size.  Useless.
	 */
1689
	struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
1690
	struct desc_struct *descs;
1691

1692
	descs = (void *)gdt->address;
1693 1694 1695 1696
	descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
	load_TR_desc();
}

A
Avi Kivity 已提交
1697
static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
1698
{
R
Roel Kluin 已提交
1699
	u64 guest_efer;
1700 1701
	u64 ignore_bits;

1702
	guest_efer = vmx->vcpu.arch.efer;
R
Roel Kluin 已提交
1703

1704
	/*
G
Guo Chao 已提交
1705
	 * NX is emulated; LMA and LME handled by hardware; SCE meaningless
1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716
	 * outside long mode
	 */
	ignore_bits = EFER_NX | EFER_SCE;
#ifdef CONFIG_X86_64
	ignore_bits |= EFER_LMA | EFER_LME;
	/* SCE is meaningful only in long mode on Intel */
	if (guest_efer & EFER_LMA)
		ignore_bits &= ~(u64)EFER_SCE;
#endif
	guest_efer &= ~ignore_bits;
	guest_efer |= host_efer & ignore_bits;
1717
	vmx->guest_msrs[efer_offset].data = guest_efer;
1718
	vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
1719 1720

	clear_atomic_switch_msr(vmx, MSR_EFER);
1721 1722 1723 1724 1725 1726 1727 1728

	/*
	 * On EPT, we can't emulate NX, so we must switch EFER atomically.
	 * On CPUs that support "load IA32_EFER", always switch EFER
	 * atomically, since it's faster than switching it manually.
	 */
	if (cpu_has_load_ia32_efer ||
	    (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
1729 1730 1731
		guest_efer = vmx->vcpu.arch.efer;
		if (!(guest_efer & EFER_LMA))
			guest_efer &= ~EFER_LME;
1732 1733 1734
		if (guest_efer != host_efer)
			add_atomic_switch_msr(vmx, MSR_EFER,
					      guest_efer, host_efer);
1735 1736 1737
		return false;
	}

1738
	return true;
1739 1740
}

1741 1742
static unsigned long segment_base(u16 selector)
{
1743
	struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
1744 1745 1746 1747 1748 1749 1750
	struct desc_struct *d;
	unsigned long table_base;
	unsigned long v;

	if (!(selector & ~3))
		return 0;

1751
	table_base = gdt->address;
1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776

	if (selector & 4) {           /* from ldt */
		u16 ldt_selector = kvm_read_ldt();

		if (!(ldt_selector & ~3))
			return 0;

		table_base = segment_base(ldt_selector);
	}
	d = (struct desc_struct *)(table_base + (selector & ~7));
	v = get_desc_base(d);
#ifdef CONFIG_X86_64
       if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
               v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
#endif
	return v;
}

static inline unsigned long kvm_read_tr_base(void)
{
	u16 tr;
	asm("str %0" : "=g"(tr));
	return segment_base(tr);
}

1777
static void vmx_save_host_state(struct kvm_vcpu *vcpu)
1778
{
1779
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1780
	int i;
1781

1782
	if (vmx->host_state.loaded)
1783 1784
		return;

1785
	vmx->host_state.loaded = 1;
1786 1787 1788 1789
	/*
	 * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
	 * allow segment selectors with cpl > 0 or ti == 1.
	 */
1790
	vmx->host_state.ldt_sel = kvm_read_ldt();
1791
	vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
1792
	savesegment(fs, vmx->host_state.fs_sel);
1793
	if (!(vmx->host_state.fs_sel & 7)) {
1794
		vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
1795 1796
		vmx->host_state.fs_reload_needed = 0;
	} else {
1797
		vmcs_write16(HOST_FS_SELECTOR, 0);
1798
		vmx->host_state.fs_reload_needed = 1;
1799
	}
1800
	savesegment(gs, vmx->host_state.gs_sel);
1801 1802
	if (!(vmx->host_state.gs_sel & 7))
		vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
1803 1804
	else {
		vmcs_write16(HOST_GS_SELECTOR, 0);
1805
		vmx->host_state.gs_ldt_reload_needed = 1;
1806 1807
	}

A
Avi Kivity 已提交
1808 1809 1810 1811 1812
#ifdef CONFIG_X86_64
	savesegment(ds, vmx->host_state.ds_sel);
	savesegment(es, vmx->host_state.es_sel);
#endif

1813 1814 1815 1816
#ifdef CONFIG_X86_64
	vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
	vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
#else
1817 1818
	vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
	vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
1819
#endif
1820 1821

#ifdef CONFIG_X86_64
1822 1823
	rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
	if (is_long_mode(&vmx->vcpu))
1824
		wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1825
#endif
1826 1827
	if (boot_cpu_has(X86_FEATURE_MPX))
		rdmsrl(MSR_IA32_BNDCFGS, vmx->host_state.msr_host_bndcfgs);
1828 1829
	for (i = 0; i < vmx->save_nmsrs; ++i)
		kvm_set_shared_msr(vmx->guest_msrs[i].index,
1830 1831
				   vmx->guest_msrs[i].data,
				   vmx->guest_msrs[i].mask);
1832 1833
}

1834
static void __vmx_load_host_state(struct vcpu_vmx *vmx)
1835
{
1836
	if (!vmx->host_state.loaded)
1837 1838
		return;

1839
	++vmx->vcpu.stat.host_state_reload;
1840
	vmx->host_state.loaded = 0;
1841 1842 1843 1844
#ifdef CONFIG_X86_64
	if (is_long_mode(&vmx->vcpu))
		rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
#endif
1845
	if (vmx->host_state.gs_ldt_reload_needed) {
1846
		kvm_load_ldt(vmx->host_state.ldt_sel);
1847
#ifdef CONFIG_X86_64
1848 1849 1850
		load_gs_index(vmx->host_state.gs_sel);
#else
		loadsegment(gs, vmx->host_state.gs_sel);
1851 1852
#endif
	}
1853 1854
	if (vmx->host_state.fs_reload_needed)
		loadsegment(fs, vmx->host_state.fs_sel);
A
Avi Kivity 已提交
1855 1856 1857 1858 1859 1860
#ifdef CONFIG_X86_64
	if (unlikely(vmx->host_state.ds_sel | vmx->host_state.es_sel)) {
		loadsegment(ds, vmx->host_state.ds_sel);
		loadsegment(es, vmx->host_state.es_sel);
	}
#endif
1861
	reload_tss();
1862
#ifdef CONFIG_X86_64
1863
	wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1864
#endif
1865 1866
	if (vmx->host_state.msr_host_bndcfgs)
		wrmsrl(MSR_IA32_BNDCFGS, vmx->host_state.msr_host_bndcfgs);
1867 1868 1869 1870 1871 1872
	/*
	 * If the FPU is not active (through the host task or
	 * the guest vcpu), then restore the cr0.TS bit.
	 */
	if (!user_has_fpu() && !vmx->vcpu.guest_fpu_loaded)
		stts();
1873
	load_gdt(this_cpu_ptr(&host_gdt));
1874 1875
}

1876 1877 1878 1879 1880 1881 1882
static void vmx_load_host_state(struct vcpu_vmx *vmx)
{
	preempt_disable();
	__vmx_load_host_state(vmx);
	preempt_enable();
}

A
Avi Kivity 已提交
1883 1884 1885 1886
/*
 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
 * vcpu mutex is already taken.
 */
1887
static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
A
Avi Kivity 已提交
1888
{
1889
	struct vcpu_vmx *vmx = to_vmx(vcpu);
1890
	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
A
Avi Kivity 已提交
1891

1892 1893
	if (!vmm_exclusive)
		kvm_cpu_vmxon(phys_addr);
1894 1895
	else if (vmx->loaded_vmcs->cpu != cpu)
		loaded_vmcs_clear(vmx->loaded_vmcs);
A
Avi Kivity 已提交
1896

1897 1898 1899
	if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
		per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
		vmcs_load(vmx->loaded_vmcs->vmcs);
A
Avi Kivity 已提交
1900 1901
	}

1902
	if (vmx->loaded_vmcs->cpu != cpu) {
1903
		struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
A
Avi Kivity 已提交
1904 1905
		unsigned long sysenter_esp;

1906
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1907
		local_irq_disable();
1908
		crash_disable_local_vmclear(cpu);
1909 1910 1911 1912 1913 1914 1915 1916

		/*
		 * Read loaded_vmcs->cpu should be before fetching
		 * loaded_vmcs->loaded_vmcss_on_cpu_link.
		 * See the comments in __loaded_vmcs_clear().
		 */
		smp_rmb();

1917 1918
		list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
			 &per_cpu(loaded_vmcss_on_cpu, cpu));
1919
		crash_enable_local_vmclear(cpu);
1920 1921
		local_irq_enable();

A
Avi Kivity 已提交
1922 1923 1924 1925
		/*
		 * Linux uses per-cpu TSS and GDT, so set these when switching
		 * processors.
		 */
1926
		vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
1927
		vmcs_writel(HOST_GDTR_BASE, gdt->address);   /* 22.2.4 */
A
Avi Kivity 已提交
1928 1929 1930

		rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
		vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
1931
		vmx->loaded_vmcs->cpu = cpu;
A
Avi Kivity 已提交
1932 1933 1934 1935 1936
	}
}

static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
{
1937
	__vmx_load_host_state(to_vmx(vcpu));
1938
	if (!vmm_exclusive) {
1939 1940
		__loaded_vmcs_clear(to_vmx(vcpu)->loaded_vmcs);
		vcpu->cpu = -1;
1941 1942
		kvm_cpu_vmxoff();
	}
A
Avi Kivity 已提交
1943 1944
}

1945 1946
static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
{
1947 1948
	ulong cr0;

1949 1950 1951
	if (vcpu->fpu_active)
		return;
	vcpu->fpu_active = 1;
1952 1953 1954 1955
	cr0 = vmcs_readl(GUEST_CR0);
	cr0 &= ~(X86_CR0_TS | X86_CR0_MP);
	cr0 |= kvm_read_cr0_bits(vcpu, X86_CR0_TS | X86_CR0_MP);
	vmcs_writel(GUEST_CR0, cr0);
1956
	update_exception_bitmap(vcpu);
1957
	vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
1958 1959 1960
	if (is_guest_mode(vcpu))
		vcpu->arch.cr0_guest_owned_bits &=
			~get_vmcs12(vcpu)->cr0_guest_host_mask;
1961
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
1962 1963
}

1964 1965
static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);

1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981
/*
 * Return the cr0 value that a nested guest would read. This is a combination
 * of the real cr0 used to run the guest (guest_cr0), and the bits shadowed by
 * its hypervisor (cr0_read_shadow).
 */
static inline unsigned long nested_read_cr0(struct vmcs12 *fields)
{
	return (fields->guest_cr0 & ~fields->cr0_guest_host_mask) |
		(fields->cr0_read_shadow & fields->cr0_guest_host_mask);
}
static inline unsigned long nested_read_cr4(struct vmcs12 *fields)
{
	return (fields->guest_cr4 & ~fields->cr4_guest_host_mask) |
		(fields->cr4_read_shadow & fields->cr4_guest_host_mask);
}

1982 1983
static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
{
1984 1985 1986
	/* Note that there is no vcpu->fpu_active = 0 here. The caller must
	 * set this *before* calling this function.
	 */
1987
	vmx_decache_cr0_guest_bits(vcpu);
1988
	vmcs_set_bits(GUEST_CR0, X86_CR0_TS | X86_CR0_MP);
1989
	update_exception_bitmap(vcpu);
1990 1991
	vcpu->arch.cr0_guest_owned_bits = 0;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006
	if (is_guest_mode(vcpu)) {
		/*
		 * L1's specified read shadow might not contain the TS bit,
		 * so now that we turned on shadowing of this bit, we need to
		 * set this bit of the shadow. Like in nested_vmx_run we need
		 * nested_read_cr0(vmcs12), but vmcs12->guest_cr0 is not yet
		 * up-to-date here because we just decached cr0.TS (and we'll
		 * only update vmcs12->guest_cr0 on nested exit).
		 */
		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
		vmcs12->guest_cr0 = (vmcs12->guest_cr0 & ~X86_CR0_TS) |
			(vcpu->arch.cr0 & X86_CR0_TS);
		vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
	} else
		vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
2007 2008
}

A
Avi Kivity 已提交
2009 2010
static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
{
2011
	unsigned long rflags, save_rflags;
2012

A
Avi Kivity 已提交
2013 2014 2015 2016 2017 2018 2019 2020 2021
	if (!test_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail)) {
		__set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
		rflags = vmcs_readl(GUEST_RFLAGS);
		if (to_vmx(vcpu)->rmode.vm86_active) {
			rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
			save_rflags = to_vmx(vcpu)->rmode.save_rflags;
			rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
		}
		to_vmx(vcpu)->rflags = rflags;
2022
	}
A
Avi Kivity 已提交
2023
	return to_vmx(vcpu)->rflags;
A
Avi Kivity 已提交
2024 2025 2026 2027
}

static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
A
Avi Kivity 已提交
2028 2029
	__set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
	to_vmx(vcpu)->rflags = rflags;
2030 2031
	if (to_vmx(vcpu)->rmode.vm86_active) {
		to_vmx(vcpu)->rmode.save_rflags = rflags;
2032
		rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
2033
	}
A
Avi Kivity 已提交
2034 2035 2036
	vmcs_writel(GUEST_RFLAGS, rflags);
}

2037
static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
2038 2039 2040 2041 2042
{
	u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	int ret = 0;

	if (interruptibility & GUEST_INTR_STATE_STI)
2043
		ret |= KVM_X86_SHADOW_INT_STI;
2044
	if (interruptibility & GUEST_INTR_STATE_MOV_SS)
2045
		ret |= KVM_X86_SHADOW_INT_MOV_SS;
2046

2047
	return ret;
2048 2049 2050 2051 2052 2053 2054 2055 2056
}

static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	u32 interruptibility = interruptibility_old;

	interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);

2057
	if (mask & KVM_X86_SHADOW_INT_MOV_SS)
2058
		interruptibility |= GUEST_INTR_STATE_MOV_SS;
2059
	else if (mask & KVM_X86_SHADOW_INT_STI)
2060 2061 2062 2063 2064 2065
		interruptibility |= GUEST_INTR_STATE_STI;

	if ((interruptibility != interruptibility_old))
		vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
}

A
Avi Kivity 已提交
2066 2067 2068 2069
static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
	unsigned long rip;

2070
	rip = kvm_rip_read(vcpu);
A
Avi Kivity 已提交
2071
	rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
2072
	kvm_rip_write(vcpu, rip);
A
Avi Kivity 已提交
2073

2074 2075
	/* skipping an emulated instruction also counts */
	vmx_set_interrupt_shadow(vcpu, 0);
A
Avi Kivity 已提交
2076 2077
}

2078 2079 2080 2081
/*
 * KVM wants to inject page-faults which it got to the guest. This function
 * checks whether in a nested guest, we need to inject them to L1 or L2.
 */
2082
static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned nr)
2083 2084 2085
{
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

2086
	if (!(vmcs12->exception_bitmap & (1u << nr)))
2087 2088
		return 0;

2089 2090 2091
	nested_vmx_vmexit(vcpu, to_vmx(vcpu)->exit_reason,
			  vmcs_read32(VM_EXIT_INTR_INFO),
			  vmcs_readl(EXIT_QUALIFICATION));
2092 2093 2094
	return 1;
}

2095
static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
2096 2097
				bool has_error_code, u32 error_code,
				bool reinject)
2098
{
2099
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2100
	u32 intr_info = nr | INTR_INFO_VALID_MASK;
2101

2102 2103
	if (!reinject && is_guest_mode(vcpu) &&
	    nested_vmx_check_exception(vcpu, nr))
2104 2105
		return;

2106
	if (has_error_code) {
2107
		vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
2108 2109
		intr_info |= INTR_INFO_DELIVER_CODE_MASK;
	}
2110

2111
	if (vmx->rmode.vm86_active) {
2112 2113 2114 2115
		int inc_eip = 0;
		if (kvm_exception_is_soft(nr))
			inc_eip = vcpu->arch.event_exit_inst_len;
		if (kvm_inject_realmode_interrupt(vcpu, nr, inc_eip) != EMULATE_DONE)
2116
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2117 2118 2119
		return;
	}

2120 2121 2122
	if (kvm_exception_is_soft(nr)) {
		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
			     vmx->vcpu.arch.event_exit_inst_len);
2123 2124 2125 2126 2127
		intr_info |= INTR_TYPE_SOFT_EXCEPTION;
	} else
		intr_info |= INTR_TYPE_HARD_EXCEPTION;

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
2128 2129
}

2130 2131 2132 2133 2134
static bool vmx_rdtscp_supported(void)
{
	return cpu_has_vmx_rdtscp();
}

2135 2136 2137 2138 2139
static bool vmx_invpcid_supported(void)
{
	return cpu_has_vmx_invpcid() && enable_ept;
}

2140 2141 2142
/*
 * Swap MSR entry in host/guest MSR entry array.
 */
R
Rusty Russell 已提交
2143
static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
2144
{
2145
	struct shared_msr_entry tmp;
2146 2147 2148 2149

	tmp = vmx->guest_msrs[to];
	vmx->guest_msrs[to] = vmx->guest_msrs[from];
	vmx->guest_msrs[from] = tmp;
2150 2151
}

2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170
static void vmx_set_msr_bitmap(struct kvm_vcpu *vcpu)
{
	unsigned long *msr_bitmap;

	if (irqchip_in_kernel(vcpu->kvm) && apic_x2apic_mode(vcpu->arch.apic)) {
		if (is_long_mode(vcpu))
			msr_bitmap = vmx_msr_bitmap_longmode_x2apic;
		else
			msr_bitmap = vmx_msr_bitmap_legacy_x2apic;
	} else {
		if (is_long_mode(vcpu))
			msr_bitmap = vmx_msr_bitmap_longmode;
		else
			msr_bitmap = vmx_msr_bitmap_legacy;
	}

	vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
}

2171 2172 2173 2174 2175
/*
 * Set up the vmcs to automatically save and restore system
 * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
 * mode, as fiddling with msrs is very expensive.
 */
R
Rusty Russell 已提交
2176
static void setup_msrs(struct vcpu_vmx *vmx)
2177
{
2178
	int save_nmsrs, index;
2179

2180 2181
	save_nmsrs = 0;
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
2182 2183
	if (is_long_mode(&vmx->vcpu)) {
		index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
2184
		if (index >= 0)
R
Rusty Russell 已提交
2185 2186
			move_msr_up(vmx, index, save_nmsrs++);
		index = __find_msr_index(vmx, MSR_LSTAR);
2187
		if (index >= 0)
R
Rusty Russell 已提交
2188 2189
			move_msr_up(vmx, index, save_nmsrs++);
		index = __find_msr_index(vmx, MSR_CSTAR);
2190
		if (index >= 0)
R
Rusty Russell 已提交
2191
			move_msr_up(vmx, index, save_nmsrs++);
2192 2193 2194
		index = __find_msr_index(vmx, MSR_TSC_AUX);
		if (index >= 0 && vmx->rdtscp_enabled)
			move_msr_up(vmx, index, save_nmsrs++);
2195
		/*
B
Brian Gerst 已提交
2196
		 * MSR_STAR is only needed on long mode guests, and only
2197 2198
		 * if efer.sce is enabled.
		 */
B
Brian Gerst 已提交
2199
		index = __find_msr_index(vmx, MSR_STAR);
2200
		if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
R
Rusty Russell 已提交
2201
			move_msr_up(vmx, index, save_nmsrs++);
2202 2203
	}
#endif
A
Avi Kivity 已提交
2204 2205
	index = __find_msr_index(vmx, MSR_EFER);
	if (index >= 0 && update_transition_efer(vmx, index))
2206
		move_msr_up(vmx, index, save_nmsrs++);
2207

2208
	vmx->save_nmsrs = save_nmsrs;
2209

2210 2211
	if (cpu_has_vmx_msr_bitmap())
		vmx_set_msr_bitmap(&vmx->vcpu);
2212 2213
}

A
Avi Kivity 已提交
2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226
/*
 * reads and returns guest's timestamp counter "register"
 * guest_tsc = host_tsc + tsc_offset    -- 21.3
 */
static u64 guest_read_tsc(void)
{
	u64 host_tsc, tsc_offset;

	rdtscll(host_tsc);
	tsc_offset = vmcs_read64(TSC_OFFSET);
	return host_tsc + tsc_offset;
}

N
Nadav Har'El 已提交
2227 2228 2229 2230
/*
 * Like guest_read_tsc, but always returns L1's notion of the timestamp
 * counter, even if a nested guest (L2) is currently running.
 */
2231
static u64 vmx_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
N
Nadav Har'El 已提交
2232
{
2233
	u64 tsc_offset;
N
Nadav Har'El 已提交
2234 2235 2236 2237 2238 2239 2240

	tsc_offset = is_guest_mode(vcpu) ?
		to_vmx(vcpu)->nested.vmcs01_tsc_offset :
		vmcs_read64(TSC_OFFSET);
	return host_tsc + tsc_offset;
}

2241
/*
2242 2243
 * Engage any workarounds for mis-matched TSC rates.  Currently limited to
 * software catchup for faster rates on slower CPUs.
2244
 */
2245
static void vmx_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
2246
{
2247 2248 2249 2250 2251 2252 2253 2254
	if (!scale)
		return;

	if (user_tsc_khz > tsc_khz) {
		vcpu->arch.tsc_catchup = 1;
		vcpu->arch.tsc_always_catchup = 1;
	} else
		WARN(1, "user requested TSC rate below hardware speed\n");
2255 2256
}

W
Will Auld 已提交
2257 2258 2259 2260 2261
static u64 vmx_read_tsc_offset(struct kvm_vcpu *vcpu)
{
	return vmcs_read64(TSC_OFFSET);
}

A
Avi Kivity 已提交
2262
/*
2263
 * writes 'offset' into guest's timestamp counter offset register
A
Avi Kivity 已提交
2264
 */
2265
static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
A
Avi Kivity 已提交
2266
{
2267
	if (is_guest_mode(vcpu)) {
2268
		/*
2269 2270 2271 2272
		 * We're here if L1 chose not to trap WRMSR to TSC. According
		 * to the spec, this should set L1's TSC; The offset that L1
		 * set for L2 remains unchanged, and still needs to be added
		 * to the newly set TSC to get L2's TSC.
2273
		 */
2274 2275 2276 2277 2278 2279 2280 2281
		struct vmcs12 *vmcs12;
		to_vmx(vcpu)->nested.vmcs01_tsc_offset = offset;
		/* recalculate vmcs02.TSC_OFFSET: */
		vmcs12 = get_vmcs12(vcpu);
		vmcs_write64(TSC_OFFSET, offset +
			(nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETING) ?
			 vmcs12->tsc_offset : 0));
	} else {
2282 2283
		trace_kvm_write_tsc_offset(vcpu->vcpu_id,
					   vmcs_read64(TSC_OFFSET), offset);
2284 2285
		vmcs_write64(TSC_OFFSET, offset);
	}
A
Avi Kivity 已提交
2286 2287
}

2288
static void vmx_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment, bool host)
Z
Zachary Amsden 已提交
2289 2290
{
	u64 offset = vmcs_read64(TSC_OFFSET);
2291

Z
Zachary Amsden 已提交
2292
	vmcs_write64(TSC_OFFSET, offset + adjustment);
2293 2294 2295
	if (is_guest_mode(vcpu)) {
		/* Even when running L2, the adjustment needs to apply to L1 */
		to_vmx(vcpu)->nested.vmcs01_tsc_offset += adjustment;
2296 2297 2298
	} else
		trace_kvm_write_tsc_offset(vcpu->vcpu_id, offset,
					   offset + adjustment);
Z
Zachary Amsden 已提交
2299 2300
}

2301 2302 2303 2304 2305
static u64 vmx_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
{
	return target_tsc - native_read_tsc();
}

2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322
static bool guest_cpuid_has_vmx(struct kvm_vcpu *vcpu)
{
	struct kvm_cpuid_entry2 *best = kvm_find_cpuid_entry(vcpu, 1, 0);
	return best && (best->ecx & (1 << (X86_FEATURE_VMX & 31)));
}

/*
 * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
 * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
 * all guests if the "nested" module option is off, and can also be disabled
 * for a single guest by disabling its VMX cpuid bit.
 */
static inline bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
{
	return nested && guest_cpuid_has_vmx(vcpu);
}

2323 2324 2325 2326 2327 2328 2329 2330 2331 2332
/*
 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
 * returned for the various VMX controls MSRs when nested VMX is enabled.
 * The same values should also be used to verify that vmcs12 control fields are
 * valid during nested entry from L1 to L2.
 * Each of these control msrs has a low and high 32-bit half: A low bit is on
 * if the corresponding bit in the (32-bit) control field *must* be on, and a
 * bit in the high half is on if the corresponding bit in the control field
 * may be on. See also vmx_control_verify().
 */
2333
static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350
{
	/*
	 * Note that as a general rule, the high half of the MSRs (bits in
	 * the control fields which may be 1) should be initialized by the
	 * intersection of the underlying hardware's MSR (i.e., features which
	 * can be supported) and the list of features we want to expose -
	 * because they are known to be properly supported in our code.
	 * Also, usually, the low half of the MSRs (bits which must be 1) can
	 * be set to 0, meaning that L1 may turn off any of these bits. The
	 * reason is that if one of these bits is necessary, it will appear
	 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
	 * fields of vmcs01 and vmcs02, will turn these bits off - and
	 * nested_vmx_exit_handled() will not pass related exits to L1.
	 * These rules have exceptions below.
	 */

	/* pin-based controls */
2351
	rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
2352 2353 2354 2355 2356 2357 2358 2359 2360 2361
		vmx->nested.nested_vmx_pinbased_ctls_low,
		vmx->nested.nested_vmx_pinbased_ctls_high);
	vmx->nested.nested_vmx_pinbased_ctls_low |=
		PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
	vmx->nested.nested_vmx_pinbased_ctls_high &=
		PIN_BASED_EXT_INTR_MASK |
		PIN_BASED_NMI_EXITING |
		PIN_BASED_VIRTUAL_NMIS;
	vmx->nested.nested_vmx_pinbased_ctls_high |=
		PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
2362
		PIN_BASED_VMX_PREEMPTION_TIMER;
2363

2364
	/* exit controls */
2365
	rdmsr(MSR_IA32_VMX_EXIT_CTLS,
2366 2367 2368 2369
		vmx->nested.nested_vmx_exit_ctls_low,
		vmx->nested.nested_vmx_exit_ctls_high);
	vmx->nested.nested_vmx_exit_ctls_low =
		VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
2370

2371
	vmx->nested.nested_vmx_exit_ctls_high &=
2372
#ifdef CONFIG_X86_64
2373
		VM_EXIT_HOST_ADDR_SPACE_SIZE |
2374
#endif
2375
		VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
2376 2377
	vmx->nested.nested_vmx_exit_ctls_high |=
		VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
2378
		VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
2379 2380
		VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;

2381
	if (vmx_mpx_supported())
2382
		vmx->nested.nested_vmx_exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
2383

2384
	/* We support free control of debug control saving. */
2385 2386
	vmx->nested.nested_vmx_true_exit_ctls_low =
		vmx->nested.nested_vmx_exit_ctls_low &
2387 2388
		~VM_EXIT_SAVE_DEBUG_CONTROLS;

2389 2390
	/* entry controls */
	rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
2391 2392 2393 2394 2395
		vmx->nested.nested_vmx_entry_ctls_low,
		vmx->nested.nested_vmx_entry_ctls_high);
	vmx->nested.nested_vmx_entry_ctls_low =
		VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
	vmx->nested.nested_vmx_entry_ctls_high &=
2396 2397 2398 2399
#ifdef CONFIG_X86_64
		VM_ENTRY_IA32E_MODE |
#endif
		VM_ENTRY_LOAD_IA32_PAT;
2400 2401
	vmx->nested.nested_vmx_entry_ctls_high |=
		(VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
2402
	if (vmx_mpx_supported())
2403
		vmx->nested.nested_vmx_entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
2404

2405
	/* We support free control of debug control loading. */
2406 2407
	vmx->nested.nested_vmx_true_entry_ctls_low =
		vmx->nested.nested_vmx_entry_ctls_low &
2408 2409
		~VM_ENTRY_LOAD_DEBUG_CONTROLS;

2410 2411
	/* cpu-based controls */
	rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
2412 2413 2414 2415 2416
		vmx->nested.nested_vmx_procbased_ctls_low,
		vmx->nested.nested_vmx_procbased_ctls_high);
	vmx->nested.nested_vmx_procbased_ctls_low =
		CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
	vmx->nested.nested_vmx_procbased_ctls_high &=
2417 2418
		CPU_BASED_VIRTUAL_INTR_PENDING |
		CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING |
2419 2420 2421 2422 2423 2424 2425 2426
		CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
		CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
		CPU_BASED_CR3_STORE_EXITING |
#ifdef CONFIG_X86_64
		CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
#endif
		CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
		CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_EXITING |
2427
		CPU_BASED_RDPMC_EXITING | CPU_BASED_RDTSC_EXITING |
2428
		CPU_BASED_PAUSE_EXITING | CPU_BASED_TPR_SHADOW |
2429 2430 2431 2432 2433 2434 2435
		CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
	/*
	 * We can allow some features even when not supported by the
	 * hardware. For example, L1 can specify an MSR bitmap - and we
	 * can use it to avoid exits to L1 - even when L0 runs L2
	 * without MSR bitmaps.
	 */
2436 2437
	vmx->nested.nested_vmx_procbased_ctls_high |=
		CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
2438
		CPU_BASED_USE_MSR_BITMAPS;
2439

2440
	/* We support free control of CR3 access interception. */
2441 2442
	vmx->nested.nested_vmx_true_procbased_ctls_low =
		vmx->nested.nested_vmx_procbased_ctls_low &
2443 2444
		~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);

2445 2446
	/* secondary cpu-based controls */
	rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
2447 2448 2449 2450
		vmx->nested.nested_vmx_secondary_ctls_low,
		vmx->nested.nested_vmx_secondary_ctls_high);
	vmx->nested.nested_vmx_secondary_ctls_low = 0;
	vmx->nested.nested_vmx_secondary_ctls_high &=
2451
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2452
		SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2453
		SECONDARY_EXEC_APIC_REGISTER_VIRT |
2454
		SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2455 2456
		SECONDARY_EXEC_WBINVD_EXITING |
		SECONDARY_EXEC_XSAVES;
2457

2458 2459
	if (enable_ept) {
		/* nested EPT: emulate EPT also to L1 */
2460 2461
		vmx->nested.nested_vmx_secondary_ctls_high |=
			SECONDARY_EXEC_ENABLE_EPT |
2462
			SECONDARY_EXEC_UNRESTRICTED_GUEST;
2463
		vmx->nested.nested_vmx_ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
2464 2465
			 VMX_EPTP_WB_BIT | VMX_EPT_2MB_PAGE_BIT |
			 VMX_EPT_INVEPT_BIT;
2466
		vmx->nested.nested_vmx_ept_caps &= vmx_capability.ept;
2467
		/*
2468 2469 2470
		 * For nested guests, we don't do anything specific
		 * for single context invalidation. Hence, only advertise
		 * support for global context invalidation.
2471
		 */
2472
		vmx->nested.nested_vmx_ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT;
2473
	} else
2474
		vmx->nested.nested_vmx_ept_caps = 0;
2475

2476
	/* miscellaneous data */
2477 2478 2479 2480 2481 2482
	rdmsr(MSR_IA32_VMX_MISC,
		vmx->nested.nested_vmx_misc_low,
		vmx->nested.nested_vmx_misc_high);
	vmx->nested.nested_vmx_misc_low &= VMX_MISC_SAVE_EFER_LMA;
	vmx->nested.nested_vmx_misc_low |=
		VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
2483
		VMX_MISC_ACTIVITY_HLT;
2484
	vmx->nested.nested_vmx_misc_high = 0;
2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499
}

static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
{
	/*
	 * Bits 0 in high must be 0, and bits 1 in low must be 1.
	 */
	return ((control & high) | low) == control;
}

static inline u64 vmx_control_msr(u32 low, u32 high)
{
	return low | ((u64)high << 32);
}

2500
/* Returns 0 on success, non-0 otherwise. */
2501 2502
static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
2503 2504
	struct vcpu_vmx *vmx = to_vmx(vcpu);

2505 2506 2507 2508 2509 2510 2511 2512
	switch (msr_index) {
	case MSR_IA32_VMX_BASIC:
		/*
		 * This MSR reports some information about VMX support. We
		 * should return information about the VMX we emulate for the
		 * guest, and the VMCS structure we give it - not about the
		 * VMX support of the underlying hardware.
		 */
2513
		*pdata = VMCS12_REVISION | VMX_BASIC_TRUE_CTLS |
2514 2515 2516 2517 2518
			   ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
			   (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
		break;
	case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
	case MSR_IA32_VMX_PINBASED_CTLS:
2519 2520 2521
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_pinbased_ctls_low,
			vmx->nested.nested_vmx_pinbased_ctls_high);
2522 2523
		break;
	case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
2524 2525 2526
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_true_procbased_ctls_low,
			vmx->nested.nested_vmx_procbased_ctls_high);
2527
		break;
2528
	case MSR_IA32_VMX_PROCBASED_CTLS:
2529 2530 2531
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_procbased_ctls_low,
			vmx->nested.nested_vmx_procbased_ctls_high);
2532 2533
		break;
	case MSR_IA32_VMX_TRUE_EXIT_CTLS:
2534 2535 2536
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_true_exit_ctls_low,
			vmx->nested.nested_vmx_exit_ctls_high);
2537
		break;
2538
	case MSR_IA32_VMX_EXIT_CTLS:
2539 2540 2541
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_exit_ctls_low,
			vmx->nested.nested_vmx_exit_ctls_high);
2542 2543
		break;
	case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
2544 2545 2546
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_true_entry_ctls_low,
			vmx->nested.nested_vmx_entry_ctls_high);
2547
		break;
2548
	case MSR_IA32_VMX_ENTRY_CTLS:
2549 2550 2551
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_entry_ctls_low,
			vmx->nested.nested_vmx_entry_ctls_high);
2552 2553
		break;
	case MSR_IA32_VMX_MISC:
2554 2555 2556
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_misc_low,
			vmx->nested.nested_vmx_misc_high);
2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577
		break;
	/*
	 * These MSRs specify bits which the guest must keep fixed (on or off)
	 * while L1 is in VMXON mode (in L1's root mode, or running an L2).
	 * We picked the standard core2 setting.
	 */
#define VMXON_CR0_ALWAYSON	(X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
#define VMXON_CR4_ALWAYSON	X86_CR4_VMXE
	case MSR_IA32_VMX_CR0_FIXED0:
		*pdata = VMXON_CR0_ALWAYSON;
		break;
	case MSR_IA32_VMX_CR0_FIXED1:
		*pdata = -1ULL;
		break;
	case MSR_IA32_VMX_CR4_FIXED0:
		*pdata = VMXON_CR4_ALWAYSON;
		break;
	case MSR_IA32_VMX_CR4_FIXED1:
		*pdata = -1ULL;
		break;
	case MSR_IA32_VMX_VMCS_ENUM:
2578
		*pdata = 0x2e; /* highest index: VMX_PREEMPTION_TIMER_VALUE */
2579 2580
		break;
	case MSR_IA32_VMX_PROCBASED_CTLS2:
2581 2582 2583
		*pdata = vmx_control_msr(
			vmx->nested.nested_vmx_secondary_ctls_low,
			vmx->nested.nested_vmx_secondary_ctls_high);
2584 2585
		break;
	case MSR_IA32_VMX_EPT_VPID_CAP:
2586
		/* Currently, no nested vpid support */
2587
		*pdata = vmx->nested.nested_vmx_ept_caps;
2588 2589 2590
		break;
	default:
		return 1;
2591 2592
	}

2593 2594 2595
	return 0;
}

A
Avi Kivity 已提交
2596 2597 2598 2599 2600 2601 2602 2603
/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
	u64 data;
2604
	struct shared_msr_entry *msr;
A
Avi Kivity 已提交
2605 2606 2607 2608 2609 2610 2611

	if (!pdata) {
		printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
		return -EINVAL;
	}

	switch (msr_index) {
2612
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2613 2614 2615 2616 2617 2618
	case MSR_FS_BASE:
		data = vmcs_readl(GUEST_FS_BASE);
		break;
	case MSR_GS_BASE:
		data = vmcs_readl(GUEST_GS_BASE);
		break;
2619 2620 2621 2622
	case MSR_KERNEL_GS_BASE:
		vmx_load_host_state(to_vmx(vcpu));
		data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
		break;
2623
#endif
A
Avi Kivity 已提交
2624
	case MSR_EFER:
2625
		return kvm_get_msr_common(vcpu, msr_index, pdata);
2626
	case MSR_IA32_TSC:
A
Avi Kivity 已提交
2627 2628 2629 2630 2631 2632
		data = guest_read_tsc();
		break;
	case MSR_IA32_SYSENTER_CS:
		data = vmcs_read32(GUEST_SYSENTER_CS);
		break;
	case MSR_IA32_SYSENTER_EIP:
A
Avi Kivity 已提交
2633
		data = vmcs_readl(GUEST_SYSENTER_EIP);
A
Avi Kivity 已提交
2634 2635
		break;
	case MSR_IA32_SYSENTER_ESP:
A
Avi Kivity 已提交
2636
		data = vmcs_readl(GUEST_SYSENTER_ESP);
A
Avi Kivity 已提交
2637
		break;
2638
	case MSR_IA32_BNDCFGS:
2639 2640
		if (!vmx_mpx_supported())
			return 1;
2641 2642
		data = vmcs_read64(GUEST_BNDCFGS);
		break;
2643 2644 2645 2646 2647 2648 2649 2650 2651
	case MSR_IA32_FEATURE_CONTROL:
		if (!nested_vmx_allowed(vcpu))
			return 1;
		data = to_vmx(vcpu)->nested.msr_ia32_feature_control;
		break;
	case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
		if (!nested_vmx_allowed(vcpu))
			return 1;
		return vmx_get_vmx_msr(vcpu, msr_index, pdata);
W
Wanpeng Li 已提交
2652 2653 2654 2655 2656
	case MSR_IA32_XSS:
		if (!vmx_xsaves_supported())
			return 1;
		data = vcpu->arch.ia32_xss;
		break;
2657 2658 2659 2660
	case MSR_TSC_AUX:
		if (!to_vmx(vcpu)->rdtscp_enabled)
			return 1;
		/* Otherwise falls through */
A
Avi Kivity 已提交
2661
	default:
R
Rusty Russell 已提交
2662
		msr = find_msr_entry(to_vmx(vcpu), msr_index);
2663 2664 2665
		if (msr) {
			data = msr->data;
			break;
A
Avi Kivity 已提交
2666
		}
2667
		return kvm_get_msr_common(vcpu, msr_index, pdata);
A
Avi Kivity 已提交
2668 2669 2670 2671 2672 2673
	}

	*pdata = data;
	return 0;
}

2674 2675
static void vmx_leave_nested(struct kvm_vcpu *vcpu);

A
Avi Kivity 已提交
2676 2677 2678 2679 2680
/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
2681
static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
A
Avi Kivity 已提交
2682
{
2683
	struct vcpu_vmx *vmx = to_vmx(vcpu);
2684
	struct shared_msr_entry *msr;
2685
	int ret = 0;
2686 2687
	u32 msr_index = msr_info->index;
	u64 data = msr_info->data;
2688

A
Avi Kivity 已提交
2689
	switch (msr_index) {
2690
	case MSR_EFER:
2691
		ret = kvm_set_msr_common(vcpu, msr_info);
2692
		break;
2693
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2694
	case MSR_FS_BASE:
A
Avi Kivity 已提交
2695
		vmx_segment_cache_clear(vmx);
A
Avi Kivity 已提交
2696 2697 2698
		vmcs_writel(GUEST_FS_BASE, data);
		break;
	case MSR_GS_BASE:
A
Avi Kivity 已提交
2699
		vmx_segment_cache_clear(vmx);
A
Avi Kivity 已提交
2700 2701
		vmcs_writel(GUEST_GS_BASE, data);
		break;
2702 2703 2704 2705
	case MSR_KERNEL_GS_BASE:
		vmx_load_host_state(vmx);
		vmx->msr_guest_kernel_gs_base = data;
		break;
A
Avi Kivity 已提交
2706 2707 2708 2709 2710
#endif
	case MSR_IA32_SYSENTER_CS:
		vmcs_write32(GUEST_SYSENTER_CS, data);
		break;
	case MSR_IA32_SYSENTER_EIP:
A
Avi Kivity 已提交
2711
		vmcs_writel(GUEST_SYSENTER_EIP, data);
A
Avi Kivity 已提交
2712 2713
		break;
	case MSR_IA32_SYSENTER_ESP:
A
Avi Kivity 已提交
2714
		vmcs_writel(GUEST_SYSENTER_ESP, data);
A
Avi Kivity 已提交
2715
		break;
2716
	case MSR_IA32_BNDCFGS:
2717 2718
		if (!vmx_mpx_supported())
			return 1;
2719 2720
		vmcs_write64(GUEST_BNDCFGS, data);
		break;
2721
	case MSR_IA32_TSC:
2722
		kvm_write_tsc(vcpu, msr_info);
A
Avi Kivity 已提交
2723
		break;
S
Sheng Yang 已提交
2724 2725
	case MSR_IA32_CR_PAT:
		if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2726 2727
			if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
				return 1;
S
Sheng Yang 已提交
2728 2729 2730 2731
			vmcs_write64(GUEST_IA32_PAT, data);
			vcpu->arch.pat = data;
			break;
		}
2732
		ret = kvm_set_msr_common(vcpu, msr_info);
2733
		break;
W
Will Auld 已提交
2734 2735
	case MSR_IA32_TSC_ADJUST:
		ret = kvm_set_msr_common(vcpu, msr_info);
2736
		break;
2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747
	case MSR_IA32_FEATURE_CONTROL:
		if (!nested_vmx_allowed(vcpu) ||
		    (to_vmx(vcpu)->nested.msr_ia32_feature_control &
		     FEATURE_CONTROL_LOCKED && !msr_info->host_initiated))
			return 1;
		vmx->nested.msr_ia32_feature_control = data;
		if (msr_info->host_initiated && data == 0)
			vmx_leave_nested(vcpu);
		break;
	case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
		return 1; /* they are read-only */
W
Wanpeng Li 已提交
2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763
	case MSR_IA32_XSS:
		if (!vmx_xsaves_supported())
			return 1;
		/*
		 * The only supported bit as of Skylake is bit 8, but
		 * it is not supported on KVM.
		 */
		if (data != 0)
			return 1;
		vcpu->arch.ia32_xss = data;
		if (vcpu->arch.ia32_xss != host_xss)
			add_atomic_switch_msr(vmx, MSR_IA32_XSS,
				vcpu->arch.ia32_xss, host_xss);
		else
			clear_atomic_switch_msr(vmx, MSR_IA32_XSS);
		break;
2764 2765 2766 2767 2768 2769 2770
	case MSR_TSC_AUX:
		if (!vmx->rdtscp_enabled)
			return 1;
		/* Check reserved bit, higher 32 bits should be zero */
		if ((data >> 32) != 0)
			return 1;
		/* Otherwise falls through */
A
Avi Kivity 已提交
2771
	default:
R
Rusty Russell 已提交
2772
		msr = find_msr_entry(vmx, msr_index);
2773
		if (msr) {
2774
			u64 old_msr_data = msr->data;
2775
			msr->data = data;
2776 2777
			if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
				preempt_disable();
2778 2779
				ret = kvm_set_shared_msr(msr->index, msr->data,
							 msr->mask);
2780
				preempt_enable();
2781 2782
				if (ret)
					msr->data = old_msr_data;
2783
			}
2784
			break;
A
Avi Kivity 已提交
2785
		}
2786
		ret = kvm_set_msr_common(vcpu, msr_info);
A
Avi Kivity 已提交
2787 2788
	}

2789
	return ret;
A
Avi Kivity 已提交
2790 2791
}

2792
static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
A
Avi Kivity 已提交
2793
{
2794 2795 2796 2797 2798 2799 2800 2801
	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
	switch (reg) {
	case VCPU_REGS_RSP:
		vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
		break;
	case VCPU_REGS_RIP:
		vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
		break;
A
Avi Kivity 已提交
2802 2803 2804 2805
	case VCPU_EXREG_PDPTR:
		if (enable_ept)
			ept_save_pdptrs(vcpu);
		break;
2806 2807 2808
	default:
		break;
	}
A
Avi Kivity 已提交
2809 2810 2811 2812
}

static __init int cpu_has_kvm_support(void)
{
2813
	return cpu_has_vmx();
A
Avi Kivity 已提交
2814 2815 2816 2817 2818 2819 2820
}

static __init int vmx_disabled_by_bios(void)
{
	u64 msr;

	rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
2821
	if (msr & FEATURE_CONTROL_LOCKED) {
2822
		/* launched w/ TXT and VMX disabled */
2823 2824 2825
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
			&& tboot_enabled())
			return 1;
2826
		/* launched w/o TXT and VMX only enabled w/ TXT */
2827
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
2828
			&& (msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
2829 2830
			&& !tboot_enabled()) {
			printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
2831
				"activate TXT before enabling KVM\n");
2832
			return 1;
2833
		}
2834 2835 2836 2837
		/* launched w/o TXT and VMX disabled */
		if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
			&& !tboot_enabled())
			return 1;
2838 2839 2840
	}

	return 0;
A
Avi Kivity 已提交
2841 2842
}

2843 2844 2845 2846 2847 2848 2849
static void kvm_cpu_vmxon(u64 addr)
{
	asm volatile (ASM_VMX_VMXON_RAX
			: : "a"(&addr), "m"(addr)
			: "memory", "cc");
}

2850
static int hardware_enable(void)
A
Avi Kivity 已提交
2851 2852 2853
{
	int cpu = raw_smp_processor_id();
	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
2854
	u64 old, test_bits;
A
Avi Kivity 已提交
2855

2856 2857 2858
	if (read_cr4() & X86_CR4_VMXE)
		return -EBUSY;

2859
	INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871

	/*
	 * Now we can enable the vmclear operation in kdump
	 * since the loaded_vmcss_on_cpu list on this cpu
	 * has been initialized.
	 *
	 * Though the cpu is not in VMX operation now, there
	 * is no problem to enable the vmclear operation
	 * for the loaded_vmcss_on_cpu list is empty!
	 */
	crash_enable_local_vmclear(cpu);

A
Avi Kivity 已提交
2872
	rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
2873 2874 2875 2876 2877 2878 2879

	test_bits = FEATURE_CONTROL_LOCKED;
	test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
	if (tboot_enabled())
		test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;

	if ((old & test_bits) != test_bits) {
A
Avi Kivity 已提交
2880
		/* enable and lock */
2881 2882
		wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
	}
2883
	write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
2884

2885 2886 2887 2888
	if (vmm_exclusive) {
		kvm_cpu_vmxon(phys_addr);
		ept_sync_global();
	}
2889

2890
	native_store_gdt(this_cpu_ptr(&host_gdt));
2891

2892
	return 0;
A
Avi Kivity 已提交
2893 2894
}

2895
static void vmclear_local_loaded_vmcss(void)
2896 2897
{
	int cpu = raw_smp_processor_id();
2898
	struct loaded_vmcs *v, *n;
2899

2900 2901 2902
	list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
				 loaded_vmcss_on_cpu_link)
		__loaded_vmcs_clear(v);
2903 2904
}

2905 2906 2907 2908 2909

/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
 * tricks.
 */
static void kvm_cpu_vmxoff(void)
A
Avi Kivity 已提交
2910
{
2911
	asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
A
Avi Kivity 已提交
2912 2913
}

2914
static void hardware_disable(void)
2915
{
2916
	if (vmm_exclusive) {
2917
		vmclear_local_loaded_vmcss();
2918 2919
		kvm_cpu_vmxoff();
	}
2920
	write_cr4(read_cr4() & ~X86_CR4_VMXE);
2921 2922
}

2923
static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
M
Mike Day 已提交
2924
				      u32 msr, u32 *result)
2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935
{
	u32 vmx_msr_low, vmx_msr_high;
	u32 ctl = ctl_min | ctl_opt;

	rdmsr(msr, vmx_msr_low, vmx_msr_high);

	ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
	ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */

	/* Ensure minimum (required) set of control bits are supported. */
	if (ctl_min & ~ctl)
Y
Yang, Sheng 已提交
2936
		return -EIO;
2937 2938 2939 2940 2941

	*result = ctl;
	return 0;
}

A
Avi Kivity 已提交
2942 2943 2944 2945 2946 2947 2948 2949
static __init bool allow_1_setting(u32 msr, u32 ctl)
{
	u32 vmx_msr_low, vmx_msr_high;

	rdmsr(msr, vmx_msr_low, vmx_msr_high);
	return vmx_msr_high & ctl;
}

Y
Yang, Sheng 已提交
2950
static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
A
Avi Kivity 已提交
2951 2952
{
	u32 vmx_msr_low, vmx_msr_high;
S
Sheng Yang 已提交
2953
	u32 min, opt, min2, opt2;
2954 2955
	u32 _pin_based_exec_control = 0;
	u32 _cpu_based_exec_control = 0;
2956
	u32 _cpu_based_2nd_exec_control = 0;
2957 2958 2959
	u32 _vmexit_control = 0;
	u32 _vmentry_control = 0;

R
Raghavendra K T 已提交
2960
	min = CPU_BASED_HLT_EXITING |
2961 2962 2963 2964
#ifdef CONFIG_X86_64
	      CPU_BASED_CR8_LOAD_EXITING |
	      CPU_BASED_CR8_STORE_EXITING |
#endif
S
Sheng Yang 已提交
2965 2966
	      CPU_BASED_CR3_LOAD_EXITING |
	      CPU_BASED_CR3_STORE_EXITING |
2967 2968
	      CPU_BASED_USE_IO_BITMAPS |
	      CPU_BASED_MOV_DR_EXITING |
M
Marcelo Tosatti 已提交
2969
	      CPU_BASED_USE_TSC_OFFSETING |
2970 2971
	      CPU_BASED_MWAIT_EXITING |
	      CPU_BASED_MONITOR_EXITING |
A
Avi Kivity 已提交
2972 2973
	      CPU_BASED_INVLPG_EXITING |
	      CPU_BASED_RDPMC_EXITING;
2974

2975
	opt = CPU_BASED_TPR_SHADOW |
S
Sheng Yang 已提交
2976
	      CPU_BASED_USE_MSR_BITMAPS |
2977
	      CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
2978 2979
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
				&_cpu_based_exec_control) < 0)
Y
Yang, Sheng 已提交
2980
		return -EIO;
2981 2982 2983 2984 2985
#ifdef CONFIG_X86_64
	if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
		_cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
					   ~CPU_BASED_CR8_STORE_EXITING;
#endif
2986
	if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
S
Sheng Yang 已提交
2987 2988
		min2 = 0;
		opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2989
			SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2990
			SECONDARY_EXEC_WBINVD_EXITING |
S
Sheng Yang 已提交
2991
			SECONDARY_EXEC_ENABLE_VPID |
2992
			SECONDARY_EXEC_ENABLE_EPT |
2993
			SECONDARY_EXEC_UNRESTRICTED_GUEST |
2994
			SECONDARY_EXEC_PAUSE_LOOP_EXITING |
2995
			SECONDARY_EXEC_RDTSCP |
2996
			SECONDARY_EXEC_ENABLE_INVPCID |
2997
			SECONDARY_EXEC_APIC_REGISTER_VIRT |
2998
			SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
W
Wanpeng Li 已提交
2999
			SECONDARY_EXEC_SHADOW_VMCS |
K
Kai Huang 已提交
3000 3001
			SECONDARY_EXEC_XSAVES |
			SECONDARY_EXEC_ENABLE_PML;
S
Sheng Yang 已提交
3002 3003
		if (adjust_vmx_controls(min2, opt2,
					MSR_IA32_VMX_PROCBASED_CTLS2,
3004 3005 3006 3007 3008 3009 3010 3011
					&_cpu_based_2nd_exec_control) < 0)
			return -EIO;
	}
#ifndef CONFIG_X86_64
	if (!(_cpu_based_2nd_exec_control &
				SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
		_cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
#endif
3012 3013 3014

	if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
		_cpu_based_2nd_exec_control &= ~(
3015
				SECONDARY_EXEC_APIC_REGISTER_VIRT |
3016 3017
				SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
				SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
3018

S
Sheng Yang 已提交
3019
	if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
M
Marcelo Tosatti 已提交
3020 3021
		/* CR3 accesses and invlpg don't need to cause VM Exits when EPT
		   enabled */
3022 3023 3024
		_cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
					     CPU_BASED_CR3_STORE_EXITING |
					     CPU_BASED_INVLPG_EXITING);
S
Sheng Yang 已提交
3025 3026 3027
		rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
		      vmx_capability.ept, vmx_capability.vpid);
	}
3028

3029
	min = VM_EXIT_SAVE_DEBUG_CONTROLS;
3030 3031 3032
#ifdef CONFIG_X86_64
	min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
#endif
3033
	opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT |
3034
		VM_EXIT_ACK_INTR_ON_EXIT | VM_EXIT_CLEAR_BNDCFGS;
3035 3036
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
				&_vmexit_control) < 0)
Y
Yang, Sheng 已提交
3037
		return -EIO;
3038

3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049
	min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
	opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR;
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
				&_pin_based_exec_control) < 0)
		return -EIO;

	if (!(_cpu_based_2nd_exec_control &
		SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) ||
		!(_vmexit_control & VM_EXIT_ACK_INTR_ON_EXIT))
		_pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;

3050
	min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
3051
	opt = VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS;
3052 3053
	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
				&_vmentry_control) < 0)
Y
Yang, Sheng 已提交
3054
		return -EIO;
A
Avi Kivity 已提交
3055

N
Nguyen Anh Quynh 已提交
3056
	rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
3057 3058 3059

	/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
	if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
Y
Yang, Sheng 已提交
3060
		return -EIO;
3061 3062 3063 3064

#ifdef CONFIG_X86_64
	/* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
	if (vmx_msr_high & (1u<<16))
Y
Yang, Sheng 已提交
3065
		return -EIO;
3066 3067 3068 3069
#endif

	/* Require Write-Back (WB) memory type for VMCS accesses. */
	if (((vmx_msr_high >> 18) & 15) != 6)
Y
Yang, Sheng 已提交
3070
		return -EIO;
3071

Y
Yang, Sheng 已提交
3072 3073 3074
	vmcs_conf->size = vmx_msr_high & 0x1fff;
	vmcs_conf->order = get_order(vmcs_config.size);
	vmcs_conf->revision_id = vmx_msr_low;
3075

Y
Yang, Sheng 已提交
3076 3077
	vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
	vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
3078
	vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
Y
Yang, Sheng 已提交
3079 3080
	vmcs_conf->vmexit_ctrl         = _vmexit_control;
	vmcs_conf->vmentry_ctrl        = _vmentry_control;
3081

A
Avi Kivity 已提交
3082 3083 3084 3085 3086 3087
	cpu_has_load_ia32_efer =
		allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
				VM_ENTRY_LOAD_IA32_EFER)
		&& allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
				   VM_EXIT_LOAD_IA32_EFER);

3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
	cpu_has_load_perf_global_ctrl =
		allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
				VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
		&& allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
				   VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);

	/*
	 * Some cpus support VM_ENTRY_(LOAD|SAVE)_IA32_PERF_GLOBAL_CTRL
	 * but due to arrata below it can't be used. Workaround is to use
	 * msr load mechanism to switch IA32_PERF_GLOBAL_CTRL.
	 *
	 * VM Exit May Incorrectly Clear IA32_PERF_GLOBAL_CTRL [34:32]
	 *
	 * AAK155             (model 26)
	 * AAP115             (model 30)
	 * AAT100             (model 37)
	 * BC86,AAY89,BD102   (model 44)
	 * BA97               (model 46)
	 *
	 */
	if (cpu_has_load_perf_global_ctrl && boot_cpu_data.x86 == 0x6) {
		switch (boot_cpu_data.x86_model) {
		case 26:
		case 30:
		case 37:
		case 44:
		case 46:
			cpu_has_load_perf_global_ctrl = false;
			printk_once(KERN_WARNING"kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
					"does not work properly. Using workaround\n");
			break;
		default:
			break;
		}
	}

W
Wanpeng Li 已提交
3124 3125 3126
	if (cpu_has_xsaves)
		rdmsrl(MSR_IA32_XSS, host_xss);

3127
	return 0;
N
Nguyen Anh Quynh 已提交
3128
}
A
Avi Kivity 已提交
3129 3130 3131 3132 3133 3134 3135

static struct vmcs *alloc_vmcs_cpu(int cpu)
{
	int node = cpu_to_node(cpu);
	struct page *pages;
	struct vmcs *vmcs;

3136
	pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order);
A
Avi Kivity 已提交
3137 3138 3139
	if (!pages)
		return NULL;
	vmcs = page_address(pages);
3140 3141
	memset(vmcs, 0, vmcs_config.size);
	vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
A
Avi Kivity 已提交
3142 3143 3144 3145 3146
	return vmcs;
}

static struct vmcs *alloc_vmcs(void)
{
3147
	return alloc_vmcs_cpu(raw_smp_processor_id());
A
Avi Kivity 已提交
3148 3149 3150 3151
}

static void free_vmcs(struct vmcs *vmcs)
{
3152
	free_pages((unsigned long)vmcs, vmcs_config.order);
A
Avi Kivity 已提交
3153 3154
}

3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166
/*
 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
 */
static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
{
	if (!loaded_vmcs->vmcs)
		return;
	loaded_vmcs_clear(loaded_vmcs);
	free_vmcs(loaded_vmcs->vmcs);
	loaded_vmcs->vmcs = NULL;
}

3167
static void free_kvm_area(void)
A
Avi Kivity 已提交
3168 3169 3170
{
	int cpu;

Z
Zachary Amsden 已提交
3171
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
3172
		free_vmcs(per_cpu(vmxarea, cpu));
Z
Zachary Amsden 已提交
3173 3174
		per_cpu(vmxarea, cpu) = NULL;
	}
A
Avi Kivity 已提交
3175 3176
}

3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211
static void init_vmcs_shadow_fields(void)
{
	int i, j;

	/* No checks for read only fields yet */

	for (i = j = 0; i < max_shadow_read_write_fields; i++) {
		switch (shadow_read_write_fields[i]) {
		case GUEST_BNDCFGS:
			if (!vmx_mpx_supported())
				continue;
			break;
		default:
			break;
		}

		if (j < i)
			shadow_read_write_fields[j] =
				shadow_read_write_fields[i];
		j++;
	}
	max_shadow_read_write_fields = j;

	/* shadowed fields guest access without vmexit */
	for (i = 0; i < max_shadow_read_write_fields; i++) {
		clear_bit(shadow_read_write_fields[i],
			  vmx_vmwrite_bitmap);
		clear_bit(shadow_read_write_fields[i],
			  vmx_vmread_bitmap);
	}
	for (i = 0; i < max_shadow_read_only_fields; i++)
		clear_bit(shadow_read_only_fields[i],
			  vmx_vmread_bitmap);
}

A
Avi Kivity 已提交
3212 3213 3214 3215
static __init int alloc_kvm_area(void)
{
	int cpu;

Z
Zachary Amsden 已提交
3216
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229
		struct vmcs *vmcs;

		vmcs = alloc_vmcs_cpu(cpu);
		if (!vmcs) {
			free_kvm_area();
			return -ENOMEM;
		}

		per_cpu(vmxarea, cpu) = vmcs;
	}
	return 0;
}

3230 3231 3232 3233 3234
static bool emulation_required(struct kvm_vcpu *vcpu)
{
	return emulate_invalid_guest_state && !guest_state_valid(vcpu);
}

3235
static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
3236
		struct kvm_segment *save)
A
Avi Kivity 已提交
3237
{
3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249
	if (!emulate_invalid_guest_state) {
		/*
		 * CS and SS RPL should be equal during guest entry according
		 * to VMX spec, but in reality it is not always so. Since vcpu
		 * is in the middle of the transition from real mode to
		 * protected mode it is safe to assume that RPL 0 is a good
		 * default value.
		 */
		if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
			save->selector &= ~SELECTOR_RPL_MASK;
		save->dpl = save->selector & SELECTOR_RPL_MASK;
		save->s = 1;
A
Avi Kivity 已提交
3250
	}
3251
	vmx_set_segment(vcpu, save, seg);
A
Avi Kivity 已提交
3252 3253 3254 3255 3256
}

static void enter_pmode(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
3257
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
3258

3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269
	/*
	 * Update real mode segment cache. It may be not up-to-date if sement
	 * register was written while vcpu was in a guest mode.
	 */
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);

3270
	vmx->rmode.vm86_active = 0;
A
Avi Kivity 已提交
3271

A
Avi Kivity 已提交
3272 3273
	vmx_segment_cache_clear(vmx);

3274
	vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
A
Avi Kivity 已提交
3275 3276

	flags = vmcs_readl(GUEST_RFLAGS);
3277 3278
	flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
	flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
A
Avi Kivity 已提交
3279 3280
	vmcs_writel(GUEST_RFLAGS, flags);

3281 3282
	vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
			(vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
A
Avi Kivity 已提交
3283 3284 3285

	update_exception_bitmap(vcpu);

3286 3287 3288 3289 3290 3291
	fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
	fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
	fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
	fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
	fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
	fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
A
Avi Kivity 已提交
3292 3293
}

3294
static void fix_rmode_seg(int seg, struct kvm_segment *save)
A
Avi Kivity 已提交
3295
{
3296
	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319
	struct kvm_segment var = *save;

	var.dpl = 0x3;
	if (seg == VCPU_SREG_CS)
		var.type = 0x3;

	if (!emulate_invalid_guest_state) {
		var.selector = var.base >> 4;
		var.base = var.base & 0xffff0;
		var.limit = 0xffff;
		var.g = 0;
		var.db = 0;
		var.present = 1;
		var.s = 1;
		var.l = 0;
		var.unusable = 0;
		var.type = 0x3;
		var.avl = 0;
		if (save->base & 0xf)
			printk_once(KERN_WARNING "kvm: segment base is not "
					"paragraph aligned when entering "
					"protected mode (seg=%d)", seg);
	}
A
Avi Kivity 已提交
3320

3321 3322 3323 3324
	vmcs_write16(sf->selector, var.selector);
	vmcs_write32(sf->base, var.base);
	vmcs_write32(sf->limit, var.limit);
	vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
A
Avi Kivity 已提交
3325 3326 3327 3328 3329
}

static void enter_rmode(struct kvm_vcpu *vcpu)
{
	unsigned long flags;
3330
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
3331

3332 3333 3334 3335 3336
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
3337 3338
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
3339

3340
	vmx->rmode.vm86_active = 1;
A
Avi Kivity 已提交
3341

3342 3343
	/*
	 * Very old userspace does not call KVM_SET_TSS_ADDR before entering
3344
	 * vcpu. Warn the user that an update is overdue.
3345
	 */
3346
	if (!vcpu->kvm->arch.tss_addr)
3347 3348 3349
		printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
			     "called before entering vcpu\n");

A
Avi Kivity 已提交
3350 3351
	vmx_segment_cache_clear(vmx);

3352
	vmcs_writel(GUEST_TR_BASE, vcpu->kvm->arch.tss_addr);
A
Avi Kivity 已提交
3353 3354 3355 3356
	vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);

	flags = vmcs_readl(GUEST_RFLAGS);
3357
	vmx->rmode.save_rflags = flags;
A
Avi Kivity 已提交
3358

3359
	flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
A
Avi Kivity 已提交
3360 3361

	vmcs_writel(GUEST_RFLAGS, flags);
3362
	vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
A
Avi Kivity 已提交
3363 3364
	update_exception_bitmap(vcpu);

3365 3366 3367 3368 3369 3370
	fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
	fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
	fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
	fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
	fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
	fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
3371

3372
	kvm_mmu_reset_context(vcpu);
A
Avi Kivity 已提交
3373 3374
}

3375 3376 3377
static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3378 3379 3380 3381
	struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);

	if (!msr)
		return;
3382

3383 3384 3385 3386 3387
	/*
	 * Force kernel_gs_base reloading before EFER changes, as control
	 * of this msr depends on is_long_mode().
	 */
	vmx_load_host_state(to_vmx(vcpu));
3388
	vcpu->arch.efer = efer;
3389
	if (efer & EFER_LMA) {
3390
		vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
3391 3392
		msr->data = efer;
	} else {
3393
		vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
3394 3395 3396 3397 3398 3399

		msr->data = efer & ~EFER_LME;
	}
	setup_msrs(vmx);
}

3400
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
3401 3402 3403 3404 3405

static void enter_lmode(struct kvm_vcpu *vcpu)
{
	u32 guest_tr_ar;

A
Avi Kivity 已提交
3406 3407
	vmx_segment_cache_clear(to_vmx(vcpu));

A
Avi Kivity 已提交
3408 3409
	guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
	if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
3410 3411
		pr_debug_ratelimited("%s: tss fixup for long mode. \n",
				     __func__);
A
Avi Kivity 已提交
3412 3413 3414 3415
		vmcs_write32(GUEST_TR_AR_BYTES,
			     (guest_tr_ar & ~AR_TYPE_MASK)
			     | AR_TYPE_BUSY_64_TSS);
	}
3416
	vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
A
Avi Kivity 已提交
3417 3418 3419 3420
}

static void exit_lmode(struct kvm_vcpu *vcpu)
{
3421
	vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
3422
	vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
A
Avi Kivity 已提交
3423 3424 3425 3426
}

#endif

3427 3428
static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
{
3429
	vpid_sync_context(to_vmx(vcpu));
3430 3431 3432
	if (enable_ept) {
		if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
			return;
3433
		ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
3434
	}
3435 3436
}

3437 3438 3439 3440 3441 3442 3443 3444
static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
{
	ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;

	vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
	vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
}

3445 3446 3447 3448 3449 3450 3451
static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
{
	if (enable_ept && is_paging(vcpu))
		vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
}

3452
static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
3453
{
3454 3455 3456 3457
	ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;

	vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
	vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
3458 3459
}

3460 3461
static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
{
G
Gleb Natapov 已提交
3462 3463
	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;

A
Avi Kivity 已提交
3464 3465 3466 3467
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_dirty))
		return;

3468
	if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
G
Gleb Natapov 已提交
3469 3470 3471 3472
		vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
		vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
		vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
		vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
3473 3474 3475
	}
}

3476 3477
static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
{
G
Gleb Natapov 已提交
3478 3479
	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;

3480
	if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
G
Gleb Natapov 已提交
3481 3482 3483 3484
		mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
		mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
		mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
		mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
3485
	}
A
Avi Kivity 已提交
3486 3487 3488 3489 3490

	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
3491 3492
}

3493
static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
3494 3495 3496 3497 3498

static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
					unsigned long cr0,
					struct kvm_vcpu *vcpu)
{
3499 3500
	if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
		vmx_decache_cr3(vcpu);
3501 3502 3503
	if (!(cr0 & X86_CR0_PG)) {
		/* From paging/starting to nonpaging */
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
3504
			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
3505 3506 3507
			     (CPU_BASED_CR3_LOAD_EXITING |
			      CPU_BASED_CR3_STORE_EXITING));
		vcpu->arch.cr0 = cr0;
3508
		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3509 3510 3511
	} else if (!is_paging(vcpu)) {
		/* From nonpaging to paging */
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
3512
			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
3513 3514 3515
			     ~(CPU_BASED_CR3_LOAD_EXITING |
			       CPU_BASED_CR3_STORE_EXITING));
		vcpu->arch.cr0 = cr0;
3516
		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3517
	}
3518 3519 3520

	if (!(cr0 & X86_CR0_WP))
		*hw_cr0 &= ~X86_CR0_WP;
3521 3522
}

A
Avi Kivity 已提交
3523 3524
static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
3525
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3526 3527
	unsigned long hw_cr0;

G
Gleb Natapov 已提交
3528
	hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK);
3529
	if (enable_unrestricted_guest)
G
Gleb Natapov 已提交
3530
		hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
3531
	else {
G
Gleb Natapov 已提交
3532
		hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
3533

3534 3535
		if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
			enter_pmode(vcpu);
A
Avi Kivity 已提交
3536

3537 3538 3539
		if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
			enter_rmode(vcpu);
	}
A
Avi Kivity 已提交
3540

3541
#ifdef CONFIG_X86_64
3542
	if (vcpu->arch.efer & EFER_LME) {
3543
		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
A
Avi Kivity 已提交
3544
			enter_lmode(vcpu);
3545
		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
A
Avi Kivity 已提交
3546 3547 3548 3549
			exit_lmode(vcpu);
	}
#endif

3550
	if (enable_ept)
3551 3552
		ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);

3553
	if (!vcpu->fpu_active)
3554
		hw_cr0 |= X86_CR0_TS | X86_CR0_MP;
3555

A
Avi Kivity 已提交
3556
	vmcs_writel(CR0_READ_SHADOW, cr0);
3557
	vmcs_writel(GUEST_CR0, hw_cr0);
3558
	vcpu->arch.cr0 = cr0;
3559 3560 3561

	/* depends on vcpu->arch.cr0 to be set to a new value */
	vmx->emulation_required = emulation_required(vcpu);
A
Avi Kivity 已提交
3562 3563
}

3564 3565 3566 3567 3568 3569 3570
static u64 construct_eptp(unsigned long root_hpa)
{
	u64 eptp;

	/* TODO write the value reading from MSR */
	eptp = VMX_EPT_DEFAULT_MT |
		VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
3571 3572
	if (enable_ept_ad_bits)
		eptp |= VMX_EPT_AD_ENABLE_BIT;
3573 3574 3575 3576 3577
	eptp |= (root_hpa & PAGE_MASK);

	return eptp;
}

A
Avi Kivity 已提交
3578 3579
static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
{
3580 3581 3582 3583
	unsigned long guest_cr3;
	u64 eptp;

	guest_cr3 = cr3;
3584
	if (enable_ept) {
3585 3586
		eptp = construct_eptp(cr3);
		vmcs_write64(EPT_POINTER, eptp);
3587 3588 3589 3590
		if (is_paging(vcpu) || is_guest_mode(vcpu))
			guest_cr3 = kvm_read_cr3(vcpu);
		else
			guest_cr3 = vcpu->kvm->arch.ept_identity_map_addr;
3591
		ept_load_pdptrs(vcpu);
3592 3593
	}

3594
	vmx_flush_tlb(vcpu);
3595
	vmcs_writel(GUEST_CR3, guest_cr3);
A
Avi Kivity 已提交
3596 3597
}

3598
static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
A
Avi Kivity 已提交
3599
{
3600
	unsigned long hw_cr4 = cr4 | (to_vmx(vcpu)->rmode.vm86_active ?
3601 3602
		    KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);

3603 3604 3605 3606 3607 3608 3609 3610 3611
	if (cr4 & X86_CR4_VMXE) {
		/*
		 * To use VMXON (and later other VMX instructions), a guest
		 * must first be able to turn on cr4.VMXE (see handle_vmon()).
		 * So basically the check on whether to allow nested VMX
		 * is here.
		 */
		if (!nested_vmx_allowed(vcpu))
			return 1;
3612 3613 3614
	}
	if (to_vmx(vcpu)->nested.vmxon &&
	    ((cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON))
3615 3616
		return 1;

3617
	vcpu->arch.cr4 = cr4;
3618 3619 3620 3621
	if (enable_ept) {
		if (!is_paging(vcpu)) {
			hw_cr4 &= ~X86_CR4_PAE;
			hw_cr4 |= X86_CR4_PSE;
3622
			/*
3623 3624
			 * SMEP/SMAP is disabled if CPU is in non-paging mode
			 * in hardware. However KVM always uses paging mode to
3625
			 * emulate guest non-paging mode with TDP.
3626 3627 3628
			 * To emulate this behavior, SMEP/SMAP needs to be
			 * manually disabled when guest switches to non-paging
			 * mode.
3629
			 */
3630
			hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP);
3631 3632 3633 3634
		} else if (!(cr4 & X86_CR4_PAE)) {
			hw_cr4 &= ~X86_CR4_PAE;
		}
	}
3635 3636 3637

	vmcs_writel(CR4_READ_SHADOW, cr4);
	vmcs_writel(GUEST_CR4, hw_cr4);
3638
	return 0;
A
Avi Kivity 已提交
3639 3640 3641 3642 3643
}

static void vmx_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
3644
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
3645 3646
	u32 ar;

3647
	if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
3648
		*var = vmx->rmode.segs[seg];
3649
		if (seg == VCPU_SREG_TR
A
Avi Kivity 已提交
3650
		    || var->selector == vmx_read_guest_seg_selector(vmx, seg))
3651
			return;
3652 3653 3654
		var->base = vmx_read_guest_seg_base(vmx, seg);
		var->selector = vmx_read_guest_seg_selector(vmx, seg);
		return;
3655
	}
A
Avi Kivity 已提交
3656 3657 3658 3659
	var->base = vmx_read_guest_seg_base(vmx, seg);
	var->limit = vmx_read_guest_seg_limit(vmx, seg);
	var->selector = vmx_read_guest_seg_selector(vmx, seg);
	ar = vmx_read_guest_seg_ar(vmx, seg);
3660
	var->unusable = (ar >> 16) & 1;
A
Avi Kivity 已提交
3661 3662 3663
	var->type = ar & 15;
	var->s = (ar >> 4) & 1;
	var->dpl = (ar >> 5) & 3;
3664 3665 3666 3667 3668 3669 3670 3671
	/*
	 * Some userspaces do not preserve unusable property. Since usable
	 * segment has to be present according to VMX spec we can use present
	 * property to amend userspace bug by making unusable segment always
	 * nonpresent. vmx_segment_access_rights() already marks nonpresent
	 * segment as unusable.
	 */
	var->present = !var->unusable;
A
Avi Kivity 已提交
3672 3673 3674 3675 3676 3677
	var->avl = (ar >> 12) & 1;
	var->l = (ar >> 13) & 1;
	var->db = (ar >> 14) & 1;
	var->g = (ar >> 15) & 1;
}

3678 3679 3680 3681 3682 3683 3684 3685
static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment s;

	if (to_vmx(vcpu)->rmode.vm86_active) {
		vmx_get_segment(vcpu, &s, seg);
		return s.base;
	}
A
Avi Kivity 已提交
3686
	return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
3687 3688
}

3689
static int vmx_get_cpl(struct kvm_vcpu *vcpu)
3690
{
3691 3692
	struct vcpu_vmx *vmx = to_vmx(vcpu);

P
Paolo Bonzini 已提交
3693
	if (unlikely(vmx->rmode.vm86_active))
3694
		return 0;
P
Paolo Bonzini 已提交
3695 3696 3697
	else {
		int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
		return AR_DPL(ar);
A
Avi Kivity 已提交
3698 3699 3700
	}
}

3701
static u32 vmx_segment_access_rights(struct kvm_segment *var)
A
Avi Kivity 已提交
3702 3703 3704
{
	u32 ar;

3705
	if (var->unusable || !var->present)
A
Avi Kivity 已提交
3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716
		ar = 1 << 16;
	else {
		ar = var->type & 15;
		ar |= (var->s & 1) << 4;
		ar |= (var->dpl & 3) << 5;
		ar |= (var->present & 1) << 7;
		ar |= (var->avl & 1) << 12;
		ar |= (var->l & 1) << 13;
		ar |= (var->db & 1) << 14;
		ar |= (var->g & 1) << 15;
	}
3717 3718 3719 3720 3721 3722 3723

	return ar;
}

static void vmx_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
3724
	struct vcpu_vmx *vmx = to_vmx(vcpu);
3725
	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3726

A
Avi Kivity 已提交
3727 3728
	vmx_segment_cache_clear(vmx);

3729 3730 3731 3732 3733 3734
	if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
		vmx->rmode.segs[seg] = *var;
		if (seg == VCPU_SREG_TR)
			vmcs_write16(sf->selector, var->selector);
		else if (var->s)
			fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
3735
		goto out;
3736
	}
3737

3738 3739 3740
	vmcs_writel(sf->base, var->base);
	vmcs_write32(sf->limit, var->limit);
	vmcs_write16(sf->selector, var->selector);
3741 3742 3743 3744 3745 3746

	/*
	 *   Fix the "Accessed" bit in AR field of segment registers for older
	 * qemu binaries.
	 *   IA32 arch specifies that at the time of processor reset the
	 * "Accessed" bit in the AR field of segment registers is 1. And qemu
G
Guo Chao 已提交
3747
	 * is setting it to 0 in the userland code. This causes invalid guest
3748 3749 3750 3751 3752 3753
	 * state vmexit when "unrestricted guest" mode is turned on.
	 *    Fix for this setup issue in cpu_reset is being pushed in the qemu
	 * tree. Newer qemu binaries with that qemu fix would not need this
	 * kvm hack.
	 */
	if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
3754
		var->type |= 0x1; /* Accessed */
3755

3756
	vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
3757 3758

out:
3759
	vmx->emulation_required = emulation_required(vcpu);
A
Avi Kivity 已提交
3760 3761 3762 3763
}

static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
A
Avi Kivity 已提交
3764
	u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
A
Avi Kivity 已提交
3765 3766 3767 3768 3769

	*db = (ar >> 14) & 1;
	*l = (ar >> 13) & 1;
}

3770
static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
3771
{
3772 3773
	dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
	dt->address = vmcs_readl(GUEST_IDTR_BASE);
A
Avi Kivity 已提交
3774 3775
}

3776
static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
3777
{
3778 3779
	vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
	vmcs_writel(GUEST_IDTR_BASE, dt->address);
A
Avi Kivity 已提交
3780 3781
}

3782
static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
3783
{
3784 3785
	dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
	dt->address = vmcs_readl(GUEST_GDTR_BASE);
A
Avi Kivity 已提交
3786 3787
}

3788
static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
3789
{
3790 3791
	vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
	vmcs_writel(GUEST_GDTR_BASE, dt->address);
A
Avi Kivity 已提交
3792 3793
}

3794 3795 3796 3797 3798 3799
static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment var;
	u32 ar;

	vmx_get_segment(vcpu, &var, seg);
3800
	var.dpl = 0x3;
3801 3802
	if (seg == VCPU_SREG_CS)
		var.type = 0x3;
3803 3804 3805 3806
	ar = vmx_segment_access_rights(&var);

	if (var.base != (var.selector << 4))
		return false;
3807
	if (var.limit != 0xffff)
3808
		return false;
3809
	if (ar != 0xf3)
3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822
		return false;

	return true;
}

static bool code_segment_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	unsigned int cs_rpl;

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs_rpl = cs.selector & SELECTOR_RPL_MASK;

3823 3824
	if (cs.unusable)
		return false;
3825 3826 3827 3828
	if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK))
		return false;
	if (!cs.s)
		return false;
3829
	if (cs.type & AR_TYPE_WRITEABLE_MASK) {
3830 3831
		if (cs.dpl > cs_rpl)
			return false;
3832
	} else {
3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850
		if (cs.dpl != cs_rpl)
			return false;
	}
	if (!cs.present)
		return false;

	/* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
	return true;
}

static bool stack_segment_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment ss;
	unsigned int ss_rpl;

	vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
	ss_rpl = ss.selector & SELECTOR_RPL_MASK;

3851 3852 3853
	if (ss.unusable)
		return true;
	if (ss.type != 3 && ss.type != 7)
3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872
		return false;
	if (!ss.s)
		return false;
	if (ss.dpl != ss_rpl) /* DPL != RPL */
		return false;
	if (!ss.present)
		return false;

	return true;
}

static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
{
	struct kvm_segment var;
	unsigned int rpl;

	vmx_get_segment(vcpu, &var, seg);
	rpl = var.selector & SELECTOR_RPL_MASK;

3873 3874
	if (var.unusable)
		return true;
3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895
	if (!var.s)
		return false;
	if (!var.present)
		return false;
	if (~var.type & (AR_TYPE_CODE_MASK|AR_TYPE_WRITEABLE_MASK)) {
		if (var.dpl < rpl) /* DPL < RPL */
			return false;
	}

	/* TODO: Add other members to kvm_segment_field to allow checking for other access
	 * rights flags
	 */
	return true;
}

static bool tr_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment tr;

	vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);

3896 3897
	if (tr.unusable)
		return false;
3898 3899
	if (tr.selector & SELECTOR_TI_MASK)	/* TI = 1 */
		return false;
3900
	if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913
		return false;
	if (!tr.present)
		return false;

	return true;
}

static bool ldtr_valid(struct kvm_vcpu *vcpu)
{
	struct kvm_segment ldtr;

	vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);

3914 3915
	if (ldtr.unusable)
		return true;
3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943
	if (ldtr.selector & SELECTOR_TI_MASK)	/* TI = 1 */
		return false;
	if (ldtr.type != 2)
		return false;
	if (!ldtr.present)
		return false;

	return true;
}

static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs, ss;

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);

	return ((cs.selector & SELECTOR_RPL_MASK) ==
		 (ss.selector & SELECTOR_RPL_MASK));
}

/*
 * Check if guest state is valid. Returns true if valid, false if
 * not.
 * We assume that registers are always usable
 */
static bool guest_state_valid(struct kvm_vcpu *vcpu)
{
3944 3945 3946
	if (enable_unrestricted_guest)
		return true;

3947
	/* real mode guest state checks */
3948
	if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989
		if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
			return false;
		if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
			return false;
	} else {
	/* protected mode guest state checks */
		if (!cs_ss_rpl_check(vcpu))
			return false;
		if (!code_segment_valid(vcpu))
			return false;
		if (!stack_segment_valid(vcpu))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_DS))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_ES))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_FS))
			return false;
		if (!data_segment_valid(vcpu, VCPU_SREG_GS))
			return false;
		if (!tr_valid(vcpu))
			return false;
		if (!ldtr_valid(vcpu))
			return false;
	}
	/* TODO:
	 * - Add checks on RIP
	 * - Add checks on RFLAGS
	 */

	return true;
}

M
Mike Day 已提交
3990
static int init_rmode_tss(struct kvm *kvm)
A
Avi Kivity 已提交
3991
{
3992
	gfn_t fn;
3993
	u16 data = 0;
3994
	int idx, r;
A
Avi Kivity 已提交
3995

3996
	idx = srcu_read_lock(&kvm->srcu);
3997
	fn = kvm->arch.tss_addr >> PAGE_SHIFT;
3998 3999
	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
	if (r < 0)
4000
		goto out;
4001
	data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
4002 4003
	r = kvm_write_guest_page(kvm, fn++, &data,
			TSS_IOPB_BASE_OFFSET, sizeof(u16));
4004
	if (r < 0)
4005
		goto out;
4006 4007
	r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
	if (r < 0)
4008
		goto out;
4009 4010
	r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
	if (r < 0)
4011
		goto out;
4012
	data = ~0;
4013 4014 4015 4016
	r = kvm_write_guest_page(kvm, fn, &data,
				 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
				 sizeof(u8));
out:
4017
	srcu_read_unlock(&kvm->srcu, idx);
4018
	return r;
A
Avi Kivity 已提交
4019 4020
}

4021 4022
static int init_rmode_identity_map(struct kvm *kvm)
{
4023
	int i, idx, r = 0;
4024 4025 4026
	pfn_t identity_map_pfn;
	u32 tmp;

4027
	if (!enable_ept)
4028
		return 0;
4029 4030 4031 4032

	/* Protect kvm->arch.ept_identity_pagetable_done. */
	mutex_lock(&kvm->slots_lock);

4033
	if (likely(kvm->arch.ept_identity_pagetable_done))
4034 4035
		goto out2;

4036
	identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
4037 4038

	r = alloc_identity_pagetable(kvm);
4039
	if (r < 0)
4040 4041
		goto out2;

4042
	idx = srcu_read_lock(&kvm->srcu);
4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055
	r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
	if (r < 0)
		goto out;
	/* Set up identity-mapping pagetable for EPT in real mode */
	for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
		tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
			_PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
		r = kvm_write_guest_page(kvm, identity_map_pfn,
				&tmp, i * sizeof(tmp), sizeof(tmp));
		if (r < 0)
			goto out;
	}
	kvm->arch.ept_identity_pagetable_done = true;
4056

4057
out:
4058
	srcu_read_unlock(&kvm->srcu, idx);
4059 4060 4061

out2:
	mutex_unlock(&kvm->slots_lock);
4062
	return r;
4063 4064
}

A
Avi Kivity 已提交
4065 4066
static void seg_setup(int seg)
{
4067
	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
4068
	unsigned int ar;
A
Avi Kivity 已提交
4069 4070 4071 4072

	vmcs_write16(sf->selector, 0);
	vmcs_writel(sf->base, 0);
	vmcs_write32(sf->limit, 0xffff);
4073 4074 4075
	ar = 0x93;
	if (seg == VCPU_SREG_CS)
		ar |= 0x08; /* code segment */
4076 4077

	vmcs_write32(sf->ar_bytes, ar);
A
Avi Kivity 已提交
4078 4079
}

4080 4081
static int alloc_apic_access_page(struct kvm *kvm)
{
4082
	struct page *page;
4083 4084 4085
	struct kvm_userspace_memory_region kvm_userspace_mem;
	int r = 0;

4086
	mutex_lock(&kvm->slots_lock);
4087
	if (kvm->arch.apic_access_page_done)
4088 4089 4090
		goto out;
	kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
	kvm_userspace_mem.flags = 0;
4091
	kvm_userspace_mem.guest_phys_addr = APIC_DEFAULT_PHYS_BASE;
4092
	kvm_userspace_mem.memory_size = PAGE_SIZE;
4093
	r = __kvm_set_memory_region(kvm, &kvm_userspace_mem);
4094 4095
	if (r)
		goto out;
4096

4097
	page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
4098 4099 4100 4101 4102
	if (is_error_page(page)) {
		r = -EFAULT;
		goto out;
	}

4103 4104 4105 4106 4107 4108
	/*
	 * Do not pin the page in memory, so that memory hot-unplug
	 * is able to migrate it.
	 */
	put_page(page);
	kvm->arch.apic_access_page_done = true;
4109
out:
4110
	mutex_unlock(&kvm->slots_lock);
4111 4112 4113
	return r;
}

4114 4115
static int alloc_identity_pagetable(struct kvm *kvm)
{
4116 4117
	/* Called with kvm->slots_lock held. */

4118 4119 4120
	struct kvm_userspace_memory_region kvm_userspace_mem;
	int r = 0;

4121 4122
	BUG_ON(kvm->arch.ept_identity_pagetable_done);

4123 4124
	kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
	kvm_userspace_mem.flags = 0;
4125 4126
	kvm_userspace_mem.guest_phys_addr =
		kvm->arch.ept_identity_map_addr;
4127
	kvm_userspace_mem.memory_size = PAGE_SIZE;
4128
	r = __kvm_set_memory_region(kvm, &kvm_userspace_mem);
4129 4130 4131 4132

	return r;
}

4133 4134 4135 4136 4137
static void allocate_vpid(struct vcpu_vmx *vmx)
{
	int vpid;

	vmx->vpid = 0;
4138
	if (!enable_vpid)
4139 4140 4141 4142 4143 4144 4145 4146 4147 4148
		return;
	spin_lock(&vmx_vpid_lock);
	vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
	if (vpid < VMX_NR_VPIDS) {
		vmx->vpid = vpid;
		__set_bit(vpid, vmx_vpid_bitmap);
	}
	spin_unlock(&vmx_vpid_lock);
}

4149 4150 4151 4152 4153 4154 4155 4156 4157 4158
static void free_vpid(struct vcpu_vmx *vmx)
{
	if (!enable_vpid)
		return;
	spin_lock(&vmx_vpid_lock);
	if (vmx->vpid != 0)
		__clear_bit(vmx->vpid, vmx_vpid_bitmap);
	spin_unlock(&vmx_vpid_lock);
}

4159 4160 4161 4162
#define MSR_TYPE_R	1
#define MSR_TYPE_W	2
static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
						u32 msr, int type)
S
Sheng Yang 已提交
4163
{
4164
	int f = sizeof(unsigned long);
S
Sheng Yang 已提交
4165 4166 4167 4168 4169 4170 4171 4172 4173 4174

	if (!cpu_has_vmx_msr_bitmap())
		return;

	/*
	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
	 * have the write-low and read-high bitmap offsets the wrong way round.
	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
	 */
	if (msr <= 0x1fff) {
4175 4176 4177 4178 4179 4180 4181 4182
		if (type & MSR_TYPE_R)
			/* read-low */
			__clear_bit(msr, msr_bitmap + 0x000 / f);

		if (type & MSR_TYPE_W)
			/* write-low */
			__clear_bit(msr, msr_bitmap + 0x800 / f);

S
Sheng Yang 已提交
4183 4184
	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
		msr &= 0x1fff;
4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227
		if (type & MSR_TYPE_R)
			/* read-high */
			__clear_bit(msr, msr_bitmap + 0x400 / f);

		if (type & MSR_TYPE_W)
			/* write-high */
			__clear_bit(msr, msr_bitmap + 0xc00 / f);

	}
}

static void __vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
						u32 msr, int type)
{
	int f = sizeof(unsigned long);

	if (!cpu_has_vmx_msr_bitmap())
		return;

	/*
	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
	 * have the write-low and read-high bitmap offsets the wrong way round.
	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
	 */
	if (msr <= 0x1fff) {
		if (type & MSR_TYPE_R)
			/* read-low */
			__set_bit(msr, msr_bitmap + 0x000 / f);

		if (type & MSR_TYPE_W)
			/* write-low */
			__set_bit(msr, msr_bitmap + 0x800 / f);

	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
		msr &= 0x1fff;
		if (type & MSR_TYPE_R)
			/* read-high */
			__set_bit(msr, msr_bitmap + 0x400 / f);

		if (type & MSR_TYPE_W)
			/* write-high */
			__set_bit(msr, msr_bitmap + 0xc00 / f);

S
Sheng Yang 已提交
4228 4229 4230
	}
}

4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276
/*
 * If a msr is allowed by L0, we should check whether it is allowed by L1.
 * The corresponding bit will be cleared unless both of L0 and L1 allow it.
 */
static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
					       unsigned long *msr_bitmap_nested,
					       u32 msr, int type)
{
	int f = sizeof(unsigned long);

	if (!cpu_has_vmx_msr_bitmap()) {
		WARN_ON(1);
		return;
	}

	/*
	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
	 * have the write-low and read-high bitmap offsets the wrong way round.
	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
	 */
	if (msr <= 0x1fff) {
		if (type & MSR_TYPE_R &&
		   !test_bit(msr, msr_bitmap_l1 + 0x000 / f))
			/* read-low */
			__clear_bit(msr, msr_bitmap_nested + 0x000 / f);

		if (type & MSR_TYPE_W &&
		   !test_bit(msr, msr_bitmap_l1 + 0x800 / f))
			/* write-low */
			__clear_bit(msr, msr_bitmap_nested + 0x800 / f);

	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
		msr &= 0x1fff;
		if (type & MSR_TYPE_R &&
		   !test_bit(msr, msr_bitmap_l1 + 0x400 / f))
			/* read-high */
			__clear_bit(msr, msr_bitmap_nested + 0x400 / f);

		if (type & MSR_TYPE_W &&
		   !test_bit(msr, msr_bitmap_l1 + 0xc00 / f))
			/* write-high */
			__clear_bit(msr, msr_bitmap_nested + 0xc00 / f);

	}
}

4277 4278 4279
static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
{
	if (!longmode_only)
4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307
		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy,
						msr, MSR_TYPE_R | MSR_TYPE_W);
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode,
						msr, MSR_TYPE_R | MSR_TYPE_W);
}

static void vmx_enable_intercept_msr_read_x2apic(u32 msr)
{
	__vmx_enable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
			msr, MSR_TYPE_R);
	__vmx_enable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
			msr, MSR_TYPE_R);
}

static void vmx_disable_intercept_msr_read_x2apic(u32 msr)
{
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
			msr, MSR_TYPE_R);
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
			msr, MSR_TYPE_R);
}

static void vmx_disable_intercept_msr_write_x2apic(u32 msr)
{
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
			msr, MSR_TYPE_W);
	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
			msr, MSR_TYPE_W);
4308 4309
}

4310 4311 4312 4313 4314
static int vmx_vm_has_apicv(struct kvm *kvm)
{
	return enable_apicv && irqchip_in_kernel(kvm);
}

4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331
/*
 * Send interrupt to vcpu via posted interrupt way.
 * 1. If target vcpu is running(non-root mode), send posted interrupt
 * notification to vcpu and hardware will sync PIR to vIRR atomically.
 * 2. If target vcpu isn't running(root mode), kick it to pick up the
 * interrupt from PIR in next vmentry.
 */
static void vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int r;

	if (pi_test_and_set_pir(vector, &vmx->pi_desc))
		return;

	r = pi_test_and_set_on(&vmx->pi_desc);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
4332
#ifdef CONFIG_SMP
4333 4334 4335 4336
	if (!r && (vcpu->mode == IN_GUEST_MODE))
		apic->send_IPI_mask(get_cpu_mask(vcpu->cpu),
				POSTED_INTR_VECTOR);
	else
4337
#endif
4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355
		kvm_vcpu_kick(vcpu);
}

static void vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (!pi_test_and_clear_on(&vmx->pi_desc))
		return;

	kvm_apic_update_irr(vcpu, vmx->pi_desc.pir);
}

static void vmx_sync_pir_to_irr_dummy(struct kvm_vcpu *vcpu)
{
	return;
}

4356 4357 4358 4359 4360 4361
/*
 * Set up the vmcs's constant host-state fields, i.e., host-state fields that
 * will not change in the lifetime of the guest.
 * Note that host-state that does change is set elsewhere. E.g., host-state
 * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
 */
4362
static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
4363 4364 4365 4366
{
	u32 low32, high32;
	unsigned long tmpl;
	struct desc_ptr dt;
4367
	unsigned long cr4;
4368

4369
	vmcs_writel(HOST_CR0, read_cr0() & ~X86_CR0_TS);  /* 22.2.3 */
4370 4371
	vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */

4372 4373 4374 4375 4376
	/* Save the most likely value for this task's CR4 in the VMCS. */
	cr4 = read_cr4();
	vmcs_writel(HOST_CR4, cr4);			/* 22.2.3, 22.2.5 */
	vmx->host_state.vmcs_host_cr4 = cr4;

4377
	vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
A
Avi Kivity 已提交
4378 4379 4380 4381 4382 4383 4384 4385 4386
#ifdef CONFIG_X86_64
	/*
	 * Load null selectors, so we can avoid reloading them in
	 * __vmx_load_host_state(), in case userspace uses the null selectors
	 * too (the expected case).
	 */
	vmcs_write16(HOST_DS_SELECTOR, 0);
	vmcs_write16(HOST_ES_SELECTOR, 0);
#else
4387 4388
	vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
	vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
A
Avi Kivity 已提交
4389
#endif
4390 4391 4392 4393 4394
	vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
	vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */

	native_store_idt(&dt);
	vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */
4395
	vmx->host_idt_base = dt.address;
4396

A
Avi Kivity 已提交
4397
	vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409

	rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
	vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
	rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
	vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */

	if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
		rdmsr(MSR_IA32_CR_PAT, low32, high32);
		vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
	}
}

4410 4411 4412 4413 4414
static void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
{
	vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
	if (enable_ept)
		vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
4415 4416 4417
	if (is_guest_mode(&vmx->vcpu))
		vmx->vcpu.arch.cr4_guest_owned_bits &=
			~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask;
4418 4419 4420
	vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
}

4421 4422 4423 4424 4425 4426 4427 4428 4429
static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
{
	u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;

	if (!vmx_vm_has_apicv(vmx->vcpu.kvm))
		pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
	return pin_based_exec_ctrl;
}

4430 4431 4432
static u32 vmx_exec_control(struct vcpu_vmx *vmx)
{
	u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
4433 4434 4435 4436

	if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
		exec_control &= ~CPU_BASED_MOV_DR_EXITING;

4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460
	if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
		exec_control &= ~CPU_BASED_TPR_SHADOW;
#ifdef CONFIG_X86_64
		exec_control |= CPU_BASED_CR8_STORE_EXITING |
				CPU_BASED_CR8_LOAD_EXITING;
#endif
	}
	if (!enable_ept)
		exec_control |= CPU_BASED_CR3_STORE_EXITING |
				CPU_BASED_CR3_LOAD_EXITING  |
				CPU_BASED_INVLPG_EXITING;
	return exec_control;
}

static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
{
	u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
	if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
		exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
	if (vmx->vpid == 0)
		exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
	if (!enable_ept) {
		exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
		enable_unrestricted_guest = 0;
4461 4462
		/* Enable INVPCID for non-ept guests may cause performance regression. */
		exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
4463 4464 4465 4466 4467
	}
	if (!enable_unrestricted_guest)
		exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
	if (!ple_gap)
		exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
4468 4469 4470
	if (!vmx_vm_has_apicv(vmx->vcpu.kvm))
		exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
				  SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
4471
	exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
4472 4473 4474 4475 4476 4477
	/* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
	   (handle_vmptrld).
	   We can NOT enable shadow_vmcs here because we don't have yet
	   a current VMCS12
	*/
	exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
K
Kai Huang 已提交
4478 4479 4480
	/* PML is enabled/disabled in creating/destorying vcpu */
	exec_control &= ~SECONDARY_EXEC_ENABLE_PML;

4481 4482 4483
	return exec_control;
}

4484 4485 4486 4487 4488
static void ept_set_mmio_spte_mask(void)
{
	/*
	 * EPT Misconfigurations can be generated if the value of bits 2:0
	 * of an EPT paging-structure entry is 110b (write/execute).
4489
	 * Also, magic bits (0x3ull << 62) is set to quickly identify mmio
4490 4491
	 * spte.
	 */
4492
	kvm_mmu_set_mmio_spte_mask((0x3ull << 62) | 0x6ull);
4493 4494
}

4495
#define VMX_XSS_EXIT_BITMAP 0
A
Avi Kivity 已提交
4496 4497 4498
/*
 * Sets up the vmcs for emulated real mode.
 */
R
Rusty Russell 已提交
4499
static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
A
Avi Kivity 已提交
4500
{
4501
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
4502
	unsigned long a;
4503
#endif
A
Avi Kivity 已提交
4504 4505 4506
	int i;

	/* I/O */
4507 4508
	vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
	vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
A
Avi Kivity 已提交
4509

4510 4511 4512 4513
	if (enable_shadow_vmcs) {
		vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
		vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
	}
S
Sheng Yang 已提交
4514
	if (cpu_has_vmx_msr_bitmap())
4515
		vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
S
Sheng Yang 已提交
4516

A
Avi Kivity 已提交
4517 4518 4519
	vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */

	/* Control */
4520
	vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
4521

4522
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx_exec_control(vmx));
A
Avi Kivity 已提交
4523

4524
	if (cpu_has_secondary_exec_ctrls()) {
4525 4526
		vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
				vmx_secondary_exec_control(vmx));
4527
	}
4528

4529
	if (vmx_vm_has_apicv(vmx->vcpu.kvm)) {
4530 4531 4532 4533 4534 4535
		vmcs_write64(EOI_EXIT_BITMAP0, 0);
		vmcs_write64(EOI_EXIT_BITMAP1, 0);
		vmcs_write64(EOI_EXIT_BITMAP2, 0);
		vmcs_write64(EOI_EXIT_BITMAP3, 0);

		vmcs_write16(GUEST_INTR_STATUS, 0);
4536 4537 4538

		vmcs_write64(POSTED_INTR_NV, POSTED_INTR_VECTOR);
		vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
4539 4540
	}

4541 4542
	if (ple_gap) {
		vmcs_write32(PLE_GAP, ple_gap);
4543 4544
		vmx->ple_window = ple_window;
		vmx->ple_window_dirty = true;
4545 4546
	}

4547 4548
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
A
Avi Kivity 已提交
4549 4550
	vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */

4551 4552
	vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
	vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
4553
	vmx_set_constant_host_state(vmx);
4554
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
4555 4556 4557 4558 4559 4560 4561 4562 4563
	rdmsrl(MSR_FS_BASE, a);
	vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
	rdmsrl(MSR_GS_BASE, a);
	vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
#else
	vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
	vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
#endif

4564 4565
	vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
4566
	vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
4567
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
4568
	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
A
Avi Kivity 已提交
4569

S
Sheng Yang 已提交
4570
	if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
4571 4572
		u32 msr_low, msr_high;
		u64 host_pat;
S
Sheng Yang 已提交
4573 4574 4575 4576 4577 4578 4579 4580
		rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
		host_pat = msr_low | ((u64) msr_high << 32);
		/* Write the default value follow host pat */
		vmcs_write64(GUEST_IA32_PAT, host_pat);
		/* Keep arch.pat sync with GUEST_IA32_PAT */
		vmx->vcpu.arch.pat = host_pat;
	}

4581
	for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) {
A
Avi Kivity 已提交
4582 4583
		u32 index = vmx_msr_index[i];
		u32 data_low, data_high;
4584
		int j = vmx->nmsrs;
A
Avi Kivity 已提交
4585 4586 4587

		if (rdmsr_safe(index, &data_low, &data_high) < 0)
			continue;
4588 4589
		if (wrmsr_safe(index, data_low, data_high) < 0)
			continue;
4590 4591
		vmx->guest_msrs[j].index = i;
		vmx->guest_msrs[j].data = 0;
4592
		vmx->guest_msrs[j].mask = -1ull;
4593
		++vmx->nmsrs;
A
Avi Kivity 已提交
4594 4595
	}

4596 4597

	vm_exit_controls_init(vmx, vmcs_config.vmexit_ctrl);
A
Avi Kivity 已提交
4598 4599

	/* 22.2.1, 20.8.1 */
4600
	vm_entry_controls_init(vmx, vmcs_config.vmentry_ctrl);
4601

4602
	vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
4603
	set_cr4_guest_host_mask(vmx);
4604

4605 4606 4607
	if (vmx_xsaves_supported())
		vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);

4608 4609 4610
	return 0;
}

4611
static void vmx_vcpu_reset(struct kvm_vcpu *vcpu)
4612 4613
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
4614
	struct msr_data apic_base_msr;
4615

4616
	vmx->rmode.vm86_active = 0;
4617

4618 4619
	vmx->soft_vnmi_blocked = 0;

4620
	vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
4621
	kvm_set_cr8(&vmx->vcpu, 0);
4622
	apic_base_msr.data = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE;
4623
	if (kvm_vcpu_is_bsp(&vmx->vcpu))
4624 4625 4626
		apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
	apic_base_msr.host_initiated = true;
	kvm_set_apic_base(&vmx->vcpu, &apic_base_msr);
4627

A
Avi Kivity 已提交
4628 4629
	vmx_segment_cache_clear(vmx);

4630
	seg_setup(VCPU_SREG_CS);
4631
	vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
4632
	vmcs_write32(GUEST_CS_BASE, 0xffff0000);
4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654

	seg_setup(VCPU_SREG_DS);
	seg_setup(VCPU_SREG_ES);
	seg_setup(VCPU_SREG_FS);
	seg_setup(VCPU_SREG_GS);
	seg_setup(VCPU_SREG_SS);

	vmcs_write16(GUEST_TR_SELECTOR, 0);
	vmcs_writel(GUEST_TR_BASE, 0);
	vmcs_write32(GUEST_TR_LIMIT, 0xffff);
	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);

	vmcs_write16(GUEST_LDTR_SELECTOR, 0);
	vmcs_writel(GUEST_LDTR_BASE, 0);
	vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
	vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);

	vmcs_write32(GUEST_SYSENTER_CS, 0);
	vmcs_writel(GUEST_SYSENTER_ESP, 0);
	vmcs_writel(GUEST_SYSENTER_EIP, 0);

	vmcs_writel(GUEST_RFLAGS, 0x02);
4655
	kvm_rip_write(vcpu, 0xfff0);
4656 4657 4658 4659 4660 4661 4662

	vmcs_writel(GUEST_GDTR_BASE, 0);
	vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);

	vmcs_writel(GUEST_IDTR_BASE, 0);
	vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);

4663
	vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
4664 4665 4666 4667 4668 4669 4670 4671
	vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
	vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);

	/* Special registers */
	vmcs_write64(GUEST_IA32_DEBUGCTL, 0);

	setup_msrs(vmx);

A
Avi Kivity 已提交
4672 4673
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */

4674 4675 4676 4677
	if (cpu_has_vmx_tpr_shadow()) {
		vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
		if (vm_need_tpr_shadow(vmx->vcpu.kvm))
			vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
4678
				     __pa(vmx->vcpu.arch.apic->regs));
4679 4680 4681
		vmcs_write32(TPR_THRESHOLD, 0);
	}

4682
	kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
A
Avi Kivity 已提交
4683

4684 4685 4686
	if (vmx_vm_has_apicv(vcpu->kvm))
		memset(&vmx->pi_desc, 0, sizeof(struct pi_desc));

4687 4688 4689
	if (vmx->vpid != 0)
		vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);

4690
	vmx->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
4691
	vmx_set_cr0(&vmx->vcpu, kvm_read_cr0(vcpu)); /* enter rmode */
R
Rusty Russell 已提交
4692 4693 4694 4695
	vmx_set_cr4(&vmx->vcpu, 0);
	vmx_set_efer(&vmx->vcpu, 0);
	vmx_fpu_activate(&vmx->vcpu);
	update_exception_bitmap(&vmx->vcpu);
A
Avi Kivity 已提交
4696

4697
	vpid_sync_context(vmx);
A
Avi Kivity 已提交
4698 4699
}

4700 4701 4702 4703 4704 4705 4706 4707 4708 4709
/*
 * In nested virtualization, check if L1 asked to exit on external interrupts.
 * For most existing hypervisors, this will always return true.
 */
static bool nested_exit_on_intr(struct kvm_vcpu *vcpu)
{
	return get_vmcs12(vcpu)->pin_based_vm_exec_control &
		PIN_BASED_EXT_INTR_MASK;
}

4710 4711 4712 4713 4714 4715 4716 4717 4718 4719
/*
 * In nested virtualization, check if L1 has set
 * VM_EXIT_ACK_INTR_ON_EXIT
 */
static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
{
	return get_vmcs12(vcpu)->vm_exit_controls &
		VM_EXIT_ACK_INTR_ON_EXIT;
}

4720 4721 4722 4723 4724 4725
static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu)
{
	return get_vmcs12(vcpu)->pin_based_vm_exec_control &
		PIN_BASED_NMI_EXITING;
}

4726
static void enable_irq_window(struct kvm_vcpu *vcpu)
4727 4728
{
	u32 cpu_based_vm_exec_control;
4729

4730 4731 4732 4733 4734
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

4735
static void enable_nmi_window(struct kvm_vcpu *vcpu)
4736 4737 4738
{
	u32 cpu_based_vm_exec_control;

4739 4740 4741 4742 4743
	if (!cpu_has_virtual_nmis() ||
	    vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
		enable_irq_window(vcpu);
		return;
	}
4744 4745 4746 4747 4748 4749

	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

4750
static void vmx_inject_irq(struct kvm_vcpu *vcpu)
4751
{
4752
	struct vcpu_vmx *vmx = to_vmx(vcpu);
4753 4754
	uint32_t intr;
	int irq = vcpu->arch.interrupt.nr;
4755

4756
	trace_kvm_inj_virq(irq);
F
Feng (Eric) Liu 已提交
4757

4758
	++vcpu->stat.irq_injections;
4759
	if (vmx->rmode.vm86_active) {
4760 4761 4762 4763
		int inc_eip = 0;
		if (vcpu->arch.interrupt.soft)
			inc_eip = vcpu->arch.event_exit_inst_len;
		if (kvm_inject_realmode_interrupt(vcpu, irq, inc_eip) != EMULATE_DONE)
4764
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4765 4766
		return;
	}
4767 4768 4769 4770 4771 4772 4773 4774
	intr = irq | INTR_INFO_VALID_MASK;
	if (vcpu->arch.interrupt.soft) {
		intr |= INTR_TYPE_SOFT_INTR;
		vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
			     vmx->vcpu.arch.event_exit_inst_len);
	} else
		intr |= INTR_TYPE_EXT_INTR;
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
4775 4776
}

4777 4778
static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
{
J
Jan Kiszka 已提交
4779 4780
	struct vcpu_vmx *vmx = to_vmx(vcpu);

4781 4782 4783
	if (is_guest_mode(vcpu))
		return;

4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796
	if (!cpu_has_virtual_nmis()) {
		/*
		 * Tracking the NMI-blocked state in software is built upon
		 * finding the next open IRQ window. This, in turn, depends on
		 * well-behaving guests: They have to keep IRQs disabled at
		 * least as long as the NMI handler runs. Otherwise we may
		 * cause NMI nesting, maybe breaking the guest. But as this is
		 * highly unlikely, we can live with the residual risk.
		 */
		vmx->soft_vnmi_blocked = 1;
		vmx->vnmi_blocked_time = 0;
	}

4797
	++vcpu->stat.nmi_injections;
4798
	vmx->nmi_known_unmasked = false;
4799
	if (vmx->rmode.vm86_active) {
4800
		if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0) != EMULATE_DONE)
4801
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
J
Jan Kiszka 已提交
4802 4803
		return;
	}
4804 4805 4806 4807
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
			INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
}

J
Jan Kiszka 已提交
4808 4809 4810 4811
static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
{
	if (!cpu_has_virtual_nmis())
		return to_vmx(vcpu)->soft_vnmi_blocked;
4812 4813
	if (to_vmx(vcpu)->nmi_known_unmasked)
		return false;
4814
	return vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)	& GUEST_INTR_STATE_NMI;
J
Jan Kiszka 已提交
4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826
}

static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (!cpu_has_virtual_nmis()) {
		if (vmx->soft_vnmi_blocked != masked) {
			vmx->soft_vnmi_blocked = masked;
			vmx->vnmi_blocked_time = 0;
		}
	} else {
4827
		vmx->nmi_known_unmasked = !masked;
J
Jan Kiszka 已提交
4828 4829 4830 4831 4832 4833 4834 4835 4836
		if (masked)
			vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				      GUEST_INTR_STATE_NMI);
		else
			vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
					GUEST_INTR_STATE_NMI);
	}
}

4837 4838
static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
{
4839 4840
	if (to_vmx(vcpu)->nested.nested_run_pending)
		return 0;
4841

4842 4843 4844 4845 4846 4847 4848 4849
	if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
		return 0;

	return	!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
		  (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
		   | GUEST_INTR_STATE_NMI));
}

4850 4851
static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
{
4852 4853
	return (!to_vmx(vcpu)->nested.nested_run_pending &&
		vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
4854 4855
		!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
			(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
4856 4857
}

4858 4859 4860 4861
static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
{
	int ret;
	struct kvm_userspace_memory_region tss_mem = {
4862
		.slot = TSS_PRIVATE_MEMSLOT,
4863 4864 4865 4866 4867
		.guest_phys_addr = addr,
		.memory_size = PAGE_SIZE * 3,
		.flags = 0,
	};

4868
	ret = kvm_set_memory_region(kvm, &tss_mem);
4869 4870
	if (ret)
		return ret;
4871
	kvm->arch.tss_addr = addr;
4872
	return init_rmode_tss(kvm);
4873 4874
}

4875
static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
A
Avi Kivity 已提交
4876
{
4877 4878
	switch (vec) {
	case BP_VECTOR:
4879 4880 4881 4882 4883 4884
		/*
		 * Update instruction length as we may reinject the exception
		 * from user space while in guest debugging mode.
		 */
		to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
			vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
J
Jan Kiszka 已提交
4885
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
4886 4887 4888 4889 4890 4891
			return false;
		/* fall through */
	case DB_VECTOR:
		if (vcpu->guest_debug &
			(KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
			return false;
J
Jan Kiszka 已提交
4892 4893
		/* fall through */
	case DE_VECTOR:
4894 4895 4896 4897 4898 4899 4900
	case OF_VECTOR:
	case BR_VECTOR:
	case UD_VECTOR:
	case DF_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
	case MF_VECTOR:
4901 4902
		return true;
	break;
4903
	}
4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931
	return false;
}

static int handle_rmode_exception(struct kvm_vcpu *vcpu,
				  int vec, u32 err_code)
{
	/*
	 * Instruction with address size override prefix opcode 0x67
	 * Cause the #SS fault with 0 error code in VM86 mode.
	 */
	if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
		if (emulate_instruction(vcpu, 0) == EMULATE_DONE) {
			if (vcpu->arch.halt_request) {
				vcpu->arch.halt_request = 0;
				return kvm_emulate_halt(vcpu);
			}
			return 1;
		}
		return 0;
	}

	/*
	 * Forward all other exceptions that are valid in real mode.
	 * FIXME: Breaks guest debugging in real mode, needs to be fixed with
	 *        the required debugging infrastructure rework.
	 */
	kvm_queue_exception(vcpu, vec);
	return 1;
A
Avi Kivity 已提交
4932 4933
}

A
Andi Kleen 已提交
4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952
/*
 * Trigger machine check on the host. We assume all the MSRs are already set up
 * by the CPU and that we still run on the same CPU as the MCE occurred on.
 * We pass a fake environment to the machine check handler because we want
 * the guest to be always treated like user space, no matter what context
 * it used internally.
 */
static void kvm_machine_check(void)
{
#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
	struct pt_regs regs = {
		.cs = 3, /* Fake ring 3 no matter what the guest ran on */
		.flags = X86_EFLAGS_IF,
	};

	do_machine_check(&regs, 0);
#endif
}

A
Avi Kivity 已提交
4953
static int handle_machine_check(struct kvm_vcpu *vcpu)
A
Andi Kleen 已提交
4954 4955 4956 4957 4958
{
	/* already handled by vcpu_run */
	return 1;
}

A
Avi Kivity 已提交
4959
static int handle_exception(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4960
{
4961
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Avi Kivity 已提交
4962
	struct kvm_run *kvm_run = vcpu->run;
J
Jan Kiszka 已提交
4963
	u32 intr_info, ex_no, error_code;
4964
	unsigned long cr2, rip, dr6;
A
Avi Kivity 已提交
4965 4966 4967
	u32 vect_info;
	enum emulation_result er;

4968
	vect_info = vmx->idt_vectoring_info;
4969
	intr_info = vmx->exit_intr_info;
A
Avi Kivity 已提交
4970

A
Andi Kleen 已提交
4971
	if (is_machine_check(intr_info))
A
Avi Kivity 已提交
4972
		return handle_machine_check(vcpu);
A
Andi Kleen 已提交
4973

4974
	if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR)
4975
		return 1;  /* already handled by vmx_vcpu_run() */
4976 4977

	if (is_no_device(intr_info)) {
4978
		vmx_fpu_activate(vcpu);
4979 4980 4981
		return 1;
	}

4982
	if (is_invalid_opcode(intr_info)) {
4983
		er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
4984
		if (er != EMULATE_DONE)
4985
			kvm_queue_exception(vcpu, UD_VECTOR);
4986 4987 4988
		return 1;
	}

A
Avi Kivity 已提交
4989
	error_code = 0;
4990
	if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
A
Avi Kivity 已提交
4991
		error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007

	/*
	 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
	 * MMIO, it is better to report an internal error.
	 * See the comments in vmx_handle_exit.
	 */
	if ((vect_info & VECTORING_INFO_VALID_MASK) &&
	    !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
		vcpu->run->internal.ndata = 2;
		vcpu->run->internal.data[0] = vect_info;
		vcpu->run->internal.data[1] = intr_info;
		return 0;
	}

A
Avi Kivity 已提交
5008
	if (is_page_fault(intr_info)) {
5009
		/* EPT won't cause page fault directly */
J
Julia Lawall 已提交
5010
		BUG_ON(enable_ept);
A
Avi Kivity 已提交
5011
		cr2 = vmcs_readl(EXIT_QUALIFICATION);
5012 5013
		trace_kvm_page_fault(cr2, error_code);

5014
		if (kvm_event_needs_reinjection(vcpu))
5015
			kvm_mmu_unprotect_page_virt(vcpu, cr2);
5016
		return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0);
A
Avi Kivity 已提交
5017 5018
	}

J
Jan Kiszka 已提交
5019
	ex_no = intr_info & INTR_INFO_VECTOR_MASK;
5020 5021 5022 5023

	if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
		return handle_rmode_exception(vcpu, ex_no, error_code);

5024 5025 5026 5027 5028
	switch (ex_no) {
	case DB_VECTOR:
		dr6 = vmcs_readl(EXIT_QUALIFICATION);
		if (!(vcpu->guest_debug &
		      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
5029
			vcpu->arch.dr6 &= ~15;
5030
			vcpu->arch.dr6 |= dr6 | DR6_RTM;
5031 5032 5033
			if (!(dr6 & ~DR6_RESERVED)) /* icebp */
				skip_emulated_instruction(vcpu);

5034 5035 5036 5037 5038 5039 5040
			kvm_queue_exception(vcpu, DB_VECTOR);
			return 1;
		}
		kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
		kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
		/* fall through */
	case BP_VECTOR:
5041 5042 5043 5044 5045 5046 5047
		/*
		 * Update instruction length as we may reinject #BP from
		 * user space while in guest debugging mode. Reading it for
		 * #DB as well causes no harm, it is not used in that case.
		 */
		vmx->vcpu.arch.event_exit_inst_len =
			vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
A
Avi Kivity 已提交
5048
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
5049
		rip = kvm_rip_read(vcpu);
J
Jan Kiszka 已提交
5050 5051
		kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
		kvm_run->debug.arch.exception = ex_no;
5052 5053
		break;
	default:
J
Jan Kiszka 已提交
5054 5055 5056
		kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
		kvm_run->ex.exception = ex_no;
		kvm_run->ex.error_code = error_code;
5057
		break;
A
Avi Kivity 已提交
5058 5059 5060 5061
	}
	return 0;
}

A
Avi Kivity 已提交
5062
static int handle_external_interrupt(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5063
{
A
Avi Kivity 已提交
5064
	++vcpu->stat.irq_exits;
A
Avi Kivity 已提交
5065 5066 5067
	return 1;
}

A
Avi Kivity 已提交
5068
static int handle_triple_fault(struct kvm_vcpu *vcpu)
5069
{
A
Avi Kivity 已提交
5070
	vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
5071 5072
	return 0;
}
A
Avi Kivity 已提交
5073

A
Avi Kivity 已提交
5074
static int handle_io(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5075
{
5076
	unsigned long exit_qualification;
5077
	int size, in, string;
5078
	unsigned port;
A
Avi Kivity 已提交
5079

5080
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5081
	string = (exit_qualification & 16) != 0;
5082
	in = (exit_qualification & 8) != 0;
5083

5084
	++vcpu->stat.io_exits;
5085

5086
	if (string || in)
5087
		return emulate_instruction(vcpu, 0) == EMULATE_DONE;
5088

5089 5090
	port = exit_qualification >> 16;
	size = (exit_qualification & 7) + 1;
5091
	skip_emulated_instruction(vcpu);
5092 5093

	return kvm_fast_pio_out(vcpu, size, port);
A
Avi Kivity 已提交
5094 5095
}

I
Ingo Molnar 已提交
5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106
static void
vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
{
	/*
	 * Patch in the VMCALL instruction:
	 */
	hypercall[0] = 0x0f;
	hypercall[1] = 0x01;
	hypercall[2] = 0xc1;
}

5107
static bool nested_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
5108 5109
{
	unsigned long always_on = VMXON_CR0_ALWAYSON;
5110
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5111

5112
	if (to_vmx(vcpu)->nested.nested_vmx_secondary_ctls_high &
5113 5114 5115 5116 5117 5118
		SECONDARY_EXEC_UNRESTRICTED_GUEST &&
	    nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST))
		always_on &= ~(X86_CR0_PE | X86_CR0_PG);
	return (val & always_on) == always_on;
}

G
Guo Chao 已提交
5119
/* called to set cr0 as appropriate for a mov-to-cr0 exit. */
5120 5121 5122
static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
{
	if (is_guest_mode(vcpu)) {
5123 5124 5125
		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
		unsigned long orig_val = val;

5126 5127 5128
		/*
		 * We get here when L2 changed cr0 in a way that did not change
		 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
5129 5130 5131 5132
		 * but did change L0 shadowed bits. So we first calculate the
		 * effective cr0 value that L1 would like to write into the
		 * hardware. It consists of the L2-owned bits from the new
		 * value combined with the L1-owned bits from L1's guest_cr0.
5133
		 */
5134 5135 5136
		val = (val & ~vmcs12->cr0_guest_host_mask) |
			(vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);

5137
		if (!nested_cr0_valid(vcpu, val))
5138
			return 1;
5139 5140 5141 5142

		if (kvm_set_cr0(vcpu, val))
			return 1;
		vmcs_writel(CR0_READ_SHADOW, orig_val);
5143
		return 0;
5144 5145 5146 5147
	} else {
		if (to_vmx(vcpu)->nested.vmxon &&
		    ((val & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON))
			return 1;
5148
		return kvm_set_cr0(vcpu, val);
5149
	}
5150 5151 5152 5153 5154
}

static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
{
	if (is_guest_mode(vcpu)) {
5155 5156 5157 5158 5159 5160 5161
		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
		unsigned long orig_val = val;

		/* analogously to handle_set_cr0 */
		val = (val & ~vmcs12->cr4_guest_host_mask) |
			(vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
		if (kvm_set_cr4(vcpu, val))
5162
			return 1;
5163
		vmcs_writel(CR4_READ_SHADOW, orig_val);
5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184
		return 0;
	} else
		return kvm_set_cr4(vcpu, val);
}

/* called to set cr0 as approriate for clts instruction exit. */
static void handle_clts(struct kvm_vcpu *vcpu)
{
	if (is_guest_mode(vcpu)) {
		/*
		 * We get here when L2 did CLTS, and L1 didn't shadow CR0.TS
		 * but we did (!fpu_active). We need to keep GUEST_CR0.TS on,
		 * just pretend it's off (also in arch.cr0 for fpu_activate).
		 */
		vmcs_writel(CR0_READ_SHADOW,
			vmcs_readl(CR0_READ_SHADOW) & ~X86_CR0_TS);
		vcpu->arch.cr0 &= ~X86_CR0_TS;
	} else
		vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
}

A
Avi Kivity 已提交
5185
static int handle_cr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5186
{
5187
	unsigned long exit_qualification, val;
A
Avi Kivity 已提交
5188 5189
	int cr;
	int reg;
5190
	int err;
A
Avi Kivity 已提交
5191

5192
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
A
Avi Kivity 已提交
5193 5194 5195 5196
	cr = exit_qualification & 15;
	reg = (exit_qualification >> 8) & 15;
	switch ((exit_qualification >> 4) & 3) {
	case 0: /* mov to cr */
5197
		val = kvm_register_readl(vcpu, reg);
5198
		trace_kvm_cr_write(cr, val);
A
Avi Kivity 已提交
5199 5200
		switch (cr) {
		case 0:
5201
			err = handle_set_cr0(vcpu, val);
5202
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
5203 5204
			return 1;
		case 3:
5205
			err = kvm_set_cr3(vcpu, val);
5206
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
5207 5208
			return 1;
		case 4:
5209
			err = handle_set_cr4(vcpu, val);
5210
			kvm_complete_insn_gp(vcpu, err);
A
Avi Kivity 已提交
5211
			return 1;
5212 5213
		case 8: {
				u8 cr8_prev = kvm_get_cr8(vcpu);
5214
				u8 cr8 = (u8)val;
A
Andre Przywara 已提交
5215
				err = kvm_set_cr8(vcpu, cr8);
5216
				kvm_complete_insn_gp(vcpu, err);
5217 5218 5219 5220
				if (irqchip_in_kernel(vcpu->kvm))
					return 1;
				if (cr8_prev <= cr8)
					return 1;
A
Avi Kivity 已提交
5221
				vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
5222 5223
				return 0;
			}
5224
		}
A
Avi Kivity 已提交
5225
		break;
5226
	case 2: /* clts */
5227
		handle_clts(vcpu);
5228
		trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
5229
		skip_emulated_instruction(vcpu);
A
Avi Kivity 已提交
5230
		vmx_fpu_activate(vcpu);
5231
		return 1;
A
Avi Kivity 已提交
5232 5233 5234
	case 1: /*mov from cr*/
		switch (cr) {
		case 3:
5235 5236 5237
			val = kvm_read_cr3(vcpu);
			kvm_register_write(vcpu, reg, val);
			trace_kvm_cr_read(cr, val);
A
Avi Kivity 已提交
5238 5239 5240
			skip_emulated_instruction(vcpu);
			return 1;
		case 8:
5241 5242 5243
			val = kvm_get_cr8(vcpu);
			kvm_register_write(vcpu, reg, val);
			trace_kvm_cr_read(cr, val);
A
Avi Kivity 已提交
5244 5245 5246 5247 5248
			skip_emulated_instruction(vcpu);
			return 1;
		}
		break;
	case 3: /* lmsw */
5249
		val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
5250
		trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
5251
		kvm_lmsw(vcpu, val);
A
Avi Kivity 已提交
5252 5253 5254 5255 5256 5257

		skip_emulated_instruction(vcpu);
		return 1;
	default:
		break;
	}
A
Avi Kivity 已提交
5258
	vcpu->run->exit_reason = 0;
5259
	vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
A
Avi Kivity 已提交
5260 5261 5262 5263
	       (int)(exit_qualification >> 4) & 3, cr);
	return 0;
}

A
Avi Kivity 已提交
5264
static int handle_dr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5265
{
5266
	unsigned long exit_qualification;
5267 5268 5269 5270 5271 5272 5273 5274
	int dr, dr7, reg;

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;

	/* First, if DR does not exist, trigger UD */
	if (!kvm_require_dr(vcpu, dr))
		return 1;
A
Avi Kivity 已提交
5275

5276
	/* Do not handle if the CPL > 0, will trigger GP on re-entry */
5277 5278
	if (!kvm_require_cpl(vcpu, 0))
		return 1;
5279 5280
	dr7 = vmcs_readl(GUEST_DR7);
	if (dr7 & DR7_GD) {
5281 5282 5283 5284 5285 5286
		/*
		 * As the vm-exit takes precedence over the debug trap, we
		 * need to emulate the latter, either for the host or the
		 * guest debugging itself.
		 */
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
A
Avi Kivity 已提交
5287
			vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
5288
			vcpu->run->debug.arch.dr7 = dr7;
5289
			vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
A
Avi Kivity 已提交
5290 5291
			vcpu->run->debug.arch.exception = DB_VECTOR;
			vcpu->run->exit_reason = KVM_EXIT_DEBUG;
5292 5293
			return 0;
		} else {
5294
			vcpu->arch.dr6 &= ~15;
5295
			vcpu->arch.dr6 |= DR6_BD | DR6_RTM;
5296 5297 5298 5299 5300
			kvm_queue_exception(vcpu, DB_VECTOR);
			return 1;
		}
	}

5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316
	if (vcpu->guest_debug == 0) {
		u32 cpu_based_vm_exec_control;

		cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
		cpu_based_vm_exec_control &= ~CPU_BASED_MOV_DR_EXITING;
		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);

		/*
		 * No more DR vmexits; force a reload of the debug registers
		 * and reenter on this instruction.  The next vmexit will
		 * retrieve the full state of the debug registers.
		 */
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
		return 1;
	}

5317 5318
	reg = DEBUG_REG_ACCESS_REG(exit_qualification);
	if (exit_qualification & TYPE_MOV_FROM_DR) {
5319
		unsigned long val;
5320 5321 5322 5323

		if (kvm_get_dr(vcpu, dr, &val))
			return 1;
		kvm_register_write(vcpu, reg, val);
5324
	} else
5325
		if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
5326 5327
			return 1;

A
Avi Kivity 已提交
5328 5329 5330 5331
	skip_emulated_instruction(vcpu);
	return 1;
}

J
Jan Kiszka 已提交
5332 5333 5334 5335 5336 5337 5338 5339 5340
static u64 vmx_get_dr6(struct kvm_vcpu *vcpu)
{
	return vcpu->arch.dr6;
}

static void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
{
}

5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358
static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
{
	u32 cpu_based_vm_exec_control;

	get_debugreg(vcpu->arch.db[0], 0);
	get_debugreg(vcpu->arch.db[1], 1);
	get_debugreg(vcpu->arch.db[2], 2);
	get_debugreg(vcpu->arch.db[3], 3);
	get_debugreg(vcpu->arch.dr6, 6);
	vcpu->arch.dr7 = vmcs_readl(GUEST_DR7);

	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;

	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control |= CPU_BASED_MOV_DR_EXITING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
}

5359 5360 5361 5362 5363
static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
{
	vmcs_writel(GUEST_DR7, val);
}

A
Avi Kivity 已提交
5364
static int handle_cpuid(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5365
{
5366 5367
	kvm_emulate_cpuid(vcpu);
	return 1;
A
Avi Kivity 已提交
5368 5369
}

A
Avi Kivity 已提交
5370
static int handle_rdmsr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5371
{
5372
	u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
A
Avi Kivity 已提交
5373 5374 5375
	u64 data;

	if (vmx_get_msr(vcpu, ecx, &data)) {
5376
		trace_kvm_msr_read_ex(ecx);
5377
		kvm_inject_gp(vcpu, 0);
A
Avi Kivity 已提交
5378 5379 5380
		return 1;
	}

5381
	trace_kvm_msr_read(ecx, data);
F
Feng (Eric) Liu 已提交
5382

A
Avi Kivity 已提交
5383
	/* FIXME: handling of bits 32:63 of rax, rdx */
5384 5385
	vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u;
	vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
A
Avi Kivity 已提交
5386 5387 5388 5389
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
5390
static int handle_wrmsr(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5391
{
5392
	struct msr_data msr;
5393 5394 5395
	u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
	u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
		| ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
A
Avi Kivity 已提交
5396

5397 5398 5399
	msr.data = data;
	msr.index = ecx;
	msr.host_initiated = false;
5400
	if (kvm_set_msr(vcpu, &msr) != 0) {
5401
		trace_kvm_msr_write_ex(ecx, data);
5402
		kvm_inject_gp(vcpu, 0);
A
Avi Kivity 已提交
5403 5404 5405
		return 1;
	}

5406
	trace_kvm_msr_write(ecx, data);
A
Avi Kivity 已提交
5407 5408 5409 5410
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
5411
static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
5412
{
5413
	kvm_make_request(KVM_REQ_EVENT, vcpu);
5414 5415 5416
	return 1;
}

A
Avi Kivity 已提交
5417
static int handle_interrupt_window(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5418
{
5419 5420 5421 5422 5423 5424
	u32 cpu_based_vm_exec_control;

	/* clear pending irq */
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
F
Feng (Eric) Liu 已提交
5425

5426 5427
	kvm_make_request(KVM_REQ_EVENT, vcpu);

5428
	++vcpu->stat.irq_window_exits;
F
Feng (Eric) Liu 已提交
5429

5430 5431 5432 5433
	/*
	 * If the user space waits to inject interrupts, exit as soon as
	 * possible
	 */
5434
	if (!irqchip_in_kernel(vcpu->kvm) &&
A
Avi Kivity 已提交
5435
	    vcpu->run->request_interrupt_window &&
5436
	    !kvm_cpu_has_interrupt(vcpu)) {
A
Avi Kivity 已提交
5437
		vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
5438 5439
		return 0;
	}
A
Avi Kivity 已提交
5440 5441 5442
	return 1;
}

A
Avi Kivity 已提交
5443
static int handle_halt(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5444 5445
{
	skip_emulated_instruction(vcpu);
5446
	return kvm_emulate_halt(vcpu);
A
Avi Kivity 已提交
5447 5448
}

A
Avi Kivity 已提交
5449
static int handle_vmcall(struct kvm_vcpu *vcpu)
5450
{
5451
	skip_emulated_instruction(vcpu);
5452 5453
	kvm_emulate_hypercall(vcpu);
	return 1;
5454 5455
}

5456 5457
static int handle_invd(struct kvm_vcpu *vcpu)
{
5458
	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
5459 5460
}

A
Avi Kivity 已提交
5461
static int handle_invlpg(struct kvm_vcpu *vcpu)
M
Marcelo Tosatti 已提交
5462
{
5463
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
M
Marcelo Tosatti 已提交
5464 5465 5466 5467 5468 5469

	kvm_mmu_invlpg(vcpu, exit_qualification);
	skip_emulated_instruction(vcpu);
	return 1;
}

A
Avi Kivity 已提交
5470 5471 5472 5473 5474 5475 5476 5477 5478 5479
static int handle_rdpmc(struct kvm_vcpu *vcpu)
{
	int err;

	err = kvm_rdpmc(vcpu);
	kvm_complete_insn_gp(vcpu, err);

	return 1;
}

A
Avi Kivity 已提交
5480
static int handle_wbinvd(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
5481 5482
{
	skip_emulated_instruction(vcpu);
5483
	kvm_emulate_wbinvd(vcpu);
E
Eddie Dong 已提交
5484 5485 5486
	return 1;
}

5487 5488 5489 5490 5491 5492 5493 5494 5495 5496
static int handle_xsetbv(struct kvm_vcpu *vcpu)
{
	u64 new_bv = kvm_read_edx_eax(vcpu);
	u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);

	if (kvm_set_xcr(vcpu, index, new_bv) == 0)
		skip_emulated_instruction(vcpu);
	return 1;
}

5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510
static int handle_xsaves(struct kvm_vcpu *vcpu)
{
	skip_emulated_instruction(vcpu);
	WARN(1, "this should never happen\n");
	return 1;
}

static int handle_xrstors(struct kvm_vcpu *vcpu)
{
	skip_emulated_instruction(vcpu);
	WARN(1, "this should never happen\n");
	return 1;
}

A
Avi Kivity 已提交
5511
static int handle_apic_access(struct kvm_vcpu *vcpu)
5512
{
5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530
	if (likely(fasteoi)) {
		unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
		int access_type, offset;

		access_type = exit_qualification & APIC_ACCESS_TYPE;
		offset = exit_qualification & APIC_ACCESS_OFFSET;
		/*
		 * Sane guest uses MOV to write EOI, with written value
		 * not cared. So make a short-circuit here by avoiding
		 * heavy instruction emulation.
		 */
		if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
		    (offset == APIC_EOI)) {
			kvm_lapic_set_eoi(vcpu);
			skip_emulated_instruction(vcpu);
			return 1;
		}
	}
5531
	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
5532 5533
}

5534 5535 5536 5537 5538 5539 5540 5541 5542 5543
static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
{
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	int vector = exit_qualification & 0xff;

	/* EOI-induced VM exit is trap-like and thus no need to adjust IP */
	kvm_apic_set_eoi_accelerated(vcpu, vector);
	return 1;
}

5544 5545 5546 5547 5548 5549 5550 5551 5552 5553
static int handle_apic_write(struct kvm_vcpu *vcpu)
{
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 offset = exit_qualification & 0xfff;

	/* APIC-write VM exit is trap-like and thus no need to adjust IP */
	kvm_apic_write_nodecode(vcpu, offset);
	return 1;
}

A
Avi Kivity 已提交
5554
static int handle_task_switch(struct kvm_vcpu *vcpu)
5555
{
J
Jan Kiszka 已提交
5556
	struct vcpu_vmx *vmx = to_vmx(vcpu);
5557
	unsigned long exit_qualification;
5558 5559
	bool has_error_code = false;
	u32 error_code = 0;
5560
	u16 tss_selector;
5561
	int reason, type, idt_v, idt_index;
5562 5563

	idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
5564
	idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
5565
	type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
5566 5567 5568 5569

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	reason = (u32)exit_qualification >> 30;
5570 5571 5572 5573
	if (reason == TASK_SWITCH_GATE && idt_v) {
		switch (type) {
		case INTR_TYPE_NMI_INTR:
			vcpu->arch.nmi_injected = false;
5574
			vmx_set_nmi_mask(vcpu, true);
5575 5576
			break;
		case INTR_TYPE_EXT_INTR:
5577
		case INTR_TYPE_SOFT_INTR:
5578 5579 5580
			kvm_clear_interrupt_queue(vcpu);
			break;
		case INTR_TYPE_HARD_EXCEPTION:
5581 5582 5583 5584 5585 5586 5587
			if (vmx->idt_vectoring_info &
			    VECTORING_INFO_DELIVER_CODE_MASK) {
				has_error_code = true;
				error_code =
					vmcs_read32(IDT_VECTORING_ERROR_CODE);
			}
			/* fall through */
5588 5589 5590 5591 5592 5593
		case INTR_TYPE_SOFT_EXCEPTION:
			kvm_clear_exception_queue(vcpu);
			break;
		default:
			break;
		}
J
Jan Kiszka 已提交
5594
	}
5595 5596
	tss_selector = exit_qualification;

5597 5598 5599 5600 5601
	if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
		       type != INTR_TYPE_EXT_INTR &&
		       type != INTR_TYPE_NMI_INTR))
		skip_emulated_instruction(vcpu);

5602 5603 5604
	if (kvm_task_switch(vcpu, tss_selector,
			    type == INTR_TYPE_SOFT_INTR ? idt_index : -1, reason,
			    has_error_code, error_code) == EMULATE_FAIL) {
5605 5606 5607
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
5608
		return 0;
5609
	}
5610 5611

	/* clear all local breakpoint enable flags */
5612
	vmcs_writel(GUEST_DR7, vmcs_readl(GUEST_DR7) & ~0x155);
5613 5614 5615 5616 5617 5618 5619

	/*
	 * TODO: What about debug traps on tss switch?
	 *       Are we supposed to inject them and update dr6?
	 */

	return 1;
5620 5621
}

A
Avi Kivity 已提交
5622
static int handle_ept_violation(struct kvm_vcpu *vcpu)
5623
{
5624
	unsigned long exit_qualification;
5625
	gpa_t gpa;
5626
	u32 error_code;
5627 5628
	int gla_validity;

5629
	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5630 5631 5632 5633 5634 5635

	gla_validity = (exit_qualification >> 7) & 0x3;
	if (gla_validity != 0x3 && gla_validity != 0x1 && gla_validity != 0) {
		printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
		printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
			(long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
5636
			vmcs_readl(GUEST_LINEAR_ADDRESS));
5637 5638
		printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
			(long unsigned int)exit_qualification);
A
Avi Kivity 已提交
5639 5640
		vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
		vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
5641
		return 0;
5642 5643
	}

5644 5645 5646 5647 5648 5649
	/*
	 * EPT violation happened while executing iret from NMI,
	 * "blocked by NMI" bit has to be set before next VM entry.
	 * There are errata that may cause this bit to not be set:
	 * AAK134, BY25.
	 */
5650 5651 5652
	if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
			cpu_has_virtual_nmis() &&
			(exit_qualification & INTR_INFO_UNBLOCK_NMI))
5653 5654
		vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);

5655
	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5656
	trace_kvm_page_fault(gpa, exit_qualification);
5657 5658

	/* It is a write fault? */
5659
	error_code = exit_qualification & PFERR_WRITE_MASK;
5660
	/* It is a fetch fault? */
5661
	error_code |= (exit_qualification << 2) & PFERR_FETCH_MASK;
5662
	/* ept page table is present? */
5663
	error_code |= (exit_qualification >> 3) & PFERR_PRESENT_MASK;
5664

5665 5666
	vcpu->arch.exit_qualification = exit_qualification;

5667
	return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
5668 5669
}

5670 5671 5672 5673 5674 5675 5676 5677
static u64 ept_rsvd_mask(u64 spte, int level)
{
	int i;
	u64 mask = 0;

	for (i = 51; i > boot_cpu_data.x86_phys_bits; i--)
		mask |= (1ULL << i);

5678
	if (level == 4)
5679 5680
		/* bits 7:3 reserved */
		mask |= 0xf8;
5681 5682 5683 5684 5685 5686 5687 5688 5689
	else if (spte & (1ULL << 7))
		/*
		 * 1GB/2MB page, bits 29:12 or 20:12 reserved respectively,
		 * level == 1 if the hypervisor is using the ignored bit 7.
		 */
		mask |= (PAGE_SIZE << ((level - 1) * 9)) - PAGE_SIZE;
	else if (level > 1)
		/* bits 6:3 reserved */
		mask |= 0x78;
5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718

	return mask;
}

static void ept_misconfig_inspect_spte(struct kvm_vcpu *vcpu, u64 spte,
				       int level)
{
	printk(KERN_ERR "%s: spte 0x%llx level %d\n", __func__, spte, level);

	/* 010b (write-only) */
	WARN_ON((spte & 0x7) == 0x2);

	/* 110b (write/execute) */
	WARN_ON((spte & 0x7) == 0x6);

	/* 100b (execute-only) and value not supported by logical processor */
	if (!cpu_has_vmx_ept_execute_only())
		WARN_ON((spte & 0x7) == 0x4);

	/* not 000b */
	if ((spte & 0x7)) {
		u64 rsvd_bits = spte & ept_rsvd_mask(spte, level);

		if (rsvd_bits != 0) {
			printk(KERN_ERR "%s: rsvd_bits = 0x%llx\n",
					 __func__, rsvd_bits);
			WARN_ON(1);
		}

5719 5720
		/* bits 5:3 are _not_ reserved for large page or leaf page */
		if ((rsvd_bits & 0x38) == 0) {
5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732
			u64 ept_mem_type = (spte & 0x38) >> 3;

			if (ept_mem_type == 2 || ept_mem_type == 3 ||
			    ept_mem_type == 7) {
				printk(KERN_ERR "%s: ept_mem_type=0x%llx\n",
						__func__, ept_mem_type);
				WARN_ON(1);
			}
		}
	}
}

A
Avi Kivity 已提交
5733
static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
5734 5735
{
	u64 sptes[4];
5736
	int nr_sptes, i, ret;
5737 5738 5739
	gpa_t gpa;

	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5740 5741 5742 5743
	if (!kvm_io_bus_write(vcpu->kvm, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
		skip_emulated_instruction(vcpu);
		return 1;
	}
5744

5745
	ret = handle_mmio_page_fault_common(vcpu, gpa, true);
5746
	if (likely(ret == RET_MMIO_PF_EMULATE))
5747 5748
		return x86_emulate_instruction(vcpu, gpa, 0, NULL, 0) ==
					      EMULATE_DONE;
5749 5750 5751 5752

	if (unlikely(ret == RET_MMIO_PF_INVALID))
		return kvm_mmu_page_fault(vcpu, gpa, 0, NULL, 0);

5753
	if (unlikely(ret == RET_MMIO_PF_RETRY))
5754 5755 5756
		return 1;

	/* It is the real ept misconfig */
5757 5758 5759 5760 5761 5762 5763 5764
	printk(KERN_ERR "EPT: Misconfiguration.\n");
	printk(KERN_ERR "EPT: GPA: 0x%llx\n", gpa);

	nr_sptes = kvm_mmu_get_spte_hierarchy(vcpu, gpa, sptes);

	for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i)
		ept_misconfig_inspect_spte(vcpu, sptes[i-1], i);

A
Avi Kivity 已提交
5765 5766
	vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
	vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
5767 5768 5769 5770

	return 0;
}

A
Avi Kivity 已提交
5771
static int handle_nmi_window(struct kvm_vcpu *vcpu)
5772 5773 5774 5775 5776 5777 5778 5779
{
	u32 cpu_based_vm_exec_control;

	/* clear pending NMI */
	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
	++vcpu->stat.nmi_window_exits;
5780
	kvm_make_request(KVM_REQ_EVENT, vcpu);
5781 5782 5783 5784

	return 1;
}

5785
static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
5786
{
5787 5788
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	enum emulation_result err = EMULATE_DONE;
5789
	int ret = 1;
5790 5791
	u32 cpu_exec_ctrl;
	bool intr_window_requested;
5792
	unsigned count = 130;
5793 5794 5795

	cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
	intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
5796

5797
	while (vmx->emulation_required && count-- != 0) {
5798
		if (intr_window_requested && vmx_interrupt_allowed(vcpu))
5799 5800
			return handle_interrupt_window(&vmx->vcpu);

5801 5802 5803
		if (test_bit(KVM_REQ_EVENT, &vcpu->requests))
			return 1;

5804
		err = emulate_instruction(vcpu, EMULTYPE_NO_REEXECUTE);
5805

P
Paolo Bonzini 已提交
5806
		if (err == EMULATE_USER_EXIT) {
5807
			++vcpu->stat.mmio_exits;
5808 5809 5810
			ret = 0;
			goto out;
		}
5811

5812 5813 5814 5815
		if (err != EMULATE_DONE) {
			vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
			vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
			vcpu->run->internal.ndata = 0;
5816
			return 0;
5817
		}
5818

5819 5820 5821 5822 5823 5824
		if (vcpu->arch.halt_request) {
			vcpu->arch.halt_request = 0;
			ret = kvm_emulate_halt(vcpu);
			goto out;
		}

5825
		if (signal_pending(current))
5826
			goto out;
5827 5828 5829 5830
		if (need_resched())
			schedule();
	}

5831 5832
out:
	return ret;
5833 5834
}

R
Radim Krčmář 已提交
5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871
static int __grow_ple_window(int val)
{
	if (ple_window_grow < 1)
		return ple_window;

	val = min(val, ple_window_actual_max);

	if (ple_window_grow < ple_window)
		val *= ple_window_grow;
	else
		val += ple_window_grow;

	return val;
}

static int __shrink_ple_window(int val, int modifier, int minimum)
{
	if (modifier < 1)
		return ple_window;

	if (modifier < ple_window)
		val /= modifier;
	else
		val -= modifier;

	return max(val, minimum);
}

static void grow_ple_window(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int old = vmx->ple_window;

	vmx->ple_window = __grow_ple_window(old);

	if (vmx->ple_window != old)
		vmx->ple_window_dirty = true;
5872 5873

	trace_kvm_ple_window_grow(vcpu->vcpu_id, vmx->ple_window, old);
R
Radim Krčmář 已提交
5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885
}

static void shrink_ple_window(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int old = vmx->ple_window;

	vmx->ple_window = __shrink_ple_window(old,
	                                      ple_window_shrink, ple_window);

	if (vmx->ple_window != old)
		vmx->ple_window_dirty = true;
5886 5887

	trace_kvm_ple_window_shrink(vcpu->vcpu_id, vmx->ple_window, old);
R
Radim Krčmář 已提交
5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904
}

/*
 * ple_window_actual_max is computed to be one grow_ple_window() below
 * ple_window_max. (See __grow_ple_window for the reason.)
 * This prevents overflows, because ple_window_max is int.
 * ple_window_max effectively rounded down to a multiple of ple_window_grow in
 * this process.
 * ple_window_max is also prevented from setting vmx->ple_window < ple_window.
 */
static void update_ple_window_actual_max(void)
{
	ple_window_actual_max =
			__shrink_ple_window(max(ple_window_max, ple_window),
			                    ple_window_grow, INT_MIN);
}

5905 5906
static __init int hardware_setup(void)
{
5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938
	int r = -ENOMEM, i, msr;

	rdmsrl_safe(MSR_EFER, &host_efer);

	for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i)
		kvm_define_shared_msr(i, vmx_msr_index[i]);

	vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_io_bitmap_a)
		return r;

	vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_io_bitmap_b)
		goto out;

	vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_legacy)
		goto out1;

	vmx_msr_bitmap_legacy_x2apic =
				(unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_legacy_x2apic)
		goto out2;

	vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_longmode)
		goto out3;

	vmx_msr_bitmap_longmode_x2apic =
				(unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_msr_bitmap_longmode_x2apic)
		goto out4;
5939 5940 5941 5942 5943 5944 5945 5946

	if (nested) {
		vmx_msr_bitmap_nested =
			(unsigned long *)__get_free_page(GFP_KERNEL);
		if (!vmx_msr_bitmap_nested)
			goto out5;
	}

5947 5948
	vmx_vmread_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_vmread_bitmap)
5949
		goto out6;
5950 5951 5952

	vmx_vmwrite_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
	if (!vmx_vmwrite_bitmap)
5953
		goto out7;
5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968

	memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
	memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);

	/*
	 * Allow direct access to the PC debug port (it is often used for I/O
	 * delays, but the vmexits simply slow things down).
	 */
	memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
	clear_bit(0x80, vmx_io_bitmap_a);

	memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);

	memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
	memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
5969 5970
	if (nested)
		memset(vmx_msr_bitmap_nested, 0xff, PAGE_SIZE);
5971 5972 5973

	if (setup_vmcs_config(&vmcs_config) < 0) {
		r = -EIO;
5974
		goto out8;
5975
	}
5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999

	if (boot_cpu_has(X86_FEATURE_NX))
		kvm_enable_efer_bits(EFER_NX);

	if (!cpu_has_vmx_vpid())
		enable_vpid = 0;
	if (!cpu_has_vmx_shadow_vmcs())
		enable_shadow_vmcs = 0;
	if (enable_shadow_vmcs)
		init_vmcs_shadow_fields();

	if (!cpu_has_vmx_ept() ||
	    !cpu_has_vmx_ept_4levels()) {
		enable_ept = 0;
		enable_unrestricted_guest = 0;
		enable_ept_ad_bits = 0;
	}

	if (!cpu_has_vmx_ept_ad_bits())
		enable_ept_ad_bits = 0;

	if (!cpu_has_vmx_unrestricted_guest())
		enable_unrestricted_guest = 0;

6000
	if (!cpu_has_vmx_flexpriority())
6001 6002
		flexpriority_enabled = 0;

6003 6004 6005 6006 6007 6008
	/*
	 * set_apic_access_page_addr() is used to reload apic access
	 * page upon invalidation.  No need to do anything if not
	 * using the APIC_ACCESS_ADDR VMCS field.
	 */
	if (!flexpriority_enabled)
6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026
		kvm_x86_ops->set_apic_access_page_addr = NULL;

	if (!cpu_has_vmx_tpr_shadow())
		kvm_x86_ops->update_cr8_intercept = NULL;

	if (enable_ept && !cpu_has_vmx_ept_2m_page())
		kvm_disable_largepages();

	if (!cpu_has_vmx_ple())
		ple_gap = 0;

	if (!cpu_has_vmx_apicv())
		enable_apicv = 0;

	if (enable_apicv)
		kvm_x86_ops->update_cr8_intercept = NULL;
	else {
		kvm_x86_ops->hwapic_irr_update = NULL;
6027
		kvm_x86_ops->hwapic_isr_update = NULL;
6028 6029 6030 6031
		kvm_x86_ops->deliver_posted_interrupt = NULL;
		kvm_x86_ops->sync_pir_to_irr = vmx_sync_pir_to_irr_dummy;
	}

6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074
	vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
	vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
	vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
	vmx_disable_intercept_for_msr(MSR_IA32_BNDCFGS, true);

	memcpy(vmx_msr_bitmap_legacy_x2apic,
			vmx_msr_bitmap_legacy, PAGE_SIZE);
	memcpy(vmx_msr_bitmap_longmode_x2apic,
			vmx_msr_bitmap_longmode, PAGE_SIZE);

	if (enable_apicv) {
		for (msr = 0x800; msr <= 0x8ff; msr++)
			vmx_disable_intercept_msr_read_x2apic(msr);

		/* According SDM, in x2apic mode, the whole id reg is used.
		 * But in KVM, it only use the highest eight bits. Need to
		 * intercept it */
		vmx_enable_intercept_msr_read_x2apic(0x802);
		/* TMCCT */
		vmx_enable_intercept_msr_read_x2apic(0x839);
		/* TPR */
		vmx_disable_intercept_msr_write_x2apic(0x808);
		/* EOI */
		vmx_disable_intercept_msr_write_x2apic(0x80b);
		/* SELF-IPI */
		vmx_disable_intercept_msr_write_x2apic(0x83f);
	}

	if (enable_ept) {
		kvm_mmu_set_mask_ptes(0ull,
			(enable_ept_ad_bits) ? VMX_EPT_ACCESS_BIT : 0ull,
			(enable_ept_ad_bits) ? VMX_EPT_DIRTY_BIT : 0ull,
			0ull, VMX_EPT_EXECUTABLE_MASK);
		ept_set_mmio_spte_mask();
		kvm_enable_tdp();
	} else
		kvm_disable_tdp();

	update_ple_window_actual_max();

K
Kai Huang 已提交
6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088
	/*
	 * Only enable PML when hardware supports PML feature, and both EPT
	 * and EPT A/D bit features are enabled -- PML depends on them to work.
	 */
	if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml())
		enable_pml = 0;

	if (!enable_pml) {
		kvm_x86_ops->slot_enable_log_dirty = NULL;
		kvm_x86_ops->slot_disable_log_dirty = NULL;
		kvm_x86_ops->flush_log_dirty = NULL;
		kvm_x86_ops->enable_log_dirty_pt_masked = NULL;
	}

6089
	return alloc_kvm_area();
6090

6091
out8:
6092
	free_page((unsigned long)vmx_vmwrite_bitmap);
6093
out7:
6094
	free_page((unsigned long)vmx_vmread_bitmap);
6095 6096 6097
out6:
	if (nested)
		free_page((unsigned long)vmx_msr_bitmap_nested);
6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111
out5:
	free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic);
out4:
	free_page((unsigned long)vmx_msr_bitmap_longmode);
out3:
	free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic);
out2:
	free_page((unsigned long)vmx_msr_bitmap_legacy);
out1:
	free_page((unsigned long)vmx_io_bitmap_b);
out:
	free_page((unsigned long)vmx_io_bitmap_a);

    return r;
6112 6113 6114 6115
}

static __exit void hardware_unsetup(void)
{
6116 6117 6118 6119 6120 6121 6122 6123
	free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic);
	free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic);
	free_page((unsigned long)vmx_msr_bitmap_legacy);
	free_page((unsigned long)vmx_msr_bitmap_longmode);
	free_page((unsigned long)vmx_io_bitmap_b);
	free_page((unsigned long)vmx_io_bitmap_a);
	free_page((unsigned long)vmx_vmwrite_bitmap);
	free_page((unsigned long)vmx_vmread_bitmap);
6124 6125
	if (nested)
		free_page((unsigned long)vmx_msr_bitmap_nested);
6126

6127 6128 6129
	free_kvm_area();
}

6130 6131 6132 6133
/*
 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
 * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
 */
6134
static int handle_pause(struct kvm_vcpu *vcpu)
6135
{
R
Radim Krčmář 已提交
6136 6137 6138
	if (ple_gap)
		grow_ple_window(vcpu);

6139 6140 6141 6142 6143 6144
	skip_emulated_instruction(vcpu);
	kvm_vcpu_on_spin(vcpu);

	return 1;
}

6145
static int handle_nop(struct kvm_vcpu *vcpu)
6146
{
6147
	skip_emulated_instruction(vcpu);
6148 6149 6150
	return 1;
}

6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162
static int handle_mwait(struct kvm_vcpu *vcpu)
{
	printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
	return handle_nop(vcpu);
}

static int handle_monitor(struct kvm_vcpu *vcpu)
{
	printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
	return handle_nop(vcpu);
}

6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195
/*
 * To run an L2 guest, we need a vmcs02 based on the L1-specified vmcs12.
 * We could reuse a single VMCS for all the L2 guests, but we also want the
 * option to allocate a separate vmcs02 for each separate loaded vmcs12 - this
 * allows keeping them loaded on the processor, and in the future will allow
 * optimizations where prepare_vmcs02 doesn't need to set all the fields on
 * every entry if they never change.
 * So we keep, in vmx->nested.vmcs02_pool, a cache of size VMCS02_POOL_SIZE
 * (>=0) with a vmcs02 for each recently loaded vmcs12s, most recent first.
 *
 * The following functions allocate and free a vmcs02 in this pool.
 */

/* Get a VMCS from the pool to use as vmcs02 for the current vmcs12. */
static struct loaded_vmcs *nested_get_current_vmcs02(struct vcpu_vmx *vmx)
{
	struct vmcs02_list *item;
	list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
		if (item->vmptr == vmx->nested.current_vmptr) {
			list_move(&item->list, &vmx->nested.vmcs02_pool);
			return &item->vmcs02;
		}

	if (vmx->nested.vmcs02_num >= max(VMCS02_POOL_SIZE, 1)) {
		/* Recycle the least recently used VMCS. */
		item = list_entry(vmx->nested.vmcs02_pool.prev,
			struct vmcs02_list, list);
		item->vmptr = vmx->nested.current_vmptr;
		list_move(&item->list, &vmx->nested.vmcs02_pool);
		return &item->vmcs02;
	}

	/* Create a new VMCS */
6196
	item = kmalloc(sizeof(struct vmcs02_list), GFP_KERNEL);
6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226
	if (!item)
		return NULL;
	item->vmcs02.vmcs = alloc_vmcs();
	if (!item->vmcs02.vmcs) {
		kfree(item);
		return NULL;
	}
	loaded_vmcs_init(&item->vmcs02);
	item->vmptr = vmx->nested.current_vmptr;
	list_add(&(item->list), &(vmx->nested.vmcs02_pool));
	vmx->nested.vmcs02_num++;
	return &item->vmcs02;
}

/* Free and remove from pool a vmcs02 saved for a vmcs12 (if there is one) */
static void nested_free_vmcs02(struct vcpu_vmx *vmx, gpa_t vmptr)
{
	struct vmcs02_list *item;
	list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
		if (item->vmptr == vmptr) {
			free_loaded_vmcs(&item->vmcs02);
			list_del(&item->list);
			kfree(item);
			vmx->nested.vmcs02_num--;
			return;
		}
}

/*
 * Free all VMCSs saved for this vcpu, except the one pointed by
6227 6228
 * vmx->loaded_vmcs. We must be running L1, so vmx->loaded_vmcs
 * must be &vmx->vmcs01.
6229 6230 6231 6232
 */
static void nested_free_all_saved_vmcss(struct vcpu_vmx *vmx)
{
	struct vmcs02_list *item, *n;
6233 6234

	WARN_ON(vmx->loaded_vmcs != &vmx->vmcs01);
6235
	list_for_each_entry_safe(item, n, &vmx->nested.vmcs02_pool, list) {
6236 6237 6238 6239 6240 6241 6242 6243
		/*
		 * Something will leak if the above WARN triggers.  Better than
		 * a use-after-free.
		 */
		if (vmx->loaded_vmcs == &item->vmcs02)
			continue;

		free_loaded_vmcs(&item->vmcs02);
6244 6245
		list_del(&item->list);
		kfree(item);
6246
		vmx->nested.vmcs02_num--;
6247 6248 6249
	}
}

6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269
/*
 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
 * set the success or error code of an emulated VMX instruction, as specified
 * by Vol 2B, VMX Instruction Reference, "Conventions".
 */
static void nested_vmx_succeed(struct kvm_vcpu *vcpu)
{
	vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
			& ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
			    X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
}

static void nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
{
	vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
			& ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
			    X86_EFLAGS_SF | X86_EFLAGS_OF))
			| X86_EFLAGS_CF);
}

A
Abel Gordon 已提交
6270
static void nested_vmx_failValid(struct kvm_vcpu *vcpu,
6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290
					u32 vm_instruction_error)
{
	if (to_vmx(vcpu)->nested.current_vmptr == -1ull) {
		/*
		 * failValid writes the error number to the current VMCS, which
		 * can't be done there isn't a current VMCS.
		 */
		nested_vmx_failInvalid(vcpu);
		return;
	}
	vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
			& ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
			    X86_EFLAGS_SF | X86_EFLAGS_OF))
			| X86_EFLAGS_ZF);
	get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
	/*
	 * We don't need to force a shadow sync because
	 * VM_INSTRUCTION_ERROR is not shadowed
	 */
}
A
Abel Gordon 已提交
6291

6292 6293 6294 6295 6296 6297 6298
static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
{
	/* TODO: not to reset guest simply here. */
	kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
	pr_warn("kvm: nested vmx abort, indicator %d\n", indicator);
}

6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310
static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
{
	struct vcpu_vmx *vmx =
		container_of(timer, struct vcpu_vmx, nested.preemption_timer);

	vmx->nested.preemption_timer_expired = true;
	kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
	kvm_vcpu_kick(&vmx->vcpu);

	return HRTIMER_NORESTART;
}

6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363
/*
 * Decode the memory-address operand of a vmx instruction, as recorded on an
 * exit caused by such an instruction (run by a guest hypervisor).
 * On success, returns 0. When the operand is invalid, returns 1 and throws
 * #UD or #GP.
 */
static int get_vmx_mem_address(struct kvm_vcpu *vcpu,
				 unsigned long exit_qualification,
				 u32 vmx_instruction_info, gva_t *ret)
{
	/*
	 * According to Vol. 3B, "Information for VM Exits Due to Instruction
	 * Execution", on an exit, vmx_instruction_info holds most of the
	 * addressing components of the operand. Only the displacement part
	 * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
	 * For how an actual address is calculated from all these components,
	 * refer to Vol. 1, "Operand Addressing".
	 */
	int  scaling = vmx_instruction_info & 3;
	int  addr_size = (vmx_instruction_info >> 7) & 7;
	bool is_reg = vmx_instruction_info & (1u << 10);
	int  seg_reg = (vmx_instruction_info >> 15) & 7;
	int  index_reg = (vmx_instruction_info >> 18) & 0xf;
	bool index_is_valid = !(vmx_instruction_info & (1u << 22));
	int  base_reg       = (vmx_instruction_info >> 23) & 0xf;
	bool base_is_valid  = !(vmx_instruction_info & (1u << 27));

	if (is_reg) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	/* Addr = segment_base + offset */
	/* offset = base + [index * scale] + displacement */
	*ret = vmx_get_segment_base(vcpu, seg_reg);
	if (base_is_valid)
		*ret += kvm_register_read(vcpu, base_reg);
	if (index_is_valid)
		*ret += kvm_register_read(vcpu, index_reg)<<scaling;
	*ret += exit_qualification; /* holds the displacement */

	if (addr_size == 1) /* 32 bit */
		*ret &= 0xffffffff;

	/*
	 * TODO: throw #GP (and return 1) in various cases that the VM*
	 * instructions require it - e.g., offset beyond segment limit,
	 * unusable or unreadable/unwritable segment, non-canonical 64-bit
	 * address, and so on. Currently these are not checked.
	 */
	return 0;
}

6364 6365 6366 6367 6368
/*
 * This function performs the various checks including
 * - if it's 4KB aligned
 * - No bits beyond the physical address width are set
 * - Returns 0 on success or else 1
6369
 * (Intel SDM Section 30.3)
6370
 */
6371 6372
static int nested_vmx_check_vmptr(struct kvm_vcpu *vcpu, int exit_reason,
				  gpa_t *vmpointer)
6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402
{
	gva_t gva;
	gpa_t vmptr;
	struct x86_exception e;
	struct page *page;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int maxphyaddr = cpuid_maxphyaddr(vcpu);

	if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
			vmcs_read32(VMX_INSTRUCTION_INFO), &gva))
		return 1;

	if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &vmptr,
				sizeof(vmptr), &e)) {
		kvm_inject_page_fault(vcpu, &e);
		return 1;
	}

	switch (exit_reason) {
	case EXIT_REASON_VMON:
		/*
		 * SDM 3: 24.11.5
		 * The first 4 bytes of VMXON region contain the supported
		 * VMCS revision identifier
		 *
		 * Note - IA32_VMX_BASIC[48] will never be 1
		 * for the nested case;
		 * which replaces physical address width with 32
		 *
		 */
6403
		if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419
			nested_vmx_failInvalid(vcpu);
			skip_emulated_instruction(vcpu);
			return 1;
		}

		page = nested_get_page(vcpu, vmptr);
		if (page == NULL ||
		    *(u32 *)kmap(page) != VMCS12_REVISION) {
			nested_vmx_failInvalid(vcpu);
			kunmap(page);
			skip_emulated_instruction(vcpu);
			return 1;
		}
		kunmap(page);
		vmx->nested.vmxon_ptr = vmptr;
		break;
6420
	case EXIT_REASON_VMCLEAR:
6421
		if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435
			nested_vmx_failValid(vcpu,
					     VMXERR_VMCLEAR_INVALID_ADDRESS);
			skip_emulated_instruction(vcpu);
			return 1;
		}

		if (vmptr == vmx->nested.vmxon_ptr) {
			nested_vmx_failValid(vcpu,
					     VMXERR_VMCLEAR_VMXON_POINTER);
			skip_emulated_instruction(vcpu);
			return 1;
		}
		break;
	case EXIT_REASON_VMPTRLD:
6436
		if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
6437 6438 6439 6440 6441
			nested_vmx_failValid(vcpu,
					     VMXERR_VMPTRLD_INVALID_ADDRESS);
			skip_emulated_instruction(vcpu);
			return 1;
		}
6442

6443 6444 6445 6446 6447 6448 6449
		if (vmptr == vmx->nested.vmxon_ptr) {
			nested_vmx_failValid(vcpu,
					     VMXERR_VMCLEAR_VMXON_POINTER);
			skip_emulated_instruction(vcpu);
			return 1;
		}
		break;
6450 6451 6452 6453
	default:
		return 1; /* shouldn't happen */
	}

6454 6455
	if (vmpointer)
		*vmpointer = vmptr;
6456 6457 6458
	return 0;
}

6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470
/*
 * Emulate the VMXON instruction.
 * Currently, we just remember that VMX is active, and do not save or even
 * inspect the argument to VMXON (the so-called "VMXON pointer") because we
 * do not currently need to store anything in that guest-allocated memory
 * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
 * argument is different from the VMXON pointer (which the spec says they do).
 */
static int handle_vmon(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Abel Gordon 已提交
6471
	struct vmcs *shadow_vmcs;
6472 6473
	const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED
		| FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496

	/* The Intel VMX Instruction Reference lists a bunch of bits that
	 * are prerequisite to running VMXON, most notably cr4.VMXE must be
	 * set to 1 (see vmx_set_cr4() for when we allow the guest to set this).
	 * Otherwise, we should fail with #UD. We test these now:
	 */
	if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE) ||
	    !kvm_read_cr0_bits(vcpu, X86_CR0_PE) ||
	    (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	if (is_long_mode(vcpu) && !cs.l) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	if (vmx_get_cpl(vcpu)) {
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
6497

6498
	if (nested_vmx_check_vmptr(vcpu, EXIT_REASON_VMON, NULL))
6499 6500
		return 1;

A
Abel Gordon 已提交
6501 6502 6503 6504 6505
	if (vmx->nested.vmxon) {
		nested_vmx_failValid(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
		skip_emulated_instruction(vcpu);
		return 1;
	}
6506 6507 6508 6509 6510 6511 6512

	if ((vmx->nested.msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
			!= VMXON_NEEDED_FEATURES) {
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

A
Abel Gordon 已提交
6513 6514 6515 6516 6517 6518 6519 6520 6521 6522
	if (enable_shadow_vmcs) {
		shadow_vmcs = alloc_vmcs();
		if (!shadow_vmcs)
			return -ENOMEM;
		/* mark vmcs as shadow */
		shadow_vmcs->revision_id |= (1u << 31);
		/* init shadow vmcs */
		vmcs_clear(shadow_vmcs);
		vmx->nested.current_shadow_vmcs = shadow_vmcs;
	}
6523

6524 6525 6526
	INIT_LIST_HEAD(&(vmx->nested.vmcs02_pool));
	vmx->nested.vmcs02_num = 0;

6527 6528 6529 6530
	hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
		     HRTIMER_MODE_REL);
	vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;

6531 6532 6533
	vmx->nested.vmxon = true;

	skip_emulated_instruction(vcpu);
6534
	nested_vmx_succeed(vcpu);
6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567
	return 1;
}

/*
 * Intel's VMX Instruction Reference specifies a common set of prerequisites
 * for running VMX instructions (except VMXON, whose prerequisites are
 * slightly different). It also specifies what exception to inject otherwise.
 */
static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
{
	struct kvm_segment cs;
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (!vmx->nested.vmxon) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 0;
	}

	vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
	if ((vmx_get_rflags(vcpu) & X86_EFLAGS_VM) ||
	    (is_long_mode(vcpu) && !cs.l)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 0;
	}

	if (vmx_get_cpl(vcpu)) {
		kvm_inject_gp(vcpu, 0);
		return 0;
	}

	return 1;
}

A
Abel Gordon 已提交
6568 6569
static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
{
6570
	u32 exec_control;
6571 6572 6573 6574 6575 6576 6577
	if (vmx->nested.current_vmptr == -1ull)
		return;

	/* current_vmptr and current_vmcs12 are always set/reset together */
	if (WARN_ON(vmx->nested.current_vmcs12 == NULL))
		return;

6578
	if (enable_shadow_vmcs) {
6579 6580 6581 6582 6583 6584 6585 6586
		/* copy to memory all shadowed fields in case
		   they were modified */
		copy_shadow_to_vmcs12(vmx);
		vmx->nested.sync_shadow_vmcs = false;
		exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
		exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
		vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
		vmcs_write64(VMCS_LINK_POINTER, -1ull);
6587
	}
A
Abel Gordon 已提交
6588 6589
	kunmap(vmx->nested.current_vmcs12_page);
	nested_release_page(vmx->nested.current_vmcs12_page);
6590 6591
	vmx->nested.current_vmptr = -1ull;
	vmx->nested.current_vmcs12 = NULL;
A
Abel Gordon 已提交
6592 6593
}

6594 6595 6596 6597 6598 6599 6600 6601
/*
 * Free whatever needs to be freed from vmx->nested when L1 goes down, or
 * just stops using VMX.
 */
static void free_nested(struct vcpu_vmx *vmx)
{
	if (!vmx->nested.vmxon)
		return;
6602

6603
	vmx->nested.vmxon = false;
6604
	nested_release_vmcs12(vmx);
A
Abel Gordon 已提交
6605 6606
	if (enable_shadow_vmcs)
		free_vmcs(vmx->nested.current_shadow_vmcs);
6607 6608 6609
	/* Unpin physical memory we referred to in current vmcs02 */
	if (vmx->nested.apic_access_page) {
		nested_release_page(vmx->nested.apic_access_page);
6610
		vmx->nested.apic_access_page = NULL;
6611
	}
6612 6613
	if (vmx->nested.virtual_apic_page) {
		nested_release_page(vmx->nested.virtual_apic_page);
6614
		vmx->nested.virtual_apic_page = NULL;
6615
	}
6616 6617

	nested_free_all_saved_vmcss(vmx);
6618 6619 6620 6621 6622 6623 6624 6625 6626
}

/* Emulate the VMXOFF instruction */
static int handle_vmoff(struct kvm_vcpu *vcpu)
{
	if (!nested_vmx_check_permission(vcpu))
		return 1;
	free_nested(to_vmx(vcpu));
	skip_emulated_instruction(vcpu);
6627
	nested_vmx_succeed(vcpu);
6628 6629 6630
	return 1;
}

N
Nadav Har'El 已提交
6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641
/* Emulate the VMCLEAR instruction */
static int handle_vmclear(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	gpa_t vmptr;
	struct vmcs12 *vmcs12;
	struct page *page;

	if (!nested_vmx_check_permission(vcpu))
		return 1;

6642
	if (nested_vmx_check_vmptr(vcpu, EXIT_REASON_VMCLEAR, &vmptr))
N
Nadav Har'El 已提交
6643 6644
		return 1;

6645
	if (vmptr == vmx->nested.current_vmptr)
A
Abel Gordon 已提交
6646
		nested_release_vmcs12(vmx);
N
Nadav Har'El 已提交
6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671

	page = nested_get_page(vcpu, vmptr);
	if (page == NULL) {
		/*
		 * For accurate processor emulation, VMCLEAR beyond available
		 * physical memory should do nothing at all. However, it is
		 * possible that a nested vmx bug, not a guest hypervisor bug,
		 * resulted in this case, so let's shut down before doing any
		 * more damage:
		 */
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
		return 1;
	}
	vmcs12 = kmap(page);
	vmcs12->launch_state = 0;
	kunmap(page);
	nested_release_page(page);

	nested_free_vmcs02(vmx, vmptr);

	skip_emulated_instruction(vcpu);
	nested_vmx_succeed(vcpu);
	return 1;
}

6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686
static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch);

/* Emulate the VMLAUNCH instruction */
static int handle_vmlaunch(struct kvm_vcpu *vcpu)
{
	return nested_vmx_run(vcpu, true);
}

/* Emulate the VMRESUME instruction */
static int handle_vmresume(struct kvm_vcpu *vcpu)
{

	return nested_vmx_run(vcpu, false);
}

6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712
enum vmcs_field_type {
	VMCS_FIELD_TYPE_U16 = 0,
	VMCS_FIELD_TYPE_U64 = 1,
	VMCS_FIELD_TYPE_U32 = 2,
	VMCS_FIELD_TYPE_NATURAL_WIDTH = 3
};

static inline int vmcs_field_type(unsigned long field)
{
	if (0x1 & field)	/* the *_HIGH fields are all 32 bit */
		return VMCS_FIELD_TYPE_U32;
	return (field >> 13) & 0x3 ;
}

static inline int vmcs_field_readonly(unsigned long field)
{
	return (((field >> 10) & 0x3) == 1);
}

/*
 * Read a vmcs12 field. Since these can have varying lengths and we return
 * one type, we chose the biggest type (u64) and zero-extend the return value
 * to that size. Note that the caller, handle_vmread, might need to use only
 * some of the bits we return here (e.g., on 32-bit guests, only 32 bits of
 * 64-bit fields are to be returned).
 */
6713 6714
static inline int vmcs12_read_any(struct kvm_vcpu *vcpu,
				  unsigned long field, u64 *ret)
6715 6716 6717 6718 6719
{
	short offset = vmcs_field_to_offset(field);
	char *p;

	if (offset < 0)
6720
		return offset;
6721 6722 6723 6724 6725 6726

	p = ((char *)(get_vmcs12(vcpu))) + offset;

	switch (vmcs_field_type(field)) {
	case VMCS_FIELD_TYPE_NATURAL_WIDTH:
		*ret = *((natural_width *)p);
6727
		return 0;
6728 6729
	case VMCS_FIELD_TYPE_U16:
		*ret = *((u16 *)p);
6730
		return 0;
6731 6732
	case VMCS_FIELD_TYPE_U32:
		*ret = *((u32 *)p);
6733
		return 0;
6734 6735
	case VMCS_FIELD_TYPE_U64:
		*ret = *((u64 *)p);
6736
		return 0;
6737
	default:
6738 6739
		WARN_ON(1);
		return -ENOENT;
6740 6741 6742
	}
}

A
Abel Gordon 已提交
6743

6744 6745
static inline int vmcs12_write_any(struct kvm_vcpu *vcpu,
				   unsigned long field, u64 field_value){
A
Abel Gordon 已提交
6746 6747 6748
	short offset = vmcs_field_to_offset(field);
	char *p = ((char *) get_vmcs12(vcpu)) + offset;
	if (offset < 0)
6749
		return offset;
A
Abel Gordon 已提交
6750 6751 6752 6753

	switch (vmcs_field_type(field)) {
	case VMCS_FIELD_TYPE_U16:
		*(u16 *)p = field_value;
6754
		return 0;
A
Abel Gordon 已提交
6755 6756
	case VMCS_FIELD_TYPE_U32:
		*(u32 *)p = field_value;
6757
		return 0;
A
Abel Gordon 已提交
6758 6759
	case VMCS_FIELD_TYPE_U64:
		*(u64 *)p = field_value;
6760
		return 0;
A
Abel Gordon 已提交
6761 6762
	case VMCS_FIELD_TYPE_NATURAL_WIDTH:
		*(natural_width *)p = field_value;
6763
		return 0;
A
Abel Gordon 已提交
6764
	default:
6765 6766
		WARN_ON(1);
		return -ENOENT;
A
Abel Gordon 已提交
6767 6768 6769 6770
	}

}

6771 6772 6773 6774 6775 6776
static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
{
	int i;
	unsigned long field;
	u64 field_value;
	struct vmcs *shadow_vmcs = vmx->nested.current_shadow_vmcs;
6777 6778
	const unsigned long *fields = shadow_read_write_fields;
	const int num_fields = max_shadow_read_write_fields;
6779

6780 6781
	preempt_disable();

6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798
	vmcs_load(shadow_vmcs);

	for (i = 0; i < num_fields; i++) {
		field = fields[i];
		switch (vmcs_field_type(field)) {
		case VMCS_FIELD_TYPE_U16:
			field_value = vmcs_read16(field);
			break;
		case VMCS_FIELD_TYPE_U32:
			field_value = vmcs_read32(field);
			break;
		case VMCS_FIELD_TYPE_U64:
			field_value = vmcs_read64(field);
			break;
		case VMCS_FIELD_TYPE_NATURAL_WIDTH:
			field_value = vmcs_readl(field);
			break;
6799 6800 6801
		default:
			WARN_ON(1);
			continue;
6802 6803 6804 6805 6806 6807
		}
		vmcs12_write_any(&vmx->vcpu, field, field_value);
	}

	vmcs_clear(shadow_vmcs);
	vmcs_load(vmx->loaded_vmcs->vmcs);
6808 6809

	preempt_enable();
6810 6811
}

6812 6813
static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
{
6814 6815 6816
	const unsigned long *fields[] = {
		shadow_read_write_fields,
		shadow_read_only_fields
6817
	};
6818
	const int max_fields[] = {
6819 6820 6821 6822 6823 6824 6825 6826 6827 6828
		max_shadow_read_write_fields,
		max_shadow_read_only_fields
	};
	int i, q;
	unsigned long field;
	u64 field_value = 0;
	struct vmcs *shadow_vmcs = vmx->nested.current_shadow_vmcs;

	vmcs_load(shadow_vmcs);

6829
	for (q = 0; q < ARRAY_SIZE(fields); q++) {
6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846
		for (i = 0; i < max_fields[q]; i++) {
			field = fields[q][i];
			vmcs12_read_any(&vmx->vcpu, field, &field_value);

			switch (vmcs_field_type(field)) {
			case VMCS_FIELD_TYPE_U16:
				vmcs_write16(field, (u16)field_value);
				break;
			case VMCS_FIELD_TYPE_U32:
				vmcs_write32(field, (u32)field_value);
				break;
			case VMCS_FIELD_TYPE_U64:
				vmcs_write64(field, (u64)field_value);
				break;
			case VMCS_FIELD_TYPE_NATURAL_WIDTH:
				vmcs_writel(field, (long)field_value);
				break;
6847 6848 6849
			default:
				WARN_ON(1);
				break;
6850 6851 6852 6853 6854 6855 6856 6857
			}
		}
	}

	vmcs_clear(shadow_vmcs);
	vmcs_load(vmx->loaded_vmcs->vmcs);
}

6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885
/*
 * VMX instructions which assume a current vmcs12 (i.e., that VMPTRLD was
 * used before) all generate the same failure when it is missing.
 */
static int nested_vmx_check_vmcs12(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	if (vmx->nested.current_vmptr == -1ull) {
		nested_vmx_failInvalid(vcpu);
		skip_emulated_instruction(vcpu);
		return 0;
	}
	return 1;
}

static int handle_vmread(struct kvm_vcpu *vcpu)
{
	unsigned long field;
	u64 field_value;
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
	gva_t gva = 0;

	if (!nested_vmx_check_permission(vcpu) ||
	    !nested_vmx_check_vmcs12(vcpu))
		return 1;

	/* Decode instruction info and find the field to read */
6886
	field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
6887
	/* Read the field, zero-extended to a u64 field_value */
6888
	if (vmcs12_read_any(vcpu, field, &field_value) < 0) {
6889 6890 6891 6892 6893 6894 6895 6896 6897 6898
		nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
		skip_emulated_instruction(vcpu);
		return 1;
	}
	/*
	 * Now copy part of this value to register or memory, as requested.
	 * Note that the number of bits actually copied is 32 or 64 depending
	 * on the guest's mode (32 or 64 bit), not on the given field's length.
	 */
	if (vmx_instruction_info & (1u << 10)) {
6899
		kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf),
6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935
			field_value);
	} else {
		if (get_vmx_mem_address(vcpu, exit_qualification,
				vmx_instruction_info, &gva))
			return 1;
		/* _system ok, as nested_vmx_check_permission verified cpl=0 */
		kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, gva,
			     &field_value, (is_long_mode(vcpu) ? 8 : 4), NULL);
	}

	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}


static int handle_vmwrite(struct kvm_vcpu *vcpu)
{
	unsigned long field;
	gva_t gva;
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
	/* The value to write might be 32 or 64 bits, depending on L1's long
	 * mode, and eventually we need to write that into a field of several
	 * possible lengths. The code below first zero-extends the value to 64
	 * bit (field_value), and then copies only the approriate number of
	 * bits into the vmcs12 field.
	 */
	u64 field_value = 0;
	struct x86_exception e;

	if (!nested_vmx_check_permission(vcpu) ||
	    !nested_vmx_check_vmcs12(vcpu))
		return 1;

	if (vmx_instruction_info & (1u << 10))
6936
		field_value = kvm_register_readl(vcpu,
6937 6938 6939 6940 6941 6942
			(((vmx_instruction_info) >> 3) & 0xf));
	else {
		if (get_vmx_mem_address(vcpu, exit_qualification,
				vmx_instruction_info, &gva))
			return 1;
		if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva,
6943
			   &field_value, (is_64_bit_mode(vcpu) ? 8 : 4), &e)) {
6944 6945 6946 6947 6948 6949
			kvm_inject_page_fault(vcpu, &e);
			return 1;
		}
	}


6950
	field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
6951 6952 6953 6954 6955 6956 6957
	if (vmcs_field_readonly(field)) {
		nested_vmx_failValid(vcpu,
			VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
		skip_emulated_instruction(vcpu);
		return 1;
	}

6958
	if (vmcs12_write_any(vcpu, field, field_value) < 0) {
6959 6960 6961 6962 6963 6964 6965 6966 6967 6968
		nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
		skip_emulated_instruction(vcpu);
		return 1;
	}

	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}

N
Nadav Har'El 已提交
6969 6970 6971 6972 6973
/* Emulate the VMPTRLD instruction */
static int handle_vmptrld(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	gpa_t vmptr;
6974
	u32 exec_control;
N
Nadav Har'El 已提交
6975 6976 6977 6978

	if (!nested_vmx_check_permission(vcpu))
		return 1;

6979
	if (nested_vmx_check_vmptr(vcpu, EXIT_REASON_VMPTRLD, &vmptr))
N
Nadav Har'El 已提交
6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000
		return 1;

	if (vmx->nested.current_vmptr != vmptr) {
		struct vmcs12 *new_vmcs12;
		struct page *page;
		page = nested_get_page(vcpu, vmptr);
		if (page == NULL) {
			nested_vmx_failInvalid(vcpu);
			skip_emulated_instruction(vcpu);
			return 1;
		}
		new_vmcs12 = kmap(page);
		if (new_vmcs12->revision_id != VMCS12_REVISION) {
			kunmap(page);
			nested_release_page_clean(page);
			nested_vmx_failValid(vcpu,
				VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
			skip_emulated_instruction(vcpu);
			return 1;
		}

7001
		nested_release_vmcs12(vmx);
N
Nadav Har'El 已提交
7002 7003 7004
		vmx->nested.current_vmptr = vmptr;
		vmx->nested.current_vmcs12 = new_vmcs12;
		vmx->nested.current_vmcs12_page = page;
7005
		if (enable_shadow_vmcs) {
7006 7007 7008 7009 7010
			exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
			exec_control |= SECONDARY_EXEC_SHADOW_VMCS;
			vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
			vmcs_write64(VMCS_LINK_POINTER,
				     __pa(vmx->nested.current_shadow_vmcs));
7011 7012
			vmx->nested.sync_shadow_vmcs = true;
		}
N
Nadav Har'El 已提交
7013 7014 7015 7016 7017 7018 7019
	}

	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}

N
Nadav Har'El 已提交
7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045
/* Emulate the VMPTRST instruction */
static int handle_vmptrst(struct kvm_vcpu *vcpu)
{
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
	gva_t vmcs_gva;
	struct x86_exception e;

	if (!nested_vmx_check_permission(vcpu))
		return 1;

	if (get_vmx_mem_address(vcpu, exit_qualification,
			vmx_instruction_info, &vmcs_gva))
		return 1;
	/* ok to use *_system, as nested_vmx_check_permission verified cpl=0 */
	if (kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, vmcs_gva,
				 (void *)&to_vmx(vcpu)->nested.current_vmptr,
				 sizeof(u64), &e)) {
		kvm_inject_page_fault(vcpu, &e);
		return 1;
	}
	nested_vmx_succeed(vcpu);
	skip_emulated_instruction(vcpu);
	return 1;
}

N
Nadav Har'El 已提交
7046 7047 7048
/* Emulate the INVEPT instruction */
static int handle_invept(struct kvm_vcpu *vcpu)
{
7049
	struct vcpu_vmx *vmx = to_vmx(vcpu);
N
Nadav Har'El 已提交
7050 7051 7052 7053 7054 7055 7056 7057
	u32 vmx_instruction_info, types;
	unsigned long type;
	gva_t gva;
	struct x86_exception e;
	struct {
		u64 eptp, gpa;
	} operand;

7058 7059 7060
	if (!(vmx->nested.nested_vmx_secondary_ctls_high &
	      SECONDARY_EXEC_ENABLE_EPT) ||
	    !(vmx->nested.nested_vmx_ept_caps & VMX_EPT_INVEPT_BIT)) {
N
Nadav Har'El 已提交
7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	if (!nested_vmx_check_permission(vcpu))
		return 1;

	if (!kvm_read_cr0_bits(vcpu, X86_CR0_PE)) {
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
	}

	vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
7074
	type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
N
Nadav Har'El 已提交
7075

7076
	types = (vmx->nested.nested_vmx_ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
N
Nadav Har'El 已提交
7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098

	if (!(types & (1UL << type))) {
		nested_vmx_failValid(vcpu,
				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
		return 1;
	}

	/* According to the Intel VMX instruction reference, the memory
	 * operand is read even if it isn't needed (e.g., for type==global)
	 */
	if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
			vmx_instruction_info, &gva))
		return 1;
	if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &operand,
				sizeof(operand), &e)) {
		kvm_inject_page_fault(vcpu, &e);
		return 1;
	}

	switch (type) {
	case VMX_EPT_EXTENT_GLOBAL:
		kvm_mmu_sync_roots(vcpu);
7099
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
N
Nadav Har'El 已提交
7100 7101 7102
		nested_vmx_succeed(vcpu);
		break;
	default:
7103
		/* Trap single context invalidation invept calls */
N
Nadav Har'El 已提交
7104 7105 7106 7107 7108 7109 7110 7111
		BUG_ON(1);
		break;
	}

	skip_emulated_instruction(vcpu);
	return 1;
}

7112 7113 7114 7115 7116 7117
static int handle_invvpid(struct kvm_vcpu *vcpu)
{
	kvm_queue_exception(vcpu, UD_VECTOR);
	return 1;
}

K
Kai Huang 已提交
7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142
static int handle_pml_full(struct kvm_vcpu *vcpu)
{
	unsigned long exit_qualification;

	trace_kvm_pml_full(vcpu->vcpu_id);

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	/*
	 * PML buffer FULL happened while executing iret from NMI,
	 * "blocked by NMI" bit has to be set before next VM entry.
	 */
	if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
			cpu_has_virtual_nmis() &&
			(exit_qualification & INTR_INFO_UNBLOCK_NMI))
		vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				GUEST_INTR_STATE_NMI);

	/*
	 * PML buffer already flushed at beginning of VMEXIT. Nothing to do
	 * here.., and there's no userspace involvement needed for PML.
	 */
	return 1;
}

A
Avi Kivity 已提交
7143 7144 7145 7146 7147
/*
 * The exit handlers return 1 if the exit was handled fully and guest execution
 * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
 * to be done to userspace and return 0.
 */
7148
static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
A
Avi Kivity 已提交
7149 7150
	[EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
	[EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
7151
	[EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
7152
	[EXIT_REASON_NMI_WINDOW]	      = handle_nmi_window,
A
Avi Kivity 已提交
7153 7154 7155 7156 7157 7158 7159 7160
	[EXIT_REASON_IO_INSTRUCTION]          = handle_io,
	[EXIT_REASON_CR_ACCESS]               = handle_cr,
	[EXIT_REASON_DR_ACCESS]               = handle_dr,
	[EXIT_REASON_CPUID]                   = handle_cpuid,
	[EXIT_REASON_MSR_READ]                = handle_rdmsr,
	[EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
	[EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
	[EXIT_REASON_HLT]                     = handle_halt,
7161
	[EXIT_REASON_INVD]		      = handle_invd,
M
Marcelo Tosatti 已提交
7162
	[EXIT_REASON_INVLPG]		      = handle_invlpg,
A
Avi Kivity 已提交
7163
	[EXIT_REASON_RDPMC]                   = handle_rdpmc,
7164
	[EXIT_REASON_VMCALL]                  = handle_vmcall,
N
Nadav Har'El 已提交
7165
	[EXIT_REASON_VMCLEAR]	              = handle_vmclear,
7166
	[EXIT_REASON_VMLAUNCH]                = handle_vmlaunch,
N
Nadav Har'El 已提交
7167
	[EXIT_REASON_VMPTRLD]                 = handle_vmptrld,
N
Nadav Har'El 已提交
7168
	[EXIT_REASON_VMPTRST]                 = handle_vmptrst,
7169
	[EXIT_REASON_VMREAD]                  = handle_vmread,
7170
	[EXIT_REASON_VMRESUME]                = handle_vmresume,
7171
	[EXIT_REASON_VMWRITE]                 = handle_vmwrite,
7172 7173
	[EXIT_REASON_VMOFF]                   = handle_vmoff,
	[EXIT_REASON_VMON]                    = handle_vmon,
7174 7175
	[EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
	[EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
7176
	[EXIT_REASON_APIC_WRITE]              = handle_apic_write,
7177
	[EXIT_REASON_EOI_INDUCED]             = handle_apic_eoi_induced,
E
Eddie Dong 已提交
7178
	[EXIT_REASON_WBINVD]                  = handle_wbinvd,
7179
	[EXIT_REASON_XSETBV]                  = handle_xsetbv,
7180
	[EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
A
Andi Kleen 已提交
7181
	[EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
7182 7183
	[EXIT_REASON_EPT_VIOLATION]	      = handle_ept_violation,
	[EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
7184
	[EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
7185 7186
	[EXIT_REASON_MWAIT_INSTRUCTION]	      = handle_mwait,
	[EXIT_REASON_MONITOR_INSTRUCTION]     = handle_monitor,
N
Nadav Har'El 已提交
7187
	[EXIT_REASON_INVEPT]                  = handle_invept,
7188
	[EXIT_REASON_INVVPID]                 = handle_invvpid,
7189 7190
	[EXIT_REASON_XSAVES]                  = handle_xsaves,
	[EXIT_REASON_XRSTORS]                 = handle_xrstors,
K
Kai Huang 已提交
7191
	[EXIT_REASON_PML_FULL]		      = handle_pml_full,
A
Avi Kivity 已提交
7192 7193 7194
};

static const int kvm_vmx_max_exit_handlers =
7195
	ARRAY_SIZE(kvm_vmx_exit_handlers);
A
Avi Kivity 已提交
7196

7197 7198 7199 7200 7201 7202 7203 7204 7205 7206
static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
				       struct vmcs12 *vmcs12)
{
	unsigned long exit_qualification;
	gpa_t bitmap, last_bitmap;
	unsigned int port;
	int size;
	u8 b;

	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
7207
		return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239

	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);

	port = exit_qualification >> 16;
	size = (exit_qualification & 7) + 1;

	last_bitmap = (gpa_t)-1;
	b = -1;

	while (size > 0) {
		if (port < 0x8000)
			bitmap = vmcs12->io_bitmap_a;
		else if (port < 0x10000)
			bitmap = vmcs12->io_bitmap_b;
		else
			return 1;
		bitmap += (port & 0x7fff) / 8;

		if (last_bitmap != bitmap)
			if (kvm_read_guest(vcpu->kvm, bitmap, &b, 1))
				return 1;
		if (b & (1 << (port & 7)))
			return 1;

		port++;
		size--;
		last_bitmap = bitmap;
	}

	return 0;
}

7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251
/*
 * Return 1 if we should exit from L2 to L1 to handle an MSR access access,
 * rather than handle it ourselves in L0. I.e., check whether L1 expressed
 * disinterest in the current event (read or write a specific MSR) by using an
 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
 */
static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
	struct vmcs12 *vmcs12, u32 exit_reason)
{
	u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX];
	gpa_t bitmap;

7252
	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270
		return 1;

	/*
	 * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
	 * for the four combinations of read/write and low/high MSR numbers.
	 * First we need to figure out which of the four to use:
	 */
	bitmap = vmcs12->msr_bitmap;
	if (exit_reason == EXIT_REASON_MSR_WRITE)
		bitmap += 2048;
	if (msr_index >= 0xc0000000) {
		msr_index -= 0xc0000000;
		bitmap += 1024;
	}

	/* Then read the msr_index'th bit from this bitmap: */
	if (msr_index < 1024*8) {
		unsigned char b;
7271 7272
		if (kvm_read_guest(vcpu->kvm, bitmap + msr_index/8, &b, 1))
			return 1;
7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288
		return 1 & (b >> (msr_index & 7));
	} else
		return 1; /* let L1 handle the wrong parameter */
}

/*
 * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
 * rather than handle it ourselves in L0. I.e., check if L1 wanted to
 * intercept (via guest_host_mask etc.) the current event.
 */
static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
	struct vmcs12 *vmcs12)
{
	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	int cr = exit_qualification & 15;
	int reg = (exit_qualification >> 8) & 15;
7289
	unsigned long val = kvm_register_readl(vcpu, reg);
7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368

	switch ((exit_qualification >> 4) & 3) {
	case 0: /* mov to cr */
		switch (cr) {
		case 0:
			if (vmcs12->cr0_guest_host_mask &
			    (val ^ vmcs12->cr0_read_shadow))
				return 1;
			break;
		case 3:
			if ((vmcs12->cr3_target_count >= 1 &&
					vmcs12->cr3_target_value0 == val) ||
				(vmcs12->cr3_target_count >= 2 &&
					vmcs12->cr3_target_value1 == val) ||
				(vmcs12->cr3_target_count >= 3 &&
					vmcs12->cr3_target_value2 == val) ||
				(vmcs12->cr3_target_count >= 4 &&
					vmcs12->cr3_target_value3 == val))
				return 0;
			if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
				return 1;
			break;
		case 4:
			if (vmcs12->cr4_guest_host_mask &
			    (vmcs12->cr4_read_shadow ^ val))
				return 1;
			break;
		case 8:
			if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
				return 1;
			break;
		}
		break;
	case 2: /* clts */
		if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
		    (vmcs12->cr0_read_shadow & X86_CR0_TS))
			return 1;
		break;
	case 1: /* mov from cr */
		switch (cr) {
		case 3:
			if (vmcs12->cpu_based_vm_exec_control &
			    CPU_BASED_CR3_STORE_EXITING)
				return 1;
			break;
		case 8:
			if (vmcs12->cpu_based_vm_exec_control &
			    CPU_BASED_CR8_STORE_EXITING)
				return 1;
			break;
		}
		break;
	case 3: /* lmsw */
		/*
		 * lmsw can change bits 1..3 of cr0, and only set bit 0 of
		 * cr0. Other attempted changes are ignored, with no exit.
		 */
		if (vmcs12->cr0_guest_host_mask & 0xe &
		    (val ^ vmcs12->cr0_read_shadow))
			return 1;
		if ((vmcs12->cr0_guest_host_mask & 0x1) &&
		    !(vmcs12->cr0_read_shadow & 0x1) &&
		    (val & 0x1))
			return 1;
		break;
	}
	return 0;
}

/*
 * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we
 * should handle it ourselves in L0 (and then continue L2). Only call this
 * when in is_guest_mode (L2).
 */
static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
{
	u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
J
Jan Kiszka 已提交
7369
	u32 exit_reason = vmx->exit_reason;
7370

7371 7372 7373 7374 7375 7376 7377
	trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason,
				vmcs_readl(EXIT_QUALIFICATION),
				vmx->idt_vectoring_info,
				intr_info,
				vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
				KVM_ISA_VMX);

7378 7379 7380 7381
	if (vmx->nested.nested_run_pending)
		return 0;

	if (unlikely(vmx->fail)) {
7382 7383
		pr_info_ratelimited("%s failed vm entry %x\n", __func__,
				    vmcs_read32(VM_INSTRUCTION_ERROR));
7384 7385 7386 7387 7388 7389 7390 7391 7392
		return 1;
	}

	switch (exit_reason) {
	case EXIT_REASON_EXCEPTION_NMI:
		if (!is_exception(intr_info))
			return 0;
		else if (is_page_fault(intr_info))
			return enable_ept;
7393
		else if (is_no_device(intr_info) &&
7394
			 !(vmcs12->guest_cr0 & X86_CR0_TS))
7395
			return 0;
7396 7397 7398 7399 7400 7401 7402
		return vmcs12->exception_bitmap &
				(1u << (intr_info & INTR_INFO_VECTOR_MASK));
	case EXIT_REASON_EXTERNAL_INTERRUPT:
		return 0;
	case EXIT_REASON_TRIPLE_FAULT:
		return 1;
	case EXIT_REASON_PENDING_INTERRUPT:
7403
		return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING);
7404
	case EXIT_REASON_NMI_WINDOW:
7405
		return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING);
7406 7407 7408
	case EXIT_REASON_TASK_SWITCH:
		return 1;
	case EXIT_REASON_CPUID:
7409 7410
		if (kvm_register_read(vcpu, VCPU_REGS_RAX) == 0xa)
			return 0;
7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426
		return 1;
	case EXIT_REASON_HLT:
		return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
	case EXIT_REASON_INVD:
		return 1;
	case EXIT_REASON_INVLPG:
		return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
	case EXIT_REASON_RDPMC:
		return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
	case EXIT_REASON_RDTSC:
		return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
	case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
	case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
	case EXIT_REASON_VMPTRST: case EXIT_REASON_VMREAD:
	case EXIT_REASON_VMRESUME: case EXIT_REASON_VMWRITE:
	case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
7427
	case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
7428 7429 7430 7431 7432 7433 7434 7435 7436 7437
		/*
		 * VMX instructions trap unconditionally. This allows L1 to
		 * emulate them for its L2 guest, i.e., allows 3-level nesting!
		 */
		return 1;
	case EXIT_REASON_CR_ACCESS:
		return nested_vmx_exit_handled_cr(vcpu, vmcs12);
	case EXIT_REASON_DR_ACCESS:
		return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
	case EXIT_REASON_IO_INSTRUCTION:
7438
		return nested_vmx_exit_handled_io(vcpu, vmcs12);
7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454
	case EXIT_REASON_MSR_READ:
	case EXIT_REASON_MSR_WRITE:
		return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
	case EXIT_REASON_INVALID_STATE:
		return 1;
	case EXIT_REASON_MWAIT_INSTRUCTION:
		return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
	case EXIT_REASON_MONITOR_INSTRUCTION:
		return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
	case EXIT_REASON_PAUSE_INSTRUCTION:
		return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
			nested_cpu_has2(vmcs12,
				SECONDARY_EXEC_PAUSE_LOOP_EXITING);
	case EXIT_REASON_MCE_DURING_VMENTRY:
		return 0;
	case EXIT_REASON_TPR_BELOW_THRESHOLD:
7455
		return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
7456 7457 7458
	case EXIT_REASON_APIC_ACCESS:
		return nested_cpu_has2(vmcs12,
			SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
7459
	case EXIT_REASON_APIC_WRITE:
7460 7461
	case EXIT_REASON_EOI_INDUCED:
		/* apic_write and eoi_induced should exit unconditionally. */
7462
		return 1;
7463
	case EXIT_REASON_EPT_VIOLATION:
N
Nadav Har'El 已提交
7464 7465 7466 7467 7468 7469 7470
		/*
		 * L0 always deals with the EPT violation. If nested EPT is
		 * used, and the nested mmu code discovers that the address is
		 * missing in the guest EPT table (EPT12), the EPT violation
		 * will be injected with nested_ept_inject_page_fault()
		 */
		return 0;
7471
	case EXIT_REASON_EPT_MISCONFIG:
N
Nadav Har'El 已提交
7472 7473 7474 7475 7476 7477
		/*
		 * L2 never uses directly L1's EPT, but rather L0's own EPT
		 * table (shadow on EPT) or a merged EPT table that L0 built
		 * (EPT on EPT). So any problems with the structure of the
		 * table is L0's fault.
		 */
7478 7479 7480 7481 7482
		return 0;
	case EXIT_REASON_WBINVD:
		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
	case EXIT_REASON_XSETBV:
		return 1;
7483 7484 7485 7486 7487 7488 7489 7490
	case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
		/*
		 * This should never happen, since it is not possible to
		 * set XSS to a non-zero value---neither in L1 nor in L2.
		 * If if it were, XSS would have to be checked against
		 * the XSS exit bitmap in vmcs12.
		 */
		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
7491 7492 7493 7494 7495
	default:
		return 1;
	}
}

7496 7497 7498 7499 7500 7501
static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
{
	*info1 = vmcs_readl(EXIT_QUALIFICATION);
	*info2 = vmcs_read32(VM_EXIT_INTR_INFO);
}

K
Kai Huang 已提交
7502 7503 7504 7505 7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584
static int vmx_enable_pml(struct vcpu_vmx *vmx)
{
	struct page *pml_pg;
	u32 exec_control;

	pml_pg = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!pml_pg)
		return -ENOMEM;

	vmx->pml_pg = pml_pg;

	vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
	vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);

	exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
	exec_control |= SECONDARY_EXEC_ENABLE_PML;
	vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);

	return 0;
}

static void vmx_disable_pml(struct vcpu_vmx *vmx)
{
	u32 exec_control;

	ASSERT(vmx->pml_pg);
	__free_page(vmx->pml_pg);
	vmx->pml_pg = NULL;

	exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
	exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
	vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
}

static void vmx_flush_pml_buffer(struct vcpu_vmx *vmx)
{
	struct kvm *kvm = vmx->vcpu.kvm;
	u64 *pml_buf;
	u16 pml_idx;

	pml_idx = vmcs_read16(GUEST_PML_INDEX);

	/* Do nothing if PML buffer is empty */
	if (pml_idx == (PML_ENTITY_NUM - 1))
		return;

	/* PML index always points to next available PML buffer entity */
	if (pml_idx >= PML_ENTITY_NUM)
		pml_idx = 0;
	else
		pml_idx++;

	pml_buf = page_address(vmx->pml_pg);
	for (; pml_idx < PML_ENTITY_NUM; pml_idx++) {
		u64 gpa;

		gpa = pml_buf[pml_idx];
		WARN_ON(gpa & (PAGE_SIZE - 1));
		mark_page_dirty(kvm, gpa >> PAGE_SHIFT);
	}

	/* reset PML index */
	vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
}

/*
 * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap.
 * Called before reporting dirty_bitmap to userspace.
 */
static void kvm_flush_pml_buffers(struct kvm *kvm)
{
	int i;
	struct kvm_vcpu *vcpu;
	/*
	 * We only need to kick vcpu out of guest mode here, as PML buffer
	 * is flushed at beginning of all VMEXITs, and it's obvious that only
	 * vcpus running in guest are possible to have unflushed GPAs in PML
	 * buffer.
	 */
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_vcpu_kick(vcpu);
}

A
Avi Kivity 已提交
7585 7586 7587 7588
/*
 * The guest has exited.  See if we can fix it or if we need userspace
 * assistance.
 */
A
Avi Kivity 已提交
7589
static int vmx_handle_exit(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
7590
{
7591
	struct vcpu_vmx *vmx = to_vmx(vcpu);
A
Andi Kleen 已提交
7592
	u32 exit_reason = vmx->exit_reason;
7593
	u32 vectoring_info = vmx->idt_vectoring_info;
7594

K
Kai Huang 已提交
7595 7596 7597 7598 7599 7600 7601 7602 7603 7604
	/*
	 * Flush logged GPAs PML buffer, this will make dirty_bitmap more
	 * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before
	 * querying dirty_bitmap, we only need to kick all vcpus out of guest
	 * mode as if vcpus is in root mode, the PML buffer must has been
	 * flushed already.
	 */
	if (enable_pml)
		vmx_flush_pml_buffer(vmx);

7605
	/* If guest state is invalid, start emulating */
7606
	if (vmx->emulation_required)
7607
		return handle_invalid_guest_state(vcpu);
7608

7609
	if (is_guest_mode(vcpu) && nested_vmx_exit_handled(vcpu)) {
7610 7611 7612
		nested_vmx_vmexit(vcpu, exit_reason,
				  vmcs_read32(VM_EXIT_INTR_INFO),
				  vmcs_readl(EXIT_QUALIFICATION));
7613 7614 7615
		return 1;
	}

7616 7617 7618 7619 7620 7621 7622
	if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		vcpu->run->fail_entry.hardware_entry_failure_reason
			= exit_reason;
		return 0;
	}

7623
	if (unlikely(vmx->fail)) {
A
Avi Kivity 已提交
7624 7625
		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		vcpu->run->fail_entry.hardware_entry_failure_reason
7626 7627 7628
			= vmcs_read32(VM_INSTRUCTION_ERROR);
		return 0;
	}
A
Avi Kivity 已提交
7629

7630 7631 7632 7633 7634 7635 7636
	/*
	 * Note:
	 * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
	 * delivery event since it indicates guest is accessing MMIO.
	 * The vm-exit can be triggered again after return to guest that
	 * will cause infinite loop.
	 */
M
Mike Day 已提交
7637
	if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
7638
			(exit_reason != EXIT_REASON_EXCEPTION_NMI &&
J
Jan Kiszka 已提交
7639
			exit_reason != EXIT_REASON_EPT_VIOLATION &&
7640 7641 7642 7643 7644 7645 7646 7647
			exit_reason != EXIT_REASON_TASK_SWITCH)) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
		vcpu->run->internal.ndata = 2;
		vcpu->run->internal.data[0] = vectoring_info;
		vcpu->run->internal.data[1] = exit_reason;
		return 0;
	}
7648

7649 7650
	if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked &&
	    !(is_guest_mode(vcpu) && nested_cpu_has_virtual_nmis(
N
Nadav Har'El 已提交
7651
					get_vmcs12(vcpu))))) {
7652
		if (vmx_interrupt_allowed(vcpu)) {
7653 7654
			vmx->soft_vnmi_blocked = 0;
		} else if (vmx->vnmi_blocked_time > 1000000000LL &&
7655
			   vcpu->arch.nmi_pending) {
7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668
			/*
			 * This CPU don't support us in finding the end of an
			 * NMI-blocked window if the guest runs with IRQs
			 * disabled. So we pull the trigger after 1 s of
			 * futile waiting, but inform the user about this.
			 */
			printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
			       "state on VCPU %d after 1 s timeout\n",
			       __func__, vcpu->vcpu_id);
			vmx->soft_vnmi_blocked = 0;
		}
	}

A
Avi Kivity 已提交
7669 7670
	if (exit_reason < kvm_vmx_max_exit_handlers
	    && kvm_vmx_exit_handlers[exit_reason])
A
Avi Kivity 已提交
7671
		return kvm_vmx_exit_handlers[exit_reason](vcpu);
A
Avi Kivity 已提交
7672
	else {
7673 7674 7675
		WARN_ONCE(1, "vmx: unexpected exit reason 0x%x\n", exit_reason);
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
A
Avi Kivity 已提交
7676 7677 7678
	}
}

7679
static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
7680
{
7681 7682 7683 7684 7685 7686
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

	if (is_guest_mode(vcpu) &&
		nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
		return;

7687
	if (irr == -1 || tpr < irr) {
7688 7689 7690 7691
		vmcs_write32(TPR_THRESHOLD, 0);
		return;
	}

7692
	vmcs_write32(TPR_THRESHOLD, irr);
7693 7694
}

7695 7696 7697 7698 7699 7700 7701 7702
static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
{
	u32 sec_exec_control;

	/*
	 * There is not point to enable virtualize x2apic without enable
	 * apicv
	 */
7703 7704
	if (!cpu_has_vmx_virtualize_x2apic_mode() ||
				!vmx_vm_has_apicv(vcpu->kvm))
7705 7706 7707 7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723
		return;

	if (!vm_need_tpr_shadow(vcpu->kvm))
		return;

	sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);

	if (set) {
		sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
		sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
	} else {
		sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
		sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
	}
	vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);

	vmx_set_msr_bitmap(vcpu);
}

7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738 7739 7740 7741 7742 7743 7744 7745 7746
static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	/*
	 * Currently we do not handle the nested case where L2 has an
	 * APIC access page of its own; that page is still pinned.
	 * Hence, we skip the case where the VCPU is in guest mode _and_
	 * L1 prepared an APIC access page for L2.
	 *
	 * For the case where L1 and L2 share the same APIC access page
	 * (flexpriority=Y but SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES clear
	 * in the vmcs12), this function will only update either the vmcs01
	 * or the vmcs02.  If the former, the vmcs02 will be updated by
	 * prepare_vmcs02.  If the latter, the vmcs01 will be updated in
	 * the next L2->L1 exit.
	 */
	if (!is_guest_mode(vcpu) ||
	    !nested_cpu_has2(vmx->nested.current_vmcs12,
			     SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
		vmcs_write64(APIC_ACCESS_ADDR, hpa);
}

7747 7748 7749 7750 7751 7752 7753 7754 7755 7756 7757 7758 7759 7760 7761 7762 7763 7764 7765 7766 7767 7768
static void vmx_hwapic_isr_update(struct kvm *kvm, int isr)
{
	u16 status;
	u8 old;

	if (isr == -1)
		isr = 0;

	status = vmcs_read16(GUEST_INTR_STATUS);
	old = status >> 8;
	if (isr != old) {
		status &= 0xff;
		status |= isr << 8;
		vmcs_write16(GUEST_INTR_STATUS, status);
	}
}

static void vmx_set_rvi(int vector)
{
	u16 status;
	u8 old;

W
Wei Wang 已提交
7769 7770 7771
	if (vector == -1)
		vector = 0;

7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782
	status = vmcs_read16(GUEST_INTR_STATUS);
	old = (u8)status & 0xff;
	if ((u8)vector != old) {
		status &= ~0xff;
		status |= (u8)vector;
		vmcs_write16(GUEST_INTR_STATUS, status);
	}
}

static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
{
W
Wei Wang 已提交
7783 7784 7785 7786 7787
	if (!is_guest_mode(vcpu)) {
		vmx_set_rvi(max_irr);
		return;
	}

7788 7789 7790
	if (max_irr == -1)
		return;

7791
	/*
W
Wei Wang 已提交
7792 7793
	 * In guest mode.  If a vmexit is needed, vmx_check_nested_events
	 * handles it.
7794
	 */
W
Wei Wang 已提交
7795
	if (nested_exit_on_intr(vcpu))
7796 7797 7798
		return;

	/*
W
Wei Wang 已提交
7799
	 * Else, fall back to pre-APICv interrupt injection since L2
7800 7801 7802 7803 7804 7805 7806
	 * is run without virtual interrupt delivery.
	 */
	if (!kvm_event_needs_reinjection(vcpu) &&
	    vmx_interrupt_allowed(vcpu)) {
		kvm_queue_interrupt(vcpu, max_irr, false);
		vmx_inject_irq(vcpu);
	}
7807 7808 7809 7810
}

static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
{
7811 7812 7813
	if (!vmx_vm_has_apicv(vcpu->kvm))
		return;

7814 7815 7816 7817 7818 7819
	vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
	vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
	vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
	vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
}

7820
static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
7821
{
7822 7823 7824 7825 7826 7827
	u32 exit_intr_info;

	if (!(vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY
	      || vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI))
		return;

7828
	vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
7829
	exit_intr_info = vmx->exit_intr_info;
A
Andi Kleen 已提交
7830 7831

	/* Handle machine checks before interrupts are enabled */
7832
	if (is_machine_check(exit_intr_info))
A
Andi Kleen 已提交
7833 7834
		kvm_machine_check();

7835
	/* We need to handle NMIs before interrupts are enabled */
7836
	if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
7837 7838
	    (exit_intr_info & INTR_INFO_VALID_MASK)) {
		kvm_before_handle_nmi(&vmx->vcpu);
7839
		asm("int $2");
7840 7841
		kvm_after_handle_nmi(&vmx->vcpu);
	}
7842
}
7843

7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888 7889
static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
{
	u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);

	/*
	 * If external interrupt exists, IF bit is set in rflags/eflags on the
	 * interrupt stack frame, and interrupt will be enabled on a return
	 * from interrupt handler.
	 */
	if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_INTR_TYPE_MASK))
			== (INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR)) {
		unsigned int vector;
		unsigned long entry;
		gate_desc *desc;
		struct vcpu_vmx *vmx = to_vmx(vcpu);
#ifdef CONFIG_X86_64
		unsigned long tmp;
#endif

		vector =  exit_intr_info & INTR_INFO_VECTOR_MASK;
		desc = (gate_desc *)vmx->host_idt_base + vector;
		entry = gate_offset(*desc);
		asm volatile(
#ifdef CONFIG_X86_64
			"mov %%" _ASM_SP ", %[sp]\n\t"
			"and $0xfffffffffffffff0, %%" _ASM_SP "\n\t"
			"push $%c[ss]\n\t"
			"push %[sp]\n\t"
#endif
			"pushf\n\t"
			"orl $0x200, (%%" _ASM_SP ")\n\t"
			__ASM_SIZE(push) " $%c[cs]\n\t"
			"call *%[entry]\n\t"
			:
#ifdef CONFIG_X86_64
			[sp]"=&r"(tmp)
#endif
			:
			[entry]"r"(entry),
			[ss]"i"(__KERNEL_DS),
			[cs]"i"(__KERNEL_CS)
			);
	} else
		local_irq_enable();
}

7890 7891 7892 7893 7894 7895
static bool vmx_mpx_supported(void)
{
	return (vmcs_config.vmexit_ctrl & VM_EXIT_CLEAR_BNDCFGS) &&
		(vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_BNDCFGS);
}

7896 7897 7898 7899 7900 7901
static bool vmx_xsaves_supported(void)
{
	return vmcs_config.cpu_based_2nd_exec_ctrl &
		SECONDARY_EXEC_XSAVES;
}

7902 7903
static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
{
7904
	u32 exit_intr_info;
7905 7906 7907 7908 7909
	bool unblock_nmi;
	u8 vector;
	bool idtv_info_valid;

	idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
7910

7911
	if (cpu_has_virtual_nmis()) {
7912 7913
		if (vmx->nmi_known_unmasked)
			return;
7914 7915 7916 7917 7918
		/*
		 * Can't use vmx->exit_intr_info since we're not sure what
		 * the exit reason is.
		 */
		exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
7919 7920 7921
		unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
		vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
		/*
7922
		 * SDM 3: 27.7.1.2 (September 2008)
7923 7924
		 * Re-set bit "block by NMI" before VM entry if vmexit caused by
		 * a guest IRET fault.
7925 7926 7927 7928 7929
		 * SDM 3: 23.2.2 (September 2008)
		 * Bit 12 is undefined in any of the following cases:
		 *  If the VM exit sets the valid bit in the IDT-vectoring
		 *   information field.
		 *  If the VM exit is due to a double fault.
7930
		 */
7931 7932
		if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
		    vector != DF_VECTOR && !idtv_info_valid)
7933 7934
			vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
				      GUEST_INTR_STATE_NMI);
7935 7936 7937 7938
		else
			vmx->nmi_known_unmasked =
				!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
				  & GUEST_INTR_STATE_NMI);
7939 7940 7941
	} else if (unlikely(vmx->soft_vnmi_blocked))
		vmx->vnmi_blocked_time +=
			ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
7942 7943
}

7944
static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
7945 7946 7947
				      u32 idt_vectoring_info,
				      int instr_len_field,
				      int error_code_field)
7948 7949 7950 7951 7952 7953
{
	u8 vector;
	int type;
	bool idtv_info_valid;

	idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
7954

7955 7956 7957
	vcpu->arch.nmi_injected = false;
	kvm_clear_exception_queue(vcpu);
	kvm_clear_interrupt_queue(vcpu);
7958 7959 7960 7961

	if (!idtv_info_valid)
		return;

7962
	kvm_make_request(KVM_REQ_EVENT, vcpu);
7963

7964 7965
	vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
	type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
7966

7967
	switch (type) {
7968
	case INTR_TYPE_NMI_INTR:
7969
		vcpu->arch.nmi_injected = true;
7970
		/*
7971
		 * SDM 3: 27.7.1.2 (September 2008)
7972 7973
		 * Clear bit "block by NMI" before VM entry if a NMI
		 * delivery faulted.
7974
		 */
7975
		vmx_set_nmi_mask(vcpu, false);
7976 7977
		break;
	case INTR_TYPE_SOFT_EXCEPTION:
7978
		vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
7979 7980
		/* fall through */
	case INTR_TYPE_HARD_EXCEPTION:
7981
		if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
7982
			u32 err = vmcs_read32(error_code_field);
7983
			kvm_requeue_exception_e(vcpu, vector, err);
7984
		} else
7985
			kvm_requeue_exception(vcpu, vector);
7986
		break;
7987
	case INTR_TYPE_SOFT_INTR:
7988
		vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
7989
		/* fall through */
7990
	case INTR_TYPE_EXT_INTR:
7991
		kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR);
7992 7993 7994
		break;
	default:
		break;
7995
	}
7996 7997
}

7998 7999
static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
{
8000
	__vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info,
8001 8002 8003 8004
				  VM_EXIT_INSTRUCTION_LEN,
				  IDT_VECTORING_ERROR_CODE);
}

A
Avi Kivity 已提交
8005 8006
static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
{
8007
	__vmx_complete_interrupts(vcpu,
A
Avi Kivity 已提交
8008 8009 8010 8011 8012 8013 8014
				  vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
				  VM_ENTRY_INSTRUCTION_LEN,
				  VM_ENTRY_EXCEPTION_ERROR_CODE);

	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
}

8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032
static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
{
	int i, nr_msrs;
	struct perf_guest_switch_msr *msrs;

	msrs = perf_guest_get_msrs(&nr_msrs);

	if (!msrs)
		return;

	for (i = 0; i < nr_msrs; i++)
		if (msrs[i].host == msrs[i].guest)
			clear_atomic_switch_msr(vmx, msrs[i].msr);
		else
			add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
					msrs[i].host);
}

8033
static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
8034
{
8035
	struct vcpu_vmx *vmx = to_vmx(vcpu);
8036
	unsigned long debugctlmsr, cr4;
8037 8038 8039 8040 8041 8042 8043

	/* Record the guest's net vcpu time for enforced NMI injections. */
	if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
		vmx->entry_time = ktime_get();

	/* Don't enter VMX if guest state is invalid, let the exit handler
	   start emulation until we arrive back to a valid state */
8044
	if (vmx->emulation_required)
8045 8046
		return;

8047 8048 8049 8050 8051
	if (vmx->ple_window_dirty) {
		vmx->ple_window_dirty = false;
		vmcs_write32(PLE_WINDOW, vmx->ple_window);
	}

8052 8053 8054 8055 8056
	if (vmx->nested.sync_shadow_vmcs) {
		copy_vmcs12_to_shadow(vmx);
		vmx->nested.sync_shadow_vmcs = false;
	}

8057 8058 8059 8060 8061
	if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
		vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
	if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
		vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);

8062 8063 8064 8065 8066 8067
	cr4 = read_cr4();
	if (unlikely(cr4 != vmx->host_state.vmcs_host_cr4)) {
		vmcs_writel(HOST_CR4, cr4);
		vmx->host_state.vmcs_host_cr4 = cr4;
	}

8068 8069 8070 8071 8072 8073 8074 8075
	/* When single-stepping over STI and MOV SS, we must clear the
	 * corresponding interruptibility bits in the guest state. Otherwise
	 * vmentry fails as it then expects bit 14 (BS) in pending debug
	 * exceptions being set, but that's not correct for the guest debugging
	 * case. */
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vmx_set_interrupt_shadow(vcpu, 0);

8076
	atomic_switch_perf_msrs(vmx);
8077
	debugctlmsr = get_debugctlmsr();
8078

8079
	vmx->__launched = vmx->loaded_vmcs->launched;
8080
	asm(
A
Avi Kivity 已提交
8081
		/* Store host registers */
A
Avi Kivity 已提交
8082 8083 8084 8085
		"push %%" _ASM_DX "; push %%" _ASM_BP ";"
		"push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */
		"push %%" _ASM_CX " \n\t"
		"cmp %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
8086
		"je 1f \n\t"
A
Avi Kivity 已提交
8087
		"mov %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
8088
		__ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
8089
		"1: \n\t"
8090
		/* Reload cr2 if changed */
A
Avi Kivity 已提交
8091 8092 8093
		"mov %c[cr2](%0), %%" _ASM_AX " \n\t"
		"mov %%cr2, %%" _ASM_DX " \n\t"
		"cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t"
8094
		"je 2f \n\t"
A
Avi Kivity 已提交
8095
		"mov %%" _ASM_AX", %%cr2 \n\t"
8096
		"2: \n\t"
A
Avi Kivity 已提交
8097
		/* Check if vmlaunch of vmresume is needed */
8098
		"cmpl $0, %c[launched](%0) \n\t"
A
Avi Kivity 已提交
8099
		/* Load guest registers.  Don't clobber flags. */
A
Avi Kivity 已提交
8100 8101 8102 8103 8104 8105
		"mov %c[rax](%0), %%" _ASM_AX " \n\t"
		"mov %c[rbx](%0), %%" _ASM_BX " \n\t"
		"mov %c[rdx](%0), %%" _ASM_DX " \n\t"
		"mov %c[rsi](%0), %%" _ASM_SI " \n\t"
		"mov %c[rdi](%0), %%" _ASM_DI " \n\t"
		"mov %c[rbp](%0), %%" _ASM_BP " \n\t"
8106
#ifdef CONFIG_X86_64
8107 8108 8109 8110 8111 8112 8113 8114
		"mov %c[r8](%0),  %%r8  \n\t"
		"mov %c[r9](%0),  %%r9  \n\t"
		"mov %c[r10](%0), %%r10 \n\t"
		"mov %c[r11](%0), %%r11 \n\t"
		"mov %c[r12](%0), %%r12 \n\t"
		"mov %c[r13](%0), %%r13 \n\t"
		"mov %c[r14](%0), %%r14 \n\t"
		"mov %c[r15](%0), %%r15 \n\t"
A
Avi Kivity 已提交
8115
#endif
A
Avi Kivity 已提交
8116
		"mov %c[rcx](%0), %%" _ASM_CX " \n\t" /* kills %0 (ecx) */
8117

A
Avi Kivity 已提交
8118
		/* Enter guest mode */
A
Avi Kivity 已提交
8119
		"jne 1f \n\t"
8120
		__ex(ASM_VMX_VMLAUNCH) "\n\t"
A
Avi Kivity 已提交
8121 8122 8123
		"jmp 2f \n\t"
		"1: " __ex(ASM_VMX_VMRESUME) "\n\t"
		"2: "
A
Avi Kivity 已提交
8124
		/* Save guest registers, load host registers, keep flags */
A
Avi Kivity 已提交
8125
		"mov %0, %c[wordsize](%%" _ASM_SP ") \n\t"
8126
		"pop %0 \n\t"
A
Avi Kivity 已提交
8127 8128 8129 8130 8131 8132 8133
		"mov %%" _ASM_AX ", %c[rax](%0) \n\t"
		"mov %%" _ASM_BX ", %c[rbx](%0) \n\t"
		__ASM_SIZE(pop) " %c[rcx](%0) \n\t"
		"mov %%" _ASM_DX ", %c[rdx](%0) \n\t"
		"mov %%" _ASM_SI ", %c[rsi](%0) \n\t"
		"mov %%" _ASM_DI ", %c[rdi](%0) \n\t"
		"mov %%" _ASM_BP ", %c[rbp](%0) \n\t"
8134
#ifdef CONFIG_X86_64
8135 8136 8137 8138 8139 8140 8141 8142
		"mov %%r8,  %c[r8](%0) \n\t"
		"mov %%r9,  %c[r9](%0) \n\t"
		"mov %%r10, %c[r10](%0) \n\t"
		"mov %%r11, %c[r11](%0) \n\t"
		"mov %%r12, %c[r12](%0) \n\t"
		"mov %%r13, %c[r13](%0) \n\t"
		"mov %%r14, %c[r14](%0) \n\t"
		"mov %%r15, %c[r15](%0) \n\t"
A
Avi Kivity 已提交
8143
#endif
A
Avi Kivity 已提交
8144 8145
		"mov %%cr2, %%" _ASM_AX "   \n\t"
		"mov %%" _ASM_AX ", %c[cr2](%0) \n\t"
8146

A
Avi Kivity 已提交
8147
		"pop  %%" _ASM_BP "; pop  %%" _ASM_DX " \n\t"
8148
		"setbe %c[fail](%0) \n\t"
A
Avi Kivity 已提交
8149 8150 8151 8152
		".pushsection .rodata \n\t"
		".global vmx_return \n\t"
		"vmx_return: " _ASM_PTR " 2b \n\t"
		".popsection"
8153
	      : : "c"(vmx), "d"((unsigned long)HOST_RSP),
8154
		[launched]"i"(offsetof(struct vcpu_vmx, __launched)),
8155
		[fail]"i"(offsetof(struct vcpu_vmx, fail)),
8156
		[host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
8157 8158 8159 8160 8161 8162 8163
		[rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
		[rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
		[rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
		[rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
		[rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
		[rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
		[rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
8164
#ifdef CONFIG_X86_64
8165 8166 8167 8168 8169 8170 8171 8172
		[r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
		[r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
		[r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
		[r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
		[r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
		[r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
		[r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
		[r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
A
Avi Kivity 已提交
8173
#endif
8174 8175
		[cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
		[wordsize]"i"(sizeof(ulong))
8176 8177
	      : "cc", "memory"
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
8178
		, "rax", "rbx", "rdi", "rsi"
8179
		, "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
A
Avi Kivity 已提交
8180 8181
#else
		, "eax", "ebx", "edi", "esi"
8182 8183
#endif
	      );
A
Avi Kivity 已提交
8184

8185 8186 8187 8188
	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
	if (debugctlmsr)
		update_debugctlmsr(debugctlmsr);

8189 8190 8191 8192 8193 8194 8195 8196 8197 8198 8199 8200 8201
#ifndef CONFIG_X86_64
	/*
	 * The sysexit path does not restore ds/es, so we must set them to
	 * a reasonable value ourselves.
	 *
	 * We can't defer this to vmx_load_host_state() since that function
	 * may be executed in interrupt context, which saves and restore segments
	 * around it, nullifying its effect.
	 */
	loadsegment(ds, __USER_DS);
	loadsegment(es, __USER_DS);
#endif

A
Avi Kivity 已提交
8202
	vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
A
Avi Kivity 已提交
8203
				  | (1 << VCPU_EXREG_RFLAGS)
8204
				  | (1 << VCPU_EXREG_PDPTR)
A
Avi Kivity 已提交
8205
				  | (1 << VCPU_EXREG_SEGMENTS)
8206
				  | (1 << VCPU_EXREG_CR3));
8207 8208
	vcpu->arch.regs_dirty = 0;

8209 8210
	vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);

8211
	vmx->loaded_vmcs->launched = 1;
8212

8213
	vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
8214
	trace_kvm_exit(vmx->exit_reason, vcpu, KVM_ISA_VMX);
8215

8216 8217 8218 8219 8220 8221 8222 8223 8224 8225
	/*
	 * the KVM_REQ_EVENT optimization bit is only on for one entry, and if
	 * we did not inject a still-pending event to L1 now because of
	 * nested_run_pending, we need to re-enable this bit.
	 */
	if (vmx->nested.nested_run_pending)
		kvm_make_request(KVM_REQ_EVENT, vcpu);

	vmx->nested.nested_run_pending = 0;

8226 8227
	vmx_complete_atomic_exit(vmx);
	vmx_recover_nmi_blocking(vmx);
8228
	vmx_complete_interrupts(vmx);
A
Avi Kivity 已提交
8229 8230
}

8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241 8242 8243 8244 8245 8246
static void vmx_load_vmcs01(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int cpu;

	if (vmx->loaded_vmcs == &vmx->vmcs01)
		return;

	cpu = get_cpu();
	vmx->loaded_vmcs = &vmx->vmcs01;
	vmx_vcpu_put(vcpu);
	vmx_vcpu_load(vcpu, cpu);
	vcpu->cpu = cpu;
	put_cpu();
}

A
Avi Kivity 已提交
8247 8248
static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
8249 8250
	struct vcpu_vmx *vmx = to_vmx(vcpu);

K
Kai Huang 已提交
8251 8252
	if (enable_pml)
		vmx_disable_pml(vmx);
8253
	free_vpid(vmx);
8254 8255
	leave_guest_mode(vcpu);
	vmx_load_vmcs01(vcpu);
8256
	free_nested(vmx);
8257
	free_loaded_vmcs(vmx->loaded_vmcs);
R
Rusty Russell 已提交
8258 8259
	kfree(vmx->guest_msrs);
	kvm_vcpu_uninit(vcpu);
8260
	kmem_cache_free(kvm_vcpu_cache, vmx);
A
Avi Kivity 已提交
8261 8262
}

R
Rusty Russell 已提交
8263
static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
A
Avi Kivity 已提交
8264
{
R
Rusty Russell 已提交
8265
	int err;
8266
	struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
8267
	int cpu;
A
Avi Kivity 已提交
8268

8269
	if (!vmx)
R
Rusty Russell 已提交
8270 8271
		return ERR_PTR(-ENOMEM);

8272 8273
	allocate_vpid(vmx);

R
Rusty Russell 已提交
8274 8275 8276
	err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
	if (err)
		goto free_vcpu;
8277

8278
	vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
8279 8280
	BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) * sizeof(vmx->guest_msrs[0])
		     > PAGE_SIZE);
8281

8282
	err = -ENOMEM;
R
Rusty Russell 已提交
8283 8284 8285
	if (!vmx->guest_msrs) {
		goto uninit_vcpu;
	}
8286

8287 8288 8289
	vmx->loaded_vmcs = &vmx->vmcs01;
	vmx->loaded_vmcs->vmcs = alloc_vmcs();
	if (!vmx->loaded_vmcs->vmcs)
R
Rusty Russell 已提交
8290
		goto free_msrs;
8291 8292 8293 8294 8295
	if (!vmm_exclusive)
		kvm_cpu_vmxon(__pa(per_cpu(vmxarea, raw_smp_processor_id())));
	loaded_vmcs_init(vmx->loaded_vmcs);
	if (!vmm_exclusive)
		kvm_cpu_vmxoff();
8296

8297 8298
	cpu = get_cpu();
	vmx_vcpu_load(&vmx->vcpu, cpu);
Z
Zachary Amsden 已提交
8299
	vmx->vcpu.cpu = cpu;
R
Rusty Russell 已提交
8300
	err = vmx_vcpu_setup(vmx);
R
Rusty Russell 已提交
8301
	vmx_vcpu_put(&vmx->vcpu);
8302
	put_cpu();
R
Rusty Russell 已提交
8303 8304
	if (err)
		goto free_vmcs;
8305
	if (vm_need_virtualize_apic_accesses(kvm)) {
8306 8307
		err = alloc_apic_access_page(kvm);
		if (err)
8308
			goto free_vmcs;
8309
	}
R
Rusty Russell 已提交
8310

8311 8312 8313 8314
	if (enable_ept) {
		if (!kvm->arch.ept_identity_map_addr)
			kvm->arch.ept_identity_map_addr =
				VMX_EPT_IDENTITY_PAGETABLE_ADDR;
8315 8316
		err = init_rmode_identity_map(kvm);
		if (err)
8317
			goto free_vmcs;
8318
	}
8319

8320 8321 8322
	if (nested)
		nested_vmx_setup_ctls_msrs(vmx);

8323 8324 8325
	vmx->nested.current_vmptr = -1ull;
	vmx->nested.current_vmcs12 = NULL;

K
Kai Huang 已提交
8326 8327 8328 8329 8330 8331 8332 8333 8334 8335 8336 8337
	/*
	 * If PML is turned on, failure on enabling PML just results in failure
	 * of creating the vcpu, therefore we can simplify PML logic (by
	 * avoiding dealing with cases, such as enabling PML partially on vcpus
	 * for the guest, etc.
	 */
	if (enable_pml) {
		err = vmx_enable_pml(vmx);
		if (err)
			goto free_vmcs;
	}

R
Rusty Russell 已提交
8338 8339 8340
	return &vmx->vcpu;

free_vmcs:
8341
	free_loaded_vmcs(vmx->loaded_vmcs);
R
Rusty Russell 已提交
8342 8343 8344 8345 8346
free_msrs:
	kfree(vmx->guest_msrs);
uninit_vcpu:
	kvm_vcpu_uninit(&vmx->vcpu);
free_vcpu:
8347
	free_vpid(vmx);
8348
	kmem_cache_free(kvm_vcpu_cache, vmx);
R
Rusty Russell 已提交
8349
	return ERR_PTR(err);
A
Avi Kivity 已提交
8350 8351
}

Y
Yang, Sheng 已提交
8352 8353 8354 8355 8356 8357 8358 8359 8360 8361 8362 8363 8364 8365
static void __init vmx_check_processor_compat(void *rtn)
{
	struct vmcs_config vmcs_conf;

	*(int *)rtn = 0;
	if (setup_vmcs_config(&vmcs_conf) < 0)
		*(int *)rtn = -EIO;
	if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
		printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
				smp_processor_id());
		*(int *)rtn = -EIO;
	}
}

8366 8367 8368 8369 8370
static int get_ept_level(void)
{
	return VMX_EPT_DEFAULT_GAW + 1;
}

8371
static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
S
Sheng Yang 已提交
8372
{
8373 8374
	u64 ret;

8375 8376 8377 8378 8379 8380 8381 8382
	/* For VT-d and EPT combination
	 * 1. MMIO: always map as UC
	 * 2. EPT with VT-d:
	 *   a. VT-d without snooping control feature: can't guarantee the
	 *	result, try to trust guest.
	 *   b. VT-d with snooping control feature: snooping control feature of
	 *	VT-d engine can guarantee the cache correctness. Just set it
	 *	to WB to keep consistent with host. So the same as item 3.
8383
	 * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
8384 8385
	 *    consistent with host MTRR
	 */
8386 8387
	if (is_mmio)
		ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
8388
	else if (kvm_arch_has_noncoherent_dma(vcpu->kvm))
8389 8390
		ret = kvm_get_guest_memory_type(vcpu, gfn) <<
		      VMX_EPT_MT_EPTE_SHIFT;
8391
	else
8392
		ret = (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT)
8393
			| VMX_EPT_IPAT_BIT;
8394 8395

	return ret;
S
Sheng Yang 已提交
8396 8397
}

8398
static int vmx_get_lpage_level(void)
8399
{
8400 8401 8402 8403 8404
	if (enable_ept && !cpu_has_vmx_ept_1g_page())
		return PT_DIRECTORY_LEVEL;
	else
		/* For shadow and EPT supported 1GB page */
		return PT_PDPE_LEVEL;
8405 8406
}

8407 8408
static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
{
8409 8410 8411 8412 8413 8414 8415 8416 8417 8418 8419 8420 8421 8422 8423 8424 8425 8426
	struct kvm_cpuid_entry2 *best;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u32 exec_control;

	vmx->rdtscp_enabled = false;
	if (vmx_rdtscp_supported()) {
		exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
		if (exec_control & SECONDARY_EXEC_RDTSCP) {
			best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
			if (best && (best->edx & bit(X86_FEATURE_RDTSCP)))
				vmx->rdtscp_enabled = true;
			else {
				exec_control &= ~SECONDARY_EXEC_RDTSCP;
				vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
						exec_control);
			}
		}
	}
8427 8428 8429 8430

	/* Exposing INVPCID only when PCID is exposed */
	best = kvm_find_cpuid_entry(vcpu, 0x7, 0);
	if (vmx_invpcid_supported() &&
8431
	    best && (best->ebx & bit(X86_FEATURE_INVPCID)) &&
8432
	    guest_cpuid_has_pcid(vcpu)) {
8433
		exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
8434 8435 8436 8437
		exec_control |= SECONDARY_EXEC_ENABLE_INVPCID;
		vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
			     exec_control);
	} else {
8438 8439 8440 8441 8442 8443
		if (cpu_has_secondary_exec_ctrls()) {
			exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
			exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
			vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
				     exec_control);
		}
8444
		if (best)
8445
			best->ebx &= ~bit(X86_FEATURE_INVPCID);
8446
	}
8447 8448
}

8449 8450
static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
{
8451 8452
	if (func == 1 && nested)
		entry->ecx |= bit(X86_FEATURE_VMX);
8453 8454
}

8455 8456 8457
static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
		struct x86_exception *fault)
{
8458 8459
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
	u32 exit_reason;
8460 8461

	if (fault->error_code & PFERR_RSVD_MASK)
8462
		exit_reason = EXIT_REASON_EPT_MISCONFIG;
8463
	else
8464 8465
		exit_reason = EXIT_REASON_EPT_VIOLATION;
	nested_vmx_vmexit(vcpu, exit_reason, 0, vcpu->arch.exit_qualification);
8466 8467 8468
	vmcs12->guest_physical_address = fault->address;
}

N
Nadav Har'El 已提交
8469 8470 8471 8472 8473 8474 8475 8476
/* Callbacks for nested_ept_init_mmu_context: */

static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu)
{
	/* return the page table to be shadowed - in our case, EPT12 */
	return get_vmcs12(vcpu)->ept_pointer;
}

8477
static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
N
Nadav Har'El 已提交
8478
{
8479 8480
	WARN_ON(mmu_is_nested(vcpu));
	kvm_init_shadow_ept_mmu(vcpu,
8481 8482
			to_vmx(vcpu)->nested.nested_vmx_ept_caps &
			VMX_EPT_EXECUTE_ONLY_BIT);
N
Nadav Har'El 已提交
8483 8484 8485 8486 8487 8488 8489 8490 8491 8492 8493 8494
	vcpu->arch.mmu.set_cr3           = vmx_set_cr3;
	vcpu->arch.mmu.get_cr3           = nested_ept_get_cr3;
	vcpu->arch.mmu.inject_page_fault = nested_ept_inject_page_fault;

	vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
}

static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
	vcpu->arch.walk_mmu = &vcpu->arch.mmu;
}

8495 8496 8497 8498 8499 8500 8501 8502 8503 8504 8505 8506
static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
					    u16 error_code)
{
	bool inequality, bit;

	bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
	inequality =
		(error_code & vmcs12->page_fault_error_code_mask) !=
		 vmcs12->page_fault_error_code_match;
	return inequality ^ bit;
}

8507 8508 8509 8510 8511 8512 8513
static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
		struct x86_exception *fault)
{
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

	WARN_ON(!is_guest_mode(vcpu));

8514
	if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code))
8515 8516 8517
		nested_vmx_vmexit(vcpu, to_vmx(vcpu)->exit_reason,
				  vmcs_read32(VM_EXIT_INTR_INFO),
				  vmcs_readl(EXIT_QUALIFICATION));
8518 8519 8520 8521
	else
		kvm_inject_page_fault(vcpu, fault);
}

8522 8523 8524 8525 8526 8527
static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
					struct vmcs12 *vmcs12)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
8528
		/* TODO: Also verify bits beyond physical address width are 0 */
8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542
		if (!PAGE_ALIGNED(vmcs12->apic_access_addr))
			return false;

		/*
		 * Translate L1 physical address to host physical
		 * address for vmcs02. Keep the page pinned, so this
		 * physical address remains valid. We keep a reference
		 * to it so we can release it later.
		 */
		if (vmx->nested.apic_access_page) /* shouldn't happen */
			nested_release_page(vmx->nested.apic_access_page);
		vmx->nested.apic_access_page =
			nested_get_page(vcpu, vmcs12->apic_access_addr);
	}
8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557 8558 8559 8560 8561 8562 8563 8564 8565 8566 8567

	if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
		/* TODO: Also verify bits beyond physical address width are 0 */
		if (!PAGE_ALIGNED(vmcs12->virtual_apic_page_addr))
			return false;

		if (vmx->nested.virtual_apic_page) /* shouldn't happen */
			nested_release_page(vmx->nested.virtual_apic_page);
		vmx->nested.virtual_apic_page =
			nested_get_page(vcpu, vmcs12->virtual_apic_page_addr);

		/*
		 * Failing the vm entry is _not_ what the processor does
		 * but it's basically the only possibility we have.
		 * We could still enter the guest if CR8 load exits are
		 * enabled, CR8 store exits are enabled, and virtualize APIC
		 * access is disabled; in this case the processor would never
		 * use the TPR shadow and we could simply clear the bit from
		 * the execution control.  But such a configuration is useless,
		 * so let's keep the code simple.
		 */
		if (!vmx->nested.virtual_apic_page)
			return false;
	}

8568 8569 8570
	return true;
}

8571 8572 8573 8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591 8592
static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu)
{
	u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value;
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (vcpu->arch.virtual_tsc_khz == 0)
		return;

	/* Make sure short timeouts reliably trigger an immediate vmexit.
	 * hrtimer_start does not guarantee this. */
	if (preemption_timeout <= 1) {
		vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
		return;
	}

	preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
	preemption_timeout *= 1000000;
	do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
	hrtimer_start(&vmx->nested.preemption_timer,
		      ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL);
}

8593 8594 8595 8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615 8616 8617 8618 8619 8620 8621
static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
						struct vmcs12 *vmcs12)
{
	int maxphyaddr;
	u64 addr;

	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
		return 0;

	if (vmcs12_read_any(vcpu, MSR_BITMAP, &addr)) {
		WARN_ON(1);
		return -EINVAL;
	}
	maxphyaddr = cpuid_maxphyaddr(vcpu);

	if (!PAGE_ALIGNED(vmcs12->msr_bitmap) ||
	   ((addr + PAGE_SIZE) >> maxphyaddr))
		return -EINVAL;

	return 0;
}

/*
 * Merge L0's and L1's MSR bitmap, return false to indicate that
 * we do not use the hardware.
 */
static inline bool nested_vmx_merge_msr_bitmap(struct kvm_vcpu *vcpu,
					       struct vmcs12 *vmcs12)
{
8622
	int msr;
8623 8624 8625 8626 8627 8628 8629 8630 8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641
	struct page *page;
	unsigned long *msr_bitmap;

	if (!nested_cpu_has_virt_x2apic_mode(vmcs12))
		return false;

	page = nested_get_page(vcpu, vmcs12->msr_bitmap);
	if (!page) {
		WARN_ON(1);
		return false;
	}
	msr_bitmap = (unsigned long *)kmap(page);
	if (!msr_bitmap) {
		nested_release_page_clean(page);
		WARN_ON(1);
		return false;
	}

	if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
8642 8643 8644 8645 8646 8647
		if (nested_cpu_has_apic_reg_virt(vmcs12))
			for (msr = 0x800; msr <= 0x8ff; msr++)
				nested_vmx_disable_intercept_for_msr(
					msr_bitmap,
					vmx_msr_bitmap_nested,
					msr, MSR_TYPE_R);
8648 8649 8650 8651 8652
		/* TPR is allowed */
		nested_vmx_disable_intercept_for_msr(msr_bitmap,
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_TASKPRI >> 4),
				MSR_TYPE_R | MSR_TYPE_W);
8653 8654 8655 8656 8657 8658 8659 8660 8661 8662 8663 8664 8665
		if (nested_cpu_has_vid(vmcs12)) {
			/* EOI and self-IPI are allowed */
			nested_vmx_disable_intercept_for_msr(
				msr_bitmap,
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_EOI >> 4),
				MSR_TYPE_W);
			nested_vmx_disable_intercept_for_msr(
				msr_bitmap,
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_SELF_IPI >> 4),
				MSR_TYPE_W);
		}
8666 8667 8668 8669 8670 8671 8672 8673 8674 8675 8676 8677 8678
	} else {
		/*
		 * Enable reading intercept of all the x2apic
		 * MSRs. We should not rely on vmcs12 to do any
		 * optimizations here, it may have been modified
		 * by L1.
		 */
		for (msr = 0x800; msr <= 0x8ff; msr++)
			__vmx_enable_intercept_for_msr(
				vmx_msr_bitmap_nested,
				msr,
				MSR_TYPE_R);

8679 8680 8681
		__vmx_enable_intercept_for_msr(
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_TASKPRI >> 4),
8682
				MSR_TYPE_W);
8683 8684 8685 8686 8687 8688 8689 8690
		__vmx_enable_intercept_for_msr(
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_EOI >> 4),
				MSR_TYPE_W);
		__vmx_enable_intercept_for_msr(
				vmx_msr_bitmap_nested,
				APIC_BASE_MSR + (APIC_SELF_IPI >> 4),
				MSR_TYPE_W);
8691
	}
8692 8693 8694 8695 8696 8697 8698 8699 8700
	kunmap(page);
	nested_release_page_clean(page);

	return true;
}

static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
					   struct vmcs12 *vmcs12)
{
8701
	if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
8702 8703
	    !nested_cpu_has_apic_reg_virt(vmcs12) &&
	    !nested_cpu_has_vid(vmcs12))
8704 8705 8706 8707 8708 8709
		return 0;

	/*
	 * If virtualize x2apic mode is enabled,
	 * virtualize apic access must be disabled.
	 */
8710 8711
	if (nested_cpu_has_virt_x2apic_mode(vmcs12) &&
	    nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
8712 8713
		return -EINVAL;

8714 8715 8716 8717 8718 8719 8720 8721
	/*
	 * If virtual interrupt delivery is enabled,
	 * we must exit on external interrupts.
	 */
	if (nested_cpu_has_vid(vmcs12) &&
	   !nested_exit_on_intr(vcpu))
		return -EINVAL;

8722 8723 8724 8725 8726
	/* tpr shadow is needed by all apicv features. */
	if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
		return -EINVAL;

	return 0;
8727 8728
}

8729 8730 8731 8732
static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
				       unsigned long count_field,
				       unsigned long addr_field,
				       int maxphyaddr)
8733
{
8734 8735 8736 8737 8738 8739 8740 8741 8742 8743 8744 8745 8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781 8782
	u64 count, addr;

	if (vmcs12_read_any(vcpu, count_field, &count) ||
	    vmcs12_read_any(vcpu, addr_field, &addr)) {
		WARN_ON(1);
		return -EINVAL;
	}
	if (count == 0)
		return 0;
	if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
	    (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) {
		pr_warn_ratelimited(
			"nVMX: invalid MSR switch (0x%lx, %d, %llu, 0x%08llx)",
			addr_field, maxphyaddr, count, addr);
		return -EINVAL;
	}
	return 0;
}

static int nested_vmx_check_msr_switch_controls(struct kvm_vcpu *vcpu,
						struct vmcs12 *vmcs12)
{
	int maxphyaddr;

	if (vmcs12->vm_exit_msr_load_count == 0 &&
	    vmcs12->vm_exit_msr_store_count == 0 &&
	    vmcs12->vm_entry_msr_load_count == 0)
		return 0; /* Fast path */
	maxphyaddr = cpuid_maxphyaddr(vcpu);
	if (nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_LOAD_COUNT,
					VM_EXIT_MSR_LOAD_ADDR, maxphyaddr) ||
	    nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_STORE_COUNT,
					VM_EXIT_MSR_STORE_ADDR, maxphyaddr) ||
	    nested_vmx_check_msr_switch(vcpu, VM_ENTRY_MSR_LOAD_COUNT,
					VM_ENTRY_MSR_LOAD_ADDR, maxphyaddr))
		return -EINVAL;
	return 0;
}

static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
				       struct vmx_msr_entry *e)
{
	/* x2APIC MSR accesses are not allowed */
	if (apic_x2apic_mode(vcpu->arch.apic) && e->index >> 8 == 0x8)
		return -EINVAL;
	if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */
	    e->index == MSR_IA32_UCODE_REV)
		return -EINVAL;
	if (e->reserved != 0)
8783 8784 8785 8786
		return -EINVAL;
	return 0;
}

8787 8788
static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
				     struct vmx_msr_entry *e)
8789 8790 8791
{
	if (e->index == MSR_FS_BASE ||
	    e->index == MSR_GS_BASE ||
8792 8793 8794 8795 8796 8797 8798 8799 8800 8801 8802
	    e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */
	    nested_vmx_msr_check_common(vcpu, e))
		return -EINVAL;
	return 0;
}

static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
				      struct vmx_msr_entry *e)
{
	if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */
	    nested_vmx_msr_check_common(vcpu, e))
8803 8804 8805 8806 8807 8808 8809 8810 8811 8812 8813 8814 8815 8816 8817 8818
		return -EINVAL;
	return 0;
}

/*
 * Load guest's/host's msr at nested entry/exit.
 * return 0 for success, entry index for failure.
 */
static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
{
	u32 i;
	struct vmx_msr_entry e;
	struct msr_data msr;

	msr.host_initiated = false;
	for (i = 0; i < count; i++) {
8819 8820 8821 8822 8823
		if (kvm_read_guest(vcpu->kvm, gpa + i * sizeof(e),
				   &e, sizeof(e))) {
			pr_warn_ratelimited(
				"%s cannot read MSR entry (%u, 0x%08llx)\n",
				__func__, i, gpa + i * sizeof(e));
8824
			goto fail;
8825 8826 8827 8828 8829 8830 8831
		}
		if (nested_vmx_load_msr_check(vcpu, &e)) {
			pr_warn_ratelimited(
				"%s check failed (%u, 0x%x, 0x%x)\n",
				__func__, i, e.index, e.reserved);
			goto fail;
		}
8832 8833
		msr.index = e.index;
		msr.data = e.value;
8834 8835 8836 8837
		if (kvm_set_msr(vcpu, &msr)) {
			pr_warn_ratelimited(
				"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
				__func__, i, e.index, e.value);
8838
			goto fail;
8839
		}
8840 8841 8842 8843 8844 8845 8846 8847 8848 8849 8850 8851
	}
	return 0;
fail:
	return i + 1;
}

static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
{
	u32 i;
	struct vmx_msr_entry e;

	for (i = 0; i < count; i++) {
8852 8853 8854 8855 8856 8857
		if (kvm_read_guest(vcpu->kvm,
				   gpa + i * sizeof(e),
				   &e, 2 * sizeof(u32))) {
			pr_warn_ratelimited(
				"%s cannot read MSR entry (%u, 0x%08llx)\n",
				__func__, i, gpa + i * sizeof(e));
8858
			return -EINVAL;
8859 8860 8861 8862 8863
		}
		if (nested_vmx_store_msr_check(vcpu, &e)) {
			pr_warn_ratelimited(
				"%s check failed (%u, 0x%x, 0x%x)\n",
				__func__, i, e.index, e.reserved);
8864
			return -EINVAL;
8865 8866 8867 8868 8869 8870 8871 8872 8873
		}
		if (kvm_get_msr(vcpu, e.index, &e.value)) {
			pr_warn_ratelimited(
				"%s cannot read MSR (%u, 0x%x)\n",
				__func__, i, e.index);
			return -EINVAL;
		}
		if (kvm_write_guest(vcpu->kvm,
				    gpa + i * sizeof(e) +
8874
					offsetof(struct vmx_msr_entry, value),
8875 8876 8877 8878 8879 8880
				    &e.value, sizeof(e.value))) {
			pr_warn_ratelimited(
				"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
				__func__, i, e.index, e.value);
			return -EINVAL;
		}
8881 8882 8883 8884
	}
	return 0;
}

8885 8886 8887
/*
 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
T
Tiejun Chen 已提交
8888
 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
8889 8890 8891 8892 8893 8894 8895 8896 8897 8898 8899 8900 8901 8902 8903 8904 8905 8906 8907 8908 8909 8910 8911 8912 8913 8914 8915 8916 8917 8918 8919 8920 8921 8922 8923 8924 8925 8926 8927 8928 8929 8930 8931 8932 8933 8934 8935
 * guest in a way that will both be appropriate to L1's requests, and our
 * needs. In addition to modifying the active vmcs (which is vmcs02), this
 * function also has additional necessary side-effects, like setting various
 * vcpu->arch fields.
 */
static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	u32 exec_control;

	vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
	vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
	vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
	vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
	vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
	vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
	vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
	vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
	vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
	vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
	vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
	vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
	vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
	vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
	vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
	vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
	vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
	vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
	vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
	vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
	vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
	vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
	vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
	vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
	vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
	vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
	vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
	vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
	vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
	vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
	vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
	vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
	vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
	vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
	vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
	vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);

8936 8937 8938 8939 8940 8941 8942
	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) {
		kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
		vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
	} else {
		kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
		vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
	}
8943 8944 8945 8946 8947 8948 8949 8950 8951
	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
		vmcs12->vm_entry_intr_info_field);
	vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
		vmcs12->vm_entry_exception_error_code);
	vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
		vmcs12->vm_entry_instruction_len);
	vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
		vmcs12->guest_interruptibility_info);
	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
8952
	vmx_set_rflags(vcpu, vmcs12->guest_rflags);
8953 8954 8955 8956 8957
	vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
		vmcs12->guest_pending_dbg_exceptions);
	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
	vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);

8958 8959
	if (nested_cpu_has_xsaves(vmcs12))
		vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
8960 8961
	vmcs_write64(VMCS_LINK_POINTER, -1ull);

8962 8963
	exec_control = vmcs12->pin_based_vm_exec_control;
	exec_control |= vmcs_config.pin_based_exec_ctrl;
8964 8965
	exec_control &= ~(PIN_BASED_VMX_PREEMPTION_TIMER |
                          PIN_BASED_POSTED_INTR);
8966
	vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, exec_control);
8967

8968 8969 8970
	vmx->nested.preemption_timer_expired = false;
	if (nested_cpu_has_preemption_timer(vmcs12))
		vmx_start_preemption_timer(vcpu);
8971

8972 8973 8974 8975 8976 8977 8978 8979 8980 8981 8982 8983 8984 8985 8986 8987 8988 8989 8990 8991 8992 8993 8994 8995 8996 8997
	/*
	 * Whether page-faults are trapped is determined by a combination of
	 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
	 * If enable_ept, L0 doesn't care about page faults and we should
	 * set all of these to L1's desires. However, if !enable_ept, L0 does
	 * care about (at least some) page faults, and because it is not easy
	 * (if at all possible?) to merge L0 and L1's desires, we simply ask
	 * to exit on each and every L2 page fault. This is done by setting
	 * MASK=MATCH=0 and (see below) EB.PF=1.
	 * Note that below we don't need special code to set EB.PF beyond the
	 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
	 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
	 * !enable_ept, EB.PF is 1, so the "or" will always be 1.
	 *
	 * A problem with this approach (when !enable_ept) is that L1 may be
	 * injected with more page faults than it asked for. This could have
	 * caused problems, but in practice existing hypervisors don't care.
	 * To fix this, we will need to emulate the PFEC checking (on the L1
	 * page tables), using walk_addr(), when injecting PFs to L1.
	 */
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
		enable_ept ? vmcs12->page_fault_error_code_mask : 0);
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
		enable_ept ? vmcs12->page_fault_error_code_match : 0);

	if (cpu_has_secondary_exec_ctrls()) {
8998
		exec_control = vmx_secondary_exec_control(vmx);
8999 9000 9001
		if (!vmx->rdtscp_enabled)
			exec_control &= ~SECONDARY_EXEC_RDTSCP;
		/* Take the following fields only from vmcs12 */
9002 9003 9004
		exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
				  SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
                                  SECONDARY_EXEC_APIC_REGISTER_VIRT);
9005 9006 9007 9008 9009 9010 9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021
		if (nested_cpu_has(vmcs12,
				CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
			exec_control |= vmcs12->secondary_vm_exec_control;

		if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) {
			/*
			 * If translation failed, no matter: This feature asks
			 * to exit when accessing the given address, and if it
			 * can never be accessed, this feature won't do
			 * anything anyway.
			 */
			if (!vmx->nested.apic_access_page)
				exec_control &=
				  ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
			else
				vmcs_write64(APIC_ACCESS_ADDR,
				  page_to_phys(vmx->nested.apic_access_page));
9022 9023
		} else if (!(nested_cpu_has_virt_x2apic_mode(vmcs12)) &&
			    (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))) {
9024 9025
			exec_control |=
				SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
9026
			kvm_vcpu_reload_apic_access_page(vcpu);
9027 9028
		}

9029 9030 9031 9032 9033 9034 9035 9036 9037 9038 9039 9040 9041
		if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) {
			vmcs_write64(EOI_EXIT_BITMAP0,
				vmcs12->eoi_exit_bitmap0);
			vmcs_write64(EOI_EXIT_BITMAP1,
				vmcs12->eoi_exit_bitmap1);
			vmcs_write64(EOI_EXIT_BITMAP2,
				vmcs12->eoi_exit_bitmap2);
			vmcs_write64(EOI_EXIT_BITMAP3,
				vmcs12->eoi_exit_bitmap3);
			vmcs_write16(GUEST_INTR_STATUS,
				vmcs12->guest_intr_status);
		}

9042 9043 9044 9045 9046 9047 9048 9049 9050 9051
		vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
	}


	/*
	 * Set host-state according to L0's settings (vmcs12 is irrelevant here)
	 * Some constant fields are set here by vmx_set_constant_host_state().
	 * Other fields are different per CPU, and will be set later when
	 * vmx_vcpu_load() is called, and when vmx_save_host_state() is called.
	 */
9052
	vmx_set_constant_host_state(vmx);
9053 9054 9055 9056 9057 9058 9059 9060 9061 9062 9063 9064 9065 9066 9067

	/*
	 * HOST_RSP is normally set correctly in vmx_vcpu_run() just before
	 * entry, but only if the current (host) sp changed from the value
	 * we wrote last (vmx->host_rsp). This cache is no longer relevant
	 * if we switch vmcs, and rather than hold a separate cache per vmcs,
	 * here we just force the write to happen on entry.
	 */
	vmx->host_rsp = 0;

	exec_control = vmx_exec_control(vmx); /* L0's desires */
	exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
	exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
	exec_control &= ~CPU_BASED_TPR_SHADOW;
	exec_control |= vmcs12->cpu_based_vm_exec_control;
9068 9069 9070 9071 9072 9073 9074

	if (exec_control & CPU_BASED_TPR_SHADOW) {
		vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
				page_to_phys(vmx->nested.virtual_apic_page));
		vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
	}

9075 9076 9077 9078 9079 9080 9081
	if (cpu_has_vmx_msr_bitmap() &&
	    exec_control & CPU_BASED_USE_MSR_BITMAPS &&
	    nested_vmx_merge_msr_bitmap(vcpu, vmcs12)) {
		vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_nested));
	} else
		exec_control &= ~CPU_BASED_USE_MSR_BITMAPS;

9082
	/*
9083
	 * Merging of IO bitmap not currently supported.
9084 9085 9086 9087 9088 9089 9090 9091 9092 9093 9094 9095 9096 9097 9098
	 * Rather, exit every time.
	 */
	exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
	exec_control |= CPU_BASED_UNCOND_IO_EXITING;

	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);

	/* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
	 * bitwise-or of what L1 wants to trap for L2, and what we want to
	 * trap. Note that CR0.TS also needs updating - we do this later.
	 */
	update_exception_bitmap(vcpu);
	vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);

9099 9100 9101 9102
	/* L2->L1 exit controls are emulated - the hardware exit is to L0 so
	 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
	 * bits are further modified by vmx_set_efer() below.
	 */
9103
	vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
9104 9105 9106 9107

	/* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE are
	 * emulated by vmx_set_efer(), below.
	 */
9108
	vm_entry_controls_init(vmx, 
9109 9110
		(vmcs12->vm_entry_controls & ~VM_ENTRY_LOAD_IA32_EFER &
			~VM_ENTRY_IA32E_MODE) |
9111 9112
		(vmcs_config.vmentry_ctrl & ~VM_ENTRY_IA32E_MODE));

9113
	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) {
9114
		vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
9115 9116
		vcpu->arch.pat = vmcs12->guest_ia32_pat;
	} else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
9117 9118 9119 9120 9121
		vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);


	set_cr4_guest_host_mask(vmx);

9122 9123 9124
	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)
		vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);

9125 9126 9127 9128 9129
	if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
		vmcs_write64(TSC_OFFSET,
			vmx->nested.vmcs01_tsc_offset + vmcs12->tsc_offset);
	else
		vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset);
9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140

	if (enable_vpid) {
		/*
		 * Trivially support vpid by letting L2s share their parent
		 * L1's vpid. TODO: move to a more elaborate solution, giving
		 * each L2 its own vpid and exposing the vpid feature to L1.
		 */
		vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
		vmx_flush_tlb(vcpu);
	}

N
Nadav Har'El 已提交
9141 9142 9143 9144 9145
	if (nested_cpu_has_ept(vmcs12)) {
		kvm_mmu_unload(vcpu);
		nested_ept_init_mmu_context(vcpu);
	}

9146 9147
	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)
		vcpu->arch.efer = vmcs12->guest_ia32_efer;
9148
	else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
9149 9150 9151 9152 9153 9154 9155 9156 9157 9158 9159 9160 9161 9162 9163 9164 9165 9166 9167 9168 9169 9170 9171 9172
		vcpu->arch.efer |= (EFER_LMA | EFER_LME);
	else
		vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
	/* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
	vmx_set_efer(vcpu, vcpu->arch.efer);

	/*
	 * This sets GUEST_CR0 to vmcs12->guest_cr0, with possibly a modified
	 * TS bit (for lazy fpu) and bits which we consider mandatory enabled.
	 * The CR0_READ_SHADOW is what L2 should have expected to read given
	 * the specifications by L1; It's not enough to take
	 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
	 * have more bits than L1 expected.
	 */
	vmx_set_cr0(vcpu, vmcs12->guest_cr0);
	vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));

	vmx_set_cr4(vcpu, vmcs12->guest_cr4);
	vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));

	/* shadow page tables on either EPT or shadow page tables */
	kvm_set_cr3(vcpu, vmcs12->guest_cr3);
	kvm_mmu_reset_context(vcpu);

9173 9174 9175
	if (!enable_ept)
		vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;

9176 9177 9178 9179 9180 9181 9182 9183 9184 9185
	/*
	 * L1 may access the L2's PDPTR, so save them to construct vmcs12
	 */
	if (enable_ept) {
		vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
		vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
		vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
		vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
	}

9186 9187 9188 9189
	kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp);
	kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip);
}

9190 9191 9192 9193 9194 9195 9196 9197 9198 9199
/*
 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
 * for running an L2 nested guest.
 */
static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
{
	struct vmcs12 *vmcs12;
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	int cpu;
	struct loaded_vmcs *vmcs02;
9200
	bool ia32e;
9201
	u32 msr_entry_idx;
9202 9203 9204 9205 9206 9207 9208 9209

	if (!nested_vmx_check_permission(vcpu) ||
	    !nested_vmx_check_vmcs12(vcpu))
		return 1;

	skip_emulated_instruction(vcpu);
	vmcs12 = get_vmcs12(vcpu);

9210 9211 9212
	if (enable_shadow_vmcs)
		copy_shadow_to_vmcs12(vmx);

9213 9214 9215 9216 9217 9218 9219 9220 9221 9222 9223 9224 9225 9226 9227 9228 9229
	/*
	 * The nested entry process starts with enforcing various prerequisites
	 * on vmcs12 as required by the Intel SDM, and act appropriately when
	 * they fail: As the SDM explains, some conditions should cause the
	 * instruction to fail, while others will cause the instruction to seem
	 * to succeed, but return an EXIT_REASON_INVALID_STATE.
	 * To speed up the normal (success) code path, we should avoid checking
	 * for misconfigurations which will anyway be caught by the processor
	 * when using the merged vmcs02.
	 */
	if (vmcs12->launch_state == launch) {
		nested_vmx_failValid(vcpu,
			launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
			       : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
		return 1;
	}

9230 9231
	if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
	    vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) {
9232 9233 9234 9235
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

9236
	if (!nested_get_vmcs12_pages(vcpu, vmcs12)) {
9237 9238 9239 9240 9241
		/*TODO: Also verify bits beyond physical address width are 0*/
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

9242
	if (nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12)) {
9243 9244 9245 9246
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

9247 9248 9249 9250 9251
	if (nested_vmx_check_apicv_controls(vcpu, vmcs12)) {
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

9252 9253 9254 9255 9256
	if (nested_vmx_check_msr_switch_controls(vcpu, vmcs12)) {
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

9257
	if (!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
9258 9259
				vmx->nested.nested_vmx_true_procbased_ctls_low,
				vmx->nested.nested_vmx_procbased_ctls_high) ||
9260
	    !vmx_control_verify(vmcs12->secondary_vm_exec_control,
9261 9262
				vmx->nested.nested_vmx_secondary_ctls_low,
				vmx->nested.nested_vmx_secondary_ctls_high) ||
9263
	    !vmx_control_verify(vmcs12->pin_based_vm_exec_control,
9264 9265
				vmx->nested.nested_vmx_pinbased_ctls_low,
				vmx->nested.nested_vmx_pinbased_ctls_high) ||
9266
	    !vmx_control_verify(vmcs12->vm_exit_controls,
9267 9268
				vmx->nested.nested_vmx_true_exit_ctls_low,
				vmx->nested.nested_vmx_exit_ctls_high) ||
9269
	    !vmx_control_verify(vmcs12->vm_entry_controls,
9270 9271
				vmx->nested.nested_vmx_true_entry_ctls_low,
				vmx->nested.nested_vmx_entry_ctls_high))
9272 9273 9274 9275 9276 9277 9278 9279 9280 9281 9282 9283
	{
		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
		return 1;
	}

	if (((vmcs12->host_cr0 & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON) ||
	    ((vmcs12->host_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) {
		nested_vmx_failValid(vcpu,
			VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
		return 1;
	}

9284
	if (!nested_cr0_valid(vcpu, vmcs12->guest_cr0) ||
9285 9286 9287 9288 9289 9290 9291 9292 9293 9294 9295
	    ((vmcs12->guest_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) {
		nested_vmx_entry_failure(vcpu, vmcs12,
			EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
		return 1;
	}
	if (vmcs12->vmcs_link_pointer != -1ull) {
		nested_vmx_entry_failure(vcpu, vmcs12,
			EXIT_REASON_INVALID_STATE, ENTRY_FAIL_VMCS_LINK_PTR);
		return 1;
	}

9296
	/*
9297
	 * If the load IA32_EFER VM-entry control is 1, the following checks
9298 9299 9300 9301 9302 9303 9304 9305 9306 9307 9308 9309 9310 9311 9312 9313 9314 9315 9316 9317 9318 9319 9320 9321 9322 9323 9324 9325 9326 9327 9328 9329 9330 9331 9332 9333 9334
	 * are performed on the field for the IA32_EFER MSR:
	 * - Bits reserved in the IA32_EFER MSR must be 0.
	 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
	 *   the IA-32e mode guest VM-exit control. It must also be identical
	 *   to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
	 *   CR0.PG) is 1.
	 */
	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER) {
		ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
		if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) ||
		    ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) ||
		    ((vmcs12->guest_cr0 & X86_CR0_PG) &&
		     ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))) {
			nested_vmx_entry_failure(vcpu, vmcs12,
				EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
			return 1;
		}
	}

	/*
	 * If the load IA32_EFER VM-exit control is 1, bits reserved in the
	 * IA32_EFER MSR must be 0 in the field for that register. In addition,
	 * the values of the LMA and LME bits in the field must each be that of
	 * the host address-space size VM-exit control.
	 */
	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
		ia32e = (vmcs12->vm_exit_controls &
			 VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0;
		if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) ||
		    ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) ||
		    ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)) {
			nested_vmx_entry_failure(vcpu, vmcs12,
				EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
			return 1;
		}
	}

9335 9336 9337 9338 9339
	/*
	 * We're finally done with prerequisite checking, and can start with
	 * the nested entry.
	 */

9340 9341 9342 9343 9344 9345 9346 9347
	vmcs02 = nested_get_current_vmcs02(vmx);
	if (!vmcs02)
		return -ENOMEM;

	enter_guest_mode(vcpu);

	vmx->nested.vmcs01_tsc_offset = vmcs_read64(TSC_OFFSET);

9348 9349 9350
	if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
		vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);

9351 9352 9353 9354 9355 9356 9357
	cpu = get_cpu();
	vmx->loaded_vmcs = vmcs02;
	vmx_vcpu_put(vcpu);
	vmx_vcpu_load(vcpu, cpu);
	vcpu->cpu = cpu;
	put_cpu();

9358 9359
	vmx_segment_cache_clear(vmx);

9360 9361
	prepare_vmcs02(vcpu, vmcs12);

9362 9363 9364 9365 9366 9367 9368 9369 9370 9371 9372 9373 9374
	msr_entry_idx = nested_vmx_load_msr(vcpu,
					    vmcs12->vm_entry_msr_load_addr,
					    vmcs12->vm_entry_msr_load_count);
	if (msr_entry_idx) {
		leave_guest_mode(vcpu);
		vmx_load_vmcs01(vcpu);
		nested_vmx_entry_failure(vcpu, vmcs12,
				EXIT_REASON_MSR_LOAD_FAIL, msr_entry_idx);
		return 1;
	}

	vmcs12->launch_state = 1;

9375 9376 9377
	if (vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT)
		return kvm_emulate_halt(vcpu);

9378 9379
	vmx->nested.nested_run_pending = 1;

9380 9381 9382 9383 9384 9385 9386 9387 9388
	/*
	 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
	 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
	 * returned as far as L1 is concerned. It will only return (and set
	 * the success flag) when L2 exits (see nested_vmx_vmexit()).
	 */
	return 1;
}

N
Nadav Har'El 已提交
9389 9390 9391 9392 9393 9394 9395 9396 9397 9398 9399 9400 9401 9402 9403 9404 9405 9406 9407 9408 9409 9410 9411 9412 9413 9414 9415 9416 9417 9418 9419 9420 9421 9422 9423 9424 9425
/*
 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
 * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK).
 * This function returns the new value we should put in vmcs12.guest_cr0.
 * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
 *  1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
 *     available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
 *     didn't trap the bit, because if L1 did, so would L0).
 *  2. Bits that L1 asked to trap (and therefore L0 also did) could not have
 *     been modified by L2, and L1 knows it. So just leave the old value of
 *     the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
 *     isn't relevant, because if L0 traps this bit it can set it to anything.
 *  3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
 *     changed these bits, and therefore they need to be updated, but L0
 *     didn't necessarily allow them to be changed in GUEST_CR0 - and rather
 *     put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
 */
static inline unsigned long
vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	return
	/*1*/	(vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
	/*2*/	(vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
	/*3*/	(vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
			vcpu->arch.cr0_guest_owned_bits));
}

static inline unsigned long
vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
{
	return
	/*1*/	(vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
	/*2*/	(vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
	/*3*/	(vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
			vcpu->arch.cr4_guest_owned_bits));
}

9426 9427 9428 9429 9430 9431
static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
				       struct vmcs12 *vmcs12)
{
	u32 idt_vectoring;
	unsigned int nr;

9432
	if (vcpu->arch.exception.pending && vcpu->arch.exception.reinject) {
9433 9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444 9445 9446 9447 9448 9449
		nr = vcpu->arch.exception.nr;
		idt_vectoring = nr | VECTORING_INFO_VALID_MASK;

		if (kvm_exception_is_soft(nr)) {
			vmcs12->vm_exit_instruction_len =
				vcpu->arch.event_exit_inst_len;
			idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
		} else
			idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;

		if (vcpu->arch.exception.has_error_code) {
			idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
			vmcs12->idt_vectoring_error_code =
				vcpu->arch.exception.error_code;
		}

		vmcs12->idt_vectoring_info_field = idt_vectoring;
J
Jan Kiszka 已提交
9450
	} else if (vcpu->arch.nmi_injected) {
9451 9452 9453 9454 9455 9456 9457 9458 9459 9460 9461 9462 9463 9464 9465 9466 9467
		vmcs12->idt_vectoring_info_field =
			INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
	} else if (vcpu->arch.interrupt.pending) {
		nr = vcpu->arch.interrupt.nr;
		idt_vectoring = nr | VECTORING_INFO_VALID_MASK;

		if (vcpu->arch.interrupt.soft) {
			idt_vectoring |= INTR_TYPE_SOFT_INTR;
			vmcs12->vm_entry_instruction_len =
				vcpu->arch.event_exit_inst_len;
		} else
			idt_vectoring |= INTR_TYPE_EXT_INTR;

		vmcs12->idt_vectoring_info_field = idt_vectoring;
	}
}

9468 9469 9470 9471
static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

9472 9473 9474 9475 9476 9477 9478 9479
	if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
	    vmx->nested.preemption_timer_expired) {
		if (vmx->nested.nested_run_pending)
			return -EBUSY;
		nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
		return 0;
	}

9480
	if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) {
9481 9482
		if (vmx->nested.nested_run_pending ||
		    vcpu->arch.interrupt.pending)
9483 9484 9485 9486 9487 9488 9489 9490 9491 9492 9493 9494 9495 9496 9497 9498 9499 9500 9501 9502 9503 9504 9505
			return -EBUSY;
		nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
				  NMI_VECTOR | INTR_TYPE_NMI_INTR |
				  INTR_INFO_VALID_MASK, 0);
		/*
		 * The NMI-triggered VM exit counts as injection:
		 * clear this one and block further NMIs.
		 */
		vcpu->arch.nmi_pending = 0;
		vmx_set_nmi_mask(vcpu, true);
		return 0;
	}

	if ((kvm_cpu_has_interrupt(vcpu) || external_intr) &&
	    nested_exit_on_intr(vcpu)) {
		if (vmx->nested.nested_run_pending)
			return -EBUSY;
		nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
	}

	return 0;
}

9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519
static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
{
	ktime_t remaining =
		hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
	u64 value;

	if (ktime_to_ns(remaining) <= 0)
		return 0;

	value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
	do_div(value, 1000000);
	return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
}

N
Nadav Har'El 已提交
9520 9521 9522 9523 9524 9525 9526 9527 9528 9529 9530
/*
 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
 * and this function updates it to reflect the changes to the guest state while
 * L2 was running (and perhaps made some exits which were handled directly by L0
 * without going back to L1), and to reflect the exit reason.
 * Note that we do not have to copy here all VMCS fields, just those that
 * could have changed by the L2 guest or the exit - i.e., the guest-state and
 * exit-information fields only. Other fields are modified by L1 with VMWRITE,
 * which already writes to vmcs12 directly.
 */
9531 9532 9533
static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
			   u32 exit_reason, u32 exit_intr_info,
			   unsigned long exit_qualification)
N
Nadav Har'El 已提交
9534 9535 9536 9537 9538 9539 9540 9541 9542 9543 9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556 9557 9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577 9578 9579 9580 9581 9582 9583
{
	/* update guest state fields: */
	vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
	vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);

	vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
	vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP);
	vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);

	vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
	vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
	vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
	vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
	vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
	vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
	vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
	vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
	vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
	vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
	vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
	vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
	vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
	vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
	vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
	vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
	vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
	vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
	vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
	vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
	vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
	vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
	vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
	vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
	vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
	vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
	vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
	vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
	vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
	vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
	vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
	vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
	vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
	vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
	vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
	vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);

	vmcs12->guest_interruptibility_info =
		vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
	vmcs12->guest_pending_dbg_exceptions =
		vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
9584 9585 9586 9587
	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
		vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
	else
		vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
N
Nadav Har'El 已提交
9588

9589 9590 9591 9592 9593 9594 9595
	if (nested_cpu_has_preemption_timer(vmcs12)) {
		if (vmcs12->vm_exit_controls &
		    VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
			vmcs12->vmx_preemption_timer_value =
				vmx_get_preemption_timer_value(vcpu);
		hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
	}
9596

9597 9598 9599 9600 9601 9602 9603 9604 9605 9606 9607 9608 9609 9610 9611 9612
	/*
	 * In some cases (usually, nested EPT), L2 is allowed to change its
	 * own CR3 without exiting. If it has changed it, we must keep it.
	 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
	 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
	 *
	 * Additionally, restore L2's PDPTR to vmcs12.
	 */
	if (enable_ept) {
		vmcs12->guest_cr3 = vmcs_read64(GUEST_CR3);
		vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
		vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
		vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
		vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
	}

9613 9614 9615
	if (nested_cpu_has_vid(vmcs12))
		vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);

9616 9617
	vmcs12->vm_entry_controls =
		(vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
9618
		(vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
9619

9620 9621 9622 9623 9624
	if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) {
		kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
		vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
	}

N
Nadav Har'El 已提交
9625 9626
	/* TODO: These cannot have changed unless we have MSR bitmaps and
	 * the relevant bit asks not to trap the change */
9627
	if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT)
N
Nadav Har'El 已提交
9628
		vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT);
9629 9630
	if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
		vmcs12->guest_ia32_efer = vcpu->arch.efer;
N
Nadav Har'El 已提交
9631 9632 9633
	vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS);
	vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP);
	vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP);
9634 9635
	if (vmx_mpx_supported())
		vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
9636 9637
	if (nested_cpu_has_xsaves(vmcs12))
		vmcs12->xss_exit_bitmap = vmcs_read64(XSS_EXIT_BITMAP);
N
Nadav Har'El 已提交
9638 9639 9640

	/* update exit information fields: */

9641 9642
	vmcs12->vm_exit_reason = exit_reason;
	vmcs12->exit_qualification = exit_qualification;
N
Nadav Har'El 已提交
9643

9644
	vmcs12->vm_exit_intr_info = exit_intr_info;
9645 9646 9647 9648 9649
	if ((vmcs12->vm_exit_intr_info &
	     (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) ==
	    (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK))
		vmcs12->vm_exit_intr_error_code =
			vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
9650
	vmcs12->idt_vectoring_info_field = 0;
N
Nadav Har'El 已提交
9651 9652 9653
	vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
	vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);

9654 9655 9656
	if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
		/* vm_entry_intr_info_field is cleared on exit. Emulate this
		 * instead of reading the real value. */
N
Nadav Har'El 已提交
9657
		vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
9658 9659 9660 9661 9662 9663 9664 9665 9666 9667 9668 9669 9670 9671 9672

		/*
		 * Transfer the event that L0 or L1 may wanted to inject into
		 * L2 to IDT_VECTORING_INFO_FIELD.
		 */
		vmcs12_save_pending_event(vcpu, vmcs12);
	}

	/*
	 * Drop what we picked up for L2 via vmx_complete_interrupts. It is
	 * preserved above and would only end up incorrectly in L1.
	 */
	vcpu->arch.nmi_injected = false;
	kvm_clear_exception_queue(vcpu);
	kvm_clear_interrupt_queue(vcpu);
N
Nadav Har'El 已提交
9673 9674 9675 9676 9677 9678 9679 9680 9681 9682 9683
}

/*
 * A part of what we need to when the nested L2 guest exits and we want to
 * run its L1 parent, is to reset L1's guest state to the host state specified
 * in vmcs12.
 * This function is to be called not only on normal nested exit, but also on
 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
 * Failures During or After Loading Guest State").
 * This function should be called when the active VMCS is L1's (vmcs01).
 */
9684 9685
static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
				   struct vmcs12 *vmcs12)
N
Nadav Har'El 已提交
9686
{
9687 9688
	struct kvm_segment seg;

N
Nadav Har'El 已提交
9689 9690
	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
		vcpu->arch.efer = vmcs12->host_ia32_efer;
9691
	else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
N
Nadav Har'El 已提交
9692 9693 9694 9695 9696 9697 9698
		vcpu->arch.efer |= (EFER_LMA | EFER_LME);
	else
		vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
	vmx_set_efer(vcpu, vcpu->arch.efer);

	kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp);
	kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip);
9699
	vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
N
Nadav Har'El 已提交
9700 9701 9702 9703 9704 9705
	/*
	 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
	 * actually changed, because it depends on the current state of
	 * fpu_active (which may have changed).
	 * Note that vmx_set_cr0 refers to efer set above.
	 */
9706
	vmx_set_cr0(vcpu, vmcs12->host_cr0);
N
Nadav Har'El 已提交
9707 9708 9709 9710 9711 9712 9713 9714 9715 9716 9717 9718 9719 9720 9721 9722
	/*
	 * If we did fpu_activate()/fpu_deactivate() during L2's run, we need
	 * to apply the same changes to L1's vmcs. We just set cr0 correctly,
	 * but we also need to update cr0_guest_host_mask and exception_bitmap.
	 */
	update_exception_bitmap(vcpu);
	vcpu->arch.cr0_guest_owned_bits = (vcpu->fpu_active ? X86_CR0_TS : 0);
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);

	/*
	 * Note that CR4_GUEST_HOST_MASK is already set in the original vmcs01
	 * (KVM doesn't change it)- no reason to call set_cr4_guest_host_mask();
	 */
	vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
	kvm_set_cr4(vcpu, vmcs12->host_cr4);

9723
	nested_ept_uninit_mmu_context(vcpu);
N
Nadav Har'El 已提交
9724

N
Nadav Har'El 已提交
9725 9726 9727
	kvm_set_cr3(vcpu, vmcs12->host_cr3);
	kvm_mmu_reset_context(vcpu);

9728 9729 9730
	if (!enable_ept)
		vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;

N
Nadav Har'El 已提交
9731 9732 9733 9734 9735 9736 9737 9738 9739 9740 9741 9742 9743 9744 9745 9746
	if (enable_vpid) {
		/*
		 * Trivially support vpid by letting L2s share their parent
		 * L1's vpid. TODO: move to a more elaborate solution, giving
		 * each L2 its own vpid and exposing the vpid feature to L1.
		 */
		vmx_flush_tlb(vcpu);
	}


	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
	vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
	vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
	vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);

9747 9748 9749 9750
	/* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1.  */
	if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
		vmcs_write64(GUEST_BNDCFGS, 0);

9751
	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
N
Nadav Har'El 已提交
9752
		vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
9753 9754
		vcpu->arch.pat = vmcs12->host_ia32_pat;
	}
N
Nadav Har'El 已提交
9755 9756 9757
	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
		vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL,
			vmcs12->host_ia32_perf_global_ctrl);
9758

9759 9760 9761 9762 9763 9764 9765 9766 9767 9768 9769 9770 9771 9772 9773 9774 9775 9776 9777 9778 9779 9780 9781 9782 9783 9784 9785 9786 9787 9788 9789 9790 9791 9792 9793 9794 9795 9796
	/* Set L1 segment info according to Intel SDM
	    27.5.2 Loading Host Segment and Descriptor-Table Registers */
	seg = (struct kvm_segment) {
		.base = 0,
		.limit = 0xFFFFFFFF,
		.selector = vmcs12->host_cs_selector,
		.type = 11,
		.present = 1,
		.s = 1,
		.g = 1
	};
	if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
		seg.l = 1;
	else
		seg.db = 1;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
	seg = (struct kvm_segment) {
		.base = 0,
		.limit = 0xFFFFFFFF,
		.type = 3,
		.present = 1,
		.s = 1,
		.db = 1,
		.g = 1
	};
	seg.selector = vmcs12->host_ds_selector;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
	seg.selector = vmcs12->host_es_selector;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
	seg.selector = vmcs12->host_ss_selector;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
	seg.selector = vmcs12->host_fs_selector;
	seg.base = vmcs12->host_fs_base;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
	seg.selector = vmcs12->host_gs_selector;
	seg.base = vmcs12->host_gs_base;
	vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
	seg = (struct kvm_segment) {
9797
		.base = vmcs12->host_tr_base,
9798 9799 9800 9801 9802 9803 9804
		.limit = 0x67,
		.selector = vmcs12->host_tr_selector,
		.type = 11,
		.present = 1
	};
	vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);

9805 9806
	kvm_set_dr(vcpu, 7, 0x400);
	vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
9807

9808 9809 9810
	if (cpu_has_vmx_msr_bitmap())
		vmx_set_msr_bitmap(vcpu);

9811 9812 9813
	if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
				vmcs12->vm_exit_msr_load_count))
		nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
N
Nadav Har'El 已提交
9814 9815 9816 9817 9818 9819 9820
}

/*
 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
 * and modify vmcs12 to make it see what it would expect to see there if
 * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
 */
9821 9822 9823
static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
			      u32 exit_intr_info,
			      unsigned long exit_qualification)
N
Nadav Har'El 已提交
9824 9825 9826 9827
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);
	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);

9828 9829 9830
	/* trying to cancel vmlaunch/vmresume is a bug */
	WARN_ON_ONCE(vmx->nested.nested_run_pending);

N
Nadav Har'El 已提交
9831
	leave_guest_mode(vcpu);
9832 9833
	prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info,
		       exit_qualification);
N
Nadav Har'El 已提交
9834

9835 9836 9837 9838
	if (nested_vmx_store_msr(vcpu, vmcs12->vm_exit_msr_store_addr,
				 vmcs12->vm_exit_msr_store_count))
		nested_vmx_abort(vcpu, VMX_ABORT_SAVE_GUEST_MSR_FAIL);

9839 9840
	vmx_load_vmcs01(vcpu);

9841 9842 9843 9844 9845 9846 9847 9848
	if ((exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)
	    && nested_exit_intr_ack_set(vcpu)) {
		int irq = kvm_cpu_get_interrupt(vcpu);
		WARN_ON(irq < 0);
		vmcs12->vm_exit_intr_info = irq |
			INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR;
	}

9849 9850 9851 9852 9853 9854
	trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
				       vmcs12->exit_qualification,
				       vmcs12->idt_vectoring_info_field,
				       vmcs12->vm_exit_intr_info,
				       vmcs12->vm_exit_intr_error_code,
				       KVM_ISA_VMX);
N
Nadav Har'El 已提交
9855

9856 9857
	vm_entry_controls_init(vmx, vmcs_read32(VM_ENTRY_CONTROLS));
	vm_exit_controls_init(vmx, vmcs_read32(VM_EXIT_CONTROLS));
9858 9859
	vmx_segment_cache_clear(vmx);

N
Nadav Har'El 已提交
9860 9861 9862 9863 9864 9865
	/* if no vmcs02 cache requested, remove the one we used */
	if (VMCS02_POOL_SIZE == 0)
		nested_free_vmcs02(vmx, vmx->nested.current_vmptr);

	load_vmcs12_host_state(vcpu, vmcs12);

9866
	/* Update TSC_OFFSET if TSC was changed while L2 ran */
N
Nadav Har'El 已提交
9867 9868 9869 9870 9871 9872 9873 9874
	vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset);

	/* This is needed for same reason as it was needed in prepare_vmcs02 */
	vmx->host_rsp = 0;

	/* Unpin physical memory we referred to in vmcs02 */
	if (vmx->nested.apic_access_page) {
		nested_release_page(vmx->nested.apic_access_page);
9875
		vmx->nested.apic_access_page = NULL;
N
Nadav Har'El 已提交
9876
	}
9877 9878
	if (vmx->nested.virtual_apic_page) {
		nested_release_page(vmx->nested.virtual_apic_page);
9879
		vmx->nested.virtual_apic_page = NULL;
9880
	}
N
Nadav Har'El 已提交
9881

9882 9883 9884 9885 9886 9887
	/*
	 * We are now running in L2, mmu_notifier will force to reload the
	 * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1.
	 */
	kvm_vcpu_reload_apic_access_page(vcpu);

N
Nadav Har'El 已提交
9888 9889 9890 9891 9892 9893 9894 9895 9896 9897
	/*
	 * Exiting from L2 to L1, we're now back to L1 which thinks it just
	 * finished a VMLAUNCH or VMRESUME instruction, so we need to set the
	 * success or failure flag accordingly.
	 */
	if (unlikely(vmx->fail)) {
		vmx->fail = 0;
		nested_vmx_failValid(vcpu, vmcs_read32(VM_INSTRUCTION_ERROR));
	} else
		nested_vmx_succeed(vcpu);
9898 9899
	if (enable_shadow_vmcs)
		vmx->nested.sync_shadow_vmcs = true;
9900 9901 9902

	/* in case we halted in L2 */
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
N
Nadav Har'El 已提交
9903 9904
}

9905 9906 9907 9908 9909 9910
/*
 * Forcibly leave nested mode in order to be able to reset the VCPU later on.
 */
static void vmx_leave_nested(struct kvm_vcpu *vcpu)
{
	if (is_guest_mode(vcpu))
9911
		nested_vmx_vmexit(vcpu, -1, 0, 0);
9912 9913 9914
	free_nested(to_vmx(vcpu));
}

9915 9916 9917 9918 9919 9920 9921 9922 9923 9924 9925 9926 9927 9928 9929
/*
 * L1's failure to enter L2 is a subset of a normal exit, as explained in
 * 23.7 "VM-entry failures during or after loading guest state" (this also
 * lists the acceptable exit-reason and exit-qualification parameters).
 * It should only be called before L2 actually succeeded to run, and when
 * vmcs01 is current (it doesn't leave_guest_mode() or switch vmcss).
 */
static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
			struct vmcs12 *vmcs12,
			u32 reason, unsigned long qualification)
{
	load_vmcs12_host_state(vcpu, vmcs12);
	vmcs12->vm_exit_reason = reason | VMX_EXIT_REASONS_FAILED_VMENTRY;
	vmcs12->exit_qualification = qualification;
	nested_vmx_succeed(vcpu);
9930 9931
	if (enable_shadow_vmcs)
		to_vmx(vcpu)->nested.sync_shadow_vmcs = true;
9932 9933
}

9934 9935 9936 9937 9938 9939 9940
static int vmx_check_intercept(struct kvm_vcpu *vcpu,
			       struct x86_instruction_info *info,
			       enum x86_intercept_stage stage)
{
	return X86EMUL_CONTINUE;
}

9941
static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
9942
{
R
Radim Krčmář 已提交
9943 9944
	if (ple_gap)
		shrink_ple_window(vcpu);
9945 9946
}

K
Kai Huang 已提交
9947 9948 9949 9950 9951 9952 9953 9954 9955 9956 9957 9958 9959 9960 9961 9962 9963 9964 9965 9966 9967 9968 9969 9970 9971
static void vmx_slot_enable_log_dirty(struct kvm *kvm,
				     struct kvm_memory_slot *slot)
{
	kvm_mmu_slot_leaf_clear_dirty(kvm, slot);
	kvm_mmu_slot_largepage_remove_write_access(kvm, slot);
}

static void vmx_slot_disable_log_dirty(struct kvm *kvm,
				       struct kvm_memory_slot *slot)
{
	kvm_mmu_slot_set_dirty(kvm, slot);
}

static void vmx_flush_log_dirty(struct kvm *kvm)
{
	kvm_flush_pml_buffers(kvm);
}

static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm,
					   struct kvm_memory_slot *memslot,
					   gfn_t offset, unsigned long mask)
{
	kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask);
}

9972
static struct kvm_x86_ops vmx_x86_ops = {
A
Avi Kivity 已提交
9973 9974 9975 9976
	.cpu_has_kvm_support = cpu_has_kvm_support,
	.disabled_by_bios = vmx_disabled_by_bios,
	.hardware_setup = hardware_setup,
	.hardware_unsetup = hardware_unsetup,
Y
Yang, Sheng 已提交
9977
	.check_processor_compatibility = vmx_check_processor_compat,
A
Avi Kivity 已提交
9978 9979
	.hardware_enable = hardware_enable,
	.hardware_disable = hardware_disable,
9980
	.cpu_has_accelerated_tpr = report_flexpriority,
A
Avi Kivity 已提交
9981 9982 9983

	.vcpu_create = vmx_create_vcpu,
	.vcpu_free = vmx_free_vcpu,
9984
	.vcpu_reset = vmx_vcpu_reset,
A
Avi Kivity 已提交
9985

9986
	.prepare_guest_switch = vmx_save_host_state,
A
Avi Kivity 已提交
9987 9988 9989
	.vcpu_load = vmx_vcpu_load,
	.vcpu_put = vmx_vcpu_put,

9990
	.update_db_bp_intercept = update_exception_bitmap,
A
Avi Kivity 已提交
9991 9992 9993 9994 9995
	.get_msr = vmx_get_msr,
	.set_msr = vmx_set_msr,
	.get_segment_base = vmx_get_segment_base,
	.get_segment = vmx_get_segment,
	.set_segment = vmx_set_segment,
9996
	.get_cpl = vmx_get_cpl,
A
Avi Kivity 已提交
9997
	.get_cs_db_l_bits = vmx_get_cs_db_l_bits,
9998
	.decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
9999
	.decache_cr3 = vmx_decache_cr3,
10000
	.decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
A
Avi Kivity 已提交
10001 10002 10003 10004 10005 10006 10007 10008
	.set_cr0 = vmx_set_cr0,
	.set_cr3 = vmx_set_cr3,
	.set_cr4 = vmx_set_cr4,
	.set_efer = vmx_set_efer,
	.get_idt = vmx_get_idt,
	.set_idt = vmx_set_idt,
	.get_gdt = vmx_get_gdt,
	.set_gdt = vmx_set_gdt,
J
Jan Kiszka 已提交
10009 10010
	.get_dr6 = vmx_get_dr6,
	.set_dr6 = vmx_set_dr6,
10011
	.set_dr7 = vmx_set_dr7,
10012
	.sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
10013
	.cache_reg = vmx_cache_reg,
A
Avi Kivity 已提交
10014 10015
	.get_rflags = vmx_get_rflags,
	.set_rflags = vmx_set_rflags,
10016
	.fpu_deactivate = vmx_fpu_deactivate,
A
Avi Kivity 已提交
10017 10018 10019 10020

	.tlb_flush = vmx_flush_tlb,

	.run = vmx_vcpu_run,
10021
	.handle_exit = vmx_handle_exit,
A
Avi Kivity 已提交
10022
	.skip_emulated_instruction = skip_emulated_instruction,
10023 10024
	.set_interrupt_shadow = vmx_set_interrupt_shadow,
	.get_interrupt_shadow = vmx_get_interrupt_shadow,
I
Ingo Molnar 已提交
10025
	.patch_hypercall = vmx_patch_hypercall,
E
Eddie Dong 已提交
10026
	.set_irq = vmx_inject_irq,
10027
	.set_nmi = vmx_inject_nmi,
10028
	.queue_exception = vmx_queue_exception,
A
Avi Kivity 已提交
10029
	.cancel_injection = vmx_cancel_injection,
10030
	.interrupt_allowed = vmx_interrupt_allowed,
10031
	.nmi_allowed = vmx_nmi_allowed,
J
Jan Kiszka 已提交
10032 10033
	.get_nmi_mask = vmx_get_nmi_mask,
	.set_nmi_mask = vmx_set_nmi_mask,
10034 10035 10036
	.enable_nmi_window = enable_nmi_window,
	.enable_irq_window = enable_irq_window,
	.update_cr8_intercept = update_cr8_intercept,
10037
	.set_virtual_x2apic_mode = vmx_set_virtual_x2apic_mode,
10038
	.set_apic_access_page_addr = vmx_set_apic_access_page_addr,
10039 10040 10041 10042
	.vm_has_apicv = vmx_vm_has_apicv,
	.load_eoi_exitmap = vmx_load_eoi_exitmap,
	.hwapic_irr_update = vmx_hwapic_irr_update,
	.hwapic_isr_update = vmx_hwapic_isr_update,
10043 10044
	.sync_pir_to_irr = vmx_sync_pir_to_irr,
	.deliver_posted_interrupt = vmx_deliver_posted_interrupt,
10045

10046
	.set_tss_addr = vmx_set_tss_addr,
10047
	.get_tdp_level = get_ept_level,
10048
	.get_mt_mask = vmx_get_mt_mask,
10049

10050 10051
	.get_exit_info = vmx_get_exit_info,

10052
	.get_lpage_level = vmx_get_lpage_level,
10053 10054

	.cpuid_update = vmx_cpuid_update,
10055 10056

	.rdtscp_supported = vmx_rdtscp_supported,
10057
	.invpcid_supported = vmx_invpcid_supported,
10058 10059

	.set_supported_cpuid = vmx_set_supported_cpuid,
10060 10061

	.has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
10062

10063
	.set_tsc_khz = vmx_set_tsc_khz,
W
Will Auld 已提交
10064
	.read_tsc_offset = vmx_read_tsc_offset,
10065
	.write_tsc_offset = vmx_write_tsc_offset,
Z
Zachary Amsden 已提交
10066
	.adjust_tsc_offset = vmx_adjust_tsc_offset,
10067
	.compute_tsc_offset = vmx_compute_tsc_offset,
N
Nadav Har'El 已提交
10068
	.read_l1_tsc = vmx_read_l1_tsc,
10069 10070

	.set_tdp_cr3 = vmx_set_cr3,
10071 10072

	.check_intercept = vmx_check_intercept,
10073
	.handle_external_intr = vmx_handle_external_intr,
10074
	.mpx_supported = vmx_mpx_supported,
10075
	.xsaves_supported = vmx_xsaves_supported,
10076 10077

	.check_nested_events = vmx_check_nested_events,
10078 10079

	.sched_in = vmx_sched_in,
K
Kai Huang 已提交
10080 10081 10082 10083 10084

	.slot_enable_log_dirty = vmx_slot_enable_log_dirty,
	.slot_disable_log_dirty = vmx_slot_disable_log_dirty,
	.flush_log_dirty = vmx_flush_log_dirty,
	.enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked,
A
Avi Kivity 已提交
10085 10086 10087 10088
};

static int __init vmx_init(void)
{
10089 10090
	int r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
                     __alignof__(struct vcpu_vmx), THIS_MODULE);
10091
	if (r)
10092
		return r;
S
Sheng Yang 已提交
10093

10094 10095 10096 10097 10098
#ifdef CONFIG_KEXEC
	rcu_assign_pointer(crash_vmclear_loaded_vmcss,
			   crash_vmclear_local_loaded_vmcss);
#endif

10099
	return 0;
A
Avi Kivity 已提交
10100 10101 10102 10103
}

static void __exit vmx_exit(void)
{
10104
#ifdef CONFIG_KEXEC
10105
	RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
10106 10107 10108
	synchronize_rcu();
#endif

10109
	kvm_exit();
A
Avi Kivity 已提交
10110 10111 10112 10113
}

module_init(vmx_init)
module_exit(vmx_exit)