svm.c 80.1 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * AMD SVM support
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
16 17
#include <linux/kvm_host.h>

18
#include "irq.h"
19
#include "mmu.h"
20
#include "kvm_cache_regs.h"
21
#include "x86.h"
A
Avi Kivity 已提交
22

A
Avi Kivity 已提交
23
#include <linux/module.h>
24
#include <linux/kernel.h>
A
Avi Kivity 已提交
25 26
#include <linux/vmalloc.h>
#include <linux/highmem.h>
A
Alexey Dobriyan 已提交
27
#include <linux/sched.h>
28
#include <linux/ftrace_event.h>
29
#include <linux/slab.h>
A
Avi Kivity 已提交
30

A
Avi Kivity 已提交
31
#include <asm/desc.h>
A
Avi Kivity 已提交
32

33
#include <asm/virtext.h>
34
#include "trace.h"
35

36 37
#define __ex(x) __kvm_handle_fault_on_reboot(x)

A
Avi Kivity 已提交
38 39 40 41 42 43 44 45 46
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

#define IOPM_ALLOC_ORDER 2
#define MSRPM_ALLOC_ORDER 1

#define SEG_TYPE_LDT 2
#define SEG_TYPE_BUSY_TSS16 3

47 48
#define SVM_FEATURE_NPT  (1 << 0)
#define SVM_FEATURE_LBRV (1 << 1)
A
Amit Shah 已提交
49
#define SVM_FEATURE_SVML (1 << 2)
50
#define SVM_FEATURE_NRIP (1 << 3)
51
#define SVM_FEATURE_PAUSE_FILTER (1 << 10)
52

53 54 55 56
#define NESTED_EXIT_HOST	0	/* Exit handled on host level */
#define NESTED_EXIT_DONE	1	/* Exit caused nested vmexit  */
#define NESTED_EXIT_CONTINUE	2	/* Further checks needed      */

57 58
#define DEBUGCTL_RESERVED_BITS (~(0x3fULL))

A
Avi Kivity 已提交
59 60 61 62 63 64 65 66 67 68 69 70
static const u32 host_save_user_msrs[] = {
#ifdef CONFIG_X86_64
	MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
	MSR_FS_BASE,
#endif
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
};

#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)

struct kvm_vcpu;

71 72 73 74 75 76 77 78 79 80
struct nested_state {
	struct vmcb *hsave;
	u64 hsave_msr;
	u64 vmcb;

	/* These are the merged vectors */
	u32 *msrpm;

	/* gpa pointers to the real vectors */
	u64 vmcb_msrpm;
J
Joerg Roedel 已提交
81

82 83 84
	/* A VMEXIT is required but not yet emulated */
	bool exit_required;

J
Joerg Roedel 已提交
85 86 87 88 89 90 91 92
	/* cache for intercepts of the guest */
	u16 intercept_cr_read;
	u16 intercept_cr_write;
	u16 intercept_dr_read;
	u16 intercept_dr_write;
	u32 intercept_exceptions;
	u64 intercept;

93 94
};

A
Avi Kivity 已提交
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
struct vcpu_svm {
	struct kvm_vcpu vcpu;
	struct vmcb *vmcb;
	unsigned long vmcb_pa;
	struct svm_cpu_data *svm_data;
	uint64_t asid_generation;
	uint64_t sysenter_esp;
	uint64_t sysenter_eip;

	u64 next_rip;

	u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
	u64 host_gs_base;

	u32 *msrpm;

111
	struct nested_state nested;
J
Jan Kiszka 已提交
112 113

	bool nmi_singlestep;
114 115 116

	unsigned int3_injected;
	unsigned long int3_rip;
A
Avi Kivity 已提交
117 118
};

119 120 121 122
/* enable NPT for AMD64 and X86 with PAE */
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
static bool npt_enabled = true;
#else
J
Joerg Roedel 已提交
123
static bool npt_enabled;
124
#endif
125 126 127
static int npt = 1;

module_param(npt, int, S_IRUGO);
128

129
static int nested = 1;
130 131
module_param(nested, int, S_IRUGO);

132
static void svm_flush_tlb(struct kvm_vcpu *vcpu);
133
static void svm_complete_interrupts(struct vcpu_svm *svm);
134

135
static int nested_svm_exit_handled(struct vcpu_svm *svm);
136
static int nested_svm_intercept(struct vcpu_svm *svm);
137 138 139 140
static int nested_svm_vmexit(struct vcpu_svm *svm);
static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
				      bool has_error_code, u32 error_code);

141 142
static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
143
	return container_of(vcpu, struct vcpu_svm, vcpu);
144 145
}

A
Alexander Graf 已提交
146 147
static inline bool is_nested(struct vcpu_svm *svm)
{
148
	return svm->nested.vmcb;
A
Alexander Graf 已提交
149 150
}

151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
static inline void enable_gif(struct vcpu_svm *svm)
{
	svm->vcpu.arch.hflags |= HF_GIF_MASK;
}

static inline void disable_gif(struct vcpu_svm *svm)
{
	svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
}

static inline bool gif_set(struct vcpu_svm *svm)
{
	return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
}

166
static unsigned long iopm_base;
A
Avi Kivity 已提交
167 168 169 170

struct kvm_ldttss_desc {
	u16 limit0;
	u16 base0;
J
Joerg Roedel 已提交
171 172
	unsigned base1:8, type:5, dpl:2, p:1;
	unsigned limit1:4, zero0:3, g:1, base2:8;
A
Avi Kivity 已提交
173 174 175 176 177 178 179
	u32 base3;
	u32 zero1;
} __attribute__((packed));

struct svm_cpu_data {
	int cpu;

A
Avi Kivity 已提交
180 181 182
	u64 asid_generation;
	u32 max_asid;
	u32 next_asid;
A
Avi Kivity 已提交
183 184 185 186 187 188
	struct kvm_ldttss_desc *tss_desc;

	struct page *save_area;
};

static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
189
static uint32_t svm_features;
A
Avi Kivity 已提交
190 191 192 193 194 195 196 197

struct svm_init_data {
	int cpu;
	int r;
};

static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};

198
#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
A
Avi Kivity 已提交
199 200 201 202 203
#define MSRS_RANGE_SIZE 2048
#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)

#define MAX_INST_SIZE 15

204 205 206 207 208
static inline u32 svm_has(u32 feat)
{
	return svm_features & feat;
}

A
Avi Kivity 已提交
209 210
static inline void clgi(void)
{
211
	asm volatile (__ex(SVM_CLGI));
A
Avi Kivity 已提交
212 213 214 215
}

static inline void stgi(void)
{
216
	asm volatile (__ex(SVM_STGI));
A
Avi Kivity 已提交
217 218 219 220
}

static inline void invlpga(unsigned long addr, u32 asid)
{
J
Joerg Roedel 已提交
221
	asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
A
Avi Kivity 已提交
222 223 224 225
}

static inline void force_new_asid(struct kvm_vcpu *vcpu)
{
226
	to_svm(vcpu)->asid_generation--;
A
Avi Kivity 已提交
227 228 229 230 231 232 233 234 235
}

static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
{
	force_new_asid(vcpu);
}

static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
236
	if (!npt_enabled && !(efer & EFER_LMA))
237
		efer &= ~EFER_LME;
A
Avi Kivity 已提交
238

239
	to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
240
	vcpu->arch.efer = efer;
A
Avi Kivity 已提交
241 242 243 244 245 246 247 248
}

static int is_external_interrupt(u32 info)
{
	info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
	return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
}

249 250 251 252 253 254
static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u32 ret = 0;

	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
255
		ret |= KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
256 257 258 259 260 261 262 263 264 265 266 267 268 269
	return ret & mask;
}

static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (mask == 0)
		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
	else
		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;

}

A
Avi Kivity 已提交
270 271
static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
272 273 274
	struct vcpu_svm *svm = to_svm(vcpu);

	if (!svm->next_rip) {
A
Avi Kivity 已提交
275
		if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) !=
276 277
				EMULATE_DONE)
			printk(KERN_DEBUG "%s: NOP\n", __func__);
A
Avi Kivity 已提交
278 279
		return;
	}
280 281 282
	if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
		printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
		       __func__, kvm_rip_read(vcpu), svm->next_rip);
A
Avi Kivity 已提交
283

284
	kvm_rip_write(vcpu, svm->next_rip);
285
	svm_set_interrupt_shadow(vcpu, 0);
A
Avi Kivity 已提交
286 287
}

J
Jan Kiszka 已提交
288 289 290 291 292
static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
				bool has_error_code, u32 error_code)
{
	struct vcpu_svm *svm = to_svm(vcpu);

J
Joerg Roedel 已提交
293 294 295 296
	/*
	 * If we are within a nested VM we'd better #VMEXIT and let the guest
	 * handle the exception
	 */
J
Jan Kiszka 已提交
297 298 299
	if (nested_svm_check_exception(svm, nr, has_error_code, error_code))
		return;

300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
	if (nr == BP_VECTOR && !svm_has(SVM_FEATURE_NRIP)) {
		unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);

		/*
		 * For guest debugging where we have to reinject #BP if some
		 * INT3 is guest-owned:
		 * Emulate nRIP by moving RIP forward. Will fail if injection
		 * raises a fault that is not intercepted. Still better than
		 * failing in all cases.
		 */
		skip_emulated_instruction(&svm->vcpu);
		rip = kvm_rip_read(&svm->vcpu);
		svm->int3_rip = rip + svm->vmcb->save.cs.base;
		svm->int3_injected = rip - old_rip;
	}

J
Jan Kiszka 已提交
316 317 318 319 320 321 322
	svm->vmcb->control.event_inj = nr
		| SVM_EVTINJ_VALID
		| (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
		| SVM_EVTINJ_TYPE_EXEPT;
	svm->vmcb->control.event_inj_err = error_code;
}

A
Avi Kivity 已提交
323 324
static int has_svm(void)
{
325
	const char *msg;
A
Avi Kivity 已提交
326

327
	if (!cpu_has_svm(&msg)) {
J
Joe Perches 已提交
328
		printk(KERN_INFO "has_svm: %s\n", msg);
A
Avi Kivity 已提交
329 330 331 332 333 334 335 336
		return 0;
	}

	return 1;
}

static void svm_hardware_disable(void *garbage)
{
337
	cpu_svm_disable();
A
Avi Kivity 已提交
338 339
}

340
static int svm_hardware_enable(void *garbage)
A
Avi Kivity 已提交
341 342
{

343
	struct svm_cpu_data *sd;
A
Avi Kivity 已提交
344
	uint64_t efer;
345
	struct desc_ptr gdt_descr;
A
Avi Kivity 已提交
346 347 348
	struct desc_struct *gdt;
	int me = raw_smp_processor_id();

349 350 351 352
	rdmsrl(MSR_EFER, efer);
	if (efer & EFER_SVME)
		return -EBUSY;

A
Avi Kivity 已提交
353
	if (!has_svm()) {
354 355
		printk(KERN_ERR "svm_hardware_enable: err EOPNOTSUPP on %d\n",
		       me);
356
		return -EINVAL;
A
Avi Kivity 已提交
357
	}
358
	sd = per_cpu(svm_data, me);
A
Avi Kivity 已提交
359

360
	if (!sd) {
361
		printk(KERN_ERR "svm_hardware_enable: svm_data is NULL on %d\n",
A
Avi Kivity 已提交
362
		       me);
363
		return -EINVAL;
A
Avi Kivity 已提交
364 365
	}

366 367 368
	sd->asid_generation = 1;
	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
	sd->next_asid = sd->max_asid + 1;
A
Avi Kivity 已提交
369

370
	kvm_get_gdt(&gdt_descr);
371
	gdt = (struct desc_struct *)gdt_descr.address;
372
	sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
A
Avi Kivity 已提交
373

374
	wrmsrl(MSR_EFER, efer | EFER_SVME);
A
Avi Kivity 已提交
375

376
	wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
377 378

	return 0;
A
Avi Kivity 已提交
379 380
}

381 382
static void svm_cpu_uninit(int cpu)
{
383
	struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
384

385
	if (!sd)
386 387 388
		return;

	per_cpu(svm_data, raw_smp_processor_id()) = NULL;
389 390
	__free_page(sd->save_area);
	kfree(sd);
391 392
}

A
Avi Kivity 已提交
393 394
static int svm_cpu_init(int cpu)
{
395
	struct svm_cpu_data *sd;
A
Avi Kivity 已提交
396 397
	int r;

398 399
	sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
	if (!sd)
A
Avi Kivity 已提交
400
		return -ENOMEM;
401 402
	sd->cpu = cpu;
	sd->save_area = alloc_page(GFP_KERNEL);
A
Avi Kivity 已提交
403
	r = -ENOMEM;
404
	if (!sd->save_area)
A
Avi Kivity 已提交
405 406
		goto err_1;

407
	per_cpu(svm_data, cpu) = sd;
A
Avi Kivity 已提交
408 409 410 411

	return 0;

err_1:
412
	kfree(sd);
A
Avi Kivity 已提交
413 414 415 416
	return r;

}

417 418
static void set_msr_interception(u32 *msrpm, unsigned msr,
				 int read, int write)
A
Avi Kivity 已提交
419 420 421 422 423 424 425 426 427 428 429 430 431 432
{
	int i;

	for (i = 0; i < NUM_MSR_MAPS; i++) {
		if (msr >= msrpm_ranges[i] &&
		    msr < msrpm_ranges[i] + MSRS_IN_RANGE) {
			u32 msr_offset = (i * MSRS_IN_RANGE + msr -
					  msrpm_ranges[i]) * 2;

			u32 *base = msrpm + (msr_offset / 32);
			u32 msr_shift = msr_offset % 32;
			u32 mask = ((write) ? 0 : 2) | ((read) ? 0 : 1);
			*base = (*base & ~(0x3 << msr_shift)) |
				(mask << msr_shift);
433
			return;
A
Avi Kivity 已提交
434 435
		}
	}
436
	BUG();
A
Avi Kivity 已提交
437 438
}

439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
static void svm_vcpu_init_msrpm(u32 *msrpm)
{
	memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));

#ifdef CONFIG_X86_64
	set_msr_interception(msrpm, MSR_GS_BASE, 1, 1);
	set_msr_interception(msrpm, MSR_FS_BASE, 1, 1);
	set_msr_interception(msrpm, MSR_KERNEL_GS_BASE, 1, 1);
	set_msr_interception(msrpm, MSR_LSTAR, 1, 1);
	set_msr_interception(msrpm, MSR_CSTAR, 1, 1);
	set_msr_interception(msrpm, MSR_SYSCALL_MASK, 1, 1);
#endif
	set_msr_interception(msrpm, MSR_K6_STAR, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_SYSENTER_CS, 1, 1);
}

455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
static void svm_enable_lbrv(struct vcpu_svm *svm)
{
	u32 *msrpm = svm->msrpm;

	svm->vmcb->control.lbr_ctl = 1;
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
}

static void svm_disable_lbrv(struct vcpu_svm *svm)
{
	u32 *msrpm = svm->msrpm;

	svm->vmcb->control.lbr_ctl = 0;
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
}

A
Avi Kivity 已提交
477 478 479 480
static __init int svm_hardware_setup(void)
{
	int cpu;
	struct page *iopm_pages;
481
	void *iopm_va;
A
Avi Kivity 已提交
482 483 484 485 486 487
	int r;

	iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);

	if (!iopm_pages)
		return -ENOMEM;
488 489 490

	iopm_va = page_address(iopm_pages);
	memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
A
Avi Kivity 已提交
491 492
	iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;

493 494 495
	if (boot_cpu_has(X86_FEATURE_NX))
		kvm_enable_efer_bits(EFER_NX);

A
Alexander Graf 已提交
496 497 498
	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
		kvm_enable_efer_bits(EFER_FFXSR);

499 500 501 502 503
	if (nested) {
		printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
		kvm_enable_efer_bits(EFER_SVME);
	}

Z
Zachary Amsden 已提交
504
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
505 506
		r = svm_cpu_init(cpu);
		if (r)
507
			goto err;
A
Avi Kivity 已提交
508
	}
509 510 511

	svm_features = cpuid_edx(SVM_CPUID_FUNC);

512 513 514
	if (!svm_has(SVM_FEATURE_NPT))
		npt_enabled = false;

515 516 517 518 519
	if (npt_enabled && !npt) {
		printk(KERN_INFO "kvm: Nested Paging disabled\n");
		npt_enabled = false;
	}

520
	if (npt_enabled) {
521
		printk(KERN_INFO "kvm: Nested Paging enabled\n");
522
		kvm_enable_tdp();
523 524
	} else
		kvm_disable_tdp();
525

A
Avi Kivity 已提交
526 527
	return 0;

528
err:
A
Avi Kivity 已提交
529 530 531 532 533 534 535
	__free_pages(iopm_pages, IOPM_ALLOC_ORDER);
	iopm_base = 0;
	return r;
}

static __exit void svm_hardware_unsetup(void)
{
536 537
	int cpu;

Z
Zachary Amsden 已提交
538
	for_each_possible_cpu(cpu)
539 540
		svm_cpu_uninit(cpu);

A
Avi Kivity 已提交
541
	__free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
542
	iopm_base = 0;
A
Avi Kivity 已提交
543 544 545 546 547 548
}

static void init_seg(struct vmcb_seg *seg)
{
	seg->selector = 0;
	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
J
Joerg Roedel 已提交
549
		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
A
Avi Kivity 已提交
550 551 552 553 554 555 556 557 558 559 560 561
	seg->limit = 0xffff;
	seg->base = 0;
}

static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
{
	seg->selector = 0;
	seg->attrib = SVM_SELECTOR_P_MASK | type;
	seg->limit = 0xffff;
	seg->base = 0;
}

562
static void init_vmcb(struct vcpu_svm *svm)
A
Avi Kivity 已提交
563
{
564 565
	struct vmcb_control_area *control = &svm->vmcb->control;
	struct vmcb_save_area *save = &svm->vmcb->save;
A
Avi Kivity 已提交
566

567 568
	svm->vcpu.fpu_active = 1;

J
Joerg Roedel 已提交
569
	control->intercept_cr_read =	INTERCEPT_CR0_MASK |
A
Avi Kivity 已提交
570
					INTERCEPT_CR3_MASK |
571
					INTERCEPT_CR4_MASK;
A
Avi Kivity 已提交
572

J
Joerg Roedel 已提交
573
	control->intercept_cr_write =	INTERCEPT_CR0_MASK |
A
Avi Kivity 已提交
574
					INTERCEPT_CR3_MASK |
575 576
					INTERCEPT_CR4_MASK |
					INTERCEPT_CR8_MASK;
A
Avi Kivity 已提交
577

J
Joerg Roedel 已提交
578
	control->intercept_dr_read =	INTERCEPT_DR0_MASK |
A
Avi Kivity 已提交
579 580
					INTERCEPT_DR1_MASK |
					INTERCEPT_DR2_MASK |
581 582 583 584 585
					INTERCEPT_DR3_MASK |
					INTERCEPT_DR4_MASK |
					INTERCEPT_DR5_MASK |
					INTERCEPT_DR6_MASK |
					INTERCEPT_DR7_MASK;
A
Avi Kivity 已提交
586

J
Joerg Roedel 已提交
587
	control->intercept_dr_write =	INTERCEPT_DR0_MASK |
A
Avi Kivity 已提交
588 589 590
					INTERCEPT_DR1_MASK |
					INTERCEPT_DR2_MASK |
					INTERCEPT_DR3_MASK |
591
					INTERCEPT_DR4_MASK |
A
Avi Kivity 已提交
592
					INTERCEPT_DR5_MASK |
593
					INTERCEPT_DR6_MASK |
A
Avi Kivity 已提交
594 595
					INTERCEPT_DR7_MASK;

596
	control->intercept_exceptions = (1 << PF_VECTOR) |
597 598
					(1 << UD_VECTOR) |
					(1 << MC_VECTOR);
A
Avi Kivity 已提交
599 600


J
Joerg Roedel 已提交
601
	control->intercept =	(1ULL << INTERCEPT_INTR) |
A
Avi Kivity 已提交
602
				(1ULL << INTERCEPT_NMI) |
603
				(1ULL << INTERCEPT_SMI) |
A
Avi Kivity 已提交
604
				(1ULL << INTERCEPT_SELECTIVE_CR0) |
A
Avi Kivity 已提交
605
				(1ULL << INTERCEPT_CPUID) |
606
				(1ULL << INTERCEPT_INVD) |
A
Avi Kivity 已提交
607
				(1ULL << INTERCEPT_HLT) |
M
Marcelo Tosatti 已提交
608
				(1ULL << INTERCEPT_INVLPG) |
A
Avi Kivity 已提交
609 610 611 612
				(1ULL << INTERCEPT_INVLPGA) |
				(1ULL << INTERCEPT_IOIO_PROT) |
				(1ULL << INTERCEPT_MSR_PROT) |
				(1ULL << INTERCEPT_TASK_SWITCH) |
613
				(1ULL << INTERCEPT_SHUTDOWN) |
A
Avi Kivity 已提交
614 615 616 617 618 619
				(1ULL << INTERCEPT_VMRUN) |
				(1ULL << INTERCEPT_VMMCALL) |
				(1ULL << INTERCEPT_VMLOAD) |
				(1ULL << INTERCEPT_VMSAVE) |
				(1ULL << INTERCEPT_STGI) |
				(1ULL << INTERCEPT_CLGI) |
620
				(1ULL << INTERCEPT_SKINIT) |
621
				(1ULL << INTERCEPT_WBINVD) |
622 623
				(1ULL << INTERCEPT_MONITOR) |
				(1ULL << INTERCEPT_MWAIT);
A
Avi Kivity 已提交
624 625

	control->iopm_base_pa = iopm_base;
626
	control->msrpm_base_pa = __pa(svm->msrpm);
627
	control->tsc_offset = 0;
A
Avi Kivity 已提交
628 629 630 631 632 633 634 635 636 637 638 639 640
	control->int_ctl = V_INTR_MASKING_MASK;

	init_seg(&save->es);
	init_seg(&save->ss);
	init_seg(&save->ds);
	init_seg(&save->fs);
	init_seg(&save->gs);

	save->cs.selector = 0xf000;
	/* Executable/Readable Code Segment */
	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
	save->cs.limit = 0xffff;
641 642 643 644 645 646 647
	/*
	 * cs.base should really be 0xffff0000, but vmx can't handle that, so
	 * be consistent with it.
	 *
	 * Replace when we have real mode working for vmx.
	 */
	save->cs.base = 0xf0000;
A
Avi Kivity 已提交
648 649 650 651 652 653 654

	save->gdtr.limit = 0xffff;
	save->idtr.limit = 0xffff;

	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);

655
	save->efer = EFER_SVME;
M
Mike Day 已提交
656
	save->dr6 = 0xffff0ff0;
A
Avi Kivity 已提交
657 658 659
	save->dr7 = 0x400;
	save->rflags = 2;
	save->rip = 0x0000fff0;
660
	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
A
Avi Kivity 已提交
661

J
Joerg Roedel 已提交
662 663
	/*
	 * This is the guest-visible cr0 value.
664
	 * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
A
Avi Kivity 已提交
665
	 */
666 667 668
	svm->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
	kvm_set_cr0(&svm->vcpu, svm->vcpu.arch.cr0);

669
	save->cr4 = X86_CR4_PAE;
A
Avi Kivity 已提交
670
	/* rdx = ?? */
671 672 673 674

	if (npt_enabled) {
		/* Setup VMCB for Nested Paging */
		control->nested_ctl = 1;
M
Marcelo Tosatti 已提交
675 676
		control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) |
					(1ULL << INTERCEPT_INVLPG));
677
		control->intercept_exceptions &= ~(1 << PF_VECTOR);
678 679
		control->intercept_cr_read &= ~INTERCEPT_CR3_MASK;
		control->intercept_cr_write &= ~INTERCEPT_CR3_MASK;
680 681 682 683
		save->g_pat = 0x0007040600070406ULL;
		save->cr3 = 0;
		save->cr4 = 0;
	}
684
	force_new_asid(&svm->vcpu);
685

686
	svm->nested.vmcb = 0;
687 688
	svm->vcpu.arch.hflags = 0;

689 690 691 692 693
	if (svm_has(SVM_FEATURE_PAUSE_FILTER)) {
		control->pause_filter_count = 3000;
		control->intercept |= (1ULL << INTERCEPT_PAUSE);
	}

694
	enable_gif(svm);
A
Avi Kivity 已提交
695 696
}

697
static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
698 699 700
{
	struct vcpu_svm *svm = to_svm(vcpu);

701
	init_vmcb(svm);
A
Avi Kivity 已提交
702

703
	if (!kvm_vcpu_is_bsp(vcpu)) {
704
		kvm_rip_write(vcpu, 0);
705 706
		svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;
		svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;
A
Avi Kivity 已提交
707
	}
708 709
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;
710 711

	return 0;
712 713
}

R
Rusty Russell 已提交
714
static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
A
Avi Kivity 已提交
715
{
716
	struct vcpu_svm *svm;
A
Avi Kivity 已提交
717
	struct page *page;
718
	struct page *msrpm_pages;
A
Alexander Graf 已提交
719
	struct page *hsave_page;
A
Alexander Graf 已提交
720
	struct page *nested_msrpm_pages;
R
Rusty Russell 已提交
721
	int err;
A
Avi Kivity 已提交
722

723
	svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
R
Rusty Russell 已提交
724 725 726 727 728 729 730 731 732
	if (!svm) {
		err = -ENOMEM;
		goto out;
	}

	err = kvm_vcpu_init(&svm->vcpu, kvm, id);
	if (err)
		goto free_svm;

733
	err = -ENOMEM;
A
Avi Kivity 已提交
734
	page = alloc_page(GFP_KERNEL);
735
	if (!page)
R
Rusty Russell 已提交
736
		goto uninit;
A
Avi Kivity 已提交
737

738 739
	msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
	if (!msrpm_pages)
740
		goto free_page1;
A
Alexander Graf 已提交
741 742 743

	nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
	if (!nested_msrpm_pages)
744
		goto free_page2;
745

A
Alexander Graf 已提交
746 747
	hsave_page = alloc_page(GFP_KERNEL);
	if (!hsave_page)
748 749
		goto free_page3;

750
	svm->nested.hsave = page_address(hsave_page);
A
Alexander Graf 已提交
751

752 753 754
	svm->msrpm = page_address(msrpm_pages);
	svm_vcpu_init_msrpm(svm->msrpm);

755
	svm->nested.msrpm = page_address(nested_msrpm_pages);
A
Alexander Graf 已提交
756

757 758 759 760
	svm->vmcb = page_address(page);
	clear_page(svm->vmcb);
	svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
	svm->asid_generation = 0;
761
	init_vmcb(svm);
762

R
Rusty Russell 已提交
763
	fx_init(&svm->vcpu);
764
	svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
765
	if (kvm_vcpu_is_bsp(&svm->vcpu))
766
		svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
A
Avi Kivity 已提交
767

R
Rusty Russell 已提交
768
	return &svm->vcpu;
769

770 771 772 773 774 775
free_page3:
	__free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
free_page2:
	__free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
free_page1:
	__free_page(page);
R
Rusty Russell 已提交
776 777 778
uninit:
	kvm_vcpu_uninit(&svm->vcpu);
free_svm:
779
	kmem_cache_free(kvm_vcpu_cache, svm);
R
Rusty Russell 已提交
780 781
out:
	return ERR_PTR(err);
A
Avi Kivity 已提交
782 783 784 785
}

static void svm_free_vcpu(struct kvm_vcpu *vcpu)
{
786 787
	struct vcpu_svm *svm = to_svm(vcpu);

R
Rusty Russell 已提交
788
	__free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
789
	__free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
790 791
	__free_page(virt_to_page(svm->nested.hsave));
	__free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
R
Rusty Russell 已提交
792
	kvm_vcpu_uninit(vcpu);
793
	kmem_cache_free(kvm_vcpu_cache, svm);
A
Avi Kivity 已提交
794 795
}

796
static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
A
Avi Kivity 已提交
797
{
798
	struct vcpu_svm *svm = to_svm(vcpu);
799
	int i;
800 801

	if (unlikely(cpu != vcpu->cpu)) {
802
		u64 delta;
803

804 805 806 807 808 809 810 811 812 813
		if (check_tsc_unstable()) {
			/*
			 * Make sure that the guest sees a monotonically
			 * increasing TSC.
			 */
			delta = vcpu->arch.host_tsc - native_read_tsc();
			svm->vmcb->control.tsc_offset += delta;
			if (is_nested(svm))
				svm->nested.hsave->control.tsc_offset += delta;
		}
814
		vcpu->cpu = cpu;
M
Marcelo Tosatti 已提交
815
		kvm_migrate_timers(vcpu);
816
		svm->asid_generation = 0;
817
	}
818 819

	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
820
		rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
A
Avi Kivity 已提交
821 822 823 824
}

static void svm_vcpu_put(struct kvm_vcpu *vcpu)
{
825
	struct vcpu_svm *svm = to_svm(vcpu);
826 827
	int i;

828
	++vcpu->stat.host_state_reload;
829
	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
830
		wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
831

832
	vcpu->arch.host_tsc = native_read_tsc();
A
Avi Kivity 已提交
833 834 835 836
}

static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
{
837
	return to_svm(vcpu)->vmcb->save.rflags;
A
Avi Kivity 已提交
838 839 840 841
}

static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
842
	to_svm(vcpu)->vmcb->save.rflags = rflags;
A
Avi Kivity 已提交
843 844
}

A
Avi Kivity 已提交
845 846 847 848 849 850 851 852 853 854 855 856
static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
{
	switch (reg) {
	case VCPU_EXREG_PDPTR:
		BUG_ON(!npt_enabled);
		load_pdptrs(vcpu, vcpu->arch.cr3);
		break;
	default:
		BUG();
	}
}

857 858 859 860 861 862 863 864 865 866
static void svm_set_vintr(struct vcpu_svm *svm)
{
	svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR;
}

static void svm_clear_vintr(struct vcpu_svm *svm)
{
	svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
}

A
Avi Kivity 已提交
867 868
static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
{
869
	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
A
Avi Kivity 已提交
870 871 872 873 874 875 876 877 878 879 880 881

	switch (seg) {
	case VCPU_SREG_CS: return &save->cs;
	case VCPU_SREG_DS: return &save->ds;
	case VCPU_SREG_ES: return &save->es;
	case VCPU_SREG_FS: return &save->fs;
	case VCPU_SREG_GS: return &save->gs;
	case VCPU_SREG_SS: return &save->ss;
	case VCPU_SREG_TR: return &save->tr;
	case VCPU_SREG_LDTR: return &save->ldtr;
	}
	BUG();
A
Al Viro 已提交
882
	return NULL;
A
Avi Kivity 已提交
883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907
}

static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	return s->base;
}

static void svm_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	var->base = s->base;
	var->limit = s->limit;
	var->selector = s->selector;
	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
	var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1;
908

J
Joerg Roedel 已提交
909 910
	/*
	 * AMD's VMCB does not have an explicit unusable field, so emulate it
911 912 913 914
	 * for cross vendor migration purposes by "not present"
	 */
	var->unusable = !var->present || (var->type == 0);

915 916 917 918 919 920 921
	switch (seg) {
	case VCPU_SREG_CS:
		/*
		 * SVM always stores 0 for the 'G' bit in the CS selector in
		 * the VMCB on a VMEXIT. This hurts cross-vendor migration:
		 * Intel's VMENTRY has a check on the 'G' bit.
		 */
922
		var->g = s->limit > 0xfffff;
923 924 925 926 927 928
		break;
	case VCPU_SREG_TR:
		/*
		 * Work around a bug where the busy flag in the tr selector
		 * isn't exposed
		 */
929
		var->type |= 0x2;
930 931 932 933 934 935 936 937 938 939 940 941 942 943 944
		break;
	case VCPU_SREG_DS:
	case VCPU_SREG_ES:
	case VCPU_SREG_FS:
	case VCPU_SREG_GS:
		/*
		 * The accessed bit must always be set in the segment
		 * descriptor cache, although it can be cleared in the
		 * descriptor, the cached bit always remains at 1. Since
		 * Intel has a check on this, set it here to support
		 * cross-vendor migration.
		 */
		if (!var->unusable)
			var->type |= 0x1;
		break;
945
	case VCPU_SREG_SS:
J
Joerg Roedel 已提交
946 947
		/*
		 * On AMD CPUs sometimes the DB bit in the segment
948 949 950 951 952 953 954
		 * descriptor is left as 1, although the whole segment has
		 * been made unusable. Clear it here to pass an Intel VMX
		 * entry check when cross vendor migrating.
		 */
		if (var->unusable)
			var->db = 0;
		break;
955
	}
A
Avi Kivity 已提交
956 957
}

958 959 960 961 962 963 964
static int svm_get_cpl(struct kvm_vcpu *vcpu)
{
	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;

	return save->cpl;
}

965
static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
966
{
967 968
	struct vcpu_svm *svm = to_svm(vcpu);

969 970
	dt->size = svm->vmcb->save.idtr.limit;
	dt->address = svm->vmcb->save.idtr.base;
A
Avi Kivity 已提交
971 972
}

973
static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
974
{
975 976
	struct vcpu_svm *svm = to_svm(vcpu);

977 978
	svm->vmcb->save.idtr.limit = dt->size;
	svm->vmcb->save.idtr.base = dt->address ;
A
Avi Kivity 已提交
979 980
}

981
static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
982
{
983 984
	struct vcpu_svm *svm = to_svm(vcpu);

985 986
	dt->size = svm->vmcb->save.gdtr.limit;
	dt->address = svm->vmcb->save.gdtr.base;
A
Avi Kivity 已提交
987 988
}

989
static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
990
{
991 992
	struct vcpu_svm *svm = to_svm(vcpu);

993 994
	svm->vmcb->save.gdtr.limit = dt->size;
	svm->vmcb->save.gdtr.base = dt->address ;
A
Avi Kivity 已提交
995 996
}

997 998 999 1000
static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
{
}

1001
static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1002 1003 1004
{
}

A
Avi Kivity 已提交
1005 1006
static void update_cr0_intercept(struct vcpu_svm *svm)
{
1007
	struct vmcb *vmcb = svm->vmcb;
A
Avi Kivity 已提交
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
	ulong gcr0 = svm->vcpu.arch.cr0;
	u64 *hcr0 = &svm->vmcb->save.cr0;

	if (!svm->vcpu.fpu_active)
		*hcr0 |= SVM_CR0_SELECTIVE_MASK;
	else
		*hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
			| (gcr0 & SVM_CR0_SELECTIVE_MASK);


	if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
		vmcb->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK;
		vmcb->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
		if (is_nested(svm)) {
			struct vmcb *hsave = svm->nested.hsave;

			hsave->control.intercept_cr_read  &= ~INTERCEPT_CR0_MASK;
			hsave->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
			vmcb->control.intercept_cr_read  |= svm->nested.intercept_cr_read;
			vmcb->control.intercept_cr_write |= svm->nested.intercept_cr_write;
		}
A
Avi Kivity 已提交
1029 1030 1031
	} else {
		svm->vmcb->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
		svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
1032 1033 1034 1035 1036 1037
		if (is_nested(svm)) {
			struct vmcb *hsave = svm->nested.hsave;

			hsave->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
			hsave->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
		}
A
Avi Kivity 已提交
1038 1039 1040
	}
}

A
Avi Kivity 已提交
1041 1042
static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
1043 1044
	struct vcpu_svm *svm = to_svm(vcpu);

1045
#ifdef CONFIG_X86_64
1046
	if (vcpu->arch.efer & EFER_LME) {
1047
		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1048
			vcpu->arch.efer |= EFER_LMA;
1049
			svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
A
Avi Kivity 已提交
1050 1051
		}

M
Mike Day 已提交
1052
		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1053
			vcpu->arch.efer &= ~EFER_LMA;
1054
			svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
A
Avi Kivity 已提交
1055 1056 1057
		}
	}
#endif
1058
	vcpu->arch.cr0 = cr0;
1059 1060 1061

	if (!npt_enabled)
		cr0 |= X86_CR0_PG | X86_CR0_WP;
1062 1063

	if (!vcpu->fpu_active)
J
Joerg Roedel 已提交
1064
		cr0 |= X86_CR0_TS;
1065 1066 1067 1068 1069 1070
	/*
	 * re-enable caching here because the QEMU bios
	 * does not do it - this results in some delay at
	 * reboot
	 */
	cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1071
	svm->vmcb->save.cr0 = cr0;
A
Avi Kivity 已提交
1072
	update_cr0_intercept(svm);
A
Avi Kivity 已提交
1073 1074 1075 1076
}

static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
1077
	unsigned long host_cr4_mce = read_cr4() & X86_CR4_MCE;
1078 1079 1080 1081
	unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;

	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
		force_new_asid(vcpu);
1082

1083 1084 1085
	vcpu->arch.cr4 = cr4;
	if (!npt_enabled)
		cr4 |= X86_CR4_PAE;
1086
	cr4 |= host_cr4_mce;
1087
	to_svm(vcpu)->vmcb->save.cr4 = cr4;
A
Avi Kivity 已提交
1088 1089 1090 1091 1092
}

static void svm_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
1093
	struct vcpu_svm *svm = to_svm(vcpu);
A
Avi Kivity 已提交
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	s->base = var->base;
	s->limit = var->limit;
	s->selector = var->selector;
	if (var->unusable)
		s->attrib = 0;
	else {
		s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
		s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
		s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
		s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
		s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
		s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
		s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
		s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
	}
	if (seg == VCPU_SREG_CS)
1112 1113
		svm->vmcb->save.cpl
			= (svm->vmcb->save.cs.attrib
A
Avi Kivity 已提交
1114 1115 1116 1117
			   >> SVM_SELECTOR_DPL_SHIFT) & 3;

}

1118
static void update_db_intercept(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1119
{
J
Jan Kiszka 已提交
1120 1121 1122 1123
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->control.intercept_exceptions &=
		~((1 << DB_VECTOR) | (1 << BP_VECTOR));
1124

J
Jan Kiszka 已提交
1125
	if (svm->nmi_singlestep)
1126 1127
		svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR);

J
Jan Kiszka 已提交
1128 1129 1130 1131 1132 1133 1134 1135 1136 1137
	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
		if (vcpu->guest_debug &
		    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
			svm->vmcb->control.intercept_exceptions |=
				1 << DB_VECTOR;
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
			svm->vmcb->control.intercept_exceptions |=
				1 << BP_VECTOR;
	} else
		vcpu->guest_debug = 0;
1138 1139
}

1140
static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
1141 1142 1143
{
	struct vcpu_svm *svm = to_svm(vcpu);

1144 1145 1146 1147 1148
	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		svm->vmcb->save.dr7 = dbg->arch.debugreg[7];
	else
		svm->vmcb->save.dr7 = vcpu->arch.dr7;

1149
	update_db_intercept(vcpu);
A
Avi Kivity 已提交
1150 1151 1152 1153
}

static void load_host_msrs(struct kvm_vcpu *vcpu)
{
1154
#ifdef CONFIG_X86_64
1155
	wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
1156
#endif
A
Avi Kivity 已提交
1157 1158 1159 1160
}

static void save_host_msrs(struct kvm_vcpu *vcpu)
{
1161
#ifdef CONFIG_X86_64
1162
	rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
1163
#endif
A
Avi Kivity 已提交
1164 1165
}

1166
static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
A
Avi Kivity 已提交
1167
{
1168 1169 1170
	if (sd->next_asid > sd->max_asid) {
		++sd->asid_generation;
		sd->next_asid = 1;
1171
		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
A
Avi Kivity 已提交
1172 1173
	}

1174 1175
	svm->asid_generation = sd->asid_generation;
	svm->vmcb->control.asid = sd->next_asid++;
A
Avi Kivity 已提交
1176 1177
}

1178
static int svm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *dest)
A
Avi Kivity 已提交
1179
{
1180 1181 1182 1183
	struct vcpu_svm *svm = to_svm(vcpu);

	switch (dr) {
	case 0 ... 3:
1184
		*dest = vcpu->arch.db[dr];
1185
		break;
1186 1187 1188 1189
	case 4:
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
			return EMULATE_FAIL; /* will re-inject UD */
		/* fall through */
1190 1191
	case 6:
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1192
			*dest = vcpu->arch.dr6;
1193
		else
1194
			*dest = svm->vmcb->save.dr6;
1195
		break;
1196 1197 1198 1199
	case 5:
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
			return EMULATE_FAIL; /* will re-inject UD */
		/* fall through */
1200 1201
	case 7:
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1202
			*dest = vcpu->arch.dr7;
1203
		else
1204
			*dest = svm->vmcb->save.dr7;
1205 1206 1207
		break;
	}

1208
	return EMULATE_DONE;
A
Avi Kivity 已提交
1209 1210
}

1211
static int svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value)
A
Avi Kivity 已提交
1212
{
1213 1214
	struct vcpu_svm *svm = to_svm(vcpu);

A
Avi Kivity 已提交
1215 1216
	switch (dr) {
	case 0 ... 3:
1217 1218 1219
		vcpu->arch.db[dr] = value;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = value;
1220 1221 1222 1223 1224
		break;
	case 4:
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
			return EMULATE_FAIL; /* will re-inject UD */
		/* fall through */
1225 1226
	case 6:
		vcpu->arch.dr6 = (value & DR6_VOLATILE) | DR6_FIXED_1;
1227 1228 1229 1230 1231
		break;
	case 5:
		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
			return EMULATE_FAIL; /* will re-inject UD */
		/* fall through */
1232 1233 1234 1235 1236 1237
	case 7:
		vcpu->arch.dr7 = (value & DR7_VOLATILE) | DR7_FIXED_1;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
			svm->vmcb->save.dr7 = vcpu->arch.dr7;
			vcpu->arch.switch_db_regs = (value & DR7_BP_EN_MASK);
		}
1238
		break;
A
Avi Kivity 已提交
1239
	}
1240 1241

	return EMULATE_DONE;
A
Avi Kivity 已提交
1242 1243
}

A
Avi Kivity 已提交
1244
static int pf_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
1245 1246 1247 1248
{
	u64 fault_address;
	u32 error_code;

1249 1250
	fault_address  = svm->vmcb->control.exit_info_2;
	error_code = svm->vmcb->control.exit_info_1;
1251

1252
	trace_kvm_page_fault(fault_address, error_code);
1253 1254
	if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
		kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
1255
	return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
A
Avi Kivity 已提交
1256 1257
}

A
Avi Kivity 已提交
1258
static int db_interception(struct vcpu_svm *svm)
J
Jan Kiszka 已提交
1259
{
A
Avi Kivity 已提交
1260 1261
	struct kvm_run *kvm_run = svm->vcpu.run;

J
Jan Kiszka 已提交
1262
	if (!(svm->vcpu.guest_debug &
1263
	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
J
Jan Kiszka 已提交
1264
		!svm->nmi_singlestep) {
J
Jan Kiszka 已提交
1265 1266 1267
		kvm_queue_exception(&svm->vcpu, DB_VECTOR);
		return 1;
	}
1268

J
Jan Kiszka 已提交
1269 1270
	if (svm->nmi_singlestep) {
		svm->nmi_singlestep = false;
1271 1272 1273 1274 1275 1276 1277
		if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
			svm->vmcb->save.rflags &=
				~(X86_EFLAGS_TF | X86_EFLAGS_RF);
		update_db_intercept(&svm->vcpu);
	}

	if (svm->vcpu.guest_debug &
J
Joerg Roedel 已提交
1278
	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1279 1280 1281 1282 1283 1284 1285 1286
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
		kvm_run->debug.arch.pc =
			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
		kvm_run->debug.arch.exception = DB_VECTOR;
		return 0;
	}

	return 1;
J
Jan Kiszka 已提交
1287 1288
}

A
Avi Kivity 已提交
1289
static int bp_interception(struct vcpu_svm *svm)
J
Jan Kiszka 已提交
1290
{
A
Avi Kivity 已提交
1291 1292
	struct kvm_run *kvm_run = svm->vcpu.run;

J
Jan Kiszka 已提交
1293 1294 1295 1296 1297 1298
	kvm_run->exit_reason = KVM_EXIT_DEBUG;
	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
	kvm_run->debug.arch.exception = BP_VECTOR;
	return 0;
}

A
Avi Kivity 已提交
1299
static int ud_interception(struct vcpu_svm *svm)
1300 1301 1302
{
	int er;

A
Avi Kivity 已提交
1303
	er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD);
1304
	if (er != EMULATE_DONE)
1305
		kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1306 1307 1308
	return 1;
}

A
Avi Kivity 已提交
1309
static void svm_fpu_activate(struct kvm_vcpu *vcpu)
A
Anthony Liguori 已提交
1310
{
A
Avi Kivity 已提交
1311
	struct vcpu_svm *svm = to_svm(vcpu);
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
	u32 excp;

	if (is_nested(svm)) {
		u32 h_excp, n_excp;

		h_excp  = svm->nested.hsave->control.intercept_exceptions;
		n_excp  = svm->nested.intercept_exceptions;
		h_excp &= ~(1 << NM_VECTOR);
		excp    = h_excp | n_excp;
	} else {
		excp  = svm->vmcb->control.intercept_exceptions;
J
Joerg Roedel 已提交
1323
		excp &= ~(1 << NM_VECTOR);
1324 1325 1326 1327
	}

	svm->vmcb->control.intercept_exceptions = excp;

R
Rusty Russell 已提交
1328
	svm->vcpu.fpu_active = 1;
A
Avi Kivity 已提交
1329
	update_cr0_intercept(svm);
A
Avi Kivity 已提交
1330
}
1331

A
Avi Kivity 已提交
1332 1333 1334
static int nm_interception(struct vcpu_svm *svm)
{
	svm_fpu_activate(&svm->vcpu);
1335
	return 1;
A
Anthony Liguori 已提交
1336 1337
}

A
Avi Kivity 已提交
1338
static int mc_interception(struct vcpu_svm *svm)
1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350
{
	/*
	 * On an #MC intercept the MCE handler is not called automatically in
	 * the host. So do it by hand here.
	 */
	asm volatile (
		"int $0x12\n");
	/* not sure if we ever come back to this point */

	return 1;
}

A
Avi Kivity 已提交
1351
static int shutdown_interception(struct vcpu_svm *svm)
1352
{
A
Avi Kivity 已提交
1353 1354
	struct kvm_run *kvm_run = svm->vcpu.run;

1355 1356 1357 1358
	/*
	 * VMCB is undefined after a SHUTDOWN intercept
	 * so reinitialize it.
	 */
1359
	clear_page(svm->vmcb);
1360
	init_vmcb(svm);
1361 1362 1363 1364 1365

	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
	return 0;
}

A
Avi Kivity 已提交
1366
static int io_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
1367
{
M
Mike Day 已提交
1368
	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
1369
	int size, in, string;
1370
	unsigned port;
A
Avi Kivity 已提交
1371

R
Rusty Russell 已提交
1372
	++svm->vcpu.stat.io_exits;
A
Avi Kivity 已提交
1373

1374
	svm->next_rip = svm->vmcb->control.exit_info_2;
A
Avi Kivity 已提交
1375

1376 1377 1378
	string = (io_info & SVM_IOIO_STR_MASK) != 0;

	if (string) {
1379
		if (emulate_instruction(&svm->vcpu,
A
Avi Kivity 已提交
1380
					0, 0, 0) == EMULATE_DO_MMIO)
1381 1382 1383 1384
			return 0;
		return 1;
	}

1385 1386 1387
	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
	port = io_info >> 16;
	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
A
Avi Kivity 已提交
1388

1389
	skip_emulated_instruction(&svm->vcpu);
A
Avi Kivity 已提交
1390
	return kvm_emulate_pio(&svm->vcpu, in, size, port);
A
Avi Kivity 已提交
1391 1392
}

A
Avi Kivity 已提交
1393
static int nmi_interception(struct vcpu_svm *svm)
1394 1395 1396 1397
{
	return 1;
}

A
Avi Kivity 已提交
1398
static int intr_interception(struct vcpu_svm *svm)
1399 1400 1401 1402 1403
{
	++svm->vcpu.stat.irq_exits;
	return 1;
}

A
Avi Kivity 已提交
1404
static int nop_on_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
1405 1406 1407 1408
{
	return 1;
}

A
Avi Kivity 已提交
1409
static int halt_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
1410
{
1411
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
R
Rusty Russell 已提交
1412 1413
	skip_emulated_instruction(&svm->vcpu);
	return kvm_emulate_halt(&svm->vcpu);
A
Avi Kivity 已提交
1414 1415
}

A
Avi Kivity 已提交
1416
static int vmmcall_interception(struct vcpu_svm *svm)
1417
{
1418
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
R
Rusty Russell 已提交
1419
	skip_emulated_instruction(&svm->vcpu);
1420 1421
	kvm_emulate_hypercall(&svm->vcpu);
	return 1;
1422 1423
}

1424 1425
static int nested_svm_check_permissions(struct vcpu_svm *svm)
{
1426
	if (!(svm->vcpu.arch.efer & EFER_SVME)
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439
	    || !is_paging(&svm->vcpu)) {
		kvm_queue_exception(&svm->vcpu, UD_VECTOR);
		return 1;
	}

	if (svm->vmcb->save.cpl) {
		kvm_inject_gp(&svm->vcpu, 0);
		return 1;
	}

       return 0;
}

1440 1441 1442
static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
				      bool has_error_code, u32 error_code)
{
1443 1444
	int vmexit;

1445 1446
	if (!is_nested(svm))
		return 0;
1447

1448 1449 1450 1451 1452
	svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
	svm->vmcb->control.exit_code_hi = 0;
	svm->vmcb->control.exit_info_1 = error_code;
	svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;

1453 1454 1455 1456 1457
	vmexit = nested_svm_intercept(svm);
	if (vmexit == NESTED_EXIT_DONE)
		svm->nested.exit_required = true;

	return vmexit;
1458 1459
}

1460 1461
/* This function returns true if it is save to enable the irq window */
static inline bool nested_svm_intr(struct vcpu_svm *svm)
1462
{
1463
	if (!is_nested(svm))
1464
		return true;
1465

1466
	if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
1467
		return true;
1468

1469
	if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
1470
		return false;
1471

1472 1473
	svm->vmcb->control.exit_code = SVM_EXIT_INTR;

1474 1475 1476 1477 1478 1479 1480 1481
	if (svm->nested.intercept & 1ULL) {
		/*
		 * The #vmexit can't be emulated here directly because this
		 * code path runs with irqs and preemtion disabled. A
		 * #vmexit emulation might sleep. Only signal request for
		 * the #vmexit here.
		 */
		svm->nested.exit_required = true;
1482
		trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
1483
		return false;
1484 1485
	}

1486
	return true;
1487 1488
}

1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
/* This function returns true if it is save to enable the nmi window */
static inline bool nested_svm_nmi(struct vcpu_svm *svm)
{
	if (!is_nested(svm))
		return true;

	if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
		return true;

	svm->vmcb->control.exit_code = SVM_EXIT_NMI;
	svm->nested.exit_required = true;

	return false;
}

1504
static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
1505 1506 1507
{
	struct page *page;

1508 1509
	might_sleep();

1510 1511 1512 1513
	page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT);
	if (is_error_page(page))
		goto error;

1514 1515 1516
	*_page = page;

	return kmap(page);
1517 1518 1519 1520 1521 1522 1523 1524

error:
	kvm_release_page_clean(page);
	kvm_inject_gp(&svm->vcpu, 0);

	return NULL;
}

1525
static void nested_svm_unmap(struct page *page)
1526
{
1527
	kunmap(page);
1528 1529 1530
	kvm_release_page_dirty(page);
}

1531
static bool nested_svm_exit_handled_msr(struct vcpu_svm *svm)
1532 1533
{
	u32 param = svm->vmcb->control.exit_info_1 & 1;
1534 1535 1536
	u32 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
	bool ret = false;
	u32 t0, t1;
1537
	u8 val;
1538

1539 1540 1541
	if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
		return false;

1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557
	switch (msr) {
	case 0 ... 0x1fff:
		t0 = (msr * 2) % 8;
		t1 = msr / 8;
		break;
	case 0xc0000000 ... 0xc0001fff:
		t0 = (8192 + msr - 0xc0000000) * 2;
		t1 = (t0 / 8);
		t0 %= 8;
		break;
	case 0xc0010000 ... 0xc0011fff:
		t0 = (16384 + msr - 0xc0010000) * 2;
		t1 = (t0 / 8);
		t0 %= 8;
		break;
	default:
1558 1559
		ret = true;
		goto out;
1560 1561
	}

1562 1563
	if (!kvm_read_guest(svm->vcpu.kvm, svm->nested.vmcb_msrpm + t1, &val, 1))
		ret = val & ((1 << param) << t0);
1564 1565 1566

out:
	return ret;
1567 1568
}

1569
static int nested_svm_exit_special(struct vcpu_svm *svm)
1570 1571
{
	u32 exit_code = svm->vmcb->control.exit_code;
1572

1573 1574 1575 1576 1577
	switch (exit_code) {
	case SVM_EXIT_INTR:
	case SVM_EXIT_NMI:
		return NESTED_EXIT_HOST;
	case SVM_EXIT_NPF:
J
Joerg Roedel 已提交
1578
		/* For now we are always handling NPFs when using them */
1579 1580 1581 1582
		if (npt_enabled)
			return NESTED_EXIT_HOST;
		break;
	case SVM_EXIT_EXCP_BASE + PF_VECTOR:
J
Joerg Roedel 已提交
1583
		/* When we're shadowing, trap PFs */
1584 1585 1586
		if (!npt_enabled)
			return NESTED_EXIT_HOST;
		break;
1587 1588 1589
	case SVM_EXIT_EXCP_BASE + NM_VECTOR:
		nm_interception(svm);
		break;
1590 1591
	default:
		break;
1592 1593
	}

1594 1595 1596 1597 1598 1599
	return NESTED_EXIT_CONTINUE;
}

/*
 * If this function returns true, this #vmexit was already handled
 */
1600
static int nested_svm_intercept(struct vcpu_svm *svm)
1601 1602 1603 1604
{
	u32 exit_code = svm->vmcb->control.exit_code;
	int vmexit = NESTED_EXIT_HOST;

1605
	switch (exit_code) {
1606
	case SVM_EXIT_MSR:
1607
		vmexit = nested_svm_exit_handled_msr(svm);
1608
		break;
1609 1610
	case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: {
		u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0);
J
Joerg Roedel 已提交
1611
		if (svm->nested.intercept_cr_read & cr_bits)
1612
			vmexit = NESTED_EXIT_DONE;
1613 1614 1615 1616
		break;
	}
	case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: {
		u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0);
J
Joerg Roedel 已提交
1617
		if (svm->nested.intercept_cr_write & cr_bits)
1618
			vmexit = NESTED_EXIT_DONE;
1619 1620 1621 1622
		break;
	}
	case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: {
		u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0);
J
Joerg Roedel 已提交
1623
		if (svm->nested.intercept_dr_read & dr_bits)
1624
			vmexit = NESTED_EXIT_DONE;
1625 1626 1627 1628
		break;
	}
	case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: {
		u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0);
J
Joerg Roedel 已提交
1629
		if (svm->nested.intercept_dr_write & dr_bits)
1630
			vmexit = NESTED_EXIT_DONE;
1631 1632 1633 1634
		break;
	}
	case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
		u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
J
Joerg Roedel 已提交
1635
		if (svm->nested.intercept_exceptions & excp_bits)
1636
			vmexit = NESTED_EXIT_DONE;
1637 1638 1639 1640
		break;
	}
	default: {
		u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
J
Joerg Roedel 已提交
1641
		if (svm->nested.intercept & exit_bits)
1642
			vmexit = NESTED_EXIT_DONE;
1643 1644 1645
	}
	}

1646 1647 1648 1649 1650 1651 1652 1653 1654 1655
	return vmexit;
}

static int nested_svm_exit_handled(struct vcpu_svm *svm)
{
	int vmexit;

	vmexit = nested_svm_intercept(svm);

	if (vmexit == NESTED_EXIT_DONE)
1656 1657 1658
		nested_svm_vmexit(svm);

	return vmexit;
1659 1660
}

1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692
static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
{
	struct vmcb_control_area *dst  = &dst_vmcb->control;
	struct vmcb_control_area *from = &from_vmcb->control;

	dst->intercept_cr_read    = from->intercept_cr_read;
	dst->intercept_cr_write   = from->intercept_cr_write;
	dst->intercept_dr_read    = from->intercept_dr_read;
	dst->intercept_dr_write   = from->intercept_dr_write;
	dst->intercept_exceptions = from->intercept_exceptions;
	dst->intercept            = from->intercept;
	dst->iopm_base_pa         = from->iopm_base_pa;
	dst->msrpm_base_pa        = from->msrpm_base_pa;
	dst->tsc_offset           = from->tsc_offset;
	dst->asid                 = from->asid;
	dst->tlb_ctl              = from->tlb_ctl;
	dst->int_ctl              = from->int_ctl;
	dst->int_vector           = from->int_vector;
	dst->int_state            = from->int_state;
	dst->exit_code            = from->exit_code;
	dst->exit_code_hi         = from->exit_code_hi;
	dst->exit_info_1          = from->exit_info_1;
	dst->exit_info_2          = from->exit_info_2;
	dst->exit_int_info        = from->exit_int_info;
	dst->exit_int_info_err    = from->exit_int_info_err;
	dst->nested_ctl           = from->nested_ctl;
	dst->event_inj            = from->event_inj;
	dst->event_inj_err        = from->event_inj_err;
	dst->nested_cr3           = from->nested_cr3;
	dst->lbr_ctl              = from->lbr_ctl;
}

1693
static int nested_svm_vmexit(struct vcpu_svm *svm)
1694
{
1695
	struct vmcb *nested_vmcb;
1696
	struct vmcb *hsave = svm->nested.hsave;
J
Joerg Roedel 已提交
1697
	struct vmcb *vmcb = svm->vmcb;
1698
	struct page *page;
1699

1700 1701 1702 1703 1704 1705
	trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
				       vmcb->control.exit_info_1,
				       vmcb->control.exit_info_2,
				       vmcb->control.exit_int_info,
				       vmcb->control.exit_int_info_err);

1706
	nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
1707 1708 1709
	if (!nested_vmcb)
		return 1;

1710 1711 1712
	/* Exit nested SVM mode */
	svm->nested.vmcb = 0;

1713
	/* Give the current vmcb to the guest */
J
Joerg Roedel 已提交
1714 1715 1716 1717 1718 1719 1720 1721
	disable_gif(svm);

	nested_vmcb->save.es     = vmcb->save.es;
	nested_vmcb->save.cs     = vmcb->save.cs;
	nested_vmcb->save.ss     = vmcb->save.ss;
	nested_vmcb->save.ds     = vmcb->save.ds;
	nested_vmcb->save.gdtr   = vmcb->save.gdtr;
	nested_vmcb->save.idtr   = vmcb->save.idtr;
1722
	nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
J
Joerg Roedel 已提交
1723 1724
	if (npt_enabled)
		nested_vmcb->save.cr3    = vmcb->save.cr3;
1725 1726
	else
		nested_vmcb->save.cr3    = svm->vcpu.arch.cr3;
J
Joerg Roedel 已提交
1727
	nested_vmcb->save.cr2    = vmcb->save.cr2;
1728
	nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
J
Joerg Roedel 已提交
1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745
	nested_vmcb->save.rflags = vmcb->save.rflags;
	nested_vmcb->save.rip    = vmcb->save.rip;
	nested_vmcb->save.rsp    = vmcb->save.rsp;
	nested_vmcb->save.rax    = vmcb->save.rax;
	nested_vmcb->save.dr7    = vmcb->save.dr7;
	nested_vmcb->save.dr6    = vmcb->save.dr6;
	nested_vmcb->save.cpl    = vmcb->save.cpl;

	nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
	nested_vmcb->control.int_vector        = vmcb->control.int_vector;
	nested_vmcb->control.int_state         = vmcb->control.int_state;
	nested_vmcb->control.exit_code         = vmcb->control.exit_code;
	nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
	nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
	nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
	nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
	nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761

	/*
	 * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
	 * to make sure that we do not lose injected events. So check event_inj
	 * here and copy it to exit_int_info if it is valid.
	 * Exit_int_info and event_inj can't be both valid because the case
	 * below only happens on a VMRUN instruction intercept which has
	 * no valid exit_int_info set.
	 */
	if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
		struct vmcb_control_area *nc = &nested_vmcb->control;

		nc->exit_int_info     = vmcb->control.event_inj;
		nc->exit_int_info_err = vmcb->control.event_inj_err;
	}

J
Joerg Roedel 已提交
1762 1763 1764
	nested_vmcb->control.tlb_ctl           = 0;
	nested_vmcb->control.event_inj         = 0;
	nested_vmcb->control.event_inj_err     = 0;
1765 1766 1767 1768 1769 1770

	/* We always set V_INTR_MASKING and remember the old value in hflags */
	if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
		nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;

	/* Restore the original control entries */
1771
	copy_vmcb_control_area(vmcb, hsave);
1772

1773 1774
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);
1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799

	/* Restore selected save entries */
	svm->vmcb->save.es = hsave->save.es;
	svm->vmcb->save.cs = hsave->save.cs;
	svm->vmcb->save.ss = hsave->save.ss;
	svm->vmcb->save.ds = hsave->save.ds;
	svm->vmcb->save.gdtr = hsave->save.gdtr;
	svm->vmcb->save.idtr = hsave->save.idtr;
	svm->vmcb->save.rflags = hsave->save.rflags;
	svm_set_efer(&svm->vcpu, hsave->save.efer);
	svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
	svm_set_cr4(&svm->vcpu, hsave->save.cr4);
	if (npt_enabled) {
		svm->vmcb->save.cr3 = hsave->save.cr3;
		svm->vcpu.arch.cr3 = hsave->save.cr3;
	} else {
		kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
	}
	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
	svm->vmcb->save.dr7 = 0;
	svm->vmcb->save.cpl = 0;
	svm->vmcb->control.exit_int_info = 0;

1800
	nested_svm_unmap(page);
1801 1802 1803 1804 1805 1806

	kvm_mmu_reset_context(&svm->vcpu);
	kvm_mmu_load(&svm->vcpu);

	return 0;
}
A
Alexander Graf 已提交
1807

1808
static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
A
Alexander Graf 已提交
1809
{
1810
	u32 *nested_msrpm;
1811
	struct page *page;
A
Alexander Graf 已提交
1812
	int i;
1813

1814
	nested_msrpm = nested_svm_map(svm, svm->nested.vmcb_msrpm, &page);
1815 1816 1817
	if (!nested_msrpm)
		return false;

J
Joerg Roedel 已提交
1818
	for (i = 0; i < PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER) / 4; i++)
1819
		svm->nested.msrpm[i] = svm->msrpm[i] | nested_msrpm[i];
1820

1821
	svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
A
Alexander Graf 已提交
1822

1823
	nested_svm_unmap(page);
1824 1825

	return true;
A
Alexander Graf 已提交
1826 1827
}

1828
static bool nested_svm_vmrun(struct vcpu_svm *svm)
A
Alexander Graf 已提交
1829
{
1830
	struct vmcb *nested_vmcb;
1831
	struct vmcb *hsave = svm->nested.hsave;
J
Joerg Roedel 已提交
1832
	struct vmcb *vmcb = svm->vmcb;
1833
	struct page *page;
1834 1835 1836
	u64 vmcb_gpa;

	vmcb_gpa = svm->vmcb->save.rax;
A
Alexander Graf 已提交
1837

1838
	nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
1839 1840 1841
	if (!nested_vmcb)
		return false;

1842 1843 1844 1845 1846 1847
	trace_kvm_nested_vmrun(svm->vmcb->save.rip - 3, svm->nested.vmcb,
			       nested_vmcb->save.rip,
			       nested_vmcb->control.int_ctl,
			       nested_vmcb->control.event_inj,
			       nested_vmcb->control.nested_ctl);

A
Alexander Graf 已提交
1848
	/* Clear internal status */
1849 1850
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);
A
Alexander Graf 已提交
1851

J
Joerg Roedel 已提交
1852 1853 1854 1855
	/*
	 * Save the old vmcb, so we don't need to pick what we save, but can
	 * restore everything when a VMEXIT occurs
	 */
J
Joerg Roedel 已提交
1856 1857 1858 1859 1860 1861
	hsave->save.es     = vmcb->save.es;
	hsave->save.cs     = vmcb->save.cs;
	hsave->save.ss     = vmcb->save.ss;
	hsave->save.ds     = vmcb->save.ds;
	hsave->save.gdtr   = vmcb->save.gdtr;
	hsave->save.idtr   = vmcb->save.idtr;
1862
	hsave->save.efer   = svm->vcpu.arch.efer;
1863
	hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
J
Joerg Roedel 已提交
1864 1865 1866 1867 1868 1869 1870 1871 1872 1873
	hsave->save.cr4    = svm->vcpu.arch.cr4;
	hsave->save.rflags = vmcb->save.rflags;
	hsave->save.rip    = svm->next_rip;
	hsave->save.rsp    = vmcb->save.rsp;
	hsave->save.rax    = vmcb->save.rax;
	if (npt_enabled)
		hsave->save.cr3    = vmcb->save.cr3;
	else
		hsave->save.cr3    = svm->vcpu.arch.cr3;

1874
	copy_vmcb_control_area(hsave, vmcb);
A
Alexander Graf 已提交
1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894

	if (svm->vmcb->save.rflags & X86_EFLAGS_IF)
		svm->vcpu.arch.hflags |= HF_HIF_MASK;
	else
		svm->vcpu.arch.hflags &= ~HF_HIF_MASK;

	/* Load the nested guest state */
	svm->vmcb->save.es = nested_vmcb->save.es;
	svm->vmcb->save.cs = nested_vmcb->save.cs;
	svm->vmcb->save.ss = nested_vmcb->save.ss;
	svm->vmcb->save.ds = nested_vmcb->save.ds;
	svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
	svm->vmcb->save.idtr = nested_vmcb->save.idtr;
	svm->vmcb->save.rflags = nested_vmcb->save.rflags;
	svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
	svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
	svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
	if (npt_enabled) {
		svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
		svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
1895
	} else
A
Alexander Graf 已提交
1896
		kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
1897 1898 1899 1900

	/* Guest paging mode is active - reset mmu */
	kvm_mmu_reset_context(&svm->vcpu);

J
Joerg Roedel 已提交
1901
	svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
A
Alexander Graf 已提交
1902 1903 1904
	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
J
Joerg Roedel 已提交
1905

A
Alexander Graf 已提交
1906 1907 1908 1909 1910 1911 1912 1913
	/* In case we don't even reach vcpu_run, the fields are not updated */
	svm->vmcb->save.rax = nested_vmcb->save.rax;
	svm->vmcb->save.rsp = nested_vmcb->save.rsp;
	svm->vmcb->save.rip = nested_vmcb->save.rip;
	svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
	svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
	svm->vmcb->save.cpl = nested_vmcb->save.cpl;

1914
	svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa;
A
Alexander Graf 已提交
1915

J
Joerg Roedel 已提交
1916 1917 1918 1919 1920 1921 1922 1923
	/* cache intercepts */
	svm->nested.intercept_cr_read    = nested_vmcb->control.intercept_cr_read;
	svm->nested.intercept_cr_write   = nested_vmcb->control.intercept_cr_write;
	svm->nested.intercept_dr_read    = nested_vmcb->control.intercept_dr_read;
	svm->nested.intercept_dr_write   = nested_vmcb->control.intercept_dr_write;
	svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
	svm->nested.intercept            = nested_vmcb->control.intercept;

A
Alexander Graf 已提交
1924 1925 1926 1927 1928 1929 1930
	force_new_asid(&svm->vcpu);
	svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
	if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
		svm->vcpu.arch.hflags |= HF_VINTR_MASK;
	else
		svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;

1931 1932 1933 1934 1935 1936
	if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
		/* We only want the cr8 intercept bits of the guest */
		svm->vmcb->control.intercept_cr_read &= ~INTERCEPT_CR8_MASK;
		svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
	}

J
Joerg Roedel 已提交
1937 1938 1939 1940
	/*
	 * We don't want a nested guest to be more powerful than the guest, so
	 * all intercepts are ORed
	 */
1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954
	svm->vmcb->control.intercept_cr_read |=
		nested_vmcb->control.intercept_cr_read;
	svm->vmcb->control.intercept_cr_write |=
		nested_vmcb->control.intercept_cr_write;
	svm->vmcb->control.intercept_dr_read |=
		nested_vmcb->control.intercept_dr_read;
	svm->vmcb->control.intercept_dr_write |=
		nested_vmcb->control.intercept_dr_write;
	svm->vmcb->control.intercept_exceptions |=
		nested_vmcb->control.intercept_exceptions;

	svm->vmcb->control.intercept |= nested_vmcb->control.intercept;

	svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
A
Alexander Graf 已提交
1955 1956 1957 1958 1959 1960
	svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
	svm->vmcb->control.int_state = nested_vmcb->control.int_state;
	svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
	svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
	svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;

1961
	nested_svm_unmap(page);
1962

1963 1964 1965
	/* nested_vmcb is our indicator if nested SVM is activated */
	svm->nested.vmcb = vmcb_gpa;

1966
	enable_gif(svm);
A
Alexander Graf 已提交
1967

1968
	return true;
A
Alexander Graf 已提交
1969 1970
}

1971
static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986
{
	to_vmcb->save.fs = from_vmcb->save.fs;
	to_vmcb->save.gs = from_vmcb->save.gs;
	to_vmcb->save.tr = from_vmcb->save.tr;
	to_vmcb->save.ldtr = from_vmcb->save.ldtr;
	to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
	to_vmcb->save.star = from_vmcb->save.star;
	to_vmcb->save.lstar = from_vmcb->save.lstar;
	to_vmcb->save.cstar = from_vmcb->save.cstar;
	to_vmcb->save.sfmask = from_vmcb->save.sfmask;
	to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
	to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
	to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
}

A
Avi Kivity 已提交
1987
static int vmload_interception(struct vcpu_svm *svm)
1988
{
1989
	struct vmcb *nested_vmcb;
1990
	struct page *page;
1991

1992 1993 1994 1995 1996 1997
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

1998
	nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
1999 2000 2001 2002
	if (!nested_vmcb)
		return 1;

	nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
2003
	nested_svm_unmap(page);
2004 2005 2006 2007

	return 1;
}

A
Avi Kivity 已提交
2008
static int vmsave_interception(struct vcpu_svm *svm)
2009
{
2010
	struct vmcb *nested_vmcb;
2011
	struct page *page;
2012

2013 2014 2015 2016 2017 2018
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

2019
	nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2020 2021 2022 2023
	if (!nested_vmcb)
		return 1;

	nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
2024
	nested_svm_unmap(page);
2025 2026 2027 2028

	return 1;
}

A
Avi Kivity 已提交
2029
static int vmrun_interception(struct vcpu_svm *svm)
A
Alexander Graf 已提交
2030 2031 2032 2033 2034 2035 2036
{
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

2037
	if (!nested_svm_vmrun(svm))
A
Alexander Graf 已提交
2038 2039
		return 1;

2040
	if (!nested_svm_vmrun_msrpm(svm))
2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052
		goto failed;

	return 1;

failed:

	svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
	svm->vmcb->control.exit_code_hi = 0;
	svm->vmcb->control.exit_info_1  = 0;
	svm->vmcb->control.exit_info_2  = 0;

	nested_svm_vmexit(svm);
A
Alexander Graf 已提交
2053 2054 2055 2056

	return 1;
}

A
Avi Kivity 已提交
2057
static int stgi_interception(struct vcpu_svm *svm)
2058 2059 2060 2061 2062 2063 2064
{
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

2065
	enable_gif(svm);
2066 2067 2068 2069

	return 1;
}

A
Avi Kivity 已提交
2070
static int clgi_interception(struct vcpu_svm *svm)
2071 2072 2073 2074 2075 2076 2077
{
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

2078
	disable_gif(svm);
2079 2080 2081 2082 2083 2084 2085 2086

	/* After a CLGI no interrupts should come */
	svm_clear_vintr(svm);
	svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;

	return 1;
}

A
Avi Kivity 已提交
2087
static int invlpga_interception(struct vcpu_svm *svm)
A
Alexander Graf 已提交
2088 2089 2090
{
	struct kvm_vcpu *vcpu = &svm->vcpu;

2091 2092 2093
	trace_kvm_invlpga(svm->vmcb->save.rip, vcpu->arch.regs[VCPU_REGS_RCX],
			  vcpu->arch.regs[VCPU_REGS_RAX]);

A
Alexander Graf 已提交
2094 2095 2096 2097 2098 2099 2100 2101
	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
	kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]);

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);
	return 1;
}

2102 2103 2104 2105 2106 2107 2108 2109
static int skinit_interception(struct vcpu_svm *svm)
{
	trace_kvm_skinit(svm->vmcb->save.rip, svm->vcpu.arch.regs[VCPU_REGS_RAX]);

	kvm_queue_exception(&svm->vcpu, UD_VECTOR);
	return 1;
}

A
Avi Kivity 已提交
2110
static int invalid_op_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2111
{
2112
	kvm_queue_exception(&svm->vcpu, UD_VECTOR);
A
Avi Kivity 已提交
2113 2114 2115
	return 1;
}

A
Avi Kivity 已提交
2116
static int task_switch_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2117
{
2118
	u16 tss_selector;
2119 2120 2121
	int reason;
	int int_type = svm->vmcb->control.exit_int_info &
		SVM_EXITINTINFO_TYPE_MASK;
2122
	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2123 2124 2125 2126
	uint32_t type =
		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
	uint32_t idt_v =
		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2127 2128

	tss_selector = (u16)svm->vmcb->control.exit_info_1;
2129

2130 2131
	if (svm->vmcb->control.exit_info_2 &
	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2132 2133 2134 2135
		reason = TASK_SWITCH_IRET;
	else if (svm->vmcb->control.exit_info_2 &
		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
		reason = TASK_SWITCH_JMP;
2136
	else if (idt_v)
2137 2138 2139 2140
		reason = TASK_SWITCH_GATE;
	else
		reason = TASK_SWITCH_CALL;

2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155
	if (reason == TASK_SWITCH_GATE) {
		switch (type) {
		case SVM_EXITINTINFO_TYPE_NMI:
			svm->vcpu.arch.nmi_injected = false;
			break;
		case SVM_EXITINTINFO_TYPE_EXEPT:
			kvm_clear_exception_queue(&svm->vcpu);
			break;
		case SVM_EXITINTINFO_TYPE_INTR:
			kvm_clear_interrupt_queue(&svm->vcpu);
			break;
		default:
			break;
		}
	}
2156

2157 2158 2159
	if (reason != TASK_SWITCH_GATE ||
	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2160 2161
	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
		skip_emulated_instruction(&svm->vcpu);
2162 2163

	return kvm_task_switch(&svm->vcpu, tss_selector, reason);
A
Avi Kivity 已提交
2164 2165
}

A
Avi Kivity 已提交
2166
static int cpuid_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2167
{
2168
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
R
Rusty Russell 已提交
2169
	kvm_emulate_cpuid(&svm->vcpu);
2170
	return 1;
A
Avi Kivity 已提交
2171 2172
}

A
Avi Kivity 已提交
2173
static int iret_interception(struct vcpu_svm *svm)
2174 2175 2176
{
	++svm->vcpu.stat.nmi_window_exits;
	svm->vmcb->control.intercept &= ~(1UL << INTERCEPT_IRET);
2177
	svm->vcpu.arch.hflags |= HF_IRET_MASK;
2178 2179 2180
	return 1;
}

A
Avi Kivity 已提交
2181
static int invlpg_interception(struct vcpu_svm *svm)
M
Marcelo Tosatti 已提交
2182
{
A
Avi Kivity 已提交
2183
	if (emulate_instruction(&svm->vcpu, 0, 0, 0) != EMULATE_DONE)
M
Marcelo Tosatti 已提交
2184 2185 2186 2187
		pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
	return 1;
}

A
Avi Kivity 已提交
2188
static int emulate_on_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2189
{
A
Avi Kivity 已提交
2190
	if (emulate_instruction(&svm->vcpu, 0, 0, 0) != EMULATE_DONE)
2191
		pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
A
Avi Kivity 已提交
2192 2193 2194
	return 1;
}

A
Avi Kivity 已提交
2195
static int cr8_write_interception(struct vcpu_svm *svm)
2196
{
A
Avi Kivity 已提交
2197 2198
	struct kvm_run *kvm_run = svm->vcpu.run;

2199 2200
	u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
	/* instruction emulation calls kvm_set_cr8() */
A
Avi Kivity 已提交
2201
	emulate_instruction(&svm->vcpu, 0, 0, 0);
2202 2203
	if (irqchip_in_kernel(svm->vcpu.kvm)) {
		svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
2204
		return 1;
2205
	}
2206 2207
	if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
		return 1;
2208 2209 2210 2211
	kvm_run->exit_reason = KVM_EXIT_SET_TPR;
	return 0;
}

A
Avi Kivity 已提交
2212 2213
static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
{
2214 2215
	struct vcpu_svm *svm = to_svm(vcpu);

A
Avi Kivity 已提交
2216
	switch (ecx) {
2217
	case MSR_IA32_TSC: {
2218
		u64 tsc_offset;
A
Avi Kivity 已提交
2219

2220 2221 2222 2223 2224 2225
		if (is_nested(svm))
			tsc_offset = svm->nested.hsave->control.tsc_offset;
		else
			tsc_offset = svm->vmcb->control.tsc_offset;

		*data = tsc_offset + native_read_tsc();
A
Avi Kivity 已提交
2226 2227
		break;
	}
2228
	case MSR_K6_STAR:
2229
		*data = svm->vmcb->save.star;
A
Avi Kivity 已提交
2230
		break;
2231
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2232
	case MSR_LSTAR:
2233
		*data = svm->vmcb->save.lstar;
A
Avi Kivity 已提交
2234 2235
		break;
	case MSR_CSTAR:
2236
		*data = svm->vmcb->save.cstar;
A
Avi Kivity 已提交
2237 2238
		break;
	case MSR_KERNEL_GS_BASE:
2239
		*data = svm->vmcb->save.kernel_gs_base;
A
Avi Kivity 已提交
2240 2241
		break;
	case MSR_SYSCALL_MASK:
2242
		*data = svm->vmcb->save.sfmask;
A
Avi Kivity 已提交
2243 2244 2245
		break;
#endif
	case MSR_IA32_SYSENTER_CS:
2246
		*data = svm->vmcb->save.sysenter_cs;
A
Avi Kivity 已提交
2247 2248
		break;
	case MSR_IA32_SYSENTER_EIP:
2249
		*data = svm->sysenter_eip;
A
Avi Kivity 已提交
2250 2251
		break;
	case MSR_IA32_SYSENTER_ESP:
2252
		*data = svm->sysenter_esp;
A
Avi Kivity 已提交
2253
		break;
J
Joerg Roedel 已提交
2254 2255 2256 2257 2258
	/*
	 * Nobody will change the following 5 values in the VMCB so we can
	 * safely return them on rdmsr. They will always be 0 until LBRV is
	 * implemented.
	 */
2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273
	case MSR_IA32_DEBUGCTLMSR:
		*data = svm->vmcb->save.dbgctl;
		break;
	case MSR_IA32_LASTBRANCHFROMIP:
		*data = svm->vmcb->save.br_from;
		break;
	case MSR_IA32_LASTBRANCHTOIP:
		*data = svm->vmcb->save.br_to;
		break;
	case MSR_IA32_LASTINTFROMIP:
		*data = svm->vmcb->save.last_excp_from;
		break;
	case MSR_IA32_LASTINTTOIP:
		*data = svm->vmcb->save.last_excp_to;
		break;
A
Alexander Graf 已提交
2274
	case MSR_VM_HSAVE_PA:
2275
		*data = svm->nested.hsave_msr;
A
Alexander Graf 已提交
2276
		break;
2277 2278 2279
	case MSR_VM_CR:
		*data = 0;
		break;
2280 2281 2282
	case MSR_IA32_UCODE_REV:
		*data = 0x01000065;
		break;
A
Avi Kivity 已提交
2283
	default:
2284
		return kvm_get_msr_common(vcpu, ecx, data);
A
Avi Kivity 已提交
2285 2286 2287 2288
	}
	return 0;
}

A
Avi Kivity 已提交
2289
static int rdmsr_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2290
{
2291
	u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
A
Avi Kivity 已提交
2292 2293
	u64 data;

2294 2295
	if (svm_get_msr(&svm->vcpu, ecx, &data)) {
		trace_kvm_msr_read_ex(ecx);
2296
		kvm_inject_gp(&svm->vcpu, 0);
2297
	} else {
2298
		trace_kvm_msr_read(ecx, data);
2299

2300
		svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff;
2301
		svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;
2302
		svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
R
Rusty Russell 已提交
2303
		skip_emulated_instruction(&svm->vcpu);
A
Avi Kivity 已提交
2304 2305 2306 2307 2308 2309
	}
	return 1;
}

static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
{
2310 2311
	struct vcpu_svm *svm = to_svm(vcpu);

A
Avi Kivity 已提交
2312
	switch (ecx) {
2313
	case MSR_IA32_TSC: {
2314 2315 2316 2317 2318 2319 2320 2321 2322 2323
		u64 tsc_offset = data - native_read_tsc();
		u64 g_tsc_offset = 0;

		if (is_nested(svm)) {
			g_tsc_offset = svm->vmcb->control.tsc_offset -
				       svm->nested.hsave->control.tsc_offset;
			svm->nested.hsave->control.tsc_offset = tsc_offset;
		}

		svm->vmcb->control.tsc_offset = tsc_offset + g_tsc_offset;
A
Avi Kivity 已提交
2324 2325 2326

		break;
	}
2327
	case MSR_K6_STAR:
2328
		svm->vmcb->save.star = data;
A
Avi Kivity 已提交
2329
		break;
2330
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2331
	case MSR_LSTAR:
2332
		svm->vmcb->save.lstar = data;
A
Avi Kivity 已提交
2333 2334
		break;
	case MSR_CSTAR:
2335
		svm->vmcb->save.cstar = data;
A
Avi Kivity 已提交
2336 2337
		break;
	case MSR_KERNEL_GS_BASE:
2338
		svm->vmcb->save.kernel_gs_base = data;
A
Avi Kivity 已提交
2339 2340
		break;
	case MSR_SYSCALL_MASK:
2341
		svm->vmcb->save.sfmask = data;
A
Avi Kivity 已提交
2342 2343 2344
		break;
#endif
	case MSR_IA32_SYSENTER_CS:
2345
		svm->vmcb->save.sysenter_cs = data;
A
Avi Kivity 已提交
2346 2347
		break;
	case MSR_IA32_SYSENTER_EIP:
2348
		svm->sysenter_eip = data;
2349
		svm->vmcb->save.sysenter_eip = data;
A
Avi Kivity 已提交
2350 2351
		break;
	case MSR_IA32_SYSENTER_ESP:
2352
		svm->sysenter_esp = data;
2353
		svm->vmcb->save.sysenter_esp = data;
A
Avi Kivity 已提交
2354
		break;
2355
	case MSR_IA32_DEBUGCTLMSR:
2356 2357
		if (!svm_has(SVM_FEATURE_LBRV)) {
			pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
2358
					__func__, data);
2359 2360 2361 2362 2363 2364 2365 2366 2367 2368
			break;
		}
		if (data & DEBUGCTL_RESERVED_BITS)
			return 1;

		svm->vmcb->save.dbgctl = data;
		if (data & (1ULL<<0))
			svm_enable_lbrv(svm);
		else
			svm_disable_lbrv(svm);
2369
		break;
A
Alexander Graf 已提交
2370
	case MSR_VM_HSAVE_PA:
2371
		svm->nested.hsave_msr = data;
2372
		break;
2373 2374 2375 2376
	case MSR_VM_CR:
	case MSR_VM_IGNNE:
		pr_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
		break;
A
Avi Kivity 已提交
2377
	default:
2378
		return kvm_set_msr_common(vcpu, ecx, data);
A
Avi Kivity 已提交
2379 2380 2381 2382
	}
	return 0;
}

A
Avi Kivity 已提交
2383
static int wrmsr_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2384
{
2385
	u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2386
	u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u)
2387
		| ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
2388 2389


2390
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
2391 2392
	if (svm_set_msr(&svm->vcpu, ecx, data)) {
		trace_kvm_msr_write_ex(ecx, data);
2393
		kvm_inject_gp(&svm->vcpu, 0);
2394 2395
	} else {
		trace_kvm_msr_write(ecx, data);
R
Rusty Russell 已提交
2396
		skip_emulated_instruction(&svm->vcpu);
2397
	}
A
Avi Kivity 已提交
2398 2399 2400
	return 1;
}

A
Avi Kivity 已提交
2401
static int msr_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2402
{
R
Rusty Russell 已提交
2403
	if (svm->vmcb->control.exit_info_1)
A
Avi Kivity 已提交
2404
		return wrmsr_interception(svm);
A
Avi Kivity 已提交
2405
	else
A
Avi Kivity 已提交
2406
		return rdmsr_interception(svm);
A
Avi Kivity 已提交
2407 2408
}

A
Avi Kivity 已提交
2409
static int interrupt_window_interception(struct vcpu_svm *svm)
2410
{
A
Avi Kivity 已提交
2411 2412
	struct kvm_run *kvm_run = svm->vcpu.run;

2413
	svm_clear_vintr(svm);
2414
	svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2415 2416 2417 2418
	/*
	 * If the user space waits to inject interrupts, exit as soon as
	 * possible
	 */
2419 2420 2421
	if (!irqchip_in_kernel(svm->vcpu.kvm) &&
	    kvm_run->request_interrupt_window &&
	    !kvm_cpu_has_interrupt(&svm->vcpu)) {
R
Rusty Russell 已提交
2422
		++svm->vcpu.stat.irq_window_exits;
2423 2424 2425 2426 2427 2428 2429
		kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
		return 0;
	}

	return 1;
}

2430 2431 2432 2433 2434 2435
static int pause_interception(struct vcpu_svm *svm)
{
	kvm_vcpu_on_spin(&(svm->vcpu));
	return 1;
}

A
Avi Kivity 已提交
2436
static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
J
Joerg Roedel 已提交
2437 2438 2439 2440
	[SVM_EXIT_READ_CR0]			= emulate_on_interception,
	[SVM_EXIT_READ_CR3]			= emulate_on_interception,
	[SVM_EXIT_READ_CR4]			= emulate_on_interception,
	[SVM_EXIT_READ_CR8]			= emulate_on_interception,
A
Avi Kivity 已提交
2441
	[SVM_EXIT_CR0_SEL_WRITE]		= emulate_on_interception,
J
Joerg Roedel 已提交
2442 2443 2444 2445 2446
	[SVM_EXIT_WRITE_CR0]			= emulate_on_interception,
	[SVM_EXIT_WRITE_CR3]			= emulate_on_interception,
	[SVM_EXIT_WRITE_CR4]			= emulate_on_interception,
	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
	[SVM_EXIT_READ_DR0]			= emulate_on_interception,
A
Avi Kivity 已提交
2447 2448 2449
	[SVM_EXIT_READ_DR1]			= emulate_on_interception,
	[SVM_EXIT_READ_DR2]			= emulate_on_interception,
	[SVM_EXIT_READ_DR3]			= emulate_on_interception,
2450 2451 2452 2453
	[SVM_EXIT_READ_DR4]			= emulate_on_interception,
	[SVM_EXIT_READ_DR5]			= emulate_on_interception,
	[SVM_EXIT_READ_DR6]			= emulate_on_interception,
	[SVM_EXIT_READ_DR7]			= emulate_on_interception,
A
Avi Kivity 已提交
2454 2455 2456 2457
	[SVM_EXIT_WRITE_DR0]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR1]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR2]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR3]			= emulate_on_interception,
2458
	[SVM_EXIT_WRITE_DR4]			= emulate_on_interception,
A
Avi Kivity 已提交
2459
	[SVM_EXIT_WRITE_DR5]			= emulate_on_interception,
2460
	[SVM_EXIT_WRITE_DR6]			= emulate_on_interception,
A
Avi Kivity 已提交
2461
	[SVM_EXIT_WRITE_DR7]			= emulate_on_interception,
J
Jan Kiszka 已提交
2462 2463
	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
2464
	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
J
Joerg Roedel 已提交
2465 2466 2467 2468
	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
	[SVM_EXIT_EXCP_BASE + NM_VECTOR]	= nm_interception,
	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
	[SVM_EXIT_INTR]				= intr_interception,
2469
	[SVM_EXIT_NMI]				= nmi_interception,
A
Avi Kivity 已提交
2470 2471
	[SVM_EXIT_SMI]				= nop_on_interception,
	[SVM_EXIT_INIT]				= nop_on_interception,
2472
	[SVM_EXIT_VINTR]			= interrupt_window_interception,
A
Avi Kivity 已提交
2473
	[SVM_EXIT_CPUID]			= cpuid_interception,
2474
	[SVM_EXIT_IRET]                         = iret_interception,
2475
	[SVM_EXIT_INVD]                         = emulate_on_interception,
2476
	[SVM_EXIT_PAUSE]			= pause_interception,
A
Avi Kivity 已提交
2477
	[SVM_EXIT_HLT]				= halt_interception,
M
Marcelo Tosatti 已提交
2478
	[SVM_EXIT_INVLPG]			= invlpg_interception,
A
Alexander Graf 已提交
2479
	[SVM_EXIT_INVLPGA]			= invlpga_interception,
J
Joerg Roedel 已提交
2480
	[SVM_EXIT_IOIO]				= io_interception,
A
Avi Kivity 已提交
2481 2482
	[SVM_EXIT_MSR]				= msr_interception,
	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
2483
	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
A
Alexander Graf 已提交
2484
	[SVM_EXIT_VMRUN]			= vmrun_interception,
2485
	[SVM_EXIT_VMMCALL]			= vmmcall_interception,
2486 2487
	[SVM_EXIT_VMLOAD]			= vmload_interception,
	[SVM_EXIT_VMSAVE]			= vmsave_interception,
2488 2489
	[SVM_EXIT_STGI]				= stgi_interception,
	[SVM_EXIT_CLGI]				= clgi_interception,
2490
	[SVM_EXIT_SKINIT]			= skinit_interception,
2491
	[SVM_EXIT_WBINVD]                       = emulate_on_interception,
2492 2493
	[SVM_EXIT_MONITOR]			= invalid_op_interception,
	[SVM_EXIT_MWAIT]			= invalid_op_interception,
2494
	[SVM_EXIT_NPF]				= pf_interception,
A
Avi Kivity 已提交
2495 2496
};

A
Avi Kivity 已提交
2497
static int handle_exit(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2498
{
2499
	struct vcpu_svm *svm = to_svm(vcpu);
A
Avi Kivity 已提交
2500
	struct kvm_run *kvm_run = vcpu->run;
2501
	u32 exit_code = svm->vmcb->control.exit_code;
A
Avi Kivity 已提交
2502

2503
	trace_kvm_exit(exit_code, svm->vmcb->save.rip);
2504

2505 2506 2507 2508 2509 2510 2511
	if (unlikely(svm->nested.exit_required)) {
		nested_svm_vmexit(svm);
		svm->nested.exit_required = false;

		return 1;
	}

2512
	if (is_nested(svm)) {
2513 2514
		int vmexit;

2515 2516 2517 2518 2519 2520
		trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
					svm->vmcb->control.exit_info_1,
					svm->vmcb->control.exit_info_2,
					svm->vmcb->control.exit_int_info,
					svm->vmcb->control.exit_int_info_err);

2521 2522 2523 2524 2525 2526
		vmexit = nested_svm_exit_special(svm);

		if (vmexit == NESTED_EXIT_CONTINUE)
			vmexit = nested_svm_exit_handled(svm);

		if (vmexit == NESTED_EXIT_DONE)
2527 2528 2529
			return 1;
	}

2530 2531
	svm_complete_interrupts(svm);

2532
	if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR0_MASK))
2533
		vcpu->arch.cr0 = svm->vmcb->save.cr0;
2534
	if (npt_enabled)
2535
		vcpu->arch.cr3 = svm->vmcb->save.cr3;
2536 2537 2538 2539 2540 2541 2542 2543

	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		kvm_run->fail_entry.hardware_entry_failure_reason
			= svm->vmcb->control.exit_code;
		return 0;
	}

2544
	if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
2545
	    exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
2546
	    exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH)
A
Avi Kivity 已提交
2547 2548
		printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
		       "exit_code 0x%x\n",
2549
		       __func__, svm->vmcb->control.exit_int_info,
A
Avi Kivity 已提交
2550 2551
		       exit_code);

2552
	if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
J
Joe Perches 已提交
2553
	    || !svm_exit_handlers[exit_code]) {
A
Avi Kivity 已提交
2554
		kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
2555
		kvm_run->hw.hardware_exit_reason = exit_code;
A
Avi Kivity 已提交
2556 2557 2558
		return 0;
	}

A
Avi Kivity 已提交
2559
	return svm_exit_handlers[exit_code](svm);
A
Avi Kivity 已提交
2560 2561 2562 2563 2564 2565
}

static void reload_tss(struct kvm_vcpu *vcpu)
{
	int cpu = raw_smp_processor_id();

2566 2567
	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
	sd->tss_desc->type = 9; /* available 32/64-bit TSS */
A
Avi Kivity 已提交
2568 2569 2570
	load_TR_desc();
}

R
Rusty Russell 已提交
2571
static void pre_svm_run(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2572 2573 2574
{
	int cpu = raw_smp_processor_id();

2575
	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
A
Avi Kivity 已提交
2576

2577
	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
2578
	/* FIXME: handle wraparound of asid_generation */
2579 2580
	if (svm->asid_generation != sd->asid_generation)
		new_asid(svm, sd);
A
Avi Kivity 已提交
2581 2582
}

2583 2584 2585 2586 2587 2588 2589 2590 2591
static void svm_inject_nmi(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
	vcpu->arch.hflags |= HF_NMI_MASK;
	svm->vmcb->control.intercept |= (1UL << INTERCEPT_IRET);
	++vcpu->stat.nmi_injections;
}
A
Avi Kivity 已提交
2592

2593
static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
A
Avi Kivity 已提交
2594 2595 2596
{
	struct vmcb_control_area *control;

2597
	trace_kvm_inj_virq(irq);
2598

2599
	++svm->vcpu.stat.irq_injections;
R
Rusty Russell 已提交
2600
	control = &svm->vmcb->control;
2601
	control->int_vector = irq;
A
Avi Kivity 已提交
2602 2603 2604 2605 2606
	control->int_ctl &= ~V_INTR_PRIO_MASK;
	control->int_ctl |= V_IRQ_MASK |
		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
}

2607
static void svm_set_irq(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
2608 2609 2610
{
	struct vcpu_svm *svm = to_svm(vcpu);

2611
	BUG_ON(!(gif_set(svm)));
2612

2613 2614
	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
		SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
E
Eddie Dong 已提交
2615 2616
}

2617
static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
2618 2619 2620
{
	struct vcpu_svm *svm = to_svm(vcpu);

2621 2622 2623
	if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
		return;

2624
	if (irr == -1)
2625 2626
		return;

2627 2628 2629
	if (tpr >= irr)
		svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK;
}
2630

2631 2632 2633 2634 2635 2636
static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb *vmcb = svm->vmcb;
	return !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
		!(svm->vcpu.arch.hflags & HF_NMI_MASK);
2637 2638
}

J
Jan Kiszka 已提交
2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658
static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
}

static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (masked) {
		svm->vcpu.arch.hflags |= HF_NMI_MASK;
		svm->vmcb->control.intercept |= (1UL << INTERCEPT_IRET);
	} else {
		svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
		svm->vmcb->control.intercept &= ~(1UL << INTERCEPT_IRET);
	}
}

2659 2660 2661 2662
static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb *vmcb = svm->vmcb;
2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674
	int ret;

	if (!gif_set(svm) ||
	     (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
		return 0;

	ret = !!(vmcb->save.rflags & X86_EFLAGS_IF);

	if (is_nested(svm))
		return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);

	return ret;
2675 2676
}

2677
static void enable_irq_window(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2678
{
2679 2680
	struct vcpu_svm *svm = to_svm(vcpu);

J
Joerg Roedel 已提交
2681 2682 2683 2684 2685 2686
	/*
	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
	 * get that intercept, this function will be called again though and
	 * we'll get the vintr intercept.
	 */
2687
	if (gif_set(svm) && nested_svm_intr(svm)) {
2688 2689 2690
		svm_set_vintr(svm);
		svm_inject_irq(svm, 0x0);
	}
2691 2692
}

2693
static void enable_nmi_window(struct kvm_vcpu *vcpu)
2694
{
2695
	struct vcpu_svm *svm = to_svm(vcpu);
2696

2697 2698 2699 2700
	if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
	    == HF_NMI_MASK)
		return; /* IRET will cause a vm exit */

J
Joerg Roedel 已提交
2701 2702 2703 2704
	/*
	 * Something prevents NMI from been injected. Single step over possible
	 * problem (IRET or exception injection or interrupt shadow)
	 */
2705 2706 2707 2708 2709
	if (gif_set(svm) && nested_svm_nmi(svm)) {
		svm->nmi_singlestep = true;
		svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
		update_db_intercept(vcpu);
	}
2710 2711
}

2712 2713 2714 2715 2716
static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
{
	return 0;
}

2717 2718 2719 2720 2721
static void svm_flush_tlb(struct kvm_vcpu *vcpu)
{
	force_new_asid(vcpu);
}

2722 2723 2724 2725
static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
{
}

2726 2727 2728 2729
static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

2730 2731 2732
	if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
		return;

2733 2734
	if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) {
		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
2735
		kvm_set_cr8(vcpu, cr8);
2736 2737 2738
	}
}

2739 2740 2741 2742 2743
static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u64 cr8;

2744 2745 2746
	if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
		return;

2747 2748 2749 2750 2751
	cr8 = kvm_get_cr8(vcpu);
	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
}

2752 2753 2754 2755 2756
static void svm_complete_interrupts(struct vcpu_svm *svm)
{
	u8 vector;
	int type;
	u32 exitintinfo = svm->vmcb->control.exit_int_info;
2757 2758 2759
	unsigned int3_injected = svm->int3_injected;

	svm->int3_injected = 0;
2760

2761 2762 2763
	if (svm->vcpu.arch.hflags & HF_IRET_MASK)
		svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);

2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778
	svm->vcpu.arch.nmi_injected = false;
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);

	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
		return;

	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;

	switch (type) {
	case SVM_EXITINTINFO_TYPE_NMI:
		svm->vcpu.arch.nmi_injected = true;
		break;
	case SVM_EXITINTINFO_TYPE_EXEPT:
2779 2780
		if (is_nested(svm))
			break;
2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791
		/*
		 * In case of software exceptions, do not reinject the vector,
		 * but re-execute the instruction instead. Rewind RIP first
		 * if we emulated INT3 before.
		 */
		if (kvm_exception_is_soft(vector)) {
			if (vector == BP_VECTOR && int3_injected &&
			    kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
				kvm_rip_write(&svm->vcpu,
					      kvm_rip_read(&svm->vcpu) -
					      int3_injected);
2792
			break;
2793
		}
2794 2795 2796 2797 2798 2799 2800 2801
		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
			u32 err = svm->vmcb->control.exit_int_info_err;
			kvm_queue_exception_e(&svm->vcpu, vector, err);

		} else
			kvm_queue_exception(&svm->vcpu, vector);
		break;
	case SVM_EXITINTINFO_TYPE_INTR:
2802
		kvm_queue_interrupt(&svm->vcpu, vector, false);
2803 2804 2805 2806 2807 2808
		break;
	default:
		break;
	}
}

2809 2810 2811 2812 2813 2814
#ifdef CONFIG_X86_64
#define R "r"
#else
#define R "e"
#endif

A
Avi Kivity 已提交
2815
static void svm_vcpu_run(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2816
{
2817
	struct vcpu_svm *svm = to_svm(vcpu);
A
Avi Kivity 已提交
2818 2819 2820
	u16 fs_selector;
	u16 gs_selector;
	u16 ldt_selector;
2821

2822 2823 2824 2825 2826 2827 2828
	/*
	 * A vmexit emulation is required before the vcpu can be executed
	 * again.
	 */
	if (unlikely(svm->nested.exit_required))
		return;

2829 2830 2831 2832
	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];

R
Rusty Russell 已提交
2833
	pre_svm_run(svm);
A
Avi Kivity 已提交
2834

2835 2836
	sync_lapic_to_cr8(vcpu);

A
Avi Kivity 已提交
2837
	save_host_msrs(vcpu);
2838 2839 2840
	fs_selector = kvm_read_fs();
	gs_selector = kvm_read_gs();
	ldt_selector = kvm_read_ldt();
2841
	svm->vmcb->save.cr2 = vcpu->arch.cr2;
2842 2843 2844
	/* required for live migration with NPT */
	if (npt_enabled)
		svm->vmcb->save.cr3 = vcpu->arch.cr3;
A
Avi Kivity 已提交
2845

2846 2847 2848
	clgi();

	local_irq_enable();
2849

A
Avi Kivity 已提交
2850
	asm volatile (
2851 2852 2853 2854 2855 2856 2857
		"push %%"R"bp; \n\t"
		"mov %c[rbx](%[svm]), %%"R"bx \n\t"
		"mov %c[rcx](%[svm]), %%"R"cx \n\t"
		"mov %c[rdx](%[svm]), %%"R"dx \n\t"
		"mov %c[rsi](%[svm]), %%"R"si \n\t"
		"mov %c[rdi](%[svm]), %%"R"di \n\t"
		"mov %c[rbp](%[svm]), %%"R"bp \n\t"
2858
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
2859 2860 2861 2862 2863 2864 2865 2866
		"mov %c[r8](%[svm]),  %%r8  \n\t"
		"mov %c[r9](%[svm]),  %%r9  \n\t"
		"mov %c[r10](%[svm]), %%r10 \n\t"
		"mov %c[r11](%[svm]), %%r11 \n\t"
		"mov %c[r12](%[svm]), %%r12 \n\t"
		"mov %c[r13](%[svm]), %%r13 \n\t"
		"mov %c[r14](%[svm]), %%r14 \n\t"
		"mov %c[r15](%[svm]), %%r15 \n\t"
A
Avi Kivity 已提交
2867 2868 2869
#endif

		/* Enter guest mode */
2870 2871
		"push %%"R"ax \n\t"
		"mov %c[vmcb](%[svm]), %%"R"ax \n\t"
2872 2873 2874
		__ex(SVM_VMLOAD) "\n\t"
		__ex(SVM_VMRUN) "\n\t"
		__ex(SVM_VMSAVE) "\n\t"
2875
		"pop %%"R"ax \n\t"
A
Avi Kivity 已提交
2876 2877

		/* Save guest registers, load host registers */
2878 2879 2880 2881 2882 2883
		"mov %%"R"bx, %c[rbx](%[svm]) \n\t"
		"mov %%"R"cx, %c[rcx](%[svm]) \n\t"
		"mov %%"R"dx, %c[rdx](%[svm]) \n\t"
		"mov %%"R"si, %c[rsi](%[svm]) \n\t"
		"mov %%"R"di, %c[rdi](%[svm]) \n\t"
		"mov %%"R"bp, %c[rbp](%[svm]) \n\t"
2884
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
2885 2886 2887 2888 2889 2890 2891 2892
		"mov %%r8,  %c[r8](%[svm]) \n\t"
		"mov %%r9,  %c[r9](%[svm]) \n\t"
		"mov %%r10, %c[r10](%[svm]) \n\t"
		"mov %%r11, %c[r11](%[svm]) \n\t"
		"mov %%r12, %c[r12](%[svm]) \n\t"
		"mov %%r13, %c[r13](%[svm]) \n\t"
		"mov %%r14, %c[r14](%[svm]) \n\t"
		"mov %%r15, %c[r15](%[svm]) \n\t"
A
Avi Kivity 已提交
2893
#endif
2894
		"pop %%"R"bp"
A
Avi Kivity 已提交
2895
		:
R
Rusty Russell 已提交
2896
		: [svm]"a"(svm),
A
Avi Kivity 已提交
2897
		  [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
2898 2899 2900 2901 2902 2903
		  [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
		  [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
		  [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
		  [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
		  [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
		  [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
2904
#ifdef CONFIG_X86_64
2905 2906 2907 2908 2909 2910 2911 2912
		  , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
		  [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
		  [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
		  [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
		  [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
		  [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
		  [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
		  [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
A
Avi Kivity 已提交
2913
#endif
2914
		: "cc", "memory"
2915
		, R"bx", R"cx", R"dx", R"si", R"di"
2916 2917 2918 2919
#ifdef CONFIG_X86_64
		, "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
#endif
		);
A
Avi Kivity 已提交
2920

2921
	vcpu->arch.cr2 = svm->vmcb->save.cr2;
2922 2923 2924
	vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
	vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
	vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
A
Avi Kivity 已提交
2925

2926 2927 2928
	kvm_load_fs(fs_selector);
	kvm_load_gs(gs_selector);
	kvm_load_ldt(ldt_selector);
A
Avi Kivity 已提交
2929 2930 2931 2932
	load_host_msrs(vcpu);

	reload_tss(vcpu);

2933 2934 2935 2936
	local_irq_disable();

	stgi();

2937 2938
	sync_cr8_to_lapic(vcpu);

2939
	svm->next_rip = 0;
2940

A
Avi Kivity 已提交
2941 2942 2943 2944
	if (npt_enabled) {
		vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
		vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
	}
A
Avi Kivity 已提交
2945 2946
}

2947 2948
#undef R

A
Avi Kivity 已提交
2949 2950
static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
{
2951 2952
	struct vcpu_svm *svm = to_svm(vcpu);

2953 2954 2955 2956 2957 2958
	if (npt_enabled) {
		svm->vmcb->control.nested_cr3 = root;
		force_new_asid(vcpu);
		return;
	}

2959
	svm->vmcb->save.cr3 = root;
A
Avi Kivity 已提交
2960 2961 2962 2963 2964
	force_new_asid(vcpu);
}

static int is_disabled(void)
{
2965 2966 2967 2968 2969 2970
	u64 vm_cr;

	rdmsrl(MSR_VM_CR, vm_cr);
	if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
		return 1;

A
Avi Kivity 已提交
2971 2972 2973
	return 0;
}

I
Ingo Molnar 已提交
2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984
static void
svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
{
	/*
	 * Patch in the VMMCALL instruction:
	 */
	hypercall[0] = 0x0f;
	hypercall[1] = 0x01;
	hypercall[2] = 0xd9;
}

Y
Yang, Sheng 已提交
2985 2986 2987 2988 2989
static void svm_check_processor_compat(void *rtn)
{
	*(int *)rtn = 0;
}

2990 2991 2992 2993 2994
static bool svm_cpu_has_accelerated_tpr(void)
{
	return false;
}

2995 2996 2997 2998 2999 3000 3001 3002 3003
static int get_npt_level(void)
{
#ifdef CONFIG_X86_64
	return PT64_ROOT_LEVEL;
#else
	return PT32E_ROOT_LEVEL;
#endif
}

3004
static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
S
Sheng Yang 已提交
3005 3006 3007 3008
{
	return 0;
}

3009 3010 3011 3012
static void svm_cpuid_update(struct kvm_vcpu *vcpu)
{
}

3013
static const struct trace_print_flags svm_exit_reasons_str[] = {
J
Joerg Roedel 已提交
3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031
	{ SVM_EXIT_READ_CR0,			"read_cr0" },
	{ SVM_EXIT_READ_CR3,			"read_cr3" },
	{ SVM_EXIT_READ_CR4,			"read_cr4" },
	{ SVM_EXIT_READ_CR8,			"read_cr8" },
	{ SVM_EXIT_WRITE_CR0,			"write_cr0" },
	{ SVM_EXIT_WRITE_CR3,			"write_cr3" },
	{ SVM_EXIT_WRITE_CR4,			"write_cr4" },
	{ SVM_EXIT_WRITE_CR8,			"write_cr8" },
	{ SVM_EXIT_READ_DR0,			"read_dr0" },
	{ SVM_EXIT_READ_DR1,			"read_dr1" },
	{ SVM_EXIT_READ_DR2,			"read_dr2" },
	{ SVM_EXIT_READ_DR3,			"read_dr3" },
	{ SVM_EXIT_WRITE_DR0,			"write_dr0" },
	{ SVM_EXIT_WRITE_DR1,			"write_dr1" },
	{ SVM_EXIT_WRITE_DR2,			"write_dr2" },
	{ SVM_EXIT_WRITE_DR3,			"write_dr3" },
	{ SVM_EXIT_WRITE_DR5,			"write_dr5" },
	{ SVM_EXIT_WRITE_DR7,			"write_dr7" },
3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065
	{ SVM_EXIT_EXCP_BASE + DB_VECTOR,	"DB excp" },
	{ SVM_EXIT_EXCP_BASE + BP_VECTOR,	"BP excp" },
	{ SVM_EXIT_EXCP_BASE + UD_VECTOR,	"UD excp" },
	{ SVM_EXIT_EXCP_BASE + PF_VECTOR,	"PF excp" },
	{ SVM_EXIT_EXCP_BASE + NM_VECTOR,	"NM excp" },
	{ SVM_EXIT_EXCP_BASE + MC_VECTOR,	"MC excp" },
	{ SVM_EXIT_INTR,			"interrupt" },
	{ SVM_EXIT_NMI,				"nmi" },
	{ SVM_EXIT_SMI,				"smi" },
	{ SVM_EXIT_INIT,			"init" },
	{ SVM_EXIT_VINTR,			"vintr" },
	{ SVM_EXIT_CPUID,			"cpuid" },
	{ SVM_EXIT_INVD,			"invd" },
	{ SVM_EXIT_HLT,				"hlt" },
	{ SVM_EXIT_INVLPG,			"invlpg" },
	{ SVM_EXIT_INVLPGA,			"invlpga" },
	{ SVM_EXIT_IOIO,			"io" },
	{ SVM_EXIT_MSR,				"msr" },
	{ SVM_EXIT_TASK_SWITCH,			"task_switch" },
	{ SVM_EXIT_SHUTDOWN,			"shutdown" },
	{ SVM_EXIT_VMRUN,			"vmrun" },
	{ SVM_EXIT_VMMCALL,			"hypercall" },
	{ SVM_EXIT_VMLOAD,			"vmload" },
	{ SVM_EXIT_VMSAVE,			"vmsave" },
	{ SVM_EXIT_STGI,			"stgi" },
	{ SVM_EXIT_CLGI,			"clgi" },
	{ SVM_EXIT_SKINIT,			"skinit" },
	{ SVM_EXIT_WBINVD,			"wbinvd" },
	{ SVM_EXIT_MONITOR,			"monitor" },
	{ SVM_EXIT_MWAIT,			"mwait" },
	{ SVM_EXIT_NPF,				"npf" },
	{ -1, NULL }
};

3066
static int svm_get_lpage_level(void)
3067
{
3068
	return PT_PDPE_LEVEL;
3069 3070
}

3071 3072 3073 3074 3075
static bool svm_rdtscp_supported(void)
{
	return false;
}

3076 3077 3078 3079 3080
static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->control.intercept_exceptions |= 1 << NM_VECTOR;
3081 3082 3083
	if (is_nested(svm))
		svm->nested.hsave->control.intercept_exceptions |= 1 << NM_VECTOR;
	update_cr0_intercept(svm);
3084 3085
}

3086
static struct kvm_x86_ops svm_x86_ops = {
A
Avi Kivity 已提交
3087 3088 3089 3090
	.cpu_has_kvm_support = has_svm,
	.disabled_by_bios = is_disabled,
	.hardware_setup = svm_hardware_setup,
	.hardware_unsetup = svm_hardware_unsetup,
Y
Yang, Sheng 已提交
3091
	.check_processor_compatibility = svm_check_processor_compat,
A
Avi Kivity 已提交
3092 3093
	.hardware_enable = svm_hardware_enable,
	.hardware_disable = svm_hardware_disable,
3094
	.cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
A
Avi Kivity 已提交
3095 3096 3097

	.vcpu_create = svm_create_vcpu,
	.vcpu_free = svm_free_vcpu,
3098
	.vcpu_reset = svm_vcpu_reset,
A
Avi Kivity 已提交
3099

3100
	.prepare_guest_switch = svm_prepare_guest_switch,
A
Avi Kivity 已提交
3101 3102 3103 3104 3105 3106 3107 3108 3109
	.vcpu_load = svm_vcpu_load,
	.vcpu_put = svm_vcpu_put,

	.set_guest_debug = svm_guest_debug,
	.get_msr = svm_get_msr,
	.set_msr = svm_set_msr,
	.get_segment_base = svm_get_segment_base,
	.get_segment = svm_get_segment,
	.set_segment = svm_set_segment,
3110
	.get_cpl = svm_get_cpl,
3111
	.get_cs_db_l_bits = kvm_get_cs_db_l_bits,
3112
	.decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
3113
	.decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
A
Avi Kivity 已提交
3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
	.set_cr0 = svm_set_cr0,
	.set_cr3 = svm_set_cr3,
	.set_cr4 = svm_set_cr4,
	.set_efer = svm_set_efer,
	.get_idt = svm_get_idt,
	.set_idt = svm_set_idt,
	.get_gdt = svm_get_gdt,
	.set_gdt = svm_set_gdt,
	.get_dr = svm_get_dr,
	.set_dr = svm_set_dr,
A
Avi Kivity 已提交
3124
	.cache_reg = svm_cache_reg,
A
Avi Kivity 已提交
3125 3126
	.get_rflags = svm_get_rflags,
	.set_rflags = svm_set_rflags,
A
Avi Kivity 已提交
3127
	.fpu_activate = svm_fpu_activate,
3128
	.fpu_deactivate = svm_fpu_deactivate,
A
Avi Kivity 已提交
3129 3130 3131 3132

	.tlb_flush = svm_flush_tlb,

	.run = svm_vcpu_run,
3133
	.handle_exit = handle_exit,
A
Avi Kivity 已提交
3134
	.skip_emulated_instruction = skip_emulated_instruction,
3135 3136
	.set_interrupt_shadow = svm_set_interrupt_shadow,
	.get_interrupt_shadow = svm_get_interrupt_shadow,
I
Ingo Molnar 已提交
3137
	.patch_hypercall = svm_patch_hypercall,
E
Eddie Dong 已提交
3138
	.set_irq = svm_set_irq,
3139
	.set_nmi = svm_inject_nmi,
3140
	.queue_exception = svm_queue_exception,
3141
	.interrupt_allowed = svm_interrupt_allowed,
3142
	.nmi_allowed = svm_nmi_allowed,
J
Jan Kiszka 已提交
3143 3144
	.get_nmi_mask = svm_get_nmi_mask,
	.set_nmi_mask = svm_set_nmi_mask,
3145 3146 3147
	.enable_nmi_window = enable_nmi_window,
	.enable_irq_window = enable_irq_window,
	.update_cr8_intercept = update_cr8_intercept,
3148 3149

	.set_tss_addr = svm_set_tss_addr,
3150
	.get_tdp_level = get_npt_level,
3151
	.get_mt_mask = svm_get_mt_mask,
3152 3153

	.exit_reasons_str = svm_exit_reasons_str,
3154
	.get_lpage_level = svm_get_lpage_level,
3155 3156

	.cpuid_update = svm_cpuid_update,
3157 3158

	.rdtscp_supported = svm_rdtscp_supported,
A
Avi Kivity 已提交
3159 3160 3161 3162
};

static int __init svm_init(void)
{
3163
	return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
3164
			      THIS_MODULE);
A
Avi Kivity 已提交
3165 3166 3167 3168
}

static void __exit svm_exit(void)
{
3169
	kvm_exit();
A
Avi Kivity 已提交
3170 3171 3172 3173
}

module_init(svm_init)
module_exit(svm_exit)