svm.c 72.7 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * AMD SVM support
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
16 17
#include <linux/kvm_host.h>

18
#include "irq.h"
19
#include "mmu.h"
20
#include "kvm_cache_regs.h"
21
#include "x86.h"
A
Avi Kivity 已提交
22

A
Avi Kivity 已提交
23
#include <linux/module.h>
24
#include <linux/kernel.h>
A
Avi Kivity 已提交
25 26
#include <linux/vmalloc.h>
#include <linux/highmem.h>
A
Alexey Dobriyan 已提交
27
#include <linux/sched.h>
28
#include <linux/ftrace_event.h>
A
Avi Kivity 已提交
29

A
Avi Kivity 已提交
30
#include <asm/desc.h>
A
Avi Kivity 已提交
31

32
#include <asm/virtext.h>
33
#include "trace.h"
34

35 36
#define __ex(x) __kvm_handle_fault_on_reboot(x)

A
Avi Kivity 已提交
37 38 39 40 41 42 43 44 45
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

#define IOPM_ALLOC_ORDER 2
#define MSRPM_ALLOC_ORDER 1

#define SEG_TYPE_LDT 2
#define SEG_TYPE_BUSY_TSS16 3

46 47
#define SVM_FEATURE_NPT  (1 << 0)
#define SVM_FEATURE_LBRV (1 << 1)
A
Amit Shah 已提交
48
#define SVM_FEATURE_SVML (1 << 2)
49

50 51
#define DEBUGCTL_RESERVED_BITS (~(0x3fULL))

52 53 54 55 56 57 58 59 60
/* Turn on to get debugging output*/
/* #define NESTED_DEBUG */

#ifdef NESTED_DEBUG
#define nsvm_printk(fmt, args...) printk(KERN_INFO fmt, ## args)
#else
#define nsvm_printk(fmt, args...) do {} while(0)
#endif

A
Avi Kivity 已提交
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
static const u32 host_save_user_msrs[] = {
#ifdef CONFIG_X86_64
	MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
	MSR_FS_BASE,
#endif
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
};

#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)

struct kvm_vcpu;

struct vcpu_svm {
	struct kvm_vcpu vcpu;
	struct vmcb *vmcb;
	unsigned long vmcb_pa;
	struct svm_cpu_data *svm_data;
	uint64_t asid_generation;
	uint64_t sysenter_esp;
	uint64_t sysenter_eip;

	u64 next_rip;

	u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
	u64 host_gs_base;

	u32 *msrpm;
	struct vmcb *hsave;
	u64 hsave_msr;

	u64 nested_vmcb;

	/* These are the merged vectors */
	u32 *nested_msrpm;

	/* gpa pointers to the real vectors */
	u64 nested_vmcb_msrpm;
};

100 101 102 103
/* enable NPT for AMD64 and X86 with PAE */
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
static bool npt_enabled = true;
#else
104
static bool npt_enabled = false;
105
#endif
106 107 108
static int npt = 1;

module_param(npt, int, S_IRUGO);
109

110 111 112
static int nested = 0;
module_param(nested, int, S_IRUGO);

113
static void svm_flush_tlb(struct kvm_vcpu *vcpu);
114

115 116 117 118 119 120 121
static int nested_svm_exit_handled(struct vcpu_svm *svm, bool kvm_override);
static int nested_svm_vmexit(struct vcpu_svm *svm);
static int nested_svm_vmsave(struct vcpu_svm *svm, void *nested_vmcb,
			     void *arg2, void *opaque);
static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
				      bool has_error_code, u32 error_code);

122 123
static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
124
	return container_of(vcpu, struct vcpu_svm, vcpu);
125 126
}

A
Alexander Graf 已提交
127 128 129 130 131
static inline bool is_nested(struct vcpu_svm *svm)
{
	return svm->nested_vmcb;
}

132
static unsigned long iopm_base;
A
Avi Kivity 已提交
133 134 135 136 137 138 139 140 141 142 143 144 145

struct kvm_ldttss_desc {
	u16 limit0;
	u16 base0;
	unsigned base1 : 8, type : 5, dpl : 2, p : 1;
	unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8;
	u32 base3;
	u32 zero1;
} __attribute__((packed));

struct svm_cpu_data {
	int cpu;

A
Avi Kivity 已提交
146 147 148
	u64 asid_generation;
	u32 max_asid;
	u32 next_asid;
A
Avi Kivity 已提交
149 150 151 152 153 154
	struct kvm_ldttss_desc *tss_desc;

	struct page *save_area;
};

static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
155
static uint32_t svm_features;
A
Avi Kivity 已提交
156 157 158 159 160 161 162 163

struct svm_init_data {
	int cpu;
	int r;
};

static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};

164
#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
A
Avi Kivity 已提交
165 166 167 168 169
#define MSRS_RANGE_SIZE 2048
#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)

#define MAX_INST_SIZE 15

170 171 172 173 174
static inline u32 svm_has(u32 feat)
{
	return svm_features & feat;
}

A
Avi Kivity 已提交
175 176
static inline void clgi(void)
{
177
	asm volatile (__ex(SVM_CLGI));
A
Avi Kivity 已提交
178 179 180 181
}

static inline void stgi(void)
{
182
	asm volatile (__ex(SVM_STGI));
A
Avi Kivity 已提交
183 184 185 186
}

static inline void invlpga(unsigned long addr, u32 asid)
{
187
	asm volatile (__ex(SVM_INVLPGA) :: "a"(addr), "c"(asid));
A
Avi Kivity 已提交
188 189 190 191
}

static inline void force_new_asid(struct kvm_vcpu *vcpu)
{
192
	to_svm(vcpu)->asid_generation--;
A
Avi Kivity 已提交
193 194 195 196 197 198 199 200 201
}

static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
{
	force_new_asid(vcpu);
}

static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
202
	if (!npt_enabled && !(efer & EFER_LMA))
203
		efer &= ~EFER_LME;
A
Avi Kivity 已提交
204

205
	to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
206
	vcpu->arch.shadow_efer = efer;
A
Avi Kivity 已提交
207 208
}

209 210 211 212 213
static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
				bool has_error_code, u32 error_code)
{
	struct vcpu_svm *svm = to_svm(vcpu);

214 215 216 217 218
	/* If we are within a nested VM we'd better #VMEXIT and let the
	   guest handle the exception */
	if (nested_svm_check_exception(svm, nr, has_error_code, error_code))
		return;

219 220 221 222 223 224 225
	svm->vmcb->control.event_inj = nr
		| SVM_EVTINJ_VALID
		| (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
		| SVM_EVTINJ_TYPE_EXEPT;
	svm->vmcb->control.event_inj_err = error_code;
}

A
Avi Kivity 已提交
226 227 228 229 230 231
static int is_external_interrupt(u32 info)
{
	info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
	return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
}

232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u32 ret = 0;

	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
		ret |= X86_SHADOW_INT_STI | X86_SHADOW_INT_MOV_SS;
	return ret & mask;
}

static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (mask == 0)
		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
	else
		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;

}

A
Avi Kivity 已提交
253 254
static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
255 256 257
	struct vcpu_svm *svm = to_svm(vcpu);

	if (!svm->next_rip) {
258 259 260
		if (emulate_instruction(vcpu, vcpu->run, 0, 0, EMULTYPE_SKIP) !=
				EMULATE_DONE)
			printk(KERN_DEBUG "%s: NOP\n", __func__);
A
Avi Kivity 已提交
261 262
		return;
	}
263 264 265
	if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
		printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
		       __func__, kvm_rip_read(vcpu), svm->next_rip);
A
Avi Kivity 已提交
266

267
	kvm_rip_write(vcpu, svm->next_rip);
268
	svm_set_interrupt_shadow(vcpu, 0);
A
Avi Kivity 已提交
269 270 271 272
}

static int has_svm(void)
{
273
	const char *msg;
A
Avi Kivity 已提交
274

275
	if (!cpu_has_svm(&msg)) {
J
Joe Perches 已提交
276
		printk(KERN_INFO "has_svm: %s\n", msg);
A
Avi Kivity 已提交
277 278 279 280 281 282 283 284
		return 0;
	}

	return 1;
}

static void svm_hardware_disable(void *garbage)
{
285
	cpu_svm_disable();
A
Avi Kivity 已提交
286 287 288 289 290 291 292
}

static void svm_hardware_enable(void *garbage)
{

	struct svm_cpu_data *svm_data;
	uint64_t efer;
293
	struct descriptor_table gdt_descr;
A
Avi Kivity 已提交
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
	struct desc_struct *gdt;
	int me = raw_smp_processor_id();

	if (!has_svm()) {
		printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me);
		return;
	}
	svm_data = per_cpu(svm_data, me);

	if (!svm_data) {
		printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n",
		       me);
		return;
	}

	svm_data->asid_generation = 1;
	svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
	svm_data->next_asid = svm_data->max_asid + 1;

313 314
	kvm_get_gdt(&gdt_descr);
	gdt = (struct desc_struct *)gdt_descr.base;
A
Avi Kivity 已提交
315 316 317
	svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);

	rdmsrl(MSR_EFER, efer);
318
	wrmsrl(MSR_EFER, efer | EFER_SVME);
A
Avi Kivity 已提交
319 320 321 322 323

	wrmsrl(MSR_VM_HSAVE_PA,
	       page_to_pfn(svm_data->save_area) << PAGE_SHIFT);
}

324 325 326 327 328 329 330 331 332 333 334 335 336
static void svm_cpu_uninit(int cpu)
{
	struct svm_cpu_data *svm_data
		= per_cpu(svm_data, raw_smp_processor_id());

	if (!svm_data)
		return;

	per_cpu(svm_data, raw_smp_processor_id()) = NULL;
	__free_page(svm_data->save_area);
	kfree(svm_data);
}

A
Avi Kivity 已提交
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
static int svm_cpu_init(int cpu)
{
	struct svm_cpu_data *svm_data;
	int r;

	svm_data = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
	if (!svm_data)
		return -ENOMEM;
	svm_data->cpu = cpu;
	svm_data->save_area = alloc_page(GFP_KERNEL);
	r = -ENOMEM;
	if (!svm_data->save_area)
		goto err_1;

	per_cpu(svm_data, cpu) = svm_data;

	return 0;

err_1:
	kfree(svm_data);
	return r;

}

361 362
static void set_msr_interception(u32 *msrpm, unsigned msr,
				 int read, int write)
A
Avi Kivity 已提交
363 364 365 366 367 368 369 370 371 372 373 374 375 376
{
	int i;

	for (i = 0; i < NUM_MSR_MAPS; i++) {
		if (msr >= msrpm_ranges[i] &&
		    msr < msrpm_ranges[i] + MSRS_IN_RANGE) {
			u32 msr_offset = (i * MSRS_IN_RANGE + msr -
					  msrpm_ranges[i]) * 2;

			u32 *base = msrpm + (msr_offset / 32);
			u32 msr_shift = msr_offset % 32;
			u32 mask = ((write) ? 0 : 2) | ((read) ? 0 : 1);
			*base = (*base & ~(0x3 << msr_shift)) |
				(mask << msr_shift);
377
			return;
A
Avi Kivity 已提交
378 379
		}
	}
380
	BUG();
A
Avi Kivity 已提交
381 382
}

383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
static void svm_vcpu_init_msrpm(u32 *msrpm)
{
	memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));

#ifdef CONFIG_X86_64
	set_msr_interception(msrpm, MSR_GS_BASE, 1, 1);
	set_msr_interception(msrpm, MSR_FS_BASE, 1, 1);
	set_msr_interception(msrpm, MSR_KERNEL_GS_BASE, 1, 1);
	set_msr_interception(msrpm, MSR_LSTAR, 1, 1);
	set_msr_interception(msrpm, MSR_CSTAR, 1, 1);
	set_msr_interception(msrpm, MSR_SYSCALL_MASK, 1, 1);
#endif
	set_msr_interception(msrpm, MSR_K6_STAR, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_SYSENTER_CS, 1, 1);
}

399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
static void svm_enable_lbrv(struct vcpu_svm *svm)
{
	u32 *msrpm = svm->msrpm;

	svm->vmcb->control.lbr_ctl = 1;
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
}

static void svm_disable_lbrv(struct vcpu_svm *svm)
{
	u32 *msrpm = svm->msrpm;

	svm->vmcb->control.lbr_ctl = 0;
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
}

A
Avi Kivity 已提交
421 422 423 424
static __init int svm_hardware_setup(void)
{
	int cpu;
	struct page *iopm_pages;
425
	void *iopm_va;
A
Avi Kivity 已提交
426 427 428 429 430 431
	int r;

	iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);

	if (!iopm_pages)
		return -ENOMEM;
432 433 434

	iopm_va = page_address(iopm_pages);
	memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
A
Avi Kivity 已提交
435 436
	iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;

437 438 439
	if (boot_cpu_has(X86_FEATURE_NX))
		kvm_enable_efer_bits(EFER_NX);

A
Alexander Graf 已提交
440 441 442
	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
		kvm_enable_efer_bits(EFER_FFXSR);

443 444 445 446 447
	if (nested) {
		printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
		kvm_enable_efer_bits(EFER_SVME);
	}

A
Avi Kivity 已提交
448 449 450
	for_each_online_cpu(cpu) {
		r = svm_cpu_init(cpu);
		if (r)
451
			goto err;
A
Avi Kivity 已提交
452
	}
453 454 455

	svm_features = cpuid_edx(SVM_CPUID_FUNC);

456 457 458
	if (!svm_has(SVM_FEATURE_NPT))
		npt_enabled = false;

459 460 461 462 463
	if (npt_enabled && !npt) {
		printk(KERN_INFO "kvm: Nested Paging disabled\n");
		npt_enabled = false;
	}

464
	if (npt_enabled) {
465
		printk(KERN_INFO "kvm: Nested Paging enabled\n");
466
		kvm_enable_tdp();
467 468
	} else
		kvm_disable_tdp();
469

A
Avi Kivity 已提交
470 471
	return 0;

472
err:
A
Avi Kivity 已提交
473 474 475 476 477 478 479
	__free_pages(iopm_pages, IOPM_ALLOC_ORDER);
	iopm_base = 0;
	return r;
}

static __exit void svm_hardware_unsetup(void)
{
480 481 482 483 484
	int cpu;

	for_each_online_cpu(cpu)
		svm_cpu_uninit(cpu);

A
Avi Kivity 已提交
485
	__free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
486
	iopm_base = 0;
A
Avi Kivity 已提交
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
}

static void init_seg(struct vmcb_seg *seg)
{
	seg->selector = 0;
	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
		SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
	seg->limit = 0xffff;
	seg->base = 0;
}

static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
{
	seg->selector = 0;
	seg->attrib = SVM_SELECTOR_P_MASK | type;
	seg->limit = 0xffff;
	seg->base = 0;
}

506
static void init_vmcb(struct vcpu_svm *svm)
A
Avi Kivity 已提交
507
{
508 509
	struct vmcb_control_area *control = &svm->vmcb->control;
	struct vmcb_save_area *save = &svm->vmcb->save;
A
Avi Kivity 已提交
510 511 512

	control->intercept_cr_read = 	INTERCEPT_CR0_MASK |
					INTERCEPT_CR3_MASK |
513
					INTERCEPT_CR4_MASK;
A
Avi Kivity 已提交
514 515 516

	control->intercept_cr_write = 	INTERCEPT_CR0_MASK |
					INTERCEPT_CR3_MASK |
517 518
					INTERCEPT_CR4_MASK |
					INTERCEPT_CR8_MASK;
A
Avi Kivity 已提交
519 520 521 522 523 524 525 526 527 528 529 530 531

	control->intercept_dr_read = 	INTERCEPT_DR0_MASK |
					INTERCEPT_DR1_MASK |
					INTERCEPT_DR2_MASK |
					INTERCEPT_DR3_MASK;

	control->intercept_dr_write = 	INTERCEPT_DR0_MASK |
					INTERCEPT_DR1_MASK |
					INTERCEPT_DR2_MASK |
					INTERCEPT_DR3_MASK |
					INTERCEPT_DR5_MASK |
					INTERCEPT_DR7_MASK;

532
	control->intercept_exceptions = (1 << PF_VECTOR) |
533 534
					(1 << UD_VECTOR) |
					(1 << MC_VECTOR);
A
Avi Kivity 已提交
535 536 537 538


	control->intercept = 	(1ULL << INTERCEPT_INTR) |
				(1ULL << INTERCEPT_NMI) |
539
				(1ULL << INTERCEPT_SMI) |
A
Avi Kivity 已提交
540
				(1ULL << INTERCEPT_CPUID) |
541
				(1ULL << INTERCEPT_INVD) |
A
Avi Kivity 已提交
542
				(1ULL << INTERCEPT_HLT) |
M
Marcelo Tosatti 已提交
543
				(1ULL << INTERCEPT_INVLPG) |
A
Avi Kivity 已提交
544 545 546 547
				(1ULL << INTERCEPT_INVLPGA) |
				(1ULL << INTERCEPT_IOIO_PROT) |
				(1ULL << INTERCEPT_MSR_PROT) |
				(1ULL << INTERCEPT_TASK_SWITCH) |
548
				(1ULL << INTERCEPT_SHUTDOWN) |
A
Avi Kivity 已提交
549 550 551 552 553 554
				(1ULL << INTERCEPT_VMRUN) |
				(1ULL << INTERCEPT_VMMCALL) |
				(1ULL << INTERCEPT_VMLOAD) |
				(1ULL << INTERCEPT_VMSAVE) |
				(1ULL << INTERCEPT_STGI) |
				(1ULL << INTERCEPT_CLGI) |
555
				(1ULL << INTERCEPT_SKINIT) |
556
				(1ULL << INTERCEPT_WBINVD) |
557 558
				(1ULL << INTERCEPT_MONITOR) |
				(1ULL << INTERCEPT_MWAIT);
A
Avi Kivity 已提交
559 560

	control->iopm_base_pa = iopm_base;
561
	control->msrpm_base_pa = __pa(svm->msrpm);
562
	control->tsc_offset = 0;
A
Avi Kivity 已提交
563 564 565 566 567 568 569 570 571 572 573 574 575
	control->int_ctl = V_INTR_MASKING_MASK;

	init_seg(&save->es);
	init_seg(&save->ss);
	init_seg(&save->ds);
	init_seg(&save->fs);
	init_seg(&save->gs);

	save->cs.selector = 0xf000;
	/* Executable/Readable Code Segment */
	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
	save->cs.limit = 0xffff;
576 577 578 579 580 581 582
	/*
	 * cs.base should really be 0xffff0000, but vmx can't handle that, so
	 * be consistent with it.
	 *
	 * Replace when we have real mode working for vmx.
	 */
	save->cs.base = 0xf0000;
A
Avi Kivity 已提交
583 584 585 586 587 588 589

	save->gdtr.limit = 0xffff;
	save->idtr.limit = 0xffff;

	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);

590
	save->efer = EFER_SVME;
M
Mike Day 已提交
591
	save->dr6 = 0xffff0ff0;
A
Avi Kivity 已提交
592 593 594
	save->dr7 = 0x400;
	save->rflags = 2;
	save->rip = 0x0000fff0;
595
	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
A
Avi Kivity 已提交
596 597 598 599 600

	/*
	 * cr0 val on cpu init should be 0x60000010, we enable cpu
	 * cache by default. the orderly way is to enable cache in bios.
	 */
601
	save->cr0 = 0x00000010 | X86_CR0_PG | X86_CR0_WP;
602
	save->cr4 = X86_CR4_PAE;
A
Avi Kivity 已提交
603
	/* rdx = ?? */
604 605 606 607

	if (npt_enabled) {
		/* Setup VMCB for Nested Paging */
		control->nested_ctl = 1;
M
Marcelo Tosatti 已提交
608 609
		control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) |
					(1ULL << INTERCEPT_INVLPG));
610 611 612 613 614 615 616 617 618 619 620
		control->intercept_exceptions &= ~(1 << PF_VECTOR);
		control->intercept_cr_read &= ~(INTERCEPT_CR0_MASK|
						INTERCEPT_CR3_MASK);
		control->intercept_cr_write &= ~(INTERCEPT_CR0_MASK|
						 INTERCEPT_CR3_MASK);
		save->g_pat = 0x0007040600070406ULL;
		/* enable caching because the QEMU Bios doesn't enable it */
		save->cr0 = X86_CR0_ET;
		save->cr3 = 0;
		save->cr4 = 0;
	}
621
	force_new_asid(&svm->vcpu);
622

A
Alexander Graf 已提交
623
	svm->nested_vmcb = 0;
624
	svm->vcpu.arch.hflags = HF_GIF_MASK;
A
Avi Kivity 已提交
625 626
}

627
static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
628 629 630
{
	struct vcpu_svm *svm = to_svm(vcpu);

631
	init_vmcb(svm);
A
Avi Kivity 已提交
632

633
	if (!kvm_vcpu_is_bsp(vcpu)) {
634
		kvm_rip_write(vcpu, 0);
635 636
		svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;
		svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;
A
Avi Kivity 已提交
637
	}
638 639
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;
640 641

	return 0;
642 643
}

R
Rusty Russell 已提交
644
static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
A
Avi Kivity 已提交
645
{
646
	struct vcpu_svm *svm;
A
Avi Kivity 已提交
647
	struct page *page;
648
	struct page *msrpm_pages;
A
Alexander Graf 已提交
649
	struct page *hsave_page;
A
Alexander Graf 已提交
650
	struct page *nested_msrpm_pages;
R
Rusty Russell 已提交
651
	int err;
A
Avi Kivity 已提交
652

653
	svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
R
Rusty Russell 已提交
654 655 656 657 658 659 660 661 662
	if (!svm) {
		err = -ENOMEM;
		goto out;
	}

	err = kvm_vcpu_init(&svm->vcpu, kvm, id);
	if (err)
		goto free_svm;

A
Avi Kivity 已提交
663
	page = alloc_page(GFP_KERNEL);
R
Rusty Russell 已提交
664 665 666 667
	if (!page) {
		err = -ENOMEM;
		goto uninit;
	}
A
Avi Kivity 已提交
668

669 670 671 672
	err = -ENOMEM;
	msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
	if (!msrpm_pages)
		goto uninit;
A
Alexander Graf 已提交
673 674 675 676 677

	nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
	if (!nested_msrpm_pages)
		goto uninit;

678 679 680
	svm->msrpm = page_address(msrpm_pages);
	svm_vcpu_init_msrpm(svm->msrpm);

A
Alexander Graf 已提交
681 682 683 684 685
	hsave_page = alloc_page(GFP_KERNEL);
	if (!hsave_page)
		goto uninit;
	svm->hsave = page_address(hsave_page);

A
Alexander Graf 已提交
686 687
	svm->nested_msrpm = page_address(nested_msrpm_pages);

688 689 690 691
	svm->vmcb = page_address(page);
	clear_page(svm->vmcb);
	svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
	svm->asid_generation = 0;
692
	init_vmcb(svm);
693

R
Rusty Russell 已提交
694 695
	fx_init(&svm->vcpu);
	svm->vcpu.fpu_active = 1;
696
	svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
697
	if (kvm_vcpu_is_bsp(&svm->vcpu))
698
		svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
A
Avi Kivity 已提交
699

R
Rusty Russell 已提交
700
	return &svm->vcpu;
701

R
Rusty Russell 已提交
702 703 704
uninit:
	kvm_vcpu_uninit(&svm->vcpu);
free_svm:
705
	kmem_cache_free(kvm_vcpu_cache, svm);
R
Rusty Russell 已提交
706 707
out:
	return ERR_PTR(err);
A
Avi Kivity 已提交
708 709 710 711
}

static void svm_free_vcpu(struct kvm_vcpu *vcpu)
{
712 713
	struct vcpu_svm *svm = to_svm(vcpu);

R
Rusty Russell 已提交
714
	__free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
715
	__free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
A
Alexander Graf 已提交
716
	__free_page(virt_to_page(svm->hsave));
A
Alexander Graf 已提交
717
	__free_pages(virt_to_page(svm->nested_msrpm), MSRPM_ALLOC_ORDER);
R
Rusty Russell 已提交
718
	kvm_vcpu_uninit(vcpu);
719
	kmem_cache_free(kvm_vcpu_cache, svm);
A
Avi Kivity 已提交
720 721
}

722
static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
A
Avi Kivity 已提交
723
{
724
	struct vcpu_svm *svm = to_svm(vcpu);
725
	int i;
726 727 728 729 730 731 732 733 734

	if (unlikely(cpu != vcpu->cpu)) {
		u64 tsc_this, delta;

		/*
		 * Make sure that the guest sees a monotonically
		 * increasing TSC.
		 */
		rdtscll(tsc_this);
735
		delta = vcpu->arch.host_tsc - tsc_this;
736
		svm->vmcb->control.tsc_offset += delta;
737
		vcpu->cpu = cpu;
M
Marcelo Tosatti 已提交
738
		kvm_migrate_timers(vcpu);
739
		svm->asid_generation = 0;
740
	}
741 742

	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
743
		rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
A
Avi Kivity 已提交
744 745 746 747
}

static void svm_vcpu_put(struct kvm_vcpu *vcpu)
{
748
	struct vcpu_svm *svm = to_svm(vcpu);
749 750
	int i;

751
	++vcpu->stat.host_state_reload;
752
	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
753
		wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
754

755
	rdtscll(vcpu->arch.host_tsc);
A
Avi Kivity 已提交
756 757 758 759
}

static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
{
760
	return to_svm(vcpu)->vmcb->save.rflags;
A
Avi Kivity 已提交
761 762 763 764
}

static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
765
	to_svm(vcpu)->vmcb->save.rflags = rflags;
A
Avi Kivity 已提交
766 767
}

A
Avi Kivity 已提交
768 769 770 771 772 773 774 775 776 777 778 779
static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
{
	switch (reg) {
	case VCPU_EXREG_PDPTR:
		BUG_ON(!npt_enabled);
		load_pdptrs(vcpu, vcpu->arch.cr3);
		break;
	default:
		BUG();
	}
}

780 781 782 783 784 785 786 787 788 789
static void svm_set_vintr(struct vcpu_svm *svm)
{
	svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR;
}

static void svm_clear_vintr(struct vcpu_svm *svm)
{
	svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
}

A
Avi Kivity 已提交
790 791
static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
{
792
	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
A
Avi Kivity 已提交
793 794 795 796 797 798 799 800 801 802 803 804

	switch (seg) {
	case VCPU_SREG_CS: return &save->cs;
	case VCPU_SREG_DS: return &save->ds;
	case VCPU_SREG_ES: return &save->es;
	case VCPU_SREG_FS: return &save->fs;
	case VCPU_SREG_GS: return &save->gs;
	case VCPU_SREG_SS: return &save->ss;
	case VCPU_SREG_TR: return &save->tr;
	case VCPU_SREG_LDTR: return &save->ldtr;
	}
	BUG();
A
Al Viro 已提交
805
	return NULL;
A
Avi Kivity 已提交
806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
}

static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	return s->base;
}

static void svm_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	var->base = s->base;
	var->limit = s->limit;
	var->selector = s->selector;
	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
	var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1;
831

832 833 834 835 836
	/* AMD's VMCB does not have an explicit unusable field, so emulate it
	 * for cross vendor migration purposes by "not present"
	 */
	var->unusable = !var->present || (var->type == 0);

837 838 839 840 841 842 843
	switch (seg) {
	case VCPU_SREG_CS:
		/*
		 * SVM always stores 0 for the 'G' bit in the CS selector in
		 * the VMCB on a VMEXIT. This hurts cross-vendor migration:
		 * Intel's VMENTRY has a check on the 'G' bit.
		 */
844
		var->g = s->limit > 0xfffff;
845 846 847 848 849 850
		break;
	case VCPU_SREG_TR:
		/*
		 * Work around a bug where the busy flag in the tr selector
		 * isn't exposed
		 */
851
		var->type |= 0x2;
852 853 854 855 856 857 858 859 860 861 862 863 864 865 866
		break;
	case VCPU_SREG_DS:
	case VCPU_SREG_ES:
	case VCPU_SREG_FS:
	case VCPU_SREG_GS:
		/*
		 * The accessed bit must always be set in the segment
		 * descriptor cache, although it can be cleared in the
		 * descriptor, the cached bit always remains at 1. Since
		 * Intel has a check on this, set it here to support
		 * cross-vendor migration.
		 */
		if (!var->unusable)
			var->type |= 0x1;
		break;
867 868 869 870 871 872 873 874 875
	case VCPU_SREG_SS:
		/* On AMD CPUs sometimes the DB bit in the segment
		 * descriptor is left as 1, although the whole segment has
		 * been made unusable. Clear it here to pass an Intel VMX
		 * entry check when cross vendor migrating.
		 */
		if (var->unusable)
			var->db = 0;
		break;
876
	}
A
Avi Kivity 已提交
877 878
}

879 880 881 882 883 884 885
static int svm_get_cpl(struct kvm_vcpu *vcpu)
{
	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;

	return save->cpl;
}

A
Avi Kivity 已提交
886 887
static void svm_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
{
888 889 890 891
	struct vcpu_svm *svm = to_svm(vcpu);

	dt->limit = svm->vmcb->save.idtr.limit;
	dt->base = svm->vmcb->save.idtr.base;
A
Avi Kivity 已提交
892 893 894 895
}

static void svm_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
{
896 897 898 899
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->save.idtr.limit = dt->limit;
	svm->vmcb->save.idtr.base = dt->base ;
A
Avi Kivity 已提交
900 901 902 903
}

static void svm_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
{
904 905 906 907
	struct vcpu_svm *svm = to_svm(vcpu);

	dt->limit = svm->vmcb->save.gdtr.limit;
	dt->base = svm->vmcb->save.gdtr.base;
A
Avi Kivity 已提交
908 909 910 911
}

static void svm_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
{
912 913 914 915
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->save.gdtr.limit = dt->limit;
	svm->vmcb->save.gdtr.base = dt->base ;
A
Avi Kivity 已提交
916 917
}

918
static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
919 920 921
{
}

A
Avi Kivity 已提交
922 923
static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
924 925
	struct vcpu_svm *svm = to_svm(vcpu);

926
#ifdef CONFIG_X86_64
927
	if (vcpu->arch.shadow_efer & EFER_LME) {
928
		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
929
			vcpu->arch.shadow_efer |= EFER_LMA;
930
			svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
A
Avi Kivity 已提交
931 932
		}

M
Mike Day 已提交
933
		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
934
			vcpu->arch.shadow_efer &= ~EFER_LMA;
935
			svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
A
Avi Kivity 已提交
936 937 938
		}
	}
#endif
939 940 941
	if (npt_enabled)
		goto set;

942
	if ((vcpu->arch.cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) {
943
		svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR);
A
Anthony Liguori 已提交
944 945 946
		vcpu->fpu_active = 1;
	}

947
	vcpu->arch.cr0 = cr0;
948
	cr0 |= X86_CR0_PG | X86_CR0_WP;
949 950
	if (!vcpu->fpu_active) {
		svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR);
J
Joerg Roedel 已提交
951
		cr0 |= X86_CR0_TS;
952
	}
953 954 955 956 957 958 959
set:
	/*
	 * re-enable caching here because the QEMU bios
	 * does not do it - this results in some delay at
	 * reboot
	 */
	cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
960
	svm->vmcb->save.cr0 = cr0;
A
Avi Kivity 已提交
961 962 963 964
}

static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
965
	unsigned long host_cr4_mce = read_cr4() & X86_CR4_MCE;
966 967 968 969
	unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;

	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
		force_new_asid(vcpu);
970

971 972 973
	vcpu->arch.cr4 = cr4;
	if (!npt_enabled)
		cr4 |= X86_CR4_PAE;
974
	cr4 |= host_cr4_mce;
975
	to_svm(vcpu)->vmcb->save.cr4 = cr4;
A
Avi Kivity 已提交
976 977 978 979 980
}

static void svm_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
981
	struct vcpu_svm *svm = to_svm(vcpu);
A
Avi Kivity 已提交
982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	s->base = var->base;
	s->limit = var->limit;
	s->selector = var->selector;
	if (var->unusable)
		s->attrib = 0;
	else {
		s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
		s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
		s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
		s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
		s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
		s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
		s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
		s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
	}
	if (seg == VCPU_SREG_CS)
1000 1001
		svm->vmcb->save.cpl
			= (svm->vmcb->save.cs.attrib
A
Avi Kivity 已提交
1002 1003 1004 1005
			   >> SVM_SELECTOR_DPL_SHIFT) & 3;

}

1006
static void update_db_intercept(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1007
{
J
Jan Kiszka 已提交
1008 1009 1010 1011
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->control.intercept_exceptions &=
		~((1 << DB_VECTOR) | (1 << BP_VECTOR));
1012 1013 1014 1015

	if (vcpu->arch.singlestep)
		svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR);

J
Jan Kiszka 已提交
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
		if (vcpu->guest_debug &
		    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
			svm->vmcb->control.intercept_exceptions |=
				1 << DB_VECTOR;
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
			svm->vmcb->control.intercept_exceptions |=
				1 << BP_VECTOR;
	} else
		vcpu->guest_debug = 0;
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
}

static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
{
	int old_debug = vcpu->guest_debug;
	struct vcpu_svm *svm = to_svm(vcpu);

	vcpu->guest_debug = dbg->control;

	update_db_intercept(vcpu);
J
Jan Kiszka 已提交
1036

1037 1038 1039 1040 1041
	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		svm->vmcb->save.dr7 = dbg->arch.debugreg[7];
	else
		svm->vmcb->save.dr7 = vcpu->arch.dr7;

J
Jan Kiszka 已提交
1042 1043 1044 1045 1046 1047
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		svm->vmcb->save.rflags |= X86_EFLAGS_TF | X86_EFLAGS_RF;
	else if (old_debug & KVM_GUESTDBG_SINGLESTEP)
		svm->vmcb->save.rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);

	return 0;
A
Avi Kivity 已提交
1048 1049 1050 1051
}

static void load_host_msrs(struct kvm_vcpu *vcpu)
{
1052
#ifdef CONFIG_X86_64
1053
	wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
1054
#endif
A
Avi Kivity 已提交
1055 1056 1057 1058
}

static void save_host_msrs(struct kvm_vcpu *vcpu)
{
1059
#ifdef CONFIG_X86_64
1060
	rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
1061
#endif
A
Avi Kivity 已提交
1062 1063
}

R
Rusty Russell 已提交
1064
static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *svm_data)
A
Avi Kivity 已提交
1065 1066 1067 1068
{
	if (svm_data->next_asid > svm_data->max_asid) {
		++svm_data->asid_generation;
		svm_data->next_asid = 1;
1069
		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
A
Avi Kivity 已提交
1070 1071
	}

1072 1073
	svm->asid_generation = svm_data->asid_generation;
	svm->vmcb->control.asid = svm_data->next_asid++;
A
Avi Kivity 已提交
1074 1075 1076 1077
}

static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr)
{
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
	struct vcpu_svm *svm = to_svm(vcpu);
	unsigned long val;

	switch (dr) {
	case 0 ... 3:
		val = vcpu->arch.db[dr];
		break;
	case 6:
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
			val = vcpu->arch.dr6;
		else
			val = svm->vmcb->save.dr6;
		break;
	case 7:
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
			val = vcpu->arch.dr7;
		else
			val = svm->vmcb->save.dr7;
		break;
	default:
		val = 0;
	}

1101
	return val;
A
Avi Kivity 已提交
1102 1103 1104 1105 1106
}

static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value,
		       int *exception)
{
1107 1108
	struct vcpu_svm *svm = to_svm(vcpu);

1109
	*exception = 0;
A
Avi Kivity 已提交
1110 1111 1112

	switch (dr) {
	case 0 ... 3:
1113 1114 1115
		vcpu->arch.db[dr] = value;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = value;
A
Avi Kivity 已提交
1116 1117
		return;
	case 4 ... 5:
1118
		if (vcpu->arch.cr4 & X86_CR4_DE)
A
Avi Kivity 已提交
1119
			*exception = UD_VECTOR;
1120 1121 1122 1123
		return;
	case 6:
		if (value & 0xffffffff00000000ULL) {
			*exception = GP_VECTOR;
A
Avi Kivity 已提交
1124 1125
			return;
		}
1126 1127 1128 1129
		vcpu->arch.dr6 = (value & DR6_VOLATILE) | DR6_FIXED_1;
		return;
	case 7:
		if (value & 0xffffffff00000000ULL) {
A
Avi Kivity 已提交
1130 1131 1132
			*exception = GP_VECTOR;
			return;
		}
1133 1134 1135 1136 1137
		vcpu->arch.dr7 = (value & DR7_VOLATILE) | DR7_FIXED_1;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
			svm->vmcb->save.dr7 = vcpu->arch.dr7;
			vcpu->arch.switch_db_regs = (value & DR7_BP_EN_MASK);
		}
A
Avi Kivity 已提交
1138 1139
		return;
	default:
1140
		/* FIXME: Possible case? */
A
Avi Kivity 已提交
1141
		printk(KERN_DEBUG "%s: unexpected dr %u\n",
1142
		       __func__, dr);
A
Avi Kivity 已提交
1143 1144 1145 1146 1147
		*exception = UD_VECTOR;
		return;
	}
}

R
Rusty Russell 已提交
1148
static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
1149 1150 1151 1152
{
	u64 fault_address;
	u32 error_code;

1153 1154
	fault_address  = svm->vmcb->control.exit_info_2;
	error_code = svm->vmcb->control.exit_info_1;
1155

1156
	trace_kvm_page_fault(fault_address, error_code);
1157 1158 1159 1160 1161 1162 1163
	/*
	 * FIXME: Tis shouldn't be necessary here, but there is a flush
	 * missing in the MMU code. Until we find this bug, flush the
	 * complete TLB here on an NPF
	 */
	if (npt_enabled)
		svm_flush_tlb(&svm->vcpu);
1164
	else {
1165
		if (kvm_event_needs_reinjection(&svm->vcpu))
1166 1167
			kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
	}
1168
	return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
A
Avi Kivity 已提交
1169 1170
}

J
Jan Kiszka 已提交
1171 1172 1173
static int db_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	if (!(svm->vcpu.guest_debug &
1174 1175
	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
		!svm->vcpu.arch.singlestep) {
J
Jan Kiszka 已提交
1176 1177 1178
		kvm_queue_exception(&svm->vcpu, DB_VECTOR);
		return 1;
	}
1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197

	if (svm->vcpu.arch.singlestep) {
		svm->vcpu.arch.singlestep = false;
		if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
			svm->vmcb->save.rflags &=
				~(X86_EFLAGS_TF | X86_EFLAGS_RF);
		update_db_intercept(&svm->vcpu);
	}

	if (svm->vcpu.guest_debug &
	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)){
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
		kvm_run->debug.arch.pc =
			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
		kvm_run->debug.arch.exception = DB_VECTOR;
		return 0;
	}

	return 1;
J
Jan Kiszka 已提交
1198 1199 1200 1201 1202 1203 1204 1205 1206 1207
}

static int bp_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	kvm_run->exit_reason = KVM_EXIT_DEBUG;
	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
	kvm_run->debug.arch.exception = BP_VECTOR;
	return 0;
}

1208 1209 1210 1211
static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	int er;

1212
	er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, EMULTYPE_TRAP_UD);
1213
	if (er != EMULATE_DONE)
1214
		kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1215 1216 1217
	return 1;
}

R
Rusty Russell 已提交
1218
static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
A
Anthony Liguori 已提交
1219
{
1220
	svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR);
1221
	if (!(svm->vcpu.arch.cr0 & X86_CR0_TS))
1222
		svm->vmcb->save.cr0 &= ~X86_CR0_TS;
R
Rusty Russell 已提交
1223
	svm->vcpu.fpu_active = 1;
1224 1225

	return 1;
A
Anthony Liguori 已提交
1226 1227
}

1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240
static int mc_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	/*
	 * On an #MC intercept the MCE handler is not called automatically in
	 * the host. So do it by hand here.
	 */
	asm volatile (
		"int $0x12\n");
	/* not sure if we ever come back to this point */

	return 1;
}

R
Rusty Russell 已提交
1241
static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
1242 1243 1244 1245 1246
{
	/*
	 * VMCB is undefined after a SHUTDOWN intercept
	 * so reinitialize it.
	 */
1247
	clear_page(svm->vmcb);
1248
	init_vmcb(svm);
1249 1250 1251 1252 1253

	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
	return 0;
}

R
Rusty Russell 已提交
1254
static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
1255
{
M
Mike Day 已提交
1256
	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
1257
	int size, in, string;
1258
	unsigned port;
A
Avi Kivity 已提交
1259

R
Rusty Russell 已提交
1260
	++svm->vcpu.stat.io_exits;
A
Avi Kivity 已提交
1261

1262
	svm->next_rip = svm->vmcb->control.exit_info_2;
A
Avi Kivity 已提交
1263

1264 1265 1266
	string = (io_info & SVM_IOIO_STR_MASK) != 0;

	if (string) {
1267 1268
		if (emulate_instruction(&svm->vcpu,
					kvm_run, 0, 0, 0) == EMULATE_DO_MMIO)
1269 1270 1271 1272
			return 0;
		return 1;
	}

1273 1274 1275
	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
	port = io_info >> 16;
	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
A
Avi Kivity 已提交
1276

1277
	skip_emulated_instruction(&svm->vcpu);
L
Laurent Vivier 已提交
1278
	return kvm_emulate_pio(&svm->vcpu, kvm_run, in, size, port);
A
Avi Kivity 已提交
1279 1280
}

1281 1282 1283 1284 1285
static int nmi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	return 1;
}

1286 1287 1288 1289 1290 1291
static int intr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	++svm->vcpu.stat.irq_exits;
	return 1;
}

R
Rusty Russell 已提交
1292
static int nop_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
1293 1294 1295 1296
{
	return 1;
}

R
Rusty Russell 已提交
1297
static int halt_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
1298
{
1299
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
R
Rusty Russell 已提交
1300 1301
	skip_emulated_instruction(&svm->vcpu);
	return kvm_emulate_halt(&svm->vcpu);
A
Avi Kivity 已提交
1302 1303
}

R
Rusty Russell 已提交
1304
static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
1305
{
1306
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
R
Rusty Russell 已提交
1307
	skip_emulated_instruction(&svm->vcpu);
1308 1309
	kvm_emulate_hypercall(&svm->vcpu);
	return 1;
1310 1311
}

1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327
static int nested_svm_check_permissions(struct vcpu_svm *svm)
{
	if (!(svm->vcpu.arch.shadow_efer & EFER_SVME)
	    || !is_paging(&svm->vcpu)) {
		kvm_queue_exception(&svm->vcpu, UD_VECTOR);
		return 1;
	}

	if (svm->vmcb->save.cpl) {
		kvm_inject_gp(&svm->vcpu, 0);
		return 1;
	}

       return 0;
}

1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367
static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
				      bool has_error_code, u32 error_code)
{
	if (is_nested(svm)) {
		svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
		svm->vmcb->control.exit_code_hi = 0;
		svm->vmcb->control.exit_info_1 = error_code;
		svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
		if (nested_svm_exit_handled(svm, false)) {
			nsvm_printk("VMexit -> EXCP 0x%x\n", nr);

			nested_svm_vmexit(svm);
			return 1;
		}
	}

	return 0;
}

static inline int nested_svm_intr(struct vcpu_svm *svm)
{
	if (is_nested(svm)) {
		if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
			return 0;

		if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
			return 0;

		svm->vmcb->control.exit_code = SVM_EXIT_INTR;

		if (nested_svm_exit_handled(svm, false)) {
			nsvm_printk("VMexit -> INTR\n");
			nested_svm_vmexit(svm);
			return 1;
		}
	}

	return 0;
}

1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
static struct page *nested_svm_get_page(struct vcpu_svm *svm, u64 gpa)
{
	struct page *page;

	down_read(&current->mm->mmap_sem);
	page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT);
	up_read(&current->mm->mmap_sem);

	if (is_error_page(page)) {
		printk(KERN_INFO "%s: could not find page at 0x%llx\n",
		       __func__, gpa);
		kvm_release_page_clean(page);
		kvm_inject_gp(&svm->vcpu, 0);
		return NULL;
	}
	return page;
}

static int nested_svm_do(struct vcpu_svm *svm,
			 u64 arg1_gpa, u64 arg2_gpa, void *opaque,
			 int (*handler)(struct vcpu_svm *svm,
					void *arg1,
					void *arg2,
					void *opaque))
{
	struct page *arg1_page;
	struct page *arg2_page = NULL;
	void *arg1;
	void *arg2 = NULL;
	int retval;

	arg1_page = nested_svm_get_page(svm, arg1_gpa);
	if(arg1_page == NULL)
		return 1;

	if (arg2_gpa) {
		arg2_page = nested_svm_get_page(svm, arg2_gpa);
		if(arg2_page == NULL) {
			kvm_release_page_clean(arg1_page);
			return 1;
		}
	}

	arg1 = kmap_atomic(arg1_page, KM_USER0);
	if (arg2_gpa)
		arg2 = kmap_atomic(arg2_page, KM_USER1);

	retval = handler(svm, arg1, arg2, opaque);

	kunmap_atomic(arg1, KM_USER0);
	if (arg2_gpa)
		kunmap_atomic(arg2, KM_USER1);

	kvm_release_page_dirty(arg1_page);
	if (arg2_gpa)
		kvm_release_page_dirty(arg2_page);

	return retval;
}

1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604
static int nested_svm_exit_handled_real(struct vcpu_svm *svm,
					void *arg1,
					void *arg2,
					void *opaque)
{
	struct vmcb *nested_vmcb = (struct vmcb *)arg1;
	bool kvm_overrides = *(bool *)opaque;
	u32 exit_code = svm->vmcb->control.exit_code;

	if (kvm_overrides) {
		switch (exit_code) {
		case SVM_EXIT_INTR:
		case SVM_EXIT_NMI:
			return 0;
		/* For now we are always handling NPFs when using them */
		case SVM_EXIT_NPF:
			if (npt_enabled)
				return 0;
			break;
		/* When we're shadowing, trap PFs */
		case SVM_EXIT_EXCP_BASE + PF_VECTOR:
			if (!npt_enabled)
				return 0;
			break;
		default:
			break;
		}
	}

	switch (exit_code) {
	case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: {
		u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0);
		if (nested_vmcb->control.intercept_cr_read & cr_bits)
			return 1;
		break;
	}
	case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: {
		u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0);
		if (nested_vmcb->control.intercept_cr_write & cr_bits)
			return 1;
		break;
	}
	case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: {
		u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0);
		if (nested_vmcb->control.intercept_dr_read & dr_bits)
			return 1;
		break;
	}
	case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: {
		u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0);
		if (nested_vmcb->control.intercept_dr_write & dr_bits)
			return 1;
		break;
	}
	case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
		u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
		if (nested_vmcb->control.intercept_exceptions & excp_bits)
			return 1;
		break;
	}
	default: {
		u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
		nsvm_printk("exit code: 0x%x\n", exit_code);
		if (nested_vmcb->control.intercept & exit_bits)
			return 1;
	}
	}

	return 0;
}

static int nested_svm_exit_handled_msr(struct vcpu_svm *svm,
				       void *arg1, void *arg2,
				       void *opaque)
{
	struct vmcb *nested_vmcb = (struct vmcb *)arg1;
	u8 *msrpm = (u8 *)arg2;
        u32 t0, t1;
	u32 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
	u32 param = svm->vmcb->control.exit_info_1 & 1;

	if (!(nested_vmcb->control.intercept & (1ULL << INTERCEPT_MSR_PROT)))
		return 0;

	switch(msr) {
	case 0 ... 0x1fff:
		t0 = (msr * 2) % 8;
		t1 = msr / 8;
		break;
	case 0xc0000000 ... 0xc0001fff:
		t0 = (8192 + msr - 0xc0000000) * 2;
		t1 = (t0 / 8);
		t0 %= 8;
		break;
	case 0xc0010000 ... 0xc0011fff:
		t0 = (16384 + msr - 0xc0010000) * 2;
		t1 = (t0 / 8);
		t0 %= 8;
		break;
	default:
		return 1;
		break;
	}
	if (msrpm[t1] & ((1 << param) << t0))
		return 1;

	return 0;
}

static int nested_svm_exit_handled(struct vcpu_svm *svm, bool kvm_override)
{
	bool k = kvm_override;

	switch (svm->vmcb->control.exit_code) {
	case SVM_EXIT_MSR:
		return nested_svm_do(svm, svm->nested_vmcb,
				     svm->nested_vmcb_msrpm, NULL,
				     nested_svm_exit_handled_msr);
	default: break;
	}

	return nested_svm_do(svm, svm->nested_vmcb, 0, &k,
			     nested_svm_exit_handled_real);
}

static int nested_svm_vmexit_real(struct vcpu_svm *svm, void *arg1,
				  void *arg2, void *opaque)
{
	struct vmcb *nested_vmcb = (struct vmcb *)arg1;
	struct vmcb *hsave = svm->hsave;
	u64 nested_save[] = { nested_vmcb->save.cr0,
			      nested_vmcb->save.cr3,
			      nested_vmcb->save.cr4,
			      nested_vmcb->save.efer,
			      nested_vmcb->control.intercept_cr_read,
			      nested_vmcb->control.intercept_cr_write,
			      nested_vmcb->control.intercept_dr_read,
			      nested_vmcb->control.intercept_dr_write,
			      nested_vmcb->control.intercept_exceptions,
			      nested_vmcb->control.intercept,
			      nested_vmcb->control.msrpm_base_pa,
			      nested_vmcb->control.iopm_base_pa,
			      nested_vmcb->control.tsc_offset };

	/* Give the current vmcb to the guest */
	memcpy(nested_vmcb, svm->vmcb, sizeof(struct vmcb));
	nested_vmcb->save.cr0 = nested_save[0];
	if (!npt_enabled)
		nested_vmcb->save.cr3 = nested_save[1];
	nested_vmcb->save.cr4 = nested_save[2];
	nested_vmcb->save.efer = nested_save[3];
	nested_vmcb->control.intercept_cr_read = nested_save[4];
	nested_vmcb->control.intercept_cr_write = nested_save[5];
	nested_vmcb->control.intercept_dr_read = nested_save[6];
	nested_vmcb->control.intercept_dr_write = nested_save[7];
	nested_vmcb->control.intercept_exceptions = nested_save[8];
	nested_vmcb->control.intercept = nested_save[9];
	nested_vmcb->control.msrpm_base_pa = nested_save[10];
	nested_vmcb->control.iopm_base_pa = nested_save[11];
	nested_vmcb->control.tsc_offset = nested_save[12];

	/* We always set V_INTR_MASKING and remember the old value in hflags */
	if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
		nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;

	if ((nested_vmcb->control.int_ctl & V_IRQ_MASK) &&
	    (nested_vmcb->control.int_vector)) {
		nsvm_printk("WARNING: IRQ 0x%x still enabled on #VMEXIT\n",
				nested_vmcb->control.int_vector);
	}

	/* Restore the original control entries */
	svm->vmcb->control = hsave->control;

	/* Kill any pending exceptions */
	if (svm->vcpu.arch.exception.pending == true)
		nsvm_printk("WARNING: Pending Exception\n");
1605 1606
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);
1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650

	/* Restore selected save entries */
	svm->vmcb->save.es = hsave->save.es;
	svm->vmcb->save.cs = hsave->save.cs;
	svm->vmcb->save.ss = hsave->save.ss;
	svm->vmcb->save.ds = hsave->save.ds;
	svm->vmcb->save.gdtr = hsave->save.gdtr;
	svm->vmcb->save.idtr = hsave->save.idtr;
	svm->vmcb->save.rflags = hsave->save.rflags;
	svm_set_efer(&svm->vcpu, hsave->save.efer);
	svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
	svm_set_cr4(&svm->vcpu, hsave->save.cr4);
	if (npt_enabled) {
		svm->vmcb->save.cr3 = hsave->save.cr3;
		svm->vcpu.arch.cr3 = hsave->save.cr3;
	} else {
		kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
	}
	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
	svm->vmcb->save.dr7 = 0;
	svm->vmcb->save.cpl = 0;
	svm->vmcb->control.exit_int_info = 0;

	svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
	/* Exit nested SVM mode */
	svm->nested_vmcb = 0;

	return 0;
}

static int nested_svm_vmexit(struct vcpu_svm *svm)
{
	nsvm_printk("VMexit\n");
	if (nested_svm_do(svm, svm->nested_vmcb, 0,
			  NULL, nested_svm_vmexit_real))
		return 1;

	kvm_mmu_reset_context(&svm->vcpu);
	kvm_mmu_load(&svm->vcpu);

	return 0;
}
A
Alexander Graf 已提交
1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673

static int nested_svm_vmrun_msrpm(struct vcpu_svm *svm, void *arg1,
				  void *arg2, void *opaque)
{
	int i;
	u32 *nested_msrpm = (u32*)arg1;
	for (i=0; i< PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER) / 4; i++)
		svm->nested_msrpm[i] = svm->msrpm[i] | nested_msrpm[i];
	svm->vmcb->control.msrpm_base_pa = __pa(svm->nested_msrpm);

	return 0;
}

static int nested_svm_vmrun(struct vcpu_svm *svm, void *arg1,
			    void *arg2, void *opaque)
{
	struct vmcb *nested_vmcb = (struct vmcb *)arg1;
	struct vmcb *hsave = svm->hsave;

	/* nested_vmcb is our indicator if nested SVM is activated */
	svm->nested_vmcb = svm->vmcb->save.rax;

	/* Clear internal status */
1674 1675
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);
A
Alexander Graf 已提交
1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768

	/* Save the old vmcb, so we don't need to pick what we save, but
	   can restore everything when a VMEXIT occurs */
	memcpy(hsave, svm->vmcb, sizeof(struct vmcb));
	/* We need to remember the original CR3 in the SPT case */
	if (!npt_enabled)
		hsave->save.cr3 = svm->vcpu.arch.cr3;
	hsave->save.cr4 = svm->vcpu.arch.cr4;
	hsave->save.rip = svm->next_rip;

	if (svm->vmcb->save.rflags & X86_EFLAGS_IF)
		svm->vcpu.arch.hflags |= HF_HIF_MASK;
	else
		svm->vcpu.arch.hflags &= ~HF_HIF_MASK;

	/* Load the nested guest state */
	svm->vmcb->save.es = nested_vmcb->save.es;
	svm->vmcb->save.cs = nested_vmcb->save.cs;
	svm->vmcb->save.ss = nested_vmcb->save.ss;
	svm->vmcb->save.ds = nested_vmcb->save.ds;
	svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
	svm->vmcb->save.idtr = nested_vmcb->save.idtr;
	svm->vmcb->save.rflags = nested_vmcb->save.rflags;
	svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
	svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
	svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
	if (npt_enabled) {
		svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
		svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
	} else {
		kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
		kvm_mmu_reset_context(&svm->vcpu);
	}
	svm->vmcb->save.cr2 = nested_vmcb->save.cr2;
	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
	/* In case we don't even reach vcpu_run, the fields are not updated */
	svm->vmcb->save.rax = nested_vmcb->save.rax;
	svm->vmcb->save.rsp = nested_vmcb->save.rsp;
	svm->vmcb->save.rip = nested_vmcb->save.rip;
	svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
	svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
	svm->vmcb->save.cpl = nested_vmcb->save.cpl;

	/* We don't want a nested guest to be more powerful than the guest,
	   so all intercepts are ORed */
	svm->vmcb->control.intercept_cr_read |=
		nested_vmcb->control.intercept_cr_read;
	svm->vmcb->control.intercept_cr_write |=
		nested_vmcb->control.intercept_cr_write;
	svm->vmcb->control.intercept_dr_read |=
		nested_vmcb->control.intercept_dr_read;
	svm->vmcb->control.intercept_dr_write |=
		nested_vmcb->control.intercept_dr_write;
	svm->vmcb->control.intercept_exceptions |=
		nested_vmcb->control.intercept_exceptions;

	svm->vmcb->control.intercept |= nested_vmcb->control.intercept;

	svm->nested_vmcb_msrpm = nested_vmcb->control.msrpm_base_pa;

	force_new_asid(&svm->vcpu);
	svm->vmcb->control.exit_int_info = nested_vmcb->control.exit_int_info;
	svm->vmcb->control.exit_int_info_err = nested_vmcb->control.exit_int_info_err;
	svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
	if (nested_vmcb->control.int_ctl & V_IRQ_MASK) {
		nsvm_printk("nSVM Injecting Interrupt: 0x%x\n",
				nested_vmcb->control.int_ctl);
	}
	if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
		svm->vcpu.arch.hflags |= HF_VINTR_MASK;
	else
		svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;

	nsvm_printk("nSVM exit_int_info: 0x%x | int_state: 0x%x\n",
			nested_vmcb->control.exit_int_info,
			nested_vmcb->control.int_state);

	svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
	svm->vmcb->control.int_state = nested_vmcb->control.int_state;
	svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
	if (nested_vmcb->control.event_inj & SVM_EVTINJ_VALID)
		nsvm_printk("Injecting Event: 0x%x\n",
				nested_vmcb->control.event_inj);
	svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
	svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;

	svm->vcpu.arch.hflags |= HF_GIF_MASK;

	return 0;
}

1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824
static int nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
{
	to_vmcb->save.fs = from_vmcb->save.fs;
	to_vmcb->save.gs = from_vmcb->save.gs;
	to_vmcb->save.tr = from_vmcb->save.tr;
	to_vmcb->save.ldtr = from_vmcb->save.ldtr;
	to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
	to_vmcb->save.star = from_vmcb->save.star;
	to_vmcb->save.lstar = from_vmcb->save.lstar;
	to_vmcb->save.cstar = from_vmcb->save.cstar;
	to_vmcb->save.sfmask = from_vmcb->save.sfmask;
	to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
	to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
	to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;

	return 1;
}

static int nested_svm_vmload(struct vcpu_svm *svm, void *nested_vmcb,
			     void *arg2, void *opaque)
{
	return nested_svm_vmloadsave((struct vmcb *)nested_vmcb, svm->vmcb);
}

static int nested_svm_vmsave(struct vcpu_svm *svm, void *nested_vmcb,
			     void *arg2, void *opaque)
{
	return nested_svm_vmloadsave(svm->vmcb, (struct vmcb *)nested_vmcb);
}

static int vmload_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

	nested_svm_do(svm, svm->vmcb->save.rax, 0, NULL, nested_svm_vmload);

	return 1;
}

static int vmsave_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

	nested_svm_do(svm, svm->vmcb->save.rax, 0, NULL, nested_svm_vmsave);

	return 1;
}

A
Alexander Graf 已提交
1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844
static int vmrun_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	nsvm_printk("VMrun\n");
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

	if (nested_svm_do(svm, svm->vmcb->save.rax, 0,
			  NULL, nested_svm_vmrun))
		return 1;

	if (nested_svm_do(svm, svm->nested_vmcb_msrpm, 0,
		      NULL, nested_svm_vmrun_msrpm))
		return 1;

	return 1;
}

1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874
static int stgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

	svm->vcpu.arch.hflags |= HF_GIF_MASK;

	return 1;
}

static int clgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

	svm->vcpu.arch.hflags &= ~HF_GIF_MASK;

	/* After a CLGI no interrupts should come */
	svm_clear_vintr(svm);
	svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;

	return 1;
}

A
Alexander Graf 已提交
1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887
static int invlpga_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	struct kvm_vcpu *vcpu = &svm->vcpu;
	nsvm_printk("INVLPGA\n");

	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
	kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]);

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);
	return 1;
}

R
Rusty Russell 已提交
1888 1889
static int invalid_op_interception(struct vcpu_svm *svm,
				   struct kvm_run *kvm_run)
A
Avi Kivity 已提交
1890
{
1891
	kvm_queue_exception(&svm->vcpu, UD_VECTOR);
A
Avi Kivity 已提交
1892 1893 1894
	return 1;
}

R
Rusty Russell 已提交
1895 1896
static int task_switch_interception(struct vcpu_svm *svm,
				    struct kvm_run *kvm_run)
A
Avi Kivity 已提交
1897
{
1898
	u16 tss_selector;
1899 1900 1901
	int reason;
	int int_type = svm->vmcb->control.exit_int_info &
		SVM_EXITINTINFO_TYPE_MASK;
1902
	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
1903 1904 1905 1906
	uint32_t type =
		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
	uint32_t idt_v =
		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
1907 1908

	tss_selector = (u16)svm->vmcb->control.exit_info_1;
1909

1910 1911
	if (svm->vmcb->control.exit_info_2 &
	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
1912 1913 1914 1915
		reason = TASK_SWITCH_IRET;
	else if (svm->vmcb->control.exit_info_2 &
		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
		reason = TASK_SWITCH_JMP;
1916
	else if (idt_v)
1917 1918 1919 1920
		reason = TASK_SWITCH_GATE;
	else
		reason = TASK_SWITCH_CALL;

1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935
	if (reason == TASK_SWITCH_GATE) {
		switch (type) {
		case SVM_EXITINTINFO_TYPE_NMI:
			svm->vcpu.arch.nmi_injected = false;
			break;
		case SVM_EXITINTINFO_TYPE_EXEPT:
			kvm_clear_exception_queue(&svm->vcpu);
			break;
		case SVM_EXITINTINFO_TYPE_INTR:
			kvm_clear_interrupt_queue(&svm->vcpu);
			break;
		default:
			break;
		}
	}
1936

1937 1938 1939
	if (reason != TASK_SWITCH_GATE ||
	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
1940 1941
	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
		skip_emulated_instruction(&svm->vcpu);
1942 1943

	return kvm_task_switch(&svm->vcpu, tss_selector, reason);
A
Avi Kivity 已提交
1944 1945
}

R
Rusty Russell 已提交
1946
static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
1947
{
1948
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
R
Rusty Russell 已提交
1949
	kvm_emulate_cpuid(&svm->vcpu);
1950
	return 1;
A
Avi Kivity 已提交
1951 1952
}

1953 1954 1955 1956
static int iret_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	++svm->vcpu.stat.nmi_window_exits;
	svm->vmcb->control.intercept &= ~(1UL << INTERCEPT_IRET);
1957
	svm->vcpu.arch.hflags |= HF_IRET_MASK;
1958 1959 1960
	return 1;
}

M
Marcelo Tosatti 已提交
1961 1962 1963 1964 1965 1966 1967
static int invlpg_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
	if (emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0) != EMULATE_DONE)
		pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
	return 1;
}

R
Rusty Russell 已提交
1968 1969
static int emulate_on_interception(struct vcpu_svm *svm,
				   struct kvm_run *kvm_run)
A
Avi Kivity 已提交
1970
{
1971
	if (emulate_instruction(&svm->vcpu, NULL, 0, 0, 0) != EMULATE_DONE)
1972
		pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
A
Avi Kivity 已提交
1973 1974 1975
	return 1;
}

1976 1977
static int cr8_write_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
{
1978 1979
	u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
	/* instruction emulation calls kvm_set_cr8() */
1980
	emulate_instruction(&svm->vcpu, NULL, 0, 0, 0);
1981 1982
	if (irqchip_in_kernel(svm->vcpu.kvm)) {
		svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
1983
		return 1;
1984
	}
1985 1986
	if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
		return 1;
1987 1988 1989 1990
	kvm_run->exit_reason = KVM_EXIT_SET_TPR;
	return 0;
}

A
Avi Kivity 已提交
1991 1992
static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
{
1993 1994
	struct vcpu_svm *svm = to_svm(vcpu);

A
Avi Kivity 已提交
1995
	switch (ecx) {
1996
	case MSR_IA32_TSC: {
A
Avi Kivity 已提交
1997 1998 1999
		u64 tsc;

		rdtscll(tsc);
2000
		*data = svm->vmcb->control.tsc_offset + tsc;
A
Avi Kivity 已提交
2001 2002
		break;
	}
2003
	case MSR_K6_STAR:
2004
		*data = svm->vmcb->save.star;
A
Avi Kivity 已提交
2005
		break;
2006
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2007
	case MSR_LSTAR:
2008
		*data = svm->vmcb->save.lstar;
A
Avi Kivity 已提交
2009 2010
		break;
	case MSR_CSTAR:
2011
		*data = svm->vmcb->save.cstar;
A
Avi Kivity 已提交
2012 2013
		break;
	case MSR_KERNEL_GS_BASE:
2014
		*data = svm->vmcb->save.kernel_gs_base;
A
Avi Kivity 已提交
2015 2016
		break;
	case MSR_SYSCALL_MASK:
2017
		*data = svm->vmcb->save.sfmask;
A
Avi Kivity 已提交
2018 2019 2020
		break;
#endif
	case MSR_IA32_SYSENTER_CS:
2021
		*data = svm->vmcb->save.sysenter_cs;
A
Avi Kivity 已提交
2022 2023
		break;
	case MSR_IA32_SYSENTER_EIP:
2024
		*data = svm->sysenter_eip;
A
Avi Kivity 已提交
2025 2026
		break;
	case MSR_IA32_SYSENTER_ESP:
2027
		*data = svm->sysenter_esp;
A
Avi Kivity 已提交
2028
		break;
2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046
	/* Nobody will change the following 5 values in the VMCB so
	   we can safely return them on rdmsr. They will always be 0
	   until LBRV is implemented. */
	case MSR_IA32_DEBUGCTLMSR:
		*data = svm->vmcb->save.dbgctl;
		break;
	case MSR_IA32_LASTBRANCHFROMIP:
		*data = svm->vmcb->save.br_from;
		break;
	case MSR_IA32_LASTBRANCHTOIP:
		*data = svm->vmcb->save.br_to;
		break;
	case MSR_IA32_LASTINTFROMIP:
		*data = svm->vmcb->save.last_excp_from;
		break;
	case MSR_IA32_LASTINTTOIP:
		*data = svm->vmcb->save.last_excp_to;
		break;
A
Alexander Graf 已提交
2047 2048 2049
	case MSR_VM_HSAVE_PA:
		*data = svm->hsave_msr;
		break;
2050 2051 2052
	case MSR_VM_CR:
		*data = 0;
		break;
2053 2054 2055
	case MSR_IA32_UCODE_REV:
		*data = 0x01000065;
		break;
A
Avi Kivity 已提交
2056
	default:
2057
		return kvm_get_msr_common(vcpu, ecx, data);
A
Avi Kivity 已提交
2058 2059 2060 2061
	}
	return 0;
}

R
Rusty Russell 已提交
2062
static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
2063
{
2064
	u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
A
Avi Kivity 已提交
2065 2066
	u64 data;

R
Rusty Russell 已提交
2067
	if (svm_get_msr(&svm->vcpu, ecx, &data))
2068
		kvm_inject_gp(&svm->vcpu, 0);
A
Avi Kivity 已提交
2069
	else {
2070
		trace_kvm_msr_read(ecx, data);
2071

2072
		svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff;
2073
		svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;
2074
		svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
R
Rusty Russell 已提交
2075
		skip_emulated_instruction(&svm->vcpu);
A
Avi Kivity 已提交
2076 2077 2078 2079 2080 2081
	}
	return 1;
}

static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
{
2082 2083
	struct vcpu_svm *svm = to_svm(vcpu);

A
Avi Kivity 已提交
2084
	switch (ecx) {
2085
	case MSR_IA32_TSC: {
A
Avi Kivity 已提交
2086 2087 2088
		u64 tsc;

		rdtscll(tsc);
2089
		svm->vmcb->control.tsc_offset = data - tsc;
A
Avi Kivity 已提交
2090 2091
		break;
	}
2092
	case MSR_K6_STAR:
2093
		svm->vmcb->save.star = data;
A
Avi Kivity 已提交
2094
		break;
2095
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2096
	case MSR_LSTAR:
2097
		svm->vmcb->save.lstar = data;
A
Avi Kivity 已提交
2098 2099
		break;
	case MSR_CSTAR:
2100
		svm->vmcb->save.cstar = data;
A
Avi Kivity 已提交
2101 2102
		break;
	case MSR_KERNEL_GS_BASE:
2103
		svm->vmcb->save.kernel_gs_base = data;
A
Avi Kivity 已提交
2104 2105
		break;
	case MSR_SYSCALL_MASK:
2106
		svm->vmcb->save.sfmask = data;
A
Avi Kivity 已提交
2107 2108 2109
		break;
#endif
	case MSR_IA32_SYSENTER_CS:
2110
		svm->vmcb->save.sysenter_cs = data;
A
Avi Kivity 已提交
2111 2112
		break;
	case MSR_IA32_SYSENTER_EIP:
2113
		svm->sysenter_eip = data;
2114
		svm->vmcb->save.sysenter_eip = data;
A
Avi Kivity 已提交
2115 2116
		break;
	case MSR_IA32_SYSENTER_ESP:
2117
		svm->sysenter_esp = data;
2118
		svm->vmcb->save.sysenter_esp = data;
A
Avi Kivity 已提交
2119
		break;
2120
	case MSR_IA32_DEBUGCTLMSR:
2121 2122
		if (!svm_has(SVM_FEATURE_LBRV)) {
			pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
2123
					__func__, data);
2124 2125 2126 2127 2128 2129 2130 2131 2132 2133
			break;
		}
		if (data & DEBUGCTL_RESERVED_BITS)
			return 1;

		svm->vmcb->save.dbgctl = data;
		if (data & (1ULL<<0))
			svm_enable_lbrv(svm);
		else
			svm_disable_lbrv(svm);
2134
		break;
A
Alexander Graf 已提交
2135 2136
	case MSR_VM_HSAVE_PA:
		svm->hsave_msr = data;
2137
		break;
2138 2139 2140 2141
	case MSR_VM_CR:
	case MSR_VM_IGNNE:
		pr_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
		break;
A
Avi Kivity 已提交
2142
	default:
2143
		return kvm_set_msr_common(vcpu, ecx, data);
A
Avi Kivity 已提交
2144 2145 2146 2147
	}
	return 0;
}

R
Rusty Russell 已提交
2148
static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
2149
{
2150
	u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2151
	u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u)
2152
		| ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
2153

2154
	trace_kvm_msr_write(ecx, data);
2155

2156
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
R
Rusty Russell 已提交
2157
	if (svm_set_msr(&svm->vcpu, ecx, data))
2158
		kvm_inject_gp(&svm->vcpu, 0);
A
Avi Kivity 已提交
2159
	else
R
Rusty Russell 已提交
2160
		skip_emulated_instruction(&svm->vcpu);
A
Avi Kivity 已提交
2161 2162 2163
	return 1;
}

R
Rusty Russell 已提交
2164
static int msr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
2165
{
R
Rusty Russell 已提交
2166 2167
	if (svm->vmcb->control.exit_info_1)
		return wrmsr_interception(svm, kvm_run);
A
Avi Kivity 已提交
2168
	else
R
Rusty Russell 已提交
2169
		return rdmsr_interception(svm, kvm_run);
A
Avi Kivity 已提交
2170 2171
}

R
Rusty Russell 已提交
2172
static int interrupt_window_interception(struct vcpu_svm *svm,
2173 2174
				   struct kvm_run *kvm_run)
{
2175
	svm_clear_vintr(svm);
2176
	svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2177 2178 2179 2180
	/*
	 * If the user space waits to inject interrupts, exit as soon as
	 * possible
	 */
2181 2182 2183
	if (!irqchip_in_kernel(svm->vcpu.kvm) &&
	    kvm_run->request_interrupt_window &&
	    !kvm_cpu_has_interrupt(&svm->vcpu)) {
R
Rusty Russell 已提交
2184
		++svm->vcpu.stat.irq_window_exits;
2185 2186 2187 2188 2189 2190 2191
		kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
		return 0;
	}

	return 1;
}

R
Rusty Russell 已提交
2192
static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
A
Avi Kivity 已提交
2193 2194 2195 2196
				      struct kvm_run *kvm_run) = {
	[SVM_EXIT_READ_CR0]           		= emulate_on_interception,
	[SVM_EXIT_READ_CR3]           		= emulate_on_interception,
	[SVM_EXIT_READ_CR4]           		= emulate_on_interception,
2197
	[SVM_EXIT_READ_CR8]           		= emulate_on_interception,
A
Avi Kivity 已提交
2198 2199 2200 2201
	/* for now: */
	[SVM_EXIT_WRITE_CR0]          		= emulate_on_interception,
	[SVM_EXIT_WRITE_CR3]          		= emulate_on_interception,
	[SVM_EXIT_WRITE_CR4]          		= emulate_on_interception,
2202
	[SVM_EXIT_WRITE_CR8]          		= cr8_write_interception,
A
Avi Kivity 已提交
2203 2204 2205 2206 2207 2208 2209 2210 2211 2212
	[SVM_EXIT_READ_DR0] 			= emulate_on_interception,
	[SVM_EXIT_READ_DR1]			= emulate_on_interception,
	[SVM_EXIT_READ_DR2]			= emulate_on_interception,
	[SVM_EXIT_READ_DR3]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR0]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR1]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR2]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR3]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR5]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR7]			= emulate_on_interception,
J
Jan Kiszka 已提交
2213 2214
	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
2215
	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
A
Avi Kivity 已提交
2216
	[SVM_EXIT_EXCP_BASE + PF_VECTOR] 	= pf_interception,
A
Anthony Liguori 已提交
2217
	[SVM_EXIT_EXCP_BASE + NM_VECTOR] 	= nm_interception,
2218
	[SVM_EXIT_EXCP_BASE + MC_VECTOR] 	= mc_interception,
2219
	[SVM_EXIT_INTR] 			= intr_interception,
2220
	[SVM_EXIT_NMI]				= nmi_interception,
A
Avi Kivity 已提交
2221 2222
	[SVM_EXIT_SMI]				= nop_on_interception,
	[SVM_EXIT_INIT]				= nop_on_interception,
2223
	[SVM_EXIT_VINTR]			= interrupt_window_interception,
A
Avi Kivity 已提交
2224 2225
	/* [SVM_EXIT_CR0_SEL_WRITE]		= emulate_on_interception, */
	[SVM_EXIT_CPUID]			= cpuid_interception,
2226
	[SVM_EXIT_IRET]                         = iret_interception,
2227
	[SVM_EXIT_INVD]                         = emulate_on_interception,
A
Avi Kivity 已提交
2228
	[SVM_EXIT_HLT]				= halt_interception,
M
Marcelo Tosatti 已提交
2229
	[SVM_EXIT_INVLPG]			= invlpg_interception,
A
Alexander Graf 已提交
2230
	[SVM_EXIT_INVLPGA]			= invlpga_interception,
A
Avi Kivity 已提交
2231 2232 2233
	[SVM_EXIT_IOIO] 		  	= io_interception,
	[SVM_EXIT_MSR]				= msr_interception,
	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
2234
	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
A
Alexander Graf 已提交
2235
	[SVM_EXIT_VMRUN]			= vmrun_interception,
2236
	[SVM_EXIT_VMMCALL]			= vmmcall_interception,
2237 2238
	[SVM_EXIT_VMLOAD]			= vmload_interception,
	[SVM_EXIT_VMSAVE]			= vmsave_interception,
2239 2240
	[SVM_EXIT_STGI]				= stgi_interception,
	[SVM_EXIT_CLGI]				= clgi_interception,
A
Avi Kivity 已提交
2241
	[SVM_EXIT_SKINIT]			= invalid_op_interception,
2242
	[SVM_EXIT_WBINVD]                       = emulate_on_interception,
2243 2244
	[SVM_EXIT_MONITOR]			= invalid_op_interception,
	[SVM_EXIT_MWAIT]			= invalid_op_interception,
2245
	[SVM_EXIT_NPF]				= pf_interception,
A
Avi Kivity 已提交
2246 2247
};

2248
static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2249
{
2250
	struct vcpu_svm *svm = to_svm(vcpu);
2251
	u32 exit_code = svm->vmcb->control.exit_code;
A
Avi Kivity 已提交
2252

2253
	trace_kvm_exit(exit_code, svm->vmcb->save.rip);
2254

2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265
	if (is_nested(svm)) {
		nsvm_printk("nested handle_exit: 0x%x | 0x%lx | 0x%lx | 0x%lx\n",
			    exit_code, svm->vmcb->control.exit_info_1,
			    svm->vmcb->control.exit_info_2, svm->vmcb->save.rip);
		if (nested_svm_exit_handled(svm, true)) {
			nested_svm_vmexit(svm);
			nsvm_printk("-> #VMEXIT\n");
			return 1;
		}
	}

2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279
	if (npt_enabled) {
		int mmu_reload = 0;
		if ((vcpu->arch.cr0 ^ svm->vmcb->save.cr0) & X86_CR0_PG) {
			svm_set_cr0(vcpu, svm->vmcb->save.cr0);
			mmu_reload = 1;
		}
		vcpu->arch.cr0 = svm->vmcb->save.cr0;
		vcpu->arch.cr3 = svm->vmcb->save.cr3;
		if (mmu_reload) {
			kvm_mmu_reset_context(vcpu);
			kvm_mmu_load(vcpu);
		}
	}

2280 2281 2282 2283 2284 2285 2286 2287

	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		kvm_run->fail_entry.hardware_entry_failure_reason
			= svm->vmcb->control.exit_code;
		return 0;
	}

2288
	if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
2289
	    exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
2290
	    exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH)
A
Avi Kivity 已提交
2291 2292
		printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
		       "exit_code 0x%x\n",
2293
		       __func__, svm->vmcb->control.exit_int_info,
A
Avi Kivity 已提交
2294 2295
		       exit_code);

2296
	if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
J
Joe Perches 已提交
2297
	    || !svm_exit_handlers[exit_code]) {
A
Avi Kivity 已提交
2298
		kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
2299
		kvm_run->hw.hardware_exit_reason = exit_code;
A
Avi Kivity 已提交
2300 2301 2302
		return 0;
	}

R
Rusty Russell 已提交
2303
	return svm_exit_handlers[exit_code](svm, kvm_run);
A
Avi Kivity 已提交
2304 2305 2306 2307 2308 2309 2310
}

static void reload_tss(struct kvm_vcpu *vcpu)
{
	int cpu = raw_smp_processor_id();

	struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
M
Mike Day 已提交
2311
	svm_data->tss_desc->type = 9; /* available 32/64-bit TSS */
A
Avi Kivity 已提交
2312 2313 2314
	load_TR_desc();
}

R
Rusty Russell 已提交
2315
static void pre_svm_run(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2316 2317 2318 2319 2320
{
	int cpu = raw_smp_processor_id();

	struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);

2321
	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
2322 2323
	/* FIXME: handle wraparound of asid_generation */
	if (svm->asid_generation != svm_data->asid_generation)
R
Rusty Russell 已提交
2324
		new_asid(svm, svm_data);
A
Avi Kivity 已提交
2325 2326
}

2327 2328 2329 2330 2331 2332 2333 2334 2335
static void svm_inject_nmi(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
	vcpu->arch.hflags |= HF_NMI_MASK;
	svm->vmcb->control.intercept |= (1UL << INTERCEPT_IRET);
	++vcpu->stat.nmi_injections;
}
A
Avi Kivity 已提交
2336

2337
static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
A
Avi Kivity 已提交
2338 2339 2340
{
	struct vmcb_control_area *control;

2341
	trace_kvm_inj_virq(irq);
2342

2343
	++svm->vcpu.stat.irq_injections;
R
Rusty Russell 已提交
2344
	control = &svm->vmcb->control;
2345
	control->int_vector = irq;
A
Avi Kivity 已提交
2346 2347 2348 2349 2350
	control->int_ctl &= ~V_INTR_PRIO_MASK;
	control->int_ctl |= V_IRQ_MASK |
		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
}

2351
static void svm_set_irq(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
2352 2353 2354
{
	struct vcpu_svm *svm = to_svm(vcpu);

2355
	BUG_ON(!(svm->vcpu.arch.hflags & HF_GIF_MASK));
2356

2357 2358
	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
		SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
E
Eddie Dong 已提交
2359 2360
}

2361
static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
2362 2363 2364
{
	struct vcpu_svm *svm = to_svm(vcpu);

2365
	if (irr == -1)
2366 2367
		return;

2368 2369 2370
	if (tpr >= irr)
		svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK;
}
2371

2372 2373 2374 2375 2376 2377
static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb *vmcb = svm->vmcb;
	return !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
		!(svm->vcpu.arch.hflags & HF_NMI_MASK);
2378 2379
}

2380 2381 2382 2383 2384 2385
static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb *vmcb = svm->vmcb;
	return (vmcb->save.rflags & X86_EFLAGS_IF) &&
		!(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
2386 2387
		(svm->vcpu.arch.hflags & HF_GIF_MASK) &&
		!is_nested(svm);
2388 2389
}

2390
static void enable_irq_window(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2391
{
2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404
	struct vcpu_svm *svm = to_svm(vcpu);
	nsvm_printk("Trying to open IRQ window\n");

	nested_svm_intr(svm);

	/* In case GIF=0 we can't rely on the CPU to tell us when
	 * GIF becomes 1, because that's a separate STGI/VMRUN intercept.
	 * The next time we get that intercept, this function will be
	 * called again though and we'll get the vintr intercept. */
	if (svm->vcpu.arch.hflags & HF_GIF_MASK) {
		svm_set_vintr(svm);
		svm_inject_irq(svm, 0x0);
	}
2405 2406
}

2407
static void enable_nmi_window(struct kvm_vcpu *vcpu)
2408
{
2409
	struct vcpu_svm *svm = to_svm(vcpu);
2410

2411 2412 2413 2414 2415 2416 2417 2418 2419 2420
	if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
	    == HF_NMI_MASK)
		return; /* IRET will cause a vm exit */

	/* Something prevents NMI from been injected. Single step over
	   possible problem (IRET or exception injection or interrupt
	   shadow) */
	vcpu->arch.singlestep = true;
	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
	update_db_intercept(vcpu);
2421 2422
}

2423 2424 2425 2426 2427
static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
{
	return 0;
}

2428 2429 2430 2431 2432
static void svm_flush_tlb(struct kvm_vcpu *vcpu)
{
	force_new_asid(vcpu);
}

2433 2434 2435 2436
static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
{
}

2437 2438 2439 2440 2441 2442
static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) {
		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
2443
		kvm_set_cr8(vcpu, cr8);
2444 2445 2446
	}
}

2447 2448 2449 2450 2451 2452 2453 2454 2455 2456
static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u64 cr8;

	cr8 = kvm_get_cr8(vcpu);
	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
}

2457 2458 2459 2460 2461 2462
static void svm_complete_interrupts(struct vcpu_svm *svm)
{
	u8 vector;
	int type;
	u32 exitintinfo = svm->vmcb->control.exit_int_info;

2463 2464 2465
	if (svm->vcpu.arch.hflags & HF_IRET_MASK)
		svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);

2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482
	svm->vcpu.arch.nmi_injected = false;
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);

	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
		return;

	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;

	switch (type) {
	case SVM_EXITINTINFO_TYPE_NMI:
		svm->vcpu.arch.nmi_injected = true;
		break;
	case SVM_EXITINTINFO_TYPE_EXEPT:
		/* In case of software exception do not reinject an exception
		   vector, but re-execute and instruction instead */
2483 2484
		if (is_nested(svm))
			break;
2485
		if (kvm_exception_is_soft(vector))
2486 2487 2488 2489 2490 2491 2492 2493 2494
			break;
		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
			u32 err = svm->vmcb->control.exit_int_info_err;
			kvm_queue_exception_e(&svm->vcpu, vector, err);

		} else
			kvm_queue_exception(&svm->vcpu, vector);
		break;
	case SVM_EXITINTINFO_TYPE_INTR:
2495
		kvm_queue_interrupt(&svm->vcpu, vector, false);
2496 2497 2498 2499 2500 2501
		break;
	default:
		break;
	}
}

2502 2503 2504 2505 2506 2507
#ifdef CONFIG_X86_64
#define R "r"
#else
#define R "e"
#endif

2508
static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
A
Avi Kivity 已提交
2509
{
2510
	struct vcpu_svm *svm = to_svm(vcpu);
A
Avi Kivity 已提交
2511 2512 2513
	u16 fs_selector;
	u16 gs_selector;
	u16 ldt_selector;
2514

2515 2516 2517 2518
	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];

R
Rusty Russell 已提交
2519
	pre_svm_run(svm);
A
Avi Kivity 已提交
2520

2521 2522
	sync_lapic_to_cr8(vcpu);

A
Avi Kivity 已提交
2523
	save_host_msrs(vcpu);
2524 2525 2526
	fs_selector = kvm_read_fs();
	gs_selector = kvm_read_gs();
	ldt_selector = kvm_read_ldt();
A
Alexander Graf 已提交
2527 2528
	if (!is_nested(svm))
		svm->vmcb->save.cr2 = vcpu->arch.cr2;
2529 2530 2531
	/* required for live migration with NPT */
	if (npt_enabled)
		svm->vmcb->save.cr3 = vcpu->arch.cr3;
A
Avi Kivity 已提交
2532

2533 2534 2535
	clgi();

	local_irq_enable();
2536

A
Avi Kivity 已提交
2537
	asm volatile (
2538 2539 2540 2541 2542 2543 2544
		"push %%"R"bp; \n\t"
		"mov %c[rbx](%[svm]), %%"R"bx \n\t"
		"mov %c[rcx](%[svm]), %%"R"cx \n\t"
		"mov %c[rdx](%[svm]), %%"R"dx \n\t"
		"mov %c[rsi](%[svm]), %%"R"si \n\t"
		"mov %c[rdi](%[svm]), %%"R"di \n\t"
		"mov %c[rbp](%[svm]), %%"R"bp \n\t"
2545
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
2546 2547 2548 2549 2550 2551 2552 2553
		"mov %c[r8](%[svm]),  %%r8  \n\t"
		"mov %c[r9](%[svm]),  %%r9  \n\t"
		"mov %c[r10](%[svm]), %%r10 \n\t"
		"mov %c[r11](%[svm]), %%r11 \n\t"
		"mov %c[r12](%[svm]), %%r12 \n\t"
		"mov %c[r13](%[svm]), %%r13 \n\t"
		"mov %c[r14](%[svm]), %%r14 \n\t"
		"mov %c[r15](%[svm]), %%r15 \n\t"
A
Avi Kivity 已提交
2554 2555 2556
#endif

		/* Enter guest mode */
2557 2558
		"push %%"R"ax \n\t"
		"mov %c[vmcb](%[svm]), %%"R"ax \n\t"
2559 2560 2561
		__ex(SVM_VMLOAD) "\n\t"
		__ex(SVM_VMRUN) "\n\t"
		__ex(SVM_VMSAVE) "\n\t"
2562
		"pop %%"R"ax \n\t"
A
Avi Kivity 已提交
2563 2564

		/* Save guest registers, load host registers */
2565 2566 2567 2568 2569 2570
		"mov %%"R"bx, %c[rbx](%[svm]) \n\t"
		"mov %%"R"cx, %c[rcx](%[svm]) \n\t"
		"mov %%"R"dx, %c[rdx](%[svm]) \n\t"
		"mov %%"R"si, %c[rsi](%[svm]) \n\t"
		"mov %%"R"di, %c[rdi](%[svm]) \n\t"
		"mov %%"R"bp, %c[rbp](%[svm]) \n\t"
2571
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
2572 2573 2574 2575 2576 2577 2578 2579
		"mov %%r8,  %c[r8](%[svm]) \n\t"
		"mov %%r9,  %c[r9](%[svm]) \n\t"
		"mov %%r10, %c[r10](%[svm]) \n\t"
		"mov %%r11, %c[r11](%[svm]) \n\t"
		"mov %%r12, %c[r12](%[svm]) \n\t"
		"mov %%r13, %c[r13](%[svm]) \n\t"
		"mov %%r14, %c[r14](%[svm]) \n\t"
		"mov %%r15, %c[r15](%[svm]) \n\t"
A
Avi Kivity 已提交
2580
#endif
2581
		"pop %%"R"bp"
A
Avi Kivity 已提交
2582
		:
R
Rusty Russell 已提交
2583
		: [svm]"a"(svm),
A
Avi Kivity 已提交
2584
		  [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
2585 2586 2587 2588 2589 2590
		  [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
		  [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
		  [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
		  [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
		  [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
		  [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
2591
#ifdef CONFIG_X86_64
2592 2593 2594 2595 2596 2597 2598 2599
		  , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
		  [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
		  [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
		  [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
		  [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
		  [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
		  [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
		  [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
A
Avi Kivity 已提交
2600
#endif
2601
		: "cc", "memory"
2602
		, R"bx", R"cx", R"dx", R"si", R"di"
2603 2604 2605 2606
#ifdef CONFIG_X86_64
		, "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
#endif
		);
A
Avi Kivity 已提交
2607

2608
	vcpu->arch.cr2 = svm->vmcb->save.cr2;
2609 2610 2611
	vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
	vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
	vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
A
Avi Kivity 已提交
2612

2613 2614 2615
	kvm_load_fs(fs_selector);
	kvm_load_gs(gs_selector);
	kvm_load_ldt(ldt_selector);
A
Avi Kivity 已提交
2616 2617 2618 2619
	load_host_msrs(vcpu);

	reload_tss(vcpu);

2620 2621 2622 2623
	local_irq_disable();

	stgi();

2624 2625
	sync_cr8_to_lapic(vcpu);

2626
	svm->next_rip = 0;
2627

A
Avi Kivity 已提交
2628 2629 2630 2631 2632
	if (npt_enabled) {
		vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
		vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
	}

2633
	svm_complete_interrupts(svm);
A
Avi Kivity 已提交
2634 2635
}

2636 2637
#undef R

A
Avi Kivity 已提交
2638 2639
static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
{
2640 2641
	struct vcpu_svm *svm = to_svm(vcpu);

2642 2643 2644 2645 2646 2647
	if (npt_enabled) {
		svm->vmcb->control.nested_cr3 = root;
		force_new_asid(vcpu);
		return;
	}

2648
	svm->vmcb->save.cr3 = root;
A
Avi Kivity 已提交
2649
	force_new_asid(vcpu);
A
Anthony Liguori 已提交
2650 2651

	if (vcpu->fpu_active) {
2652 2653
		svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR);
		svm->vmcb->save.cr0 |= X86_CR0_TS;
A
Anthony Liguori 已提交
2654 2655
		vcpu->fpu_active = 0;
	}
A
Avi Kivity 已提交
2656 2657 2658 2659
}

static int is_disabled(void)
{
2660 2661 2662 2663 2664 2665
	u64 vm_cr;

	rdmsrl(MSR_VM_CR, vm_cr);
	if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
		return 1;

A
Avi Kivity 已提交
2666 2667 2668
	return 0;
}

I
Ingo Molnar 已提交
2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679
static void
svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
{
	/*
	 * Patch in the VMMCALL instruction:
	 */
	hypercall[0] = 0x0f;
	hypercall[1] = 0x01;
	hypercall[2] = 0xd9;
}

Y
Yang, Sheng 已提交
2680 2681 2682 2683 2684
static void svm_check_processor_compat(void *rtn)
{
	*(int *)rtn = 0;
}

2685 2686 2687 2688 2689
static bool svm_cpu_has_accelerated_tpr(void)
{
	return false;
}

2690 2691 2692 2693 2694 2695 2696 2697 2698
static int get_npt_level(void)
{
#ifdef CONFIG_X86_64
	return PT64_ROOT_LEVEL;
#else
	return PT32E_ROOT_LEVEL;
#endif
}

2699
static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
S
Sheng Yang 已提交
2700 2701 2702 2703
{
	return 0;
}

2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756
static const struct trace_print_flags svm_exit_reasons_str[] = {
	{ SVM_EXIT_READ_CR0,           		"read_cr0" },
	{ SVM_EXIT_READ_CR3,	      		"read_cr3" },
	{ SVM_EXIT_READ_CR4,	      		"read_cr4" },
	{ SVM_EXIT_READ_CR8,  	      		"read_cr8" },
	{ SVM_EXIT_WRITE_CR0,          		"write_cr0" },
	{ SVM_EXIT_WRITE_CR3,	      		"write_cr3" },
	{ SVM_EXIT_WRITE_CR4,          		"write_cr4" },
	{ SVM_EXIT_WRITE_CR8, 	      		"write_cr8" },
	{ SVM_EXIT_READ_DR0, 	      		"read_dr0" },
	{ SVM_EXIT_READ_DR1,	      		"read_dr1" },
	{ SVM_EXIT_READ_DR2,	      		"read_dr2" },
	{ SVM_EXIT_READ_DR3,	      		"read_dr3" },
	{ SVM_EXIT_WRITE_DR0,	      		"write_dr0" },
	{ SVM_EXIT_WRITE_DR1,	      		"write_dr1" },
	{ SVM_EXIT_WRITE_DR2,	      		"write_dr2" },
	{ SVM_EXIT_WRITE_DR3,	      		"write_dr3" },
	{ SVM_EXIT_WRITE_DR5,	      		"write_dr5" },
	{ SVM_EXIT_WRITE_DR7,	      		"write_dr7" },
	{ SVM_EXIT_EXCP_BASE + DB_VECTOR,	"DB excp" },
	{ SVM_EXIT_EXCP_BASE + BP_VECTOR,	"BP excp" },
	{ SVM_EXIT_EXCP_BASE + UD_VECTOR,	"UD excp" },
	{ SVM_EXIT_EXCP_BASE + PF_VECTOR,	"PF excp" },
	{ SVM_EXIT_EXCP_BASE + NM_VECTOR,	"NM excp" },
	{ SVM_EXIT_EXCP_BASE + MC_VECTOR,	"MC excp" },
	{ SVM_EXIT_INTR,			"interrupt" },
	{ SVM_EXIT_NMI,				"nmi" },
	{ SVM_EXIT_SMI,				"smi" },
	{ SVM_EXIT_INIT,			"init" },
	{ SVM_EXIT_VINTR,			"vintr" },
	{ SVM_EXIT_CPUID,			"cpuid" },
	{ SVM_EXIT_INVD,			"invd" },
	{ SVM_EXIT_HLT,				"hlt" },
	{ SVM_EXIT_INVLPG,			"invlpg" },
	{ SVM_EXIT_INVLPGA,			"invlpga" },
	{ SVM_EXIT_IOIO,			"io" },
	{ SVM_EXIT_MSR,				"msr" },
	{ SVM_EXIT_TASK_SWITCH,			"task_switch" },
	{ SVM_EXIT_SHUTDOWN,			"shutdown" },
	{ SVM_EXIT_VMRUN,			"vmrun" },
	{ SVM_EXIT_VMMCALL,			"hypercall" },
	{ SVM_EXIT_VMLOAD,			"vmload" },
	{ SVM_EXIT_VMSAVE,			"vmsave" },
	{ SVM_EXIT_STGI,			"stgi" },
	{ SVM_EXIT_CLGI,			"clgi" },
	{ SVM_EXIT_SKINIT,			"skinit" },
	{ SVM_EXIT_WBINVD,			"wbinvd" },
	{ SVM_EXIT_MONITOR,			"monitor" },
	{ SVM_EXIT_MWAIT,			"mwait" },
	{ SVM_EXIT_NPF,				"npf" },
	{ -1, NULL }
};

2757 2758 2759 2760 2761
static bool svm_gb_page_enable(void)
{
	return true;
}

2762
static struct kvm_x86_ops svm_x86_ops = {
A
Avi Kivity 已提交
2763 2764 2765 2766
	.cpu_has_kvm_support = has_svm,
	.disabled_by_bios = is_disabled,
	.hardware_setup = svm_hardware_setup,
	.hardware_unsetup = svm_hardware_unsetup,
Y
Yang, Sheng 已提交
2767
	.check_processor_compatibility = svm_check_processor_compat,
A
Avi Kivity 已提交
2768 2769
	.hardware_enable = svm_hardware_enable,
	.hardware_disable = svm_hardware_disable,
2770
	.cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
A
Avi Kivity 已提交
2771 2772 2773

	.vcpu_create = svm_create_vcpu,
	.vcpu_free = svm_free_vcpu,
2774
	.vcpu_reset = svm_vcpu_reset,
A
Avi Kivity 已提交
2775

2776
	.prepare_guest_switch = svm_prepare_guest_switch,
A
Avi Kivity 已提交
2777 2778 2779 2780 2781 2782 2783 2784 2785
	.vcpu_load = svm_vcpu_load,
	.vcpu_put = svm_vcpu_put,

	.set_guest_debug = svm_guest_debug,
	.get_msr = svm_get_msr,
	.set_msr = svm_set_msr,
	.get_segment_base = svm_get_segment_base,
	.get_segment = svm_get_segment,
	.set_segment = svm_set_segment,
2786
	.get_cpl = svm_get_cpl,
2787
	.get_cs_db_l_bits = kvm_get_cs_db_l_bits,
2788
	.decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
A
Avi Kivity 已提交
2789 2790 2791 2792 2793 2794 2795 2796 2797 2798
	.set_cr0 = svm_set_cr0,
	.set_cr3 = svm_set_cr3,
	.set_cr4 = svm_set_cr4,
	.set_efer = svm_set_efer,
	.get_idt = svm_get_idt,
	.set_idt = svm_set_idt,
	.get_gdt = svm_get_gdt,
	.set_gdt = svm_set_gdt,
	.get_dr = svm_get_dr,
	.set_dr = svm_set_dr,
A
Avi Kivity 已提交
2799
	.cache_reg = svm_cache_reg,
A
Avi Kivity 已提交
2800 2801 2802 2803 2804 2805
	.get_rflags = svm_get_rflags,
	.set_rflags = svm_set_rflags,

	.tlb_flush = svm_flush_tlb,

	.run = svm_vcpu_run,
2806
	.handle_exit = handle_exit,
A
Avi Kivity 已提交
2807
	.skip_emulated_instruction = skip_emulated_instruction,
2808 2809
	.set_interrupt_shadow = svm_set_interrupt_shadow,
	.get_interrupt_shadow = svm_get_interrupt_shadow,
I
Ingo Molnar 已提交
2810
	.patch_hypercall = svm_patch_hypercall,
E
Eddie Dong 已提交
2811
	.set_irq = svm_set_irq,
2812
	.set_nmi = svm_inject_nmi,
2813
	.queue_exception = svm_queue_exception,
2814
	.interrupt_allowed = svm_interrupt_allowed,
2815 2816 2817 2818
	.nmi_allowed = svm_nmi_allowed,
	.enable_nmi_window = enable_nmi_window,
	.enable_irq_window = enable_irq_window,
	.update_cr8_intercept = update_cr8_intercept,
2819 2820

	.set_tss_addr = svm_set_tss_addr,
2821
	.get_tdp_level = get_npt_level,
2822
	.get_mt_mask = svm_get_mt_mask,
2823 2824

	.exit_reasons_str = svm_exit_reasons_str,
2825
	.gb_page_enable = svm_gb_page_enable,
A
Avi Kivity 已提交
2826 2827 2828 2829
};

static int __init svm_init(void)
{
2830
	return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
2831
			      THIS_MODULE);
A
Avi Kivity 已提交
2832 2833 2834 2835
}

static void __exit svm_exit(void)
{
2836
	kvm_exit();
A
Avi Kivity 已提交
2837 2838 2839 2840
}

module_init(svm_init)
module_exit(svm_exit)