svm.c 90.7 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * AMD SVM support
 *
 * Copyright (C) 2006 Qumranet, Inc.
A
Avi Kivity 已提交
7
 * Copyright 2010 Red Hat, Inc. and/or its affilates.
A
Avi Kivity 已提交
8 9 10 11 12 13 14 15 16
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
17 18
#include <linux/kvm_host.h>

19
#include "irq.h"
20
#include "mmu.h"
21
#include "kvm_cache_regs.h"
22
#include "x86.h"
A
Avi Kivity 已提交
23

A
Avi Kivity 已提交
24
#include <linux/module.h>
25
#include <linux/kernel.h>
A
Avi Kivity 已提交
26 27
#include <linux/vmalloc.h>
#include <linux/highmem.h>
A
Alexey Dobriyan 已提交
28
#include <linux/sched.h>
29
#include <linux/ftrace_event.h>
30
#include <linux/slab.h>
A
Avi Kivity 已提交
31

32
#include <asm/tlbflush.h>
A
Avi Kivity 已提交
33
#include <asm/desc.h>
A
Avi Kivity 已提交
34

35
#include <asm/virtext.h>
36
#include "trace.h"
37

38 39
#define __ex(x) __kvm_handle_fault_on_reboot(x)

A
Avi Kivity 已提交
40 41 42 43 44 45 46 47 48
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

#define IOPM_ALLOC_ORDER 2
#define MSRPM_ALLOC_ORDER 1

#define SEG_TYPE_LDT 2
#define SEG_TYPE_BUSY_TSS16 3

49 50 51 52 53
#define SVM_FEATURE_NPT            (1 <<  0)
#define SVM_FEATURE_LBRV           (1 <<  1)
#define SVM_FEATURE_SVML           (1 <<  2)
#define SVM_FEATURE_NRIP           (1 <<  3)
#define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
54

55 56 57 58
#define NESTED_EXIT_HOST	0	/* Exit handled on host level */
#define NESTED_EXIT_DONE	1	/* Exit caused nested vmexit  */
#define NESTED_EXIT_CONTINUE	2	/* Further checks needed      */

59 60
#define DEBUGCTL_RESERVED_BITS (~(0x3fULL))

61 62
static bool erratum_383_found __read_mostly;

A
Avi Kivity 已提交
63 64 65 66 67 68 69 70 71 72 73 74
static const u32 host_save_user_msrs[] = {
#ifdef CONFIG_X86_64
	MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
	MSR_FS_BASE,
#endif
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
};

#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)

struct kvm_vcpu;

75 76 77
struct nested_state {
	struct vmcb *hsave;
	u64 hsave_msr;
78
	u64 vm_cr_msr;
79 80 81 82 83 84 85
	u64 vmcb;

	/* These are the merged vectors */
	u32 *msrpm;

	/* gpa pointers to the real vectors */
	u64 vmcb_msrpm;
86
	u64 vmcb_iopm;
J
Joerg Roedel 已提交
87

88 89 90
	/* A VMEXIT is required but not yet emulated */
	bool exit_required;

J
Joerg Roedel 已提交
91 92 93 94 95 96 97 98
	/* cache for intercepts of the guest */
	u16 intercept_cr_read;
	u16 intercept_cr_write;
	u16 intercept_dr_read;
	u16 intercept_dr_write;
	u32 intercept_exceptions;
	u64 intercept;

99 100
};

101 102 103
#define MSRPM_OFFSETS	16
static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;

A
Avi Kivity 已提交
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
struct vcpu_svm {
	struct kvm_vcpu vcpu;
	struct vmcb *vmcb;
	unsigned long vmcb_pa;
	struct svm_cpu_data *svm_data;
	uint64_t asid_generation;
	uint64_t sysenter_esp;
	uint64_t sysenter_eip;

	u64 next_rip;

	u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
	u64 host_gs_base;

	u32 *msrpm;

120
	struct nested_state nested;
J
Jan Kiszka 已提交
121 122

	bool nmi_singlestep;
123 124 125

	unsigned int3_injected;
	unsigned long int3_rip;
A
Avi Kivity 已提交
126 127
};

128 129
#define MSR_INVALID			0xffffffffU

130 131 132 133
static struct svm_direct_access_msrs {
	u32 index;   /* Index of the MSR */
	bool always; /* True if intercept is always on */
} direct_access_msrs[] = {
B
Brian Gerst 已提交
134
	{ .index = MSR_STAR,				.always = true  },
135 136 137 138 139 140 141 142 143 144 145 146 147 148
	{ .index = MSR_IA32_SYSENTER_CS,		.always = true  },
#ifdef CONFIG_X86_64
	{ .index = MSR_GS_BASE,				.always = true  },
	{ .index = MSR_FS_BASE,				.always = true  },
	{ .index = MSR_KERNEL_GS_BASE,			.always = true  },
	{ .index = MSR_LSTAR,				.always = true  },
	{ .index = MSR_CSTAR,				.always = true  },
	{ .index = MSR_SYSCALL_MASK,			.always = true  },
#endif
	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
	{ .index = MSR_IA32_LASTINTTOIP,		.always = false },
	{ .index = MSR_INVALID,				.always = false },
A
Avi Kivity 已提交
149 150
};

151 152 153 154
/* enable NPT for AMD64 and X86 with PAE */
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
static bool npt_enabled = true;
#else
J
Joerg Roedel 已提交
155
static bool npt_enabled;
156
#endif
157 158 159
static int npt = 1;

module_param(npt, int, S_IRUGO);
160

161
static int nested = 1;
162 163
module_param(nested, int, S_IRUGO);

164
static void svm_flush_tlb(struct kvm_vcpu *vcpu);
165
static void svm_complete_interrupts(struct vcpu_svm *svm);
166

167
static int nested_svm_exit_handled(struct vcpu_svm *svm);
168
static int nested_svm_intercept(struct vcpu_svm *svm);
169 170 171 172
static int nested_svm_vmexit(struct vcpu_svm *svm);
static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
				      bool has_error_code, u32 error_code);

173 174
static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
175
	return container_of(vcpu, struct vcpu_svm, vcpu);
176 177
}

A
Alexander Graf 已提交
178 179
static inline bool is_nested(struct vcpu_svm *svm)
{
180
	return svm->nested.vmcb;
A
Alexander Graf 已提交
181 182
}

183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
static inline void enable_gif(struct vcpu_svm *svm)
{
	svm->vcpu.arch.hflags |= HF_GIF_MASK;
}

static inline void disable_gif(struct vcpu_svm *svm)
{
	svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
}

static inline bool gif_set(struct vcpu_svm *svm)
{
	return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
}

198
static unsigned long iopm_base;
A
Avi Kivity 已提交
199 200 201 202

struct kvm_ldttss_desc {
	u16 limit0;
	u16 base0;
J
Joerg Roedel 已提交
203 204
	unsigned base1:8, type:5, dpl:2, p:1;
	unsigned limit1:4, zero0:3, g:1, base2:8;
A
Avi Kivity 已提交
205 206 207 208 209 210 211
	u32 base3;
	u32 zero1;
} __attribute__((packed));

struct svm_cpu_data {
	int cpu;

A
Avi Kivity 已提交
212 213 214
	u64 asid_generation;
	u32 max_asid;
	u32 next_asid;
A
Avi Kivity 已提交
215 216 217 218 219 220
	struct kvm_ldttss_desc *tss_desc;

	struct page *save_area;
};

static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
221
static uint32_t svm_features;
A
Avi Kivity 已提交
222 223 224 225 226 227 228 229

struct svm_init_data {
	int cpu;
	int r;
};

static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};

230
#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
A
Avi Kivity 已提交
231 232 233
#define MSRS_RANGE_SIZE 2048
#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)

234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
static u32 svm_msrpm_offset(u32 msr)
{
	u32 offset;
	int i;

	for (i = 0; i < NUM_MSR_MAPS; i++) {
		if (msr < msrpm_ranges[i] ||
		    msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
			continue;

		offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
		offset += (i * MSRS_RANGE_SIZE);       /* add range offset */

		/* Now we have the u8 offset - but need the u32 offset */
		return offset / 4;
	}

	/* MSR not in any range */
	return MSR_INVALID;
}

A
Avi Kivity 已提交
255 256
#define MAX_INST_SIZE 15

257 258 259 260 261
static inline u32 svm_has(u32 feat)
{
	return svm_features & feat;
}

A
Avi Kivity 已提交
262 263
static inline void clgi(void)
{
264
	asm volatile (__ex(SVM_CLGI));
A
Avi Kivity 已提交
265 266 267 268
}

static inline void stgi(void)
{
269
	asm volatile (__ex(SVM_STGI));
A
Avi Kivity 已提交
270 271 272 273
}

static inline void invlpga(unsigned long addr, u32 asid)
{
J
Joerg Roedel 已提交
274
	asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
A
Avi Kivity 已提交
275 276 277 278
}

static inline void force_new_asid(struct kvm_vcpu *vcpu)
{
279
	to_svm(vcpu)->asid_generation--;
A
Avi Kivity 已提交
280 281 282 283 284 285 286 287 288
}

static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
{
	force_new_asid(vcpu);
}

static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
289
	vcpu->arch.efer = efer;
290
	if (!npt_enabled && !(efer & EFER_LMA))
291
		efer &= ~EFER_LME;
A
Avi Kivity 已提交
292

293
	to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
A
Avi Kivity 已提交
294 295 296 297 298 299 300 301
}

static int is_external_interrupt(u32 info)
{
	info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
	return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
}

302 303 304 305 306 307
static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u32 ret = 0;

	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
308
		ret |= KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
309 310 311 312 313 314 315 316 317 318 319 320 321 322
	return ret & mask;
}

static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (mask == 0)
		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
	else
		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;

}

A
Avi Kivity 已提交
323 324
static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
325 326
	struct vcpu_svm *svm = to_svm(vcpu);

327 328 329
	if (svm->vmcb->control.next_rip != 0)
		svm->next_rip = svm->vmcb->control.next_rip;

330
	if (!svm->next_rip) {
A
Avi Kivity 已提交
331
		if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) !=
332 333
				EMULATE_DONE)
			printk(KERN_DEBUG "%s: NOP\n", __func__);
A
Avi Kivity 已提交
334 335
		return;
	}
336 337 338
	if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
		printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
		       __func__, kvm_rip_read(vcpu), svm->next_rip);
A
Avi Kivity 已提交
339

340
	kvm_rip_write(vcpu, svm->next_rip);
341
	svm_set_interrupt_shadow(vcpu, 0);
A
Avi Kivity 已提交
342 343
}

J
Jan Kiszka 已提交
344
static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
345 346
				bool has_error_code, u32 error_code,
				bool reinject)
J
Jan Kiszka 已提交
347 348 349
{
	struct vcpu_svm *svm = to_svm(vcpu);

J
Joerg Roedel 已提交
350 351 352 353
	/*
	 * If we are within a nested VM we'd better #VMEXIT and let the guest
	 * handle the exception
	 */
354 355
	if (!reinject &&
	    nested_svm_check_exception(svm, nr, has_error_code, error_code))
J
Jan Kiszka 已提交
356 357
		return;

358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
	if (nr == BP_VECTOR && !svm_has(SVM_FEATURE_NRIP)) {
		unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);

		/*
		 * For guest debugging where we have to reinject #BP if some
		 * INT3 is guest-owned:
		 * Emulate nRIP by moving RIP forward. Will fail if injection
		 * raises a fault that is not intercepted. Still better than
		 * failing in all cases.
		 */
		skip_emulated_instruction(&svm->vcpu);
		rip = kvm_rip_read(&svm->vcpu);
		svm->int3_rip = rip + svm->vmcb->save.cs.base;
		svm->int3_injected = rip - old_rip;
	}

J
Jan Kiszka 已提交
374 375 376 377 378 379 380
	svm->vmcb->control.event_inj = nr
		| SVM_EVTINJ_VALID
		| (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
		| SVM_EVTINJ_TYPE_EXEPT;
	svm->vmcb->control.event_inj_err = error_code;
}

381 382 383 384 385 386
static void svm_init_erratum_383(void)
{
	u32 low, high;
	int err;
	u64 val;

387
	if (!cpu_has_amd_erratum(amd_erratum_383))
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404
		return;

	/* Use _safe variants to not break nested virtualization */
	val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
	if (err)
		return;

	val |= (1ULL << 47);

	low  = lower_32_bits(val);
	high = upper_32_bits(val);

	native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);

	erratum_383_found = true;
}

A
Avi Kivity 已提交
405 406
static int has_svm(void)
{
407
	const char *msg;
A
Avi Kivity 已提交
408

409
	if (!cpu_has_svm(&msg)) {
J
Joe Perches 已提交
410
		printk(KERN_INFO "has_svm: %s\n", msg);
A
Avi Kivity 已提交
411 412 413 414 415 416 417 418
		return 0;
	}

	return 1;
}

static void svm_hardware_disable(void *garbage)
{
419
	cpu_svm_disable();
A
Avi Kivity 已提交
420 421
}

422
static int svm_hardware_enable(void *garbage)
A
Avi Kivity 已提交
423 424
{

425
	struct svm_cpu_data *sd;
A
Avi Kivity 已提交
426
	uint64_t efer;
427
	struct desc_ptr gdt_descr;
A
Avi Kivity 已提交
428 429 430
	struct desc_struct *gdt;
	int me = raw_smp_processor_id();

431 432 433 434
	rdmsrl(MSR_EFER, efer);
	if (efer & EFER_SVME)
		return -EBUSY;

A
Avi Kivity 已提交
435
	if (!has_svm()) {
436 437
		printk(KERN_ERR "svm_hardware_enable: err EOPNOTSUPP on %d\n",
		       me);
438
		return -EINVAL;
A
Avi Kivity 已提交
439
	}
440
	sd = per_cpu(svm_data, me);
A
Avi Kivity 已提交
441

442
	if (!sd) {
443
		printk(KERN_ERR "svm_hardware_enable: svm_data is NULL on %d\n",
A
Avi Kivity 已提交
444
		       me);
445
		return -EINVAL;
A
Avi Kivity 已提交
446 447
	}

448 449 450
	sd->asid_generation = 1;
	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
	sd->next_asid = sd->max_asid + 1;
A
Avi Kivity 已提交
451

452
	native_store_gdt(&gdt_descr);
453
	gdt = (struct desc_struct *)gdt_descr.address;
454
	sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
A
Avi Kivity 已提交
455

456
	wrmsrl(MSR_EFER, efer | EFER_SVME);
A
Avi Kivity 已提交
457

458
	wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
459

460 461
	svm_init_erratum_383();

462
	return 0;
A
Avi Kivity 已提交
463 464
}

465 466
static void svm_cpu_uninit(int cpu)
{
467
	struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
468

469
	if (!sd)
470 471 472
		return;

	per_cpu(svm_data, raw_smp_processor_id()) = NULL;
473 474
	__free_page(sd->save_area);
	kfree(sd);
475 476
}

A
Avi Kivity 已提交
477 478
static int svm_cpu_init(int cpu)
{
479
	struct svm_cpu_data *sd;
A
Avi Kivity 已提交
480 481
	int r;

482 483
	sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
	if (!sd)
A
Avi Kivity 已提交
484
		return -ENOMEM;
485 486
	sd->cpu = cpu;
	sd->save_area = alloc_page(GFP_KERNEL);
A
Avi Kivity 已提交
487
	r = -ENOMEM;
488
	if (!sd->save_area)
A
Avi Kivity 已提交
489 490
		goto err_1;

491
	per_cpu(svm_data, cpu) = sd;
A
Avi Kivity 已提交
492 493 494 495

	return 0;

err_1:
496
	kfree(sd);
A
Avi Kivity 已提交
497 498 499 500
	return r;

}

501 502 503 504 505 506 507 508 509 510 511
static bool valid_msr_intercept(u32 index)
{
	int i;

	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
		if (direct_access_msrs[i].index == index)
			return true;

	return false;
}

512 513
static void set_msr_interception(u32 *msrpm, unsigned msr,
				 int read, int write)
A
Avi Kivity 已提交
514
{
515 516 517
	u8 bit_read, bit_write;
	unsigned long tmp;
	u32 offset;
A
Avi Kivity 已提交
518

519 520 521 522 523 524
	/*
	 * If this warning triggers extend the direct_access_msrs list at the
	 * beginning of the file
	 */
	WARN_ON(!valid_msr_intercept(msr));

525 526 527 528 529 530 531 532 533 534 535
	offset    = svm_msrpm_offset(msr);
	bit_read  = 2 * (msr & 0x0f);
	bit_write = 2 * (msr & 0x0f) + 1;
	tmp       = msrpm[offset];

	BUG_ON(offset == MSR_INVALID);

	read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
	write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);

	msrpm[offset] = tmp;
A
Avi Kivity 已提交
536 537
}

538
static void svm_vcpu_init_msrpm(u32 *msrpm)
A
Avi Kivity 已提交
539 540 541
{
	int i;

542 543
	memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));

544 545 546 547 548 549
	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
		if (!direct_access_msrs[i].always)
			continue;

		set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
	}
550 551
}

552 553 554 555 556 557 558 559
static void add_msr_offset(u32 offset)
{
	int i;

	for (i = 0; i < MSRPM_OFFSETS; ++i) {

		/* Offset already in list? */
		if (msrpm_offsets[i] == offset)
560
			return;
561 562 563 564 565 566 567 568 569

		/* Slot used by another offset? */
		if (msrpm_offsets[i] != MSR_INVALID)
			continue;

		/* Add offset to list */
		msrpm_offsets[i] = offset;

		return;
A
Avi Kivity 已提交
570
	}
571 572 573 574 575

	/*
	 * If this BUG triggers the msrpm_offsets table has an overflow. Just
	 * increase MSRPM_OFFSETS in this case.
	 */
576
	BUG();
A
Avi Kivity 已提交
577 578
}

579
static void init_msrpm_offsets(void)
580
{
581
	int i;
582

583 584 585 586 587 588 589 590 591 592
	memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));

	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
		u32 offset;

		offset = svm_msrpm_offset(direct_access_msrs[i].index);
		BUG_ON(offset == MSR_INVALID);

		add_msr_offset(offset);
	}
593 594
}

595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616
static void svm_enable_lbrv(struct vcpu_svm *svm)
{
	u32 *msrpm = svm->msrpm;

	svm->vmcb->control.lbr_ctl = 1;
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
}

static void svm_disable_lbrv(struct vcpu_svm *svm)
{
	u32 *msrpm = svm->msrpm;

	svm->vmcb->control.lbr_ctl = 0;
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
}

A
Avi Kivity 已提交
617 618 619 620
static __init int svm_hardware_setup(void)
{
	int cpu;
	struct page *iopm_pages;
621
	void *iopm_va;
A
Avi Kivity 已提交
622 623 624 625 626 627
	int r;

	iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);

	if (!iopm_pages)
		return -ENOMEM;
628 629 630

	iopm_va = page_address(iopm_pages);
	memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
A
Avi Kivity 已提交
631 632
	iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;

633 634
	init_msrpm_offsets();

635 636 637
	if (boot_cpu_has(X86_FEATURE_NX))
		kvm_enable_efer_bits(EFER_NX);

A
Alexander Graf 已提交
638 639 640
	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
		kvm_enable_efer_bits(EFER_FFXSR);

641 642
	if (nested) {
		printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
643
		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
644 645
	}

Z
Zachary Amsden 已提交
646
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
647 648
		r = svm_cpu_init(cpu);
		if (r)
649
			goto err;
A
Avi Kivity 已提交
650
	}
651 652 653

	svm_features = cpuid_edx(SVM_CPUID_FUNC);

654 655 656
	if (!svm_has(SVM_FEATURE_NPT))
		npt_enabled = false;

657 658 659 660 661
	if (npt_enabled && !npt) {
		printk(KERN_INFO "kvm: Nested Paging disabled\n");
		npt_enabled = false;
	}

662
	if (npt_enabled) {
663
		printk(KERN_INFO "kvm: Nested Paging enabled\n");
664
		kvm_enable_tdp();
665 666
	} else
		kvm_disable_tdp();
667

A
Avi Kivity 已提交
668 669
	return 0;

670
err:
A
Avi Kivity 已提交
671 672 673 674 675 676 677
	__free_pages(iopm_pages, IOPM_ALLOC_ORDER);
	iopm_base = 0;
	return r;
}

static __exit void svm_hardware_unsetup(void)
{
678 679
	int cpu;

Z
Zachary Amsden 已提交
680
	for_each_possible_cpu(cpu)
681 682
		svm_cpu_uninit(cpu);

A
Avi Kivity 已提交
683
	__free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
684
	iopm_base = 0;
A
Avi Kivity 已提交
685 686 687 688 689 690
}

static void init_seg(struct vmcb_seg *seg)
{
	seg->selector = 0;
	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
J
Joerg Roedel 已提交
691
		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
A
Avi Kivity 已提交
692 693 694 695 696 697 698 699 700 701 702 703
	seg->limit = 0xffff;
	seg->base = 0;
}

static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
{
	seg->selector = 0;
	seg->attrib = SVM_SELECTOR_P_MASK | type;
	seg->limit = 0xffff;
	seg->base = 0;
}

704
static void init_vmcb(struct vcpu_svm *svm)
A
Avi Kivity 已提交
705
{
706 707
	struct vmcb_control_area *control = &svm->vmcb->control;
	struct vmcb_save_area *save = &svm->vmcb->save;
A
Avi Kivity 已提交
708

709 710
	svm->vcpu.fpu_active = 1;

J
Joerg Roedel 已提交
711
	control->intercept_cr_read =	INTERCEPT_CR0_MASK |
A
Avi Kivity 已提交
712
					INTERCEPT_CR3_MASK |
713
					INTERCEPT_CR4_MASK;
A
Avi Kivity 已提交
714

J
Joerg Roedel 已提交
715
	control->intercept_cr_write =	INTERCEPT_CR0_MASK |
A
Avi Kivity 已提交
716
					INTERCEPT_CR3_MASK |
717 718
					INTERCEPT_CR4_MASK |
					INTERCEPT_CR8_MASK;
A
Avi Kivity 已提交
719

J
Joerg Roedel 已提交
720
	control->intercept_dr_read =	INTERCEPT_DR0_MASK |
A
Avi Kivity 已提交
721 722
					INTERCEPT_DR1_MASK |
					INTERCEPT_DR2_MASK |
723 724 725 726 727
					INTERCEPT_DR3_MASK |
					INTERCEPT_DR4_MASK |
					INTERCEPT_DR5_MASK |
					INTERCEPT_DR6_MASK |
					INTERCEPT_DR7_MASK;
A
Avi Kivity 已提交
728

J
Joerg Roedel 已提交
729
	control->intercept_dr_write =	INTERCEPT_DR0_MASK |
A
Avi Kivity 已提交
730 731 732
					INTERCEPT_DR1_MASK |
					INTERCEPT_DR2_MASK |
					INTERCEPT_DR3_MASK |
733
					INTERCEPT_DR4_MASK |
A
Avi Kivity 已提交
734
					INTERCEPT_DR5_MASK |
735
					INTERCEPT_DR6_MASK |
A
Avi Kivity 已提交
736 737
					INTERCEPT_DR7_MASK;

738
	control->intercept_exceptions = (1 << PF_VECTOR) |
739 740
					(1 << UD_VECTOR) |
					(1 << MC_VECTOR);
A
Avi Kivity 已提交
741 742


J
Joerg Roedel 已提交
743
	control->intercept =	(1ULL << INTERCEPT_INTR) |
A
Avi Kivity 已提交
744
				(1ULL << INTERCEPT_NMI) |
745
				(1ULL << INTERCEPT_SMI) |
A
Avi Kivity 已提交
746
				(1ULL << INTERCEPT_SELECTIVE_CR0) |
A
Avi Kivity 已提交
747
				(1ULL << INTERCEPT_CPUID) |
748
				(1ULL << INTERCEPT_INVD) |
A
Avi Kivity 已提交
749
				(1ULL << INTERCEPT_HLT) |
M
Marcelo Tosatti 已提交
750
				(1ULL << INTERCEPT_INVLPG) |
A
Avi Kivity 已提交
751 752 753 754
				(1ULL << INTERCEPT_INVLPGA) |
				(1ULL << INTERCEPT_IOIO_PROT) |
				(1ULL << INTERCEPT_MSR_PROT) |
				(1ULL << INTERCEPT_TASK_SWITCH) |
755
				(1ULL << INTERCEPT_SHUTDOWN) |
A
Avi Kivity 已提交
756 757 758 759 760 761
				(1ULL << INTERCEPT_VMRUN) |
				(1ULL << INTERCEPT_VMMCALL) |
				(1ULL << INTERCEPT_VMLOAD) |
				(1ULL << INTERCEPT_VMSAVE) |
				(1ULL << INTERCEPT_STGI) |
				(1ULL << INTERCEPT_CLGI) |
762
				(1ULL << INTERCEPT_SKINIT) |
763
				(1ULL << INTERCEPT_WBINVD) |
764 765
				(1ULL << INTERCEPT_MONITOR) |
				(1ULL << INTERCEPT_MWAIT);
A
Avi Kivity 已提交
766 767

	control->iopm_base_pa = iopm_base;
768
	control->msrpm_base_pa = __pa(svm->msrpm);
Z
Zachary Amsden 已提交
769
	control->tsc_offset = 0-native_read_tsc();
A
Avi Kivity 已提交
770 771 772 773 774 775 776 777 778 779 780 781 782
	control->int_ctl = V_INTR_MASKING_MASK;

	init_seg(&save->es);
	init_seg(&save->ss);
	init_seg(&save->ds);
	init_seg(&save->fs);
	init_seg(&save->gs);

	save->cs.selector = 0xf000;
	/* Executable/Readable Code Segment */
	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
	save->cs.limit = 0xffff;
783 784 785 786 787 788 789
	/*
	 * cs.base should really be 0xffff0000, but vmx can't handle that, so
	 * be consistent with it.
	 *
	 * Replace when we have real mode working for vmx.
	 */
	save->cs.base = 0xf0000;
A
Avi Kivity 已提交
790 791 792 793 794 795 796

	save->gdtr.limit = 0xffff;
	save->idtr.limit = 0xffff;

	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);

797
	save->efer = EFER_SVME;
M
Mike Day 已提交
798
	save->dr6 = 0xffff0ff0;
A
Avi Kivity 已提交
799 800 801
	save->dr7 = 0x400;
	save->rflags = 2;
	save->rip = 0x0000fff0;
802
	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
A
Avi Kivity 已提交
803

J
Joerg Roedel 已提交
804 805
	/*
	 * This is the guest-visible cr0 value.
806
	 * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
A
Avi Kivity 已提交
807
	 */
808
	svm->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
809
	(void)kvm_set_cr0(&svm->vcpu, svm->vcpu.arch.cr0);
810

811
	save->cr4 = X86_CR4_PAE;
A
Avi Kivity 已提交
812
	/* rdx = ?? */
813 814 815 816

	if (npt_enabled) {
		/* Setup VMCB for Nested Paging */
		control->nested_ctl = 1;
M
Marcelo Tosatti 已提交
817 818
		control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) |
					(1ULL << INTERCEPT_INVLPG));
819
		control->intercept_exceptions &= ~(1 << PF_VECTOR);
820 821
		control->intercept_cr_read &= ~INTERCEPT_CR3_MASK;
		control->intercept_cr_write &= ~INTERCEPT_CR3_MASK;
822 823 824 825
		save->g_pat = 0x0007040600070406ULL;
		save->cr3 = 0;
		save->cr4 = 0;
	}
826
	force_new_asid(&svm->vcpu);
827

828
	svm->nested.vmcb = 0;
829 830
	svm->vcpu.arch.hflags = 0;

831 832 833 834 835
	if (svm_has(SVM_FEATURE_PAUSE_FILTER)) {
		control->pause_filter_count = 3000;
		control->intercept |= (1ULL << INTERCEPT_PAUSE);
	}

836
	enable_gif(svm);
A
Avi Kivity 已提交
837 838
}

839
static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
840 841 842
{
	struct vcpu_svm *svm = to_svm(vcpu);

843
	init_vmcb(svm);
A
Avi Kivity 已提交
844

845
	if (!kvm_vcpu_is_bsp(vcpu)) {
846
		kvm_rip_write(vcpu, 0);
847 848
		svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;
		svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;
A
Avi Kivity 已提交
849
	}
850 851
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;
852 853

	return 0;
854 855
}

R
Rusty Russell 已提交
856
static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
A
Avi Kivity 已提交
857
{
858
	struct vcpu_svm *svm;
A
Avi Kivity 已提交
859
	struct page *page;
860
	struct page *msrpm_pages;
A
Alexander Graf 已提交
861
	struct page *hsave_page;
A
Alexander Graf 已提交
862
	struct page *nested_msrpm_pages;
R
Rusty Russell 已提交
863
	int err;
A
Avi Kivity 已提交
864

865
	svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
R
Rusty Russell 已提交
866 867 868 869 870 871 872 873 874
	if (!svm) {
		err = -ENOMEM;
		goto out;
	}

	err = kvm_vcpu_init(&svm->vcpu, kvm, id);
	if (err)
		goto free_svm;

875
	err = -ENOMEM;
A
Avi Kivity 已提交
876
	page = alloc_page(GFP_KERNEL);
877
	if (!page)
R
Rusty Russell 已提交
878
		goto uninit;
A
Avi Kivity 已提交
879

880 881
	msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
	if (!msrpm_pages)
882
		goto free_page1;
A
Alexander Graf 已提交
883 884 885

	nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
	if (!nested_msrpm_pages)
886
		goto free_page2;
887

A
Alexander Graf 已提交
888 889
	hsave_page = alloc_page(GFP_KERNEL);
	if (!hsave_page)
890 891
		goto free_page3;

892
	svm->nested.hsave = page_address(hsave_page);
A
Alexander Graf 已提交
893

894 895 896
	svm->msrpm = page_address(msrpm_pages);
	svm_vcpu_init_msrpm(svm->msrpm);

897
	svm->nested.msrpm = page_address(nested_msrpm_pages);
898
	svm_vcpu_init_msrpm(svm->nested.msrpm);
A
Alexander Graf 已提交
899

900 901 902 903
	svm->vmcb = page_address(page);
	clear_page(svm->vmcb);
	svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
	svm->asid_generation = 0;
904
	init_vmcb(svm);
905

906 907 908 909
	err = fx_init(&svm->vcpu);
	if (err)
		goto free_page4;

910
	svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
911
	if (kvm_vcpu_is_bsp(&svm->vcpu))
912
		svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
A
Avi Kivity 已提交
913

R
Rusty Russell 已提交
914
	return &svm->vcpu;
915

916 917
free_page4:
	__free_page(hsave_page);
918 919 920 921 922 923
free_page3:
	__free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
free_page2:
	__free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
free_page1:
	__free_page(page);
R
Rusty Russell 已提交
924 925 926
uninit:
	kvm_vcpu_uninit(&svm->vcpu);
free_svm:
927
	kmem_cache_free(kvm_vcpu_cache, svm);
R
Rusty Russell 已提交
928 929
out:
	return ERR_PTR(err);
A
Avi Kivity 已提交
930 931 932 933
}

static void svm_free_vcpu(struct kvm_vcpu *vcpu)
{
934 935
	struct vcpu_svm *svm = to_svm(vcpu);

R
Rusty Russell 已提交
936
	__free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
937
	__free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
938 939
	__free_page(virt_to_page(svm->nested.hsave));
	__free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
R
Rusty Russell 已提交
940
	kvm_vcpu_uninit(vcpu);
941
	kmem_cache_free(kvm_vcpu_cache, svm);
A
Avi Kivity 已提交
942 943
}

944
static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
A
Avi Kivity 已提交
945
{
946
	struct vcpu_svm *svm = to_svm(vcpu);
947
	int i;
948 949

	if (unlikely(cpu != vcpu->cpu)) {
950
		u64 delta;
951

952 953 954 955 956 957 958 959 960 961
		if (check_tsc_unstable()) {
			/*
			 * Make sure that the guest sees a monotonically
			 * increasing TSC.
			 */
			delta = vcpu->arch.host_tsc - native_read_tsc();
			svm->vmcb->control.tsc_offset += delta;
			if (is_nested(svm))
				svm->nested.hsave->control.tsc_offset += delta;
		}
962
		vcpu->cpu = cpu;
M
Marcelo Tosatti 已提交
963
		kvm_migrate_timers(vcpu);
964
		svm->asid_generation = 0;
965
	}
966 967

	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
968
		rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
A
Avi Kivity 已提交
969 970 971 972
}

static void svm_vcpu_put(struct kvm_vcpu *vcpu)
{
973
	struct vcpu_svm *svm = to_svm(vcpu);
974 975
	int i;

976
	++vcpu->stat.host_state_reload;
977
	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
978
		wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
979

980
	vcpu->arch.host_tsc = native_read_tsc();
A
Avi Kivity 已提交
981 982 983 984
}

static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
{
985
	return to_svm(vcpu)->vmcb->save.rflags;
A
Avi Kivity 已提交
986 987 988 989
}

static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
990
	to_svm(vcpu)->vmcb->save.rflags = rflags;
A
Avi Kivity 已提交
991 992
}

A
Avi Kivity 已提交
993 994 995 996 997 998 999 1000 1001 1002 1003 1004
static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
{
	switch (reg) {
	case VCPU_EXREG_PDPTR:
		BUG_ON(!npt_enabled);
		load_pdptrs(vcpu, vcpu->arch.cr3);
		break;
	default:
		BUG();
	}
}

1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
static void svm_set_vintr(struct vcpu_svm *svm)
{
	svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR;
}

static void svm_clear_vintr(struct vcpu_svm *svm)
{
	svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
}

A
Avi Kivity 已提交
1015 1016
static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
{
1017
	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
A
Avi Kivity 已提交
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029

	switch (seg) {
	case VCPU_SREG_CS: return &save->cs;
	case VCPU_SREG_DS: return &save->ds;
	case VCPU_SREG_ES: return &save->es;
	case VCPU_SREG_FS: return &save->fs;
	case VCPU_SREG_GS: return &save->gs;
	case VCPU_SREG_SS: return &save->ss;
	case VCPU_SREG_TR: return &save->tr;
	case VCPU_SREG_LDTR: return &save->ldtr;
	}
	BUG();
A
Al Viro 已提交
1030
	return NULL;
A
Avi Kivity 已提交
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055
}

static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	return s->base;
}

static void svm_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	var->base = s->base;
	var->limit = s->limit;
	var->selector = s->selector;
	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
	var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1;
1056

J
Joerg Roedel 已提交
1057 1058
	/*
	 * AMD's VMCB does not have an explicit unusable field, so emulate it
1059 1060 1061 1062
	 * for cross vendor migration purposes by "not present"
	 */
	var->unusable = !var->present || (var->type == 0);

1063 1064 1065 1066 1067 1068 1069
	switch (seg) {
	case VCPU_SREG_CS:
		/*
		 * SVM always stores 0 for the 'G' bit in the CS selector in
		 * the VMCB on a VMEXIT. This hurts cross-vendor migration:
		 * Intel's VMENTRY has a check on the 'G' bit.
		 */
1070
		var->g = s->limit > 0xfffff;
1071 1072 1073 1074 1075 1076
		break;
	case VCPU_SREG_TR:
		/*
		 * Work around a bug where the busy flag in the tr selector
		 * isn't exposed
		 */
1077
		var->type |= 0x2;
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
		break;
	case VCPU_SREG_DS:
	case VCPU_SREG_ES:
	case VCPU_SREG_FS:
	case VCPU_SREG_GS:
		/*
		 * The accessed bit must always be set in the segment
		 * descriptor cache, although it can be cleared in the
		 * descriptor, the cached bit always remains at 1. Since
		 * Intel has a check on this, set it here to support
		 * cross-vendor migration.
		 */
		if (!var->unusable)
			var->type |= 0x1;
		break;
1093
	case VCPU_SREG_SS:
J
Joerg Roedel 已提交
1094 1095
		/*
		 * On AMD CPUs sometimes the DB bit in the segment
1096 1097 1098 1099 1100 1101 1102
		 * descriptor is left as 1, although the whole segment has
		 * been made unusable. Clear it here to pass an Intel VMX
		 * entry check when cross vendor migrating.
		 */
		if (var->unusable)
			var->db = 0;
		break;
1103
	}
A
Avi Kivity 已提交
1104 1105
}

1106 1107 1108 1109 1110 1111 1112
static int svm_get_cpl(struct kvm_vcpu *vcpu)
{
	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;

	return save->cpl;
}

1113
static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
1114
{
1115 1116
	struct vcpu_svm *svm = to_svm(vcpu);

1117 1118
	dt->size = svm->vmcb->save.idtr.limit;
	dt->address = svm->vmcb->save.idtr.base;
A
Avi Kivity 已提交
1119 1120
}

1121
static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
1122
{
1123 1124
	struct vcpu_svm *svm = to_svm(vcpu);

1125 1126
	svm->vmcb->save.idtr.limit = dt->size;
	svm->vmcb->save.idtr.base = dt->address ;
A
Avi Kivity 已提交
1127 1128
}

1129
static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
1130
{
1131 1132
	struct vcpu_svm *svm = to_svm(vcpu);

1133 1134
	dt->size = svm->vmcb->save.gdtr.limit;
	dt->address = svm->vmcb->save.gdtr.base;
A
Avi Kivity 已提交
1135 1136
}

1137
static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
1138
{
1139 1140
	struct vcpu_svm *svm = to_svm(vcpu);

1141 1142
	svm->vmcb->save.gdtr.limit = dt->size;
	svm->vmcb->save.gdtr.base = dt->address ;
A
Avi Kivity 已提交
1143 1144
}

1145 1146 1147 1148
static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
{
}

1149
static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1150 1151 1152
{
}

A
Avi Kivity 已提交
1153 1154
static void update_cr0_intercept(struct vcpu_svm *svm)
{
1155
	struct vmcb *vmcb = svm->vmcb;
A
Avi Kivity 已提交
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
	ulong gcr0 = svm->vcpu.arch.cr0;
	u64 *hcr0 = &svm->vmcb->save.cr0;

	if (!svm->vcpu.fpu_active)
		*hcr0 |= SVM_CR0_SELECTIVE_MASK;
	else
		*hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
			| (gcr0 & SVM_CR0_SELECTIVE_MASK);


	if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
		vmcb->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK;
		vmcb->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
		if (is_nested(svm)) {
			struct vmcb *hsave = svm->nested.hsave;

			hsave->control.intercept_cr_read  &= ~INTERCEPT_CR0_MASK;
			hsave->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
			vmcb->control.intercept_cr_read  |= svm->nested.intercept_cr_read;
			vmcb->control.intercept_cr_write |= svm->nested.intercept_cr_write;
		}
A
Avi Kivity 已提交
1177 1178 1179
	} else {
		svm->vmcb->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
		svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
1180 1181 1182 1183 1184 1185
		if (is_nested(svm)) {
			struct vmcb *hsave = svm->nested.hsave;

			hsave->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
			hsave->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
		}
A
Avi Kivity 已提交
1186 1187 1188
	}
}

A
Avi Kivity 已提交
1189 1190
static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
1191 1192
	struct vcpu_svm *svm = to_svm(vcpu);

1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
	if (is_nested(svm)) {
		/*
		 * We are here because we run in nested mode, the host kvm
		 * intercepts cr0 writes but the l1 hypervisor does not.
		 * But the L1 hypervisor may intercept selective cr0 writes.
		 * This needs to be checked here.
		 */
		unsigned long old, new;

		/* Remove bits that would trigger a real cr0 write intercept */
		old = vcpu->arch.cr0 & SVM_CR0_SELECTIVE_MASK;
		new = cr0 & SVM_CR0_SELECTIVE_MASK;

		if (old == new) {
			/* cr0 write with ts and mp unchanged */
			svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
			if (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE)
				return;
		}
	}

1214
#ifdef CONFIG_X86_64
1215
	if (vcpu->arch.efer & EFER_LME) {
1216
		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1217
			vcpu->arch.efer |= EFER_LMA;
1218
			svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
A
Avi Kivity 已提交
1219 1220
		}

M
Mike Day 已提交
1221
		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1222
			vcpu->arch.efer &= ~EFER_LMA;
1223
			svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
A
Avi Kivity 已提交
1224 1225 1226
		}
	}
#endif
1227
	vcpu->arch.cr0 = cr0;
1228 1229 1230

	if (!npt_enabled)
		cr0 |= X86_CR0_PG | X86_CR0_WP;
1231 1232

	if (!vcpu->fpu_active)
J
Joerg Roedel 已提交
1233
		cr0 |= X86_CR0_TS;
1234 1235 1236 1237 1238 1239
	/*
	 * re-enable caching here because the QEMU bios
	 * does not do it - this results in some delay at
	 * reboot
	 */
	cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1240
	svm->vmcb->save.cr0 = cr0;
A
Avi Kivity 已提交
1241
	update_cr0_intercept(svm);
A
Avi Kivity 已提交
1242 1243 1244 1245
}

static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
1246
	unsigned long host_cr4_mce = read_cr4() & X86_CR4_MCE;
1247 1248 1249 1250
	unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;

	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
		force_new_asid(vcpu);
1251

1252 1253 1254
	vcpu->arch.cr4 = cr4;
	if (!npt_enabled)
		cr4 |= X86_CR4_PAE;
1255
	cr4 |= host_cr4_mce;
1256
	to_svm(vcpu)->vmcb->save.cr4 = cr4;
A
Avi Kivity 已提交
1257 1258 1259 1260 1261
}

static void svm_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
1262
	struct vcpu_svm *svm = to_svm(vcpu);
A
Avi Kivity 已提交
1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	s->base = var->base;
	s->limit = var->limit;
	s->selector = var->selector;
	if (var->unusable)
		s->attrib = 0;
	else {
		s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
		s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
		s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
		s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
		s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
		s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
		s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
		s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
	}
	if (seg == VCPU_SREG_CS)
1281 1282
		svm->vmcb->save.cpl
			= (svm->vmcb->save.cs.attrib
A
Avi Kivity 已提交
1283 1284 1285 1286
			   >> SVM_SELECTOR_DPL_SHIFT) & 3;

}

1287
static void update_db_intercept(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1288
{
J
Jan Kiszka 已提交
1289 1290 1291 1292
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->control.intercept_exceptions &=
		~((1 << DB_VECTOR) | (1 << BP_VECTOR));
1293

J
Jan Kiszka 已提交
1294
	if (svm->nmi_singlestep)
1295 1296
		svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR);

J
Jan Kiszka 已提交
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
		if (vcpu->guest_debug &
		    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
			svm->vmcb->control.intercept_exceptions |=
				1 << DB_VECTOR;
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
			svm->vmcb->control.intercept_exceptions |=
				1 << BP_VECTOR;
	} else
		vcpu->guest_debug = 0;
1307 1308
}

1309
static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
1310 1311 1312
{
	struct vcpu_svm *svm = to_svm(vcpu);

1313 1314 1315 1316 1317
	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		svm->vmcb->save.dr7 = dbg->arch.debugreg[7];
	else
		svm->vmcb->save.dr7 = vcpu->arch.dr7;

1318
	update_db_intercept(vcpu);
A
Avi Kivity 已提交
1319 1320 1321 1322
}

static void load_host_msrs(struct kvm_vcpu *vcpu)
{
1323
#ifdef CONFIG_X86_64
1324
	wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
1325
#endif
A
Avi Kivity 已提交
1326 1327 1328 1329
}

static void save_host_msrs(struct kvm_vcpu *vcpu)
{
1330
#ifdef CONFIG_X86_64
1331
	rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
1332
#endif
A
Avi Kivity 已提交
1333 1334
}

1335
static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
A
Avi Kivity 已提交
1336
{
1337 1338 1339
	if (sd->next_asid > sd->max_asid) {
		++sd->asid_generation;
		sd->next_asid = 1;
1340
		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
A
Avi Kivity 已提交
1341 1342
	}

1343 1344
	svm->asid_generation = sd->asid_generation;
	svm->vmcb->control.asid = sd->next_asid++;
A
Avi Kivity 已提交
1345 1346
}

1347
static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
A
Avi Kivity 已提交
1348
{
1349 1350
	struct vcpu_svm *svm = to_svm(vcpu);

1351
	svm->vmcb->save.dr7 = value;
A
Avi Kivity 已提交
1352 1353
}

A
Avi Kivity 已提交
1354
static int pf_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
1355 1356 1357 1358
{
	u64 fault_address;
	u32 error_code;

1359 1360
	fault_address  = svm->vmcb->control.exit_info_2;
	error_code = svm->vmcb->control.exit_info_1;
1361

1362
	trace_kvm_page_fault(fault_address, error_code);
1363 1364
	if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
		kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
1365
	return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
A
Avi Kivity 已提交
1366 1367
}

A
Avi Kivity 已提交
1368
static int db_interception(struct vcpu_svm *svm)
J
Jan Kiszka 已提交
1369
{
A
Avi Kivity 已提交
1370 1371
	struct kvm_run *kvm_run = svm->vcpu.run;

J
Jan Kiszka 已提交
1372
	if (!(svm->vcpu.guest_debug &
1373
	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
J
Jan Kiszka 已提交
1374
		!svm->nmi_singlestep) {
J
Jan Kiszka 已提交
1375 1376 1377
		kvm_queue_exception(&svm->vcpu, DB_VECTOR);
		return 1;
	}
1378

J
Jan Kiszka 已提交
1379 1380
	if (svm->nmi_singlestep) {
		svm->nmi_singlestep = false;
1381 1382 1383 1384 1385 1386 1387
		if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
			svm->vmcb->save.rflags &=
				~(X86_EFLAGS_TF | X86_EFLAGS_RF);
		update_db_intercept(&svm->vcpu);
	}

	if (svm->vcpu.guest_debug &
J
Joerg Roedel 已提交
1388
	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1389 1390 1391 1392 1393 1394 1395 1396
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
		kvm_run->debug.arch.pc =
			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
		kvm_run->debug.arch.exception = DB_VECTOR;
		return 0;
	}

	return 1;
J
Jan Kiszka 已提交
1397 1398
}

A
Avi Kivity 已提交
1399
static int bp_interception(struct vcpu_svm *svm)
J
Jan Kiszka 已提交
1400
{
A
Avi Kivity 已提交
1401 1402
	struct kvm_run *kvm_run = svm->vcpu.run;

J
Jan Kiszka 已提交
1403 1404 1405 1406 1407 1408
	kvm_run->exit_reason = KVM_EXIT_DEBUG;
	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
	kvm_run->debug.arch.exception = BP_VECTOR;
	return 0;
}

A
Avi Kivity 已提交
1409
static int ud_interception(struct vcpu_svm *svm)
1410 1411 1412
{
	int er;

A
Avi Kivity 已提交
1413
	er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD);
1414
	if (er != EMULATE_DONE)
1415
		kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1416 1417 1418
	return 1;
}

A
Avi Kivity 已提交
1419
static void svm_fpu_activate(struct kvm_vcpu *vcpu)
A
Anthony Liguori 已提交
1420
{
A
Avi Kivity 已提交
1421
	struct vcpu_svm *svm = to_svm(vcpu);
1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
	u32 excp;

	if (is_nested(svm)) {
		u32 h_excp, n_excp;

		h_excp  = svm->nested.hsave->control.intercept_exceptions;
		n_excp  = svm->nested.intercept_exceptions;
		h_excp &= ~(1 << NM_VECTOR);
		excp    = h_excp | n_excp;
	} else {
		excp  = svm->vmcb->control.intercept_exceptions;
J
Joerg Roedel 已提交
1433
		excp &= ~(1 << NM_VECTOR);
1434 1435 1436 1437
	}

	svm->vmcb->control.intercept_exceptions = excp;

R
Rusty Russell 已提交
1438
	svm->vcpu.fpu_active = 1;
A
Avi Kivity 已提交
1439
	update_cr0_intercept(svm);
A
Avi Kivity 已提交
1440
}
1441

A
Avi Kivity 已提交
1442 1443 1444
static int nm_interception(struct vcpu_svm *svm)
{
	svm_fpu_activate(&svm->vcpu);
1445
	return 1;
A
Anthony Liguori 已提交
1446 1447
}

1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
static bool is_erratum_383(void)
{
	int err, i;
	u64 value;

	if (!erratum_383_found)
		return false;

	value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
	if (err)
		return false;

	/* Bit 62 may or may not be set for this mce */
	value &= ~(1ULL << 62);

	if (value != 0xb600000000010015ULL)
		return false;

	/* Clear MCi_STATUS registers */
	for (i = 0; i < 6; ++i)
		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);

	value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
	if (!err) {
		u32 low, high;

		value &= ~(1ULL << 2);
		low    = lower_32_bits(value);
		high   = upper_32_bits(value);

		native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
	}

	/* Flush tlb to evict multi-match entries */
	__flush_tlb_all();

	return true;
}

1487
static void svm_handle_mce(struct vcpu_svm *svm)
1488
{
1489 1490 1491 1492 1493 1494 1495
	if (is_erratum_383()) {
		/*
		 * Erratum 383 triggered. Guest state is corrupt so kill the
		 * guest.
		 */
		pr_err("KVM: Guest triggered AMD Erratum 383\n");

1496
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
1497 1498 1499 1500

		return;
	}

1501 1502 1503 1504 1505 1506 1507 1508
	/*
	 * On an #MC intercept the MCE handler is not called automatically in
	 * the host. So do it by hand here.
	 */
	asm volatile (
		"int $0x12\n");
	/* not sure if we ever come back to this point */

1509 1510 1511 1512 1513
	return;
}

static int mc_interception(struct vcpu_svm *svm)
{
1514 1515 1516
	return 1;
}

A
Avi Kivity 已提交
1517
static int shutdown_interception(struct vcpu_svm *svm)
1518
{
A
Avi Kivity 已提交
1519 1520
	struct kvm_run *kvm_run = svm->vcpu.run;

1521 1522 1523 1524
	/*
	 * VMCB is undefined after a SHUTDOWN intercept
	 * so reinitialize it.
	 */
1525
	clear_page(svm->vmcb);
1526
	init_vmcb(svm);
1527 1528 1529 1530 1531

	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
	return 0;
}

A
Avi Kivity 已提交
1532
static int io_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
1533
{
1534
	struct kvm_vcpu *vcpu = &svm->vcpu;
M
Mike Day 已提交
1535
	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
1536
	int size, in, string;
1537
	unsigned port;
A
Avi Kivity 已提交
1538

R
Rusty Russell 已提交
1539
	++svm->vcpu.stat.io_exits;
1540
	string = (io_info & SVM_IOIO_STR_MASK) != 0;
1541
	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
1542
	if (string || in)
1543
		return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
1544

1545 1546
	port = io_info >> 16;
	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
1547
	svm->next_rip = svm->vmcb->control.exit_info_2;
1548
	skip_emulated_instruction(&svm->vcpu);
1549 1550

	return kvm_fast_pio_out(vcpu, size, port);
A
Avi Kivity 已提交
1551 1552
}

A
Avi Kivity 已提交
1553
static int nmi_interception(struct vcpu_svm *svm)
1554 1555 1556 1557
{
	return 1;
}

A
Avi Kivity 已提交
1558
static int intr_interception(struct vcpu_svm *svm)
1559 1560 1561 1562 1563
{
	++svm->vcpu.stat.irq_exits;
	return 1;
}

A
Avi Kivity 已提交
1564
static int nop_on_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
1565 1566 1567 1568
{
	return 1;
}

A
Avi Kivity 已提交
1569
static int halt_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
1570
{
1571
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
R
Rusty Russell 已提交
1572 1573
	skip_emulated_instruction(&svm->vcpu);
	return kvm_emulate_halt(&svm->vcpu);
A
Avi Kivity 已提交
1574 1575
}

A
Avi Kivity 已提交
1576
static int vmmcall_interception(struct vcpu_svm *svm)
1577
{
1578
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
R
Rusty Russell 已提交
1579
	skip_emulated_instruction(&svm->vcpu);
1580 1581
	kvm_emulate_hypercall(&svm->vcpu);
	return 1;
1582 1583
}

1584 1585
static int nested_svm_check_permissions(struct vcpu_svm *svm)
{
1586
	if (!(svm->vcpu.arch.efer & EFER_SVME)
1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599
	    || !is_paging(&svm->vcpu)) {
		kvm_queue_exception(&svm->vcpu, UD_VECTOR);
		return 1;
	}

	if (svm->vmcb->save.cpl) {
		kvm_inject_gp(&svm->vcpu, 0);
		return 1;
	}

       return 0;
}

1600 1601 1602
static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
				      bool has_error_code, u32 error_code)
{
1603 1604
	int vmexit;

1605 1606
	if (!is_nested(svm))
		return 0;
1607

1608 1609 1610 1611 1612
	svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
	svm->vmcb->control.exit_code_hi = 0;
	svm->vmcb->control.exit_info_1 = error_code;
	svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;

1613 1614 1615 1616 1617
	vmexit = nested_svm_intercept(svm);
	if (vmexit == NESTED_EXIT_DONE)
		svm->nested.exit_required = true;

	return vmexit;
1618 1619
}

1620 1621
/* This function returns true if it is save to enable the irq window */
static inline bool nested_svm_intr(struct vcpu_svm *svm)
1622
{
1623
	if (!is_nested(svm))
1624
		return true;
1625

1626
	if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
1627
		return true;
1628

1629
	if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
1630
		return false;
1631

1632 1633 1634
	svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
	svm->vmcb->control.exit_info_1 = 0;
	svm->vmcb->control.exit_info_2 = 0;
1635

1636 1637 1638 1639 1640 1641 1642 1643
	if (svm->nested.intercept & 1ULL) {
		/*
		 * The #vmexit can't be emulated here directly because this
		 * code path runs with irqs and preemtion disabled. A
		 * #vmexit emulation might sleep. Only signal request for
		 * the #vmexit here.
		 */
		svm->nested.exit_required = true;
1644
		trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
1645
		return false;
1646 1647
	}

1648
	return true;
1649 1650
}

1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
/* This function returns true if it is save to enable the nmi window */
static inline bool nested_svm_nmi(struct vcpu_svm *svm)
{
	if (!is_nested(svm))
		return true;

	if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
		return true;

	svm->vmcb->control.exit_code = SVM_EXIT_NMI;
	svm->nested.exit_required = true;

	return false;
1664 1665
}

1666
static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
1667 1668 1669
{
	struct page *page;

1670 1671
	might_sleep();

1672 1673 1674 1675
	page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT);
	if (is_error_page(page))
		goto error;

1676 1677 1678
	*_page = page;

	return kmap(page);
1679 1680 1681 1682 1683 1684 1685 1686

error:
	kvm_release_page_clean(page);
	kvm_inject_gp(&svm->vcpu, 0);

	return NULL;
}

1687
static void nested_svm_unmap(struct page *page)
1688
{
1689
	kunmap(page);
1690 1691 1692
	kvm_release_page_dirty(page);
}

1693 1694 1695 1696 1697
static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
{
	unsigned port;
	u8 val, bit;
	u64 gpa;
1698

1699 1700
	if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
		return NESTED_EXIT_HOST;
1701

1702 1703 1704 1705 1706 1707 1708 1709 1710
	port = svm->vmcb->control.exit_info_1 >> 16;
	gpa  = svm->nested.vmcb_iopm + (port / 8);
	bit  = port % 8;
	val  = 0;

	if (kvm_read_guest(svm->vcpu.kvm, gpa, &val, 1))
		val &= (1 << bit);

	return val ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
1711 1712
}

1713
static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
1714
{
1715 1716
	u32 offset, msr, value;
	int write, mask;
1717

1718
	if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
1719
		return NESTED_EXIT_HOST;
1720

1721 1722 1723 1724
	msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
	offset = svm_msrpm_offset(msr);
	write  = svm->vmcb->control.exit_info_1 & 1;
	mask   = 1 << ((2 * (msr & 0xf)) + write);
1725

1726 1727
	if (offset == MSR_INVALID)
		return NESTED_EXIT_DONE;
1728

1729 1730
	/* Offset is in 32 bit units but need in 8 bit units */
	offset *= 4;
1731

1732 1733
	if (kvm_read_guest(svm->vcpu.kvm, svm->nested.vmcb_msrpm + offset, &value, 4))
		return NESTED_EXIT_DONE;
1734

1735
	return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
1736 1737
}

1738
static int nested_svm_exit_special(struct vcpu_svm *svm)
1739 1740
{
	u32 exit_code = svm->vmcb->control.exit_code;
1741

1742 1743 1744
	switch (exit_code) {
	case SVM_EXIT_INTR:
	case SVM_EXIT_NMI:
1745
	case SVM_EXIT_EXCP_BASE + MC_VECTOR:
1746 1747
		return NESTED_EXIT_HOST;
	case SVM_EXIT_NPF:
J
Joerg Roedel 已提交
1748
		/* For now we are always handling NPFs when using them */
1749 1750 1751 1752
		if (npt_enabled)
			return NESTED_EXIT_HOST;
		break;
	case SVM_EXIT_EXCP_BASE + PF_VECTOR:
J
Joerg Roedel 已提交
1753
		/* When we're shadowing, trap PFs */
1754 1755 1756
		if (!npt_enabled)
			return NESTED_EXIT_HOST;
		break;
1757 1758 1759
	case SVM_EXIT_EXCP_BASE + NM_VECTOR:
		nm_interception(svm);
		break;
1760 1761
	default:
		break;
1762 1763
	}

1764 1765 1766 1767 1768 1769
	return NESTED_EXIT_CONTINUE;
}

/*
 * If this function returns true, this #vmexit was already handled
 */
1770
static int nested_svm_intercept(struct vcpu_svm *svm)
1771 1772 1773 1774
{
	u32 exit_code = svm->vmcb->control.exit_code;
	int vmexit = NESTED_EXIT_HOST;

1775
	switch (exit_code) {
1776
	case SVM_EXIT_MSR:
1777
		vmexit = nested_svm_exit_handled_msr(svm);
1778
		break;
1779 1780 1781
	case SVM_EXIT_IOIO:
		vmexit = nested_svm_intercept_ioio(svm);
		break;
1782 1783
	case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: {
		u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0);
J
Joerg Roedel 已提交
1784
		if (svm->nested.intercept_cr_read & cr_bits)
1785
			vmexit = NESTED_EXIT_DONE;
1786 1787 1788 1789
		break;
	}
	case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: {
		u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0);
J
Joerg Roedel 已提交
1790
		if (svm->nested.intercept_cr_write & cr_bits)
1791
			vmexit = NESTED_EXIT_DONE;
1792 1793 1794 1795
		break;
	}
	case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: {
		u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0);
J
Joerg Roedel 已提交
1796
		if (svm->nested.intercept_dr_read & dr_bits)
1797
			vmexit = NESTED_EXIT_DONE;
1798 1799 1800 1801
		break;
	}
	case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: {
		u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0);
J
Joerg Roedel 已提交
1802
		if (svm->nested.intercept_dr_write & dr_bits)
1803
			vmexit = NESTED_EXIT_DONE;
1804 1805 1806 1807
		break;
	}
	case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
		u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
J
Joerg Roedel 已提交
1808
		if (svm->nested.intercept_exceptions & excp_bits)
1809
			vmexit = NESTED_EXIT_DONE;
1810 1811
		break;
	}
1812 1813 1814 1815
	case SVM_EXIT_ERR: {
		vmexit = NESTED_EXIT_DONE;
		break;
	}
1816 1817
	default: {
		u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
J
Joerg Roedel 已提交
1818
		if (svm->nested.intercept & exit_bits)
1819
			vmexit = NESTED_EXIT_DONE;
1820 1821 1822
	}
	}

1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
	return vmexit;
}

static int nested_svm_exit_handled(struct vcpu_svm *svm)
{
	int vmexit;

	vmexit = nested_svm_intercept(svm);

	if (vmexit == NESTED_EXIT_DONE)
1833 1834 1835
		nested_svm_vmexit(svm);

	return vmexit;
1836 1837
}

1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
{
	struct vmcb_control_area *dst  = &dst_vmcb->control;
	struct vmcb_control_area *from = &from_vmcb->control;

	dst->intercept_cr_read    = from->intercept_cr_read;
	dst->intercept_cr_write   = from->intercept_cr_write;
	dst->intercept_dr_read    = from->intercept_dr_read;
	dst->intercept_dr_write   = from->intercept_dr_write;
	dst->intercept_exceptions = from->intercept_exceptions;
	dst->intercept            = from->intercept;
	dst->iopm_base_pa         = from->iopm_base_pa;
	dst->msrpm_base_pa        = from->msrpm_base_pa;
	dst->tsc_offset           = from->tsc_offset;
	dst->asid                 = from->asid;
	dst->tlb_ctl              = from->tlb_ctl;
	dst->int_ctl              = from->int_ctl;
	dst->int_vector           = from->int_vector;
	dst->int_state            = from->int_state;
	dst->exit_code            = from->exit_code;
	dst->exit_code_hi         = from->exit_code_hi;
	dst->exit_info_1          = from->exit_info_1;
	dst->exit_info_2          = from->exit_info_2;
	dst->exit_int_info        = from->exit_int_info;
	dst->exit_int_info_err    = from->exit_int_info_err;
	dst->nested_ctl           = from->nested_ctl;
	dst->event_inj            = from->event_inj;
	dst->event_inj_err        = from->event_inj_err;
	dst->nested_cr3           = from->nested_cr3;
	dst->lbr_ctl              = from->lbr_ctl;
}

1870
static int nested_svm_vmexit(struct vcpu_svm *svm)
1871
{
1872
	struct vmcb *nested_vmcb;
1873
	struct vmcb *hsave = svm->nested.hsave;
J
Joerg Roedel 已提交
1874
	struct vmcb *vmcb = svm->vmcb;
1875
	struct page *page;
1876

1877 1878 1879 1880 1881 1882
	trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
				       vmcb->control.exit_info_1,
				       vmcb->control.exit_info_2,
				       vmcb->control.exit_int_info,
				       vmcb->control.exit_int_info_err);

1883
	nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
1884 1885 1886
	if (!nested_vmcb)
		return 1;

1887 1888 1889
	/* Exit nested SVM mode */
	svm->nested.vmcb = 0;

1890
	/* Give the current vmcb to the guest */
J
Joerg Roedel 已提交
1891 1892 1893 1894 1895 1896 1897 1898
	disable_gif(svm);

	nested_vmcb->save.es     = vmcb->save.es;
	nested_vmcb->save.cs     = vmcb->save.cs;
	nested_vmcb->save.ss     = vmcb->save.ss;
	nested_vmcb->save.ds     = vmcb->save.ds;
	nested_vmcb->save.gdtr   = vmcb->save.gdtr;
	nested_vmcb->save.idtr   = vmcb->save.idtr;
1899
	nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
1900
	nested_vmcb->save.cr3    = svm->vcpu.arch.cr3;
J
Joerg Roedel 已提交
1901
	nested_vmcb->save.cr2    = vmcb->save.cr2;
1902
	nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
J
Joerg Roedel 已提交
1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919
	nested_vmcb->save.rflags = vmcb->save.rflags;
	nested_vmcb->save.rip    = vmcb->save.rip;
	nested_vmcb->save.rsp    = vmcb->save.rsp;
	nested_vmcb->save.rax    = vmcb->save.rax;
	nested_vmcb->save.dr7    = vmcb->save.dr7;
	nested_vmcb->save.dr6    = vmcb->save.dr6;
	nested_vmcb->save.cpl    = vmcb->save.cpl;

	nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
	nested_vmcb->control.int_vector        = vmcb->control.int_vector;
	nested_vmcb->control.int_state         = vmcb->control.int_state;
	nested_vmcb->control.exit_code         = vmcb->control.exit_code;
	nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
	nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
	nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
	nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
	nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935

	/*
	 * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
	 * to make sure that we do not lose injected events. So check event_inj
	 * here and copy it to exit_int_info if it is valid.
	 * Exit_int_info and event_inj can't be both valid because the case
	 * below only happens on a VMRUN instruction intercept which has
	 * no valid exit_int_info set.
	 */
	if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
		struct vmcb_control_area *nc = &nested_vmcb->control;

		nc->exit_int_info     = vmcb->control.event_inj;
		nc->exit_int_info_err = vmcb->control.event_inj_err;
	}

J
Joerg Roedel 已提交
1936 1937 1938
	nested_vmcb->control.tlb_ctl           = 0;
	nested_vmcb->control.event_inj         = 0;
	nested_vmcb->control.event_inj_err     = 0;
1939 1940 1941 1942 1943 1944

	/* We always set V_INTR_MASKING and remember the old value in hflags */
	if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
		nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;

	/* Restore the original control entries */
1945
	copy_vmcb_control_area(vmcb, hsave);
1946

1947 1948
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);
1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964

	/* Restore selected save entries */
	svm->vmcb->save.es = hsave->save.es;
	svm->vmcb->save.cs = hsave->save.cs;
	svm->vmcb->save.ss = hsave->save.ss;
	svm->vmcb->save.ds = hsave->save.ds;
	svm->vmcb->save.gdtr = hsave->save.gdtr;
	svm->vmcb->save.idtr = hsave->save.idtr;
	svm->vmcb->save.rflags = hsave->save.rflags;
	svm_set_efer(&svm->vcpu, hsave->save.efer);
	svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
	svm_set_cr4(&svm->vcpu, hsave->save.cr4);
	if (npt_enabled) {
		svm->vmcb->save.cr3 = hsave->save.cr3;
		svm->vcpu.arch.cr3 = hsave->save.cr3;
	} else {
1965
		(void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
1966 1967 1968 1969 1970 1971 1972 1973
	}
	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
	svm->vmcb->save.dr7 = 0;
	svm->vmcb->save.cpl = 0;
	svm->vmcb->control.exit_int_info = 0;

1974
	nested_svm_unmap(page);
1975 1976 1977 1978 1979 1980

	kvm_mmu_reset_context(&svm->vcpu);
	kvm_mmu_load(&svm->vcpu);

	return 0;
}
A
Alexander Graf 已提交
1981

1982
static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
A
Alexander Graf 已提交
1983
{
1984 1985 1986 1987 1988
	/*
	 * This function merges the msr permission bitmaps of kvm and the
	 * nested vmcb. It is omptimized in that it only merges the parts where
	 * the kvm msr permission bitmap may contain zero bits
	 */
A
Alexander Graf 已提交
1989
	int i;
1990

1991 1992
	if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
		return true;
1993

1994 1995 1996
	for (i = 0; i < MSRPM_OFFSETS; i++) {
		u32 value, p;
		u64 offset;
1997

1998 1999
		if (msrpm_offsets[i] == 0xffffffff)
			break;
A
Alexander Graf 已提交
2000

2001 2002
		p      = msrpm_offsets[i];
		offset = svm->nested.vmcb_msrpm + (p * 4);
2003 2004 2005 2006 2007 2008

		if (kvm_read_guest(svm->vcpu.kvm, offset, &value, 4))
			return false;

		svm->nested.msrpm[p] = svm->msrpm[p] | value;
	}
A
Alexander Graf 已提交
2009

2010
	svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
2011 2012

	return true;
A
Alexander Graf 已提交
2013 2014
}

2015
static bool nested_svm_vmrun(struct vcpu_svm *svm)
A
Alexander Graf 已提交
2016
{
2017
	struct vmcb *nested_vmcb;
2018
	struct vmcb *hsave = svm->nested.hsave;
J
Joerg Roedel 已提交
2019
	struct vmcb *vmcb = svm->vmcb;
2020
	struct page *page;
2021
	u64 vmcb_gpa;
A
Alexander Graf 已提交
2022

2023
	vmcb_gpa = svm->vmcb->save.rax;
A
Alexander Graf 已提交
2024

2025
	nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2026 2027 2028
	if (!nested_vmcb)
		return false;

2029
	trace_kvm_nested_vmrun(svm->vmcb->save.rip - 3, vmcb_gpa,
2030 2031 2032 2033 2034
			       nested_vmcb->save.rip,
			       nested_vmcb->control.int_ctl,
			       nested_vmcb->control.event_inj,
			       nested_vmcb->control.nested_ctl);

2035 2036 2037 2038 2039
	trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr_read,
				    nested_vmcb->control.intercept_cr_write,
				    nested_vmcb->control.intercept_exceptions,
				    nested_vmcb->control.intercept);

A
Alexander Graf 已提交
2040
	/* Clear internal status */
2041 2042
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);
A
Alexander Graf 已提交
2043

J
Joerg Roedel 已提交
2044 2045 2046 2047
	/*
	 * Save the old vmcb, so we don't need to pick what we save, but can
	 * restore everything when a VMEXIT occurs
	 */
J
Joerg Roedel 已提交
2048 2049 2050 2051 2052 2053
	hsave->save.es     = vmcb->save.es;
	hsave->save.cs     = vmcb->save.cs;
	hsave->save.ss     = vmcb->save.ss;
	hsave->save.ds     = vmcb->save.ds;
	hsave->save.gdtr   = vmcb->save.gdtr;
	hsave->save.idtr   = vmcb->save.idtr;
2054
	hsave->save.efer   = svm->vcpu.arch.efer;
2055
	hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
J
Joerg Roedel 已提交
2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
	hsave->save.cr4    = svm->vcpu.arch.cr4;
	hsave->save.rflags = vmcb->save.rflags;
	hsave->save.rip    = svm->next_rip;
	hsave->save.rsp    = vmcb->save.rsp;
	hsave->save.rax    = vmcb->save.rax;
	if (npt_enabled)
		hsave->save.cr3    = vmcb->save.cr3;
	else
		hsave->save.cr3    = svm->vcpu.arch.cr3;

2066
	copy_vmcb_control_area(hsave, vmcb);
A
Alexander Graf 已提交
2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086

	if (svm->vmcb->save.rflags & X86_EFLAGS_IF)
		svm->vcpu.arch.hflags |= HF_HIF_MASK;
	else
		svm->vcpu.arch.hflags &= ~HF_HIF_MASK;

	/* Load the nested guest state */
	svm->vmcb->save.es = nested_vmcb->save.es;
	svm->vmcb->save.cs = nested_vmcb->save.cs;
	svm->vmcb->save.ss = nested_vmcb->save.ss;
	svm->vmcb->save.ds = nested_vmcb->save.ds;
	svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
	svm->vmcb->save.idtr = nested_vmcb->save.idtr;
	svm->vmcb->save.rflags = nested_vmcb->save.rflags;
	svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
	svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
	svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
	if (npt_enabled) {
		svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
		svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
2087
	} else
2088
		(void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
2089 2090 2091 2092

	/* Guest paging mode is active - reset mmu */
	kvm_mmu_reset_context(&svm->vcpu);

J
Joerg Roedel 已提交
2093
	svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
A
Alexander Graf 已提交
2094 2095 2096
	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
J
Joerg Roedel 已提交
2097

A
Alexander Graf 已提交
2098 2099 2100 2101 2102 2103 2104 2105
	/* In case we don't even reach vcpu_run, the fields are not updated */
	svm->vmcb->save.rax = nested_vmcb->save.rax;
	svm->vmcb->save.rsp = nested_vmcb->save.rsp;
	svm->vmcb->save.rip = nested_vmcb->save.rip;
	svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
	svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
	svm->vmcb->save.cpl = nested_vmcb->save.cpl;

2106
	svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
2107
	svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
A
Alexander Graf 已提交
2108

J
Joerg Roedel 已提交
2109 2110 2111 2112 2113 2114 2115 2116
	/* cache intercepts */
	svm->nested.intercept_cr_read    = nested_vmcb->control.intercept_cr_read;
	svm->nested.intercept_cr_write   = nested_vmcb->control.intercept_cr_write;
	svm->nested.intercept_dr_read    = nested_vmcb->control.intercept_dr_read;
	svm->nested.intercept_dr_write   = nested_vmcb->control.intercept_dr_write;
	svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
	svm->nested.intercept            = nested_vmcb->control.intercept;

A
Alexander Graf 已提交
2117 2118 2119 2120 2121 2122 2123
	force_new_asid(&svm->vcpu);
	svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
	if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
		svm->vcpu.arch.hflags |= HF_VINTR_MASK;
	else
		svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;

2124 2125 2126 2127 2128 2129
	if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
		/* We only want the cr8 intercept bits of the guest */
		svm->vmcb->control.intercept_cr_read &= ~INTERCEPT_CR8_MASK;
		svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
	}

2130 2131 2132
	/* We don't want to see VMMCALLs from a nested guest */
	svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMMCALL);

J
Joerg Roedel 已提交
2133 2134 2135 2136
	/*
	 * We don't want a nested guest to be more powerful than the guest, so
	 * all intercepts are ORed
	 */
2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150
	svm->vmcb->control.intercept_cr_read |=
		nested_vmcb->control.intercept_cr_read;
	svm->vmcb->control.intercept_cr_write |=
		nested_vmcb->control.intercept_cr_write;
	svm->vmcb->control.intercept_dr_read |=
		nested_vmcb->control.intercept_dr_read;
	svm->vmcb->control.intercept_dr_write |=
		nested_vmcb->control.intercept_dr_write;
	svm->vmcb->control.intercept_exceptions |=
		nested_vmcb->control.intercept_exceptions;

	svm->vmcb->control.intercept |= nested_vmcb->control.intercept;

	svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
A
Alexander Graf 已提交
2151 2152 2153 2154 2155 2156
	svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
	svm->vmcb->control.int_state = nested_vmcb->control.int_state;
	svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
	svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
	svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;

2157
	nested_svm_unmap(page);
2158

2159 2160
	/* nested_vmcb is our indicator if nested SVM is activated */
	svm->nested.vmcb = vmcb_gpa;
2161

2162
	enable_gif(svm);
A
Alexander Graf 已提交
2163

2164
	return true;
A
Alexander Graf 已提交
2165 2166
}

2167
static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182
{
	to_vmcb->save.fs = from_vmcb->save.fs;
	to_vmcb->save.gs = from_vmcb->save.gs;
	to_vmcb->save.tr = from_vmcb->save.tr;
	to_vmcb->save.ldtr = from_vmcb->save.ldtr;
	to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
	to_vmcb->save.star = from_vmcb->save.star;
	to_vmcb->save.lstar = from_vmcb->save.lstar;
	to_vmcb->save.cstar = from_vmcb->save.cstar;
	to_vmcb->save.sfmask = from_vmcb->save.sfmask;
	to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
	to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
	to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
}

A
Avi Kivity 已提交
2183
static int vmload_interception(struct vcpu_svm *svm)
2184
{
2185
	struct vmcb *nested_vmcb;
2186
	struct page *page;
2187

2188 2189 2190 2191 2192 2193
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

2194
	nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2195 2196 2197 2198
	if (!nested_vmcb)
		return 1;

	nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
2199
	nested_svm_unmap(page);
2200 2201 2202 2203

	return 1;
}

A
Avi Kivity 已提交
2204
static int vmsave_interception(struct vcpu_svm *svm)
2205
{
2206
	struct vmcb *nested_vmcb;
2207
	struct page *page;
2208

2209 2210 2211 2212 2213 2214
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

2215
	nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2216 2217 2218 2219
	if (!nested_vmcb)
		return 1;

	nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
2220
	nested_svm_unmap(page);
2221 2222 2223 2224

	return 1;
}

A
Avi Kivity 已提交
2225
static int vmrun_interception(struct vcpu_svm *svm)
A
Alexander Graf 已提交
2226 2227 2228 2229 2230 2231 2232
{
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

2233
	if (!nested_svm_vmrun(svm))
A
Alexander Graf 已提交
2234 2235
		return 1;

2236
	if (!nested_svm_vmrun_msrpm(svm))
2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248
		goto failed;

	return 1;

failed:

	svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
	svm->vmcb->control.exit_code_hi = 0;
	svm->vmcb->control.exit_info_1  = 0;
	svm->vmcb->control.exit_info_2  = 0;

	nested_svm_vmexit(svm);
A
Alexander Graf 已提交
2249 2250 2251 2252

	return 1;
}

A
Avi Kivity 已提交
2253
static int stgi_interception(struct vcpu_svm *svm)
2254 2255 2256 2257 2258 2259 2260
{
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

2261
	enable_gif(svm);
2262 2263 2264 2265

	return 1;
}

A
Avi Kivity 已提交
2266
static int clgi_interception(struct vcpu_svm *svm)
2267 2268 2269 2270 2271 2272 2273
{
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);

2274
	disable_gif(svm);
2275 2276 2277 2278 2279 2280 2281 2282

	/* After a CLGI no interrupts should come */
	svm_clear_vintr(svm);
	svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;

	return 1;
}

A
Avi Kivity 已提交
2283
static int invlpga_interception(struct vcpu_svm *svm)
A
Alexander Graf 已提交
2284 2285 2286
{
	struct kvm_vcpu *vcpu = &svm->vcpu;

2287 2288 2289
	trace_kvm_invlpga(svm->vmcb->save.rip, vcpu->arch.regs[VCPU_REGS_RCX],
			  vcpu->arch.regs[VCPU_REGS_RAX]);

A
Alexander Graf 已提交
2290 2291 2292 2293 2294 2295 2296 2297
	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
	kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]);

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
	skip_emulated_instruction(&svm->vcpu);
	return 1;
}

2298 2299 2300 2301 2302 2303 2304 2305
static int skinit_interception(struct vcpu_svm *svm)
{
	trace_kvm_skinit(svm->vmcb->save.rip, svm->vcpu.arch.regs[VCPU_REGS_RAX]);

	kvm_queue_exception(&svm->vcpu, UD_VECTOR);
	return 1;
}

A
Avi Kivity 已提交
2306
static int invalid_op_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2307
{
2308
	kvm_queue_exception(&svm->vcpu, UD_VECTOR);
A
Avi Kivity 已提交
2309 2310 2311
	return 1;
}

A
Avi Kivity 已提交
2312
static int task_switch_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2313
{
2314
	u16 tss_selector;
2315 2316 2317
	int reason;
	int int_type = svm->vmcb->control.exit_int_info &
		SVM_EXITINTINFO_TYPE_MASK;
2318
	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2319 2320 2321 2322
	uint32_t type =
		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
	uint32_t idt_v =
		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2323 2324
	bool has_error_code = false;
	u32 error_code = 0;
2325 2326

	tss_selector = (u16)svm->vmcb->control.exit_info_1;
2327

2328 2329
	if (svm->vmcb->control.exit_info_2 &
	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2330 2331 2332 2333
		reason = TASK_SWITCH_IRET;
	else if (svm->vmcb->control.exit_info_2 &
		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
		reason = TASK_SWITCH_JMP;
2334
	else if (idt_v)
2335 2336 2337 2338
		reason = TASK_SWITCH_GATE;
	else
		reason = TASK_SWITCH_CALL;

2339 2340 2341 2342 2343 2344
	if (reason == TASK_SWITCH_GATE) {
		switch (type) {
		case SVM_EXITINTINFO_TYPE_NMI:
			svm->vcpu.arch.nmi_injected = false;
			break;
		case SVM_EXITINTINFO_TYPE_EXEPT:
2345 2346 2347 2348 2349 2350
			if (svm->vmcb->control.exit_info_2 &
			    (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
				has_error_code = true;
				error_code =
					(u32)svm->vmcb->control.exit_info_2;
			}
2351 2352 2353 2354 2355 2356 2357 2358 2359
			kvm_clear_exception_queue(&svm->vcpu);
			break;
		case SVM_EXITINTINFO_TYPE_INTR:
			kvm_clear_interrupt_queue(&svm->vcpu);
			break;
		default:
			break;
		}
	}
2360

2361 2362 2363
	if (reason != TASK_SWITCH_GATE ||
	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2364 2365
	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
		skip_emulated_instruction(&svm->vcpu);
2366

2367 2368 2369 2370 2371 2372 2373 2374
	if (kvm_task_switch(&svm->vcpu, tss_selector, reason,
				has_error_code, error_code) == EMULATE_FAIL) {
		svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		svm->vcpu.run->internal.ndata = 0;
		return 0;
	}
	return 1;
A
Avi Kivity 已提交
2375 2376
}

A
Avi Kivity 已提交
2377
static int cpuid_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2378
{
2379
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
R
Rusty Russell 已提交
2380
	kvm_emulate_cpuid(&svm->vcpu);
2381
	return 1;
A
Avi Kivity 已提交
2382 2383
}

A
Avi Kivity 已提交
2384
static int iret_interception(struct vcpu_svm *svm)
2385 2386
{
	++svm->vcpu.stat.nmi_window_exits;
2387
	svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
2388
	svm->vcpu.arch.hflags |= HF_IRET_MASK;
2389 2390 2391
	return 1;
}

A
Avi Kivity 已提交
2392
static int invlpg_interception(struct vcpu_svm *svm)
M
Marcelo Tosatti 已提交
2393
{
2394
	return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE;
M
Marcelo Tosatti 已提交
2395 2396
}

A
Avi Kivity 已提交
2397
static int emulate_on_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2398
{
2399
	return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE;
A
Avi Kivity 已提交
2400 2401
}

A
Avi Kivity 已提交
2402
static int cr8_write_interception(struct vcpu_svm *svm)
2403
{
A
Avi Kivity 已提交
2404 2405
	struct kvm_run *kvm_run = svm->vcpu.run;

2406 2407
	u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
	/* instruction emulation calls kvm_set_cr8() */
A
Avi Kivity 已提交
2408
	emulate_instruction(&svm->vcpu, 0, 0, 0);
2409 2410
	if (irqchip_in_kernel(svm->vcpu.kvm)) {
		svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
2411
		return 1;
2412
	}
2413 2414
	if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
		return 1;
2415 2416 2417 2418
	kvm_run->exit_reason = KVM_EXIT_SET_TPR;
	return 0;
}

A
Avi Kivity 已提交
2419 2420
static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
{
2421 2422
	struct vcpu_svm *svm = to_svm(vcpu);

A
Avi Kivity 已提交
2423
	switch (ecx) {
2424
	case MSR_IA32_TSC: {
2425
		u64 tsc_offset;
A
Avi Kivity 已提交
2426

2427 2428 2429 2430 2431 2432
		if (is_nested(svm))
			tsc_offset = svm->nested.hsave->control.tsc_offset;
		else
			tsc_offset = svm->vmcb->control.tsc_offset;

		*data = tsc_offset + native_read_tsc();
A
Avi Kivity 已提交
2433 2434
		break;
	}
B
Brian Gerst 已提交
2435
	case MSR_STAR:
2436
		*data = svm->vmcb->save.star;
A
Avi Kivity 已提交
2437
		break;
2438
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2439
	case MSR_LSTAR:
2440
		*data = svm->vmcb->save.lstar;
A
Avi Kivity 已提交
2441 2442
		break;
	case MSR_CSTAR:
2443
		*data = svm->vmcb->save.cstar;
A
Avi Kivity 已提交
2444 2445
		break;
	case MSR_KERNEL_GS_BASE:
2446
		*data = svm->vmcb->save.kernel_gs_base;
A
Avi Kivity 已提交
2447 2448
		break;
	case MSR_SYSCALL_MASK:
2449
		*data = svm->vmcb->save.sfmask;
A
Avi Kivity 已提交
2450 2451 2452
		break;
#endif
	case MSR_IA32_SYSENTER_CS:
2453
		*data = svm->vmcb->save.sysenter_cs;
A
Avi Kivity 已提交
2454 2455
		break;
	case MSR_IA32_SYSENTER_EIP:
2456
		*data = svm->sysenter_eip;
A
Avi Kivity 已提交
2457 2458
		break;
	case MSR_IA32_SYSENTER_ESP:
2459
		*data = svm->sysenter_esp;
A
Avi Kivity 已提交
2460
		break;
J
Joerg Roedel 已提交
2461 2462 2463 2464 2465
	/*
	 * Nobody will change the following 5 values in the VMCB so we can
	 * safely return them on rdmsr. They will always be 0 until LBRV is
	 * implemented.
	 */
2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480
	case MSR_IA32_DEBUGCTLMSR:
		*data = svm->vmcb->save.dbgctl;
		break;
	case MSR_IA32_LASTBRANCHFROMIP:
		*data = svm->vmcb->save.br_from;
		break;
	case MSR_IA32_LASTBRANCHTOIP:
		*data = svm->vmcb->save.br_to;
		break;
	case MSR_IA32_LASTINTFROMIP:
		*data = svm->vmcb->save.last_excp_from;
		break;
	case MSR_IA32_LASTINTTOIP:
		*data = svm->vmcb->save.last_excp_to;
		break;
A
Alexander Graf 已提交
2481
	case MSR_VM_HSAVE_PA:
2482
		*data = svm->nested.hsave_msr;
A
Alexander Graf 已提交
2483
		break;
2484
	case MSR_VM_CR:
2485
		*data = svm->nested.vm_cr_msr;
2486
		break;
2487 2488 2489
	case MSR_IA32_UCODE_REV:
		*data = 0x01000065;
		break;
A
Avi Kivity 已提交
2490
	default:
2491
		return kvm_get_msr_common(vcpu, ecx, data);
A
Avi Kivity 已提交
2492 2493 2494 2495
	}
	return 0;
}

A
Avi Kivity 已提交
2496
static int rdmsr_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2497
{
2498
	u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
A
Avi Kivity 已提交
2499 2500
	u64 data;

2501 2502
	if (svm_get_msr(&svm->vcpu, ecx, &data)) {
		trace_kvm_msr_read_ex(ecx);
2503
		kvm_inject_gp(&svm->vcpu, 0);
2504
	} else {
2505
		trace_kvm_msr_read(ecx, data);
2506

2507
		svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff;
2508
		svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;
2509
		svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
R
Rusty Russell 已提交
2510
		skip_emulated_instruction(&svm->vcpu);
A
Avi Kivity 已提交
2511 2512 2513 2514
	}
	return 1;
}

2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539
static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	int svm_dis, chg_mask;

	if (data & ~SVM_VM_CR_VALID_MASK)
		return 1;

	chg_mask = SVM_VM_CR_VALID_MASK;

	if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
		chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);

	svm->nested.vm_cr_msr &= ~chg_mask;
	svm->nested.vm_cr_msr |= (data & chg_mask);

	svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;

	/* check for svm_disable while efer.svme is set */
	if (svm_dis && (vcpu->arch.efer & EFER_SVME))
		return 1;

	return 0;
}

A
Avi Kivity 已提交
2540 2541
static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
{
2542 2543
	struct vcpu_svm *svm = to_svm(vcpu);

A
Avi Kivity 已提交
2544
	switch (ecx) {
2545
	case MSR_IA32_TSC: {
2546 2547 2548 2549 2550 2551 2552 2553 2554 2555
		u64 tsc_offset = data - native_read_tsc();
		u64 g_tsc_offset = 0;

		if (is_nested(svm)) {
			g_tsc_offset = svm->vmcb->control.tsc_offset -
				       svm->nested.hsave->control.tsc_offset;
			svm->nested.hsave->control.tsc_offset = tsc_offset;
		}

		svm->vmcb->control.tsc_offset = tsc_offset + g_tsc_offset;
A
Avi Kivity 已提交
2556 2557 2558

		break;
	}
B
Brian Gerst 已提交
2559
	case MSR_STAR:
2560
		svm->vmcb->save.star = data;
A
Avi Kivity 已提交
2561
		break;
2562
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
2563
	case MSR_LSTAR:
2564
		svm->vmcb->save.lstar = data;
A
Avi Kivity 已提交
2565 2566
		break;
	case MSR_CSTAR:
2567
		svm->vmcb->save.cstar = data;
A
Avi Kivity 已提交
2568 2569
		break;
	case MSR_KERNEL_GS_BASE:
2570
		svm->vmcb->save.kernel_gs_base = data;
A
Avi Kivity 已提交
2571 2572
		break;
	case MSR_SYSCALL_MASK:
2573
		svm->vmcb->save.sfmask = data;
A
Avi Kivity 已提交
2574 2575 2576
		break;
#endif
	case MSR_IA32_SYSENTER_CS:
2577
		svm->vmcb->save.sysenter_cs = data;
A
Avi Kivity 已提交
2578 2579
		break;
	case MSR_IA32_SYSENTER_EIP:
2580
		svm->sysenter_eip = data;
2581
		svm->vmcb->save.sysenter_eip = data;
A
Avi Kivity 已提交
2582 2583
		break;
	case MSR_IA32_SYSENTER_ESP:
2584
		svm->sysenter_esp = data;
2585
		svm->vmcb->save.sysenter_esp = data;
A
Avi Kivity 已提交
2586
		break;
2587
	case MSR_IA32_DEBUGCTLMSR:
2588 2589
		if (!svm_has(SVM_FEATURE_LBRV)) {
			pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
2590
					__func__, data);
2591 2592 2593 2594 2595 2596 2597 2598 2599 2600
			break;
		}
		if (data & DEBUGCTL_RESERVED_BITS)
			return 1;

		svm->vmcb->save.dbgctl = data;
		if (data & (1ULL<<0))
			svm_enable_lbrv(svm);
		else
			svm_disable_lbrv(svm);
2601
		break;
A
Alexander Graf 已提交
2602
	case MSR_VM_HSAVE_PA:
2603
		svm->nested.hsave_msr = data;
2604
		break;
2605
	case MSR_VM_CR:
2606
		return svm_set_vm_cr(vcpu, data);
2607 2608 2609
	case MSR_VM_IGNNE:
		pr_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
		break;
A
Avi Kivity 已提交
2610
	default:
2611
		return kvm_set_msr_common(vcpu, ecx, data);
A
Avi Kivity 已提交
2612 2613 2614 2615
	}
	return 0;
}

A
Avi Kivity 已提交
2616
static int wrmsr_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2617
{
2618
	u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2619
	u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u)
2620
		| ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
2621 2622


2623
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
2624 2625
	if (svm_set_msr(&svm->vcpu, ecx, data)) {
		trace_kvm_msr_write_ex(ecx, data);
2626
		kvm_inject_gp(&svm->vcpu, 0);
2627 2628
	} else {
		trace_kvm_msr_write(ecx, data);
R
Rusty Russell 已提交
2629
		skip_emulated_instruction(&svm->vcpu);
2630
	}
A
Avi Kivity 已提交
2631 2632 2633
	return 1;
}

A
Avi Kivity 已提交
2634
static int msr_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2635
{
R
Rusty Russell 已提交
2636
	if (svm->vmcb->control.exit_info_1)
A
Avi Kivity 已提交
2637
		return wrmsr_interception(svm);
A
Avi Kivity 已提交
2638
	else
A
Avi Kivity 已提交
2639
		return rdmsr_interception(svm);
A
Avi Kivity 已提交
2640 2641
}

A
Avi Kivity 已提交
2642
static int interrupt_window_interception(struct vcpu_svm *svm)
2643
{
A
Avi Kivity 已提交
2644 2645
	struct kvm_run *kvm_run = svm->vcpu.run;

2646
	svm_clear_vintr(svm);
2647
	svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2648 2649 2650 2651
	/*
	 * If the user space waits to inject interrupts, exit as soon as
	 * possible
	 */
2652 2653 2654
	if (!irqchip_in_kernel(svm->vcpu.kvm) &&
	    kvm_run->request_interrupt_window &&
	    !kvm_cpu_has_interrupt(&svm->vcpu)) {
R
Rusty Russell 已提交
2655
		++svm->vcpu.stat.irq_window_exits;
2656 2657 2658 2659 2660 2661 2662
		kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
		return 0;
	}

	return 1;
}

2663 2664 2665 2666 2667 2668
static int pause_interception(struct vcpu_svm *svm)
{
	kvm_vcpu_on_spin(&(svm->vcpu));
	return 1;
}

A
Avi Kivity 已提交
2669
static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
J
Joerg Roedel 已提交
2670 2671 2672 2673
	[SVM_EXIT_READ_CR0]			= emulate_on_interception,
	[SVM_EXIT_READ_CR3]			= emulate_on_interception,
	[SVM_EXIT_READ_CR4]			= emulate_on_interception,
	[SVM_EXIT_READ_CR8]			= emulate_on_interception,
A
Avi Kivity 已提交
2674
	[SVM_EXIT_CR0_SEL_WRITE]		= emulate_on_interception,
J
Joerg Roedel 已提交
2675 2676 2677 2678 2679
	[SVM_EXIT_WRITE_CR0]			= emulate_on_interception,
	[SVM_EXIT_WRITE_CR3]			= emulate_on_interception,
	[SVM_EXIT_WRITE_CR4]			= emulate_on_interception,
	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
	[SVM_EXIT_READ_DR0]			= emulate_on_interception,
A
Avi Kivity 已提交
2680 2681 2682
	[SVM_EXIT_READ_DR1]			= emulate_on_interception,
	[SVM_EXIT_READ_DR2]			= emulate_on_interception,
	[SVM_EXIT_READ_DR3]			= emulate_on_interception,
2683 2684 2685 2686
	[SVM_EXIT_READ_DR4]			= emulate_on_interception,
	[SVM_EXIT_READ_DR5]			= emulate_on_interception,
	[SVM_EXIT_READ_DR6]			= emulate_on_interception,
	[SVM_EXIT_READ_DR7]			= emulate_on_interception,
A
Avi Kivity 已提交
2687 2688 2689 2690
	[SVM_EXIT_WRITE_DR0]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR1]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR2]			= emulate_on_interception,
	[SVM_EXIT_WRITE_DR3]			= emulate_on_interception,
2691
	[SVM_EXIT_WRITE_DR4]			= emulate_on_interception,
A
Avi Kivity 已提交
2692
	[SVM_EXIT_WRITE_DR5]			= emulate_on_interception,
2693
	[SVM_EXIT_WRITE_DR6]			= emulate_on_interception,
A
Avi Kivity 已提交
2694
	[SVM_EXIT_WRITE_DR7]			= emulate_on_interception,
J
Jan Kiszka 已提交
2695 2696
	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
2697
	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
J
Joerg Roedel 已提交
2698 2699 2700 2701
	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
	[SVM_EXIT_EXCP_BASE + NM_VECTOR]	= nm_interception,
	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
	[SVM_EXIT_INTR]				= intr_interception,
2702
	[SVM_EXIT_NMI]				= nmi_interception,
A
Avi Kivity 已提交
2703 2704
	[SVM_EXIT_SMI]				= nop_on_interception,
	[SVM_EXIT_INIT]				= nop_on_interception,
2705
	[SVM_EXIT_VINTR]			= interrupt_window_interception,
A
Avi Kivity 已提交
2706
	[SVM_EXIT_CPUID]			= cpuid_interception,
2707
	[SVM_EXIT_IRET]                         = iret_interception,
2708
	[SVM_EXIT_INVD]                         = emulate_on_interception,
2709
	[SVM_EXIT_PAUSE]			= pause_interception,
A
Avi Kivity 已提交
2710
	[SVM_EXIT_HLT]				= halt_interception,
M
Marcelo Tosatti 已提交
2711
	[SVM_EXIT_INVLPG]			= invlpg_interception,
A
Alexander Graf 已提交
2712
	[SVM_EXIT_INVLPGA]			= invlpga_interception,
J
Joerg Roedel 已提交
2713
	[SVM_EXIT_IOIO]				= io_interception,
A
Avi Kivity 已提交
2714 2715
	[SVM_EXIT_MSR]				= msr_interception,
	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
2716
	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
A
Alexander Graf 已提交
2717
	[SVM_EXIT_VMRUN]			= vmrun_interception,
2718
	[SVM_EXIT_VMMCALL]			= vmmcall_interception,
2719 2720
	[SVM_EXIT_VMLOAD]			= vmload_interception,
	[SVM_EXIT_VMSAVE]			= vmsave_interception,
2721 2722
	[SVM_EXIT_STGI]				= stgi_interception,
	[SVM_EXIT_CLGI]				= clgi_interception,
2723
	[SVM_EXIT_SKINIT]			= skinit_interception,
2724
	[SVM_EXIT_WBINVD]                       = emulate_on_interception,
2725 2726
	[SVM_EXIT_MONITOR]			= invalid_op_interception,
	[SVM_EXIT_MWAIT]			= invalid_op_interception,
2727
	[SVM_EXIT_NPF]				= pf_interception,
A
Avi Kivity 已提交
2728 2729
};

2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822
void dump_vmcb(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb_control_area *control = &svm->vmcb->control;
	struct vmcb_save_area *save = &svm->vmcb->save;

	pr_err("VMCB Control Area:\n");
	pr_err("cr_read:            %04x\n", control->intercept_cr_read);
	pr_err("cr_write:           %04x\n", control->intercept_cr_write);
	pr_err("dr_read:            %04x\n", control->intercept_dr_read);
	pr_err("dr_write:           %04x\n", control->intercept_dr_write);
	pr_err("exceptions:         %08x\n", control->intercept_exceptions);
	pr_err("intercepts:         %016llx\n", control->intercept);
	pr_err("pause filter count: %d\n", control->pause_filter_count);
	pr_err("iopm_base_pa:       %016llx\n", control->iopm_base_pa);
	pr_err("msrpm_base_pa:      %016llx\n", control->msrpm_base_pa);
	pr_err("tsc_offset:         %016llx\n", control->tsc_offset);
	pr_err("asid:               %d\n", control->asid);
	pr_err("tlb_ctl:            %d\n", control->tlb_ctl);
	pr_err("int_ctl:            %08x\n", control->int_ctl);
	pr_err("int_vector:         %08x\n", control->int_vector);
	pr_err("int_state:          %08x\n", control->int_state);
	pr_err("exit_code:          %08x\n", control->exit_code);
	pr_err("exit_info1:         %016llx\n", control->exit_info_1);
	pr_err("exit_info2:         %016llx\n", control->exit_info_2);
	pr_err("exit_int_info:      %08x\n", control->exit_int_info);
	pr_err("exit_int_info_err:  %08x\n", control->exit_int_info_err);
	pr_err("nested_ctl:         %lld\n", control->nested_ctl);
	pr_err("nested_cr3:         %016llx\n", control->nested_cr3);
	pr_err("event_inj:          %08x\n", control->event_inj);
	pr_err("event_inj_err:      %08x\n", control->event_inj_err);
	pr_err("lbr_ctl:            %lld\n", control->lbr_ctl);
	pr_err("next_rip:           %016llx\n", control->next_rip);
	pr_err("VMCB State Save Area:\n");
	pr_err("es:   s: %04x a: %04x l: %08x b: %016llx\n",
		save->es.selector, save->es.attrib,
		save->es.limit, save->es.base);
	pr_err("cs:   s: %04x a: %04x l: %08x b: %016llx\n",
		save->cs.selector, save->cs.attrib,
		save->cs.limit, save->cs.base);
	pr_err("ss:   s: %04x a: %04x l: %08x b: %016llx\n",
		save->ss.selector, save->ss.attrib,
		save->ss.limit, save->ss.base);
	pr_err("ds:   s: %04x a: %04x l: %08x b: %016llx\n",
		save->ds.selector, save->ds.attrib,
		save->ds.limit, save->ds.base);
	pr_err("fs:   s: %04x a: %04x l: %08x b: %016llx\n",
		save->fs.selector, save->fs.attrib,
		save->fs.limit, save->fs.base);
	pr_err("gs:   s: %04x a: %04x l: %08x b: %016llx\n",
		save->gs.selector, save->gs.attrib,
		save->gs.limit, save->gs.base);
	pr_err("gdtr: s: %04x a: %04x l: %08x b: %016llx\n",
		save->gdtr.selector, save->gdtr.attrib,
		save->gdtr.limit, save->gdtr.base);
	pr_err("ldtr: s: %04x a: %04x l: %08x b: %016llx\n",
		save->ldtr.selector, save->ldtr.attrib,
		save->ldtr.limit, save->ldtr.base);
	pr_err("idtr: s: %04x a: %04x l: %08x b: %016llx\n",
		save->idtr.selector, save->idtr.attrib,
		save->idtr.limit, save->idtr.base);
	pr_err("tr:   s: %04x a: %04x l: %08x b: %016llx\n",
		save->tr.selector, save->tr.attrib,
		save->tr.limit, save->tr.base);
	pr_err("cpl:            %d                efer:         %016llx\n",
		save->cpl, save->efer);
	pr_err("cr0:            %016llx cr2:          %016llx\n",
		save->cr0, save->cr2);
	pr_err("cr3:            %016llx cr4:          %016llx\n",
		save->cr3, save->cr4);
	pr_err("dr6:            %016llx dr7:          %016llx\n",
		save->dr6, save->dr7);
	pr_err("rip:            %016llx rflags:       %016llx\n",
		save->rip, save->rflags);
	pr_err("rsp:            %016llx rax:          %016llx\n",
		save->rsp, save->rax);
	pr_err("star:           %016llx lstar:        %016llx\n",
		save->star, save->lstar);
	pr_err("cstar:          %016llx sfmask:       %016llx\n",
		save->cstar, save->sfmask);
	pr_err("kernel_gs_base: %016llx sysenter_cs:  %016llx\n",
		save->kernel_gs_base, save->sysenter_cs);
	pr_err("sysenter_esp:   %016llx sysenter_eip: %016llx\n",
		save->sysenter_esp, save->sysenter_eip);
	pr_err("gpat:           %016llx dbgctl:       %016llx\n",
		save->g_pat, save->dbgctl);
	pr_err("br_from:        %016llx br_to:        %016llx\n",
		save->br_from, save->br_to);
	pr_err("excp_from:      %016llx excp_to:      %016llx\n",
		save->last_excp_from, save->last_excp_to);

}

A
Avi Kivity 已提交
2823
static int handle_exit(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2824
{
2825
	struct vcpu_svm *svm = to_svm(vcpu);
A
Avi Kivity 已提交
2826
	struct kvm_run *kvm_run = vcpu->run;
2827
	u32 exit_code = svm->vmcb->control.exit_code;
A
Avi Kivity 已提交
2828

2829
	trace_kvm_exit(exit_code, vcpu);
2830

2831 2832 2833 2834
	if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR0_MASK))
		vcpu->arch.cr0 = svm->vmcb->save.cr0;
	if (npt_enabled)
		vcpu->arch.cr3 = svm->vmcb->save.cr3;
2835

2836 2837 2838 2839 2840 2841 2842
	if (unlikely(svm->nested.exit_required)) {
		nested_svm_vmexit(svm);
		svm->nested.exit_required = false;

		return 1;
	}

2843
	if (is_nested(svm)) {
2844 2845
		int vmexit;

2846 2847 2848 2849 2850 2851
		trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
					svm->vmcb->control.exit_info_1,
					svm->vmcb->control.exit_info_2,
					svm->vmcb->control.exit_int_info,
					svm->vmcb->control.exit_int_info_err);

2852 2853 2854 2855 2856 2857
		vmexit = nested_svm_exit_special(svm);

		if (vmexit == NESTED_EXIT_CONTINUE)
			vmexit = nested_svm_exit_handled(svm);

		if (vmexit == NESTED_EXIT_DONE)
2858 2859 2860
			return 1;
	}

2861 2862
	svm_complete_interrupts(svm);

2863 2864 2865 2866
	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		kvm_run->fail_entry.hardware_entry_failure_reason
			= svm->vmcb->control.exit_code;
2867 2868
		pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
		dump_vmcb(vcpu);
2869 2870 2871
		return 0;
	}

2872
	if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
2873
	    exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
2874
	    exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH)
A
Avi Kivity 已提交
2875 2876
		printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
		       "exit_code 0x%x\n",
2877
		       __func__, svm->vmcb->control.exit_int_info,
A
Avi Kivity 已提交
2878 2879
		       exit_code);

2880
	if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
J
Joe Perches 已提交
2881
	    || !svm_exit_handlers[exit_code]) {
A
Avi Kivity 已提交
2882
		kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
2883
		kvm_run->hw.hardware_exit_reason = exit_code;
A
Avi Kivity 已提交
2884 2885 2886
		return 0;
	}

A
Avi Kivity 已提交
2887
	return svm_exit_handlers[exit_code](svm);
A
Avi Kivity 已提交
2888 2889 2890 2891 2892 2893
}

static void reload_tss(struct kvm_vcpu *vcpu)
{
	int cpu = raw_smp_processor_id();

2894 2895
	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
	sd->tss_desc->type = 9; /* available 32/64-bit TSS */
A
Avi Kivity 已提交
2896 2897 2898
	load_TR_desc();
}

R
Rusty Russell 已提交
2899
static void pre_svm_run(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2900 2901 2902
{
	int cpu = raw_smp_processor_id();

2903
	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
A
Avi Kivity 已提交
2904

2905
	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
2906
	/* FIXME: handle wraparound of asid_generation */
2907 2908
	if (svm->asid_generation != sd->asid_generation)
		new_asid(svm, sd);
A
Avi Kivity 已提交
2909 2910
}

2911 2912 2913 2914 2915 2916
static void svm_inject_nmi(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
	vcpu->arch.hflags |= HF_NMI_MASK;
2917
	svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
2918 2919
	++vcpu->stat.nmi_injections;
}
A
Avi Kivity 已提交
2920

2921
static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
A
Avi Kivity 已提交
2922 2923 2924
{
	struct vmcb_control_area *control;

R
Rusty Russell 已提交
2925
	control = &svm->vmcb->control;
2926
	control->int_vector = irq;
A
Avi Kivity 已提交
2927 2928 2929 2930 2931
	control->int_ctl &= ~V_INTR_PRIO_MASK;
	control->int_ctl |= V_IRQ_MASK |
		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
}

2932
static void svm_set_irq(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
2933 2934 2935
{
	struct vcpu_svm *svm = to_svm(vcpu);

2936
	BUG_ON(!(gif_set(svm)));
2937

2938 2939 2940
	trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
	++vcpu->stat.irq_injections;

2941 2942
	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
		SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
E
Eddie Dong 已提交
2943 2944
}

2945
static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
2946 2947 2948
{
	struct vcpu_svm *svm = to_svm(vcpu);

2949 2950 2951
	if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
		return;

2952
	if (irr == -1)
2953 2954
		return;

2955 2956 2957
	if (tpr >= irr)
		svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK;
}
2958

2959 2960 2961 2962
static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb *vmcb = svm->vmcb;
J
Joerg Roedel 已提交
2963 2964 2965 2966 2967 2968
	int ret;
	ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
	      !(svm->vcpu.arch.hflags & HF_NMI_MASK);
	ret = ret && gif_set(svm) && nested_svm_nmi(svm);

	return ret;
2969 2970
}

J
Jan Kiszka 已提交
2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983
static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
}

static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (masked) {
		svm->vcpu.arch.hflags |= HF_NMI_MASK;
2984
		svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
J
Jan Kiszka 已提交
2985 2986
	} else {
		svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
2987
		svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
J
Jan Kiszka 已提交
2988 2989 2990
	}
}

2991 2992 2993 2994
static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb *vmcb = svm->vmcb;
2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006
	int ret;

	if (!gif_set(svm) ||
	     (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
		return 0;

	ret = !!(vmcb->save.rflags & X86_EFLAGS_IF);

	if (is_nested(svm))
		return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);

	return ret;
3007 3008
}

3009
static void enable_irq_window(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3010
{
3011 3012
	struct vcpu_svm *svm = to_svm(vcpu);

J
Joerg Roedel 已提交
3013 3014 3015 3016 3017 3018
	/*
	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
	 * get that intercept, this function will be called again though and
	 * we'll get the vintr intercept.
	 */
3019
	if (gif_set(svm) && nested_svm_intr(svm)) {
3020 3021 3022
		svm_set_vintr(svm);
		svm_inject_irq(svm, 0x0);
	}
3023 3024
}

3025
static void enable_nmi_window(struct kvm_vcpu *vcpu)
3026
{
3027
	struct vcpu_svm *svm = to_svm(vcpu);
3028

3029 3030 3031 3032
	if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
	    == HF_NMI_MASK)
		return; /* IRET will cause a vm exit */

J
Joerg Roedel 已提交
3033 3034 3035 3036
	/*
	 * Something prevents NMI from been injected. Single step over possible
	 * problem (IRET or exception injection or interrupt shadow)
	 */
J
Jan Kiszka 已提交
3037
	svm->nmi_singlestep = true;
3038 3039
	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
	update_db_intercept(vcpu);
3040 3041
}

3042 3043 3044 3045 3046
static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
{
	return 0;
}

3047 3048 3049 3050 3051
static void svm_flush_tlb(struct kvm_vcpu *vcpu)
{
	force_new_asid(vcpu);
}

3052 3053 3054 3055
static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
{
}

3056 3057 3058 3059
static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

3060 3061 3062
	if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
		return;

3063 3064
	if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) {
		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3065
		kvm_set_cr8(vcpu, cr8);
3066 3067 3068
	}
}

3069 3070 3071 3072 3073
static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u64 cr8;

3074 3075 3076
	if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
		return;

3077 3078 3079 3080 3081
	cr8 = kvm_get_cr8(vcpu);
	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
}

3082 3083 3084 3085 3086
static void svm_complete_interrupts(struct vcpu_svm *svm)
{
	u8 vector;
	int type;
	u32 exitintinfo = svm->vmcb->control.exit_int_info;
3087 3088 3089
	unsigned int3_injected = svm->int3_injected;

	svm->int3_injected = 0;
3090

3091 3092 3093
	if (svm->vcpu.arch.hflags & HF_IRET_MASK)
		svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);

3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108
	svm->vcpu.arch.nmi_injected = false;
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);

	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
		return;

	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;

	switch (type) {
	case SVM_EXITINTINFO_TYPE_NMI:
		svm->vcpu.arch.nmi_injected = true;
		break;
	case SVM_EXITINTINFO_TYPE_EXEPT:
3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119
		/*
		 * In case of software exceptions, do not reinject the vector,
		 * but re-execute the instruction instead. Rewind RIP first
		 * if we emulated INT3 before.
		 */
		if (kvm_exception_is_soft(vector)) {
			if (vector == BP_VECTOR && int3_injected &&
			    kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
				kvm_rip_write(&svm->vcpu,
					      kvm_rip_read(&svm->vcpu) -
					      int3_injected);
3120
			break;
3121
		}
3122 3123
		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
			u32 err = svm->vmcb->control.exit_int_info_err;
3124
			kvm_requeue_exception_e(&svm->vcpu, vector, err);
3125 3126

		} else
3127
			kvm_requeue_exception(&svm->vcpu, vector);
3128 3129
		break;
	case SVM_EXITINTINFO_TYPE_INTR:
3130
		kvm_queue_interrupt(&svm->vcpu, vector, false);
3131 3132 3133 3134 3135 3136
		break;
	default:
		break;
	}
}

3137 3138 3139 3140 3141 3142
#ifdef CONFIG_X86_64
#define R "r"
#else
#define R "e"
#endif

A
Avi Kivity 已提交
3143
static void svm_vcpu_run(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3144
{
3145
	struct vcpu_svm *svm = to_svm(vcpu);
A
Avi Kivity 已提交
3146 3147 3148
	u16 fs_selector;
	u16 gs_selector;
	u16 ldt_selector;
3149

3150 3151 3152 3153
	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];

3154 3155 3156 3157 3158 3159 3160
	/*
	 * A vmexit emulation is required before the vcpu can be executed
	 * again.
	 */
	if (unlikely(svm->nested.exit_required))
		return;

R
Rusty Russell 已提交
3161
	pre_svm_run(svm);
A
Avi Kivity 已提交
3162

3163 3164
	sync_lapic_to_cr8(vcpu);

A
Avi Kivity 已提交
3165
	save_host_msrs(vcpu);
3166 3167 3168
	fs_selector = kvm_read_fs();
	gs_selector = kvm_read_gs();
	ldt_selector = kvm_read_ldt();
3169
	svm->vmcb->save.cr2 = vcpu->arch.cr2;
3170 3171 3172
	/* required for live migration with NPT */
	if (npt_enabled)
		svm->vmcb->save.cr3 = vcpu->arch.cr3;
A
Avi Kivity 已提交
3173

3174 3175 3176
	clgi();

	local_irq_enable();
3177

A
Avi Kivity 已提交
3178
	asm volatile (
3179 3180 3181 3182 3183 3184 3185
		"push %%"R"bp; \n\t"
		"mov %c[rbx](%[svm]), %%"R"bx \n\t"
		"mov %c[rcx](%[svm]), %%"R"cx \n\t"
		"mov %c[rdx](%[svm]), %%"R"dx \n\t"
		"mov %c[rsi](%[svm]), %%"R"si \n\t"
		"mov %c[rdi](%[svm]), %%"R"di \n\t"
		"mov %c[rbp](%[svm]), %%"R"bp \n\t"
3186
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
3187 3188 3189 3190 3191 3192 3193 3194
		"mov %c[r8](%[svm]),  %%r8  \n\t"
		"mov %c[r9](%[svm]),  %%r9  \n\t"
		"mov %c[r10](%[svm]), %%r10 \n\t"
		"mov %c[r11](%[svm]), %%r11 \n\t"
		"mov %c[r12](%[svm]), %%r12 \n\t"
		"mov %c[r13](%[svm]), %%r13 \n\t"
		"mov %c[r14](%[svm]), %%r14 \n\t"
		"mov %c[r15](%[svm]), %%r15 \n\t"
A
Avi Kivity 已提交
3195 3196 3197
#endif

		/* Enter guest mode */
3198 3199
		"push %%"R"ax \n\t"
		"mov %c[vmcb](%[svm]), %%"R"ax \n\t"
3200 3201 3202
		__ex(SVM_VMLOAD) "\n\t"
		__ex(SVM_VMRUN) "\n\t"
		__ex(SVM_VMSAVE) "\n\t"
3203
		"pop %%"R"ax \n\t"
A
Avi Kivity 已提交
3204 3205

		/* Save guest registers, load host registers */
3206 3207 3208 3209 3210 3211
		"mov %%"R"bx, %c[rbx](%[svm]) \n\t"
		"mov %%"R"cx, %c[rcx](%[svm]) \n\t"
		"mov %%"R"dx, %c[rdx](%[svm]) \n\t"
		"mov %%"R"si, %c[rsi](%[svm]) \n\t"
		"mov %%"R"di, %c[rdi](%[svm]) \n\t"
		"mov %%"R"bp, %c[rbp](%[svm]) \n\t"
3212
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
3213 3214 3215 3216 3217 3218 3219 3220
		"mov %%r8,  %c[r8](%[svm]) \n\t"
		"mov %%r9,  %c[r9](%[svm]) \n\t"
		"mov %%r10, %c[r10](%[svm]) \n\t"
		"mov %%r11, %c[r11](%[svm]) \n\t"
		"mov %%r12, %c[r12](%[svm]) \n\t"
		"mov %%r13, %c[r13](%[svm]) \n\t"
		"mov %%r14, %c[r14](%[svm]) \n\t"
		"mov %%r15, %c[r15](%[svm]) \n\t"
A
Avi Kivity 已提交
3221
#endif
3222
		"pop %%"R"bp"
A
Avi Kivity 已提交
3223
		:
R
Rusty Russell 已提交
3224
		: [svm]"a"(svm),
A
Avi Kivity 已提交
3225
		  [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
3226 3227 3228 3229 3230 3231
		  [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
		  [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
		  [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
		  [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
		  [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
		  [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
3232
#ifdef CONFIG_X86_64
3233 3234 3235 3236 3237 3238 3239 3240
		  , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
		  [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
		  [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
		  [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
		  [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
		  [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
		  [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
		  [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
A
Avi Kivity 已提交
3241
#endif
3242
		: "cc", "memory"
3243
		, R"bx", R"cx", R"dx", R"si", R"di"
3244 3245 3246 3247
#ifdef CONFIG_X86_64
		, "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
#endif
		);
A
Avi Kivity 已提交
3248

3249
	vcpu->arch.cr2 = svm->vmcb->save.cr2;
3250 3251 3252
	vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
	vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
	vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
A
Avi Kivity 已提交
3253

3254 3255 3256
	kvm_load_fs(fs_selector);
	kvm_load_gs(gs_selector);
	kvm_load_ldt(ldt_selector);
A
Avi Kivity 已提交
3257 3258 3259 3260
	load_host_msrs(vcpu);

	reload_tss(vcpu);

3261 3262 3263 3264
	local_irq_disable();

	stgi();

3265 3266
	sync_cr8_to_lapic(vcpu);

3267
	svm->next_rip = 0;
3268

A
Avi Kivity 已提交
3269 3270 3271 3272
	if (npt_enabled) {
		vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
		vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
	}
3273 3274 3275 3276 3277 3278 3279 3280

	/*
	 * We need to handle MC intercepts here before the vcpu has a chance to
	 * change the physical cpu
	 */
	if (unlikely(svm->vmcb->control.exit_code ==
		     SVM_EXIT_EXCP_BASE + MC_VECTOR))
		svm_handle_mce(svm);
A
Avi Kivity 已提交
3281 3282
}

3283 3284
#undef R

A
Avi Kivity 已提交
3285 3286
static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
{
3287 3288
	struct vcpu_svm *svm = to_svm(vcpu);

3289 3290 3291 3292 3293 3294
	if (npt_enabled) {
		svm->vmcb->control.nested_cr3 = root;
		force_new_asid(vcpu);
		return;
	}

3295
	svm->vmcb->save.cr3 = root;
A
Avi Kivity 已提交
3296 3297 3298 3299 3300
	force_new_asid(vcpu);
}

static int is_disabled(void)
{
3301 3302 3303 3304 3305 3306
	u64 vm_cr;

	rdmsrl(MSR_VM_CR, vm_cr);
	if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
		return 1;

A
Avi Kivity 已提交
3307 3308 3309
	return 0;
}

I
Ingo Molnar 已提交
3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320
static void
svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
{
	/*
	 * Patch in the VMMCALL instruction:
	 */
	hypercall[0] = 0x0f;
	hypercall[1] = 0x01;
	hypercall[2] = 0xd9;
}

Y
Yang, Sheng 已提交
3321 3322 3323 3324 3325
static void svm_check_processor_compat(void *rtn)
{
	*(int *)rtn = 0;
}

3326 3327 3328 3329 3330
static bool svm_cpu_has_accelerated_tpr(void)
{
	return false;
}

3331 3332 3333 3334 3335 3336 3337 3338 3339
static int get_npt_level(void)
{
#ifdef CONFIG_X86_64
	return PT64_ROOT_LEVEL;
#else
	return PT32E_ROOT_LEVEL;
#endif
}

3340
static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
S
Sheng Yang 已提交
3341 3342 3343 3344
{
	return 0;
}

3345 3346 3347 3348
static void svm_cpuid_update(struct kvm_vcpu *vcpu)
{
}

3349 3350
static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
{
3351 3352 3353 3354 3355 3356 3357 3358 3359 3360
	switch (func) {
	case 0x8000000A:
		entry->eax = 1; /* SVM revision 1 */
		entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
				   ASID emulation to nested SVM */
		entry->ecx = 0; /* Reserved */
		entry->edx = 0; /* Do not support any additional features */

		break;
	}
3361 3362
}

3363
static const struct trace_print_flags svm_exit_reasons_str[] = {
J
Joerg Roedel 已提交
3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381
	{ SVM_EXIT_READ_CR0,			"read_cr0" },
	{ SVM_EXIT_READ_CR3,			"read_cr3" },
	{ SVM_EXIT_READ_CR4,			"read_cr4" },
	{ SVM_EXIT_READ_CR8,			"read_cr8" },
	{ SVM_EXIT_WRITE_CR0,			"write_cr0" },
	{ SVM_EXIT_WRITE_CR3,			"write_cr3" },
	{ SVM_EXIT_WRITE_CR4,			"write_cr4" },
	{ SVM_EXIT_WRITE_CR8,			"write_cr8" },
	{ SVM_EXIT_READ_DR0,			"read_dr0" },
	{ SVM_EXIT_READ_DR1,			"read_dr1" },
	{ SVM_EXIT_READ_DR2,			"read_dr2" },
	{ SVM_EXIT_READ_DR3,			"read_dr3" },
	{ SVM_EXIT_WRITE_DR0,			"write_dr0" },
	{ SVM_EXIT_WRITE_DR1,			"write_dr1" },
	{ SVM_EXIT_WRITE_DR2,			"write_dr2" },
	{ SVM_EXIT_WRITE_DR3,			"write_dr3" },
	{ SVM_EXIT_WRITE_DR5,			"write_dr5" },
	{ SVM_EXIT_WRITE_DR7,			"write_dr7" },
3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415
	{ SVM_EXIT_EXCP_BASE + DB_VECTOR,	"DB excp" },
	{ SVM_EXIT_EXCP_BASE + BP_VECTOR,	"BP excp" },
	{ SVM_EXIT_EXCP_BASE + UD_VECTOR,	"UD excp" },
	{ SVM_EXIT_EXCP_BASE + PF_VECTOR,	"PF excp" },
	{ SVM_EXIT_EXCP_BASE + NM_VECTOR,	"NM excp" },
	{ SVM_EXIT_EXCP_BASE + MC_VECTOR,	"MC excp" },
	{ SVM_EXIT_INTR,			"interrupt" },
	{ SVM_EXIT_NMI,				"nmi" },
	{ SVM_EXIT_SMI,				"smi" },
	{ SVM_EXIT_INIT,			"init" },
	{ SVM_EXIT_VINTR,			"vintr" },
	{ SVM_EXIT_CPUID,			"cpuid" },
	{ SVM_EXIT_INVD,			"invd" },
	{ SVM_EXIT_HLT,				"hlt" },
	{ SVM_EXIT_INVLPG,			"invlpg" },
	{ SVM_EXIT_INVLPGA,			"invlpga" },
	{ SVM_EXIT_IOIO,			"io" },
	{ SVM_EXIT_MSR,				"msr" },
	{ SVM_EXIT_TASK_SWITCH,			"task_switch" },
	{ SVM_EXIT_SHUTDOWN,			"shutdown" },
	{ SVM_EXIT_VMRUN,			"vmrun" },
	{ SVM_EXIT_VMMCALL,			"hypercall" },
	{ SVM_EXIT_VMLOAD,			"vmload" },
	{ SVM_EXIT_VMSAVE,			"vmsave" },
	{ SVM_EXIT_STGI,			"stgi" },
	{ SVM_EXIT_CLGI,			"clgi" },
	{ SVM_EXIT_SKINIT,			"skinit" },
	{ SVM_EXIT_WBINVD,			"wbinvd" },
	{ SVM_EXIT_MONITOR,			"monitor" },
	{ SVM_EXIT_MWAIT,			"mwait" },
	{ SVM_EXIT_NPF,				"npf" },
	{ -1, NULL }
};

3416
static int svm_get_lpage_level(void)
3417
{
3418
	return PT_PDPE_LEVEL;
3419 3420
}

3421 3422 3423 3424 3425
static bool svm_rdtscp_supported(void)
{
	return false;
}

3426 3427 3428 3429 3430
static bool svm_has_wbinvd_exit(void)
{
	return true;
}

3431 3432 3433 3434 3435
static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->control.intercept_exceptions |= 1 << NM_VECTOR;
3436 3437 3438
	if (is_nested(svm))
		svm->nested.hsave->control.intercept_exceptions |= 1 << NM_VECTOR;
	update_cr0_intercept(svm);
3439 3440
}

3441
static struct kvm_x86_ops svm_x86_ops = {
A
Avi Kivity 已提交
3442 3443 3444 3445
	.cpu_has_kvm_support = has_svm,
	.disabled_by_bios = is_disabled,
	.hardware_setup = svm_hardware_setup,
	.hardware_unsetup = svm_hardware_unsetup,
Y
Yang, Sheng 已提交
3446
	.check_processor_compatibility = svm_check_processor_compat,
A
Avi Kivity 已提交
3447 3448
	.hardware_enable = svm_hardware_enable,
	.hardware_disable = svm_hardware_disable,
3449
	.cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
A
Avi Kivity 已提交
3450 3451 3452

	.vcpu_create = svm_create_vcpu,
	.vcpu_free = svm_free_vcpu,
3453
	.vcpu_reset = svm_vcpu_reset,
A
Avi Kivity 已提交
3454

3455
	.prepare_guest_switch = svm_prepare_guest_switch,
A
Avi Kivity 已提交
3456 3457 3458 3459 3460 3461 3462 3463 3464
	.vcpu_load = svm_vcpu_load,
	.vcpu_put = svm_vcpu_put,

	.set_guest_debug = svm_guest_debug,
	.get_msr = svm_get_msr,
	.set_msr = svm_set_msr,
	.get_segment_base = svm_get_segment_base,
	.get_segment = svm_get_segment,
	.set_segment = svm_set_segment,
3465
	.get_cpl = svm_get_cpl,
3466
	.get_cs_db_l_bits = kvm_get_cs_db_l_bits,
3467
	.decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
3468
	.decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
A
Avi Kivity 已提交
3469 3470 3471 3472 3473 3474 3475 3476
	.set_cr0 = svm_set_cr0,
	.set_cr3 = svm_set_cr3,
	.set_cr4 = svm_set_cr4,
	.set_efer = svm_set_efer,
	.get_idt = svm_get_idt,
	.set_idt = svm_set_idt,
	.get_gdt = svm_get_gdt,
	.set_gdt = svm_set_gdt,
3477
	.set_dr7 = svm_set_dr7,
A
Avi Kivity 已提交
3478
	.cache_reg = svm_cache_reg,
A
Avi Kivity 已提交
3479 3480
	.get_rflags = svm_get_rflags,
	.set_rflags = svm_set_rflags,
A
Avi Kivity 已提交
3481
	.fpu_activate = svm_fpu_activate,
3482
	.fpu_deactivate = svm_fpu_deactivate,
A
Avi Kivity 已提交
3483 3484 3485 3486

	.tlb_flush = svm_flush_tlb,

	.run = svm_vcpu_run,
3487
	.handle_exit = handle_exit,
A
Avi Kivity 已提交
3488
	.skip_emulated_instruction = skip_emulated_instruction,
3489 3490
	.set_interrupt_shadow = svm_set_interrupt_shadow,
	.get_interrupt_shadow = svm_get_interrupt_shadow,
I
Ingo Molnar 已提交
3491
	.patch_hypercall = svm_patch_hypercall,
E
Eddie Dong 已提交
3492
	.set_irq = svm_set_irq,
3493
	.set_nmi = svm_inject_nmi,
3494
	.queue_exception = svm_queue_exception,
3495
	.interrupt_allowed = svm_interrupt_allowed,
3496
	.nmi_allowed = svm_nmi_allowed,
J
Jan Kiszka 已提交
3497 3498
	.get_nmi_mask = svm_get_nmi_mask,
	.set_nmi_mask = svm_set_nmi_mask,
3499 3500 3501
	.enable_nmi_window = enable_nmi_window,
	.enable_irq_window = enable_irq_window,
	.update_cr8_intercept = update_cr8_intercept,
3502 3503

	.set_tss_addr = svm_set_tss_addr,
3504
	.get_tdp_level = get_npt_level,
3505
	.get_mt_mask = svm_get_mt_mask,
3506 3507

	.exit_reasons_str = svm_exit_reasons_str,
3508
	.get_lpage_level = svm_get_lpage_level,
3509 3510

	.cpuid_update = svm_cpuid_update,
3511 3512

	.rdtscp_supported = svm_rdtscp_supported,
3513 3514

	.set_supported_cpuid = svm_set_supported_cpuid,
3515 3516

	.has_wbinvd_exit = svm_has_wbinvd_exit,
A
Avi Kivity 已提交
3517 3518 3519 3520
};

static int __init svm_init(void)
{
3521
	return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
3522
			__alignof__(struct vcpu_svm), THIS_MODULE);
A
Avi Kivity 已提交
3523 3524 3525 3526
}

static void __exit svm_exit(void)
{
3527
	kvm_exit();
A
Avi Kivity 已提交
3528 3529 3530 3531
}

module_init(svm_init)
module_exit(svm_exit)