svm.c 175.4 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * AMD SVM support
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
7
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
8 9 10 11 12 13 14 15 16
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
17 18 19

#define pr_fmt(fmt) "SVM: " fmt

20 21
#include <linux/kvm_host.h>

22
#include "irq.h"
23
#include "mmu.h"
24
#include "kvm_cache_regs.h"
25
#include "x86.h"
26
#include "cpuid.h"
27
#include "pmu.h"
A
Avi Kivity 已提交
28

A
Avi Kivity 已提交
29
#include <linux/module.h>
30
#include <linux/mod_devicetable.h>
31
#include <linux/kernel.h>
A
Avi Kivity 已提交
32 33
#include <linux/vmalloc.h>
#include <linux/highmem.h>
A
Alexey Dobriyan 已提交
34
#include <linux/sched.h>
35
#include <linux/trace_events.h>
36
#include <linux/slab.h>
37 38
#include <linux/amd-iommu.h>
#include <linux/hashtable.h>
39
#include <linux/frame.h>
B
Brijesh Singh 已提交
40
#include <linux/psp-sev.h>
B
Brijesh Singh 已提交
41
#include <linux/file.h>
42 43
#include <linux/pagemap.h>
#include <linux/swap.h>
A
Avi Kivity 已提交
44

45
#include <asm/apic.h>
46
#include <asm/perf_event.h>
47
#include <asm/tlbflush.h>
A
Avi Kivity 已提交
48
#include <asm/desc.h>
49
#include <asm/debugreg.h>
G
Gleb Natapov 已提交
50
#include <asm/kvm_para.h>
51
#include <asm/irq_remapping.h>
52
#include <asm/microcode.h>
53
#include <asm/nospec-branch.h>
A
Avi Kivity 已提交
54

55
#include <asm/virtext.h>
56
#include "trace.h"
57

58 59
#define __ex(x) __kvm_handle_fault_on_reboot(x)

A
Avi Kivity 已提交
60 61 62
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");

63 64 65 66 67 68
static const struct x86_cpu_id svm_cpu_id[] = {
	X86_FEATURE_MATCH(X86_FEATURE_SVM),
	{}
};
MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);

A
Avi Kivity 已提交
69 70 71 72 73 74
#define IOPM_ALLOC_ORDER 2
#define MSRPM_ALLOC_ORDER 1

#define SEG_TYPE_LDT 2
#define SEG_TYPE_BUSY_TSS16 3

75 76 77 78
#define SVM_FEATURE_NPT            (1 <<  0)
#define SVM_FEATURE_LBRV           (1 <<  1)
#define SVM_FEATURE_SVML           (1 <<  2)
#define SVM_FEATURE_NRIP           (1 <<  3)
79 80 81 82
#define SVM_FEATURE_TSC_RATE       (1 <<  4)
#define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
#define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
#define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
83
#define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
84

85 86
#define SVM_AVIC_DOORBELL	0xc001011b

87 88 89 90
#define NESTED_EXIT_HOST	0	/* Exit handled on host level */
#define NESTED_EXIT_DONE	1	/* Exit caused nested vmexit  */
#define NESTED_EXIT_CONTINUE	2	/* Further checks needed      */

91 92
#define DEBUGCTL_RESERVED_BITS (~(0x3fULL))

93
#define TSC_RATIO_RSVD          0xffffff0000000000ULL
94 95
#define TSC_RATIO_MIN		0x0000000000000001ULL
#define TSC_RATIO_MAX		0x000000ffffffffffULL
96

D
Dan Carpenter 已提交
97
#define AVIC_HPA_MASK	~((0xFFFULL << 52) | 0xFFF)
98 99 100 101 102 103 104

/*
 * 0xff is broadcast, so the max index allowed for physical APIC ID
 * table is 0xfe.  APIC IDs above 0xff are reserved.
 */
#define AVIC_MAX_PHYSICAL_ID_COUNT	255

105 106 107 108
#define AVIC_UNACCEL_ACCESS_WRITE_MASK		1
#define AVIC_UNACCEL_ACCESS_OFFSET_MASK		0xFF0
#define AVIC_UNACCEL_ACCESS_VECTOR_MASK		0xFFFFFFFF

109 110 111 112 113 114 115 116 117 118 119 120 121
/* AVIC GATAG is encoded using VM and VCPU IDs */
#define AVIC_VCPU_ID_BITS		8
#define AVIC_VCPU_ID_MASK		((1 << AVIC_VCPU_ID_BITS) - 1)

#define AVIC_VM_ID_BITS			24
#define AVIC_VM_ID_NR			(1 << AVIC_VM_ID_BITS)
#define AVIC_VM_ID_MASK			((1 << AVIC_VM_ID_BITS) - 1)

#define AVIC_GATAG(x, y)		(((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
						(y & AVIC_VCPU_ID_MASK))
#define AVIC_GATAG_TO_VMID(x)		((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
#define AVIC_GATAG_TO_VCPUID(x)		(x & AVIC_VCPU_ID_MASK)

122 123
static bool erratum_383_found __read_mostly;

A
Avi Kivity 已提交
124 125 126 127 128 129
static const u32 host_save_user_msrs[] = {
#ifdef CONFIG_X86_64
	MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
	MSR_FS_BASE,
#endif
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
P
Paolo Bonzini 已提交
130
	MSR_TSC_AUX,
A
Avi Kivity 已提交
131 132 133 134 135 136
};

#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)

struct kvm_vcpu;

137 138 139
struct nested_state {
	struct vmcb *hsave;
	u64 hsave_msr;
140
	u64 vm_cr_msr;
141 142 143 144 145 146 147
	u64 vmcb;

	/* These are the merged vectors */
	u32 *msrpm;

	/* gpa pointers to the real vectors */
	u64 vmcb_msrpm;
148
	u64 vmcb_iopm;
J
Joerg Roedel 已提交
149

150 151 152
	/* A VMEXIT is required but not yet emulated */
	bool exit_required;

J
Joerg Roedel 已提交
153
	/* cache for intercepts of the guest */
154
	u32 intercept_cr;
155
	u32 intercept_dr;
J
Joerg Roedel 已提交
156 157 158
	u32 intercept_exceptions;
	u64 intercept;

159 160
	/* Nested Paging related state */
	u64 nested_cr3;
161 162
};

163 164 165
#define MSRPM_OFFSETS	16
static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;

166 167 168 169 170 171
/*
 * Set osvw_len to higher value when updated Revision Guides
 * are published and we know what the new status bits are
 */
static uint64_t osvw_len = 4, osvw_status;

A
Avi Kivity 已提交
172 173 174 175 176 177 178 179
struct vcpu_svm {
	struct kvm_vcpu vcpu;
	struct vmcb *vmcb;
	unsigned long vmcb_pa;
	struct svm_cpu_data *svm_data;
	uint64_t asid_generation;
	uint64_t sysenter_esp;
	uint64_t sysenter_eip;
P
Paolo Bonzini 已提交
180
	uint64_t tsc_aux;
A
Avi Kivity 已提交
181

182 183
	u64 msr_decfg;

A
Avi Kivity 已提交
184 185 186
	u64 next_rip;

	u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
187
	struct {
188 189 190
		u16 fs;
		u16 gs;
		u16 ldt;
191 192
		u64 gs_base;
	} host;
A
Avi Kivity 已提交
193

194 195
	u64 spec_ctrl;

A
Avi Kivity 已提交
196 197
	u32 *msrpm;

198 199
	ulong nmi_iret_rip;

200
	struct nested_state nested;
J
Jan Kiszka 已提交
201 202

	bool nmi_singlestep;
203
	u64 nmi_singlestep_guest_rflags;
204 205 206

	unsigned int3_injected;
	unsigned long int3_rip;
207

208 209
	/* cached guest cpuid flags for faster access */
	bool nrips_enabled	: 1;
210

211
	u32 ldr_reg;
212 213
	struct page *avic_backing_page;
	u64 *avic_physical_id_cache;
214
	bool avic_is_running;
215 216 217 218 219 220 221 222 223

	/*
	 * Per-vcpu list of struct amd_svm_iommu_ir:
	 * This is used mainly to store interrupt remapping information used
	 * when update the vcpu affinity. This avoids the need to scan for
	 * IRTE and try to match ga_tag in the IOMMU driver.
	 */
	struct list_head ir_list;
	spinlock_t ir_list_lock;
224 225 226

	/* which host CPU was used for running this vcpu */
	unsigned int last_cpu;
227 228 229 230 231 232 233 234
};

/*
 * This is a wrapper of struct amd_iommu_ir_data.
 */
struct amd_svm_iommu_ir {
	struct list_head node;	/* Used by SVM for per-vcpu ir_list */
	void *data;		/* Storing pointer to struct amd_ir_data */
A
Avi Kivity 已提交
235 236
};

237 238 239 240 241 242 243 244
#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK	(0xFF)
#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK		(1 << 31)

#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK	(0xFFULL)
#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK	(0xFFFFFFFFFFULL << 12)
#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK		(1ULL << 62)
#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK		(1ULL << 63)

245 246 247
static DEFINE_PER_CPU(u64, current_tsc_ratio);
#define TSC_RATIO_DEFAULT	0x0100000000ULL

248 249
#define MSR_INVALID			0xffffffffU

250
static const struct svm_direct_access_msrs {
251 252 253
	u32 index;   /* Index of the MSR */
	bool always; /* True if intercept is always on */
} direct_access_msrs[] = {
B
Brian Gerst 已提交
254
	{ .index = MSR_STAR,				.always = true  },
255 256 257 258 259 260 261 262 263
	{ .index = MSR_IA32_SYSENTER_CS,		.always = true  },
#ifdef CONFIG_X86_64
	{ .index = MSR_GS_BASE,				.always = true  },
	{ .index = MSR_FS_BASE,				.always = true  },
	{ .index = MSR_KERNEL_GS_BASE,			.always = true  },
	{ .index = MSR_LSTAR,				.always = true  },
	{ .index = MSR_CSTAR,				.always = true  },
	{ .index = MSR_SYSCALL_MASK,			.always = true  },
#endif
264
	{ .index = MSR_IA32_SPEC_CTRL,			.always = false },
A
Ashok Raj 已提交
265
	{ .index = MSR_IA32_PRED_CMD,			.always = false },
266 267 268 269 270
	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
	{ .index = MSR_IA32_LASTINTTOIP,		.always = false },
	{ .index = MSR_INVALID,				.always = false },
A
Avi Kivity 已提交
271 272
};

273 274 275 276
/* enable NPT for AMD64 and X86 with PAE */
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
static bool npt_enabled = true;
#else
J
Joerg Roedel 已提交
277
static bool npt_enabled;
278
#endif
279

280 281
/* allow nested paging (virtualized MMU) for all guests */
static int npt = true;
282
module_param(npt, int, S_IRUGO);
283

284 285
/* allow nested virtualization in KVM/SVM */
static int nested = true;
286 287
module_param(nested, int, S_IRUGO);

288 289
/* enable / disable AVIC */
static int avic;
290
#ifdef CONFIG_X86_LOCAL_APIC
291
module_param(avic, int, S_IRUGO);
292
#endif
293

294 295 296 297
/* enable/disable Virtual VMLOAD VMSAVE */
static int vls = true;
module_param(vls, int, 0444);

298 299 300
/* enable/disable Virtual GIF */
static int vgif = true;
module_param(vgif, int, 0444);
301

B
Brijesh Singh 已提交
302 303 304 305
/* enable/disable SEV support */
static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
module_param(sev, int, 0444);

B
Brijesh Singh 已提交
306 307
static u8 rsm_ins_bytes[] = "\x0f\xaa";

308
static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
309
static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa);
310
static void svm_complete_interrupts(struct vcpu_svm *svm);
311

312
static int nested_svm_exit_handled(struct vcpu_svm *svm);
313
static int nested_svm_intercept(struct vcpu_svm *svm);
314 315 316 317
static int nested_svm_vmexit(struct vcpu_svm *svm);
static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
				      bool has_error_code, u32 error_code);

318
enum {
319 320
	VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
			    pause filter count */
321
	VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
322
	VMCB_ASID,	 /* ASID */
323
	VMCB_INTR,	 /* int_ctl, int_vector */
324
	VMCB_NPT,        /* npt_en, nCR3, gPAT */
325
	VMCB_CR,	 /* CR0, CR3, CR4, EFER */
326
	VMCB_DR,         /* DR6, DR7 */
327
	VMCB_DT,         /* GDT, IDT */
328
	VMCB_SEG,        /* CS, DS, SS, ES, CPL */
329
	VMCB_CR2,        /* CR2 only */
330
	VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
331 332 333 334
	VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
			  * AVIC PHYSICAL_TABLE pointer,
			  * AVIC LOGICAL_TABLE pointer
			  */
335 336 337
	VMCB_DIRTY_MAX,
};

338 339
/* TPR and CR2 are always written before VMRUN */
#define VMCB_ALWAYS_DIRTY_MASK	((1U << VMCB_INTR) | (1U << VMCB_CR2))
340

341 342
#define VMCB_AVIC_APIC_BAR_MASK		0xFFFFFFFFFF000ULL

343
static unsigned int max_sev_asid;
B
Brijesh Singh 已提交
344 345
static unsigned int min_sev_asid;
static unsigned long *sev_asid_bitmap;
346
#define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
B
Brijesh Singh 已提交
347

348 349 350 351 352 353 354 355
struct enc_region {
	struct list_head list;
	unsigned long npages;
	struct page **pages;
	unsigned long uaddr;
	unsigned long size;
};

B
Brijesh Singh 已提交
356 357 358 359 360 361 362 363 364 365 366
static inline bool svm_sev_enabled(void)
{
	return max_sev_asid;
}

static inline bool sev_guest(struct kvm *kvm)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;

	return sev->active;
}
367

368 369 370 371 372 373 374
static inline int sev_get_asid(struct kvm *kvm)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;

	return sev->asid;
}

375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390
static inline void mark_all_dirty(struct vmcb *vmcb)
{
	vmcb->control.clean = 0;
}

static inline void mark_all_clean(struct vmcb *vmcb)
{
	vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
			       & ~VMCB_ALWAYS_DIRTY_MASK;
}

static inline void mark_dirty(struct vmcb *vmcb, int bit)
{
	vmcb->control.clean &= ~(1 << bit);
}

391 392
static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
{
R
Rusty Russell 已提交
393
	return container_of(vcpu, struct vcpu_svm, vcpu);
394 395
}

396 397 398 399 400 401
static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
{
	svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
	mark_dirty(svm->vmcb, VMCB_AVIC);
}

402 403 404 405 406 407 408 409 410 411 412
static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u64 *entry = svm->avic_physical_id_cache;

	if (!entry)
		return false;

	return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
}

413 414 415 416 417
static void recalc_intercepts(struct vcpu_svm *svm)
{
	struct vmcb_control_area *c, *h;
	struct nested_state *g;

418 419
	mark_dirty(svm->vmcb, VMCB_INTERCEPTS);

420 421 422 423 424 425 426
	if (!is_guest_mode(&svm->vcpu))
		return;

	c = &svm->vmcb->control;
	h = &svm->nested.hsave->control;
	g = &svm->nested;

427
	c->intercept_cr = h->intercept_cr | g->intercept_cr;
428
	c->intercept_dr = h->intercept_dr | g->intercept_dr;
429
	c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
430 431 432
	c->intercept = h->intercept | g->intercept;
}

433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
{
	if (is_guest_mode(&svm->vcpu))
		return svm->nested.hsave;
	else
		return svm->vmcb;
}

static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept_cr |= (1U << bit);

	recalc_intercepts(svm);
}

static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept_cr &= ~(1U << bit);

	recalc_intercepts(svm);
}

static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	return vmcb->control.intercept_cr & (1U << bit);
}

466
static inline void set_dr_intercepts(struct vcpu_svm *svm)
467 468 469
{
	struct vmcb *vmcb = get_host_vmcb(svm);

470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
	vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
		| (1 << INTERCEPT_DR1_READ)
		| (1 << INTERCEPT_DR2_READ)
		| (1 << INTERCEPT_DR3_READ)
		| (1 << INTERCEPT_DR4_READ)
		| (1 << INTERCEPT_DR5_READ)
		| (1 << INTERCEPT_DR6_READ)
		| (1 << INTERCEPT_DR7_READ)
		| (1 << INTERCEPT_DR0_WRITE)
		| (1 << INTERCEPT_DR1_WRITE)
		| (1 << INTERCEPT_DR2_WRITE)
		| (1 << INTERCEPT_DR3_WRITE)
		| (1 << INTERCEPT_DR4_WRITE)
		| (1 << INTERCEPT_DR5_WRITE)
		| (1 << INTERCEPT_DR6_WRITE)
		| (1 << INTERCEPT_DR7_WRITE);
486 487 488 489

	recalc_intercepts(svm);
}

490
static inline void clr_dr_intercepts(struct vcpu_svm *svm)
491 492 493
{
	struct vmcb *vmcb = get_host_vmcb(svm);

494
	vmcb->control.intercept_dr = 0;
495 496 497 498

	recalc_intercepts(svm);
}

499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept_exceptions |= (1U << bit);

	recalc_intercepts(svm);
}

static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept_exceptions &= ~(1U << bit);

	recalc_intercepts(svm);
}

517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
static inline void set_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept |= (1ULL << bit);

	recalc_intercepts(svm);
}

static inline void clr_intercept(struct vcpu_svm *svm, int bit)
{
	struct vmcb *vmcb = get_host_vmcb(svm);

	vmcb->control.intercept &= ~(1ULL << bit);

	recalc_intercepts(svm);
}

535 536 537 538 539
static inline bool vgif_enabled(struct vcpu_svm *svm)
{
	return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK);
}

540 541
static inline void enable_gif(struct vcpu_svm *svm)
{
542 543 544 545
	if (vgif_enabled(svm))
		svm->vmcb->control.int_ctl |= V_GIF_MASK;
	else
		svm->vcpu.arch.hflags |= HF_GIF_MASK;
546 547 548 549
}

static inline void disable_gif(struct vcpu_svm *svm)
{
550 551 552 553
	if (vgif_enabled(svm))
		svm->vmcb->control.int_ctl &= ~V_GIF_MASK;
	else
		svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
554 555 556 557
}

static inline bool gif_set(struct vcpu_svm *svm)
{
558 559 560 561
	if (vgif_enabled(svm))
		return !!(svm->vmcb->control.int_ctl & V_GIF_MASK);
	else
		return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
562 563
}

564
static unsigned long iopm_base;
A
Avi Kivity 已提交
565 566 567 568

struct kvm_ldttss_desc {
	u16 limit0;
	u16 base0;
J
Joerg Roedel 已提交
569 570
	unsigned base1:8, type:5, dpl:2, p:1;
	unsigned limit1:4, zero0:3, g:1, base2:8;
A
Avi Kivity 已提交
571 572 573 574 575 576 577
	u32 base3;
	u32 zero1;
} __attribute__((packed));

struct svm_cpu_data {
	int cpu;

A
Avi Kivity 已提交
578 579 580
	u64 asid_generation;
	u32 max_asid;
	u32 next_asid;
581
	u32 min_asid;
A
Avi Kivity 已提交
582 583 584
	struct kvm_ldttss_desc *tss_desc;

	struct page *save_area;
A
Ashok Raj 已提交
585
	struct vmcb *current_vmcb;
586 587 588

	/* index = sev_asid, value = vmcb pointer */
	struct vmcb **sev_vmcbs;
A
Avi Kivity 已提交
589 590 591 592 593 594 595 596 597
};

static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);

struct svm_init_data {
	int cpu;
	int r;
};

598
static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
A
Avi Kivity 已提交
599

600
#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
A
Avi Kivity 已提交
601 602 603
#define MSRS_RANGE_SIZE 2048
#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)

604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624
static u32 svm_msrpm_offset(u32 msr)
{
	u32 offset;
	int i;

	for (i = 0; i < NUM_MSR_MAPS; i++) {
		if (msr < msrpm_ranges[i] ||
		    msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
			continue;

		offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
		offset += (i * MSRS_RANGE_SIZE);       /* add range offset */

		/* Now we have the u8 offset - but need the u32 offset */
		return offset / 4;
	}

	/* MSR not in any range */
	return MSR_INVALID;
}

A
Avi Kivity 已提交
625 626 627 628
#define MAX_INST_SIZE 15

static inline void clgi(void)
{
629
	asm volatile (__ex(SVM_CLGI));
A
Avi Kivity 已提交
630 631 632 633
}

static inline void stgi(void)
{
634
	asm volatile (__ex(SVM_STGI));
A
Avi Kivity 已提交
635 636 637 638
}

static inline void invlpga(unsigned long addr, u32 asid)
{
J
Joerg Roedel 已提交
639
	asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
A
Avi Kivity 已提交
640 641
}

642
static int get_npt_level(struct kvm_vcpu *vcpu)
643 644
{
#ifdef CONFIG_X86_64
645
	return PT64_ROOT_4LEVEL;
646 647 648 649 650
#else
	return PT32E_ROOT_LEVEL;
#endif
}

A
Avi Kivity 已提交
651 652
static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
653
	vcpu->arch.efer = efer;
654
	if (!npt_enabled && !(efer & EFER_LMA))
655
		efer &= ~EFER_LME;
A
Avi Kivity 已提交
656

657
	to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
658
	mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
A
Avi Kivity 已提交
659 660 661 662 663 664 665 666
}

static int is_external_interrupt(u32 info)
{
	info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
	return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
}

667
static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
668 669 670 671 672
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u32 ret = 0;

	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
673 674
		ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
	return ret;
675 676 677 678 679 680 681 682 683 684 685 686 687
}

static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (mask == 0)
		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
	else
		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;

}

A
Avi Kivity 已提交
688 689
static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
690 691
	struct vcpu_svm *svm = to_svm(vcpu);

692
	if (svm->vmcb->control.next_rip != 0) {
693
		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
694
		svm->next_rip = svm->vmcb->control.next_rip;
695
	}
696

697
	if (!svm->next_rip) {
698
		if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
699 700
				EMULATE_DONE)
			printk(KERN_DEBUG "%s: NOP\n", __func__);
A
Avi Kivity 已提交
701 702
		return;
	}
703 704 705
	if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
		printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
		       __func__, kvm_rip_read(vcpu), svm->next_rip);
A
Avi Kivity 已提交
706

707
	kvm_rip_write(vcpu, svm->next_rip);
708
	svm_set_interrupt_shadow(vcpu, 0);
A
Avi Kivity 已提交
709 710
}

711
static void svm_queue_exception(struct kvm_vcpu *vcpu)
J
Jan Kiszka 已提交
712 713
{
	struct vcpu_svm *svm = to_svm(vcpu);
714 715
	unsigned nr = vcpu->arch.exception.nr;
	bool has_error_code = vcpu->arch.exception.has_error_code;
716
	bool reinject = vcpu->arch.exception.injected;
717
	u32 error_code = vcpu->arch.exception.error_code;
J
Jan Kiszka 已提交
718

J
Joerg Roedel 已提交
719 720 721 722
	/*
	 * If we are within a nested VM we'd better #VMEXIT and let the guest
	 * handle the exception
	 */
723 724
	if (!reinject &&
	    nested_svm_check_exception(svm, nr, has_error_code, error_code))
J
Jan Kiszka 已提交
725 726
		return;

727
	if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
		unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);

		/*
		 * For guest debugging where we have to reinject #BP if some
		 * INT3 is guest-owned:
		 * Emulate nRIP by moving RIP forward. Will fail if injection
		 * raises a fault that is not intercepted. Still better than
		 * failing in all cases.
		 */
		skip_emulated_instruction(&svm->vcpu);
		rip = kvm_rip_read(&svm->vcpu);
		svm->int3_rip = rip + svm->vmcb->save.cs.base;
		svm->int3_injected = rip - old_rip;
	}

J
Jan Kiszka 已提交
743 744 745 746 747 748 749
	svm->vmcb->control.event_inj = nr
		| SVM_EVTINJ_VALID
		| (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
		| SVM_EVTINJ_TYPE_EXEPT;
	svm->vmcb->control.event_inj_err = error_code;
}

750 751 752 753 754 755
static void svm_init_erratum_383(void)
{
	u32 low, high;
	int err;
	u64 val;

756
	if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
		return;

	/* Use _safe variants to not break nested virtualization */
	val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
	if (err)
		return;

	val |= (1ULL << 47);

	low  = lower_32_bits(val);
	high = upper_32_bits(val);

	native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);

	erratum_383_found = true;
}

774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
static void svm_init_osvw(struct kvm_vcpu *vcpu)
{
	/*
	 * Guests should see errata 400 and 415 as fixed (assuming that
	 * HLT and IO instructions are intercepted).
	 */
	vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
	vcpu->arch.osvw.status = osvw_status & ~(6ULL);

	/*
	 * By increasing VCPU's osvw.length to 3 we are telling the guest that
	 * all osvw.status bits inside that length, including bit 0 (which is
	 * reserved for erratum 298), are valid. However, if host processor's
	 * osvw_len is 0 then osvw_status[0] carries no information. We need to
	 * be conservative here and therefore we tell the guest that erratum 298
	 * is present (because we really don't know).
	 */
	if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
		vcpu->arch.osvw.status |= 1;
}

A
Avi Kivity 已提交
795 796
static int has_svm(void)
{
797
	const char *msg;
A
Avi Kivity 已提交
798

799
	if (!cpu_has_svm(&msg)) {
J
Joe Perches 已提交
800
		printk(KERN_INFO "has_svm: %s\n", msg);
A
Avi Kivity 已提交
801 802 803 804 805 806
		return 0;
	}

	return 1;
}

807
static void svm_hardware_disable(void)
A
Avi Kivity 已提交
808
{
809 810 811 812
	/* Make sure we clean up behind us */
	if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
		wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);

813
	cpu_svm_disable();
814 815

	amd_pmu_disable_virt();
A
Avi Kivity 已提交
816 817
}

818
static int svm_hardware_enable(void)
A
Avi Kivity 已提交
819 820
{

821
	struct svm_cpu_data *sd;
A
Avi Kivity 已提交
822 823 824 825
	uint64_t efer;
	struct desc_struct *gdt;
	int me = raw_smp_processor_id();

826 827 828 829
	rdmsrl(MSR_EFER, efer);
	if (efer & EFER_SVME)
		return -EBUSY;

A
Avi Kivity 已提交
830
	if (!has_svm()) {
831
		pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
832
		return -EINVAL;
A
Avi Kivity 已提交
833
	}
834 835
	sd = per_cpu(svm_data, me);
	if (!sd) {
836
		pr_err("%s: svm_data is NULL on %d\n", __func__, me);
837
		return -EINVAL;
A
Avi Kivity 已提交
838 839
	}

840 841 842
	sd->asid_generation = 1;
	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
	sd->next_asid = sd->max_asid + 1;
843
	sd->min_asid = max_sev_asid + 1;
A
Avi Kivity 已提交
844

845
	gdt = get_current_gdt_rw();
846
	sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
A
Avi Kivity 已提交
847

848
	wrmsrl(MSR_EFER, efer | EFER_SVME);
A
Avi Kivity 已提交
849

850
	wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
851

852 853
	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
		wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
854
		__this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
855 856
	}

857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886

	/*
	 * Get OSVW bits.
	 *
	 * Note that it is possible to have a system with mixed processor
	 * revisions and therefore different OSVW bits. If bits are not the same
	 * on different processors then choose the worst case (i.e. if erratum
	 * is present on one processor and not on another then assume that the
	 * erratum is present everywhere).
	 */
	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
		uint64_t len, status = 0;
		int err;

		len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
		if (!err)
			status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
						      &err);

		if (err)
			osvw_status = osvw_len = 0;
		else {
			if (len < osvw_len)
				osvw_len = len;
			osvw_status |= status;
			osvw_status &= (1ULL << osvw_len) - 1;
		}
	} else
		osvw_status = osvw_len = 0;

887 888
	svm_init_erratum_383();

889 890
	amd_pmu_enable_virt();

891
	return 0;
A
Avi Kivity 已提交
892 893
}

894 895
static void svm_cpu_uninit(int cpu)
{
896
	struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
897

898
	if (!sd)
899 900 901
		return;

	per_cpu(svm_data, raw_smp_processor_id()) = NULL;
902
	kfree(sd->sev_vmcbs);
903 904
	__free_page(sd->save_area);
	kfree(sd);
905 906
}

A
Avi Kivity 已提交
907 908
static int svm_cpu_init(int cpu)
{
909
	struct svm_cpu_data *sd;
A
Avi Kivity 已提交
910 911
	int r;

912 913
	sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
	if (!sd)
A
Avi Kivity 已提交
914
		return -ENOMEM;
915
	sd->cpu = cpu;
A
Avi Kivity 已提交
916
	r = -ENOMEM;
917
	sd->save_area = alloc_page(GFP_KERNEL);
918
	if (!sd->save_area)
A
Avi Kivity 已提交
919 920
		goto err_1;

921 922 923 924 925 926 927
	if (svm_sev_enabled()) {
		r = -ENOMEM;
		sd->sev_vmcbs = kmalloc((max_sev_asid + 1) * sizeof(void *), GFP_KERNEL);
		if (!sd->sev_vmcbs)
			goto err_1;
	}

928
	per_cpu(svm_data, cpu) = sd;
A
Avi Kivity 已提交
929 930 931 932

	return 0;

err_1:
933
	kfree(sd);
A
Avi Kivity 已提交
934 935 936 937
	return r;

}

938 939 940 941 942 943 944 945 946 947 948
static bool valid_msr_intercept(u32 index)
{
	int i;

	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
		if (direct_access_msrs[i].index == index)
			return true;

	return false;
}

949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967
static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
{
	u8 bit_write;
	unsigned long tmp;
	u32 offset;
	u32 *msrpm;

	msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
				      to_svm(vcpu)->msrpm;

	offset    = svm_msrpm_offset(msr);
	bit_write = 2 * (msr & 0x0f) + 1;
	tmp       = msrpm[offset];

	BUG_ON(offset == MSR_INVALID);

	return !!test_bit(bit_write,  &tmp);
}

968 969
static void set_msr_interception(u32 *msrpm, unsigned msr,
				 int read, int write)
A
Avi Kivity 已提交
970
{
971 972 973
	u8 bit_read, bit_write;
	unsigned long tmp;
	u32 offset;
A
Avi Kivity 已提交
974

975 976 977 978 979 980
	/*
	 * If this warning triggers extend the direct_access_msrs list at the
	 * beginning of the file
	 */
	WARN_ON(!valid_msr_intercept(msr));

981 982 983 984 985 986 987 988 989 990 991
	offset    = svm_msrpm_offset(msr);
	bit_read  = 2 * (msr & 0x0f);
	bit_write = 2 * (msr & 0x0f) + 1;
	tmp       = msrpm[offset];

	BUG_ON(offset == MSR_INVALID);

	read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
	write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);

	msrpm[offset] = tmp;
A
Avi Kivity 已提交
992 993
}

994
static void svm_vcpu_init_msrpm(u32 *msrpm)
A
Avi Kivity 已提交
995 996 997
{
	int i;

998 999
	memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));

1000 1001 1002 1003 1004 1005
	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
		if (!direct_access_msrs[i].always)
			continue;

		set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
	}
1006 1007
}

1008 1009 1010 1011 1012 1013 1014 1015
static void add_msr_offset(u32 offset)
{
	int i;

	for (i = 0; i < MSRPM_OFFSETS; ++i) {

		/* Offset already in list? */
		if (msrpm_offsets[i] == offset)
1016
			return;
1017 1018 1019 1020 1021 1022 1023 1024 1025

		/* Slot used by another offset? */
		if (msrpm_offsets[i] != MSR_INVALID)
			continue;

		/* Add offset to list */
		msrpm_offsets[i] = offset;

		return;
A
Avi Kivity 已提交
1026
	}
1027 1028 1029 1030 1031

	/*
	 * If this BUG triggers the msrpm_offsets table has an overflow. Just
	 * increase MSRPM_OFFSETS in this case.
	 */
1032
	BUG();
A
Avi Kivity 已提交
1033 1034
}

1035
static void init_msrpm_offsets(void)
1036
{
1037
	int i;
1038

1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
	memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));

	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
		u32 offset;

		offset = svm_msrpm_offset(direct_access_msrs[i].index);
		BUG_ON(offset == MSR_INVALID);

		add_msr_offset(offset);
	}
1049 1050
}

1051 1052 1053 1054
static void svm_enable_lbrv(struct vcpu_svm *svm)
{
	u32 *msrpm = svm->msrpm;

1055
	svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
	set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
}

static void svm_disable_lbrv(struct vcpu_svm *svm)
{
	u32 *msrpm = svm->msrpm;

1066
	svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1067 1068 1069 1070 1071 1072
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
	set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
}

1073 1074 1075
static void disable_nmi_singlestep(struct vcpu_svm *svm)
{
	svm->nmi_singlestep = false;
1076

1077 1078 1079 1080 1081 1082 1083
	if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
		/* Clear our flags if they were not set by the guest */
		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
	}
1084 1085
}

1086 1087 1088 1089 1090 1091
/* Note:
 * This hash table is used to map VM_ID to a struct kvm_arch,
 * when handling AMD IOMMU GALOG notification to schedule in
 * a particular vCPU.
 */
#define SVM_VM_DATA_HASH_BITS	8
1092
static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
1093 1094
static u32 next_vm_id = 0;
static bool next_vm_id_wrapped = 0;
1095
static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127

/* Note:
 * This function is called from IOMMU driver to notify
 * SVM to schedule in a particular vCPU of a particular VM.
 */
static int avic_ga_log_notifier(u32 ga_tag)
{
	unsigned long flags;
	struct kvm_arch *ka = NULL;
	struct kvm_vcpu *vcpu = NULL;
	u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
	u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);

	pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);

	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
	hash_for_each_possible(svm_vm_data_hash, ka, hnode, vm_id) {
		struct kvm *kvm = container_of(ka, struct kvm, arch);
		struct kvm_arch *vm_data = &kvm->arch;

		if (vm_data->avic_vm_id != vm_id)
			continue;
		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
		break;
	}
	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);

	/* Note:
	 * At this point, the IOMMU should have already set the pending
	 * bit in the vAPIC backing page. So, we just need to schedule
	 * in the vcpu.
	 */
1128
	if (vcpu)
1129 1130 1131 1132 1133
		kvm_vcpu_wake_up(vcpu);

	return 0;
}

B
Brijesh Singh 已提交
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
static __init int sev_hardware_setup(void)
{
	struct sev_user_data_status *status;
	int rc;

	/* Maximum number of encrypted guests supported simultaneously */
	max_sev_asid = cpuid_ecx(0x8000001F);

	if (!max_sev_asid)
		return 1;

B
Brijesh Singh 已提交
1145 1146 1147 1148 1149 1150 1151 1152 1153
	/* Minimum ASID value that should be used for SEV guest */
	min_sev_asid = cpuid_edx(0x8000001F);

	/* Initialize SEV ASID bitmap */
	sev_asid_bitmap = kcalloc(BITS_TO_LONGS(max_sev_asid),
				sizeof(unsigned long), GFP_KERNEL);
	if (!sev_asid_bitmap)
		return 1;

B
Brijesh Singh 已提交
1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
	status = kmalloc(sizeof(*status), GFP_KERNEL);
	if (!status)
		return 1;

	/*
	 * Check SEV platform status.
	 *
	 * PLATFORM_STATUS can be called in any state, if we failed to query
	 * the PLATFORM status then either PSP firmware does not support SEV
	 * feature or SEV firmware is dead.
	 */
	rc = sev_platform_status(status, NULL);
	if (rc)
		goto err;

	pr_info("SEV supported\n");

err:
	kfree(status);
	return rc;
}

A
Avi Kivity 已提交
1176 1177 1178 1179
static __init int svm_hardware_setup(void)
{
	int cpu;
	struct page *iopm_pages;
1180
	void *iopm_va;
A
Avi Kivity 已提交
1181 1182 1183 1184 1185 1186
	int r;

	iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);

	if (!iopm_pages)
		return -ENOMEM;
1187 1188 1189

	iopm_va = page_address(iopm_pages);
	memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
A
Avi Kivity 已提交
1190 1191
	iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;

1192 1193
	init_msrpm_offsets();

1194 1195 1196
	if (boot_cpu_has(X86_FEATURE_NX))
		kvm_enable_efer_bits(EFER_NX);

A
Alexander Graf 已提交
1197 1198 1199
	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
		kvm_enable_efer_bits(EFER_FFXSR);

1200 1201
	if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
		kvm_has_tsc_control = true;
1202 1203
		kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
		kvm_tsc_scaling_ratio_frac_bits = 32;
1204 1205
	}

1206 1207
	if (nested) {
		printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
1208
		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
1209 1210
	}

B
Brijesh Singh 已提交
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
	if (sev) {
		if (boot_cpu_has(X86_FEATURE_SEV) &&
		    IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
			r = sev_hardware_setup();
			if (r)
				sev = false;
		} else {
			sev = false;
		}
	}

Z
Zachary Amsden 已提交
1222
	for_each_possible_cpu(cpu) {
A
Avi Kivity 已提交
1223 1224
		r = svm_cpu_init(cpu);
		if (r)
1225
			goto err;
A
Avi Kivity 已提交
1226
	}
1227

1228
	if (!boot_cpu_has(X86_FEATURE_NPT))
1229 1230
		npt_enabled = false;

1231 1232 1233 1234 1235
	if (npt_enabled && !npt) {
		printk(KERN_INFO "kvm: Nested Paging disabled\n");
		npt_enabled = false;
	}

1236
	if (npt_enabled) {
1237
		printk(KERN_INFO "kvm: Nested Paging enabled\n");
1238
		kvm_enable_tdp();
1239 1240
	} else
		kvm_disable_tdp();
1241

1242 1243 1244
	if (avic) {
		if (!npt_enabled ||
		    !boot_cpu_has(X86_FEATURE_AVIC) ||
1245
		    !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
1246
			avic = false;
1247
		} else {
1248
			pr_info("AVIC enabled\n");
1249 1250 1251

			amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
		}
1252
	}
1253

1254 1255
	if (vls) {
		if (!npt_enabled ||
1256
		    !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1257 1258 1259 1260 1261 1262 1263
		    !IS_ENABLED(CONFIG_X86_64)) {
			vls = false;
		} else {
			pr_info("Virtual VMLOAD VMSAVE supported\n");
		}
	}

1264 1265 1266 1267 1268 1269 1270
	if (vgif) {
		if (!boot_cpu_has(X86_FEATURE_VGIF))
			vgif = false;
		else
			pr_info("Virtual GIF supported\n");
	}

A
Avi Kivity 已提交
1271 1272
	return 0;

1273
err:
A
Avi Kivity 已提交
1274 1275 1276 1277 1278 1279 1280
	__free_pages(iopm_pages, IOPM_ALLOC_ORDER);
	iopm_base = 0;
	return r;
}

static __exit void svm_hardware_unsetup(void)
{
1281 1282
	int cpu;

B
Brijesh Singh 已提交
1283 1284 1285
	if (svm_sev_enabled())
		kfree(sev_asid_bitmap);

Z
Zachary Amsden 已提交
1286
	for_each_possible_cpu(cpu)
1287 1288
		svm_cpu_uninit(cpu);

A
Avi Kivity 已提交
1289
	__free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
1290
	iopm_base = 0;
A
Avi Kivity 已提交
1291 1292 1293 1294 1295 1296
}

static void init_seg(struct vmcb_seg *seg)
{
	seg->selector = 0;
	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
J
Joerg Roedel 已提交
1297
		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
A
Avi Kivity 已提交
1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309
	seg->limit = 0xffff;
	seg->base = 0;
}

static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
{
	seg->selector = 0;
	seg->attrib = SVM_SELECTOR_P_MASK | type;
	seg->limit = 0xffff;
	seg->base = 0;
}

1310 1311 1312 1313 1314
static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u64 g_tsc_offset = 0;

1315
	if (is_guest_mode(vcpu)) {
1316 1317 1318
		g_tsc_offset = svm->vmcb->control.tsc_offset -
			       svm->nested.hsave->control.tsc_offset;
		svm->nested.hsave->control.tsc_offset = offset;
1319 1320 1321 1322
	} else
		trace_kvm_write_tsc_offset(vcpu->vcpu_id,
					   svm->vmcb->control.tsc_offset,
					   offset);
1323 1324

	svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1325 1326

	mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1327 1328
}

1329 1330 1331 1332
static void avic_init_vmcb(struct vcpu_svm *svm)
{
	struct vmcb *vmcb = svm->vmcb;
	struct kvm_arch *vm_data = &svm->vcpu.kvm->arch;
1333 1334 1335
	phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
	phys_addr_t lpa = __sme_set(page_to_phys(vm_data->avic_logical_id_table_page));
	phys_addr_t ppa = __sme_set(page_to_phys(vm_data->avic_physical_id_table_page));
1336 1337 1338 1339 1340 1341 1342 1343

	vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
	vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
	vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
	vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
	vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
}

P
Paolo Bonzini 已提交
1344
static void init_vmcb(struct vcpu_svm *svm)
A
Avi Kivity 已提交
1345
{
1346 1347
	struct vmcb_control_area *control = &svm->vmcb->control;
	struct vmcb_save_area *save = &svm->vmcb->save;
A
Avi Kivity 已提交
1348

1349
	svm->vcpu.arch.hflags = 0;
1350

1351 1352 1353 1354 1355 1356
	set_cr_intercept(svm, INTERCEPT_CR0_READ);
	set_cr_intercept(svm, INTERCEPT_CR3_READ);
	set_cr_intercept(svm, INTERCEPT_CR4_READ);
	set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
	set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
	set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1357 1358
	if (!kvm_vcpu_apicv_active(&svm->vcpu))
		set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
A
Avi Kivity 已提交
1359

1360
	set_dr_intercepts(svm);
A
Avi Kivity 已提交
1361

1362 1363 1364
	set_exception_intercept(svm, PF_VECTOR);
	set_exception_intercept(svm, UD_VECTOR);
	set_exception_intercept(svm, MC_VECTOR);
1365
	set_exception_intercept(svm, AC_VECTOR);
1366
	set_exception_intercept(svm, DB_VECTOR);
A
Avi Kivity 已提交
1367

1368 1369 1370 1371
	set_intercept(svm, INTERCEPT_INTR);
	set_intercept(svm, INTERCEPT_NMI);
	set_intercept(svm, INTERCEPT_SMI);
	set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
A
Avi Kivity 已提交
1372
	set_intercept(svm, INTERCEPT_RDPMC);
1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
	set_intercept(svm, INTERCEPT_CPUID);
	set_intercept(svm, INTERCEPT_INVD);
	set_intercept(svm, INTERCEPT_HLT);
	set_intercept(svm, INTERCEPT_INVLPG);
	set_intercept(svm, INTERCEPT_INVLPGA);
	set_intercept(svm, INTERCEPT_IOIO_PROT);
	set_intercept(svm, INTERCEPT_MSR_PROT);
	set_intercept(svm, INTERCEPT_TASK_SWITCH);
	set_intercept(svm, INTERCEPT_SHUTDOWN);
	set_intercept(svm, INTERCEPT_VMRUN);
	set_intercept(svm, INTERCEPT_VMMCALL);
	set_intercept(svm, INTERCEPT_VMLOAD);
	set_intercept(svm, INTERCEPT_VMSAVE);
	set_intercept(svm, INTERCEPT_STGI);
	set_intercept(svm, INTERCEPT_CLGI);
	set_intercept(svm, INTERCEPT_SKINIT);
	set_intercept(svm, INTERCEPT_WBINVD);
J
Joerg Roedel 已提交
1390
	set_intercept(svm, INTERCEPT_XSETBV);
B
Brijesh Singh 已提交
1391
	set_intercept(svm, INTERCEPT_RSM);
A
Avi Kivity 已提交
1392

1393 1394 1395 1396 1397
	if (!kvm_mwait_in_guest()) {
		set_intercept(svm, INTERCEPT_MONITOR);
		set_intercept(svm, INTERCEPT_MWAIT);
	}

1398 1399
	control->iopm_base_pa = __sme_set(iopm_base);
	control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
A
Avi Kivity 已提交
1400 1401 1402 1403 1404 1405 1406 1407 1408
	control->int_ctl = V_INTR_MASKING_MASK;

	init_seg(&save->es);
	init_seg(&save->ss);
	init_seg(&save->ds);
	init_seg(&save->fs);
	init_seg(&save->gs);

	save->cs.selector = 0xf000;
1409
	save->cs.base = 0xffff0000;
A
Avi Kivity 已提交
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420
	/* Executable/Readable Code Segment */
	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
	save->cs.limit = 0xffff;

	save->gdtr.limit = 0xffff;
	save->idtr.limit = 0xffff;

	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);

P
Paolo Bonzini 已提交
1421
	svm_set_efer(&svm->vcpu, 0);
M
Mike Day 已提交
1422
	save->dr6 = 0xffff0ff0;
1423
	kvm_set_rflags(&svm->vcpu, 2);
A
Avi Kivity 已提交
1424
	save->rip = 0x0000fff0;
1425
	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
A
Avi Kivity 已提交
1426

J
Joerg Roedel 已提交
1427
	/*
1428
	 * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1429
	 * It also updates the guest-visible cr0 value.
A
Avi Kivity 已提交
1430
	 */
1431
	svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1432
	kvm_mmu_reset_context(&svm->vcpu);
1433

1434
	save->cr4 = X86_CR4_PAE;
A
Avi Kivity 已提交
1435
	/* rdx = ?? */
1436 1437 1438

	if (npt_enabled) {
		/* Setup VMCB for Nested Paging */
1439
		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1440
		clr_intercept(svm, INTERCEPT_INVLPG);
1441
		clr_exception_intercept(svm, PF_VECTOR);
1442 1443
		clr_cr_intercept(svm, INTERCEPT_CR3_READ);
		clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1444
		save->g_pat = svm->vcpu.arch.pat;
1445 1446 1447
		save->cr3 = 0;
		save->cr4 = 0;
	}
1448
	svm->asid_generation = 0;
1449

1450
	svm->nested.vmcb = 0;
1451 1452
	svm->vcpu.arch.hflags = 0;

1453
	if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1454
		control->pause_filter_count = 3000;
1455
		set_intercept(svm, INTERCEPT_PAUSE);
1456 1457
	}

1458
	if (kvm_vcpu_apicv_active(&svm->vcpu))
1459 1460
		avic_init_vmcb(svm);

1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
	/*
	 * If hardware supports Virtual VMLOAD VMSAVE then enable it
	 * in VMCB and clear intercepts to avoid #VMEXIT.
	 */
	if (vls) {
		clr_intercept(svm, INTERCEPT_VMLOAD);
		clr_intercept(svm, INTERCEPT_VMSAVE);
		svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
	}

1471 1472 1473 1474 1475 1476
	if (vgif) {
		clr_intercept(svm, INTERCEPT_STGI);
		clr_intercept(svm, INTERCEPT_CLGI);
		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
	}

1477
	if (sev_guest(svm->vcpu.kvm)) {
B
Brijesh Singh 已提交
1478
		svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1479 1480
		clr_exception_intercept(svm, UD_VECTOR);
	}
B
Brijesh Singh 已提交
1481

1482 1483
	mark_all_dirty(svm->vmcb);

1484
	enable_gif(svm);
1485 1486 1487

}

1488 1489
static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
				       unsigned int index)
1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553
{
	u64 *avic_physical_id_table;
	struct kvm_arch *vm_data = &vcpu->kvm->arch;

	if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
		return NULL;

	avic_physical_id_table = page_address(vm_data->avic_physical_id_table_page);

	return &avic_physical_id_table[index];
}

/**
 * Note:
 * AVIC hardware walks the nested page table to check permissions,
 * but does not use the SPA address specified in the leaf page
 * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
 * field of the VMCB. Therefore, we set up the
 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
 */
static int avic_init_access_page(struct kvm_vcpu *vcpu)
{
	struct kvm *kvm = vcpu->kvm;
	int ret;

	if (kvm->arch.apic_access_page_done)
		return 0;

	ret = x86_set_memory_region(kvm,
				    APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
				    APIC_DEFAULT_PHYS_BASE,
				    PAGE_SIZE);
	if (ret)
		return ret;

	kvm->arch.apic_access_page_done = true;
	return 0;
}

static int avic_init_backing_page(struct kvm_vcpu *vcpu)
{
	int ret;
	u64 *entry, new_entry;
	int id = vcpu->vcpu_id;
	struct vcpu_svm *svm = to_svm(vcpu);

	ret = avic_init_access_page(vcpu);
	if (ret)
		return ret;

	if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
		return -EINVAL;

	if (!svm->vcpu.arch.apic->regs)
		return -EINVAL;

	svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);

	/* Setting AVIC backing page address in the phy APIC ID table */
	entry = avic_get_physical_id_entry(vcpu, id);
	if (!entry)
		return -EINVAL;

	new_entry = READ_ONCE(*entry);
1554 1555 1556
	new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
			      AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
			      AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
1557 1558 1559 1560 1561 1562 1563
	WRITE_ONCE(*entry, new_entry);

	svm->avic_physical_id_cache = entry;

	return 0;
}

B
Brijesh Singh 已提交
1564 1565
static void __sev_asid_free(int asid)
{
1566 1567
	struct svm_cpu_data *sd;
	int cpu, pos;
B
Brijesh Singh 已提交
1568 1569 1570

	pos = asid - 1;
	clear_bit(pos, sev_asid_bitmap);
1571 1572 1573 1574 1575

	for_each_possible_cpu(cpu) {
		sd = per_cpu(svm_data, cpu);
		sd->sev_vmcbs[pos] = NULL;
	}
B
Brijesh Singh 已提交
1576 1577 1578 1579 1580 1581 1582 1583 1584
}

static void sev_asid_free(struct kvm *kvm)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;

	__sev_asid_free(sev->asid);
}

1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
{
	struct sev_data_decommission *decommission;
	struct sev_data_deactivate *data;

	if (!handle)
		return;

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return;

	/* deactivate handle */
	data->handle = handle;
	sev_guest_deactivate(data, NULL);

	wbinvd_on_all_cpus();
	sev_guest_df_flush(NULL);
	kfree(data);

	decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
	if (!decommission)
		return;

	/* decommission handle */
	decommission->handle = handle;
	sev_guest_decommission(decommission, NULL);

	kfree(decommission);
}

1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692
static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
				    unsigned long ulen, unsigned long *n,
				    int write)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	unsigned long npages, npinned, size;
	unsigned long locked, lock_limit;
	struct page **pages;
	int first, last;

	/* Calculate number of pages. */
	first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
	last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
	npages = (last - first + 1);

	locked = sev->pages_locked + npages;
	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
		pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
		return NULL;
	}

	/* Avoid using vmalloc for smaller buffers. */
	size = npages * sizeof(struct page *);
	if (size > PAGE_SIZE)
		pages = vmalloc(size);
	else
		pages = kmalloc(size, GFP_KERNEL);

	if (!pages)
		return NULL;

	/* Pin the user virtual address. */
	npinned = get_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages);
	if (npinned != npages) {
		pr_err("SEV: Failure locking %lu pages.\n", npages);
		goto err;
	}

	*n = npages;
	sev->pages_locked = locked;

	return pages;

err:
	if (npinned > 0)
		release_pages(pages, npinned);

	kvfree(pages);
	return NULL;
}

static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
			     unsigned long npages)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;

	release_pages(pages, npages);
	kvfree(pages);
	sev->pages_locked -= npages;
}

static void sev_clflush_pages(struct page *pages[], unsigned long npages)
{
	uint8_t *page_virtual;
	unsigned long i;

	if (npages == 0 || pages == NULL)
		return;

	for (i = 0; i < npages; i++) {
		page_virtual = kmap_atomic(pages[i]);
		clflush_cache_range(page_virtual, PAGE_SIZE);
		kunmap_atomic(page_virtual);
	}
}

1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708
static void __unregister_enc_region_locked(struct kvm *kvm,
					   struct enc_region *region)
{
	/*
	 * The guest may change the memory encryption attribute from C=0 -> C=1
	 * or vice versa for this memory range. Lets make sure caches are
	 * flushed to ensure that guest data gets written into memory with
	 * correct C-bit.
	 */
	sev_clflush_pages(region->pages, region->npages);

	sev_unpin_memory(kvm, region->pages, region->npages);
	list_del(&region->list);
	kfree(region);
}

B
Brijesh Singh 已提交
1709 1710
static void sev_vm_destroy(struct kvm *kvm)
{
1711
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
1712 1713
	struct list_head *head = &sev->regions_list;
	struct list_head *pos, *q;
1714

B
Brijesh Singh 已提交
1715 1716 1717
	if (!sev_guest(kvm))
		return;

1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
	mutex_lock(&kvm->lock);

	/*
	 * if userspace was terminated before unregistering the memory regions
	 * then lets unpin all the registered memory.
	 */
	if (!list_empty(head)) {
		list_for_each_safe(pos, q, head) {
			__unregister_enc_region_locked(kvm,
				list_entry(pos, struct enc_region, list));
		}
	}

	mutex_unlock(&kvm->lock);

1733
	sev_unbind_asid(kvm, sev->handle);
B
Brijesh Singh 已提交
1734 1735 1736
	sev_asid_free(kvm);
}

1737 1738
static void avic_vm_destroy(struct kvm *kvm)
{
1739
	unsigned long flags;
1740 1741
	struct kvm_arch *vm_data = &kvm->arch;

1742 1743 1744
	if (!avic)
		return;

1745 1746 1747 1748
	if (vm_data->avic_logical_id_table_page)
		__free_page(vm_data->avic_logical_id_table_page);
	if (vm_data->avic_physical_id_table_page)
		__free_page(vm_data->avic_physical_id_table_page);
1749 1750 1751 1752

	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
	hash_del(&vm_data->hnode);
	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1753 1754
}

B
Brijesh Singh 已提交
1755 1756 1757 1758 1759 1760
static void svm_vm_destroy(struct kvm *kvm)
{
	avic_vm_destroy(kvm);
	sev_vm_destroy(kvm);
}

1761 1762
static int avic_vm_init(struct kvm *kvm)
{
1763
	unsigned long flags;
1764
	int err = -ENOMEM;
1765 1766 1767
	struct kvm_arch *vm_data = &kvm->arch;
	struct page *p_page;
	struct page *l_page;
1768 1769
	struct kvm_arch *ka;
	u32 vm_id;
1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789

	if (!avic)
		return 0;

	/* Allocating physical APIC ID table (4KB) */
	p_page = alloc_page(GFP_KERNEL);
	if (!p_page)
		goto free_avic;

	vm_data->avic_physical_id_table_page = p_page;
	clear_page(page_address(p_page));

	/* Allocating logical APIC ID table (4KB) */
	l_page = alloc_page(GFP_KERNEL);
	if (!l_page)
		goto free_avic;

	vm_data->avic_logical_id_table_page = l_page;
	clear_page(page_address(l_page));

1790
	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806
 again:
	vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
	if (vm_id == 0) { /* id is 1-based, zero is not okay */
		next_vm_id_wrapped = 1;
		goto again;
	}
	/* Is it still in use? Only possible if wrapped at least once */
	if (next_vm_id_wrapped) {
		hash_for_each_possible(svm_vm_data_hash, ka, hnode, vm_id) {
			struct kvm *k2 = container_of(ka, struct kvm, arch);
			struct kvm_arch *vd2 = &k2->arch;
			if (vd2->avic_vm_id == vm_id)
				goto again;
		}
	}
	vm_data->avic_vm_id = vm_id;
1807 1808 1809
	hash_add(svm_vm_data_hash, &vm_data->hnode, vm_data->avic_vm_id);
	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);

1810 1811 1812 1813 1814
	return 0;

free_avic:
	avic_vm_destroy(kvm);
	return err;
A
Avi Kivity 已提交
1815 1816
}

1817 1818
static inline int
avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
1819
{
1820 1821 1822
	int ret = 0;
	unsigned long flags;
	struct amd_svm_iommu_ir *ir;
1823 1824
	struct vcpu_svm *svm = to_svm(vcpu);

1825 1826
	if (!kvm_arch_has_assigned_device(vcpu->kvm))
		return 0;
1827

1828 1829 1830 1831 1832
	/*
	 * Here, we go through the per-vcpu ir_list to update all existing
	 * interrupt remapping table entry targeting this vcpu.
	 */
	spin_lock_irqsave(&svm->ir_list_lock, flags);
1833

1834 1835
	if (list_empty(&svm->ir_list))
		goto out;
1836

1837 1838 1839 1840 1841 1842 1843 1844
	list_for_each_entry(ir, &svm->ir_list, node) {
		ret = amd_iommu_update_ga(cpu, r, ir->data);
		if (ret)
			break;
	}
out:
	spin_unlock_irqrestore(&svm->ir_list_lock, flags);
	return ret;
1845 1846 1847 1848 1849 1850
}

static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
	u64 entry;
	/* ID = 0xff (broadcast), ID > 0xff (reserved) */
1851
	int h_physical_id = kvm_cpu_get_apicid(cpu);
1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
	struct vcpu_svm *svm = to_svm(vcpu);

	if (!kvm_vcpu_apicv_active(vcpu))
		return;

	if (WARN_ON(h_physical_id >= AVIC_MAX_PHYSICAL_ID_COUNT))
		return;

	entry = READ_ONCE(*(svm->avic_physical_id_cache));
	WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);

	entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
	entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);

	entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
	if (svm->avic_is_running)
		entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;

	WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
1871 1872
	avic_update_iommu_vcpu_affinity(vcpu, h_physical_id,
					svm->avic_is_running);
1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883
}

static void avic_vcpu_put(struct kvm_vcpu *vcpu)
{
	u64 entry;
	struct vcpu_svm *svm = to_svm(vcpu);

	if (!kvm_vcpu_apicv_active(vcpu))
		return;

	entry = READ_ONCE(*(svm->avic_physical_id_cache));
1884 1885 1886
	if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
		avic_update_iommu_vcpu_affinity(vcpu, -1, 0);

1887 1888
	entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
	WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
A
Avi Kivity 已提交
1889 1890
}

1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904
/**
 * This function is called during VCPU halt/unhalt.
 */
static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->avic_is_running = is_run;
	if (is_run)
		avic_vcpu_load(vcpu, vcpu->cpu);
	else
		avic_vcpu_put(vcpu);
}

1905
static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1906 1907
{
	struct vcpu_svm *svm = to_svm(vcpu);
1908 1909
	u32 dummy;
	u32 eax = 1;
1910

1911
	vcpu->arch.microcode_version = 0x01000065;
1912 1913
	svm->spec_ctrl = 0;

1914 1915 1916 1917 1918 1919
	if (!init_event) {
		svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
					   MSR_IA32_APICBASE_ENABLE;
		if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
			svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
	}
P
Paolo Bonzini 已提交
1920
	init_vmcb(svm);
A
Avi Kivity 已提交
1921

1922
	kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true);
1923
	kvm_register_write(vcpu, VCPU_REGS_RDX, eax);
1924 1925 1926

	if (kvm_vcpu_apicv_active(vcpu) && !init_event)
		avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
1927 1928
}

1929 1930 1931 1932
static int avic_init_vcpu(struct vcpu_svm *svm)
{
	int ret;

1933
	if (!kvm_vcpu_apicv_active(&svm->vcpu))
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945
		return 0;

	ret = avic_init_backing_page(&svm->vcpu);
	if (ret)
		return ret;

	INIT_LIST_HEAD(&svm->ir_list);
	spin_lock_init(&svm->ir_list_lock);

	return ret;
}

R
Rusty Russell 已提交
1946
static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
A
Avi Kivity 已提交
1947
{
1948
	struct vcpu_svm *svm;
A
Avi Kivity 已提交
1949
	struct page *page;
1950
	struct page *msrpm_pages;
A
Alexander Graf 已提交
1951
	struct page *hsave_page;
A
Alexander Graf 已提交
1952
	struct page *nested_msrpm_pages;
R
Rusty Russell 已提交
1953
	int err;
A
Avi Kivity 已提交
1954

1955
	svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
R
Rusty Russell 已提交
1956 1957 1958 1959 1960 1961 1962 1963 1964
	if (!svm) {
		err = -ENOMEM;
		goto out;
	}

	err = kvm_vcpu_init(&svm->vcpu, kvm, id);
	if (err)
		goto free_svm;

1965
	err = -ENOMEM;
A
Avi Kivity 已提交
1966
	page = alloc_page(GFP_KERNEL);
1967
	if (!page)
R
Rusty Russell 已提交
1968
		goto uninit;
A
Avi Kivity 已提交
1969

1970 1971
	msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
	if (!msrpm_pages)
1972
		goto free_page1;
A
Alexander Graf 已提交
1973 1974 1975

	nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
	if (!nested_msrpm_pages)
1976
		goto free_page2;
1977

A
Alexander Graf 已提交
1978 1979
	hsave_page = alloc_page(GFP_KERNEL);
	if (!hsave_page)
1980 1981
		goto free_page3;

1982 1983 1984
	err = avic_init_vcpu(svm);
	if (err)
		goto free_page4;
1985

1986 1987 1988 1989 1990
	/* We initialize this flag to true to make sure that the is_running
	 * bit would be set the first time the vcpu is loaded.
	 */
	svm->avic_is_running = true;

1991
	svm->nested.hsave = page_address(hsave_page);
A
Alexander Graf 已提交
1992

1993 1994 1995
	svm->msrpm = page_address(msrpm_pages);
	svm_vcpu_init_msrpm(svm->msrpm);

1996
	svm->nested.msrpm = page_address(nested_msrpm_pages);
1997
	svm_vcpu_init_msrpm(svm->nested.msrpm);
A
Alexander Graf 已提交
1998

1999 2000
	svm->vmcb = page_address(page);
	clear_page(svm->vmcb);
2001
	svm->vmcb_pa = __sme_set(page_to_pfn(page) << PAGE_SHIFT);
2002
	svm->asid_generation = 0;
P
Paolo Bonzini 已提交
2003
	init_vmcb(svm);
A
Avi Kivity 已提交
2004

2005 2006
	svm_init_osvw(&svm->vcpu);

R
Rusty Russell 已提交
2007
	return &svm->vcpu;
2008

2009 2010
free_page4:
	__free_page(hsave_page);
2011 2012 2013 2014 2015 2016
free_page3:
	__free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
free_page2:
	__free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
free_page1:
	__free_page(page);
R
Rusty Russell 已提交
2017 2018 2019
uninit:
	kvm_vcpu_uninit(&svm->vcpu);
free_svm:
2020
	kmem_cache_free(kvm_vcpu_cache, svm);
R
Rusty Russell 已提交
2021 2022
out:
	return ERR_PTR(err);
A
Avi Kivity 已提交
2023 2024 2025 2026
}

static void svm_free_vcpu(struct kvm_vcpu *vcpu)
{
2027 2028
	struct vcpu_svm *svm = to_svm(vcpu);

2029
	__free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT));
2030
	__free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
2031 2032
	__free_page(virt_to_page(svm->nested.hsave));
	__free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
R
Rusty Russell 已提交
2033
	kvm_vcpu_uninit(vcpu);
2034
	kmem_cache_free(kvm_vcpu_cache, svm);
A
Ashok Raj 已提交
2035 2036 2037 2038 2039
	/*
	 * The vmcb page can be recycled, causing a false negative in
	 * svm_vcpu_load(). So do a full IBPB now.
	 */
	indirect_branch_prediction_barrier();
A
Avi Kivity 已提交
2040 2041
}

2042
static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
A
Avi Kivity 已提交
2043
{
2044
	struct vcpu_svm *svm = to_svm(vcpu);
A
Ashok Raj 已提交
2045
	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2046
	int i;
2047 2048

	if (unlikely(cpu != vcpu->cpu)) {
2049
		svm->asid_generation = 0;
2050
		mark_all_dirty(svm->vmcb);
2051
	}
2052

2053 2054 2055
#ifdef CONFIG_X86_64
	rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
#endif
2056 2057 2058 2059
	savesegment(fs, svm->host.fs);
	savesegment(gs, svm->host.gs);
	svm->host.ldt = kvm_read_ldt();

2060
	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2061
		rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2062

2063 2064 2065 2066 2067 2068
	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
		u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
		if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
			__this_cpu_write(current_tsc_ratio, tsc_ratio);
			wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
		}
2069
	}
P
Paolo Bonzini 已提交
2070 2071 2072
	/* This assumes that the kernel never uses MSR_TSC_AUX */
	if (static_cpu_has(X86_FEATURE_RDTSCP))
		wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
2073

A
Ashok Raj 已提交
2074 2075 2076 2077
	if (sd->current_vmcb != svm->vmcb) {
		sd->current_vmcb = svm->vmcb;
		indirect_branch_prediction_barrier();
	}
2078
	avic_vcpu_load(vcpu, cpu);
A
Avi Kivity 已提交
2079 2080 2081 2082
}

static void svm_vcpu_put(struct kvm_vcpu *vcpu)
{
2083
	struct vcpu_svm *svm = to_svm(vcpu);
2084 2085
	int i;

2086 2087
	avic_vcpu_put(vcpu);

2088
	++vcpu->stat.host_state_reload;
2089 2090 2091
	kvm_load_ldt(svm->host.ldt);
#ifdef CONFIG_X86_64
	loadsegment(fs, svm->host.fs);
2092
	wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
2093
	load_gs_index(svm->host.gs);
2094
#else
2095
#ifdef CONFIG_X86_32_LAZY_GS
2096
	loadsegment(gs, svm->host.gs);
2097
#endif
2098
#endif
2099
	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2100
		wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
A
Avi Kivity 已提交
2101 2102
}

2103 2104 2105 2106 2107 2108 2109 2110 2111 2112
static void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
{
	avic_set_running(vcpu, false);
}

static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu)
{
	avic_set_running(vcpu, true);
}

A
Avi Kivity 已提交
2113 2114
static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
{
2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125
	struct vcpu_svm *svm = to_svm(vcpu);
	unsigned long rflags = svm->vmcb->save.rflags;

	if (svm->nmi_singlestep) {
		/* Hide our flags if they were not set by the guest */
		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
			rflags &= ~X86_EFLAGS_TF;
		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
			rflags &= ~X86_EFLAGS_RF;
	}
	return rflags;
A
Avi Kivity 已提交
2126 2127 2128 2129
}

static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
2130 2131 2132
	if (to_svm(vcpu)->nmi_singlestep)
		rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);

P
Paolo Bonzini 已提交
2133
       /*
A
Andrea Gelmini 已提交
2134
        * Any change of EFLAGS.VM is accompanied by a reload of SS
P
Paolo Bonzini 已提交
2135 2136 2137
        * (caused by either a task switch or an inter-privilege IRET),
        * so we do not need to update the CPL here.
        */
2138
	to_svm(vcpu)->vmcb->save.rflags = rflags;
A
Avi Kivity 已提交
2139 2140
}

A
Avi Kivity 已提交
2141 2142 2143 2144 2145
static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
{
	switch (reg) {
	case VCPU_EXREG_PDPTR:
		BUG_ON(!npt_enabled);
2146
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
A
Avi Kivity 已提交
2147 2148 2149 2150 2151 2152
		break;
	default:
		BUG();
	}
}

2153 2154
static void svm_set_vintr(struct vcpu_svm *svm)
{
2155
	set_intercept(svm, INTERCEPT_VINTR);
2156 2157 2158 2159
}

static void svm_clear_vintr(struct vcpu_svm *svm)
{
2160
	clr_intercept(svm, INTERCEPT_VINTR);
2161 2162
}

A
Avi Kivity 已提交
2163 2164
static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
{
2165
	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
A
Avi Kivity 已提交
2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177

	switch (seg) {
	case VCPU_SREG_CS: return &save->cs;
	case VCPU_SREG_DS: return &save->ds;
	case VCPU_SREG_ES: return &save->es;
	case VCPU_SREG_FS: return &save->fs;
	case VCPU_SREG_GS: return &save->gs;
	case VCPU_SREG_SS: return &save->ss;
	case VCPU_SREG_TR: return &save->tr;
	case VCPU_SREG_LDTR: return &save->ldtr;
	}
	BUG();
A
Al Viro 已提交
2178
	return NULL;
A
Avi Kivity 已提交
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202
}

static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	return s->base;
}

static void svm_get_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	var->base = s->base;
	var->limit = s->limit;
	var->selector = s->selector;
	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
2203 2204 2205 2206 2207 2208 2209 2210 2211 2212

	/*
	 * AMD CPUs circa 2014 track the G bit for all segments except CS.
	 * However, the SVM spec states that the G bit is not observed by the
	 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
	 * So let's synthesize a legal G bit for all segments, this helps
	 * running KVM nested. It also helps cross-vendor migration, because
	 * Intel's vmentry has a check on the 'G' bit.
	 */
	var->g = s->limit > 0xfffff;
2213

J
Joerg Roedel 已提交
2214 2215
	/*
	 * AMD's VMCB does not have an explicit unusable field, so emulate it
2216 2217
	 * for cross vendor migration purposes by "not present"
	 */
2218
	var->unusable = !var->present;
2219

2220 2221 2222 2223 2224 2225
	switch (seg) {
	case VCPU_SREG_TR:
		/*
		 * Work around a bug where the busy flag in the tr selector
		 * isn't exposed
		 */
2226
		var->type |= 0x2;
2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241
		break;
	case VCPU_SREG_DS:
	case VCPU_SREG_ES:
	case VCPU_SREG_FS:
	case VCPU_SREG_GS:
		/*
		 * The accessed bit must always be set in the segment
		 * descriptor cache, although it can be cleared in the
		 * descriptor, the cached bit always remains at 1. Since
		 * Intel has a check on this, set it here to support
		 * cross-vendor migration.
		 */
		if (!var->unusable)
			var->type |= 0x1;
		break;
2242
	case VCPU_SREG_SS:
J
Joerg Roedel 已提交
2243 2244
		/*
		 * On AMD CPUs sometimes the DB bit in the segment
2245 2246 2247 2248 2249 2250
		 * descriptor is left as 1, although the whole segment has
		 * been made unusable. Clear it here to pass an Intel VMX
		 * entry check when cross vendor migrating.
		 */
		if (var->unusable)
			var->db = 0;
2251
		/* This is symmetric with svm_set_segment() */
J
Jan Kiszka 已提交
2252
		var->dpl = to_svm(vcpu)->vmcb->save.cpl;
2253
		break;
2254
	}
A
Avi Kivity 已提交
2255 2256
}

2257 2258 2259 2260 2261 2262 2263
static int svm_get_cpl(struct kvm_vcpu *vcpu)
{
	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;

	return save->cpl;
}

2264
static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2265
{
2266 2267
	struct vcpu_svm *svm = to_svm(vcpu);

2268 2269
	dt->size = svm->vmcb->save.idtr.limit;
	dt->address = svm->vmcb->save.idtr.base;
A
Avi Kivity 已提交
2270 2271
}

2272
static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2273
{
2274 2275
	struct vcpu_svm *svm = to_svm(vcpu);

2276 2277
	svm->vmcb->save.idtr.limit = dt->size;
	svm->vmcb->save.idtr.base = dt->address ;
2278
	mark_dirty(svm->vmcb, VMCB_DT);
A
Avi Kivity 已提交
2279 2280
}

2281
static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2282
{
2283 2284
	struct vcpu_svm *svm = to_svm(vcpu);

2285 2286
	dt->size = svm->vmcb->save.gdtr.limit;
	dt->address = svm->vmcb->save.gdtr.base;
A
Avi Kivity 已提交
2287 2288
}

2289
static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
A
Avi Kivity 已提交
2290
{
2291 2292
	struct vcpu_svm *svm = to_svm(vcpu);

2293 2294
	svm->vmcb->save.gdtr.limit = dt->size;
	svm->vmcb->save.gdtr.base = dt->address ;
2295
	mark_dirty(svm->vmcb, VMCB_DT);
A
Avi Kivity 已提交
2296 2297
}

2298 2299 2300 2301
static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
{
}

2302 2303 2304 2305
static void svm_decache_cr3(struct kvm_vcpu *vcpu)
{
}

2306
static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
2307 2308 2309
{
}

A
Avi Kivity 已提交
2310 2311 2312 2313 2314
static void update_cr0_intercept(struct vcpu_svm *svm)
{
	ulong gcr0 = svm->vcpu.arch.cr0;
	u64 *hcr0 = &svm->vmcb->save.cr0;

2315 2316
	*hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
		| (gcr0 & SVM_CR0_SELECTIVE_MASK);
A
Avi Kivity 已提交
2317

2318
	mark_dirty(svm->vmcb, VMCB_CR);
A
Avi Kivity 已提交
2319

2320
	if (gcr0 == *hcr0) {
2321 2322
		clr_cr_intercept(svm, INTERCEPT_CR0_READ);
		clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
A
Avi Kivity 已提交
2323
	} else {
2324 2325
		set_cr_intercept(svm, INTERCEPT_CR0_READ);
		set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
A
Avi Kivity 已提交
2326 2327 2328
	}
}

A
Avi Kivity 已提交
2329 2330
static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
2331 2332
	struct vcpu_svm *svm = to_svm(vcpu);

2333
#ifdef CONFIG_X86_64
2334
	if (vcpu->arch.efer & EFER_LME) {
2335
		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
2336
			vcpu->arch.efer |= EFER_LMA;
2337
			svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
A
Avi Kivity 已提交
2338 2339
		}

M
Mike Day 已提交
2340
		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
2341
			vcpu->arch.efer &= ~EFER_LMA;
2342
			svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
A
Avi Kivity 已提交
2343 2344 2345
		}
	}
#endif
2346
	vcpu->arch.cr0 = cr0;
2347 2348 2349

	if (!npt_enabled)
		cr0 |= X86_CR0_PG | X86_CR0_WP;
2350

2351 2352 2353 2354 2355 2356 2357
	/*
	 * re-enable caching here because the QEMU bios
	 * does not do it - this results in some delay at
	 * reboot
	 */
	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
		cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
2358
	svm->vmcb->save.cr0 = cr0;
2359
	mark_dirty(svm->vmcb, VMCB_CR);
A
Avi Kivity 已提交
2360
	update_cr0_intercept(svm);
A
Avi Kivity 已提交
2361 2362
}

2363
static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
A
Avi Kivity 已提交
2364
{
2365
	unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
2366 2367
	unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;

2368 2369 2370
	if (cr4 & X86_CR4_VMXE)
		return 1;

2371
	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
2372
		svm_flush_tlb(vcpu, true);
2373

2374 2375 2376
	vcpu->arch.cr4 = cr4;
	if (!npt_enabled)
		cr4 |= X86_CR4_PAE;
2377
	cr4 |= host_cr4_mce;
2378
	to_svm(vcpu)->vmcb->save.cr4 = cr4;
2379
	mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
2380
	return 0;
A
Avi Kivity 已提交
2381 2382 2383 2384 2385
}

static void svm_set_segment(struct kvm_vcpu *vcpu,
			    struct kvm_segment *var, int seg)
{
2386
	struct vcpu_svm *svm = to_svm(vcpu);
A
Avi Kivity 已提交
2387 2388 2389 2390 2391
	struct vmcb_seg *s = svm_seg(vcpu, seg);

	s->base = var->base;
	s->limit = var->limit;
	s->selector = var->selector;
2392 2393 2394 2395 2396 2397 2398 2399
	s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
	s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
	s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
	s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
	s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
	s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
	s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
	s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
P
Paolo Bonzini 已提交
2400 2401 2402 2403 2404 2405 2406 2407

	/*
	 * This is always accurate, except if SYSRET returned to a segment
	 * with SS.DPL != 3.  Intel does not have this quirk, and always
	 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
	 * would entail passing the CPL to userspace and back.
	 */
	if (seg == VCPU_SREG_SS)
2408 2409
		/* This is symmetric with svm_get_segment() */
		svm->vmcb->save.cpl = (var->dpl & 3);
A
Avi Kivity 已提交
2410

2411
	mark_dirty(svm->vmcb, VMCB_SEG);
A
Avi Kivity 已提交
2412 2413
}

2414
static void update_bp_intercept(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2415
{
J
Jan Kiszka 已提交
2416 2417
	struct vcpu_svm *svm = to_svm(vcpu);

2418
	clr_exception_intercept(svm, BP_VECTOR);
2419

J
Jan Kiszka 已提交
2420 2421
	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2422
			set_exception_intercept(svm, BP_VECTOR);
J
Jan Kiszka 已提交
2423 2424
	} else
		vcpu->guest_debug = 0;
2425 2426
}

2427
static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
A
Avi Kivity 已提交
2428
{
2429 2430
	if (sd->next_asid > sd->max_asid) {
		++sd->asid_generation;
2431
		sd->next_asid = sd->min_asid;
2432
		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
A
Avi Kivity 已提交
2433 2434
	}

2435 2436
	svm->asid_generation = sd->asid_generation;
	svm->vmcb->control.asid = sd->next_asid++;
2437 2438

	mark_dirty(svm->vmcb, VMCB_ASID);
A
Avi Kivity 已提交
2439 2440
}

J
Jan Kiszka 已提交
2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453
static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
{
	return to_svm(vcpu)->vmcb->save.dr6;
}

static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->save.dr6 = value;
	mark_dirty(svm->vmcb, VMCB_DR);
}

2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468
static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	get_debugreg(vcpu->arch.db[0], 0);
	get_debugreg(vcpu->arch.db[1], 1);
	get_debugreg(vcpu->arch.db[2], 2);
	get_debugreg(vcpu->arch.db[3], 3);
	vcpu->arch.dr6 = svm_get_dr6(vcpu);
	vcpu->arch.dr7 = svm->vmcb->save.dr7;

	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
	set_dr_intercepts(svm);
}

2469
static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
A
Avi Kivity 已提交
2470
{
2471 2472
	struct vcpu_svm *svm = to_svm(vcpu);

2473
	svm->vmcb->save.dr7 = value;
2474
	mark_dirty(svm->vmcb, VMCB_DR);
A
Avi Kivity 已提交
2475 2476
}

A
Avi Kivity 已提交
2477
static int pf_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2478
{
2479
	u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2480
	u64 error_code = svm->vmcb->control.exit_info_1;
A
Avi Kivity 已提交
2481

2482
	return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
2483 2484
			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
			svm->vmcb->control.insn_bytes : NULL,
2485 2486 2487 2488 2489
			svm->vmcb->control.insn_len);
}

static int npf_interception(struct vcpu_svm *svm)
{
2490
	u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2491 2492 2493 2494
	u64 error_code = svm->vmcb->control.exit_info_1;

	trace_kvm_page_fault(fault_address, error_code);
	return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
2495 2496
			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
			svm->vmcb->control.insn_bytes : NULL,
2497
			svm->vmcb->control.insn_len);
A
Avi Kivity 已提交
2498 2499
}

A
Avi Kivity 已提交
2500
static int db_interception(struct vcpu_svm *svm)
J
Jan Kiszka 已提交
2501
{
A
Avi Kivity 已提交
2502 2503
	struct kvm_run *kvm_run = svm->vcpu.run;

J
Jan Kiszka 已提交
2504
	if (!(svm->vcpu.guest_debug &
2505
	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
J
Jan Kiszka 已提交
2506
		!svm->nmi_singlestep) {
J
Jan Kiszka 已提交
2507 2508 2509
		kvm_queue_exception(&svm->vcpu, DB_VECTOR);
		return 1;
	}
2510

J
Jan Kiszka 已提交
2511
	if (svm->nmi_singlestep) {
2512
		disable_nmi_singlestep(svm);
2513 2514 2515
	}

	if (svm->vcpu.guest_debug &
J
Joerg Roedel 已提交
2516
	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2517 2518 2519 2520 2521 2522 2523 2524
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
		kvm_run->debug.arch.pc =
			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
		kvm_run->debug.arch.exception = DB_VECTOR;
		return 0;
	}

	return 1;
J
Jan Kiszka 已提交
2525 2526
}

A
Avi Kivity 已提交
2527
static int bp_interception(struct vcpu_svm *svm)
J
Jan Kiszka 已提交
2528
{
A
Avi Kivity 已提交
2529 2530
	struct kvm_run *kvm_run = svm->vcpu.run;

J
Jan Kiszka 已提交
2531 2532 2533 2534 2535 2536
	kvm_run->exit_reason = KVM_EXIT_DEBUG;
	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
	kvm_run->debug.arch.exception = BP_VECTOR;
	return 0;
}

A
Avi Kivity 已提交
2537
static int ud_interception(struct vcpu_svm *svm)
2538 2539 2540
{
	int er;

2541
	er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
2542 2543
	if (er == EMULATE_USER_EXIT)
		return 0;
2544
	if (er != EMULATE_DONE)
2545
		kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2546 2547 2548
	return 1;
}

2549 2550 2551 2552 2553 2554
static int ac_interception(struct vcpu_svm *svm)
{
	kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
	return 1;
}

2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593
static bool is_erratum_383(void)
{
	int err, i;
	u64 value;

	if (!erratum_383_found)
		return false;

	value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
	if (err)
		return false;

	/* Bit 62 may or may not be set for this mce */
	value &= ~(1ULL << 62);

	if (value != 0xb600000000010015ULL)
		return false;

	/* Clear MCi_STATUS registers */
	for (i = 0; i < 6; ++i)
		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);

	value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
	if (!err) {
		u32 low, high;

		value &= ~(1ULL << 2);
		low    = lower_32_bits(value);
		high   = upper_32_bits(value);

		native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
	}

	/* Flush tlb to evict multi-match entries */
	__flush_tlb_all();

	return true;
}

2594
static void svm_handle_mce(struct vcpu_svm *svm)
2595
{
2596 2597 2598 2599 2600 2601 2602
	if (is_erratum_383()) {
		/*
		 * Erratum 383 triggered. Guest state is corrupt so kill the
		 * guest.
		 */
		pr_err("KVM: Guest triggered AMD Erratum 383\n");

2603
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
2604 2605 2606 2607

		return;
	}

2608 2609 2610 2611 2612 2613 2614 2615
	/*
	 * On an #MC intercept the MCE handler is not called automatically in
	 * the host. So do it by hand here.
	 */
	asm volatile (
		"int $0x12\n");
	/* not sure if we ever come back to this point */

2616 2617 2618 2619 2620
	return;
}

static int mc_interception(struct vcpu_svm *svm)
{
2621 2622 2623
	return 1;
}

A
Avi Kivity 已提交
2624
static int shutdown_interception(struct vcpu_svm *svm)
2625
{
A
Avi Kivity 已提交
2626 2627
	struct kvm_run *kvm_run = svm->vcpu.run;

2628 2629 2630 2631
	/*
	 * VMCB is undefined after a SHUTDOWN intercept
	 * so reinitialize it.
	 */
2632
	clear_page(svm->vmcb);
P
Paolo Bonzini 已提交
2633
	init_vmcb(svm);
2634 2635 2636 2637 2638

	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
	return 0;
}

A
Avi Kivity 已提交
2639
static int io_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2640
{
2641
	struct kvm_vcpu *vcpu = &svm->vcpu;
M
Mike Day 已提交
2642
	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2643
	int size, in, string;
2644
	unsigned port;
A
Avi Kivity 已提交
2645

R
Rusty Russell 已提交
2646
	++svm->vcpu.stat.io_exits;
2647
	string = (io_info & SVM_IOIO_STR_MASK) != 0;
2648
	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2649
	if (string)
2650
		return emulate_instruction(vcpu, 0) == EMULATE_DONE;
2651

2652 2653
	port = io_info >> 16;
	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2654 2655
	svm->next_rip = svm->vmcb->control.exit_info_2;

2656
	return kvm_fast_pio(&svm->vcpu, size, port, in);
A
Avi Kivity 已提交
2657 2658
}

A
Avi Kivity 已提交
2659
static int nmi_interception(struct vcpu_svm *svm)
2660 2661 2662 2663
{
	return 1;
}

A
Avi Kivity 已提交
2664
static int intr_interception(struct vcpu_svm *svm)
2665 2666 2667 2668 2669
{
	++svm->vcpu.stat.irq_exits;
	return 1;
}

A
Avi Kivity 已提交
2670
static int nop_on_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2671 2672 2673 2674
{
	return 1;
}

A
Avi Kivity 已提交
2675
static int halt_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
2676
{
2677
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
R
Rusty Russell 已提交
2678
	return kvm_emulate_halt(&svm->vcpu);
A
Avi Kivity 已提交
2679 2680
}

A
Avi Kivity 已提交
2681
static int vmmcall_interception(struct vcpu_svm *svm)
2682
{
2683
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2684
	return kvm_emulate_hypercall(&svm->vcpu);
2685 2686
}

2687 2688 2689 2690 2691 2692 2693
static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	return svm->nested.nested_cr3;
}

2694 2695 2696 2697 2698 2699 2700
static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u64 cr3 = svm->nested.nested_cr3;
	u64 pdpte;
	int ret;

2701
	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte,
2702
				       offset_in_page(cr3) + index * 8, 8);
2703 2704 2705 2706 2707
	if (ret)
		return 0;
	return pdpte;
}

2708 2709 2710 2711 2712
static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
				   unsigned long root)
{
	struct vcpu_svm *svm = to_svm(vcpu);

2713
	svm->vmcb->control.nested_cr3 = __sme_set(root);
2714
	mark_dirty(svm->vmcb, VMCB_NPT);
2715
	svm_flush_tlb(vcpu, true);
2716 2717
}

2718 2719
static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
				       struct x86_exception *fault)
2720 2721 2722
{
	struct vcpu_svm *svm = to_svm(vcpu);

2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742
	if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
		/*
		 * TODO: track the cause of the nested page fault, and
		 * correctly fill in the high bits of exit_info_1.
		 */
		svm->vmcb->control.exit_code = SVM_EXIT_NPF;
		svm->vmcb->control.exit_code_hi = 0;
		svm->vmcb->control.exit_info_1 = (1ULL << 32);
		svm->vmcb->control.exit_info_2 = fault->address;
	}

	svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
	svm->vmcb->control.exit_info_1 |= fault->error_code;

	/*
	 * The present bit is always zero for page structure faults on real
	 * hardware.
	 */
	if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
		svm->vmcb->control.exit_info_1 &= ~1;
2743 2744 2745 2746

	nested_svm_vmexit(svm);
}

2747
static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
2748
{
2749 2750
	WARN_ON(mmu_is_nested(vcpu));
	kvm_init_shadow_mmu(vcpu);
2751 2752
	vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
	vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
2753
	vcpu->arch.mmu.get_pdptr         = nested_svm_get_tdp_pdptr;
2754
	vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
2755
	vcpu->arch.mmu.shadow_root_level = get_npt_level(vcpu);
2756
	reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
2757 2758 2759 2760 2761 2762 2763 2764
	vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
}

static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
{
	vcpu->arch.walk_mmu = &vcpu->arch.mmu;
}

2765 2766
static int nested_svm_check_permissions(struct vcpu_svm *svm)
{
2767 2768
	if (!(svm->vcpu.arch.efer & EFER_SVME) ||
	    !is_paging(&svm->vcpu)) {
2769 2770 2771 2772 2773 2774 2775 2776 2777
		kvm_queue_exception(&svm->vcpu, UD_VECTOR);
		return 1;
	}

	if (svm->vmcb->save.cpl) {
		kvm_inject_gp(&svm->vcpu, 0);
		return 1;
	}

2778
	return 0;
2779 2780
}

2781 2782 2783
static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
				      bool has_error_code, u32 error_code)
{
2784 2785
	int vmexit;

2786
	if (!is_guest_mode(&svm->vcpu))
2787
		return 0;
2788

2789 2790 2791 2792
	vmexit = nested_svm_intercept(svm);
	if (vmexit != NESTED_EXIT_DONE)
		return 0;

2793 2794 2795
	svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
	svm->vmcb->control.exit_code_hi = 0;
	svm->vmcb->control.exit_info_1 = error_code;
2796 2797 2798 2799 2800 2801 2802 2803 2804 2805

	/*
	 * FIXME: we should not write CR2 when L1 intercepts an L2 #PF exception.
	 * The fix is to add the ancillary datum (CR2 or DR6) to structs
	 * kvm_queued_exception and kvm_vcpu_events, so that CR2 and DR6 can be
	 * written only when inject_pending_event runs (DR6 would written here
	 * too).  This should be conditional on a new capability---if the
	 * capability is disabled, kvm_multiple_exception would write the
	 * ancillary information to CR2 or DR6, for backwards ABI-compatibility.
	 */
2806 2807 2808 2809
	if (svm->vcpu.arch.exception.nested_apf)
		svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
	else
		svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
2810

2811
	svm->nested.exit_required = true;
2812
	return vmexit;
2813 2814
}

2815 2816
/* This function returns true if it is save to enable the irq window */
static inline bool nested_svm_intr(struct vcpu_svm *svm)
2817
{
2818
	if (!is_guest_mode(&svm->vcpu))
2819
		return true;
2820

2821
	if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2822
		return true;
2823

2824
	if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
2825
		return false;
2826

2827 2828 2829 2830 2831 2832 2833 2834
	/*
	 * if vmexit was already requested (by intercepted exception
	 * for instance) do not overwrite it with "external interrupt"
	 * vmexit.
	 */
	if (svm->nested.exit_required)
		return false;

2835 2836 2837
	svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
	svm->vmcb->control.exit_info_1 = 0;
	svm->vmcb->control.exit_info_2 = 0;
2838

2839 2840 2841
	if (svm->nested.intercept & 1ULL) {
		/*
		 * The #vmexit can't be emulated here directly because this
G
Guo Chao 已提交
2842
		 * code path runs with irqs and preemption disabled. A
2843 2844 2845 2846
		 * #vmexit emulation might sleep. Only signal request for
		 * the #vmexit here.
		 */
		svm->nested.exit_required = true;
2847
		trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
2848
		return false;
2849 2850
	}

2851
	return true;
2852 2853
}

2854 2855 2856
/* This function returns true if it is save to enable the nmi window */
static inline bool nested_svm_nmi(struct vcpu_svm *svm)
{
2857
	if (!is_guest_mode(&svm->vcpu))
2858 2859 2860 2861 2862 2863 2864 2865 2866
		return true;

	if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
		return true;

	svm->vmcb->control.exit_code = SVM_EXIT_NMI;
	svm->nested.exit_required = true;

	return false;
2867 2868
}

2869
static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
2870 2871 2872
{
	struct page *page;

2873 2874
	might_sleep();

2875
	page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
2876 2877 2878
	if (is_error_page(page))
		goto error;

2879 2880 2881
	*_page = page;

	return kmap(page);
2882 2883 2884 2885 2886 2887 2888

error:
	kvm_inject_gp(&svm->vcpu, 0);

	return NULL;
}

2889
static void nested_svm_unmap(struct page *page)
2890
{
2891
	kunmap(page);
2892 2893 2894
	kvm_release_page_dirty(page);
}

2895 2896
static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
{
2897 2898 2899
	unsigned port, size, iopm_len;
	u16 val, mask;
	u8 start_bit;
2900
	u64 gpa;
2901

2902 2903
	if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
		return NESTED_EXIT_HOST;
2904

2905
	port = svm->vmcb->control.exit_info_1 >> 16;
2906 2907
	size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
		SVM_IOIO_SIZE_SHIFT;
2908
	gpa  = svm->nested.vmcb_iopm + (port / 8);
2909 2910 2911 2912
	start_bit = port % 8;
	iopm_len = (start_bit + size > 8) ? 2 : 1;
	mask = (0xf >> (4 - size)) << start_bit;
	val = 0;
2913

2914
	if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
2915
		return NESTED_EXIT_DONE;
2916

2917
	return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2918 2919
}

2920
static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
2921
{
2922 2923
	u32 offset, msr, value;
	int write, mask;
2924

2925
	if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2926
		return NESTED_EXIT_HOST;
2927

2928 2929 2930 2931
	msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
	offset = svm_msrpm_offset(msr);
	write  = svm->vmcb->control.exit_info_1 & 1;
	mask   = 1 << ((2 * (msr & 0xf)) + write);
2932

2933 2934
	if (offset == MSR_INVALID)
		return NESTED_EXIT_DONE;
2935

2936 2937
	/* Offset is in 32 bit units but need in 8 bit units */
	offset *= 4;
2938

2939
	if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
2940
		return NESTED_EXIT_DONE;
2941

2942
	return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2943 2944
}

2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969
/* DB exceptions for our internal use must not cause vmexit */
static int nested_svm_intercept_db(struct vcpu_svm *svm)
{
	unsigned long dr6;

	/* if we're not singlestepping, it's not ours */
	if (!svm->nmi_singlestep)
		return NESTED_EXIT_DONE;

	/* if it's not a singlestep exception, it's not ours */
	if (kvm_get_dr(&svm->vcpu, 6, &dr6))
		return NESTED_EXIT_DONE;
	if (!(dr6 & DR6_BS))
		return NESTED_EXIT_DONE;

	/* if the guest is singlestepping, it should get the vmexit */
	if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) {
		disable_nmi_singlestep(svm);
		return NESTED_EXIT_DONE;
	}

	/* it's ours, the nested hypervisor must not see this one */
	return NESTED_EXIT_HOST;
}

2970
static int nested_svm_exit_special(struct vcpu_svm *svm)
2971 2972
{
	u32 exit_code = svm->vmcb->control.exit_code;
2973

2974 2975 2976
	switch (exit_code) {
	case SVM_EXIT_INTR:
	case SVM_EXIT_NMI:
2977
	case SVM_EXIT_EXCP_BASE + MC_VECTOR:
2978 2979
		return NESTED_EXIT_HOST;
	case SVM_EXIT_NPF:
J
Joerg Roedel 已提交
2980
		/* For now we are always handling NPFs when using them */
2981 2982 2983 2984
		if (npt_enabled)
			return NESTED_EXIT_HOST;
		break;
	case SVM_EXIT_EXCP_BASE + PF_VECTOR:
G
Gleb Natapov 已提交
2985
		/* When we're shadowing, trap PFs, but not async PF */
2986
		if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0)
2987 2988 2989 2990
			return NESTED_EXIT_HOST;
		break;
	default:
		break;
2991 2992
	}

2993 2994 2995 2996 2997 2998
	return NESTED_EXIT_CONTINUE;
}

/*
 * If this function returns true, this #vmexit was already handled
 */
2999
static int nested_svm_intercept(struct vcpu_svm *svm)
3000 3001 3002 3003
{
	u32 exit_code = svm->vmcb->control.exit_code;
	int vmexit = NESTED_EXIT_HOST;

3004
	switch (exit_code) {
3005
	case SVM_EXIT_MSR:
3006
		vmexit = nested_svm_exit_handled_msr(svm);
3007
		break;
3008 3009 3010
	case SVM_EXIT_IOIO:
		vmexit = nested_svm_intercept_ioio(svm);
		break;
3011 3012 3013
	case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
		u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
		if (svm->nested.intercept_cr & bit)
3014
			vmexit = NESTED_EXIT_DONE;
3015 3016
		break;
	}
3017 3018 3019
	case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
		u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
		if (svm->nested.intercept_dr & bit)
3020
			vmexit = NESTED_EXIT_DONE;
3021 3022 3023 3024
		break;
	}
	case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
		u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
3025 3026 3027 3028 3029 3030
		if (svm->nested.intercept_exceptions & excp_bits) {
			if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR)
				vmexit = nested_svm_intercept_db(svm);
			else
				vmexit = NESTED_EXIT_DONE;
		}
G
Gleb Natapov 已提交
3031 3032
		/* async page fault always cause vmexit */
		else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
3033
			 svm->vcpu.arch.exception.nested_apf != 0)
G
Gleb Natapov 已提交
3034
			vmexit = NESTED_EXIT_DONE;
3035 3036
		break;
	}
3037 3038 3039 3040
	case SVM_EXIT_ERR: {
		vmexit = NESTED_EXIT_DONE;
		break;
	}
3041 3042
	default: {
		u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
J
Joerg Roedel 已提交
3043
		if (svm->nested.intercept & exit_bits)
3044
			vmexit = NESTED_EXIT_DONE;
3045 3046 3047
	}
	}

3048 3049 3050 3051 3052 3053 3054 3055 3056 3057
	return vmexit;
}

static int nested_svm_exit_handled(struct vcpu_svm *svm)
{
	int vmexit;

	vmexit = nested_svm_intercept(svm);

	if (vmexit == NESTED_EXIT_DONE)
3058 3059 3060
		nested_svm_vmexit(svm);

	return vmexit;
3061 3062
}

3063 3064 3065 3066 3067
static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
{
	struct vmcb_control_area *dst  = &dst_vmcb->control;
	struct vmcb_control_area *from = &from_vmcb->control;

3068
	dst->intercept_cr         = from->intercept_cr;
3069
	dst->intercept_dr         = from->intercept_dr;
3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089
	dst->intercept_exceptions = from->intercept_exceptions;
	dst->intercept            = from->intercept;
	dst->iopm_base_pa         = from->iopm_base_pa;
	dst->msrpm_base_pa        = from->msrpm_base_pa;
	dst->tsc_offset           = from->tsc_offset;
	dst->asid                 = from->asid;
	dst->tlb_ctl              = from->tlb_ctl;
	dst->int_ctl              = from->int_ctl;
	dst->int_vector           = from->int_vector;
	dst->int_state            = from->int_state;
	dst->exit_code            = from->exit_code;
	dst->exit_code_hi         = from->exit_code_hi;
	dst->exit_info_1          = from->exit_info_1;
	dst->exit_info_2          = from->exit_info_2;
	dst->exit_int_info        = from->exit_int_info;
	dst->exit_int_info_err    = from->exit_int_info_err;
	dst->nested_ctl           = from->nested_ctl;
	dst->event_inj            = from->event_inj;
	dst->event_inj_err        = from->event_inj_err;
	dst->nested_cr3           = from->nested_cr3;
3090
	dst->virt_ext              = from->virt_ext;
3091 3092
}

3093
static int nested_svm_vmexit(struct vcpu_svm *svm)
3094
{
3095
	struct vmcb *nested_vmcb;
3096
	struct vmcb *hsave = svm->nested.hsave;
J
Joerg Roedel 已提交
3097
	struct vmcb *vmcb = svm->vmcb;
3098
	struct page *page;
3099

3100 3101 3102 3103
	trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
				       vmcb->control.exit_info_1,
				       vmcb->control.exit_info_2,
				       vmcb->control.exit_int_info,
3104 3105
				       vmcb->control.exit_int_info_err,
				       KVM_ISA_SVM);
3106

3107
	nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
3108 3109 3110
	if (!nested_vmcb)
		return 1;

3111 3112
	/* Exit Guest-Mode */
	leave_guest_mode(&svm->vcpu);
3113 3114
	svm->nested.vmcb = 0;

3115
	/* Give the current vmcb to the guest */
J
Joerg Roedel 已提交
3116 3117 3118 3119 3120 3121 3122 3123
	disable_gif(svm);

	nested_vmcb->save.es     = vmcb->save.es;
	nested_vmcb->save.cs     = vmcb->save.cs;
	nested_vmcb->save.ss     = vmcb->save.ss;
	nested_vmcb->save.ds     = vmcb->save.ds;
	nested_vmcb->save.gdtr   = vmcb->save.gdtr;
	nested_vmcb->save.idtr   = vmcb->save.idtr;
3124
	nested_vmcb->save.efer   = svm->vcpu.arch.efer;
3125
	nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
3126
	nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
J
Joerg Roedel 已提交
3127
	nested_vmcb->save.cr2    = vmcb->save.cr2;
3128
	nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
3129
	nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
J
Joerg Roedel 已提交
3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145
	nested_vmcb->save.rip    = vmcb->save.rip;
	nested_vmcb->save.rsp    = vmcb->save.rsp;
	nested_vmcb->save.rax    = vmcb->save.rax;
	nested_vmcb->save.dr7    = vmcb->save.dr7;
	nested_vmcb->save.dr6    = vmcb->save.dr6;
	nested_vmcb->save.cpl    = vmcb->save.cpl;

	nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
	nested_vmcb->control.int_vector        = vmcb->control.int_vector;
	nested_vmcb->control.int_state         = vmcb->control.int_state;
	nested_vmcb->control.exit_code         = vmcb->control.exit_code;
	nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
	nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
	nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
	nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
	nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
3146 3147 3148

	if (svm->nrips_enabled)
		nested_vmcb->control.next_rip  = vmcb->control.next_rip;
3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164

	/*
	 * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
	 * to make sure that we do not lose injected events. So check event_inj
	 * here and copy it to exit_int_info if it is valid.
	 * Exit_int_info and event_inj can't be both valid because the case
	 * below only happens on a VMRUN instruction intercept which has
	 * no valid exit_int_info set.
	 */
	if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
		struct vmcb_control_area *nc = &nested_vmcb->control;

		nc->exit_int_info     = vmcb->control.event_inj;
		nc->exit_int_info_err = vmcb->control.event_inj_err;
	}

J
Joerg Roedel 已提交
3165 3166 3167
	nested_vmcb->control.tlb_ctl           = 0;
	nested_vmcb->control.event_inj         = 0;
	nested_vmcb->control.event_inj_err     = 0;
3168 3169 3170 3171 3172 3173

	/* We always set V_INTR_MASKING and remember the old value in hflags */
	if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
		nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;

	/* Restore the original control entries */
3174
	copy_vmcb_control_area(vmcb, hsave);
3175

3176 3177
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);
3178

3179 3180
	svm->nested.nested_cr3 = 0;

3181 3182 3183 3184 3185 3186 3187
	/* Restore selected save entries */
	svm->vmcb->save.es = hsave->save.es;
	svm->vmcb->save.cs = hsave->save.cs;
	svm->vmcb->save.ss = hsave->save.ss;
	svm->vmcb->save.ds = hsave->save.ds;
	svm->vmcb->save.gdtr = hsave->save.gdtr;
	svm->vmcb->save.idtr = hsave->save.idtr;
3188
	kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
3189 3190 3191 3192 3193 3194 3195
	svm_set_efer(&svm->vcpu, hsave->save.efer);
	svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
	svm_set_cr4(&svm->vcpu, hsave->save.cr4);
	if (npt_enabled) {
		svm->vmcb->save.cr3 = hsave->save.cr3;
		svm->vcpu.arch.cr3 = hsave->save.cr3;
	} else {
3196
		(void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
3197 3198 3199 3200 3201 3202 3203 3204
	}
	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
	svm->vmcb->save.dr7 = 0;
	svm->vmcb->save.cpl = 0;
	svm->vmcb->control.exit_int_info = 0;

3205 3206
	mark_all_dirty(svm->vmcb);

3207
	nested_svm_unmap(page);
3208

3209
	nested_svm_uninit_mmu_context(&svm->vcpu);
3210 3211 3212 3213 3214
	kvm_mmu_reset_context(&svm->vcpu);
	kvm_mmu_load(&svm->vcpu);

	return 0;
}
A
Alexander Graf 已提交
3215

3216
static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
A
Alexander Graf 已提交
3217
{
3218 3219
	/*
	 * This function merges the msr permission bitmaps of kvm and the
G
Guo Chao 已提交
3220
	 * nested vmcb. It is optimized in that it only merges the parts where
3221 3222
	 * the kvm msr permission bitmap may contain zero bits
	 */
A
Alexander Graf 已提交
3223
	int i;
3224

3225 3226
	if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
		return true;
3227

3228 3229 3230
	for (i = 0; i < MSRPM_OFFSETS; i++) {
		u32 value, p;
		u64 offset;
3231

3232 3233
		if (msrpm_offsets[i] == 0xffffffff)
			break;
A
Alexander Graf 已提交
3234

3235 3236
		p      = msrpm_offsets[i];
		offset = svm->nested.vmcb_msrpm + (p * 4);
3237

3238
		if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
3239 3240 3241 3242
			return false;

		svm->nested.msrpm[p] = svm->msrpm[p] | value;
	}
A
Alexander Graf 已提交
3243

3244
	svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
3245 3246

	return true;
A
Alexander Graf 已提交
3247 3248
}

3249 3250 3251 3252 3253
static bool nested_vmcb_checks(struct vmcb *vmcb)
{
	if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
		return false;

3254 3255 3256
	if (vmcb->control.asid == 0)
		return false;

3257 3258
	if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) &&
	    !npt_enabled)
3259 3260
		return false;

3261 3262 3263
	return true;
}

L
Ladi Prosek 已提交
3264 3265
static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa,
				 struct vmcb *nested_vmcb, struct page *page)
A
Alexander Graf 已提交
3266
{
3267
	if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
A
Alexander Graf 已提交
3268 3269 3270 3271
		svm->vcpu.arch.hflags |= HF_HIF_MASK;
	else
		svm->vcpu.arch.hflags &= ~HF_HIF_MASK;

3272
	if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) {
3273 3274 3275 3276 3277
		kvm_mmu_unload(&svm->vcpu);
		svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
		nested_svm_init_mmu_context(&svm->vcpu);
	}

A
Alexander Graf 已提交
3278 3279 3280 3281 3282 3283 3284
	/* Load the nested guest state */
	svm->vmcb->save.es = nested_vmcb->save.es;
	svm->vmcb->save.cs = nested_vmcb->save.cs;
	svm->vmcb->save.ss = nested_vmcb->save.ss;
	svm->vmcb->save.ds = nested_vmcb->save.ds;
	svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
	svm->vmcb->save.idtr = nested_vmcb->save.idtr;
3285
	kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
A
Alexander Graf 已提交
3286 3287 3288 3289 3290 3291
	svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
	svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
	svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
	if (npt_enabled) {
		svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
		svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
3292
	} else
3293
		(void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
3294 3295 3296 3297

	/* Guest paging mode is active - reset mmu */
	kvm_mmu_reset_context(&svm->vcpu);

J
Joerg Roedel 已提交
3298
	svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
A
Alexander Graf 已提交
3299 3300 3301
	kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
	kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
J
Joerg Roedel 已提交
3302

A
Alexander Graf 已提交
3303 3304 3305 3306 3307 3308 3309 3310
	/* In case we don't even reach vcpu_run, the fields are not updated */
	svm->vmcb->save.rax = nested_vmcb->save.rax;
	svm->vmcb->save.rsp = nested_vmcb->save.rsp;
	svm->vmcb->save.rip = nested_vmcb->save.rip;
	svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
	svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
	svm->vmcb->save.cpl = nested_vmcb->save.cpl;

3311
	svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
3312
	svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
A
Alexander Graf 已提交
3313

J
Joerg Roedel 已提交
3314
	/* cache intercepts */
3315
	svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
3316
	svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
J
Joerg Roedel 已提交
3317 3318 3319
	svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
	svm->nested.intercept            = nested_vmcb->control.intercept;

3320
	svm_flush_tlb(&svm->vcpu, true);
A
Alexander Graf 已提交
3321 3322 3323 3324 3325 3326
	svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
	if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
		svm->vcpu.arch.hflags |= HF_VINTR_MASK;
	else
		svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;

3327 3328
	if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
		/* We only want the cr8 intercept bits of the guest */
3329 3330
		clr_cr_intercept(svm, INTERCEPT_CR8_READ);
		clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3331 3332
	}

3333
	/* We don't want to see VMMCALLs from a nested guest */
3334
	clr_intercept(svm, INTERCEPT_VMMCALL);
3335

3336
	svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext;
A
Alexander Graf 已提交
3337 3338 3339 3340 3341 3342
	svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
	svm->vmcb->control.int_state = nested_vmcb->control.int_state;
	svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
	svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
	svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;

3343
	nested_svm_unmap(page);
3344

3345 3346 3347
	/* Enter Guest-Mode */
	enter_guest_mode(&svm->vcpu);

3348 3349 3350 3351 3352 3353
	/*
	 * Merge guest and host intercepts - must be called  with vcpu in
	 * guest-mode to take affect here
	 */
	recalc_intercepts(svm);

3354
	svm->nested.vmcb = vmcb_gpa;
3355

3356
	enable_gif(svm);
A
Alexander Graf 已提交
3357

3358
	mark_all_dirty(svm->vmcb);
L
Ladi Prosek 已提交
3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425
}

static bool nested_svm_vmrun(struct vcpu_svm *svm)
{
	struct vmcb *nested_vmcb;
	struct vmcb *hsave = svm->nested.hsave;
	struct vmcb *vmcb = svm->vmcb;
	struct page *page;
	u64 vmcb_gpa;

	vmcb_gpa = svm->vmcb->save.rax;

	nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
	if (!nested_vmcb)
		return false;

	if (!nested_vmcb_checks(nested_vmcb)) {
		nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
		nested_vmcb->control.exit_code_hi = 0;
		nested_vmcb->control.exit_info_1  = 0;
		nested_vmcb->control.exit_info_2  = 0;

		nested_svm_unmap(page);

		return false;
	}

	trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
			       nested_vmcb->save.rip,
			       nested_vmcb->control.int_ctl,
			       nested_vmcb->control.event_inj,
			       nested_vmcb->control.nested_ctl);

	trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
				    nested_vmcb->control.intercept_cr >> 16,
				    nested_vmcb->control.intercept_exceptions,
				    nested_vmcb->control.intercept);

	/* Clear internal status */
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);

	/*
	 * Save the old vmcb, so we don't need to pick what we save, but can
	 * restore everything when a VMEXIT occurs
	 */
	hsave->save.es     = vmcb->save.es;
	hsave->save.cs     = vmcb->save.cs;
	hsave->save.ss     = vmcb->save.ss;
	hsave->save.ds     = vmcb->save.ds;
	hsave->save.gdtr   = vmcb->save.gdtr;
	hsave->save.idtr   = vmcb->save.idtr;
	hsave->save.efer   = svm->vcpu.arch.efer;
	hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
	hsave->save.cr4    = svm->vcpu.arch.cr4;
	hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
	hsave->save.rip    = kvm_rip_read(&svm->vcpu);
	hsave->save.rsp    = vmcb->save.rsp;
	hsave->save.rax    = vmcb->save.rax;
	if (npt_enabled)
		hsave->save.cr3    = vmcb->save.cr3;
	else
		hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);

	copy_vmcb_control_area(hsave, vmcb);

	enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, page);
3426

3427
	return true;
A
Alexander Graf 已提交
3428 3429
}

3430
static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445
{
	to_vmcb->save.fs = from_vmcb->save.fs;
	to_vmcb->save.gs = from_vmcb->save.gs;
	to_vmcb->save.tr = from_vmcb->save.tr;
	to_vmcb->save.ldtr = from_vmcb->save.ldtr;
	to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
	to_vmcb->save.star = from_vmcb->save.star;
	to_vmcb->save.lstar = from_vmcb->save.lstar;
	to_vmcb->save.cstar = from_vmcb->save.cstar;
	to_vmcb->save.sfmask = from_vmcb->save.sfmask;
	to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
	to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
	to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
}

A
Avi Kivity 已提交
3446
static int vmload_interception(struct vcpu_svm *svm)
3447
{
3448
	struct vmcb *nested_vmcb;
3449
	struct page *page;
3450
	int ret;
3451

3452 3453 3454
	if (nested_svm_check_permissions(svm))
		return 1;

3455
	nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3456 3457 3458
	if (!nested_vmcb)
		return 1;

3459
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3460
	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3461

3462
	nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
3463
	nested_svm_unmap(page);
3464

3465
	return ret;
3466 3467
}

A
Avi Kivity 已提交
3468
static int vmsave_interception(struct vcpu_svm *svm)
3469
{
3470
	struct vmcb *nested_vmcb;
3471
	struct page *page;
3472
	int ret;
3473

3474 3475 3476
	if (nested_svm_check_permissions(svm))
		return 1;

3477
	nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3478 3479 3480
	if (!nested_vmcb)
		return 1;

3481
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3482
	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3483

3484
	nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
3485
	nested_svm_unmap(page);
3486

3487
	return ret;
3488 3489
}

A
Avi Kivity 已提交
3490
static int vmrun_interception(struct vcpu_svm *svm)
A
Alexander Graf 已提交
3491 3492 3493 3494
{
	if (nested_svm_check_permissions(svm))
		return 1;

3495 3496
	/* Save rip after vmrun instruction */
	kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
A
Alexander Graf 已提交
3497

3498
	if (!nested_svm_vmrun(svm))
A
Alexander Graf 已提交
3499 3500
		return 1;

3501
	if (!nested_svm_vmrun_msrpm(svm))
3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513
		goto failed;

	return 1;

failed:

	svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
	svm->vmcb->control.exit_code_hi = 0;
	svm->vmcb->control.exit_info_1  = 0;
	svm->vmcb->control.exit_info_2  = 0;

	nested_svm_vmexit(svm);
A
Alexander Graf 已提交
3514 3515 3516 3517

	return 1;
}

A
Avi Kivity 已提交
3518
static int stgi_interception(struct vcpu_svm *svm)
3519
{
3520 3521
	int ret;

3522 3523 3524
	if (nested_svm_check_permissions(svm))
		return 1;

3525 3526
	/*
	 * If VGIF is enabled, the STGI intercept is only added to
3527
	 * detect the opening of the SMI/NMI window; remove it now.
3528 3529 3530 3531
	 */
	if (vgif_enabled(svm))
		clr_intercept(svm, INTERCEPT_STGI);

3532
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3533
	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3534
	kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3535

3536
	enable_gif(svm);
3537

3538
	return ret;
3539 3540
}

A
Avi Kivity 已提交
3541
static int clgi_interception(struct vcpu_svm *svm)
3542
{
3543 3544
	int ret;

3545 3546 3547 3548
	if (nested_svm_check_permissions(svm))
		return 1;

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3549
	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3550

3551
	disable_gif(svm);
3552 3553

	/* After a CLGI no interrupts should come */
3554 3555 3556 3557 3558
	if (!kvm_vcpu_apicv_active(&svm->vcpu)) {
		svm_clear_vintr(svm);
		svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
		mark_dirty(svm->vmcb, VMCB_INTR);
	}
3559

3560
	return ret;
3561 3562
}

A
Avi Kivity 已提交
3563
static int invlpga_interception(struct vcpu_svm *svm)
A
Alexander Graf 已提交
3564 3565 3566
{
	struct kvm_vcpu *vcpu = &svm->vcpu;

3567 3568
	trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
			  kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3569

A
Alexander Graf 已提交
3570
	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
3571
	kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
A
Alexander Graf 已提交
3572 3573

	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3574
	return kvm_skip_emulated_instruction(&svm->vcpu);
A
Alexander Graf 已提交
3575 3576
}

3577 3578
static int skinit_interception(struct vcpu_svm *svm)
{
3579
	trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3580 3581 3582 3583 3584

	kvm_queue_exception(&svm->vcpu, UD_VECTOR);
	return 1;
}

D
David Kaplan 已提交
3585 3586
static int wbinvd_interception(struct vcpu_svm *svm)
{
3587
	return kvm_emulate_wbinvd(&svm->vcpu);
D
David Kaplan 已提交
3588 3589
}

J
Joerg Roedel 已提交
3590 3591 3592 3593 3594 3595 3596
static int xsetbv_interception(struct vcpu_svm *svm)
{
	u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
	u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);

	if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
		svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3597
		return kvm_skip_emulated_instruction(&svm->vcpu);
J
Joerg Roedel 已提交
3598 3599 3600 3601 3602
	}

	return 1;
}

A
Avi Kivity 已提交
3603
static int task_switch_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
3604
{
3605
	u16 tss_selector;
3606 3607 3608
	int reason;
	int int_type = svm->vmcb->control.exit_int_info &
		SVM_EXITINTINFO_TYPE_MASK;
3609
	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
3610 3611 3612 3613
	uint32_t type =
		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
	uint32_t idt_v =
		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
3614 3615
	bool has_error_code = false;
	u32 error_code = 0;
3616 3617

	tss_selector = (u16)svm->vmcb->control.exit_info_1;
3618

3619 3620
	if (svm->vmcb->control.exit_info_2 &
	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
3621 3622 3623 3624
		reason = TASK_SWITCH_IRET;
	else if (svm->vmcb->control.exit_info_2 &
		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
		reason = TASK_SWITCH_JMP;
3625
	else if (idt_v)
3626 3627 3628 3629
		reason = TASK_SWITCH_GATE;
	else
		reason = TASK_SWITCH_CALL;

3630 3631 3632 3633 3634 3635
	if (reason == TASK_SWITCH_GATE) {
		switch (type) {
		case SVM_EXITINTINFO_TYPE_NMI:
			svm->vcpu.arch.nmi_injected = false;
			break;
		case SVM_EXITINTINFO_TYPE_EXEPT:
3636 3637 3638 3639 3640 3641
			if (svm->vmcb->control.exit_info_2 &
			    (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
				has_error_code = true;
				error_code =
					(u32)svm->vmcb->control.exit_info_2;
			}
3642 3643 3644 3645 3646 3647 3648 3649 3650
			kvm_clear_exception_queue(&svm->vcpu);
			break;
		case SVM_EXITINTINFO_TYPE_INTR:
			kvm_clear_interrupt_queue(&svm->vcpu);
			break;
		default:
			break;
		}
	}
3651

3652 3653 3654
	if (reason != TASK_SWITCH_GATE ||
	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
3655 3656
	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
		skip_emulated_instruction(&svm->vcpu);
3657

3658 3659 3660 3661
	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
		int_vec = -1;

	if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
3662 3663 3664 3665 3666 3667 3668
				has_error_code, error_code) == EMULATE_FAIL) {
		svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		svm->vcpu.run->internal.ndata = 0;
		return 0;
	}
	return 1;
A
Avi Kivity 已提交
3669 3670
}

A
Avi Kivity 已提交
3671
static int cpuid_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
3672
{
3673
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3674
	return kvm_emulate_cpuid(&svm->vcpu);
A
Avi Kivity 已提交
3675 3676
}

A
Avi Kivity 已提交
3677
static int iret_interception(struct vcpu_svm *svm)
3678 3679
{
	++svm->vcpu.stat.nmi_window_exits;
3680
	clr_intercept(svm, INTERCEPT_IRET);
3681
	svm->vcpu.arch.hflags |= HF_IRET_MASK;
3682
	svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
3683
	kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3684 3685 3686
	return 1;
}

A
Avi Kivity 已提交
3687
static int invlpg_interception(struct vcpu_svm *svm)
M
Marcelo Tosatti 已提交
3688
{
3689 3690 3691 3692
	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
		return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;

	kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
3693
	return kvm_skip_emulated_instruction(&svm->vcpu);
M
Marcelo Tosatti 已提交
3694 3695
}

A
Avi Kivity 已提交
3696
static int emulate_on_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
3697
{
3698
	return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
A
Avi Kivity 已提交
3699 3700
}

B
Brijesh Singh 已提交
3701 3702 3703 3704 3705 3706
static int rsm_interception(struct vcpu_svm *svm)
{
	return x86_emulate_instruction(&svm->vcpu, 0, 0,
				       rsm_ins_bytes, 2) == EMULATE_DONE;
}

A
Avi Kivity 已提交
3707 3708 3709 3710 3711 3712 3713 3714
static int rdpmc_interception(struct vcpu_svm *svm)
{
	int err;

	if (!static_cpu_has(X86_FEATURE_NRIPS))
		return emulate_on_interception(svm);

	err = kvm_rdpmc(&svm->vcpu);
3715
	return kvm_complete_insn_gp(&svm->vcpu, err);
A
Avi Kivity 已提交
3716 3717
}

3718 3719
static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
					    unsigned long val)
3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741
{
	unsigned long cr0 = svm->vcpu.arch.cr0;
	bool ret = false;
	u64 intercept;

	intercept = svm->nested.intercept;

	if (!is_guest_mode(&svm->vcpu) ||
	    (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
		return false;

	cr0 &= ~SVM_CR0_SELECTIVE_MASK;
	val &= ~SVM_CR0_SELECTIVE_MASK;

	if (cr0 ^ val) {
		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
		ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
	}

	return ret;
}

3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756
#define CR_VALID (1ULL << 63)

static int cr_interception(struct vcpu_svm *svm)
{
	int reg, cr;
	unsigned long val;
	int err;

	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
		return emulate_on_interception(svm);

	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
		return emulate_on_interception(svm);

	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3757 3758 3759 3760
	if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
		cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
	else
		cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
3761 3762 3763 3764 3765 3766 3767

	err = 0;
	if (cr >= 16) { /* mov to cr */
		cr -= 16;
		val = kvm_register_read(&svm->vcpu, reg);
		switch (cr) {
		case 0:
3768 3769
			if (!check_selective_cr0_intercepted(svm, val))
				err = kvm_set_cr0(&svm->vcpu, val);
3770 3771 3772
			else
				return 1;

3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796
			break;
		case 3:
			err = kvm_set_cr3(&svm->vcpu, val);
			break;
		case 4:
			err = kvm_set_cr4(&svm->vcpu, val);
			break;
		case 8:
			err = kvm_set_cr8(&svm->vcpu, val);
			break;
		default:
			WARN(1, "unhandled write to CR%d", cr);
			kvm_queue_exception(&svm->vcpu, UD_VECTOR);
			return 1;
		}
	} else { /* mov from cr */
		switch (cr) {
		case 0:
			val = kvm_read_cr0(&svm->vcpu);
			break;
		case 2:
			val = svm->vcpu.arch.cr2;
			break;
		case 3:
3797
			val = kvm_read_cr3(&svm->vcpu);
3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811
			break;
		case 4:
			val = kvm_read_cr4(&svm->vcpu);
			break;
		case 8:
			val = kvm_get_cr8(&svm->vcpu);
			break;
		default:
			WARN(1, "unhandled read from CR%d", cr);
			kvm_queue_exception(&svm->vcpu, UD_VECTOR);
			return 1;
		}
		kvm_register_write(&svm->vcpu, reg, val);
	}
3812
	return kvm_complete_insn_gp(&svm->vcpu, err);
3813 3814
}

3815 3816 3817 3818 3819
static int dr_interception(struct vcpu_svm *svm)
{
	int reg, dr;
	unsigned long val;

3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830
	if (svm->vcpu.guest_debug == 0) {
		/*
		 * No more DR vmexits; force a reload of the debug registers
		 * and reenter on this instruction.  The next vmexit will
		 * retrieve the full state of the debug registers.
		 */
		clr_dr_intercepts(svm);
		svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
		return 1;
	}

3831 3832 3833 3834 3835 3836 3837
	if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
		return emulate_on_interception(svm);

	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;

	if (dr >= 16) { /* mov to DRn */
3838 3839
		if (!kvm_require_dr(&svm->vcpu, dr - 16))
			return 1;
3840 3841 3842
		val = kvm_register_read(&svm->vcpu, reg);
		kvm_set_dr(&svm->vcpu, dr - 16, val);
	} else {
3843 3844 3845 3846
		if (!kvm_require_dr(&svm->vcpu, dr))
			return 1;
		kvm_get_dr(&svm->vcpu, dr, &val);
		kvm_register_write(&svm->vcpu, reg, val);
3847 3848
	}

3849
	return kvm_skip_emulated_instruction(&svm->vcpu);
3850 3851
}

A
Avi Kivity 已提交
3852
static int cr8_write_interception(struct vcpu_svm *svm)
3853
{
A
Avi Kivity 已提交
3854
	struct kvm_run *kvm_run = svm->vcpu.run;
A
Andre Przywara 已提交
3855
	int r;
A
Avi Kivity 已提交
3856

3857 3858
	u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
	/* instruction emulation calls kvm_set_cr8() */
3859
	r = cr_interception(svm);
3860
	if (lapic_in_kernel(&svm->vcpu))
3861
		return r;
3862
	if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
3863
		return r;
3864 3865 3866 3867
	kvm_run->exit_reason = KVM_EXIT_SET_TPR;
	return 0;
}

3868 3869
static int svm_get_msr_feature(struct kvm_msr_entry *msr)
{
3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881
	msr->data = 0;

	switch (msr->index) {
	case MSR_F10H_DECFG:
		if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
			msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
		break;
	default:
		return 1;
	}

	return 0;
3882 3883
}

3884
static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
A
Avi Kivity 已提交
3885
{
3886 3887
	struct vcpu_svm *svm = to_svm(vcpu);

3888
	switch (msr_info->index) {
3889
	case MSR_IA32_TSC: {
3890
		msr_info->data = svm->vmcb->control.tsc_offset +
3891
			kvm_scale_tsc(vcpu, rdtsc());
3892

A
Avi Kivity 已提交
3893 3894
		break;
	}
B
Brian Gerst 已提交
3895
	case MSR_STAR:
3896
		msr_info->data = svm->vmcb->save.star;
A
Avi Kivity 已提交
3897
		break;
3898
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
3899
	case MSR_LSTAR:
3900
		msr_info->data = svm->vmcb->save.lstar;
A
Avi Kivity 已提交
3901 3902
		break;
	case MSR_CSTAR:
3903
		msr_info->data = svm->vmcb->save.cstar;
A
Avi Kivity 已提交
3904 3905
		break;
	case MSR_KERNEL_GS_BASE:
3906
		msr_info->data = svm->vmcb->save.kernel_gs_base;
A
Avi Kivity 已提交
3907 3908
		break;
	case MSR_SYSCALL_MASK:
3909
		msr_info->data = svm->vmcb->save.sfmask;
A
Avi Kivity 已提交
3910 3911 3912
		break;
#endif
	case MSR_IA32_SYSENTER_CS:
3913
		msr_info->data = svm->vmcb->save.sysenter_cs;
A
Avi Kivity 已提交
3914 3915
		break;
	case MSR_IA32_SYSENTER_EIP:
3916
		msr_info->data = svm->sysenter_eip;
A
Avi Kivity 已提交
3917 3918
		break;
	case MSR_IA32_SYSENTER_ESP:
3919
		msr_info->data = svm->sysenter_esp;
A
Avi Kivity 已提交
3920
		break;
P
Paolo Bonzini 已提交
3921 3922 3923 3924 3925
	case MSR_TSC_AUX:
		if (!boot_cpu_has(X86_FEATURE_RDTSCP))
			return 1;
		msr_info->data = svm->tsc_aux;
		break;
J
Joerg Roedel 已提交
3926 3927 3928 3929 3930
	/*
	 * Nobody will change the following 5 values in the VMCB so we can
	 * safely return them on rdmsr. They will always be 0 until LBRV is
	 * implemented.
	 */
3931
	case MSR_IA32_DEBUGCTLMSR:
3932
		msr_info->data = svm->vmcb->save.dbgctl;
3933 3934
		break;
	case MSR_IA32_LASTBRANCHFROMIP:
3935
		msr_info->data = svm->vmcb->save.br_from;
3936 3937
		break;
	case MSR_IA32_LASTBRANCHTOIP:
3938
		msr_info->data = svm->vmcb->save.br_to;
3939 3940
		break;
	case MSR_IA32_LASTINTFROMIP:
3941
		msr_info->data = svm->vmcb->save.last_excp_from;
3942 3943
		break;
	case MSR_IA32_LASTINTTOIP:
3944
		msr_info->data = svm->vmcb->save.last_excp_to;
3945
		break;
A
Alexander Graf 已提交
3946
	case MSR_VM_HSAVE_PA:
3947
		msr_info->data = svm->nested.hsave_msr;
A
Alexander Graf 已提交
3948
		break;
3949
	case MSR_VM_CR:
3950
		msr_info->data = svm->nested.vm_cr_msr;
3951
		break;
3952 3953 3954 3955 3956 3957 3958
	case MSR_IA32_SPEC_CTRL:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_IBRS))
			return 1;

		msr_info->data = svm->spec_ctrl;
		break;
3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975
	case MSR_F15H_IC_CFG: {

		int family, model;

		family = guest_cpuid_family(vcpu);
		model  = guest_cpuid_model(vcpu);

		if (family < 0 || model < 0)
			return kvm_get_msr_common(vcpu, msr_info);

		msr_info->data = 0;

		if (family == 0x15 &&
		    (model >= 0x2 && model < 0x20))
			msr_info->data = 0x1E;
		}
		break;
3976 3977 3978
	case MSR_F10H_DECFG:
		msr_info->data = svm->msr_decfg;
		break;
A
Avi Kivity 已提交
3979
	default:
3980
		return kvm_get_msr_common(vcpu, msr_info);
A
Avi Kivity 已提交
3981 3982 3983 3984
	}
	return 0;
}

A
Avi Kivity 已提交
3985
static int rdmsr_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
3986
{
3987
	u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3988
	struct msr_data msr_info;
A
Avi Kivity 已提交
3989

3990 3991 3992
	msr_info.index = ecx;
	msr_info.host_initiated = false;
	if (svm_get_msr(&svm->vcpu, &msr_info)) {
3993
		trace_kvm_msr_read_ex(ecx);
3994
		kvm_inject_gp(&svm->vcpu, 0);
3995
		return 1;
3996
	} else {
3997
		trace_kvm_msr_read(ecx, msr_info.data);
3998

3999 4000 4001 4002
		kvm_register_write(&svm->vcpu, VCPU_REGS_RAX,
				   msr_info.data & 0xffffffff);
		kvm_register_write(&svm->vcpu, VCPU_REGS_RDX,
				   msr_info.data >> 32);
4003
		svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
4004
		return kvm_skip_emulated_instruction(&svm->vcpu);
A
Avi Kivity 已提交
4005 4006 4007
	}
}

4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032
static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	int svm_dis, chg_mask;

	if (data & ~SVM_VM_CR_VALID_MASK)
		return 1;

	chg_mask = SVM_VM_CR_VALID_MASK;

	if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
		chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);

	svm->nested.vm_cr_msr &= ~chg_mask;
	svm->nested.vm_cr_msr |= (data & chg_mask);

	svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;

	/* check for svm_disable while efer.svme is set */
	if (svm_dis && (vcpu->arch.efer & EFER_SVME))
		return 1;

	return 0;
}

4033
static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
A
Avi Kivity 已提交
4034
{
4035 4036
	struct vcpu_svm *svm = to_svm(vcpu);

4037 4038
	u32 ecx = msr->index;
	u64 data = msr->data;
A
Avi Kivity 已提交
4039
	switch (ecx) {
P
Paolo Bonzini 已提交
4040 4041 4042 4043 4044 4045 4046
	case MSR_IA32_CR_PAT:
		if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
			return 1;
		vcpu->arch.pat = data;
		svm->vmcb->save.g_pat = data;
		mark_dirty(svm->vmcb, VMCB_NPT);
		break;
4047
	case MSR_IA32_TSC:
4048
		kvm_write_tsc(vcpu, msr);
A
Avi Kivity 已提交
4049
		break;
4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076
	case MSR_IA32_SPEC_CTRL:
		if (!msr->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_IBRS))
			return 1;

		/* The STIBP bit doesn't fault even if it's not advertised */
		if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP))
			return 1;

		svm->spec_ctrl = data;

		if (!data)
			break;

		/*
		 * For non-nested:
		 * When it's written (to non-zero) for the first time, pass
		 * it through.
		 *
		 * For nested:
		 * The handling of the MSR bitmap for L2 guests is done in
		 * nested_svm_vmrun_msrpm.
		 * We update the L1 MSR bit as well since it will end up
		 * touching the MSR anyway now.
		 */
		set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
		break;
A
Ashok Raj 已提交
4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092
	case MSR_IA32_PRED_CMD:
		if (!msr->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_IBPB))
			return 1;

		if (data & ~PRED_CMD_IBPB)
			return 1;

		if (!data)
			break;

		wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
		if (is_guest_mode(vcpu))
			break;
		set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
		break;
B
Brian Gerst 已提交
4093
	case MSR_STAR:
4094
		svm->vmcb->save.star = data;
A
Avi Kivity 已提交
4095
		break;
4096
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
4097
	case MSR_LSTAR:
4098
		svm->vmcb->save.lstar = data;
A
Avi Kivity 已提交
4099 4100
		break;
	case MSR_CSTAR:
4101
		svm->vmcb->save.cstar = data;
A
Avi Kivity 已提交
4102 4103
		break;
	case MSR_KERNEL_GS_BASE:
4104
		svm->vmcb->save.kernel_gs_base = data;
A
Avi Kivity 已提交
4105 4106
		break;
	case MSR_SYSCALL_MASK:
4107
		svm->vmcb->save.sfmask = data;
A
Avi Kivity 已提交
4108 4109 4110
		break;
#endif
	case MSR_IA32_SYSENTER_CS:
4111
		svm->vmcb->save.sysenter_cs = data;
A
Avi Kivity 已提交
4112 4113
		break;
	case MSR_IA32_SYSENTER_EIP:
4114
		svm->sysenter_eip = data;
4115
		svm->vmcb->save.sysenter_eip = data;
A
Avi Kivity 已提交
4116 4117
		break;
	case MSR_IA32_SYSENTER_ESP:
4118
		svm->sysenter_esp = data;
4119
		svm->vmcb->save.sysenter_esp = data;
A
Avi Kivity 已提交
4120
		break;
P
Paolo Bonzini 已提交
4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132
	case MSR_TSC_AUX:
		if (!boot_cpu_has(X86_FEATURE_RDTSCP))
			return 1;

		/*
		 * This is rare, so we update the MSR here instead of using
		 * direct_access_msrs.  Doing that would require a rdmsr in
		 * svm_vcpu_put.
		 */
		svm->tsc_aux = data;
		wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
		break;
4133
	case MSR_IA32_DEBUGCTLMSR:
4134
		if (!boot_cpu_has(X86_FEATURE_LBRV)) {
4135 4136
			vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
				    __func__, data);
4137 4138 4139 4140 4141 4142
			break;
		}
		if (data & DEBUGCTL_RESERVED_BITS)
			return 1;

		svm->vmcb->save.dbgctl = data;
4143
		mark_dirty(svm->vmcb, VMCB_LBR);
4144 4145 4146 4147
		if (data & (1ULL<<0))
			svm_enable_lbrv(svm);
		else
			svm_disable_lbrv(svm);
4148
		break;
A
Alexander Graf 已提交
4149
	case MSR_VM_HSAVE_PA:
4150
		svm->nested.hsave_msr = data;
4151
		break;
4152
	case MSR_VM_CR:
4153
		return svm_set_vm_cr(vcpu, data);
4154
	case MSR_VM_IGNNE:
4155
		vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
4156
		break;
4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174
	case MSR_F10H_DECFG: {
		struct kvm_msr_entry msr_entry;

		msr_entry.index = msr->index;
		if (svm_get_msr_feature(&msr_entry))
			return 1;

		/* Check the supported bits */
		if (data & ~msr_entry.data)
			return 1;

		/* Don't allow the guest to change a bit, #GP */
		if (!msr->host_initiated && (data ^ msr_entry.data))
			return 1;

		svm->msr_decfg = data;
		break;
	}
4175 4176 4177 4178
	case MSR_IA32_APICBASE:
		if (kvm_vcpu_apicv_active(vcpu))
			avic_update_vapic_bar(to_svm(vcpu), data);
		/* Follow through */
A
Avi Kivity 已提交
4179
	default:
4180
		return kvm_set_msr_common(vcpu, msr);
A
Avi Kivity 已提交
4181 4182 4183 4184
	}
	return 0;
}

A
Avi Kivity 已提交
4185
static int wrmsr_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
4186
{
4187
	struct msr_data msr;
4188 4189
	u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
	u64 data = kvm_read_edx_eax(&svm->vcpu);
4190

4191 4192 4193
	msr.data = data;
	msr.index = ecx;
	msr.host_initiated = false;
4194

4195
	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
4196
	if (kvm_set_msr(&svm->vcpu, &msr)) {
4197
		trace_kvm_msr_write_ex(ecx, data);
4198
		kvm_inject_gp(&svm->vcpu, 0);
4199
		return 1;
4200 4201
	} else {
		trace_kvm_msr_write(ecx, data);
4202
		return kvm_skip_emulated_instruction(&svm->vcpu);
4203
	}
A
Avi Kivity 已提交
4204 4205
}

A
Avi Kivity 已提交
4206
static int msr_interception(struct vcpu_svm *svm)
A
Avi Kivity 已提交
4207
{
R
Rusty Russell 已提交
4208
	if (svm->vmcb->control.exit_info_1)
A
Avi Kivity 已提交
4209
		return wrmsr_interception(svm);
A
Avi Kivity 已提交
4210
	else
A
Avi Kivity 已提交
4211
		return rdmsr_interception(svm);
A
Avi Kivity 已提交
4212 4213
}

A
Avi Kivity 已提交
4214
static int interrupt_window_interception(struct vcpu_svm *svm)
4215
{
4216
	kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
4217
	svm_clear_vintr(svm);
4218
	svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
4219
	mark_dirty(svm->vmcb, VMCB_INTR);
4220
	++svm->vcpu.stat.irq_window_exits;
4221 4222 4223
	return 1;
}

4224 4225
static int pause_interception(struct vcpu_svm *svm)
{
4226 4227 4228 4229
	struct kvm_vcpu *vcpu = &svm->vcpu;
	bool in_kernel = (svm_get_cpl(vcpu) == 0);

	kvm_vcpu_on_spin(vcpu, in_kernel);
4230 4231 4232
	return 1;
}

4233 4234
static int nop_interception(struct vcpu_svm *svm)
{
4235
	return kvm_skip_emulated_instruction(&(svm->vcpu));
4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249
}

static int monitor_interception(struct vcpu_svm *svm)
{
	printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
	return nop_interception(svm);
}

static int mwait_interception(struct vcpu_svm *svm)
{
	printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
	return nop_interception(svm);
}

4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261
enum avic_ipi_failure_cause {
	AVIC_IPI_FAILURE_INVALID_INT_TYPE,
	AVIC_IPI_FAILURE_TARGET_NOT_RUNNING,
	AVIC_IPI_FAILURE_INVALID_TARGET,
	AVIC_IPI_FAILURE_INVALID_BACKING_PAGE,
};

static int avic_incomplete_ipi_interception(struct vcpu_svm *svm)
{
	u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
	u32 icrl = svm->vmcb->control.exit_info_1;
	u32 id = svm->vmcb->control.exit_info_2 >> 32;
D
Dan Carpenter 已提交
4262
	u32 index = svm->vmcb->control.exit_info_2 & 0xFF;
4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521
	struct kvm_lapic *apic = svm->vcpu.arch.apic;

	trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index);

	switch (id) {
	case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
		/*
		 * AVIC hardware handles the generation of
		 * IPIs when the specified Message Type is Fixed
		 * (also known as fixed delivery mode) and
		 * the Trigger Mode is edge-triggered. The hardware
		 * also supports self and broadcast delivery modes
		 * specified via the Destination Shorthand(DSH)
		 * field of the ICRL. Logical and physical APIC ID
		 * formats are supported. All other IPI types cause
		 * a #VMEXIT, which needs to emulated.
		 */
		kvm_lapic_reg_write(apic, APIC_ICR2, icrh);
		kvm_lapic_reg_write(apic, APIC_ICR, icrl);
		break;
	case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: {
		int i;
		struct kvm_vcpu *vcpu;
		struct kvm *kvm = svm->vcpu.kvm;
		struct kvm_lapic *apic = svm->vcpu.arch.apic;

		/*
		 * At this point, we expect that the AVIC HW has already
		 * set the appropriate IRR bits on the valid target
		 * vcpus. So, we just need to kick the appropriate vcpu.
		 */
		kvm_for_each_vcpu(i, vcpu, kvm) {
			bool m = kvm_apic_match_dest(vcpu, apic,
						     icrl & KVM_APIC_SHORT_MASK,
						     GET_APIC_DEST_FIELD(icrh),
						     icrl & KVM_APIC_DEST_MASK);

			if (m && !avic_vcpu_is_running(vcpu))
				kvm_vcpu_wake_up(vcpu);
		}
		break;
	}
	case AVIC_IPI_FAILURE_INVALID_TARGET:
		break;
	case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
		WARN_ONCE(1, "Invalid backing page\n");
		break;
	default:
		pr_err("Unknown IPI interception\n");
	}

	return 1;
}

static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
{
	struct kvm_arch *vm_data = &vcpu->kvm->arch;
	int index;
	u32 *logical_apic_id_table;
	int dlid = GET_APIC_LOGICAL_ID(ldr);

	if (!dlid)
		return NULL;

	if (flat) { /* flat */
		index = ffs(dlid) - 1;
		if (index > 7)
			return NULL;
	} else { /* cluster */
		int cluster = (dlid & 0xf0) >> 4;
		int apic = ffs(dlid & 0x0f) - 1;

		if ((apic < 0) || (apic > 7) ||
		    (cluster >= 0xf))
			return NULL;
		index = (cluster << 2) + apic;
	}

	logical_apic_id_table = (u32 *) page_address(vm_data->avic_logical_id_table_page);

	return &logical_apic_id_table[index];
}

static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr,
			  bool valid)
{
	bool flat;
	u32 *entry, new_entry;

	flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
	entry = avic_get_logical_id_entry(vcpu, ldr, flat);
	if (!entry)
		return -EINVAL;

	new_entry = READ_ONCE(*entry);
	new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
	new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
	if (valid)
		new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
	else
		new_entry &= ~AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
	WRITE_ONCE(*entry, new_entry);

	return 0;
}

static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
{
	int ret;
	struct vcpu_svm *svm = to_svm(vcpu);
	u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);

	if (!ldr)
		return 1;

	ret = avic_ldr_write(vcpu, vcpu->vcpu_id, ldr, true);
	if (ret && svm->ldr_reg) {
		avic_ldr_write(vcpu, 0, svm->ldr_reg, false);
		svm->ldr_reg = 0;
	} else {
		svm->ldr_reg = ldr;
	}
	return ret;
}

static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu)
{
	u64 *old, *new;
	struct vcpu_svm *svm = to_svm(vcpu);
	u32 apic_id_reg = kvm_lapic_get_reg(vcpu->arch.apic, APIC_ID);
	u32 id = (apic_id_reg >> 24) & 0xff;

	if (vcpu->vcpu_id == id)
		return 0;

	old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id);
	new = avic_get_physical_id_entry(vcpu, id);
	if (!new || !old)
		return 1;

	/* We need to move physical_id_entry to new offset */
	*new = *old;
	*old = 0ULL;
	to_svm(vcpu)->avic_physical_id_cache = new;

	/*
	 * Also update the guest physical APIC ID in the logical
	 * APIC ID table entry if already setup the LDR.
	 */
	if (svm->ldr_reg)
		avic_handle_ldr_update(vcpu);

	return 0;
}

static int avic_handle_dfr_update(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct kvm_arch *vm_data = &vcpu->kvm->arch;
	u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
	u32 mod = (dfr >> 28) & 0xf;

	/*
	 * We assume that all local APICs are using the same type.
	 * If this changes, we need to flush the AVIC logical
	 * APID id table.
	 */
	if (vm_data->ldr_mode == mod)
		return 0;

	clear_page(page_address(vm_data->avic_logical_id_table_page));
	vm_data->ldr_mode = mod;

	if (svm->ldr_reg)
		avic_handle_ldr_update(vcpu);
	return 0;
}

static int avic_unaccel_trap_write(struct vcpu_svm *svm)
{
	struct kvm_lapic *apic = svm->vcpu.arch.apic;
	u32 offset = svm->vmcb->control.exit_info_1 &
				AVIC_UNACCEL_ACCESS_OFFSET_MASK;

	switch (offset) {
	case APIC_ID:
		if (avic_handle_apic_id_update(&svm->vcpu))
			return 0;
		break;
	case APIC_LDR:
		if (avic_handle_ldr_update(&svm->vcpu))
			return 0;
		break;
	case APIC_DFR:
		avic_handle_dfr_update(&svm->vcpu);
		break;
	default:
		break;
	}

	kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset));

	return 1;
}

static bool is_avic_unaccelerated_access_trap(u32 offset)
{
	bool ret = false;

	switch (offset) {
	case APIC_ID:
	case APIC_EOI:
	case APIC_RRR:
	case APIC_LDR:
	case APIC_DFR:
	case APIC_SPIV:
	case APIC_ESR:
	case APIC_ICR:
	case APIC_LVTT:
	case APIC_LVTTHMR:
	case APIC_LVTPC:
	case APIC_LVT0:
	case APIC_LVT1:
	case APIC_LVTERR:
	case APIC_TMICT:
	case APIC_TDCR:
		ret = true;
		break;
	default:
		break;
	}
	return ret;
}

static int avic_unaccelerated_access_interception(struct vcpu_svm *svm)
{
	int ret = 0;
	u32 offset = svm->vmcb->control.exit_info_1 &
		     AVIC_UNACCEL_ACCESS_OFFSET_MASK;
	u32 vector = svm->vmcb->control.exit_info_2 &
		     AVIC_UNACCEL_ACCESS_VECTOR_MASK;
	bool write = (svm->vmcb->control.exit_info_1 >> 32) &
		     AVIC_UNACCEL_ACCESS_WRITE_MASK;
	bool trap = is_avic_unaccelerated_access_trap(offset);

	trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset,
					    trap, write, vector);
	if (trap) {
		/* Handling Trap */
		WARN_ONCE(!write, "svm: Handling trap read.\n");
		ret = avic_unaccel_trap_write(svm);
	} else {
		/* Handling Fault */
		ret = (emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE);
	}

	return ret;
}

4522
static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
4523 4524 4525 4526
	[SVM_EXIT_READ_CR0]			= cr_interception,
	[SVM_EXIT_READ_CR3]			= cr_interception,
	[SVM_EXIT_READ_CR4]			= cr_interception,
	[SVM_EXIT_READ_CR8]			= cr_interception,
4527
	[SVM_EXIT_CR0_SEL_WRITE]		= cr_interception,
4528
	[SVM_EXIT_WRITE_CR0]			= cr_interception,
4529 4530
	[SVM_EXIT_WRITE_CR3]			= cr_interception,
	[SVM_EXIT_WRITE_CR4]			= cr_interception,
J
Joerg Roedel 已提交
4531
	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547
	[SVM_EXIT_READ_DR0]			= dr_interception,
	[SVM_EXIT_READ_DR1]			= dr_interception,
	[SVM_EXIT_READ_DR2]			= dr_interception,
	[SVM_EXIT_READ_DR3]			= dr_interception,
	[SVM_EXIT_READ_DR4]			= dr_interception,
	[SVM_EXIT_READ_DR5]			= dr_interception,
	[SVM_EXIT_READ_DR6]			= dr_interception,
	[SVM_EXIT_READ_DR7]			= dr_interception,
	[SVM_EXIT_WRITE_DR0]			= dr_interception,
	[SVM_EXIT_WRITE_DR1]			= dr_interception,
	[SVM_EXIT_WRITE_DR2]			= dr_interception,
	[SVM_EXIT_WRITE_DR3]			= dr_interception,
	[SVM_EXIT_WRITE_DR4]			= dr_interception,
	[SVM_EXIT_WRITE_DR5]			= dr_interception,
	[SVM_EXIT_WRITE_DR6]			= dr_interception,
	[SVM_EXIT_WRITE_DR7]			= dr_interception,
J
Jan Kiszka 已提交
4548 4549
	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
4550
	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
J
Joerg Roedel 已提交
4551 4552
	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
4553
	[SVM_EXIT_EXCP_BASE + AC_VECTOR]	= ac_interception,
J
Joerg Roedel 已提交
4554
	[SVM_EXIT_INTR]				= intr_interception,
4555
	[SVM_EXIT_NMI]				= nmi_interception,
A
Avi Kivity 已提交
4556 4557
	[SVM_EXIT_SMI]				= nop_on_interception,
	[SVM_EXIT_INIT]				= nop_on_interception,
4558
	[SVM_EXIT_VINTR]			= interrupt_window_interception,
A
Avi Kivity 已提交
4559
	[SVM_EXIT_RDPMC]			= rdpmc_interception,
A
Avi Kivity 已提交
4560
	[SVM_EXIT_CPUID]			= cpuid_interception,
4561
	[SVM_EXIT_IRET]                         = iret_interception,
4562
	[SVM_EXIT_INVD]                         = emulate_on_interception,
4563
	[SVM_EXIT_PAUSE]			= pause_interception,
A
Avi Kivity 已提交
4564
	[SVM_EXIT_HLT]				= halt_interception,
M
Marcelo Tosatti 已提交
4565
	[SVM_EXIT_INVLPG]			= invlpg_interception,
A
Alexander Graf 已提交
4566
	[SVM_EXIT_INVLPGA]			= invlpga_interception,
J
Joerg Roedel 已提交
4567
	[SVM_EXIT_IOIO]				= io_interception,
A
Avi Kivity 已提交
4568 4569
	[SVM_EXIT_MSR]				= msr_interception,
	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
4570
	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
A
Alexander Graf 已提交
4571
	[SVM_EXIT_VMRUN]			= vmrun_interception,
4572
	[SVM_EXIT_VMMCALL]			= vmmcall_interception,
4573 4574
	[SVM_EXIT_VMLOAD]			= vmload_interception,
	[SVM_EXIT_VMSAVE]			= vmsave_interception,
4575 4576
	[SVM_EXIT_STGI]				= stgi_interception,
	[SVM_EXIT_CLGI]				= clgi_interception,
4577
	[SVM_EXIT_SKINIT]			= skinit_interception,
D
David Kaplan 已提交
4578
	[SVM_EXIT_WBINVD]                       = wbinvd_interception,
4579 4580
	[SVM_EXIT_MONITOR]			= monitor_interception,
	[SVM_EXIT_MWAIT]			= mwait_interception,
J
Joerg Roedel 已提交
4581
	[SVM_EXIT_XSETBV]			= xsetbv_interception,
4582
	[SVM_EXIT_NPF]				= npf_interception,
B
Brijesh Singh 已提交
4583
	[SVM_EXIT_RSM]                          = rsm_interception,
4584 4585
	[SVM_EXIT_AVIC_INCOMPLETE_IPI]		= avic_incomplete_ipi_interception,
	[SVM_EXIT_AVIC_UNACCELERATED_ACCESS]	= avic_unaccelerated_access_interception,
A
Avi Kivity 已提交
4586 4587
};

4588
static void dump_vmcb(struct kvm_vcpu *vcpu)
4589 4590 4591 4592 4593 4594
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb_control_area *control = &svm->vmcb->control;
	struct vmcb_save_area *save = &svm->vmcb->save;

	pr_err("VMCB Control Area:\n");
4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616
	pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
	pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
	pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
	pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
	pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
	pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
	pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
	pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
	pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
	pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
	pr_err("%-20s%d\n", "asid:", control->asid);
	pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
	pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
	pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
	pr_err("%-20s%08x\n", "int_state:", control->int_state);
	pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
	pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
	pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
4617
	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
4618 4619
	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
	pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
4620
	pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
4621
	pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
4622 4623 4624
	pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
	pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
	pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
4625
	pr_err("VMCB State Save Area:\n");
4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665
	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
	       "es:",
	       save->es.selector, save->es.attrib,
	       save->es.limit, save->es.base);
	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
	       "cs:",
	       save->cs.selector, save->cs.attrib,
	       save->cs.limit, save->cs.base);
	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
	       "ss:",
	       save->ss.selector, save->ss.attrib,
	       save->ss.limit, save->ss.base);
	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
	       "ds:",
	       save->ds.selector, save->ds.attrib,
	       save->ds.limit, save->ds.base);
	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
	       "fs:",
	       save->fs.selector, save->fs.attrib,
	       save->fs.limit, save->fs.base);
	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
	       "gs:",
	       save->gs.selector, save->gs.attrib,
	       save->gs.limit, save->gs.base);
	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
	       "gdtr:",
	       save->gdtr.selector, save->gdtr.attrib,
	       save->gdtr.limit, save->gdtr.base);
	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
	       "ldtr:",
	       save->ldtr.selector, save->ldtr.attrib,
	       save->ldtr.limit, save->ldtr.base);
	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
	       "idtr:",
	       save->idtr.selector, save->idtr.attrib,
	       save->idtr.limit, save->idtr.base);
	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
	       "tr:",
	       save->tr.selector, save->tr.attrib,
	       save->tr.limit, save->tr.base);
4666 4667
	pr_err("cpl:            %d                efer:         %016llx\n",
		save->cpl, save->efer);
4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "cr0:", save->cr0, "cr2:", save->cr2);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "cr3:", save->cr3, "cr4:", save->cr4);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "dr6:", save->dr6, "dr7:", save->dr7);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "rip:", save->rip, "rflags:", save->rflags);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "rsp:", save->rsp, "rax:", save->rax);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "star:", save->star, "lstar:", save->lstar);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "cstar:", save->cstar, "sfmask:", save->sfmask);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "kernel_gs_base:", save->kernel_gs_base,
	       "sysenter_cs:", save->sysenter_cs);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "sysenter_esp:", save->sysenter_esp,
	       "sysenter_eip:", save->sysenter_eip);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "br_from:", save->br_from, "br_to:", save->br_to);
	pr_err("%-15s %016llx %-13s %016llx\n",
	       "excp_from:", save->last_excp_from,
	       "excp_to:", save->last_excp_to);
4695 4696
}

4697 4698 4699 4700 4701 4702 4703 4704
static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
{
	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;

	*info1 = control->exit_info_1;
	*info2 = control->exit_info_2;
}

A
Avi Kivity 已提交
4705
static int handle_exit(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4706
{
4707
	struct vcpu_svm *svm = to_svm(vcpu);
A
Avi Kivity 已提交
4708
	struct kvm_run *kvm_run = vcpu->run;
4709
	u32 exit_code = svm->vmcb->control.exit_code;
A
Avi Kivity 已提交
4710

4711 4712
	trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);

4713
	if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
4714 4715 4716
		vcpu->arch.cr0 = svm->vmcb->save.cr0;
	if (npt_enabled)
		vcpu->arch.cr3 = svm->vmcb->save.cr3;
4717

4718 4719 4720 4721 4722 4723 4724
	if (unlikely(svm->nested.exit_required)) {
		nested_svm_vmexit(svm);
		svm->nested.exit_required = false;

		return 1;
	}

4725
	if (is_guest_mode(vcpu)) {
4726 4727
		int vmexit;

4728 4729 4730 4731
		trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
					svm->vmcb->control.exit_info_1,
					svm->vmcb->control.exit_info_2,
					svm->vmcb->control.exit_int_info,
4732 4733
					svm->vmcb->control.exit_int_info_err,
					KVM_ISA_SVM);
4734

4735 4736 4737 4738 4739 4740
		vmexit = nested_svm_exit_special(svm);

		if (vmexit == NESTED_EXIT_CONTINUE)
			vmexit = nested_svm_exit_handled(svm);

		if (vmexit == NESTED_EXIT_DONE)
4741 4742 4743
			return 1;
	}

4744 4745
	svm_complete_interrupts(svm);

4746 4747 4748 4749
	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		kvm_run->fail_entry.hardware_entry_failure_reason
			= svm->vmcb->control.exit_code;
4750 4751
		pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
		dump_vmcb(vcpu);
4752 4753 4754
		return 0;
	}

4755
	if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
4756
	    exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
4757 4758
	    exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
	    exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
4759
		printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
A
Avi Kivity 已提交
4760
		       "exit_code 0x%x\n",
4761
		       __func__, svm->vmcb->control.exit_int_info,
A
Avi Kivity 已提交
4762 4763
		       exit_code);

4764
	if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
J
Joe Perches 已提交
4765
	    || !svm_exit_handlers[exit_code]) {
4766
		WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
4767 4768
		kvm_queue_exception(vcpu, UD_VECTOR);
		return 1;
A
Avi Kivity 已提交
4769 4770
	}

A
Avi Kivity 已提交
4771
	return svm_exit_handlers[exit_code](svm);
A
Avi Kivity 已提交
4772 4773 4774 4775 4776 4777
}

static void reload_tss(struct kvm_vcpu *vcpu)
{
	int cpu = raw_smp_processor_id();

4778 4779
	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
	sd->tss_desc->type = 9; /* available 32/64-bit TSS */
A
Avi Kivity 已提交
4780 4781 4782
	load_TR_desc();
}

4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806
static void pre_sev_run(struct vcpu_svm *svm, int cpu)
{
	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
	int asid = sev_get_asid(svm->vcpu.kvm);

	/* Assign the asid allocated with this SEV guest */
	svm->vmcb->control.asid = asid;

	/*
	 * Flush guest TLB:
	 *
	 * 1) when different VMCB for the same ASID is to be run on the same host CPU.
	 * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
	 */
	if (sd->sev_vmcbs[asid] == svm->vmcb &&
	    svm->last_cpu == cpu)
		return;

	svm->last_cpu = cpu;
	sd->sev_vmcbs[asid] = svm->vmcb;
	svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
	mark_dirty(svm->vmcb, VMCB_ASID);
}

R
Rusty Russell 已提交
4807
static void pre_svm_run(struct vcpu_svm *svm)
A
Avi Kivity 已提交
4808 4809 4810
{
	int cpu = raw_smp_processor_id();

4811
	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
A
Avi Kivity 已提交
4812

4813 4814 4815
	if (sev_guest(svm->vcpu.kvm))
		return pre_sev_run(svm, cpu);

4816
	/* FIXME: handle wraparound of asid_generation */
4817 4818
	if (svm->asid_generation != sd->asid_generation)
		new_asid(svm, sd);
A
Avi Kivity 已提交
4819 4820
}

4821 4822 4823 4824 4825 4826
static void svm_inject_nmi(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
	vcpu->arch.hflags |= HF_NMI_MASK;
4827
	set_intercept(svm, INTERCEPT_IRET);
4828 4829
	++vcpu->stat.nmi_injections;
}
A
Avi Kivity 已提交
4830

4831
static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
A
Avi Kivity 已提交
4832 4833 4834
{
	struct vmcb_control_area *control;

4835
	/* The following fields are ignored when AVIC is enabled */
R
Rusty Russell 已提交
4836
	control = &svm->vmcb->control;
4837
	control->int_vector = irq;
A
Avi Kivity 已提交
4838 4839 4840
	control->int_ctl &= ~V_INTR_PRIO_MASK;
	control->int_ctl |= V_IRQ_MASK |
		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
4841
	mark_dirty(svm->vmcb, VMCB_INTR);
A
Avi Kivity 已提交
4842 4843
}

4844
static void svm_set_irq(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
4845 4846 4847
{
	struct vcpu_svm *svm = to_svm(vcpu);

4848
	BUG_ON(!(gif_set(svm)));
4849

4850 4851 4852
	trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
	++vcpu->stat.irq_injections;

4853 4854
	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
		SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
E
Eddie Dong 已提交
4855 4856
}

4857 4858 4859 4860 4861
static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu)
{
	return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK);
}

4862
static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
4863 4864 4865
{
	struct vcpu_svm *svm = to_svm(vcpu);

4866 4867
	if (svm_nested_virtualize_tpr(vcpu) ||
	    kvm_vcpu_apicv_active(vcpu))
4868 4869
		return;

4870 4871
	clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);

4872
	if (irr == -1)
4873 4874
		return;

4875
	if (tpr >= irr)
4876
		set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
4877
}
4878

4879 4880 4881 4882 4883
static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
{
	return;
}

4884
static bool svm_get_enable_apicv(struct kvm_vcpu *vcpu)
4885
{
4886
	return avic && irqchip_split(vcpu->kvm);
4887 4888 4889 4890
}

static void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
{
4891 4892
}

4893
static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
4894
{
4895 4896
}

4897
/* Note: Currently only used by Hyper-V. */
4898
static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
4899
{
4900 4901 4902
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb *vmcb = svm->vmcb;

4903
	if (!kvm_vcpu_apicv_active(&svm->vcpu))
4904 4905 4906 4907
		return;

	vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
	mark_dirty(vmcb, VMCB_INTR);
4908 4909
}

4910
static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
4911 4912 4913 4914
{
	return;
}

4915 4916 4917 4918 4919 4920 4921
static void svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec)
{
	kvm_lapic_set_irr(vec, vcpu->arch.apic);
	smp_mb__after_atomic();

	if (avic_vcpu_is_running(vcpu))
		wrmsrl(SVM_AVIC_DOORBELL,
4922
		       kvm_cpu_get_apicid(vcpu->cpu));
4923 4924 4925 4926
	else
		kvm_vcpu_wake_up(vcpu);
}

4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015
static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
{
	unsigned long flags;
	struct amd_svm_iommu_ir *cur;

	spin_lock_irqsave(&svm->ir_list_lock, flags);
	list_for_each_entry(cur, &svm->ir_list, node) {
		if (cur->data != pi->ir_data)
			continue;
		list_del(&cur->node);
		kfree(cur);
		break;
	}
	spin_unlock_irqrestore(&svm->ir_list_lock, flags);
}

static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
{
	int ret = 0;
	unsigned long flags;
	struct amd_svm_iommu_ir *ir;

	/**
	 * In some cases, the existing irte is updaed and re-set,
	 * so we need to check here if it's already been * added
	 * to the ir_list.
	 */
	if (pi->ir_data && (pi->prev_ga_tag != 0)) {
		struct kvm *kvm = svm->vcpu.kvm;
		u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
		struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
		struct vcpu_svm *prev_svm;

		if (!prev_vcpu) {
			ret = -EINVAL;
			goto out;
		}

		prev_svm = to_svm(prev_vcpu);
		svm_ir_list_del(prev_svm, pi);
	}

	/**
	 * Allocating new amd_iommu_pi_data, which will get
	 * add to the per-vcpu ir_list.
	 */
	ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL);
	if (!ir) {
		ret = -ENOMEM;
		goto out;
	}
	ir->data = pi->ir_data;

	spin_lock_irqsave(&svm->ir_list_lock, flags);
	list_add(&ir->node, &svm->ir_list);
	spin_unlock_irqrestore(&svm->ir_list_lock, flags);
out:
	return ret;
}

/**
 * Note:
 * The HW cannot support posting multicast/broadcast
 * interrupts to a vCPU. So, we still use legacy interrupt
 * remapping for these kind of interrupts.
 *
 * For lowest-priority interrupts, we only support
 * those with single CPU as the destination, e.g. user
 * configures the interrupts via /proc/irq or uses
 * irqbalance to make the interrupts single-CPU.
 */
static int
get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
		 struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
{
	struct kvm_lapic_irq irq;
	struct kvm_vcpu *vcpu = NULL;

	kvm_set_msi_irq(kvm, e, &irq);

	if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
		pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
			 __func__, irq.vector);
		return -1;
	}

	pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
		 irq.vector);
	*svm = to_svm(vcpu);
5016
	vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066
	vcpu_info->vector = irq.vector;

	return 0;
}

/*
 * svm_update_pi_irte - set IRTE for Posted-Interrupts
 *
 * @kvm: kvm
 * @host_irq: host irq of the interrupt
 * @guest_irq: gsi of the interrupt
 * @set: set or unset PI
 * returns 0 on success, < 0 on failure
 */
static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
			      uint32_t guest_irq, bool set)
{
	struct kvm_kernel_irq_routing_entry *e;
	struct kvm_irq_routing_table *irq_rt;
	int idx, ret = -EINVAL;

	if (!kvm_arch_has_assigned_device(kvm) ||
	    !irq_remapping_cap(IRQ_POSTING_CAP))
		return 0;

	pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
		 __func__, host_irq, guest_irq, set);

	idx = srcu_read_lock(&kvm->irq_srcu);
	irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
	WARN_ON(guest_irq >= irq_rt->nr_rt_entries);

	hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
		struct vcpu_data vcpu_info;
		struct vcpu_svm *svm = NULL;

		if (e->type != KVM_IRQ_ROUTING_MSI)
			continue;

		/**
		 * Here, we setup with legacy mode in the following cases:
		 * 1. When cannot target interrupt to a specific vcpu.
		 * 2. Unsetting posted interrupt.
		 * 3. APIC virtialization is disabled for the vcpu.
		 */
		if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
		    kvm_vcpu_apicv_active(&svm->vcpu)) {
			struct amd_iommu_pi_data pi;

			/* Try to enable guest_mode in IRTE */
5067 5068
			pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
					    AVIC_HPA_MASK);
5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130
			pi.ga_tag = AVIC_GATAG(kvm->arch.avic_vm_id,
						     svm->vcpu.vcpu_id);
			pi.is_guest_mode = true;
			pi.vcpu_data = &vcpu_info;
			ret = irq_set_vcpu_affinity(host_irq, &pi);

			/**
			 * Here, we successfully setting up vcpu affinity in
			 * IOMMU guest mode. Now, we need to store the posted
			 * interrupt information in a per-vcpu ir_list so that
			 * we can reference to them directly when we update vcpu
			 * scheduling information in IOMMU irte.
			 */
			if (!ret && pi.is_guest_mode)
				svm_ir_list_add(svm, &pi);
		} else {
			/* Use legacy mode in IRTE */
			struct amd_iommu_pi_data pi;

			/**
			 * Here, pi is used to:
			 * - Tell IOMMU to use legacy mode for this interrupt.
			 * - Retrieve ga_tag of prior interrupt remapping data.
			 */
			pi.is_guest_mode = false;
			ret = irq_set_vcpu_affinity(host_irq, &pi);

			/**
			 * Check if the posted interrupt was previously
			 * setup with the guest_mode by checking if the ga_tag
			 * was cached. If so, we need to clean up the per-vcpu
			 * ir_list.
			 */
			if (!ret && pi.prev_ga_tag) {
				int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
				struct kvm_vcpu *vcpu;

				vcpu = kvm_get_vcpu_by_id(kvm, id);
				if (vcpu)
					svm_ir_list_del(to_svm(vcpu), &pi);
			}
		}

		if (!ret && svm) {
			trace_kvm_pi_irte_update(svm->vcpu.vcpu_id,
						 host_irq, e->gsi,
						 vcpu_info.vector,
						 vcpu_info.pi_desc_addr, set);
		}

		if (ret < 0) {
			pr_err("%s: failed to update PI IRTE\n", __func__);
			goto out;
		}
	}

	ret = 0;
out:
	srcu_read_unlock(&kvm->irq_srcu, idx);
	return ret;
}

5131 5132 5133 5134
static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb *vmcb = svm->vmcb;
J
Joerg Roedel 已提交
5135 5136 5137 5138 5139 5140
	int ret;
	ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
	      !(svm->vcpu.arch.hflags & HF_NMI_MASK);
	ret = ret && gif_set(svm) && nested_svm_nmi(svm);

	return ret;
5141 5142
}

J
Jan Kiszka 已提交
5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155
static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
}

static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (masked) {
		svm->vcpu.arch.hflags |= HF_NMI_MASK;
5156
		set_intercept(svm, INTERCEPT_IRET);
J
Jan Kiszka 已提交
5157 5158
	} else {
		svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
5159
		clr_intercept(svm, INTERCEPT_IRET);
J
Jan Kiszka 已提交
5160 5161 5162
	}
}

5163 5164 5165 5166
static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb *vmcb = svm->vmcb;
5167 5168 5169 5170 5171 5172
	int ret;

	if (!gif_set(svm) ||
	     (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
		return 0;

5173
	ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
5174

5175
	if (is_guest_mode(vcpu))
5176 5177 5178
		return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);

	return ret;
5179 5180
}

5181
static void enable_irq_window(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5182
{
5183 5184
	struct vcpu_svm *svm = to_svm(vcpu);

5185 5186 5187
	if (kvm_vcpu_apicv_active(vcpu))
		return;

J
Joerg Roedel 已提交
5188 5189 5190 5191
	/*
	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
	 * get that intercept, this function will be called again though and
5192 5193 5194
	 * we'll get the vintr intercept. However, if the vGIF feature is
	 * enabled, the STGI interception will not occur. Enable the irq
	 * window under the assumption that the hardware will set the GIF.
J
Joerg Roedel 已提交
5195
	 */
5196
	if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) {
5197 5198 5199
		svm_set_vintr(svm);
		svm_inject_irq(svm, 0x0);
	}
5200 5201
}

5202
static void enable_nmi_window(struct kvm_vcpu *vcpu)
5203
{
5204
	struct vcpu_svm *svm = to_svm(vcpu);
5205

5206 5207
	if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
	    == HF_NMI_MASK)
5208
		return; /* IRET will cause a vm exit */
5209

5210 5211 5212
	if (!gif_set(svm)) {
		if (vgif_enabled(svm))
			set_intercept(svm, INTERCEPT_STGI);
5213
		return; /* STGI will cause a vm exit */
5214
	}
5215 5216 5217 5218

	if (svm->nested.exit_required)
		return; /* we're not going to run the guest yet */

J
Joerg Roedel 已提交
5219 5220 5221 5222
	/*
	 * Something prevents NMI from been injected. Single step over possible
	 * problem (IRET or exception injection or interrupt shadow)
	 */
5223
	svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
J
Jan Kiszka 已提交
5224
	svm->nmi_singlestep = true;
5225
	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
5226 5227
}

5228 5229 5230 5231 5232
static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
{
	return 0;
}

5233
static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
5234
{
5235 5236 5237 5238 5239 5240
	struct vcpu_svm *svm = to_svm(vcpu);

	if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
	else
		svm->asid_generation--;
5241 5242
}

5243 5244 5245 5246
static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
{
}

5247 5248 5249 5250
static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

5251
	if (svm_nested_virtualize_tpr(vcpu))
5252 5253
		return;

5254
	if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
5255
		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
5256
		kvm_set_cr8(vcpu, cr8);
5257 5258 5259
	}
}

5260 5261 5262 5263 5264
static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	u64 cr8;

5265 5266
	if (svm_nested_virtualize_tpr(vcpu) ||
	    kvm_vcpu_apicv_active(vcpu))
5267 5268
		return;

5269 5270 5271 5272 5273
	cr8 = kvm_get_cr8(vcpu);
	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
}

5274 5275 5276 5277 5278
static void svm_complete_interrupts(struct vcpu_svm *svm)
{
	u8 vector;
	int type;
	u32 exitintinfo = svm->vmcb->control.exit_int_info;
5279 5280 5281
	unsigned int3_injected = svm->int3_injected;

	svm->int3_injected = 0;
5282

5283 5284 5285 5286 5287 5288
	/*
	 * If we've made progress since setting HF_IRET_MASK, we've
	 * executed an IRET and can allow NMI injection.
	 */
	if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
	    && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
5289
		svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
5290 5291
		kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
	}
5292

5293 5294 5295 5296 5297 5298 5299
	svm->vcpu.arch.nmi_injected = false;
	kvm_clear_exception_queue(&svm->vcpu);
	kvm_clear_interrupt_queue(&svm->vcpu);

	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
		return;

5300 5301
	kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);

5302 5303 5304 5305 5306 5307 5308 5309
	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;

	switch (type) {
	case SVM_EXITINTINFO_TYPE_NMI:
		svm->vcpu.arch.nmi_injected = true;
		break;
	case SVM_EXITINTINFO_TYPE_EXEPT:
5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320
		/*
		 * In case of software exceptions, do not reinject the vector,
		 * but re-execute the instruction instead. Rewind RIP first
		 * if we emulated INT3 before.
		 */
		if (kvm_exception_is_soft(vector)) {
			if (vector == BP_VECTOR && int3_injected &&
			    kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
				kvm_rip_write(&svm->vcpu,
					      kvm_rip_read(&svm->vcpu) -
					      int3_injected);
5321
			break;
5322
		}
5323 5324
		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
			u32 err = svm->vmcb->control.exit_int_info_err;
5325
			kvm_requeue_exception_e(&svm->vcpu, vector, err);
5326 5327

		} else
5328
			kvm_requeue_exception(&svm->vcpu, vector);
5329 5330
		break;
	case SVM_EXITINTINFO_TYPE_INTR:
5331
		kvm_queue_interrupt(&svm->vcpu, vector, false);
5332 5333 5334 5335 5336 5337
		break;
	default:
		break;
	}
}

A
Avi Kivity 已提交
5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348
static void svm_cancel_injection(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb_control_area *control = &svm->vmcb->control;

	control->exit_int_info = control->event_inj;
	control->exit_int_info_err = control->event_inj_err;
	control->event_inj = 0;
	svm_complete_interrupts(svm);
}

A
Avi Kivity 已提交
5349
static void svm_vcpu_run(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
5350
{
5351
	struct vcpu_svm *svm = to_svm(vcpu);
5352

5353 5354 5355 5356
	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];

5357 5358 5359 5360 5361 5362 5363
	/*
	 * A vmexit emulation is required before the vcpu can be executed
	 * again.
	 */
	if (unlikely(svm->nested.exit_required))
		return;

5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379
	/*
	 * Disable singlestep if we're injecting an interrupt/exception.
	 * We don't want our modified rflags to be pushed on the stack where
	 * we might not be able to easily reset them if we disabled NMI
	 * singlestep later.
	 */
	if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
		/*
		 * Event injection happens before external interrupts cause a
		 * vmexit and interrupts are disabled here, so smp_send_reschedule
		 * is enough to force an immediate vmexit.
		 */
		disable_nmi_singlestep(svm);
		smp_send_reschedule(vcpu->cpu);
	}

R
Rusty Russell 已提交
5380
	pre_svm_run(svm);
A
Avi Kivity 已提交
5381

5382 5383
	sync_lapic_to_cr8(vcpu);

5384
	svm->vmcb->save.cr2 = vcpu->arch.cr2;
A
Avi Kivity 已提交
5385

5386 5387 5388
	clgi();

	local_irq_enable();
5389

5390 5391 5392 5393 5394 5395 5396
	/*
	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
	 * is no need to worry about the conditional branch over the wrmsr
	 * being speculatively taken.
	 */
	if (svm->spec_ctrl)
5397
		native_wrmsrl(MSR_IA32_SPEC_CTRL, svm->spec_ctrl);
5398

A
Avi Kivity 已提交
5399
	asm volatile (
A
Avi Kivity 已提交
5400 5401 5402 5403 5404 5405 5406
		"push %%" _ASM_BP "; \n\t"
		"mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
		"mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
		"mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
		"mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
		"mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
		"mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
5407
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
5408 5409 5410 5411 5412 5413 5414 5415
		"mov %c[r8](%[svm]),  %%r8  \n\t"
		"mov %c[r9](%[svm]),  %%r9  \n\t"
		"mov %c[r10](%[svm]), %%r10 \n\t"
		"mov %c[r11](%[svm]), %%r11 \n\t"
		"mov %c[r12](%[svm]), %%r12 \n\t"
		"mov %c[r13](%[svm]), %%r13 \n\t"
		"mov %c[r14](%[svm]), %%r14 \n\t"
		"mov %c[r15](%[svm]), %%r15 \n\t"
A
Avi Kivity 已提交
5416 5417 5418
#endif

		/* Enter guest mode */
A
Avi Kivity 已提交
5419 5420
		"push %%" _ASM_AX " \n\t"
		"mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
5421 5422 5423
		__ex(SVM_VMLOAD) "\n\t"
		__ex(SVM_VMRUN) "\n\t"
		__ex(SVM_VMSAVE) "\n\t"
A
Avi Kivity 已提交
5424
		"pop %%" _ASM_AX " \n\t"
A
Avi Kivity 已提交
5425 5426

		/* Save guest registers, load host registers */
A
Avi Kivity 已提交
5427 5428 5429 5430 5431 5432
		"mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
		"mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
		"mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
		"mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
		"mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
		"mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
5433
#ifdef CONFIG_X86_64
R
Rusty Russell 已提交
5434 5435 5436 5437 5438 5439 5440 5441
		"mov %%r8,  %c[r8](%[svm]) \n\t"
		"mov %%r9,  %c[r9](%[svm]) \n\t"
		"mov %%r10, %c[r10](%[svm]) \n\t"
		"mov %%r11, %c[r11](%[svm]) \n\t"
		"mov %%r12, %c[r12](%[svm]) \n\t"
		"mov %%r13, %c[r13](%[svm]) \n\t"
		"mov %%r14, %c[r14](%[svm]) \n\t"
		"mov %%r15, %c[r15](%[svm]) \n\t"
5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460
#endif
		/*
		* Clear host registers marked as clobbered to prevent
		* speculative use.
		*/
		"xor %%" _ASM_BX ", %%" _ASM_BX " \n\t"
		"xor %%" _ASM_CX ", %%" _ASM_CX " \n\t"
		"xor %%" _ASM_DX ", %%" _ASM_DX " \n\t"
		"xor %%" _ASM_SI ", %%" _ASM_SI " \n\t"
		"xor %%" _ASM_DI ", %%" _ASM_DI " \n\t"
#ifdef CONFIG_X86_64
		"xor %%r8, %%r8 \n\t"
		"xor %%r9, %%r9 \n\t"
		"xor %%r10, %%r10 \n\t"
		"xor %%r11, %%r11 \n\t"
		"xor %%r12, %%r12 \n\t"
		"xor %%r13, %%r13 \n\t"
		"xor %%r14, %%r14 \n\t"
		"xor %%r15, %%r15 \n\t"
A
Avi Kivity 已提交
5461
#endif
A
Avi Kivity 已提交
5462
		"pop %%" _ASM_BP
A
Avi Kivity 已提交
5463
		:
R
Rusty Russell 已提交
5464
		: [svm]"a"(svm),
A
Avi Kivity 已提交
5465
		  [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
5466 5467 5468 5469 5470 5471
		  [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
		  [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
		  [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
		  [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
		  [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
		  [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
5472
#ifdef CONFIG_X86_64
5473 5474 5475 5476 5477 5478 5479 5480
		  , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
		  [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
		  [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
		  [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
		  [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
		  [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
		  [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
		  [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
A
Avi Kivity 已提交
5481
#endif
5482 5483
		: "cc", "memory"
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
5484
		, "rbx", "rcx", "rdx", "rsi", "rdi"
5485
		, "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
A
Avi Kivity 已提交
5486 5487
#else
		, "ebx", "ecx", "edx", "esi", "edi"
5488 5489
#endif
		);
A
Avi Kivity 已提交
5490

5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505
	/*
	 * We do not use IBRS in the kernel. If this vCPU has used the
	 * SPEC_CTRL MSR it may have left it on; save the value and
	 * turn it off. This is much more efficient than blindly adding
	 * it to the atomic save/restore list. Especially as the former
	 * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
	 *
	 * For non-nested case:
	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
	 * save it.
	 *
	 * For nested case:
	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
	 * save it.
	 */
5506
	if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
5507
		svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
5508 5509

	if (svm->spec_ctrl)
5510
		native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
5511

5512 5513 5514
	/* Eliminate branch target predictions from guest mode */
	vmexit_fill_RSB();

5515 5516 5517
#ifdef CONFIG_X86_64
	wrmsrl(MSR_GS_BASE, svm->host.gs_base);
#else
5518
	loadsegment(fs, svm->host.fs);
5519 5520 5521
#ifndef CONFIG_X86_32_LAZY_GS
	loadsegment(gs, svm->host.gs);
#endif
5522
#endif
A
Avi Kivity 已提交
5523 5524 5525

	reload_tss(vcpu);

5526 5527
	local_irq_disable();

5528 5529 5530 5531 5532
	vcpu->arch.cr2 = svm->vmcb->save.cr2;
	vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
	vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
	vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;

5533 5534 5535 5536 5537 5538 5539 5540 5541 5542
	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
		kvm_before_handle_nmi(&svm->vcpu);

	stgi();

	/* Any pending NMI will happen here */

	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
		kvm_after_handle_nmi(&svm->vcpu);

5543 5544
	sync_cr8_to_lapic(vcpu);

5545
	svm->next_rip = 0;
5546

5547 5548
	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;

G
Gleb Natapov 已提交
5549 5550
	/* if exit due to PF check for async PF */
	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
5551
		svm->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
G
Gleb Natapov 已提交
5552

A
Avi Kivity 已提交
5553 5554 5555 5556
	if (npt_enabled) {
		vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
		vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
	}
5557 5558 5559 5560 5561 5562 5563 5564

	/*
	 * We need to handle MC intercepts here before the vcpu has a chance to
	 * change the physical cpu
	 */
	if (unlikely(svm->vmcb->control.exit_code ==
		     SVM_EXIT_EXCP_BASE + MC_VECTOR))
		svm_handle_mce(svm);
5565 5566

	mark_all_clean(svm->vmcb);
A
Avi Kivity 已提交
5567
}
5568
STACK_FRAME_NON_STANDARD(svm_vcpu_run);
A
Avi Kivity 已提交
5569 5570 5571

static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
{
5572 5573
	struct vcpu_svm *svm = to_svm(vcpu);

5574
	svm->vmcb->save.cr3 = __sme_set(root);
5575
	mark_dirty(svm->vmcb, VMCB_CR);
5576
	svm_flush_tlb(vcpu, true);
A
Avi Kivity 已提交
5577 5578
}

5579 5580 5581 5582
static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
{
	struct vcpu_svm *svm = to_svm(vcpu);

5583
	svm->vmcb->control.nested_cr3 = __sme_set(root);
5584
	mark_dirty(svm->vmcb, VMCB_NPT);
5585 5586

	/* Also sync guest cr3 here in case we live migrate */
5587
	svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
5588
	mark_dirty(svm->vmcb, VMCB_CR);
5589

5590
	svm_flush_tlb(vcpu, true);
5591 5592
}

A
Avi Kivity 已提交
5593 5594
static int is_disabled(void)
{
5595 5596 5597 5598 5599 5600
	u64 vm_cr;

	rdmsrl(MSR_VM_CR, vm_cr);
	if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
		return 1;

A
Avi Kivity 已提交
5601 5602 5603
	return 0;
}

I
Ingo Molnar 已提交
5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614
static void
svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
{
	/*
	 * Patch in the VMMCALL instruction:
	 */
	hypercall[0] = 0x0f;
	hypercall[1] = 0x01;
	hypercall[2] = 0xd9;
}

Y
Yang, Sheng 已提交
5615 5616 5617 5618 5619
static void svm_check_processor_compat(void *rtn)
{
	*(int *)rtn = 0;
}

5620 5621 5622 5623 5624
static bool svm_cpu_has_accelerated_tpr(void)
{
	return false;
}

5625 5626 5627 5628 5629
static bool svm_has_high_real_mode_segbase(void)
{
	return true;
}

5630 5631 5632 5633 5634
static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
{
	return 0;
}

5635 5636
static void svm_cpuid_update(struct kvm_vcpu *vcpu)
{
5637 5638 5639
	struct vcpu_svm *svm = to_svm(vcpu);

	/* Update nrips enabled cache */
5640
	svm->nrips_enabled = !!guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS);
5641 5642 5643 5644

	if (!kvm_vcpu_apicv_active(vcpu))
		return;

5645
	guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
5646 5647
}

5648 5649
static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
{
5650
	switch (func) {
5651 5652 5653 5654
	case 0x1:
		if (avic)
			entry->ecx &= ~bit(X86_FEATURE_X2APIC);
		break;
5655 5656 5657 5658
	case 0x80000001:
		if (nested)
			entry->ecx |= (1 << 2); /* Set SVM bit */
		break;
5659 5660 5661 5662 5663
	case 0x8000000A:
		entry->eax = 1; /* SVM revision 1 */
		entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
				   ASID emulation to nested SVM */
		entry->ecx = 0; /* Reserved */
5664 5665 5666 5667
		entry->edx = 0; /* Per default do not support any
				   additional features */

		/* Support next_rip if host supports it */
5668
		if (boot_cpu_has(X86_FEATURE_NRIPS))
5669
			entry->edx |= SVM_FEATURE_NRIP;
5670

5671 5672 5673 5674
		/* Support NPT for the guest if enabled */
		if (npt_enabled)
			entry->edx |= SVM_FEATURE_NPT;

5675
		break;
5676 5677 5678 5679 5680 5681
	case 0x8000001F:
		/* Support memory encryption cpuid if host supports it */
		if (boot_cpu_has(X86_FEATURE_SEV))
			cpuid(0x8000001f, &entry->eax, &entry->ebx,
				&entry->ecx, &entry->edx);

5682
	}
5683 5684
}

5685
static int svm_get_lpage_level(void)
5686
{
5687
	return PT_PDPE_LEVEL;
5688 5689
}

5690 5691
static bool svm_rdtscp_supported(void)
{
P
Paolo Bonzini 已提交
5692
	return boot_cpu_has(X86_FEATURE_RDTSCP);
5693 5694
}

5695 5696 5697 5698 5699
static bool svm_invpcid_supported(void)
{
	return false;
}

5700 5701 5702 5703 5704
static bool svm_mpx_supported(void)
{
	return false;
}

5705 5706 5707 5708 5709
static bool svm_xsaves_supported(void)
{
	return false;
}

5710 5711 5712 5713 5714
static bool svm_umip_emulated(void)
{
	return false;
}

5715 5716 5717 5718 5719
static bool svm_has_wbinvd_exit(void)
{
	return true;
}

5720
#define PRE_EX(exit)  { .exit_code = (exit), \
5721
			.stage = X86_ICPT_PRE_EXCEPT, }
5722
#define POST_EX(exit) { .exit_code = (exit), \
5723
			.stage = X86_ICPT_POST_EXCEPT, }
5724
#define POST_MEM(exit) { .exit_code = (exit), \
5725
			.stage = X86_ICPT_POST_MEMACCESS, }
5726

5727
static const struct __x86_intercept {
5728 5729 5730 5731 5732 5733 5734 5735
	u32 exit_code;
	enum x86_intercept_stage stage;
} x86_intercept_map[] = {
	[x86_intercept_cr_read]		= POST_EX(SVM_EXIT_READ_CR0),
	[x86_intercept_cr_write]	= POST_EX(SVM_EXIT_WRITE_CR0),
	[x86_intercept_clts]		= POST_EX(SVM_EXIT_WRITE_CR0),
	[x86_intercept_lmsw]		= POST_EX(SVM_EXIT_WRITE_CR0),
	[x86_intercept_smsw]		= POST_EX(SVM_EXIT_READ_CR0),
5736 5737
	[x86_intercept_dr_read]		= POST_EX(SVM_EXIT_READ_DR0),
	[x86_intercept_dr_write]	= POST_EX(SVM_EXIT_WRITE_DR0),
5738 5739 5740 5741 5742 5743 5744 5745
	[x86_intercept_sldt]		= POST_EX(SVM_EXIT_LDTR_READ),
	[x86_intercept_str]		= POST_EX(SVM_EXIT_TR_READ),
	[x86_intercept_lldt]		= POST_EX(SVM_EXIT_LDTR_WRITE),
	[x86_intercept_ltr]		= POST_EX(SVM_EXIT_TR_WRITE),
	[x86_intercept_sgdt]		= POST_EX(SVM_EXIT_GDTR_READ),
	[x86_intercept_sidt]		= POST_EX(SVM_EXIT_IDTR_READ),
	[x86_intercept_lgdt]		= POST_EX(SVM_EXIT_GDTR_WRITE),
	[x86_intercept_lidt]		= POST_EX(SVM_EXIT_IDTR_WRITE),
5746 5747 5748 5749 5750 5751 5752 5753
	[x86_intercept_vmrun]		= POST_EX(SVM_EXIT_VMRUN),
	[x86_intercept_vmmcall]		= POST_EX(SVM_EXIT_VMMCALL),
	[x86_intercept_vmload]		= POST_EX(SVM_EXIT_VMLOAD),
	[x86_intercept_vmsave]		= POST_EX(SVM_EXIT_VMSAVE),
	[x86_intercept_stgi]		= POST_EX(SVM_EXIT_STGI),
	[x86_intercept_clgi]		= POST_EX(SVM_EXIT_CLGI),
	[x86_intercept_skinit]		= POST_EX(SVM_EXIT_SKINIT),
	[x86_intercept_invlpga]		= POST_EX(SVM_EXIT_INVLPGA),
5754 5755 5756
	[x86_intercept_rdtscp]		= POST_EX(SVM_EXIT_RDTSCP),
	[x86_intercept_monitor]		= POST_MEM(SVM_EXIT_MONITOR),
	[x86_intercept_mwait]		= POST_EX(SVM_EXIT_MWAIT),
5757 5758 5759 5760 5761 5762 5763 5764 5765
	[x86_intercept_invlpg]		= POST_EX(SVM_EXIT_INVLPG),
	[x86_intercept_invd]		= POST_EX(SVM_EXIT_INVD),
	[x86_intercept_wbinvd]		= POST_EX(SVM_EXIT_WBINVD),
	[x86_intercept_wrmsr]		= POST_EX(SVM_EXIT_MSR),
	[x86_intercept_rdtsc]		= POST_EX(SVM_EXIT_RDTSC),
	[x86_intercept_rdmsr]		= POST_EX(SVM_EXIT_MSR),
	[x86_intercept_rdpmc]		= POST_EX(SVM_EXIT_RDPMC),
	[x86_intercept_cpuid]		= PRE_EX(SVM_EXIT_CPUID),
	[x86_intercept_rsm]		= PRE_EX(SVM_EXIT_RSM),
5766 5767 5768 5769 5770 5771 5772
	[x86_intercept_pause]		= PRE_EX(SVM_EXIT_PAUSE),
	[x86_intercept_pushf]		= PRE_EX(SVM_EXIT_PUSHF),
	[x86_intercept_popf]		= PRE_EX(SVM_EXIT_POPF),
	[x86_intercept_intn]		= PRE_EX(SVM_EXIT_SWINT),
	[x86_intercept_iret]		= PRE_EX(SVM_EXIT_IRET),
	[x86_intercept_icebp]		= PRE_EX(SVM_EXIT_ICEBP),
	[x86_intercept_hlt]		= POST_EX(SVM_EXIT_HLT),
5773 5774 5775 5776
	[x86_intercept_in]		= POST_EX(SVM_EXIT_IOIO),
	[x86_intercept_ins]		= POST_EX(SVM_EXIT_IOIO),
	[x86_intercept_out]		= POST_EX(SVM_EXIT_IOIO),
	[x86_intercept_outs]		= POST_EX(SVM_EXIT_IOIO),
5777 5778
};

5779
#undef PRE_EX
5780
#undef POST_EX
5781
#undef POST_MEM
5782

5783 5784 5785 5786
static int svm_check_intercept(struct kvm_vcpu *vcpu,
			       struct x86_instruction_info *info,
			       enum x86_intercept_stage stage)
{
5787 5788 5789 5790 5791 5792 5793 5794 5795 5796
	struct vcpu_svm *svm = to_svm(vcpu);
	int vmexit, ret = X86EMUL_CONTINUE;
	struct __x86_intercept icpt_info;
	struct vmcb *vmcb = svm->vmcb;

	if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
		goto out;

	icpt_info = x86_intercept_map[info->intercept];

5797
	if (stage != icpt_info.stage)
5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811
		goto out;

	switch (icpt_info.exit_code) {
	case SVM_EXIT_READ_CR0:
		if (info->intercept == x86_intercept_cr_read)
			icpt_info.exit_code += info->modrm_reg;
		break;
	case SVM_EXIT_WRITE_CR0: {
		unsigned long cr0, val;
		u64 intercept;

		if (info->intercept == x86_intercept_cr_write)
			icpt_info.exit_code += info->modrm_reg;

5812 5813
		if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
		    info->intercept == x86_intercept_clts)
5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836
			break;

		intercept = svm->nested.intercept;

		if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
			break;

		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;

		if (info->intercept == x86_intercept_lmsw) {
			cr0 &= 0xfUL;
			val &= 0xfUL;
			/* lmsw can't clear PE - catch this here */
			if (cr0 & X86_CR0_PE)
				val |= X86_CR0_PE;
		}

		if (cr0 ^ val)
			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;

		break;
	}
5837 5838 5839 5840
	case SVM_EXIT_READ_DR0:
	case SVM_EXIT_WRITE_DR0:
		icpt_info.exit_code += info->modrm_reg;
		break;
5841 5842 5843 5844 5845 5846
	case SVM_EXIT_MSR:
		if (info->intercept == x86_intercept_wrmsr)
			vmcb->control.exit_info_1 = 1;
		else
			vmcb->control.exit_info_1 = 0;
		break;
5847 5848 5849 5850 5851 5852 5853
	case SVM_EXIT_PAUSE:
		/*
		 * We get this for NOP only, but pause
		 * is rep not, check this here
		 */
		if (info->rep_prefix != REPE_PREFIX)
			goto out;
5854
		break;
5855 5856 5857 5858 5859 5860
	case SVM_EXIT_IOIO: {
		u64 exit_info;
		u32 bytes;

		if (info->intercept == x86_intercept_in ||
		    info->intercept == x86_intercept_ins) {
5861 5862
			exit_info = ((info->src_val & 0xffff) << 16) |
				SVM_IOIO_TYPE_MASK;
5863
			bytes = info->dst_bytes;
5864
		} else {
5865
			exit_info = (info->dst_val & 0xffff) << 16;
5866
			bytes = info->src_bytes;
5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886
		}

		if (info->intercept == x86_intercept_outs ||
		    info->intercept == x86_intercept_ins)
			exit_info |= SVM_IOIO_STR_MASK;

		if (info->rep_prefix)
			exit_info |= SVM_IOIO_REP_MASK;

		bytes = min(bytes, 4u);

		exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;

		exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);

		vmcb->control.exit_info_1 = exit_info;
		vmcb->control.exit_info_2 = info->next_rip;

		break;
	}
5887 5888 5889 5890
	default:
		break;
	}

5891 5892 5893
	/* TODO: Advertise NRIPS to guest hypervisor unconditionally */
	if (static_cpu_has(X86_FEATURE_NRIPS))
		vmcb->control.next_rip  = info->next_rip;
5894 5895 5896 5897 5898 5899 5900 5901
	vmcb->control.exit_code = icpt_info.exit_code;
	vmexit = nested_svm_exit_handled(svm);

	ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
					   : X86EMUL_CONTINUE;

out:
	return ret;
5902 5903
}

5904 5905 5906
static void svm_handle_external_intr(struct kvm_vcpu *vcpu)
{
	local_irq_enable();
P
Paolo Bonzini 已提交
5907 5908 5909 5910 5911 5912
	/*
	 * We must have an instruction with interrupts enabled, so
	 * the timer interrupt isn't delayed by the interrupt shadow.
	 */
	asm("nop");
	local_irq_disable();
5913 5914
}

5915 5916 5917 5918
static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
{
}

5919 5920 5921 5922 5923 5924 5925 5926 5927
static inline void avic_post_state_restore(struct kvm_vcpu *vcpu)
{
	if (avic_handle_apic_id_update(vcpu) != 0)
		return;
	if (avic_handle_dfr_update(vcpu) != 0)
		return;
	avic_handle_ldr_update(vcpu);
}

5928 5929 5930 5931 5932 5933
static void svm_setup_mce(struct kvm_vcpu *vcpu)
{
	/* [63:9] are reserved. */
	vcpu->arch.mcg_cap &= 0x1ff;
}

5934 5935
static int svm_smi_allowed(struct kvm_vcpu *vcpu)
{
5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949
	struct vcpu_svm *svm = to_svm(vcpu);

	/* Per APM Vol.2 15.22.2 "Response to SMI" */
	if (!gif_set(svm))
		return 0;

	if (is_guest_mode(&svm->vcpu) &&
	    svm->nested.intercept & (1ULL << INTERCEPT_SMI)) {
		/* TODO: Might need to set exit_info_1 and exit_info_2 here */
		svm->vmcb->control.exit_code = SVM_EXIT_SMI;
		svm->nested.exit_required = true;
		return 0;
	}

5950 5951 5952
	return 1;
}

5953 5954
static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
{
5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971
	struct vcpu_svm *svm = to_svm(vcpu);
	int ret;

	if (is_guest_mode(vcpu)) {
		/* FED8h - SVM Guest */
		put_smstate(u64, smstate, 0x7ed8, 1);
		/* FEE0h - SVM Guest VMCB Physical Address */
		put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb);

		svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
		svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
		svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];

		ret = nested_svm_vmexit(svm);
		if (ret)
			return ret;
	}
5972 5973 5974 5975 5976
	return 0;
}

static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, u64 smbase)
{
5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000
	struct vcpu_svm *svm = to_svm(vcpu);
	struct vmcb *nested_vmcb;
	struct page *page;
	struct {
		u64 guest;
		u64 vmcb;
	} svm_state_save;
	int ret;

	ret = kvm_vcpu_read_guest(vcpu, smbase + 0xfed8, &svm_state_save,
				  sizeof(svm_state_save));
	if (ret)
		return ret;

	if (svm_state_save.guest) {
		vcpu->arch.hflags &= ~HF_SMM_MASK;
		nested_vmcb = nested_svm_map(svm, svm_state_save.vmcb, &page);
		if (nested_vmcb)
			enter_svm_guest_mode(svm, svm_state_save.vmcb, nested_vmcb, page);
		else
			ret = 1;
		vcpu->arch.hflags |= HF_SMM_MASK;
	}
	return ret;
6001 6002
}

6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015
static int enable_smi_window(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	if (!gif_set(svm)) {
		if (vgif_enabled(svm))
			set_intercept(svm, INTERCEPT_STGI);
		/* STGI will cause a vm exit */
		return 1;
	}
	return 0;
}

B
Brijesh Singh 已提交
6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046
static int sev_asid_new(void)
{
	int pos;

	/*
	 * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid.
	 */
	pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1);
	if (pos >= max_sev_asid)
		return -EBUSY;

	set_bit(pos, sev_asid_bitmap);
	return pos + 1;
}

static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	int asid, ret;

	ret = -EBUSY;
	asid = sev_asid_new();
	if (asid < 0)
		return ret;

	ret = sev_platform_init(&argp->error);
	if (ret)
		goto e_free;

	sev->active = true;
	sev->asid = asid;
6047
	INIT_LIST_HEAD(&sev->regions_list);
B
Brijesh Singh 已提交
6048 6049 6050 6051 6052 6053 6054 6055

	return 0;

e_free:
	__sev_asid_free(asid);
	return ret;
}

6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080
static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
{
	struct sev_data_activate *data;
	int asid = sev_get_asid(kvm);
	int ret;

	wbinvd_on_all_cpus();

	ret = sev_guest_df_flush(error);
	if (ret)
		return ret;

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	/* activate ASID on the given handle */
	data->handle = handle;
	data->asid   = asid;
	ret = sev_guest_activate(data, error);
	kfree(data);

	return ret;
}

6081
static int __sev_issue_cmd(int fd, int id, void *data, int *error)
6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095
{
	struct fd f;
	int ret;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = sev_issue_cmd_external_user(f.file, id, data, error);

	fdput(f);
	return ret;
}

6096 6097 6098 6099 6100 6101 6102
static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;

	return __sev_issue_cmd(sev->fd, id, data, error);
}

6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149
static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	struct sev_data_launch_start *start;
	struct kvm_sev_launch_start params;
	void *dh_blob, *session_blob;
	int *error = &argp->error;
	int ret;

	if (!sev_guest(kvm))
		return -ENOTTY;

	if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
		return -EFAULT;

	start = kzalloc(sizeof(*start), GFP_KERNEL);
	if (!start)
		return -ENOMEM;

	dh_blob = NULL;
	if (params.dh_uaddr) {
		dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
		if (IS_ERR(dh_blob)) {
			ret = PTR_ERR(dh_blob);
			goto e_free;
		}

		start->dh_cert_address = __sme_set(__pa(dh_blob));
		start->dh_cert_len = params.dh_len;
	}

	session_blob = NULL;
	if (params.session_uaddr) {
		session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
		if (IS_ERR(session_blob)) {
			ret = PTR_ERR(session_blob);
			goto e_free_dh;
		}

		start->session_address = __sme_set(__pa(session_blob));
		start->session_len = params.session_len;
	}

	start->handle = params.handle;
	start->policy = params.policy;

	/* create memory encryption context */
6150
	ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error);
6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178
	if (ret)
		goto e_free_session;

	/* Bind ASID to this guest */
	ret = sev_bind_asid(kvm, start->handle, error);
	if (ret)
		goto e_free_session;

	/* return handle to userspace */
	params.handle = start->handle;
	if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params))) {
		sev_unbind_asid(kvm, start->handle);
		ret = -EFAULT;
		goto e_free_session;
	}

	sev->handle = start->handle;
	sev->fd = argp->sev_fd;

e_free_session:
	kfree(session_blob);
e_free_dh:
	kfree(dh_blob);
e_free:
	kfree(start);
	return ret;
}

6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275
static int get_num_contig_pages(int idx, struct page **inpages,
				unsigned long npages)
{
	unsigned long paddr, next_paddr;
	int i = idx + 1, pages = 1;

	/* find the number of contiguous pages starting from idx */
	paddr = __sme_page_pa(inpages[idx]);
	while (i < npages) {
		next_paddr = __sme_page_pa(inpages[i++]);
		if ((paddr + PAGE_SIZE) == next_paddr) {
			pages++;
			paddr = next_paddr;
			continue;
		}
		break;
	}

	return pages;
}

static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
{
	unsigned long vaddr, vaddr_end, next_vaddr, npages, size;
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	struct kvm_sev_launch_update_data params;
	struct sev_data_launch_update_data *data;
	struct page **inpages;
	int i, ret, pages;

	if (!sev_guest(kvm))
		return -ENOTTY;

	if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
		return -EFAULT;

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	vaddr = params.uaddr;
	size = params.len;
	vaddr_end = vaddr + size;

	/* Lock the user memory. */
	inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
	if (!inpages) {
		ret = -ENOMEM;
		goto e_free;
	}

	/*
	 * The LAUNCH_UPDATE command will perform in-place encryption of the
	 * memory content (i.e it will write the same memory region with C=1).
	 * It's possible that the cache may contain the data with C=0, i.e.,
	 * unencrypted so invalidate it first.
	 */
	sev_clflush_pages(inpages, npages);

	for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
		int offset, len;

		/*
		 * If the user buffer is not page-aligned, calculate the offset
		 * within the page.
		 */
		offset = vaddr & (PAGE_SIZE - 1);

		/* Calculate the number of pages that can be encrypted in one go. */
		pages = get_num_contig_pages(i, inpages, npages);

		len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);

		data->handle = sev->handle;
		data->len = len;
		data->address = __sme_page_pa(inpages[i]) + offset;
		ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error);
		if (ret)
			goto e_unpin;

		size -= len;
		next_vaddr = vaddr + len;
	}

e_unpin:
	/* content of memory is updated, mark pages dirty */
	for (i = 0; i < npages; i++) {
		set_page_dirty_lock(inpages[i]);
		mark_page_accessed(inpages[i]);
	}
	/* unlock the user pages */
	sev_unpin_memory(kvm, inpages, npages);
e_free:
	kfree(data);
	return ret;
}

6276 6277
static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
{
6278
	void __user *measure = (void __user *)(uintptr_t)argp->data;
6279 6280 6281
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	struct sev_data_launch_measure *data;
	struct kvm_sev_launch_measure params;
6282
	void __user *p = NULL;
6283 6284 6285 6286 6287 6288
	void *blob = NULL;
	int ret;

	if (!sev_guest(kvm))
		return -ENOTTY;

6289
	if (copy_from_user(&params, measure, sizeof(params)))
6290 6291 6292 6293 6294 6295 6296 6297 6298 6299
		return -EFAULT;

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	/* User wants to query the blob length */
	if (!params.len)
		goto cmd;

6300 6301
	p = (void __user *)(uintptr_t)params.uaddr;
	if (p) {
6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329
		if (params.len > SEV_FW_BLOB_MAX_SIZE) {
			ret = -EINVAL;
			goto e_free;
		}

		ret = -ENOMEM;
		blob = kmalloc(params.len, GFP_KERNEL);
		if (!blob)
			goto e_free;

		data->address = __psp_pa(blob);
		data->len = params.len;
	}

cmd:
	data->handle = sev->handle;
	ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error);

	/*
	 * If we query the session length, FW responded with expected data.
	 */
	if (!params.len)
		goto done;

	if (ret)
		goto e_free_blob;

	if (blob) {
6330
		if (copy_to_user(p, blob, params.len))
6331 6332 6333 6334 6335
			ret = -EFAULT;
	}

done:
	params.len = data->len;
6336
	if (copy_to_user(measure, &params, sizeof(params)))
6337 6338 6339 6340 6341 6342 6343 6344
		ret = -EFAULT;
e_free_blob:
	kfree(blob);
e_free:
	kfree(data);
	return ret;
}

6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364
static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	struct sev_data_launch_finish *data;
	int ret;

	if (!sev_guest(kvm))
		return -ENOTTY;

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	data->handle = sev->handle;
	ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error);

	kfree(data);
	return ret;
}

6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394
static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	struct kvm_sev_guest_status params;
	struct sev_data_guest_status *data;
	int ret;

	if (!sev_guest(kvm))
		return -ENOTTY;

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	data->handle = sev->handle;
	ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error);
	if (ret)
		goto e_free;

	params.policy = data->policy;
	params.state = data->state;
	params.handle = data->handle;

	if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params)))
		ret = -EFAULT;
e_free:
	kfree(data);
	return ret;
}

6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471
static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
			       unsigned long dst, int size,
			       int *error, bool enc)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	struct sev_data_dbg *data;
	int ret;

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	data->handle = sev->handle;
	data->dst_addr = dst;
	data->src_addr = src;
	data->len = size;

	ret = sev_issue_cmd(kvm,
			    enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
			    data, error);
	kfree(data);
	return ret;
}

static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
			     unsigned long dst_paddr, int sz, int *err)
{
	int offset;

	/*
	 * Its safe to read more than we are asked, caller should ensure that
	 * destination has enough space.
	 */
	src_paddr = round_down(src_paddr, 16);
	offset = src_paddr & 15;
	sz = round_up(sz + offset, 16);

	return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
}

static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
				  unsigned long __user dst_uaddr,
				  unsigned long dst_paddr,
				  int size, int *err)
{
	struct page *tpage = NULL;
	int ret, offset;

	/* if inputs are not 16-byte then use intermediate buffer */
	if (!IS_ALIGNED(dst_paddr, 16) ||
	    !IS_ALIGNED(paddr,     16) ||
	    !IS_ALIGNED(size,      16)) {
		tpage = (void *)alloc_page(GFP_KERNEL);
		if (!tpage)
			return -ENOMEM;

		dst_paddr = __sme_page_pa(tpage);
	}

	ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
	if (ret)
		goto e_free;

	if (tpage) {
		offset = paddr & 15;
		if (copy_to_user((void __user *)(uintptr_t)dst_uaddr,
				 page_address(tpage) + offset, size))
			ret = -EFAULT;
	}

e_free:
	if (tpage)
		__free_page(tpage);

	return ret;
}

6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548
static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
				  unsigned long __user vaddr,
				  unsigned long dst_paddr,
				  unsigned long __user dst_vaddr,
				  int size, int *error)
{
	struct page *src_tpage = NULL;
	struct page *dst_tpage = NULL;
	int ret, len = size;

	/* If source buffer is not aligned then use an intermediate buffer */
	if (!IS_ALIGNED(vaddr, 16)) {
		src_tpage = alloc_page(GFP_KERNEL);
		if (!src_tpage)
			return -ENOMEM;

		if (copy_from_user(page_address(src_tpage),
				(void __user *)(uintptr_t)vaddr, size)) {
			__free_page(src_tpage);
			return -EFAULT;
		}

		paddr = __sme_page_pa(src_tpage);
	}

	/*
	 *  If destination buffer or length is not aligned then do read-modify-write:
	 *   - decrypt destination in an intermediate buffer
	 *   - copy the source buffer in an intermediate buffer
	 *   - use the intermediate buffer as source buffer
	 */
	if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
		int dst_offset;

		dst_tpage = alloc_page(GFP_KERNEL);
		if (!dst_tpage) {
			ret = -ENOMEM;
			goto e_free;
		}

		ret = __sev_dbg_decrypt(kvm, dst_paddr,
					__sme_page_pa(dst_tpage), size, error);
		if (ret)
			goto e_free;

		/*
		 *  If source is kernel buffer then use memcpy() otherwise
		 *  copy_from_user().
		 */
		dst_offset = dst_paddr & 15;

		if (src_tpage)
			memcpy(page_address(dst_tpage) + dst_offset,
			       page_address(src_tpage), size);
		else {
			if (copy_from_user(page_address(dst_tpage) + dst_offset,
					   (void __user *)(uintptr_t)vaddr, size)) {
				ret = -EFAULT;
				goto e_free;
			}
		}

		paddr = __sme_page_pa(dst_tpage);
		dst_paddr = round_down(dst_paddr, 16);
		len = round_up(size, 16);
	}

	ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);

e_free:
	if (src_tpage)
		__free_page(src_tpage);
	if (dst_tpage)
		__free_page(dst_tpage);
	return ret;
}

6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600
static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
{
	unsigned long vaddr, vaddr_end, next_vaddr;
	unsigned long dst_vaddr, dst_vaddr_end;
	struct page **src_p, **dst_p;
	struct kvm_sev_dbg debug;
	unsigned long n;
	int ret, size;

	if (!sev_guest(kvm))
		return -ENOTTY;

	if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
		return -EFAULT;

	vaddr = debug.src_uaddr;
	size = debug.len;
	vaddr_end = vaddr + size;
	dst_vaddr = debug.dst_uaddr;
	dst_vaddr_end = dst_vaddr + size;

	for (; vaddr < vaddr_end; vaddr = next_vaddr) {
		int len, s_off, d_off;

		/* lock userspace source and destination page */
		src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
		if (!src_p)
			return -EFAULT;

		dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
		if (!dst_p) {
			sev_unpin_memory(kvm, src_p, n);
			return -EFAULT;
		}

		/*
		 * The DBG_{DE,EN}CRYPT commands will perform {dec,en}cryption of the
		 * memory content (i.e it will write the same memory region with C=1).
		 * It's possible that the cache may contain the data with C=0, i.e.,
		 * unencrypted so invalidate it first.
		 */
		sev_clflush_pages(src_p, 1);
		sev_clflush_pages(dst_p, 1);

		/*
		 * Since user buffer may not be page aligned, calculate the
		 * offset within the page.
		 */
		s_off = vaddr & ~PAGE_MASK;
		d_off = dst_vaddr & ~PAGE_MASK;
		len = min_t(size_t, (PAGE_SIZE - s_off), size);

6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613
		if (dec)
			ret = __sev_dbg_decrypt_user(kvm,
						     __sme_page_pa(src_p[0]) + s_off,
						     dst_vaddr,
						     __sme_page_pa(dst_p[0]) + d_off,
						     len, &argp->error);
		else
			ret = __sev_dbg_encrypt_user(kvm,
						     __sme_page_pa(src_p[0]) + s_off,
						     vaddr,
						     __sme_page_pa(dst_p[0]) + d_off,
						     dst_vaddr,
						     len, &argp->error);
6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628

		sev_unpin_memory(kvm, src_p, 1);
		sev_unpin_memory(kvm, dst_p, 1);

		if (ret)
			goto err;

		next_vaddr = vaddr + len;
		dst_vaddr = dst_vaddr + len;
		size -= len;
	}
err:
	return ret;
}

6629 6630 6631 6632 6633 6634 6635 6636
static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	struct sev_data_launch_secret *data;
	struct kvm_sev_launch_secret params;
	struct page **pages;
	void *blob, *hdr;
	unsigned long n;
6637
	int ret, offset;
6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662

	if (!sev_guest(kvm))
		return -ENOTTY;

	if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
		return -EFAULT;

	pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
	if (!pages)
		return -ENOMEM;

	/*
	 * The secret must be copied into contiguous memory region, lets verify
	 * that userspace memory pages are contiguous before we issue command.
	 */
	if (get_num_contig_pages(0, pages, n) != n) {
		ret = -EINVAL;
		goto e_unpin_memory;
	}

	ret = -ENOMEM;
	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		goto e_unpin_memory;

6663 6664 6665 6666
	offset = params.guest_uaddr & (PAGE_SIZE - 1);
	data->guest_address = __sme_page_pa(pages[0]) + offset;
	data->guest_len = params.guest_len;

6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680
	blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
	if (IS_ERR(blob)) {
		ret = PTR_ERR(blob);
		goto e_free;
	}

	data->trans_address = __psp_pa(blob);
	data->trans_len = params.trans_len;

	hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
	if (IS_ERR(hdr)) {
		ret = PTR_ERR(hdr);
		goto e_free_blob;
	}
6681 6682
	data->hdr_address = __psp_pa(hdr);
	data->hdr_len = params.hdr_len;
6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697

	data->handle = sev->handle;
	ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error);

	kfree(hdr);

e_free_blob:
	kfree(blob);
e_free:
	kfree(data);
e_unpin_memory:
	sev_unpin_memory(kvm, pages, n);
	return ret;
}

B
Brijesh Singh 已提交
6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714
static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
{
	struct kvm_sev_cmd sev_cmd;
	int r;

	if (!svm_sev_enabled())
		return -ENOTTY;

	if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
		return -EFAULT;

	mutex_lock(&kvm->lock);

	switch (sev_cmd.id) {
	case KVM_SEV_INIT:
		r = sev_guest_init(kvm, &sev_cmd);
		break;
6715 6716 6717
	case KVM_SEV_LAUNCH_START:
		r = sev_launch_start(kvm, &sev_cmd);
		break;
6718 6719 6720
	case KVM_SEV_LAUNCH_UPDATE_DATA:
		r = sev_launch_update_data(kvm, &sev_cmd);
		break;
6721 6722 6723
	case KVM_SEV_LAUNCH_MEASURE:
		r = sev_launch_measure(kvm, &sev_cmd);
		break;
6724 6725 6726
	case KVM_SEV_LAUNCH_FINISH:
		r = sev_launch_finish(kvm, &sev_cmd);
		break;
6727 6728 6729
	case KVM_SEV_GUEST_STATUS:
		r = sev_guest_status(kvm, &sev_cmd);
		break;
6730 6731 6732
	case KVM_SEV_DBG_DECRYPT:
		r = sev_dbg_crypt(kvm, &sev_cmd, true);
		break;
6733 6734 6735
	case KVM_SEV_DBG_ENCRYPT:
		r = sev_dbg_crypt(kvm, &sev_cmd, false);
		break;
6736 6737 6738
	case KVM_SEV_LAUNCH_SECRET:
		r = sev_launch_secret(kvm, &sev_cmd);
		break;
B
Brijesh Singh 已提交
6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751
	default:
		r = -EINVAL;
		goto out;
	}

	if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
		r = -EFAULT;

out:
	mutex_unlock(&kvm->lock);
	return r;
}

6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839
static int svm_register_enc_region(struct kvm *kvm,
				   struct kvm_enc_region *range)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	struct enc_region *region;
	int ret = 0;

	if (!sev_guest(kvm))
		return -ENOTTY;

	region = kzalloc(sizeof(*region), GFP_KERNEL);
	if (!region)
		return -ENOMEM;

	region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
	if (!region->pages) {
		ret = -ENOMEM;
		goto e_free;
	}

	/*
	 * The guest may change the memory encryption attribute from C=0 -> C=1
	 * or vice versa for this memory range. Lets make sure caches are
	 * flushed to ensure that guest data gets written into memory with
	 * correct C-bit.
	 */
	sev_clflush_pages(region->pages, region->npages);

	region->uaddr = range->addr;
	region->size = range->size;

	mutex_lock(&kvm->lock);
	list_add_tail(&region->list, &sev->regions_list);
	mutex_unlock(&kvm->lock);

	return ret;

e_free:
	kfree(region);
	return ret;
}

static struct enc_region *
find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
{
	struct kvm_sev_info *sev = &kvm->arch.sev_info;
	struct list_head *head = &sev->regions_list;
	struct enc_region *i;

	list_for_each_entry(i, head, list) {
		if (i->uaddr == range->addr &&
		    i->size == range->size)
			return i;
	}

	return NULL;
}


static int svm_unregister_enc_region(struct kvm *kvm,
				     struct kvm_enc_region *range)
{
	struct enc_region *region;
	int ret;

	mutex_lock(&kvm->lock);

	if (!sev_guest(kvm)) {
		ret = -ENOTTY;
		goto failed;
	}

	region = find_enc_region(kvm, range);
	if (!region) {
		ret = -EINVAL;
		goto failed;
	}

	__unregister_enc_region_locked(kvm, region);

	mutex_unlock(&kvm->lock);
	return 0;

failed:
	mutex_unlock(&kvm->lock);
	return ret;
}

6840
static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
A
Avi Kivity 已提交
6841 6842 6843 6844
	.cpu_has_kvm_support = has_svm,
	.disabled_by_bios = is_disabled,
	.hardware_setup = svm_hardware_setup,
	.hardware_unsetup = svm_hardware_unsetup,
Y
Yang, Sheng 已提交
6845
	.check_processor_compatibility = svm_check_processor_compat,
A
Avi Kivity 已提交
6846 6847
	.hardware_enable = svm_hardware_enable,
	.hardware_disable = svm_hardware_disable,
6848
	.cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
6849
	.cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
A
Avi Kivity 已提交
6850 6851 6852

	.vcpu_create = svm_create_vcpu,
	.vcpu_free = svm_free_vcpu,
6853
	.vcpu_reset = svm_vcpu_reset,
A
Avi Kivity 已提交
6854

6855
	.vm_init = avic_vm_init,
B
Brijesh Singh 已提交
6856
	.vm_destroy = svm_vm_destroy,
6857

6858
	.prepare_guest_switch = svm_prepare_guest_switch,
A
Avi Kivity 已提交
6859 6860
	.vcpu_load = svm_vcpu_load,
	.vcpu_put = svm_vcpu_put,
6861 6862
	.vcpu_blocking = svm_vcpu_blocking,
	.vcpu_unblocking = svm_vcpu_unblocking,
A
Avi Kivity 已提交
6863

6864
	.update_bp_intercept = update_bp_intercept,
6865
	.get_msr_feature = svm_get_msr_feature,
A
Avi Kivity 已提交
6866 6867 6868 6869 6870
	.get_msr = svm_get_msr,
	.set_msr = svm_set_msr,
	.get_segment_base = svm_get_segment_base,
	.get_segment = svm_get_segment,
	.set_segment = svm_set_segment,
6871
	.get_cpl = svm_get_cpl,
6872
	.get_cs_db_l_bits = kvm_get_cs_db_l_bits,
6873
	.decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
6874
	.decache_cr3 = svm_decache_cr3,
6875
	.decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
A
Avi Kivity 已提交
6876 6877 6878 6879 6880 6881 6882 6883
	.set_cr0 = svm_set_cr0,
	.set_cr3 = svm_set_cr3,
	.set_cr4 = svm_set_cr4,
	.set_efer = svm_set_efer,
	.get_idt = svm_get_idt,
	.set_idt = svm_set_idt,
	.get_gdt = svm_get_gdt,
	.set_gdt = svm_set_gdt,
J
Jan Kiszka 已提交
6884 6885
	.get_dr6 = svm_get_dr6,
	.set_dr6 = svm_set_dr6,
6886
	.set_dr7 = svm_set_dr7,
6887
	.sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
A
Avi Kivity 已提交
6888
	.cache_reg = svm_cache_reg,
A
Avi Kivity 已提交
6889 6890
	.get_rflags = svm_get_rflags,
	.set_rflags = svm_set_rflags,
6891

A
Avi Kivity 已提交
6892 6893 6894
	.tlb_flush = svm_flush_tlb,

	.run = svm_vcpu_run,
6895
	.handle_exit = handle_exit,
A
Avi Kivity 已提交
6896
	.skip_emulated_instruction = skip_emulated_instruction,
6897 6898
	.set_interrupt_shadow = svm_set_interrupt_shadow,
	.get_interrupt_shadow = svm_get_interrupt_shadow,
I
Ingo Molnar 已提交
6899
	.patch_hypercall = svm_patch_hypercall,
E
Eddie Dong 已提交
6900
	.set_irq = svm_set_irq,
6901
	.set_nmi = svm_inject_nmi,
6902
	.queue_exception = svm_queue_exception,
A
Avi Kivity 已提交
6903
	.cancel_injection = svm_cancel_injection,
6904
	.interrupt_allowed = svm_interrupt_allowed,
6905
	.nmi_allowed = svm_nmi_allowed,
J
Jan Kiszka 已提交
6906 6907
	.get_nmi_mask = svm_get_nmi_mask,
	.set_nmi_mask = svm_set_nmi_mask,
6908 6909 6910
	.enable_nmi_window = enable_nmi_window,
	.enable_irq_window = enable_irq_window,
	.update_cr8_intercept = update_cr8_intercept,
6911
	.set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
6912 6913
	.get_enable_apicv = svm_get_enable_apicv,
	.refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
6914
	.load_eoi_exitmap = svm_load_eoi_exitmap,
6915 6916
	.hwapic_irr_update = svm_hwapic_irr_update,
	.hwapic_isr_update = svm_hwapic_isr_update,
6917
	.sync_pir_to_irr = kvm_lapic_find_highest_irr,
6918
	.apicv_post_state_restore = avic_post_state_restore,
6919 6920

	.set_tss_addr = svm_set_tss_addr,
6921
	.get_tdp_level = get_npt_level,
6922
	.get_mt_mask = svm_get_mt_mask,
6923

6924 6925
	.get_exit_info = svm_get_exit_info,

6926
	.get_lpage_level = svm_get_lpage_level,
6927 6928

	.cpuid_update = svm_cpuid_update,
6929 6930

	.rdtscp_supported = svm_rdtscp_supported,
6931
	.invpcid_supported = svm_invpcid_supported,
6932
	.mpx_supported = svm_mpx_supported,
6933
	.xsaves_supported = svm_xsaves_supported,
6934
	.umip_emulated = svm_umip_emulated,
6935 6936

	.set_supported_cpuid = svm_set_supported_cpuid,
6937 6938

	.has_wbinvd_exit = svm_has_wbinvd_exit,
6939 6940

	.write_tsc_offset = svm_write_tsc_offset,
6941 6942

	.set_tdp_cr3 = set_tdp_cr3,
6943 6944

	.check_intercept = svm_check_intercept,
6945
	.handle_external_intr = svm_handle_external_intr,
6946 6947

	.sched_in = svm_sched_in,
6948 6949

	.pmu_ops = &amd_pmu_ops,
6950
	.deliver_posted_interrupt = svm_deliver_avic_intr,
6951
	.update_pi_irte = svm_update_pi_irte,
6952
	.setup_mce = svm_setup_mce,
6953

6954
	.smi_allowed = svm_smi_allowed,
6955 6956
	.pre_enter_smm = svm_pre_enter_smm,
	.pre_leave_smm = svm_pre_leave_smm,
6957
	.enable_smi_window = enable_smi_window,
B
Brijesh Singh 已提交
6958 6959

	.mem_enc_op = svm_mem_enc_op,
6960 6961
	.mem_enc_reg_region = svm_register_enc_region,
	.mem_enc_unreg_region = svm_unregister_enc_region,
A
Avi Kivity 已提交
6962 6963 6964 6965
};

static int __init svm_init(void)
{
6966
	return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
6967
			__alignof__(struct vcpu_svm), THIS_MODULE);
A
Avi Kivity 已提交
6968 6969 6970 6971
}

static void __exit svm_exit(void)
{
6972
	kvm_exit();
A
Avi Kivity 已提交
6973 6974 6975 6976
}

module_init(svm_init)
module_exit(svm_exit)