mmu.c 158.5 KB
Newer Older
1 2 3 4 5 6 7 8 9
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11 12 13 14 15 16 17 18 19
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
A
Avi Kivity 已提交
20

21
#include "irq.h"
22
#include "mmu.h"
23
#include "x86.h"
A
Avi Kivity 已提交
24
#include "kvm_cache_regs.h"
25
#include "cpuid.h"
A
Avi Kivity 已提交
26

27
#include <linux/kvm_host.h>
28 29 30 31
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
32 33
#include <linux/moduleparam.h>
#include <linux/export.h>
34
#include <linux/swap.h>
35
#include <linux/hugetlb.h>
36
#include <linux/compiler.h>
37
#include <linux/srcu.h>
38
#include <linux/slab.h>
39
#include <linux/sched/signal.h>
40
#include <linux/uaccess.h>
41
#include <linux/hash.h>
42
#include <linux/kern_levels.h>
43

A
Avi Kivity 已提交
44
#include <asm/page.h>
45
#include <asm/pat.h>
A
Avi Kivity 已提交
46
#include <asm/cmpxchg.h>
47
#include <asm/e820/api.h>
48
#include <asm/io.h>
49
#include <asm/vmx.h>
50
#include <asm/kvm_page_track.h>
51
#include "trace.h"
52

53 54 55 56 57 58 59
/*
 * When setting this variable to true it enables Two-Dimensional-Paging
 * where the hardware walks 2 page tables:
 * 1. the guest-virtual to guest-physical
 * 2. while doing 1. it walks guest-physical to host-physical
 * If the hardware supports that we don't need to do shadow paging.
 */
60
bool tdp_enabled = false;
61

62 63 64 65
enum {
	AUDIT_PRE_PAGE_FAULT,
	AUDIT_POST_PAGE_FAULT,
	AUDIT_PRE_PTE_WRITE,
66 67 68
	AUDIT_POST_PTE_WRITE,
	AUDIT_PRE_SYNC,
	AUDIT_POST_SYNC
69
};
70

71
#undef MMU_DEBUG
72 73

#ifdef MMU_DEBUG
74 75
static bool dbg = 0;
module_param(dbg, bool, 0644);
76 77 78

#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
79
#define MMU_WARN_ON(x) WARN_ON(x)
80 81 82
#else
#define pgprintk(x...) do { } while (0)
#define rmap_printk(x...) do { } while (0)
83
#define MMU_WARN_ON(x) do { } while (0)
84
#endif
85

86 87
#define PTE_PREFETCH_NUM		8

88
#define PT_FIRST_AVAIL_BITS_SHIFT 10
89 90 91 92 93
#define PT64_SECOND_AVAIL_BITS_SHIFT 52

#define PT64_LEVEL_BITS 9

#define PT64_LEVEL_SHIFT(level) \
M
Mike Day 已提交
94
		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
95 96 97 98 99 100 101 102

#define PT64_INDEX(address, level)\
	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))


#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
M
Mike Day 已提交
103
		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
104

105 106 107
#define PT32_LVL_OFFSET_MASK(level) \
	(PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT32_LEVEL_BITS))) - 1))
108 109 110 111 112

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


113 114 115 116 117
#ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
#define PT64_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1))
#else
#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
#endif
118 119 120 121 122 123
#define PT64_LVL_ADDR_MASK(level) \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
#define PT64_LVL_OFFSET_MASK(level) \
	(PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
124 125 126 127

#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
128 129 130
#define PT32_LVL_ADDR_MASK(level) \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
					    * PT32_LEVEL_BITS))) - 1))
131

132
#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
133
			| shadow_x_mask | shadow_nx_mask | shadow_me_mask)
134

135 136 137 138 139
#define ACC_EXEC_MASK    1
#define ACC_WRITE_MASK   PT_WRITABLE_MASK
#define ACC_USER_MASK    PT_USER_MASK
#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)

140 141 142 143
/* The mask for the R/X bits in EPT PTEs */
#define PT64_EPT_READABLE_MASK			0x1ull
#define PT64_EPT_EXECUTABLE_MASK		0x4ull

144 145
#include <trace/events/kvm.h>

146 147 148
#define CREATE_TRACE_POINTS
#include "mmutrace.h"

149 150
#define SPTE_HOST_WRITEABLE	(1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
#define SPTE_MMU_WRITEABLE	(1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
151

152 153
#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)

154 155 156
/* make pte_list_desc fit well in cache line */
#define PTE_LIST_EXT 3

157 158 159 160 161 162 163 164 165 166 167 168 169 170
/*
 * Return values of handle_mmio_page_fault and mmu.page_fault:
 * RET_PF_RETRY: let CPU fault again on the address.
 * RET_PF_EMULATE: mmio page fault, emulate the instruction directly.
 *
 * For handle_mmio_page_fault only:
 * RET_PF_INVALID: the spte is invalid, let the real page fault path update it.
 */
enum {
	RET_PF_RETRY = 0,
	RET_PF_EMULATE = 1,
	RET_PF_INVALID = 2,
};

171 172 173
struct pte_list_desc {
	u64 *sptes[PTE_LIST_EXT];
	struct pte_list_desc *more;
174 175
};

176 177 178 179
struct kvm_shadow_walk_iterator {
	u64 addr;
	hpa_t shadow_addr;
	u64 *sptep;
180
	int level;
181 182 183
	unsigned index;
};

184 185
static const union kvm_mmu_page_role mmu_base_role_mask = {
	.cr0_wp = 1,
186
	.gpte_is_8_bytes = 1,
187 188 189 190 191 192 193 194
	.nxe = 1,
	.smep_andnot_wp = 1,
	.smap_andnot_wp = 1,
	.smm = 1,
	.guest_mode = 1,
	.ad_disabled = 1,
};

195 196 197 198 199 200 201
#define for_each_shadow_entry_using_root(_vcpu, _root, _addr, _walker)     \
	for (shadow_walk_init_using_root(&(_walker), (_vcpu),              \
					 (_root), (_addr));                \
	     shadow_walk_okay(&(_walker));			           \
	     shadow_walk_next(&(_walker)))

#define for_each_shadow_entry(_vcpu, _addr, _walker)            \
202 203 204 205
	for (shadow_walk_init(&(_walker), _vcpu, _addr);	\
	     shadow_walk_okay(&(_walker));			\
	     shadow_walk_next(&(_walker)))

206 207 208 209 210 211
#define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)	\
	for (shadow_walk_init(&(_walker), _vcpu, _addr);		\
	     shadow_walk_okay(&(_walker)) &&				\
		({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });	\
	     __shadow_walk_next(&(_walker), spte))

212
static struct kmem_cache *pte_list_desc_cache;
213
static struct kmem_cache *mmu_page_header_cache;
214
static struct percpu_counter kvm_total_used_mmu_pages;
215

S
Sheng Yang 已提交
216 217 218 219 220
static u64 __read_mostly shadow_nx_mask;
static u64 __read_mostly shadow_x_mask;	/* mutual exclusive with nx_mask */
static u64 __read_mostly shadow_user_mask;
static u64 __read_mostly shadow_accessed_mask;
static u64 __read_mostly shadow_dirty_mask;
221
static u64 __read_mostly shadow_mmio_mask;
222
static u64 __read_mostly shadow_mmio_value;
223
static u64 __read_mostly shadow_present_mask;
224
static u64 __read_mostly shadow_me_mask;
225

226
/*
227 228 229
 * SPTEs used by MMUs without A/D bits are marked with shadow_acc_track_value.
 * Non-present SPTEs with shadow_acc_track_value set are in place for access
 * tracking.
230 231 232 233 234 235 236 237 238 239 240 241 242 243
 */
static u64 __read_mostly shadow_acc_track_mask;
static const u64 shadow_acc_track_value = SPTE_SPECIAL_MASK;

/*
 * The mask/shift to use for saving the original R/X bits when marking the PTE
 * as not-present for access tracking purposes. We do not save the W bit as the
 * PTEs being access tracked also need to be dirty tracked, so the W bit will be
 * restored only when a write is attempted to the page.
 */
static const u64 shadow_acc_track_saved_bits_mask = PT64_EPT_READABLE_MASK |
						    PT64_EPT_EXECUTABLE_MASK;
static const u64 shadow_acc_track_saved_bits_shift = PT64_SECOND_AVAIL_BITS_SHIFT;

244 245 246 247 248 249 250 251 252 253 254
/*
 * This mask must be set on all non-zero Non-Present or Reserved SPTEs in order
 * to guard against L1TF attacks.
 */
static u64 __read_mostly shadow_nonpresent_or_rsvd_mask;

/*
 * The number of high-order 1 bits to use in the mask above.
 */
static const u64 shadow_nonpresent_or_rsvd_mask_len = 5;

255 256 257 258 259 260 261 262 263 264 265
/*
 * In some cases, we need to preserve the GFN of a non-present or reserved
 * SPTE when we usurp the upper five bits of the physical address space to
 * defend against L1TF, e.g. for MMIO SPTEs.  To preserve the GFN, we'll
 * shift bits of the GFN that overlap with shadow_nonpresent_or_rsvd_mask
 * left into the reserved bits, i.e. the GFN in the SPTE will be split into
 * high and low parts.  This mask covers the lower bits of the GFN.
 */
static u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask;


266
static void mmu_spte_set(u64 *sptep, u64 spte);
267 268
static union kvm_mmu_page_role
kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu);
269

270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298

static inline bool kvm_available_flush_tlb_with_range(void)
{
	return kvm_x86_ops->tlb_remote_flush_with_range;
}

static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,
		struct kvm_tlb_range *range)
{
	int ret = -ENOTSUPP;

	if (range && kvm_x86_ops->tlb_remote_flush_with_range)
		ret = kvm_x86_ops->tlb_remote_flush_with_range(kvm, range);

	if (ret)
		kvm_flush_remote_tlbs(kvm);
}

static void kvm_flush_remote_tlbs_with_address(struct kvm *kvm,
		u64 start_gfn, u64 pages)
{
	struct kvm_tlb_range range;

	range.start_gfn = start_gfn;
	range.pages = pages;

	kvm_flush_remote_tlbs_with_range(kvm, &range);
}

299
void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value)
300
{
301 302
	BUG_ON((mmio_mask & mmio_value) != mmio_value);
	shadow_mmio_value = mmio_value | SPTE_SPECIAL_MASK;
303
	shadow_mmio_mask = mmio_mask | SPTE_SPECIAL_MASK;
304 305 306
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);

307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
{
	return sp->role.ad_disabled;
}

static inline bool spte_ad_enabled(u64 spte)
{
	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	return !(spte & shadow_acc_track_value);
}

static inline u64 spte_shadow_accessed_mask(u64 spte)
{
	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
}

static inline u64 spte_shadow_dirty_mask(u64 spte)
{
	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
}

330 331
static inline bool is_access_track_spte(u64 spte)
{
332
	return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
333 334
}

335
/*
336 337
 * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of
 * the memslots generation and is derived as follows:
338
 *
339 340
 * Bits 0-8 of the MMIO generation are propagated to spte bits 3-11
 * Bits 9-18 of the MMIO generation are propagated to spte bits 52-61
341
 *
342 343 344 345 346 347
 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in
 * the MMIO generation number, as doing so would require stealing a bit from
 * the "real" generation number and thus effectively halve the maximum number
 * of MMIO generations that can be handled before encountering a wrap (which
 * requires a full MMU zap).  The flag is instead explicitly queried when
 * checking for MMIO spte cache hits.
348
 */
349
#define MMIO_SPTE_GEN_MASK		GENMASK_ULL(18, 0)
350

351 352 353 354
#define MMIO_SPTE_GEN_LOW_START		3
#define MMIO_SPTE_GEN_LOW_END		11
#define MMIO_SPTE_GEN_LOW_MASK		GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \
						    MMIO_SPTE_GEN_LOW_START)
355

356 357 358 359
#define MMIO_SPTE_GEN_HIGH_START	52
#define MMIO_SPTE_GEN_HIGH_END		61
#define MMIO_SPTE_GEN_HIGH_MASK		GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \
						    MMIO_SPTE_GEN_HIGH_START)
360
static u64 generation_mmio_spte_mask(u64 gen)
361 362 363
{
	u64 mask;

364
	WARN_ON(gen & ~MMIO_SPTE_GEN_MASK);
365

366 367
	mask = (gen << MMIO_SPTE_GEN_LOW_START) & MMIO_SPTE_GEN_LOW_MASK;
	mask |= (gen << MMIO_SPTE_GEN_HIGH_START) & MMIO_SPTE_GEN_HIGH_MASK;
368 369 370
	return mask;
}

371
static u64 get_mmio_spte_generation(u64 spte)
372
{
373
	u64 gen;
374 375 376

	spte &= ~shadow_mmio_mask;

377 378
	gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_START;
	gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_START;
379 380 381
	return gen;
}

382
static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
383
			   unsigned access)
384
{
385
	u64 gen = kvm_vcpu_memslots(vcpu)->generation & MMIO_SPTE_GEN_MASK;
386
	u64 mask = generation_mmio_spte_mask(gen);
387
	u64 gpa = gfn << PAGE_SHIFT;
388

389
	access &= ACC_WRITE_MASK | ACC_USER_MASK;
390 391 392 393
	mask |= shadow_mmio_value | access;
	mask |= gpa | shadow_nonpresent_or_rsvd_mask;
	mask |= (gpa & shadow_nonpresent_or_rsvd_mask)
		<< shadow_nonpresent_or_rsvd_mask_len;
394

395 396
	page_header(__pa(sptep))->mmio_cached = true;

397
	trace_mark_mmio_spte(sptep, gfn, access, gen);
398
	mmu_spte_set(sptep, mask);
399 400 401 402
}

static bool is_mmio_spte(u64 spte)
{
403
	return (spte & shadow_mmio_mask) == shadow_mmio_value;
404 405 406 407
}

static gfn_t get_mmio_spte_gfn(u64 spte)
{
408
	u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask;
409 410 411 412 413

	gpa |= (spte >> shadow_nonpresent_or_rsvd_mask_len)
	       & shadow_nonpresent_or_rsvd_mask;

	return gpa >> PAGE_SHIFT;
414 415 416 417
}

static unsigned get_mmio_spte_access(u64 spte)
{
418
	u64 mask = generation_mmio_spte_mask(MMIO_SPTE_GEN_MASK) | shadow_mmio_mask;
419
	return (spte & ~mask) & ~PAGE_MASK;
420 421
}

422
static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
423
			  kvm_pfn_t pfn, unsigned access)
424 425
{
	if (unlikely(is_noslot_pfn(pfn))) {
426
		mark_mmio_spte(vcpu, sptep, gfn, access);
427 428 429 430 431
		return true;
	}

	return false;
}
432

433
static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
434
{
435
	u64 kvm_gen, spte_gen, gen;
436

437 438 439
	gen = kvm_vcpu_memslots(vcpu)->generation;
	if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS))
		return false;
440

441
	kvm_gen = gen & MMIO_SPTE_GEN_MASK;
442 443 444 445
	spte_gen = get_mmio_spte_generation(spte);

	trace_check_mmio_spte(spte, kvm_gen, spte_gen);
	return likely(kvm_gen == spte_gen);
446 447
}

448 449 450 451 452 453 454
/*
 * Sets the shadow PTE masks used by the MMU.
 *
 * Assumptions:
 *  - Setting either @accessed_mask or @dirty_mask requires setting both
 *  - At least one of @accessed_mask or @acc_track_mask must be set
 */
S
Sheng Yang 已提交
455
void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
456
		u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask,
457
		u64 acc_track_mask, u64 me_mask)
S
Sheng Yang 已提交
458
{
459 460
	BUG_ON(!dirty_mask != !accessed_mask);
	BUG_ON(!accessed_mask && !acc_track_mask);
461
	BUG_ON(acc_track_mask & shadow_acc_track_value);
462

S
Sheng Yang 已提交
463 464 465 466 467
	shadow_user_mask = user_mask;
	shadow_accessed_mask = accessed_mask;
	shadow_dirty_mask = dirty_mask;
	shadow_nx_mask = nx_mask;
	shadow_x_mask = x_mask;
468
	shadow_present_mask = p_mask;
469
	shadow_acc_track_mask = acc_track_mask;
470
	shadow_me_mask = me_mask;
S
Sheng Yang 已提交
471 472 473
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);

474
static void kvm_mmu_reset_all_pte_masks(void)
475
{
476 477
	u8 low_phys_bits;

478 479 480 481 482 483 484 485
	shadow_user_mask = 0;
	shadow_accessed_mask = 0;
	shadow_dirty_mask = 0;
	shadow_nx_mask = 0;
	shadow_x_mask = 0;
	shadow_mmio_mask = 0;
	shadow_present_mask = 0;
	shadow_acc_track_mask = 0;
486 487 488 489 490

	/*
	 * If the CPU has 46 or less physical address bits, then set an
	 * appropriate mask to guard against L1TF attacks. Otherwise, it is
	 * assumed that the CPU is not vulnerable to L1TF.
491 492 493 494 495
	 *
	 * Some Intel CPUs address the L1 cache using more PA bits than are
	 * reported by CPUID. Use the PA width of the L1 cache when possible
	 * to achieve more effective mitigation, e.g. if system RAM overlaps
	 * the most significant bits of legal physical address space.
496
	 */
497 498 499
	shadow_nonpresent_or_rsvd_mask = 0;
	low_phys_bits = boot_cpu_data.x86_cache_bits;
	if (boot_cpu_data.x86_cache_bits <
500
	    52 - shadow_nonpresent_or_rsvd_mask_len) {
501
		shadow_nonpresent_or_rsvd_mask =
502
			rsvd_bits(boot_cpu_data.x86_cache_bits -
503
				  shadow_nonpresent_or_rsvd_mask_len,
504
				  boot_cpu_data.x86_cache_bits - 1);
505
		low_phys_bits -= shadow_nonpresent_or_rsvd_mask_len;
506 507 508
	} else
		WARN_ON_ONCE(boot_cpu_has_bug(X86_BUG_L1TF));

509 510
	shadow_nonpresent_or_rsvd_lower_gfn_mask =
		GENMASK_ULL(low_phys_bits - 1, PAGE_SHIFT);
511 512
}

513 514 515 516 517
static int is_cpuid_PSE36(void)
{
	return 1;
}

518 519
static int is_nx(struct kvm_vcpu *vcpu)
{
520
	return vcpu->arch.efer & EFER_NX;
521 522
}

523 524
static int is_shadow_present_pte(u64 pte)
{
525
	return (pte != 0) && !is_mmio_spte(pte);
526 527
}

528 529 530 531 532
static int is_large_pte(u64 pte)
{
	return pte & PT_PAGE_SIZE_MASK;
}

533 534 535 536
static int is_last_spte(u64 pte, int level)
{
	if (level == PT_PAGE_TABLE_LEVEL)
		return 1;
537
	if (is_large_pte(pte))
538 539 540 541
		return 1;
	return 0;
}

542 543 544 545 546
static bool is_executable_pte(u64 spte)
{
	return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
}

547
static kvm_pfn_t spte_to_pfn(u64 pte)
548
{
549
	return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
550 551
}

552 553 554 555 556 557 558
static gfn_t pse36_gfn_delta(u32 gpte)
{
	int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;

	return (gpte & PT32_DIR_PSE36_MASK) << shift;
}

559
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
560
static void __set_spte(u64 *sptep, u64 spte)
561
{
562
	WRITE_ONCE(*sptep, spte);
563 564
}

565
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
566
{
567
	WRITE_ONCE(*sptep, spte);
568 569 570 571 572 573
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	return xchg(sptep, spte);
}
574 575 576

static u64 __get_spte_lockless(u64 *sptep)
{
577
	return READ_ONCE(*sptep);
578
}
579
#else
580 581 582 583 584 585 586
union split_spte {
	struct {
		u32 spte_low;
		u32 spte_high;
	};
	u64 spte;
};
587

588 589 590 591 592 593 594 595 596 597 598 599
static void count_spte_clear(u64 *sptep, u64 spte)
{
	struct kvm_mmu_page *sp =  page_header(__pa(sptep));

	if (is_shadow_present_pte(spte))
		return;

	/* Ensure the spte is completely set before we increase the count */
	smp_wmb();
	sp->clear_spte_count++;
}

600 601 602
static void __set_spte(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;
603

604 605 606 607 608 609 610 611 612 613 614 615
	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	ssptep->spte_high = sspte.spte_high;

	/*
	 * If we map the spte from nonpresent to present, We should store
	 * the high bits firstly, then set present bit, so cpu can not
	 * fetch this spte while we are setting the spte.
	 */
	smp_wmb();

616
	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
617 618
}

619 620 621 622 623 624 625
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

626
	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
627 628 629 630 631 632 633 634

	/*
	 * If we map the spte from present to nonpresent, we should clear
	 * present bit firstly to avoid vcpu fetch the old high bits.
	 */
	smp_wmb();

	ssptep->spte_high = sspte.spte_high;
635
	count_spte_clear(sptep, spte);
636 637 638 639 640 641 642 643 644 645 646
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte, orig;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	/* xchg acts as a barrier before the setting of the high bits */
	orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
647 648
	orig.spte_high = ssptep->spte_high;
	ssptep->spte_high = sspte.spte_high;
649
	count_spte_clear(sptep, spte);
650 651 652

	return orig.spte;
}
653 654 655 656

/*
 * The idea using the light way get the spte on x86_32 guest is from
 * gup_get_pte(arch/x86/mm/gup.c).
657 658 659 660 661 662 663 664 665 666 667 668 669 670
 *
 * An spte tlb flush may be pending, because kvm_set_pte_rmapp
 * coalesces them and we are running out of the MMU lock.  Therefore
 * we need to protect against in-progress updates of the spte.
 *
 * Reading the spte while an update is in progress may get the old value
 * for the high part of the spte.  The race is fine for a present->non-present
 * change (because the high part of the spte is ignored for non-present spte),
 * but for a present->present change we must reread the spte.
 *
 * All such changes are done in two steps (present->non-present and
 * non-present->present), hence it is enough to count the number of
 * present->non-present updates: if it changed while reading the spte,
 * we might have hit the race.  This is done using clear_spte_count.
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
 */
static u64 __get_spte_lockless(u64 *sptep)
{
	struct kvm_mmu_page *sp =  page_header(__pa(sptep));
	union split_spte spte, *orig = (union split_spte *)sptep;
	int count;

retry:
	count = sp->clear_spte_count;
	smp_rmb();

	spte.spte_low = orig->spte_low;
	smp_rmb();

	spte.spte_high = orig->spte_high;
	smp_rmb();

	if (unlikely(spte.spte_low != orig->spte_low ||
	      count != sp->clear_spte_count))
		goto retry;

	return spte.spte;
}
694 695
#endif

696
static bool spte_can_locklessly_be_made_writable(u64 spte)
697
{
698 699
	return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
		(SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
700 701
}

702 703
static bool spte_has_volatile_bits(u64 spte)
{
704 705 706
	if (!is_shadow_present_pte(spte))
		return false;

707
	/*
708
	 * Always atomically update spte if it can be updated
709 710 711 712
	 * out of mmu-lock, it can ensure dirty bit is not lost,
	 * also, it can help us to get a stable is_writable_pte()
	 * to ensure tlb flush is not missed.
	 */
713 714
	if (spte_can_locklessly_be_made_writable(spte) ||
	    is_access_track_spte(spte))
715 716
		return true;

717
	if (spte_ad_enabled(spte)) {
718 719 720 721
		if ((spte & shadow_accessed_mask) == 0 ||
	    	    (is_writable_pte(spte) && (spte & shadow_dirty_mask) == 0))
			return true;
	}
722

723
	return false;
724 725
}

726
static bool is_accessed_spte(u64 spte)
727
{
728 729 730 731
	u64 accessed_mask = spte_shadow_accessed_mask(spte);

	return accessed_mask ? spte & accessed_mask
			     : !is_access_track_spte(spte);
732 733
}

734
static bool is_dirty_spte(u64 spte)
735
{
736 737 738
	u64 dirty_mask = spte_shadow_dirty_mask(spte);

	return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK;
739 740
}

741 742 743 744 745 746 747 748 749 750 751 752
/* Rules for using mmu_spte_set:
 * Set the sptep from nonpresent to present.
 * Note: the sptep being assigned *must* be either not present
 * or in a state where the hardware will not attempt to update
 * the spte.
 */
static void mmu_spte_set(u64 *sptep, u64 new_spte)
{
	WARN_ON(is_shadow_present_pte(*sptep));
	__set_spte(sptep, new_spte);
}

753 754 755
/*
 * Update the SPTE (excluding the PFN), but do not track changes in its
 * accessed/dirty status.
756
 */
757
static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte)
758
{
759
	u64 old_spte = *sptep;
760

761
	WARN_ON(!is_shadow_present_pte(new_spte));
762

763 764
	if (!is_shadow_present_pte(old_spte)) {
		mmu_spte_set(sptep, new_spte);
765
		return old_spte;
766
	}
767

768
	if (!spte_has_volatile_bits(old_spte))
769
		__update_clear_spte_fast(sptep, new_spte);
770
	else
771
		old_spte = __update_clear_spte_slow(sptep, new_spte);
772

773 774
	WARN_ON(spte_to_pfn(old_spte) != spte_to_pfn(new_spte));

775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796
	return old_spte;
}

/* Rules for using mmu_spte_update:
 * Update the state bits, it means the mapped pfn is not changed.
 *
 * Whenever we overwrite a writable spte with a read-only one we
 * should flush remote TLBs. Otherwise rmap_write_protect
 * will find a read-only spte, even though the writable spte
 * might be cached on a CPU's TLB, the return value indicates this
 * case.
 *
 * Returns true if the TLB needs to be flushed
 */
static bool mmu_spte_update(u64 *sptep, u64 new_spte)
{
	bool flush = false;
	u64 old_spte = mmu_spte_update_no_track(sptep, new_spte);

	if (!is_shadow_present_pte(old_spte))
		return false;

797 798
	/*
	 * For the spte updated out of mmu-lock is safe, since
799
	 * we always atomically update it, see the comments in
800 801
	 * spte_has_volatile_bits().
	 */
802
	if (spte_can_locklessly_be_made_writable(old_spte) &&
803
	      !is_writable_pte(new_spte))
804
		flush = true;
805

806
	/*
807
	 * Flush TLB when accessed/dirty states are changed in the page tables,
808 809 810
	 * to guarantee consistency between TLB and page tables.
	 */

811 812
	if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) {
		flush = true;
813
		kvm_set_pfn_accessed(spte_to_pfn(old_spte));
814 815 816 817
	}

	if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) {
		flush = true;
818
		kvm_set_pfn_dirty(spte_to_pfn(old_spte));
819
	}
820

821
	return flush;
822 823
}

824 825 826 827
/*
 * Rules for using mmu_spte_clear_track_bits:
 * It sets the sptep from present to nonpresent, and track the
 * state bits, it is used to clear the last level sptep.
828
 * Returns non-zero if the PTE was previously valid.
829 830 831
 */
static int mmu_spte_clear_track_bits(u64 *sptep)
{
832
	kvm_pfn_t pfn;
833 834 835
	u64 old_spte = *sptep;

	if (!spte_has_volatile_bits(old_spte))
836
		__update_clear_spte_fast(sptep, 0ull);
837
	else
838
		old_spte = __update_clear_spte_slow(sptep, 0ull);
839

840
	if (!is_shadow_present_pte(old_spte))
841 842 843
		return 0;

	pfn = spte_to_pfn(old_spte);
844 845 846 847 848 849

	/*
	 * KVM does not hold the refcount of the page used by
	 * kvm mmu, before reclaiming the page, we should
	 * unmap it from mmu first.
	 */
850
	WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
851

852
	if (is_accessed_spte(old_spte))
853
		kvm_set_pfn_accessed(pfn);
854 855

	if (is_dirty_spte(old_spte))
856
		kvm_set_pfn_dirty(pfn);
857

858 859 860 861 862 863 864 865 866 867
	return 1;
}

/*
 * Rules for using mmu_spte_clear_no_track:
 * Directly clear spte without caring the state bits of sptep,
 * it is used to set the upper level spte.
 */
static void mmu_spte_clear_no_track(u64 *sptep)
{
868
	__update_clear_spte_fast(sptep, 0ull);
869 870
}

871 872 873 874 875
static u64 mmu_spte_get_lockless(u64 *sptep)
{
	return __get_spte_lockless(sptep);
}

876 877
static u64 mark_spte_for_access_track(u64 spte)
{
878
	if (spte_ad_enabled(spte))
879 880
		return spte & ~shadow_accessed_mask;

881
	if (is_access_track_spte(spte))
882 883 884
		return spte;

	/*
885 886 887
	 * Making an Access Tracking PTE will result in removal of write access
	 * from the PTE. So, verify that we will be able to restore the write
	 * access in the fast page fault path later on.
888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903
	 */
	WARN_ONCE((spte & PT_WRITABLE_MASK) &&
		  !spte_can_locklessly_be_made_writable(spte),
		  "kvm: Writable SPTE is not locklessly dirty-trackable\n");

	WARN_ONCE(spte & (shadow_acc_track_saved_bits_mask <<
			  shadow_acc_track_saved_bits_shift),
		  "kvm: Access Tracking saved bit locations are not zero\n");

	spte |= (spte & shadow_acc_track_saved_bits_mask) <<
		shadow_acc_track_saved_bits_shift;
	spte &= ~shadow_acc_track_mask;

	return spte;
}

904 905 906 907 908 909 910
/* Restore an acc-track PTE back to a regular PTE */
static u64 restore_acc_track_spte(u64 spte)
{
	u64 new_spte = spte;
	u64 saved_bits = (spte >> shadow_acc_track_saved_bits_shift)
			 & shadow_acc_track_saved_bits_mask;

911
	WARN_ON_ONCE(spte_ad_enabled(spte));
912 913 914 915 916 917 918 919 920 921
	WARN_ON_ONCE(!is_access_track_spte(spte));

	new_spte &= ~shadow_acc_track_mask;
	new_spte &= ~(shadow_acc_track_saved_bits_mask <<
		      shadow_acc_track_saved_bits_shift);
	new_spte |= saved_bits;

	return new_spte;
}

922 923 924 925 926 927 928 929
/* Returns the Accessed status of the PTE and resets it at the same time. */
static bool mmu_spte_age(u64 *sptep)
{
	u64 spte = mmu_spte_get_lockless(sptep);

	if (!is_accessed_spte(spte))
		return false;

930
	if (spte_ad_enabled(spte)) {
931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947
		clear_bit((ffs(shadow_accessed_mask) - 1),
			  (unsigned long *)sptep);
	} else {
		/*
		 * Capture the dirty status of the page, so that it doesn't get
		 * lost when the SPTE is marked for access tracking.
		 */
		if (is_writable_pte(spte))
			kvm_set_pfn_dirty(spte_to_pfn(spte));

		spte = mark_spte_for_access_track(spte);
		mmu_spte_update_no_track(sptep, spte);
	}

	return true;
}

948 949
static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
{
950 951 952 953 954
	/*
	 * Prevent page table teardown by making any free-er wait during
	 * kvm_flush_remote_tlbs() IPI to all active vcpus.
	 */
	local_irq_disable();
955

956 957 958 959
	/*
	 * Make sure a following spte read is not reordered ahead of the write
	 * to vcpu->mode.
	 */
960
	smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
961 962 963 964
}

static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
{
965 966
	/*
	 * Make sure the write to vcpu->mode is not reordered in front of
967
	 * reads to sptes.  If it does, kvm_mmu_commit_zap_page() can see us
968 969
	 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
	 */
970
	smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
971
	local_irq_enable();
972 973
}

974
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
975
				  struct kmem_cache *base_cache, int min)
976 977 978 979
{
	void *obj;

	if (cache->nobjs >= min)
980
		return 0;
981
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
982
		obj = kmem_cache_zalloc(base_cache, GFP_KERNEL_ACCOUNT);
983
		if (!obj)
984
			return cache->nobjs >= min ? 0 : -ENOMEM;
985 986
		cache->objects[cache->nobjs++] = obj;
	}
987
	return 0;
988 989
}

990 991 992 993 994
static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
{
	return cache->nobjs;
}

995 996
static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
				  struct kmem_cache *cache)
997 998
{
	while (mc->nobjs)
999
		kmem_cache_free(cache, mc->objects[--mc->nobjs]);
1000 1001
}

1002
static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
1003
				       int min)
1004
{
1005
	void *page;
1006 1007 1008 1009

	if (cache->nobjs >= min)
		return 0;
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
1010
		page = (void *)__get_free_page(GFP_KERNEL_ACCOUNT);
1011
		if (!page)
1012
			return cache->nobjs >= min ? 0 : -ENOMEM;
1013
		cache->objects[cache->nobjs++] = page;
1014 1015 1016 1017 1018 1019 1020
	}
	return 0;
}

static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
1021
		free_page((unsigned long)mc->objects[--mc->nobjs]);
1022 1023
}

1024
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
1025
{
1026 1027
	int r;

1028
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
1029
				   pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
1030 1031
	if (r)
		goto out;
1032
	r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
1033 1034
	if (r)
		goto out;
1035
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
1036
				   mmu_page_header_cache, 4);
1037 1038
out:
	return r;
1039 1040 1041 1042
}

static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
1043 1044
	mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
				pte_list_desc_cache);
1045
	mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
1046 1047
	mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
				mmu_page_header_cache);
1048 1049
}

1050
static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
1051 1052 1053 1054 1055 1056 1057 1058
{
	void *p;

	BUG_ON(!mc->nobjs);
	p = mc->objects[--mc->nobjs];
	return p;
}

1059
static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
1060
{
1061
	return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
1062 1063
}

1064
static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
1065
{
1066
	kmem_cache_free(pte_list_desc_cache, pte_list_desc);
1067 1068
}

1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
{
	if (!sp->role.direct)
		return sp->gfns[index];

	return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
}

static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
{
	if (sp->role.direct)
		BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
	else
		sp->gfns[index] = gfn;
}

1085
/*
1086 1087
 * Return the pointer to the large page information for a given gfn,
 * handling slots that are not large page aligned.
1088
 */
1089 1090 1091
static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
					      struct kvm_memory_slot *slot,
					      int level)
1092 1093 1094
{
	unsigned long idx;

1095
	idx = gfn_to_index(gfn, slot->base_gfn, level);
1096
	return &slot->arch.lpage_info[level - 2][idx];
1097 1098
}

1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
static void update_gfn_disallow_lpage_count(struct kvm_memory_slot *slot,
					    gfn_t gfn, int count)
{
	struct kvm_lpage_info *linfo;
	int i;

	for (i = PT_DIRECTORY_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->disallow_lpage += count;
		WARN_ON(linfo->disallow_lpage < 0);
	}
}

void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn)
{
	update_gfn_disallow_lpage_count(slot, gfn, 1);
}

void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn)
{
	update_gfn_disallow_lpage_count(slot, gfn, -1);
}

1122
static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
1123
{
1124
	struct kvm_memslots *slots;
1125
	struct kvm_memory_slot *slot;
1126
	gfn_t gfn;
1127

1128
	kvm->arch.indirect_shadow_pages++;
1129
	gfn = sp->gfn;
1130 1131
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
1132 1133 1134 1135 1136 1137

	/* the non-leaf shadow pages are keeping readonly. */
	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return kvm_slot_page_track_add_page(kvm, slot, gfn,
						    KVM_PAGE_TRACK_WRITE);

1138
	kvm_mmu_gfn_disallow_lpage(slot, gfn);
1139 1140
}

1141
static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
1142
{
1143
	struct kvm_memslots *slots;
1144
	struct kvm_memory_slot *slot;
1145
	gfn_t gfn;
1146

1147
	kvm->arch.indirect_shadow_pages--;
1148
	gfn = sp->gfn;
1149 1150
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
1151 1152 1153 1154
	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return kvm_slot_page_track_remove_page(kvm, slot, gfn,
						       KVM_PAGE_TRACK_WRITE);

1155
	kvm_mmu_gfn_allow_lpage(slot, gfn);
1156 1157
}

1158 1159
static bool __mmu_gfn_lpage_is_disallowed(gfn_t gfn, int level,
					  struct kvm_memory_slot *slot)
1160
{
1161
	struct kvm_lpage_info *linfo;
1162 1163

	if (slot) {
1164
		linfo = lpage_info_slot(gfn, slot, level);
1165
		return !!linfo->disallow_lpage;
1166 1167
	}

1168
	return true;
1169 1170
}

1171 1172
static bool mmu_gfn_lpage_is_disallowed(struct kvm_vcpu *vcpu, gfn_t gfn,
					int level)
1173 1174 1175 1176
{
	struct kvm_memory_slot *slot;

	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1177
	return __mmu_gfn_lpage_is_disallowed(gfn, level, slot);
1178 1179
}

1180
static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
1181
{
1182
	unsigned long page_size;
1183
	int i, ret = 0;
1184

1185
	page_size = kvm_host_page_size(kvm, gfn);
1186

1187
	for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
1188 1189 1190 1191 1192 1193
		if (page_size >= KVM_HPAGE_SIZE(i))
			ret = i;
		else
			break;
	}

1194
	return ret;
1195 1196
}

1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207
static inline bool memslot_valid_for_gpte(struct kvm_memory_slot *slot,
					  bool no_dirty_log)
{
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
		return false;
	if (no_dirty_log && slot->dirty_bitmap)
		return false;

	return true;
}

1208 1209 1210
static struct kvm_memory_slot *
gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
			    bool no_dirty_log)
1211 1212
{
	struct kvm_memory_slot *slot;
1213

1214
	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1215
	if (!memslot_valid_for_gpte(slot, no_dirty_log))
1216 1217 1218 1219 1220
		slot = NULL;

	return slot;
}

1221 1222
static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn,
			 bool *force_pt_level)
1223 1224
{
	int host_level, level, max_level;
1225 1226
	struct kvm_memory_slot *slot;

1227 1228
	if (unlikely(*force_pt_level))
		return PT_PAGE_TABLE_LEVEL;
1229

1230 1231
	slot = kvm_vcpu_gfn_to_memslot(vcpu, large_gfn);
	*force_pt_level = !memslot_valid_for_gpte(slot, true);
1232 1233 1234
	if (unlikely(*force_pt_level))
		return PT_PAGE_TABLE_LEVEL;

1235 1236 1237 1238 1239
	host_level = host_mapping_level(vcpu->kvm, large_gfn);

	if (host_level == PT_PAGE_TABLE_LEVEL)
		return host_level;

1240
	max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
1241 1242

	for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
1243
		if (__mmu_gfn_lpage_is_disallowed(large_gfn, level, slot))
1244 1245 1246
			break;

	return level - 1;
1247 1248
}

1249
/*
1250
 * About rmap_head encoding:
1251
 *
1252 1253
 * If the bit zero of rmap_head->val is clear, then it points to the only spte
 * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
1254
 * pte_list_desc containing more mappings.
1255 1256 1257 1258
 */

/*
 * Returns the number of pointers in the rmap chain, not counting the new one.
1259
 */
1260
static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
1261
			struct kvm_rmap_head *rmap_head)
1262
{
1263
	struct pte_list_desc *desc;
1264
	int i, count = 0;
1265

1266
	if (!rmap_head->val) {
1267
		rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
1268 1269
		rmap_head->val = (unsigned long)spte;
	} else if (!(rmap_head->val & 1)) {
1270 1271
		rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
		desc = mmu_alloc_pte_list_desc(vcpu);
1272
		desc->sptes[0] = (u64 *)rmap_head->val;
A
Avi Kivity 已提交
1273
		desc->sptes[1] = spte;
1274
		rmap_head->val = (unsigned long)desc | 1;
1275
		++count;
1276
	} else {
1277
		rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
1278
		desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1279
		while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
1280
			desc = desc->more;
1281
			count += PTE_LIST_EXT;
1282
		}
1283 1284
		if (desc->sptes[PTE_LIST_EXT-1]) {
			desc->more = mmu_alloc_pte_list_desc(vcpu);
1285 1286
			desc = desc->more;
		}
A
Avi Kivity 已提交
1287
		for (i = 0; desc->sptes[i]; ++i)
1288
			++count;
A
Avi Kivity 已提交
1289
		desc->sptes[i] = spte;
1290
	}
1291
	return count;
1292 1293
}

1294
static void
1295 1296 1297
pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
			   struct pte_list_desc *desc, int i,
			   struct pte_list_desc *prev_desc)
1298 1299 1300
{
	int j;

1301
	for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
1302
		;
A
Avi Kivity 已提交
1303 1304
	desc->sptes[i] = desc->sptes[j];
	desc->sptes[j] = NULL;
1305 1306 1307
	if (j != 0)
		return;
	if (!prev_desc && !desc->more)
1308
		rmap_head->val = (unsigned long)desc->sptes[0];
1309 1310 1311 1312
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
1313
			rmap_head->val = (unsigned long)desc->more | 1;
1314
	mmu_free_pte_list_desc(desc);
1315 1316
}

1317
static void __pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
1318
{
1319 1320
	struct pte_list_desc *desc;
	struct pte_list_desc *prev_desc;
1321 1322
	int i;

1323
	if (!rmap_head->val) {
1324
		pr_err("%s: %p 0->BUG\n", __func__, spte);
1325
		BUG();
1326
	} else if (!(rmap_head->val & 1)) {
1327
		rmap_printk("%s:  %p 1->0\n", __func__, spte);
1328
		if ((u64 *)rmap_head->val != spte) {
1329
			pr_err("%s:  %p 1->BUG\n", __func__, spte);
1330 1331
			BUG();
		}
1332
		rmap_head->val = 0;
1333
	} else {
1334
		rmap_printk("%s:  %p many->many\n", __func__, spte);
1335
		desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1336 1337
		prev_desc = NULL;
		while (desc) {
1338
			for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) {
A
Avi Kivity 已提交
1339
				if (desc->sptes[i] == spte) {
1340 1341
					pte_list_desc_remove_entry(rmap_head,
							desc, i, prev_desc);
1342 1343
					return;
				}
1344
			}
1345 1346 1347
			prev_desc = desc;
			desc = desc->more;
		}
1348
		pr_err("%s: %p many->many\n", __func__, spte);
1349 1350 1351 1352
		BUG();
	}
}

1353 1354 1355 1356 1357 1358
static void pte_list_remove(struct kvm_rmap_head *rmap_head, u64 *sptep)
{
	mmu_spte_clear_track_bits(sptep);
	__pte_list_remove(sptep, rmap_head);
}

1359 1360
static struct kvm_rmap_head *__gfn_to_rmap(gfn_t gfn, int level,
					   struct kvm_memory_slot *slot)
1361
{
1362
	unsigned long idx;
1363

1364
	idx = gfn_to_index(gfn, slot->base_gfn, level);
1365
	return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
1366 1367
}

1368 1369
static struct kvm_rmap_head *gfn_to_rmap(struct kvm *kvm, gfn_t gfn,
					 struct kvm_mmu_page *sp)
1370
{
1371
	struct kvm_memslots *slots;
1372 1373
	struct kvm_memory_slot *slot;

1374 1375
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
1376
	return __gfn_to_rmap(gfn, sp->role.level, slot);
1377 1378
}

1379 1380 1381 1382 1383 1384 1385 1386
static bool rmap_can_add(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_memory_cache *cache;

	cache = &vcpu->arch.mmu_pte_list_desc_cache;
	return mmu_memory_cache_free_objects(cache);
}

1387 1388 1389
static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
{
	struct kvm_mmu_page *sp;
1390
	struct kvm_rmap_head *rmap_head;
1391 1392 1393

	sp = page_header(__pa(spte));
	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1394 1395
	rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
	return pte_list_add(vcpu, spte, rmap_head);
1396 1397 1398 1399 1400 1401
}

static void rmap_remove(struct kvm *kvm, u64 *spte)
{
	struct kvm_mmu_page *sp;
	gfn_t gfn;
1402
	struct kvm_rmap_head *rmap_head;
1403 1404 1405

	sp = page_header(__pa(spte));
	gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1406
	rmap_head = gfn_to_rmap(kvm, gfn, sp);
1407
	__pte_list_remove(spte, rmap_head);
1408 1409
}

1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426
/*
 * Used by the following functions to iterate through the sptes linked by a
 * rmap.  All fields are private and not assumed to be used outside.
 */
struct rmap_iterator {
	/* private fields */
	struct pte_list_desc *desc;	/* holds the sptep if not NULL */
	int pos;			/* index of the sptep */
};

/*
 * Iteration must be started by this function.  This should also be used after
 * removing/dropping sptes from the rmap link because in such cases the
 * information in the itererator may not be valid.
 *
 * Returns sptep if found, NULL otherwise.
 */
1427 1428
static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
			   struct rmap_iterator *iter)
1429
{
1430 1431
	u64 *sptep;

1432
	if (!rmap_head->val)
1433 1434
		return NULL;

1435
	if (!(rmap_head->val & 1)) {
1436
		iter->desc = NULL;
1437 1438
		sptep = (u64 *)rmap_head->val;
		goto out;
1439 1440
	}

1441
	iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1442
	iter->pos = 0;
1443 1444 1445 1446
	sptep = iter->desc->sptes[iter->pos];
out:
	BUG_ON(!is_shadow_present_pte(*sptep));
	return sptep;
1447 1448 1449 1450 1451 1452 1453 1454 1455
}

/*
 * Must be used with a valid iterator: e.g. after rmap_get_first().
 *
 * Returns sptep if found, NULL otherwise.
 */
static u64 *rmap_get_next(struct rmap_iterator *iter)
{
1456 1457
	u64 *sptep;

1458 1459 1460 1461 1462
	if (iter->desc) {
		if (iter->pos < PTE_LIST_EXT - 1) {
			++iter->pos;
			sptep = iter->desc->sptes[iter->pos];
			if (sptep)
1463
				goto out;
1464 1465 1466 1467 1468 1469 1470
		}

		iter->desc = iter->desc->more;

		if (iter->desc) {
			iter->pos = 0;
			/* desc->sptes[0] cannot be NULL */
1471 1472
			sptep = iter->desc->sptes[iter->pos];
			goto out;
1473 1474 1475 1476
		}
	}

	return NULL;
1477 1478 1479
out:
	BUG_ON(!is_shadow_present_pte(*sptep));
	return sptep;
1480 1481
}

1482 1483
#define for_each_rmap_spte(_rmap_head_, _iter_, _spte_)			\
	for (_spte_ = rmap_get_first(_rmap_head_, _iter_);		\
1484
	     _spte_; _spte_ = rmap_get_next(_iter_))
1485

1486
static void drop_spte(struct kvm *kvm, u64 *sptep)
1487
{
1488
	if (mmu_spte_clear_track_bits(sptep))
1489
		rmap_remove(kvm, sptep);
1490 1491
}

1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507

static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
{
	if (is_large_pte(*sptep)) {
		WARN_ON(page_header(__pa(sptep))->role.level ==
			PT_PAGE_TABLE_LEVEL);
		drop_spte(kvm, sptep);
		--kvm->stat.lpages;
		return true;
	}

	return false;
}

static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
{
1508 1509 1510 1511 1512 1513
	if (__drop_large_spte(vcpu->kvm, sptep)) {
		struct kvm_mmu_page *sp = page_header(__pa(sptep));

		kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn,
			KVM_PAGES_PER_HPAGE(sp->role.level));
	}
1514 1515 1516
}

/*
1517
 * Write-protect on the specified @sptep, @pt_protect indicates whether
1518
 * spte write-protection is caused by protecting shadow page table.
1519
 *
1520
 * Note: write protection is difference between dirty logging and spte
1521 1522 1523 1524 1525
 * protection:
 * - for dirty logging, the spte can be set to writable at anytime if
 *   its dirty bitmap is properly set.
 * - for spte protection, the spte can be writable only after unsync-ing
 *   shadow page.
1526
 *
1527
 * Return true if tlb need be flushed.
1528
 */
1529
static bool spte_write_protect(u64 *sptep, bool pt_protect)
1530 1531 1532
{
	u64 spte = *sptep;

1533
	if (!is_writable_pte(spte) &&
1534
	      !(pt_protect && spte_can_locklessly_be_made_writable(spte)))
1535 1536 1537 1538
		return false;

	rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);

1539 1540
	if (pt_protect)
		spte &= ~SPTE_MMU_WRITEABLE;
1541
	spte = spte & ~PT_WRITABLE_MASK;
1542

1543
	return mmu_spte_update(sptep, spte);
1544 1545
}

1546 1547
static bool __rmap_write_protect(struct kvm *kvm,
				 struct kvm_rmap_head *rmap_head,
1548
				 bool pt_protect)
1549
{
1550 1551
	u64 *sptep;
	struct rmap_iterator iter;
1552
	bool flush = false;
1553

1554
	for_each_rmap_spte(rmap_head, &iter, sptep)
1555
		flush |= spte_write_protect(sptep, pt_protect);
1556

1557
	return flush;
1558 1559
}

1560
static bool spte_clear_dirty(u64 *sptep)
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
{
	u64 spte = *sptep;

	rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);

	spte &= ~shadow_dirty_mask;

	return mmu_spte_update(sptep, spte);
}

1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
static bool wrprot_ad_disabled_spte(u64 *sptep)
{
	bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT,
					       (unsigned long *)sptep);
	if (was_writable)
		kvm_set_pfn_dirty(spte_to_pfn(*sptep));

	return was_writable;
}

/*
 * Gets the GFN ready for another round of dirty logging by clearing the
 *	- D bit on ad-enabled SPTEs, and
 *	- W bit on ad-disabled SPTEs.
 * Returns true iff any D or W bits were cleared.
 */
1587
static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1588 1589 1590 1591 1592
{
	u64 *sptep;
	struct rmap_iterator iter;
	bool flush = false;

1593
	for_each_rmap_spte(rmap_head, &iter, sptep)
1594 1595 1596 1597
		if (spte_ad_enabled(*sptep))
			flush |= spte_clear_dirty(sptep);
		else
			flush |= wrprot_ad_disabled_spte(sptep);
1598 1599 1600 1601

	return flush;
}

1602
static bool spte_set_dirty(u64 *sptep)
1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
{
	u64 spte = *sptep;

	rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);

	spte |= shadow_dirty_mask;

	return mmu_spte_update(sptep, spte);
}

1613
static bool __rmap_set_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1614 1615 1616 1617 1618
{
	u64 *sptep;
	struct rmap_iterator iter;
	bool flush = false;

1619
	for_each_rmap_spte(rmap_head, &iter, sptep)
1620 1621
		if (spte_ad_enabled(*sptep))
			flush |= spte_set_dirty(sptep);
1622 1623 1624 1625

	return flush;
}

1626
/**
1627
 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1628 1629 1630 1631 1632 1633 1634 1635
 * @kvm: kvm instance
 * @slot: slot to protect
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should protect
 *
 * Used when we do not need to care about huge page mappings: e.g. during dirty
 * logging we do not have any such mappings.
 */
1636
static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1637 1638
				     struct kvm_memory_slot *slot,
				     gfn_t gfn_offset, unsigned long mask)
1639
{
1640
	struct kvm_rmap_head *rmap_head;
1641

1642
	while (mask) {
1643 1644 1645
		rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
					  PT_PAGE_TABLE_LEVEL, slot);
		__rmap_write_protect(kvm, rmap_head, false);
1646

1647 1648 1649
		/* clear the first set bit */
		mask &= mask - 1;
	}
1650 1651
}

1652
/**
1653 1654
 * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write
 * protect the page if the D-bit isn't supported.
1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665
 * @kvm: kvm instance
 * @slot: slot to clear D-bit
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should clear D-bit
 *
 * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
 */
void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
				     struct kvm_memory_slot *slot,
				     gfn_t gfn_offset, unsigned long mask)
{
1666
	struct kvm_rmap_head *rmap_head;
1667 1668

	while (mask) {
1669 1670 1671
		rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
					  PT_PAGE_TABLE_LEVEL, slot);
		__rmap_clear_dirty(kvm, rmap_head);
1672 1673 1674 1675 1676 1677 1678

		/* clear the first set bit */
		mask &= mask - 1;
	}
}
EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked);

1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692
/**
 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
 * PT level pages.
 *
 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
 * enable dirty logging for them.
 *
 * Used when we do not need to care about huge page mappings: e.g. during dirty
 * logging we do not have any such mappings.
 */
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
				struct kvm_memory_slot *slot,
				gfn_t gfn_offset, unsigned long mask)
{
1693 1694 1695 1696 1697
	if (kvm_x86_ops->enable_log_dirty_pt_masked)
		kvm_x86_ops->enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
				mask);
	else
		kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1698 1699
}

1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
/**
 * kvm_arch_write_log_dirty - emulate dirty page logging
 * @vcpu: Guest mode vcpu
 *
 * Emulate arch specific page modification logging for the
 * nested hypervisor
 */
int kvm_arch_write_log_dirty(struct kvm_vcpu *vcpu)
{
	if (kvm_x86_ops->write_log_dirty)
		return kvm_x86_ops->write_log_dirty(vcpu);

	return 0;
}

1715 1716
bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
				    struct kvm_memory_slot *slot, u64 gfn)
1717
{
1718
	struct kvm_rmap_head *rmap_head;
1719
	int i;
1720
	bool write_protected = false;
1721

1722
	for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
1723
		rmap_head = __gfn_to_rmap(gfn, i, slot);
1724
		write_protected |= __rmap_write_protect(kvm, rmap_head, true);
1725 1726 1727
	}

	return write_protected;
1728 1729
}

1730 1731 1732 1733 1734 1735 1736 1737
static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
{
	struct kvm_memory_slot *slot;

	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
	return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn);
}

1738
static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1739
{
1740 1741
	u64 *sptep;
	struct rmap_iterator iter;
1742
	bool flush = false;
1743

1744
	while ((sptep = rmap_get_first(rmap_head, &iter))) {
1745
		rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);
1746

1747
		pte_list_remove(rmap_head, sptep);
1748
		flush = true;
1749
	}
1750

1751 1752 1753
	return flush;
}

1754
static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1755 1756 1757
			   struct kvm_memory_slot *slot, gfn_t gfn, int level,
			   unsigned long data)
{
1758
	return kvm_zap_rmapp(kvm, rmap_head);
1759 1760
}

1761
static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1762 1763
			     struct kvm_memory_slot *slot, gfn_t gfn, int level,
			     unsigned long data)
1764
{
1765 1766
	u64 *sptep;
	struct rmap_iterator iter;
1767
	int need_flush = 0;
1768
	u64 new_spte;
1769
	pte_t *ptep = (pte_t *)data;
1770
	kvm_pfn_t new_pfn;
1771 1772 1773

	WARN_ON(pte_huge(*ptep));
	new_pfn = pte_pfn(*ptep);
1774

1775
restart:
1776
	for_each_rmap_spte(rmap_head, &iter, sptep) {
1777
		rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
1778
			    sptep, *sptep, gfn, level);
1779

1780
		need_flush = 1;
1781

1782
		if (pte_write(*ptep)) {
1783
			pte_list_remove(rmap_head, sptep);
1784
			goto restart;
1785
		} else {
1786
			new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
1787 1788 1789 1790
			new_spte |= (u64)new_pfn << PAGE_SHIFT;

			new_spte &= ~PT_WRITABLE_MASK;
			new_spte &= ~SPTE_HOST_WRITEABLE;
1791 1792

			new_spte = mark_spte_for_access_track(new_spte);
1793 1794 1795

			mmu_spte_clear_track_bits(sptep);
			mmu_spte_set(sptep, new_spte);
1796 1797
		}
	}
1798

1799 1800 1801 1802 1803
	if (need_flush && kvm_available_flush_tlb_with_range()) {
		kvm_flush_remote_tlbs_with_address(kvm, gfn, 1);
		return 0;
	}

1804
	return need_flush;
1805 1806
}

1807 1808 1809 1810 1811 1812 1813 1814 1815 1816
struct slot_rmap_walk_iterator {
	/* input fields. */
	struct kvm_memory_slot *slot;
	gfn_t start_gfn;
	gfn_t end_gfn;
	int start_level;
	int end_level;

	/* output fields. */
	gfn_t gfn;
1817
	struct kvm_rmap_head *rmap;
1818 1819 1820
	int level;

	/* private field. */
1821
	struct kvm_rmap_head *end_rmap;
1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874
};

static void
rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)
{
	iterator->level = level;
	iterator->gfn = iterator->start_gfn;
	iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot);
	iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level,
					   iterator->slot);
}

static void
slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
		    struct kvm_memory_slot *slot, int start_level,
		    int end_level, gfn_t start_gfn, gfn_t end_gfn)
{
	iterator->slot = slot;
	iterator->start_level = start_level;
	iterator->end_level = end_level;
	iterator->start_gfn = start_gfn;
	iterator->end_gfn = end_gfn;

	rmap_walk_init_level(iterator, iterator->start_level);
}

static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
{
	return !!iterator->rmap;
}

static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
{
	if (++iterator->rmap <= iterator->end_rmap) {
		iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
		return;
	}

	if (++iterator->level > iterator->end_level) {
		iterator->rmap = NULL;
		return;
	}

	rmap_walk_init_level(iterator, iterator->level);
}

#define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_,	\
	   _start_gfn, _end_gfn, _iter_)				\
	for (slot_rmap_walk_init(_iter_, _slot_, _start_level_,		\
				 _end_level_, _start_gfn, _end_gfn);	\
	     slot_rmap_walk_okay(_iter_);				\
	     slot_rmap_walk_next(_iter_))

1875 1876 1877 1878 1879
static int kvm_handle_hva_range(struct kvm *kvm,
				unsigned long start,
				unsigned long end,
				unsigned long data,
				int (*handler)(struct kvm *kvm,
1880
					       struct kvm_rmap_head *rmap_head,
1881
					       struct kvm_memory_slot *slot,
1882 1883
					       gfn_t gfn,
					       int level,
1884
					       unsigned long data))
1885
{
1886
	struct kvm_memslots *slots;
1887
	struct kvm_memory_slot *memslot;
1888 1889
	struct slot_rmap_walk_iterator iterator;
	int ret = 0;
1890
	int i;
1891

1892 1893 1894 1895 1896
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
		kvm_for_each_memslot(memslot, slots) {
			unsigned long hva_start, hva_end;
			gfn_t gfn_start, gfn_end;
1897

1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916
			hva_start = max(start, memslot->userspace_addr);
			hva_end = min(end, memslot->userspace_addr +
				      (memslot->npages << PAGE_SHIFT));
			if (hva_start >= hva_end)
				continue;
			/*
			 * {gfn(page) | page intersects with [hva_start, hva_end)} =
			 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
			 */
			gfn_start = hva_to_gfn_memslot(hva_start, memslot);
			gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);

			for_each_slot_rmap_range(memslot, PT_PAGE_TABLE_LEVEL,
						 PT_MAX_HUGEPAGE_LEVEL,
						 gfn_start, gfn_end - 1,
						 &iterator)
				ret |= handler(kvm, iterator.rmap, memslot,
					       iterator.gfn, iterator.level, data);
		}
1917 1918
	}

1919
	return ret;
1920 1921
}

1922 1923
static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
			  unsigned long data,
1924 1925
			  int (*handler)(struct kvm *kvm,
					 struct kvm_rmap_head *rmap_head,
1926
					 struct kvm_memory_slot *slot,
1927
					 gfn_t gfn, int level,
1928 1929 1930
					 unsigned long data))
{
	return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1931 1932
}

1933 1934 1935 1936 1937
int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
{
	return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
}

1938
int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1939
{
1940
	return kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1941 1942
}

1943
static int kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1944 1945
			 struct kvm_memory_slot *slot, gfn_t gfn, int level,
			 unsigned long data)
1946
{
1947
	u64 *sptep;
1948
	struct rmap_iterator uninitialized_var(iter);
1949 1950
	int young = 0;

1951 1952
	for_each_rmap_spte(rmap_head, &iter, sptep)
		young |= mmu_spte_age(sptep);
1953

1954
	trace_kvm_age_page(gfn, level, slot, young);
1955 1956 1957
	return young;
}

1958
static int kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1959 1960
			      struct kvm_memory_slot *slot, gfn_t gfn,
			      int level, unsigned long data)
1961
{
1962 1963
	u64 *sptep;
	struct rmap_iterator iter;
1964

1965 1966 1967 1968
	for_each_rmap_spte(rmap_head, &iter, sptep)
		if (is_accessed_spte(*sptep))
			return 1;
	return 0;
1969 1970
}

1971 1972
#define RMAP_RECYCLE_THRESHOLD 1000

1973
static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1974
{
1975
	struct kvm_rmap_head *rmap_head;
1976 1977 1978
	struct kvm_mmu_page *sp;

	sp = page_header(__pa(spte));
1979

1980
	rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
1981

1982
	kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, 0);
1983 1984
	kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn,
			KVM_PAGES_PER_HPAGE(sp->role.level));
1985 1986
}

1987
int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1988
{
1989
	return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
1990 1991
}

1992 1993 1994 1995 1996
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
{
	return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
}

1997
#ifdef MMU_DEBUG
1998
static int is_empty_shadow_page(u64 *spt)
1999
{
2000 2001 2002
	u64 *pos;
	u64 *end;

2003
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
2004
		if (is_shadow_present_pte(*pos)) {
2005
			printk(KERN_ERR "%s: %p %llx\n", __func__,
2006
			       pos, *pos);
2007
			return 0;
2008
		}
2009 2010
	return 1;
}
2011
#endif
2012

2013 2014 2015 2016 2017 2018
/*
 * This value is the sum of all of the kvm instances's
 * kvm->arch.n_used_mmu_pages values.  We need a global,
 * aggregate version in order to make the slab shrinker
 * faster
 */
2019
static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, unsigned long nr)
2020 2021 2022 2023 2024
{
	kvm->arch.n_used_mmu_pages += nr;
	percpu_counter_add(&kvm_total_used_mmu_pages, nr);
}

2025
static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
2026
{
2027
	MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
2028
	hlist_del(&sp->hash_link);
2029 2030
	list_del(&sp->link);
	free_page((unsigned long)sp->spt);
2031 2032
	if (!sp->role.direct)
		free_page((unsigned long)sp->gfns);
2033
	kmem_cache_free(mmu_page_header_cache, sp);
2034 2035
}

2036 2037
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
2038
	return hash_64(gfn, KVM_MMU_HASH_SHIFT);
2039 2040
}

2041
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
2042
				    struct kvm_mmu_page *sp, u64 *parent_pte)
2043 2044 2045 2046
{
	if (!parent_pte)
		return;

2047
	pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
2048 2049
}

2050
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
2051 2052
				       u64 *parent_pte)
{
2053
	__pte_list_remove(parent_pte, &sp->parent_ptes);
2054 2055
}

2056 2057 2058 2059
static void drop_parent_pte(struct kvm_mmu_page *sp,
			    u64 *parent_pte)
{
	mmu_page_remove_parent_pte(sp, parent_pte);
2060
	mmu_spte_clear_no_track(parent_pte);
2061 2062
}

2063
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct)
2064
{
2065
	struct kvm_mmu_page *sp;
2066

2067 2068
	sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
	sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
2069
	if (!direct)
2070
		sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
2071 2072 2073 2074
	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
	kvm_mod_used_mmu_pages(vcpu->kvm, +1);
	return sp;
2075 2076
}

2077
static void mark_unsync(u64 *spte);
2078
static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
2079
{
2080 2081 2082 2083 2084 2085
	u64 *sptep;
	struct rmap_iterator iter;

	for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
		mark_unsync(sptep);
	}
2086 2087
}

2088
static void mark_unsync(u64 *spte)
2089
{
2090
	struct kvm_mmu_page *sp;
2091
	unsigned int index;
2092

2093
	sp = page_header(__pa(spte));
2094 2095
	index = spte - sp->spt;
	if (__test_and_set_bit(index, sp->unsync_child_bitmap))
2096
		return;
2097
	if (sp->unsync_children++)
2098
		return;
2099
	kvm_mmu_mark_parents_unsync(sp);
2100 2101
}

2102
static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
2103
			       struct kvm_mmu_page *sp)
2104
{
2105
	return 0;
2106 2107
}

2108
static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root)
M
Marcelo Tosatti 已提交
2109 2110 2111
{
}

2112 2113
static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
				 struct kvm_mmu_page *sp, u64 *spte,
2114
				 const void *pte)
2115 2116 2117 2118
{
	WARN_ON(1);
}

2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
#define KVM_PAGE_ARRAY_NR 16

struct kvm_mmu_pages {
	struct mmu_page_and_offset {
		struct kvm_mmu_page *sp;
		unsigned int idx;
	} page[KVM_PAGE_ARRAY_NR];
	unsigned int nr;
};

2129 2130
static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
			 int idx)
2131
{
2132
	int i;
2133

2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
	if (sp->unsync)
		for (i=0; i < pvec->nr; i++)
			if (pvec->page[i].sp == sp)
				return 0;

	pvec->page[pvec->nr].sp = sp;
	pvec->page[pvec->nr].idx = idx;
	pvec->nr++;
	return (pvec->nr == KVM_PAGE_ARRAY_NR);
}

2145 2146 2147 2148 2149 2150 2151
static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
{
	--sp->unsync_children;
	WARN_ON((int)sp->unsync_children < 0);
	__clear_bit(idx, sp->unsync_child_bitmap);
}

2152 2153 2154 2155
static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	int i, ret, nr_unsync_leaf = 0;
2156

2157
	for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
2158
		struct kvm_mmu_page *child;
2159 2160
		u64 ent = sp->spt[i];

2161 2162 2163 2164
		if (!is_shadow_present_pte(ent) || is_large_pte(ent)) {
			clear_unsync_child_bit(sp, i);
			continue;
		}
2165 2166 2167 2168 2169 2170 2171 2172

		child = page_header(ent & PT64_BASE_ADDR_MASK);

		if (child->unsync_children) {
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;

			ret = __mmu_unsync_walk(child, pvec);
2173 2174 2175 2176
			if (!ret) {
				clear_unsync_child_bit(sp, i);
				continue;
			} else if (ret > 0) {
2177
				nr_unsync_leaf += ret;
2178
			} else
2179 2180 2181 2182 2183 2184
				return ret;
		} else if (child->unsync) {
			nr_unsync_leaf++;
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;
		} else
2185
			clear_unsync_child_bit(sp, i);
2186 2187
	}

2188 2189 2190
	return nr_unsync_leaf;
}

2191 2192
#define INVALID_INDEX (-1)

2193 2194 2195
static int mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
2196
	pvec->nr = 0;
2197 2198 2199
	if (!sp->unsync_children)
		return 0;

2200
	mmu_pages_add(pvec, sp, INVALID_INDEX);
2201
	return __mmu_unsync_walk(sp, pvec);
2202 2203 2204 2205 2206
}

static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	WARN_ON(!sp->unsync);
2207
	trace_kvm_mmu_sync_page(sp);
2208 2209 2210 2211
	sp->unsync = 0;
	--kvm->stat.mmu_unsync;
}

2212 2213
static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				     struct list_head *invalid_list);
2214 2215
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list);
2216

2217

2218
#define for_each_valid_sp(_kvm, _sp, _gfn)				\
2219 2220
	hlist_for_each_entry(_sp,					\
	  &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
2221
		if ((_sp)->role.invalid) {    \
2222
		} else
2223 2224

#define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn)			\
2225 2226
	for_each_valid_sp(_kvm, _sp, _gfn)				\
		if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else
2227

2228 2229 2230 2231 2232
static inline bool is_ept_sp(struct kvm_mmu_page *sp)
{
	return sp->role.cr0_wp && sp->role.smap_andnot_wp;
}

2233
/* @sp->gfn should be write-protected at the call site */
2234 2235
static bool __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
			    struct list_head *invalid_list)
2236
{
2237 2238
	if ((!is_ept_sp(sp) && sp->role.gpte_is_8_bytes != !!is_pae(vcpu)) ||
	    vcpu->arch.mmu->sync_page(vcpu, sp) == 0) {
2239
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
2240
		return false;
2241 2242
	}

2243
	return true;
2244 2245
}

2246 2247 2248 2249
static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm,
					struct list_head *invalid_list,
					bool remote_flush)
{
2250
	if (!remote_flush && list_empty(invalid_list))
2251 2252 2253 2254 2255 2256 2257 2258 2259
		return false;

	if (!list_empty(invalid_list))
		kvm_mmu_commit_zap_page(kvm, invalid_list);
	else
		kvm_flush_remote_tlbs(kvm);
	return true;
}

2260 2261 2262
static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu,
				 struct list_head *invalid_list,
				 bool remote_flush, bool local_flush)
2263
{
2264
	if (kvm_mmu_remote_flush_or_zap(vcpu->kvm, invalid_list, remote_flush))
2265
		return;
2266

2267
	if (local_flush)
2268
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2269 2270
}

2271 2272 2273 2274 2275 2276 2277
#ifdef CONFIG_KVM_MMU_AUDIT
#include "mmu_audit.c"
#else
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
static void mmu_audit_disable(void) { }
#endif

2278
static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
2279
			 struct list_head *invalid_list)
2280
{
2281 2282
	kvm_unlink_unsync_page(vcpu->kvm, sp);
	return __kvm_sync_page(vcpu, sp, invalid_list);
2283 2284
}

2285
/* @gfn should be write-protected at the call site */
2286 2287
static bool kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn,
			   struct list_head *invalid_list)
2288 2289
{
	struct kvm_mmu_page *s;
2290
	bool ret = false;
2291

2292
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
2293
		if (!s->unsync)
2294 2295 2296
			continue;

		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
2297
		ret |= kvm_sync_page(vcpu, s, invalid_list);
2298 2299
	}

2300
	return ret;
2301 2302
}

2303
struct mmu_page_path {
2304 2305
	struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL];
	unsigned int idx[PT64_ROOT_MAX_LEVEL];
2306 2307
};

2308
#define for_each_sp(pvec, sp, parents, i)			\
2309
		for (i = mmu_pages_first(&pvec, &parents);	\
2310 2311 2312
			i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});	\
			i = mmu_pages_next(&pvec, &parents, i))

2313 2314 2315
static int mmu_pages_next(struct kvm_mmu_pages *pvec,
			  struct mmu_page_path *parents,
			  int i)
2316 2317 2318 2319 2320
{
	int n;

	for (n = i+1; n < pvec->nr; n++) {
		struct kvm_mmu_page *sp = pvec->page[n].sp;
2321 2322
		unsigned idx = pvec->page[n].idx;
		int level = sp->role.level;
2323

2324 2325 2326
		parents->idx[level-1] = idx;
		if (level == PT_PAGE_TABLE_LEVEL)
			break;
2327

2328
		parents->parent[level-2] = sp;
2329 2330 2331 2332 2333
	}

	return n;
}

2334 2335 2336 2337 2338 2339 2340 2341 2342
static int mmu_pages_first(struct kvm_mmu_pages *pvec,
			   struct mmu_page_path *parents)
{
	struct kvm_mmu_page *sp;
	int level;

	if (pvec->nr == 0)
		return 0;

2343 2344
	WARN_ON(pvec->page[0].idx != INVALID_INDEX);

2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357
	sp = pvec->page[0].sp;
	level = sp->role.level;
	WARN_ON(level == PT_PAGE_TABLE_LEVEL);

	parents->parent[level-2] = sp;

	/* Also set up a sentinel.  Further entries in pvec are all
	 * children of sp, so this element is never overwritten.
	 */
	parents->parent[level-1] = NULL;
	return mmu_pages_next(pvec, parents, 0);
}

2358
static void mmu_pages_clear_parents(struct mmu_page_path *parents)
2359
{
2360 2361 2362 2363 2364 2365 2366 2367 2368
	struct kvm_mmu_page *sp;
	unsigned int level = 0;

	do {
		unsigned int idx = parents->idx[level];
		sp = parents->parent[level];
		if (!sp)
			return;

2369
		WARN_ON(idx == INVALID_INDEX);
2370
		clear_unsync_child_bit(sp, idx);
2371
		level++;
2372
	} while (!sp->unsync_children);
2373
}
2374

2375 2376 2377 2378 2379 2380 2381
static void mmu_sync_children(struct kvm_vcpu *vcpu,
			      struct kvm_mmu_page *parent)
{
	int i;
	struct kvm_mmu_page *sp;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
2382
	LIST_HEAD(invalid_list);
2383
	bool flush = false;
2384 2385

	while (mmu_unsync_walk(parent, &pages)) {
2386
		bool protected = false;
2387 2388

		for_each_sp(pages, sp, parents, i)
2389
			protected |= rmap_write_protect(vcpu, sp->gfn);
2390

2391
		if (protected) {
2392
			kvm_flush_remote_tlbs(vcpu->kvm);
2393 2394
			flush = false;
		}
2395

2396
		for_each_sp(pages, sp, parents, i) {
2397
			flush |= kvm_sync_page(vcpu, sp, &invalid_list);
2398 2399
			mmu_pages_clear_parents(&parents);
		}
2400 2401 2402 2403 2404
		if (need_resched() || spin_needbreak(&vcpu->kvm->mmu_lock)) {
			kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
			cond_resched_lock(&vcpu->kvm->mmu_lock);
			flush = false;
		}
2405
	}
2406 2407

	kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2408 2409
}

2410 2411
static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
{
2412
	atomic_set(&sp->write_flooding_count,  0);
2413 2414 2415 2416 2417 2418 2419 2420 2421
}

static void clear_sp_write_flooding_count(u64 *spte)
{
	struct kvm_mmu_page *sp =  page_header(__pa(spte));

	__clear_sp_write_flooding_count(sp);
}

2422 2423 2424 2425
static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
2426
					     int direct,
2427
					     unsigned access)
2428 2429 2430
{
	union kvm_mmu_page_role role;
	unsigned quadrant;
2431 2432
	struct kvm_mmu_page *sp;
	bool need_sync = false;
2433
	bool flush = false;
2434
	int collisions = 0;
2435
	LIST_HEAD(invalid_list);
2436

2437
	role = vcpu->arch.mmu->mmu_role.base;
2438
	role.level = level;
2439
	role.direct = direct;
2440
	if (role.direct)
2441
		role.gpte_is_8_bytes = true;
2442
	role.access = access;
2443 2444
	if (!vcpu->arch.mmu->direct_map
	    && vcpu->arch.mmu->root_level <= PT32_ROOT_LEVEL) {
2445 2446 2447 2448
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
2449 2450 2451 2452 2453 2454
	for_each_valid_sp(vcpu->kvm, sp, gfn) {
		if (sp->gfn != gfn) {
			collisions++;
			continue;
		}

2455 2456
		if (!need_sync && sp->unsync)
			need_sync = true;
2457

2458 2459
		if (sp->role.word != role.word)
			continue;
2460

2461 2462 2463 2464 2465 2466 2467 2468 2469 2470
		if (sp->unsync) {
			/* The page is good, but __kvm_sync_page might still end
			 * up zapping it.  If so, break in order to rebuild it.
			 */
			if (!__kvm_sync_page(vcpu, sp, &invalid_list))
				break;

			WARN_ON(!list_empty(&invalid_list));
			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
		}
2471

2472
		if (sp->unsync_children)
2473
			kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2474

2475
		__clear_sp_write_flooding_count(sp);
2476
		trace_kvm_mmu_get_page(sp, false);
2477
		goto out;
2478
	}
2479

2480
	++vcpu->kvm->stat.mmu_cache_miss;
2481 2482 2483

	sp = kvm_mmu_alloc_page(vcpu, direct);

2484 2485
	sp->gfn = gfn;
	sp->role = role;
2486 2487
	hlist_add_head(&sp->hash_link,
		&vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
2488
	if (!direct) {
2489 2490 2491 2492 2493 2494 2495 2496
		/*
		 * we should do write protection before syncing pages
		 * otherwise the content of the synced shadow page may
		 * be inconsistent with guest page table.
		 */
		account_shadowed(vcpu->kvm, sp);
		if (level == PT_PAGE_TABLE_LEVEL &&
		      rmap_write_protect(vcpu, gfn))
2497
			kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1);
2498 2499

		if (level > PT_PAGE_TABLE_LEVEL && need_sync)
2500
			flush |= kvm_sync_pages(vcpu, gfn, &invalid_list);
2501
	}
2502
	clear_page(sp->spt);
2503
	trace_kvm_mmu_get_page(sp, true);
2504 2505

	kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2506 2507 2508
out:
	if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions)
		vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions;
2509
	return sp;
2510 2511
}

2512 2513 2514
static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator,
					struct kvm_vcpu *vcpu, hpa_t root,
					u64 addr)
2515 2516
{
	iterator->addr = addr;
2517
	iterator->shadow_addr = root;
2518
	iterator->level = vcpu->arch.mmu->shadow_root_level;
2519

2520
	if (iterator->level == PT64_ROOT_4LEVEL &&
2521 2522
	    vcpu->arch.mmu->root_level < PT64_ROOT_4LEVEL &&
	    !vcpu->arch.mmu->direct_map)
2523 2524
		--iterator->level;

2525
	if (iterator->level == PT32E_ROOT_LEVEL) {
2526 2527 2528 2529
		/*
		 * prev_root is currently only used for 64-bit hosts. So only
		 * the active root_hpa is valid here.
		 */
2530
		BUG_ON(root != vcpu->arch.mmu->root_hpa);
2531

2532
		iterator->shadow_addr
2533
			= vcpu->arch.mmu->pae_root[(addr >> 30) & 3];
2534 2535 2536 2537 2538 2539 2540
		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
		--iterator->level;
		if (!iterator->shadow_addr)
			iterator->level = 0;
	}
}

2541 2542 2543
static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
			     struct kvm_vcpu *vcpu, u64 addr)
{
2544
	shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root_hpa,
2545 2546 2547
				    addr);
}

2548 2549 2550 2551
static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
{
	if (iterator->level < PT_PAGE_TABLE_LEVEL)
		return false;
2552

2553 2554 2555 2556 2557
	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
	return true;
}

2558 2559
static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
			       u64 spte)
2560
{
2561
	if (is_last_spte(spte, iterator->level)) {
2562 2563 2564 2565
		iterator->level = 0;
		return;
	}

2566
	iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2567 2568 2569
	--iterator->level;
}

2570 2571
static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
{
2572
	__shadow_walk_next(iterator, *iterator->sptep);
2573 2574
}

2575 2576
static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
			     struct kvm_mmu_page *sp)
2577 2578 2579
{
	u64 spte;

2580
	BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2581

2582
	spte = __pa(sp->spt) | shadow_present_mask | PT_WRITABLE_MASK |
2583
	       shadow_user_mask | shadow_x_mask | shadow_me_mask;
2584 2585 2586 2587 2588

	if (sp_ad_disabled(sp))
		spte |= shadow_acc_track_value;
	else
		spte |= shadow_accessed_mask;
2589

2590
	mmu_spte_set(sptep, spte);
2591 2592 2593 2594 2595

	mmu_page_add_parent_pte(vcpu, sp, sptep);

	if (sp->unsync_children || sp->unsync)
		mark_unsync(sptep);
2596 2597
}

2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614
static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
				   unsigned direct_access)
{
	if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
		struct kvm_mmu_page *child;

		/*
		 * For the direct sp, if the guest pte's dirty bit
		 * changed form clean to dirty, it will corrupt the
		 * sp's access: allow writable in the read-only sp,
		 * so we should update the spte at this point to get
		 * a new sp with the correct access.
		 */
		child = page_header(*sptep & PT64_BASE_ADDR_MASK);
		if (child->role.access == direct_access)
			return;

2615
		drop_parent_pte(child, sptep);
2616
		kvm_flush_remote_tlbs_with_address(vcpu->kvm, child->gfn, 1);
2617 2618 2619
	}
}

2620
static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2621 2622 2623 2624 2625 2626 2627
			     u64 *spte)
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
	if (is_shadow_present_pte(pte)) {
2628
		if (is_last_spte(pte, sp->role.level)) {
2629
			drop_spte(kvm, spte);
2630 2631 2632
			if (is_large_pte(pte))
				--kvm->stat.lpages;
		} else {
2633
			child = page_header(pte & PT64_BASE_ADDR_MASK);
2634
			drop_parent_pte(child, spte);
2635
		}
2636 2637 2638 2639
		return true;
	}

	if (is_mmio_spte(pte))
2640
		mmu_spte_clear_no_track(spte);
2641

2642
	return false;
2643 2644
}

2645
static void kvm_mmu_page_unlink_children(struct kvm *kvm,
2646
					 struct kvm_mmu_page *sp)
2647
{
2648 2649
	unsigned i;

2650 2651
	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		mmu_page_zap_pte(kvm, sp, sp->spt + i);
2652 2653
}

2654
static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
2655
{
2656 2657
	u64 *sptep;
	struct rmap_iterator iter;
2658

2659
	while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2660
		drop_parent_pte(sp, sptep);
2661 2662
}

2663
static int mmu_zap_unsync_children(struct kvm *kvm,
2664 2665
				   struct kvm_mmu_page *parent,
				   struct list_head *invalid_list)
2666
{
2667 2668 2669
	int i, zapped = 0;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
2670

2671
	if (parent->role.level == PT_PAGE_TABLE_LEVEL)
2672
		return 0;
2673 2674 2675 2676 2677

	while (mmu_unsync_walk(parent, &pages)) {
		struct kvm_mmu_page *sp;

		for_each_sp(pages, sp, parents, i) {
2678
			kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2679
			mmu_pages_clear_parents(&parents);
2680
			zapped++;
2681 2682 2683 2684
		}
	}

	return zapped;
2685 2686
}

2687 2688 2689 2690
static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
				       struct kvm_mmu_page *sp,
				       struct list_head *invalid_list,
				       int *nr_zapped)
2691
{
2692
	bool list_unstable;
2693

2694
	trace_kvm_mmu_prepare_zap_page(sp);
2695
	++kvm->stat.mmu_shadow_zapped;
2696
	*nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list);
2697
	kvm_mmu_page_unlink_children(kvm, sp);
2698
	kvm_mmu_unlink_parents(kvm, sp);
2699

2700 2701 2702
	/* Zapping children means active_mmu_pages has become unstable. */
	list_unstable = *nr_zapped;

2703
	if (!sp->role.invalid && !sp->role.direct)
2704
		unaccount_shadowed(kvm, sp);
2705

2706 2707
	if (sp->unsync)
		kvm_unlink_unsync_page(kvm, sp);
2708
	if (!sp->root_count) {
2709
		/* Count self */
2710
		(*nr_zapped)++;
2711
		list_move(&sp->link, invalid_list);
2712
		kvm_mod_used_mmu_pages(kvm, -1);
2713
	} else {
2714
		list_move(&sp->link, &kvm->arch.active_mmu_pages);
2715

2716
		if (!sp->role.invalid)
2717
			kvm_reload_remote_mmus(kvm);
2718
	}
2719 2720

	sp->role.invalid = 1;
2721 2722 2723 2724 2725 2726 2727 2728 2729 2730
	return list_unstable;
}

static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				     struct list_head *invalid_list)
{
	int nr_zapped;

	__kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped);
	return nr_zapped;
2731 2732
}

2733 2734 2735
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list)
{
2736
	struct kvm_mmu_page *sp, *nsp;
2737 2738 2739 2740

	if (list_empty(invalid_list))
		return;

2741
	/*
2742 2743 2744 2745 2746 2747 2748
	 * We need to make sure everyone sees our modifications to
	 * the page tables and see changes to vcpu->mode here. The barrier
	 * in the kvm_flush_remote_tlbs() achieves this. This pairs
	 * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end.
	 *
	 * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
	 * guest mode and/or lockless shadow page table walks.
2749 2750
	 */
	kvm_flush_remote_tlbs(kvm);
2751

2752
	list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2753
		WARN_ON(!sp->role.invalid || sp->root_count);
2754
		kvm_mmu_free_page(sp);
2755
	}
2756 2757
}

2758 2759 2760 2761 2762 2763 2764 2765
static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
					struct list_head *invalid_list)
{
	struct kvm_mmu_page *sp;

	if (list_empty(&kvm->arch.active_mmu_pages))
		return false;

2766 2767
	sp = list_last_entry(&kvm->arch.active_mmu_pages,
			     struct kvm_mmu_page, link);
2768
	return kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2769 2770
}

2771 2772
/*
 * Changing the number of mmu pages allocated to the vm
2773
 * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2774
 */
2775
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages)
2776
{
2777
	LIST_HEAD(invalid_list);
2778

2779 2780
	spin_lock(&kvm->mmu_lock);

2781
	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2782 2783 2784 2785
		/* Need to free some mmu pages to achieve the goal. */
		while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
			if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
				break;
2786

2787
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
2788
		goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2789 2790
	}

2791
	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2792 2793

	spin_unlock(&kvm->mmu_lock);
2794 2795
}

2796
int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2797
{
2798
	struct kvm_mmu_page *sp;
2799
	LIST_HEAD(invalid_list);
2800 2801
	int r;

2802
	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2803
	r = 0;
2804
	spin_lock(&kvm->mmu_lock);
2805
	for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2806
		pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2807 2808
			 sp->role.word);
		r = 1;
2809
		kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2810
	}
2811
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
2812 2813
	spin_unlock(&kvm->mmu_lock);

2814
	return r;
2815
}
2816
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
2817

2818
static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
2819 2820 2821 2822 2823 2824 2825 2826
{
	trace_kvm_mmu_unsync_page(sp);
	++vcpu->kvm->stat.mmu_unsync;
	sp->unsync = 1;

	kvm_mmu_mark_parents_unsync(sp);
}

2827 2828
static bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
				   bool can_unsync)
2829
{
2830
	struct kvm_mmu_page *sp;
2831

2832 2833
	if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE))
		return true;
2834

2835
	for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
2836
		if (!can_unsync)
2837
			return true;
2838

2839 2840
		if (sp->unsync)
			continue;
2841

2842 2843
		WARN_ON(sp->role.level != PT_PAGE_TABLE_LEVEL);
		kvm_unsync_page(vcpu, sp);
2844
	}
2845

2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884
	/*
	 * We need to ensure that the marking of unsync pages is visible
	 * before the SPTE is updated to allow writes because
	 * kvm_mmu_sync_roots() checks the unsync flags without holding
	 * the MMU lock and so can race with this. If the SPTE was updated
	 * before the page had been marked as unsync-ed, something like the
	 * following could happen:
	 *
	 * CPU 1                    CPU 2
	 * ---------------------------------------------------------------------
	 * 1.2 Host updates SPTE
	 *     to be writable
	 *                      2.1 Guest writes a GPTE for GVA X.
	 *                          (GPTE being in the guest page table shadowed
	 *                           by the SP from CPU 1.)
	 *                          This reads SPTE during the page table walk.
	 *                          Since SPTE.W is read as 1, there is no
	 *                          fault.
	 *
	 *                      2.2 Guest issues TLB flush.
	 *                          That causes a VM Exit.
	 *
	 *                      2.3 kvm_mmu_sync_pages() reads sp->unsync.
	 *                          Since it is false, so it just returns.
	 *
	 *                      2.4 Guest accesses GVA X.
	 *                          Since the mapping in the SP was not updated,
	 *                          so the old mapping for GVA X incorrectly
	 *                          gets used.
	 * 1.1 Host marks SP
	 *     as unsync
	 *     (sp->unsync = true)
	 *
	 * The write barrier below ensures that 1.1 happens before 1.2 and thus
	 * the situation in 2.4 does not arise. The implicit barrier in 2.2
	 * pairs with this write barrier.
	 */
	smp_wmb();

2885
	return false;
2886 2887
}

2888
static bool kvm_is_mmio_pfn(kvm_pfn_t pfn)
2889 2890
{
	if (pfn_valid(pfn))
2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902
		return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn)) &&
			/*
			 * Some reserved pages, such as those from NVDIMM
			 * DAX devices, are not for MMIO, and can be mapped
			 * with cached memory type for better performance.
			 * However, the above check misconceives those pages
			 * as MMIO, and results in KVM mapping them with UC
			 * memory type, which would hurt the performance.
			 * Therefore, we check the host memory type in addition
			 * and only treat UC/UC-/WC pages as MMIO.
			 */
			(!pat_enabled() || pat_pfn_immune_to_uc_mtrr(pfn));
2903

2904 2905 2906
	return !e820__mapped_raw_any(pfn_to_hpa(pfn),
				     pfn_to_hpa(pfn + 1) - 1,
				     E820_TYPE_RAM);
2907 2908
}

2909 2910 2911 2912
/* Bits which may be returned by set_spte() */
#define SET_SPTE_WRITE_PROTECTED_PT	BIT(0)
#define SET_SPTE_NEED_REMOTE_TLB_FLUSH	BIT(1)

A
Avi Kivity 已提交
2913
static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2914
		    unsigned pte_access, int level,
2915
		    gfn_t gfn, kvm_pfn_t pfn, bool speculative,
2916
		    bool can_unsync, bool host_writable)
2917
{
2918
	u64 spte = 0;
2919
	int ret = 0;
2920
	struct kvm_mmu_page *sp;
S
Sheng Yang 已提交
2921

2922
	if (set_mmio_spte(vcpu, sptep, gfn, pfn, pte_access))
2923 2924
		return 0;

2925 2926 2927 2928
	sp = page_header(__pa(sptep));
	if (sp_ad_disabled(sp))
		spte |= shadow_acc_track_value;

2929 2930 2931 2932 2933 2934
	/*
	 * For the EPT case, shadow_present_mask is 0 if hardware
	 * supports exec-only page table entries.  In that case,
	 * ACC_USER_MASK and shadow_user_mask are used to represent
	 * read access.  See FNAME(gpte_access) in paging_tmpl.h.
	 */
2935
	spte |= shadow_present_mask;
2936
	if (!speculative)
2937
		spte |= spte_shadow_accessed_mask(spte);
2938

S
Sheng Yang 已提交
2939 2940 2941 2942
	if (pte_access & ACC_EXEC_MASK)
		spte |= shadow_x_mask;
	else
		spte |= shadow_nx_mask;
2943

2944
	if (pte_access & ACC_USER_MASK)
S
Sheng Yang 已提交
2945
		spte |= shadow_user_mask;
2946

2947
	if (level > PT_PAGE_TABLE_LEVEL)
2948
		spte |= PT_PAGE_SIZE_MASK;
2949
	if (tdp_enabled)
2950
		spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
2951
			kvm_is_mmio_pfn(pfn));
2952

2953
	if (host_writable)
2954
		spte |= SPTE_HOST_WRITEABLE;
2955 2956
	else
		pte_access &= ~ACC_WRITE_MASK;
2957

2958 2959 2960
	if (!kvm_is_mmio_pfn(pfn))
		spte |= shadow_me_mask;

2961
	spte |= (u64)pfn << PAGE_SHIFT;
2962

2963
	if (pte_access & ACC_WRITE_MASK) {
2964

2965
		/*
2966 2967 2968 2969
		 * Other vcpu creates new sp in the window between
		 * mapping_level() and acquiring mmu-lock. We can
		 * allow guest to retry the access, the mapping can
		 * be fixed if guest refault.
2970
		 */
2971
		if (level > PT_PAGE_TABLE_LEVEL &&
2972
		    mmu_gfn_lpage_is_disallowed(vcpu, gfn, level))
2973
			goto done;
2974

2975
		spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
2976

2977 2978 2979 2980 2981 2982
		/*
		 * Optimization: for pte sync, if spte was writable the hash
		 * lookup is unnecessary (and expensive). Write protection
		 * is responsibility of mmu_get_page / kvm_sync_page.
		 * Same reasoning can be applied to dirty page accounting.
		 */
2983
		if (!can_unsync && is_writable_pte(*sptep))
2984 2985
			goto set_pte;

2986
		if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2987
			pgprintk("%s: found shadow page for %llx, marking ro\n",
2988
				 __func__, gfn);
2989
			ret |= SET_SPTE_WRITE_PROTECTED_PT;
2990
			pte_access &= ~ACC_WRITE_MASK;
2991
			spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
2992 2993 2994
		}
	}

2995
	if (pte_access & ACC_WRITE_MASK) {
2996
		kvm_vcpu_mark_page_dirty(vcpu, gfn);
2997
		spte |= spte_shadow_dirty_mask(spte);
2998
	}
2999

3000 3001 3002
	if (speculative)
		spte = mark_spte_for_access_track(spte);

3003
set_pte:
3004
	if (mmu_spte_update(sptep, spte))
3005
		ret |= SET_SPTE_NEED_REMOTE_TLB_FLUSH;
3006
done:
3007 3008 3009
	return ret;
}

3010 3011 3012
static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access,
			int write_fault, int level, gfn_t gfn, kvm_pfn_t pfn,
		       	bool speculative, bool host_writable)
3013 3014
{
	int was_rmapped = 0;
3015
	int rmap_count;
3016
	int set_spte_ret;
3017
	int ret = RET_PF_RETRY;
3018
	bool flush = false;
3019

3020 3021
	pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
		 *sptep, write_fault, gfn);
3022

3023
	if (is_shadow_present_pte(*sptep)) {
3024 3025 3026 3027
		/*
		 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
		 * the parent of the now unreachable PTE.
		 */
3028 3029
		if (level > PT_PAGE_TABLE_LEVEL &&
		    !is_large_pte(*sptep)) {
3030
			struct kvm_mmu_page *child;
A
Avi Kivity 已提交
3031
			u64 pte = *sptep;
3032 3033

			child = page_header(pte & PT64_BASE_ADDR_MASK);
3034
			drop_parent_pte(child, sptep);
3035
			flush = true;
A
Avi Kivity 已提交
3036
		} else if (pfn != spte_to_pfn(*sptep)) {
3037
			pgprintk("hfn old %llx new %llx\n",
A
Avi Kivity 已提交
3038
				 spte_to_pfn(*sptep), pfn);
3039
			drop_spte(vcpu->kvm, sptep);
3040
			flush = true;
3041 3042
		} else
			was_rmapped = 1;
3043
	}
3044

3045 3046 3047
	set_spte_ret = set_spte(vcpu, sptep, pte_access, level, gfn, pfn,
				speculative, true, host_writable);
	if (set_spte_ret & SET_SPTE_WRITE_PROTECTED_PT) {
3048
		if (write_fault)
3049
			ret = RET_PF_EMULATE;
3050
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3051
	}
3052

3053
	if (set_spte_ret & SET_SPTE_NEED_REMOTE_TLB_FLUSH || flush)
3054 3055
		kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn,
				KVM_PAGES_PER_HPAGE(level));
3056

3057
	if (unlikely(is_mmio_spte(*sptep)))
3058
		ret = RET_PF_EMULATE;
3059

A
Avi Kivity 已提交
3060
	pgprintk("%s: setting spte %llx\n", __func__, *sptep);
3061
	pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
A
Avi Kivity 已提交
3062
		 is_large_pte(*sptep)? "2MB" : "4kB",
3063
		 *sptep & PT_WRITABLE_MASK ? "RW" : "R", gfn,
3064
		 *sptep, sptep);
A
Avi Kivity 已提交
3065
	if (!was_rmapped && is_large_pte(*sptep))
3066 3067
		++vcpu->kvm->stat.lpages;

3068 3069 3070 3071 3072 3073
	if (is_shadow_present_pte(*sptep)) {
		if (!was_rmapped) {
			rmap_count = rmap_add(vcpu, sptep, gfn);
			if (rmap_count > RMAP_RECYCLE_THRESHOLD)
				rmap_recycle(vcpu, sptep, gfn);
		}
3074
	}
3075

3076
	kvm_release_pfn_clean(pfn);
3077

3078
	return ret;
3079 3080
}

3081
static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
3082 3083 3084 3085
				     bool no_dirty_log)
{
	struct kvm_memory_slot *slot;

3086
	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
3087
	if (!slot)
3088
		return KVM_PFN_ERR_FAULT;
3089

3090
	return gfn_to_pfn_memslot_atomic(slot, gfn);
3091 3092 3093 3094 3095 3096 3097
}

static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *sp,
				    u64 *start, u64 *end)
{
	struct page *pages[PTE_PREFETCH_NUM];
3098
	struct kvm_memory_slot *slot;
3099 3100 3101 3102 3103
	unsigned access = sp->role.access;
	int i, ret;
	gfn_t gfn;

	gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
3104 3105
	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
	if (!slot)
3106 3107
		return -1;

3108
	ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
3109 3110 3111 3112
	if (ret <= 0)
		return -1;

	for (i = 0; i < ret; i++, gfn++, start++)
3113 3114
		mmu_set_spte(vcpu, start, access, 0, sp->role.level, gfn,
			     page_to_pfn(pages[i]), true, true);
3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130

	return 0;
}

static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
				  struct kvm_mmu_page *sp, u64 *sptep)
{
	u64 *spte, *start = NULL;
	int i;

	WARN_ON(!sp->role.direct);

	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
	spte = sp->spt + i;

	for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
3131
		if (is_shadow_present_pte(*spte) || spte == sptep) {
3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145
			if (!start)
				continue;
			if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
				break;
			start = NULL;
		} else if (!start)
			start = spte;
	}
}

static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
{
	struct kvm_mmu_page *sp;

3146 3147
	sp = page_header(__pa(sptep));

3148
	/*
3149 3150 3151
	 * Without accessed bits, there's no way to distinguish between
	 * actually accessed translations and prefetched, so disable pte
	 * prefetch if accessed bits aren't available.
3152
	 */
3153
	if (sp_ad_disabled(sp))
3154 3155 3156 3157 3158 3159 3160 3161
		return;

	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return;

	__direct_pte_prefetch(vcpu, sp, sptep);
}

3162
static int __direct_map(struct kvm_vcpu *vcpu, int write, int map_writable,
3163
			int level, gfn_t gfn, kvm_pfn_t pfn, bool prefault)
3164
{
3165
	struct kvm_shadow_walk_iterator iterator;
3166
	struct kvm_mmu_page *sp;
3167
	int emulate = 0;
3168
	gfn_t pseudo_gfn;
3169

3170
	if (!VALID_PAGE(vcpu->arch.mmu->root_hpa))
3171 3172
		return 0;

3173
	for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
3174
		if (iterator.level == level) {
3175 3176 3177
			emulate = mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
					       write, level, gfn, pfn, prefault,
					       map_writable);
3178
			direct_pte_prefetch(vcpu, iterator.sptep);
3179 3180
			++vcpu->stat.pf_fixed;
			break;
3181 3182
		}

3183
		drop_large_spte(vcpu, iterator.sptep);
3184
		if (!is_shadow_present_pte(*iterator.sptep)) {
3185 3186 3187 3188
			u64 base_addr = iterator.addr;

			base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
			pseudo_gfn = base_addr >> PAGE_SHIFT;
3189
			sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
3190
					      iterator.level - 1, 1, ACC_ALL);
3191

3192
			link_shadow_page(vcpu, iterator.sptep, sp);
3193 3194
		}
	}
3195
	return emulate;
3196 3197
}

3198
static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
3199
{
3200
	send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, PAGE_SHIFT, tsk);
3201 3202
}

3203
static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
3204
{
3205 3206 3207 3208 3209 3210
	/*
	 * Do not cache the mmio info caused by writing the readonly gfn
	 * into the spte otherwise read access on readonly gfn also can
	 * caused mmio page fault and treat it as mmio access.
	 */
	if (pfn == KVM_PFN_ERR_RO_FAULT)
3211
		return RET_PF_EMULATE;
3212

3213
	if (pfn == KVM_PFN_ERR_HWPOISON) {
3214
		kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
3215
		return RET_PF_RETRY;
3216
	}
3217

3218
	return -EFAULT;
3219 3220
}

3221
static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
3222 3223
					gfn_t *gfnp, kvm_pfn_t *pfnp,
					int *levelp)
3224
{
3225
	kvm_pfn_t pfn = *pfnp;
3226 3227 3228 3229 3230 3231 3232 3233 3234
	gfn_t gfn = *gfnp;
	int level = *levelp;

	/*
	 * Check if it's a transparent hugepage. If this would be an
	 * hugetlbfs page, level wouldn't be set to
	 * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
	 * here.
	 */
3235
	if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn) &&
3236
	    level == PT_PAGE_TABLE_LEVEL &&
3237
	    PageTransCompoundMap(pfn_to_page(pfn)) &&
3238
	    !mmu_gfn_lpage_is_disallowed(vcpu, gfn, PT_DIRECTORY_LEVEL)) {
3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256
		unsigned long mask;
		/*
		 * mmu_notifier_retry was successful and we hold the
		 * mmu_lock here, so the pmd can't become splitting
		 * from under us, and in turn
		 * __split_huge_page_refcount() can't run from under
		 * us and we can safely transfer the refcount from
		 * PG_tail to PG_head as we switch the pfn to tail to
		 * head.
		 */
		*levelp = level = PT_DIRECTORY_LEVEL;
		mask = KVM_PAGES_PER_HPAGE(level) - 1;
		VM_BUG_ON((gfn & mask) != (pfn & mask));
		if (pfn & mask) {
			gfn &= ~mask;
			*gfnp = gfn;
			kvm_release_pfn_clean(pfn);
			pfn &= ~mask;
3257
			kvm_get_pfn(pfn);
3258 3259 3260 3261 3262
			*pfnp = pfn;
		}
	}
}

3263
static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
3264
				kvm_pfn_t pfn, unsigned access, int *ret_val)
3265 3266
{
	/* The pfn is invalid, report the error! */
3267
	if (unlikely(is_error_pfn(pfn))) {
3268
		*ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
3269
		return true;
3270 3271
	}

3272
	if (unlikely(is_noslot_pfn(pfn)))
3273 3274
		vcpu_cache_mmio_info(vcpu, gva, gfn, access);

3275
	return false;
3276 3277
}

3278
static bool page_fault_can_be_fast(u32 error_code)
3279
{
3280 3281 3282 3283 3284 3285 3286
	/*
	 * Do not fix the mmio spte with invalid generation number which
	 * need to be updated by slow page fault path.
	 */
	if (unlikely(error_code & PFERR_RSVD_MASK))
		return false;

3287 3288 3289 3290 3291
	/* See if the page fault is due to an NX violation */
	if (unlikely(((error_code & (PFERR_FETCH_MASK | PFERR_PRESENT_MASK))
		      == (PFERR_FETCH_MASK | PFERR_PRESENT_MASK))))
		return false;

3292
	/*
3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303
	 * #PF can be fast if:
	 * 1. The shadow page table entry is not present, which could mean that
	 *    the fault is potentially caused by access tracking (if enabled).
	 * 2. The shadow page table entry is present and the fault
	 *    is caused by write-protect, that means we just need change the W
	 *    bit of the spte which can be done out of mmu-lock.
	 *
	 * However, if access tracking is disabled we know that a non-present
	 * page must be a genuine page fault where we have to create a new SPTE.
	 * So, if access tracking is disabled, we return true only for write
	 * accesses to a present page.
3304 3305
	 */

3306 3307 3308
	return shadow_acc_track_mask != 0 ||
	       ((error_code & (PFERR_WRITE_MASK | PFERR_PRESENT_MASK))
		== (PFERR_WRITE_MASK | PFERR_PRESENT_MASK));
3309 3310
}

3311 3312 3313 3314
/*
 * Returns true if the SPTE was fixed successfully. Otherwise,
 * someone else modified the SPTE from its original value.
 */
3315
static bool
3316
fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
3317
			u64 *sptep, u64 old_spte, u64 new_spte)
3318 3319 3320 3321 3322
{
	gfn_t gfn;

	WARN_ON(!sp->role.direct);

3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334
	/*
	 * Theoretically we could also set dirty bit (and flush TLB) here in
	 * order to eliminate unnecessary PML logging. See comments in
	 * set_spte. But fast_page_fault is very unlikely to happen with PML
	 * enabled, so we do not do this. This might result in the same GPA
	 * to be logged in PML buffer again when the write really happens, and
	 * eventually to be called by mark_page_dirty twice. But it's also no
	 * harm. This also avoids the TLB flush needed after setting dirty bit
	 * so non-PML cases won't be impacted.
	 *
	 * Compare with set_spte where instead shadow_dirty_mask is set.
	 */
3335
	if (cmpxchg64(sptep, old_spte, new_spte) != old_spte)
3336 3337
		return false;

3338
	if (is_writable_pte(new_spte) && !is_writable_pte(old_spte)) {
3339 3340 3341 3342 3343 3344 3345
		/*
		 * The gfn of direct spte is stable since it is
		 * calculated by sp->gfn.
		 */
		gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
		kvm_vcpu_mark_page_dirty(vcpu, gfn);
	}
3346 3347 3348 3349

	return true;
}

3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361
static bool is_access_allowed(u32 fault_err_code, u64 spte)
{
	if (fault_err_code & PFERR_FETCH_MASK)
		return is_executable_pte(spte);

	if (fault_err_code & PFERR_WRITE_MASK)
		return is_writable_pte(spte);

	/* Fault was on Read access */
	return spte & PT_PRESENT_MASK;
}

3362 3363 3364 3365 3366 3367 3368 3369 3370
/*
 * Return value:
 * - true: let the vcpu to access on the same address again.
 * - false: let the real page fault path to fix it.
 */
static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
			    u32 error_code)
{
	struct kvm_shadow_walk_iterator iterator;
3371
	struct kvm_mmu_page *sp;
3372
	bool fault_handled = false;
3373
	u64 spte = 0ull;
3374
	uint retry_count = 0;
3375

3376
	if (!VALID_PAGE(vcpu->arch.mmu->root_hpa))
3377 3378
		return false;

3379
	if (!page_fault_can_be_fast(error_code))
3380 3381 3382 3383
		return false;

	walk_shadow_page_lockless_begin(vcpu);

3384
	do {
3385
		u64 new_spte;
3386

3387 3388 3389 3390 3391
		for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
			if (!is_shadow_present_pte(spte) ||
			    iterator.level < level)
				break;

3392 3393 3394
		sp = page_header(__pa(iterator.sptep));
		if (!is_last_spte(spte, sp->role.level))
			break;
3395

3396
		/*
3397 3398 3399 3400 3401
		 * Check whether the memory access that caused the fault would
		 * still cause it if it were to be performed right now. If not,
		 * then this is a spurious fault caused by TLB lazily flushed,
		 * or some other CPU has already fixed the PTE after the
		 * current CPU took the fault.
3402 3403 3404 3405
		 *
		 * Need not check the access of upper level table entries since
		 * they are always ACC_ALL.
		 */
3406 3407 3408 3409
		if (is_access_allowed(error_code, spte)) {
			fault_handled = true;
			break;
		}
3410

3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424
		new_spte = spte;

		if (is_access_track_spte(spte))
			new_spte = restore_acc_track_spte(new_spte);

		/*
		 * Currently, to simplify the code, write-protection can
		 * be removed in the fast path only if the SPTE was
		 * write-protected for dirty-logging or access tracking.
		 */
		if ((error_code & PFERR_WRITE_MASK) &&
		    spte_can_locklessly_be_made_writable(spte))
		{
			new_spte |= PT_WRITABLE_MASK;
3425 3426

			/*
3427 3428 3429 3430 3431 3432 3433 3434 3435
			 * Do not fix write-permission on the large spte.  Since
			 * we only dirty the first page into the dirty-bitmap in
			 * fast_pf_fix_direct_spte(), other pages are missed
			 * if its slot has dirty logging enabled.
			 *
			 * Instead, we let the slow page fault path create a
			 * normal spte to fix the access.
			 *
			 * See the comments in kvm_arch_commit_memory_region().
3436
			 */
3437
			if (sp->role.level > PT_PAGE_TABLE_LEVEL)
3438
				break;
3439
		}
3440

3441
		/* Verify that the fault can be handled in the fast path */
3442 3443
		if (new_spte == spte ||
		    !is_access_allowed(error_code, new_spte))
3444 3445 3446 3447 3448 3449 3450 3451
			break;

		/*
		 * Currently, fast page fault only works for direct mapping
		 * since the gfn is not stable for indirect shadow page. See
		 * Documentation/virtual/kvm/locking.txt to get more detail.
		 */
		fault_handled = fast_pf_fix_direct_spte(vcpu, sp,
3452
							iterator.sptep, spte,
3453
							new_spte);
3454 3455 3456 3457 3458 3459 3460 3461 3462 3463
		if (fault_handled)
			break;

		if (++retry_count > 4) {
			printk_once(KERN_WARNING
				"kvm: Fast #PF retrying more than 4 times.\n");
			break;
		}

	} while (true);
3464

3465
	trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
3466
			      spte, fault_handled);
3467 3468
	walk_shadow_page_lockless_end(vcpu);

3469
	return fault_handled;
3470 3471
}

3472
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
3473
			 gva_t gva, kvm_pfn_t *pfn, bool write, bool *writable);
3474
static int make_mmu_pages_available(struct kvm_vcpu *vcpu);
3475

3476 3477
static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
			 gfn_t gfn, bool prefault)
3478 3479
{
	int r;
3480
	int level;
3481
	bool force_pt_level = false;
3482
	kvm_pfn_t pfn;
3483
	unsigned long mmu_seq;
3484
	bool map_writable, write = error_code & PFERR_WRITE_MASK;
3485

3486
	level = mapping_level(vcpu, gfn, &force_pt_level);
3487 3488 3489 3490 3491 3492 3493 3494
	if (likely(!force_pt_level)) {
		/*
		 * This path builds a PAE pagetable - so we can map
		 * 2mb pages at maximum. Therefore check if the level
		 * is larger than that.
		 */
		if (level > PT_DIRECTORY_LEVEL)
			level = PT_DIRECTORY_LEVEL;
3495

3496
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3497
	}
3498

3499
	if (fast_page_fault(vcpu, v, level, error_code))
3500
		return RET_PF_RETRY;
3501

3502
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
3503
	smp_rmb();
3504

3505
	if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
3506
		return RET_PF_RETRY;
3507

3508 3509
	if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
		return r;
3510

3511
	spin_lock(&vcpu->kvm->mmu_lock);
3512
	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3513
		goto out_unlock;
3514 3515
	if (make_mmu_pages_available(vcpu) < 0)
		goto out_unlock;
3516 3517
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3518
	r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3519 3520
	spin_unlock(&vcpu->kvm->mmu_lock);

3521
	return r;
3522 3523 3524 3525

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
3526
	return RET_PF_RETRY;
3527 3528
}

3529 3530
static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa,
			       struct list_head *invalid_list)
3531
{
3532
	struct kvm_mmu_page *sp;
3533

3534
	if (!VALID_PAGE(*root_hpa))
3535
		return;
3536

3537 3538 3539 3540
	sp = page_header(*root_hpa & PT64_BASE_ADDR_MASK);
	--sp->root_count;
	if (!sp->root_count && sp->role.invalid)
		kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
3541

3542 3543 3544
	*root_hpa = INVALID_PAGE;
}

3545
/* roots_to_free must be some combination of the KVM_MMU_ROOT_* flags */
3546 3547
void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			ulong roots_to_free)
3548 3549 3550
{
	int i;
	LIST_HEAD(invalid_list);
3551
	bool free_active_root = roots_to_free & KVM_MMU_ROOT_CURRENT;
3552

3553
	BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG);
3554

3555
	/* Before acquiring the MMU lock, see if we need to do any real work. */
3556 3557 3558 3559 3560 3561 3562 3563 3564
	if (!(free_active_root && VALID_PAGE(mmu->root_hpa))) {
		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
			if ((roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) &&
			    VALID_PAGE(mmu->prev_roots[i].hpa))
				break;

		if (i == KVM_MMU_NUM_PREV_ROOTS)
			return;
	}
3565 3566

	spin_lock(&vcpu->kvm->mmu_lock);
3567

3568 3569 3570 3571
	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
		if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i))
			mmu_free_root_page(vcpu->kvm, &mmu->prev_roots[i].hpa,
					   &invalid_list);
3572

3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585
	if (free_active_root) {
		if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL &&
		    (mmu->root_level >= PT64_ROOT_4LEVEL || mmu->direct_map)) {
			mmu_free_root_page(vcpu->kvm, &mmu->root_hpa,
					   &invalid_list);
		} else {
			for (i = 0; i < 4; ++i)
				if (mmu->pae_root[i] != 0)
					mmu_free_root_page(vcpu->kvm,
							   &mmu->pae_root[i],
							   &invalid_list);
			mmu->root_hpa = INVALID_PAGE;
		}
3586
		mmu->root_cr3 = 0;
3587
	}
3588

3589
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3590
	spin_unlock(&vcpu->kvm->mmu_lock);
3591
}
3592
EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);
3593

3594 3595 3596 3597 3598
static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
{
	int ret = 0;

	if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
3599
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3600 3601 3602 3603 3604 3605
		ret = 1;
	}

	return ret;
}

3606 3607 3608
static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_page *sp;
3609
	unsigned i;
3610

3611
	if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) {
3612
		spin_lock(&vcpu->kvm->mmu_lock);
3613 3614
		if(make_mmu_pages_available(vcpu) < 0) {
			spin_unlock(&vcpu->kvm->mmu_lock);
3615
			return -ENOSPC;
3616
		}
3617
		sp = kvm_mmu_get_page(vcpu, 0, 0,
3618
				vcpu->arch.mmu->shadow_root_level, 1, ACC_ALL);
3619 3620
		++sp->root_count;
		spin_unlock(&vcpu->kvm->mmu_lock);
3621 3622
		vcpu->arch.mmu->root_hpa = __pa(sp->spt);
	} else if (vcpu->arch.mmu->shadow_root_level == PT32E_ROOT_LEVEL) {
3623
		for (i = 0; i < 4; ++i) {
3624
			hpa_t root = vcpu->arch.mmu->pae_root[i];
3625

3626
			MMU_WARN_ON(VALID_PAGE(root));
3627
			spin_lock(&vcpu->kvm->mmu_lock);
3628 3629
			if (make_mmu_pages_available(vcpu) < 0) {
				spin_unlock(&vcpu->kvm->mmu_lock);
3630
				return -ENOSPC;
3631
			}
3632
			sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
3633
					i << 30, PT32_ROOT_LEVEL, 1, ACC_ALL);
3634 3635 3636
			root = __pa(sp->spt);
			++sp->root_count;
			spin_unlock(&vcpu->kvm->mmu_lock);
3637
			vcpu->arch.mmu->pae_root[i] = root | PT_PRESENT_MASK;
3638
		}
3639
		vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pae_root);
3640 3641
	} else
		BUG();
3642
	vcpu->arch.mmu->root_cr3 = vcpu->arch.mmu->get_cr3(vcpu);
3643 3644 3645 3646 3647

	return 0;
}

static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3648
{
3649
	struct kvm_mmu_page *sp;
3650
	u64 pdptr, pm_mask;
3651
	gfn_t root_gfn, root_cr3;
3652
	int i;
3653

3654 3655
	root_cr3 = vcpu->arch.mmu->get_cr3(vcpu);
	root_gfn = root_cr3 >> PAGE_SHIFT;
3656

3657 3658 3659 3660 3661 3662 3663
	if (mmu_check_root(vcpu, root_gfn))
		return 1;

	/*
	 * Do we shadow a long mode page table? If so we need to
	 * write-protect the guests page table root.
	 */
3664 3665
	if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) {
		hpa_t root = vcpu->arch.mmu->root_hpa;
3666

3667
		MMU_WARN_ON(VALID_PAGE(root));
3668

3669
		spin_lock(&vcpu->kvm->mmu_lock);
3670 3671
		if (make_mmu_pages_available(vcpu) < 0) {
			spin_unlock(&vcpu->kvm->mmu_lock);
3672
			return -ENOSPC;
3673
		}
3674
		sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
3675
				vcpu->arch.mmu->shadow_root_level, 0, ACC_ALL);
3676 3677
		root = __pa(sp->spt);
		++sp->root_count;
3678
		spin_unlock(&vcpu->kvm->mmu_lock);
3679
		vcpu->arch.mmu->root_hpa = root;
3680
		goto set_root_cr3;
3681
	}
3682

3683 3684
	/*
	 * We shadow a 32 bit page table. This may be a legacy 2-level
3685 3686
	 * or a PAE 3-level page table. In either case we need to be aware that
	 * the shadow page table may be a PAE or a long mode page table.
3687
	 */
3688
	pm_mask = PT_PRESENT_MASK;
3689
	if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL)
3690 3691
		pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;

3692
	for (i = 0; i < 4; ++i) {
3693
		hpa_t root = vcpu->arch.mmu->pae_root[i];
3694

3695
		MMU_WARN_ON(VALID_PAGE(root));
3696 3697
		if (vcpu->arch.mmu->root_level == PT32E_ROOT_LEVEL) {
			pdptr = vcpu->arch.mmu->get_pdptr(vcpu, i);
3698
			if (!(pdptr & PT_PRESENT_MASK)) {
3699
				vcpu->arch.mmu->pae_root[i] = 0;
3700 3701
				continue;
			}
A
Avi Kivity 已提交
3702
			root_gfn = pdptr >> PAGE_SHIFT;
3703 3704
			if (mmu_check_root(vcpu, root_gfn))
				return 1;
3705
		}
3706
		spin_lock(&vcpu->kvm->mmu_lock);
3707 3708
		if (make_mmu_pages_available(vcpu) < 0) {
			spin_unlock(&vcpu->kvm->mmu_lock);
3709
			return -ENOSPC;
3710
		}
3711 3712
		sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL,
				      0, ACC_ALL);
3713 3714
		root = __pa(sp->spt);
		++sp->root_count;
3715 3716
		spin_unlock(&vcpu->kvm->mmu_lock);

3717
		vcpu->arch.mmu->pae_root[i] = root | pm_mask;
3718
	}
3719
	vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pae_root);
3720 3721 3722 3723 3724

	/*
	 * If we shadow a 32 bit page table with a long mode page
	 * table we enter this path.
	 */
3725 3726
	if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) {
		if (vcpu->arch.mmu->lm_root == NULL) {
3727 3728 3729 3730 3731 3732 3733
			/*
			 * The additional page necessary for this is only
			 * allocated on demand.
			 */

			u64 *lm_root;

3734
			lm_root = (void*)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3735 3736 3737
			if (lm_root == NULL)
				return 1;

3738
			lm_root[0] = __pa(vcpu->arch.mmu->pae_root) | pm_mask;
3739

3740
			vcpu->arch.mmu->lm_root = lm_root;
3741 3742
		}

3743
		vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->lm_root);
3744 3745
	}

3746 3747 3748
set_root_cr3:
	vcpu->arch.mmu->root_cr3 = root_cr3;

3749
	return 0;
3750 3751
}

3752 3753
static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
{
3754
	if (vcpu->arch.mmu->direct_map)
3755 3756 3757 3758 3759
		return mmu_alloc_direct_roots(vcpu);
	else
		return mmu_alloc_shadow_roots(vcpu);
}

3760
void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
3761 3762 3763 3764
{
	int i;
	struct kvm_mmu_page *sp;

3765
	if (vcpu->arch.mmu->direct_map)
3766 3767
		return;

3768
	if (!VALID_PAGE(vcpu->arch.mmu->root_hpa))
3769
		return;
3770

3771
	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3772

3773 3774
	if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) {
		hpa_t root = vcpu->arch.mmu->root_hpa;
3775
		sp = page_header(root);
3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793

		/*
		 * Even if another CPU was marking the SP as unsync-ed
		 * simultaneously, any guest page table changes are not
		 * guaranteed to be visible anyway until this VCPU issues a TLB
		 * flush strictly after those changes are made. We only need to
		 * ensure that the other CPU sets these flags before any actual
		 * changes to the page tables are made. The comments in
		 * mmu_need_write_protect() describe what could go wrong if this
		 * requirement isn't satisfied.
		 */
		if (!smp_load_acquire(&sp->unsync) &&
		    !smp_load_acquire(&sp->unsync_children))
			return;

		spin_lock(&vcpu->kvm->mmu_lock);
		kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);

3794
		mmu_sync_children(vcpu, sp);
3795

3796
		kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3797
		spin_unlock(&vcpu->kvm->mmu_lock);
3798 3799
		return;
	}
3800 3801 3802 3803

	spin_lock(&vcpu->kvm->mmu_lock);
	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);

3804
	for (i = 0; i < 4; ++i) {
3805
		hpa_t root = vcpu->arch.mmu->pae_root[i];
3806

3807
		if (root && VALID_PAGE(root)) {
3808 3809 3810 3811 3812 3813
			root &= PT64_BASE_ADDR_MASK;
			sp = page_header(root);
			mmu_sync_children(vcpu, sp);
		}
	}

3814
	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3815
	spin_unlock(&vcpu->kvm->mmu_lock);
3816
}
N
Nadav Har'El 已提交
3817
EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
3818

3819
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
3820
				  u32 access, struct x86_exception *exception)
3821
{
3822 3823
	if (exception)
		exception->error_code = 0;
3824 3825 3826
	return vaddr;
}

3827
static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
3828 3829
					 u32 access,
					 struct x86_exception *exception)
3830
{
3831 3832
	if (exception)
		exception->error_code = 0;
3833
	return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception);
3834 3835
}

3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854
static bool
__is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, u64 pte, int level)
{
	int bit7 = (pte >> 7) & 1, low6 = pte & 0x3f;

	return (pte & rsvd_check->rsvd_bits_mask[bit7][level-1]) |
		((rsvd_check->bad_mt_xwr & (1ull << low6)) != 0);
}

static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
{
	return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level);
}

static bool is_shadow_zero_bits_set(struct kvm_mmu *mmu, u64 spte, int level)
{
	return __is_rsvd_bits_set(&mmu->shadow_zero_check, spte, level);
}

3855
static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3856
{
3857 3858 3859 3860 3861 3862 3863
	/*
	 * A nested guest cannot use the MMIO cache if it is using nested
	 * page tables, because cr2 is a nGPA while the cache stores GPAs.
	 */
	if (mmu_is_nested(vcpu))
		return false;

3864 3865 3866 3867 3868 3869
	if (direct)
		return vcpu_match_mmio_gpa(vcpu, addr);

	return vcpu_match_mmio_gva(vcpu, addr);
}

3870 3871 3872
/* return true if reserved bit is detected on spte. */
static bool
walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
3873 3874
{
	struct kvm_shadow_walk_iterator iterator;
3875
	u64 sptes[PT64_ROOT_MAX_LEVEL], spte = 0ull;
3876 3877
	int root, leaf;
	bool reserved = false;
3878

3879
	if (!VALID_PAGE(vcpu->arch.mmu->root_hpa))
3880
		goto exit;
3881

3882
	walk_shadow_page_lockless_begin(vcpu);
3883

3884 3885
	for (shadow_walk_init(&iterator, vcpu, addr),
		 leaf = root = iterator.level;
3886 3887 3888 3889 3890
	     shadow_walk_okay(&iterator);
	     __shadow_walk_next(&iterator, spte)) {
		spte = mmu_spte_get_lockless(iterator.sptep);

		sptes[leaf - 1] = spte;
3891
		leaf--;
3892

3893 3894
		if (!is_shadow_present_pte(spte))
			break;
3895

3896
		reserved |= is_shadow_zero_bits_set(vcpu->arch.mmu, spte,
3897
						    iterator.level);
3898 3899
	}

3900 3901
	walk_shadow_page_lockless_end(vcpu);

3902 3903 3904
	if (reserved) {
		pr_err("%s: detect reserved bits on spte, addr 0x%llx, dump hierarchy:\n",
		       __func__, addr);
3905
		while (root > leaf) {
3906 3907 3908 3909 3910 3911 3912 3913
			pr_err("------ spte 0x%llx level %d.\n",
			       sptes[root - 1], root);
			root--;
		}
	}
exit:
	*sptep = spte;
	return reserved;
3914 3915
}

3916
static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3917 3918
{
	u64 spte;
3919
	bool reserved;
3920

3921
	if (mmio_info_in_cache(vcpu, addr, direct))
3922
		return RET_PF_EMULATE;
3923

3924
	reserved = walk_shadow_page_get_mmio_spte(vcpu, addr, &spte);
3925
	if (WARN_ON(reserved))
3926
		return -EINVAL;
3927 3928 3929 3930 3931

	if (is_mmio_spte(spte)) {
		gfn_t gfn = get_mmio_spte_gfn(spte);
		unsigned access = get_mmio_spte_access(spte);

3932
		if (!check_mmio_spte(vcpu, spte))
3933
			return RET_PF_INVALID;
3934

3935 3936
		if (direct)
			addr = 0;
3937 3938

		trace_handle_mmio_page_fault(addr, gfn, access);
3939
		vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3940
		return RET_PF_EMULATE;
3941 3942 3943 3944 3945 3946
	}

	/*
	 * If the page table is zapped by other cpus, let CPU fault again on
	 * the address.
	 */
3947
	return RET_PF_RETRY;
3948 3949
}

3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969
static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
					 u32 error_code, gfn_t gfn)
{
	if (unlikely(error_code & PFERR_RSVD_MASK))
		return false;

	if (!(error_code & PFERR_PRESENT_MASK) ||
	      !(error_code & PFERR_WRITE_MASK))
		return false;

	/*
	 * guest is writing the page which is write tracked which can
	 * not be fixed by page fault handler.
	 */
	if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE))
		return true;

	return false;
}

3970 3971 3972 3973 3974
static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
{
	struct kvm_shadow_walk_iterator iterator;
	u64 spte;

3975
	if (!VALID_PAGE(vcpu->arch.mmu->root_hpa))
3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986
		return;

	walk_shadow_page_lockless_begin(vcpu);
	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {
		clear_sp_write_flooding_count(iterator.sptep);
		if (!is_shadow_present_pte(spte))
			break;
	}
	walk_shadow_page_lockless_end(vcpu);
}

3987
static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
3988
				u32 error_code, bool prefault)
3989
{
3990
	gfn_t gfn = gva >> PAGE_SHIFT;
3991
	int r;
3992

3993
	pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
3994

3995
	if (page_fault_handle_page_track(vcpu, error_code, gfn))
3996
		return RET_PF_EMULATE;
3997

3998 3999 4000
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;
4001

4002
	MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa));
4003 4004


4005
	return nonpaging_map(vcpu, gva & PAGE_MASK,
4006
			     error_code, gfn, prefault);
4007 4008
}

4009
static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
4010 4011
{
	struct kvm_arch_async_pf arch;
4012

4013
	arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
4014
	arch.gfn = gfn;
4015 4016
	arch.direct_map = vcpu->arch.mmu->direct_map;
	arch.cr3 = vcpu->arch.mmu->get_cr3(vcpu);
4017

4018
	return kvm_setup_async_pf(vcpu, gva, kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
4019 4020
}

4021
bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
4022
{
4023
	if (unlikely(!lapic_in_kernel(vcpu) ||
4024 4025
		     kvm_event_needs_reinjection(vcpu) ||
		     vcpu->arch.exception.pending))
4026 4027
		return false;

4028
	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
4029 4030
		return false;

4031 4032 4033
	return kvm_x86_ops->interrupt_allowed(vcpu);
}

4034
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
4035
			 gva_t gva, kvm_pfn_t *pfn, bool write, bool *writable)
4036
{
4037
	struct kvm_memory_slot *slot;
4038 4039
	bool async;

4040 4041 4042 4043 4044 4045 4046 4047
	/*
	 * Don't expose private memslots to L2.
	 */
	if (is_guest_mode(vcpu) && !kvm_is_visible_gfn(vcpu->kvm, gfn)) {
		*pfn = KVM_PFN_NOSLOT;
		return false;
	}

4048
	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
4049 4050
	async = false;
	*pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable);
4051 4052 4053
	if (!async)
		return false; /* *pfn has correct page already */

4054
	if (!prefault && kvm_can_do_async_pf(vcpu)) {
4055
		trace_kvm_try_async_get_page(gva, gfn);
4056 4057 4058 4059 4060 4061 4062 4063
		if (kvm_find_async_pf_gfn(vcpu, gfn)) {
			trace_kvm_async_pf_doublefault(gva, gfn);
			kvm_make_request(KVM_REQ_APF_HALT, vcpu);
			return true;
		} else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
			return true;
	}

4064
	*pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, write, writable);
4065 4066 4067
	return false;
}

4068
int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
4069
				u64 fault_address, char *insn, int insn_len)
4070 4071 4072
{
	int r = 1;

4073
	vcpu->arch.l1tf_flush_l1d = true;
4074 4075 4076 4077
	switch (vcpu->arch.apf.host_apf_reason) {
	default:
		trace_kvm_page_fault(fault_address, error_code);

4078
		if (kvm_event_needs_reinjection(vcpu))
4079 4080 4081 4082 4083 4084 4085
			kvm_mmu_unprotect_page_virt(vcpu, fault_address);
		r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
				insn_len);
		break;
	case KVM_PV_REASON_PAGE_NOT_PRESENT:
		vcpu->arch.apf.host_apf_reason = 0;
		local_irq_disable();
4086
		kvm_async_pf_task_wait(fault_address, 0);
4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099
		local_irq_enable();
		break;
	case KVM_PV_REASON_PAGE_READY:
		vcpu->arch.apf.host_apf_reason = 0;
		local_irq_disable();
		kvm_async_pf_task_wake(fault_address);
		local_irq_enable();
		break;
	}
	return r;
}
EXPORT_SYMBOL_GPL(kvm_handle_page_fault);

4100 4101 4102 4103 4104 4105 4106 4107 4108 4109
static bool
check_hugepage_cache_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, int level)
{
	int page_num = KVM_PAGES_PER_HPAGE(level);

	gfn &= ~(page_num - 1);

	return kvm_mtrr_check_gfn_range_consistency(vcpu, gfn, page_num);
}

4110
static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
4111
			  bool prefault)
4112
{
4113
	kvm_pfn_t pfn;
4114
	int r;
4115
	int level;
4116
	bool force_pt_level;
4117
	gfn_t gfn = gpa >> PAGE_SHIFT;
4118
	unsigned long mmu_seq;
4119 4120
	int write = error_code & PFERR_WRITE_MASK;
	bool map_writable;
4121

4122
	MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa));
4123

4124
	if (page_fault_handle_page_track(vcpu, error_code, gfn))
4125
		return RET_PF_EMULATE;
4126

4127 4128 4129 4130
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

4131 4132 4133
	force_pt_level = !check_hugepage_cache_consistency(vcpu, gfn,
							   PT_DIRECTORY_LEVEL);
	level = mapping_level(vcpu, gfn, &force_pt_level);
4134
	if (likely(!force_pt_level)) {
4135 4136 4137
		if (level > PT_DIRECTORY_LEVEL &&
		    !check_hugepage_cache_consistency(vcpu, gfn, level))
			level = PT_DIRECTORY_LEVEL;
4138
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
4139
	}
4140

4141
	if (fast_page_fault(vcpu, gpa, level, error_code))
4142
		return RET_PF_RETRY;
4143

4144
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
4145
	smp_rmb();
4146

4147
	if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
4148
		return RET_PF_RETRY;
4149

4150 4151 4152
	if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
		return r;

4153
	spin_lock(&vcpu->kvm->mmu_lock);
4154
	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
4155
		goto out_unlock;
4156 4157
	if (make_mmu_pages_available(vcpu) < 0)
		goto out_unlock;
4158 4159
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
4160
	r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
4161 4162 4163
	spin_unlock(&vcpu->kvm->mmu_lock);

	return r;
4164 4165 4166 4167

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
4168
	return RET_PF_RETRY;
4169 4170
}

4171 4172
static void nonpaging_init_context(struct kvm_vcpu *vcpu,
				   struct kvm_mmu *context)
4173 4174 4175
{
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
4176
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
4177
	context->invlpg = nonpaging_invlpg;
4178
	context->update_pte = nonpaging_update_pte;
4179
	context->root_level = 0;
4180
	context->shadow_root_level = PT32E_ROOT_LEVEL;
4181
	context->direct_map = true;
4182
	context->nx = false;
4183 4184
}

4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197
/*
 * Find out if a previously cached root matching the new CR3/role is available.
 * The current root is also inserted into the cache.
 * If a matching root was found, it is assigned to kvm_mmu->root_hpa and true is
 * returned.
 * Otherwise, the LRU root from the cache is assigned to kvm_mmu->root_hpa and
 * false is returned. This root should now be freed by the caller.
 */
static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_cr3,
				  union kvm_mmu_page_role new_role)
{
	uint i;
	struct kvm_mmu_root_info root;
4198
	struct kvm_mmu *mmu = vcpu->arch.mmu;
4199

4200
	root.cr3 = mmu->root_cr3;
4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212
	root.hpa = mmu->root_hpa;

	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
		swap(root, mmu->prev_roots[i]);

		if (new_cr3 == root.cr3 && VALID_PAGE(root.hpa) &&
		    page_header(root.hpa) != NULL &&
		    new_role.word == page_header(root.hpa)->role.word)
			break;
	}

	mmu->root_hpa = root.hpa;
4213
	mmu->root_cr3 = root.cr3;
4214 4215 4216 4217

	return i < KVM_MMU_NUM_PREV_ROOTS;
}

4218
static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3,
4219 4220
			    union kvm_mmu_page_role new_role,
			    bool skip_tlb_flush)
4221
{
4222
	struct kvm_mmu *mmu = vcpu->arch.mmu;
4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233

	/*
	 * For now, limit the fast switch to 64-bit hosts+VMs in order to avoid
	 * having to deal with PDPTEs. We may add support for 32-bit hosts/VMs
	 * later if necessary.
	 */
	if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL &&
	    mmu->root_level >= PT64_ROOT_4LEVEL) {
		if (mmu_check_root(vcpu, new_cr3 >> PAGE_SHIFT))
			return false;

4234
		if (cached_root_available(vcpu, new_cr3, new_role)) {
4235
			kvm_make_request(KVM_REQ_LOAD_CR3, vcpu);
4236 4237
			if (!skip_tlb_flush) {
				kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
4238
				kvm_x86_ops->tlb_flush(vcpu, true);
4239 4240 4241 4242 4243 4244 4245 4246 4247 4248
			}

			/*
			 * The last MMIO access's GVA and GPA are cached in the
			 * VCPU. When switching to a new CR3, that GVA->GPA
			 * mapping may no longer be valid. So clear any cached
			 * MMIO info even when we don't need to sync the shadow
			 * page tables.
			 */
			vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
4249

4250 4251 4252 4253 4254 4255 4256 4257
			__clear_sp_write_flooding_count(
				page_header(mmu->root_hpa));

			return true;
		}
	}

	return false;
4258 4259
}

4260
static void __kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3,
4261 4262
			      union kvm_mmu_page_role new_role,
			      bool skip_tlb_flush)
4263
{
4264
	if (!fast_cr3_switch(vcpu, new_cr3, new_role, skip_tlb_flush))
4265 4266
		kvm_mmu_free_roots(vcpu, vcpu->arch.mmu,
				   KVM_MMU_ROOT_CURRENT);
4267 4268
}

4269
void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush)
4270
{
4271 4272
	__kvm_mmu_new_cr3(vcpu, new_cr3, kvm_mmu_calc_root_page_role(vcpu),
			  skip_tlb_flush);
4273
}
4274
EXPORT_SYMBOL_GPL(kvm_mmu_new_cr3);
4275

4276 4277
static unsigned long get_cr3(struct kvm_vcpu *vcpu)
{
4278
	return kvm_read_cr3(vcpu);
4279 4280
}

4281 4282
static void inject_page_fault(struct kvm_vcpu *vcpu,
			      struct x86_exception *fault)
4283
{
4284
	vcpu->arch.mmu->inject_page_fault(vcpu, fault);
4285 4286
}

4287
static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
4288
			   unsigned access, int *nr_present)
4289 4290 4291 4292 4293 4294 4295 4296
{
	if (unlikely(is_mmio_spte(*sptep))) {
		if (gfn != get_mmio_spte_gfn(*sptep)) {
			mmu_spte_clear_no_track(sptep);
			return true;
		}

		(*nr_present)++;
4297
		mark_mmio_spte(vcpu, sptep, gfn, access);
4298 4299 4300 4301 4302 4303
		return true;
	}

	return false;
}

4304 4305
static inline bool is_last_gpte(struct kvm_mmu *mmu,
				unsigned level, unsigned gpte)
4306
{
4307 4308 4309 4310 4311 4312 4313
	/*
	 * The RHS has bit 7 set iff level < mmu->last_nonleaf_level.
	 * If it is clear, there are no large pages at this level, so clear
	 * PT_PAGE_SIZE_MASK in gpte if that is the case.
	 */
	gpte &= level - mmu->last_nonleaf_level;

4314 4315 4316 4317 4318 4319 4320
	/*
	 * PT_PAGE_TABLE_LEVEL always terminates.  The RHS has bit 7 set
	 * iff level <= PT_PAGE_TABLE_LEVEL, which for our purpose means
	 * level == PT_PAGE_TABLE_LEVEL; set PT_PAGE_SIZE_MASK in gpte then.
	 */
	gpte |= level - PT_PAGE_TABLE_LEVEL - 1;

4321
	return gpte & PT_PAGE_SIZE_MASK;
4322 4323
}

4324 4325 4326 4327 4328
#define PTTYPE_EPT 18 /* arbitrary */
#define PTTYPE PTTYPE_EPT
#include "paging_tmpl.h"
#undef PTTYPE

4329 4330 4331 4332 4333 4334 4335 4336
#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

4337 4338 4339 4340
static void
__reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
			struct rsvd_bits_validate *rsvd_check,
			int maxphyaddr, int level, bool nx, bool gbpages,
4341
			bool pse, bool amd)
4342 4343
{
	u64 exb_bit_rsvd = 0;
4344
	u64 gbpages_bit_rsvd = 0;
4345
	u64 nonleaf_bit8_rsvd = 0;
4346

4347
	rsvd_check->bad_mt_xwr = 0;
4348

4349
	if (!nx)
4350
		exb_bit_rsvd = rsvd_bits(63, 63);
4351
	if (!gbpages)
4352
		gbpages_bit_rsvd = rsvd_bits(7, 7);
4353 4354 4355 4356 4357

	/*
	 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
	 * leaf entries) on AMD CPUs only.
	 */
4358
	if (amd)
4359 4360
		nonleaf_bit8_rsvd = rsvd_bits(8, 8);

4361
	switch (level) {
4362 4363
	case PT32_ROOT_LEVEL:
		/* no rsvd bits for 2 level 4K page table entries */
4364 4365 4366 4367
		rsvd_check->rsvd_bits_mask[0][1] = 0;
		rsvd_check->rsvd_bits_mask[0][0] = 0;
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4368

4369
		if (!pse) {
4370
			rsvd_check->rsvd_bits_mask[1][1] = 0;
4371 4372 4373
			break;
		}

4374 4375
		if (is_cpuid_PSE36())
			/* 36bits PSE 4MB page */
4376
			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
4377 4378
		else
			/* 32 bits PSE 4MB page */
4379
			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
4380 4381
		break;
	case PT32E_ROOT_LEVEL:
4382
		rsvd_check->rsvd_bits_mask[0][2] =
4383
			rsvd_bits(maxphyaddr, 63) |
4384
			rsvd_bits(5, 8) | rsvd_bits(1, 2);	/* PDPTE */
4385
		rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
4386
			rsvd_bits(maxphyaddr, 62);	/* PDE */
4387
		rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
4388
			rsvd_bits(maxphyaddr, 62); 	/* PTE */
4389
		rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
4390 4391
			rsvd_bits(maxphyaddr, 62) |
			rsvd_bits(13, 20);		/* large page */
4392 4393
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4394
		break;
4395 4396 4397 4398 4399 4400
	case PT64_ROOT_5LEVEL:
		rsvd_check->rsvd_bits_mask[0][4] = exb_bit_rsvd |
			nonleaf_bit8_rsvd | rsvd_bits(7, 7) |
			rsvd_bits(maxphyaddr, 51);
		rsvd_check->rsvd_bits_mask[1][4] =
			rsvd_check->rsvd_bits_mask[0][4];
4401
		/* fall through */
4402
	case PT64_ROOT_4LEVEL:
4403 4404
		rsvd_check->rsvd_bits_mask[0][3] = exb_bit_rsvd |
			nonleaf_bit8_rsvd | rsvd_bits(7, 7) |
4405
			rsvd_bits(maxphyaddr, 51);
4406 4407
		rsvd_check->rsvd_bits_mask[0][2] = exb_bit_rsvd |
			nonleaf_bit8_rsvd | gbpages_bit_rsvd |
4408
			rsvd_bits(maxphyaddr, 51);
4409 4410 4411 4412 4413 4414 4415
		rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51);
		rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51);
		rsvd_check->rsvd_bits_mask[1][3] =
			rsvd_check->rsvd_bits_mask[0][3];
		rsvd_check->rsvd_bits_mask[1][2] = exb_bit_rsvd |
4416
			gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51) |
4417
			rsvd_bits(13, 29);
4418
		rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
4419 4420
			rsvd_bits(maxphyaddr, 51) |
			rsvd_bits(13, 20);		/* large page */
4421 4422
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4423 4424 4425 4426
		break;
	}
}

4427 4428 4429 4430 4431
static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
{
	__reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check,
				cpuid_maxphyaddr(vcpu), context->root_level,
4432 4433
				context->nx,
				guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),
4434
				is_pse(vcpu), guest_cpuid_is_amd(vcpu));
4435 4436
}

4437 4438 4439
static void
__reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
			    int maxphyaddr, bool execonly)
4440
{
4441
	u64 bad_mt_xwr;
4442

4443 4444
	rsvd_check->rsvd_bits_mask[0][4] =
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7);
4445
	rsvd_check->rsvd_bits_mask[0][3] =
4446
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7);
4447
	rsvd_check->rsvd_bits_mask[0][2] =
4448
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
4449
	rsvd_check->rsvd_bits_mask[0][1] =
4450
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
4451
	rsvd_check->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51);
4452 4453

	/* large page */
4454
	rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];
4455 4456
	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
	rsvd_check->rsvd_bits_mask[1][2] =
4457
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 29);
4458
	rsvd_check->rsvd_bits_mask[1][1] =
4459
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 20);
4460
	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
4461

4462 4463 4464 4465 4466 4467 4468 4469
	bad_mt_xwr = 0xFFull << (2 * 8);	/* bits 3..5 must not be 2 */
	bad_mt_xwr |= 0xFFull << (3 * 8);	/* bits 3..5 must not be 3 */
	bad_mt_xwr |= 0xFFull << (7 * 8);	/* bits 3..5 must not be 7 */
	bad_mt_xwr |= REPEAT_BYTE(1ull << 2);	/* bits 0..2 must not be 010 */
	bad_mt_xwr |= REPEAT_BYTE(1ull << 6);	/* bits 0..2 must not be 110 */
	if (!execonly) {
		/* bits 0..2 must not be 100 unless VMX capabilities allow it */
		bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
4470
	}
4471
	rsvd_check->bad_mt_xwr = bad_mt_xwr;
4472 4473
}

4474 4475 4476 4477 4478 4479 4480
static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
		struct kvm_mmu *context, bool execonly)
{
	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
				    cpuid_maxphyaddr(vcpu), execonly);
}

4481 4482 4483 4484 4485 4486 4487 4488
/*
 * the page table on host is the shadow page table for the page
 * table in guest or amd nested guest, its mmu features completely
 * follow the features in guest.
 */
void
reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
{
4489 4490
	bool uses_nx = context->nx ||
		context->mmu_role.base.smep_andnot_wp;
4491 4492
	struct rsvd_bits_validate *shadow_zero_check;
	int i;
4493

4494 4495 4496 4497
	/*
	 * Passing "true" to the last argument is okay; it adds a check
	 * on bit 8 of the SPTEs which KVM doesn't use anyway.
	 */
4498 4499
	shadow_zero_check = &context->shadow_zero_check;
	__reset_rsvds_bits_mask(vcpu, shadow_zero_check,
4500
				boot_cpu_data.x86_phys_bits,
4501
				context->shadow_root_level, uses_nx,
4502 4503
				guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),
				is_pse(vcpu), true);
4504 4505 4506 4507 4508 4509 4510 4511 4512

	if (!shadow_me_mask)
		return;

	for (i = context->shadow_root_level; --i >= 0;) {
		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
	}

4513 4514 4515
}
EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask);

4516 4517 4518 4519 4520 4521
static inline bool boot_cpu_is_amd(void)
{
	WARN_ON_ONCE(!tdp_enabled);
	return shadow_x_mask == 0;
}

4522 4523 4524 4525 4526 4527 4528 4529
/*
 * the direct page table on host, use as much mmu features as
 * possible, however, kvm currently does not do execution-protection.
 */
static void
reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
				struct kvm_mmu *context)
{
4530 4531 4532 4533 4534
	struct rsvd_bits_validate *shadow_zero_check;
	int i;

	shadow_zero_check = &context->shadow_zero_check;

4535
	if (boot_cpu_is_amd())
4536
		__reset_rsvds_bits_mask(vcpu, shadow_zero_check,
4537 4538
					boot_cpu_data.x86_phys_bits,
					context->shadow_root_level, false,
4539 4540
					boot_cpu_has(X86_FEATURE_GBPAGES),
					true, true);
4541
	else
4542
		__reset_rsvds_bits_mask_ept(shadow_zero_check,
4543 4544 4545
					    boot_cpu_data.x86_phys_bits,
					    false);

4546 4547 4548 4549 4550 4551 4552
	if (!shadow_me_mask)
		return;

	for (i = context->shadow_root_level; --i >= 0;) {
		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
	}
4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566
}

/*
 * as the comments in reset_shadow_zero_bits_mask() except it
 * is the shadow page table for intel nested guest.
 */
static void
reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
				struct kvm_mmu *context, bool execonly)
{
	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
				    boot_cpu_data.x86_phys_bits, execonly);
}

4567 4568 4569 4570 4571 4572 4573 4574 4575 4576
#define BYTE_MASK(access) \
	((1 & (access) ? 2 : 0) | \
	 (2 & (access) ? 4 : 0) | \
	 (3 & (access) ? 8 : 0) | \
	 (4 & (access) ? 16 : 0) | \
	 (5 & (access) ? 32 : 0) | \
	 (6 & (access) ? 64 : 0) | \
	 (7 & (access) ? 128 : 0))


4577 4578
static void update_permission_bitmask(struct kvm_vcpu *vcpu,
				      struct kvm_mmu *mmu, bool ept)
4579
{
4580 4581 4582 4583 4584 4585 4586 4587 4588
	unsigned byte;

	const u8 x = BYTE_MASK(ACC_EXEC_MASK);
	const u8 w = BYTE_MASK(ACC_WRITE_MASK);
	const u8 u = BYTE_MASK(ACC_USER_MASK);

	bool cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP) != 0;
	bool cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP) != 0;
	bool cr0_wp = is_write_protection(vcpu);
4589 4590

	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
4591 4592
		unsigned pfec = byte << 1;

4593
		/*
4594 4595
		 * Each "*f" variable has a 1 bit for each UWX value
		 * that causes a fault with the given PFEC.
4596
		 */
4597

4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628
		/* Faults from writes to non-writable pages */
		u8 wf = (pfec & PFERR_WRITE_MASK) ? ~w : 0;
		/* Faults from user mode accesses to supervisor pages */
		u8 uf = (pfec & PFERR_USER_MASK) ? ~u : 0;
		/* Faults from fetches of non-executable pages*/
		u8 ff = (pfec & PFERR_FETCH_MASK) ? ~x : 0;
		/* Faults from kernel mode fetches of user pages */
		u8 smepf = 0;
		/* Faults from kernel mode accesses of user pages */
		u8 smapf = 0;

		if (!ept) {
			/* Faults from kernel mode accesses to user pages */
			u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u;

			/* Not really needed: !nx will cause pte.nx to fault */
			if (!mmu->nx)
				ff = 0;

			/* Allow supervisor writes if !cr0.wp */
			if (!cr0_wp)
				wf = (pfec & PFERR_USER_MASK) ? wf : 0;

			/* Disallow supervisor fetches of user code if cr4.smep */
			if (cr4_smep)
				smepf = (pfec & PFERR_FETCH_MASK) ? kf : 0;

			/*
			 * SMAP:kernel-mode data accesses from user-mode
			 * mappings should fault. A fault is considered
			 * as a SMAP violation if all of the following
P
Peng Hao 已提交
4629
			 * conditions are true:
4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642
			 *   - X86_CR4_SMAP is set in CR4
			 *   - A user page is accessed
			 *   - The access is not a fetch
			 *   - Page fault in kernel mode
			 *   - if CPL = 3 or X86_EFLAGS_AC is clear
			 *
			 * Here, we cover the first three conditions.
			 * The fourth is computed dynamically in permission_fault();
			 * PFERR_RSVD_MASK bit will be set in PFEC if the access is
			 * *not* subject to SMAP restrictions.
			 */
			if (cr4_smap)
				smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
4643
		}
4644 4645

		mmu->permissions[byte] = ff | uf | wf | smepf | smapf;
4646 4647 4648
	}
}

4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723
/*
* PKU is an additional mechanism by which the paging controls access to
* user-mode addresses based on the value in the PKRU register.  Protection
* key violations are reported through a bit in the page fault error code.
* Unlike other bits of the error code, the PK bit is not known at the
* call site of e.g. gva_to_gpa; it must be computed directly in
* permission_fault based on two bits of PKRU, on some machine state (CR4,
* CR0, EFER, CPL), and on other bits of the error code and the page tables.
*
* In particular the following conditions come from the error code, the
* page tables and the machine state:
* - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
* - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
* - PK is always zero if U=0 in the page tables
* - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
*
* The PKRU bitmask caches the result of these four conditions.  The error
* code (minus the P bit) and the page table's U bit form an index into the
* PKRU bitmask.  Two bits of the PKRU bitmask are then extracted and ANDed
* with the two bits of the PKRU register corresponding to the protection key.
* For the first three conditions above the bits will be 00, thus masking
* away both AD and WD.  For all reads or if the last condition holds, WD
* only will be masked away.
*/
static void update_pkru_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
				bool ept)
{
	unsigned bit;
	bool wp;

	if (ept) {
		mmu->pkru_mask = 0;
		return;
	}

	/* PKEY is enabled only if CR4.PKE and EFER.LMA are both set. */
	if (!kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || !is_long_mode(vcpu)) {
		mmu->pkru_mask = 0;
		return;
	}

	wp = is_write_protection(vcpu);

	for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
		unsigned pfec, pkey_bits;
		bool check_pkey, check_write, ff, uf, wf, pte_user;

		pfec = bit << 1;
		ff = pfec & PFERR_FETCH_MASK;
		uf = pfec & PFERR_USER_MASK;
		wf = pfec & PFERR_WRITE_MASK;

		/* PFEC.RSVD is replaced by ACC_USER_MASK. */
		pte_user = pfec & PFERR_RSVD_MASK;

		/*
		 * Only need to check the access which is not an
		 * instruction fetch and is to a user page.
		 */
		check_pkey = (!ff && pte_user);
		/*
		 * write access is controlled by PKRU if it is a
		 * user access or CR0.WP = 1.
		 */
		check_write = check_pkey && wf && (uf || wp);

		/* PKRU.AD stops both read and write access. */
		pkey_bits = !!check_pkey;
		/* PKRU.WD stops write access. */
		pkey_bits |= (!!check_write) << 1;

		mmu->pkru_mask |= (pkey_bits & 3) << pfec;
	}
}

4724
static void update_last_nonleaf_level(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
4725
{
4726 4727 4728 4729 4730
	unsigned root_level = mmu->root_level;

	mmu->last_nonleaf_level = root_level;
	if (root_level == PT32_ROOT_LEVEL && is_pse(vcpu))
		mmu->last_nonleaf_level++;
4731 4732
}

4733 4734 4735
static void paging64_init_context_common(struct kvm_vcpu *vcpu,
					 struct kvm_mmu *context,
					 int level)
4736
{
4737
	context->nx = is_nx(vcpu);
4738
	context->root_level = level;
4739

4740
	reset_rsvds_bits_mask(vcpu, context);
4741
	update_permission_bitmask(vcpu, context, false);
4742
	update_pkru_bitmask(vcpu, context, false);
4743
	update_last_nonleaf_level(vcpu, context);
4744

4745
	MMU_WARN_ON(!is_pae(vcpu));
4746 4747
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
4748
	context->sync_page = paging64_sync_page;
M
Marcelo Tosatti 已提交
4749
	context->invlpg = paging64_invlpg;
4750
	context->update_pte = paging64_update_pte;
4751
	context->shadow_root_level = level;
4752
	context->direct_map = false;
4753 4754
}

4755 4756
static void paging64_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
4757
{
4758 4759 4760 4761
	int root_level = is_la57_mode(vcpu) ?
			 PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;

	paging64_init_context_common(vcpu, context, root_level);
4762 4763
}

4764 4765
static void paging32_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
4766
{
4767
	context->nx = false;
4768
	context->root_level = PT32_ROOT_LEVEL;
4769

4770
	reset_rsvds_bits_mask(vcpu, context);
4771
	update_permission_bitmask(vcpu, context, false);
4772
	update_pkru_bitmask(vcpu, context, false);
4773
	update_last_nonleaf_level(vcpu, context);
4774 4775 4776

	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
4777
	context->sync_page = paging32_sync_page;
M
Marcelo Tosatti 已提交
4778
	context->invlpg = paging32_invlpg;
4779
	context->update_pte = paging32_update_pte;
4780
	context->shadow_root_level = PT32E_ROOT_LEVEL;
4781
	context->direct_map = false;
4782 4783
}

4784 4785
static void paging32E_init_context(struct kvm_vcpu *vcpu,
				   struct kvm_mmu *context)
4786
{
4787
	paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
4788 4789
}

4790 4791 4792 4793
static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu)
{
	union kvm_mmu_extended_role ext = {0};

4794
	ext.cr0_pg = !!is_paging(vcpu);
4795 4796 4797 4798
	ext.cr4_smep = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
	ext.cr4_smap = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
	ext.cr4_pse = !!is_pse(vcpu);
	ext.cr4_pke = !!kvm_read_cr4_bits(vcpu, X86_CR4_PKE);
4799
	ext.cr4_la57 = !!kvm_read_cr4_bits(vcpu, X86_CR4_LA57);
4800
	ext.maxphyaddr = cpuid_maxphyaddr(vcpu);
4801 4802 4803 4804 4805 4806

	ext.valid = 1;

	return ext;
}

4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827
static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu,
						   bool base_only)
{
	union kvm_mmu_role role = {0};

	role.base.access = ACC_ALL;
	role.base.nxe = !!is_nx(vcpu);
	role.base.cr0_wp = is_write_protection(vcpu);
	role.base.smm = is_smm(vcpu);
	role.base.guest_mode = is_guest_mode(vcpu);

	if (base_only)
		return role;

	role.ext = kvm_calc_mmu_role_ext(vcpu);

	return role;
}

static union kvm_mmu_role
kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only)
4828
{
4829
	union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only);
4830

4831 4832 4833
	role.base.ad_disabled = (shadow_accessed_mask == 0);
	role.base.level = kvm_x86_ops->get_tdp_level(vcpu);
	role.base.direct = true;
4834
	role.base.gpte_is_8_bytes = true;
4835 4836 4837 4838

	return role;
}

4839
static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
4840
{
4841
	struct kvm_mmu *context = vcpu->arch.mmu;
4842 4843
	union kvm_mmu_role new_role =
		kvm_calc_tdp_mmu_root_page_role(vcpu, false);
4844

4845 4846 4847 4848 4849
	new_role.base.word &= mmu_base_role_mask.word;
	if (new_role.as_u64 == context->mmu_role.as_u64)
		return;

	context->mmu_role.as_u64 = new_role.as_u64;
4850
	context->page_fault = tdp_page_fault;
4851
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
4852
	context->invlpg = nonpaging_invlpg;
4853
	context->update_pte = nonpaging_update_pte;
4854
	context->shadow_root_level = kvm_x86_ops->get_tdp_level(vcpu);
4855
	context->direct_map = true;
4856
	context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
4857
	context->get_cr3 = get_cr3;
4858
	context->get_pdptr = kvm_pdptr_read;
4859
	context->inject_page_fault = kvm_inject_page_fault;
4860 4861

	if (!is_paging(vcpu)) {
4862
		context->nx = false;
4863 4864 4865
		context->gva_to_gpa = nonpaging_gva_to_gpa;
		context->root_level = 0;
	} else if (is_long_mode(vcpu)) {
4866
		context->nx = is_nx(vcpu);
4867 4868
		context->root_level = is_la57_mode(vcpu) ?
				PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
4869 4870
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging64_gva_to_gpa;
4871
	} else if (is_pae(vcpu)) {
4872
		context->nx = is_nx(vcpu);
4873
		context->root_level = PT32E_ROOT_LEVEL;
4874 4875
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging64_gva_to_gpa;
4876
	} else {
4877
		context->nx = false;
4878
		context->root_level = PT32_ROOT_LEVEL;
4879 4880
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging32_gva_to_gpa;
4881 4882
	}

4883
	update_permission_bitmask(vcpu, context, false);
4884
	update_pkru_bitmask(vcpu, context, false);
4885
	update_last_nonleaf_level(vcpu, context);
4886
	reset_tdp_shadow_zero_bits_mask(vcpu, context);
4887 4888
}

4889 4890 4891 4892 4893 4894 4895 4896 4897 4898
static union kvm_mmu_role
kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only)
{
	union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only);

	role.base.smep_andnot_wp = role.ext.cr4_smep &&
		!is_write_protection(vcpu);
	role.base.smap_andnot_wp = role.ext.cr4_smap &&
		!is_write_protection(vcpu);
	role.base.direct = !is_paging(vcpu);
4899
	role.base.gpte_is_8_bytes = !!is_pae(vcpu);
4900 4901

	if (!is_long_mode(vcpu))
4902
		role.base.level = PT32E_ROOT_LEVEL;
4903
	else if (is_la57_mode(vcpu))
4904
		role.base.level = PT64_ROOT_5LEVEL;
4905
	else
4906
		role.base.level = PT64_ROOT_4LEVEL;
4907 4908 4909 4910 4911 4912

	return role;
}

void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu)
{
4913
	struct kvm_mmu *context = vcpu->arch.mmu;
4914 4915 4916 4917 4918 4919
	union kvm_mmu_role new_role =
		kvm_calc_shadow_mmu_root_page_role(vcpu, false);

	new_role.base.word &= mmu_base_role_mask.word;
	if (new_role.as_u64 == context->mmu_role.as_u64)
		return;
4920 4921

	if (!is_paging(vcpu))
4922
		nonpaging_init_context(vcpu, context);
4923
	else if (is_long_mode(vcpu))
4924
		paging64_init_context(vcpu, context);
4925
	else if (is_pae(vcpu))
4926
		paging32E_init_context(vcpu, context);
4927
	else
4928
		paging32_init_context(vcpu, context);
4929

4930
	context->mmu_role.as_u64 = new_role.as_u64;
4931
	reset_shadow_zero_bits_mask(vcpu, context);
4932 4933 4934
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);

4935 4936 4937
static union kvm_mmu_role
kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty,
				   bool execonly)
4938
{
4939
	union kvm_mmu_role role = {0};
4940

4941 4942
	/* SMM flag is inherited from root_mmu */
	role.base.smm = vcpu->arch.root_mmu.mmu_role.base.smm;
4943

4944
	role.base.level = PT64_ROOT_4LEVEL;
4945
	role.base.gpte_is_8_bytes = true;
4946 4947 4948 4949
	role.base.direct = false;
	role.base.ad_disabled = !accessed_dirty;
	role.base.guest_mode = true;
	role.base.access = ACC_ALL;
4950

4951 4952 4953 4954 4955 4956 4957
	/*
	 * WP=1 and NOT_WP=1 is an impossible combination, use WP and the
	 * SMAP variation to denote shadow EPT entries.
	 */
	role.base.cr0_wp = true;
	role.base.smap_andnot_wp = true;

4958
	role.ext = kvm_calc_mmu_role_ext(vcpu);
4959
	role.ext.execonly = execonly;
4960 4961 4962 4963

	return role;
}

4964
void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
4965
			     bool accessed_dirty, gpa_t new_eptp)
4966
{
4967
	struct kvm_mmu *context = vcpu->arch.mmu;
4968 4969 4970 4971 4972 4973 4974 4975 4976
	union kvm_mmu_role new_role =
		kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
						   execonly);

	__kvm_mmu_new_cr3(vcpu, new_eptp, new_role.base, false);

	new_role.base.word &= mmu_base_role_mask.word;
	if (new_role.as_u64 == context->mmu_role.as_u64)
		return;
4977

4978
	context->shadow_root_level = PT64_ROOT_4LEVEL;
4979 4980

	context->nx = true;
4981
	context->ept_ad = accessed_dirty;
4982 4983 4984 4985 4986
	context->page_fault = ept_page_fault;
	context->gva_to_gpa = ept_gva_to_gpa;
	context->sync_page = ept_sync_page;
	context->invlpg = ept_invlpg;
	context->update_pte = ept_update_pte;
4987
	context->root_level = PT64_ROOT_4LEVEL;
4988
	context->direct_map = false;
4989
	context->mmu_role.as_u64 = new_role.as_u64;
4990

4991
	update_permission_bitmask(vcpu, context, true);
4992
	update_pkru_bitmask(vcpu, context, true);
4993
	update_last_nonleaf_level(vcpu, context);
4994
	reset_rsvds_bits_mask_ept(vcpu, context, execonly);
4995
	reset_ept_shadow_zero_bits_mask(vcpu, context, execonly);
4996 4997 4998
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);

4999
static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
5000
{
5001
	struct kvm_mmu *context = vcpu->arch.mmu;
5002 5003 5004 5005 5006 5007

	kvm_init_shadow_mmu(vcpu);
	context->set_cr3           = kvm_x86_ops->set_cr3;
	context->get_cr3           = get_cr3;
	context->get_pdptr         = kvm_pdptr_read;
	context->inject_page_fault = kvm_inject_page_fault;
5008 5009
}

5010
static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
5011
{
5012
	union kvm_mmu_role new_role = kvm_calc_mmu_role_common(vcpu, false);
5013 5014
	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;

5015 5016 5017 5018 5019
	new_role.base.word &= mmu_base_role_mask.word;
	if (new_role.as_u64 == g_context->mmu_role.as_u64)
		return;

	g_context->mmu_role.as_u64 = new_role.as_u64;
5020
	g_context->get_cr3           = get_cr3;
5021
	g_context->get_pdptr         = kvm_pdptr_read;
5022 5023 5024
	g_context->inject_page_fault = kvm_inject_page_fault;

	/*
5025
	 * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
5026 5027 5028 5029 5030
	 * L1's nested page tables (e.g. EPT12). The nested translation
	 * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
	 * L2's page tables as the first level of translation and L1's
	 * nested page tables as the second level of translation. Basically
	 * the gva_to_gpa functions between mmu and nested_mmu are swapped.
5031 5032
	 */
	if (!is_paging(vcpu)) {
5033
		g_context->nx = false;
5034 5035 5036
		g_context->root_level = 0;
		g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
	} else if (is_long_mode(vcpu)) {
5037
		g_context->nx = is_nx(vcpu);
5038 5039
		g_context->root_level = is_la57_mode(vcpu) ?
					PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
5040
		reset_rsvds_bits_mask(vcpu, g_context);
5041 5042
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else if (is_pae(vcpu)) {
5043
		g_context->nx = is_nx(vcpu);
5044
		g_context->root_level = PT32E_ROOT_LEVEL;
5045
		reset_rsvds_bits_mask(vcpu, g_context);
5046 5047
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else {
5048
		g_context->nx = false;
5049
		g_context->root_level = PT32_ROOT_LEVEL;
5050
		reset_rsvds_bits_mask(vcpu, g_context);
5051 5052 5053
		g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
	}

5054
	update_permission_bitmask(vcpu, g_context, false);
5055
	update_pkru_bitmask(vcpu, g_context, false);
5056
	update_last_nonleaf_level(vcpu, g_context);
5057 5058
}

5059
void kvm_init_mmu(struct kvm_vcpu *vcpu, bool reset_roots)
5060
{
5061
	if (reset_roots) {
5062 5063
		uint i;

5064
		vcpu->arch.mmu->root_hpa = INVALID_PAGE;
5065 5066

		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
5067
			vcpu->arch.mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
5068 5069
	}

5070
	if (mmu_is_nested(vcpu))
5071
		init_kvm_nested_mmu(vcpu);
5072
	else if (tdp_enabled)
5073
		init_kvm_tdp_mmu(vcpu);
5074
	else
5075
		init_kvm_softmmu(vcpu);
5076
}
5077
EXPORT_SYMBOL_GPL(kvm_init_mmu);
5078

5079 5080 5081
static union kvm_mmu_page_role
kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu)
{
5082 5083
	union kvm_mmu_role role;

5084
	if (tdp_enabled)
5085
		role = kvm_calc_tdp_mmu_root_page_role(vcpu, true);
5086
	else
5087 5088 5089
		role = kvm_calc_shadow_mmu_root_page_role(vcpu, true);

	return role.base;
5090
}
5091

5092
void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
5093
{
5094
	kvm_mmu_unload(vcpu);
5095
	kvm_init_mmu(vcpu, true);
5096
}
5097
EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
5098 5099

int kvm_mmu_load(struct kvm_vcpu *vcpu)
5100
{
5101 5102
	int r;

5103
	r = mmu_topup_memory_caches(vcpu);
5104 5105
	if (r)
		goto out;
5106
	r = mmu_alloc_roots(vcpu);
5107
	kvm_mmu_sync_roots(vcpu);
5108 5109
	if (r)
		goto out;
5110
	kvm_mmu_load_cr3(vcpu);
5111
	kvm_x86_ops->tlb_flush(vcpu, true);
5112 5113
out:
	return r;
5114
}
5115 5116 5117 5118
EXPORT_SYMBOL_GPL(kvm_mmu_load);

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
5119 5120 5121 5122
	kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);
	WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root_hpa));
	kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
	WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root_hpa));
5123
}
5124
EXPORT_SYMBOL_GPL(kvm_mmu_unload);
5125

5126
static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
5127 5128
				  struct kvm_mmu_page *sp, u64 *spte,
				  const void *new)
5129
{
5130
	if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
5131 5132
		++vcpu->kvm->stat.mmu_pde_zapped;
		return;
5133
        }
5134

5135
	++vcpu->kvm->stat.mmu_pte_updated;
5136
	vcpu->arch.mmu->update_pte(vcpu, sp, spte, new);
5137 5138
}

5139 5140 5141 5142 5143 5144 5145 5146
static bool need_remote_flush(u64 old, u64 new)
{
	if (!is_shadow_present_pte(old))
		return false;
	if (!is_shadow_present_pte(new))
		return true;
	if ((old ^ new) & PT64_BASE_ADDR_MASK)
		return true;
5147 5148
	old ^= shadow_nx_mask;
	new ^= shadow_nx_mask;
5149 5150 5151
	return (old & ~new & PT64_PERM_MASK) != 0;
}

5152
static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
5153
				    int *bytes)
5154
{
5155
	u64 gentry = 0;
5156
	int r;
5157 5158 5159

	/*
	 * Assume that the pte write on a page table of the same type
5160 5161
	 * as the current vcpu paging mode since we update the sptes only
	 * when they have the same mode.
5162
	 */
5163
	if (is_pae(vcpu) && *bytes == 4) {
5164
		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */
5165 5166
		*gpa &= ~(gpa_t)7;
		*bytes = 8;
5167 5168
	}

5169 5170 5171 5172
	if (*bytes == 4 || *bytes == 8) {
		r = kvm_vcpu_read_guest_atomic(vcpu, *gpa, &gentry, *bytes);
		if (r)
			gentry = 0;
5173 5174
	}

5175 5176 5177 5178 5179 5180 5181
	return gentry;
}

/*
 * If we're seeing too many writes to a page, it may no longer be a page table,
 * or we may be forking, in which case it is better to unmap the page.
 */
5182
static bool detect_write_flooding(struct kvm_mmu_page *sp)
5183
{
5184 5185 5186 5187
	/*
	 * Skip write-flooding detected for the sp whose level is 1, because
	 * it can become unsync, then the guest page is not write-protected.
	 */
5188
	if (sp->role.level == PT_PAGE_TABLE_LEVEL)
5189
		return false;
5190

5191 5192
	atomic_inc(&sp->write_flooding_count);
	return atomic_read(&sp->write_flooding_count) >= 3;
5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207
}

/*
 * Misaligned accesses are too much trouble to fix up; also, they usually
 * indicate a page is not used as a page table.
 */
static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
				    int bytes)
{
	unsigned offset, pte_size, misaligned;

	pgprintk("misaligned: gpa %llx bytes %d role %x\n",
		 gpa, bytes, sp->role.word);

	offset = offset_in_page(gpa);
5208
	pte_size = sp->role.gpte_is_8_bytes ? 8 : 4;
5209 5210 5211 5212 5213 5214 5215 5216

	/*
	 * Sometimes, the OS only writes the last one bytes to update status
	 * bits, for example, in linux, andb instruction is used in clear_bit().
	 */
	if (!(offset & (pte_size - 1)) && bytes == 1)
		return false;

5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231
	misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
	misaligned |= bytes < 4;

	return misaligned;
}

static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
{
	unsigned page_offset, quadrant;
	u64 *spte;
	int level;

	page_offset = offset_in_page(gpa);
	level = sp->role.level;
	*nspte = 1;
5232
	if (!sp->role.gpte_is_8_bytes) {
5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253
		page_offset <<= 1;	/* 32->64 */
		/*
		 * A 32-bit pde maps 4MB while the shadow pdes map
		 * only 2MB.  So we need to double the offset again
		 * and zap two pdes instead of one.
		 */
		if (level == PT32_ROOT_LEVEL) {
			page_offset &= ~7; /* kill rounding error */
			page_offset <<= 1;
			*nspte = 2;
		}
		quadrant = page_offset >> PAGE_SHIFT;
		page_offset &= ~PAGE_MASK;
		if (quadrant != sp->role.quadrant)
			return NULL;
	}

	spte = &sp->spt[page_offset / sizeof(*spte)];
	return spte;
}

5254
static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
5255 5256
			      const u8 *new, int bytes,
			      struct kvm_page_track_notifier_node *node)
5257 5258 5259 5260 5261 5262
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_mmu_page *sp;
	LIST_HEAD(invalid_list);
	u64 entry, gentry, *spte;
	int npte;
5263
	bool remote_flush, local_flush;
5264 5265 5266 5267 5268

	/*
	 * If we don't have indirect shadow pages, it means no page is
	 * write-protected, so we can exit simply.
	 */
5269
	if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
5270 5271
		return;

5272
	remote_flush = local_flush = false;
5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283

	pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);

	/*
	 * No need to care whether allocation memory is successful
	 * or not since pte prefetch is skiped if it does not have
	 * enough objects in the cache.
	 */
	mmu_topup_memory_caches(vcpu);

	spin_lock(&vcpu->kvm->mmu_lock);
5284 5285 5286

	gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes);

5287
	++vcpu->kvm->stat.mmu_pte_write;
5288
	kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
5289

5290
	for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
5291
		if (detect_write_misaligned(sp, gpa, bytes) ||
5292
		      detect_write_flooding(sp)) {
5293
			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
5294
			++vcpu->kvm->stat.mmu_flooded;
5295 5296
			continue;
		}
5297 5298 5299 5300 5301

		spte = get_written_sptes(sp, gpa, &npte);
		if (!spte)
			continue;

5302
		local_flush = true;
5303
		while (npte--) {
5304 5305
			u32 base_role = vcpu->arch.mmu->mmu_role.base.word;

5306
			entry = *spte;
5307
			mmu_page_zap_pte(vcpu->kvm, sp, spte);
5308
			if (gentry &&
5309
			      !((sp->role.word ^ base_role)
5310
			      & mmu_base_role_mask.word) && rmap_can_add(vcpu))
5311
				mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
5312
			if (need_remote_flush(entry, *spte))
5313
				remote_flush = true;
5314
			++spte;
5315 5316
		}
	}
5317
	kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush);
5318
	kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
5319
	spin_unlock(&vcpu->kvm->mmu_lock);
5320 5321
}

5322 5323
int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
5324 5325
	gpa_t gpa;
	int r;
5326

5327
	if (vcpu->arch.mmu->direct_map)
5328 5329
		return 0;

5330
	gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
5331 5332

	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
5333

5334
	return r;
5335
}
5336
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
5337

5338
static int make_mmu_pages_available(struct kvm_vcpu *vcpu)
5339
{
5340
	LIST_HEAD(invalid_list);
5341

5342
	if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES))
5343
		return 0;
5344

5345 5346 5347
	while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) {
		if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list))
			break;
5348

5349
		++vcpu->kvm->stat.mmu_recycled;
5350
	}
5351
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
5352 5353 5354 5355

	if (!kvm_mmu_available_pages(vcpu->kvm))
		return -ENOSPC;
	return 0;
5356 5357
}

5358
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code,
5359
		       void *insn, int insn_len)
5360
{
5361
	int r, emulation_type = 0;
5362
	enum emulation_result er;
5363
	bool direct = vcpu->arch.mmu->direct_map;
5364

5365
	/* With shadow page tables, fault_address contains a GVA or nGPA.  */
5366
	if (vcpu->arch.mmu->direct_map) {
5367 5368 5369
		vcpu->arch.gpa_available = true;
		vcpu->arch.gpa_val = cr2;
	}
5370

5371
	r = RET_PF_INVALID;
5372 5373
	if (unlikely(error_code & PFERR_RSVD_MASK)) {
		r = handle_mmio_page_fault(vcpu, cr2, direct);
5374
		if (r == RET_PF_EMULATE)
5375 5376
			goto emulate;
	}
5377

5378
	if (r == RET_PF_INVALID) {
5379 5380 5381
		r = vcpu->arch.mmu->page_fault(vcpu, cr2,
					       lower_32_bits(error_code),
					       false);
5382 5383 5384 5385 5386
		WARN_ON(r == RET_PF_INVALID);
	}

	if (r == RET_PF_RETRY)
		return 1;
5387
	if (r < 0)
5388
		return r;
5389

5390 5391 5392 5393 5394 5395 5396
	/*
	 * Before emulating the instruction, check if the error code
	 * was due to a RO violation while translating the guest page.
	 * This can occur when using nested virtualization with nested
	 * paging in both guests. If true, we simply unprotect the page
	 * and resume the guest.
	 */
5397
	if (vcpu->arch.mmu->direct_map &&
5398
	    (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) {
5399 5400 5401 5402
		kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2));
		return 1;
	}

5403 5404 5405 5406 5407 5408
	/*
	 * vcpu->arch.mmu.page_fault returned RET_PF_EMULATE, but we can still
	 * optimistically try to just unprotect the page and let the processor
	 * re-execute the instruction that caused the page fault.  Do not allow
	 * retrying MMIO emulation, as it's not only pointless but could also
	 * cause us to enter an infinite loop because the processor will keep
5409 5410 5411 5412
	 * faulting on the non-existent MMIO address.  Retrying an instruction
	 * from a nested guest is also pointless and dangerous as we are only
	 * explicitly shadowing L1's page tables, i.e. unprotecting something
	 * for L1 isn't going to magically fix whatever issue cause L2 to fail.
5413
	 */
5414
	if (!mmio_info_in_cache(vcpu, cr2, direct) && !is_guest_mode(vcpu))
5415
		emulation_type = EMULTYPE_ALLOW_RETRY;
5416
emulate:
5417 5418 5419 5420 5421
	/*
	 * On AMD platforms, under certain conditions insn_len may be zero on #NPF.
	 * This can happen if a guest gets a page-fault on data access but the HW
	 * table walker is not able to read the instruction page (e.g instruction
	 * page is not present in memory). In those cases we simply restart the
5422
	 * guest, with the exception of AMD Erratum 1096 which is unrecoverable.
5423
	 */
5424 5425 5426 5427
	if (unlikely(insn && !insn_len)) {
		if (!kvm_x86_ops->need_emulation_on_page_fault(vcpu))
			return 1;
	}
5428

5429
	er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
5430 5431 5432 5433

	switch (er) {
	case EMULATE_DONE:
		return 1;
5434
	case EMULATE_USER_EXIT:
5435
		++vcpu->stat.mmio_exits;
5436
		/* fall through */
5437
	case EMULATE_FAIL:
5438
		return 0;
5439 5440 5441 5442 5443 5444
	default:
		BUG();
	}
}
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);

M
Marcelo Tosatti 已提交
5445 5446
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
5447
	struct kvm_mmu *mmu = vcpu->arch.mmu;
5448
	int i;
5449

5450 5451 5452 5453
	/* INVLPG on a * non-canonical address is a NOP according to the SDM.  */
	if (is_noncanonical_address(gva, vcpu))
		return;

5454
	mmu->invlpg(vcpu, gva, mmu->root_hpa);
5455 5456 5457 5458

	/*
	 * INVLPG is required to invalidate any global mappings for the VA,
	 * irrespective of PCID. Since it would take us roughly similar amount
5459 5460 5461
	 * of work to determine whether any of the prev_root mappings of the VA
	 * is marked global, or to just sync it blindly, so we might as well
	 * just always sync it.
5462
	 *
5463 5464 5465
	 * Mappings not reachable via the current cr3 or the prev_roots will be
	 * synced when switching to that cr3, so nothing needs to be done here
	 * for them.
5466
	 */
5467 5468 5469
	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
		if (VALID_PAGE(mmu->prev_roots[i].hpa))
			mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
5470

5471
	kvm_x86_ops->tlb_flush_gva(vcpu, gva);
M
Marcelo Tosatti 已提交
5472 5473 5474 5475
	++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);

5476 5477
void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
{
5478
	struct kvm_mmu *mmu = vcpu->arch.mmu;
5479
	bool tlb_flush = false;
5480
	uint i;
5481 5482

	if (pcid == kvm_get_active_pcid(vcpu)) {
5483
		mmu->invlpg(vcpu, gva, mmu->root_hpa);
5484
		tlb_flush = true;
5485 5486
	}

5487 5488 5489 5490 5491 5492
	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
		if (VALID_PAGE(mmu->prev_roots[i].hpa) &&
		    pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].cr3)) {
			mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
			tlb_flush = true;
		}
5493
	}
5494

5495 5496 5497
	if (tlb_flush)
		kvm_x86_ops->tlb_flush_gva(vcpu, gva);

5498 5499 5500
	++vcpu->stat.invlpg;

	/*
5501 5502 5503
	 * Mappings not reachable via the current cr3 or the prev_roots will be
	 * synced when switching to that cr3, so nothing needs to be done here
	 * for them.
5504 5505 5506 5507
	 */
}
EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva);

5508 5509 5510 5511 5512 5513
void kvm_enable_tdp(void)
{
	tdp_enabled = true;
}
EXPORT_SYMBOL_GPL(kvm_enable_tdp);

5514 5515 5516 5517 5518 5519
void kvm_disable_tdp(void)
{
	tdp_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_tdp);

5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539

/* The return value indicates if tlb flush on all vcpus is needed. */
typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head);

/* The caller should hold mmu-lock before calling this function. */
static __always_inline bool
slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
			slot_level_handler fn, int start_level, int end_level,
			gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
{
	struct slot_rmap_walk_iterator iterator;
	bool flush = false;

	for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
			end_gfn, &iterator) {
		if (iterator.rmap)
			flush |= fn(kvm, iterator.rmap);

		if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
			if (flush && lock_flush_tlb) {
5540 5541 5542
				kvm_flush_remote_tlbs_with_address(kvm,
						start_gfn,
						iterator.gfn - start_gfn + 1);
5543 5544 5545 5546 5547 5548 5549
				flush = false;
			}
			cond_resched_lock(&kvm->mmu_lock);
		}
	}

	if (flush && lock_flush_tlb) {
5550 5551
		kvm_flush_remote_tlbs_with_address(kvm, start_gfn,
						   end_gfn - start_gfn + 1);
5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592
		flush = false;
	}

	return flush;
}

static __always_inline bool
slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
		  slot_level_handler fn, int start_level, int end_level,
		  bool lock_flush_tlb)
{
	return slot_handle_level_range(kvm, memslot, fn, start_level,
			end_level, memslot->base_gfn,
			memslot->base_gfn + memslot->npages - 1,
			lock_flush_tlb);
}

static __always_inline bool
slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
		      slot_level_handler fn, bool lock_flush_tlb)
{
	return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
				 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
}

static __always_inline bool
slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
			slot_level_handler fn, bool lock_flush_tlb)
{
	return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL + 1,
				 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
}

static __always_inline bool
slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
		 slot_level_handler fn, bool lock_flush_tlb)
{
	return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
				 PT_PAGE_TABLE_LEVEL, lock_flush_tlb);
}

5593 5594
static void free_mmu_pages(struct kvm_vcpu *vcpu)
{
5595 5596
	free_page((unsigned long)vcpu->arch.mmu->pae_root);
	free_page((unsigned long)vcpu->arch.mmu->lm_root);
5597 5598 5599 5600
}

static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
{
5601
	struct page *page;
5602 5603
	int i;

5604 5605 5606
	if (tdp_enabled)
		return 0;

5607 5608 5609 5610 5611
	/*
	 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
	 * Therefore we need to allocate shadow page tables in the first
	 * 4GB of memory, which happens to fit the DMA32 zone.
	 */
5612
	page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32);
5613
	if (!page)
5614 5615
		return -ENOMEM;

5616
	vcpu->arch.mmu->pae_root = page_address(page);
5617
	for (i = 0; i < 4; ++i)
5618
		vcpu->arch.mmu->pae_root[i] = INVALID_PAGE;
5619

5620 5621 5622
	return 0;
}

5623
int kvm_mmu_create(struct kvm_vcpu *vcpu)
5624
{
5625 5626
	uint i;

5627 5628
	vcpu->arch.mmu = &vcpu->arch.root_mmu;
	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
5629

5630
	vcpu->arch.root_mmu.root_hpa = INVALID_PAGE;
5631
	vcpu->arch.root_mmu.root_cr3 = 0;
5632
	vcpu->arch.root_mmu.translate_gpa = translate_gpa;
5633
	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
5634
		vcpu->arch.root_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
5635

5636
	vcpu->arch.guest_mmu.root_hpa = INVALID_PAGE;
5637
	vcpu->arch.guest_mmu.root_cr3 = 0;
5638 5639 5640
	vcpu->arch.guest_mmu.translate_gpa = translate_gpa;
	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
		vcpu->arch.guest_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
5641

5642
	vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
5643
	return alloc_mmu_pages(vcpu);
5644 5645
}

5646
static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
5647 5648
			struct kvm_memory_slot *slot,
			struct kvm_page_track_notifier_node *node)
5649
{
5650 5651 5652 5653 5654
	struct kvm_mmu_page *sp;
	LIST_HEAD(invalid_list);
	unsigned long i;
	bool flush;
	gfn_t gfn;
5655

5656
	spin_lock(&kvm->mmu_lock);
5657

5658 5659
	if (list_empty(&kvm->arch.active_mmu_pages))
		goto out_unlock;
5660

5661
	flush = slot_handle_all_level(kvm, slot, kvm_zap_rmapp, false);
5662

5663 5664
	for (i = 0; i < slot->npages; i++) {
		gfn = slot->base_gfn + i;
5665

5666 5667 5668
		for_each_valid_sp(kvm, sp, gfn) {
			if (sp->gfn != gfn)
				continue;
5669

5670 5671
			kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
		}
5672
		if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
5673 5674
			kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
			flush = false;
5675 5676 5677
			cond_resched_lock(&kvm->mmu_lock);
		}
	}
5678
	kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
5679

5680 5681
out_unlock:
	spin_unlock(&kvm->mmu_lock);
5682 5683
}

5684
void kvm_mmu_init_vm(struct kvm *kvm)
5685
{
5686
	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
5687

5688
	node->track_write = kvm_mmu_pte_write;
5689
	node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;
5690
	kvm_page_track_register_notifier(kvm, node);
5691 5692
}

5693
void kvm_mmu_uninit_vm(struct kvm *kvm)
5694
{
5695
	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
5696

5697
	kvm_page_track_unregister_notifier(kvm, node);
5698 5699
}

5700 5701 5702 5703
void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
5704
	int i;
5705 5706

	spin_lock(&kvm->mmu_lock);
5707 5708 5709 5710 5711 5712 5713 5714 5715
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
		kvm_for_each_memslot(memslot, slots) {
			gfn_t start, end;

			start = max(gfn_start, memslot->base_gfn);
			end = min(gfn_end, memslot->base_gfn + memslot->npages);
			if (start >= end)
				continue;
5716

5717 5718 5719
			slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
						PT_PAGE_TABLE_LEVEL, PT_MAX_HUGEPAGE_LEVEL,
						start, end - 1, true);
5720
		}
5721 5722 5723 5724 5725
	}

	spin_unlock(&kvm->mmu_lock);
}

5726 5727
static bool slot_rmap_write_protect(struct kvm *kvm,
				    struct kvm_rmap_head *rmap_head)
5728
{
5729
	return __rmap_write_protect(kvm, rmap_head, false);
5730 5731
}

5732 5733
void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
				      struct kvm_memory_slot *memslot)
5734
{
5735
	bool flush;
5736

5737
	spin_lock(&kvm->mmu_lock);
5738 5739
	flush = slot_handle_all_level(kvm, memslot, slot_rmap_write_protect,
				      false);
5740
	spin_unlock(&kvm->mmu_lock);
5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755

	/*
	 * kvm_mmu_slot_remove_write_access() and kvm_vm_ioctl_get_dirty_log()
	 * which do tlb flush out of mmu-lock should be serialized by
	 * kvm->slots_lock otherwise tlb flush would be missed.
	 */
	lockdep_assert_held(&kvm->slots_lock);

	/*
	 * We can flush all the TLBs out of the mmu lock without TLB
	 * corruption since we just change the spte from writable to
	 * readonly so that we only need to care the case of changing
	 * spte from present to present (changing the spte from present
	 * to nonpresent will flush all the TLBs immediately), in other
	 * words, the only case we care is mmu_spte_update() where we
W
Wei Yang 已提交
5756
	 * have checked SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE
5757 5758 5759
	 * instead of PT_WRITABLE_MASK, that means it does not depend
	 * on PT_WRITABLE_MASK anymore.
	 */
5760
	if (flush)
5761 5762
		kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn,
			memslot->npages);
5763
}
5764

5765
static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
5766
					 struct kvm_rmap_head *rmap_head)
5767 5768 5769 5770
{
	u64 *sptep;
	struct rmap_iterator iter;
	int need_tlb_flush = 0;
5771
	kvm_pfn_t pfn;
5772 5773
	struct kvm_mmu_page *sp;

5774
restart:
5775
	for_each_rmap_spte(rmap_head, &iter, sptep) {
5776 5777 5778 5779
		sp = page_header(__pa(sptep));
		pfn = spte_to_pfn(*sptep);

		/*
5780 5781 5782 5783 5784
		 * We cannot do huge page mapping for indirect shadow pages,
		 * which are found on the last rmap (level = 1) when not using
		 * tdp; such shadow pages are synced with the page table in
		 * the guest, and the guest page table is using 4K page size
		 * mapping if the indirect sp has level = 1.
5785 5786 5787
		 */
		if (sp->role.direct &&
			!kvm_is_reserved_pfn(pfn) &&
5788
			PageTransCompoundMap(pfn_to_page(pfn))) {
5789
			pte_list_remove(rmap_head, sptep);
5790 5791 5792 5793 5794 5795 5796

			if (kvm_available_flush_tlb_with_range())
				kvm_flush_remote_tlbs_with_address(kvm, sp->gfn,
					KVM_PAGES_PER_HPAGE(sp->role.level));
			else
				need_tlb_flush = 1;

5797 5798
			goto restart;
		}
5799 5800 5801 5802 5803 5804
	}

	return need_tlb_flush;
}

void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
5805
				   const struct kvm_memory_slot *memslot)
5806
{
5807
	/* FIXME: const-ify all uses of struct kvm_memory_slot.  */
5808
	spin_lock(&kvm->mmu_lock);
5809 5810
	slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot,
			 kvm_mmu_zap_collapsible_spte, true);
5811 5812 5813
	spin_unlock(&kvm->mmu_lock);
}

5814 5815 5816
void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
				   struct kvm_memory_slot *memslot)
{
5817
	bool flush;
5818 5819

	spin_lock(&kvm->mmu_lock);
5820
	flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false);
5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831
	spin_unlock(&kvm->mmu_lock);

	lockdep_assert_held(&kvm->slots_lock);

	/*
	 * It's also safe to flush TLBs out of mmu lock here as currently this
	 * function is only used for dirty logging, in which case flushing TLB
	 * out of mmu lock also guarantees no dirty pages will be lost in
	 * dirty_bitmap.
	 */
	if (flush)
5832 5833
		kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn,
				memslot->npages);
5834 5835 5836 5837 5838 5839
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty);

void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
					struct kvm_memory_slot *memslot)
{
5840
	bool flush;
5841 5842

	spin_lock(&kvm->mmu_lock);
5843 5844
	flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect,
					false);
5845 5846 5847 5848 5849 5850
	spin_unlock(&kvm->mmu_lock);

	/* see kvm_mmu_slot_remove_write_access */
	lockdep_assert_held(&kvm->slots_lock);

	if (flush)
5851 5852
		kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn,
				memslot->npages);
5853 5854 5855 5856 5857 5858
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access);

void kvm_mmu_slot_set_dirty(struct kvm *kvm,
			    struct kvm_memory_slot *memslot)
{
5859
	bool flush;
5860 5861

	spin_lock(&kvm->mmu_lock);
5862
	flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false);
5863 5864 5865 5866 5867 5868
	spin_unlock(&kvm->mmu_lock);

	lockdep_assert_held(&kvm->slots_lock);

	/* see kvm_mmu_slot_leaf_clear_dirty */
	if (flush)
5869 5870
		kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn,
				memslot->npages);
5871 5872 5873
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty);

5874
static void __kvm_mmu_zap_all(struct kvm *kvm, bool mmio_only)
5875 5876
{
	struct kvm_mmu_page *sp, *node;
5877
	LIST_HEAD(invalid_list);
5878
	int ign;
5879

5880
	spin_lock(&kvm->mmu_lock);
5881
restart:
5882
	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {
5883
		if (mmio_only && !sp->mmio_cached)
5884
			continue;
5885
		if (sp->role.invalid && sp->root_count)
5886
			continue;
5887
		if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign)) {
5888
			WARN_ON_ONCE(mmio_only);
5889 5890
			goto restart;
		}
5891
		if (cond_resched_lock(&kvm->mmu_lock))
5892 5893 5894
			goto restart;
	}

5895
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
5896 5897 5898
	spin_unlock(&kvm->mmu_lock);
}

5899
void kvm_mmu_zap_all(struct kvm *kvm)
5900
{
5901
	return __kvm_mmu_zap_all(kvm, false);
5902 5903
}

5904
void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
5905
{
5906
	WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
5907

5908
	gen &= MMIO_SPTE_GEN_MASK;
5909

5910
	/*
5911 5912 5913 5914 5915 5916 5917 5918
	 * Generation numbers are incremented in multiples of the number of
	 * address spaces in order to provide unique generations across all
	 * address spaces.  Strip what is effectively the address space
	 * modifier prior to checking for a wrap of the MMIO generation so
	 * that a wrap in any address space is detected.
	 */
	gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1);

5919
	/*
5920
	 * The very rare case: if the MMIO generation number has wrapped,
5921 5922
	 * zap all shadow pages.
	 */
5923
	if (unlikely(gen == 0)) {
5924
		kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n");
5925
		__kvm_mmu_zap_all(kvm, true);
5926
	}
5927 5928
}

5929 5930
static unsigned long
mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
5931 5932
{
	struct kvm *kvm;
5933
	int nr_to_scan = sc->nr_to_scan;
5934
	unsigned long freed = 0;
5935

5936
	spin_lock(&kvm_lock);
5937 5938

	list_for_each_entry(kvm, &vm_list, vm_list) {
5939
		int idx;
5940
		LIST_HEAD(invalid_list);
5941

5942 5943 5944 5945 5946 5947 5948 5949
		/*
		 * Never scan more than sc->nr_to_scan VM instances.
		 * Will not hit this condition practically since we do not try
		 * to shrink more than one VM and it is very unlikely to see
		 * !n_used_mmu_pages so many times.
		 */
		if (!nr_to_scan--)
			break;
5950 5951 5952 5953 5954 5955
		/*
		 * n_used_mmu_pages is accessed without holding kvm->mmu_lock
		 * here. We may skip a VM instance errorneosly, but we do not
		 * want to shrink a VM that only started to populate its MMU
		 * anyway.
		 */
5956
		if (!kvm->arch.n_used_mmu_pages)
5957 5958
			continue;

5959
		idx = srcu_read_lock(&kvm->srcu);
5960 5961
		spin_lock(&kvm->mmu_lock);

5962 5963
		if (prepare_zap_oldest_mmu_page(kvm, &invalid_list))
			freed++;
5964
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
5965

5966
		spin_unlock(&kvm->mmu_lock);
5967
		srcu_read_unlock(&kvm->srcu, idx);
5968

5969 5970 5971 5972 5973
		/*
		 * unfair on small ones
		 * per-vm shrinkers cry out
		 * sadness comes quickly
		 */
5974 5975
		list_move_tail(&kvm->vm_list, &vm_list);
		break;
5976 5977
	}

5978
	spin_unlock(&kvm_lock);
5979 5980 5981 5982 5983 5984
	return freed;
}

static unsigned long
mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
{
5985
	return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
5986 5987 5988
}

static struct shrinker mmu_shrinker = {
5989 5990
	.count_objects = mmu_shrink_count,
	.scan_objects = mmu_shrink_scan,
5991 5992 5993
	.seeks = DEFAULT_SEEKS * 10,
};

5994
static void mmu_destroy_caches(void)
5995
{
5996 5997
	kmem_cache_destroy(pte_list_desc_cache);
	kmem_cache_destroy(mmu_page_header_cache);
5998 5999 6000 6001
}

int kvm_mmu_module_init(void)
{
6002 6003
	int ret = -ENOMEM;

6004 6005 6006 6007 6008 6009 6010 6011 6012 6013
	/*
	 * MMU roles use union aliasing which is, generally speaking, an
	 * undefined behavior. However, we supposedly know how compilers behave
	 * and the current status quo is unlikely to change. Guardians below are
	 * supposed to let us know if the assumption becomes false.
	 */
	BUILD_BUG_ON(sizeof(union kvm_mmu_page_role) != sizeof(u32));
	BUILD_BUG_ON(sizeof(union kvm_mmu_extended_role) != sizeof(u32));
	BUILD_BUG_ON(sizeof(union kvm_mmu_role) != sizeof(u64));

6014
	kvm_mmu_reset_all_pte_masks();
6015

6016 6017
	pte_list_desc_cache = kmem_cache_create("pte_list_desc",
					    sizeof(struct pte_list_desc),
6018
					    0, SLAB_ACCOUNT, NULL);
6019
	if (!pte_list_desc_cache)
6020
		goto out;
6021

6022 6023
	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
6024
						  0, SLAB_ACCOUNT, NULL);
6025
	if (!mmu_page_header_cache)
6026
		goto out;
6027

6028
	if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
6029
		goto out;
6030

6031 6032 6033
	ret = register_shrinker(&mmu_shrinker);
	if (ret)
		goto out;
6034

6035 6036
	return 0;

6037
out:
6038
	mmu_destroy_caches();
6039
	return ret;
6040 6041
}

6042
/*
P
Peng Hao 已提交
6043
 * Calculate mmu pages needed for kvm.
6044
 */
6045
unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm)
6046
{
6047 6048
	unsigned long nr_mmu_pages;
	unsigned long nr_pages = 0;
6049
	struct kvm_memslots *slots;
6050
	struct kvm_memory_slot *memslot;
6051
	int i;
6052

6053 6054
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
6055

6056 6057 6058
		kvm_for_each_memslot(memslot, slots)
			nr_pages += memslot->npages;
	}
6059 6060

	nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
6061
	nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES);
6062 6063 6064 6065

	return nr_mmu_pages;
}

6066 6067
void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
6068
	kvm_mmu_unload(vcpu);
6069 6070
	free_mmu_pages(vcpu);
	mmu_free_memory_caches(vcpu);
6071 6072 6073 6074 6075 6076 6077
}

void kvm_mmu_module_exit(void)
{
	mmu_destroy_caches();
	percpu_counter_destroy(&kvm_total_used_mmu_pages);
	unregister_shrinker(&mmu_shrinker);
6078 6079
	mmu_audit_disable();
}
反馈
建议
客服 返回
顶部