mmu.c 139.3 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
11 12 13 14 15 16 17 18 19
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
A
Avi Kivity 已提交
20

21
#include "irq.h"
22
#include "mmu.h"
23
#include "x86.h"
A
Avi Kivity 已提交
24
#include "kvm_cache_regs.h"
25
#include "cpuid.h"
A
Avi Kivity 已提交
26

27
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
28 29 30 31
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
32 33
#include <linux/moduleparam.h>
#include <linux/export.h>
34
#include <linux/swap.h>
M
Marcelo Tosatti 已提交
35
#include <linux/hugetlb.h>
36
#include <linux/compiler.h>
37
#include <linux/srcu.h>
38
#include <linux/slab.h>
39
#include <linux/sched/signal.h>
40
#include <linux/uaccess.h>
41
#include <linux/hash.h>
42
#include <linux/kern_levels.h>
A
Avi Kivity 已提交
43

A
Avi Kivity 已提交
44 45
#include <asm/page.h>
#include <asm/cmpxchg.h>
46
#include <asm/io.h>
47
#include <asm/vmx.h>
48
#include <asm/kvm_page_track.h>
49
#include "trace.h"
A
Avi Kivity 已提交
50

51 52 53 54 55 56 57
/*
 * When setting this variable to true it enables Two-Dimensional-Paging
 * where the hardware walks 2 page tables:
 * 1. the guest-virtual to guest-physical
 * 2. while doing 1. it walks guest-physical to host-physical
 * If the hardware supports that we don't need to do shadow paging.
 */
58
bool tdp_enabled = false;
59

60 61 62 63
enum {
	AUDIT_PRE_PAGE_FAULT,
	AUDIT_POST_PAGE_FAULT,
	AUDIT_PRE_PTE_WRITE,
64 65 66
	AUDIT_POST_PTE_WRITE,
	AUDIT_PRE_SYNC,
	AUDIT_POST_SYNC
67
};
68

69
#undef MMU_DEBUG
70 71

#ifdef MMU_DEBUG
72 73
static bool dbg = 0;
module_param(dbg, bool, 0644);
74 75 76

#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
77
#define MMU_WARN_ON(x) WARN_ON(x)
78 79 80
#else
#define pgprintk(x...) do { } while (0)
#define rmap_printk(x...) do { } while (0)
81
#define MMU_WARN_ON(x) do { } while (0)
82
#endif
A
Avi Kivity 已提交
83

84 85
#define PTE_PREFETCH_NUM		8

86
#define PT_FIRST_AVAIL_BITS_SHIFT 10
A
Avi Kivity 已提交
87 88 89 90 91
#define PT64_SECOND_AVAIL_BITS_SHIFT 52

#define PT64_LEVEL_BITS 9

#define PT64_LEVEL_SHIFT(level) \
M
Mike Day 已提交
92
		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
A
Avi Kivity 已提交
93 94 95 96 97 98 99 100

#define PT64_INDEX(address, level)\
	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))


#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
M
Mike Day 已提交
101
		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
A
Avi Kivity 已提交
102

103 104 105
#define PT32_LVL_OFFSET_MASK(level) \
	(PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT32_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
106 107 108 109 110

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


111
#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
A
Avi Kivity 已提交
112 113
#define PT64_DIR_BASE_ADDR_MASK \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
114 115 116 117 118 119
#define PT64_LVL_ADDR_MASK(level) \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
#define PT64_LVL_OFFSET_MASK(level) \
	(PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
120 121 122 123

#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
124 125 126
#define PT32_LVL_ADDR_MASK(level) \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
					    * PT32_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
127

128 129
#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
			| shadow_x_mask | shadow_nx_mask)
A
Avi Kivity 已提交
130

131 132 133 134 135
#define ACC_EXEC_MASK    1
#define ACC_WRITE_MASK   PT_WRITABLE_MASK
#define ACC_USER_MASK    PT_USER_MASK
#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)

136 137 138 139
/* The mask for the R/X bits in EPT PTEs */
#define PT64_EPT_READABLE_MASK			0x1ull
#define PT64_EPT_EXECUTABLE_MASK		0x4ull

140 141
#include <trace/events/kvm.h>

142 143 144
#define CREATE_TRACE_POINTS
#include "mmutrace.h"

145 146
#define SPTE_HOST_WRITEABLE	(1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
#define SPTE_MMU_WRITEABLE	(1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
147

148 149
#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)

150 151 152
/* make pte_list_desc fit well in cache line */
#define PTE_LIST_EXT 3

153 154 155
struct pte_list_desc {
	u64 *sptes[PTE_LIST_EXT];
	struct pte_list_desc *more;
156 157
};

158 159 160 161
struct kvm_shadow_walk_iterator {
	u64 addr;
	hpa_t shadow_addr;
	u64 *sptep;
162
	int level;
163 164 165 166 167 168 169 170
	unsigned index;
};

#define for_each_shadow_entry(_vcpu, _addr, _walker)    \
	for (shadow_walk_init(&(_walker), _vcpu, _addr);	\
	     shadow_walk_okay(&(_walker));			\
	     shadow_walk_next(&(_walker)))

171 172 173 174 175 176
#define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)	\
	for (shadow_walk_init(&(_walker), _vcpu, _addr);		\
	     shadow_walk_okay(&(_walker)) &&				\
		({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });	\
	     __shadow_walk_next(&(_walker), spte))

177
static struct kmem_cache *pte_list_desc_cache;
178
static struct kmem_cache *mmu_page_header_cache;
179
static struct percpu_counter kvm_total_used_mmu_pages;
180

S
Sheng Yang 已提交
181 182 183 184 185
static u64 __read_mostly shadow_nx_mask;
static u64 __read_mostly shadow_x_mask;	/* mutual exclusive with nx_mask */
static u64 __read_mostly shadow_user_mask;
static u64 __read_mostly shadow_accessed_mask;
static u64 __read_mostly shadow_dirty_mask;
186
static u64 __read_mostly shadow_mmio_mask;
187
static u64 __read_mostly shadow_mmio_value;
188
static u64 __read_mostly shadow_present_mask;
189

190
/*
191 192 193
 * SPTEs used by MMUs without A/D bits are marked with shadow_acc_track_value.
 * Non-present SPTEs with shadow_acc_track_value set are in place for access
 * tracking.
194 195 196 197 198 199 200 201 202 203 204 205 206 207
 */
static u64 __read_mostly shadow_acc_track_mask;
static const u64 shadow_acc_track_value = SPTE_SPECIAL_MASK;

/*
 * The mask/shift to use for saving the original R/X bits when marking the PTE
 * as not-present for access tracking purposes. We do not save the W bit as the
 * PTEs being access tracked also need to be dirty tracked, so the W bit will be
 * restored only when a write is attempted to the page.
 */
static const u64 shadow_acc_track_saved_bits_mask = PT64_EPT_READABLE_MASK |
						    PT64_EPT_EXECUTABLE_MASK;
static const u64 shadow_acc_track_saved_bits_shift = PT64_SECOND_AVAIL_BITS_SHIFT;

208
static void mmu_spte_set(u64 *sptep, u64 spte);
209
static void mmu_free_roots(struct kvm_vcpu *vcpu);
210

211
void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value)
212
{
213 214
	BUG_ON((mmio_mask & mmio_value) != mmio_value);
	shadow_mmio_value = mmio_value | SPTE_SPECIAL_MASK;
215
	shadow_mmio_mask = mmio_mask | SPTE_SPECIAL_MASK;
216 217 218
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);

219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
{
	return sp->role.ad_disabled;
}

static inline bool spte_ad_enabled(u64 spte)
{
	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	return !(spte & shadow_acc_track_value);
}

static inline u64 spte_shadow_accessed_mask(u64 spte)
{
	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
}

static inline u64 spte_shadow_dirty_mask(u64 spte)
{
	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
}

242 243
static inline bool is_access_track_spte(u64 spte)
{
244
	return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
245 246
}

247
/*
248 249 250 251 252 253 254
 * the low bit of the generation number is always presumed to be zero.
 * This disables mmio caching during memslot updates.  The concept is
 * similar to a seqcount but instead of retrying the access we just punt
 * and ignore the cache.
 *
 * spte bits 3-11 are used as bits 1-9 of the generation number,
 * the bits 52-61 are used as bits 10-19 of the generation number.
255
 */
256
#define MMIO_SPTE_GEN_LOW_SHIFT		2
257 258
#define MMIO_SPTE_GEN_HIGH_SHIFT	52

259 260 261
#define MMIO_GEN_SHIFT			20
#define MMIO_GEN_LOW_SHIFT		10
#define MMIO_GEN_LOW_MASK		((1 << MMIO_GEN_LOW_SHIFT) - 2)
262
#define MMIO_GEN_MASK			((1 << MMIO_GEN_SHIFT) - 1)
263 264 265 266 267

static u64 generation_mmio_spte_mask(unsigned int gen)
{
	u64 mask;

T
Tiejun Chen 已提交
268
	WARN_ON(gen & ~MMIO_GEN_MASK);
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285

	mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT;
	mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT;
	return mask;
}

static unsigned int get_mmio_spte_generation(u64 spte)
{
	unsigned int gen;

	spte &= ~shadow_mmio_mask;

	gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK;
	gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT;
	return gen;
}

286
static unsigned int kvm_current_mmio_generation(struct kvm_vcpu *vcpu)
287
{
288
	return kvm_vcpu_memslots(vcpu)->generation & MMIO_GEN_MASK;
289 290
}

291
static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
292
			   unsigned access)
293
{
294
	unsigned int gen = kvm_current_mmio_generation(vcpu);
295
	u64 mask = generation_mmio_spte_mask(gen);
296

297
	access &= ACC_WRITE_MASK | ACC_USER_MASK;
298
	mask |= shadow_mmio_value | access | gfn << PAGE_SHIFT;
299

300
	trace_mark_mmio_spte(sptep, gfn, access, gen);
301
	mmu_spte_set(sptep, mask);
302 303 304 305
}

static bool is_mmio_spte(u64 spte)
{
306
	return (spte & shadow_mmio_mask) == shadow_mmio_value;
307 308 309 310
}

static gfn_t get_mmio_spte_gfn(u64 spte)
{
T
Tiejun Chen 已提交
311
	u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
312
	return (spte & ~mask) >> PAGE_SHIFT;
313 314 315 316
}

static unsigned get_mmio_spte_access(u64 spte)
{
T
Tiejun Chen 已提交
317
	u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
318
	return (spte & ~mask) & ~PAGE_MASK;
319 320
}

321
static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
D
Dan Williams 已提交
322
			  kvm_pfn_t pfn, unsigned access)
323 324
{
	if (unlikely(is_noslot_pfn(pfn))) {
325
		mark_mmio_spte(vcpu, sptep, gfn, access);
326 327 328 329 330
		return true;
	}

	return false;
}
331

332
static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
333
{
334 335
	unsigned int kvm_gen, spte_gen;

336
	kvm_gen = kvm_current_mmio_generation(vcpu);
337 338 339 340
	spte_gen = get_mmio_spte_generation(spte);

	trace_check_mmio_spte(spte, kvm_gen, spte_gen);
	return likely(kvm_gen == spte_gen);
341 342
}

343 344 345 346 347 348 349
/*
 * Sets the shadow PTE masks used by the MMU.
 *
 * Assumptions:
 *  - Setting either @accessed_mask or @dirty_mask requires setting both
 *  - At least one of @accessed_mask or @acc_track_mask must be set
 */
S
Sheng Yang 已提交
350
void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
351 352
		u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask,
		u64 acc_track_mask)
S
Sheng Yang 已提交
353
{
354 355
	BUG_ON(!dirty_mask != !accessed_mask);
	BUG_ON(!accessed_mask && !acc_track_mask);
356
	BUG_ON(acc_track_mask & shadow_acc_track_value);
357

S
Sheng Yang 已提交
358 359 360 361 362
	shadow_user_mask = user_mask;
	shadow_accessed_mask = accessed_mask;
	shadow_dirty_mask = dirty_mask;
	shadow_nx_mask = nx_mask;
	shadow_x_mask = x_mask;
363
	shadow_present_mask = p_mask;
364
	shadow_acc_track_mask = acc_track_mask;
S
Sheng Yang 已提交
365 366 367
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);

368 369 370 371 372 373 374 375 376 377 378 379
void kvm_mmu_clear_all_pte_masks(void)
{
	shadow_user_mask = 0;
	shadow_accessed_mask = 0;
	shadow_dirty_mask = 0;
	shadow_nx_mask = 0;
	shadow_x_mask = 0;
	shadow_mmio_mask = 0;
	shadow_present_mask = 0;
	shadow_acc_track_mask = 0;
}

A
Avi Kivity 已提交
380 381 382 383 384
static int is_cpuid_PSE36(void)
{
	return 1;
}

385 386
static int is_nx(struct kvm_vcpu *vcpu)
{
387
	return vcpu->arch.efer & EFER_NX;
388 389
}

390 391
static int is_shadow_present_pte(u64 pte)
{
392
	return (pte != 0) && !is_mmio_spte(pte);
393 394
}

M
Marcelo Tosatti 已提交
395 396 397 398 399
static int is_large_pte(u64 pte)
{
	return pte & PT_PAGE_SIZE_MASK;
}

400 401 402 403
static int is_last_spte(u64 pte, int level)
{
	if (level == PT_PAGE_TABLE_LEVEL)
		return 1;
404
	if (is_large_pte(pte))
405 406 407 408
		return 1;
	return 0;
}

409 410 411 412 413
static bool is_executable_pte(u64 spte)
{
	return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
}

D
Dan Williams 已提交
414
static kvm_pfn_t spte_to_pfn(u64 pte)
415
{
416
	return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
417 418
}

419 420 421 422 423 424 425
static gfn_t pse36_gfn_delta(u32 gpte)
{
	int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;

	return (gpte & PT32_DIR_PSE36_MASK) << shift;
}

426
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
427
static void __set_spte(u64 *sptep, u64 spte)
428
{
429
	WRITE_ONCE(*sptep, spte);
430 431
}

432
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
433
{
434
	WRITE_ONCE(*sptep, spte);
435 436 437 438 439 440
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	return xchg(sptep, spte);
}
441 442 443 444 445

static u64 __get_spte_lockless(u64 *sptep)
{
	return ACCESS_ONCE(*sptep);
}
446
#else
447 448 449 450 451 452 453
union split_spte {
	struct {
		u32 spte_low;
		u32 spte_high;
	};
	u64 spte;
};
454

455 456 457 458 459 460 461 462 463 464 465 466
static void count_spte_clear(u64 *sptep, u64 spte)
{
	struct kvm_mmu_page *sp =  page_header(__pa(sptep));

	if (is_shadow_present_pte(spte))
		return;

	/* Ensure the spte is completely set before we increase the count */
	smp_wmb();
	sp->clear_spte_count++;
}

467 468 469
static void __set_spte(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;
470

471 472 473 474 475 476 477 478 479 480 481 482
	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	ssptep->spte_high = sspte.spte_high;

	/*
	 * If we map the spte from nonpresent to present, We should store
	 * the high bits firstly, then set present bit, so cpu can not
	 * fetch this spte while we are setting the spte.
	 */
	smp_wmb();

483
	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
484 485
}

486 487 488 489 490 491 492
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

493
	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
494 495 496 497 498 499 500 501

	/*
	 * If we map the spte from present to nonpresent, we should clear
	 * present bit firstly to avoid vcpu fetch the old high bits.
	 */
	smp_wmb();

	ssptep->spte_high = sspte.spte_high;
502
	count_spte_clear(sptep, spte);
503 504 505 506 507 508 509 510 511 512 513
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte, orig;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	/* xchg acts as a barrier before the setting of the high bits */
	orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
514 515
	orig.spte_high = ssptep->spte_high;
	ssptep->spte_high = sspte.spte_high;
516
	count_spte_clear(sptep, spte);
517 518 519

	return orig.spte;
}
520 521 522 523

/*
 * The idea using the light way get the spte on x86_32 guest is from
 * gup_get_pte(arch/x86/mm/gup.c).
524 525 526 527 528 529 530 531 532 533 534 535 536 537
 *
 * An spte tlb flush may be pending, because kvm_set_pte_rmapp
 * coalesces them and we are running out of the MMU lock.  Therefore
 * we need to protect against in-progress updates of the spte.
 *
 * Reading the spte while an update is in progress may get the old value
 * for the high part of the spte.  The race is fine for a present->non-present
 * change (because the high part of the spte is ignored for non-present spte),
 * but for a present->present change we must reread the spte.
 *
 * All such changes are done in two steps (present->non-present and
 * non-present->present), hence it is enough to count the number of
 * present->non-present updates: if it changed while reading the spte,
 * we might have hit the race.  This is done using clear_spte_count.
538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
 */
static u64 __get_spte_lockless(u64 *sptep)
{
	struct kvm_mmu_page *sp =  page_header(__pa(sptep));
	union split_spte spte, *orig = (union split_spte *)sptep;
	int count;

retry:
	count = sp->clear_spte_count;
	smp_rmb();

	spte.spte_low = orig->spte_low;
	smp_rmb();

	spte.spte_high = orig->spte_high;
	smp_rmb();

	if (unlikely(spte.spte_low != orig->spte_low ||
	      count != sp->clear_spte_count))
		goto retry;

	return spte.spte;
}
561 562
#endif

563
static bool spte_can_locklessly_be_made_writable(u64 spte)
564
{
565 566
	return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
		(SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
567 568
}

569 570
static bool spte_has_volatile_bits(u64 spte)
{
571 572 573
	if (!is_shadow_present_pte(spte))
		return false;

574
	/*
575
	 * Always atomically update spte if it can be updated
576 577 578 579
	 * out of mmu-lock, it can ensure dirty bit is not lost,
	 * also, it can help us to get a stable is_writable_pte()
	 * to ensure tlb flush is not missed.
	 */
580 581
	if (spte_can_locklessly_be_made_writable(spte) ||
	    is_access_track_spte(spte))
582 583
		return true;

584
	if (spte_ad_enabled(spte)) {
585 586 587 588
		if ((spte & shadow_accessed_mask) == 0 ||
	    	    (is_writable_pte(spte) && (spte & shadow_dirty_mask) == 0))
			return true;
	}
589

590
	return false;
591 592
}

593
static bool is_accessed_spte(u64 spte)
594
{
595 596 597 598
	u64 accessed_mask = spte_shadow_accessed_mask(spte);

	return accessed_mask ? spte & accessed_mask
			     : !is_access_track_spte(spte);
599 600
}

601
static bool is_dirty_spte(u64 spte)
602
{
603 604 605
	u64 dirty_mask = spte_shadow_dirty_mask(spte);

	return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK;
606 607
}

608 609 610 611 612 613 614 615 616 617 618 619
/* Rules for using mmu_spte_set:
 * Set the sptep from nonpresent to present.
 * Note: the sptep being assigned *must* be either not present
 * or in a state where the hardware will not attempt to update
 * the spte.
 */
static void mmu_spte_set(u64 *sptep, u64 new_spte)
{
	WARN_ON(is_shadow_present_pte(*sptep));
	__set_spte(sptep, new_spte);
}

620 621 622
/*
 * Update the SPTE (excluding the PFN), but do not track changes in its
 * accessed/dirty status.
623
 */
624
static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte)
625
{
626
	u64 old_spte = *sptep;
627

628
	WARN_ON(!is_shadow_present_pte(new_spte));
629

630 631
	if (!is_shadow_present_pte(old_spte)) {
		mmu_spte_set(sptep, new_spte);
632
		return old_spte;
633
	}
634

635
	if (!spte_has_volatile_bits(old_spte))
636
		__update_clear_spte_fast(sptep, new_spte);
637
	else
638
		old_spte = __update_clear_spte_slow(sptep, new_spte);
639

640 641
	WARN_ON(spte_to_pfn(old_spte) != spte_to_pfn(new_spte));

642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663
	return old_spte;
}

/* Rules for using mmu_spte_update:
 * Update the state bits, it means the mapped pfn is not changed.
 *
 * Whenever we overwrite a writable spte with a read-only one we
 * should flush remote TLBs. Otherwise rmap_write_protect
 * will find a read-only spte, even though the writable spte
 * might be cached on a CPU's TLB, the return value indicates this
 * case.
 *
 * Returns true if the TLB needs to be flushed
 */
static bool mmu_spte_update(u64 *sptep, u64 new_spte)
{
	bool flush = false;
	u64 old_spte = mmu_spte_update_no_track(sptep, new_spte);

	if (!is_shadow_present_pte(old_spte))
		return false;

664 665
	/*
	 * For the spte updated out of mmu-lock is safe, since
666
	 * we always atomically update it, see the comments in
667 668
	 * spte_has_volatile_bits().
	 */
669
	if (spte_can_locklessly_be_made_writable(old_spte) &&
670
	      !is_writable_pte(new_spte))
671
		flush = true;
672

673
	/*
674
	 * Flush TLB when accessed/dirty states are changed in the page tables,
675 676 677
	 * to guarantee consistency between TLB and page tables.
	 */

678 679
	if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) {
		flush = true;
680
		kvm_set_pfn_accessed(spte_to_pfn(old_spte));
681 682 683 684
	}

	if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) {
		flush = true;
685
		kvm_set_pfn_dirty(spte_to_pfn(old_spte));
686
	}
687

688
	return flush;
689 690
}

691 692 693 694
/*
 * Rules for using mmu_spte_clear_track_bits:
 * It sets the sptep from present to nonpresent, and track the
 * state bits, it is used to clear the last level sptep.
695
 * Returns non-zero if the PTE was previously valid.
696 697 698
 */
static int mmu_spte_clear_track_bits(u64 *sptep)
{
D
Dan Williams 已提交
699
	kvm_pfn_t pfn;
700 701 702
	u64 old_spte = *sptep;

	if (!spte_has_volatile_bits(old_spte))
703
		__update_clear_spte_fast(sptep, 0ull);
704
	else
705
		old_spte = __update_clear_spte_slow(sptep, 0ull);
706

707
	if (!is_shadow_present_pte(old_spte))
708 709 710
		return 0;

	pfn = spte_to_pfn(old_spte);
711 712 713 714 715 716

	/*
	 * KVM does not hold the refcount of the page used by
	 * kvm mmu, before reclaiming the page, we should
	 * unmap it from mmu first.
	 */
717
	WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
718

719
	if (is_accessed_spte(old_spte))
720
		kvm_set_pfn_accessed(pfn);
721 722

	if (is_dirty_spte(old_spte))
723
		kvm_set_pfn_dirty(pfn);
724

725 726 727 728 729 730 731 732 733 734
	return 1;
}

/*
 * Rules for using mmu_spte_clear_no_track:
 * Directly clear spte without caring the state bits of sptep,
 * it is used to set the upper level spte.
 */
static void mmu_spte_clear_no_track(u64 *sptep)
{
735
	__update_clear_spte_fast(sptep, 0ull);
736 737
}

738 739 740 741 742
static u64 mmu_spte_get_lockless(u64 *sptep)
{
	return __get_spte_lockless(sptep);
}

743 744
static u64 mark_spte_for_access_track(u64 spte)
{
745
	if (spte_ad_enabled(spte))
746 747
		return spte & ~shadow_accessed_mask;

748
	if (is_access_track_spte(spte))
749 750 751
		return spte;

	/*
752 753 754
	 * Making an Access Tracking PTE will result in removal of write access
	 * from the PTE. So, verify that we will be able to restore the write
	 * access in the fast page fault path later on.
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770
	 */
	WARN_ONCE((spte & PT_WRITABLE_MASK) &&
		  !spte_can_locklessly_be_made_writable(spte),
		  "kvm: Writable SPTE is not locklessly dirty-trackable\n");

	WARN_ONCE(spte & (shadow_acc_track_saved_bits_mask <<
			  shadow_acc_track_saved_bits_shift),
		  "kvm: Access Tracking saved bit locations are not zero\n");

	spte |= (spte & shadow_acc_track_saved_bits_mask) <<
		shadow_acc_track_saved_bits_shift;
	spte &= ~shadow_acc_track_mask;

	return spte;
}

771 772 773 774 775 776 777
/* Restore an acc-track PTE back to a regular PTE */
static u64 restore_acc_track_spte(u64 spte)
{
	u64 new_spte = spte;
	u64 saved_bits = (spte >> shadow_acc_track_saved_bits_shift)
			 & shadow_acc_track_saved_bits_mask;

778
	WARN_ON_ONCE(spte_ad_enabled(spte));
779 780 781 782 783 784 785 786 787 788
	WARN_ON_ONCE(!is_access_track_spte(spte));

	new_spte &= ~shadow_acc_track_mask;
	new_spte &= ~(shadow_acc_track_saved_bits_mask <<
		      shadow_acc_track_saved_bits_shift);
	new_spte |= saved_bits;

	return new_spte;
}

789 790 791 792 793 794 795 796
/* Returns the Accessed status of the PTE and resets it at the same time. */
static bool mmu_spte_age(u64 *sptep)
{
	u64 spte = mmu_spte_get_lockless(sptep);

	if (!is_accessed_spte(spte))
		return false;

797
	if (spte_ad_enabled(spte)) {
798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
		clear_bit((ffs(shadow_accessed_mask) - 1),
			  (unsigned long *)sptep);
	} else {
		/*
		 * Capture the dirty status of the page, so that it doesn't get
		 * lost when the SPTE is marked for access tracking.
		 */
		if (is_writable_pte(spte))
			kvm_set_pfn_dirty(spte_to_pfn(spte));

		spte = mark_spte_for_access_track(spte);
		mmu_spte_update_no_track(sptep, spte);
	}

	return true;
}

815 816
static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
{
817 818 819 820 821
	/*
	 * Prevent page table teardown by making any free-er wait during
	 * kvm_flush_remote_tlbs() IPI to all active vcpus.
	 */
	local_irq_disable();
822

823 824 825 826
	/*
	 * Make sure a following spte read is not reordered ahead of the write
	 * to vcpu->mode.
	 */
827
	smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
828 829 830 831
}

static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
{
832 833 834 835 836
	/*
	 * Make sure the write to vcpu->mode is not reordered in front of
	 * reads to sptes.  If it does, kvm_commit_zap_page() can see us
	 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
	 */
837
	smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
838
	local_irq_enable();
839 840
}

841
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
842
				  struct kmem_cache *base_cache, int min)
843 844 845 846
{
	void *obj;

	if (cache->nobjs >= min)
847
		return 0;
848
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
849
		obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
850
		if (!obj)
851
			return -ENOMEM;
852 853
		cache->objects[cache->nobjs++] = obj;
	}
854
	return 0;
855 856
}

857 858 859 860 861
static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
{
	return cache->nobjs;
}

862 863
static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
				  struct kmem_cache *cache)
864 865
{
	while (mc->nobjs)
866
		kmem_cache_free(cache, mc->objects[--mc->nobjs]);
867 868
}

A
Avi Kivity 已提交
869
static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
870
				       int min)
A
Avi Kivity 已提交
871
{
872
	void *page;
A
Avi Kivity 已提交
873 874 875 876

	if (cache->nobjs >= min)
		return 0;
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
877
		page = (void *)__get_free_page(GFP_KERNEL);
A
Avi Kivity 已提交
878 879
		if (!page)
			return -ENOMEM;
880
		cache->objects[cache->nobjs++] = page;
A
Avi Kivity 已提交
881 882 883 884 885 886 887
	}
	return 0;
}

static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
888
		free_page((unsigned long)mc->objects[--mc->nobjs]);
A
Avi Kivity 已提交
889 890
}

891
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
892
{
893 894
	int r;

895
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
896
				   pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
897 898
	if (r)
		goto out;
899
	r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
900 901
	if (r)
		goto out;
902
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
903
				   mmu_page_header_cache, 4);
904 905
out:
	return r;
906 907 908 909
}

static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
910 911
	mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
				pte_list_desc_cache);
912
	mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
913 914
	mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
				mmu_page_header_cache);
915 916
}

917
static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
918 919 920 921 922 923 924 925
{
	void *p;

	BUG_ON(!mc->nobjs);
	p = mc->objects[--mc->nobjs];
	return p;
}

926
static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
927
{
928
	return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
929 930
}

931
static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
932
{
933
	kmem_cache_free(pte_list_desc_cache, pte_list_desc);
934 935
}

936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
{
	if (!sp->role.direct)
		return sp->gfns[index];

	return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
}

static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
{
	if (sp->role.direct)
		BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
	else
		sp->gfns[index] = gfn;
}

M
Marcelo Tosatti 已提交
952
/*
953 954
 * Return the pointer to the large page information for a given gfn,
 * handling slots that are not large page aligned.
M
Marcelo Tosatti 已提交
955
 */
956 957 958
static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
					      struct kvm_memory_slot *slot,
					      int level)
M
Marcelo Tosatti 已提交
959 960 961
{
	unsigned long idx;

962
	idx = gfn_to_index(gfn, slot->base_gfn, level);
963
	return &slot->arch.lpage_info[level - 2][idx];
M
Marcelo Tosatti 已提交
964 965
}

966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988
static void update_gfn_disallow_lpage_count(struct kvm_memory_slot *slot,
					    gfn_t gfn, int count)
{
	struct kvm_lpage_info *linfo;
	int i;

	for (i = PT_DIRECTORY_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->disallow_lpage += count;
		WARN_ON(linfo->disallow_lpage < 0);
	}
}

void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn)
{
	update_gfn_disallow_lpage_count(slot, gfn, 1);
}

void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn)
{
	update_gfn_disallow_lpage_count(slot, gfn, -1);
}

989
static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
M
Marcelo Tosatti 已提交
990
{
991
	struct kvm_memslots *slots;
992
	struct kvm_memory_slot *slot;
993
	gfn_t gfn;
M
Marcelo Tosatti 已提交
994

995
	kvm->arch.indirect_shadow_pages++;
996
	gfn = sp->gfn;
997 998
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
999 1000 1001 1002 1003 1004

	/* the non-leaf shadow pages are keeping readonly. */
	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return kvm_slot_page_track_add_page(kvm, slot, gfn,
						    KVM_PAGE_TRACK_WRITE);

1005
	kvm_mmu_gfn_disallow_lpage(slot, gfn);
M
Marcelo Tosatti 已提交
1006 1007
}

1008
static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
M
Marcelo Tosatti 已提交
1009
{
1010
	struct kvm_memslots *slots;
1011
	struct kvm_memory_slot *slot;
1012
	gfn_t gfn;
M
Marcelo Tosatti 已提交
1013

1014
	kvm->arch.indirect_shadow_pages--;
1015
	gfn = sp->gfn;
1016 1017
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
1018 1019 1020 1021
	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return kvm_slot_page_track_remove_page(kvm, slot, gfn,
						       KVM_PAGE_TRACK_WRITE);

1022
	kvm_mmu_gfn_allow_lpage(slot, gfn);
M
Marcelo Tosatti 已提交
1023 1024
}

1025 1026
static bool __mmu_gfn_lpage_is_disallowed(gfn_t gfn, int level,
					  struct kvm_memory_slot *slot)
M
Marcelo Tosatti 已提交
1027
{
1028
	struct kvm_lpage_info *linfo;
M
Marcelo Tosatti 已提交
1029 1030

	if (slot) {
1031
		linfo = lpage_info_slot(gfn, slot, level);
1032
		return !!linfo->disallow_lpage;
M
Marcelo Tosatti 已提交
1033 1034
	}

1035
	return true;
M
Marcelo Tosatti 已提交
1036 1037
}

1038 1039
static bool mmu_gfn_lpage_is_disallowed(struct kvm_vcpu *vcpu, gfn_t gfn,
					int level)
1040 1041 1042 1043
{
	struct kvm_memory_slot *slot;

	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1044
	return __mmu_gfn_lpage_is_disallowed(gfn, level, slot);
1045 1046
}

1047
static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
M
Marcelo Tosatti 已提交
1048
{
J
Joerg Roedel 已提交
1049
	unsigned long page_size;
1050
	int i, ret = 0;
M
Marcelo Tosatti 已提交
1051

J
Joerg Roedel 已提交
1052
	page_size = kvm_host_page_size(kvm, gfn);
M
Marcelo Tosatti 已提交
1053

1054
	for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
1055 1056 1057 1058 1059 1060
		if (page_size >= KVM_HPAGE_SIZE(i))
			ret = i;
		else
			break;
	}

1061
	return ret;
M
Marcelo Tosatti 已提交
1062 1063
}

1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
static inline bool memslot_valid_for_gpte(struct kvm_memory_slot *slot,
					  bool no_dirty_log)
{
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
		return false;
	if (no_dirty_log && slot->dirty_bitmap)
		return false;

	return true;
}

1075 1076 1077
static struct kvm_memory_slot *
gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
			    bool no_dirty_log)
M
Marcelo Tosatti 已提交
1078 1079
{
	struct kvm_memory_slot *slot;
1080

1081
	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1082
	if (!memslot_valid_for_gpte(slot, no_dirty_log))
1083 1084 1085 1086 1087
		slot = NULL;

	return slot;
}

1088 1089
static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn,
			 bool *force_pt_level)
1090 1091
{
	int host_level, level, max_level;
1092 1093
	struct kvm_memory_slot *slot;

1094 1095
	if (unlikely(*force_pt_level))
		return PT_PAGE_TABLE_LEVEL;
M
Marcelo Tosatti 已提交
1096

1097 1098
	slot = kvm_vcpu_gfn_to_memslot(vcpu, large_gfn);
	*force_pt_level = !memslot_valid_for_gpte(slot, true);
1099 1100 1101
	if (unlikely(*force_pt_level))
		return PT_PAGE_TABLE_LEVEL;

1102 1103 1104 1105 1106
	host_level = host_mapping_level(vcpu->kvm, large_gfn);

	if (host_level == PT_PAGE_TABLE_LEVEL)
		return host_level;

X
Xiao Guangrong 已提交
1107
	max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
1108 1109

	for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
1110
		if (__mmu_gfn_lpage_is_disallowed(large_gfn, level, slot))
1111 1112 1113
			break;

	return level - 1;
M
Marcelo Tosatti 已提交
1114 1115
}

1116
/*
1117
 * About rmap_head encoding:
1118
 *
1119 1120
 * If the bit zero of rmap_head->val is clear, then it points to the only spte
 * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
1121
 * pte_list_desc containing more mappings.
1122 1123 1124 1125
 */

/*
 * Returns the number of pointers in the rmap chain, not counting the new one.
1126
 */
1127
static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
1128
			struct kvm_rmap_head *rmap_head)
1129
{
1130
	struct pte_list_desc *desc;
1131
	int i, count = 0;
1132

1133
	if (!rmap_head->val) {
1134
		rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
1135 1136
		rmap_head->val = (unsigned long)spte;
	} else if (!(rmap_head->val & 1)) {
1137 1138
		rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
		desc = mmu_alloc_pte_list_desc(vcpu);
1139
		desc->sptes[0] = (u64 *)rmap_head->val;
A
Avi Kivity 已提交
1140
		desc->sptes[1] = spte;
1141
		rmap_head->val = (unsigned long)desc | 1;
1142
		++count;
1143
	} else {
1144
		rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
1145
		desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1146
		while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
1147
			desc = desc->more;
1148
			count += PTE_LIST_EXT;
1149
		}
1150 1151
		if (desc->sptes[PTE_LIST_EXT-1]) {
			desc->more = mmu_alloc_pte_list_desc(vcpu);
1152 1153
			desc = desc->more;
		}
A
Avi Kivity 已提交
1154
		for (i = 0; desc->sptes[i]; ++i)
1155
			++count;
A
Avi Kivity 已提交
1156
		desc->sptes[i] = spte;
1157
	}
1158
	return count;
1159 1160
}

1161
static void
1162 1163 1164
pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
			   struct pte_list_desc *desc, int i,
			   struct pte_list_desc *prev_desc)
1165 1166 1167
{
	int j;

1168
	for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
1169
		;
A
Avi Kivity 已提交
1170 1171
	desc->sptes[i] = desc->sptes[j];
	desc->sptes[j] = NULL;
1172 1173 1174
	if (j != 0)
		return;
	if (!prev_desc && !desc->more)
1175
		rmap_head->val = (unsigned long)desc->sptes[0];
1176 1177 1178 1179
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
1180
			rmap_head->val = (unsigned long)desc->more | 1;
1181
	mmu_free_pte_list_desc(desc);
1182 1183
}

1184
static void pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
1185
{
1186 1187
	struct pte_list_desc *desc;
	struct pte_list_desc *prev_desc;
1188 1189
	int i;

1190
	if (!rmap_head->val) {
1191
		printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
1192
		BUG();
1193
	} else if (!(rmap_head->val & 1)) {
1194
		rmap_printk("pte_list_remove:  %p 1->0\n", spte);
1195
		if ((u64 *)rmap_head->val != spte) {
1196
			printk(KERN_ERR "pte_list_remove:  %p 1->BUG\n", spte);
1197 1198
			BUG();
		}
1199
		rmap_head->val = 0;
1200
	} else {
1201
		rmap_printk("pte_list_remove:  %p many->many\n", spte);
1202
		desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1203 1204
		prev_desc = NULL;
		while (desc) {
1205
			for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) {
A
Avi Kivity 已提交
1206
				if (desc->sptes[i] == spte) {
1207 1208
					pte_list_desc_remove_entry(rmap_head,
							desc, i, prev_desc);
1209 1210
					return;
				}
1211
			}
1212 1213 1214
			prev_desc = desc;
			desc = desc->more;
		}
1215
		pr_err("pte_list_remove: %p many->many\n", spte);
1216 1217 1218 1219
		BUG();
	}
}

1220 1221
static struct kvm_rmap_head *__gfn_to_rmap(gfn_t gfn, int level,
					   struct kvm_memory_slot *slot)
1222
{
1223
	unsigned long idx;
1224

1225
	idx = gfn_to_index(gfn, slot->base_gfn, level);
1226
	return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
1227 1228
}

1229 1230
static struct kvm_rmap_head *gfn_to_rmap(struct kvm *kvm, gfn_t gfn,
					 struct kvm_mmu_page *sp)
1231
{
1232
	struct kvm_memslots *slots;
1233 1234
	struct kvm_memory_slot *slot;

1235 1236
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
1237
	return __gfn_to_rmap(gfn, sp->role.level, slot);
1238 1239
}

1240 1241 1242 1243 1244 1245 1246 1247
static bool rmap_can_add(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_memory_cache *cache;

	cache = &vcpu->arch.mmu_pte_list_desc_cache;
	return mmu_memory_cache_free_objects(cache);
}

1248 1249 1250
static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
{
	struct kvm_mmu_page *sp;
1251
	struct kvm_rmap_head *rmap_head;
1252 1253 1254

	sp = page_header(__pa(spte));
	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1255 1256
	rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
	return pte_list_add(vcpu, spte, rmap_head);
1257 1258 1259 1260 1261 1262
}

static void rmap_remove(struct kvm *kvm, u64 *spte)
{
	struct kvm_mmu_page *sp;
	gfn_t gfn;
1263
	struct kvm_rmap_head *rmap_head;
1264 1265 1266

	sp = page_header(__pa(spte));
	gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1267 1268
	rmap_head = gfn_to_rmap(kvm, gfn, sp);
	pte_list_remove(spte, rmap_head);
1269 1270
}

1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
/*
 * Used by the following functions to iterate through the sptes linked by a
 * rmap.  All fields are private and not assumed to be used outside.
 */
struct rmap_iterator {
	/* private fields */
	struct pte_list_desc *desc;	/* holds the sptep if not NULL */
	int pos;			/* index of the sptep */
};

/*
 * Iteration must be started by this function.  This should also be used after
 * removing/dropping sptes from the rmap link because in such cases the
 * information in the itererator may not be valid.
 *
 * Returns sptep if found, NULL otherwise.
 */
1288 1289
static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
			   struct rmap_iterator *iter)
1290
{
1291 1292
	u64 *sptep;

1293
	if (!rmap_head->val)
1294 1295
		return NULL;

1296
	if (!(rmap_head->val & 1)) {
1297
		iter->desc = NULL;
1298 1299
		sptep = (u64 *)rmap_head->val;
		goto out;
1300 1301
	}

1302
	iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1303
	iter->pos = 0;
1304 1305 1306 1307
	sptep = iter->desc->sptes[iter->pos];
out:
	BUG_ON(!is_shadow_present_pte(*sptep));
	return sptep;
1308 1309 1310 1311 1312 1313 1314 1315 1316
}

/*
 * Must be used with a valid iterator: e.g. after rmap_get_first().
 *
 * Returns sptep if found, NULL otherwise.
 */
static u64 *rmap_get_next(struct rmap_iterator *iter)
{
1317 1318
	u64 *sptep;

1319 1320 1321 1322 1323
	if (iter->desc) {
		if (iter->pos < PTE_LIST_EXT - 1) {
			++iter->pos;
			sptep = iter->desc->sptes[iter->pos];
			if (sptep)
1324
				goto out;
1325 1326 1327 1328 1329 1330 1331
		}

		iter->desc = iter->desc->more;

		if (iter->desc) {
			iter->pos = 0;
			/* desc->sptes[0] cannot be NULL */
1332 1333
			sptep = iter->desc->sptes[iter->pos];
			goto out;
1334 1335 1336 1337
		}
	}

	return NULL;
1338 1339 1340
out:
	BUG_ON(!is_shadow_present_pte(*sptep));
	return sptep;
1341 1342
}

1343 1344
#define for_each_rmap_spte(_rmap_head_, _iter_, _spte_)			\
	for (_spte_ = rmap_get_first(_rmap_head_, _iter_);		\
1345
	     _spte_; _spte_ = rmap_get_next(_iter_))
1346

1347
static void drop_spte(struct kvm *kvm, u64 *sptep)
1348
{
1349
	if (mmu_spte_clear_track_bits(sptep))
1350
		rmap_remove(kvm, sptep);
A
Avi Kivity 已提交
1351 1352
}

1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373

static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
{
	if (is_large_pte(*sptep)) {
		WARN_ON(page_header(__pa(sptep))->role.level ==
			PT_PAGE_TABLE_LEVEL);
		drop_spte(kvm, sptep);
		--kvm->stat.lpages;
		return true;
	}

	return false;
}

static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
{
	if (__drop_large_spte(vcpu->kvm, sptep))
		kvm_flush_remote_tlbs(vcpu->kvm);
}

/*
1374
 * Write-protect on the specified @sptep, @pt_protect indicates whether
1375
 * spte write-protection is caused by protecting shadow page table.
1376
 *
T
Tiejun Chen 已提交
1377
 * Note: write protection is difference between dirty logging and spte
1378 1379 1380 1381 1382
 * protection:
 * - for dirty logging, the spte can be set to writable at anytime if
 *   its dirty bitmap is properly set.
 * - for spte protection, the spte can be writable only after unsync-ing
 *   shadow page.
1383
 *
1384
 * Return true if tlb need be flushed.
1385
 */
1386
static bool spte_write_protect(u64 *sptep, bool pt_protect)
1387 1388 1389
{
	u64 spte = *sptep;

1390
	if (!is_writable_pte(spte) &&
1391
	      !(pt_protect && spte_can_locklessly_be_made_writable(spte)))
1392 1393 1394 1395
		return false;

	rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);

1396 1397
	if (pt_protect)
		spte &= ~SPTE_MMU_WRITEABLE;
1398
	spte = spte & ~PT_WRITABLE_MASK;
1399

1400
	return mmu_spte_update(sptep, spte);
1401 1402
}

1403 1404
static bool __rmap_write_protect(struct kvm *kvm,
				 struct kvm_rmap_head *rmap_head,
1405
				 bool pt_protect)
1406
{
1407 1408
	u64 *sptep;
	struct rmap_iterator iter;
1409
	bool flush = false;
1410

1411
	for_each_rmap_spte(rmap_head, &iter, sptep)
1412
		flush |= spte_write_protect(sptep, pt_protect);
1413

1414
	return flush;
1415 1416
}

1417
static bool spte_clear_dirty(u64 *sptep)
1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
{
	u64 spte = *sptep;

	rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);

	spte &= ~shadow_dirty_mask;

	return mmu_spte_update(sptep, spte);
}

1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443
static bool wrprot_ad_disabled_spte(u64 *sptep)
{
	bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT,
					       (unsigned long *)sptep);
	if (was_writable)
		kvm_set_pfn_dirty(spte_to_pfn(*sptep));

	return was_writable;
}

/*
 * Gets the GFN ready for another round of dirty logging by clearing the
 *	- D bit on ad-enabled SPTEs, and
 *	- W bit on ad-disabled SPTEs.
 * Returns true iff any D or W bits were cleared.
 */
1444
static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1445 1446 1447 1448 1449
{
	u64 *sptep;
	struct rmap_iterator iter;
	bool flush = false;

1450
	for_each_rmap_spte(rmap_head, &iter, sptep)
1451 1452 1453 1454
		if (spte_ad_enabled(*sptep))
			flush |= spte_clear_dirty(sptep);
		else
			flush |= wrprot_ad_disabled_spte(sptep);
1455 1456 1457 1458

	return flush;
}

1459
static bool spte_set_dirty(u64 *sptep)
1460 1461 1462 1463 1464 1465 1466 1467 1468 1469
{
	u64 spte = *sptep;

	rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);

	spte |= shadow_dirty_mask;

	return mmu_spte_update(sptep, spte);
}

1470
static bool __rmap_set_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1471 1472 1473 1474 1475
{
	u64 *sptep;
	struct rmap_iterator iter;
	bool flush = false;

1476
	for_each_rmap_spte(rmap_head, &iter, sptep)
1477 1478
		if (spte_ad_enabled(*sptep))
			flush |= spte_set_dirty(sptep);
1479 1480 1481 1482

	return flush;
}

1483
/**
1484
 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1485 1486 1487 1488 1489 1490 1491 1492
 * @kvm: kvm instance
 * @slot: slot to protect
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should protect
 *
 * Used when we do not need to care about huge page mappings: e.g. during dirty
 * logging we do not have any such mappings.
 */
1493
static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1494 1495
				     struct kvm_memory_slot *slot,
				     gfn_t gfn_offset, unsigned long mask)
1496
{
1497
	struct kvm_rmap_head *rmap_head;
1498

1499
	while (mask) {
1500 1501 1502
		rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
					  PT_PAGE_TABLE_LEVEL, slot);
		__rmap_write_protect(kvm, rmap_head, false);
M
Marcelo Tosatti 已提交
1503

1504 1505 1506
		/* clear the first set bit */
		mask &= mask - 1;
	}
1507 1508
}

1509
/**
1510 1511
 * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write
 * protect the page if the D-bit isn't supported.
1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522
 * @kvm: kvm instance
 * @slot: slot to clear D-bit
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should clear D-bit
 *
 * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
 */
void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
				     struct kvm_memory_slot *slot,
				     gfn_t gfn_offset, unsigned long mask)
{
1523
	struct kvm_rmap_head *rmap_head;
1524 1525

	while (mask) {
1526 1527 1528
		rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
					  PT_PAGE_TABLE_LEVEL, slot);
		__rmap_clear_dirty(kvm, rmap_head);
1529 1530 1531 1532 1533 1534 1535

		/* clear the first set bit */
		mask &= mask - 1;
	}
}
EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked);

1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
/**
 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
 * PT level pages.
 *
 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
 * enable dirty logging for them.
 *
 * Used when we do not need to care about huge page mappings: e.g. during dirty
 * logging we do not have any such mappings.
 */
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
				struct kvm_memory_slot *slot,
				gfn_t gfn_offset, unsigned long mask)
{
1550 1551 1552 1553 1554
	if (kvm_x86_ops->enable_log_dirty_pt_masked)
		kvm_x86_ops->enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
				mask);
	else
		kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1555 1556
}

1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571
/**
 * kvm_arch_write_log_dirty - emulate dirty page logging
 * @vcpu: Guest mode vcpu
 *
 * Emulate arch specific page modification logging for the
 * nested hypervisor
 */
int kvm_arch_write_log_dirty(struct kvm_vcpu *vcpu)
{
	if (kvm_x86_ops->write_log_dirty)
		return kvm_x86_ops->write_log_dirty(vcpu);

	return 0;
}

1572 1573
bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
				    struct kvm_memory_slot *slot, u64 gfn)
1574
{
1575
	struct kvm_rmap_head *rmap_head;
1576
	int i;
1577
	bool write_protected = false;
1578

1579
	for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
1580
		rmap_head = __gfn_to_rmap(gfn, i, slot);
1581
		write_protected |= __rmap_write_protect(kvm, rmap_head, true);
1582 1583 1584
	}

	return write_protected;
1585 1586
}

1587 1588 1589 1590 1591 1592 1593 1594
static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
{
	struct kvm_memory_slot *slot;

	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
	return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn);
}

1595
static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1596
{
1597 1598
	u64 *sptep;
	struct rmap_iterator iter;
1599
	bool flush = false;
1600

1601
	while ((sptep = rmap_get_first(rmap_head, &iter))) {
1602
		rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);
1603 1604

		drop_spte(kvm, sptep);
1605
		flush = true;
1606
	}
1607

1608 1609 1610
	return flush;
}

1611
static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1612 1613 1614
			   struct kvm_memory_slot *slot, gfn_t gfn, int level,
			   unsigned long data)
{
1615
	return kvm_zap_rmapp(kvm, rmap_head);
1616 1617
}

1618
static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1619 1620
			     struct kvm_memory_slot *slot, gfn_t gfn, int level,
			     unsigned long data)
1621
{
1622 1623
	u64 *sptep;
	struct rmap_iterator iter;
1624
	int need_flush = 0;
1625
	u64 new_spte;
1626
	pte_t *ptep = (pte_t *)data;
D
Dan Williams 已提交
1627
	kvm_pfn_t new_pfn;
1628 1629 1630

	WARN_ON(pte_huge(*ptep));
	new_pfn = pte_pfn(*ptep);
1631

1632
restart:
1633
	for_each_rmap_spte(rmap_head, &iter, sptep) {
1634
		rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
1635
			    sptep, *sptep, gfn, level);
1636

1637
		need_flush = 1;
1638

1639
		if (pte_write(*ptep)) {
1640
			drop_spte(kvm, sptep);
1641
			goto restart;
1642
		} else {
1643
			new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
1644 1645 1646 1647
			new_spte |= (u64)new_pfn << PAGE_SHIFT;

			new_spte &= ~PT_WRITABLE_MASK;
			new_spte &= ~SPTE_HOST_WRITEABLE;
1648 1649

			new_spte = mark_spte_for_access_track(new_spte);
1650 1651 1652

			mmu_spte_clear_track_bits(sptep);
			mmu_spte_set(sptep, new_spte);
1653 1654
		}
	}
1655

1656 1657 1658 1659 1660 1661
	if (need_flush)
		kvm_flush_remote_tlbs(kvm);

	return 0;
}

1662 1663 1664 1665 1666 1667 1668 1669 1670 1671
struct slot_rmap_walk_iterator {
	/* input fields. */
	struct kvm_memory_slot *slot;
	gfn_t start_gfn;
	gfn_t end_gfn;
	int start_level;
	int end_level;

	/* output fields. */
	gfn_t gfn;
1672
	struct kvm_rmap_head *rmap;
1673 1674 1675
	int level;

	/* private field. */
1676
	struct kvm_rmap_head *end_rmap;
1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729
};

static void
rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)
{
	iterator->level = level;
	iterator->gfn = iterator->start_gfn;
	iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot);
	iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level,
					   iterator->slot);
}

static void
slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
		    struct kvm_memory_slot *slot, int start_level,
		    int end_level, gfn_t start_gfn, gfn_t end_gfn)
{
	iterator->slot = slot;
	iterator->start_level = start_level;
	iterator->end_level = end_level;
	iterator->start_gfn = start_gfn;
	iterator->end_gfn = end_gfn;

	rmap_walk_init_level(iterator, iterator->start_level);
}

static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
{
	return !!iterator->rmap;
}

static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
{
	if (++iterator->rmap <= iterator->end_rmap) {
		iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
		return;
	}

	if (++iterator->level > iterator->end_level) {
		iterator->rmap = NULL;
		return;
	}

	rmap_walk_init_level(iterator, iterator->level);
}

#define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_,	\
	   _start_gfn, _end_gfn, _iter_)				\
	for (slot_rmap_walk_init(_iter_, _slot_, _start_level_,		\
				 _end_level_, _start_gfn, _end_gfn);	\
	     slot_rmap_walk_okay(_iter_);				\
	     slot_rmap_walk_next(_iter_))

1730 1731 1732 1733 1734
static int kvm_handle_hva_range(struct kvm *kvm,
				unsigned long start,
				unsigned long end,
				unsigned long data,
				int (*handler)(struct kvm *kvm,
1735
					       struct kvm_rmap_head *rmap_head,
1736
					       struct kvm_memory_slot *slot,
1737 1738
					       gfn_t gfn,
					       int level,
1739
					       unsigned long data))
1740
{
1741
	struct kvm_memslots *slots;
1742
	struct kvm_memory_slot *memslot;
1743 1744
	struct slot_rmap_walk_iterator iterator;
	int ret = 0;
1745
	int i;
1746

1747 1748 1749 1750 1751
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
		kvm_for_each_memslot(memslot, slots) {
			unsigned long hva_start, hva_end;
			gfn_t gfn_start, gfn_end;
1752

1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771
			hva_start = max(start, memslot->userspace_addr);
			hva_end = min(end, memslot->userspace_addr +
				      (memslot->npages << PAGE_SHIFT));
			if (hva_start >= hva_end)
				continue;
			/*
			 * {gfn(page) | page intersects with [hva_start, hva_end)} =
			 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
			 */
			gfn_start = hva_to_gfn_memslot(hva_start, memslot);
			gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);

			for_each_slot_rmap_range(memslot, PT_PAGE_TABLE_LEVEL,
						 PT_MAX_HUGEPAGE_LEVEL,
						 gfn_start, gfn_end - 1,
						 &iterator)
				ret |= handler(kvm, iterator.rmap, memslot,
					       iterator.gfn, iterator.level, data);
		}
1772 1773
	}

1774
	return ret;
1775 1776
}

1777 1778
static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
			  unsigned long data,
1779 1780
			  int (*handler)(struct kvm *kvm,
					 struct kvm_rmap_head *rmap_head,
1781
					 struct kvm_memory_slot *slot,
1782
					 gfn_t gfn, int level,
1783 1784 1785
					 unsigned long data))
{
	return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1786 1787 1788 1789
}

int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
{
1790 1791 1792
	return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
}

1793 1794 1795 1796 1797
int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
{
	return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
}

1798 1799
void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
{
F
Frederik Deweerdt 已提交
1800
	kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1801 1802
}

1803
static int kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1804 1805
			 struct kvm_memory_slot *slot, gfn_t gfn, int level,
			 unsigned long data)
1806
{
1807
	u64 *sptep;
1808
	struct rmap_iterator uninitialized_var(iter);
1809 1810
	int young = 0;

1811 1812
	for_each_rmap_spte(rmap_head, &iter, sptep)
		young |= mmu_spte_age(sptep);
1813

1814
	trace_kvm_age_page(gfn, level, slot, young);
1815 1816 1817
	return young;
}

1818
static int kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1819 1820
			      struct kvm_memory_slot *slot, gfn_t gfn,
			      int level, unsigned long data)
A
Andrea Arcangeli 已提交
1821
{
1822 1823
	u64 *sptep;
	struct rmap_iterator iter;
A
Andrea Arcangeli 已提交
1824

1825 1826 1827 1828
	for_each_rmap_spte(rmap_head, &iter, sptep)
		if (is_accessed_spte(*sptep))
			return 1;
	return 0;
A
Andrea Arcangeli 已提交
1829 1830
}

1831 1832
#define RMAP_RECYCLE_THRESHOLD 1000

1833
static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1834
{
1835
	struct kvm_rmap_head *rmap_head;
1836 1837 1838
	struct kvm_mmu_page *sp;

	sp = page_header(__pa(spte));
1839

1840
	rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
1841

1842
	kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, 0);
1843 1844 1845
	kvm_flush_remote_tlbs(vcpu->kvm);
}

A
Andres Lagar-Cavilla 已提交
1846
int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1847
{
A
Andres Lagar-Cavilla 已提交
1848
	return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
1849 1850
}

A
Andrea Arcangeli 已提交
1851 1852 1853 1854 1855
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
{
	return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
}

1856
#ifdef MMU_DEBUG
1857
static int is_empty_shadow_page(u64 *spt)
A
Avi Kivity 已提交
1858
{
1859 1860 1861
	u64 *pos;
	u64 *end;

1862
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1863
		if (is_shadow_present_pte(*pos)) {
1864
			printk(KERN_ERR "%s: %p %llx\n", __func__,
1865
			       pos, *pos);
A
Avi Kivity 已提交
1866
			return 0;
1867
		}
A
Avi Kivity 已提交
1868 1869
	return 1;
}
1870
#endif
A
Avi Kivity 已提交
1871

1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883
/*
 * This value is the sum of all of the kvm instances's
 * kvm->arch.n_used_mmu_pages values.  We need a global,
 * aggregate version in order to make the slab shrinker
 * faster
 */
static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
{
	kvm->arch.n_used_mmu_pages += nr;
	percpu_counter_add(&kvm_total_used_mmu_pages, nr);
}

1884
static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1885
{
1886
	MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
1887
	hlist_del(&sp->hash_link);
1888 1889
	list_del(&sp->link);
	free_page((unsigned long)sp->spt);
1890 1891
	if (!sp->role.direct)
		free_page((unsigned long)sp->gfns);
1892
	kmem_cache_free(mmu_page_header_cache, sp);
1893 1894
}

1895 1896
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
1897
	return hash_64(gfn, KVM_MMU_HASH_SHIFT);
1898 1899
}

1900
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1901
				    struct kvm_mmu_page *sp, u64 *parent_pte)
1902 1903 1904 1905
{
	if (!parent_pte)
		return;

1906
	pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1907 1908
}

1909
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1910 1911
				       u64 *parent_pte)
{
1912
	pte_list_remove(parent_pte, &sp->parent_ptes);
1913 1914
}

1915 1916 1917 1918
static void drop_parent_pte(struct kvm_mmu_page *sp,
			    u64 *parent_pte)
{
	mmu_page_remove_parent_pte(sp, parent_pte);
1919
	mmu_spte_clear_no_track(parent_pte);
1920 1921
}

1922
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct)
M
Marcelo Tosatti 已提交
1923
{
1924
	struct kvm_mmu_page *sp;
1925

1926 1927
	sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
	sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1928
	if (!direct)
1929
		sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1930
	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1931 1932 1933 1934 1935 1936

	/*
	 * The active_mmu_pages list is the FIFO list, do not move the
	 * page until it is zapped. kvm_zap_obsolete_pages depends on
	 * this feature. See the comments in kvm_zap_obsolete_pages().
	 */
1937 1938 1939
	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
	kvm_mod_used_mmu_pages(vcpu->kvm, +1);
	return sp;
M
Marcelo Tosatti 已提交
1940 1941
}

1942
static void mark_unsync(u64 *spte);
1943
static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1944
{
1945 1946 1947 1948 1949 1950
	u64 *sptep;
	struct rmap_iterator iter;

	for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
		mark_unsync(sptep);
	}
1951 1952
}

1953
static void mark_unsync(u64 *spte)
1954
{
1955
	struct kvm_mmu_page *sp;
1956
	unsigned int index;
1957

1958
	sp = page_header(__pa(spte));
1959 1960
	index = spte - sp->spt;
	if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1961
		return;
1962
	if (sp->unsync_children++)
1963
		return;
1964
	kvm_mmu_mark_parents_unsync(sp);
1965 1966
}

1967
static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1968
			       struct kvm_mmu_page *sp)
1969
{
1970
	return 0;
1971 1972
}

M
Marcelo Tosatti 已提交
1973 1974 1975 1976
static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
}

1977 1978
static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
				 struct kvm_mmu_page *sp, u64 *spte,
1979
				 const void *pte)
1980 1981 1982 1983
{
	WARN_ON(1);
}

1984 1985 1986 1987 1988 1989 1990 1991 1992 1993
#define KVM_PAGE_ARRAY_NR 16

struct kvm_mmu_pages {
	struct mmu_page_and_offset {
		struct kvm_mmu_page *sp;
		unsigned int idx;
	} page[KVM_PAGE_ARRAY_NR];
	unsigned int nr;
};

1994 1995
static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
			 int idx)
1996
{
1997
	int i;
1998

1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009
	if (sp->unsync)
		for (i=0; i < pvec->nr; i++)
			if (pvec->page[i].sp == sp)
				return 0;

	pvec->page[pvec->nr].sp = sp;
	pvec->page[pvec->nr].idx = idx;
	pvec->nr++;
	return (pvec->nr == KVM_PAGE_ARRAY_NR);
}

2010 2011 2012 2013 2014 2015 2016
static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
{
	--sp->unsync_children;
	WARN_ON((int)sp->unsync_children < 0);
	__clear_bit(idx, sp->unsync_child_bitmap);
}

2017 2018 2019 2020
static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	int i, ret, nr_unsync_leaf = 0;
2021

2022
	for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
2023
		struct kvm_mmu_page *child;
2024 2025
		u64 ent = sp->spt[i];

2026 2027 2028 2029
		if (!is_shadow_present_pte(ent) || is_large_pte(ent)) {
			clear_unsync_child_bit(sp, i);
			continue;
		}
2030 2031 2032 2033 2034 2035 2036 2037

		child = page_header(ent & PT64_BASE_ADDR_MASK);

		if (child->unsync_children) {
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;

			ret = __mmu_unsync_walk(child, pvec);
2038 2039 2040 2041
			if (!ret) {
				clear_unsync_child_bit(sp, i);
				continue;
			} else if (ret > 0) {
2042
				nr_unsync_leaf += ret;
2043
			} else
2044 2045 2046 2047 2048 2049
				return ret;
		} else if (child->unsync) {
			nr_unsync_leaf++;
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;
		} else
2050
			clear_unsync_child_bit(sp, i);
2051 2052
	}

2053 2054 2055
	return nr_unsync_leaf;
}

2056 2057
#define INVALID_INDEX (-1)

2058 2059 2060
static int mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
P
Paolo Bonzini 已提交
2061
	pvec->nr = 0;
2062 2063 2064
	if (!sp->unsync_children)
		return 0;

2065
	mmu_pages_add(pvec, sp, INVALID_INDEX);
2066
	return __mmu_unsync_walk(sp, pvec);
2067 2068 2069 2070 2071
}

static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	WARN_ON(!sp->unsync);
2072
	trace_kvm_mmu_sync_page(sp);
2073 2074 2075 2076
	sp->unsync = 0;
	--kvm->stat.mmu_unsync;
}

2077 2078 2079 2080
static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				    struct list_head *invalid_list);
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list);
2081

2082 2083 2084 2085 2086 2087
/*
 * NOTE: we should pay more attention on the zapped-obsolete page
 * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
 * since it has been deleted from active_mmu_pages but still can be found
 * at hast list.
 *
2088
 * for_each_valid_sp() has skipped that kind of pages.
2089
 */
2090
#define for_each_valid_sp(_kvm, _sp, _gfn)				\
2091 2092
	hlist_for_each_entry(_sp,					\
	  &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
2093 2094
		if (is_obsolete_sp((_kvm), (_sp)) || (_sp)->role.invalid) {    \
		} else
2095 2096

#define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn)			\
2097 2098
	for_each_valid_sp(_kvm, _sp, _gfn)				\
		if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else
2099

2100
/* @sp->gfn should be write-protected at the call site */
2101 2102
static bool __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
			    struct list_head *invalid_list)
2103
{
2104
	if (sp->role.cr4_pae != !!is_pae(vcpu)) {
2105
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
2106
		return false;
2107 2108
	}

2109
	if (vcpu->arch.mmu.sync_page(vcpu, sp) == 0) {
2110
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
2111
		return false;
2112 2113
	}

2114
	return true;
2115 2116
}

2117 2118 2119
static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu,
				 struct list_head *invalid_list,
				 bool remote_flush, bool local_flush)
2120
{
2121 2122 2123 2124
	if (!list_empty(invalid_list)) {
		kvm_mmu_commit_zap_page(vcpu->kvm, invalid_list);
		return;
	}
2125

2126 2127 2128 2129
	if (remote_flush)
		kvm_flush_remote_tlbs(vcpu->kvm);
	else if (local_flush)
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2130 2131
}

2132 2133 2134 2135 2136 2137 2138
#ifdef CONFIG_KVM_MMU_AUDIT
#include "mmu_audit.c"
#else
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
static void mmu_audit_disable(void) { }
#endif

2139 2140 2141 2142 2143
static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
}

2144
static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
2145
			 struct list_head *invalid_list)
2146
{
2147 2148
	kvm_unlink_unsync_page(vcpu->kvm, sp);
	return __kvm_sync_page(vcpu, sp, invalid_list);
2149 2150
}

2151
/* @gfn should be write-protected at the call site */
2152 2153
static bool kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn,
			   struct list_head *invalid_list)
2154 2155
{
	struct kvm_mmu_page *s;
2156
	bool ret = false;
2157

2158
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
2159
		if (!s->unsync)
2160 2161 2162
			continue;

		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
2163
		ret |= kvm_sync_page(vcpu, s, invalid_list);
2164 2165
	}

2166
	return ret;
2167 2168
}

2169
struct mmu_page_path {
2170 2171
	struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL];
	unsigned int idx[PT64_ROOT_MAX_LEVEL];
2172 2173
};

2174
#define for_each_sp(pvec, sp, parents, i)			\
P
Paolo Bonzini 已提交
2175
		for (i = mmu_pages_first(&pvec, &parents);	\
2176 2177 2178
			i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});	\
			i = mmu_pages_next(&pvec, &parents, i))

2179 2180 2181
static int mmu_pages_next(struct kvm_mmu_pages *pvec,
			  struct mmu_page_path *parents,
			  int i)
2182 2183 2184 2185 2186
{
	int n;

	for (n = i+1; n < pvec->nr; n++) {
		struct kvm_mmu_page *sp = pvec->page[n].sp;
P
Paolo Bonzini 已提交
2187 2188
		unsigned idx = pvec->page[n].idx;
		int level = sp->role.level;
2189

P
Paolo Bonzini 已提交
2190 2191 2192
		parents->idx[level-1] = idx;
		if (level == PT_PAGE_TABLE_LEVEL)
			break;
2193

P
Paolo Bonzini 已提交
2194
		parents->parent[level-2] = sp;
2195 2196 2197 2198 2199
	}

	return n;
}

P
Paolo Bonzini 已提交
2200 2201 2202 2203 2204 2205 2206 2207 2208
static int mmu_pages_first(struct kvm_mmu_pages *pvec,
			   struct mmu_page_path *parents)
{
	struct kvm_mmu_page *sp;
	int level;

	if (pvec->nr == 0)
		return 0;

2209 2210
	WARN_ON(pvec->page[0].idx != INVALID_INDEX);

P
Paolo Bonzini 已提交
2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223
	sp = pvec->page[0].sp;
	level = sp->role.level;
	WARN_ON(level == PT_PAGE_TABLE_LEVEL);

	parents->parent[level-2] = sp;

	/* Also set up a sentinel.  Further entries in pvec are all
	 * children of sp, so this element is never overwritten.
	 */
	parents->parent[level-1] = NULL;
	return mmu_pages_next(pvec, parents, 0);
}

2224
static void mmu_pages_clear_parents(struct mmu_page_path *parents)
2225
{
2226 2227 2228 2229 2230 2231 2232 2233 2234
	struct kvm_mmu_page *sp;
	unsigned int level = 0;

	do {
		unsigned int idx = parents->idx[level];
		sp = parents->parent[level];
		if (!sp)
			return;

2235
		WARN_ON(idx == INVALID_INDEX);
2236
		clear_unsync_child_bit(sp, idx);
2237
		level++;
P
Paolo Bonzini 已提交
2238
	} while (!sp->unsync_children);
2239
}
2240

2241 2242 2243 2244 2245 2246 2247
static void mmu_sync_children(struct kvm_vcpu *vcpu,
			      struct kvm_mmu_page *parent)
{
	int i;
	struct kvm_mmu_page *sp;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
2248
	LIST_HEAD(invalid_list);
2249
	bool flush = false;
2250 2251

	while (mmu_unsync_walk(parent, &pages)) {
2252
		bool protected = false;
2253 2254

		for_each_sp(pages, sp, parents, i)
2255
			protected |= rmap_write_protect(vcpu, sp->gfn);
2256

2257
		if (protected) {
2258
			kvm_flush_remote_tlbs(vcpu->kvm);
2259 2260
			flush = false;
		}
2261

2262
		for_each_sp(pages, sp, parents, i) {
2263
			flush |= kvm_sync_page(vcpu, sp, &invalid_list);
2264 2265
			mmu_pages_clear_parents(&parents);
		}
2266 2267 2268 2269 2270
		if (need_resched() || spin_needbreak(&vcpu->kvm->mmu_lock)) {
			kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
			cond_resched_lock(&vcpu->kvm->mmu_lock);
			flush = false;
		}
2271
	}
2272 2273

	kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2274 2275
}

2276 2277
static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
{
2278
	atomic_set(&sp->write_flooding_count,  0);
2279 2280 2281 2282 2283 2284 2285 2286 2287
}

static void clear_sp_write_flooding_count(u64 *spte)
{
	struct kvm_mmu_page *sp =  page_header(__pa(spte));

	__clear_sp_write_flooding_count(sp);
}

2288 2289 2290 2291
static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
2292
					     int direct,
2293
					     unsigned access)
2294 2295 2296
{
	union kvm_mmu_page_role role;
	unsigned quadrant;
2297 2298
	struct kvm_mmu_page *sp;
	bool need_sync = false;
2299
	bool flush = false;
2300
	int collisions = 0;
2301
	LIST_HEAD(invalid_list);
2302

2303
	role = vcpu->arch.mmu.base_role;
2304
	role.level = level;
2305
	role.direct = direct;
2306
	if (role.direct)
2307
		role.cr4_pae = 0;
2308
	role.access = access;
2309 2310
	if (!vcpu->arch.mmu.direct_map
	    && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
2311 2312 2313 2314
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
2315 2316 2317 2318 2319 2320
	for_each_valid_sp(vcpu->kvm, sp, gfn) {
		if (sp->gfn != gfn) {
			collisions++;
			continue;
		}

2321 2322
		if (!need_sync && sp->unsync)
			need_sync = true;
2323

2324 2325
		if (sp->role.word != role.word)
			continue;
2326

2327 2328 2329 2330 2331 2332 2333 2334 2335 2336
		if (sp->unsync) {
			/* The page is good, but __kvm_sync_page might still end
			 * up zapping it.  If so, break in order to rebuild it.
			 */
			if (!__kvm_sync_page(vcpu, sp, &invalid_list))
				break;

			WARN_ON(!list_empty(&invalid_list));
			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
		}
2337

2338
		if (sp->unsync_children)
2339
			kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2340

2341
		__clear_sp_write_flooding_count(sp);
2342
		trace_kvm_mmu_get_page(sp, false);
2343
		goto out;
2344
	}
2345

A
Avi Kivity 已提交
2346
	++vcpu->kvm->stat.mmu_cache_miss;
2347 2348 2349

	sp = kvm_mmu_alloc_page(vcpu, direct);

2350 2351
	sp->gfn = gfn;
	sp->role = role;
2352 2353
	hlist_add_head(&sp->hash_link,
		&vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
2354
	if (!direct) {
2355 2356 2357 2358 2359 2360 2361 2362
		/*
		 * we should do write protection before syncing pages
		 * otherwise the content of the synced shadow page may
		 * be inconsistent with guest page table.
		 */
		account_shadowed(vcpu->kvm, sp);
		if (level == PT_PAGE_TABLE_LEVEL &&
		      rmap_write_protect(vcpu, gfn))
2363
			kvm_flush_remote_tlbs(vcpu->kvm);
2364 2365

		if (level > PT_PAGE_TABLE_LEVEL && need_sync)
2366
			flush |= kvm_sync_pages(vcpu, gfn, &invalid_list);
2367
	}
2368
	sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
2369
	clear_page(sp->spt);
A
Avi Kivity 已提交
2370
	trace_kvm_mmu_get_page(sp, true);
2371 2372

	kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2373 2374 2375
out:
	if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions)
		vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions;
2376
	return sp;
2377 2378
}

2379 2380 2381 2382 2383 2384
static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
			     struct kvm_vcpu *vcpu, u64 addr)
{
	iterator->addr = addr;
	iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
	iterator->level = vcpu->arch.mmu.shadow_root_level;
2385

2386 2387
	if (iterator->level == PT64_ROOT_4LEVEL &&
	    vcpu->arch.mmu.root_level < PT64_ROOT_4LEVEL &&
2388 2389 2390
	    !vcpu->arch.mmu.direct_map)
		--iterator->level;

2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404
	if (iterator->level == PT32E_ROOT_LEVEL) {
		iterator->shadow_addr
			= vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
		--iterator->level;
		if (!iterator->shadow_addr)
			iterator->level = 0;
	}
}

static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
{
	if (iterator->level < PT_PAGE_TABLE_LEVEL)
		return false;
2405

2406 2407 2408 2409 2410
	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
	return true;
}

2411 2412
static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
			       u64 spte)
2413
{
2414
	if (is_last_spte(spte, iterator->level)) {
2415 2416 2417 2418
		iterator->level = 0;
		return;
	}

2419
	iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2420 2421 2422
	--iterator->level;
}

2423 2424 2425 2426 2427
static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
{
	return __shadow_walk_next(iterator, *iterator->sptep);
}

2428 2429
static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
			     struct kvm_mmu_page *sp)
2430 2431 2432
{
	u64 spte;

2433
	BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2434

2435
	spte = __pa(sp->spt) | shadow_present_mask | PT_WRITABLE_MASK |
2436 2437 2438 2439 2440 2441
	       shadow_user_mask | shadow_x_mask;

	if (sp_ad_disabled(sp))
		spte |= shadow_acc_track_value;
	else
		spte |= shadow_accessed_mask;
X
Xiao Guangrong 已提交
2442

2443
	mmu_spte_set(sptep, spte);
2444 2445 2446 2447 2448

	mmu_page_add_parent_pte(vcpu, sp, sptep);

	if (sp->unsync_children || sp->unsync)
		mark_unsync(sptep);
2449 2450
}

2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467
static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
				   unsigned direct_access)
{
	if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
		struct kvm_mmu_page *child;

		/*
		 * For the direct sp, if the guest pte's dirty bit
		 * changed form clean to dirty, it will corrupt the
		 * sp's access: allow writable in the read-only sp,
		 * so we should update the spte at this point to get
		 * a new sp with the correct access.
		 */
		child = page_header(*sptep & PT64_BASE_ADDR_MASK);
		if (child->role.access == direct_access)
			return;

2468
		drop_parent_pte(child, sptep);
2469 2470 2471 2472
		kvm_flush_remote_tlbs(vcpu->kvm);
	}
}

X
Xiao Guangrong 已提交
2473
static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2474 2475 2476 2477 2478 2479 2480
			     u64 *spte)
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
	if (is_shadow_present_pte(pte)) {
X
Xiao Guangrong 已提交
2481
		if (is_last_spte(pte, sp->role.level)) {
2482
			drop_spte(kvm, spte);
X
Xiao Guangrong 已提交
2483 2484 2485
			if (is_large_pte(pte))
				--kvm->stat.lpages;
		} else {
2486
			child = page_header(pte & PT64_BASE_ADDR_MASK);
2487
			drop_parent_pte(child, spte);
2488
		}
X
Xiao Guangrong 已提交
2489 2490 2491 2492
		return true;
	}

	if (is_mmio_spte(pte))
2493
		mmu_spte_clear_no_track(spte);
2494

X
Xiao Guangrong 已提交
2495
	return false;
2496 2497
}

2498
static void kvm_mmu_page_unlink_children(struct kvm *kvm,
2499
					 struct kvm_mmu_page *sp)
2500
{
2501 2502
	unsigned i;

2503 2504
	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		mmu_page_zap_pte(kvm, sp, sp->spt + i);
2505 2506
}

2507
static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
2508
{
2509 2510
	u64 *sptep;
	struct rmap_iterator iter;
2511

2512
	while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2513
		drop_parent_pte(sp, sptep);
2514 2515
}

2516
static int mmu_zap_unsync_children(struct kvm *kvm,
2517 2518
				   struct kvm_mmu_page *parent,
				   struct list_head *invalid_list)
2519
{
2520 2521 2522
	int i, zapped = 0;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
2523

2524
	if (parent->role.level == PT_PAGE_TABLE_LEVEL)
2525
		return 0;
2526 2527 2528 2529 2530

	while (mmu_unsync_walk(parent, &pages)) {
		struct kvm_mmu_page *sp;

		for_each_sp(pages, sp, parents, i) {
2531
			kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2532
			mmu_pages_clear_parents(&parents);
2533
			zapped++;
2534 2535 2536 2537
		}
	}

	return zapped;
2538 2539
}

2540 2541
static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				    struct list_head *invalid_list)
2542
{
2543
	int ret;
A
Avi Kivity 已提交
2544

2545
	trace_kvm_mmu_prepare_zap_page(sp);
2546
	++kvm->stat.mmu_shadow_zapped;
2547
	ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
2548
	kvm_mmu_page_unlink_children(kvm, sp);
2549
	kvm_mmu_unlink_parents(kvm, sp);
2550

2551
	if (!sp->role.invalid && !sp->role.direct)
2552
		unaccount_shadowed(kvm, sp);
2553

2554 2555
	if (sp->unsync)
		kvm_unlink_unsync_page(kvm, sp);
2556
	if (!sp->root_count) {
2557 2558
		/* Count self */
		ret++;
2559
		list_move(&sp->link, invalid_list);
2560
		kvm_mod_used_mmu_pages(kvm, -1);
2561
	} else {
A
Avi Kivity 已提交
2562
		list_move(&sp->link, &kvm->arch.active_mmu_pages);
2563 2564 2565 2566 2567 2568 2569

		/*
		 * The obsolete pages can not be used on any vcpus.
		 * See the comments in kvm_mmu_invalidate_zap_all_pages().
		 */
		if (!sp->role.invalid && !is_obsolete_sp(kvm, sp))
			kvm_reload_remote_mmus(kvm);
2570
	}
2571 2572

	sp->role.invalid = 1;
2573
	return ret;
2574 2575
}

2576 2577 2578
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list)
{
2579
	struct kvm_mmu_page *sp, *nsp;
2580 2581 2582 2583

	if (list_empty(invalid_list))
		return;

2584
	/*
2585 2586 2587 2588 2589 2590 2591
	 * We need to make sure everyone sees our modifications to
	 * the page tables and see changes to vcpu->mode here. The barrier
	 * in the kvm_flush_remote_tlbs() achieves this. This pairs
	 * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end.
	 *
	 * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
	 * guest mode and/or lockless shadow page table walks.
2592 2593
	 */
	kvm_flush_remote_tlbs(kvm);
2594

2595
	list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2596
		WARN_ON(!sp->role.invalid || sp->root_count);
2597
		kvm_mmu_free_page(sp);
2598
	}
2599 2600
}

2601 2602 2603 2604 2605 2606 2607 2608
static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
					struct list_head *invalid_list)
{
	struct kvm_mmu_page *sp;

	if (list_empty(&kvm->arch.active_mmu_pages))
		return false;

G
Geliang Tang 已提交
2609 2610
	sp = list_last_entry(&kvm->arch.active_mmu_pages,
			     struct kvm_mmu_page, link);
2611
	return kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2612 2613
}

2614 2615
/*
 * Changing the number of mmu pages allocated to the vm
2616
 * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2617
 */
2618
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
2619
{
2620
	LIST_HEAD(invalid_list);
2621

2622 2623
	spin_lock(&kvm->mmu_lock);

2624
	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2625 2626 2627 2628
		/* Need to free some mmu pages to achieve the goal. */
		while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
			if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
				break;
2629

2630
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
2631
		goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2632 2633
	}

2634
	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2635 2636

	spin_unlock(&kvm->mmu_lock);
2637 2638
}

2639
int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2640
{
2641
	struct kvm_mmu_page *sp;
2642
	LIST_HEAD(invalid_list);
2643 2644
	int r;

2645
	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2646
	r = 0;
2647
	spin_lock(&kvm->mmu_lock);
2648
	for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2649
		pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2650 2651
			 sp->role.word);
		r = 1;
2652
		kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2653
	}
2654
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
2655 2656
	spin_unlock(&kvm->mmu_lock);

2657
	return r;
2658
}
2659
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
2660

2661
static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
2662 2663 2664 2665 2666 2667 2668 2669
{
	trace_kvm_mmu_unsync_page(sp);
	++vcpu->kvm->stat.mmu_unsync;
	sp->unsync = 1;

	kvm_mmu_mark_parents_unsync(sp);
}

2670 2671
static bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
				   bool can_unsync)
2672
{
2673
	struct kvm_mmu_page *sp;
2674

2675 2676
	if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE))
		return true;
2677

2678
	for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
2679
		if (!can_unsync)
2680
			return true;
2681

2682 2683
		if (sp->unsync)
			continue;
2684

2685 2686
		WARN_ON(sp->role.level != PT_PAGE_TABLE_LEVEL);
		kvm_unsync_page(vcpu, sp);
2687
	}
2688 2689

	return false;
2690 2691
}

D
Dan Williams 已提交
2692
static bool kvm_is_mmio_pfn(kvm_pfn_t pfn)
2693 2694 2695 2696 2697 2698 2699
{
	if (pfn_valid(pfn))
		return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn));

	return true;
}

A
Avi Kivity 已提交
2700
static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2701
		    unsigned pte_access, int level,
D
Dan Williams 已提交
2702
		    gfn_t gfn, kvm_pfn_t pfn, bool speculative,
2703
		    bool can_unsync, bool host_writable)
2704
{
2705
	u64 spte = 0;
M
Marcelo Tosatti 已提交
2706
	int ret = 0;
2707
	struct kvm_mmu_page *sp;
S
Sheng Yang 已提交
2708

2709
	if (set_mmio_spte(vcpu, sptep, gfn, pfn, pte_access))
2710 2711
		return 0;

2712 2713 2714 2715
	sp = page_header(__pa(sptep));
	if (sp_ad_disabled(sp))
		spte |= shadow_acc_track_value;

2716 2717 2718 2719 2720 2721
	/*
	 * For the EPT case, shadow_present_mask is 0 if hardware
	 * supports exec-only page table entries.  In that case,
	 * ACC_USER_MASK and shadow_user_mask are used to represent
	 * read access.  See FNAME(gpte_access) in paging_tmpl.h.
	 */
2722
	spte |= shadow_present_mask;
2723
	if (!speculative)
2724
		spte |= spte_shadow_accessed_mask(spte);
2725

S
Sheng Yang 已提交
2726 2727 2728 2729
	if (pte_access & ACC_EXEC_MASK)
		spte |= shadow_x_mask;
	else
		spte |= shadow_nx_mask;
2730

2731
	if (pte_access & ACC_USER_MASK)
S
Sheng Yang 已提交
2732
		spte |= shadow_user_mask;
2733

2734
	if (level > PT_PAGE_TABLE_LEVEL)
M
Marcelo Tosatti 已提交
2735
		spte |= PT_PAGE_SIZE_MASK;
2736
	if (tdp_enabled)
2737
		spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
2738
			kvm_is_mmio_pfn(pfn));
2739

2740
	if (host_writable)
2741
		spte |= SPTE_HOST_WRITEABLE;
2742 2743
	else
		pte_access &= ~ACC_WRITE_MASK;
2744

2745
	spte |= (u64)pfn << PAGE_SHIFT;
2746

2747
	if (pte_access & ACC_WRITE_MASK) {
2748

X
Xiao Guangrong 已提交
2749
		/*
2750 2751 2752 2753
		 * Other vcpu creates new sp in the window between
		 * mapping_level() and acquiring mmu-lock. We can
		 * allow guest to retry the access, the mapping can
		 * be fixed if guest refault.
X
Xiao Guangrong 已提交
2754
		 */
2755
		if (level > PT_PAGE_TABLE_LEVEL &&
2756
		    mmu_gfn_lpage_is_disallowed(vcpu, gfn, level))
A
Avi Kivity 已提交
2757
			goto done;
2758

2759
		spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
2760

2761 2762 2763 2764 2765 2766
		/*
		 * Optimization: for pte sync, if spte was writable the hash
		 * lookup is unnecessary (and expensive). Write protection
		 * is responsibility of mmu_get_page / kvm_sync_page.
		 * Same reasoning can be applied to dirty page accounting.
		 */
2767
		if (!can_unsync && is_writable_pte(*sptep))
2768 2769
			goto set_pte;

2770
		if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2771
			pgprintk("%s: found shadow page for %llx, marking ro\n",
2772
				 __func__, gfn);
M
Marcelo Tosatti 已提交
2773
			ret = 1;
2774
			pte_access &= ~ACC_WRITE_MASK;
2775
			spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
2776 2777 2778
		}
	}

2779
	if (pte_access & ACC_WRITE_MASK) {
2780
		kvm_vcpu_mark_page_dirty(vcpu, gfn);
2781
		spte |= spte_shadow_dirty_mask(spte);
2782
	}
2783

2784 2785 2786
	if (speculative)
		spte = mark_spte_for_access_track(spte);

2787
set_pte:
2788
	if (mmu_spte_update(sptep, spte))
2789
		kvm_flush_remote_tlbs(vcpu->kvm);
A
Avi Kivity 已提交
2790
done:
M
Marcelo Tosatti 已提交
2791 2792 2793
	return ret;
}

2794
static bool mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access,
D
Dan Williams 已提交
2795
			 int write_fault, int level, gfn_t gfn, kvm_pfn_t pfn,
2796
			 bool speculative, bool host_writable)
M
Marcelo Tosatti 已提交
2797 2798
{
	int was_rmapped = 0;
2799
	int rmap_count;
2800
	bool emulate = false;
M
Marcelo Tosatti 已提交
2801

2802 2803
	pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
		 *sptep, write_fault, gfn);
M
Marcelo Tosatti 已提交
2804

2805
	if (is_shadow_present_pte(*sptep)) {
M
Marcelo Tosatti 已提交
2806 2807 2808 2809
		/*
		 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
		 * the parent of the now unreachable PTE.
		 */
2810 2811
		if (level > PT_PAGE_TABLE_LEVEL &&
		    !is_large_pte(*sptep)) {
M
Marcelo Tosatti 已提交
2812
			struct kvm_mmu_page *child;
A
Avi Kivity 已提交
2813
			u64 pte = *sptep;
M
Marcelo Tosatti 已提交
2814 2815

			child = page_header(pte & PT64_BASE_ADDR_MASK);
2816
			drop_parent_pte(child, sptep);
2817
			kvm_flush_remote_tlbs(vcpu->kvm);
A
Avi Kivity 已提交
2818
		} else if (pfn != spte_to_pfn(*sptep)) {
2819
			pgprintk("hfn old %llx new %llx\n",
A
Avi Kivity 已提交
2820
				 spte_to_pfn(*sptep), pfn);
2821
			drop_spte(vcpu->kvm, sptep);
2822
			kvm_flush_remote_tlbs(vcpu->kvm);
2823 2824
		} else
			was_rmapped = 1;
M
Marcelo Tosatti 已提交
2825
	}
2826

2827 2828
	if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
	      true, host_writable)) {
M
Marcelo Tosatti 已提交
2829
		if (write_fault)
2830
			emulate = true;
2831
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2832
	}
M
Marcelo Tosatti 已提交
2833

2834 2835
	if (unlikely(is_mmio_spte(*sptep)))
		emulate = true;
2836

A
Avi Kivity 已提交
2837
	pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2838
	pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
A
Avi Kivity 已提交
2839
		 is_large_pte(*sptep)? "2MB" : "4kB",
2840
		 *sptep & PT_WRITABLE_MASK ? "RW" : "R", gfn,
2841
		 *sptep, sptep);
A
Avi Kivity 已提交
2842
	if (!was_rmapped && is_large_pte(*sptep))
M
Marcelo Tosatti 已提交
2843 2844
		++vcpu->kvm->stat.lpages;

2845 2846 2847 2848 2849 2850
	if (is_shadow_present_pte(*sptep)) {
		if (!was_rmapped) {
			rmap_count = rmap_add(vcpu, sptep, gfn);
			if (rmap_count > RMAP_RECYCLE_THRESHOLD)
				rmap_recycle(vcpu, sptep, gfn);
		}
2851
	}
2852

X
Xiao Guangrong 已提交
2853
	kvm_release_pfn_clean(pfn);
2854 2855

	return emulate;
2856 2857
}

D
Dan Williams 已提交
2858
static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
2859 2860 2861 2862
				     bool no_dirty_log)
{
	struct kvm_memory_slot *slot;

2863
	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
2864
	if (!slot)
2865
		return KVM_PFN_ERR_FAULT;
2866

2867
	return gfn_to_pfn_memslot_atomic(slot, gfn);
2868 2869 2870 2871 2872 2873 2874
}

static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *sp,
				    u64 *start, u64 *end)
{
	struct page *pages[PTE_PREFETCH_NUM];
2875
	struct kvm_memory_slot *slot;
2876 2877 2878 2879 2880
	unsigned access = sp->role.access;
	int i, ret;
	gfn_t gfn;

	gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2881 2882
	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
	if (!slot)
2883 2884
		return -1;

2885
	ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
2886 2887 2888 2889
	if (ret <= 0)
		return -1;

	for (i = 0; i < ret; i++, gfn++, start++)
2890 2891
		mmu_set_spte(vcpu, start, access, 0, sp->role.level, gfn,
			     page_to_pfn(pages[i]), true, true);
2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907

	return 0;
}

static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
				  struct kvm_mmu_page *sp, u64 *sptep)
{
	u64 *spte, *start = NULL;
	int i;

	WARN_ON(!sp->role.direct);

	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
	spte = sp->spt + i;

	for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2908
		if (is_shadow_present_pte(*spte) || spte == sptep) {
2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922
			if (!start)
				continue;
			if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
				break;
			start = NULL;
		} else if (!start)
			start = spte;
	}
}

static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
{
	struct kvm_mmu_page *sp;

2923 2924
	sp = page_header(__pa(sptep));

2925
	/*
2926 2927 2928
	 * Without accessed bits, there's no way to distinguish between
	 * actually accessed translations and prefetched, so disable pte
	 * prefetch if accessed bits aren't available.
2929
	 */
2930
	if (sp_ad_disabled(sp))
2931 2932 2933 2934 2935 2936 2937 2938
		return;

	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return;

	__direct_pte_prefetch(vcpu, sp, sptep);
}

2939
static int __direct_map(struct kvm_vcpu *vcpu, int write, int map_writable,
D
Dan Williams 已提交
2940
			int level, gfn_t gfn, kvm_pfn_t pfn, bool prefault)
2941
{
2942
	struct kvm_shadow_walk_iterator iterator;
2943
	struct kvm_mmu_page *sp;
2944
	int emulate = 0;
2945
	gfn_t pseudo_gfn;
A
Avi Kivity 已提交
2946

2947 2948 2949
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return 0;

2950
	for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2951
		if (iterator.level == level) {
2952 2953 2954
			emulate = mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
					       write, level, gfn, pfn, prefault,
					       map_writable);
2955
			direct_pte_prefetch(vcpu, iterator.sptep);
2956 2957
			++vcpu->stat.pf_fixed;
			break;
A
Avi Kivity 已提交
2958 2959
		}

2960
		drop_large_spte(vcpu, iterator.sptep);
2961
		if (!is_shadow_present_pte(*iterator.sptep)) {
2962 2963 2964 2965
			u64 base_addr = iterator.addr;

			base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
			pseudo_gfn = base_addr >> PAGE_SHIFT;
2966
			sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2967
					      iterator.level - 1, 1, ACC_ALL);
2968

2969
			link_shadow_page(vcpu, iterator.sptep, sp);
2970 2971
		}
	}
2972
	return emulate;
A
Avi Kivity 已提交
2973 2974
}

H
Huang Ying 已提交
2975
static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2976
{
H
Huang Ying 已提交
2977 2978 2979 2980 2981 2982 2983
	siginfo_t info;

	info.si_signo	= SIGBUS;
	info.si_errno	= 0;
	info.si_code	= BUS_MCEERR_AR;
	info.si_addr	= (void __user *)address;
	info.si_addr_lsb = PAGE_SHIFT;
2984

H
Huang Ying 已提交
2985
	send_sig_info(SIGBUS, &info, tsk);
2986 2987
}

D
Dan Williams 已提交
2988
static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
2989
{
X
Xiao Guangrong 已提交
2990 2991 2992 2993 2994 2995 2996 2997 2998
	/*
	 * Do not cache the mmio info caused by writing the readonly gfn
	 * into the spte otherwise read access on readonly gfn also can
	 * caused mmio page fault and treat it as mmio access.
	 * Return 1 to tell kvm to emulate it.
	 */
	if (pfn == KVM_PFN_ERR_RO_FAULT)
		return 1;

2999
	if (pfn == KVM_PFN_ERR_HWPOISON) {
3000
		kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
3001
		return 0;
3002
	}
3003

3004
	return -EFAULT;
3005 3006
}

3007
static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
D
Dan Williams 已提交
3008 3009
					gfn_t *gfnp, kvm_pfn_t *pfnp,
					int *levelp)
3010
{
D
Dan Williams 已提交
3011
	kvm_pfn_t pfn = *pfnp;
3012 3013 3014 3015 3016 3017 3018 3019 3020
	gfn_t gfn = *gfnp;
	int level = *levelp;

	/*
	 * Check if it's a transparent hugepage. If this would be an
	 * hugetlbfs page, level wouldn't be set to
	 * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
	 * here.
	 */
3021
	if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn) &&
3022
	    level == PT_PAGE_TABLE_LEVEL &&
3023
	    PageTransCompoundMap(pfn_to_page(pfn)) &&
3024
	    !mmu_gfn_lpage_is_disallowed(vcpu, gfn, PT_DIRECTORY_LEVEL)) {
3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042
		unsigned long mask;
		/*
		 * mmu_notifier_retry was successful and we hold the
		 * mmu_lock here, so the pmd can't become splitting
		 * from under us, and in turn
		 * __split_huge_page_refcount() can't run from under
		 * us and we can safely transfer the refcount from
		 * PG_tail to PG_head as we switch the pfn to tail to
		 * head.
		 */
		*levelp = level = PT_DIRECTORY_LEVEL;
		mask = KVM_PAGES_PER_HPAGE(level) - 1;
		VM_BUG_ON((gfn & mask) != (pfn & mask));
		if (pfn & mask) {
			gfn &= ~mask;
			*gfnp = gfn;
			kvm_release_pfn_clean(pfn);
			pfn &= ~mask;
3043
			kvm_get_pfn(pfn);
3044 3045 3046 3047 3048
			*pfnp = pfn;
		}
	}
}

3049
static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
D
Dan Williams 已提交
3050
				kvm_pfn_t pfn, unsigned access, int *ret_val)
3051 3052
{
	/* The pfn is invalid, report the error! */
3053
	if (unlikely(is_error_pfn(pfn))) {
3054
		*ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
3055
		return true;
3056 3057
	}

3058
	if (unlikely(is_noslot_pfn(pfn)))
3059 3060
		vcpu_cache_mmio_info(vcpu, gva, gfn, access);

3061
	return false;
3062 3063
}

3064
static bool page_fault_can_be_fast(u32 error_code)
3065
{
3066 3067 3068 3069 3070 3071 3072
	/*
	 * Do not fix the mmio spte with invalid generation number which
	 * need to be updated by slow page fault path.
	 */
	if (unlikely(error_code & PFERR_RSVD_MASK))
		return false;

3073 3074 3075 3076 3077
	/* See if the page fault is due to an NX violation */
	if (unlikely(((error_code & (PFERR_FETCH_MASK | PFERR_PRESENT_MASK))
		      == (PFERR_FETCH_MASK | PFERR_PRESENT_MASK))))
		return false;

3078
	/*
3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089
	 * #PF can be fast if:
	 * 1. The shadow page table entry is not present, which could mean that
	 *    the fault is potentially caused by access tracking (if enabled).
	 * 2. The shadow page table entry is present and the fault
	 *    is caused by write-protect, that means we just need change the W
	 *    bit of the spte which can be done out of mmu-lock.
	 *
	 * However, if access tracking is disabled we know that a non-present
	 * page must be a genuine page fault where we have to create a new SPTE.
	 * So, if access tracking is disabled, we return true only for write
	 * accesses to a present page.
3090 3091
	 */

3092 3093 3094
	return shadow_acc_track_mask != 0 ||
	       ((error_code & (PFERR_WRITE_MASK | PFERR_PRESENT_MASK))
		== (PFERR_WRITE_MASK | PFERR_PRESENT_MASK));
3095 3096
}

3097 3098 3099 3100
/*
 * Returns true if the SPTE was fixed successfully. Otherwise,
 * someone else modified the SPTE from its original value.
 */
3101
static bool
3102
fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
3103
			u64 *sptep, u64 old_spte, u64 new_spte)
3104 3105 3106 3107 3108
{
	gfn_t gfn;

	WARN_ON(!sp->role.direct);

3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120
	/*
	 * Theoretically we could also set dirty bit (and flush TLB) here in
	 * order to eliminate unnecessary PML logging. See comments in
	 * set_spte. But fast_page_fault is very unlikely to happen with PML
	 * enabled, so we do not do this. This might result in the same GPA
	 * to be logged in PML buffer again when the write really happens, and
	 * eventually to be called by mark_page_dirty twice. But it's also no
	 * harm. This also avoids the TLB flush needed after setting dirty bit
	 * so non-PML cases won't be impacted.
	 *
	 * Compare with set_spte where instead shadow_dirty_mask is set.
	 */
3121
	if (cmpxchg64(sptep, old_spte, new_spte) != old_spte)
3122 3123
		return false;

3124
	if (is_writable_pte(new_spte) && !is_writable_pte(old_spte)) {
3125 3126 3127 3128 3129 3130 3131
		/*
		 * The gfn of direct spte is stable since it is
		 * calculated by sp->gfn.
		 */
		gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
		kvm_vcpu_mark_page_dirty(vcpu, gfn);
	}
3132 3133 3134 3135

	return true;
}

3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147
static bool is_access_allowed(u32 fault_err_code, u64 spte)
{
	if (fault_err_code & PFERR_FETCH_MASK)
		return is_executable_pte(spte);

	if (fault_err_code & PFERR_WRITE_MASK)
		return is_writable_pte(spte);

	/* Fault was on Read access */
	return spte & PT_PRESENT_MASK;
}

3148 3149 3150 3151 3152 3153 3154 3155 3156
/*
 * Return value:
 * - true: let the vcpu to access on the same address again.
 * - false: let the real page fault path to fix it.
 */
static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
			    u32 error_code)
{
	struct kvm_shadow_walk_iterator iterator;
3157
	struct kvm_mmu_page *sp;
3158
	bool fault_handled = false;
3159
	u64 spte = 0ull;
3160
	uint retry_count = 0;
3161

3162 3163 3164
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return false;

3165
	if (!page_fault_can_be_fast(error_code))
3166 3167 3168 3169
		return false;

	walk_shadow_page_lockless_begin(vcpu);

3170
	do {
3171
		u64 new_spte;
3172

3173 3174 3175 3176 3177
		for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
			if (!is_shadow_present_pte(spte) ||
			    iterator.level < level)
				break;

3178 3179 3180
		sp = page_header(__pa(iterator.sptep));
		if (!is_last_spte(spte, sp->role.level))
			break;
3181

3182
		/*
3183 3184 3185 3186 3187
		 * Check whether the memory access that caused the fault would
		 * still cause it if it were to be performed right now. If not,
		 * then this is a spurious fault caused by TLB lazily flushed,
		 * or some other CPU has already fixed the PTE after the
		 * current CPU took the fault.
3188 3189 3190 3191
		 *
		 * Need not check the access of upper level table entries since
		 * they are always ACC_ALL.
		 */
3192 3193 3194 3195
		if (is_access_allowed(error_code, spte)) {
			fault_handled = true;
			break;
		}
3196

3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210
		new_spte = spte;

		if (is_access_track_spte(spte))
			new_spte = restore_acc_track_spte(new_spte);

		/*
		 * Currently, to simplify the code, write-protection can
		 * be removed in the fast path only if the SPTE was
		 * write-protected for dirty-logging or access tracking.
		 */
		if ((error_code & PFERR_WRITE_MASK) &&
		    spte_can_locklessly_be_made_writable(spte))
		{
			new_spte |= PT_WRITABLE_MASK;
3211 3212

			/*
3213 3214 3215 3216 3217 3218 3219 3220 3221
			 * Do not fix write-permission on the large spte.  Since
			 * we only dirty the first page into the dirty-bitmap in
			 * fast_pf_fix_direct_spte(), other pages are missed
			 * if its slot has dirty logging enabled.
			 *
			 * Instead, we let the slow page fault path create a
			 * normal spte to fix the access.
			 *
			 * See the comments in kvm_arch_commit_memory_region().
3222
			 */
3223
			if (sp->role.level > PT_PAGE_TABLE_LEVEL)
3224
				break;
3225
		}
3226

3227
		/* Verify that the fault can be handled in the fast path */
3228 3229
		if (new_spte == spte ||
		    !is_access_allowed(error_code, new_spte))
3230 3231 3232 3233 3234 3235 3236 3237
			break;

		/*
		 * Currently, fast page fault only works for direct mapping
		 * since the gfn is not stable for indirect shadow page. See
		 * Documentation/virtual/kvm/locking.txt to get more detail.
		 */
		fault_handled = fast_pf_fix_direct_spte(vcpu, sp,
3238
							iterator.sptep, spte,
3239
							new_spte);
3240 3241 3242 3243 3244 3245 3246 3247 3248 3249
		if (fault_handled)
			break;

		if (++retry_count > 4) {
			printk_once(KERN_WARNING
				"kvm: Fast #PF retrying more than 4 times.\n");
			break;
		}

	} while (true);
3250

X
Xiao Guangrong 已提交
3251
	trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
3252
			      spte, fault_handled);
3253 3254
	walk_shadow_page_lockless_end(vcpu);

3255
	return fault_handled;
3256 3257
}

3258
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
D
Dan Williams 已提交
3259
			 gva_t gva, kvm_pfn_t *pfn, bool write, bool *writable);
3260
static int make_mmu_pages_available(struct kvm_vcpu *vcpu);
3261

3262 3263
static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
			 gfn_t gfn, bool prefault)
3264 3265
{
	int r;
3266
	int level;
3267
	bool force_pt_level = false;
D
Dan Williams 已提交
3268
	kvm_pfn_t pfn;
3269
	unsigned long mmu_seq;
3270
	bool map_writable, write = error_code & PFERR_WRITE_MASK;
3271

3272
	level = mapping_level(vcpu, gfn, &force_pt_level);
3273 3274 3275 3276 3277 3278 3279 3280
	if (likely(!force_pt_level)) {
		/*
		 * This path builds a PAE pagetable - so we can map
		 * 2mb pages at maximum. Therefore check if the level
		 * is larger than that.
		 */
		if (level > PT_DIRECTORY_LEVEL)
			level = PT_DIRECTORY_LEVEL;
3281

3282
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3283
	}
M
Marcelo Tosatti 已提交
3284

3285 3286 3287
	if (fast_page_fault(vcpu, v, level, error_code))
		return 0;

3288
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
3289
	smp_rmb();
3290

3291
	if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
3292
		return 0;
3293

3294 3295
	if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
		return r;
3296

3297
	spin_lock(&vcpu->kvm->mmu_lock);
3298
	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3299
		goto out_unlock;
3300 3301
	if (make_mmu_pages_available(vcpu) < 0)
		goto out_unlock;
3302 3303
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3304
	r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3305 3306
	spin_unlock(&vcpu->kvm->mmu_lock);

3307
	return r;
3308 3309 3310 3311 3312

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
3313 3314 3315
}


3316 3317 3318
static void mmu_free_roots(struct kvm_vcpu *vcpu)
{
	int i;
3319
	struct kvm_mmu_page *sp;
3320
	LIST_HEAD(invalid_list);
3321

3322
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
A
Avi Kivity 已提交
3323
		return;
3324

3325 3326
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_4LEVEL &&
	    (vcpu->arch.mmu.root_level == PT64_ROOT_4LEVEL ||
3327
	     vcpu->arch.mmu.direct_map)) {
3328
		hpa_t root = vcpu->arch.mmu.root_hpa;
3329

3330
		spin_lock(&vcpu->kvm->mmu_lock);
3331 3332
		sp = page_header(root);
		--sp->root_count;
3333 3334 3335 3336
		if (!sp->root_count && sp->role.invalid) {
			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
			kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
		}
3337
		spin_unlock(&vcpu->kvm->mmu_lock);
3338
		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3339 3340
		return;
	}
3341 3342

	spin_lock(&vcpu->kvm->mmu_lock);
3343
	for (i = 0; i < 4; ++i) {
3344
		hpa_t root = vcpu->arch.mmu.pae_root[i];
3345

A
Avi Kivity 已提交
3346 3347
		if (root) {
			root &= PT64_BASE_ADDR_MASK;
3348 3349
			sp = page_header(root);
			--sp->root_count;
3350
			if (!sp->root_count && sp->role.invalid)
3351 3352
				kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
							 &invalid_list);
A
Avi Kivity 已提交
3353
		}
3354
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
3355
	}
3356
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3357
	spin_unlock(&vcpu->kvm->mmu_lock);
3358
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3359 3360
}

3361 3362 3363 3364 3365
static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
{
	int ret = 0;

	if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
3366
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3367 3368 3369 3370 3371 3372
		ret = 1;
	}

	return ret;
}

3373 3374 3375
static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_page *sp;
3376
	unsigned i;
3377

3378
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_4LEVEL) {
3379
		spin_lock(&vcpu->kvm->mmu_lock);
3380 3381 3382 3383
		if(make_mmu_pages_available(vcpu) < 0) {
			spin_unlock(&vcpu->kvm->mmu_lock);
			return 1;
		}
3384
		sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_4LEVEL, 1, ACC_ALL);
3385 3386 3387 3388 3389 3390 3391
		++sp->root_count;
		spin_unlock(&vcpu->kvm->mmu_lock);
		vcpu->arch.mmu.root_hpa = __pa(sp->spt);
	} else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
		for (i = 0; i < 4; ++i) {
			hpa_t root = vcpu->arch.mmu.pae_root[i];

3392
			MMU_WARN_ON(VALID_PAGE(root));
3393
			spin_lock(&vcpu->kvm->mmu_lock);
3394 3395 3396 3397
			if (make_mmu_pages_available(vcpu) < 0) {
				spin_unlock(&vcpu->kvm->mmu_lock);
				return 1;
			}
3398
			sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
3399
					i << 30, PT32_ROOT_LEVEL, 1, ACC_ALL);
3400 3401 3402 3403 3404
			root = __pa(sp->spt);
			++sp->root_count;
			spin_unlock(&vcpu->kvm->mmu_lock);
			vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
		}
3405
		vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3406 3407 3408 3409 3410 3411 3412
	} else
		BUG();

	return 0;
}

static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3413
{
3414
	struct kvm_mmu_page *sp;
3415 3416 3417
	u64 pdptr, pm_mask;
	gfn_t root_gfn;
	int i;
3418

3419
	root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
3420

3421 3422 3423 3424 3425 3426 3427
	if (mmu_check_root(vcpu, root_gfn))
		return 1;

	/*
	 * Do we shadow a long mode page table? If so we need to
	 * write-protect the guests page table root.
	 */
3428
	if (vcpu->arch.mmu.root_level == PT64_ROOT_4LEVEL) {
3429
		hpa_t root = vcpu->arch.mmu.root_hpa;
3430

3431
		MMU_WARN_ON(VALID_PAGE(root));
3432

3433
		spin_lock(&vcpu->kvm->mmu_lock);
3434 3435 3436 3437
		if (make_mmu_pages_available(vcpu) < 0) {
			spin_unlock(&vcpu->kvm->mmu_lock);
			return 1;
		}
3438
		sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_4LEVEL,
3439
				      0, ACC_ALL);
3440 3441
		root = __pa(sp->spt);
		++sp->root_count;
3442
		spin_unlock(&vcpu->kvm->mmu_lock);
3443
		vcpu->arch.mmu.root_hpa = root;
3444
		return 0;
3445
	}
3446

3447 3448
	/*
	 * We shadow a 32 bit page table. This may be a legacy 2-level
3449 3450
	 * or a PAE 3-level page table. In either case we need to be aware that
	 * the shadow page table may be a PAE or a long mode page table.
3451
	 */
3452
	pm_mask = PT_PRESENT_MASK;
3453
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_4LEVEL)
3454 3455
		pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;

3456
	for (i = 0; i < 4; ++i) {
3457
		hpa_t root = vcpu->arch.mmu.pae_root[i];
3458

3459
		MMU_WARN_ON(VALID_PAGE(root));
3460
		if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
3461
			pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
B
Bandan Das 已提交
3462
			if (!(pdptr & PT_PRESENT_MASK)) {
3463
				vcpu->arch.mmu.pae_root[i] = 0;
A
Avi Kivity 已提交
3464 3465
				continue;
			}
A
Avi Kivity 已提交
3466
			root_gfn = pdptr >> PAGE_SHIFT;
3467 3468
			if (mmu_check_root(vcpu, root_gfn))
				return 1;
3469
		}
3470
		spin_lock(&vcpu->kvm->mmu_lock);
3471 3472 3473 3474
		if (make_mmu_pages_available(vcpu) < 0) {
			spin_unlock(&vcpu->kvm->mmu_lock);
			return 1;
		}
3475 3476
		sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL,
				      0, ACC_ALL);
3477 3478
		root = __pa(sp->spt);
		++sp->root_count;
3479 3480
		spin_unlock(&vcpu->kvm->mmu_lock);

3481
		vcpu->arch.mmu.pae_root[i] = root | pm_mask;
3482
	}
3483
	vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3484 3485 3486 3487 3488

	/*
	 * If we shadow a 32 bit page table with a long mode page
	 * table we enter this path.
	 */
3489
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_4LEVEL) {
3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509
		if (vcpu->arch.mmu.lm_root == NULL) {
			/*
			 * The additional page necessary for this is only
			 * allocated on demand.
			 */

			u64 *lm_root;

			lm_root = (void*)get_zeroed_page(GFP_KERNEL);
			if (lm_root == NULL)
				return 1;

			lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;

			vcpu->arch.mmu.lm_root = lm_root;
		}

		vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
	}

3510
	return 0;
3511 3512
}

3513 3514 3515 3516 3517 3518 3519 3520
static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
{
	if (vcpu->arch.mmu.direct_map)
		return mmu_alloc_direct_roots(vcpu);
	else
		return mmu_alloc_shadow_roots(vcpu);
}

3521 3522 3523 3524 3525
static void mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	int i;
	struct kvm_mmu_page *sp;

3526 3527 3528
	if (vcpu->arch.mmu.direct_map)
		return;

3529 3530
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return;
3531

3532
	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3533
	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3534
	if (vcpu->arch.mmu.root_level == PT64_ROOT_4LEVEL) {
3535 3536 3537
		hpa_t root = vcpu->arch.mmu.root_hpa;
		sp = page_header(root);
		mmu_sync_children(vcpu, sp);
3538
		kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3539 3540 3541 3542 3543
		return;
	}
	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->arch.mmu.pae_root[i];

3544
		if (root && VALID_PAGE(root)) {
3545 3546 3547 3548 3549
			root &= PT64_BASE_ADDR_MASK;
			sp = page_header(root);
			mmu_sync_children(vcpu, sp);
		}
	}
3550
	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3551 3552 3553 3554 3555 3556
}

void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	spin_lock(&vcpu->kvm->mmu_lock);
	mmu_sync_roots(vcpu);
3557
	spin_unlock(&vcpu->kvm->mmu_lock);
3558
}
N
Nadav Har'El 已提交
3559
EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
3560

3561
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
3562
				  u32 access, struct x86_exception *exception)
A
Avi Kivity 已提交
3563
{
3564 3565
	if (exception)
		exception->error_code = 0;
A
Avi Kivity 已提交
3566 3567 3568
	return vaddr;
}

3569
static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
3570 3571
					 u32 access,
					 struct x86_exception *exception)
3572
{
3573 3574
	if (exception)
		exception->error_code = 0;
3575
	return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception);
3576 3577
}

3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596
static bool
__is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, u64 pte, int level)
{
	int bit7 = (pte >> 7) & 1, low6 = pte & 0x3f;

	return (pte & rsvd_check->rsvd_bits_mask[bit7][level-1]) |
		((rsvd_check->bad_mt_xwr & (1ull << low6)) != 0);
}

static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
{
	return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level);
}

static bool is_shadow_zero_bits_set(struct kvm_mmu *mmu, u64 spte, int level)
{
	return __is_rsvd_bits_set(&mmu->shadow_zero_check, spte, level);
}

3597
static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3598
{
3599 3600 3601 3602 3603 3604 3605
	/*
	 * A nested guest cannot use the MMIO cache if it is using nested
	 * page tables, because cr2 is a nGPA while the cache stores GPAs.
	 */
	if (mmu_is_nested(vcpu))
		return false;

3606 3607 3608 3609 3610 3611
	if (direct)
		return vcpu_match_mmio_gpa(vcpu, addr);

	return vcpu_match_mmio_gva(vcpu, addr);
}

3612 3613 3614
/* return true if reserved bit is detected on spte. */
static bool
walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
3615 3616
{
	struct kvm_shadow_walk_iterator iterator;
3617
	u64 sptes[PT64_ROOT_MAX_LEVEL], spte = 0ull;
3618 3619
	int root, leaf;
	bool reserved = false;
3620

3621
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3622
		goto exit;
3623

3624
	walk_shadow_page_lockless_begin(vcpu);
3625

3626 3627
	for (shadow_walk_init(&iterator, vcpu, addr),
		 leaf = root = iterator.level;
3628 3629 3630 3631 3632
	     shadow_walk_okay(&iterator);
	     __shadow_walk_next(&iterator, spte)) {
		spte = mmu_spte_get_lockless(iterator.sptep);

		sptes[leaf - 1] = spte;
3633
		leaf--;
3634

3635 3636
		if (!is_shadow_present_pte(spte))
			break;
3637 3638

		reserved |= is_shadow_zero_bits_set(&vcpu->arch.mmu, spte,
3639
						    iterator.level);
3640 3641
	}

3642 3643
	walk_shadow_page_lockless_end(vcpu);

3644 3645 3646
	if (reserved) {
		pr_err("%s: detect reserved bits on spte, addr 0x%llx, dump hierarchy:\n",
		       __func__, addr);
3647
		while (root > leaf) {
3648 3649 3650 3651 3652 3653 3654 3655
			pr_err("------ spte 0x%llx level %d.\n",
			       sptes[root - 1], root);
			root--;
		}
	}
exit:
	*sptep = spte;
	return reserved;
3656 3657
}

P
Paolo Bonzini 已提交
3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674
/*
 * Return values of handle_mmio_page_fault:
 * RET_MMIO_PF_EMULATE: it is a real mmio page fault, emulate the instruction
 *			directly.
 * RET_MMIO_PF_INVALID: invalid spte is detected then let the real page
 *			fault path update the mmio spte.
 * RET_MMIO_PF_RETRY: let CPU fault again on the address.
 * RET_MMIO_PF_BUG: a bug was detected (and a WARN was printed).
 */
enum {
	RET_MMIO_PF_EMULATE = 1,
	RET_MMIO_PF_INVALID = 2,
	RET_MMIO_PF_RETRY = 0,
	RET_MMIO_PF_BUG = -1
};

static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3675 3676
{
	u64 spte;
3677
	bool reserved;
3678

3679
	if (mmio_info_in_cache(vcpu, addr, direct))
3680
		return RET_MMIO_PF_EMULATE;
3681

3682
	reserved = walk_shadow_page_get_mmio_spte(vcpu, addr, &spte);
3683
	if (WARN_ON(reserved))
3684
		return RET_MMIO_PF_BUG;
3685 3686 3687 3688 3689

	if (is_mmio_spte(spte)) {
		gfn_t gfn = get_mmio_spte_gfn(spte);
		unsigned access = get_mmio_spte_access(spte);

3690
		if (!check_mmio_spte(vcpu, spte))
3691 3692
			return RET_MMIO_PF_INVALID;

3693 3694
		if (direct)
			addr = 0;
X
Xiao Guangrong 已提交
3695 3696

		trace_handle_mmio_page_fault(addr, gfn, access);
3697
		vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3698
		return RET_MMIO_PF_EMULATE;
3699 3700 3701 3702 3703 3704
	}

	/*
	 * If the page table is zapped by other cpus, let CPU fault again on
	 * the address.
	 */
3705
	return RET_MMIO_PF_RETRY;
3706
}
3707
EXPORT_SYMBOL_GPL(handle_mmio_page_fault);
3708

3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728
static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
					 u32 error_code, gfn_t gfn)
{
	if (unlikely(error_code & PFERR_RSVD_MASK))
		return false;

	if (!(error_code & PFERR_PRESENT_MASK) ||
	      !(error_code & PFERR_WRITE_MASK))
		return false;

	/*
	 * guest is writing the page which is write tracked which can
	 * not be fixed by page fault handler.
	 */
	if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE))
		return true;

	return false;
}

3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745
static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
{
	struct kvm_shadow_walk_iterator iterator;
	u64 spte;

	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return;

	walk_shadow_page_lockless_begin(vcpu);
	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {
		clear_sp_write_flooding_count(iterator.sptep);
		if (!is_shadow_present_pte(spte))
			break;
	}
	walk_shadow_page_lockless_end(vcpu);
}

A
Avi Kivity 已提交
3746
static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
3747
				u32 error_code, bool prefault)
A
Avi Kivity 已提交
3748
{
3749
	gfn_t gfn = gva >> PAGE_SHIFT;
3750
	int r;
A
Avi Kivity 已提交
3751

3752
	pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
3753

3754 3755
	if (page_fault_handle_page_track(vcpu, error_code, gfn))
		return 1;
3756

3757 3758 3759
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;
3760

3761
	MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
3762 3763


3764
	return nonpaging_map(vcpu, gva & PAGE_MASK,
3765
			     error_code, gfn, prefault);
A
Avi Kivity 已提交
3766 3767
}

3768
static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
3769 3770
{
	struct kvm_arch_async_pf arch;
X
Xiao Guangrong 已提交
3771

3772
	arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3773
	arch.gfn = gfn;
3774
	arch.direct_map = vcpu->arch.mmu.direct_map;
X
Xiao Guangrong 已提交
3775
	arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
3776

3777
	return kvm_setup_async_pf(vcpu, gva, kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
3778 3779
}

3780
bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
3781
{
3782
	if (unlikely(!lapic_in_kernel(vcpu) ||
3783 3784 3785
		     kvm_event_needs_reinjection(vcpu)))
		return false;

3786
	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
3787 3788
		return false;

3789 3790 3791
	return kvm_x86_ops->interrupt_allowed(vcpu);
}

3792
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
D
Dan Williams 已提交
3793
			 gva_t gva, kvm_pfn_t *pfn, bool write, bool *writable)
3794
{
3795
	struct kvm_memory_slot *slot;
3796 3797
	bool async;

3798
	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3799 3800
	async = false;
	*pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable);
3801 3802 3803
	if (!async)
		return false; /* *pfn has correct page already */

3804
	if (!prefault && kvm_can_do_async_pf(vcpu)) {
3805
		trace_kvm_try_async_get_page(gva, gfn);
3806 3807 3808 3809 3810 3811 3812 3813
		if (kvm_find_async_pf_gfn(vcpu, gfn)) {
			trace_kvm_async_pf_doublefault(gva, gfn);
			kvm_make_request(KVM_REQ_APF_HALT, vcpu);
			return true;
		} else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
			return true;
	}

3814
	*pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, write, writable);
3815 3816 3817
	return false;
}

3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849
int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
				u64 fault_address, char *insn, int insn_len,
				bool need_unprotect)
{
	int r = 1;

	switch (vcpu->arch.apf.host_apf_reason) {
	default:
		trace_kvm_page_fault(fault_address, error_code);

		if (need_unprotect && kvm_event_needs_reinjection(vcpu))
			kvm_mmu_unprotect_page_virt(vcpu, fault_address);
		r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
				insn_len);
		break;
	case KVM_PV_REASON_PAGE_NOT_PRESENT:
		vcpu->arch.apf.host_apf_reason = 0;
		local_irq_disable();
		kvm_async_pf_task_wait(fault_address);
		local_irq_enable();
		break;
	case KVM_PV_REASON_PAGE_READY:
		vcpu->arch.apf.host_apf_reason = 0;
		local_irq_disable();
		kvm_async_pf_task_wake(fault_address);
		local_irq_enable();
		break;
	}
	return r;
}
EXPORT_SYMBOL_GPL(kvm_handle_page_fault);

3850 3851 3852 3853 3854 3855 3856 3857 3858 3859
static bool
check_hugepage_cache_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, int level)
{
	int page_num = KVM_PAGES_PER_HPAGE(level);

	gfn &= ~(page_num - 1);

	return kvm_mtrr_check_gfn_range_consistency(vcpu, gfn, page_num);
}

G
Gleb Natapov 已提交
3860
static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
3861
			  bool prefault)
3862
{
D
Dan Williams 已提交
3863
	kvm_pfn_t pfn;
3864
	int r;
3865
	int level;
3866
	bool force_pt_level;
M
Marcelo Tosatti 已提交
3867
	gfn_t gfn = gpa >> PAGE_SHIFT;
3868
	unsigned long mmu_seq;
3869 3870
	int write = error_code & PFERR_WRITE_MASK;
	bool map_writable;
3871

3872
	MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3873

3874 3875
	if (page_fault_handle_page_track(vcpu, error_code, gfn))
		return 1;
3876

3877 3878 3879 3880
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

3881 3882 3883
	force_pt_level = !check_hugepage_cache_consistency(vcpu, gfn,
							   PT_DIRECTORY_LEVEL);
	level = mapping_level(vcpu, gfn, &force_pt_level);
3884
	if (likely(!force_pt_level)) {
3885 3886 3887
		if (level > PT_DIRECTORY_LEVEL &&
		    !check_hugepage_cache_consistency(vcpu, gfn, level))
			level = PT_DIRECTORY_LEVEL;
3888
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3889
	}
3890

3891 3892 3893
	if (fast_page_fault(vcpu, gpa, level, error_code))
		return 0;

3894
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
3895
	smp_rmb();
3896

3897
	if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
3898 3899
		return 0;

3900 3901 3902
	if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
		return r;

3903
	spin_lock(&vcpu->kvm->mmu_lock);
3904
	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3905
		goto out_unlock;
3906 3907
	if (make_mmu_pages_available(vcpu) < 0)
		goto out_unlock;
3908 3909
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3910
	r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3911 3912 3913
	spin_unlock(&vcpu->kvm->mmu_lock);

	return r;
3914 3915 3916 3917 3918

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
3919 3920
}

3921 3922
static void nonpaging_init_context(struct kvm_vcpu *vcpu,
				   struct kvm_mmu *context)
A
Avi Kivity 已提交
3923 3924 3925
{
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
3926
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
3927
	context->invlpg = nonpaging_invlpg;
3928
	context->update_pte = nonpaging_update_pte;
3929
	context->root_level = 0;
A
Avi Kivity 已提交
3930
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
3931
	context->root_hpa = INVALID_PAGE;
3932
	context->direct_map = true;
3933
	context->nx = false;
A
Avi Kivity 已提交
3934 3935
}

3936
void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3937
{
3938
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
3939 3940
}

3941 3942
static unsigned long get_cr3(struct kvm_vcpu *vcpu)
{
3943
	return kvm_read_cr3(vcpu);
3944 3945
}

3946 3947
static void inject_page_fault(struct kvm_vcpu *vcpu,
			      struct x86_exception *fault)
A
Avi Kivity 已提交
3948
{
3949
	vcpu->arch.mmu.inject_page_fault(vcpu, fault);
A
Avi Kivity 已提交
3950 3951
}

3952
static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
3953
			   unsigned access, int *nr_present)
3954 3955 3956 3957 3958 3959 3960 3961
{
	if (unlikely(is_mmio_spte(*sptep))) {
		if (gfn != get_mmio_spte_gfn(*sptep)) {
			mmu_spte_clear_no_track(sptep);
			return true;
		}

		(*nr_present)++;
3962
		mark_mmio_spte(vcpu, sptep, gfn, access);
3963 3964 3965 3966 3967 3968
		return true;
	}

	return false;
}

3969 3970
static inline bool is_last_gpte(struct kvm_mmu *mmu,
				unsigned level, unsigned gpte)
A
Avi Kivity 已提交
3971
{
3972 3973 3974 3975 3976 3977
	/*
	 * PT_PAGE_TABLE_LEVEL always terminates.  The RHS has bit 7 set
	 * iff level <= PT_PAGE_TABLE_LEVEL, which for our purpose means
	 * level == PT_PAGE_TABLE_LEVEL; set PT_PAGE_SIZE_MASK in gpte then.
	 */
	gpte |= level - PT_PAGE_TABLE_LEVEL - 1;
A
Avi Kivity 已提交
3978

3979 3980 3981 3982 3983 3984 3985 3986
	/*
	 * The RHS has bit 7 set iff level < mmu->last_nonleaf_level.
	 * If it is clear, there are no large pages at this level, so clear
	 * PT_PAGE_SIZE_MASK in gpte if that is the case.
	 */
	gpte &= level - mmu->last_nonleaf_level;

	return gpte & PT_PAGE_SIZE_MASK;
A
Avi Kivity 已提交
3987 3988
}

3989 3990 3991 3992 3993
#define PTTYPE_EPT 18 /* arbitrary */
#define PTTYPE PTTYPE_EPT
#include "paging_tmpl.h"
#undef PTTYPE

A
Avi Kivity 已提交
3994 3995 3996 3997 3998 3999 4000 4001
#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

4002 4003 4004 4005
static void
__reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
			struct rsvd_bits_validate *rsvd_check,
			int maxphyaddr, int level, bool nx, bool gbpages,
4006
			bool pse, bool amd)
4007 4008
{
	u64 exb_bit_rsvd = 0;
4009
	u64 gbpages_bit_rsvd = 0;
4010
	u64 nonleaf_bit8_rsvd = 0;
4011

4012
	rsvd_check->bad_mt_xwr = 0;
4013

4014
	if (!nx)
4015
		exb_bit_rsvd = rsvd_bits(63, 63);
4016
	if (!gbpages)
4017
		gbpages_bit_rsvd = rsvd_bits(7, 7);
4018 4019 4020 4021 4022

	/*
	 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
	 * leaf entries) on AMD CPUs only.
	 */
4023
	if (amd)
4024 4025
		nonleaf_bit8_rsvd = rsvd_bits(8, 8);

4026
	switch (level) {
4027 4028
	case PT32_ROOT_LEVEL:
		/* no rsvd bits for 2 level 4K page table entries */
4029 4030 4031 4032
		rsvd_check->rsvd_bits_mask[0][1] = 0;
		rsvd_check->rsvd_bits_mask[0][0] = 0;
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4033

4034
		if (!pse) {
4035
			rsvd_check->rsvd_bits_mask[1][1] = 0;
4036 4037 4038
			break;
		}

4039 4040
		if (is_cpuid_PSE36())
			/* 36bits PSE 4MB page */
4041
			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
4042 4043
		else
			/* 32 bits PSE 4MB page */
4044
			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
4045 4046
		break;
	case PT32E_ROOT_LEVEL:
4047
		rsvd_check->rsvd_bits_mask[0][2] =
4048
			rsvd_bits(maxphyaddr, 63) |
4049
			rsvd_bits(5, 8) | rsvd_bits(1, 2);	/* PDPTE */
4050
		rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
4051
			rsvd_bits(maxphyaddr, 62);	/* PDE */
4052
		rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
4053
			rsvd_bits(maxphyaddr, 62); 	/* PTE */
4054
		rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
4055 4056
			rsvd_bits(maxphyaddr, 62) |
			rsvd_bits(13, 20);		/* large page */
4057 4058
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4059
		break;
4060
	case PT64_ROOT_4LEVEL:
4061 4062
		rsvd_check->rsvd_bits_mask[0][3] = exb_bit_rsvd |
			nonleaf_bit8_rsvd | rsvd_bits(7, 7) |
4063
			rsvd_bits(maxphyaddr, 51);
4064 4065
		rsvd_check->rsvd_bits_mask[0][2] = exb_bit_rsvd |
			nonleaf_bit8_rsvd | gbpages_bit_rsvd |
4066
			rsvd_bits(maxphyaddr, 51);
4067 4068 4069 4070 4071 4072 4073
		rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51);
		rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51);
		rsvd_check->rsvd_bits_mask[1][3] =
			rsvd_check->rsvd_bits_mask[0][3];
		rsvd_check->rsvd_bits_mask[1][2] = exb_bit_rsvd |
4074
			gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51) |
4075
			rsvd_bits(13, 29);
4076
		rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
4077 4078
			rsvd_bits(maxphyaddr, 51) |
			rsvd_bits(13, 20);		/* large page */
4079 4080
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4081 4082 4083 4084
		break;
	}
}

4085 4086 4087 4088 4089
static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
{
	__reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check,
				cpuid_maxphyaddr(vcpu), context->root_level,
4090 4091
				context->nx,
				guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),
4092
				is_pse(vcpu), guest_cpuid_is_amd(vcpu));
4093 4094
}

4095 4096 4097
static void
__reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
			    int maxphyaddr, bool execonly)
4098
{
4099
	u64 bad_mt_xwr;
4100

4101
	rsvd_check->rsvd_bits_mask[0][3] =
4102
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7);
4103
	rsvd_check->rsvd_bits_mask[0][2] =
4104
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
4105
	rsvd_check->rsvd_bits_mask[0][1] =
4106
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
4107
	rsvd_check->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51);
4108 4109

	/* large page */
4110 4111
	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
	rsvd_check->rsvd_bits_mask[1][2] =
4112
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 29);
4113
	rsvd_check->rsvd_bits_mask[1][1] =
4114
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 20);
4115
	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
4116

4117 4118 4119 4120 4121 4122 4123 4124
	bad_mt_xwr = 0xFFull << (2 * 8);	/* bits 3..5 must not be 2 */
	bad_mt_xwr |= 0xFFull << (3 * 8);	/* bits 3..5 must not be 3 */
	bad_mt_xwr |= 0xFFull << (7 * 8);	/* bits 3..5 must not be 7 */
	bad_mt_xwr |= REPEAT_BYTE(1ull << 2);	/* bits 0..2 must not be 010 */
	bad_mt_xwr |= REPEAT_BYTE(1ull << 6);	/* bits 0..2 must not be 110 */
	if (!execonly) {
		/* bits 0..2 must not be 100 unless VMX capabilities allow it */
		bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
4125
	}
4126
	rsvd_check->bad_mt_xwr = bad_mt_xwr;
4127 4128
}

4129 4130 4131 4132 4133 4134 4135
static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
		struct kvm_mmu *context, bool execonly)
{
	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
				    cpuid_maxphyaddr(vcpu), execonly);
}

4136 4137 4138 4139 4140 4141 4142 4143
/*
 * the page table on host is the shadow page table for the page
 * table in guest or amd nested guest, its mmu features completely
 * follow the features in guest.
 */
void
reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
{
4144 4145
	bool uses_nx = context->nx || context->base_role.smep_andnot_wp;

4146 4147 4148 4149
	/*
	 * Passing "true" to the last argument is okay; it adds a check
	 * on bit 8 of the SPTEs which KVM doesn't use anyway.
	 */
4150 4151
	__reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
				boot_cpu_data.x86_phys_bits,
4152
				context->shadow_root_level, uses_nx,
4153 4154
				guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),
				is_pse(vcpu), true);
4155 4156 4157
}
EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask);

4158 4159 4160 4161 4162 4163
static inline bool boot_cpu_is_amd(void)
{
	WARN_ON_ONCE(!tdp_enabled);
	return shadow_x_mask == 0;
}

4164 4165 4166 4167 4168 4169 4170 4171
/*
 * the direct page table on host, use as much mmu features as
 * possible, however, kvm currently does not do execution-protection.
 */
static void
reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
				struct kvm_mmu *context)
{
4172
	if (boot_cpu_is_amd())
4173 4174 4175
		__reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
					boot_cpu_data.x86_phys_bits,
					context->shadow_root_level, false,
4176 4177
					boot_cpu_has(X86_FEATURE_GBPAGES),
					true, true);
4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196
	else
		__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
					    boot_cpu_data.x86_phys_bits,
					    false);

}

/*
 * as the comments in reset_shadow_zero_bits_mask() except it
 * is the shadow page table for intel nested guest.
 */
static void
reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
				struct kvm_mmu *context, bool execonly)
{
	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
				    boot_cpu_data.x86_phys_bits, execonly);
}

4197 4198
static void update_permission_bitmask(struct kvm_vcpu *vcpu,
				      struct kvm_mmu *mmu, bool ept)
4199 4200 4201
{
	unsigned bit, byte, pfec;
	u8 map;
F
Feng Wu 已提交
4202
	bool fault, x, w, u, wf, uf, ff, smapf, cr4_smap, cr4_smep, smap = 0;
4203

F
Feng Wu 已提交
4204
	cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
F
Feng Wu 已提交
4205
	cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
4206 4207 4208 4209 4210 4211
	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
		pfec = byte << 1;
		map = 0;
		wf = pfec & PFERR_WRITE_MASK;
		uf = pfec & PFERR_USER_MASK;
		ff = pfec & PFERR_FETCH_MASK;
F
Feng Wu 已提交
4212 4213 4214 4215 4216 4217
		/*
		 * PFERR_RSVD_MASK bit is set in PFEC if the access is not
		 * subject to SMAP restrictions, and cleared otherwise. The
		 * bit is only meaningful if the SMAP bit is set in CR4.
		 */
		smapf = !(pfec & PFERR_RSVD_MASK);
4218 4219 4220 4221 4222
		for (bit = 0; bit < 8; ++bit) {
			x = bit & ACC_EXEC_MASK;
			w = bit & ACC_WRITE_MASK;
			u = bit & ACC_USER_MASK;

4223 4224 4225 4226 4227 4228
			if (!ept) {
				/* Not really needed: !nx will cause pte.nx to fault */
				x |= !mmu->nx;
				/* Allow supervisor writes if !cr0.wp */
				w |= !is_write_protection(vcpu) && !uf;
				/* Disallow supervisor fetches of user code if cr4.smep */
F
Feng Wu 已提交
4229
				x &= !(cr4_smep && u && !uf);
F
Feng Wu 已提交
4230 4231 4232 4233 4234 4235 4236

				/*
				 * SMAP:kernel-mode data accesses from user-mode
				 * mappings should fault. A fault is considered
				 * as a SMAP violation if all of the following
				 * conditions are ture:
				 *   - X86_CR4_SMAP is set in CR4
4237
				 *   - A user page is accessed
F
Feng Wu 已提交
4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249
				 *   - Page fault in kernel mode
				 *   - if CPL = 3 or X86_EFLAGS_AC is clear
				 *
				 *   Here, we cover the first three conditions.
				 *   The fourth is computed dynamically in
				 *   permission_fault() and is in smapf.
				 *
				 *   Also, SMAP does not affect instruction
				 *   fetches, add the !ff check here to make it
				 *   clearer.
				 */
				smap = cr4_smap && u && !uf && !ff;
4250
			}
4251

F
Feng Wu 已提交
4252 4253
			fault = (ff && !x) || (uf && !u) || (wf && !w) ||
				(smapf && smap);
4254 4255 4256 4257 4258 4259
			map |= fault << bit;
		}
		mmu->permissions[byte] = map;
	}
}

4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334
/*
* PKU is an additional mechanism by which the paging controls access to
* user-mode addresses based on the value in the PKRU register.  Protection
* key violations are reported through a bit in the page fault error code.
* Unlike other bits of the error code, the PK bit is not known at the
* call site of e.g. gva_to_gpa; it must be computed directly in
* permission_fault based on two bits of PKRU, on some machine state (CR4,
* CR0, EFER, CPL), and on other bits of the error code and the page tables.
*
* In particular the following conditions come from the error code, the
* page tables and the machine state:
* - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
* - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
* - PK is always zero if U=0 in the page tables
* - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
*
* The PKRU bitmask caches the result of these four conditions.  The error
* code (minus the P bit) and the page table's U bit form an index into the
* PKRU bitmask.  Two bits of the PKRU bitmask are then extracted and ANDed
* with the two bits of the PKRU register corresponding to the protection key.
* For the first three conditions above the bits will be 00, thus masking
* away both AD and WD.  For all reads or if the last condition holds, WD
* only will be masked away.
*/
static void update_pkru_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
				bool ept)
{
	unsigned bit;
	bool wp;

	if (ept) {
		mmu->pkru_mask = 0;
		return;
	}

	/* PKEY is enabled only if CR4.PKE and EFER.LMA are both set. */
	if (!kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || !is_long_mode(vcpu)) {
		mmu->pkru_mask = 0;
		return;
	}

	wp = is_write_protection(vcpu);

	for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
		unsigned pfec, pkey_bits;
		bool check_pkey, check_write, ff, uf, wf, pte_user;

		pfec = bit << 1;
		ff = pfec & PFERR_FETCH_MASK;
		uf = pfec & PFERR_USER_MASK;
		wf = pfec & PFERR_WRITE_MASK;

		/* PFEC.RSVD is replaced by ACC_USER_MASK. */
		pte_user = pfec & PFERR_RSVD_MASK;

		/*
		 * Only need to check the access which is not an
		 * instruction fetch and is to a user page.
		 */
		check_pkey = (!ff && pte_user);
		/*
		 * write access is controlled by PKRU if it is a
		 * user access or CR0.WP = 1.
		 */
		check_write = check_pkey && wf && (uf || wp);

		/* PKRU.AD stops both read and write access. */
		pkey_bits = !!check_pkey;
		/* PKRU.WD stops write access. */
		pkey_bits |= (!!check_write) << 1;

		mmu->pkru_mask |= (pkey_bits & 3) << pfec;
	}
}

4335
static void update_last_nonleaf_level(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
A
Avi Kivity 已提交
4336
{
4337 4338 4339 4340 4341
	unsigned root_level = mmu->root_level;

	mmu->last_nonleaf_level = root_level;
	if (root_level == PT32_ROOT_LEVEL && is_pse(vcpu))
		mmu->last_nonleaf_level++;
A
Avi Kivity 已提交
4342 4343
}

4344 4345 4346
static void paging64_init_context_common(struct kvm_vcpu *vcpu,
					 struct kvm_mmu *context,
					 int level)
A
Avi Kivity 已提交
4347
{
4348
	context->nx = is_nx(vcpu);
4349
	context->root_level = level;
4350

4351
	reset_rsvds_bits_mask(vcpu, context);
4352
	update_permission_bitmask(vcpu, context, false);
4353
	update_pkru_bitmask(vcpu, context, false);
4354
	update_last_nonleaf_level(vcpu, context);
A
Avi Kivity 已提交
4355

4356
	MMU_WARN_ON(!is_pae(vcpu));
A
Avi Kivity 已提交
4357 4358
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
4359
	context->sync_page = paging64_sync_page;
M
Marcelo Tosatti 已提交
4360
	context->invlpg = paging64_invlpg;
4361
	context->update_pte = paging64_update_pte;
4362
	context->shadow_root_level = level;
A
Avi Kivity 已提交
4363
	context->root_hpa = INVALID_PAGE;
4364
	context->direct_map = false;
A
Avi Kivity 已提交
4365 4366
}

4367 4368
static void paging64_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
4369
{
4370
	paging64_init_context_common(vcpu, context, PT64_ROOT_4LEVEL);
4371 4372
}

4373 4374
static void paging32_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
A
Avi Kivity 已提交
4375
{
4376
	context->nx = false;
4377
	context->root_level = PT32_ROOT_LEVEL;
4378

4379
	reset_rsvds_bits_mask(vcpu, context);
4380
	update_permission_bitmask(vcpu, context, false);
4381
	update_pkru_bitmask(vcpu, context, false);
4382
	update_last_nonleaf_level(vcpu, context);
A
Avi Kivity 已提交
4383 4384 4385

	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
4386
	context->sync_page = paging32_sync_page;
M
Marcelo Tosatti 已提交
4387
	context->invlpg = paging32_invlpg;
4388
	context->update_pte = paging32_update_pte;
A
Avi Kivity 已提交
4389
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
4390
	context->root_hpa = INVALID_PAGE;
4391
	context->direct_map = false;
A
Avi Kivity 已提交
4392 4393
}

4394 4395
static void paging32E_init_context(struct kvm_vcpu *vcpu,
				   struct kvm_mmu *context)
A
Avi Kivity 已提交
4396
{
4397
	paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
A
Avi Kivity 已提交
4398 4399
}

4400
static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
4401
{
4402
	struct kvm_mmu *context = &vcpu->arch.mmu;
4403

4404
	context->base_role.word = 0;
4405
	context->base_role.smm = is_smm(vcpu);
4406
	context->base_role.ad_disabled = (shadow_accessed_mask == 0);
4407
	context->page_fault = tdp_page_fault;
4408
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
4409
	context->invlpg = nonpaging_invlpg;
4410
	context->update_pte = nonpaging_update_pte;
4411
	context->shadow_root_level = kvm_x86_ops->get_tdp_level();
4412
	context->root_hpa = INVALID_PAGE;
4413
	context->direct_map = true;
4414
	context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
4415
	context->get_cr3 = get_cr3;
4416
	context->get_pdptr = kvm_pdptr_read;
4417
	context->inject_page_fault = kvm_inject_page_fault;
4418 4419

	if (!is_paging(vcpu)) {
4420
		context->nx = false;
4421 4422 4423
		context->gva_to_gpa = nonpaging_gva_to_gpa;
		context->root_level = 0;
	} else if (is_long_mode(vcpu)) {
4424
		context->nx = is_nx(vcpu);
4425
		context->root_level = PT64_ROOT_4LEVEL;
4426 4427
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging64_gva_to_gpa;
4428
	} else if (is_pae(vcpu)) {
4429
		context->nx = is_nx(vcpu);
4430
		context->root_level = PT32E_ROOT_LEVEL;
4431 4432
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging64_gva_to_gpa;
4433
	} else {
4434
		context->nx = false;
4435
		context->root_level = PT32_ROOT_LEVEL;
4436 4437
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging32_gva_to_gpa;
4438 4439
	}

4440
	update_permission_bitmask(vcpu, context, false);
4441
	update_pkru_bitmask(vcpu, context, false);
4442
	update_last_nonleaf_level(vcpu, context);
4443
	reset_tdp_shadow_zero_bits_mask(vcpu, context);
4444 4445
}

4446
void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4447
{
4448
	bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
4449
	bool smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
4450 4451
	struct kvm_mmu *context = &vcpu->arch.mmu;

4452
	MMU_WARN_ON(VALID_PAGE(context->root_hpa));
A
Avi Kivity 已提交
4453 4454

	if (!is_paging(vcpu))
4455
		nonpaging_init_context(vcpu, context);
A
Avi Kivity 已提交
4456
	else if (is_long_mode(vcpu))
4457
		paging64_init_context(vcpu, context);
A
Avi Kivity 已提交
4458
	else if (is_pae(vcpu))
4459
		paging32E_init_context(vcpu, context);
A
Avi Kivity 已提交
4460
	else
4461
		paging32_init_context(vcpu, context);
4462

4463 4464 4465 4466
	context->base_role.nxe = is_nx(vcpu);
	context->base_role.cr4_pae = !!is_pae(vcpu);
	context->base_role.cr0_wp  = is_write_protection(vcpu);
	context->base_role.smep_andnot_wp
4467
		= smep && !is_write_protection(vcpu);
4468 4469
	context->base_role.smap_andnot_wp
		= smap && !is_write_protection(vcpu);
4470
	context->base_role.smm = is_smm(vcpu);
4471
	reset_shadow_zero_bits_mask(vcpu, context);
4472 4473 4474
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);

4475 4476
void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
			     bool accessed_dirty)
N
Nadav Har'El 已提交
4477
{
4478 4479
	struct kvm_mmu *context = &vcpu->arch.mmu;

4480
	MMU_WARN_ON(VALID_PAGE(context->root_hpa));
N
Nadav Har'El 已提交
4481 4482 4483 4484

	context->shadow_root_level = kvm_x86_ops->get_tdp_level();

	context->nx = true;
4485
	context->ept_ad = accessed_dirty;
N
Nadav Har'El 已提交
4486 4487 4488 4489 4490 4491 4492 4493
	context->page_fault = ept_page_fault;
	context->gva_to_gpa = ept_gva_to_gpa;
	context->sync_page = ept_sync_page;
	context->invlpg = ept_invlpg;
	context->update_pte = ept_update_pte;
	context->root_level = context->shadow_root_level;
	context->root_hpa = INVALID_PAGE;
	context->direct_map = false;
4494
	context->base_role.ad_disabled = !accessed_dirty;
N
Nadav Har'El 已提交
4495 4496

	update_permission_bitmask(vcpu, context, true);
4497
	update_pkru_bitmask(vcpu, context, true);
N
Nadav Har'El 已提交
4498
	reset_rsvds_bits_mask_ept(vcpu, context, execonly);
4499
	reset_ept_shadow_zero_bits_mask(vcpu, context, execonly);
N
Nadav Har'El 已提交
4500 4501 4502
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);

4503
static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
4504
{
4505 4506 4507 4508 4509 4510 4511
	struct kvm_mmu *context = &vcpu->arch.mmu;

	kvm_init_shadow_mmu(vcpu);
	context->set_cr3           = kvm_x86_ops->set_cr3;
	context->get_cr3           = get_cr3;
	context->get_pdptr         = kvm_pdptr_read;
	context->inject_page_fault = kvm_inject_page_fault;
A
Avi Kivity 已提交
4512 4513
}

4514
static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
4515 4516 4517 4518
{
	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;

	g_context->get_cr3           = get_cr3;
4519
	g_context->get_pdptr         = kvm_pdptr_read;
4520 4521 4522
	g_context->inject_page_fault = kvm_inject_page_fault;

	/*
4523 4524 4525 4526 4527 4528
	 * Note that arch.mmu.gva_to_gpa translates l2_gpa to l1_gpa using
	 * L1's nested page tables (e.g. EPT12). The nested translation
	 * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
	 * L2's page tables as the first level of translation and L1's
	 * nested page tables as the second level of translation. Basically
	 * the gva_to_gpa functions between mmu and nested_mmu are swapped.
4529 4530
	 */
	if (!is_paging(vcpu)) {
4531
		g_context->nx = false;
4532 4533 4534
		g_context->root_level = 0;
		g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
	} else if (is_long_mode(vcpu)) {
4535
		g_context->nx = is_nx(vcpu);
4536
		g_context->root_level = PT64_ROOT_4LEVEL;
4537
		reset_rsvds_bits_mask(vcpu, g_context);
4538 4539
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else if (is_pae(vcpu)) {
4540
		g_context->nx = is_nx(vcpu);
4541
		g_context->root_level = PT32E_ROOT_LEVEL;
4542
		reset_rsvds_bits_mask(vcpu, g_context);
4543 4544
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else {
4545
		g_context->nx = false;
4546
		g_context->root_level = PT32_ROOT_LEVEL;
4547
		reset_rsvds_bits_mask(vcpu, g_context);
4548 4549 4550
		g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
	}

4551
	update_permission_bitmask(vcpu, g_context, false);
4552
	update_pkru_bitmask(vcpu, g_context, false);
4553
	update_last_nonleaf_level(vcpu, g_context);
4554 4555
}

4556
static void init_kvm_mmu(struct kvm_vcpu *vcpu)
4557
{
4558
	if (mmu_is_nested(vcpu))
4559
		init_kvm_nested_mmu(vcpu);
4560
	else if (tdp_enabled)
4561
		init_kvm_tdp_mmu(vcpu);
4562
	else
4563
		init_kvm_softmmu(vcpu);
4564 4565
}

4566
void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4567
{
4568
	kvm_mmu_unload(vcpu);
4569
	init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
4570
}
4571
EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
A
Avi Kivity 已提交
4572 4573

int kvm_mmu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4574
{
4575 4576
	int r;

4577
	r = mmu_topup_memory_caches(vcpu);
A
Avi Kivity 已提交
4578 4579
	if (r)
		goto out;
4580
	r = mmu_alloc_roots(vcpu);
4581
	kvm_mmu_sync_roots(vcpu);
4582 4583
	if (r)
		goto out;
4584
	/* set_cr3() should ensure TLB has been flushed */
4585
	vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
4586 4587
out:
	return r;
A
Avi Kivity 已提交
4588
}
A
Avi Kivity 已提交
4589 4590 4591 4592 4593
EXPORT_SYMBOL_GPL(kvm_mmu_load);

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
	mmu_free_roots(vcpu);
4594
	WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
4595
}
4596
EXPORT_SYMBOL_GPL(kvm_mmu_unload);
A
Avi Kivity 已提交
4597

4598
static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
4599 4600
				  struct kvm_mmu_page *sp, u64 *spte,
				  const void *new)
4601
{
4602
	if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
4603 4604
		++vcpu->kvm->stat.mmu_pde_zapped;
		return;
4605
        }
4606

A
Avi Kivity 已提交
4607
	++vcpu->kvm->stat.mmu_pte_updated;
4608
	vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
4609 4610
}

4611 4612 4613 4614 4615 4616 4617 4618
static bool need_remote_flush(u64 old, u64 new)
{
	if (!is_shadow_present_pte(old))
		return false;
	if (!is_shadow_present_pte(new))
		return true;
	if ((old ^ new) & PT64_BASE_ADDR_MASK)
		return true;
4619 4620
	old ^= shadow_nx_mask;
	new ^= shadow_nx_mask;
4621 4622 4623
	return (old & ~new & PT64_PERM_MASK) != 0;
}

4624 4625
static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
				    const u8 *new, int *bytes)
4626
{
4627 4628
	u64 gentry;
	int r;
4629 4630 4631

	/*
	 * Assume that the pte write on a page table of the same type
4632 4633
	 * as the current vcpu paging mode since we update the sptes only
	 * when they have the same mode.
4634
	 */
4635
	if (is_pae(vcpu) && *bytes == 4) {
4636
		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */
4637 4638
		*gpa &= ~(gpa_t)7;
		*bytes = 8;
4639
		r = kvm_vcpu_read_guest(vcpu, *gpa, &gentry, 8);
4640 4641
		if (r)
			gentry = 0;
4642 4643 4644
		new = (const u8 *)&gentry;
	}

4645
	switch (*bytes) {
4646 4647 4648 4649 4650 4651 4652 4653 4654
	case 4:
		gentry = *(const u32 *)new;
		break;
	case 8:
		gentry = *(const u64 *)new;
		break;
	default:
		gentry = 0;
		break;
4655 4656
	}

4657 4658 4659 4660 4661 4662 4663
	return gentry;
}

/*
 * If we're seeing too many writes to a page, it may no longer be a page table,
 * or we may be forking, in which case it is better to unmap the page.
 */
4664
static bool detect_write_flooding(struct kvm_mmu_page *sp)
4665
{
4666 4667 4668 4669
	/*
	 * Skip write-flooding detected for the sp whose level is 1, because
	 * it can become unsync, then the guest page is not write-protected.
	 */
4670
	if (sp->role.level == PT_PAGE_TABLE_LEVEL)
4671
		return false;
4672

4673 4674
	atomic_inc(&sp->write_flooding_count);
	return atomic_read(&sp->write_flooding_count) >= 3;
4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690
}

/*
 * Misaligned accesses are too much trouble to fix up; also, they usually
 * indicate a page is not used as a page table.
 */
static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
				    int bytes)
{
	unsigned offset, pte_size, misaligned;

	pgprintk("misaligned: gpa %llx bytes %d role %x\n",
		 gpa, bytes, sp->role.word);

	offset = offset_in_page(gpa);
	pte_size = sp->role.cr4_pae ? 8 : 4;
4691 4692 4693 4694 4695 4696 4697 4698

	/*
	 * Sometimes, the OS only writes the last one bytes to update status
	 * bits, for example, in linux, andb instruction is used in clear_bit().
	 */
	if (!(offset & (pte_size - 1)) && bytes == 1)
		return false;

4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735
	misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
	misaligned |= bytes < 4;

	return misaligned;
}

static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
{
	unsigned page_offset, quadrant;
	u64 *spte;
	int level;

	page_offset = offset_in_page(gpa);
	level = sp->role.level;
	*nspte = 1;
	if (!sp->role.cr4_pae) {
		page_offset <<= 1;	/* 32->64 */
		/*
		 * A 32-bit pde maps 4MB while the shadow pdes map
		 * only 2MB.  So we need to double the offset again
		 * and zap two pdes instead of one.
		 */
		if (level == PT32_ROOT_LEVEL) {
			page_offset &= ~7; /* kill rounding error */
			page_offset <<= 1;
			*nspte = 2;
		}
		quadrant = page_offset >> PAGE_SHIFT;
		page_offset &= ~PAGE_MASK;
		if (quadrant != sp->role.quadrant)
			return NULL;
	}

	spte = &sp->spt[page_offset / sizeof(*spte)];
	return spte;
}

4736
static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
4737 4738
			      const u8 *new, int bytes,
			      struct kvm_page_track_notifier_node *node)
4739 4740 4741 4742 4743 4744
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_mmu_page *sp;
	LIST_HEAD(invalid_list);
	u64 entry, gentry, *spte;
	int npte;
4745
	bool remote_flush, local_flush;
4746 4747 4748 4749 4750 4751 4752
	union kvm_mmu_page_role mask = { };

	mask.cr0_wp = 1;
	mask.cr4_pae = 1;
	mask.nxe = 1;
	mask.smep_andnot_wp = 1;
	mask.smap_andnot_wp = 1;
4753
	mask.smm = 1;
4754
	mask.ad_disabled = 1;
4755 4756 4757 4758 4759 4760 4761 4762

	/*
	 * If we don't have indirect shadow pages, it means no page is
	 * write-protected, so we can exit simply.
	 */
	if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
		return;

4763
	remote_flush = local_flush = false;
4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777

	pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);

	gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, new, &bytes);

	/*
	 * No need to care whether allocation memory is successful
	 * or not since pte prefetch is skiped if it does not have
	 * enough objects in the cache.
	 */
	mmu_topup_memory_caches(vcpu);

	spin_lock(&vcpu->kvm->mmu_lock);
	++vcpu->kvm->stat.mmu_pte_write;
4778
	kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
4779

4780
	for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
4781
		if (detect_write_misaligned(sp, gpa, bytes) ||
4782
		      detect_write_flooding(sp)) {
4783
			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
A
Avi Kivity 已提交
4784
			++vcpu->kvm->stat.mmu_flooded;
4785 4786
			continue;
		}
4787 4788 4789 4790 4791

		spte = get_written_sptes(sp, gpa, &npte);
		if (!spte)
			continue;

4792
		local_flush = true;
4793
		while (npte--) {
4794
			entry = *spte;
4795
			mmu_page_zap_pte(vcpu->kvm, sp, spte);
4796 4797
			if (gentry &&
			      !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
4798
			      & mask.word) && rmap_can_add(vcpu))
4799
				mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
G
Gleb Natapov 已提交
4800
			if (need_remote_flush(entry, *spte))
4801
				remote_flush = true;
4802
			++spte;
4803 4804
		}
	}
4805
	kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush);
4806
	kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
4807
	spin_unlock(&vcpu->kvm->mmu_lock);
4808 4809
}

4810 4811
int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
4812 4813
	gpa_t gpa;
	int r;
4814

4815
	if (vcpu->arch.mmu.direct_map)
4816 4817
		return 0;

4818
	gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
4819 4820

	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
4821

4822
	return r;
4823
}
4824
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
4825

4826
static int make_mmu_pages_available(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4827
{
4828
	LIST_HEAD(invalid_list);
4829

4830
	if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES))
4831
		return 0;
4832

4833 4834 4835
	while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) {
		if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list))
			break;
A
Avi Kivity 已提交
4836

A
Avi Kivity 已提交
4837
		++vcpu->kvm->stat.mmu_recycled;
A
Avi Kivity 已提交
4838
	}
4839
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
4840 4841 4842 4843

	if (!kvm_mmu_available_pages(vcpu->kvm))
		return -ENOSPC;
	return 0;
A
Avi Kivity 已提交
4844 4845
}

4846
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code,
4847
		       void *insn, int insn_len)
4848
{
4849
	int r, emulation_type = EMULTYPE_RETRY;
4850
	enum emulation_result er;
4851
	bool direct = vcpu->arch.mmu.direct_map;
4852

4853 4854 4855 4856 4857 4858
	/* With shadow page tables, fault_address contains a GVA or nGPA.  */
	if (vcpu->arch.mmu.direct_map) {
		vcpu->arch.gpa_available = true;
		vcpu->arch.gpa_val = cr2;
	}

4859 4860 4861 4862 4863 4864 4865 4866 4867 4868
	if (unlikely(error_code & PFERR_RSVD_MASK)) {
		r = handle_mmio_page_fault(vcpu, cr2, direct);
		if (r == RET_MMIO_PF_EMULATE) {
			emulation_type = 0;
			goto emulate;
		}
		if (r == RET_MMIO_PF_RETRY)
			return 1;
		if (r < 0)
			return r;
P
Paolo Bonzini 已提交
4869
		/* Must be RET_MMIO_PF_INVALID.  */
4870
	}
4871

4872 4873
	r = vcpu->arch.mmu.page_fault(vcpu, cr2, lower_32_bits(error_code),
				      false);
4874
	if (r < 0)
4875 4876 4877
		return r;
	if (!r)
		return 1;
4878

4879 4880 4881 4882 4883 4884 4885
	/*
	 * Before emulating the instruction, check if the error code
	 * was due to a RO violation while translating the guest page.
	 * This can occur when using nested virtualization with nested
	 * paging in both guests. If true, we simply unprotect the page
	 * and resume the guest.
	 */
4886
	if (vcpu->arch.mmu.direct_map &&
4887
	    (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) {
4888 4889 4890 4891
		kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2));
		return 1;
	}

4892
	if (mmio_info_in_cache(vcpu, cr2, direct))
4893
		emulation_type = 0;
4894
emulate:
4895
	er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
4896 4897 4898 4899

	switch (er) {
	case EMULATE_DONE:
		return 1;
P
Paolo Bonzini 已提交
4900
	case EMULATE_USER_EXIT:
4901
		++vcpu->stat.mmio_exits;
4902
		/* fall through */
4903
	case EMULATE_FAIL:
4904
		return 0;
4905 4906 4907 4908 4909 4910
	default:
		BUG();
	}
}
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);

M
Marcelo Tosatti 已提交
4911 4912 4913
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
	vcpu->arch.mmu.invlpg(vcpu, gva);
4914
	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
M
Marcelo Tosatti 已提交
4915 4916 4917 4918
	++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);

4919 4920 4921 4922 4923 4924
void kvm_enable_tdp(void)
{
	tdp_enabled = true;
}
EXPORT_SYMBOL_GPL(kvm_enable_tdp);

4925 4926 4927 4928 4929 4930
void kvm_disable_tdp(void)
{
	tdp_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_tdp);

A
Avi Kivity 已提交
4931 4932
static void free_mmu_pages(struct kvm_vcpu *vcpu)
{
4933
	free_page((unsigned long)vcpu->arch.mmu.pae_root);
4934 4935
	if (vcpu->arch.mmu.lm_root != NULL)
		free_page((unsigned long)vcpu->arch.mmu.lm_root);
A
Avi Kivity 已提交
4936 4937 4938 4939
}

static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
{
4940
	struct page *page;
A
Avi Kivity 已提交
4941 4942
	int i;

4943 4944 4945 4946 4947 4948 4949
	/*
	 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
	 * Therefore we need to allocate shadow page tables in the first
	 * 4GB of memory, which happens to fit the DMA32 zone.
	 */
	page = alloc_page(GFP_KERNEL | __GFP_DMA32);
	if (!page)
4950 4951
		return -ENOMEM;

4952
	vcpu->arch.mmu.pae_root = page_address(page);
4953
	for (i = 0; i < 4; ++i)
4954
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
4955

A
Avi Kivity 已提交
4956 4957 4958
	return 0;
}

4959
int kvm_mmu_create(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4960
{
4961 4962 4963 4964
	vcpu->arch.walk_mmu = &vcpu->arch.mmu;
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
	vcpu->arch.mmu.translate_gpa = translate_gpa;
	vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
A
Avi Kivity 已提交
4965

4966 4967
	return alloc_mmu_pages(vcpu);
}
A
Avi Kivity 已提交
4968

4969
void kvm_mmu_setup(struct kvm_vcpu *vcpu)
4970
{
4971
	MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
4972

4973
	init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
4974 4975
}

4976
static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
4977 4978
			struct kvm_memory_slot *slot,
			struct kvm_page_track_notifier_node *node)
4979 4980 4981 4982
{
	kvm_mmu_invalidate_zap_all_pages(kvm);
}

4983 4984 4985 4986 4987
void kvm_mmu_init_vm(struct kvm *kvm)
{
	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;

	node->track_write = kvm_mmu_pte_write;
4988
	node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;
4989 4990 4991 4992 4993 4994 4995 4996 4997 4998
	kvm_page_track_register_notifier(kvm, node);
}

void kvm_mmu_uninit_vm(struct kvm *kvm)
{
	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;

	kvm_page_track_unregister_notifier(kvm, node);
}

4999
/* The return value indicates if tlb flush on all vcpus is needed. */
5000
typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head);
5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067

/* The caller should hold mmu-lock before calling this function. */
static bool
slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
			slot_level_handler fn, int start_level, int end_level,
			gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
{
	struct slot_rmap_walk_iterator iterator;
	bool flush = false;

	for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
			end_gfn, &iterator) {
		if (iterator.rmap)
			flush |= fn(kvm, iterator.rmap);

		if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
			if (flush && lock_flush_tlb) {
				kvm_flush_remote_tlbs(kvm);
				flush = false;
			}
			cond_resched_lock(&kvm->mmu_lock);
		}
	}

	if (flush && lock_flush_tlb) {
		kvm_flush_remote_tlbs(kvm);
		flush = false;
	}

	return flush;
}

static bool
slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
		  slot_level_handler fn, int start_level, int end_level,
		  bool lock_flush_tlb)
{
	return slot_handle_level_range(kvm, memslot, fn, start_level,
			end_level, memslot->base_gfn,
			memslot->base_gfn + memslot->npages - 1,
			lock_flush_tlb);
}

static bool
slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
		      slot_level_handler fn, bool lock_flush_tlb)
{
	return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
				 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
}

static bool
slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
			slot_level_handler fn, bool lock_flush_tlb)
{
	return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL + 1,
				 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
}

static bool
slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
		 slot_level_handler fn, bool lock_flush_tlb)
{
	return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
				 PT_PAGE_TABLE_LEVEL, lock_flush_tlb);
}

X
Xiao Guangrong 已提交
5068 5069 5070 5071
void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
5072
	int i;
X
Xiao Guangrong 已提交
5073 5074

	spin_lock(&kvm->mmu_lock);
5075 5076 5077 5078 5079 5080 5081 5082 5083
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
		kvm_for_each_memslot(memslot, slots) {
			gfn_t start, end;

			start = max(gfn_start, memslot->base_gfn);
			end = min(gfn_end, memslot->base_gfn + memslot->npages);
			if (start >= end)
				continue;
X
Xiao Guangrong 已提交
5084

5085 5086 5087 5088
			slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
						PT_PAGE_TABLE_LEVEL, PT_MAX_HUGEPAGE_LEVEL,
						start, end - 1, true);
		}
X
Xiao Guangrong 已提交
5089 5090 5091 5092 5093
	}

	spin_unlock(&kvm->mmu_lock);
}

5094 5095
static bool slot_rmap_write_protect(struct kvm *kvm,
				    struct kvm_rmap_head *rmap_head)
5096
{
5097
	return __rmap_write_protect(kvm, rmap_head, false);
5098 5099
}

5100 5101
void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
				      struct kvm_memory_slot *memslot)
A
Avi Kivity 已提交
5102
{
5103
	bool flush;
A
Avi Kivity 已提交
5104

5105
	spin_lock(&kvm->mmu_lock);
5106 5107
	flush = slot_handle_all_level(kvm, memslot, slot_rmap_write_protect,
				      false);
5108
	spin_unlock(&kvm->mmu_lock);
5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127

	/*
	 * kvm_mmu_slot_remove_write_access() and kvm_vm_ioctl_get_dirty_log()
	 * which do tlb flush out of mmu-lock should be serialized by
	 * kvm->slots_lock otherwise tlb flush would be missed.
	 */
	lockdep_assert_held(&kvm->slots_lock);

	/*
	 * We can flush all the TLBs out of the mmu lock without TLB
	 * corruption since we just change the spte from writable to
	 * readonly so that we only need to care the case of changing
	 * spte from present to present (changing the spte from present
	 * to nonpresent will flush all the TLBs immediately), in other
	 * words, the only case we care is mmu_spte_update() where we
	 * haved checked SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE
	 * instead of PT_WRITABLE_MASK, that means it does not depend
	 * on PT_WRITABLE_MASK anymore.
	 */
5128 5129
	if (flush)
		kvm_flush_remote_tlbs(kvm);
A
Avi Kivity 已提交
5130
}
5131

5132
static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
5133
					 struct kvm_rmap_head *rmap_head)
5134 5135 5136 5137
{
	u64 *sptep;
	struct rmap_iterator iter;
	int need_tlb_flush = 0;
D
Dan Williams 已提交
5138
	kvm_pfn_t pfn;
5139 5140
	struct kvm_mmu_page *sp;

5141
restart:
5142
	for_each_rmap_spte(rmap_head, &iter, sptep) {
5143 5144 5145 5146
		sp = page_header(__pa(sptep));
		pfn = spte_to_pfn(*sptep);

		/*
5147 5148 5149 5150 5151
		 * We cannot do huge page mapping for indirect shadow pages,
		 * which are found on the last rmap (level = 1) when not using
		 * tdp; such shadow pages are synced with the page table in
		 * the guest, and the guest page table is using 4K page size
		 * mapping if the indirect sp has level = 1.
5152 5153 5154
		 */
		if (sp->role.direct &&
			!kvm_is_reserved_pfn(pfn) &&
5155
			PageTransCompoundMap(pfn_to_page(pfn))) {
5156 5157
			drop_spte(kvm, sptep);
			need_tlb_flush = 1;
5158 5159
			goto restart;
		}
5160 5161 5162 5163 5164 5165
	}

	return need_tlb_flush;
}

void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
5166
				   const struct kvm_memory_slot *memslot)
5167
{
5168
	/* FIXME: const-ify all uses of struct kvm_memory_slot.  */
5169
	spin_lock(&kvm->mmu_lock);
5170 5171
	slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot,
			 kvm_mmu_zap_collapsible_spte, true);
5172 5173 5174
	spin_unlock(&kvm->mmu_lock);
}

5175 5176 5177
void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
				   struct kvm_memory_slot *memslot)
{
5178
	bool flush;
5179 5180

	spin_lock(&kvm->mmu_lock);
5181
	flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false);
5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199
	spin_unlock(&kvm->mmu_lock);

	lockdep_assert_held(&kvm->slots_lock);

	/*
	 * It's also safe to flush TLBs out of mmu lock here as currently this
	 * function is only used for dirty logging, in which case flushing TLB
	 * out of mmu lock also guarantees no dirty pages will be lost in
	 * dirty_bitmap.
	 */
	if (flush)
		kvm_flush_remote_tlbs(kvm);
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty);

void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
					struct kvm_memory_slot *memslot)
{
5200
	bool flush;
5201 5202

	spin_lock(&kvm->mmu_lock);
5203 5204
	flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect,
					false);
5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217
	spin_unlock(&kvm->mmu_lock);

	/* see kvm_mmu_slot_remove_write_access */
	lockdep_assert_held(&kvm->slots_lock);

	if (flush)
		kvm_flush_remote_tlbs(kvm);
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access);

void kvm_mmu_slot_set_dirty(struct kvm *kvm,
			    struct kvm_memory_slot *memslot)
{
5218
	bool flush;
5219 5220

	spin_lock(&kvm->mmu_lock);
5221
	flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false);
5222 5223 5224 5225 5226 5227 5228 5229 5230 5231
	spin_unlock(&kvm->mmu_lock);

	lockdep_assert_held(&kvm->slots_lock);

	/* see kvm_mmu_slot_leaf_clear_dirty */
	if (flush)
		kvm_flush_remote_tlbs(kvm);
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty);

X
Xiao Guangrong 已提交
5232
#define BATCH_ZAP_PAGES	10
5233 5234 5235
static void kvm_zap_obsolete_pages(struct kvm *kvm)
{
	struct kvm_mmu_page *sp, *node;
X
Xiao Guangrong 已提交
5236
	int batch = 0;
5237 5238 5239 5240

restart:
	list_for_each_entry_safe_reverse(sp, node,
	      &kvm->arch.active_mmu_pages, link) {
X
Xiao Guangrong 已提交
5241 5242
		int ret;

5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257
		/*
		 * No obsolete page exists before new created page since
		 * active_mmu_pages is the FIFO list.
		 */
		if (!is_obsolete_sp(kvm, sp))
			break;

		/*
		 * Since we are reversely walking the list and the invalid
		 * list will be moved to the head, skip the invalid page
		 * can help us to avoid the infinity list walking.
		 */
		if (sp->role.invalid)
			continue;

5258 5259 5260 5261
		/*
		 * Need not flush tlb since we only zap the sp with invalid
		 * generation number.
		 */
X
Xiao Guangrong 已提交
5262
		if (batch >= BATCH_ZAP_PAGES &&
5263
		      cond_resched_lock(&kvm->mmu_lock)) {
X
Xiao Guangrong 已提交
5264
			batch = 0;
5265 5266 5267
			goto restart;
		}

5268 5269
		ret = kvm_mmu_prepare_zap_page(kvm, sp,
				&kvm->arch.zapped_obsolete_pages);
X
Xiao Guangrong 已提交
5270 5271 5272
		batch += ret;

		if (ret)
5273 5274 5275
			goto restart;
	}

5276 5277 5278 5279
	/*
	 * Should flush tlb before free page tables since lockless-walking
	 * may use the pages.
	 */
5280
	kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294
}

/*
 * Fast invalidate all shadow pages and use lock-break technique
 * to zap obsolete pages.
 *
 * It's required when memslot is being deleted or VM is being
 * destroyed, in these cases, we should ensure that KVM MMU does
 * not use any resource of the being-deleted slot or all slots
 * after calling the function.
 */
void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm)
{
	spin_lock(&kvm->mmu_lock);
5295
	trace_kvm_mmu_invalidate_zap_all_pages(kvm);
5296 5297
	kvm->arch.mmu_valid_gen++;

5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308
	/*
	 * Notify all vcpus to reload its shadow page table
	 * and flush TLB. Then all vcpus will switch to new
	 * shadow page table with the new mmu_valid_gen.
	 *
	 * Note: we should do this under the protection of
	 * mmu-lock, otherwise, vcpu would purge shadow page
	 * but miss tlb flush.
	 */
	kvm_reload_remote_mmus(kvm);

5309 5310 5311 5312
	kvm_zap_obsolete_pages(kvm);
	spin_unlock(&kvm->mmu_lock);
}

5313 5314 5315 5316 5317
static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
{
	return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
}

5318
void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots)
5319 5320 5321 5322 5323
{
	/*
	 * The very rare case: if the generation-number is round,
	 * zap all shadow pages.
	 */
5324
	if (unlikely((slots->generation & MMIO_GEN_MASK) == 0)) {
5325
		kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n");
5326
		kvm_mmu_invalidate_zap_all_pages(kvm);
5327
	}
5328 5329
}

5330 5331
static unsigned long
mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
5332 5333
{
	struct kvm *kvm;
5334
	int nr_to_scan = sc->nr_to_scan;
5335
	unsigned long freed = 0;
5336

5337
	spin_lock(&kvm_lock);
5338 5339

	list_for_each_entry(kvm, &vm_list, vm_list) {
5340
		int idx;
5341
		LIST_HEAD(invalid_list);
5342

5343 5344 5345 5346 5347 5348 5349 5350
		/*
		 * Never scan more than sc->nr_to_scan VM instances.
		 * Will not hit this condition practically since we do not try
		 * to shrink more than one VM and it is very unlikely to see
		 * !n_used_mmu_pages so many times.
		 */
		if (!nr_to_scan--)
			break;
5351 5352 5353 5354 5355 5356
		/*
		 * n_used_mmu_pages is accessed without holding kvm->mmu_lock
		 * here. We may skip a VM instance errorneosly, but we do not
		 * want to shrink a VM that only started to populate its MMU
		 * anyway.
		 */
5357 5358
		if (!kvm->arch.n_used_mmu_pages &&
		      !kvm_has_zapped_obsolete_pages(kvm))
5359 5360
			continue;

5361
		idx = srcu_read_lock(&kvm->srcu);
5362 5363
		spin_lock(&kvm->mmu_lock);

5364 5365 5366 5367 5368 5369
		if (kvm_has_zapped_obsolete_pages(kvm)) {
			kvm_mmu_commit_zap_page(kvm,
			      &kvm->arch.zapped_obsolete_pages);
			goto unlock;
		}

5370 5371
		if (prepare_zap_oldest_mmu_page(kvm, &invalid_list))
			freed++;
5372
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
5373

5374
unlock:
5375
		spin_unlock(&kvm->mmu_lock);
5376
		srcu_read_unlock(&kvm->srcu, idx);
5377

5378 5379 5380 5381 5382
		/*
		 * unfair on small ones
		 * per-vm shrinkers cry out
		 * sadness comes quickly
		 */
5383 5384
		list_move_tail(&kvm->vm_list, &vm_list);
		break;
5385 5386
	}

5387
	spin_unlock(&kvm_lock);
5388 5389 5390 5391 5392 5393
	return freed;
}

static unsigned long
mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
{
5394
	return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
5395 5396 5397
}

static struct shrinker mmu_shrinker = {
5398 5399
	.count_objects = mmu_shrink_count,
	.scan_objects = mmu_shrink_scan,
5400 5401 5402
	.seeks = DEFAULT_SEEKS * 10,
};

I
Ingo Molnar 已提交
5403
static void mmu_destroy_caches(void)
5404
{
5405 5406
	if (pte_list_desc_cache)
		kmem_cache_destroy(pte_list_desc_cache);
5407 5408
	if (mmu_page_header_cache)
		kmem_cache_destroy(mmu_page_header_cache);
5409 5410 5411 5412
}

int kvm_mmu_module_init(void)
{
5413 5414
	kvm_mmu_clear_all_pte_masks();

5415 5416
	pte_list_desc_cache = kmem_cache_create("pte_list_desc",
					    sizeof(struct pte_list_desc),
5417
					    0, 0, NULL);
5418
	if (!pte_list_desc_cache)
5419 5420
		goto nomem;

5421 5422
	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
5423
						  0, 0, NULL);
5424 5425 5426
	if (!mmu_page_header_cache)
		goto nomem;

5427
	if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
5428 5429
		goto nomem;

5430 5431
	register_shrinker(&mmu_shrinker);

5432 5433 5434
	return 0;

nomem:
5435
	mmu_destroy_caches();
5436 5437 5438
	return -ENOMEM;
}

5439 5440 5441 5442 5443 5444 5445
/*
 * Caculate mmu pages needed for kvm.
 */
unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
{
	unsigned int nr_mmu_pages;
	unsigned int  nr_pages = 0;
5446
	struct kvm_memslots *slots;
5447
	struct kvm_memory_slot *memslot;
5448
	int i;
5449

5450 5451
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
5452

5453 5454 5455
		kvm_for_each_memslot(memslot, slots)
			nr_pages += memslot->npages;
	}
5456 5457 5458

	nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
	nr_mmu_pages = max(nr_mmu_pages,
5459
			   (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
5460 5461 5462 5463

	return nr_mmu_pages;
}

5464 5465
void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
5466
	kvm_mmu_unload(vcpu);
5467 5468
	free_mmu_pages(vcpu);
	mmu_free_memory_caches(vcpu);
5469 5470 5471 5472 5473 5474 5475
}

void kvm_mmu_module_exit(void)
{
	mmu_destroy_caches();
	percpu_counter_destroy(&kvm_total_used_mmu_pages);
	unregister_shrinker(&mmu_shrinker);
5476 5477
	mmu_audit_disable();
}