mmu.c 138.0 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
11 12 13 14 15 16 17 18 19
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
A
Avi Kivity 已提交
20

21
#include "irq.h"
22
#include "mmu.h"
23
#include "x86.h"
A
Avi Kivity 已提交
24
#include "kvm_cache_regs.h"
25
#include "cpuid.h"
A
Avi Kivity 已提交
26

27
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
28 29 30 31
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
32 33
#include <linux/moduleparam.h>
#include <linux/export.h>
34
#include <linux/swap.h>
M
Marcelo Tosatti 已提交
35
#include <linux/hugetlb.h>
36
#include <linux/compiler.h>
37
#include <linux/srcu.h>
38
#include <linux/slab.h>
39
#include <linux/sched/signal.h>
40
#include <linux/uaccess.h>
41
#include <linux/hash.h>
42
#include <linux/kern_levels.h>
A
Avi Kivity 已提交
43

A
Avi Kivity 已提交
44 45
#include <asm/page.h>
#include <asm/cmpxchg.h>
46
#include <asm/io.h>
47
#include <asm/vmx.h>
48
#include <asm/kvm_page_track.h>
49
#include "trace.h"
A
Avi Kivity 已提交
50

51 52 53 54 55 56 57
/*
 * When setting this variable to true it enables Two-Dimensional-Paging
 * where the hardware walks 2 page tables:
 * 1. the guest-virtual to guest-physical
 * 2. while doing 1. it walks guest-physical to host-physical
 * If the hardware supports that we don't need to do shadow paging.
 */
58
bool tdp_enabled = false;
59

60 61 62 63
enum {
	AUDIT_PRE_PAGE_FAULT,
	AUDIT_POST_PAGE_FAULT,
	AUDIT_PRE_PTE_WRITE,
64 65 66
	AUDIT_POST_PTE_WRITE,
	AUDIT_PRE_SYNC,
	AUDIT_POST_SYNC
67
};
68

69
#undef MMU_DEBUG
70 71

#ifdef MMU_DEBUG
72 73
static bool dbg = 0;
module_param(dbg, bool, 0644);
74 75 76

#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
77
#define MMU_WARN_ON(x) WARN_ON(x)
78 79 80
#else
#define pgprintk(x...) do { } while (0)
#define rmap_printk(x...) do { } while (0)
81
#define MMU_WARN_ON(x) do { } while (0)
82
#endif
A
Avi Kivity 已提交
83

84 85
#define PTE_PREFETCH_NUM		8

86
#define PT_FIRST_AVAIL_BITS_SHIFT 10
A
Avi Kivity 已提交
87 88 89 90 91
#define PT64_SECOND_AVAIL_BITS_SHIFT 52

#define PT64_LEVEL_BITS 9

#define PT64_LEVEL_SHIFT(level) \
M
Mike Day 已提交
92
		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
A
Avi Kivity 已提交
93 94 95 96 97 98 99 100

#define PT64_INDEX(address, level)\
	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))


#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
M
Mike Day 已提交
101
		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
A
Avi Kivity 已提交
102

103 104 105
#define PT32_LVL_OFFSET_MASK(level) \
	(PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT32_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
106 107 108 109 110

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


111
#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
A
Avi Kivity 已提交
112 113
#define PT64_DIR_BASE_ADDR_MASK \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
114 115 116 117 118 119
#define PT64_LVL_ADDR_MASK(level) \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
#define PT64_LVL_OFFSET_MASK(level) \
	(PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
120 121 122 123

#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
124 125 126
#define PT32_LVL_ADDR_MASK(level) \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
					    * PT32_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
127

128 129
#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
			| shadow_x_mask | shadow_nx_mask)
A
Avi Kivity 已提交
130

131 132 133 134 135
#define ACC_EXEC_MASK    1
#define ACC_WRITE_MASK   PT_WRITABLE_MASK
#define ACC_USER_MASK    PT_USER_MASK
#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)

136 137 138 139
/* The mask for the R/X bits in EPT PTEs */
#define PT64_EPT_READABLE_MASK			0x1ull
#define PT64_EPT_EXECUTABLE_MASK		0x4ull

140 141
#include <trace/events/kvm.h>

142 143 144
#define CREATE_TRACE_POINTS
#include "mmutrace.h"

145 146
#define SPTE_HOST_WRITEABLE	(1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
#define SPTE_MMU_WRITEABLE	(1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
147

148 149
#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)

150 151 152
/* make pte_list_desc fit well in cache line */
#define PTE_LIST_EXT 3

153 154 155
struct pte_list_desc {
	u64 *sptes[PTE_LIST_EXT];
	struct pte_list_desc *more;
156 157
};

158 159 160 161
struct kvm_shadow_walk_iterator {
	u64 addr;
	hpa_t shadow_addr;
	u64 *sptep;
162
	int level;
163 164 165 166 167 168 169 170
	unsigned index;
};

#define for_each_shadow_entry(_vcpu, _addr, _walker)    \
	for (shadow_walk_init(&(_walker), _vcpu, _addr);	\
	     shadow_walk_okay(&(_walker));			\
	     shadow_walk_next(&(_walker)))

171 172 173 174 175 176
#define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)	\
	for (shadow_walk_init(&(_walker), _vcpu, _addr);		\
	     shadow_walk_okay(&(_walker)) &&				\
		({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });	\
	     __shadow_walk_next(&(_walker), spte))

177
static struct kmem_cache *pte_list_desc_cache;
178
static struct kmem_cache *mmu_page_header_cache;
179
static struct percpu_counter kvm_total_used_mmu_pages;
180

S
Sheng Yang 已提交
181 182 183 184 185
static u64 __read_mostly shadow_nx_mask;
static u64 __read_mostly shadow_x_mask;	/* mutual exclusive with nx_mask */
static u64 __read_mostly shadow_user_mask;
static u64 __read_mostly shadow_accessed_mask;
static u64 __read_mostly shadow_dirty_mask;
186
static u64 __read_mostly shadow_mmio_mask;
187
static u64 __read_mostly shadow_mmio_value;
188
static u64 __read_mostly shadow_present_mask;
189

190
/*
191 192 193
 * SPTEs used by MMUs without A/D bits are marked with shadow_acc_track_value.
 * Non-present SPTEs with shadow_acc_track_value set are in place for access
 * tracking.
194 195 196 197 198 199 200 201 202 203 204 205 206 207
 */
static u64 __read_mostly shadow_acc_track_mask;
static const u64 shadow_acc_track_value = SPTE_SPECIAL_MASK;

/*
 * The mask/shift to use for saving the original R/X bits when marking the PTE
 * as not-present for access tracking purposes. We do not save the W bit as the
 * PTEs being access tracked also need to be dirty tracked, so the W bit will be
 * restored only when a write is attempted to the page.
 */
static const u64 shadow_acc_track_saved_bits_mask = PT64_EPT_READABLE_MASK |
						    PT64_EPT_EXECUTABLE_MASK;
static const u64 shadow_acc_track_saved_bits_shift = PT64_SECOND_AVAIL_BITS_SHIFT;

208
static void mmu_spte_set(u64 *sptep, u64 spte);
209
static void mmu_free_roots(struct kvm_vcpu *vcpu);
210

211
void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value)
212
{
213 214
	BUG_ON((mmio_mask & mmio_value) != mmio_value);
	shadow_mmio_value = mmio_value | SPTE_SPECIAL_MASK;
215
	shadow_mmio_mask = mmio_mask | SPTE_SPECIAL_MASK;
216 217 218
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);

219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
{
	return sp->role.ad_disabled;
}

static inline bool spte_ad_enabled(u64 spte)
{
	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	return !(spte & shadow_acc_track_value);
}

static inline u64 spte_shadow_accessed_mask(u64 spte)
{
	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
}

static inline u64 spte_shadow_dirty_mask(u64 spte)
{
	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
}

242 243
static inline bool is_access_track_spte(u64 spte)
{
244
	return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
245 246
}

247
/*
248 249 250 251 252 253 254
 * the low bit of the generation number is always presumed to be zero.
 * This disables mmio caching during memslot updates.  The concept is
 * similar to a seqcount but instead of retrying the access we just punt
 * and ignore the cache.
 *
 * spte bits 3-11 are used as bits 1-9 of the generation number,
 * the bits 52-61 are used as bits 10-19 of the generation number.
255
 */
256
#define MMIO_SPTE_GEN_LOW_SHIFT		2
257 258
#define MMIO_SPTE_GEN_HIGH_SHIFT	52

259 260 261
#define MMIO_GEN_SHIFT			20
#define MMIO_GEN_LOW_SHIFT		10
#define MMIO_GEN_LOW_MASK		((1 << MMIO_GEN_LOW_SHIFT) - 2)
262
#define MMIO_GEN_MASK			((1 << MMIO_GEN_SHIFT) - 1)
263 264 265 266 267

static u64 generation_mmio_spte_mask(unsigned int gen)
{
	u64 mask;

T
Tiejun Chen 已提交
268
	WARN_ON(gen & ~MMIO_GEN_MASK);
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285

	mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT;
	mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT;
	return mask;
}

static unsigned int get_mmio_spte_generation(u64 spte)
{
	unsigned int gen;

	spte &= ~shadow_mmio_mask;

	gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK;
	gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT;
	return gen;
}

286
static unsigned int kvm_current_mmio_generation(struct kvm_vcpu *vcpu)
287
{
288
	return kvm_vcpu_memslots(vcpu)->generation & MMIO_GEN_MASK;
289 290
}

291
static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
292
			   unsigned access)
293
{
294
	unsigned int gen = kvm_current_mmio_generation(vcpu);
295
	u64 mask = generation_mmio_spte_mask(gen);
296

297
	access &= ACC_WRITE_MASK | ACC_USER_MASK;
298
	mask |= shadow_mmio_value | access | gfn << PAGE_SHIFT;
299

300
	trace_mark_mmio_spte(sptep, gfn, access, gen);
301
	mmu_spte_set(sptep, mask);
302 303 304 305
}

static bool is_mmio_spte(u64 spte)
{
306
	return (spte & shadow_mmio_mask) == shadow_mmio_value;
307 308 309 310
}

static gfn_t get_mmio_spte_gfn(u64 spte)
{
T
Tiejun Chen 已提交
311
	u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
312
	return (spte & ~mask) >> PAGE_SHIFT;
313 314 315 316
}

static unsigned get_mmio_spte_access(u64 spte)
{
T
Tiejun Chen 已提交
317
	u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
318
	return (spte & ~mask) & ~PAGE_MASK;
319 320
}

321
static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
D
Dan Williams 已提交
322
			  kvm_pfn_t pfn, unsigned access)
323 324
{
	if (unlikely(is_noslot_pfn(pfn))) {
325
		mark_mmio_spte(vcpu, sptep, gfn, access);
326 327 328 329 330
		return true;
	}

	return false;
}
331

332
static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
333
{
334 335
	unsigned int kvm_gen, spte_gen;

336
	kvm_gen = kvm_current_mmio_generation(vcpu);
337 338 339 340
	spte_gen = get_mmio_spte_generation(spte);

	trace_check_mmio_spte(spte, kvm_gen, spte_gen);
	return likely(kvm_gen == spte_gen);
341 342
}

343 344 345 346 347 348 349
/*
 * Sets the shadow PTE masks used by the MMU.
 *
 * Assumptions:
 *  - Setting either @accessed_mask or @dirty_mask requires setting both
 *  - At least one of @accessed_mask or @acc_track_mask must be set
 */
S
Sheng Yang 已提交
350
void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
351 352
		u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask,
		u64 acc_track_mask)
S
Sheng Yang 已提交
353
{
354 355
	BUG_ON(!dirty_mask != !accessed_mask);
	BUG_ON(!accessed_mask && !acc_track_mask);
356
	BUG_ON(acc_track_mask & shadow_acc_track_value);
357

S
Sheng Yang 已提交
358 359 360 361 362
	shadow_user_mask = user_mask;
	shadow_accessed_mask = accessed_mask;
	shadow_dirty_mask = dirty_mask;
	shadow_nx_mask = nx_mask;
	shadow_x_mask = x_mask;
363
	shadow_present_mask = p_mask;
364
	shadow_acc_track_mask = acc_track_mask;
S
Sheng Yang 已提交
365 366 367
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);

368 369 370 371 372 373 374 375 376 377 378 379
void kvm_mmu_clear_all_pte_masks(void)
{
	shadow_user_mask = 0;
	shadow_accessed_mask = 0;
	shadow_dirty_mask = 0;
	shadow_nx_mask = 0;
	shadow_x_mask = 0;
	shadow_mmio_mask = 0;
	shadow_present_mask = 0;
	shadow_acc_track_mask = 0;
}

A
Avi Kivity 已提交
380 381 382 383 384
static int is_cpuid_PSE36(void)
{
	return 1;
}

385 386
static int is_nx(struct kvm_vcpu *vcpu)
{
387
	return vcpu->arch.efer & EFER_NX;
388 389
}

390 391
static int is_shadow_present_pte(u64 pte)
{
392
	return (pte != 0) && !is_mmio_spte(pte);
393 394
}

M
Marcelo Tosatti 已提交
395 396 397 398 399
static int is_large_pte(u64 pte)
{
	return pte & PT_PAGE_SIZE_MASK;
}

400 401 402 403
static int is_last_spte(u64 pte, int level)
{
	if (level == PT_PAGE_TABLE_LEVEL)
		return 1;
404
	if (is_large_pte(pte))
405 406 407 408
		return 1;
	return 0;
}

409 410 411 412 413
static bool is_executable_pte(u64 spte)
{
	return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
}

D
Dan Williams 已提交
414
static kvm_pfn_t spte_to_pfn(u64 pte)
415
{
416
	return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
417 418
}

419 420 421 422 423 424 425
static gfn_t pse36_gfn_delta(u32 gpte)
{
	int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;

	return (gpte & PT32_DIR_PSE36_MASK) << shift;
}

426
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
427
static void __set_spte(u64 *sptep, u64 spte)
428
{
429
	WRITE_ONCE(*sptep, spte);
430 431
}

432
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
433
{
434
	WRITE_ONCE(*sptep, spte);
435 436 437 438 439 440
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	return xchg(sptep, spte);
}
441 442 443 444 445

static u64 __get_spte_lockless(u64 *sptep)
{
	return ACCESS_ONCE(*sptep);
}
446
#else
447 448 449 450 451 452 453
union split_spte {
	struct {
		u32 spte_low;
		u32 spte_high;
	};
	u64 spte;
};
454

455 456 457 458 459 460 461 462 463 464 465 466
static void count_spte_clear(u64 *sptep, u64 spte)
{
	struct kvm_mmu_page *sp =  page_header(__pa(sptep));

	if (is_shadow_present_pte(spte))
		return;

	/* Ensure the spte is completely set before we increase the count */
	smp_wmb();
	sp->clear_spte_count++;
}

467 468 469
static void __set_spte(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;
470

471 472 473 474 475 476 477 478 479 480 481 482
	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	ssptep->spte_high = sspte.spte_high;

	/*
	 * If we map the spte from nonpresent to present, We should store
	 * the high bits firstly, then set present bit, so cpu can not
	 * fetch this spte while we are setting the spte.
	 */
	smp_wmb();

483
	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
484 485
}

486 487 488 489 490 491 492
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

493
	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
494 495 496 497 498 499 500 501

	/*
	 * If we map the spte from present to nonpresent, we should clear
	 * present bit firstly to avoid vcpu fetch the old high bits.
	 */
	smp_wmb();

	ssptep->spte_high = sspte.spte_high;
502
	count_spte_clear(sptep, spte);
503 504 505 506 507 508 509 510 511 512 513
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte, orig;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	/* xchg acts as a barrier before the setting of the high bits */
	orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
514 515
	orig.spte_high = ssptep->spte_high;
	ssptep->spte_high = sspte.spte_high;
516
	count_spte_clear(sptep, spte);
517 518 519

	return orig.spte;
}
520 521 522 523

/*
 * The idea using the light way get the spte on x86_32 guest is from
 * gup_get_pte(arch/x86/mm/gup.c).
524 525 526 527 528 529 530 531 532 533 534 535 536 537
 *
 * An spte tlb flush may be pending, because kvm_set_pte_rmapp
 * coalesces them and we are running out of the MMU lock.  Therefore
 * we need to protect against in-progress updates of the spte.
 *
 * Reading the spte while an update is in progress may get the old value
 * for the high part of the spte.  The race is fine for a present->non-present
 * change (because the high part of the spte is ignored for non-present spte),
 * but for a present->present change we must reread the spte.
 *
 * All such changes are done in two steps (present->non-present and
 * non-present->present), hence it is enough to count the number of
 * present->non-present updates: if it changed while reading the spte,
 * we might have hit the race.  This is done using clear_spte_count.
538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
 */
static u64 __get_spte_lockless(u64 *sptep)
{
	struct kvm_mmu_page *sp =  page_header(__pa(sptep));
	union split_spte spte, *orig = (union split_spte *)sptep;
	int count;

retry:
	count = sp->clear_spte_count;
	smp_rmb();

	spte.spte_low = orig->spte_low;
	smp_rmb();

	spte.spte_high = orig->spte_high;
	smp_rmb();

	if (unlikely(spte.spte_low != orig->spte_low ||
	      count != sp->clear_spte_count))
		goto retry;

	return spte.spte;
}
561 562
#endif

563
static bool spte_can_locklessly_be_made_writable(u64 spte)
564
{
565 566
	return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
		(SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
567 568
}

569 570
static bool spte_has_volatile_bits(u64 spte)
{
571 572 573
	if (!is_shadow_present_pte(spte))
		return false;

574
	/*
575
	 * Always atomically update spte if it can be updated
576 577 578 579
	 * out of mmu-lock, it can ensure dirty bit is not lost,
	 * also, it can help us to get a stable is_writable_pte()
	 * to ensure tlb flush is not missed.
	 */
580 581
	if (spte_can_locklessly_be_made_writable(spte) ||
	    is_access_track_spte(spte))
582 583
		return true;

584
	if (spte_ad_enabled(spte)) {
585 586 587 588
		if ((spte & shadow_accessed_mask) == 0 ||
	    	    (is_writable_pte(spte) && (spte & shadow_dirty_mask) == 0))
			return true;
	}
589

590
	return false;
591 592
}

593
static bool is_accessed_spte(u64 spte)
594
{
595 596 597 598
	u64 accessed_mask = spte_shadow_accessed_mask(spte);

	return accessed_mask ? spte & accessed_mask
			     : !is_access_track_spte(spte);
599 600
}

601
static bool is_dirty_spte(u64 spte)
602
{
603 604 605
	u64 dirty_mask = spte_shadow_dirty_mask(spte);

	return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK;
606 607
}

608 609 610 611 612 613 614 615 616 617 618 619
/* Rules for using mmu_spte_set:
 * Set the sptep from nonpresent to present.
 * Note: the sptep being assigned *must* be either not present
 * or in a state where the hardware will not attempt to update
 * the spte.
 */
static void mmu_spte_set(u64 *sptep, u64 new_spte)
{
	WARN_ON(is_shadow_present_pte(*sptep));
	__set_spte(sptep, new_spte);
}

620 621 622
/*
 * Update the SPTE (excluding the PFN), but do not track changes in its
 * accessed/dirty status.
623
 */
624
static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte)
625
{
626
	u64 old_spte = *sptep;
627

628
	WARN_ON(!is_shadow_present_pte(new_spte));
629

630 631
	if (!is_shadow_present_pte(old_spte)) {
		mmu_spte_set(sptep, new_spte);
632
		return old_spte;
633
	}
634

635
	if (!spte_has_volatile_bits(old_spte))
636
		__update_clear_spte_fast(sptep, new_spte);
637
	else
638
		old_spte = __update_clear_spte_slow(sptep, new_spte);
639

640 641
	WARN_ON(spte_to_pfn(old_spte) != spte_to_pfn(new_spte));

642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663
	return old_spte;
}

/* Rules for using mmu_spte_update:
 * Update the state bits, it means the mapped pfn is not changed.
 *
 * Whenever we overwrite a writable spte with a read-only one we
 * should flush remote TLBs. Otherwise rmap_write_protect
 * will find a read-only spte, even though the writable spte
 * might be cached on a CPU's TLB, the return value indicates this
 * case.
 *
 * Returns true if the TLB needs to be flushed
 */
static bool mmu_spte_update(u64 *sptep, u64 new_spte)
{
	bool flush = false;
	u64 old_spte = mmu_spte_update_no_track(sptep, new_spte);

	if (!is_shadow_present_pte(old_spte))
		return false;

664 665
	/*
	 * For the spte updated out of mmu-lock is safe, since
666
	 * we always atomically update it, see the comments in
667 668
	 * spte_has_volatile_bits().
	 */
669
	if (spte_can_locklessly_be_made_writable(old_spte) &&
670
	      !is_writable_pte(new_spte))
671
		flush = true;
672

673
	/*
674
	 * Flush TLB when accessed/dirty states are changed in the page tables,
675 676 677
	 * to guarantee consistency between TLB and page tables.
	 */

678 679
	if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) {
		flush = true;
680
		kvm_set_pfn_accessed(spte_to_pfn(old_spte));
681 682 683 684
	}

	if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) {
		flush = true;
685
		kvm_set_pfn_dirty(spte_to_pfn(old_spte));
686
	}
687

688
	return flush;
689 690
}

691 692 693 694
/*
 * Rules for using mmu_spte_clear_track_bits:
 * It sets the sptep from present to nonpresent, and track the
 * state bits, it is used to clear the last level sptep.
695
 * Returns non-zero if the PTE was previously valid.
696 697 698
 */
static int mmu_spte_clear_track_bits(u64 *sptep)
{
D
Dan Williams 已提交
699
	kvm_pfn_t pfn;
700 701 702
	u64 old_spte = *sptep;

	if (!spte_has_volatile_bits(old_spte))
703
		__update_clear_spte_fast(sptep, 0ull);
704
	else
705
		old_spte = __update_clear_spte_slow(sptep, 0ull);
706

707
	if (!is_shadow_present_pte(old_spte))
708 709 710
		return 0;

	pfn = spte_to_pfn(old_spte);
711 712 713 714 715 716

	/*
	 * KVM does not hold the refcount of the page used by
	 * kvm mmu, before reclaiming the page, we should
	 * unmap it from mmu first.
	 */
717
	WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
718

719
	if (is_accessed_spte(old_spte))
720
		kvm_set_pfn_accessed(pfn);
721 722

	if (is_dirty_spte(old_spte))
723
		kvm_set_pfn_dirty(pfn);
724

725 726 727 728 729 730 731 732 733 734
	return 1;
}

/*
 * Rules for using mmu_spte_clear_no_track:
 * Directly clear spte without caring the state bits of sptep,
 * it is used to set the upper level spte.
 */
static void mmu_spte_clear_no_track(u64 *sptep)
{
735
	__update_clear_spte_fast(sptep, 0ull);
736 737
}

738 739 740 741 742
static u64 mmu_spte_get_lockless(u64 *sptep)
{
	return __get_spte_lockless(sptep);
}

743 744
static u64 mark_spte_for_access_track(u64 spte)
{
745
	if (spte_ad_enabled(spte))
746 747
		return spte & ~shadow_accessed_mask;

748
	if (is_access_track_spte(spte))
749 750 751
		return spte;

	/*
752 753 754
	 * Making an Access Tracking PTE will result in removal of write access
	 * from the PTE. So, verify that we will be able to restore the write
	 * access in the fast page fault path later on.
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770
	 */
	WARN_ONCE((spte & PT_WRITABLE_MASK) &&
		  !spte_can_locklessly_be_made_writable(spte),
		  "kvm: Writable SPTE is not locklessly dirty-trackable\n");

	WARN_ONCE(spte & (shadow_acc_track_saved_bits_mask <<
			  shadow_acc_track_saved_bits_shift),
		  "kvm: Access Tracking saved bit locations are not zero\n");

	spte |= (spte & shadow_acc_track_saved_bits_mask) <<
		shadow_acc_track_saved_bits_shift;
	spte &= ~shadow_acc_track_mask;

	return spte;
}

771 772 773 774 775 776 777
/* Restore an acc-track PTE back to a regular PTE */
static u64 restore_acc_track_spte(u64 spte)
{
	u64 new_spte = spte;
	u64 saved_bits = (spte >> shadow_acc_track_saved_bits_shift)
			 & shadow_acc_track_saved_bits_mask;

778
	WARN_ON_ONCE(spte_ad_enabled(spte));
779 780 781 782 783 784 785 786 787 788
	WARN_ON_ONCE(!is_access_track_spte(spte));

	new_spte &= ~shadow_acc_track_mask;
	new_spte &= ~(shadow_acc_track_saved_bits_mask <<
		      shadow_acc_track_saved_bits_shift);
	new_spte |= saved_bits;

	return new_spte;
}

789 790 791 792 793 794 795 796
/* Returns the Accessed status of the PTE and resets it at the same time. */
static bool mmu_spte_age(u64 *sptep)
{
	u64 spte = mmu_spte_get_lockless(sptep);

	if (!is_accessed_spte(spte))
		return false;

797
	if (spte_ad_enabled(spte)) {
798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
		clear_bit((ffs(shadow_accessed_mask) - 1),
			  (unsigned long *)sptep);
	} else {
		/*
		 * Capture the dirty status of the page, so that it doesn't get
		 * lost when the SPTE is marked for access tracking.
		 */
		if (is_writable_pte(spte))
			kvm_set_pfn_dirty(spte_to_pfn(spte));

		spte = mark_spte_for_access_track(spte);
		mmu_spte_update_no_track(sptep, spte);
	}

	return true;
}

815 816
static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
{
817 818 819 820 821
	/*
	 * Prevent page table teardown by making any free-er wait during
	 * kvm_flush_remote_tlbs() IPI to all active vcpus.
	 */
	local_irq_disable();
822

823 824 825 826
	/*
	 * Make sure a following spte read is not reordered ahead of the write
	 * to vcpu->mode.
	 */
827
	smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
828 829 830 831
}

static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
{
832 833 834 835 836
	/*
	 * Make sure the write to vcpu->mode is not reordered in front of
	 * reads to sptes.  If it does, kvm_commit_zap_page() can see us
	 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
	 */
837
	smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
838
	local_irq_enable();
839 840
}

841
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
842
				  struct kmem_cache *base_cache, int min)
843 844 845 846
{
	void *obj;

	if (cache->nobjs >= min)
847
		return 0;
848
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
849
		obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
850
		if (!obj)
851
			return -ENOMEM;
852 853
		cache->objects[cache->nobjs++] = obj;
	}
854
	return 0;
855 856
}

857 858 859 860 861
static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
{
	return cache->nobjs;
}

862 863
static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
				  struct kmem_cache *cache)
864 865
{
	while (mc->nobjs)
866
		kmem_cache_free(cache, mc->objects[--mc->nobjs]);
867 868
}

A
Avi Kivity 已提交
869
static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
870
				       int min)
A
Avi Kivity 已提交
871
{
872
	void *page;
A
Avi Kivity 已提交
873 874 875 876

	if (cache->nobjs >= min)
		return 0;
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
877
		page = (void *)__get_free_page(GFP_KERNEL);
A
Avi Kivity 已提交
878 879
		if (!page)
			return -ENOMEM;
880
		cache->objects[cache->nobjs++] = page;
A
Avi Kivity 已提交
881 882 883 884 885 886 887
	}
	return 0;
}

static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
888
		free_page((unsigned long)mc->objects[--mc->nobjs]);
A
Avi Kivity 已提交
889 890
}

891
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
892
{
893 894
	int r;

895
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
896
				   pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
897 898
	if (r)
		goto out;
899
	r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
900 901
	if (r)
		goto out;
902
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
903
				   mmu_page_header_cache, 4);
904 905
out:
	return r;
906 907 908 909
}

static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
910 911
	mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
				pte_list_desc_cache);
912
	mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
913 914
	mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
				mmu_page_header_cache);
915 916
}

917
static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
918 919 920 921 922 923 924 925
{
	void *p;

	BUG_ON(!mc->nobjs);
	p = mc->objects[--mc->nobjs];
	return p;
}

926
static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
927
{
928
	return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
929 930
}

931
static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
932
{
933
	kmem_cache_free(pte_list_desc_cache, pte_list_desc);
934 935
}

936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
{
	if (!sp->role.direct)
		return sp->gfns[index];

	return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
}

static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
{
	if (sp->role.direct)
		BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
	else
		sp->gfns[index] = gfn;
}

M
Marcelo Tosatti 已提交
952
/*
953 954
 * Return the pointer to the large page information for a given gfn,
 * handling slots that are not large page aligned.
M
Marcelo Tosatti 已提交
955
 */
956 957 958
static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
					      struct kvm_memory_slot *slot,
					      int level)
M
Marcelo Tosatti 已提交
959 960 961
{
	unsigned long idx;

962
	idx = gfn_to_index(gfn, slot->base_gfn, level);
963
	return &slot->arch.lpage_info[level - 2][idx];
M
Marcelo Tosatti 已提交
964 965
}

966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988
static void update_gfn_disallow_lpage_count(struct kvm_memory_slot *slot,
					    gfn_t gfn, int count)
{
	struct kvm_lpage_info *linfo;
	int i;

	for (i = PT_DIRECTORY_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->disallow_lpage += count;
		WARN_ON(linfo->disallow_lpage < 0);
	}
}

void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn)
{
	update_gfn_disallow_lpage_count(slot, gfn, 1);
}

void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn)
{
	update_gfn_disallow_lpage_count(slot, gfn, -1);
}

989
static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
M
Marcelo Tosatti 已提交
990
{
991
	struct kvm_memslots *slots;
992
	struct kvm_memory_slot *slot;
993
	gfn_t gfn;
M
Marcelo Tosatti 已提交
994

995
	kvm->arch.indirect_shadow_pages++;
996
	gfn = sp->gfn;
997 998
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
999 1000 1001 1002 1003 1004

	/* the non-leaf shadow pages are keeping readonly. */
	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return kvm_slot_page_track_add_page(kvm, slot, gfn,
						    KVM_PAGE_TRACK_WRITE);

1005
	kvm_mmu_gfn_disallow_lpage(slot, gfn);
M
Marcelo Tosatti 已提交
1006 1007
}

1008
static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
M
Marcelo Tosatti 已提交
1009
{
1010
	struct kvm_memslots *slots;
1011
	struct kvm_memory_slot *slot;
1012
	gfn_t gfn;
M
Marcelo Tosatti 已提交
1013

1014
	kvm->arch.indirect_shadow_pages--;
1015
	gfn = sp->gfn;
1016 1017
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
1018 1019 1020 1021
	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return kvm_slot_page_track_remove_page(kvm, slot, gfn,
						       KVM_PAGE_TRACK_WRITE);

1022
	kvm_mmu_gfn_allow_lpage(slot, gfn);
M
Marcelo Tosatti 已提交
1023 1024
}

1025 1026
static bool __mmu_gfn_lpage_is_disallowed(gfn_t gfn, int level,
					  struct kvm_memory_slot *slot)
M
Marcelo Tosatti 已提交
1027
{
1028
	struct kvm_lpage_info *linfo;
M
Marcelo Tosatti 已提交
1029 1030

	if (slot) {
1031
		linfo = lpage_info_slot(gfn, slot, level);
1032
		return !!linfo->disallow_lpage;
M
Marcelo Tosatti 已提交
1033 1034
	}

1035
	return true;
M
Marcelo Tosatti 已提交
1036 1037
}

1038 1039
static bool mmu_gfn_lpage_is_disallowed(struct kvm_vcpu *vcpu, gfn_t gfn,
					int level)
1040 1041 1042 1043
{
	struct kvm_memory_slot *slot;

	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1044
	return __mmu_gfn_lpage_is_disallowed(gfn, level, slot);
1045 1046
}

1047
static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
M
Marcelo Tosatti 已提交
1048
{
J
Joerg Roedel 已提交
1049
	unsigned long page_size;
1050
	int i, ret = 0;
M
Marcelo Tosatti 已提交
1051

J
Joerg Roedel 已提交
1052
	page_size = kvm_host_page_size(kvm, gfn);
M
Marcelo Tosatti 已提交
1053

1054
	for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
1055 1056 1057 1058 1059 1060
		if (page_size >= KVM_HPAGE_SIZE(i))
			ret = i;
		else
			break;
	}

1061
	return ret;
M
Marcelo Tosatti 已提交
1062 1063
}

1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
static inline bool memslot_valid_for_gpte(struct kvm_memory_slot *slot,
					  bool no_dirty_log)
{
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
		return false;
	if (no_dirty_log && slot->dirty_bitmap)
		return false;

	return true;
}

1075 1076 1077
static struct kvm_memory_slot *
gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
			    bool no_dirty_log)
M
Marcelo Tosatti 已提交
1078 1079
{
	struct kvm_memory_slot *slot;
1080

1081
	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1082
	if (!memslot_valid_for_gpte(slot, no_dirty_log))
1083 1084 1085 1086 1087
		slot = NULL;

	return slot;
}

1088 1089
static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn,
			 bool *force_pt_level)
1090 1091
{
	int host_level, level, max_level;
1092 1093
	struct kvm_memory_slot *slot;

1094 1095
	if (unlikely(*force_pt_level))
		return PT_PAGE_TABLE_LEVEL;
M
Marcelo Tosatti 已提交
1096

1097 1098
	slot = kvm_vcpu_gfn_to_memslot(vcpu, large_gfn);
	*force_pt_level = !memslot_valid_for_gpte(slot, true);
1099 1100 1101
	if (unlikely(*force_pt_level))
		return PT_PAGE_TABLE_LEVEL;

1102 1103 1104 1105 1106
	host_level = host_mapping_level(vcpu->kvm, large_gfn);

	if (host_level == PT_PAGE_TABLE_LEVEL)
		return host_level;

X
Xiao Guangrong 已提交
1107
	max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
1108 1109

	for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
1110
		if (__mmu_gfn_lpage_is_disallowed(large_gfn, level, slot))
1111 1112 1113
			break;

	return level - 1;
M
Marcelo Tosatti 已提交
1114 1115
}

1116
/*
1117
 * About rmap_head encoding:
1118
 *
1119 1120
 * If the bit zero of rmap_head->val is clear, then it points to the only spte
 * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
1121
 * pte_list_desc containing more mappings.
1122 1123 1124 1125
 */

/*
 * Returns the number of pointers in the rmap chain, not counting the new one.
1126
 */
1127
static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
1128
			struct kvm_rmap_head *rmap_head)
1129
{
1130
	struct pte_list_desc *desc;
1131
	int i, count = 0;
1132

1133
	if (!rmap_head->val) {
1134
		rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
1135 1136
		rmap_head->val = (unsigned long)spte;
	} else if (!(rmap_head->val & 1)) {
1137 1138
		rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
		desc = mmu_alloc_pte_list_desc(vcpu);
1139
		desc->sptes[0] = (u64 *)rmap_head->val;
A
Avi Kivity 已提交
1140
		desc->sptes[1] = spte;
1141
		rmap_head->val = (unsigned long)desc | 1;
1142
		++count;
1143
	} else {
1144
		rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
1145
		desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1146
		while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
1147
			desc = desc->more;
1148
			count += PTE_LIST_EXT;
1149
		}
1150 1151
		if (desc->sptes[PTE_LIST_EXT-1]) {
			desc->more = mmu_alloc_pte_list_desc(vcpu);
1152 1153
			desc = desc->more;
		}
A
Avi Kivity 已提交
1154
		for (i = 0; desc->sptes[i]; ++i)
1155
			++count;
A
Avi Kivity 已提交
1156
		desc->sptes[i] = spte;
1157
	}
1158
	return count;
1159 1160
}

1161
static void
1162 1163 1164
pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
			   struct pte_list_desc *desc, int i,
			   struct pte_list_desc *prev_desc)
1165 1166 1167
{
	int j;

1168
	for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
1169
		;
A
Avi Kivity 已提交
1170 1171
	desc->sptes[i] = desc->sptes[j];
	desc->sptes[j] = NULL;
1172 1173 1174
	if (j != 0)
		return;
	if (!prev_desc && !desc->more)
1175
		rmap_head->val = (unsigned long)desc->sptes[0];
1176 1177 1178 1179
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
1180
			rmap_head->val = (unsigned long)desc->more | 1;
1181
	mmu_free_pte_list_desc(desc);
1182 1183
}

1184
static void pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
1185
{
1186 1187
	struct pte_list_desc *desc;
	struct pte_list_desc *prev_desc;
1188 1189
	int i;

1190
	if (!rmap_head->val) {
1191
		printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
1192
		BUG();
1193
	} else if (!(rmap_head->val & 1)) {
1194
		rmap_printk("pte_list_remove:  %p 1->0\n", spte);
1195
		if ((u64 *)rmap_head->val != spte) {
1196
			printk(KERN_ERR "pte_list_remove:  %p 1->BUG\n", spte);
1197 1198
			BUG();
		}
1199
		rmap_head->val = 0;
1200
	} else {
1201
		rmap_printk("pte_list_remove:  %p many->many\n", spte);
1202
		desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1203 1204
		prev_desc = NULL;
		while (desc) {
1205
			for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) {
A
Avi Kivity 已提交
1206
				if (desc->sptes[i] == spte) {
1207 1208
					pte_list_desc_remove_entry(rmap_head,
							desc, i, prev_desc);
1209 1210
					return;
				}
1211
			}
1212 1213 1214
			prev_desc = desc;
			desc = desc->more;
		}
1215
		pr_err("pte_list_remove: %p many->many\n", spte);
1216 1217 1218 1219
		BUG();
	}
}

1220 1221
static struct kvm_rmap_head *__gfn_to_rmap(gfn_t gfn, int level,
					   struct kvm_memory_slot *slot)
1222
{
1223
	unsigned long idx;
1224

1225
	idx = gfn_to_index(gfn, slot->base_gfn, level);
1226
	return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
1227 1228
}

1229 1230
static struct kvm_rmap_head *gfn_to_rmap(struct kvm *kvm, gfn_t gfn,
					 struct kvm_mmu_page *sp)
1231
{
1232
	struct kvm_memslots *slots;
1233 1234
	struct kvm_memory_slot *slot;

1235 1236
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
1237
	return __gfn_to_rmap(gfn, sp->role.level, slot);
1238 1239
}

1240 1241 1242 1243 1244 1245 1246 1247
static bool rmap_can_add(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_memory_cache *cache;

	cache = &vcpu->arch.mmu_pte_list_desc_cache;
	return mmu_memory_cache_free_objects(cache);
}

1248 1249 1250
static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
{
	struct kvm_mmu_page *sp;
1251
	struct kvm_rmap_head *rmap_head;
1252 1253 1254

	sp = page_header(__pa(spte));
	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1255 1256
	rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
	return pte_list_add(vcpu, spte, rmap_head);
1257 1258 1259 1260 1261 1262
}

static void rmap_remove(struct kvm *kvm, u64 *spte)
{
	struct kvm_mmu_page *sp;
	gfn_t gfn;
1263
	struct kvm_rmap_head *rmap_head;
1264 1265 1266

	sp = page_header(__pa(spte));
	gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1267 1268
	rmap_head = gfn_to_rmap(kvm, gfn, sp);
	pte_list_remove(spte, rmap_head);
1269 1270
}

1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
/*
 * Used by the following functions to iterate through the sptes linked by a
 * rmap.  All fields are private and not assumed to be used outside.
 */
struct rmap_iterator {
	/* private fields */
	struct pte_list_desc *desc;	/* holds the sptep if not NULL */
	int pos;			/* index of the sptep */
};

/*
 * Iteration must be started by this function.  This should also be used after
 * removing/dropping sptes from the rmap link because in such cases the
 * information in the itererator may not be valid.
 *
 * Returns sptep if found, NULL otherwise.
 */
1288 1289
static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
			   struct rmap_iterator *iter)
1290
{
1291 1292
	u64 *sptep;

1293
	if (!rmap_head->val)
1294 1295
		return NULL;

1296
	if (!(rmap_head->val & 1)) {
1297
		iter->desc = NULL;
1298 1299
		sptep = (u64 *)rmap_head->val;
		goto out;
1300 1301
	}

1302
	iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1303
	iter->pos = 0;
1304 1305 1306 1307
	sptep = iter->desc->sptes[iter->pos];
out:
	BUG_ON(!is_shadow_present_pte(*sptep));
	return sptep;
1308 1309 1310 1311 1312 1313 1314 1315 1316
}

/*
 * Must be used with a valid iterator: e.g. after rmap_get_first().
 *
 * Returns sptep if found, NULL otherwise.
 */
static u64 *rmap_get_next(struct rmap_iterator *iter)
{
1317 1318
	u64 *sptep;

1319 1320 1321 1322 1323
	if (iter->desc) {
		if (iter->pos < PTE_LIST_EXT - 1) {
			++iter->pos;
			sptep = iter->desc->sptes[iter->pos];
			if (sptep)
1324
				goto out;
1325 1326 1327 1328 1329 1330 1331
		}

		iter->desc = iter->desc->more;

		if (iter->desc) {
			iter->pos = 0;
			/* desc->sptes[0] cannot be NULL */
1332 1333
			sptep = iter->desc->sptes[iter->pos];
			goto out;
1334 1335 1336 1337
		}
	}

	return NULL;
1338 1339 1340
out:
	BUG_ON(!is_shadow_present_pte(*sptep));
	return sptep;
1341 1342
}

1343 1344
#define for_each_rmap_spte(_rmap_head_, _iter_, _spte_)			\
	for (_spte_ = rmap_get_first(_rmap_head_, _iter_);		\
1345
	     _spte_; _spte_ = rmap_get_next(_iter_))
1346

1347
static void drop_spte(struct kvm *kvm, u64 *sptep)
1348
{
1349
	if (mmu_spte_clear_track_bits(sptep))
1350
		rmap_remove(kvm, sptep);
A
Avi Kivity 已提交
1351 1352
}

1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373

static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
{
	if (is_large_pte(*sptep)) {
		WARN_ON(page_header(__pa(sptep))->role.level ==
			PT_PAGE_TABLE_LEVEL);
		drop_spte(kvm, sptep);
		--kvm->stat.lpages;
		return true;
	}

	return false;
}

static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
{
	if (__drop_large_spte(vcpu->kvm, sptep))
		kvm_flush_remote_tlbs(vcpu->kvm);
}

/*
1374
 * Write-protect on the specified @sptep, @pt_protect indicates whether
1375
 * spte write-protection is caused by protecting shadow page table.
1376
 *
T
Tiejun Chen 已提交
1377
 * Note: write protection is difference between dirty logging and spte
1378 1379 1380 1381 1382
 * protection:
 * - for dirty logging, the spte can be set to writable at anytime if
 *   its dirty bitmap is properly set.
 * - for spte protection, the spte can be writable only after unsync-ing
 *   shadow page.
1383
 *
1384
 * Return true if tlb need be flushed.
1385
 */
1386
static bool spte_write_protect(u64 *sptep, bool pt_protect)
1387 1388 1389
{
	u64 spte = *sptep;

1390
	if (!is_writable_pte(spte) &&
1391
	      !(pt_protect && spte_can_locklessly_be_made_writable(spte)))
1392 1393 1394 1395
		return false;

	rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);

1396 1397
	if (pt_protect)
		spte &= ~SPTE_MMU_WRITEABLE;
1398
	spte = spte & ~PT_WRITABLE_MASK;
1399

1400
	return mmu_spte_update(sptep, spte);
1401 1402
}

1403 1404
static bool __rmap_write_protect(struct kvm *kvm,
				 struct kvm_rmap_head *rmap_head,
1405
				 bool pt_protect)
1406
{
1407 1408
	u64 *sptep;
	struct rmap_iterator iter;
1409
	bool flush = false;
1410

1411
	for_each_rmap_spte(rmap_head, &iter, sptep)
1412
		flush |= spte_write_protect(sptep, pt_protect);
1413

1414
	return flush;
1415 1416
}

1417
static bool spte_clear_dirty(u64 *sptep)
1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
{
	u64 spte = *sptep;

	rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);

	spte &= ~shadow_dirty_mask;

	return mmu_spte_update(sptep, spte);
}

1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443
static bool wrprot_ad_disabled_spte(u64 *sptep)
{
	bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT,
					       (unsigned long *)sptep);
	if (was_writable)
		kvm_set_pfn_dirty(spte_to_pfn(*sptep));

	return was_writable;
}

/*
 * Gets the GFN ready for another round of dirty logging by clearing the
 *	- D bit on ad-enabled SPTEs, and
 *	- W bit on ad-disabled SPTEs.
 * Returns true iff any D or W bits were cleared.
 */
1444
static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1445 1446 1447 1448 1449
{
	u64 *sptep;
	struct rmap_iterator iter;
	bool flush = false;

1450
	for_each_rmap_spte(rmap_head, &iter, sptep)
1451 1452 1453 1454
		if (spte_ad_enabled(*sptep))
			flush |= spte_clear_dirty(sptep);
		else
			flush |= wrprot_ad_disabled_spte(sptep);
1455 1456 1457 1458

	return flush;
}

1459
static bool spte_set_dirty(u64 *sptep)
1460 1461 1462 1463 1464 1465 1466 1467 1468 1469
{
	u64 spte = *sptep;

	rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);

	spte |= shadow_dirty_mask;

	return mmu_spte_update(sptep, spte);
}

1470
static bool __rmap_set_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1471 1472 1473 1474 1475
{
	u64 *sptep;
	struct rmap_iterator iter;
	bool flush = false;

1476
	for_each_rmap_spte(rmap_head, &iter, sptep)
1477 1478
		if (spte_ad_enabled(*sptep))
			flush |= spte_set_dirty(sptep);
1479 1480 1481 1482

	return flush;
}

1483
/**
1484
 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1485 1486 1487 1488 1489 1490 1491 1492
 * @kvm: kvm instance
 * @slot: slot to protect
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should protect
 *
 * Used when we do not need to care about huge page mappings: e.g. during dirty
 * logging we do not have any such mappings.
 */
1493
static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1494 1495
				     struct kvm_memory_slot *slot,
				     gfn_t gfn_offset, unsigned long mask)
1496
{
1497
	struct kvm_rmap_head *rmap_head;
1498

1499
	while (mask) {
1500 1501 1502
		rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
					  PT_PAGE_TABLE_LEVEL, slot);
		__rmap_write_protect(kvm, rmap_head, false);
M
Marcelo Tosatti 已提交
1503

1504 1505 1506
		/* clear the first set bit */
		mask &= mask - 1;
	}
1507 1508
}

1509
/**
1510 1511
 * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write
 * protect the page if the D-bit isn't supported.
1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522
 * @kvm: kvm instance
 * @slot: slot to clear D-bit
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should clear D-bit
 *
 * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
 */
void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
				     struct kvm_memory_slot *slot,
				     gfn_t gfn_offset, unsigned long mask)
{
1523
	struct kvm_rmap_head *rmap_head;
1524 1525

	while (mask) {
1526 1527 1528
		rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
					  PT_PAGE_TABLE_LEVEL, slot);
		__rmap_clear_dirty(kvm, rmap_head);
1529 1530 1531 1532 1533 1534 1535

		/* clear the first set bit */
		mask &= mask - 1;
	}
}
EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked);

1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
/**
 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
 * PT level pages.
 *
 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
 * enable dirty logging for them.
 *
 * Used when we do not need to care about huge page mappings: e.g. during dirty
 * logging we do not have any such mappings.
 */
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
				struct kvm_memory_slot *slot,
				gfn_t gfn_offset, unsigned long mask)
{
1550 1551 1552 1553 1554
	if (kvm_x86_ops->enable_log_dirty_pt_masked)
		kvm_x86_ops->enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
				mask);
	else
		kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1555 1556
}

1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571
/**
 * kvm_arch_write_log_dirty - emulate dirty page logging
 * @vcpu: Guest mode vcpu
 *
 * Emulate arch specific page modification logging for the
 * nested hypervisor
 */
int kvm_arch_write_log_dirty(struct kvm_vcpu *vcpu)
{
	if (kvm_x86_ops->write_log_dirty)
		return kvm_x86_ops->write_log_dirty(vcpu);

	return 0;
}

1572 1573
bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
				    struct kvm_memory_slot *slot, u64 gfn)
1574
{
1575
	struct kvm_rmap_head *rmap_head;
1576
	int i;
1577
	bool write_protected = false;
1578

1579
	for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
1580
		rmap_head = __gfn_to_rmap(gfn, i, slot);
1581
		write_protected |= __rmap_write_protect(kvm, rmap_head, true);
1582 1583 1584
	}

	return write_protected;
1585 1586
}

1587 1588 1589 1590 1591 1592 1593 1594
static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
{
	struct kvm_memory_slot *slot;

	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
	return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn);
}

1595
static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1596
{
1597 1598
	u64 *sptep;
	struct rmap_iterator iter;
1599
	bool flush = false;
1600

1601
	while ((sptep = rmap_get_first(rmap_head, &iter))) {
1602
		rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);
1603 1604

		drop_spte(kvm, sptep);
1605
		flush = true;
1606
	}
1607

1608 1609 1610
	return flush;
}

1611
static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1612 1613 1614
			   struct kvm_memory_slot *slot, gfn_t gfn, int level,
			   unsigned long data)
{
1615
	return kvm_zap_rmapp(kvm, rmap_head);
1616 1617
}

1618
static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1619 1620
			     struct kvm_memory_slot *slot, gfn_t gfn, int level,
			     unsigned long data)
1621
{
1622 1623
	u64 *sptep;
	struct rmap_iterator iter;
1624
	int need_flush = 0;
1625
	u64 new_spte;
1626
	pte_t *ptep = (pte_t *)data;
D
Dan Williams 已提交
1627
	kvm_pfn_t new_pfn;
1628 1629 1630

	WARN_ON(pte_huge(*ptep));
	new_pfn = pte_pfn(*ptep);
1631

1632
restart:
1633
	for_each_rmap_spte(rmap_head, &iter, sptep) {
1634
		rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
1635
			    sptep, *sptep, gfn, level);
1636

1637
		need_flush = 1;
1638

1639
		if (pte_write(*ptep)) {
1640
			drop_spte(kvm, sptep);
1641
			goto restart;
1642
		} else {
1643
			new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
1644 1645 1646 1647
			new_spte |= (u64)new_pfn << PAGE_SHIFT;

			new_spte &= ~PT_WRITABLE_MASK;
			new_spte &= ~SPTE_HOST_WRITEABLE;
1648 1649

			new_spte = mark_spte_for_access_track(new_spte);
1650 1651 1652

			mmu_spte_clear_track_bits(sptep);
			mmu_spte_set(sptep, new_spte);
1653 1654
		}
	}
1655

1656 1657 1658 1659 1660 1661
	if (need_flush)
		kvm_flush_remote_tlbs(kvm);

	return 0;
}

1662 1663 1664 1665 1666 1667 1668 1669 1670 1671
struct slot_rmap_walk_iterator {
	/* input fields. */
	struct kvm_memory_slot *slot;
	gfn_t start_gfn;
	gfn_t end_gfn;
	int start_level;
	int end_level;

	/* output fields. */
	gfn_t gfn;
1672
	struct kvm_rmap_head *rmap;
1673 1674 1675
	int level;

	/* private field. */
1676
	struct kvm_rmap_head *end_rmap;
1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729
};

static void
rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)
{
	iterator->level = level;
	iterator->gfn = iterator->start_gfn;
	iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot);
	iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level,
					   iterator->slot);
}

static void
slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
		    struct kvm_memory_slot *slot, int start_level,
		    int end_level, gfn_t start_gfn, gfn_t end_gfn)
{
	iterator->slot = slot;
	iterator->start_level = start_level;
	iterator->end_level = end_level;
	iterator->start_gfn = start_gfn;
	iterator->end_gfn = end_gfn;

	rmap_walk_init_level(iterator, iterator->start_level);
}

static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
{
	return !!iterator->rmap;
}

static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
{
	if (++iterator->rmap <= iterator->end_rmap) {
		iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
		return;
	}

	if (++iterator->level > iterator->end_level) {
		iterator->rmap = NULL;
		return;
	}

	rmap_walk_init_level(iterator, iterator->level);
}

#define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_,	\
	   _start_gfn, _end_gfn, _iter_)				\
	for (slot_rmap_walk_init(_iter_, _slot_, _start_level_,		\
				 _end_level_, _start_gfn, _end_gfn);	\
	     slot_rmap_walk_okay(_iter_);				\
	     slot_rmap_walk_next(_iter_))

1730 1731 1732 1733 1734
static int kvm_handle_hva_range(struct kvm *kvm,
				unsigned long start,
				unsigned long end,
				unsigned long data,
				int (*handler)(struct kvm *kvm,
1735
					       struct kvm_rmap_head *rmap_head,
1736
					       struct kvm_memory_slot *slot,
1737 1738
					       gfn_t gfn,
					       int level,
1739
					       unsigned long data))
1740
{
1741
	struct kvm_memslots *slots;
1742
	struct kvm_memory_slot *memslot;
1743 1744
	struct slot_rmap_walk_iterator iterator;
	int ret = 0;
1745
	int i;
1746

1747 1748 1749 1750 1751
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
		kvm_for_each_memslot(memslot, slots) {
			unsigned long hva_start, hva_end;
			gfn_t gfn_start, gfn_end;
1752

1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771
			hva_start = max(start, memslot->userspace_addr);
			hva_end = min(end, memslot->userspace_addr +
				      (memslot->npages << PAGE_SHIFT));
			if (hva_start >= hva_end)
				continue;
			/*
			 * {gfn(page) | page intersects with [hva_start, hva_end)} =
			 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
			 */
			gfn_start = hva_to_gfn_memslot(hva_start, memslot);
			gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);

			for_each_slot_rmap_range(memslot, PT_PAGE_TABLE_LEVEL,
						 PT_MAX_HUGEPAGE_LEVEL,
						 gfn_start, gfn_end - 1,
						 &iterator)
				ret |= handler(kvm, iterator.rmap, memslot,
					       iterator.gfn, iterator.level, data);
		}
1772 1773
	}

1774
	return ret;
1775 1776
}

1777 1778
static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
			  unsigned long data,
1779 1780
			  int (*handler)(struct kvm *kvm,
					 struct kvm_rmap_head *rmap_head,
1781
					 struct kvm_memory_slot *slot,
1782
					 gfn_t gfn, int level,
1783 1784 1785
					 unsigned long data))
{
	return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1786 1787 1788 1789
}

int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
{
1790 1791 1792
	return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
}

1793 1794 1795 1796 1797
int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
{
	return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
}

1798 1799
void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
{
F
Frederik Deweerdt 已提交
1800
	kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1801 1802
}

1803
static int kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1804 1805
			 struct kvm_memory_slot *slot, gfn_t gfn, int level,
			 unsigned long data)
1806
{
1807
	u64 *sptep;
1808
	struct rmap_iterator uninitialized_var(iter);
1809 1810
	int young = 0;

1811 1812
	for_each_rmap_spte(rmap_head, &iter, sptep)
		young |= mmu_spte_age(sptep);
1813

1814
	trace_kvm_age_page(gfn, level, slot, young);
1815 1816 1817
	return young;
}

1818
static int kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1819 1820
			      struct kvm_memory_slot *slot, gfn_t gfn,
			      int level, unsigned long data)
A
Andrea Arcangeli 已提交
1821
{
1822 1823
	u64 *sptep;
	struct rmap_iterator iter;
A
Andrea Arcangeli 已提交
1824

1825 1826 1827 1828
	for_each_rmap_spte(rmap_head, &iter, sptep)
		if (is_accessed_spte(*sptep))
			return 1;
	return 0;
A
Andrea Arcangeli 已提交
1829 1830
}

1831 1832
#define RMAP_RECYCLE_THRESHOLD 1000

1833
static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1834
{
1835
	struct kvm_rmap_head *rmap_head;
1836 1837 1838
	struct kvm_mmu_page *sp;

	sp = page_header(__pa(spte));
1839

1840
	rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
1841

1842
	kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, 0);
1843 1844 1845
	kvm_flush_remote_tlbs(vcpu->kvm);
}

A
Andres Lagar-Cavilla 已提交
1846
int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1847
{
A
Andres Lagar-Cavilla 已提交
1848
	return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
1849 1850
}

A
Andrea Arcangeli 已提交
1851 1852 1853 1854 1855
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
{
	return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
}

1856
#ifdef MMU_DEBUG
1857
static int is_empty_shadow_page(u64 *spt)
A
Avi Kivity 已提交
1858
{
1859 1860 1861
	u64 *pos;
	u64 *end;

1862
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1863
		if (is_shadow_present_pte(*pos)) {
1864
			printk(KERN_ERR "%s: %p %llx\n", __func__,
1865
			       pos, *pos);
A
Avi Kivity 已提交
1866
			return 0;
1867
		}
A
Avi Kivity 已提交
1868 1869
	return 1;
}
1870
#endif
A
Avi Kivity 已提交
1871

1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883
/*
 * This value is the sum of all of the kvm instances's
 * kvm->arch.n_used_mmu_pages values.  We need a global,
 * aggregate version in order to make the slab shrinker
 * faster
 */
static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
{
	kvm->arch.n_used_mmu_pages += nr;
	percpu_counter_add(&kvm_total_used_mmu_pages, nr);
}

1884
static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1885
{
1886
	MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
1887
	hlist_del(&sp->hash_link);
1888 1889
	list_del(&sp->link);
	free_page((unsigned long)sp->spt);
1890 1891
	if (!sp->role.direct)
		free_page((unsigned long)sp->gfns);
1892
	kmem_cache_free(mmu_page_header_cache, sp);
1893 1894
}

1895 1896
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
1897
	return hash_64(gfn, KVM_MMU_HASH_SHIFT);
1898 1899
}

1900
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1901
				    struct kvm_mmu_page *sp, u64 *parent_pte)
1902 1903 1904 1905
{
	if (!parent_pte)
		return;

1906
	pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1907 1908
}

1909
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1910 1911
				       u64 *parent_pte)
{
1912
	pte_list_remove(parent_pte, &sp->parent_ptes);
1913 1914
}

1915 1916 1917 1918
static void drop_parent_pte(struct kvm_mmu_page *sp,
			    u64 *parent_pte)
{
	mmu_page_remove_parent_pte(sp, parent_pte);
1919
	mmu_spte_clear_no_track(parent_pte);
1920 1921
}

1922
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct)
M
Marcelo Tosatti 已提交
1923
{
1924
	struct kvm_mmu_page *sp;
1925

1926 1927
	sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
	sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1928
	if (!direct)
1929
		sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1930
	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1931 1932 1933 1934 1935 1936

	/*
	 * The active_mmu_pages list is the FIFO list, do not move the
	 * page until it is zapped. kvm_zap_obsolete_pages depends on
	 * this feature. See the comments in kvm_zap_obsolete_pages().
	 */
1937 1938 1939
	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
	kvm_mod_used_mmu_pages(vcpu->kvm, +1);
	return sp;
M
Marcelo Tosatti 已提交
1940 1941
}

1942
static void mark_unsync(u64 *spte);
1943
static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1944
{
1945 1946 1947 1948 1949 1950
	u64 *sptep;
	struct rmap_iterator iter;

	for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
		mark_unsync(sptep);
	}
1951 1952
}

1953
static void mark_unsync(u64 *spte)
1954
{
1955
	struct kvm_mmu_page *sp;
1956
	unsigned int index;
1957

1958
	sp = page_header(__pa(spte));
1959 1960
	index = spte - sp->spt;
	if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1961
		return;
1962
	if (sp->unsync_children++)
1963
		return;
1964
	kvm_mmu_mark_parents_unsync(sp);
1965 1966
}

1967
static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1968
			       struct kvm_mmu_page *sp)
1969
{
1970
	return 0;
1971 1972
}

M
Marcelo Tosatti 已提交
1973 1974 1975 1976
static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
}

1977 1978
static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
				 struct kvm_mmu_page *sp, u64 *spte,
1979
				 const void *pte)
1980 1981 1982 1983
{
	WARN_ON(1);
}

1984 1985 1986 1987 1988 1989 1990 1991 1992 1993
#define KVM_PAGE_ARRAY_NR 16

struct kvm_mmu_pages {
	struct mmu_page_and_offset {
		struct kvm_mmu_page *sp;
		unsigned int idx;
	} page[KVM_PAGE_ARRAY_NR];
	unsigned int nr;
};

1994 1995
static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
			 int idx)
1996
{
1997
	int i;
1998

1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009
	if (sp->unsync)
		for (i=0; i < pvec->nr; i++)
			if (pvec->page[i].sp == sp)
				return 0;

	pvec->page[pvec->nr].sp = sp;
	pvec->page[pvec->nr].idx = idx;
	pvec->nr++;
	return (pvec->nr == KVM_PAGE_ARRAY_NR);
}

2010 2011 2012 2013 2014 2015 2016
static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
{
	--sp->unsync_children;
	WARN_ON((int)sp->unsync_children < 0);
	__clear_bit(idx, sp->unsync_child_bitmap);
}

2017 2018 2019 2020
static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	int i, ret, nr_unsync_leaf = 0;
2021

2022
	for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
2023
		struct kvm_mmu_page *child;
2024 2025
		u64 ent = sp->spt[i];

2026 2027 2028 2029
		if (!is_shadow_present_pte(ent) || is_large_pte(ent)) {
			clear_unsync_child_bit(sp, i);
			continue;
		}
2030 2031 2032 2033 2034 2035 2036 2037

		child = page_header(ent & PT64_BASE_ADDR_MASK);

		if (child->unsync_children) {
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;

			ret = __mmu_unsync_walk(child, pvec);
2038 2039 2040 2041
			if (!ret) {
				clear_unsync_child_bit(sp, i);
				continue;
			} else if (ret > 0) {
2042
				nr_unsync_leaf += ret;
2043
			} else
2044 2045 2046 2047 2048 2049
				return ret;
		} else if (child->unsync) {
			nr_unsync_leaf++;
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;
		} else
2050
			clear_unsync_child_bit(sp, i);
2051 2052
	}

2053 2054 2055
	return nr_unsync_leaf;
}

2056 2057
#define INVALID_INDEX (-1)

2058 2059 2060
static int mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
P
Paolo Bonzini 已提交
2061
	pvec->nr = 0;
2062 2063 2064
	if (!sp->unsync_children)
		return 0;

2065
	mmu_pages_add(pvec, sp, INVALID_INDEX);
2066
	return __mmu_unsync_walk(sp, pvec);
2067 2068 2069 2070 2071
}

static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	WARN_ON(!sp->unsync);
2072
	trace_kvm_mmu_sync_page(sp);
2073 2074 2075 2076
	sp->unsync = 0;
	--kvm->stat.mmu_unsync;
}

2077 2078 2079 2080
static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				    struct list_head *invalid_list);
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list);
2081

2082 2083 2084 2085 2086 2087
/*
 * NOTE: we should pay more attention on the zapped-obsolete page
 * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
 * since it has been deleted from active_mmu_pages but still can be found
 * at hast list.
 *
2088
 * for_each_valid_sp() has skipped that kind of pages.
2089
 */
2090
#define for_each_valid_sp(_kvm, _sp, _gfn)				\
2091 2092
	hlist_for_each_entry(_sp,					\
	  &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
2093 2094
		if (is_obsolete_sp((_kvm), (_sp)) || (_sp)->role.invalid) {    \
		} else
2095 2096

#define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn)			\
2097 2098
	for_each_valid_sp(_kvm, _sp, _gfn)				\
		if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else
2099

2100
/* @sp->gfn should be write-protected at the call site */
2101 2102
static bool __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
			    struct list_head *invalid_list)
2103
{
2104
	if (sp->role.cr4_pae != !!is_pae(vcpu)) {
2105
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
2106
		return false;
2107 2108
	}

2109
	if (vcpu->arch.mmu.sync_page(vcpu, sp) == 0) {
2110
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
2111
		return false;
2112 2113
	}

2114
	return true;
2115 2116
}

2117 2118 2119
static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu,
				 struct list_head *invalid_list,
				 bool remote_flush, bool local_flush)
2120
{
2121 2122 2123 2124
	if (!list_empty(invalid_list)) {
		kvm_mmu_commit_zap_page(vcpu->kvm, invalid_list);
		return;
	}
2125

2126 2127 2128 2129
	if (remote_flush)
		kvm_flush_remote_tlbs(vcpu->kvm);
	else if (local_flush)
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2130 2131
}

2132 2133 2134 2135 2136 2137 2138
#ifdef CONFIG_KVM_MMU_AUDIT
#include "mmu_audit.c"
#else
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
static void mmu_audit_disable(void) { }
#endif

2139 2140 2141 2142 2143
static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
}

2144
static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
2145
			 struct list_head *invalid_list)
2146
{
2147 2148
	kvm_unlink_unsync_page(vcpu->kvm, sp);
	return __kvm_sync_page(vcpu, sp, invalid_list);
2149 2150
}

2151
/* @gfn should be write-protected at the call site */
2152 2153
static bool kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn,
			   struct list_head *invalid_list)
2154 2155
{
	struct kvm_mmu_page *s;
2156
	bool ret = false;
2157

2158
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
2159
		if (!s->unsync)
2160 2161 2162
			continue;

		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
2163
		ret |= kvm_sync_page(vcpu, s, invalid_list);
2164 2165
	}

2166
	return ret;
2167 2168
}

2169
struct mmu_page_path {
P
Paolo Bonzini 已提交
2170 2171
	struct kvm_mmu_page *parent[PT64_ROOT_LEVEL];
	unsigned int idx[PT64_ROOT_LEVEL];
2172 2173
};

2174
#define for_each_sp(pvec, sp, parents, i)			\
P
Paolo Bonzini 已提交
2175
		for (i = mmu_pages_first(&pvec, &parents);	\
2176 2177 2178
			i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});	\
			i = mmu_pages_next(&pvec, &parents, i))

2179 2180 2181
static int mmu_pages_next(struct kvm_mmu_pages *pvec,
			  struct mmu_page_path *parents,
			  int i)
2182 2183 2184 2185 2186
{
	int n;

	for (n = i+1; n < pvec->nr; n++) {
		struct kvm_mmu_page *sp = pvec->page[n].sp;
P
Paolo Bonzini 已提交
2187 2188
		unsigned idx = pvec->page[n].idx;
		int level = sp->role.level;
2189

P
Paolo Bonzini 已提交
2190 2191 2192
		parents->idx[level-1] = idx;
		if (level == PT_PAGE_TABLE_LEVEL)
			break;
2193

P
Paolo Bonzini 已提交
2194
		parents->parent[level-2] = sp;
2195 2196 2197 2198 2199
	}

	return n;
}

P
Paolo Bonzini 已提交
2200 2201 2202 2203 2204 2205 2206 2207 2208
static int mmu_pages_first(struct kvm_mmu_pages *pvec,
			   struct mmu_page_path *parents)
{
	struct kvm_mmu_page *sp;
	int level;

	if (pvec->nr == 0)
		return 0;

2209 2210
	WARN_ON(pvec->page[0].idx != INVALID_INDEX);

P
Paolo Bonzini 已提交
2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223
	sp = pvec->page[0].sp;
	level = sp->role.level;
	WARN_ON(level == PT_PAGE_TABLE_LEVEL);

	parents->parent[level-2] = sp;

	/* Also set up a sentinel.  Further entries in pvec are all
	 * children of sp, so this element is never overwritten.
	 */
	parents->parent[level-1] = NULL;
	return mmu_pages_next(pvec, parents, 0);
}

2224
static void mmu_pages_clear_parents(struct mmu_page_path *parents)
2225
{
2226 2227 2228 2229 2230 2231 2232 2233 2234
	struct kvm_mmu_page *sp;
	unsigned int level = 0;

	do {
		unsigned int idx = parents->idx[level];
		sp = parents->parent[level];
		if (!sp)
			return;

2235
		WARN_ON(idx == INVALID_INDEX);
2236
		clear_unsync_child_bit(sp, idx);
2237
		level++;
P
Paolo Bonzini 已提交
2238
	} while (!sp->unsync_children);
2239
}
2240

2241 2242 2243 2244 2245 2246 2247
static void mmu_sync_children(struct kvm_vcpu *vcpu,
			      struct kvm_mmu_page *parent)
{
	int i;
	struct kvm_mmu_page *sp;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
2248
	LIST_HEAD(invalid_list);
2249
	bool flush = false;
2250 2251

	while (mmu_unsync_walk(parent, &pages)) {
2252
		bool protected = false;
2253 2254

		for_each_sp(pages, sp, parents, i)
2255
			protected |= rmap_write_protect(vcpu, sp->gfn);
2256

2257
		if (protected) {
2258
			kvm_flush_remote_tlbs(vcpu->kvm);
2259 2260
			flush = false;
		}
2261

2262
		for_each_sp(pages, sp, parents, i) {
2263
			flush |= kvm_sync_page(vcpu, sp, &invalid_list);
2264 2265
			mmu_pages_clear_parents(&parents);
		}
2266 2267 2268 2269 2270
		if (need_resched() || spin_needbreak(&vcpu->kvm->mmu_lock)) {
			kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
			cond_resched_lock(&vcpu->kvm->mmu_lock);
			flush = false;
		}
2271
	}
2272 2273

	kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2274 2275
}

2276 2277
static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
{
2278
	atomic_set(&sp->write_flooding_count,  0);
2279 2280 2281 2282 2283 2284 2285 2286 2287
}

static void clear_sp_write_flooding_count(u64 *spte)
{
	struct kvm_mmu_page *sp =  page_header(__pa(spte));

	__clear_sp_write_flooding_count(sp);
}

2288 2289 2290 2291
static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
2292
					     int direct,
2293
					     unsigned access)
2294 2295 2296
{
	union kvm_mmu_page_role role;
	unsigned quadrant;
2297 2298
	struct kvm_mmu_page *sp;
	bool need_sync = false;
2299
	bool flush = false;
2300
	int collisions = 0;
2301
	LIST_HEAD(invalid_list);
2302

2303
	role = vcpu->arch.mmu.base_role;
2304
	role.level = level;
2305
	role.direct = direct;
2306
	if (role.direct)
2307
		role.cr4_pae = 0;
2308
	role.access = access;
2309 2310
	if (!vcpu->arch.mmu.direct_map
	    && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
2311 2312 2313 2314
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
2315 2316 2317 2318 2319 2320
	for_each_valid_sp(vcpu->kvm, sp, gfn) {
		if (sp->gfn != gfn) {
			collisions++;
			continue;
		}

2321 2322
		if (!need_sync && sp->unsync)
			need_sync = true;
2323

2324 2325
		if (sp->role.word != role.word)
			continue;
2326

2327 2328 2329 2330 2331 2332 2333 2334 2335 2336
		if (sp->unsync) {
			/* The page is good, but __kvm_sync_page might still end
			 * up zapping it.  If so, break in order to rebuild it.
			 */
			if (!__kvm_sync_page(vcpu, sp, &invalid_list))
				break;

			WARN_ON(!list_empty(&invalid_list));
			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
		}
2337

2338
		if (sp->unsync_children)
2339
			kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2340

2341
		__clear_sp_write_flooding_count(sp);
2342
		trace_kvm_mmu_get_page(sp, false);
2343
		goto out;
2344
	}
2345

A
Avi Kivity 已提交
2346
	++vcpu->kvm->stat.mmu_cache_miss;
2347 2348 2349

	sp = kvm_mmu_alloc_page(vcpu, direct);

2350 2351
	sp->gfn = gfn;
	sp->role = role;
2352 2353
	hlist_add_head(&sp->hash_link,
		&vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
2354
	if (!direct) {
2355 2356 2357 2358 2359 2360 2361 2362
		/*
		 * we should do write protection before syncing pages
		 * otherwise the content of the synced shadow page may
		 * be inconsistent with guest page table.
		 */
		account_shadowed(vcpu->kvm, sp);
		if (level == PT_PAGE_TABLE_LEVEL &&
		      rmap_write_protect(vcpu, gfn))
2363
			kvm_flush_remote_tlbs(vcpu->kvm);
2364 2365

		if (level > PT_PAGE_TABLE_LEVEL && need_sync)
2366
			flush |= kvm_sync_pages(vcpu, gfn, &invalid_list);
2367
	}
2368
	sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
2369
	clear_page(sp->spt);
A
Avi Kivity 已提交
2370
	trace_kvm_mmu_get_page(sp, true);
2371 2372

	kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2373 2374 2375
out:
	if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions)
		vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions;
2376
	return sp;
2377 2378
}

2379 2380 2381 2382 2383 2384
static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
			     struct kvm_vcpu *vcpu, u64 addr)
{
	iterator->addr = addr;
	iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
	iterator->level = vcpu->arch.mmu.shadow_root_level;
2385 2386 2387 2388 2389 2390

	if (iterator->level == PT64_ROOT_LEVEL &&
	    vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
	    !vcpu->arch.mmu.direct_map)
		--iterator->level;

2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404
	if (iterator->level == PT32E_ROOT_LEVEL) {
		iterator->shadow_addr
			= vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
		--iterator->level;
		if (!iterator->shadow_addr)
			iterator->level = 0;
	}
}

static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
{
	if (iterator->level < PT_PAGE_TABLE_LEVEL)
		return false;
2405

2406 2407 2408 2409 2410
	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
	return true;
}

2411 2412
static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
			       u64 spte)
2413
{
2414
	if (is_last_spte(spte, iterator->level)) {
2415 2416 2417 2418
		iterator->level = 0;
		return;
	}

2419
	iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2420 2421 2422
	--iterator->level;
}

2423 2424 2425 2426 2427
static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
{
	return __shadow_walk_next(iterator, *iterator->sptep);
}

2428 2429
static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
			     struct kvm_mmu_page *sp)
2430 2431 2432
{
	u64 spte;

2433
	BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2434

2435
	spte = __pa(sp->spt) | shadow_present_mask | PT_WRITABLE_MASK |
2436 2437 2438 2439 2440 2441
	       shadow_user_mask | shadow_x_mask;

	if (sp_ad_disabled(sp))
		spte |= shadow_acc_track_value;
	else
		spte |= shadow_accessed_mask;
X
Xiao Guangrong 已提交
2442

2443
	mmu_spte_set(sptep, spte);
2444 2445 2446 2447 2448

	mmu_page_add_parent_pte(vcpu, sp, sptep);

	if (sp->unsync_children || sp->unsync)
		mark_unsync(sptep);
2449 2450
}

2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467
static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
				   unsigned direct_access)
{
	if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
		struct kvm_mmu_page *child;

		/*
		 * For the direct sp, if the guest pte's dirty bit
		 * changed form clean to dirty, it will corrupt the
		 * sp's access: allow writable in the read-only sp,
		 * so we should update the spte at this point to get
		 * a new sp with the correct access.
		 */
		child = page_header(*sptep & PT64_BASE_ADDR_MASK);
		if (child->role.access == direct_access)
			return;

2468
		drop_parent_pte(child, sptep);
2469 2470 2471 2472
		kvm_flush_remote_tlbs(vcpu->kvm);
	}
}

X
Xiao Guangrong 已提交
2473
static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2474 2475 2476 2477 2478 2479 2480
			     u64 *spte)
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
	if (is_shadow_present_pte(pte)) {
X
Xiao Guangrong 已提交
2481
		if (is_last_spte(pte, sp->role.level)) {
2482
			drop_spte(kvm, spte);
X
Xiao Guangrong 已提交
2483 2484 2485
			if (is_large_pte(pte))
				--kvm->stat.lpages;
		} else {
2486
			child = page_header(pte & PT64_BASE_ADDR_MASK);
2487
			drop_parent_pte(child, spte);
2488
		}
X
Xiao Guangrong 已提交
2489 2490 2491 2492
		return true;
	}

	if (is_mmio_spte(pte))
2493
		mmu_spte_clear_no_track(spte);
2494

X
Xiao Guangrong 已提交
2495
	return false;
2496 2497
}

2498
static void kvm_mmu_page_unlink_children(struct kvm *kvm,
2499
					 struct kvm_mmu_page *sp)
2500
{
2501 2502
	unsigned i;

2503 2504
	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		mmu_page_zap_pte(kvm, sp, sp->spt + i);
2505 2506
}

2507
static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
2508
{
2509 2510
	u64 *sptep;
	struct rmap_iterator iter;
2511

2512
	while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2513
		drop_parent_pte(sp, sptep);
2514 2515
}

2516
static int mmu_zap_unsync_children(struct kvm *kvm,
2517 2518
				   struct kvm_mmu_page *parent,
				   struct list_head *invalid_list)
2519
{
2520 2521 2522
	int i, zapped = 0;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
2523

2524
	if (parent->role.level == PT_PAGE_TABLE_LEVEL)
2525
		return 0;
2526 2527 2528 2529 2530

	while (mmu_unsync_walk(parent, &pages)) {
		struct kvm_mmu_page *sp;

		for_each_sp(pages, sp, parents, i) {
2531
			kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2532
			mmu_pages_clear_parents(&parents);
2533
			zapped++;
2534 2535 2536 2537
		}
	}

	return zapped;
2538 2539
}

2540 2541
static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				    struct list_head *invalid_list)
2542
{
2543
	int ret;
A
Avi Kivity 已提交
2544

2545
	trace_kvm_mmu_prepare_zap_page(sp);
2546
	++kvm->stat.mmu_shadow_zapped;
2547
	ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
2548
	kvm_mmu_page_unlink_children(kvm, sp);
2549
	kvm_mmu_unlink_parents(kvm, sp);
2550

2551
	if (!sp->role.invalid && !sp->role.direct)
2552
		unaccount_shadowed(kvm, sp);
2553

2554 2555
	if (sp->unsync)
		kvm_unlink_unsync_page(kvm, sp);
2556
	if (!sp->root_count) {
2557 2558
		/* Count self */
		ret++;
2559
		list_move(&sp->link, invalid_list);
2560
		kvm_mod_used_mmu_pages(kvm, -1);
2561
	} else {
A
Avi Kivity 已提交
2562
		list_move(&sp->link, &kvm->arch.active_mmu_pages);
2563 2564 2565 2566 2567 2568 2569

		/*
		 * The obsolete pages can not be used on any vcpus.
		 * See the comments in kvm_mmu_invalidate_zap_all_pages().
		 */
		if (!sp->role.invalid && !is_obsolete_sp(kvm, sp))
			kvm_reload_remote_mmus(kvm);
2570
	}
2571 2572

	sp->role.invalid = 1;
2573
	return ret;
2574 2575
}

2576 2577 2578
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list)
{
2579
	struct kvm_mmu_page *sp, *nsp;
2580 2581 2582 2583

	if (list_empty(invalid_list))
		return;

2584
	/*
2585 2586 2587 2588 2589 2590 2591
	 * We need to make sure everyone sees our modifications to
	 * the page tables and see changes to vcpu->mode here. The barrier
	 * in the kvm_flush_remote_tlbs() achieves this. This pairs
	 * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end.
	 *
	 * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
	 * guest mode and/or lockless shadow page table walks.
2592 2593
	 */
	kvm_flush_remote_tlbs(kvm);
2594

2595
	list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2596
		WARN_ON(!sp->role.invalid || sp->root_count);
2597
		kvm_mmu_free_page(sp);
2598
	}
2599 2600
}

2601 2602 2603 2604 2605 2606 2607 2608
static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
					struct list_head *invalid_list)
{
	struct kvm_mmu_page *sp;

	if (list_empty(&kvm->arch.active_mmu_pages))
		return false;

G
Geliang Tang 已提交
2609 2610
	sp = list_last_entry(&kvm->arch.active_mmu_pages,
			     struct kvm_mmu_page, link);
2611 2612 2613 2614 2615
	kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);

	return true;
}

2616 2617
/*
 * Changing the number of mmu pages allocated to the vm
2618
 * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2619
 */
2620
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
2621
{
2622
	LIST_HEAD(invalid_list);
2623

2624 2625
	spin_lock(&kvm->mmu_lock);

2626
	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2627 2628 2629 2630
		/* Need to free some mmu pages to achieve the goal. */
		while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
			if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
				break;
2631

2632
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
2633
		goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2634 2635
	}

2636
	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2637 2638

	spin_unlock(&kvm->mmu_lock);
2639 2640
}

2641
int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2642
{
2643
	struct kvm_mmu_page *sp;
2644
	LIST_HEAD(invalid_list);
2645 2646
	int r;

2647
	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2648
	r = 0;
2649
	spin_lock(&kvm->mmu_lock);
2650
	for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2651
		pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2652 2653
			 sp->role.word);
		r = 1;
2654
		kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2655
	}
2656
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
2657 2658
	spin_unlock(&kvm->mmu_lock);

2659
	return r;
2660
}
2661
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
2662

2663
static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
2664 2665 2666 2667 2668 2669 2670 2671
{
	trace_kvm_mmu_unsync_page(sp);
	++vcpu->kvm->stat.mmu_unsync;
	sp->unsync = 1;

	kvm_mmu_mark_parents_unsync(sp);
}

2672 2673
static bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
				   bool can_unsync)
2674
{
2675
	struct kvm_mmu_page *sp;
2676

2677 2678
	if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE))
		return true;
2679

2680
	for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
2681
		if (!can_unsync)
2682
			return true;
2683

2684 2685
		if (sp->unsync)
			continue;
2686

2687 2688
		WARN_ON(sp->role.level != PT_PAGE_TABLE_LEVEL);
		kvm_unsync_page(vcpu, sp);
2689
	}
2690 2691

	return false;
2692 2693
}

D
Dan Williams 已提交
2694
static bool kvm_is_mmio_pfn(kvm_pfn_t pfn)
2695 2696 2697 2698 2699 2700 2701
{
	if (pfn_valid(pfn))
		return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn));

	return true;
}

A
Avi Kivity 已提交
2702
static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2703
		    unsigned pte_access, int level,
D
Dan Williams 已提交
2704
		    gfn_t gfn, kvm_pfn_t pfn, bool speculative,
2705
		    bool can_unsync, bool host_writable)
2706
{
2707
	u64 spte = 0;
M
Marcelo Tosatti 已提交
2708
	int ret = 0;
2709
	struct kvm_mmu_page *sp;
S
Sheng Yang 已提交
2710

2711
	if (set_mmio_spte(vcpu, sptep, gfn, pfn, pte_access))
2712 2713
		return 0;

2714 2715 2716 2717
	sp = page_header(__pa(sptep));
	if (sp_ad_disabled(sp))
		spte |= shadow_acc_track_value;

2718 2719 2720 2721 2722 2723
	/*
	 * For the EPT case, shadow_present_mask is 0 if hardware
	 * supports exec-only page table entries.  In that case,
	 * ACC_USER_MASK and shadow_user_mask are used to represent
	 * read access.  See FNAME(gpte_access) in paging_tmpl.h.
	 */
2724
	spte |= shadow_present_mask;
2725
	if (!speculative)
2726
		spte |= spte_shadow_accessed_mask(spte);
2727

S
Sheng Yang 已提交
2728 2729 2730 2731
	if (pte_access & ACC_EXEC_MASK)
		spte |= shadow_x_mask;
	else
		spte |= shadow_nx_mask;
2732

2733
	if (pte_access & ACC_USER_MASK)
S
Sheng Yang 已提交
2734
		spte |= shadow_user_mask;
2735

2736
	if (level > PT_PAGE_TABLE_LEVEL)
M
Marcelo Tosatti 已提交
2737
		spte |= PT_PAGE_SIZE_MASK;
2738
	if (tdp_enabled)
2739
		spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
2740
			kvm_is_mmio_pfn(pfn));
2741

2742
	if (host_writable)
2743
		spte |= SPTE_HOST_WRITEABLE;
2744 2745
	else
		pte_access &= ~ACC_WRITE_MASK;
2746

2747
	spte |= (u64)pfn << PAGE_SHIFT;
2748

2749
	if (pte_access & ACC_WRITE_MASK) {
2750

X
Xiao Guangrong 已提交
2751
		/*
2752 2753 2754 2755
		 * Other vcpu creates new sp in the window between
		 * mapping_level() and acquiring mmu-lock. We can
		 * allow guest to retry the access, the mapping can
		 * be fixed if guest refault.
X
Xiao Guangrong 已提交
2756
		 */
2757
		if (level > PT_PAGE_TABLE_LEVEL &&
2758
		    mmu_gfn_lpage_is_disallowed(vcpu, gfn, level))
A
Avi Kivity 已提交
2759
			goto done;
2760

2761
		spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
2762

2763 2764 2765 2766 2767 2768
		/*
		 * Optimization: for pte sync, if spte was writable the hash
		 * lookup is unnecessary (and expensive). Write protection
		 * is responsibility of mmu_get_page / kvm_sync_page.
		 * Same reasoning can be applied to dirty page accounting.
		 */
2769
		if (!can_unsync && is_writable_pte(*sptep))
2770 2771
			goto set_pte;

2772
		if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2773
			pgprintk("%s: found shadow page for %llx, marking ro\n",
2774
				 __func__, gfn);
M
Marcelo Tosatti 已提交
2775
			ret = 1;
2776
			pte_access &= ~ACC_WRITE_MASK;
2777
			spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
2778 2779 2780
		}
	}

2781
	if (pte_access & ACC_WRITE_MASK) {
2782
		kvm_vcpu_mark_page_dirty(vcpu, gfn);
2783
		spte |= spte_shadow_dirty_mask(spte);
2784
	}
2785

2786 2787 2788
	if (speculative)
		spte = mark_spte_for_access_track(spte);

2789
set_pte:
2790
	if (mmu_spte_update(sptep, spte))
2791
		kvm_flush_remote_tlbs(vcpu->kvm);
A
Avi Kivity 已提交
2792
done:
M
Marcelo Tosatti 已提交
2793 2794 2795
	return ret;
}

2796
static bool mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access,
D
Dan Williams 已提交
2797
			 int write_fault, int level, gfn_t gfn, kvm_pfn_t pfn,
2798
			 bool speculative, bool host_writable)
M
Marcelo Tosatti 已提交
2799 2800
{
	int was_rmapped = 0;
2801
	int rmap_count;
2802
	bool emulate = false;
M
Marcelo Tosatti 已提交
2803

2804 2805
	pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
		 *sptep, write_fault, gfn);
M
Marcelo Tosatti 已提交
2806

2807
	if (is_shadow_present_pte(*sptep)) {
M
Marcelo Tosatti 已提交
2808 2809 2810 2811
		/*
		 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
		 * the parent of the now unreachable PTE.
		 */
2812 2813
		if (level > PT_PAGE_TABLE_LEVEL &&
		    !is_large_pte(*sptep)) {
M
Marcelo Tosatti 已提交
2814
			struct kvm_mmu_page *child;
A
Avi Kivity 已提交
2815
			u64 pte = *sptep;
M
Marcelo Tosatti 已提交
2816 2817

			child = page_header(pte & PT64_BASE_ADDR_MASK);
2818
			drop_parent_pte(child, sptep);
2819
			kvm_flush_remote_tlbs(vcpu->kvm);
A
Avi Kivity 已提交
2820
		} else if (pfn != spte_to_pfn(*sptep)) {
2821
			pgprintk("hfn old %llx new %llx\n",
A
Avi Kivity 已提交
2822
				 spte_to_pfn(*sptep), pfn);
2823
			drop_spte(vcpu->kvm, sptep);
2824
			kvm_flush_remote_tlbs(vcpu->kvm);
2825 2826
		} else
			was_rmapped = 1;
M
Marcelo Tosatti 已提交
2827
	}
2828

2829 2830
	if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
	      true, host_writable)) {
M
Marcelo Tosatti 已提交
2831
		if (write_fault)
2832
			emulate = true;
2833
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2834
	}
M
Marcelo Tosatti 已提交
2835

2836 2837
	if (unlikely(is_mmio_spte(*sptep)))
		emulate = true;
2838

A
Avi Kivity 已提交
2839
	pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2840
	pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
A
Avi Kivity 已提交
2841
		 is_large_pte(*sptep)? "2MB" : "4kB",
2842
		 *sptep & PT_WRITABLE_MASK ? "RW" : "R", gfn,
2843
		 *sptep, sptep);
A
Avi Kivity 已提交
2844
	if (!was_rmapped && is_large_pte(*sptep))
M
Marcelo Tosatti 已提交
2845 2846
		++vcpu->kvm->stat.lpages;

2847 2848 2849 2850 2851 2852
	if (is_shadow_present_pte(*sptep)) {
		if (!was_rmapped) {
			rmap_count = rmap_add(vcpu, sptep, gfn);
			if (rmap_count > RMAP_RECYCLE_THRESHOLD)
				rmap_recycle(vcpu, sptep, gfn);
		}
2853
	}
2854

X
Xiao Guangrong 已提交
2855
	kvm_release_pfn_clean(pfn);
2856 2857

	return emulate;
2858 2859
}

D
Dan Williams 已提交
2860
static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
2861 2862 2863 2864
				     bool no_dirty_log)
{
	struct kvm_memory_slot *slot;

2865
	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
2866
	if (!slot)
2867
		return KVM_PFN_ERR_FAULT;
2868

2869
	return gfn_to_pfn_memslot_atomic(slot, gfn);
2870 2871 2872 2873 2874 2875 2876
}

static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *sp,
				    u64 *start, u64 *end)
{
	struct page *pages[PTE_PREFETCH_NUM];
2877
	struct kvm_memory_slot *slot;
2878 2879 2880 2881 2882
	unsigned access = sp->role.access;
	int i, ret;
	gfn_t gfn;

	gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2883 2884
	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
	if (!slot)
2885 2886
		return -1;

2887
	ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
2888 2889 2890 2891
	if (ret <= 0)
		return -1;

	for (i = 0; i < ret; i++, gfn++, start++)
2892 2893
		mmu_set_spte(vcpu, start, access, 0, sp->role.level, gfn,
			     page_to_pfn(pages[i]), true, true);
2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909

	return 0;
}

static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
				  struct kvm_mmu_page *sp, u64 *sptep)
{
	u64 *spte, *start = NULL;
	int i;

	WARN_ON(!sp->role.direct);

	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
	spte = sp->spt + i;

	for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2910
		if (is_shadow_present_pte(*spte) || spte == sptep) {
2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924
			if (!start)
				continue;
			if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
				break;
			start = NULL;
		} else if (!start)
			start = spte;
	}
}

static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
{
	struct kvm_mmu_page *sp;

2925 2926
	sp = page_header(__pa(sptep));

2927
	/*
2928 2929 2930
	 * Without accessed bits, there's no way to distinguish between
	 * actually accessed translations and prefetched, so disable pte
	 * prefetch if accessed bits aren't available.
2931
	 */
2932
	if (sp_ad_disabled(sp))
2933 2934 2935 2936 2937 2938 2939 2940
		return;

	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return;

	__direct_pte_prefetch(vcpu, sp, sptep);
}

2941
static int __direct_map(struct kvm_vcpu *vcpu, int write, int map_writable,
D
Dan Williams 已提交
2942
			int level, gfn_t gfn, kvm_pfn_t pfn, bool prefault)
2943
{
2944
	struct kvm_shadow_walk_iterator iterator;
2945
	struct kvm_mmu_page *sp;
2946
	int emulate = 0;
2947
	gfn_t pseudo_gfn;
A
Avi Kivity 已提交
2948

2949 2950 2951
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return 0;

2952
	for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2953
		if (iterator.level == level) {
2954 2955 2956
			emulate = mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
					       write, level, gfn, pfn, prefault,
					       map_writable);
2957
			direct_pte_prefetch(vcpu, iterator.sptep);
2958 2959
			++vcpu->stat.pf_fixed;
			break;
A
Avi Kivity 已提交
2960 2961
		}

2962
		drop_large_spte(vcpu, iterator.sptep);
2963
		if (!is_shadow_present_pte(*iterator.sptep)) {
2964 2965 2966 2967
			u64 base_addr = iterator.addr;

			base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
			pseudo_gfn = base_addr >> PAGE_SHIFT;
2968
			sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2969
					      iterator.level - 1, 1, ACC_ALL);
2970

2971
			link_shadow_page(vcpu, iterator.sptep, sp);
2972 2973
		}
	}
2974
	return emulate;
A
Avi Kivity 已提交
2975 2976
}

H
Huang Ying 已提交
2977
static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2978
{
H
Huang Ying 已提交
2979 2980 2981 2982 2983 2984 2985
	siginfo_t info;

	info.si_signo	= SIGBUS;
	info.si_errno	= 0;
	info.si_code	= BUS_MCEERR_AR;
	info.si_addr	= (void __user *)address;
	info.si_addr_lsb = PAGE_SHIFT;
2986

H
Huang Ying 已提交
2987
	send_sig_info(SIGBUS, &info, tsk);
2988 2989
}

D
Dan Williams 已提交
2990
static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
2991
{
X
Xiao Guangrong 已提交
2992 2993 2994 2995 2996 2997 2998 2999 3000
	/*
	 * Do not cache the mmio info caused by writing the readonly gfn
	 * into the spte otherwise read access on readonly gfn also can
	 * caused mmio page fault and treat it as mmio access.
	 * Return 1 to tell kvm to emulate it.
	 */
	if (pfn == KVM_PFN_ERR_RO_FAULT)
		return 1;

3001
	if (pfn == KVM_PFN_ERR_HWPOISON) {
3002
		kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
3003
		return 0;
3004
	}
3005

3006
	return -EFAULT;
3007 3008
}

3009
static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
D
Dan Williams 已提交
3010 3011
					gfn_t *gfnp, kvm_pfn_t *pfnp,
					int *levelp)
3012
{
D
Dan Williams 已提交
3013
	kvm_pfn_t pfn = *pfnp;
3014 3015 3016 3017 3018 3019 3020 3021 3022
	gfn_t gfn = *gfnp;
	int level = *levelp;

	/*
	 * Check if it's a transparent hugepage. If this would be an
	 * hugetlbfs page, level wouldn't be set to
	 * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
	 * here.
	 */
3023
	if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn) &&
3024
	    level == PT_PAGE_TABLE_LEVEL &&
3025
	    PageTransCompoundMap(pfn_to_page(pfn)) &&
3026
	    !mmu_gfn_lpage_is_disallowed(vcpu, gfn, PT_DIRECTORY_LEVEL)) {
3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044
		unsigned long mask;
		/*
		 * mmu_notifier_retry was successful and we hold the
		 * mmu_lock here, so the pmd can't become splitting
		 * from under us, and in turn
		 * __split_huge_page_refcount() can't run from under
		 * us and we can safely transfer the refcount from
		 * PG_tail to PG_head as we switch the pfn to tail to
		 * head.
		 */
		*levelp = level = PT_DIRECTORY_LEVEL;
		mask = KVM_PAGES_PER_HPAGE(level) - 1;
		VM_BUG_ON((gfn & mask) != (pfn & mask));
		if (pfn & mask) {
			gfn &= ~mask;
			*gfnp = gfn;
			kvm_release_pfn_clean(pfn);
			pfn &= ~mask;
3045
			kvm_get_pfn(pfn);
3046 3047 3048 3049 3050
			*pfnp = pfn;
		}
	}
}

3051
static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
D
Dan Williams 已提交
3052
				kvm_pfn_t pfn, unsigned access, int *ret_val)
3053 3054
{
	/* The pfn is invalid, report the error! */
3055
	if (unlikely(is_error_pfn(pfn))) {
3056
		*ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
3057
		return true;
3058 3059
	}

3060
	if (unlikely(is_noslot_pfn(pfn)))
3061 3062
		vcpu_cache_mmio_info(vcpu, gva, gfn, access);

3063
	return false;
3064 3065
}

3066
static bool page_fault_can_be_fast(u32 error_code)
3067
{
3068 3069 3070 3071 3072 3073 3074
	/*
	 * Do not fix the mmio spte with invalid generation number which
	 * need to be updated by slow page fault path.
	 */
	if (unlikely(error_code & PFERR_RSVD_MASK))
		return false;

3075 3076 3077 3078 3079
	/* See if the page fault is due to an NX violation */
	if (unlikely(((error_code & (PFERR_FETCH_MASK | PFERR_PRESENT_MASK))
		      == (PFERR_FETCH_MASK | PFERR_PRESENT_MASK))))
		return false;

3080
	/*
3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091
	 * #PF can be fast if:
	 * 1. The shadow page table entry is not present, which could mean that
	 *    the fault is potentially caused by access tracking (if enabled).
	 * 2. The shadow page table entry is present and the fault
	 *    is caused by write-protect, that means we just need change the W
	 *    bit of the spte which can be done out of mmu-lock.
	 *
	 * However, if access tracking is disabled we know that a non-present
	 * page must be a genuine page fault where we have to create a new SPTE.
	 * So, if access tracking is disabled, we return true only for write
	 * accesses to a present page.
3092 3093
	 */

3094 3095 3096
	return shadow_acc_track_mask != 0 ||
	       ((error_code & (PFERR_WRITE_MASK | PFERR_PRESENT_MASK))
		== (PFERR_WRITE_MASK | PFERR_PRESENT_MASK));
3097 3098
}

3099 3100 3101 3102
/*
 * Returns true if the SPTE was fixed successfully. Otherwise,
 * someone else modified the SPTE from its original value.
 */
3103
static bool
3104
fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
3105
			u64 *sptep, u64 old_spte, u64 new_spte)
3106 3107 3108 3109 3110
{
	gfn_t gfn;

	WARN_ON(!sp->role.direct);

3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122
	/*
	 * Theoretically we could also set dirty bit (and flush TLB) here in
	 * order to eliminate unnecessary PML logging. See comments in
	 * set_spte. But fast_page_fault is very unlikely to happen with PML
	 * enabled, so we do not do this. This might result in the same GPA
	 * to be logged in PML buffer again when the write really happens, and
	 * eventually to be called by mark_page_dirty twice. But it's also no
	 * harm. This also avoids the TLB flush needed after setting dirty bit
	 * so non-PML cases won't be impacted.
	 *
	 * Compare with set_spte where instead shadow_dirty_mask is set.
	 */
3123
	if (cmpxchg64(sptep, old_spte, new_spte) != old_spte)
3124 3125
		return false;

3126
	if (is_writable_pte(new_spte) && !is_writable_pte(old_spte)) {
3127 3128 3129 3130 3131 3132 3133
		/*
		 * The gfn of direct spte is stable since it is
		 * calculated by sp->gfn.
		 */
		gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
		kvm_vcpu_mark_page_dirty(vcpu, gfn);
	}
3134 3135 3136 3137

	return true;
}

3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149
static bool is_access_allowed(u32 fault_err_code, u64 spte)
{
	if (fault_err_code & PFERR_FETCH_MASK)
		return is_executable_pte(spte);

	if (fault_err_code & PFERR_WRITE_MASK)
		return is_writable_pte(spte);

	/* Fault was on Read access */
	return spte & PT_PRESENT_MASK;
}

3150 3151 3152 3153 3154 3155 3156 3157 3158
/*
 * Return value:
 * - true: let the vcpu to access on the same address again.
 * - false: let the real page fault path to fix it.
 */
static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
			    u32 error_code)
{
	struct kvm_shadow_walk_iterator iterator;
3159
	struct kvm_mmu_page *sp;
3160
	bool fault_handled = false;
3161
	u64 spte = 0ull;
3162
	uint retry_count = 0;
3163

3164 3165 3166
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return false;

3167
	if (!page_fault_can_be_fast(error_code))
3168 3169 3170 3171
		return false;

	walk_shadow_page_lockless_begin(vcpu);

3172
	do {
3173
		u64 new_spte;
3174

3175 3176 3177 3178 3179
		for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
			if (!is_shadow_present_pte(spte) ||
			    iterator.level < level)
				break;

3180 3181 3182
		sp = page_header(__pa(iterator.sptep));
		if (!is_last_spte(spte, sp->role.level))
			break;
3183

3184
		/*
3185 3186 3187 3188 3189
		 * Check whether the memory access that caused the fault would
		 * still cause it if it were to be performed right now. If not,
		 * then this is a spurious fault caused by TLB lazily flushed,
		 * or some other CPU has already fixed the PTE after the
		 * current CPU took the fault.
3190 3191 3192 3193
		 *
		 * Need not check the access of upper level table entries since
		 * they are always ACC_ALL.
		 */
3194 3195 3196 3197
		if (is_access_allowed(error_code, spte)) {
			fault_handled = true;
			break;
		}
3198

3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212
		new_spte = spte;

		if (is_access_track_spte(spte))
			new_spte = restore_acc_track_spte(new_spte);

		/*
		 * Currently, to simplify the code, write-protection can
		 * be removed in the fast path only if the SPTE was
		 * write-protected for dirty-logging or access tracking.
		 */
		if ((error_code & PFERR_WRITE_MASK) &&
		    spte_can_locklessly_be_made_writable(spte))
		{
			new_spte |= PT_WRITABLE_MASK;
3213 3214

			/*
3215 3216 3217 3218 3219 3220 3221 3222 3223
			 * Do not fix write-permission on the large spte.  Since
			 * we only dirty the first page into the dirty-bitmap in
			 * fast_pf_fix_direct_spte(), other pages are missed
			 * if its slot has dirty logging enabled.
			 *
			 * Instead, we let the slow page fault path create a
			 * normal spte to fix the access.
			 *
			 * See the comments in kvm_arch_commit_memory_region().
3224
			 */
3225
			if (sp->role.level > PT_PAGE_TABLE_LEVEL)
3226
				break;
3227
		}
3228

3229
		/* Verify that the fault can be handled in the fast path */
3230 3231
		if (new_spte == spte ||
		    !is_access_allowed(error_code, new_spte))
3232 3233 3234 3235 3236 3237 3238 3239
			break;

		/*
		 * Currently, fast page fault only works for direct mapping
		 * since the gfn is not stable for indirect shadow page. See
		 * Documentation/virtual/kvm/locking.txt to get more detail.
		 */
		fault_handled = fast_pf_fix_direct_spte(vcpu, sp,
3240
							iterator.sptep, spte,
3241
							new_spte);
3242 3243 3244 3245 3246 3247 3248 3249 3250 3251
		if (fault_handled)
			break;

		if (++retry_count > 4) {
			printk_once(KERN_WARNING
				"kvm: Fast #PF retrying more than 4 times.\n");
			break;
		}

	} while (true);
3252

X
Xiao Guangrong 已提交
3253
	trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
3254
			      spte, fault_handled);
3255 3256
	walk_shadow_page_lockless_end(vcpu);

3257
	return fault_handled;
3258 3259
}

3260
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
D
Dan Williams 已提交
3261
			 gva_t gva, kvm_pfn_t *pfn, bool write, bool *writable);
3262
static void make_mmu_pages_available(struct kvm_vcpu *vcpu);
3263

3264 3265
static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
			 gfn_t gfn, bool prefault)
3266 3267
{
	int r;
3268
	int level;
3269
	bool force_pt_level = false;
D
Dan Williams 已提交
3270
	kvm_pfn_t pfn;
3271
	unsigned long mmu_seq;
3272
	bool map_writable, write = error_code & PFERR_WRITE_MASK;
3273

3274
	level = mapping_level(vcpu, gfn, &force_pt_level);
3275 3276 3277 3278 3279 3280 3281 3282
	if (likely(!force_pt_level)) {
		/*
		 * This path builds a PAE pagetable - so we can map
		 * 2mb pages at maximum. Therefore check if the level
		 * is larger than that.
		 */
		if (level > PT_DIRECTORY_LEVEL)
			level = PT_DIRECTORY_LEVEL;
3283

3284
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3285
	}
M
Marcelo Tosatti 已提交
3286

3287 3288 3289
	if (fast_page_fault(vcpu, v, level, error_code))
		return 0;

3290
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
3291
	smp_rmb();
3292

3293
	if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
3294
		return 0;
3295

3296 3297
	if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
		return r;
3298

3299
	spin_lock(&vcpu->kvm->mmu_lock);
3300
	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3301
		goto out_unlock;
3302
	make_mmu_pages_available(vcpu);
3303 3304
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3305
	r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3306 3307
	spin_unlock(&vcpu->kvm->mmu_lock);

3308
	return r;
3309 3310 3311 3312 3313

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
3314 3315 3316
}


3317 3318 3319
static void mmu_free_roots(struct kvm_vcpu *vcpu)
{
	int i;
3320
	struct kvm_mmu_page *sp;
3321
	LIST_HEAD(invalid_list);
3322

3323
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
A
Avi Kivity 已提交
3324
		return;
3325

3326 3327 3328
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
	    (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
	     vcpu->arch.mmu.direct_map)) {
3329
		hpa_t root = vcpu->arch.mmu.root_hpa;
3330

3331
		spin_lock(&vcpu->kvm->mmu_lock);
3332 3333
		sp = page_header(root);
		--sp->root_count;
3334 3335 3336 3337
		if (!sp->root_count && sp->role.invalid) {
			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
			kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
		}
3338
		spin_unlock(&vcpu->kvm->mmu_lock);
3339
		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3340 3341
		return;
	}
3342 3343

	spin_lock(&vcpu->kvm->mmu_lock);
3344
	for (i = 0; i < 4; ++i) {
3345
		hpa_t root = vcpu->arch.mmu.pae_root[i];
3346

A
Avi Kivity 已提交
3347 3348
		if (root) {
			root &= PT64_BASE_ADDR_MASK;
3349 3350
			sp = page_header(root);
			--sp->root_count;
3351
			if (!sp->root_count && sp->role.invalid)
3352 3353
				kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
							 &invalid_list);
A
Avi Kivity 已提交
3354
		}
3355
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
3356
	}
3357
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3358
	spin_unlock(&vcpu->kvm->mmu_lock);
3359
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3360 3361
}

3362 3363 3364 3365 3366
static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
{
	int ret = 0;

	if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
3367
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3368 3369 3370 3371 3372 3373
		ret = 1;
	}

	return ret;
}

3374 3375 3376
static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_page *sp;
3377
	unsigned i;
3378 3379 3380

	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		spin_lock(&vcpu->kvm->mmu_lock);
3381
		make_mmu_pages_available(vcpu);
3382
		sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL, 1, ACC_ALL);
3383 3384 3385 3386 3387 3388 3389
		++sp->root_count;
		spin_unlock(&vcpu->kvm->mmu_lock);
		vcpu->arch.mmu.root_hpa = __pa(sp->spt);
	} else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
		for (i = 0; i < 4; ++i) {
			hpa_t root = vcpu->arch.mmu.pae_root[i];

3390
			MMU_WARN_ON(VALID_PAGE(root));
3391
			spin_lock(&vcpu->kvm->mmu_lock);
3392
			make_mmu_pages_available(vcpu);
3393
			sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
3394
					i << 30, PT32_ROOT_LEVEL, 1, ACC_ALL);
3395 3396 3397 3398 3399
			root = __pa(sp->spt);
			++sp->root_count;
			spin_unlock(&vcpu->kvm->mmu_lock);
			vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
		}
3400
		vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3401 3402 3403 3404 3405 3406 3407
	} else
		BUG();

	return 0;
}

static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3408
{
3409
	struct kvm_mmu_page *sp;
3410 3411 3412
	u64 pdptr, pm_mask;
	gfn_t root_gfn;
	int i;
3413

3414
	root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
3415

3416 3417 3418 3419 3420 3421 3422 3423
	if (mmu_check_root(vcpu, root_gfn))
		return 1;

	/*
	 * Do we shadow a long mode page table? If so we need to
	 * write-protect the guests page table root.
	 */
	if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3424
		hpa_t root = vcpu->arch.mmu.root_hpa;
3425

3426
		MMU_WARN_ON(VALID_PAGE(root));
3427

3428
		spin_lock(&vcpu->kvm->mmu_lock);
3429
		make_mmu_pages_available(vcpu);
3430
		sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
3431
				      0, ACC_ALL);
3432 3433
		root = __pa(sp->spt);
		++sp->root_count;
3434
		spin_unlock(&vcpu->kvm->mmu_lock);
3435
		vcpu->arch.mmu.root_hpa = root;
3436
		return 0;
3437
	}
3438

3439 3440
	/*
	 * We shadow a 32 bit page table. This may be a legacy 2-level
3441 3442
	 * or a PAE 3-level page table. In either case we need to be aware that
	 * the shadow page table may be a PAE or a long mode page table.
3443
	 */
3444 3445 3446 3447
	pm_mask = PT_PRESENT_MASK;
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
		pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;

3448
	for (i = 0; i < 4; ++i) {
3449
		hpa_t root = vcpu->arch.mmu.pae_root[i];
3450

3451
		MMU_WARN_ON(VALID_PAGE(root));
3452
		if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
3453
			pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
B
Bandan Das 已提交
3454
			if (!(pdptr & PT_PRESENT_MASK)) {
3455
				vcpu->arch.mmu.pae_root[i] = 0;
A
Avi Kivity 已提交
3456 3457
				continue;
			}
A
Avi Kivity 已提交
3458
			root_gfn = pdptr >> PAGE_SHIFT;
3459 3460
			if (mmu_check_root(vcpu, root_gfn))
				return 1;
3461
		}
3462
		spin_lock(&vcpu->kvm->mmu_lock);
3463
		make_mmu_pages_available(vcpu);
3464 3465
		sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL,
				      0, ACC_ALL);
3466 3467
		root = __pa(sp->spt);
		++sp->root_count;
3468 3469
		spin_unlock(&vcpu->kvm->mmu_lock);

3470
		vcpu->arch.mmu.pae_root[i] = root | pm_mask;
3471
	}
3472
	vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498

	/*
	 * If we shadow a 32 bit page table with a long mode page
	 * table we enter this path.
	 */
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		if (vcpu->arch.mmu.lm_root == NULL) {
			/*
			 * The additional page necessary for this is only
			 * allocated on demand.
			 */

			u64 *lm_root;

			lm_root = (void*)get_zeroed_page(GFP_KERNEL);
			if (lm_root == NULL)
				return 1;

			lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;

			vcpu->arch.mmu.lm_root = lm_root;
		}

		vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
	}

3499
	return 0;
3500 3501
}

3502 3503 3504 3505 3506 3507 3508 3509
static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
{
	if (vcpu->arch.mmu.direct_map)
		return mmu_alloc_direct_roots(vcpu);
	else
		return mmu_alloc_shadow_roots(vcpu);
}

3510 3511 3512 3513 3514
static void mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	int i;
	struct kvm_mmu_page *sp;

3515 3516 3517
	if (vcpu->arch.mmu.direct_map)
		return;

3518 3519
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return;
3520

3521
	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3522
	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3523
	if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3524 3525 3526
		hpa_t root = vcpu->arch.mmu.root_hpa;
		sp = page_header(root);
		mmu_sync_children(vcpu, sp);
3527
		kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3528 3529 3530 3531 3532
		return;
	}
	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->arch.mmu.pae_root[i];

3533
		if (root && VALID_PAGE(root)) {
3534 3535 3536 3537 3538
			root &= PT64_BASE_ADDR_MASK;
			sp = page_header(root);
			mmu_sync_children(vcpu, sp);
		}
	}
3539
	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3540 3541 3542 3543 3544 3545
}

void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	spin_lock(&vcpu->kvm->mmu_lock);
	mmu_sync_roots(vcpu);
3546
	spin_unlock(&vcpu->kvm->mmu_lock);
3547
}
N
Nadav Har'El 已提交
3548
EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
3549

3550
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
3551
				  u32 access, struct x86_exception *exception)
A
Avi Kivity 已提交
3552
{
3553 3554
	if (exception)
		exception->error_code = 0;
A
Avi Kivity 已提交
3555 3556 3557
	return vaddr;
}

3558
static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
3559 3560
					 u32 access,
					 struct x86_exception *exception)
3561
{
3562 3563
	if (exception)
		exception->error_code = 0;
3564
	return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception);
3565 3566
}

3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585
static bool
__is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, u64 pte, int level)
{
	int bit7 = (pte >> 7) & 1, low6 = pte & 0x3f;

	return (pte & rsvd_check->rsvd_bits_mask[bit7][level-1]) |
		((rsvd_check->bad_mt_xwr & (1ull << low6)) != 0);
}

static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
{
	return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level);
}

static bool is_shadow_zero_bits_set(struct kvm_mmu *mmu, u64 spte, int level)
{
	return __is_rsvd_bits_set(&mmu->shadow_zero_check, spte, level);
}

3586
static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3587 3588 3589 3590 3591 3592 3593
{
	if (direct)
		return vcpu_match_mmio_gpa(vcpu, addr);

	return vcpu_match_mmio_gva(vcpu, addr);
}

3594 3595 3596
/* return true if reserved bit is detected on spte. */
static bool
walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
3597 3598
{
	struct kvm_shadow_walk_iterator iterator;
3599 3600 3601
	u64 sptes[PT64_ROOT_LEVEL], spte = 0ull;
	int root, leaf;
	bool reserved = false;
3602

3603
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3604
		goto exit;
3605

3606
	walk_shadow_page_lockless_begin(vcpu);
3607

3608 3609
	for (shadow_walk_init(&iterator, vcpu, addr),
		 leaf = root = iterator.level;
3610 3611 3612 3613 3614
	     shadow_walk_okay(&iterator);
	     __shadow_walk_next(&iterator, spte)) {
		spte = mmu_spte_get_lockless(iterator.sptep);

		sptes[leaf - 1] = spte;
3615
		leaf--;
3616

3617 3618
		if (!is_shadow_present_pte(spte))
			break;
3619 3620

		reserved |= is_shadow_zero_bits_set(&vcpu->arch.mmu, spte,
3621
						    iterator.level);
3622 3623
	}

3624 3625
	walk_shadow_page_lockless_end(vcpu);

3626 3627 3628
	if (reserved) {
		pr_err("%s: detect reserved bits on spte, addr 0x%llx, dump hierarchy:\n",
		       __func__, addr);
3629
		while (root > leaf) {
3630 3631 3632 3633 3634 3635 3636 3637
			pr_err("------ spte 0x%llx level %d.\n",
			       sptes[root - 1], root);
			root--;
		}
	}
exit:
	*sptep = spte;
	return reserved;
3638 3639
}

3640
int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3641 3642
{
	u64 spte;
3643
	bool reserved;
3644

3645
	if (mmio_info_in_cache(vcpu, addr, direct))
3646
		return RET_MMIO_PF_EMULATE;
3647

3648
	reserved = walk_shadow_page_get_mmio_spte(vcpu, addr, &spte);
3649
	if (WARN_ON(reserved))
3650
		return RET_MMIO_PF_BUG;
3651 3652 3653 3654 3655

	if (is_mmio_spte(spte)) {
		gfn_t gfn = get_mmio_spte_gfn(spte);
		unsigned access = get_mmio_spte_access(spte);

3656
		if (!check_mmio_spte(vcpu, spte))
3657 3658
			return RET_MMIO_PF_INVALID;

3659 3660
		if (direct)
			addr = 0;
X
Xiao Guangrong 已提交
3661 3662

		trace_handle_mmio_page_fault(addr, gfn, access);
3663
		vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3664
		return RET_MMIO_PF_EMULATE;
3665 3666 3667 3668 3669 3670
	}

	/*
	 * If the page table is zapped by other cpus, let CPU fault again on
	 * the address.
	 */
3671
	return RET_MMIO_PF_RETRY;
3672
}
3673
EXPORT_SYMBOL_GPL(handle_mmio_page_fault);
3674

3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694
static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
					 u32 error_code, gfn_t gfn)
{
	if (unlikely(error_code & PFERR_RSVD_MASK))
		return false;

	if (!(error_code & PFERR_PRESENT_MASK) ||
	      !(error_code & PFERR_WRITE_MASK))
		return false;

	/*
	 * guest is writing the page which is write tracked which can
	 * not be fixed by page fault handler.
	 */
	if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE))
		return true;

	return false;
}

3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711
static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
{
	struct kvm_shadow_walk_iterator iterator;
	u64 spte;

	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return;

	walk_shadow_page_lockless_begin(vcpu);
	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {
		clear_sp_write_flooding_count(iterator.sptep);
		if (!is_shadow_present_pte(spte))
			break;
	}
	walk_shadow_page_lockless_end(vcpu);
}

A
Avi Kivity 已提交
3712
static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
3713
				u32 error_code, bool prefault)
A
Avi Kivity 已提交
3714
{
3715
	gfn_t gfn = gva >> PAGE_SHIFT;
3716
	int r;
A
Avi Kivity 已提交
3717

3718
	pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
3719

3720 3721
	if (page_fault_handle_page_track(vcpu, error_code, gfn))
		return 1;
3722

3723 3724 3725
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;
3726

3727
	MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
3728 3729


3730
	return nonpaging_map(vcpu, gva & PAGE_MASK,
3731
			     error_code, gfn, prefault);
A
Avi Kivity 已提交
3732 3733
}

3734
static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
3735 3736
{
	struct kvm_arch_async_pf arch;
X
Xiao Guangrong 已提交
3737

3738
	arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3739
	arch.gfn = gfn;
3740
	arch.direct_map = vcpu->arch.mmu.direct_map;
X
Xiao Guangrong 已提交
3741
	arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
3742

3743
	return kvm_setup_async_pf(vcpu, gva, kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
3744 3745
}

3746
bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
3747
{
3748
	if (unlikely(!lapic_in_kernel(vcpu) ||
3749 3750 3751
		     kvm_event_needs_reinjection(vcpu)))
		return false;

3752
	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
3753 3754
		return false;

3755 3756 3757
	return kvm_x86_ops->interrupt_allowed(vcpu);
}

3758
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
D
Dan Williams 已提交
3759
			 gva_t gva, kvm_pfn_t *pfn, bool write, bool *writable)
3760
{
3761
	struct kvm_memory_slot *slot;
3762 3763
	bool async;

3764
	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3765 3766
	async = false;
	*pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable);
3767 3768 3769
	if (!async)
		return false; /* *pfn has correct page already */

3770
	if (!prefault && kvm_can_do_async_pf(vcpu)) {
3771
		trace_kvm_try_async_get_page(gva, gfn);
3772 3773 3774 3775 3776 3777 3778 3779
		if (kvm_find_async_pf_gfn(vcpu, gfn)) {
			trace_kvm_async_pf_doublefault(gva, gfn);
			kvm_make_request(KVM_REQ_APF_HALT, vcpu);
			return true;
		} else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
			return true;
	}

3780
	*pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, write, writable);
3781 3782 3783
	return false;
}

3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815
int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
				u64 fault_address, char *insn, int insn_len,
				bool need_unprotect)
{
	int r = 1;

	switch (vcpu->arch.apf.host_apf_reason) {
	default:
		trace_kvm_page_fault(fault_address, error_code);

		if (need_unprotect && kvm_event_needs_reinjection(vcpu))
			kvm_mmu_unprotect_page_virt(vcpu, fault_address);
		r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
				insn_len);
		break;
	case KVM_PV_REASON_PAGE_NOT_PRESENT:
		vcpu->arch.apf.host_apf_reason = 0;
		local_irq_disable();
		kvm_async_pf_task_wait(fault_address);
		local_irq_enable();
		break;
	case KVM_PV_REASON_PAGE_READY:
		vcpu->arch.apf.host_apf_reason = 0;
		local_irq_disable();
		kvm_async_pf_task_wake(fault_address);
		local_irq_enable();
		break;
	}
	return r;
}
EXPORT_SYMBOL_GPL(kvm_handle_page_fault);

3816 3817 3818 3819 3820 3821 3822 3823 3824 3825
static bool
check_hugepage_cache_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, int level)
{
	int page_num = KVM_PAGES_PER_HPAGE(level);

	gfn &= ~(page_num - 1);

	return kvm_mtrr_check_gfn_range_consistency(vcpu, gfn, page_num);
}

G
Gleb Natapov 已提交
3826
static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
3827
			  bool prefault)
3828
{
D
Dan Williams 已提交
3829
	kvm_pfn_t pfn;
3830
	int r;
3831
	int level;
3832
	bool force_pt_level;
M
Marcelo Tosatti 已提交
3833
	gfn_t gfn = gpa >> PAGE_SHIFT;
3834
	unsigned long mmu_seq;
3835 3836
	int write = error_code & PFERR_WRITE_MASK;
	bool map_writable;
3837

3838
	MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3839

3840 3841
	if (page_fault_handle_page_track(vcpu, error_code, gfn))
		return 1;
3842

3843 3844 3845 3846
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

3847 3848 3849
	force_pt_level = !check_hugepage_cache_consistency(vcpu, gfn,
							   PT_DIRECTORY_LEVEL);
	level = mapping_level(vcpu, gfn, &force_pt_level);
3850
	if (likely(!force_pt_level)) {
3851 3852 3853
		if (level > PT_DIRECTORY_LEVEL &&
		    !check_hugepage_cache_consistency(vcpu, gfn, level))
			level = PT_DIRECTORY_LEVEL;
3854
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3855
	}
3856

3857 3858 3859
	if (fast_page_fault(vcpu, gpa, level, error_code))
		return 0;

3860
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
3861
	smp_rmb();
3862

3863
	if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
3864 3865
		return 0;

3866 3867 3868
	if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
		return r;

3869
	spin_lock(&vcpu->kvm->mmu_lock);
3870
	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3871
		goto out_unlock;
3872
	make_mmu_pages_available(vcpu);
3873 3874
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3875
	r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3876 3877 3878
	spin_unlock(&vcpu->kvm->mmu_lock);

	return r;
3879 3880 3881 3882 3883

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
3884 3885
}

3886 3887
static void nonpaging_init_context(struct kvm_vcpu *vcpu,
				   struct kvm_mmu *context)
A
Avi Kivity 已提交
3888 3889 3890
{
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
3891
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
3892
	context->invlpg = nonpaging_invlpg;
3893
	context->update_pte = nonpaging_update_pte;
3894
	context->root_level = 0;
A
Avi Kivity 已提交
3895
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
3896
	context->root_hpa = INVALID_PAGE;
3897
	context->direct_map = true;
3898
	context->nx = false;
A
Avi Kivity 已提交
3899 3900
}

3901
void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3902
{
3903
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
3904 3905
}

3906 3907
static unsigned long get_cr3(struct kvm_vcpu *vcpu)
{
3908
	return kvm_read_cr3(vcpu);
3909 3910
}

3911 3912
static void inject_page_fault(struct kvm_vcpu *vcpu,
			      struct x86_exception *fault)
A
Avi Kivity 已提交
3913
{
3914
	vcpu->arch.mmu.inject_page_fault(vcpu, fault);
A
Avi Kivity 已提交
3915 3916
}

3917
static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
3918
			   unsigned access, int *nr_present)
3919 3920 3921 3922 3923 3924 3925 3926
{
	if (unlikely(is_mmio_spte(*sptep))) {
		if (gfn != get_mmio_spte_gfn(*sptep)) {
			mmu_spte_clear_no_track(sptep);
			return true;
		}

		(*nr_present)++;
3927
		mark_mmio_spte(vcpu, sptep, gfn, access);
3928 3929 3930 3931 3932 3933
		return true;
	}

	return false;
}

3934 3935
static inline bool is_last_gpte(struct kvm_mmu *mmu,
				unsigned level, unsigned gpte)
A
Avi Kivity 已提交
3936
{
3937 3938 3939 3940 3941 3942
	/*
	 * PT_PAGE_TABLE_LEVEL always terminates.  The RHS has bit 7 set
	 * iff level <= PT_PAGE_TABLE_LEVEL, which for our purpose means
	 * level == PT_PAGE_TABLE_LEVEL; set PT_PAGE_SIZE_MASK in gpte then.
	 */
	gpte |= level - PT_PAGE_TABLE_LEVEL - 1;
A
Avi Kivity 已提交
3943

3944 3945 3946 3947 3948 3949 3950 3951
	/*
	 * The RHS has bit 7 set iff level < mmu->last_nonleaf_level.
	 * If it is clear, there are no large pages at this level, so clear
	 * PT_PAGE_SIZE_MASK in gpte if that is the case.
	 */
	gpte &= level - mmu->last_nonleaf_level;

	return gpte & PT_PAGE_SIZE_MASK;
A
Avi Kivity 已提交
3952 3953
}

3954 3955 3956 3957 3958
#define PTTYPE_EPT 18 /* arbitrary */
#define PTTYPE PTTYPE_EPT
#include "paging_tmpl.h"
#undef PTTYPE

A
Avi Kivity 已提交
3959 3960 3961 3962 3963 3964 3965 3966
#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

3967 3968 3969 3970
static void
__reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
			struct rsvd_bits_validate *rsvd_check,
			int maxphyaddr, int level, bool nx, bool gbpages,
3971
			bool pse, bool amd)
3972 3973
{
	u64 exb_bit_rsvd = 0;
3974
	u64 gbpages_bit_rsvd = 0;
3975
	u64 nonleaf_bit8_rsvd = 0;
3976

3977
	rsvd_check->bad_mt_xwr = 0;
3978

3979
	if (!nx)
3980
		exb_bit_rsvd = rsvd_bits(63, 63);
3981
	if (!gbpages)
3982
		gbpages_bit_rsvd = rsvd_bits(7, 7);
3983 3984 3985 3986 3987

	/*
	 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
	 * leaf entries) on AMD CPUs only.
	 */
3988
	if (amd)
3989 3990
		nonleaf_bit8_rsvd = rsvd_bits(8, 8);

3991
	switch (level) {
3992 3993
	case PT32_ROOT_LEVEL:
		/* no rsvd bits for 2 level 4K page table entries */
3994 3995 3996 3997
		rsvd_check->rsvd_bits_mask[0][1] = 0;
		rsvd_check->rsvd_bits_mask[0][0] = 0;
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
3998

3999
		if (!pse) {
4000
			rsvd_check->rsvd_bits_mask[1][1] = 0;
4001 4002 4003
			break;
		}

4004 4005
		if (is_cpuid_PSE36())
			/* 36bits PSE 4MB page */
4006
			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
4007 4008
		else
			/* 32 bits PSE 4MB page */
4009
			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
4010 4011
		break;
	case PT32E_ROOT_LEVEL:
4012
		rsvd_check->rsvd_bits_mask[0][2] =
4013
			rsvd_bits(maxphyaddr, 63) |
4014
			rsvd_bits(5, 8) | rsvd_bits(1, 2);	/* PDPTE */
4015
		rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
4016
			rsvd_bits(maxphyaddr, 62);	/* PDE */
4017
		rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
4018
			rsvd_bits(maxphyaddr, 62); 	/* PTE */
4019
		rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
4020 4021
			rsvd_bits(maxphyaddr, 62) |
			rsvd_bits(13, 20);		/* large page */
4022 4023
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4024 4025
		break;
	case PT64_ROOT_LEVEL:
4026 4027
		rsvd_check->rsvd_bits_mask[0][3] = exb_bit_rsvd |
			nonleaf_bit8_rsvd | rsvd_bits(7, 7) |
4028
			rsvd_bits(maxphyaddr, 51);
4029 4030
		rsvd_check->rsvd_bits_mask[0][2] = exb_bit_rsvd |
			nonleaf_bit8_rsvd | gbpages_bit_rsvd |
4031
			rsvd_bits(maxphyaddr, 51);
4032 4033 4034 4035 4036 4037 4038
		rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51);
		rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51);
		rsvd_check->rsvd_bits_mask[1][3] =
			rsvd_check->rsvd_bits_mask[0][3];
		rsvd_check->rsvd_bits_mask[1][2] = exb_bit_rsvd |
4039
			gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51) |
4040
			rsvd_bits(13, 29);
4041
		rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
4042 4043
			rsvd_bits(maxphyaddr, 51) |
			rsvd_bits(13, 20);		/* large page */
4044 4045
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4046 4047 4048 4049
		break;
	}
}

4050 4051 4052 4053 4054
static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
{
	__reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check,
				cpuid_maxphyaddr(vcpu), context->root_level,
4055 4056
				context->nx,
				guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),
4057
				is_pse(vcpu), guest_cpuid_is_amd(vcpu));
4058 4059
}

4060 4061 4062
static void
__reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
			    int maxphyaddr, bool execonly)
4063
{
4064
	u64 bad_mt_xwr;
4065

4066
	rsvd_check->rsvd_bits_mask[0][3] =
4067
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7);
4068
	rsvd_check->rsvd_bits_mask[0][2] =
4069
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
4070
	rsvd_check->rsvd_bits_mask[0][1] =
4071
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
4072
	rsvd_check->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51);
4073 4074

	/* large page */
4075 4076
	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
	rsvd_check->rsvd_bits_mask[1][2] =
4077
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 29);
4078
	rsvd_check->rsvd_bits_mask[1][1] =
4079
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 20);
4080
	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
4081

4082 4083 4084 4085 4086 4087 4088 4089
	bad_mt_xwr = 0xFFull << (2 * 8);	/* bits 3..5 must not be 2 */
	bad_mt_xwr |= 0xFFull << (3 * 8);	/* bits 3..5 must not be 3 */
	bad_mt_xwr |= 0xFFull << (7 * 8);	/* bits 3..5 must not be 7 */
	bad_mt_xwr |= REPEAT_BYTE(1ull << 2);	/* bits 0..2 must not be 010 */
	bad_mt_xwr |= REPEAT_BYTE(1ull << 6);	/* bits 0..2 must not be 110 */
	if (!execonly) {
		/* bits 0..2 must not be 100 unless VMX capabilities allow it */
		bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
4090
	}
4091
	rsvd_check->bad_mt_xwr = bad_mt_xwr;
4092 4093
}

4094 4095 4096 4097 4098 4099 4100
static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
		struct kvm_mmu *context, bool execonly)
{
	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
				    cpuid_maxphyaddr(vcpu), execonly);
}

4101 4102 4103 4104 4105 4106 4107 4108
/*
 * the page table on host is the shadow page table for the page
 * table in guest or amd nested guest, its mmu features completely
 * follow the features in guest.
 */
void
reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
{
4109 4110
	bool uses_nx = context->nx || context->base_role.smep_andnot_wp;

4111 4112 4113 4114
	/*
	 * Passing "true" to the last argument is okay; it adds a check
	 * on bit 8 of the SPTEs which KVM doesn't use anyway.
	 */
4115 4116
	__reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
				boot_cpu_data.x86_phys_bits,
4117
				context->shadow_root_level, uses_nx,
4118 4119
				guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),
				is_pse(vcpu), true);
4120 4121 4122
}
EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask);

4123 4124 4125 4126 4127 4128
static inline bool boot_cpu_is_amd(void)
{
	WARN_ON_ONCE(!tdp_enabled);
	return shadow_x_mask == 0;
}

4129 4130 4131 4132 4133 4134 4135 4136
/*
 * the direct page table on host, use as much mmu features as
 * possible, however, kvm currently does not do execution-protection.
 */
static void
reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
				struct kvm_mmu *context)
{
4137
	if (boot_cpu_is_amd())
4138 4139 4140
		__reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
					boot_cpu_data.x86_phys_bits,
					context->shadow_root_level, false,
4141 4142
					boot_cpu_has(X86_FEATURE_GBPAGES),
					true, true);
4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161
	else
		__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
					    boot_cpu_data.x86_phys_bits,
					    false);

}

/*
 * as the comments in reset_shadow_zero_bits_mask() except it
 * is the shadow page table for intel nested guest.
 */
static void
reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
				struct kvm_mmu *context, bool execonly)
{
	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
				    boot_cpu_data.x86_phys_bits, execonly);
}

4162 4163
static void update_permission_bitmask(struct kvm_vcpu *vcpu,
				      struct kvm_mmu *mmu, bool ept)
4164 4165 4166
{
	unsigned bit, byte, pfec;
	u8 map;
F
Feng Wu 已提交
4167
	bool fault, x, w, u, wf, uf, ff, smapf, cr4_smap, cr4_smep, smap = 0;
4168

F
Feng Wu 已提交
4169
	cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
F
Feng Wu 已提交
4170
	cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
4171 4172 4173 4174 4175 4176
	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
		pfec = byte << 1;
		map = 0;
		wf = pfec & PFERR_WRITE_MASK;
		uf = pfec & PFERR_USER_MASK;
		ff = pfec & PFERR_FETCH_MASK;
F
Feng Wu 已提交
4177 4178 4179 4180 4181 4182
		/*
		 * PFERR_RSVD_MASK bit is set in PFEC if the access is not
		 * subject to SMAP restrictions, and cleared otherwise. The
		 * bit is only meaningful if the SMAP bit is set in CR4.
		 */
		smapf = !(pfec & PFERR_RSVD_MASK);
4183 4184 4185 4186 4187
		for (bit = 0; bit < 8; ++bit) {
			x = bit & ACC_EXEC_MASK;
			w = bit & ACC_WRITE_MASK;
			u = bit & ACC_USER_MASK;

4188 4189 4190 4191 4192 4193
			if (!ept) {
				/* Not really needed: !nx will cause pte.nx to fault */
				x |= !mmu->nx;
				/* Allow supervisor writes if !cr0.wp */
				w |= !is_write_protection(vcpu) && !uf;
				/* Disallow supervisor fetches of user code if cr4.smep */
F
Feng Wu 已提交
4194
				x &= !(cr4_smep && u && !uf);
F
Feng Wu 已提交
4195 4196 4197 4198 4199 4200 4201

				/*
				 * SMAP:kernel-mode data accesses from user-mode
				 * mappings should fault. A fault is considered
				 * as a SMAP violation if all of the following
				 * conditions are ture:
				 *   - X86_CR4_SMAP is set in CR4
4202
				 *   - A user page is accessed
F
Feng Wu 已提交
4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214
				 *   - Page fault in kernel mode
				 *   - if CPL = 3 or X86_EFLAGS_AC is clear
				 *
				 *   Here, we cover the first three conditions.
				 *   The fourth is computed dynamically in
				 *   permission_fault() and is in smapf.
				 *
				 *   Also, SMAP does not affect instruction
				 *   fetches, add the !ff check here to make it
				 *   clearer.
				 */
				smap = cr4_smap && u && !uf && !ff;
4215
			}
4216

F
Feng Wu 已提交
4217 4218
			fault = (ff && !x) || (uf && !u) || (wf && !w) ||
				(smapf && smap);
4219 4220 4221 4222 4223 4224
			map |= fault << bit;
		}
		mmu->permissions[byte] = map;
	}
}

4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299
/*
* PKU is an additional mechanism by which the paging controls access to
* user-mode addresses based on the value in the PKRU register.  Protection
* key violations are reported through a bit in the page fault error code.
* Unlike other bits of the error code, the PK bit is not known at the
* call site of e.g. gva_to_gpa; it must be computed directly in
* permission_fault based on two bits of PKRU, on some machine state (CR4,
* CR0, EFER, CPL), and on other bits of the error code and the page tables.
*
* In particular the following conditions come from the error code, the
* page tables and the machine state:
* - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
* - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
* - PK is always zero if U=0 in the page tables
* - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
*
* The PKRU bitmask caches the result of these four conditions.  The error
* code (minus the P bit) and the page table's U bit form an index into the
* PKRU bitmask.  Two bits of the PKRU bitmask are then extracted and ANDed
* with the two bits of the PKRU register corresponding to the protection key.
* For the first three conditions above the bits will be 00, thus masking
* away both AD and WD.  For all reads or if the last condition holds, WD
* only will be masked away.
*/
static void update_pkru_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
				bool ept)
{
	unsigned bit;
	bool wp;

	if (ept) {
		mmu->pkru_mask = 0;
		return;
	}

	/* PKEY is enabled only if CR4.PKE and EFER.LMA are both set. */
	if (!kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || !is_long_mode(vcpu)) {
		mmu->pkru_mask = 0;
		return;
	}

	wp = is_write_protection(vcpu);

	for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
		unsigned pfec, pkey_bits;
		bool check_pkey, check_write, ff, uf, wf, pte_user;

		pfec = bit << 1;
		ff = pfec & PFERR_FETCH_MASK;
		uf = pfec & PFERR_USER_MASK;
		wf = pfec & PFERR_WRITE_MASK;

		/* PFEC.RSVD is replaced by ACC_USER_MASK. */
		pte_user = pfec & PFERR_RSVD_MASK;

		/*
		 * Only need to check the access which is not an
		 * instruction fetch and is to a user page.
		 */
		check_pkey = (!ff && pte_user);
		/*
		 * write access is controlled by PKRU if it is a
		 * user access or CR0.WP = 1.
		 */
		check_write = check_pkey && wf && (uf || wp);

		/* PKRU.AD stops both read and write access. */
		pkey_bits = !!check_pkey;
		/* PKRU.WD stops write access. */
		pkey_bits |= (!!check_write) << 1;

		mmu->pkru_mask |= (pkey_bits & 3) << pfec;
	}
}

4300
static void update_last_nonleaf_level(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
A
Avi Kivity 已提交
4301
{
4302 4303 4304 4305 4306
	unsigned root_level = mmu->root_level;

	mmu->last_nonleaf_level = root_level;
	if (root_level == PT32_ROOT_LEVEL && is_pse(vcpu))
		mmu->last_nonleaf_level++;
A
Avi Kivity 已提交
4307 4308
}

4309 4310 4311
static void paging64_init_context_common(struct kvm_vcpu *vcpu,
					 struct kvm_mmu *context,
					 int level)
A
Avi Kivity 已提交
4312
{
4313
	context->nx = is_nx(vcpu);
4314
	context->root_level = level;
4315

4316
	reset_rsvds_bits_mask(vcpu, context);
4317
	update_permission_bitmask(vcpu, context, false);
4318
	update_pkru_bitmask(vcpu, context, false);
4319
	update_last_nonleaf_level(vcpu, context);
A
Avi Kivity 已提交
4320

4321
	MMU_WARN_ON(!is_pae(vcpu));
A
Avi Kivity 已提交
4322 4323
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
4324
	context->sync_page = paging64_sync_page;
M
Marcelo Tosatti 已提交
4325
	context->invlpg = paging64_invlpg;
4326
	context->update_pte = paging64_update_pte;
4327
	context->shadow_root_level = level;
A
Avi Kivity 已提交
4328
	context->root_hpa = INVALID_PAGE;
4329
	context->direct_map = false;
A
Avi Kivity 已提交
4330 4331
}

4332 4333
static void paging64_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
4334
{
4335
	paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
4336 4337
}

4338 4339
static void paging32_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
A
Avi Kivity 已提交
4340
{
4341
	context->nx = false;
4342
	context->root_level = PT32_ROOT_LEVEL;
4343

4344
	reset_rsvds_bits_mask(vcpu, context);
4345
	update_permission_bitmask(vcpu, context, false);
4346
	update_pkru_bitmask(vcpu, context, false);
4347
	update_last_nonleaf_level(vcpu, context);
A
Avi Kivity 已提交
4348 4349 4350

	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
4351
	context->sync_page = paging32_sync_page;
M
Marcelo Tosatti 已提交
4352
	context->invlpg = paging32_invlpg;
4353
	context->update_pte = paging32_update_pte;
A
Avi Kivity 已提交
4354
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
4355
	context->root_hpa = INVALID_PAGE;
4356
	context->direct_map = false;
A
Avi Kivity 已提交
4357 4358
}

4359 4360
static void paging32E_init_context(struct kvm_vcpu *vcpu,
				   struct kvm_mmu *context)
A
Avi Kivity 已提交
4361
{
4362
	paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
A
Avi Kivity 已提交
4363 4364
}

4365
static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
4366
{
4367
	struct kvm_mmu *context = &vcpu->arch.mmu;
4368

4369
	context->base_role.word = 0;
4370
	context->base_role.smm = is_smm(vcpu);
4371
	context->base_role.ad_disabled = (shadow_accessed_mask == 0);
4372
	context->page_fault = tdp_page_fault;
4373
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
4374
	context->invlpg = nonpaging_invlpg;
4375
	context->update_pte = nonpaging_update_pte;
4376
	context->shadow_root_level = kvm_x86_ops->get_tdp_level();
4377
	context->root_hpa = INVALID_PAGE;
4378
	context->direct_map = true;
4379
	context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
4380
	context->get_cr3 = get_cr3;
4381
	context->get_pdptr = kvm_pdptr_read;
4382
	context->inject_page_fault = kvm_inject_page_fault;
4383 4384

	if (!is_paging(vcpu)) {
4385
		context->nx = false;
4386 4387 4388
		context->gva_to_gpa = nonpaging_gva_to_gpa;
		context->root_level = 0;
	} else if (is_long_mode(vcpu)) {
4389
		context->nx = is_nx(vcpu);
4390
		context->root_level = PT64_ROOT_LEVEL;
4391 4392
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging64_gva_to_gpa;
4393
	} else if (is_pae(vcpu)) {
4394
		context->nx = is_nx(vcpu);
4395
		context->root_level = PT32E_ROOT_LEVEL;
4396 4397
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging64_gva_to_gpa;
4398
	} else {
4399
		context->nx = false;
4400
		context->root_level = PT32_ROOT_LEVEL;
4401 4402
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging32_gva_to_gpa;
4403 4404
	}

4405
	update_permission_bitmask(vcpu, context, false);
4406
	update_pkru_bitmask(vcpu, context, false);
4407
	update_last_nonleaf_level(vcpu, context);
4408
	reset_tdp_shadow_zero_bits_mask(vcpu, context);
4409 4410
}

4411
void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4412
{
4413
	bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
4414
	bool smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
4415 4416
	struct kvm_mmu *context = &vcpu->arch.mmu;

4417
	MMU_WARN_ON(VALID_PAGE(context->root_hpa));
A
Avi Kivity 已提交
4418 4419

	if (!is_paging(vcpu))
4420
		nonpaging_init_context(vcpu, context);
A
Avi Kivity 已提交
4421
	else if (is_long_mode(vcpu))
4422
		paging64_init_context(vcpu, context);
A
Avi Kivity 已提交
4423
	else if (is_pae(vcpu))
4424
		paging32E_init_context(vcpu, context);
A
Avi Kivity 已提交
4425
	else
4426
		paging32_init_context(vcpu, context);
4427

4428 4429 4430 4431
	context->base_role.nxe = is_nx(vcpu);
	context->base_role.cr4_pae = !!is_pae(vcpu);
	context->base_role.cr0_wp  = is_write_protection(vcpu);
	context->base_role.smep_andnot_wp
4432
		= smep && !is_write_protection(vcpu);
4433 4434
	context->base_role.smap_andnot_wp
		= smap && !is_write_protection(vcpu);
4435
	context->base_role.smm = is_smm(vcpu);
4436
	reset_shadow_zero_bits_mask(vcpu, context);
4437 4438 4439
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);

4440 4441
void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
			     bool accessed_dirty)
N
Nadav Har'El 已提交
4442
{
4443 4444
	struct kvm_mmu *context = &vcpu->arch.mmu;

4445
	MMU_WARN_ON(VALID_PAGE(context->root_hpa));
N
Nadav Har'El 已提交
4446 4447 4448 4449

	context->shadow_root_level = kvm_x86_ops->get_tdp_level();

	context->nx = true;
4450
	context->ept_ad = accessed_dirty;
N
Nadav Har'El 已提交
4451 4452 4453 4454 4455 4456 4457 4458
	context->page_fault = ept_page_fault;
	context->gva_to_gpa = ept_gva_to_gpa;
	context->sync_page = ept_sync_page;
	context->invlpg = ept_invlpg;
	context->update_pte = ept_update_pte;
	context->root_level = context->shadow_root_level;
	context->root_hpa = INVALID_PAGE;
	context->direct_map = false;
4459
	context->base_role.ad_disabled = !accessed_dirty;
N
Nadav Har'El 已提交
4460 4461

	update_permission_bitmask(vcpu, context, true);
4462
	update_pkru_bitmask(vcpu, context, true);
N
Nadav Har'El 已提交
4463
	reset_rsvds_bits_mask_ept(vcpu, context, execonly);
4464
	reset_ept_shadow_zero_bits_mask(vcpu, context, execonly);
N
Nadav Har'El 已提交
4465 4466 4467
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);

4468
static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
4469
{
4470 4471 4472 4473 4474 4475 4476
	struct kvm_mmu *context = &vcpu->arch.mmu;

	kvm_init_shadow_mmu(vcpu);
	context->set_cr3           = kvm_x86_ops->set_cr3;
	context->get_cr3           = get_cr3;
	context->get_pdptr         = kvm_pdptr_read;
	context->inject_page_fault = kvm_inject_page_fault;
A
Avi Kivity 已提交
4477 4478
}

4479
static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
4480 4481 4482 4483
{
	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;

	g_context->get_cr3           = get_cr3;
4484
	g_context->get_pdptr         = kvm_pdptr_read;
4485 4486 4487
	g_context->inject_page_fault = kvm_inject_page_fault;

	/*
4488 4489 4490 4491 4492 4493
	 * Note that arch.mmu.gva_to_gpa translates l2_gpa to l1_gpa using
	 * L1's nested page tables (e.g. EPT12). The nested translation
	 * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
	 * L2's page tables as the first level of translation and L1's
	 * nested page tables as the second level of translation. Basically
	 * the gva_to_gpa functions between mmu and nested_mmu are swapped.
4494 4495
	 */
	if (!is_paging(vcpu)) {
4496
		g_context->nx = false;
4497 4498 4499
		g_context->root_level = 0;
		g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
	} else if (is_long_mode(vcpu)) {
4500
		g_context->nx = is_nx(vcpu);
4501
		g_context->root_level = PT64_ROOT_LEVEL;
4502
		reset_rsvds_bits_mask(vcpu, g_context);
4503 4504
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else if (is_pae(vcpu)) {
4505
		g_context->nx = is_nx(vcpu);
4506
		g_context->root_level = PT32E_ROOT_LEVEL;
4507
		reset_rsvds_bits_mask(vcpu, g_context);
4508 4509
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else {
4510
		g_context->nx = false;
4511
		g_context->root_level = PT32_ROOT_LEVEL;
4512
		reset_rsvds_bits_mask(vcpu, g_context);
4513 4514 4515
		g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
	}

4516
	update_permission_bitmask(vcpu, g_context, false);
4517
	update_pkru_bitmask(vcpu, g_context, false);
4518
	update_last_nonleaf_level(vcpu, g_context);
4519 4520
}

4521
static void init_kvm_mmu(struct kvm_vcpu *vcpu)
4522
{
4523
	if (mmu_is_nested(vcpu))
4524
		init_kvm_nested_mmu(vcpu);
4525
	else if (tdp_enabled)
4526
		init_kvm_tdp_mmu(vcpu);
4527
	else
4528
		init_kvm_softmmu(vcpu);
4529 4530
}

4531
void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4532
{
4533
	kvm_mmu_unload(vcpu);
4534
	init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
4535
}
4536
EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
A
Avi Kivity 已提交
4537 4538

int kvm_mmu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4539
{
4540 4541
	int r;

4542
	r = mmu_topup_memory_caches(vcpu);
A
Avi Kivity 已提交
4543 4544
	if (r)
		goto out;
4545
	r = mmu_alloc_roots(vcpu);
4546
	kvm_mmu_sync_roots(vcpu);
4547 4548
	if (r)
		goto out;
4549
	/* set_cr3() should ensure TLB has been flushed */
4550
	vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
4551 4552
out:
	return r;
A
Avi Kivity 已提交
4553
}
A
Avi Kivity 已提交
4554 4555 4556 4557 4558
EXPORT_SYMBOL_GPL(kvm_mmu_load);

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
	mmu_free_roots(vcpu);
4559
	WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
4560
}
4561
EXPORT_SYMBOL_GPL(kvm_mmu_unload);
A
Avi Kivity 已提交
4562

4563
static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
4564 4565
				  struct kvm_mmu_page *sp, u64 *spte,
				  const void *new)
4566
{
4567
	if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
4568 4569
		++vcpu->kvm->stat.mmu_pde_zapped;
		return;
4570
        }
4571

A
Avi Kivity 已提交
4572
	++vcpu->kvm->stat.mmu_pte_updated;
4573
	vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
4574 4575
}

4576 4577 4578 4579 4580 4581 4582 4583
static bool need_remote_flush(u64 old, u64 new)
{
	if (!is_shadow_present_pte(old))
		return false;
	if (!is_shadow_present_pte(new))
		return true;
	if ((old ^ new) & PT64_BASE_ADDR_MASK)
		return true;
4584 4585
	old ^= shadow_nx_mask;
	new ^= shadow_nx_mask;
4586 4587 4588
	return (old & ~new & PT64_PERM_MASK) != 0;
}

4589 4590
static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
				    const u8 *new, int *bytes)
4591
{
4592 4593
	u64 gentry;
	int r;
4594 4595 4596

	/*
	 * Assume that the pte write on a page table of the same type
4597 4598
	 * as the current vcpu paging mode since we update the sptes only
	 * when they have the same mode.
4599
	 */
4600
	if (is_pae(vcpu) && *bytes == 4) {
4601
		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */
4602 4603
		*gpa &= ~(gpa_t)7;
		*bytes = 8;
4604
		r = kvm_vcpu_read_guest(vcpu, *gpa, &gentry, 8);
4605 4606
		if (r)
			gentry = 0;
4607 4608 4609
		new = (const u8 *)&gentry;
	}

4610
	switch (*bytes) {
4611 4612 4613 4614 4615 4616 4617 4618 4619
	case 4:
		gentry = *(const u32 *)new;
		break;
	case 8:
		gentry = *(const u64 *)new;
		break;
	default:
		gentry = 0;
		break;
4620 4621
	}

4622 4623 4624 4625 4626 4627 4628
	return gentry;
}

/*
 * If we're seeing too many writes to a page, it may no longer be a page table,
 * or we may be forking, in which case it is better to unmap the page.
 */
4629
static bool detect_write_flooding(struct kvm_mmu_page *sp)
4630
{
4631 4632 4633 4634
	/*
	 * Skip write-flooding detected for the sp whose level is 1, because
	 * it can become unsync, then the guest page is not write-protected.
	 */
4635
	if (sp->role.level == PT_PAGE_TABLE_LEVEL)
4636
		return false;
4637

4638 4639
	atomic_inc(&sp->write_flooding_count);
	return atomic_read(&sp->write_flooding_count) >= 3;
4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655
}

/*
 * Misaligned accesses are too much trouble to fix up; also, they usually
 * indicate a page is not used as a page table.
 */
static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
				    int bytes)
{
	unsigned offset, pte_size, misaligned;

	pgprintk("misaligned: gpa %llx bytes %d role %x\n",
		 gpa, bytes, sp->role.word);

	offset = offset_in_page(gpa);
	pte_size = sp->role.cr4_pae ? 8 : 4;
4656 4657 4658 4659 4660 4661 4662 4663

	/*
	 * Sometimes, the OS only writes the last one bytes to update status
	 * bits, for example, in linux, andb instruction is used in clear_bit().
	 */
	if (!(offset & (pte_size - 1)) && bytes == 1)
		return false;

4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700
	misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
	misaligned |= bytes < 4;

	return misaligned;
}

static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
{
	unsigned page_offset, quadrant;
	u64 *spte;
	int level;

	page_offset = offset_in_page(gpa);
	level = sp->role.level;
	*nspte = 1;
	if (!sp->role.cr4_pae) {
		page_offset <<= 1;	/* 32->64 */
		/*
		 * A 32-bit pde maps 4MB while the shadow pdes map
		 * only 2MB.  So we need to double the offset again
		 * and zap two pdes instead of one.
		 */
		if (level == PT32_ROOT_LEVEL) {
			page_offset &= ~7; /* kill rounding error */
			page_offset <<= 1;
			*nspte = 2;
		}
		quadrant = page_offset >> PAGE_SHIFT;
		page_offset &= ~PAGE_MASK;
		if (quadrant != sp->role.quadrant)
			return NULL;
	}

	spte = &sp->spt[page_offset / sizeof(*spte)];
	return spte;
}

4701
static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
4702 4703
			      const u8 *new, int bytes,
			      struct kvm_page_track_notifier_node *node)
4704 4705 4706 4707 4708 4709
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_mmu_page *sp;
	LIST_HEAD(invalid_list);
	u64 entry, gentry, *spte;
	int npte;
4710
	bool remote_flush, local_flush;
4711 4712 4713 4714 4715 4716 4717
	union kvm_mmu_page_role mask = { };

	mask.cr0_wp = 1;
	mask.cr4_pae = 1;
	mask.nxe = 1;
	mask.smep_andnot_wp = 1;
	mask.smap_andnot_wp = 1;
4718
	mask.smm = 1;
4719
	mask.ad_disabled = 1;
4720 4721 4722 4723 4724 4725 4726 4727

	/*
	 * If we don't have indirect shadow pages, it means no page is
	 * write-protected, so we can exit simply.
	 */
	if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
		return;

4728
	remote_flush = local_flush = false;
4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742

	pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);

	gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, new, &bytes);

	/*
	 * No need to care whether allocation memory is successful
	 * or not since pte prefetch is skiped if it does not have
	 * enough objects in the cache.
	 */
	mmu_topup_memory_caches(vcpu);

	spin_lock(&vcpu->kvm->mmu_lock);
	++vcpu->kvm->stat.mmu_pte_write;
4743
	kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
4744

4745
	for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
4746
		if (detect_write_misaligned(sp, gpa, bytes) ||
4747
		      detect_write_flooding(sp)) {
4748
			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
A
Avi Kivity 已提交
4749
			++vcpu->kvm->stat.mmu_flooded;
4750 4751
			continue;
		}
4752 4753 4754 4755 4756

		spte = get_written_sptes(sp, gpa, &npte);
		if (!spte)
			continue;

4757
		local_flush = true;
4758
		while (npte--) {
4759
			entry = *spte;
4760
			mmu_page_zap_pte(vcpu->kvm, sp, spte);
4761 4762
			if (gentry &&
			      !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
4763
			      & mask.word) && rmap_can_add(vcpu))
4764
				mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
G
Gleb Natapov 已提交
4765
			if (need_remote_flush(entry, *spte))
4766
				remote_flush = true;
4767
			++spte;
4768 4769
		}
	}
4770
	kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush);
4771
	kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
4772
	spin_unlock(&vcpu->kvm->mmu_lock);
4773 4774
}

4775 4776
int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
4777 4778
	gpa_t gpa;
	int r;
4779

4780
	if (vcpu->arch.mmu.direct_map)
4781 4782
		return 0;

4783
	gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
4784 4785

	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
4786

4787
	return r;
4788
}
4789
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
4790

4791
static void make_mmu_pages_available(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4792
{
4793
	LIST_HEAD(invalid_list);
4794

4795 4796 4797
	if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES))
		return;

4798 4799 4800
	while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) {
		if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list))
			break;
A
Avi Kivity 已提交
4801

A
Avi Kivity 已提交
4802
		++vcpu->kvm->stat.mmu_recycled;
A
Avi Kivity 已提交
4803
	}
4804
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
A
Avi Kivity 已提交
4805 4806
}

4807
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code,
4808
		       void *insn, int insn_len)
4809
{
4810
	int r, emulation_type = EMULTYPE_RETRY;
4811
	enum emulation_result er;
4812
	bool direct = vcpu->arch.mmu.direct_map || mmu_is_nested(vcpu);
4813

4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824
	if (unlikely(error_code & PFERR_RSVD_MASK)) {
		r = handle_mmio_page_fault(vcpu, cr2, direct);
		if (r == RET_MMIO_PF_EMULATE) {
			emulation_type = 0;
			goto emulate;
		}
		if (r == RET_MMIO_PF_RETRY)
			return 1;
		if (r < 0)
			return r;
	}
4825

4826 4827
	r = vcpu->arch.mmu.page_fault(vcpu, cr2, lower_32_bits(error_code),
				      false);
4828
	if (r < 0)
4829 4830 4831
		return r;
	if (!r)
		return 1;
4832

4833 4834 4835 4836 4837 4838 4839
	/*
	 * Before emulating the instruction, check if the error code
	 * was due to a RO violation while translating the guest page.
	 * This can occur when using nested virtualization with nested
	 * paging in both guests. If true, we simply unprotect the page
	 * and resume the guest.
	 */
4840
	if (vcpu->arch.mmu.direct_map &&
4841
	    (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) {
4842 4843 4844 4845
		kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2));
		return 1;
	}

4846
	if (mmio_info_in_cache(vcpu, cr2, direct))
4847
		emulation_type = 0;
4848
emulate:
4849
	er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
4850 4851 4852 4853

	switch (er) {
	case EMULATE_DONE:
		return 1;
P
Paolo Bonzini 已提交
4854
	case EMULATE_USER_EXIT:
4855
		++vcpu->stat.mmio_exits;
4856
		/* fall through */
4857
	case EMULATE_FAIL:
4858
		return 0;
4859 4860 4861 4862 4863 4864
	default:
		BUG();
	}
}
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);

M
Marcelo Tosatti 已提交
4865 4866 4867
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
	vcpu->arch.mmu.invlpg(vcpu, gva);
4868
	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
M
Marcelo Tosatti 已提交
4869 4870 4871 4872
	++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);

4873 4874 4875 4876 4877 4878
void kvm_enable_tdp(void)
{
	tdp_enabled = true;
}
EXPORT_SYMBOL_GPL(kvm_enable_tdp);

4879 4880 4881 4882 4883 4884
void kvm_disable_tdp(void)
{
	tdp_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_tdp);

A
Avi Kivity 已提交
4885 4886
static void free_mmu_pages(struct kvm_vcpu *vcpu)
{
4887
	free_page((unsigned long)vcpu->arch.mmu.pae_root);
4888 4889
	if (vcpu->arch.mmu.lm_root != NULL)
		free_page((unsigned long)vcpu->arch.mmu.lm_root);
A
Avi Kivity 已提交
4890 4891 4892 4893
}

static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
{
4894
	struct page *page;
A
Avi Kivity 已提交
4895 4896
	int i;

4897 4898 4899 4900 4901 4902 4903
	/*
	 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
	 * Therefore we need to allocate shadow page tables in the first
	 * 4GB of memory, which happens to fit the DMA32 zone.
	 */
	page = alloc_page(GFP_KERNEL | __GFP_DMA32);
	if (!page)
4904 4905
		return -ENOMEM;

4906
	vcpu->arch.mmu.pae_root = page_address(page);
4907
	for (i = 0; i < 4; ++i)
4908
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
4909

A
Avi Kivity 已提交
4910 4911 4912
	return 0;
}

4913
int kvm_mmu_create(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4914
{
4915 4916 4917 4918
	vcpu->arch.walk_mmu = &vcpu->arch.mmu;
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
	vcpu->arch.mmu.translate_gpa = translate_gpa;
	vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
A
Avi Kivity 已提交
4919

4920 4921
	return alloc_mmu_pages(vcpu);
}
A
Avi Kivity 已提交
4922

4923
void kvm_mmu_setup(struct kvm_vcpu *vcpu)
4924
{
4925
	MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
4926

4927
	init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
4928 4929
}

4930
static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
4931 4932
			struct kvm_memory_slot *slot,
			struct kvm_page_track_notifier_node *node)
4933 4934 4935 4936
{
	kvm_mmu_invalidate_zap_all_pages(kvm);
}

4937 4938 4939 4940 4941
void kvm_mmu_init_vm(struct kvm *kvm)
{
	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;

	node->track_write = kvm_mmu_pte_write;
4942
	node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;
4943 4944 4945 4946 4947 4948 4949 4950 4951 4952
	kvm_page_track_register_notifier(kvm, node);
}

void kvm_mmu_uninit_vm(struct kvm *kvm)
{
	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;

	kvm_page_track_unregister_notifier(kvm, node);
}

4953
/* The return value indicates if tlb flush on all vcpus is needed. */
4954
typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head);
4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021

/* The caller should hold mmu-lock before calling this function. */
static bool
slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
			slot_level_handler fn, int start_level, int end_level,
			gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
{
	struct slot_rmap_walk_iterator iterator;
	bool flush = false;

	for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
			end_gfn, &iterator) {
		if (iterator.rmap)
			flush |= fn(kvm, iterator.rmap);

		if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
			if (flush && lock_flush_tlb) {
				kvm_flush_remote_tlbs(kvm);
				flush = false;
			}
			cond_resched_lock(&kvm->mmu_lock);
		}
	}

	if (flush && lock_flush_tlb) {
		kvm_flush_remote_tlbs(kvm);
		flush = false;
	}

	return flush;
}

static bool
slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
		  slot_level_handler fn, int start_level, int end_level,
		  bool lock_flush_tlb)
{
	return slot_handle_level_range(kvm, memslot, fn, start_level,
			end_level, memslot->base_gfn,
			memslot->base_gfn + memslot->npages - 1,
			lock_flush_tlb);
}

static bool
slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
		      slot_level_handler fn, bool lock_flush_tlb)
{
	return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
				 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
}

static bool
slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
			slot_level_handler fn, bool lock_flush_tlb)
{
	return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL + 1,
				 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
}

static bool
slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
		 slot_level_handler fn, bool lock_flush_tlb)
{
	return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
				 PT_PAGE_TABLE_LEVEL, lock_flush_tlb);
}

X
Xiao Guangrong 已提交
5022 5023 5024 5025
void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
5026
	int i;
X
Xiao Guangrong 已提交
5027 5028

	spin_lock(&kvm->mmu_lock);
5029 5030 5031 5032 5033 5034 5035 5036 5037
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
		kvm_for_each_memslot(memslot, slots) {
			gfn_t start, end;

			start = max(gfn_start, memslot->base_gfn);
			end = min(gfn_end, memslot->base_gfn + memslot->npages);
			if (start >= end)
				continue;
X
Xiao Guangrong 已提交
5038

5039 5040 5041 5042
			slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
						PT_PAGE_TABLE_LEVEL, PT_MAX_HUGEPAGE_LEVEL,
						start, end - 1, true);
		}
X
Xiao Guangrong 已提交
5043 5044 5045 5046 5047
	}

	spin_unlock(&kvm->mmu_lock);
}

5048 5049
static bool slot_rmap_write_protect(struct kvm *kvm,
				    struct kvm_rmap_head *rmap_head)
5050
{
5051
	return __rmap_write_protect(kvm, rmap_head, false);
5052 5053
}

5054 5055
void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
				      struct kvm_memory_slot *memslot)
A
Avi Kivity 已提交
5056
{
5057
	bool flush;
A
Avi Kivity 已提交
5058

5059
	spin_lock(&kvm->mmu_lock);
5060 5061
	flush = slot_handle_all_level(kvm, memslot, slot_rmap_write_protect,
				      false);
5062
	spin_unlock(&kvm->mmu_lock);
5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081

	/*
	 * kvm_mmu_slot_remove_write_access() and kvm_vm_ioctl_get_dirty_log()
	 * which do tlb flush out of mmu-lock should be serialized by
	 * kvm->slots_lock otherwise tlb flush would be missed.
	 */
	lockdep_assert_held(&kvm->slots_lock);

	/*
	 * We can flush all the TLBs out of the mmu lock without TLB
	 * corruption since we just change the spte from writable to
	 * readonly so that we only need to care the case of changing
	 * spte from present to present (changing the spte from present
	 * to nonpresent will flush all the TLBs immediately), in other
	 * words, the only case we care is mmu_spte_update() where we
	 * haved checked SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE
	 * instead of PT_WRITABLE_MASK, that means it does not depend
	 * on PT_WRITABLE_MASK anymore.
	 */
5082 5083
	if (flush)
		kvm_flush_remote_tlbs(kvm);
A
Avi Kivity 已提交
5084
}
5085

5086
static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
5087
					 struct kvm_rmap_head *rmap_head)
5088 5089 5090 5091
{
	u64 *sptep;
	struct rmap_iterator iter;
	int need_tlb_flush = 0;
D
Dan Williams 已提交
5092
	kvm_pfn_t pfn;
5093 5094
	struct kvm_mmu_page *sp;

5095
restart:
5096
	for_each_rmap_spte(rmap_head, &iter, sptep) {
5097 5098 5099 5100
		sp = page_header(__pa(sptep));
		pfn = spte_to_pfn(*sptep);

		/*
5101 5102 5103 5104 5105
		 * We cannot do huge page mapping for indirect shadow pages,
		 * which are found on the last rmap (level = 1) when not using
		 * tdp; such shadow pages are synced with the page table in
		 * the guest, and the guest page table is using 4K page size
		 * mapping if the indirect sp has level = 1.
5106 5107 5108
		 */
		if (sp->role.direct &&
			!kvm_is_reserved_pfn(pfn) &&
5109
			PageTransCompoundMap(pfn_to_page(pfn))) {
5110 5111
			drop_spte(kvm, sptep);
			need_tlb_flush = 1;
5112 5113
			goto restart;
		}
5114 5115 5116 5117 5118 5119
	}

	return need_tlb_flush;
}

void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
5120
				   const struct kvm_memory_slot *memslot)
5121
{
5122
	/* FIXME: const-ify all uses of struct kvm_memory_slot.  */
5123
	spin_lock(&kvm->mmu_lock);
5124 5125
	slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot,
			 kvm_mmu_zap_collapsible_spte, true);
5126 5127 5128
	spin_unlock(&kvm->mmu_lock);
}

5129 5130 5131
void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
				   struct kvm_memory_slot *memslot)
{
5132
	bool flush;
5133 5134

	spin_lock(&kvm->mmu_lock);
5135
	flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false);
5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153
	spin_unlock(&kvm->mmu_lock);

	lockdep_assert_held(&kvm->slots_lock);

	/*
	 * It's also safe to flush TLBs out of mmu lock here as currently this
	 * function is only used for dirty logging, in which case flushing TLB
	 * out of mmu lock also guarantees no dirty pages will be lost in
	 * dirty_bitmap.
	 */
	if (flush)
		kvm_flush_remote_tlbs(kvm);
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty);

void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
					struct kvm_memory_slot *memslot)
{
5154
	bool flush;
5155 5156

	spin_lock(&kvm->mmu_lock);
5157 5158
	flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect,
					false);
5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171
	spin_unlock(&kvm->mmu_lock);

	/* see kvm_mmu_slot_remove_write_access */
	lockdep_assert_held(&kvm->slots_lock);

	if (flush)
		kvm_flush_remote_tlbs(kvm);
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access);

void kvm_mmu_slot_set_dirty(struct kvm *kvm,
			    struct kvm_memory_slot *memslot)
{
5172
	bool flush;
5173 5174

	spin_lock(&kvm->mmu_lock);
5175
	flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false);
5176 5177 5178 5179 5180 5181 5182 5183 5184 5185
	spin_unlock(&kvm->mmu_lock);

	lockdep_assert_held(&kvm->slots_lock);

	/* see kvm_mmu_slot_leaf_clear_dirty */
	if (flush)
		kvm_flush_remote_tlbs(kvm);
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty);

X
Xiao Guangrong 已提交
5186
#define BATCH_ZAP_PAGES	10
5187 5188 5189
static void kvm_zap_obsolete_pages(struct kvm *kvm)
{
	struct kvm_mmu_page *sp, *node;
X
Xiao Guangrong 已提交
5190
	int batch = 0;
5191 5192 5193 5194

restart:
	list_for_each_entry_safe_reverse(sp, node,
	      &kvm->arch.active_mmu_pages, link) {
X
Xiao Guangrong 已提交
5195 5196
		int ret;

5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211
		/*
		 * No obsolete page exists before new created page since
		 * active_mmu_pages is the FIFO list.
		 */
		if (!is_obsolete_sp(kvm, sp))
			break;

		/*
		 * Since we are reversely walking the list and the invalid
		 * list will be moved to the head, skip the invalid page
		 * can help us to avoid the infinity list walking.
		 */
		if (sp->role.invalid)
			continue;

5212 5213 5214 5215
		/*
		 * Need not flush tlb since we only zap the sp with invalid
		 * generation number.
		 */
X
Xiao Guangrong 已提交
5216
		if (batch >= BATCH_ZAP_PAGES &&
5217
		      cond_resched_lock(&kvm->mmu_lock)) {
X
Xiao Guangrong 已提交
5218
			batch = 0;
5219 5220 5221
			goto restart;
		}

5222 5223
		ret = kvm_mmu_prepare_zap_page(kvm, sp,
				&kvm->arch.zapped_obsolete_pages);
X
Xiao Guangrong 已提交
5224 5225 5226
		batch += ret;

		if (ret)
5227 5228 5229
			goto restart;
	}

5230 5231 5232 5233
	/*
	 * Should flush tlb before free page tables since lockless-walking
	 * may use the pages.
	 */
5234
	kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248
}

/*
 * Fast invalidate all shadow pages and use lock-break technique
 * to zap obsolete pages.
 *
 * It's required when memslot is being deleted or VM is being
 * destroyed, in these cases, we should ensure that KVM MMU does
 * not use any resource of the being-deleted slot or all slots
 * after calling the function.
 */
void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm)
{
	spin_lock(&kvm->mmu_lock);
5249
	trace_kvm_mmu_invalidate_zap_all_pages(kvm);
5250 5251
	kvm->arch.mmu_valid_gen++;

5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262
	/*
	 * Notify all vcpus to reload its shadow page table
	 * and flush TLB. Then all vcpus will switch to new
	 * shadow page table with the new mmu_valid_gen.
	 *
	 * Note: we should do this under the protection of
	 * mmu-lock, otherwise, vcpu would purge shadow page
	 * but miss tlb flush.
	 */
	kvm_reload_remote_mmus(kvm);

5263 5264 5265 5266
	kvm_zap_obsolete_pages(kvm);
	spin_unlock(&kvm->mmu_lock);
}

5267 5268 5269 5270 5271
static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
{
	return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
}

5272
void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots)
5273 5274 5275 5276 5277
{
	/*
	 * The very rare case: if the generation-number is round,
	 * zap all shadow pages.
	 */
5278
	if (unlikely((slots->generation & MMIO_GEN_MASK) == 0)) {
5279
		kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n");
5280
		kvm_mmu_invalidate_zap_all_pages(kvm);
5281
	}
5282 5283
}

5284 5285
static unsigned long
mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
5286 5287
{
	struct kvm *kvm;
5288
	int nr_to_scan = sc->nr_to_scan;
5289
	unsigned long freed = 0;
5290

5291
	spin_lock(&kvm_lock);
5292 5293

	list_for_each_entry(kvm, &vm_list, vm_list) {
5294
		int idx;
5295
		LIST_HEAD(invalid_list);
5296

5297 5298 5299 5300 5301 5302 5303 5304
		/*
		 * Never scan more than sc->nr_to_scan VM instances.
		 * Will not hit this condition practically since we do not try
		 * to shrink more than one VM and it is very unlikely to see
		 * !n_used_mmu_pages so many times.
		 */
		if (!nr_to_scan--)
			break;
5305 5306 5307 5308 5309 5310
		/*
		 * n_used_mmu_pages is accessed without holding kvm->mmu_lock
		 * here. We may skip a VM instance errorneosly, but we do not
		 * want to shrink a VM that only started to populate its MMU
		 * anyway.
		 */
5311 5312
		if (!kvm->arch.n_used_mmu_pages &&
		      !kvm_has_zapped_obsolete_pages(kvm))
5313 5314
			continue;

5315
		idx = srcu_read_lock(&kvm->srcu);
5316 5317
		spin_lock(&kvm->mmu_lock);

5318 5319 5320 5321 5322 5323
		if (kvm_has_zapped_obsolete_pages(kvm)) {
			kvm_mmu_commit_zap_page(kvm,
			      &kvm->arch.zapped_obsolete_pages);
			goto unlock;
		}

5324 5325
		if (prepare_zap_oldest_mmu_page(kvm, &invalid_list))
			freed++;
5326
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
5327

5328
unlock:
5329
		spin_unlock(&kvm->mmu_lock);
5330
		srcu_read_unlock(&kvm->srcu, idx);
5331

5332 5333 5334 5335 5336
		/*
		 * unfair on small ones
		 * per-vm shrinkers cry out
		 * sadness comes quickly
		 */
5337 5338
		list_move_tail(&kvm->vm_list, &vm_list);
		break;
5339 5340
	}

5341
	spin_unlock(&kvm_lock);
5342 5343 5344 5345 5346 5347
	return freed;
}

static unsigned long
mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
{
5348
	return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
5349 5350 5351
}

static struct shrinker mmu_shrinker = {
5352 5353
	.count_objects = mmu_shrink_count,
	.scan_objects = mmu_shrink_scan,
5354 5355 5356
	.seeks = DEFAULT_SEEKS * 10,
};

I
Ingo Molnar 已提交
5357
static void mmu_destroy_caches(void)
5358
{
5359 5360
	if (pte_list_desc_cache)
		kmem_cache_destroy(pte_list_desc_cache);
5361 5362
	if (mmu_page_header_cache)
		kmem_cache_destroy(mmu_page_header_cache);
5363 5364 5365 5366
}

int kvm_mmu_module_init(void)
{
5367 5368
	kvm_mmu_clear_all_pte_masks();

5369 5370
	pte_list_desc_cache = kmem_cache_create("pte_list_desc",
					    sizeof(struct pte_list_desc),
5371
					    0, 0, NULL);
5372
	if (!pte_list_desc_cache)
5373 5374
		goto nomem;

5375 5376
	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
5377
						  0, 0, NULL);
5378 5379 5380
	if (!mmu_page_header_cache)
		goto nomem;

5381
	if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
5382 5383
		goto nomem;

5384 5385
	register_shrinker(&mmu_shrinker);

5386 5387 5388
	return 0;

nomem:
5389
	mmu_destroy_caches();
5390 5391 5392
	return -ENOMEM;
}

5393 5394 5395 5396 5397 5398 5399
/*
 * Caculate mmu pages needed for kvm.
 */
unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
{
	unsigned int nr_mmu_pages;
	unsigned int  nr_pages = 0;
5400
	struct kvm_memslots *slots;
5401
	struct kvm_memory_slot *memslot;
5402
	int i;
5403

5404 5405
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
5406

5407 5408 5409
		kvm_for_each_memslot(memslot, slots)
			nr_pages += memslot->npages;
	}
5410 5411 5412

	nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
	nr_mmu_pages = max(nr_mmu_pages,
5413
			   (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
5414 5415 5416 5417

	return nr_mmu_pages;
}

5418 5419
void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
5420
	kvm_mmu_unload(vcpu);
5421 5422
	free_mmu_pages(vcpu);
	mmu_free_memory_caches(vcpu);
5423 5424 5425 5426 5427 5428 5429
}

void kvm_mmu_module_exit(void)
{
	mmu_destroy_caches();
	percpu_counter_destroy(&kvm_total_used_mmu_pages);
	unregister_shrinker(&mmu_shrinker);
5430 5431
	mmu_audit_disable();
}