mmu.c 121.6 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
11 12 13 14 15 16 17 18 19
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
A
Avi Kivity 已提交
20

21
#include "irq.h"
22
#include "mmu.h"
23
#include "x86.h"
A
Avi Kivity 已提交
24
#include "kvm_cache_regs.h"
25
#include "cpuid.h"
A
Avi Kivity 已提交
26

27
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
28 29 30 31 32
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/module.h>
33
#include <linux/swap.h>
M
Marcelo Tosatti 已提交
34
#include <linux/hugetlb.h>
35
#include <linux/compiler.h>
36
#include <linux/srcu.h>
37
#include <linux/slab.h>
38
#include <linux/uaccess.h>
A
Avi Kivity 已提交
39

A
Avi Kivity 已提交
40 41
#include <asm/page.h>
#include <asm/cmpxchg.h>
42
#include <asm/io.h>
43
#include <asm/vmx.h>
A
Avi Kivity 已提交
44

45 46 47 48 49 50 51
/*
 * When setting this variable to true it enables Two-Dimensional-Paging
 * where the hardware walks 2 page tables:
 * 1. the guest-virtual to guest-physical
 * 2. while doing 1. it walks guest-physical to host-physical
 * If the hardware supports that we don't need to do shadow paging.
 */
52
bool tdp_enabled = false;
53

54 55 56 57
enum {
	AUDIT_PRE_PAGE_FAULT,
	AUDIT_POST_PAGE_FAULT,
	AUDIT_PRE_PTE_WRITE,
58 59 60
	AUDIT_POST_PTE_WRITE,
	AUDIT_PRE_SYNC,
	AUDIT_POST_SYNC
61
};
62

63
#undef MMU_DEBUG
64 65

#ifdef MMU_DEBUG
66 67
static bool dbg = 0;
module_param(dbg, bool, 0644);
68 69 70

#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
71
#define MMU_WARN_ON(x) WARN_ON(x)
72 73 74
#else
#define pgprintk(x...) do { } while (0)
#define rmap_printk(x...) do { } while (0)
75
#define MMU_WARN_ON(x) do { } while (0)
76
#endif
A
Avi Kivity 已提交
77

78 79
#define PTE_PREFETCH_NUM		8

80
#define PT_FIRST_AVAIL_BITS_SHIFT 10
A
Avi Kivity 已提交
81 82 83 84 85
#define PT64_SECOND_AVAIL_BITS_SHIFT 52

#define PT64_LEVEL_BITS 9

#define PT64_LEVEL_SHIFT(level) \
M
Mike Day 已提交
86
		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
A
Avi Kivity 已提交
87 88 89 90 91 92 93 94

#define PT64_INDEX(address, level)\
	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))


#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
M
Mike Day 已提交
95
		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
A
Avi Kivity 已提交
96

97 98 99
#define PT32_LVL_OFFSET_MASK(level) \
	(PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT32_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
100 101 102 103 104

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


105
#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
A
Avi Kivity 已提交
106 107
#define PT64_DIR_BASE_ADDR_MASK \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
108 109 110 111 112 113
#define PT64_LVL_ADDR_MASK(level) \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
#define PT64_LVL_OFFSET_MASK(level) \
	(PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
114 115 116 117

#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
118 119 120
#define PT32_LVL_ADDR_MASK(level) \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
					    * PT32_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
121

122 123
#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
			| shadow_x_mask | shadow_nx_mask)
A
Avi Kivity 已提交
124

125 126 127 128 129
#define ACC_EXEC_MASK    1
#define ACC_WRITE_MASK   PT_WRITABLE_MASK
#define ACC_USER_MASK    PT_USER_MASK
#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)

130 131
#include <trace/events/kvm.h>

132 133 134
#define CREATE_TRACE_POINTS
#include "mmutrace.h"

135 136
#define SPTE_HOST_WRITEABLE	(1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
#define SPTE_MMU_WRITEABLE	(1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
137

138 139
#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)

140 141 142
/* make pte_list_desc fit well in cache line */
#define PTE_LIST_EXT 3

143 144 145
struct pte_list_desc {
	u64 *sptes[PTE_LIST_EXT];
	struct pte_list_desc *more;
146 147
};

148 149 150 151
struct kvm_shadow_walk_iterator {
	u64 addr;
	hpa_t shadow_addr;
	u64 *sptep;
152
	int level;
153 154 155 156 157 158 159 160
	unsigned index;
};

#define for_each_shadow_entry(_vcpu, _addr, _walker)    \
	for (shadow_walk_init(&(_walker), _vcpu, _addr);	\
	     shadow_walk_okay(&(_walker));			\
	     shadow_walk_next(&(_walker)))

161 162 163 164 165 166
#define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)	\
	for (shadow_walk_init(&(_walker), _vcpu, _addr);		\
	     shadow_walk_okay(&(_walker)) &&				\
		({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });	\
	     __shadow_walk_next(&(_walker), spte))

167
static struct kmem_cache *pte_list_desc_cache;
168
static struct kmem_cache *mmu_page_header_cache;
169
static struct percpu_counter kvm_total_used_mmu_pages;
170

S
Sheng Yang 已提交
171 172 173 174 175
static u64 __read_mostly shadow_nx_mask;
static u64 __read_mostly shadow_x_mask;	/* mutual exclusive with nx_mask */
static u64 __read_mostly shadow_user_mask;
static u64 __read_mostly shadow_accessed_mask;
static u64 __read_mostly shadow_dirty_mask;
176 177 178
static u64 __read_mostly shadow_mmio_mask;

static void mmu_spte_set(u64 *sptep, u64 spte);
179
static void mmu_free_roots(struct kvm_vcpu *vcpu);
180 181 182 183 184 185 186

void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask)
{
	shadow_mmio_mask = mmio_mask;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);

187
/*
188 189 190 191 192 193 194
 * the low bit of the generation number is always presumed to be zero.
 * This disables mmio caching during memslot updates.  The concept is
 * similar to a seqcount but instead of retrying the access we just punt
 * and ignore the cache.
 *
 * spte bits 3-11 are used as bits 1-9 of the generation number,
 * the bits 52-61 are used as bits 10-19 of the generation number.
195
 */
196
#define MMIO_SPTE_GEN_LOW_SHIFT		2
197 198
#define MMIO_SPTE_GEN_HIGH_SHIFT	52

199 200 201
#define MMIO_GEN_SHIFT			20
#define MMIO_GEN_LOW_SHIFT		10
#define MMIO_GEN_LOW_MASK		((1 << MMIO_GEN_LOW_SHIFT) - 2)
202
#define MMIO_GEN_MASK			((1 << MMIO_GEN_SHIFT) - 1)
203 204 205 206 207

static u64 generation_mmio_spte_mask(unsigned int gen)
{
	u64 mask;

T
Tiejun Chen 已提交
208
	WARN_ON(gen & ~MMIO_GEN_MASK);
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225

	mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT;
	mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT;
	return mask;
}

static unsigned int get_mmio_spte_generation(u64 spte)
{
	unsigned int gen;

	spte &= ~shadow_mmio_mask;

	gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK;
	gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT;
	return gen;
}

226 227
static unsigned int kvm_current_mmio_generation(struct kvm *kvm)
{
228
	return kvm_memslots(kvm)->generation & MMIO_GEN_MASK;
229 230
}

231 232
static void mark_mmio_spte(struct kvm *kvm, u64 *sptep, u64 gfn,
			   unsigned access)
233
{
234 235
	unsigned int gen = kvm_current_mmio_generation(kvm);
	u64 mask = generation_mmio_spte_mask(gen);
236

237
	access &= ACC_WRITE_MASK | ACC_USER_MASK;
238 239
	mask |= shadow_mmio_mask | access | gfn << PAGE_SHIFT;

240
	trace_mark_mmio_spte(sptep, gfn, access, gen);
241
	mmu_spte_set(sptep, mask);
242 243 244 245 246 247 248 249 250
}

static bool is_mmio_spte(u64 spte)
{
	return (spte & shadow_mmio_mask) == shadow_mmio_mask;
}

static gfn_t get_mmio_spte_gfn(u64 spte)
{
T
Tiejun Chen 已提交
251
	u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
252
	return (spte & ~mask) >> PAGE_SHIFT;
253 254 255 256
}

static unsigned get_mmio_spte_access(u64 spte)
{
T
Tiejun Chen 已提交
257
	u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
258
	return (spte & ~mask) & ~PAGE_MASK;
259 260
}

261 262
static bool set_mmio_spte(struct kvm *kvm, u64 *sptep, gfn_t gfn,
			  pfn_t pfn, unsigned access)
263 264
{
	if (unlikely(is_noslot_pfn(pfn))) {
265
		mark_mmio_spte(kvm, sptep, gfn, access);
266 267 268 269 270
		return true;
	}

	return false;
}
271

272 273
static bool check_mmio_spte(struct kvm *kvm, u64 spte)
{
274 275 276 277 278 279 280
	unsigned int kvm_gen, spte_gen;

	kvm_gen = kvm_current_mmio_generation(kvm);
	spte_gen = get_mmio_spte_generation(spte);

	trace_check_mmio_spte(spte, kvm_gen, spte_gen);
	return likely(kvm_gen == spte_gen);
281 282
}

S
Sheng Yang 已提交
283
void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
284
		u64 dirty_mask, u64 nx_mask, u64 x_mask)
S
Sheng Yang 已提交
285 286 287 288 289 290 291 292 293
{
	shadow_user_mask = user_mask;
	shadow_accessed_mask = accessed_mask;
	shadow_dirty_mask = dirty_mask;
	shadow_nx_mask = nx_mask;
	shadow_x_mask = x_mask;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);

A
Avi Kivity 已提交
294 295 296 297 298
static int is_cpuid_PSE36(void)
{
	return 1;
}

299 300
static int is_nx(struct kvm_vcpu *vcpu)
{
301
	return vcpu->arch.efer & EFER_NX;
302 303
}

304 305
static int is_shadow_present_pte(u64 pte)
{
306
	return pte & PT_PRESENT_MASK && !is_mmio_spte(pte);
307 308
}

M
Marcelo Tosatti 已提交
309 310 311 312 313
static int is_large_pte(u64 pte)
{
	return pte & PT_PAGE_SIZE_MASK;
}

314
static int is_rmap_spte(u64 pte)
315
{
316
	return is_shadow_present_pte(pte);
317 318
}

319 320 321 322
static int is_last_spte(u64 pte, int level)
{
	if (level == PT_PAGE_TABLE_LEVEL)
		return 1;
323
	if (is_large_pte(pte))
324 325 326 327
		return 1;
	return 0;
}

328
static pfn_t spte_to_pfn(u64 pte)
329
{
330
	return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
331 332
}

333 334 335 336 337 338 339
static gfn_t pse36_gfn_delta(u32 gpte)
{
	int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;

	return (gpte & PT32_DIR_PSE36_MASK) << shift;
}

340
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
341
static void __set_spte(u64 *sptep, u64 spte)
342
{
343
	*sptep = spte;
344 345
}

346
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
347
{
348 349 350 351 352 353 354
	*sptep = spte;
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	return xchg(sptep, spte);
}
355 356 357 358 359

static u64 __get_spte_lockless(u64 *sptep)
{
	return ACCESS_ONCE(*sptep);
}
360 361 362 363 364 365

static bool __check_direct_spte_mmio_pf(u64 spte)
{
	/* It is valid if the spte is zapped. */
	return spte == 0ull;
}
366
#else
367 368 369 370 371 372 373
union split_spte {
	struct {
		u32 spte_low;
		u32 spte_high;
	};
	u64 spte;
};
374

375 376 377 378 379 380 381 382 383 384 385 386
static void count_spte_clear(u64 *sptep, u64 spte)
{
	struct kvm_mmu_page *sp =  page_header(__pa(sptep));

	if (is_shadow_present_pte(spte))
		return;

	/* Ensure the spte is completely set before we increase the count */
	smp_wmb();
	sp->clear_spte_count++;
}

387 388 389
static void __set_spte(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;
390

391 392 393 394 395 396 397 398 399 400 401 402 403
	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	ssptep->spte_high = sspte.spte_high;

	/*
	 * If we map the spte from nonpresent to present, We should store
	 * the high bits firstly, then set present bit, so cpu can not
	 * fetch this spte while we are setting the spte.
	 */
	smp_wmb();

	ssptep->spte_low = sspte.spte_low;
404 405
}

406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	ssptep->spte_low = sspte.spte_low;

	/*
	 * If we map the spte from present to nonpresent, we should clear
	 * present bit firstly to avoid vcpu fetch the old high bits.
	 */
	smp_wmb();

	ssptep->spte_high = sspte.spte_high;
422
	count_spte_clear(sptep, spte);
423 424 425 426 427 428 429 430 431 432 433
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte, orig;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	/* xchg acts as a barrier before the setting of the high bits */
	orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
434 435
	orig.spte_high = ssptep->spte_high;
	ssptep->spte_high = sspte.spte_high;
436
	count_spte_clear(sptep, spte);
437 438 439

	return orig.spte;
}
440 441 442 443

/*
 * The idea using the light way get the spte on x86_32 guest is from
 * gup_get_pte(arch/x86/mm/gup.c).
444 445 446 447 448 449 450 451 452 453 454 455 456 457
 *
 * An spte tlb flush may be pending, because kvm_set_pte_rmapp
 * coalesces them and we are running out of the MMU lock.  Therefore
 * we need to protect against in-progress updates of the spte.
 *
 * Reading the spte while an update is in progress may get the old value
 * for the high part of the spte.  The race is fine for a present->non-present
 * change (because the high part of the spte is ignored for non-present spte),
 * but for a present->present change we must reread the spte.
 *
 * All such changes are done in two steps (present->non-present and
 * non-present->present), hence it is enough to count the number of
 * present->non-present updates: if it changed while reading the spte,
 * we might have hit the race.  This is done using clear_spte_count.
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
 */
static u64 __get_spte_lockless(u64 *sptep)
{
	struct kvm_mmu_page *sp =  page_header(__pa(sptep));
	union split_spte spte, *orig = (union split_spte *)sptep;
	int count;

retry:
	count = sp->clear_spte_count;
	smp_rmb();

	spte.spte_low = orig->spte_low;
	smp_rmb();

	spte.spte_high = orig->spte_high;
	smp_rmb();

	if (unlikely(spte.spte_low != orig->spte_low ||
	      count != sp->clear_spte_count))
		goto retry;

	return spte.spte;
}
481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497

static bool __check_direct_spte_mmio_pf(u64 spte)
{
	union split_spte sspte = (union split_spte)spte;
	u32 high_mmio_mask = shadow_mmio_mask >> 32;

	/* It is valid if the spte is zapped. */
	if (spte == 0ull)
		return true;

	/* It is valid if the spte is being zapped. */
	if (sspte.spte_low == 0ull &&
	    (sspte.spte_high & high_mmio_mask) == high_mmio_mask)
		return true;

	return false;
}
498 499
#endif

500 501
static bool spte_is_locklessly_modifiable(u64 spte)
{
502 503
	return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
		(SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
504 505
}

506 507
static bool spte_has_volatile_bits(u64 spte)
{
508 509 510 511 512 513 514 515 516
	/*
	 * Always atomicly update spte if it can be updated
	 * out of mmu-lock, it can ensure dirty bit is not lost,
	 * also, it can help us to get a stable is_writable_pte()
	 * to ensure tlb flush is not missed.
	 */
	if (spte_is_locklessly_modifiable(spte))
		return true;

517 518 519 520 521 522
	if (!shadow_accessed_mask)
		return false;

	if (!is_shadow_present_pte(spte))
		return false;

523 524
	if ((spte & shadow_accessed_mask) &&
	      (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
525 526 527 528 529
		return false;

	return true;
}

530 531 532 533 534
static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
{
	return (old_spte & bit_mask) && !(new_spte & bit_mask);
}

535 536 537 538 539
static bool spte_is_bit_changed(u64 old_spte, u64 new_spte, u64 bit_mask)
{
	return (old_spte & bit_mask) != (new_spte & bit_mask);
}

540 541 542 543 544 545 546 547 548 549 550 551 552 553
/* Rules for using mmu_spte_set:
 * Set the sptep from nonpresent to present.
 * Note: the sptep being assigned *must* be either not present
 * or in a state where the hardware will not attempt to update
 * the spte.
 */
static void mmu_spte_set(u64 *sptep, u64 new_spte)
{
	WARN_ON(is_shadow_present_pte(*sptep));
	__set_spte(sptep, new_spte);
}

/* Rules for using mmu_spte_update:
 * Update the state bits, it means the mapped pfn is not changged.
554 555 556 557 558 559
 *
 * Whenever we overwrite a writable spte with a read-only one we
 * should flush remote TLBs. Otherwise rmap_write_protect
 * will find a read-only spte, even though the writable spte
 * might be cached on a CPU's TLB, the return value indicates this
 * case.
560
 */
561
static bool mmu_spte_update(u64 *sptep, u64 new_spte)
562
{
563
	u64 old_spte = *sptep;
564
	bool ret = false;
565 566

	WARN_ON(!is_rmap_spte(new_spte));
567

568 569 570 571
	if (!is_shadow_present_pte(old_spte)) {
		mmu_spte_set(sptep, new_spte);
		return ret;
	}
572

573
	if (!spte_has_volatile_bits(old_spte))
574
		__update_clear_spte_fast(sptep, new_spte);
575
	else
576
		old_spte = __update_clear_spte_slow(sptep, new_spte);
577

578 579 580 581 582
	/*
	 * For the spte updated out of mmu-lock is safe, since
	 * we always atomicly update it, see the comments in
	 * spte_has_volatile_bits().
	 */
583 584
	if (spte_is_locklessly_modifiable(old_spte) &&
	      !is_writable_pte(new_spte))
585 586
		ret = true;

587
	if (!shadow_accessed_mask)
588
		return ret;
589

590 591 592 593 594 595 596 597
	/*
	 * Flush TLB when accessed/dirty bits are changed in the page tables,
	 * to guarantee consistency between TLB and page tables.
	 */
	if (spte_is_bit_changed(old_spte, new_spte,
                                shadow_accessed_mask | shadow_dirty_mask))
		ret = true;

598 599 600 601
	if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
		kvm_set_pfn_accessed(spte_to_pfn(old_spte));
	if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
		kvm_set_pfn_dirty(spte_to_pfn(old_spte));
602 603

	return ret;
604 605
}

606 607 608 609 610 611 612 613 614 615 616
/*
 * Rules for using mmu_spte_clear_track_bits:
 * It sets the sptep from present to nonpresent, and track the
 * state bits, it is used to clear the last level sptep.
 */
static int mmu_spte_clear_track_bits(u64 *sptep)
{
	pfn_t pfn;
	u64 old_spte = *sptep;

	if (!spte_has_volatile_bits(old_spte))
617
		__update_clear_spte_fast(sptep, 0ull);
618
	else
619
		old_spte = __update_clear_spte_slow(sptep, 0ull);
620 621 622 623 624

	if (!is_rmap_spte(old_spte))
		return 0;

	pfn = spte_to_pfn(old_spte);
625 626 627 628 629 630

	/*
	 * KVM does not hold the refcount of the page used by
	 * kvm mmu, before reclaiming the page, we should
	 * unmap it from mmu first.
	 */
631
	WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
632

633 634 635 636 637 638 639 640 641 642 643 644 645 646
	if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
		kvm_set_pfn_accessed(pfn);
	if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
		kvm_set_pfn_dirty(pfn);
	return 1;
}

/*
 * Rules for using mmu_spte_clear_no_track:
 * Directly clear spte without caring the state bits of sptep,
 * it is used to set the upper level spte.
 */
static void mmu_spte_clear_no_track(u64 *sptep)
{
647
	__update_clear_spte_fast(sptep, 0ull);
648 649
}

650 651 652 653 654 655 656
static u64 mmu_spte_get_lockless(u64 *sptep)
{
	return __get_spte_lockless(sptep);
}

static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
{
657 658 659 660 661 662 663 664 665 666 667
	/*
	 * Prevent page table teardown by making any free-er wait during
	 * kvm_flush_remote_tlbs() IPI to all active vcpus.
	 */
	local_irq_disable();
	vcpu->mode = READING_SHADOW_PAGE_TABLES;
	/*
	 * Make sure a following spte read is not reordered ahead of the write
	 * to vcpu->mode.
	 */
	smp_mb();
668 669 670 671
}

static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
{
672 673 674 675 676 677 678 679
	/*
	 * Make sure the write to vcpu->mode is not reordered in front of
	 * reads to sptes.  If it does, kvm_commit_zap_page() can see us
	 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
	 */
	smp_mb();
	vcpu->mode = OUTSIDE_GUEST_MODE;
	local_irq_enable();
680 681
}

682
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
683
				  struct kmem_cache *base_cache, int min)
684 685 686 687
{
	void *obj;

	if (cache->nobjs >= min)
688
		return 0;
689
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
690
		obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
691
		if (!obj)
692
			return -ENOMEM;
693 694
		cache->objects[cache->nobjs++] = obj;
	}
695
	return 0;
696 697
}

698 699 700 701 702
static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
{
	return cache->nobjs;
}

703 704
static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
				  struct kmem_cache *cache)
705 706
{
	while (mc->nobjs)
707
		kmem_cache_free(cache, mc->objects[--mc->nobjs]);
708 709
}

A
Avi Kivity 已提交
710
static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
711
				       int min)
A
Avi Kivity 已提交
712
{
713
	void *page;
A
Avi Kivity 已提交
714 715 716 717

	if (cache->nobjs >= min)
		return 0;
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
718
		page = (void *)__get_free_page(GFP_KERNEL);
A
Avi Kivity 已提交
719 720
		if (!page)
			return -ENOMEM;
721
		cache->objects[cache->nobjs++] = page;
A
Avi Kivity 已提交
722 723 724 725 726 727 728
	}
	return 0;
}

static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
729
		free_page((unsigned long)mc->objects[--mc->nobjs]);
A
Avi Kivity 已提交
730 731
}

732
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
733
{
734 735
	int r;

736
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
737
				   pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
738 739
	if (r)
		goto out;
740
	r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
741 742
	if (r)
		goto out;
743
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
744
				   mmu_page_header_cache, 4);
745 746
out:
	return r;
747 748 749 750
}

static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
751 752
	mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
				pte_list_desc_cache);
753
	mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
754 755
	mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
				mmu_page_header_cache);
756 757
}

758
static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
759 760 761 762 763 764 765 766
{
	void *p;

	BUG_ON(!mc->nobjs);
	p = mc->objects[--mc->nobjs];
	return p;
}

767
static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
768
{
769
	return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
770 771
}

772
static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
773
{
774
	kmem_cache_free(pte_list_desc_cache, pte_list_desc);
775 776
}

777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792
static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
{
	if (!sp->role.direct)
		return sp->gfns[index];

	return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
}

static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
{
	if (sp->role.direct)
		BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
	else
		sp->gfns[index] = gfn;
}

M
Marcelo Tosatti 已提交
793
/*
794 795
 * Return the pointer to the large page information for a given gfn,
 * handling slots that are not large page aligned.
M
Marcelo Tosatti 已提交
796
 */
797 798 799
static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
					      struct kvm_memory_slot *slot,
					      int level)
M
Marcelo Tosatti 已提交
800 801 802
{
	unsigned long idx;

803
	idx = gfn_to_index(gfn, slot->base_gfn, level);
804
	return &slot->arch.lpage_info[level - 2][idx];
M
Marcelo Tosatti 已提交
805 806 807 808
}

static void account_shadowed(struct kvm *kvm, gfn_t gfn)
{
809
	struct kvm_memory_slot *slot;
810
	struct kvm_lpage_info *linfo;
811
	int i;
M
Marcelo Tosatti 已提交
812

A
Avi Kivity 已提交
813
	slot = gfn_to_memslot(kvm, gfn);
814 815
	for (i = PT_DIRECTORY_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
816 817
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->write_count += 1;
818
	}
819
	kvm->arch.indirect_shadow_pages++;
M
Marcelo Tosatti 已提交
820 821 822 823
}

static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
{
824
	struct kvm_memory_slot *slot;
825
	struct kvm_lpage_info *linfo;
826
	int i;
M
Marcelo Tosatti 已提交
827

A
Avi Kivity 已提交
828
	slot = gfn_to_memslot(kvm, gfn);
829 830
	for (i = PT_DIRECTORY_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
831 832 833
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->write_count -= 1;
		WARN_ON(linfo->write_count < 0);
834
	}
835
	kvm->arch.indirect_shadow_pages--;
M
Marcelo Tosatti 已提交
836 837
}

838 839 840
static int has_wrprotected_page(struct kvm *kvm,
				gfn_t gfn,
				int level)
M
Marcelo Tosatti 已提交
841
{
842
	struct kvm_memory_slot *slot;
843
	struct kvm_lpage_info *linfo;
M
Marcelo Tosatti 已提交
844

A
Avi Kivity 已提交
845
	slot = gfn_to_memslot(kvm, gfn);
M
Marcelo Tosatti 已提交
846
	if (slot) {
847 848
		linfo = lpage_info_slot(gfn, slot, level);
		return linfo->write_count;
M
Marcelo Tosatti 已提交
849 850 851 852 853
	}

	return 1;
}

854
static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
M
Marcelo Tosatti 已提交
855
{
J
Joerg Roedel 已提交
856
	unsigned long page_size;
857
	int i, ret = 0;
M
Marcelo Tosatti 已提交
858

J
Joerg Roedel 已提交
859
	page_size = kvm_host_page_size(kvm, gfn);
M
Marcelo Tosatti 已提交
860

861 862 863 864 865 866 867 868
	for (i = PT_PAGE_TABLE_LEVEL;
	     i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
		if (page_size >= KVM_HPAGE_SIZE(i))
			ret = i;
		else
			break;
	}

869
	return ret;
M
Marcelo Tosatti 已提交
870 871
}

872 873 874
static struct kvm_memory_slot *
gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
			    bool no_dirty_log)
M
Marcelo Tosatti 已提交
875 876
{
	struct kvm_memory_slot *slot;
877 878 879 880 881 882 883 884 885 886 887

	slot = gfn_to_memslot(vcpu->kvm, gfn);
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
	      (no_dirty_log && slot->dirty_bitmap))
		slot = NULL;

	return slot;
}

static bool mapping_level_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t large_gfn)
{
888
	return !gfn_to_memslot_dirty_bitmap(vcpu, large_gfn, true);
889 890 891 892 893
}

static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
{
	int host_level, level, max_level;
M
Marcelo Tosatti 已提交
894

895 896 897 898 899
	host_level = host_mapping_level(vcpu->kvm, large_gfn);

	if (host_level == PT_PAGE_TABLE_LEVEL)
		return host_level;

X
Xiao Guangrong 已提交
900
	max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
901 902

	for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
903 904 905 906
		if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
			break;

	return level - 1;
M
Marcelo Tosatti 已提交
907 908
}

909
/*
910
 * Pte mapping structures:
911
 *
912
 * If pte_list bit zero is zero, then pte_list point to the spte.
913
 *
914 915
 * If pte_list bit zero is one, (then pte_list & ~1) points to a struct
 * pte_list_desc containing more mappings.
916
 *
917
 * Returns the number of pte entries before the spte was added or zero if
918 919
 * the spte was not added.
 *
920
 */
921 922
static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
			unsigned long *pte_list)
923
{
924
	struct pte_list_desc *desc;
925
	int i, count = 0;
926

927 928 929 930 931 932 933
	if (!*pte_list) {
		rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
		*pte_list = (unsigned long)spte;
	} else if (!(*pte_list & 1)) {
		rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
		desc = mmu_alloc_pte_list_desc(vcpu);
		desc->sptes[0] = (u64 *)*pte_list;
A
Avi Kivity 已提交
934
		desc->sptes[1] = spte;
935
		*pte_list = (unsigned long)desc | 1;
936
		++count;
937
	} else {
938 939 940
		rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
		desc = (struct pte_list_desc *)(*pte_list & ~1ul);
		while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
941
			desc = desc->more;
942
			count += PTE_LIST_EXT;
943
		}
944 945
		if (desc->sptes[PTE_LIST_EXT-1]) {
			desc->more = mmu_alloc_pte_list_desc(vcpu);
946 947
			desc = desc->more;
		}
A
Avi Kivity 已提交
948
		for (i = 0; desc->sptes[i]; ++i)
949
			++count;
A
Avi Kivity 已提交
950
		desc->sptes[i] = spte;
951
	}
952
	return count;
953 954
}

955 956 957
static void
pte_list_desc_remove_entry(unsigned long *pte_list, struct pte_list_desc *desc,
			   int i, struct pte_list_desc *prev_desc)
958 959 960
{
	int j;

961
	for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
962
		;
A
Avi Kivity 已提交
963 964
	desc->sptes[i] = desc->sptes[j];
	desc->sptes[j] = NULL;
965 966 967
	if (j != 0)
		return;
	if (!prev_desc && !desc->more)
968
		*pte_list = (unsigned long)desc->sptes[0];
969 970 971 972
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
973 974
			*pte_list = (unsigned long)desc->more | 1;
	mmu_free_pte_list_desc(desc);
975 976
}

977
static void pte_list_remove(u64 *spte, unsigned long *pte_list)
978
{
979 980
	struct pte_list_desc *desc;
	struct pte_list_desc *prev_desc;
981 982
	int i;

983 984
	if (!*pte_list) {
		printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
985
		BUG();
986 987 988 989
	} else if (!(*pte_list & 1)) {
		rmap_printk("pte_list_remove:  %p 1->0\n", spte);
		if ((u64 *)*pte_list != spte) {
			printk(KERN_ERR "pte_list_remove:  %p 1->BUG\n", spte);
990 991
			BUG();
		}
992
		*pte_list = 0;
993
	} else {
994 995
		rmap_printk("pte_list_remove:  %p many->many\n", spte);
		desc = (struct pte_list_desc *)(*pte_list & ~1ul);
996 997
		prev_desc = NULL;
		while (desc) {
998
			for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
A
Avi Kivity 已提交
999
				if (desc->sptes[i] == spte) {
1000
					pte_list_desc_remove_entry(pte_list,
1001
							       desc, i,
1002 1003 1004 1005 1006 1007
							       prev_desc);
					return;
				}
			prev_desc = desc;
			desc = desc->more;
		}
1008
		pr_err("pte_list_remove: %p many->many\n", spte);
1009 1010 1011 1012
		BUG();
	}
}

1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
typedef void (*pte_list_walk_fn) (u64 *spte);
static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
{
	struct pte_list_desc *desc;
	int i;

	if (!*pte_list)
		return;

	if (!(*pte_list & 1))
		return fn((u64 *)*pte_list);

	desc = (struct pte_list_desc *)(*pte_list & ~1ul);
	while (desc) {
		for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
			fn(desc->sptes[i]);
		desc = desc->more;
	}
}

1033
static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,
1034
				    struct kvm_memory_slot *slot)
1035
{
1036
	unsigned long idx;
1037

1038
	idx = gfn_to_index(gfn, slot->base_gfn, level);
1039
	return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
1040 1041
}

1042 1043 1044 1045 1046 1047 1048 1049
/*
 * Take gfn and return the reverse mapping to it.
 */
static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
{
	struct kvm_memory_slot *slot;

	slot = gfn_to_memslot(kvm, gfn);
1050
	return __gfn_to_rmap(gfn, level, slot);
1051 1052
}

1053 1054 1055 1056 1057 1058 1059 1060
static bool rmap_can_add(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_memory_cache *cache;

	cache = &vcpu->arch.mmu_pte_list_desc_cache;
	return mmu_memory_cache_free_objects(cache);
}

1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
{
	struct kvm_mmu_page *sp;
	unsigned long *rmapp;

	sp = page_header(__pa(spte));
	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
	return pte_list_add(vcpu, spte, rmapp);
}

static void rmap_remove(struct kvm *kvm, u64 *spte)
{
	struct kvm_mmu_page *sp;
	gfn_t gfn;
	unsigned long *rmapp;

	sp = page_header(__pa(spte));
	gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
	rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
	pte_list_remove(spte, rmapp);
}

1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
/*
 * Used by the following functions to iterate through the sptes linked by a
 * rmap.  All fields are private and not assumed to be used outside.
 */
struct rmap_iterator {
	/* private fields */
	struct pte_list_desc *desc;	/* holds the sptep if not NULL */
	int pos;			/* index of the sptep */
};

/*
 * Iteration must be started by this function.  This should also be used after
 * removing/dropping sptes from the rmap link because in such cases the
 * information in the itererator may not be valid.
 *
 * Returns sptep if found, NULL otherwise.
 */
static u64 *rmap_get_first(unsigned long rmap, struct rmap_iterator *iter)
{
	if (!rmap)
		return NULL;

	if (!(rmap & 1)) {
		iter->desc = NULL;
		return (u64 *)rmap;
	}

	iter->desc = (struct pte_list_desc *)(rmap & ~1ul);
	iter->pos = 0;
	return iter->desc->sptes[iter->pos];
}

/*
 * Must be used with a valid iterator: e.g. after rmap_get_first().
 *
 * Returns sptep if found, NULL otherwise.
 */
static u64 *rmap_get_next(struct rmap_iterator *iter)
{
	if (iter->desc) {
		if (iter->pos < PTE_LIST_EXT - 1) {
			u64 *sptep;

			++iter->pos;
			sptep = iter->desc->sptes[iter->pos];
			if (sptep)
				return sptep;
		}

		iter->desc = iter->desc->more;

		if (iter->desc) {
			iter->pos = 0;
			/* desc->sptes[0] cannot be NULL */
			return iter->desc->sptes[iter->pos];
		}
	}

	return NULL;
}

1145
static void drop_spte(struct kvm *kvm, u64 *sptep)
1146
{
1147
	if (mmu_spte_clear_track_bits(sptep))
1148
		rmap_remove(kvm, sptep);
A
Avi Kivity 已提交
1149 1150
}

1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171

static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
{
	if (is_large_pte(*sptep)) {
		WARN_ON(page_header(__pa(sptep))->role.level ==
			PT_PAGE_TABLE_LEVEL);
		drop_spte(kvm, sptep);
		--kvm->stat.lpages;
		return true;
	}

	return false;
}

static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
{
	if (__drop_large_spte(vcpu->kvm, sptep))
		kvm_flush_remote_tlbs(vcpu->kvm);
}

/*
1172
 * Write-protect on the specified @sptep, @pt_protect indicates whether
1173
 * spte write-protection is caused by protecting shadow page table.
1174
 *
T
Tiejun Chen 已提交
1175
 * Note: write protection is difference between dirty logging and spte
1176 1177 1178 1179 1180
 * protection:
 * - for dirty logging, the spte can be set to writable at anytime if
 *   its dirty bitmap is properly set.
 * - for spte protection, the spte can be writable only after unsync-ing
 *   shadow page.
1181
 *
1182
 * Return true if tlb need be flushed.
1183
 */
1184
static bool spte_write_protect(struct kvm *kvm, u64 *sptep, bool pt_protect)
1185 1186 1187
{
	u64 spte = *sptep;

1188 1189
	if (!is_writable_pte(spte) &&
	      !(pt_protect && spte_is_locklessly_modifiable(spte)))
1190 1191 1192 1193
		return false;

	rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);

1194 1195
	if (pt_protect)
		spte &= ~SPTE_MMU_WRITEABLE;
1196
	spte = spte & ~PT_WRITABLE_MASK;
1197

1198
	return mmu_spte_update(sptep, spte);
1199 1200
}

1201
static bool __rmap_write_protect(struct kvm *kvm, unsigned long *rmapp,
1202
				 bool pt_protect)
1203
{
1204 1205
	u64 *sptep;
	struct rmap_iterator iter;
1206
	bool flush = false;
1207

1208 1209
	for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
		BUG_ON(!(*sptep & PT_PRESENT_MASK));
1210

1211
		flush |= spte_write_protect(kvm, sptep, pt_protect);
1212
		sptep = rmap_get_next(&iter);
1213
	}
1214

1215
	return flush;
1216 1217
}

1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
static bool spte_clear_dirty(struct kvm *kvm, u64 *sptep)
{
	u64 spte = *sptep;

	rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);

	spte &= ~shadow_dirty_mask;

	return mmu_spte_update(sptep, spte);
}

static bool __rmap_clear_dirty(struct kvm *kvm, unsigned long *rmapp)
{
	u64 *sptep;
	struct rmap_iterator iter;
	bool flush = false;

	for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
		BUG_ON(!(*sptep & PT_PRESENT_MASK));

		flush |= spte_clear_dirty(kvm, sptep);
		sptep = rmap_get_next(&iter);
	}

	return flush;
}

static bool spte_set_dirty(struct kvm *kvm, u64 *sptep)
{
	u64 spte = *sptep;

	rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);

	spte |= shadow_dirty_mask;

	return mmu_spte_update(sptep, spte);
}

static bool __rmap_set_dirty(struct kvm *kvm, unsigned long *rmapp)
{
	u64 *sptep;
	struct rmap_iterator iter;
	bool flush = false;

	for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
		BUG_ON(!(*sptep & PT_PRESENT_MASK));

		flush |= spte_set_dirty(kvm, sptep);
		sptep = rmap_get_next(&iter);
	}

	return flush;
}

1272
/**
1273
 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1274 1275 1276 1277 1278 1279 1280 1281
 * @kvm: kvm instance
 * @slot: slot to protect
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should protect
 *
 * Used when we do not need to care about huge page mappings: e.g. during dirty
 * logging we do not have any such mappings.
 */
1282
static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1283 1284
				     struct kvm_memory_slot *slot,
				     gfn_t gfn_offset, unsigned long mask)
1285 1286 1287
{
	unsigned long *rmapp;

1288
	while (mask) {
1289 1290
		rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
				      PT_PAGE_TABLE_LEVEL, slot);
1291
		__rmap_write_protect(kvm, rmapp, false);
M
Marcelo Tosatti 已提交
1292

1293 1294 1295
		/* clear the first set bit */
		mask &= mask - 1;
	}
1296 1297
}

1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
/**
 * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages
 * @kvm: kvm instance
 * @slot: slot to clear D-bit
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should clear D-bit
 *
 * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
 */
void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
				     struct kvm_memory_slot *slot,
				     gfn_t gfn_offset, unsigned long mask)
{
	unsigned long *rmapp;

	while (mask) {
		rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
				      PT_PAGE_TABLE_LEVEL, slot);
		__rmap_clear_dirty(kvm, rmapp);

		/* clear the first set bit */
		mask &= mask - 1;
	}
}
EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked);

1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
/**
 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
 * PT level pages.
 *
 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
 * enable dirty logging for them.
 *
 * Used when we do not need to care about huge page mappings: e.g. during dirty
 * logging we do not have any such mappings.
 */
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
				struct kvm_memory_slot *slot,
				gfn_t gfn_offset, unsigned long mask)
{
1338 1339 1340 1341 1342
	if (kvm_x86_ops->enable_log_dirty_pt_masked)
		kvm_x86_ops->enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
				mask);
	else
		kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1343 1344
}

1345
static bool rmap_write_protect(struct kvm *kvm, u64 gfn)
1346 1347
{
	struct kvm_memory_slot *slot;
1348 1349
	unsigned long *rmapp;
	int i;
1350
	bool write_protected = false;
1351 1352

	slot = gfn_to_memslot(kvm, gfn);
1353 1354 1355 1356

	for (i = PT_PAGE_TABLE_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
		rmapp = __gfn_to_rmap(gfn, i, slot);
1357
		write_protected |= __rmap_write_protect(kvm, rmapp, true);
1358 1359 1360
	}

	return write_protected;
1361 1362
}

F
Frederik Deweerdt 已提交
1363
static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
1364 1365
			   struct kvm_memory_slot *slot, gfn_t gfn, int level,
			   unsigned long data)
1366
{
1367 1368
	u64 *sptep;
	struct rmap_iterator iter;
1369 1370
	int need_tlb_flush = 0;

1371 1372
	while ((sptep = rmap_get_first(*rmapp, &iter))) {
		BUG_ON(!(*sptep & PT_PRESENT_MASK));
1373 1374
		rmap_printk("kvm_rmap_unmap_hva: spte %p %llx gfn %llx (%d)\n",
			     sptep, *sptep, gfn, level);
1375 1376

		drop_spte(kvm, sptep);
1377 1378
		need_tlb_flush = 1;
	}
1379

1380 1381 1382
	return need_tlb_flush;
}

F
Frederik Deweerdt 已提交
1383
static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
1384 1385
			     struct kvm_memory_slot *slot, gfn_t gfn, int level,
			     unsigned long data)
1386
{
1387 1388
	u64 *sptep;
	struct rmap_iterator iter;
1389
	int need_flush = 0;
1390
	u64 new_spte;
1391 1392 1393 1394 1395
	pte_t *ptep = (pte_t *)data;
	pfn_t new_pfn;

	WARN_ON(pte_huge(*ptep));
	new_pfn = pte_pfn(*ptep);
1396 1397 1398

	for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
		BUG_ON(!is_shadow_present_pte(*sptep));
1399 1400
		rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
			     sptep, *sptep, gfn, level);
1401

1402
		need_flush = 1;
1403

1404
		if (pte_write(*ptep)) {
1405 1406
			drop_spte(kvm, sptep);
			sptep = rmap_get_first(*rmapp, &iter);
1407
		} else {
1408
			new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
1409 1410 1411 1412
			new_spte |= (u64)new_pfn << PAGE_SHIFT;

			new_spte &= ~PT_WRITABLE_MASK;
			new_spte &= ~SPTE_HOST_WRITEABLE;
1413
			new_spte &= ~shadow_accessed_mask;
1414 1415 1416 1417

			mmu_spte_clear_track_bits(sptep);
			mmu_spte_set(sptep, new_spte);
			sptep = rmap_get_next(&iter);
1418 1419
		}
	}
1420

1421 1422 1423 1424 1425 1426
	if (need_flush)
		kvm_flush_remote_tlbs(kvm);

	return 0;
}

1427 1428 1429 1430 1431 1432
static int kvm_handle_hva_range(struct kvm *kvm,
				unsigned long start,
				unsigned long end,
				unsigned long data,
				int (*handler)(struct kvm *kvm,
					       unsigned long *rmapp,
1433
					       struct kvm_memory_slot *slot,
1434 1435
					       gfn_t gfn,
					       int level,
1436
					       unsigned long data))
1437
{
1438
	int j;
1439
	int ret = 0;
1440
	struct kvm_memslots *slots;
1441
	struct kvm_memory_slot *memslot;
1442

1443
	slots = kvm_memslots(kvm);
1444

1445
	kvm_for_each_memslot(memslot, slots) {
1446
		unsigned long hva_start, hva_end;
1447
		gfn_t gfn_start, gfn_end;
1448

1449 1450 1451 1452 1453 1454 1455
		hva_start = max(start, memslot->userspace_addr);
		hva_end = min(end, memslot->userspace_addr +
					(memslot->npages << PAGE_SHIFT));
		if (hva_start >= hva_end)
			continue;
		/*
		 * {gfn(page) | page intersects with [hva_start, hva_end)} =
1456
		 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
1457
		 */
1458
		gfn_start = hva_to_gfn_memslot(hva_start, memslot);
1459
		gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
1460

1461 1462 1463 1464
		for (j = PT_PAGE_TABLE_LEVEL;
		     j < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++j) {
			unsigned long idx, idx_end;
			unsigned long *rmapp;
1465
			gfn_t gfn = gfn_start;
1466

1467 1468 1469 1470 1471 1472
			/*
			 * {idx(page_j) | page_j intersects with
			 *  [hva_start, hva_end)} = {idx, idx+1, ..., idx_end}.
			 */
			idx = gfn_to_index(gfn_start, memslot->base_gfn, j);
			idx_end = gfn_to_index(gfn_end - 1, memslot->base_gfn, j);
1473

1474
			rmapp = __gfn_to_rmap(gfn_start, j, memslot);
1475

1476 1477 1478 1479
			for (; idx <= idx_end;
			       ++idx, gfn += (1UL << KVM_HPAGE_GFN_SHIFT(j)))
				ret |= handler(kvm, rmapp++, memslot,
					       gfn, j, data);
1480 1481 1482
		}
	}

1483
	return ret;
1484 1485
}

1486 1487 1488
static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
			  unsigned long data,
			  int (*handler)(struct kvm *kvm, unsigned long *rmapp,
1489
					 struct kvm_memory_slot *slot,
1490
					 gfn_t gfn, int level,
1491 1492 1493
					 unsigned long data))
{
	return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1494 1495 1496 1497
}

int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
{
1498 1499 1500
	return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
}

1501 1502 1503 1504 1505
int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
{
	return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
}

1506 1507
void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
{
F
Frederik Deweerdt 已提交
1508
	kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1509 1510
}

F
Frederik Deweerdt 已提交
1511
static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1512 1513
			 struct kvm_memory_slot *slot, gfn_t gfn, int level,
			 unsigned long data)
1514
{
1515
	u64 *sptep;
1516
	struct rmap_iterator uninitialized_var(iter);
1517 1518
	int young = 0;

A
Andres Lagar-Cavilla 已提交
1519
	BUG_ON(!shadow_accessed_mask);
1520

1521 1522
	for (sptep = rmap_get_first(*rmapp, &iter); sptep;
	     sptep = rmap_get_next(&iter)) {
1523
		BUG_ON(!is_shadow_present_pte(*sptep));
1524

1525
		if (*sptep & shadow_accessed_mask) {
1526
			young = 1;
1527 1528
			clear_bit((ffs(shadow_accessed_mask) - 1),
				 (unsigned long *)sptep);
1529 1530
		}
	}
1531
	trace_kvm_age_page(gfn, level, slot, young);
1532 1533 1534
	return young;
}

A
Andrea Arcangeli 已提交
1535
static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1536 1537
			      struct kvm_memory_slot *slot, gfn_t gfn,
			      int level, unsigned long data)
A
Andrea Arcangeli 已提交
1538
{
1539 1540
	u64 *sptep;
	struct rmap_iterator iter;
A
Andrea Arcangeli 已提交
1541 1542 1543 1544 1545 1546 1547 1548 1549 1550
	int young = 0;

	/*
	 * If there's no access bit in the secondary pte set by the
	 * hardware it's up to gup-fast/gup to set the access bit in
	 * the primary pte or in the page structure.
	 */
	if (!shadow_accessed_mask)
		goto out;

1551 1552
	for (sptep = rmap_get_first(*rmapp, &iter); sptep;
	     sptep = rmap_get_next(&iter)) {
1553
		BUG_ON(!is_shadow_present_pte(*sptep));
1554

1555
		if (*sptep & shadow_accessed_mask) {
A
Andrea Arcangeli 已提交
1556 1557 1558 1559 1560 1561 1562 1563
			young = 1;
			break;
		}
	}
out:
	return young;
}

1564 1565
#define RMAP_RECYCLE_THRESHOLD 1000

1566
static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1567 1568
{
	unsigned long *rmapp;
1569 1570 1571
	struct kvm_mmu_page *sp;

	sp = page_header(__pa(spte));
1572

1573
	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
1574

1575
	kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, gfn, sp->role.level, 0);
1576 1577 1578
	kvm_flush_remote_tlbs(vcpu->kvm);
}

A
Andres Lagar-Cavilla 已提交
1579
int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1580
{
A
Andres Lagar-Cavilla 已提交
1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601
	/*
	 * In case of absence of EPT Access and Dirty Bits supports,
	 * emulate the accessed bit for EPT, by checking if this page has
	 * an EPT mapping, and clearing it if it does. On the next access,
	 * a new EPT mapping will be established.
	 * This has some overhead, but not as much as the cost of swapping
	 * out actively used pages or breaking up actively used hugepages.
	 */
	if (!shadow_accessed_mask) {
		/*
		 * We are holding the kvm->mmu_lock, and we are blowing up
		 * shadow PTEs. MMU notifier consumers need to be kept at bay.
		 * This is correct as long as we don't decouple the mmu_lock
		 * protected regions (like invalidate_range_start|end does).
		 */
		kvm->mmu_notifier_seq++;
		return kvm_handle_hva_range(kvm, start, end, 0,
					    kvm_unmap_rmapp);
	}

	return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
1602 1603
}

A
Andrea Arcangeli 已提交
1604 1605 1606 1607 1608
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
{
	return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
}

1609
#ifdef MMU_DEBUG
1610
static int is_empty_shadow_page(u64 *spt)
A
Avi Kivity 已提交
1611
{
1612 1613 1614
	u64 *pos;
	u64 *end;

1615
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1616
		if (is_shadow_present_pte(*pos)) {
1617
			printk(KERN_ERR "%s: %p %llx\n", __func__,
1618
			       pos, *pos);
A
Avi Kivity 已提交
1619
			return 0;
1620
		}
A
Avi Kivity 已提交
1621 1622
	return 1;
}
1623
#endif
A
Avi Kivity 已提交
1624

1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636
/*
 * This value is the sum of all of the kvm instances's
 * kvm->arch.n_used_mmu_pages values.  We need a global,
 * aggregate version in order to make the slab shrinker
 * faster
 */
static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
{
	kvm->arch.n_used_mmu_pages += nr;
	percpu_counter_add(&kvm_total_used_mmu_pages, nr);
}

1637
static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1638
{
1639
	MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
1640
	hlist_del(&sp->hash_link);
1641 1642
	list_del(&sp->link);
	free_page((unsigned long)sp->spt);
1643 1644
	if (!sp->role.direct)
		free_page((unsigned long)sp->gfns);
1645
	kmem_cache_free(mmu_page_header_cache, sp);
1646 1647
}

1648 1649
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
1650
	return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
1651 1652
}

1653
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1654
				    struct kvm_mmu_page *sp, u64 *parent_pte)
1655 1656 1657 1658
{
	if (!parent_pte)
		return;

1659
	pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1660 1661
}

1662
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1663 1664
				       u64 *parent_pte)
{
1665
	pte_list_remove(parent_pte, &sp->parent_ptes);
1666 1667
}

1668 1669 1670 1671
static void drop_parent_pte(struct kvm_mmu_page *sp,
			    u64 *parent_pte)
{
	mmu_page_remove_parent_pte(sp, parent_pte);
1672
	mmu_spte_clear_no_track(parent_pte);
1673 1674
}

1675 1676
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
					       u64 *parent_pte, int direct)
M
Marcelo Tosatti 已提交
1677
{
1678
	struct kvm_mmu_page *sp;
1679

1680 1681
	sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
	sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1682
	if (!direct)
1683
		sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1684
	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1685 1686 1687 1688 1689 1690

	/*
	 * The active_mmu_pages list is the FIFO list, do not move the
	 * page until it is zapped. kvm_zap_obsolete_pages depends on
	 * this feature. See the comments in kvm_zap_obsolete_pages().
	 */
1691 1692 1693 1694 1695
	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
	sp->parent_ptes = 0;
	mmu_page_add_parent_pte(vcpu, sp, parent_pte);
	kvm_mod_used_mmu_pages(vcpu->kvm, +1);
	return sp;
M
Marcelo Tosatti 已提交
1696 1697
}

1698
static void mark_unsync(u64 *spte);
1699
static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1700
{
1701
	pte_list_walk(&sp->parent_ptes, mark_unsync);
1702 1703
}

1704
static void mark_unsync(u64 *spte)
1705
{
1706
	struct kvm_mmu_page *sp;
1707
	unsigned int index;
1708

1709
	sp = page_header(__pa(spte));
1710 1711
	index = spte - sp->spt;
	if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1712
		return;
1713
	if (sp->unsync_children++)
1714
		return;
1715
	kvm_mmu_mark_parents_unsync(sp);
1716 1717
}

1718
static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1719
			       struct kvm_mmu_page *sp)
1720 1721 1722 1723
{
	return 1;
}

M
Marcelo Tosatti 已提交
1724 1725 1726 1727
static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
}

1728 1729
static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
				 struct kvm_mmu_page *sp, u64 *spte,
1730
				 const void *pte)
1731 1732 1733 1734
{
	WARN_ON(1);
}

1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
#define KVM_PAGE_ARRAY_NR 16

struct kvm_mmu_pages {
	struct mmu_page_and_offset {
		struct kvm_mmu_page *sp;
		unsigned int idx;
	} page[KVM_PAGE_ARRAY_NR];
	unsigned int nr;
};

1745 1746
static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
			 int idx)
1747
{
1748
	int i;
1749

1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764
	if (sp->unsync)
		for (i=0; i < pvec->nr; i++)
			if (pvec->page[i].sp == sp)
				return 0;

	pvec->page[pvec->nr].sp = sp;
	pvec->page[pvec->nr].idx = idx;
	pvec->nr++;
	return (pvec->nr == KVM_PAGE_ARRAY_NR);
}

static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	int i, ret, nr_unsync_leaf = 0;
1765

1766
	for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
1767
		struct kvm_mmu_page *child;
1768 1769
		u64 ent = sp->spt[i];

1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798
		if (!is_shadow_present_pte(ent) || is_large_pte(ent))
			goto clear_child_bitmap;

		child = page_header(ent & PT64_BASE_ADDR_MASK);

		if (child->unsync_children) {
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;

			ret = __mmu_unsync_walk(child, pvec);
			if (!ret)
				goto clear_child_bitmap;
			else if (ret > 0)
				nr_unsync_leaf += ret;
			else
				return ret;
		} else if (child->unsync) {
			nr_unsync_leaf++;
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;
		} else
			 goto clear_child_bitmap;

		continue;

clear_child_bitmap:
		__clear_bit(i, sp->unsync_child_bitmap);
		sp->unsync_children--;
		WARN_ON((int)sp->unsync_children < 0);
1799 1800 1801
	}


1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812
	return nr_unsync_leaf;
}

static int mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	if (!sp->unsync_children)
		return 0;

	mmu_pages_add(pvec, sp, 0);
	return __mmu_unsync_walk(sp, pvec);
1813 1814 1815 1816 1817
}

static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	WARN_ON(!sp->unsync);
1818
	trace_kvm_mmu_sync_page(sp);
1819 1820 1821 1822
	sp->unsync = 0;
	--kvm->stat.mmu_unsync;
}

1823 1824 1825 1826
static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				    struct list_head *invalid_list);
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list);
1827

1828 1829 1830 1831 1832 1833 1834 1835 1836 1837
/*
 * NOTE: we should pay more attention on the zapped-obsolete page
 * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
 * since it has been deleted from active_mmu_pages but still can be found
 * at hast list.
 *
 * for_each_gfn_indirect_valid_sp has skipped that kind of page and
 * kvm_mmu_get_page(), the only user of for_each_gfn_sp(), has skipped
 * all the obsolete pages.
 */
1838 1839 1840 1841 1842 1843 1844 1845
#define for_each_gfn_sp(_kvm, _sp, _gfn)				\
	hlist_for_each_entry(_sp,					\
	  &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
		if ((_sp)->gfn != (_gfn)) {} else

#define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn)			\
	for_each_gfn_sp(_kvm, _sp, _gfn)				\
		if ((_sp)->role.direct || (_sp)->role.invalid) {} else
1846

1847
/* @sp->gfn should be write-protected at the call site */
1848
static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1849
			   struct list_head *invalid_list, bool clear_unsync)
1850
{
1851
	if (sp->role.cr4_pae != !!is_pae(vcpu)) {
1852
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1853 1854 1855
		return 1;
	}

1856
	if (clear_unsync)
1857 1858
		kvm_unlink_unsync_page(vcpu->kvm, sp);

1859
	if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
1860
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1861 1862 1863
		return 1;
	}

1864
	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1865 1866 1867
	return 0;
}

1868 1869 1870
static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
				   struct kvm_mmu_page *sp)
{
1871
	LIST_HEAD(invalid_list);
1872 1873
	int ret;

1874
	ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
1875
	if (ret)
1876 1877
		kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);

1878 1879 1880
	return ret;
}

1881 1882 1883 1884 1885 1886 1887
#ifdef CONFIG_KVM_MMU_AUDIT
#include "mmu_audit.c"
#else
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
static void mmu_audit_disable(void) { }
#endif

1888 1889
static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
			 struct list_head *invalid_list)
1890
{
1891
	return __kvm_sync_page(vcpu, sp, invalid_list, true);
1892 1893
}

1894 1895 1896 1897
/* @gfn should be write-protected at the call site */
static void kvm_sync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
{
	struct kvm_mmu_page *s;
1898
	LIST_HEAD(invalid_list);
1899 1900
	bool flush = false;

1901
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
1902
		if (!s->unsync)
1903 1904 1905
			continue;

		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1906
		kvm_unlink_unsync_page(vcpu->kvm, s);
1907
		if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
1908
			(vcpu->arch.mmu.sync_page(vcpu, s))) {
1909
			kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
1910 1911 1912 1913 1914
			continue;
		}
		flush = true;
	}

1915
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1916
	if (flush)
1917
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1918 1919
}

1920 1921 1922
struct mmu_page_path {
	struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
	unsigned int idx[PT64_ROOT_LEVEL-1];
1923 1924
};

1925 1926 1927 1928 1929 1930
#define for_each_sp(pvec, sp, parents, i)			\
		for (i = mmu_pages_next(&pvec, &parents, -1),	\
			sp = pvec.page[i].sp;			\
			i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});	\
			i = mmu_pages_next(&pvec, &parents, i))

1931 1932 1933
static int mmu_pages_next(struct kvm_mmu_pages *pvec,
			  struct mmu_page_path *parents,
			  int i)
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951
{
	int n;

	for (n = i+1; n < pvec->nr; n++) {
		struct kvm_mmu_page *sp = pvec->page[n].sp;

		if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
			parents->idx[0] = pvec->page[n].idx;
			return n;
		}

		parents->parent[sp->role.level-2] = sp;
		parents->idx[sp->role.level-1] = pvec->page[n].idx;
	}

	return n;
}

1952
static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1953
{
1954 1955 1956 1957 1958
	struct kvm_mmu_page *sp;
	unsigned int level = 0;

	do {
		unsigned int idx = parents->idx[level];
1959

1960 1961 1962 1963 1964 1965 1966 1967 1968
		sp = parents->parent[level];
		if (!sp)
			return;

		--sp->unsync_children;
		WARN_ON((int)sp->unsync_children < 0);
		__clear_bit(idx, sp->unsync_child_bitmap);
		level++;
	} while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1969 1970
}

1971 1972 1973
static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
			       struct mmu_page_path *parents,
			       struct kvm_mmu_pages *pvec)
1974
{
1975 1976 1977
	parents->parent[parent->role.level-1] = NULL;
	pvec->nr = 0;
}
1978

1979 1980 1981 1982 1983 1984 1985
static void mmu_sync_children(struct kvm_vcpu *vcpu,
			      struct kvm_mmu_page *parent)
{
	int i;
	struct kvm_mmu_page *sp;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
1986
	LIST_HEAD(invalid_list);
1987 1988 1989

	kvm_mmu_pages_init(parent, &parents, &pages);
	while (mmu_unsync_walk(parent, &pages)) {
1990
		bool protected = false;
1991 1992 1993 1994 1995 1996 1997

		for_each_sp(pages, sp, parents, i)
			protected |= rmap_write_protect(vcpu->kvm, sp->gfn);

		if (protected)
			kvm_flush_remote_tlbs(vcpu->kvm);

1998
		for_each_sp(pages, sp, parents, i) {
1999
			kvm_sync_page(vcpu, sp, &invalid_list);
2000 2001
			mmu_pages_clear_parents(&parents);
		}
2002
		kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2003
		cond_resched_lock(&vcpu->kvm->mmu_lock);
2004 2005
		kvm_mmu_pages_init(parent, &parents, &pages);
	}
2006 2007
}

2008 2009 2010 2011 2012 2013 2014 2015
static void init_shadow_page_table(struct kvm_mmu_page *sp)
{
	int i;

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		sp->spt[i] = 0ull;
}

2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027
static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
{
	sp->write_flooding_count = 0;
}

static void clear_sp_write_flooding_count(u64 *spte)
{
	struct kvm_mmu_page *sp =  page_header(__pa(spte));

	__clear_sp_write_flooding_count(sp);
}

2028 2029 2030 2031 2032
static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
}

2033 2034 2035 2036
static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
2037
					     int direct,
2038
					     unsigned access,
2039
					     u64 *parent_pte)
2040 2041 2042
{
	union kvm_mmu_page_role role;
	unsigned quadrant;
2043 2044
	struct kvm_mmu_page *sp;
	bool need_sync = false;
2045

2046
	role = vcpu->arch.mmu.base_role;
2047
	role.level = level;
2048
	role.direct = direct;
2049
	if (role.direct)
2050
		role.cr4_pae = 0;
2051
	role.access = access;
2052 2053
	if (!vcpu->arch.mmu.direct_map
	    && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
2054 2055 2056 2057
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
2058
	for_each_gfn_sp(vcpu->kvm, sp, gfn) {
2059 2060 2061
		if (is_obsolete_sp(vcpu->kvm, sp))
			continue;

2062 2063
		if (!need_sync && sp->unsync)
			need_sync = true;
2064

2065 2066
		if (sp->role.word != role.word)
			continue;
2067

2068 2069
		if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
			break;
2070

2071 2072
		mmu_page_add_parent_pte(vcpu, sp, parent_pte);
		if (sp->unsync_children) {
2073
			kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2074 2075 2076
			kvm_mmu_mark_parents_unsync(sp);
		} else if (sp->unsync)
			kvm_mmu_mark_parents_unsync(sp);
2077

2078
		__clear_sp_write_flooding_count(sp);
2079 2080 2081
		trace_kvm_mmu_get_page(sp, false);
		return sp;
	}
A
Avi Kivity 已提交
2082
	++vcpu->kvm->stat.mmu_cache_miss;
2083
	sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
2084 2085 2086 2087
	if (!sp)
		return sp;
	sp->gfn = gfn;
	sp->role = role;
2088 2089
	hlist_add_head(&sp->hash_link,
		&vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
2090
	if (!direct) {
2091 2092
		if (rmap_write_protect(vcpu->kvm, gfn))
			kvm_flush_remote_tlbs(vcpu->kvm);
2093 2094 2095
		if (level > PT_PAGE_TABLE_LEVEL && need_sync)
			kvm_sync_pages(vcpu, gfn);

2096 2097
		account_shadowed(vcpu->kvm, gfn);
	}
2098
	sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
2099
	init_shadow_page_table(sp);
A
Avi Kivity 已提交
2100
	trace_kvm_mmu_get_page(sp, true);
2101
	return sp;
2102 2103
}

2104 2105 2106 2107 2108 2109
static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
			     struct kvm_vcpu *vcpu, u64 addr)
{
	iterator->addr = addr;
	iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
	iterator->level = vcpu->arch.mmu.shadow_root_level;
2110 2111 2112 2113 2114 2115

	if (iterator->level == PT64_ROOT_LEVEL &&
	    vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
	    !vcpu->arch.mmu.direct_map)
		--iterator->level;

2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129
	if (iterator->level == PT32E_ROOT_LEVEL) {
		iterator->shadow_addr
			= vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
		--iterator->level;
		if (!iterator->shadow_addr)
			iterator->level = 0;
	}
}

static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
{
	if (iterator->level < PT_PAGE_TABLE_LEVEL)
		return false;
2130

2131 2132 2133 2134 2135
	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
	return true;
}

2136 2137
static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
			       u64 spte)
2138
{
2139
	if (is_last_spte(spte, iterator->level)) {
2140 2141 2142 2143
		iterator->level = 0;
		return;
	}

2144
	iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2145 2146 2147
	--iterator->level;
}

2148 2149 2150 2151 2152
static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
{
	return __shadow_walk_next(iterator, *iterator->sptep);
}

2153
static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp, bool accessed)
2154 2155 2156
{
	u64 spte;

2157 2158 2159
	BUILD_BUG_ON(VMX_EPT_READABLE_MASK != PT_PRESENT_MASK ||
			VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);

X
Xiao Guangrong 已提交
2160
	spte = __pa(sp->spt) | PT_PRESENT_MASK | PT_WRITABLE_MASK |
2161 2162 2163 2164
	       shadow_user_mask | shadow_x_mask;

	if (accessed)
		spte |= shadow_accessed_mask;
X
Xiao Guangrong 已提交
2165

2166
	mmu_spte_set(sptep, spte);
2167 2168
}

2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185
static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
				   unsigned direct_access)
{
	if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
		struct kvm_mmu_page *child;

		/*
		 * For the direct sp, if the guest pte's dirty bit
		 * changed form clean to dirty, it will corrupt the
		 * sp's access: allow writable in the read-only sp,
		 * so we should update the spte at this point to get
		 * a new sp with the correct access.
		 */
		child = page_header(*sptep & PT64_BASE_ADDR_MASK);
		if (child->role.access == direct_access)
			return;

2186
		drop_parent_pte(child, sptep);
2187 2188 2189 2190
		kvm_flush_remote_tlbs(vcpu->kvm);
	}
}

X
Xiao Guangrong 已提交
2191
static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2192 2193 2194 2195 2196 2197 2198
			     u64 *spte)
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
	if (is_shadow_present_pte(pte)) {
X
Xiao Guangrong 已提交
2199
		if (is_last_spte(pte, sp->role.level)) {
2200
			drop_spte(kvm, spte);
X
Xiao Guangrong 已提交
2201 2202 2203
			if (is_large_pte(pte))
				--kvm->stat.lpages;
		} else {
2204
			child = page_header(pte & PT64_BASE_ADDR_MASK);
2205
			drop_parent_pte(child, spte);
2206
		}
X
Xiao Guangrong 已提交
2207 2208 2209 2210
		return true;
	}

	if (is_mmio_spte(pte))
2211
		mmu_spte_clear_no_track(spte);
2212

X
Xiao Guangrong 已提交
2213
	return false;
2214 2215
}

2216
static void kvm_mmu_page_unlink_children(struct kvm *kvm,
2217
					 struct kvm_mmu_page *sp)
2218
{
2219 2220
	unsigned i;

2221 2222
	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		mmu_page_zap_pte(kvm, sp, sp->spt + i);
2223 2224
}

2225
static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
2226
{
2227
	mmu_page_remove_parent_pte(sp, parent_pte);
2228 2229
}

2230
static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
2231
{
2232 2233
	u64 *sptep;
	struct rmap_iterator iter;
2234

2235 2236
	while ((sptep = rmap_get_first(sp->parent_ptes, &iter)))
		drop_parent_pte(sp, sptep);
2237 2238
}

2239
static int mmu_zap_unsync_children(struct kvm *kvm,
2240 2241
				   struct kvm_mmu_page *parent,
				   struct list_head *invalid_list)
2242
{
2243 2244 2245
	int i, zapped = 0;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
2246

2247
	if (parent->role.level == PT_PAGE_TABLE_LEVEL)
2248
		return 0;
2249 2250 2251 2252 2253 2254

	kvm_mmu_pages_init(parent, &parents, &pages);
	while (mmu_unsync_walk(parent, &pages)) {
		struct kvm_mmu_page *sp;

		for_each_sp(pages, sp, parents, i) {
2255
			kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2256
			mmu_pages_clear_parents(&parents);
2257
			zapped++;
2258 2259 2260 2261 2262
		}
		kvm_mmu_pages_init(parent, &parents, &pages);
	}

	return zapped;
2263 2264
}

2265 2266
static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				    struct list_head *invalid_list)
2267
{
2268
	int ret;
A
Avi Kivity 已提交
2269

2270
	trace_kvm_mmu_prepare_zap_page(sp);
2271
	++kvm->stat.mmu_shadow_zapped;
2272
	ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
2273
	kvm_mmu_page_unlink_children(kvm, sp);
2274
	kvm_mmu_unlink_parents(kvm, sp);
2275

2276
	if (!sp->role.invalid && !sp->role.direct)
A
Avi Kivity 已提交
2277
		unaccount_shadowed(kvm, sp->gfn);
2278

2279 2280
	if (sp->unsync)
		kvm_unlink_unsync_page(kvm, sp);
2281
	if (!sp->root_count) {
2282 2283
		/* Count self */
		ret++;
2284
		list_move(&sp->link, invalid_list);
2285
		kvm_mod_used_mmu_pages(kvm, -1);
2286
	} else {
A
Avi Kivity 已提交
2287
		list_move(&sp->link, &kvm->arch.active_mmu_pages);
2288 2289 2290 2291 2292 2293 2294

		/*
		 * The obsolete pages can not be used on any vcpus.
		 * See the comments in kvm_mmu_invalidate_zap_all_pages().
		 */
		if (!sp->role.invalid && !is_obsolete_sp(kvm, sp))
			kvm_reload_remote_mmus(kvm);
2295
	}
2296 2297

	sp->role.invalid = 1;
2298
	return ret;
2299 2300
}

2301 2302 2303
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list)
{
2304
	struct kvm_mmu_page *sp, *nsp;
2305 2306 2307 2308

	if (list_empty(invalid_list))
		return;

2309 2310 2311 2312 2313
	/*
	 * wmb: make sure everyone sees our modifications to the page tables
	 * rmb: make sure we see changes to vcpu->mode
	 */
	smp_mb();
X
Xiao Guangrong 已提交
2314

2315 2316 2317 2318 2319
	/*
	 * Wait for all vcpus to exit guest mode and/or lockless shadow
	 * page table walks.
	 */
	kvm_flush_remote_tlbs(kvm);
2320

2321
	list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2322
		WARN_ON(!sp->role.invalid || sp->root_count);
2323
		kvm_mmu_free_page(sp);
2324
	}
2325 2326
}

2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341
static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
					struct list_head *invalid_list)
{
	struct kvm_mmu_page *sp;

	if (list_empty(&kvm->arch.active_mmu_pages))
		return false;

	sp = list_entry(kvm->arch.active_mmu_pages.prev,
			struct kvm_mmu_page, link);
	kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);

	return true;
}

2342 2343
/*
 * Changing the number of mmu pages allocated to the vm
2344
 * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2345
 */
2346
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
2347
{
2348
	LIST_HEAD(invalid_list);
2349

2350 2351
	spin_lock(&kvm->mmu_lock);

2352
	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2353 2354 2355 2356
		/* Need to free some mmu pages to achieve the goal. */
		while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
			if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
				break;
2357

2358
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
2359
		goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2360 2361
	}

2362
	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2363 2364

	spin_unlock(&kvm->mmu_lock);
2365 2366
}

2367
int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2368
{
2369
	struct kvm_mmu_page *sp;
2370
	LIST_HEAD(invalid_list);
2371 2372
	int r;

2373
	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2374
	r = 0;
2375
	spin_lock(&kvm->mmu_lock);
2376
	for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2377
		pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2378 2379
			 sp->role.word);
		r = 1;
2380
		kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2381
	}
2382
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
2383 2384
	spin_unlock(&kvm->mmu_lock);

2385
	return r;
2386
}
2387
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
2388

2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481
/*
 * The function is based on mtrr_type_lookup() in
 * arch/x86/kernel/cpu/mtrr/generic.c
 */
static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
			 u64 start, u64 end)
{
	int i;
	u64 base, mask;
	u8 prev_match, curr_match;
	int num_var_ranges = KVM_NR_VAR_MTRR;

	if (!mtrr_state->enabled)
		return 0xFF;

	/* Make end inclusive end, instead of exclusive */
	end--;

	/* Look in fixed ranges. Just return the type as per start */
	if (mtrr_state->have_fixed && (start < 0x100000)) {
		int idx;

		if (start < 0x80000) {
			idx = 0;
			idx += (start >> 16);
			return mtrr_state->fixed_ranges[idx];
		} else if (start < 0xC0000) {
			idx = 1 * 8;
			idx += ((start - 0x80000) >> 14);
			return mtrr_state->fixed_ranges[idx];
		} else if (start < 0x1000000) {
			idx = 3 * 8;
			idx += ((start - 0xC0000) >> 12);
			return mtrr_state->fixed_ranges[idx];
		}
	}

	/*
	 * Look in variable ranges
	 * Look of multiple ranges matching this address and pick type
	 * as per MTRR precedence
	 */
	if (!(mtrr_state->enabled & 2))
		return mtrr_state->def_type;

	prev_match = 0xFF;
	for (i = 0; i < num_var_ranges; ++i) {
		unsigned short start_state, end_state;

		if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
			continue;

		base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
		       (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
		mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
		       (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);

		start_state = ((start & mask) == (base & mask));
		end_state = ((end & mask) == (base & mask));
		if (start_state != end_state)
			return 0xFE;

		if ((start & mask) != (base & mask))
			continue;

		curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
		if (prev_match == 0xFF) {
			prev_match = curr_match;
			continue;
		}

		if (prev_match == MTRR_TYPE_UNCACHABLE ||
		    curr_match == MTRR_TYPE_UNCACHABLE)
			return MTRR_TYPE_UNCACHABLE;

		if ((prev_match == MTRR_TYPE_WRBACK &&
		     curr_match == MTRR_TYPE_WRTHROUGH) ||
		    (prev_match == MTRR_TYPE_WRTHROUGH &&
		     curr_match == MTRR_TYPE_WRBACK)) {
			prev_match = MTRR_TYPE_WRTHROUGH;
			curr_match = MTRR_TYPE_WRTHROUGH;
		}

		if (prev_match != curr_match)
			return MTRR_TYPE_UNCACHABLE;
	}

	if (prev_match != 0xFF)
		return prev_match;

	return mtrr_state->def_type;
}

2482
u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
2483 2484 2485 2486 2487 2488 2489 2490 2491
{
	u8 mtrr;

	mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
			     (gfn << PAGE_SHIFT) + PAGE_SIZE);
	if (mtrr == 0xfe || mtrr == 0xff)
		mtrr = MTRR_TYPE_WRBACK;
	return mtrr;
}
2492
EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
2493

2494 2495 2496 2497 2498 2499 2500 2501 2502 2503
static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
{
	trace_kvm_mmu_unsync_page(sp);
	++vcpu->kvm->stat.mmu_unsync;
	sp->unsync = 1;

	kvm_mmu_mark_parents_unsync(sp);
}

static void kvm_unsync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
2504 2505
{
	struct kvm_mmu_page *s;
2506

2507
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
2508
		if (s->unsync)
2509
			continue;
2510 2511
		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
		__kvm_unsync_page(vcpu, s);
2512 2513 2514 2515 2516 2517
	}
}

static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
				  bool can_unsync)
{
2518 2519 2520
	struct kvm_mmu_page *s;
	bool need_unsync = false;

2521
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
2522 2523 2524
		if (!can_unsync)
			return 1;

2525
		if (s->role.level != PT_PAGE_TABLE_LEVEL)
2526
			return 1;
2527

G
Gleb Natapov 已提交
2528
		if (!s->unsync)
2529
			need_unsync = true;
2530
	}
2531 2532
	if (need_unsync)
		kvm_unsync_pages(vcpu, gfn);
2533 2534 2535
	return 0;
}

A
Avi Kivity 已提交
2536
static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2537
		    unsigned pte_access, int level,
2538
		    gfn_t gfn, pfn_t pfn, bool speculative,
2539
		    bool can_unsync, bool host_writable)
2540
{
2541
	u64 spte;
M
Marcelo Tosatti 已提交
2542
	int ret = 0;
S
Sheng Yang 已提交
2543

2544
	if (set_mmio_spte(vcpu->kvm, sptep, gfn, pfn, pte_access))
2545 2546
		return 0;

2547
	spte = PT_PRESENT_MASK;
2548
	if (!speculative)
2549
		spte |= shadow_accessed_mask;
2550

S
Sheng Yang 已提交
2551 2552 2553 2554
	if (pte_access & ACC_EXEC_MASK)
		spte |= shadow_x_mask;
	else
		spte |= shadow_nx_mask;
2555

2556
	if (pte_access & ACC_USER_MASK)
S
Sheng Yang 已提交
2557
		spte |= shadow_user_mask;
2558

2559
	if (level > PT_PAGE_TABLE_LEVEL)
M
Marcelo Tosatti 已提交
2560
		spte |= PT_PAGE_SIZE_MASK;
2561
	if (tdp_enabled)
2562
		spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
2563
			kvm_is_reserved_pfn(pfn));
2564

2565
	if (host_writable)
2566
		spte |= SPTE_HOST_WRITEABLE;
2567 2568
	else
		pte_access &= ~ACC_WRITE_MASK;
2569

2570
	spte |= (u64)pfn << PAGE_SHIFT;
2571

2572
	if (pte_access & ACC_WRITE_MASK) {
2573

X
Xiao Guangrong 已提交
2574
		/*
2575 2576 2577 2578
		 * Other vcpu creates new sp in the window between
		 * mapping_level() and acquiring mmu-lock. We can
		 * allow guest to retry the access, the mapping can
		 * be fixed if guest refault.
X
Xiao Guangrong 已提交
2579
		 */
2580
		if (level > PT_PAGE_TABLE_LEVEL &&
X
Xiao Guangrong 已提交
2581
		    has_wrprotected_page(vcpu->kvm, gfn, level))
A
Avi Kivity 已提交
2582
			goto done;
2583

2584
		spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
2585

2586 2587 2588 2589 2590 2591
		/*
		 * Optimization: for pte sync, if spte was writable the hash
		 * lookup is unnecessary (and expensive). Write protection
		 * is responsibility of mmu_get_page / kvm_sync_page.
		 * Same reasoning can be applied to dirty page accounting.
		 */
2592
		if (!can_unsync && is_writable_pte(*sptep))
2593 2594
			goto set_pte;

2595
		if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2596
			pgprintk("%s: found shadow page for %llx, marking ro\n",
2597
				 __func__, gfn);
M
Marcelo Tosatti 已提交
2598
			ret = 1;
2599
			pte_access &= ~ACC_WRITE_MASK;
2600
			spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
2601 2602 2603
		}
	}

2604
	if (pte_access & ACC_WRITE_MASK) {
2605
		mark_page_dirty(vcpu->kvm, gfn);
2606 2607
		spte |= shadow_dirty_mask;
	}
2608

2609
set_pte:
2610
	if (mmu_spte_update(sptep, spte))
2611
		kvm_flush_remote_tlbs(vcpu->kvm);
A
Avi Kivity 已提交
2612
done:
M
Marcelo Tosatti 已提交
2613 2614 2615
	return ret;
}

A
Avi Kivity 已提交
2616
static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2617 2618 2619
			 unsigned pte_access, int write_fault, int *emulate,
			 int level, gfn_t gfn, pfn_t pfn, bool speculative,
			 bool host_writable)
M
Marcelo Tosatti 已提交
2620 2621
{
	int was_rmapped = 0;
2622
	int rmap_count;
M
Marcelo Tosatti 已提交
2623

2624 2625
	pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
		 *sptep, write_fault, gfn);
M
Marcelo Tosatti 已提交
2626

A
Avi Kivity 已提交
2627
	if (is_rmap_spte(*sptep)) {
M
Marcelo Tosatti 已提交
2628 2629 2630 2631
		/*
		 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
		 * the parent of the now unreachable PTE.
		 */
2632 2633
		if (level > PT_PAGE_TABLE_LEVEL &&
		    !is_large_pte(*sptep)) {
M
Marcelo Tosatti 已提交
2634
			struct kvm_mmu_page *child;
A
Avi Kivity 已提交
2635
			u64 pte = *sptep;
M
Marcelo Tosatti 已提交
2636 2637

			child = page_header(pte & PT64_BASE_ADDR_MASK);
2638
			drop_parent_pte(child, sptep);
2639
			kvm_flush_remote_tlbs(vcpu->kvm);
A
Avi Kivity 已提交
2640
		} else if (pfn != spte_to_pfn(*sptep)) {
2641
			pgprintk("hfn old %llx new %llx\n",
A
Avi Kivity 已提交
2642
				 spte_to_pfn(*sptep), pfn);
2643
			drop_spte(vcpu->kvm, sptep);
2644
			kvm_flush_remote_tlbs(vcpu->kvm);
2645 2646
		} else
			was_rmapped = 1;
M
Marcelo Tosatti 已提交
2647
	}
2648

2649 2650
	if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
	      true, host_writable)) {
M
Marcelo Tosatti 已提交
2651
		if (write_fault)
2652
			*emulate = 1;
2653
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2654
	}
M
Marcelo Tosatti 已提交
2655

2656 2657 2658
	if (unlikely(is_mmio_spte(*sptep) && emulate))
		*emulate = 1;

A
Avi Kivity 已提交
2659
	pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2660
	pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
A
Avi Kivity 已提交
2661
		 is_large_pte(*sptep)? "2MB" : "4kB",
2662 2663
		 *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
		 *sptep, sptep);
A
Avi Kivity 已提交
2664
	if (!was_rmapped && is_large_pte(*sptep))
M
Marcelo Tosatti 已提交
2665 2666
		++vcpu->kvm->stat.lpages;

2667 2668 2669 2670 2671 2672
	if (is_shadow_present_pte(*sptep)) {
		if (!was_rmapped) {
			rmap_count = rmap_add(vcpu, sptep, gfn);
			if (rmap_count > RMAP_RECYCLE_THRESHOLD)
				rmap_recycle(vcpu, sptep, gfn);
		}
2673
	}
2674

X
Xiao Guangrong 已提交
2675
	kvm_release_pfn_clean(pfn);
2676 2677
}

2678 2679 2680 2681 2682
static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
				     bool no_dirty_log)
{
	struct kvm_memory_slot *slot;

2683
	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
2684
	if (!slot)
2685
		return KVM_PFN_ERR_FAULT;
2686

2687
	return gfn_to_pfn_memslot_atomic(slot, gfn);
2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699
}

static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *sp,
				    u64 *start, u64 *end)
{
	struct page *pages[PTE_PREFETCH_NUM];
	unsigned access = sp->role.access;
	int i, ret;
	gfn_t gfn;

	gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2700
	if (!gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK))
2701 2702 2703 2704 2705 2706 2707
		return -1;

	ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
	if (ret <= 0)
		return -1;

	for (i = 0; i < ret; i++, gfn++, start++)
2708
		mmu_set_spte(vcpu, start, access, 0, NULL,
2709 2710
			     sp->role.level, gfn, page_to_pfn(pages[i]),
			     true, true);
2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726

	return 0;
}

static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
				  struct kvm_mmu_page *sp, u64 *sptep)
{
	u64 *spte, *start = NULL;
	int i;

	WARN_ON(!sp->role.direct);

	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
	spte = sp->spt + i;

	for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2727
		if (is_shadow_present_pte(*spte) || spte == sptep) {
2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757
			if (!start)
				continue;
			if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
				break;
			start = NULL;
		} else if (!start)
			start = spte;
	}
}

static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
{
	struct kvm_mmu_page *sp;

	/*
	 * Since it's no accessed bit on EPT, it's no way to
	 * distinguish between actually accessed translations
	 * and prefetched, so disable pte prefetch if EPT is
	 * enabled.
	 */
	if (!shadow_accessed_mask)
		return;

	sp = page_header(__pa(sptep));
	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return;

	__direct_pte_prefetch(vcpu, sp, sptep);
}

2758
static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
2759 2760
			int map_writable, int level, gfn_t gfn, pfn_t pfn,
			bool prefault)
2761
{
2762
	struct kvm_shadow_walk_iterator iterator;
2763
	struct kvm_mmu_page *sp;
2764
	int emulate = 0;
2765
	gfn_t pseudo_gfn;
A
Avi Kivity 已提交
2766

2767 2768 2769
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return 0;

2770
	for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2771
		if (iterator.level == level) {
2772
			mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
2773 2774
				     write, &emulate, level, gfn, pfn,
				     prefault, map_writable);
2775
			direct_pte_prefetch(vcpu, iterator.sptep);
2776 2777
			++vcpu->stat.pf_fixed;
			break;
A
Avi Kivity 已提交
2778 2779
		}

2780
		drop_large_spte(vcpu, iterator.sptep);
2781
		if (!is_shadow_present_pte(*iterator.sptep)) {
2782 2783 2784 2785
			u64 base_addr = iterator.addr;

			base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
			pseudo_gfn = base_addr >> PAGE_SHIFT;
2786 2787 2788
			sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
					      iterator.level - 1,
					      1, ACC_ALL, iterator.sptep);
2789

2790
			link_shadow_page(iterator.sptep, sp, true);
2791 2792
		}
	}
2793
	return emulate;
A
Avi Kivity 已提交
2794 2795
}

H
Huang Ying 已提交
2796
static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2797
{
H
Huang Ying 已提交
2798 2799 2800 2801 2802 2803 2804
	siginfo_t info;

	info.si_signo	= SIGBUS;
	info.si_errno	= 0;
	info.si_code	= BUS_MCEERR_AR;
	info.si_addr	= (void __user *)address;
	info.si_addr_lsb = PAGE_SHIFT;
2805

H
Huang Ying 已提交
2806
	send_sig_info(SIGBUS, &info, tsk);
2807 2808
}

2809
static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn)
2810
{
X
Xiao Guangrong 已提交
2811 2812 2813 2814 2815 2816 2817 2818 2819
	/*
	 * Do not cache the mmio info caused by writing the readonly gfn
	 * into the spte otherwise read access on readonly gfn also can
	 * caused mmio page fault and treat it as mmio access.
	 * Return 1 to tell kvm to emulate it.
	 */
	if (pfn == KVM_PFN_ERR_RO_FAULT)
		return 1;

2820
	if (pfn == KVM_PFN_ERR_HWPOISON) {
2821
		kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current);
2822
		return 0;
2823
	}
2824

2825
	return -EFAULT;
2826 2827
}

2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840
static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
					gfn_t *gfnp, pfn_t *pfnp, int *levelp)
{
	pfn_t pfn = *pfnp;
	gfn_t gfn = *gfnp;
	int level = *levelp;

	/*
	 * Check if it's a transparent hugepage. If this would be an
	 * hugetlbfs page, level wouldn't be set to
	 * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
	 * here.
	 */
2841
	if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn) &&
2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862
	    level == PT_PAGE_TABLE_LEVEL &&
	    PageTransCompound(pfn_to_page(pfn)) &&
	    !has_wrprotected_page(vcpu->kvm, gfn, PT_DIRECTORY_LEVEL)) {
		unsigned long mask;
		/*
		 * mmu_notifier_retry was successful and we hold the
		 * mmu_lock here, so the pmd can't become splitting
		 * from under us, and in turn
		 * __split_huge_page_refcount() can't run from under
		 * us and we can safely transfer the refcount from
		 * PG_tail to PG_head as we switch the pfn to tail to
		 * head.
		 */
		*levelp = level = PT_DIRECTORY_LEVEL;
		mask = KVM_PAGES_PER_HPAGE(level) - 1;
		VM_BUG_ON((gfn & mask) != (pfn & mask));
		if (pfn & mask) {
			gfn &= ~mask;
			*gfnp = gfn;
			kvm_release_pfn_clean(pfn);
			pfn &= ~mask;
2863
			kvm_get_pfn(pfn);
2864 2865 2866 2867 2868
			*pfnp = pfn;
		}
	}
}

2869 2870 2871 2872 2873 2874
static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
				pfn_t pfn, unsigned access, int *ret_val)
{
	bool ret = true;

	/* The pfn is invalid, report the error! */
2875
	if (unlikely(is_error_pfn(pfn))) {
2876 2877 2878 2879
		*ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
		goto exit;
	}

2880
	if (unlikely(is_noslot_pfn(pfn)))
2881 2882 2883 2884 2885 2886 2887
		vcpu_cache_mmio_info(vcpu, gva, gfn, access);

	ret = false;
exit:
	return ret;
}

2888
static bool page_fault_can_be_fast(u32 error_code)
2889
{
2890 2891 2892 2893 2894 2895 2896
	/*
	 * Do not fix the mmio spte with invalid generation number which
	 * need to be updated by slow page fault path.
	 */
	if (unlikely(error_code & PFERR_RSVD_MASK))
		return false;

2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909
	/*
	 * #PF can be fast only if the shadow page table is present and it
	 * is caused by write-protect, that means we just need change the
	 * W bit of the spte which can be done out of mmu-lock.
	 */
	if (!(error_code & PFERR_PRESENT_MASK) ||
	      !(error_code & PFERR_WRITE_MASK))
		return false;

	return true;
}

static bool
2910 2911
fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
			u64 *sptep, u64 spte)
2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922
{
	gfn_t gfn;

	WARN_ON(!sp->role.direct);

	/*
	 * The gfn of direct spte is stable since it is calculated
	 * by sp->gfn.
	 */
	gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);

2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934
	/*
	 * Theoretically we could also set dirty bit (and flush TLB) here in
	 * order to eliminate unnecessary PML logging. See comments in
	 * set_spte. But fast_page_fault is very unlikely to happen with PML
	 * enabled, so we do not do this. This might result in the same GPA
	 * to be logged in PML buffer again when the write really happens, and
	 * eventually to be called by mark_page_dirty twice. But it's also no
	 * harm. This also avoids the TLB flush needed after setting dirty bit
	 * so non-PML cases won't be impacted.
	 *
	 * Compare with set_spte where instead shadow_dirty_mask is set.
	 */
2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949
	if (cmpxchg64(sptep, spte, spte | PT_WRITABLE_MASK) == spte)
		mark_page_dirty(vcpu->kvm, gfn);

	return true;
}

/*
 * Return value:
 * - true: let the vcpu to access on the same address again.
 * - false: let the real page fault path to fix it.
 */
static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
			    u32 error_code)
{
	struct kvm_shadow_walk_iterator iterator;
2950
	struct kvm_mmu_page *sp;
2951 2952 2953
	bool ret = false;
	u64 spte = 0ull;

2954 2955 2956
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return false;

2957
	if (!page_fault_can_be_fast(error_code))
2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973
		return false;

	walk_shadow_page_lockless_begin(vcpu);
	for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
		if (!is_shadow_present_pte(spte) || iterator.level < level)
			break;

	/*
	 * If the mapping has been changed, let the vcpu fault on the
	 * same address again.
	 */
	if (!is_rmap_spte(spte)) {
		ret = true;
		goto exit;
	}

2974 2975
	sp = page_header(__pa(iterator.sptep));
	if (!is_last_spte(spte, sp->role.level))
2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995
		goto exit;

	/*
	 * Check if it is a spurious fault caused by TLB lazily flushed.
	 *
	 * Need not check the access of upper level table entries since
	 * they are always ACC_ALL.
	 */
	 if (is_writable_pte(spte)) {
		ret = true;
		goto exit;
	}

	/*
	 * Currently, to simplify the code, only the spte write-protected
	 * by dirty-log can be fast fixed.
	 */
	if (!spte_is_locklessly_modifiable(spte))
		goto exit;

2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008
	/*
	 * Do not fix write-permission on the large spte since we only dirty
	 * the first page into the dirty-bitmap in fast_pf_fix_direct_spte()
	 * that means other pages are missed if its slot is dirty-logged.
	 *
	 * Instead, we let the slow page fault path create a normal spte to
	 * fix the access.
	 *
	 * See the comments in kvm_arch_commit_memory_region().
	 */
	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		goto exit;

3009 3010 3011 3012 3013
	/*
	 * Currently, fast page fault only works for direct mapping since
	 * the gfn is not stable for indirect shadow page.
	 * See Documentation/virtual/kvm/locking.txt to get more detail.
	 */
3014
	ret = fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte);
3015
exit:
X
Xiao Guangrong 已提交
3016 3017
	trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
			      spte, ret);
3018 3019 3020 3021 3022
	walk_shadow_page_lockless_end(vcpu);

	return ret;
}

3023
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
3024
			 gva_t gva, pfn_t *pfn, bool write, bool *writable);
3025
static void make_mmu_pages_available(struct kvm_vcpu *vcpu);
3026

3027 3028
static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
			 gfn_t gfn, bool prefault)
3029 3030
{
	int r;
3031
	int level;
3032
	int force_pt_level;
3033
	pfn_t pfn;
3034
	unsigned long mmu_seq;
3035
	bool map_writable, write = error_code & PFERR_WRITE_MASK;
3036

3037 3038 3039 3040 3041 3042 3043 3044 3045 3046
	force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
	if (likely(!force_pt_level)) {
		level = mapping_level(vcpu, gfn);
		/*
		 * This path builds a PAE pagetable - so we can map
		 * 2mb pages at maximum. Therefore check if the level
		 * is larger than that.
		 */
		if (level > PT_DIRECTORY_LEVEL)
			level = PT_DIRECTORY_LEVEL;
3047

3048 3049 3050
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
	} else
		level = PT_PAGE_TABLE_LEVEL;
M
Marcelo Tosatti 已提交
3051

3052 3053 3054
	if (fast_page_fault(vcpu, v, level, error_code))
		return 0;

3055
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
3056
	smp_rmb();
3057

3058
	if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
3059
		return 0;
3060

3061 3062
	if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
		return r;
3063

3064
	spin_lock(&vcpu->kvm->mmu_lock);
3065
	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3066
		goto out_unlock;
3067
	make_mmu_pages_available(vcpu);
3068 3069
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3070 3071
	r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
			 prefault);
3072 3073 3074
	spin_unlock(&vcpu->kvm->mmu_lock);


3075
	return r;
3076 3077 3078 3079 3080

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
3081 3082 3083
}


3084 3085 3086
static void mmu_free_roots(struct kvm_vcpu *vcpu)
{
	int i;
3087
	struct kvm_mmu_page *sp;
3088
	LIST_HEAD(invalid_list);
3089

3090
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
A
Avi Kivity 已提交
3091
		return;
3092

3093 3094 3095
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
	    (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
	     vcpu->arch.mmu.direct_map)) {
3096
		hpa_t root = vcpu->arch.mmu.root_hpa;
3097

3098
		spin_lock(&vcpu->kvm->mmu_lock);
3099 3100
		sp = page_header(root);
		--sp->root_count;
3101 3102 3103 3104
		if (!sp->root_count && sp->role.invalid) {
			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
			kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
		}
3105
		spin_unlock(&vcpu->kvm->mmu_lock);
3106
		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3107 3108
		return;
	}
3109 3110

	spin_lock(&vcpu->kvm->mmu_lock);
3111
	for (i = 0; i < 4; ++i) {
3112
		hpa_t root = vcpu->arch.mmu.pae_root[i];
3113

A
Avi Kivity 已提交
3114 3115
		if (root) {
			root &= PT64_BASE_ADDR_MASK;
3116 3117
			sp = page_header(root);
			--sp->root_count;
3118
			if (!sp->root_count && sp->role.invalid)
3119 3120
				kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
							 &invalid_list);
A
Avi Kivity 已提交
3121
		}
3122
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
3123
	}
3124
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3125
	spin_unlock(&vcpu->kvm->mmu_lock);
3126
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3127 3128
}

3129 3130 3131 3132 3133
static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
{
	int ret = 0;

	if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
3134
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3135 3136 3137 3138 3139 3140
		ret = 1;
	}

	return ret;
}

3141 3142 3143
static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_page *sp;
3144
	unsigned i;
3145 3146 3147

	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		spin_lock(&vcpu->kvm->mmu_lock);
3148
		make_mmu_pages_available(vcpu);
3149 3150 3151 3152 3153 3154 3155 3156 3157
		sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
				      1, ACC_ALL, NULL);
		++sp->root_count;
		spin_unlock(&vcpu->kvm->mmu_lock);
		vcpu->arch.mmu.root_hpa = __pa(sp->spt);
	} else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
		for (i = 0; i < 4; ++i) {
			hpa_t root = vcpu->arch.mmu.pae_root[i];

3158
			MMU_WARN_ON(VALID_PAGE(root));
3159
			spin_lock(&vcpu->kvm->mmu_lock);
3160
			make_mmu_pages_available(vcpu);
3161 3162
			sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
					      i << 30,
3163 3164 3165 3166 3167 3168 3169
					      PT32_ROOT_LEVEL, 1, ACC_ALL,
					      NULL);
			root = __pa(sp->spt);
			++sp->root_count;
			spin_unlock(&vcpu->kvm->mmu_lock);
			vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
		}
3170
		vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3171 3172 3173 3174 3175 3176 3177
	} else
		BUG();

	return 0;
}

static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3178
{
3179
	struct kvm_mmu_page *sp;
3180 3181 3182
	u64 pdptr, pm_mask;
	gfn_t root_gfn;
	int i;
3183

3184
	root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
3185

3186 3187 3188 3189 3190 3191 3192 3193
	if (mmu_check_root(vcpu, root_gfn))
		return 1;

	/*
	 * Do we shadow a long mode page table? If so we need to
	 * write-protect the guests page table root.
	 */
	if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3194
		hpa_t root = vcpu->arch.mmu.root_hpa;
3195

3196
		MMU_WARN_ON(VALID_PAGE(root));
3197

3198
		spin_lock(&vcpu->kvm->mmu_lock);
3199
		make_mmu_pages_available(vcpu);
3200 3201
		sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
				      0, ACC_ALL, NULL);
3202 3203
		root = __pa(sp->spt);
		++sp->root_count;
3204
		spin_unlock(&vcpu->kvm->mmu_lock);
3205
		vcpu->arch.mmu.root_hpa = root;
3206
		return 0;
3207
	}
3208

3209 3210
	/*
	 * We shadow a 32 bit page table. This may be a legacy 2-level
3211 3212
	 * or a PAE 3-level page table. In either case we need to be aware that
	 * the shadow page table may be a PAE or a long mode page table.
3213
	 */
3214 3215 3216 3217
	pm_mask = PT_PRESENT_MASK;
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
		pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;

3218
	for (i = 0; i < 4; ++i) {
3219
		hpa_t root = vcpu->arch.mmu.pae_root[i];
3220

3221
		MMU_WARN_ON(VALID_PAGE(root));
3222
		if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
3223
			pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
3224
			if (!is_present_gpte(pdptr)) {
3225
				vcpu->arch.mmu.pae_root[i] = 0;
A
Avi Kivity 已提交
3226 3227
				continue;
			}
A
Avi Kivity 已提交
3228
			root_gfn = pdptr >> PAGE_SHIFT;
3229 3230
			if (mmu_check_root(vcpu, root_gfn))
				return 1;
3231
		}
3232
		spin_lock(&vcpu->kvm->mmu_lock);
3233
		make_mmu_pages_available(vcpu);
3234
		sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
3235
				      PT32_ROOT_LEVEL, 0,
3236
				      ACC_ALL, NULL);
3237 3238
		root = __pa(sp->spt);
		++sp->root_count;
3239 3240
		spin_unlock(&vcpu->kvm->mmu_lock);

3241
		vcpu->arch.mmu.pae_root[i] = root | pm_mask;
3242
	}
3243
	vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269

	/*
	 * If we shadow a 32 bit page table with a long mode page
	 * table we enter this path.
	 */
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		if (vcpu->arch.mmu.lm_root == NULL) {
			/*
			 * The additional page necessary for this is only
			 * allocated on demand.
			 */

			u64 *lm_root;

			lm_root = (void*)get_zeroed_page(GFP_KERNEL);
			if (lm_root == NULL)
				return 1;

			lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;

			vcpu->arch.mmu.lm_root = lm_root;
		}

		vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
	}

3270
	return 0;
3271 3272
}

3273 3274 3275 3276 3277 3278 3279 3280
static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
{
	if (vcpu->arch.mmu.direct_map)
		return mmu_alloc_direct_roots(vcpu);
	else
		return mmu_alloc_shadow_roots(vcpu);
}

3281 3282 3283 3284 3285
static void mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	int i;
	struct kvm_mmu_page *sp;

3286 3287 3288
	if (vcpu->arch.mmu.direct_map)
		return;

3289 3290
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return;
3291

3292
	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3293
	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3294
	if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3295 3296 3297
		hpa_t root = vcpu->arch.mmu.root_hpa;
		sp = page_header(root);
		mmu_sync_children(vcpu, sp);
3298
		kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3299 3300 3301 3302 3303
		return;
	}
	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->arch.mmu.pae_root[i];

3304
		if (root && VALID_PAGE(root)) {
3305 3306 3307 3308 3309
			root &= PT64_BASE_ADDR_MASK;
			sp = page_header(root);
			mmu_sync_children(vcpu, sp);
		}
	}
3310
	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3311 3312 3313 3314 3315 3316
}

void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	spin_lock(&vcpu->kvm->mmu_lock);
	mmu_sync_roots(vcpu);
3317
	spin_unlock(&vcpu->kvm->mmu_lock);
3318
}
N
Nadav Har'El 已提交
3319
EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
3320

3321
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
3322
				  u32 access, struct x86_exception *exception)
A
Avi Kivity 已提交
3323
{
3324 3325
	if (exception)
		exception->error_code = 0;
A
Avi Kivity 已提交
3326 3327 3328
	return vaddr;
}

3329
static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
3330 3331
					 u32 access,
					 struct x86_exception *exception)
3332
{
3333 3334
	if (exception)
		exception->error_code = 0;
3335
	return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception);
3336 3337
}

3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365
static bool quickly_check_mmio_pf(struct kvm_vcpu *vcpu, u64 addr, bool direct)
{
	if (direct)
		return vcpu_match_mmio_gpa(vcpu, addr);

	return vcpu_match_mmio_gva(vcpu, addr);
}


/*
 * On direct hosts, the last spte is only allows two states
 * for mmio page fault:
 *   - It is the mmio spte
 *   - It is zapped or it is being zapped.
 *
 * This function completely checks the spte when the last spte
 * is not the mmio spte.
 */
static bool check_direct_spte_mmio_pf(u64 spte)
{
	return __check_direct_spte_mmio_pf(spte);
}

static u64 walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr)
{
	struct kvm_shadow_walk_iterator iterator;
	u64 spte = 0ull;

3366 3367 3368
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return spte;

3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382
	walk_shadow_page_lockless_begin(vcpu);
	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
		if (!is_shadow_present_pte(spte))
			break;
	walk_shadow_page_lockless_end(vcpu);

	return spte;
}

int handle_mmio_page_fault_common(struct kvm_vcpu *vcpu, u64 addr, bool direct)
{
	u64 spte;

	if (quickly_check_mmio_pf(vcpu, addr, direct))
3383
		return RET_MMIO_PF_EMULATE;
3384 3385 3386 3387 3388 3389 3390

	spte = walk_shadow_page_get_mmio_spte(vcpu, addr);

	if (is_mmio_spte(spte)) {
		gfn_t gfn = get_mmio_spte_gfn(spte);
		unsigned access = get_mmio_spte_access(spte);

3391 3392 3393
		if (!check_mmio_spte(vcpu->kvm, spte))
			return RET_MMIO_PF_INVALID;

3394 3395
		if (direct)
			addr = 0;
X
Xiao Guangrong 已提交
3396 3397

		trace_handle_mmio_page_fault(addr, gfn, access);
3398
		vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3399
		return RET_MMIO_PF_EMULATE;
3400 3401 3402 3403 3404 3405 3406
	}

	/*
	 * It's ok if the gva is remapped by other cpus on shadow guest,
	 * it's a BUG if the gfn is not a mmio page.
	 */
	if (direct && !check_direct_spte_mmio_pf(spte))
3407
		return RET_MMIO_PF_BUG;
3408 3409 3410 3411 3412

	/*
	 * If the page table is zapped by other cpus, let CPU fault again on
	 * the address.
	 */
3413
	return RET_MMIO_PF_RETRY;
3414 3415 3416 3417 3418 3419 3420 3421 3422
}
EXPORT_SYMBOL_GPL(handle_mmio_page_fault_common);

static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr,
				  u32 error_code, bool direct)
{
	int ret;

	ret = handle_mmio_page_fault_common(vcpu, addr, direct);
3423
	WARN_ON(ret == RET_MMIO_PF_BUG);
3424 3425 3426
	return ret;
}

A
Avi Kivity 已提交
3427
static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
3428
				u32 error_code, bool prefault)
A
Avi Kivity 已提交
3429
{
3430
	gfn_t gfn;
3431
	int r;
A
Avi Kivity 已提交
3432

3433
	pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
3434

3435 3436 3437 3438 3439 3440
	if (unlikely(error_code & PFERR_RSVD_MASK)) {
		r = handle_mmio_page_fault(vcpu, gva, error_code, true);

		if (likely(r != RET_MMIO_PF_INVALID))
			return r;
	}
3441

3442 3443 3444
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;
3445

3446
	MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
3447

3448
	gfn = gva >> PAGE_SHIFT;
A
Avi Kivity 已提交
3449

3450
	return nonpaging_map(vcpu, gva & PAGE_MASK,
3451
			     error_code, gfn, prefault);
A
Avi Kivity 已提交
3452 3453
}

3454
static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
3455 3456
{
	struct kvm_arch_async_pf arch;
X
Xiao Guangrong 已提交
3457

3458
	arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3459
	arch.gfn = gfn;
3460
	arch.direct_map = vcpu->arch.mmu.direct_map;
X
Xiao Guangrong 已提交
3461
	arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
3462

3463
	return kvm_setup_async_pf(vcpu, gva, gfn_to_hva(vcpu->kvm, gfn), &arch);
3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474
}

static bool can_do_async_pf(struct kvm_vcpu *vcpu)
{
	if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
		     kvm_event_needs_reinjection(vcpu)))
		return false;

	return kvm_x86_ops->interrupt_allowed(vcpu);
}

3475
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
3476
			 gva_t gva, pfn_t *pfn, bool write, bool *writable)
3477 3478 3479
{
	bool async;

3480
	*pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable);
3481 3482 3483 3484

	if (!async)
		return false; /* *pfn has correct page already */

3485
	if (!prefault && can_do_async_pf(vcpu)) {
3486
		trace_kvm_try_async_get_page(gva, gfn);
3487 3488 3489 3490 3491 3492 3493 3494
		if (kvm_find_async_pf_gfn(vcpu, gfn)) {
			trace_kvm_async_pf_doublefault(gva, gfn);
			kvm_make_request(KVM_REQ_APF_HALT, vcpu);
			return true;
		} else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
			return true;
	}

3495
	*pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable);
3496 3497 3498 3499

	return false;
}

G
Gleb Natapov 已提交
3500
static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
3501
			  bool prefault)
3502
{
3503
	pfn_t pfn;
3504
	int r;
3505
	int level;
3506
	int force_pt_level;
M
Marcelo Tosatti 已提交
3507
	gfn_t gfn = gpa >> PAGE_SHIFT;
3508
	unsigned long mmu_seq;
3509 3510
	int write = error_code & PFERR_WRITE_MASK;
	bool map_writable;
3511

3512
	MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3513

3514 3515 3516 3517 3518 3519
	if (unlikely(error_code & PFERR_RSVD_MASK)) {
		r = handle_mmio_page_fault(vcpu, gpa, error_code, true);

		if (likely(r != RET_MMIO_PF_INVALID))
			return r;
	}
3520

3521 3522 3523 3524
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

3525 3526 3527 3528 3529 3530
	force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
	if (likely(!force_pt_level)) {
		level = mapping_level(vcpu, gfn);
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
	} else
		level = PT_PAGE_TABLE_LEVEL;
3531

3532 3533 3534
	if (fast_page_fault(vcpu, gpa, level, error_code))
		return 0;

3535
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
3536
	smp_rmb();
3537

3538
	if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
3539 3540
		return 0;

3541 3542 3543
	if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
		return r;

3544
	spin_lock(&vcpu->kvm->mmu_lock);
3545
	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3546
		goto out_unlock;
3547
	make_mmu_pages_available(vcpu);
3548 3549
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3550
	r = __direct_map(vcpu, gpa, write, map_writable,
3551
			 level, gfn, pfn, prefault);
3552 3553 3554
	spin_unlock(&vcpu->kvm->mmu_lock);

	return r;
3555 3556 3557 3558 3559

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
3560 3561
}

3562 3563
static void nonpaging_init_context(struct kvm_vcpu *vcpu,
				   struct kvm_mmu *context)
A
Avi Kivity 已提交
3564 3565 3566
{
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
3567
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
3568
	context->invlpg = nonpaging_invlpg;
3569
	context->update_pte = nonpaging_update_pte;
3570
	context->root_level = 0;
A
Avi Kivity 已提交
3571
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
3572
	context->root_hpa = INVALID_PAGE;
3573
	context->direct_map = true;
3574
	context->nx = false;
A
Avi Kivity 已提交
3575 3576
}

3577
void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3578
{
3579
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
3580 3581
}

3582 3583
static unsigned long get_cr3(struct kvm_vcpu *vcpu)
{
3584
	return kvm_read_cr3(vcpu);
3585 3586
}

3587 3588
static void inject_page_fault(struct kvm_vcpu *vcpu,
			      struct x86_exception *fault)
A
Avi Kivity 已提交
3589
{
3590
	vcpu->arch.mmu.inject_page_fault(vcpu, fault);
A
Avi Kivity 已提交
3591 3592
}

3593 3594
static bool sync_mmio_spte(struct kvm *kvm, u64 *sptep, gfn_t gfn,
			   unsigned access, int *nr_present)
3595 3596 3597 3598 3599 3600 3601 3602
{
	if (unlikely(is_mmio_spte(*sptep))) {
		if (gfn != get_mmio_spte_gfn(*sptep)) {
			mmu_spte_clear_no_track(sptep);
			return true;
		}

		(*nr_present)++;
3603
		mark_mmio_spte(kvm, sptep, gfn, access);
3604 3605 3606 3607 3608 3609
		return true;
	}

	return false;
}

A
Avi Kivity 已提交
3610 3611 3612 3613 3614 3615 3616 3617 3618
static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)
{
	unsigned index;

	index = level - 1;
	index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
	return mmu->last_pte_bitmap & (1 << index);
}

3619 3620 3621 3622 3623
#define PTTYPE_EPT 18 /* arbitrary */
#define PTTYPE PTTYPE_EPT
#include "paging_tmpl.h"
#undef PTTYPE

A
Avi Kivity 已提交
3624 3625 3626 3627 3628 3629 3630 3631
#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

3632
static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3633
				  struct kvm_mmu *context)
3634 3635 3636
{
	int maxphyaddr = cpuid_maxphyaddr(vcpu);
	u64 exb_bit_rsvd = 0;
3637
	u64 gbpages_bit_rsvd = 0;
3638
	u64 nonleaf_bit8_rsvd = 0;
3639

3640 3641
	context->bad_mt_xwr = 0;

3642
	if (!context->nx)
3643
		exb_bit_rsvd = rsvd_bits(63, 63);
3644 3645
	if (!guest_cpuid_has_gbpages(vcpu))
		gbpages_bit_rsvd = rsvd_bits(7, 7);
3646 3647 3648 3649 3650 3651 3652 3653

	/*
	 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
	 * leaf entries) on AMD CPUs only.
	 */
	if (guest_cpuid_is_amd(vcpu))
		nonleaf_bit8_rsvd = rsvd_bits(8, 8);

3654
	switch (context->root_level) {
3655 3656 3657 3658
	case PT32_ROOT_LEVEL:
		/* no rsvd bits for 2 level 4K page table entries */
		context->rsvd_bits_mask[0][1] = 0;
		context->rsvd_bits_mask[0][0] = 0;
3659 3660 3661 3662 3663 3664 3665
		context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];

		if (!is_pse(vcpu)) {
			context->rsvd_bits_mask[1][1] = 0;
			break;
		}

3666 3667 3668 3669 3670 3671 3672 3673
		if (is_cpuid_PSE36())
			/* 36bits PSE 4MB page */
			context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
		else
			/* 32 bits PSE 4MB page */
			context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
		break;
	case PT32E_ROOT_LEVEL:
3674 3675
		context->rsvd_bits_mask[0][2] =
			rsvd_bits(maxphyaddr, 63) |
3676
			rsvd_bits(5, 8) | rsvd_bits(1, 2);	/* PDPTE */
3677
		context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3678
			rsvd_bits(maxphyaddr, 62);	/* PDE */
3679 3680 3681 3682 3683
		context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 62); 	/* PTE */
		context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 62) |
			rsvd_bits(13, 20);		/* large page */
3684
		context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
3685 3686 3687
		break;
	case PT64_ROOT_LEVEL:
		context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
3688
			nonleaf_bit8_rsvd | rsvd_bits(7, 7) | rsvd_bits(maxphyaddr, 51);
3689
		context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
3690
			nonleaf_bit8_rsvd | gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51);
3691
		context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3692
			rsvd_bits(maxphyaddr, 51);
3693 3694 3695
		context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51);
		context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
3696
		context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
3697
			gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51) |
3698
			rsvd_bits(13, 29);
3699
		context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
3700 3701
			rsvd_bits(maxphyaddr, 51) |
			rsvd_bits(13, 20);		/* large page */
3702
		context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
3703 3704 3705 3706
		break;
	}
}

3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738
static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
		struct kvm_mmu *context, bool execonly)
{
	int maxphyaddr = cpuid_maxphyaddr(vcpu);
	int pte;

	context->rsvd_bits_mask[0][3] =
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7);
	context->rsvd_bits_mask[0][2] =
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
	context->rsvd_bits_mask[0][1] =
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
	context->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51);

	/* large page */
	context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
	context->rsvd_bits_mask[1][2] =
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 29);
	context->rsvd_bits_mask[1][1] =
		rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 20);
	context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];

	for (pte = 0; pte < 64; pte++) {
		int rwx_bits = pte & 7;
		int mt = pte >> 3;
		if (mt == 0x2 || mt == 0x3 || mt == 0x7 ||
				rwx_bits == 0x2 || rwx_bits == 0x6 ||
				(rwx_bits == 0x4 && !execonly))
			context->bad_mt_xwr |= (1ull << pte);
	}
}

3739 3740
static void update_permission_bitmask(struct kvm_vcpu *vcpu,
				      struct kvm_mmu *mmu, bool ept)
3741 3742 3743
{
	unsigned bit, byte, pfec;
	u8 map;
F
Feng Wu 已提交
3744
	bool fault, x, w, u, wf, uf, ff, smapf, cr4_smap, cr4_smep, smap = 0;
3745

F
Feng Wu 已提交
3746
	cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
F
Feng Wu 已提交
3747
	cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
3748 3749 3750 3751 3752 3753
	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
		pfec = byte << 1;
		map = 0;
		wf = pfec & PFERR_WRITE_MASK;
		uf = pfec & PFERR_USER_MASK;
		ff = pfec & PFERR_FETCH_MASK;
F
Feng Wu 已提交
3754 3755 3756 3757 3758 3759
		/*
		 * PFERR_RSVD_MASK bit is set in PFEC if the access is not
		 * subject to SMAP restrictions, and cleared otherwise. The
		 * bit is only meaningful if the SMAP bit is set in CR4.
		 */
		smapf = !(pfec & PFERR_RSVD_MASK);
3760 3761 3762 3763 3764
		for (bit = 0; bit < 8; ++bit) {
			x = bit & ACC_EXEC_MASK;
			w = bit & ACC_WRITE_MASK;
			u = bit & ACC_USER_MASK;

3765 3766 3767 3768 3769 3770
			if (!ept) {
				/* Not really needed: !nx will cause pte.nx to fault */
				x |= !mmu->nx;
				/* Allow supervisor writes if !cr0.wp */
				w |= !is_write_protection(vcpu) && !uf;
				/* Disallow supervisor fetches of user code if cr4.smep */
F
Feng Wu 已提交
3771
				x &= !(cr4_smep && u && !uf);
F
Feng Wu 已提交
3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791

				/*
				 * SMAP:kernel-mode data accesses from user-mode
				 * mappings should fault. A fault is considered
				 * as a SMAP violation if all of the following
				 * conditions are ture:
				 *   - X86_CR4_SMAP is set in CR4
				 *   - An user page is accessed
				 *   - Page fault in kernel mode
				 *   - if CPL = 3 or X86_EFLAGS_AC is clear
				 *
				 *   Here, we cover the first three conditions.
				 *   The fourth is computed dynamically in
				 *   permission_fault() and is in smapf.
				 *
				 *   Also, SMAP does not affect instruction
				 *   fetches, add the !ff check here to make it
				 *   clearer.
				 */
				smap = cr4_smap && u && !uf && !ff;
3792 3793 3794
			} else
				/* Not really needed: no U/S accesses on ept  */
				u = 1;
3795

F
Feng Wu 已提交
3796 3797
			fault = (ff && !x) || (uf && !u) || (wf && !w) ||
				(smapf && smap);
3798 3799 3800 3801 3802 3803
			map |= fault << bit;
		}
		mmu->permissions[byte] = map;
	}
}

A
Avi Kivity 已提交
3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821
static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
{
	u8 map;
	unsigned level, root_level = mmu->root_level;
	const unsigned ps_set_index = 1 << 2;  /* bit 2 of index: ps */

	if (root_level == PT32E_ROOT_LEVEL)
		--root_level;
	/* PT_PAGE_TABLE_LEVEL always terminates */
	map = 1 | (1 << ps_set_index);
	for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
		if (level <= PT_PDPE_LEVEL
		    && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
			map |= 1 << (ps_set_index | (level - 1));
	}
	mmu->last_pte_bitmap = map;
}

3822 3823 3824
static void paging64_init_context_common(struct kvm_vcpu *vcpu,
					 struct kvm_mmu *context,
					 int level)
A
Avi Kivity 已提交
3825
{
3826
	context->nx = is_nx(vcpu);
3827
	context->root_level = level;
3828

3829
	reset_rsvds_bits_mask(vcpu, context);
3830
	update_permission_bitmask(vcpu, context, false);
A
Avi Kivity 已提交
3831
	update_last_pte_bitmap(vcpu, context);
A
Avi Kivity 已提交
3832

3833
	MMU_WARN_ON(!is_pae(vcpu));
A
Avi Kivity 已提交
3834 3835
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
3836
	context->sync_page = paging64_sync_page;
M
Marcelo Tosatti 已提交
3837
	context->invlpg = paging64_invlpg;
3838
	context->update_pte = paging64_update_pte;
3839
	context->shadow_root_level = level;
A
Avi Kivity 已提交
3840
	context->root_hpa = INVALID_PAGE;
3841
	context->direct_map = false;
A
Avi Kivity 已提交
3842 3843
}

3844 3845
static void paging64_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
3846
{
3847
	paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
3848 3849
}

3850 3851
static void paging32_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
A
Avi Kivity 已提交
3852
{
3853
	context->nx = false;
3854
	context->root_level = PT32_ROOT_LEVEL;
3855

3856
	reset_rsvds_bits_mask(vcpu, context);
3857
	update_permission_bitmask(vcpu, context, false);
A
Avi Kivity 已提交
3858
	update_last_pte_bitmap(vcpu, context);
A
Avi Kivity 已提交
3859 3860 3861

	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
3862
	context->sync_page = paging32_sync_page;
M
Marcelo Tosatti 已提交
3863
	context->invlpg = paging32_invlpg;
3864
	context->update_pte = paging32_update_pte;
A
Avi Kivity 已提交
3865
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
3866
	context->root_hpa = INVALID_PAGE;
3867
	context->direct_map = false;
A
Avi Kivity 已提交
3868 3869
}

3870 3871
static void paging32E_init_context(struct kvm_vcpu *vcpu,
				   struct kvm_mmu *context)
A
Avi Kivity 已提交
3872
{
3873
	paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
A
Avi Kivity 已提交
3874 3875
}

3876
static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
3877
{
3878
	struct kvm_mmu *context = &vcpu->arch.mmu;
3879

3880
	context->base_role.word = 0;
3881
	context->page_fault = tdp_page_fault;
3882
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
3883
	context->invlpg = nonpaging_invlpg;
3884
	context->update_pte = nonpaging_update_pte;
3885
	context->shadow_root_level = kvm_x86_ops->get_tdp_level();
3886
	context->root_hpa = INVALID_PAGE;
3887
	context->direct_map = true;
3888
	context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
3889
	context->get_cr3 = get_cr3;
3890
	context->get_pdptr = kvm_pdptr_read;
3891
	context->inject_page_fault = kvm_inject_page_fault;
3892 3893

	if (!is_paging(vcpu)) {
3894
		context->nx = false;
3895 3896 3897
		context->gva_to_gpa = nonpaging_gva_to_gpa;
		context->root_level = 0;
	} else if (is_long_mode(vcpu)) {
3898
		context->nx = is_nx(vcpu);
3899
		context->root_level = PT64_ROOT_LEVEL;
3900 3901
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging64_gva_to_gpa;
3902
	} else if (is_pae(vcpu)) {
3903
		context->nx = is_nx(vcpu);
3904
		context->root_level = PT32E_ROOT_LEVEL;
3905 3906
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging64_gva_to_gpa;
3907
	} else {
3908
		context->nx = false;
3909
		context->root_level = PT32_ROOT_LEVEL;
3910 3911
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging32_gva_to_gpa;
3912 3913
	}

3914
	update_permission_bitmask(vcpu, context, false);
A
Avi Kivity 已提交
3915
	update_last_pte_bitmap(vcpu, context);
3916 3917
}

3918
void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3919
{
3920
	bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
3921
	bool smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
3922 3923
	struct kvm_mmu *context = &vcpu->arch.mmu;

3924
	MMU_WARN_ON(VALID_PAGE(context->root_hpa));
A
Avi Kivity 已提交
3925 3926

	if (!is_paging(vcpu))
3927
		nonpaging_init_context(vcpu, context);
A
Avi Kivity 已提交
3928
	else if (is_long_mode(vcpu))
3929
		paging64_init_context(vcpu, context);
A
Avi Kivity 已提交
3930
	else if (is_pae(vcpu))
3931
		paging32E_init_context(vcpu, context);
A
Avi Kivity 已提交
3932
	else
3933
		paging32_init_context(vcpu, context);
3934

3935 3936 3937 3938
	context->base_role.nxe = is_nx(vcpu);
	context->base_role.cr4_pae = !!is_pae(vcpu);
	context->base_role.cr0_wp  = is_write_protection(vcpu);
	context->base_role.smep_andnot_wp
3939
		= smep && !is_write_protection(vcpu);
3940 3941
	context->base_role.smap_andnot_wp
		= smap && !is_write_protection(vcpu);
3942 3943 3944
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);

3945
void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly)
N
Nadav Har'El 已提交
3946
{
3947 3948
	struct kvm_mmu *context = &vcpu->arch.mmu;

3949
	MMU_WARN_ON(VALID_PAGE(context->root_hpa));
N
Nadav Har'El 已提交
3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967

	context->shadow_root_level = kvm_x86_ops->get_tdp_level();

	context->nx = true;
	context->page_fault = ept_page_fault;
	context->gva_to_gpa = ept_gva_to_gpa;
	context->sync_page = ept_sync_page;
	context->invlpg = ept_invlpg;
	context->update_pte = ept_update_pte;
	context->root_level = context->shadow_root_level;
	context->root_hpa = INVALID_PAGE;
	context->direct_map = false;

	update_permission_bitmask(vcpu, context, true);
	reset_rsvds_bits_mask_ept(vcpu, context, execonly);
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);

3968
static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
3969
{
3970 3971 3972 3973 3974 3975 3976
	struct kvm_mmu *context = &vcpu->arch.mmu;

	kvm_init_shadow_mmu(vcpu);
	context->set_cr3           = kvm_x86_ops->set_cr3;
	context->get_cr3           = get_cr3;
	context->get_pdptr         = kvm_pdptr_read;
	context->inject_page_fault = kvm_inject_page_fault;
A
Avi Kivity 已提交
3977 3978
}

3979
static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
3980 3981 3982 3983
{
	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;

	g_context->get_cr3           = get_cr3;
3984
	g_context->get_pdptr         = kvm_pdptr_read;
3985 3986 3987 3988 3989 3990 3991 3992 3993
	g_context->inject_page_fault = kvm_inject_page_fault;

	/*
	 * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
	 * translation of l2_gpa to l1_gpa addresses is done using the
	 * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
	 * functions between mmu and nested_mmu are swapped.
	 */
	if (!is_paging(vcpu)) {
3994
		g_context->nx = false;
3995 3996 3997
		g_context->root_level = 0;
		g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
	} else if (is_long_mode(vcpu)) {
3998
		g_context->nx = is_nx(vcpu);
3999
		g_context->root_level = PT64_ROOT_LEVEL;
4000
		reset_rsvds_bits_mask(vcpu, g_context);
4001 4002
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else if (is_pae(vcpu)) {
4003
		g_context->nx = is_nx(vcpu);
4004
		g_context->root_level = PT32E_ROOT_LEVEL;
4005
		reset_rsvds_bits_mask(vcpu, g_context);
4006 4007
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else {
4008
		g_context->nx = false;
4009
		g_context->root_level = PT32_ROOT_LEVEL;
4010
		reset_rsvds_bits_mask(vcpu, g_context);
4011 4012 4013
		g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
	}

4014
	update_permission_bitmask(vcpu, g_context, false);
A
Avi Kivity 已提交
4015
	update_last_pte_bitmap(vcpu, g_context);
4016 4017
}

4018
static void init_kvm_mmu(struct kvm_vcpu *vcpu)
4019
{
4020
	if (mmu_is_nested(vcpu))
4021
		init_kvm_nested_mmu(vcpu);
4022
	else if (tdp_enabled)
4023
		init_kvm_tdp_mmu(vcpu);
4024
	else
4025
		init_kvm_softmmu(vcpu);
4026 4027
}

4028
void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4029
{
4030
	kvm_mmu_unload(vcpu);
4031
	init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
4032
}
4033
EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
A
Avi Kivity 已提交
4034 4035

int kvm_mmu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4036
{
4037 4038
	int r;

4039
	r = mmu_topup_memory_caches(vcpu);
A
Avi Kivity 已提交
4040 4041
	if (r)
		goto out;
4042
	r = mmu_alloc_roots(vcpu);
4043
	kvm_mmu_sync_roots(vcpu);
4044 4045
	if (r)
		goto out;
4046
	/* set_cr3() should ensure TLB has been flushed */
4047
	vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
4048 4049
out:
	return r;
A
Avi Kivity 已提交
4050
}
A
Avi Kivity 已提交
4051 4052 4053 4054 4055
EXPORT_SYMBOL_GPL(kvm_mmu_load);

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
	mmu_free_roots(vcpu);
4056
	WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
4057
}
4058
EXPORT_SYMBOL_GPL(kvm_mmu_unload);
A
Avi Kivity 已提交
4059

4060
static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
4061 4062
				  struct kvm_mmu_page *sp, u64 *spte,
				  const void *new)
4063
{
4064
	if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
4065 4066
		++vcpu->kvm->stat.mmu_pde_zapped;
		return;
4067
        }
4068

A
Avi Kivity 已提交
4069
	++vcpu->kvm->stat.mmu_pte_updated;
4070
	vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
4071 4072
}

4073 4074 4075 4076 4077 4078 4079 4080
static bool need_remote_flush(u64 old, u64 new)
{
	if (!is_shadow_present_pte(old))
		return false;
	if (!is_shadow_present_pte(new))
		return true;
	if ((old ^ new) & PT64_BASE_ADDR_MASK)
		return true;
4081 4082
	old ^= shadow_nx_mask;
	new ^= shadow_nx_mask;
4083 4084 4085
	return (old & ~new & PT64_PERM_MASK) != 0;
}

4086 4087
static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
				    bool remote_flush, bool local_flush)
4088
{
4089 4090 4091 4092
	if (zap_page)
		return;

	if (remote_flush)
4093
		kvm_flush_remote_tlbs(vcpu->kvm);
4094
	else if (local_flush)
4095
		kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4096 4097
}

4098 4099
static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
				    const u8 *new, int *bytes)
4100
{
4101 4102
	u64 gentry;
	int r;
4103 4104 4105

	/*
	 * Assume that the pte write on a page table of the same type
4106 4107
	 * as the current vcpu paging mode since we update the sptes only
	 * when they have the same mode.
4108
	 */
4109
	if (is_pae(vcpu) && *bytes == 4) {
4110
		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */
4111 4112
		*gpa &= ~(gpa_t)7;
		*bytes = 8;
4113
		r = kvm_read_guest(vcpu->kvm, *gpa, &gentry, 8);
4114 4115
		if (r)
			gentry = 0;
4116 4117 4118
		new = (const u8 *)&gentry;
	}

4119
	switch (*bytes) {
4120 4121 4122 4123 4124 4125 4126 4127 4128
	case 4:
		gentry = *(const u32 *)new;
		break;
	case 8:
		gentry = *(const u64 *)new;
		break;
	default:
		gentry = 0;
		break;
4129 4130
	}

4131 4132 4133 4134 4135 4136 4137
	return gentry;
}

/*
 * If we're seeing too many writes to a page, it may no longer be a page table,
 * or we may be forking, in which case it is better to unmap the page.
 */
4138
static bool detect_write_flooding(struct kvm_mmu_page *sp)
4139
{
4140 4141 4142 4143
	/*
	 * Skip write-flooding detected for the sp whose level is 1, because
	 * it can become unsync, then the guest page is not write-protected.
	 */
4144
	if (sp->role.level == PT_PAGE_TABLE_LEVEL)
4145
		return false;
4146

4147
	return ++sp->write_flooding_count >= 3;
4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163
}

/*
 * Misaligned accesses are too much trouble to fix up; also, they usually
 * indicate a page is not used as a page table.
 */
static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
				    int bytes)
{
	unsigned offset, pte_size, misaligned;

	pgprintk("misaligned: gpa %llx bytes %d role %x\n",
		 gpa, bytes, sp->role.word);

	offset = offset_in_page(gpa);
	pte_size = sp->role.cr4_pae ? 8 : 4;
4164 4165 4166 4167 4168 4169 4170 4171

	/*
	 * Sometimes, the OS only writes the last one bytes to update status
	 * bits, for example, in linux, andb instruction is used in clear_bit().
	 */
	if (!(offset & (pte_size - 1)) && bytes == 1)
		return false;

4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216
	misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
	misaligned |= bytes < 4;

	return misaligned;
}

static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
{
	unsigned page_offset, quadrant;
	u64 *spte;
	int level;

	page_offset = offset_in_page(gpa);
	level = sp->role.level;
	*nspte = 1;
	if (!sp->role.cr4_pae) {
		page_offset <<= 1;	/* 32->64 */
		/*
		 * A 32-bit pde maps 4MB while the shadow pdes map
		 * only 2MB.  So we need to double the offset again
		 * and zap two pdes instead of one.
		 */
		if (level == PT32_ROOT_LEVEL) {
			page_offset &= ~7; /* kill rounding error */
			page_offset <<= 1;
			*nspte = 2;
		}
		quadrant = page_offset >> PAGE_SHIFT;
		page_offset &= ~PAGE_MASK;
		if (quadrant != sp->role.quadrant)
			return NULL;
	}

	spte = &sp->spt[page_offset / sizeof(*spte)];
	return spte;
}

void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
		       const u8 *new, int bytes)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_mmu_page *sp;
	LIST_HEAD(invalid_list);
	u64 entry, gentry, *spte;
	int npte;
4217
	bool remote_flush, local_flush, zap_page;
4218 4219 4220 4221 4222 4223 4224
	union kvm_mmu_page_role mask = (union kvm_mmu_page_role) {
		.cr0_wp = 1,
		.cr4_pae = 1,
		.nxe = 1,
		.smep_andnot_wp = 1,
		.smap_andnot_wp = 1,
	};
4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247

	/*
	 * If we don't have indirect shadow pages, it means no page is
	 * write-protected, so we can exit simply.
	 */
	if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
		return;

	zap_page = remote_flush = local_flush = false;

	pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);

	gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, new, &bytes);

	/*
	 * No need to care whether allocation memory is successful
	 * or not since pte prefetch is skiped if it does not have
	 * enough objects in the cache.
	 */
	mmu_topup_memory_caches(vcpu);

	spin_lock(&vcpu->kvm->mmu_lock);
	++vcpu->kvm->stat.mmu_pte_write;
4248
	kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
4249

4250
	for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
4251
		if (detect_write_misaligned(sp, gpa, bytes) ||
4252
		      detect_write_flooding(sp)) {
4253
			zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
4254
						     &invalid_list);
A
Avi Kivity 已提交
4255
			++vcpu->kvm->stat.mmu_flooded;
4256 4257
			continue;
		}
4258 4259 4260 4261 4262

		spte = get_written_sptes(sp, gpa, &npte);
		if (!spte)
			continue;

4263
		local_flush = true;
4264
		while (npte--) {
4265
			entry = *spte;
4266
			mmu_page_zap_pte(vcpu->kvm, sp, spte);
4267 4268
			if (gentry &&
			      !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
4269
			      & mask.word) && rmap_can_add(vcpu))
4270
				mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
G
Gleb Natapov 已提交
4271
			if (need_remote_flush(entry, *spte))
4272
				remote_flush = true;
4273
			++spte;
4274 4275
		}
	}
4276
	mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
4277
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
4278
	kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
4279
	spin_unlock(&vcpu->kvm->mmu_lock);
4280 4281
}

4282 4283
int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
4284 4285
	gpa_t gpa;
	int r;
4286

4287
	if (vcpu->arch.mmu.direct_map)
4288 4289
		return 0;

4290
	gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
4291 4292

	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
4293

4294
	return r;
4295
}
4296
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
4297

4298
static void make_mmu_pages_available(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4299
{
4300
	LIST_HEAD(invalid_list);
4301

4302 4303 4304
	if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES))
		return;

4305 4306 4307
	while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) {
		if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list))
			break;
A
Avi Kivity 已提交
4308

A
Avi Kivity 已提交
4309
		++vcpu->kvm->stat.mmu_recycled;
A
Avi Kivity 已提交
4310
	}
4311
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
A
Avi Kivity 已提交
4312 4313
}

4314 4315 4316 4317 4318 4319 4320 4321
static bool is_mmio_page_fault(struct kvm_vcpu *vcpu, gva_t addr)
{
	if (vcpu->arch.mmu.direct_map || mmu_is_nested(vcpu))
		return vcpu_match_mmio_gpa(vcpu, addr);

	return vcpu_match_mmio_gva(vcpu, addr);
}

4322 4323
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
		       void *insn, int insn_len)
4324
{
4325
	int r, emulation_type = EMULTYPE_RETRY;
4326 4327
	enum emulation_result er;

G
Gleb Natapov 已提交
4328
	r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
4329 4330 4331 4332 4333 4334 4335 4336
	if (r < 0)
		goto out;

	if (!r) {
		r = 1;
		goto out;
	}

4337 4338 4339 4340
	if (is_mmio_page_fault(vcpu, cr2))
		emulation_type = 0;

	er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
4341 4342 4343 4344

	switch (er) {
	case EMULATE_DONE:
		return 1;
P
Paolo Bonzini 已提交
4345
	case EMULATE_USER_EXIT:
4346
		++vcpu->stat.mmio_exits;
4347
		/* fall through */
4348
	case EMULATE_FAIL:
4349
		return 0;
4350 4351 4352 4353 4354 4355 4356 4357
	default:
		BUG();
	}
out:
	return r;
}
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);

M
Marcelo Tosatti 已提交
4358 4359 4360
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
	vcpu->arch.mmu.invlpg(vcpu, gva);
4361
	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
M
Marcelo Tosatti 已提交
4362 4363 4364 4365
	++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);

4366 4367 4368 4369 4370 4371
void kvm_enable_tdp(void)
{
	tdp_enabled = true;
}
EXPORT_SYMBOL_GPL(kvm_enable_tdp);

4372 4373 4374 4375 4376 4377
void kvm_disable_tdp(void)
{
	tdp_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_tdp);

A
Avi Kivity 已提交
4378 4379
static void free_mmu_pages(struct kvm_vcpu *vcpu)
{
4380
	free_page((unsigned long)vcpu->arch.mmu.pae_root);
4381 4382
	if (vcpu->arch.mmu.lm_root != NULL)
		free_page((unsigned long)vcpu->arch.mmu.lm_root);
A
Avi Kivity 已提交
4383 4384 4385 4386
}

static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
{
4387
	struct page *page;
A
Avi Kivity 已提交
4388 4389
	int i;

4390 4391 4392 4393 4394 4395 4396
	/*
	 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
	 * Therefore we need to allocate shadow page tables in the first
	 * 4GB of memory, which happens to fit the DMA32 zone.
	 */
	page = alloc_page(GFP_KERNEL | __GFP_DMA32);
	if (!page)
4397 4398
		return -ENOMEM;

4399
	vcpu->arch.mmu.pae_root = page_address(page);
4400
	for (i = 0; i < 4; ++i)
4401
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
4402

A
Avi Kivity 已提交
4403 4404 4405
	return 0;
}

4406
int kvm_mmu_create(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4407
{
4408 4409 4410 4411
	vcpu->arch.walk_mmu = &vcpu->arch.mmu;
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
	vcpu->arch.mmu.translate_gpa = translate_gpa;
	vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
A
Avi Kivity 已提交
4412

4413 4414
	return alloc_mmu_pages(vcpu);
}
A
Avi Kivity 已提交
4415

4416
void kvm_mmu_setup(struct kvm_vcpu *vcpu)
4417
{
4418
	MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
4419

4420
	init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
4421 4422
}

4423 4424
void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
				      struct kvm_memory_slot *memslot)
A
Avi Kivity 已提交
4425
{
4426 4427
	gfn_t last_gfn;
	int i;
4428
	bool flush = false;
A
Avi Kivity 已提交
4429

4430
	last_gfn = memslot->base_gfn + memslot->npages - 1;
A
Avi Kivity 已提交
4431

4432 4433
	spin_lock(&kvm->mmu_lock);

4434 4435 4436 4437
	for (i = PT_PAGE_TABLE_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
		unsigned long *rmapp;
		unsigned long last_index, index;
A
Avi Kivity 已提交
4438

4439 4440
		rmapp = memslot->arch.rmap[i - PT_PAGE_TABLE_LEVEL];
		last_index = gfn_to_index(last_gfn, memslot->base_gfn, i);
4441

4442 4443
		for (index = 0; index <= last_index; ++index, ++rmapp) {
			if (*rmapp)
4444 4445
				flush |= __rmap_write_protect(kvm, rmapp,
						false);
4446

4447
			if (need_resched() || spin_needbreak(&kvm->mmu_lock))
4448
				cond_resched_lock(&kvm->mmu_lock);
4449
		}
A
Avi Kivity 已提交
4450
	}
4451

4452
	spin_unlock(&kvm->mmu_lock);
4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471

	/*
	 * kvm_mmu_slot_remove_write_access() and kvm_vm_ioctl_get_dirty_log()
	 * which do tlb flush out of mmu-lock should be serialized by
	 * kvm->slots_lock otherwise tlb flush would be missed.
	 */
	lockdep_assert_held(&kvm->slots_lock);

	/*
	 * We can flush all the TLBs out of the mmu lock without TLB
	 * corruption since we just change the spte from writable to
	 * readonly so that we only need to care the case of changing
	 * spte from present to present (changing the spte from present
	 * to nonpresent will flush all the TLBs immediately), in other
	 * words, the only case we care is mmu_spte_update() where we
	 * haved checked SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE
	 * instead of PT_WRITABLE_MASK, that means it does not depend
	 * on PT_WRITABLE_MASK anymore.
	 */
4472 4473
	if (flush)
		kvm_flush_remote_tlbs(kvm);
A
Avi Kivity 已提交
4474
}
4475

4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491
static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
		unsigned long *rmapp)
{
	u64 *sptep;
	struct rmap_iterator iter;
	int need_tlb_flush = 0;
	pfn_t pfn;
	struct kvm_mmu_page *sp;

	for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
		BUG_ON(!(*sptep & PT_PRESENT_MASK));

		sp = page_header(__pa(sptep));
		pfn = spte_to_pfn(*sptep);

		/*
4492 4493 4494 4495 4496
		 * We cannot do huge page mapping for indirect shadow pages,
		 * which are found on the last rmap (level = 1) when not using
		 * tdp; such shadow pages are synced with the page table in
		 * the guest, and the guest page table is using 4K page size
		 * mapping if the indirect sp has level = 1.
4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520
		 */
		if (sp->role.direct &&
			!kvm_is_reserved_pfn(pfn) &&
			PageTransCompound(pfn_to_page(pfn))) {
			drop_spte(kvm, sptep);
			sptep = rmap_get_first(*rmapp, &iter);
			need_tlb_flush = 1;
		} else
			sptep = rmap_get_next(&iter);
	}

	return need_tlb_flush;
}

void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
			struct kvm_memory_slot *memslot)
{
	bool flush = false;
	unsigned long *rmapp;
	unsigned long last_index, index;

	spin_lock(&kvm->mmu_lock);

	rmapp = memslot->arch.rmap[0];
4521 4522
	last_index = gfn_to_index(memslot->base_gfn + memslot->npages - 1,
				memslot->base_gfn, PT_PAGE_TABLE_LEVEL);
4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542

	for (index = 0; index <= last_index; ++index, ++rmapp) {
		if (*rmapp)
			flush |= kvm_mmu_zap_collapsible_spte(kvm, rmapp);

		if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
			if (flush) {
				kvm_flush_remote_tlbs(kvm);
				flush = false;
			}
			cond_resched_lock(&kvm->mmu_lock);
		}
	}

	if (flush)
		kvm_flush_remote_tlbs(kvm);

	spin_unlock(&kvm->mmu_lock);
}

4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657
void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
				   struct kvm_memory_slot *memslot)
{
	gfn_t last_gfn;
	unsigned long *rmapp;
	unsigned long last_index, index;
	bool flush = false;

	last_gfn = memslot->base_gfn + memslot->npages - 1;

	spin_lock(&kvm->mmu_lock);

	rmapp = memslot->arch.rmap[PT_PAGE_TABLE_LEVEL - 1];
	last_index = gfn_to_index(last_gfn, memslot->base_gfn,
			PT_PAGE_TABLE_LEVEL);

	for (index = 0; index <= last_index; ++index, ++rmapp) {
		if (*rmapp)
			flush |= __rmap_clear_dirty(kvm, rmapp);

		if (need_resched() || spin_needbreak(&kvm->mmu_lock))
			cond_resched_lock(&kvm->mmu_lock);
	}

	spin_unlock(&kvm->mmu_lock);

	lockdep_assert_held(&kvm->slots_lock);

	/*
	 * It's also safe to flush TLBs out of mmu lock here as currently this
	 * function is only used for dirty logging, in which case flushing TLB
	 * out of mmu lock also guarantees no dirty pages will be lost in
	 * dirty_bitmap.
	 */
	if (flush)
		kvm_flush_remote_tlbs(kvm);
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty);

void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
					struct kvm_memory_slot *memslot)
{
	gfn_t last_gfn;
	int i;
	bool flush = false;

	last_gfn = memslot->base_gfn + memslot->npages - 1;

	spin_lock(&kvm->mmu_lock);

	for (i = PT_PAGE_TABLE_LEVEL + 1; /* skip rmap for 4K page */
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
		unsigned long *rmapp;
		unsigned long last_index, index;

		rmapp = memslot->arch.rmap[i - PT_PAGE_TABLE_LEVEL];
		last_index = gfn_to_index(last_gfn, memslot->base_gfn, i);

		for (index = 0; index <= last_index; ++index, ++rmapp) {
			if (*rmapp)
				flush |= __rmap_write_protect(kvm, rmapp,
						false);

			if (need_resched() || spin_needbreak(&kvm->mmu_lock))
				cond_resched_lock(&kvm->mmu_lock);
		}
	}
	spin_unlock(&kvm->mmu_lock);

	/* see kvm_mmu_slot_remove_write_access */
	lockdep_assert_held(&kvm->slots_lock);

	if (flush)
		kvm_flush_remote_tlbs(kvm);
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access);

void kvm_mmu_slot_set_dirty(struct kvm *kvm,
			    struct kvm_memory_slot *memslot)
{
	gfn_t last_gfn;
	int i;
	bool flush = false;

	last_gfn = memslot->base_gfn + memslot->npages - 1;

	spin_lock(&kvm->mmu_lock);

	for (i = PT_PAGE_TABLE_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
		unsigned long *rmapp;
		unsigned long last_index, index;

		rmapp = memslot->arch.rmap[i - PT_PAGE_TABLE_LEVEL];
		last_index = gfn_to_index(last_gfn, memslot->base_gfn, i);

		for (index = 0; index <= last_index; ++index, ++rmapp) {
			if (*rmapp)
				flush |= __rmap_set_dirty(kvm, rmapp);

			if (need_resched() || spin_needbreak(&kvm->mmu_lock))
				cond_resched_lock(&kvm->mmu_lock);
		}
	}

	spin_unlock(&kvm->mmu_lock);

	lockdep_assert_held(&kvm->slots_lock);

	/* see kvm_mmu_slot_leaf_clear_dirty */
	if (flush)
		kvm_flush_remote_tlbs(kvm);
}
EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty);

X
Xiao Guangrong 已提交
4658
#define BATCH_ZAP_PAGES	10
4659 4660 4661
static void kvm_zap_obsolete_pages(struct kvm *kvm)
{
	struct kvm_mmu_page *sp, *node;
X
Xiao Guangrong 已提交
4662
	int batch = 0;
4663 4664 4665 4666

restart:
	list_for_each_entry_safe_reverse(sp, node,
	      &kvm->arch.active_mmu_pages, link) {
X
Xiao Guangrong 已提交
4667 4668
		int ret;

4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683
		/*
		 * No obsolete page exists before new created page since
		 * active_mmu_pages is the FIFO list.
		 */
		if (!is_obsolete_sp(kvm, sp))
			break;

		/*
		 * Since we are reversely walking the list and the invalid
		 * list will be moved to the head, skip the invalid page
		 * can help us to avoid the infinity list walking.
		 */
		if (sp->role.invalid)
			continue;

4684 4685 4686 4687
		/*
		 * Need not flush tlb since we only zap the sp with invalid
		 * generation number.
		 */
X
Xiao Guangrong 已提交
4688
		if (batch >= BATCH_ZAP_PAGES &&
4689
		      cond_resched_lock(&kvm->mmu_lock)) {
X
Xiao Guangrong 已提交
4690
			batch = 0;
4691 4692 4693
			goto restart;
		}

4694 4695
		ret = kvm_mmu_prepare_zap_page(kvm, sp,
				&kvm->arch.zapped_obsolete_pages);
X
Xiao Guangrong 已提交
4696 4697 4698
		batch += ret;

		if (ret)
4699 4700 4701
			goto restart;
	}

4702 4703 4704 4705
	/*
	 * Should flush tlb before free page tables since lockless-walking
	 * may use the pages.
	 */
4706
	kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720
}

/*
 * Fast invalidate all shadow pages and use lock-break technique
 * to zap obsolete pages.
 *
 * It's required when memslot is being deleted or VM is being
 * destroyed, in these cases, we should ensure that KVM MMU does
 * not use any resource of the being-deleted slot or all slots
 * after calling the function.
 */
void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm)
{
	spin_lock(&kvm->mmu_lock);
4721
	trace_kvm_mmu_invalidate_zap_all_pages(kvm);
4722 4723
	kvm->arch.mmu_valid_gen++;

4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734
	/*
	 * Notify all vcpus to reload its shadow page table
	 * and flush TLB. Then all vcpus will switch to new
	 * shadow page table with the new mmu_valid_gen.
	 *
	 * Note: we should do this under the protection of
	 * mmu-lock, otherwise, vcpu would purge shadow page
	 * but miss tlb flush.
	 */
	kvm_reload_remote_mmus(kvm);

4735 4736 4737 4738
	kvm_zap_obsolete_pages(kvm);
	spin_unlock(&kvm->mmu_lock);
}

4739 4740 4741 4742 4743
static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
{
	return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
}

4744 4745 4746 4747 4748 4749
void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm)
{
	/*
	 * The very rare case: if the generation-number is round,
	 * zap all shadow pages.
	 */
4750
	if (unlikely(kvm_current_mmio_generation(kvm) == 0)) {
4751
		printk_ratelimited(KERN_DEBUG "kvm: zapping shadow pages for mmio generation wraparound\n");
4752
		kvm_mmu_invalidate_zap_all_pages(kvm);
4753
	}
4754 4755
}

4756 4757
static unsigned long
mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
4758 4759
{
	struct kvm *kvm;
4760
	int nr_to_scan = sc->nr_to_scan;
4761
	unsigned long freed = 0;
4762

4763
	spin_lock(&kvm_lock);
4764 4765

	list_for_each_entry(kvm, &vm_list, vm_list) {
4766
		int idx;
4767
		LIST_HEAD(invalid_list);
4768

4769 4770 4771 4772 4773 4774 4775 4776
		/*
		 * Never scan more than sc->nr_to_scan VM instances.
		 * Will not hit this condition practically since we do not try
		 * to shrink more than one VM and it is very unlikely to see
		 * !n_used_mmu_pages so many times.
		 */
		if (!nr_to_scan--)
			break;
4777 4778 4779 4780 4781 4782
		/*
		 * n_used_mmu_pages is accessed without holding kvm->mmu_lock
		 * here. We may skip a VM instance errorneosly, but we do not
		 * want to shrink a VM that only started to populate its MMU
		 * anyway.
		 */
4783 4784
		if (!kvm->arch.n_used_mmu_pages &&
		      !kvm_has_zapped_obsolete_pages(kvm))
4785 4786
			continue;

4787
		idx = srcu_read_lock(&kvm->srcu);
4788 4789
		spin_lock(&kvm->mmu_lock);

4790 4791 4792 4793 4794 4795
		if (kvm_has_zapped_obsolete_pages(kvm)) {
			kvm_mmu_commit_zap_page(kvm,
			      &kvm->arch.zapped_obsolete_pages);
			goto unlock;
		}

4796 4797
		if (prepare_zap_oldest_mmu_page(kvm, &invalid_list))
			freed++;
4798
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
4799

4800
unlock:
4801
		spin_unlock(&kvm->mmu_lock);
4802
		srcu_read_unlock(&kvm->srcu, idx);
4803

4804 4805 4806 4807 4808
		/*
		 * unfair on small ones
		 * per-vm shrinkers cry out
		 * sadness comes quickly
		 */
4809 4810
		list_move_tail(&kvm->vm_list, &vm_list);
		break;
4811 4812
	}

4813
	spin_unlock(&kvm_lock);
4814 4815 4816 4817 4818 4819
	return freed;
}

static unsigned long
mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
{
4820
	return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
4821 4822 4823
}

static struct shrinker mmu_shrinker = {
4824 4825
	.count_objects = mmu_shrink_count,
	.scan_objects = mmu_shrink_scan,
4826 4827 4828
	.seeks = DEFAULT_SEEKS * 10,
};

I
Ingo Molnar 已提交
4829
static void mmu_destroy_caches(void)
4830
{
4831 4832
	if (pte_list_desc_cache)
		kmem_cache_destroy(pte_list_desc_cache);
4833 4834
	if (mmu_page_header_cache)
		kmem_cache_destroy(mmu_page_header_cache);
4835 4836 4837 4838
}

int kvm_mmu_module_init(void)
{
4839 4840
	pte_list_desc_cache = kmem_cache_create("pte_list_desc",
					    sizeof(struct pte_list_desc),
4841
					    0, 0, NULL);
4842
	if (!pte_list_desc_cache)
4843 4844
		goto nomem;

4845 4846
	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
4847
						  0, 0, NULL);
4848 4849 4850
	if (!mmu_page_header_cache)
		goto nomem;

4851
	if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
4852 4853
		goto nomem;

4854 4855
	register_shrinker(&mmu_shrinker);

4856 4857 4858
	return 0;

nomem:
4859
	mmu_destroy_caches();
4860 4861 4862
	return -ENOMEM;
}

4863 4864 4865 4866 4867 4868 4869
/*
 * Caculate mmu pages needed for kvm.
 */
unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
{
	unsigned int nr_mmu_pages;
	unsigned int  nr_pages = 0;
4870
	struct kvm_memslots *slots;
4871
	struct kvm_memory_slot *memslot;
4872

4873 4874
	slots = kvm_memslots(kvm);

4875 4876
	kvm_for_each_memslot(memslot, slots)
		nr_pages += memslot->npages;
4877 4878 4879 4880 4881 4882 4883 4884

	nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
	nr_mmu_pages = max(nr_mmu_pages,
			(unsigned int) KVM_MIN_ALLOC_MMU_PAGES);

	return nr_mmu_pages;
}

4885 4886 4887
int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
{
	struct kvm_shadow_walk_iterator iterator;
4888
	u64 spte;
4889 4890
	int nr_sptes = 0;

4891 4892 4893
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return nr_sptes;

4894 4895 4896
	walk_shadow_page_lockless_begin(vcpu);
	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {
		sptes[iterator.level-1] = spte;
4897
		nr_sptes++;
4898
		if (!is_shadow_present_pte(spte))
4899 4900
			break;
	}
4901
	walk_shadow_page_lockless_end(vcpu);
4902 4903 4904 4905 4906

	return nr_sptes;
}
EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);

4907 4908
void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
4909
	kvm_mmu_unload(vcpu);
4910 4911
	free_mmu_pages(vcpu);
	mmu_free_memory_caches(vcpu);
4912 4913 4914 4915 4916 4917 4918
}

void kvm_mmu_module_exit(void)
{
	mmu_destroy_caches();
	percpu_counter_destroy(&kvm_total_used_mmu_pages);
	unregister_shrinker(&mmu_shrinker);
4919 4920
	mmu_audit_disable();
}