mmu.c 101.3 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
11 12 13 14 15 16 17 18 19
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
A
Avi Kivity 已提交
20

21
#include "irq.h"
22
#include "mmu.h"
23
#include "x86.h"
A
Avi Kivity 已提交
24
#include "kvm_cache_regs.h"
A
Avi Kivity 已提交
25

26
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
27 28 29 30 31
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/module.h>
32
#include <linux/swap.h>
M
Marcelo Tosatti 已提交
33
#include <linux/hugetlb.h>
34
#include <linux/compiler.h>
35
#include <linux/srcu.h>
36
#include <linux/slab.h>
37
#include <linux/uaccess.h>
A
Avi Kivity 已提交
38

A
Avi Kivity 已提交
39 40
#include <asm/page.h>
#include <asm/cmpxchg.h>
41
#include <asm/io.h>
42
#include <asm/vmx.h>
A
Avi Kivity 已提交
43

44 45 46 47 48 49 50
/*
 * When setting this variable to true it enables Two-Dimensional-Paging
 * where the hardware walks 2 page tables:
 * 1. the guest-virtual to guest-physical
 * 2. while doing 1. it walks guest-physical to host-physical
 * If the hardware supports that we don't need to do shadow paging.
 */
51
bool tdp_enabled = false;
52

53 54 55 56
enum {
	AUDIT_PRE_PAGE_FAULT,
	AUDIT_POST_PAGE_FAULT,
	AUDIT_PRE_PTE_WRITE,
57 58 59
	AUDIT_POST_PTE_WRITE,
	AUDIT_PRE_SYNC,
	AUDIT_POST_SYNC
60
};
61

62
#undef MMU_DEBUG
63 64 65 66 67 68 69 70 71 72 73 74 75

#ifdef MMU_DEBUG

#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)

#else

#define pgprintk(x...) do { } while (0)
#define rmap_printk(x...) do { } while (0)

#endif

76
#ifdef MMU_DEBUG
77
static bool dbg = 0;
78
module_param(dbg, bool, 0644);
79
#endif
A
Avi Kivity 已提交
80

81 82 83
#ifndef MMU_DEBUG
#define ASSERT(x) do { } while (0)
#else
A
Avi Kivity 已提交
84 85 86 87 88
#define ASSERT(x)							\
	if (!(x)) {							\
		printk(KERN_WARNING "assertion failed %s:%d: %s\n",	\
		       __FILE__, __LINE__, #x);				\
	}
89
#endif
A
Avi Kivity 已提交
90

91 92
#define PTE_PREFETCH_NUM		8

93
#define PT_FIRST_AVAIL_BITS_SHIFT 10
A
Avi Kivity 已提交
94 95 96 97 98
#define PT64_SECOND_AVAIL_BITS_SHIFT 52

#define PT64_LEVEL_BITS 9

#define PT64_LEVEL_SHIFT(level) \
M
Mike Day 已提交
99
		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
A
Avi Kivity 已提交
100 101 102 103 104 105 106 107

#define PT64_INDEX(address, level)\
	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))


#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
M
Mike Day 已提交
108
		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
A
Avi Kivity 已提交
109

110 111 112
#define PT32_LVL_OFFSET_MASK(level) \
	(PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT32_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
113 114 115 116 117

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


118
#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
A
Avi Kivity 已提交
119 120
#define PT64_DIR_BASE_ADDR_MASK \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
121 122 123 124 125 126
#define PT64_LVL_ADDR_MASK(level) \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
#define PT64_LVL_OFFSET_MASK(level) \
	(PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
127 128 129 130

#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
131 132 133
#define PT32_LVL_ADDR_MASK(level) \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
					    * PT32_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
134

135 136
#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
			| PT64_NX_MASK)
A
Avi Kivity 已提交
137

138 139 140 141 142
#define ACC_EXEC_MASK    1
#define ACC_WRITE_MASK   PT_WRITABLE_MASK
#define ACC_USER_MASK    PT_USER_MASK
#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)

143 144
#include <trace/events/kvm.h>

145 146 147
#define CREATE_TRACE_POINTS
#include "mmutrace.h"

148 149
#define SPTE_HOST_WRITEABLE	(1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
#define SPTE_MMU_WRITEABLE	(1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
150

151 152
#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)

153 154 155
/* make pte_list_desc fit well in cache line */
#define PTE_LIST_EXT 3

156 157 158
struct pte_list_desc {
	u64 *sptes[PTE_LIST_EXT];
	struct pte_list_desc *more;
159 160
};

161 162 163 164
struct kvm_shadow_walk_iterator {
	u64 addr;
	hpa_t shadow_addr;
	u64 *sptep;
165
	int level;
166 167 168 169 170 171 172 173
	unsigned index;
};

#define for_each_shadow_entry(_vcpu, _addr, _walker)    \
	for (shadow_walk_init(&(_walker), _vcpu, _addr);	\
	     shadow_walk_okay(&(_walker));			\
	     shadow_walk_next(&(_walker)))

174 175 176 177 178 179
#define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)	\
	for (shadow_walk_init(&(_walker), _vcpu, _addr);		\
	     shadow_walk_okay(&(_walker)) &&				\
		({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });	\
	     __shadow_walk_next(&(_walker), spte))

180
static struct kmem_cache *pte_list_desc_cache;
181
static struct kmem_cache *mmu_page_header_cache;
182
static struct percpu_counter kvm_total_used_mmu_pages;
183

S
Sheng Yang 已提交
184 185 186 187 188
static u64 __read_mostly shadow_nx_mask;
static u64 __read_mostly shadow_x_mask;	/* mutual exclusive with nx_mask */
static u64 __read_mostly shadow_user_mask;
static u64 __read_mostly shadow_accessed_mask;
static u64 __read_mostly shadow_dirty_mask;
189 190 191
static u64 __read_mostly shadow_mmio_mask;

static void mmu_spte_set(u64 *sptep, u64 spte);
192
static void mmu_free_roots(struct kvm_vcpu *vcpu);
193 194 195 196 197 198 199 200 201 202 203

void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask)
{
	shadow_mmio_mask = mmio_mask;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);

static void mark_mmio_spte(u64 *sptep, u64 gfn, unsigned access)
{
	access &= ACC_WRITE_MASK | ACC_USER_MASK;

X
Xiao Guangrong 已提交
204
	trace_mark_mmio_spte(sptep, gfn, access);
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
	mmu_spte_set(sptep, shadow_mmio_mask | access | gfn << PAGE_SHIFT);
}

static bool is_mmio_spte(u64 spte)
{
	return (spte & shadow_mmio_mask) == shadow_mmio_mask;
}

static gfn_t get_mmio_spte_gfn(u64 spte)
{
	return (spte & ~shadow_mmio_mask) >> PAGE_SHIFT;
}

static unsigned get_mmio_spte_access(u64 spte)
{
	return (spte & ~shadow_mmio_mask) & ~PAGE_MASK;
}

static bool set_mmio_spte(u64 *sptep, gfn_t gfn, pfn_t pfn, unsigned access)
{
	if (unlikely(is_noslot_pfn(pfn))) {
		mark_mmio_spte(sptep, gfn, access);
		return true;
	}

	return false;
}
232

233 234 235 236 237
static inline u64 rsvd_bits(int s, int e)
{
	return ((1ULL << (e - s + 1)) - 1) << s;
}

S
Sheng Yang 已提交
238
void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
239
		u64 dirty_mask, u64 nx_mask, u64 x_mask)
S
Sheng Yang 已提交
240 241 242 243 244 245 246 247 248
{
	shadow_user_mask = user_mask;
	shadow_accessed_mask = accessed_mask;
	shadow_dirty_mask = dirty_mask;
	shadow_nx_mask = nx_mask;
	shadow_x_mask = x_mask;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);

A
Avi Kivity 已提交
249 250 251 252 253
static int is_cpuid_PSE36(void)
{
	return 1;
}

254 255
static int is_nx(struct kvm_vcpu *vcpu)
{
256
	return vcpu->arch.efer & EFER_NX;
257 258
}

259 260
static int is_shadow_present_pte(u64 pte)
{
261
	return pte & PT_PRESENT_MASK && !is_mmio_spte(pte);
262 263
}

M
Marcelo Tosatti 已提交
264 265 266 267 268
static int is_large_pte(u64 pte)
{
	return pte & PT_PAGE_SIZE_MASK;
}

269
static int is_dirty_gpte(unsigned long pte)
270
{
A
Avi Kivity 已提交
271
	return pte & PT_DIRTY_MASK;
272 273
}

274
static int is_rmap_spte(u64 pte)
275
{
276
	return is_shadow_present_pte(pte);
277 278
}

279 280 281 282
static int is_last_spte(u64 pte, int level)
{
	if (level == PT_PAGE_TABLE_LEVEL)
		return 1;
283
	if (is_large_pte(pte))
284 285 286 287
		return 1;
	return 0;
}

288
static pfn_t spte_to_pfn(u64 pte)
289
{
290
	return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
291 292
}

293 294 295 296 297 298 299
static gfn_t pse36_gfn_delta(u32 gpte)
{
	int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;

	return (gpte & PT32_DIR_PSE36_MASK) << shift;
}

300
#ifdef CONFIG_X86_64
A
Avi Kivity 已提交
301
static void __set_spte(u64 *sptep, u64 spte)
302
{
303
	*sptep = spte;
304 305
}

306
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
307
{
308 309 310 311 312 313 314
	*sptep = spte;
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	return xchg(sptep, spte);
}
315 316 317 318 319

static u64 __get_spte_lockless(u64 *sptep)
{
	return ACCESS_ONCE(*sptep);
}
320 321 322 323 324 325

static bool __check_direct_spte_mmio_pf(u64 spte)
{
	/* It is valid if the spte is zapped. */
	return spte == 0ull;
}
326
#else
327 328 329 330 331 332 333
union split_spte {
	struct {
		u32 spte_low;
		u32 spte_high;
	};
	u64 spte;
};
334

335 336 337 338 339 340 341 342 343 344 345 346
static void count_spte_clear(u64 *sptep, u64 spte)
{
	struct kvm_mmu_page *sp =  page_header(__pa(sptep));

	if (is_shadow_present_pte(spte))
		return;

	/* Ensure the spte is completely set before we increase the count */
	smp_wmb();
	sp->clear_spte_count++;
}

347 348 349
static void __set_spte(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;
350

351 352 353 354 355 356 357 358 359 360 361 362 363
	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	ssptep->spte_high = sspte.spte_high;

	/*
	 * If we map the spte from nonpresent to present, We should store
	 * the high bits firstly, then set present bit, so cpu can not
	 * fetch this spte while we are setting the spte.
	 */
	smp_wmb();

	ssptep->spte_low = sspte.spte_low;
364 365
}

366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	ssptep->spte_low = sspte.spte_low;

	/*
	 * If we map the spte from present to nonpresent, we should clear
	 * present bit firstly to avoid vcpu fetch the old high bits.
	 */
	smp_wmb();

	ssptep->spte_high = sspte.spte_high;
382
	count_spte_clear(sptep, spte);
383 384 385 386 387 388 389 390 391 392 393
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte, orig;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	/* xchg acts as a barrier before the setting of the high bits */
	orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
394 395
	orig.spte_high = ssptep->spte_high;
	ssptep->spte_high = sspte.spte_high;
396
	count_spte_clear(sptep, spte);
397 398 399

	return orig.spte;
}
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429

/*
 * The idea using the light way get the spte on x86_32 guest is from
 * gup_get_pte(arch/x86/mm/gup.c).
 * The difference is we can not catch the spte tlb flush if we leave
 * guest mode, so we emulate it by increase clear_spte_count when spte
 * is cleared.
 */
static u64 __get_spte_lockless(u64 *sptep)
{
	struct kvm_mmu_page *sp =  page_header(__pa(sptep));
	union split_spte spte, *orig = (union split_spte *)sptep;
	int count;

retry:
	count = sp->clear_spte_count;
	smp_rmb();

	spte.spte_low = orig->spte_low;
	smp_rmb();

	spte.spte_high = orig->spte_high;
	smp_rmb();

	if (unlikely(spte.spte_low != orig->spte_low ||
	      count != sp->clear_spte_count))
		goto retry;

	return spte.spte;
}
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446

static bool __check_direct_spte_mmio_pf(u64 spte)
{
	union split_spte sspte = (union split_spte)spte;
	u32 high_mmio_mask = shadow_mmio_mask >> 32;

	/* It is valid if the spte is zapped. */
	if (spte == 0ull)
		return true;

	/* It is valid if the spte is being zapped. */
	if (sspte.spte_low == 0ull &&
	    (sspte.spte_high & high_mmio_mask) == high_mmio_mask)
		return true;

	return false;
}
447 448
#endif

449 450 451 452 453
static bool spte_is_locklessly_modifiable(u64 spte)
{
	return !(~spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE));
}

454 455
static bool spte_has_volatile_bits(u64 spte)
{
456 457 458 459 460 461 462 463 464
	/*
	 * Always atomicly update spte if it can be updated
	 * out of mmu-lock, it can ensure dirty bit is not lost,
	 * also, it can help us to get a stable is_writable_pte()
	 * to ensure tlb flush is not missed.
	 */
	if (spte_is_locklessly_modifiable(spte))
		return true;

465 466 467 468 469 470
	if (!shadow_accessed_mask)
		return false;

	if (!is_shadow_present_pte(spte))
		return false;

471 472
	if ((spte & shadow_accessed_mask) &&
	      (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
473 474 475 476 477
		return false;

	return true;
}

478 479 480 481 482
static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
{
	return (old_spte & bit_mask) && !(new_spte & bit_mask);
}

483 484 485 486 487 488 489 490 491 492 493 494 495 496
/* Rules for using mmu_spte_set:
 * Set the sptep from nonpresent to present.
 * Note: the sptep being assigned *must* be either not present
 * or in a state where the hardware will not attempt to update
 * the spte.
 */
static void mmu_spte_set(u64 *sptep, u64 new_spte)
{
	WARN_ON(is_shadow_present_pte(*sptep));
	__set_spte(sptep, new_spte);
}

/* Rules for using mmu_spte_update:
 * Update the state bits, it means the mapped pfn is not changged.
497 498 499 500 501 502
 *
 * Whenever we overwrite a writable spte with a read-only one we
 * should flush remote TLBs. Otherwise rmap_write_protect
 * will find a read-only spte, even though the writable spte
 * might be cached on a CPU's TLB, the return value indicates this
 * case.
503
 */
504
static bool mmu_spte_update(u64 *sptep, u64 new_spte)
505
{
506
	u64 old_spte = *sptep;
507
	bool ret = false;
508 509

	WARN_ON(!is_rmap_spte(new_spte));
510

511 512 513 514
	if (!is_shadow_present_pte(old_spte)) {
		mmu_spte_set(sptep, new_spte);
		return ret;
	}
515

516
	if (!spte_has_volatile_bits(old_spte))
517
		__update_clear_spte_fast(sptep, new_spte);
518
	else
519
		old_spte = __update_clear_spte_slow(sptep, new_spte);
520

521 522 523 524 525
	/*
	 * For the spte updated out of mmu-lock is safe, since
	 * we always atomicly update it, see the comments in
	 * spte_has_volatile_bits().
	 */
526 527 528
	if (is_writable_pte(old_spte) && !is_writable_pte(new_spte))
		ret = true;

529
	if (!shadow_accessed_mask)
530
		return ret;
531 532 533 534 535

	if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
		kvm_set_pfn_accessed(spte_to_pfn(old_spte));
	if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
		kvm_set_pfn_dirty(spte_to_pfn(old_spte));
536 537

	return ret;
538 539
}

540 541 542 543 544 545 546 547 548 549 550
/*
 * Rules for using mmu_spte_clear_track_bits:
 * It sets the sptep from present to nonpresent, and track the
 * state bits, it is used to clear the last level sptep.
 */
static int mmu_spte_clear_track_bits(u64 *sptep)
{
	pfn_t pfn;
	u64 old_spte = *sptep;

	if (!spte_has_volatile_bits(old_spte))
551
		__update_clear_spte_fast(sptep, 0ull);
552
	else
553
		old_spte = __update_clear_spte_slow(sptep, 0ull);
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572

	if (!is_rmap_spte(old_spte))
		return 0;

	pfn = spte_to_pfn(old_spte);
	if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
		kvm_set_pfn_accessed(pfn);
	if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
		kvm_set_pfn_dirty(pfn);
	return 1;
}

/*
 * Rules for using mmu_spte_clear_no_track:
 * Directly clear spte without caring the state bits of sptep,
 * it is used to set the upper level spte.
 */
static void mmu_spte_clear_no_track(u64 *sptep)
{
573
	__update_clear_spte_fast(sptep, 0ull);
574 575
}

576 577 578 579 580 581 582
static u64 mmu_spte_get_lockless(u64 *sptep)
{
	return __get_spte_lockless(sptep);
}

static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
{
583 584 585 586 587 588 589 590 591 592 593
	/*
	 * Prevent page table teardown by making any free-er wait during
	 * kvm_flush_remote_tlbs() IPI to all active vcpus.
	 */
	local_irq_disable();
	vcpu->mode = READING_SHADOW_PAGE_TABLES;
	/*
	 * Make sure a following spte read is not reordered ahead of the write
	 * to vcpu->mode.
	 */
	smp_mb();
594 595 596 597
}

static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
{
598 599 600 601 602 603 604 605
	/*
	 * Make sure the write to vcpu->mode is not reordered in front of
	 * reads to sptes.  If it does, kvm_commit_zap_page() can see us
	 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
	 */
	smp_mb();
	vcpu->mode = OUTSIDE_GUEST_MODE;
	local_irq_enable();
606 607
}

608
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
609
				  struct kmem_cache *base_cache, int min)
610 611 612 613
{
	void *obj;

	if (cache->nobjs >= min)
614
		return 0;
615
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
616
		obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
617
		if (!obj)
618
			return -ENOMEM;
619 620
		cache->objects[cache->nobjs++] = obj;
	}
621
	return 0;
622 623
}

624 625 626 627 628
static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
{
	return cache->nobjs;
}

629 630
static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
				  struct kmem_cache *cache)
631 632
{
	while (mc->nobjs)
633
		kmem_cache_free(cache, mc->objects[--mc->nobjs]);
634 635
}

A
Avi Kivity 已提交
636
static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
637
				       int min)
A
Avi Kivity 已提交
638
{
639
	void *page;
A
Avi Kivity 已提交
640 641 642 643

	if (cache->nobjs >= min)
		return 0;
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
644
		page = (void *)__get_free_page(GFP_KERNEL);
A
Avi Kivity 已提交
645 646
		if (!page)
			return -ENOMEM;
647
		cache->objects[cache->nobjs++] = page;
A
Avi Kivity 已提交
648 649 650 651 652 653 654
	}
	return 0;
}

static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
655
		free_page((unsigned long)mc->objects[--mc->nobjs]);
A
Avi Kivity 已提交
656 657
}

658
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
659
{
660 661
	int r;

662
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
663
				   pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
664 665
	if (r)
		goto out;
666
	r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
667 668
	if (r)
		goto out;
669
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
670
				   mmu_page_header_cache, 4);
671 672
out:
	return r;
673 674 675 676
}

static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
677 678
	mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
				pte_list_desc_cache);
679
	mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
680 681
	mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
				mmu_page_header_cache);
682 683
}

684
static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
685 686 687 688 689 690 691 692
{
	void *p;

	BUG_ON(!mc->nobjs);
	p = mc->objects[--mc->nobjs];
	return p;
}

693
static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
694
{
695
	return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
696 697
}

698
static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
699
{
700
	kmem_cache_free(pte_list_desc_cache, pte_list_desc);
701 702
}

703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
{
	if (!sp->role.direct)
		return sp->gfns[index];

	return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
}

static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
{
	if (sp->role.direct)
		BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
	else
		sp->gfns[index] = gfn;
}

M
Marcelo Tosatti 已提交
719
/*
720 721
 * Return the pointer to the large page information for a given gfn,
 * handling slots that are not large page aligned.
M
Marcelo Tosatti 已提交
722
 */
723 724 725
static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
					      struct kvm_memory_slot *slot,
					      int level)
M
Marcelo Tosatti 已提交
726 727 728
{
	unsigned long idx;

729
	idx = gfn_to_index(gfn, slot->base_gfn, level);
730
	return &slot->arch.lpage_info[level - 2][idx];
M
Marcelo Tosatti 已提交
731 732 733 734
}

static void account_shadowed(struct kvm *kvm, gfn_t gfn)
{
735
	struct kvm_memory_slot *slot;
736
	struct kvm_lpage_info *linfo;
737
	int i;
M
Marcelo Tosatti 已提交
738

A
Avi Kivity 已提交
739
	slot = gfn_to_memslot(kvm, gfn);
740 741
	for (i = PT_DIRECTORY_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
742 743
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->write_count += 1;
744
	}
745
	kvm->arch.indirect_shadow_pages++;
M
Marcelo Tosatti 已提交
746 747 748 749
}

static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
{
750
	struct kvm_memory_slot *slot;
751
	struct kvm_lpage_info *linfo;
752
	int i;
M
Marcelo Tosatti 已提交
753

A
Avi Kivity 已提交
754
	slot = gfn_to_memslot(kvm, gfn);
755 756
	for (i = PT_DIRECTORY_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
757 758 759
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->write_count -= 1;
		WARN_ON(linfo->write_count < 0);
760
	}
761
	kvm->arch.indirect_shadow_pages--;
M
Marcelo Tosatti 已提交
762 763
}

764 765 766
static int has_wrprotected_page(struct kvm *kvm,
				gfn_t gfn,
				int level)
M
Marcelo Tosatti 已提交
767
{
768
	struct kvm_memory_slot *slot;
769
	struct kvm_lpage_info *linfo;
M
Marcelo Tosatti 已提交
770

A
Avi Kivity 已提交
771
	slot = gfn_to_memslot(kvm, gfn);
M
Marcelo Tosatti 已提交
772
	if (slot) {
773 774
		linfo = lpage_info_slot(gfn, slot, level);
		return linfo->write_count;
M
Marcelo Tosatti 已提交
775 776 777 778 779
	}

	return 1;
}

780
static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
M
Marcelo Tosatti 已提交
781
{
J
Joerg Roedel 已提交
782
	unsigned long page_size;
783
	int i, ret = 0;
M
Marcelo Tosatti 已提交
784

J
Joerg Roedel 已提交
785
	page_size = kvm_host_page_size(kvm, gfn);
M
Marcelo Tosatti 已提交
786

787 788 789 790 791 792 793 794
	for (i = PT_PAGE_TABLE_LEVEL;
	     i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
		if (page_size >= KVM_HPAGE_SIZE(i))
			ret = i;
		else
			break;
	}

795
	return ret;
M
Marcelo Tosatti 已提交
796 797
}

798 799 800
static struct kvm_memory_slot *
gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
			    bool no_dirty_log)
M
Marcelo Tosatti 已提交
801 802
{
	struct kvm_memory_slot *slot;
803 804 805 806 807 808 809 810 811 812 813

	slot = gfn_to_memslot(vcpu->kvm, gfn);
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
	      (no_dirty_log && slot->dirty_bitmap))
		slot = NULL;

	return slot;
}

static bool mapping_level_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t large_gfn)
{
814
	return !gfn_to_memslot_dirty_bitmap(vcpu, large_gfn, true);
815 816 817 818 819
}

static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
{
	int host_level, level, max_level;
M
Marcelo Tosatti 已提交
820

821 822 823 824 825
	host_level = host_mapping_level(vcpu->kvm, large_gfn);

	if (host_level == PT_PAGE_TABLE_LEVEL)
		return host_level;

826 827 828 829
	max_level = kvm_x86_ops->get_lpage_level() < host_level ?
		kvm_x86_ops->get_lpage_level() : host_level;

	for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
830 831 832 833
		if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
			break;

	return level - 1;
M
Marcelo Tosatti 已提交
834 835
}

836
/*
837
 * Pte mapping structures:
838
 *
839
 * If pte_list bit zero is zero, then pte_list point to the spte.
840
 *
841 842
 * If pte_list bit zero is one, (then pte_list & ~1) points to a struct
 * pte_list_desc containing more mappings.
843
 *
844
 * Returns the number of pte entries before the spte was added or zero if
845 846
 * the spte was not added.
 *
847
 */
848 849
static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
			unsigned long *pte_list)
850
{
851
	struct pte_list_desc *desc;
852
	int i, count = 0;
853

854 855 856 857 858 859 860
	if (!*pte_list) {
		rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
		*pte_list = (unsigned long)spte;
	} else if (!(*pte_list & 1)) {
		rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
		desc = mmu_alloc_pte_list_desc(vcpu);
		desc->sptes[0] = (u64 *)*pte_list;
A
Avi Kivity 已提交
861
		desc->sptes[1] = spte;
862
		*pte_list = (unsigned long)desc | 1;
863
		++count;
864
	} else {
865 866 867
		rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
		desc = (struct pte_list_desc *)(*pte_list & ~1ul);
		while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
868
			desc = desc->more;
869
			count += PTE_LIST_EXT;
870
		}
871 872
		if (desc->sptes[PTE_LIST_EXT-1]) {
			desc->more = mmu_alloc_pte_list_desc(vcpu);
873 874
			desc = desc->more;
		}
A
Avi Kivity 已提交
875
		for (i = 0; desc->sptes[i]; ++i)
876
			++count;
A
Avi Kivity 已提交
877
		desc->sptes[i] = spte;
878
	}
879
	return count;
880 881
}

882 883 884
static void
pte_list_desc_remove_entry(unsigned long *pte_list, struct pte_list_desc *desc,
			   int i, struct pte_list_desc *prev_desc)
885 886 887
{
	int j;

888
	for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
889
		;
A
Avi Kivity 已提交
890 891
	desc->sptes[i] = desc->sptes[j];
	desc->sptes[j] = NULL;
892 893 894
	if (j != 0)
		return;
	if (!prev_desc && !desc->more)
895
		*pte_list = (unsigned long)desc->sptes[0];
896 897 898 899
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
900 901
			*pte_list = (unsigned long)desc->more | 1;
	mmu_free_pte_list_desc(desc);
902 903
}

904
static void pte_list_remove(u64 *spte, unsigned long *pte_list)
905
{
906 907
	struct pte_list_desc *desc;
	struct pte_list_desc *prev_desc;
908 909
	int i;

910 911
	if (!*pte_list) {
		printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
912
		BUG();
913 914 915 916
	} else if (!(*pte_list & 1)) {
		rmap_printk("pte_list_remove:  %p 1->0\n", spte);
		if ((u64 *)*pte_list != spte) {
			printk(KERN_ERR "pte_list_remove:  %p 1->BUG\n", spte);
917 918
			BUG();
		}
919
		*pte_list = 0;
920
	} else {
921 922
		rmap_printk("pte_list_remove:  %p many->many\n", spte);
		desc = (struct pte_list_desc *)(*pte_list & ~1ul);
923 924
		prev_desc = NULL;
		while (desc) {
925
			for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
A
Avi Kivity 已提交
926
				if (desc->sptes[i] == spte) {
927
					pte_list_desc_remove_entry(pte_list,
928
							       desc, i,
929 930 931 932 933 934
							       prev_desc);
					return;
				}
			prev_desc = desc;
			desc = desc->more;
		}
935
		pr_err("pte_list_remove: %p many->many\n", spte);
936 937 938 939
		BUG();
	}
}

940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959
typedef void (*pte_list_walk_fn) (u64 *spte);
static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
{
	struct pte_list_desc *desc;
	int i;

	if (!*pte_list)
		return;

	if (!(*pte_list & 1))
		return fn((u64 *)*pte_list);

	desc = (struct pte_list_desc *)(*pte_list & ~1ul);
	while (desc) {
		for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
			fn(desc->sptes[i]);
		desc = desc->more;
	}
}

960
static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,
961
				    struct kvm_memory_slot *slot)
962 963 964 965 966 967 968 969 970 971
{
	struct kvm_lpage_info *linfo;

	if (likely(level == PT_PAGE_TABLE_LEVEL))
		return &slot->rmap[gfn - slot->base_gfn];

	linfo = lpage_info_slot(gfn, slot, level);
	return &linfo->rmap_pde;
}

972 973 974 975 976 977 978 979
/*
 * Take gfn and return the reverse mapping to it.
 */
static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
{
	struct kvm_memory_slot *slot;

	slot = gfn_to_memslot(kvm, gfn);
980
	return __gfn_to_rmap(gfn, level, slot);
981 982
}

983 984 985 986 987 988 989 990
static bool rmap_can_add(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_memory_cache *cache;

	cache = &vcpu->arch.mmu_pte_list_desc_cache;
	return mmu_memory_cache_free_objects(cache);
}

991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
{
	struct kvm_mmu_page *sp;
	unsigned long *rmapp;

	sp = page_header(__pa(spte));
	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
	return pte_list_add(vcpu, spte, rmapp);
}

static void rmap_remove(struct kvm *kvm, u64 *spte)
{
	struct kvm_mmu_page *sp;
	gfn_t gfn;
	unsigned long *rmapp;

	sp = page_header(__pa(spte));
	gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
	rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
	pte_list_remove(spte, rmapp);
}

1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
/*
 * Used by the following functions to iterate through the sptes linked by a
 * rmap.  All fields are private and not assumed to be used outside.
 */
struct rmap_iterator {
	/* private fields */
	struct pte_list_desc *desc;	/* holds the sptep if not NULL */
	int pos;			/* index of the sptep */
};

/*
 * Iteration must be started by this function.  This should also be used after
 * removing/dropping sptes from the rmap link because in such cases the
 * information in the itererator may not be valid.
 *
 * Returns sptep if found, NULL otherwise.
 */
static u64 *rmap_get_first(unsigned long rmap, struct rmap_iterator *iter)
{
	if (!rmap)
		return NULL;

	if (!(rmap & 1)) {
		iter->desc = NULL;
		return (u64 *)rmap;
	}

	iter->desc = (struct pte_list_desc *)(rmap & ~1ul);
	iter->pos = 0;
	return iter->desc->sptes[iter->pos];
}

/*
 * Must be used with a valid iterator: e.g. after rmap_get_first().
 *
 * Returns sptep if found, NULL otherwise.
 */
static u64 *rmap_get_next(struct rmap_iterator *iter)
{
	if (iter->desc) {
		if (iter->pos < PTE_LIST_EXT - 1) {
			u64 *sptep;

			++iter->pos;
			sptep = iter->desc->sptes[iter->pos];
			if (sptep)
				return sptep;
		}

		iter->desc = iter->desc->more;

		if (iter->desc) {
			iter->pos = 0;
			/* desc->sptes[0] cannot be NULL */
			return iter->desc->sptes[iter->pos];
		}
	}

	return NULL;
}

1075
static void drop_spte(struct kvm *kvm, u64 *sptep)
1076
{
1077
	if (mmu_spte_clear_track_bits(sptep))
1078
		rmap_remove(kvm, sptep);
A
Avi Kivity 已提交
1079 1080
}

1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101

static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
{
	if (is_large_pte(*sptep)) {
		WARN_ON(page_header(__pa(sptep))->role.level ==
			PT_PAGE_TABLE_LEVEL);
		drop_spte(kvm, sptep);
		--kvm->stat.lpages;
		return true;
	}

	return false;
}

static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
{
	if (__drop_large_spte(vcpu->kvm, sptep))
		kvm_flush_remote_tlbs(vcpu->kvm);
}

/*
1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
 * Write-protect on the specified @sptep, @pt_protect indicates whether
 * spte writ-protection is caused by protecting shadow page table.
 * @flush indicates whether tlb need be flushed.
 *
 * Note: write protection is difference between drity logging and spte
 * protection:
 * - for dirty logging, the spte can be set to writable at anytime if
 *   its dirty bitmap is properly set.
 * - for spte protection, the spte can be writable only after unsync-ing
 *   shadow page.
1112 1113 1114
 *
 * Return true if the spte is dropped.
 */
1115 1116
static bool
spte_write_protect(struct kvm *kvm, u64 *sptep, bool *flush, bool pt_protect)
1117 1118 1119
{
	u64 spte = *sptep;

1120 1121
	if (!is_writable_pte(spte) &&
	      !(pt_protect && spte_is_locklessly_modifiable(spte)))
1122 1123 1124 1125
		return false;

	rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);

1126 1127
	if (__drop_large_spte(kvm, sptep)) {
		*flush |= true;
1128
		return true;
1129
	}
1130

1131 1132
	if (pt_protect)
		spte &= ~SPTE_MMU_WRITEABLE;
1133
	spte = spte & ~PT_WRITABLE_MASK;
1134 1135

	*flush |= mmu_spte_update(sptep, spte);
1136 1137 1138
	return false;
}

1139 1140
static bool __rmap_write_protect(struct kvm *kvm, unsigned long *rmapp,
				 int level, bool pt_protect)
1141
{
1142 1143
	u64 *sptep;
	struct rmap_iterator iter;
1144
	bool flush = false;
1145

1146 1147
	for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
		BUG_ON(!(*sptep & PT_PRESENT_MASK));
1148
		if (spte_write_protect(kvm, sptep, &flush, pt_protect)) {
1149
			sptep = rmap_get_first(*rmapp, &iter);
1150
			continue;
1151
		}
1152

1153
		sptep = rmap_get_next(&iter);
1154
	}
1155

1156
	return flush;
1157 1158
}

1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
/**
 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
 * @kvm: kvm instance
 * @slot: slot to protect
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should protect
 *
 * Used when we do not need to care about huge page mappings: e.g. during dirty
 * logging we do not have any such mappings.
 */
void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
				     struct kvm_memory_slot *slot,
				     gfn_t gfn_offset, unsigned long mask)
1172 1173 1174
{
	unsigned long *rmapp;

1175 1176
	while (mask) {
		rmapp = &slot->rmap[gfn_offset + __ffs(mask)];
1177
		__rmap_write_protect(kvm, rmapp, PT_PAGE_TABLE_LEVEL, false);
M
Marcelo Tosatti 已提交
1178

1179 1180 1181
		/* clear the first set bit */
		mask &= mask - 1;
	}
1182 1183
}

1184
static bool rmap_write_protect(struct kvm *kvm, u64 gfn)
1185 1186
{
	struct kvm_memory_slot *slot;
1187 1188
	unsigned long *rmapp;
	int i;
1189
	bool write_protected = false;
1190 1191

	slot = gfn_to_memslot(kvm, gfn);
1192 1193 1194 1195

	for (i = PT_PAGE_TABLE_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
		rmapp = __gfn_to_rmap(gfn, i, slot);
1196
		write_protected |= __rmap_write_protect(kvm, rmapp, i, true);
1197 1198 1199
	}

	return write_protected;
1200 1201
}

F
Frederik Deweerdt 已提交
1202 1203
static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
			   unsigned long data)
1204
{
1205 1206
	u64 *sptep;
	struct rmap_iterator iter;
1207 1208
	int need_tlb_flush = 0;

1209 1210 1211 1212 1213
	while ((sptep = rmap_get_first(*rmapp, &iter))) {
		BUG_ON(!(*sptep & PT_PRESENT_MASK));
		rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", sptep, *sptep);

		drop_spte(kvm, sptep);
1214 1215
		need_tlb_flush = 1;
	}
1216

1217 1218 1219
	return need_tlb_flush;
}

F
Frederik Deweerdt 已提交
1220 1221
static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
			     unsigned long data)
1222
{
1223 1224
	u64 *sptep;
	struct rmap_iterator iter;
1225
	int need_flush = 0;
1226
	u64 new_spte;
1227 1228 1229 1230 1231
	pte_t *ptep = (pte_t *)data;
	pfn_t new_pfn;

	WARN_ON(pte_huge(*ptep));
	new_pfn = pte_pfn(*ptep);
1232 1233 1234 1235 1236

	for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
		BUG_ON(!is_shadow_present_pte(*sptep));
		rmap_printk("kvm_set_pte_rmapp: spte %p %llx\n", sptep, *sptep);

1237
		need_flush = 1;
1238

1239
		if (pte_write(*ptep)) {
1240 1241
			drop_spte(kvm, sptep);
			sptep = rmap_get_first(*rmapp, &iter);
1242
		} else {
1243
			new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
1244 1245 1246 1247
			new_spte |= (u64)new_pfn << PAGE_SHIFT;

			new_spte &= ~PT_WRITABLE_MASK;
			new_spte &= ~SPTE_HOST_WRITEABLE;
1248
			new_spte &= ~shadow_accessed_mask;
1249 1250 1251 1252

			mmu_spte_clear_track_bits(sptep);
			mmu_spte_set(sptep, new_spte);
			sptep = rmap_get_next(&iter);
1253 1254
		}
	}
1255

1256 1257 1258 1259 1260 1261
	if (need_flush)
		kvm_flush_remote_tlbs(kvm);

	return 0;
}

F
Frederik Deweerdt 已提交
1262 1263
static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
			  unsigned long data,
1264
			  int (*handler)(struct kvm *kvm, unsigned long *rmapp,
F
Frederik Deweerdt 已提交
1265
					 unsigned long data))
1266
{
1267
	int j;
1268
	int ret;
1269
	int retval = 0;
1270
	struct kvm_memslots *slots;
1271
	struct kvm_memory_slot *memslot;
1272

1273
	slots = kvm_memslots(kvm);
1274

1275
	kvm_for_each_memslot(memslot, slots) {
1276 1277 1278 1279 1280 1281
		unsigned long start = memslot->userspace_addr;
		unsigned long end;

		end = start + (memslot->npages << PAGE_SHIFT);
		if (hva >= start && hva < end) {
			gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
1282
			gfn_t gfn = memslot->base_gfn + gfn_offset;
1283

1284
			ret = handler(kvm, &memslot->rmap[gfn_offset], data);
1285 1286

			for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
1287 1288 1289 1290 1291
				struct kvm_lpage_info *linfo;

				linfo = lpage_info_slot(gfn, memslot,
							PT_DIRECTORY_LEVEL + j);
				ret |= handler(kvm, &linfo->rmap_pde, data);
1292
			}
1293 1294
			trace_kvm_age_page(hva, memslot, ret);
			retval |= ret;
1295 1296 1297 1298 1299 1300 1301 1302
		}
	}

	return retval;
}

int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
{
1303 1304 1305 1306 1307
	return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
}

void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
{
F
Frederik Deweerdt 已提交
1308
	kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1309 1310
}

F
Frederik Deweerdt 已提交
1311 1312
static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
			 unsigned long data)
1313
{
1314
	u64 *sptep;
1315
	struct rmap_iterator uninitialized_var(iter);
1316 1317
	int young = 0;

1318
	/*
1319 1320
	 * In case of absence of EPT Access and Dirty Bits supports,
	 * emulate the accessed bit for EPT, by checking if this page has
1321 1322 1323 1324 1325
	 * an EPT mapping, and clearing it if it does. On the next access,
	 * a new EPT mapping will be established.
	 * This has some overhead, but not as much as the cost of swapping
	 * out actively used pages or breaking up actively used hugepages.
	 */
1326
	if (!shadow_accessed_mask)
1327
		return kvm_unmap_rmapp(kvm, rmapp, data);
1328

1329 1330
	for (sptep = rmap_get_first(*rmapp, &iter); sptep;
	     sptep = rmap_get_next(&iter)) {
1331
		BUG_ON(!is_shadow_present_pte(*sptep));
1332

1333
		if (*sptep & shadow_accessed_mask) {
1334
			young = 1;
1335 1336
			clear_bit((ffs(shadow_accessed_mask) - 1),
				 (unsigned long *)sptep);
1337 1338
		}
	}
1339

1340 1341 1342
	return young;
}

A
Andrea Arcangeli 已提交
1343 1344 1345
static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
			      unsigned long data)
{
1346 1347
	u64 *sptep;
	struct rmap_iterator iter;
A
Andrea Arcangeli 已提交
1348 1349 1350 1351 1352 1353 1354 1355 1356 1357
	int young = 0;

	/*
	 * If there's no access bit in the secondary pte set by the
	 * hardware it's up to gup-fast/gup to set the access bit in
	 * the primary pte or in the page structure.
	 */
	if (!shadow_accessed_mask)
		goto out;

1358 1359
	for (sptep = rmap_get_first(*rmapp, &iter); sptep;
	     sptep = rmap_get_next(&iter)) {
1360
		BUG_ON(!is_shadow_present_pte(*sptep));
1361

1362
		if (*sptep & shadow_accessed_mask) {
A
Andrea Arcangeli 已提交
1363 1364 1365 1366 1367 1368 1369 1370
			young = 1;
			break;
		}
	}
out:
	return young;
}

1371 1372
#define RMAP_RECYCLE_THRESHOLD 1000

1373
static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1374 1375
{
	unsigned long *rmapp;
1376 1377 1378
	struct kvm_mmu_page *sp;

	sp = page_header(__pa(spte));
1379

1380
	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
1381

1382
	kvm_unmap_rmapp(vcpu->kvm, rmapp, 0);
1383 1384 1385
	kvm_flush_remote_tlbs(vcpu->kvm);
}

1386 1387
int kvm_age_hva(struct kvm *kvm, unsigned long hva)
{
1388
	return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp);
1389 1390
}

A
Andrea Arcangeli 已提交
1391 1392 1393 1394 1395
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
{
	return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
}

1396
#ifdef MMU_DEBUG
1397
static int is_empty_shadow_page(u64 *spt)
A
Avi Kivity 已提交
1398
{
1399 1400 1401
	u64 *pos;
	u64 *end;

1402
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1403
		if (is_shadow_present_pte(*pos)) {
1404
			printk(KERN_ERR "%s: %p %llx\n", __func__,
1405
			       pos, *pos);
A
Avi Kivity 已提交
1406
			return 0;
1407
		}
A
Avi Kivity 已提交
1408 1409
	return 1;
}
1410
#endif
A
Avi Kivity 已提交
1411

1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423
/*
 * This value is the sum of all of the kvm instances's
 * kvm->arch.n_used_mmu_pages values.  We need a global,
 * aggregate version in order to make the slab shrinker
 * faster
 */
static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
{
	kvm->arch.n_used_mmu_pages += nr;
	percpu_counter_add(&kvm_total_used_mmu_pages, nr);
}

1424 1425 1426 1427 1428 1429 1430
/*
 * Remove the sp from shadow page cache, after call it,
 * we can not find this sp from the cache, and the shadow
 * page table is still valid.
 * It should be under the protection of mmu lock.
 */
static void kvm_mmu_isolate_page(struct kvm_mmu_page *sp)
1431
{
1432
	ASSERT(is_empty_shadow_page(sp->spt));
1433
	hlist_del(&sp->hash_link);
1434
	if (!sp->role.direct)
1435
		free_page((unsigned long)sp->gfns);
1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
}

/*
 * Free the shadow page table and the sp, we can do it
 * out of the protection of mmu lock.
 */
static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
{
	list_del(&sp->link);
	free_page((unsigned long)sp->spt);
1446
	kmem_cache_free(mmu_page_header_cache, sp);
1447 1448
}

1449 1450
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
1451
	return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
1452 1453
}

1454
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1455
				    struct kvm_mmu_page *sp, u64 *parent_pte)
1456 1457 1458 1459
{
	if (!parent_pte)
		return;

1460
	pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1461 1462
}

1463
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1464 1465
				       u64 *parent_pte)
{
1466
	pte_list_remove(parent_pte, &sp->parent_ptes);
1467 1468
}

1469 1470 1471 1472
static void drop_parent_pte(struct kvm_mmu_page *sp,
			    u64 *parent_pte)
{
	mmu_page_remove_parent_pte(sp, parent_pte);
1473
	mmu_spte_clear_no_track(parent_pte);
1474 1475
}

1476 1477
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
					       u64 *parent_pte, int direct)
M
Marcelo Tosatti 已提交
1478
{
1479
	struct kvm_mmu_page *sp;
1480 1481
	sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
	sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1482
	if (!direct)
1483
		sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1484 1485
	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1486
	bitmap_zero(sp->slot_bitmap, KVM_MEM_SLOTS_NUM);
1487 1488 1489 1490
	sp->parent_ptes = 0;
	mmu_page_add_parent_pte(vcpu, sp, parent_pte);
	kvm_mod_used_mmu_pages(vcpu->kvm, +1);
	return sp;
M
Marcelo Tosatti 已提交
1491 1492
}

1493
static void mark_unsync(u64 *spte);
1494
static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1495
{
1496
	pte_list_walk(&sp->parent_ptes, mark_unsync);
1497 1498
}

1499
static void mark_unsync(u64 *spte)
1500
{
1501
	struct kvm_mmu_page *sp;
1502
	unsigned int index;
1503

1504
	sp = page_header(__pa(spte));
1505 1506
	index = spte - sp->spt;
	if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1507
		return;
1508
	if (sp->unsync_children++)
1509
		return;
1510
	kvm_mmu_mark_parents_unsync(sp);
1511 1512
}

1513
static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1514
			       struct kvm_mmu_page *sp)
1515 1516 1517 1518
{
	return 1;
}

M
Marcelo Tosatti 已提交
1519 1520 1521 1522
static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
}

1523 1524
static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
				 struct kvm_mmu_page *sp, u64 *spte,
1525
				 const void *pte)
1526 1527 1528 1529
{
	WARN_ON(1);
}

1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
#define KVM_PAGE_ARRAY_NR 16

struct kvm_mmu_pages {
	struct mmu_page_and_offset {
		struct kvm_mmu_page *sp;
		unsigned int idx;
	} page[KVM_PAGE_ARRAY_NR];
	unsigned int nr;
};

1540 1541
static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
			 int idx)
1542
{
1543
	int i;
1544

1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559
	if (sp->unsync)
		for (i=0; i < pvec->nr; i++)
			if (pvec->page[i].sp == sp)
				return 0;

	pvec->page[pvec->nr].sp = sp;
	pvec->page[pvec->nr].idx = idx;
	pvec->nr++;
	return (pvec->nr == KVM_PAGE_ARRAY_NR);
}

static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	int i, ret, nr_unsync_leaf = 0;
1560

1561
	for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
1562
		struct kvm_mmu_page *child;
1563 1564
		u64 ent = sp->spt[i];

1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593
		if (!is_shadow_present_pte(ent) || is_large_pte(ent))
			goto clear_child_bitmap;

		child = page_header(ent & PT64_BASE_ADDR_MASK);

		if (child->unsync_children) {
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;

			ret = __mmu_unsync_walk(child, pvec);
			if (!ret)
				goto clear_child_bitmap;
			else if (ret > 0)
				nr_unsync_leaf += ret;
			else
				return ret;
		} else if (child->unsync) {
			nr_unsync_leaf++;
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;
		} else
			 goto clear_child_bitmap;

		continue;

clear_child_bitmap:
		__clear_bit(i, sp->unsync_child_bitmap);
		sp->unsync_children--;
		WARN_ON((int)sp->unsync_children < 0);
1594 1595 1596
	}


1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607
	return nr_unsync_leaf;
}

static int mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	if (!sp->unsync_children)
		return 0;

	mmu_pages_add(pvec, sp, 0);
	return __mmu_unsync_walk(sp, pvec);
1608 1609 1610 1611 1612
}

static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	WARN_ON(!sp->unsync);
1613
	trace_kvm_mmu_sync_page(sp);
1614 1615 1616 1617
	sp->unsync = 0;
	--kvm->stat.mmu_unsync;
}

1618 1619 1620 1621
static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				    struct list_head *invalid_list);
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list);
1622

1623 1624
#define for_each_gfn_sp(kvm, sp, gfn, pos)				\
  hlist_for_each_entry(sp, pos,						\
1625 1626 1627
   &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)	\
	if ((sp)->gfn != (gfn)) {} else

1628 1629
#define for_each_gfn_indirect_valid_sp(kvm, sp, gfn, pos)		\
  hlist_for_each_entry(sp, pos,						\
1630 1631 1632 1633
   &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)	\
		if ((sp)->gfn != (gfn) || (sp)->role.direct ||		\
			(sp)->role.invalid) {} else

1634
/* @sp->gfn should be write-protected at the call site */
1635
static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1636
			   struct list_head *invalid_list, bool clear_unsync)
1637
{
1638
	if (sp->role.cr4_pae != !!is_pae(vcpu)) {
1639
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1640 1641 1642
		return 1;
	}

1643
	if (clear_unsync)
1644 1645
		kvm_unlink_unsync_page(vcpu->kvm, sp);

1646
	if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
1647
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1648 1649 1650 1651 1652 1653 1654
		return 1;
	}

	kvm_mmu_flush_tlb(vcpu);
	return 0;
}

1655 1656 1657
static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
				   struct kvm_mmu_page *sp)
{
1658
	LIST_HEAD(invalid_list);
1659 1660
	int ret;

1661
	ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
1662
	if (ret)
1663 1664
		kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);

1665 1666 1667
	return ret;
}

1668 1669 1670 1671 1672 1673 1674
#ifdef CONFIG_KVM_MMU_AUDIT
#include "mmu_audit.c"
#else
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
static void mmu_audit_disable(void) { }
#endif

1675 1676
static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
			 struct list_head *invalid_list)
1677
{
1678
	return __kvm_sync_page(vcpu, sp, invalid_list, true);
1679 1680
}

1681 1682 1683 1684
/* @gfn should be write-protected at the call site */
static void kvm_sync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
{
	struct kvm_mmu_page *s;
1685
	struct hlist_node *node;
1686
	LIST_HEAD(invalid_list);
1687 1688
	bool flush = false;

1689
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1690
		if (!s->unsync)
1691 1692 1693
			continue;

		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1694
		kvm_unlink_unsync_page(vcpu->kvm, s);
1695
		if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
1696
			(vcpu->arch.mmu.sync_page(vcpu, s))) {
1697
			kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
1698 1699 1700 1701 1702
			continue;
		}
		flush = true;
	}

1703
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1704 1705 1706 1707
	if (flush)
		kvm_mmu_flush_tlb(vcpu);
}

1708 1709 1710
struct mmu_page_path {
	struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
	unsigned int idx[PT64_ROOT_LEVEL-1];
1711 1712
};

1713 1714 1715 1716 1717 1718
#define for_each_sp(pvec, sp, parents, i)			\
		for (i = mmu_pages_next(&pvec, &parents, -1),	\
			sp = pvec.page[i].sp;			\
			i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});	\
			i = mmu_pages_next(&pvec, &parents, i))

1719 1720 1721
static int mmu_pages_next(struct kvm_mmu_pages *pvec,
			  struct mmu_page_path *parents,
			  int i)
1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739
{
	int n;

	for (n = i+1; n < pvec->nr; n++) {
		struct kvm_mmu_page *sp = pvec->page[n].sp;

		if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
			parents->idx[0] = pvec->page[n].idx;
			return n;
		}

		parents->parent[sp->role.level-2] = sp;
		parents->idx[sp->role.level-1] = pvec->page[n].idx;
	}

	return n;
}

1740
static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1741
{
1742 1743 1744 1745 1746
	struct kvm_mmu_page *sp;
	unsigned int level = 0;

	do {
		unsigned int idx = parents->idx[level];
1747

1748 1749 1750 1751 1752 1753 1754 1755 1756
		sp = parents->parent[level];
		if (!sp)
			return;

		--sp->unsync_children;
		WARN_ON((int)sp->unsync_children < 0);
		__clear_bit(idx, sp->unsync_child_bitmap);
		level++;
	} while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1757 1758
}

1759 1760 1761
static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
			       struct mmu_page_path *parents,
			       struct kvm_mmu_pages *pvec)
1762
{
1763 1764 1765
	parents->parent[parent->role.level-1] = NULL;
	pvec->nr = 0;
}
1766

1767 1768 1769 1770 1771 1772 1773
static void mmu_sync_children(struct kvm_vcpu *vcpu,
			      struct kvm_mmu_page *parent)
{
	int i;
	struct kvm_mmu_page *sp;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
1774
	LIST_HEAD(invalid_list);
1775 1776 1777

	kvm_mmu_pages_init(parent, &parents, &pages);
	while (mmu_unsync_walk(parent, &pages)) {
1778
		bool protected = false;
1779 1780 1781 1782 1783 1784 1785

		for_each_sp(pages, sp, parents, i)
			protected |= rmap_write_protect(vcpu->kvm, sp->gfn);

		if (protected)
			kvm_flush_remote_tlbs(vcpu->kvm);

1786
		for_each_sp(pages, sp, parents, i) {
1787
			kvm_sync_page(vcpu, sp, &invalid_list);
1788 1789
			mmu_pages_clear_parents(&parents);
		}
1790
		kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1791
		cond_resched_lock(&vcpu->kvm->mmu_lock);
1792 1793
		kvm_mmu_pages_init(parent, &parents, &pages);
	}
1794 1795
}

1796 1797 1798 1799 1800 1801 1802 1803
static void init_shadow_page_table(struct kvm_mmu_page *sp)
{
	int i;

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		sp->spt[i] = 0ull;
}

1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815
static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
{
	sp->write_flooding_count = 0;
}

static void clear_sp_write_flooding_count(u64 *spte)
{
	struct kvm_mmu_page *sp =  page_header(__pa(spte));

	__clear_sp_write_flooding_count(sp);
}

1816 1817 1818 1819
static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
1820
					     int direct,
1821
					     unsigned access,
1822
					     u64 *parent_pte)
1823 1824 1825
{
	union kvm_mmu_page_role role;
	unsigned quadrant;
1826
	struct kvm_mmu_page *sp;
1827
	struct hlist_node *node;
1828
	bool need_sync = false;
1829

1830
	role = vcpu->arch.mmu.base_role;
1831
	role.level = level;
1832
	role.direct = direct;
1833
	if (role.direct)
1834
		role.cr4_pae = 0;
1835
	role.access = access;
1836 1837
	if (!vcpu->arch.mmu.direct_map
	    && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
1838 1839 1840 1841
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
1842
	for_each_gfn_sp(vcpu->kvm, sp, gfn, node) {
1843 1844
		if (!need_sync && sp->unsync)
			need_sync = true;
1845

1846 1847
		if (sp->role.word != role.word)
			continue;
1848

1849 1850
		if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
			break;
1851

1852 1853
		mmu_page_add_parent_pte(vcpu, sp, parent_pte);
		if (sp->unsync_children) {
1854
			kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
1855 1856 1857
			kvm_mmu_mark_parents_unsync(sp);
		} else if (sp->unsync)
			kvm_mmu_mark_parents_unsync(sp);
1858

1859
		__clear_sp_write_flooding_count(sp);
1860 1861 1862
		trace_kvm_mmu_get_page(sp, false);
		return sp;
	}
A
Avi Kivity 已提交
1863
	++vcpu->kvm->stat.mmu_cache_miss;
1864
	sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
1865 1866 1867 1868
	if (!sp)
		return sp;
	sp->gfn = gfn;
	sp->role = role;
1869 1870
	hlist_add_head(&sp->hash_link,
		&vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
1871
	if (!direct) {
1872 1873
		if (rmap_write_protect(vcpu->kvm, gfn))
			kvm_flush_remote_tlbs(vcpu->kvm);
1874 1875 1876
		if (level > PT_PAGE_TABLE_LEVEL && need_sync)
			kvm_sync_pages(vcpu, gfn);

1877 1878
		account_shadowed(vcpu->kvm, gfn);
	}
1879
	init_shadow_page_table(sp);
A
Avi Kivity 已提交
1880
	trace_kvm_mmu_get_page(sp, true);
1881
	return sp;
1882 1883
}

1884 1885 1886 1887 1888 1889
static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
			     struct kvm_vcpu *vcpu, u64 addr)
{
	iterator->addr = addr;
	iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
	iterator->level = vcpu->arch.mmu.shadow_root_level;
1890 1891 1892 1893 1894 1895

	if (iterator->level == PT64_ROOT_LEVEL &&
	    vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
	    !vcpu->arch.mmu.direct_map)
		--iterator->level;

1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909
	if (iterator->level == PT32E_ROOT_LEVEL) {
		iterator->shadow_addr
			= vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
		--iterator->level;
		if (!iterator->shadow_addr)
			iterator->level = 0;
	}
}

static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
{
	if (iterator->level < PT_PAGE_TABLE_LEVEL)
		return false;
1910

1911 1912 1913 1914 1915
	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
	return true;
}

1916 1917
static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
			       u64 spte)
1918
{
1919
	if (is_last_spte(spte, iterator->level)) {
1920 1921 1922 1923
		iterator->level = 0;
		return;
	}

1924
	iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
1925 1926 1927
	--iterator->level;
}

1928 1929 1930 1931 1932
static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
{
	return __shadow_walk_next(iterator, *iterator->sptep);
}

1933 1934 1935 1936 1937 1938 1939
static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp)
{
	u64 spte;

	spte = __pa(sp->spt)
		| PT_PRESENT_MASK | PT_ACCESSED_MASK
		| PT_WRITABLE_MASK | PT_USER_MASK;
1940
	mmu_spte_set(sptep, spte);
1941 1942
}

1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959
static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
				   unsigned direct_access)
{
	if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
		struct kvm_mmu_page *child;

		/*
		 * For the direct sp, if the guest pte's dirty bit
		 * changed form clean to dirty, it will corrupt the
		 * sp's access: allow writable in the read-only sp,
		 * so we should update the spte at this point to get
		 * a new sp with the correct access.
		 */
		child = page_header(*sptep & PT64_BASE_ADDR_MASK);
		if (child->role.access == direct_access)
			return;

1960
		drop_parent_pte(child, sptep);
1961 1962 1963 1964
		kvm_flush_remote_tlbs(vcpu->kvm);
	}
}

X
Xiao Guangrong 已提交
1965
static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
1966 1967 1968 1969 1970 1971 1972
			     u64 *spte)
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
	if (is_shadow_present_pte(pte)) {
X
Xiao Guangrong 已提交
1973
		if (is_last_spte(pte, sp->role.level)) {
1974
			drop_spte(kvm, spte);
X
Xiao Guangrong 已提交
1975 1976 1977
			if (is_large_pte(pte))
				--kvm->stat.lpages;
		} else {
1978
			child = page_header(pte & PT64_BASE_ADDR_MASK);
1979
			drop_parent_pte(child, spte);
1980
		}
X
Xiao Guangrong 已提交
1981 1982 1983 1984
		return true;
	}

	if (is_mmio_spte(pte))
1985
		mmu_spte_clear_no_track(spte);
1986

X
Xiao Guangrong 已提交
1987
	return false;
1988 1989
}

1990
static void kvm_mmu_page_unlink_children(struct kvm *kvm,
1991
					 struct kvm_mmu_page *sp)
1992
{
1993 1994
	unsigned i;

1995 1996
	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		mmu_page_zap_pte(kvm, sp, sp->spt + i);
1997 1998
}

1999
static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
2000
{
2001
	mmu_page_remove_parent_pte(sp, parent_pte);
2002 2003
}

2004
static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
2005
{
2006 2007
	u64 *sptep;
	struct rmap_iterator iter;
2008

2009 2010
	while ((sptep = rmap_get_first(sp->parent_ptes, &iter)))
		drop_parent_pte(sp, sptep);
2011 2012
}

2013
static int mmu_zap_unsync_children(struct kvm *kvm,
2014 2015
				   struct kvm_mmu_page *parent,
				   struct list_head *invalid_list)
2016
{
2017 2018 2019
	int i, zapped = 0;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
2020

2021
	if (parent->role.level == PT_PAGE_TABLE_LEVEL)
2022
		return 0;
2023 2024 2025 2026 2027 2028

	kvm_mmu_pages_init(parent, &parents, &pages);
	while (mmu_unsync_walk(parent, &pages)) {
		struct kvm_mmu_page *sp;

		for_each_sp(pages, sp, parents, i) {
2029
			kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2030
			mmu_pages_clear_parents(&parents);
2031
			zapped++;
2032 2033 2034 2035 2036
		}
		kvm_mmu_pages_init(parent, &parents, &pages);
	}

	return zapped;
2037 2038
}

2039 2040
static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				    struct list_head *invalid_list)
2041
{
2042
	int ret;
A
Avi Kivity 已提交
2043

2044
	trace_kvm_mmu_prepare_zap_page(sp);
2045
	++kvm->stat.mmu_shadow_zapped;
2046
	ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
2047
	kvm_mmu_page_unlink_children(kvm, sp);
2048
	kvm_mmu_unlink_parents(kvm, sp);
2049
	if (!sp->role.invalid && !sp->role.direct)
A
Avi Kivity 已提交
2050
		unaccount_shadowed(kvm, sp->gfn);
2051 2052
	if (sp->unsync)
		kvm_unlink_unsync_page(kvm, sp);
2053
	if (!sp->root_count) {
2054 2055
		/* Count self */
		ret++;
2056
		list_move(&sp->link, invalid_list);
2057
		kvm_mod_used_mmu_pages(kvm, -1);
2058
	} else {
A
Avi Kivity 已提交
2059
		list_move(&sp->link, &kvm->arch.active_mmu_pages);
2060 2061
		kvm_reload_remote_mmus(kvm);
	}
2062 2063

	sp->role.invalid = 1;
2064
	return ret;
2065 2066
}

2067 2068 2069 2070 2071 2072 2073 2074
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list)
{
	struct kvm_mmu_page *sp;

	if (list_empty(invalid_list))
		return;

2075 2076 2077 2078 2079
	/*
	 * wmb: make sure everyone sees our modifications to the page tables
	 * rmb: make sure we see changes to vcpu->mode
	 */
	smp_mb();
X
Xiao Guangrong 已提交
2080

2081 2082 2083 2084 2085
	/*
	 * Wait for all vcpus to exit guest mode and/or lockless shadow
	 * page table walks.
	 */
	kvm_flush_remote_tlbs(kvm);
2086

2087 2088 2089
	do {
		sp = list_first_entry(invalid_list, struct kvm_mmu_page, link);
		WARN_ON(!sp->role.invalid || sp->root_count);
2090
		kvm_mmu_isolate_page(sp);
2091
		kvm_mmu_free_page(sp);
2092 2093 2094
	} while (!list_empty(invalid_list));
}

2095 2096
/*
 * Changing the number of mmu pages allocated to the vm
2097
 * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2098
 */
2099
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
2100
{
2101
	LIST_HEAD(invalid_list);
2102 2103 2104 2105 2106 2107
	/*
	 * If we set the number of mmu pages to be smaller be than the
	 * number of actived pages , we must to free some mmu pages before we
	 * change the value
	 */

2108 2109
	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
		while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages &&
2110
			!list_empty(&kvm->arch.active_mmu_pages)) {
2111 2112
			struct kvm_mmu_page *page;

2113
			page = container_of(kvm->arch.active_mmu_pages.prev,
2114
					    struct kvm_mmu_page, link);
2115
			kvm_mmu_prepare_zap_page(kvm, page, &invalid_list);
2116
		}
2117
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
2118
		goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2119 2120
	}

2121
	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2122 2123
}

2124
int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2125
{
2126
	struct kvm_mmu_page *sp;
2127
	struct hlist_node *node;
2128
	LIST_HEAD(invalid_list);
2129 2130
	int r;

2131
	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2132
	r = 0;
2133
	spin_lock(&kvm->mmu_lock);
2134
	for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
2135
		pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2136 2137
			 sp->role.word);
		r = 1;
2138
		kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2139
	}
2140
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
2141 2142
	spin_unlock(&kvm->mmu_lock);

2143
	return r;
2144
}
2145
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
2146

2147
static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
A
Avi Kivity 已提交
2148
{
2149
	int slot = memslot_id(kvm, gfn);
2150
	struct kvm_mmu_page *sp = page_header(__pa(pte));
A
Avi Kivity 已提交
2151

2152
	__set_bit(slot, sp->slot_bitmap);
A
Avi Kivity 已提交
2153 2154
}

2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247
/*
 * The function is based on mtrr_type_lookup() in
 * arch/x86/kernel/cpu/mtrr/generic.c
 */
static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
			 u64 start, u64 end)
{
	int i;
	u64 base, mask;
	u8 prev_match, curr_match;
	int num_var_ranges = KVM_NR_VAR_MTRR;

	if (!mtrr_state->enabled)
		return 0xFF;

	/* Make end inclusive end, instead of exclusive */
	end--;

	/* Look in fixed ranges. Just return the type as per start */
	if (mtrr_state->have_fixed && (start < 0x100000)) {
		int idx;

		if (start < 0x80000) {
			idx = 0;
			idx += (start >> 16);
			return mtrr_state->fixed_ranges[idx];
		} else if (start < 0xC0000) {
			idx = 1 * 8;
			idx += ((start - 0x80000) >> 14);
			return mtrr_state->fixed_ranges[idx];
		} else if (start < 0x1000000) {
			idx = 3 * 8;
			idx += ((start - 0xC0000) >> 12);
			return mtrr_state->fixed_ranges[idx];
		}
	}

	/*
	 * Look in variable ranges
	 * Look of multiple ranges matching this address and pick type
	 * as per MTRR precedence
	 */
	if (!(mtrr_state->enabled & 2))
		return mtrr_state->def_type;

	prev_match = 0xFF;
	for (i = 0; i < num_var_ranges; ++i) {
		unsigned short start_state, end_state;

		if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
			continue;

		base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
		       (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
		mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
		       (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);

		start_state = ((start & mask) == (base & mask));
		end_state = ((end & mask) == (base & mask));
		if (start_state != end_state)
			return 0xFE;

		if ((start & mask) != (base & mask))
			continue;

		curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
		if (prev_match == 0xFF) {
			prev_match = curr_match;
			continue;
		}

		if (prev_match == MTRR_TYPE_UNCACHABLE ||
		    curr_match == MTRR_TYPE_UNCACHABLE)
			return MTRR_TYPE_UNCACHABLE;

		if ((prev_match == MTRR_TYPE_WRBACK &&
		     curr_match == MTRR_TYPE_WRTHROUGH) ||
		    (prev_match == MTRR_TYPE_WRTHROUGH &&
		     curr_match == MTRR_TYPE_WRBACK)) {
			prev_match = MTRR_TYPE_WRTHROUGH;
			curr_match = MTRR_TYPE_WRTHROUGH;
		}

		if (prev_match != curr_match)
			return MTRR_TYPE_UNCACHABLE;
	}

	if (prev_match != 0xFF)
		return prev_match;

	return mtrr_state->def_type;
}

2248
u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
2249 2250 2251 2252 2253 2254 2255 2256 2257
{
	u8 mtrr;

	mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
			     (gfn << PAGE_SHIFT) + PAGE_SIZE);
	if (mtrr == 0xfe || mtrr == 0xff)
		mtrr = MTRR_TYPE_WRBACK;
	return mtrr;
}
2258
EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
2259

2260 2261 2262 2263 2264 2265 2266 2267 2268 2269
static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
{
	trace_kvm_mmu_unsync_page(sp);
	++vcpu->kvm->stat.mmu_unsync;
	sp->unsync = 1;

	kvm_mmu_mark_parents_unsync(sp);
}

static void kvm_unsync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
2270 2271
{
	struct kvm_mmu_page *s;
2272
	struct hlist_node *node;
2273

2274
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
2275
		if (s->unsync)
2276
			continue;
2277 2278
		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
		__kvm_unsync_page(vcpu, s);
2279 2280 2281 2282 2283 2284
	}
}

static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
				  bool can_unsync)
{
2285
	struct kvm_mmu_page *s;
2286
	struct hlist_node *node;
2287 2288
	bool need_unsync = false;

2289
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
2290 2291 2292
		if (!can_unsync)
			return 1;

2293
		if (s->role.level != PT_PAGE_TABLE_LEVEL)
2294
			return 1;
2295 2296 2297 2298

		if (!need_unsync && !s->unsync) {
			need_unsync = true;
		}
2299
	}
2300 2301
	if (need_unsync)
		kvm_unsync_pages(vcpu, gfn);
2302 2303 2304
	return 0;
}

A
Avi Kivity 已提交
2305
static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
M
Marcelo Tosatti 已提交
2306
		    unsigned pte_access, int user_fault,
2307
		    int write_fault, int level,
2308
		    gfn_t gfn, pfn_t pfn, bool speculative,
2309
		    bool can_unsync, bool host_writable)
2310
{
2311
	u64 spte;
M
Marcelo Tosatti 已提交
2312
	int ret = 0;
S
Sheng Yang 已提交
2313

2314 2315 2316
	if (set_mmio_spte(sptep, gfn, pfn, pte_access))
		return 0;

2317
	spte = PT_PRESENT_MASK;
2318
	if (!speculative)
2319
		spte |= shadow_accessed_mask;
2320

S
Sheng Yang 已提交
2321 2322 2323 2324
	if (pte_access & ACC_EXEC_MASK)
		spte |= shadow_x_mask;
	else
		spte |= shadow_nx_mask;
2325

2326
	if (pte_access & ACC_USER_MASK)
S
Sheng Yang 已提交
2327
		spte |= shadow_user_mask;
2328

2329
	if (level > PT_PAGE_TABLE_LEVEL)
M
Marcelo Tosatti 已提交
2330
		spte |= PT_PAGE_SIZE_MASK;
2331
	if (tdp_enabled)
2332 2333
		spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
			kvm_is_mmio_pfn(pfn));
2334

2335
	if (host_writable)
2336
		spte |= SPTE_HOST_WRITEABLE;
2337 2338
	else
		pte_access &= ~ACC_WRITE_MASK;
2339

2340
	spte |= (u64)pfn << PAGE_SHIFT;
2341 2342

	if ((pte_access & ACC_WRITE_MASK)
2343 2344
	    || (!vcpu->arch.mmu.direct_map && write_fault
		&& !is_write_protection(vcpu) && !user_fault)) {
2345

2346 2347
		if (level > PT_PAGE_TABLE_LEVEL &&
		    has_wrprotected_page(vcpu->kvm, gfn, level)) {
2348
			ret = 1;
2349
			drop_spte(vcpu->kvm, sptep);
A
Avi Kivity 已提交
2350
			goto done;
2351 2352
		}

2353
		spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
2354

2355
		if (!vcpu->arch.mmu.direct_map
2356
		    && !(pte_access & ACC_WRITE_MASK)) {
2357
			spte &= ~PT_USER_MASK;
2358 2359 2360 2361 2362 2363 2364 2365 2366
			/*
			 * If we converted a user page to a kernel page,
			 * so that the kernel can write to it when cr0.wp=0,
			 * then we should prevent the kernel from executing it
			 * if SMEP is enabled.
			 */
			if (kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
				spte |= PT64_NX_MASK;
		}
2367

2368 2369 2370 2371 2372 2373
		/*
		 * Optimization: for pte sync, if spte was writable the hash
		 * lookup is unnecessary (and expensive). Write protection
		 * is responsibility of mmu_get_page / kvm_sync_page.
		 * Same reasoning can be applied to dirty page accounting.
		 */
2374
		if (!can_unsync && is_writable_pte(*sptep))
2375 2376
			goto set_pte;

2377
		if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2378
			pgprintk("%s: found shadow page for %llx, marking ro\n",
2379
				 __func__, gfn);
M
Marcelo Tosatti 已提交
2380
			ret = 1;
2381
			pte_access &= ~ACC_WRITE_MASK;
2382
			spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
2383 2384 2385 2386 2387 2388
		}
	}

	if (pte_access & ACC_WRITE_MASK)
		mark_page_dirty(vcpu->kvm, gfn);

2389
set_pte:
2390
	if (mmu_spte_update(sptep, spte))
2391
		kvm_flush_remote_tlbs(vcpu->kvm);
A
Avi Kivity 已提交
2392
done:
M
Marcelo Tosatti 已提交
2393 2394 2395
	return ret;
}

A
Avi Kivity 已提交
2396
static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
M
Marcelo Tosatti 已提交
2397
			 unsigned pt_access, unsigned pte_access,
2398
			 int user_fault, int write_fault,
2399
			 int *emulate, int level, gfn_t gfn,
2400
			 pfn_t pfn, bool speculative,
2401
			 bool host_writable)
M
Marcelo Tosatti 已提交
2402 2403
{
	int was_rmapped = 0;
2404
	int rmap_count;
M
Marcelo Tosatti 已提交
2405 2406

	pgprintk("%s: spte %llx access %x write_fault %d"
2407
		 " user_fault %d gfn %llx\n",
A
Avi Kivity 已提交
2408
		 __func__, *sptep, pt_access,
M
Marcelo Tosatti 已提交
2409 2410
		 write_fault, user_fault, gfn);

A
Avi Kivity 已提交
2411
	if (is_rmap_spte(*sptep)) {
M
Marcelo Tosatti 已提交
2412 2413 2414 2415
		/*
		 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
		 * the parent of the now unreachable PTE.
		 */
2416 2417
		if (level > PT_PAGE_TABLE_LEVEL &&
		    !is_large_pte(*sptep)) {
M
Marcelo Tosatti 已提交
2418
			struct kvm_mmu_page *child;
A
Avi Kivity 已提交
2419
			u64 pte = *sptep;
M
Marcelo Tosatti 已提交
2420 2421

			child = page_header(pte & PT64_BASE_ADDR_MASK);
2422
			drop_parent_pte(child, sptep);
2423
			kvm_flush_remote_tlbs(vcpu->kvm);
A
Avi Kivity 已提交
2424
		} else if (pfn != spte_to_pfn(*sptep)) {
2425
			pgprintk("hfn old %llx new %llx\n",
A
Avi Kivity 已提交
2426
				 spte_to_pfn(*sptep), pfn);
2427
			drop_spte(vcpu->kvm, sptep);
2428
			kvm_flush_remote_tlbs(vcpu->kvm);
2429 2430
		} else
			was_rmapped = 1;
M
Marcelo Tosatti 已提交
2431
	}
2432

A
Avi Kivity 已提交
2433
	if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
2434
		      level, gfn, pfn, speculative, true,
2435
		      host_writable)) {
M
Marcelo Tosatti 已提交
2436
		if (write_fault)
2437
			*emulate = 1;
2438
		kvm_mmu_flush_tlb(vcpu);
2439
	}
M
Marcelo Tosatti 已提交
2440

2441 2442 2443
	if (unlikely(is_mmio_spte(*sptep) && emulate))
		*emulate = 1;

A
Avi Kivity 已提交
2444
	pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2445
	pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
A
Avi Kivity 已提交
2446
		 is_large_pte(*sptep)? "2MB" : "4kB",
2447 2448
		 *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
		 *sptep, sptep);
A
Avi Kivity 已提交
2449
	if (!was_rmapped && is_large_pte(*sptep))
M
Marcelo Tosatti 已提交
2450 2451
		++vcpu->kvm->stat.lpages;

2452 2453 2454 2455 2456 2457 2458
	if (is_shadow_present_pte(*sptep)) {
		page_header_update_slot(vcpu->kvm, sptep, gfn);
		if (!was_rmapped) {
			rmap_count = rmap_add(vcpu, sptep, gfn);
			if (rmap_count > RMAP_RECYCLE_THRESHOLD)
				rmap_recycle(vcpu, sptep, gfn);
		}
2459
	}
2460
	kvm_release_pfn_clean(pfn);
2461 2462
}

A
Avi Kivity 已提交
2463 2464
static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
{
2465
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
2466 2467
}

2468 2469 2470 2471 2472 2473
static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
				     bool no_dirty_log)
{
	struct kvm_memory_slot *slot;
	unsigned long hva;

2474
	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
2475
	if (!slot) {
2476 2477
		get_page(fault_page);
		return page_to_pfn(fault_page);
2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494
	}

	hva = gfn_to_hva_memslot(slot, gfn);

	return hva_to_pfn_atomic(vcpu->kvm, hva);
}

static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *sp,
				    u64 *start, u64 *end)
{
	struct page *pages[PTE_PREFETCH_NUM];
	unsigned access = sp->role.access;
	int i, ret;
	gfn_t gfn;

	gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2495
	if (!gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK))
2496 2497 2498 2499 2500 2501 2502 2503
		return -1;

	ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
	if (ret <= 0)
		return -1;

	for (i = 0; i < ret; i++, gfn++, start++)
		mmu_set_spte(vcpu, start, ACC_ALL,
2504
			     access, 0, 0, NULL,
2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522
			     sp->role.level, gfn,
			     page_to_pfn(pages[i]), true, true);

	return 0;
}

static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
				  struct kvm_mmu_page *sp, u64 *sptep)
{
	u64 *spte, *start = NULL;
	int i;

	WARN_ON(!sp->role.direct);

	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
	spte = sp->spt + i;

	for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2523
		if (is_shadow_present_pte(*spte) || spte == sptep) {
2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553
			if (!start)
				continue;
			if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
				break;
			start = NULL;
		} else if (!start)
			start = spte;
	}
}

static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
{
	struct kvm_mmu_page *sp;

	/*
	 * Since it's no accessed bit on EPT, it's no way to
	 * distinguish between actually accessed translations
	 * and prefetched, so disable pte prefetch if EPT is
	 * enabled.
	 */
	if (!shadow_accessed_mask)
		return;

	sp = page_header(__pa(sptep));
	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return;

	__direct_pte_prefetch(vcpu, sp, sptep);
}

2554
static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
2555 2556
			int map_writable, int level, gfn_t gfn, pfn_t pfn,
			bool prefault)
2557
{
2558
	struct kvm_shadow_walk_iterator iterator;
2559
	struct kvm_mmu_page *sp;
2560
	int emulate = 0;
2561
	gfn_t pseudo_gfn;
A
Avi Kivity 已提交
2562

2563
	for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2564
		if (iterator.level == level) {
2565 2566 2567
			unsigned pte_access = ACC_ALL;

			mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, pte_access,
2568
				     0, write, &emulate,
2569
				     level, gfn, pfn, prefault, map_writable);
2570
			direct_pte_prefetch(vcpu, iterator.sptep);
2571 2572
			++vcpu->stat.pf_fixed;
			break;
A
Avi Kivity 已提交
2573 2574
		}

2575
		if (!is_shadow_present_pte(*iterator.sptep)) {
2576 2577 2578 2579
			u64 base_addr = iterator.addr;

			base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
			pseudo_gfn = base_addr >> PAGE_SHIFT;
2580 2581 2582 2583 2584 2585 2586 2587
			sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
					      iterator.level - 1,
					      1, ACC_ALL, iterator.sptep);
			if (!sp) {
				pgprintk("nonpaging_map: ENOMEM\n");
				kvm_release_pfn_clean(pfn);
				return -ENOMEM;
			}
2588

2589 2590 2591 2592 2593
			mmu_spte_set(iterator.sptep,
				     __pa(sp->spt)
				     | PT_PRESENT_MASK | PT_WRITABLE_MASK
				     | shadow_user_mask | shadow_x_mask
				     | shadow_accessed_mask);
2594 2595
		}
	}
2596
	return emulate;
A
Avi Kivity 已提交
2597 2598
}

H
Huang Ying 已提交
2599
static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2600
{
H
Huang Ying 已提交
2601 2602 2603 2604 2605 2606 2607
	siginfo_t info;

	info.si_signo	= SIGBUS;
	info.si_errno	= 0;
	info.si_code	= BUS_MCEERR_AR;
	info.si_addr	= (void __user *)address;
	info.si_addr_lsb = PAGE_SHIFT;
2608

H
Huang Ying 已提交
2609
	send_sig_info(SIGBUS, &info, tsk);
2610 2611
}

2612
static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn)
2613 2614 2615
{
	kvm_release_pfn_clean(pfn);
	if (is_hwpoison_pfn(pfn)) {
2616
		kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current);
2617
		return 0;
2618
	}
2619

2620
	return -EFAULT;
2621 2622
}

2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657
static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
					gfn_t *gfnp, pfn_t *pfnp, int *levelp)
{
	pfn_t pfn = *pfnp;
	gfn_t gfn = *gfnp;
	int level = *levelp;

	/*
	 * Check if it's a transparent hugepage. If this would be an
	 * hugetlbfs page, level wouldn't be set to
	 * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
	 * here.
	 */
	if (!is_error_pfn(pfn) && !kvm_is_mmio_pfn(pfn) &&
	    level == PT_PAGE_TABLE_LEVEL &&
	    PageTransCompound(pfn_to_page(pfn)) &&
	    !has_wrprotected_page(vcpu->kvm, gfn, PT_DIRECTORY_LEVEL)) {
		unsigned long mask;
		/*
		 * mmu_notifier_retry was successful and we hold the
		 * mmu_lock here, so the pmd can't become splitting
		 * from under us, and in turn
		 * __split_huge_page_refcount() can't run from under
		 * us and we can safely transfer the refcount from
		 * PG_tail to PG_head as we switch the pfn to tail to
		 * head.
		 */
		*levelp = level = PT_DIRECTORY_LEVEL;
		mask = KVM_PAGES_PER_HPAGE(level) - 1;
		VM_BUG_ON((gfn & mask) != (pfn & mask));
		if (pfn & mask) {
			gfn &= ~mask;
			*gfnp = gfn;
			kvm_release_pfn_clean(pfn);
			pfn &= ~mask;
2658
			kvm_get_pfn(pfn);
2659 2660 2661 2662 2663
			*pfnp = pfn;
		}
	}
}

2664 2665
static bool mmu_invalid_pfn(pfn_t pfn)
{
2666
	return unlikely(is_invalid_pfn(pfn));
2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679
}

static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
				pfn_t pfn, unsigned access, int *ret_val)
{
	bool ret = true;

	/* The pfn is invalid, report the error! */
	if (unlikely(is_invalid_pfn(pfn))) {
		*ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
		goto exit;
	}

2680
	if (unlikely(is_noslot_pfn(pfn)))
2681 2682 2683 2684 2685 2686 2687
		vcpu_cache_mmio_info(vcpu, gva, gfn, access);

	ret = false;
exit:
	return ret;
}

2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778
static bool page_fault_can_be_fast(struct kvm_vcpu *vcpu, u32 error_code)
{
	/*
	 * #PF can be fast only if the shadow page table is present and it
	 * is caused by write-protect, that means we just need change the
	 * W bit of the spte which can be done out of mmu-lock.
	 */
	if (!(error_code & PFERR_PRESENT_MASK) ||
	      !(error_code & PFERR_WRITE_MASK))
		return false;

	return true;
}

static bool
fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 spte)
{
	struct kvm_mmu_page *sp = page_header(__pa(sptep));
	gfn_t gfn;

	WARN_ON(!sp->role.direct);

	/*
	 * The gfn of direct spte is stable since it is calculated
	 * by sp->gfn.
	 */
	gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);

	if (cmpxchg64(sptep, spte, spte | PT_WRITABLE_MASK) == spte)
		mark_page_dirty(vcpu->kvm, gfn);

	return true;
}

/*
 * Return value:
 * - true: let the vcpu to access on the same address again.
 * - false: let the real page fault path to fix it.
 */
static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
			    u32 error_code)
{
	struct kvm_shadow_walk_iterator iterator;
	bool ret = false;
	u64 spte = 0ull;

	if (!page_fault_can_be_fast(vcpu, error_code))
		return false;

	walk_shadow_page_lockless_begin(vcpu);
	for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
		if (!is_shadow_present_pte(spte) || iterator.level < level)
			break;

	/*
	 * If the mapping has been changed, let the vcpu fault on the
	 * same address again.
	 */
	if (!is_rmap_spte(spte)) {
		ret = true;
		goto exit;
	}

	if (!is_last_spte(spte, level))
		goto exit;

	/*
	 * Check if it is a spurious fault caused by TLB lazily flushed.
	 *
	 * Need not check the access of upper level table entries since
	 * they are always ACC_ALL.
	 */
	 if (is_writable_pte(spte)) {
		ret = true;
		goto exit;
	}

	/*
	 * Currently, to simplify the code, only the spte write-protected
	 * by dirty-log can be fast fixed.
	 */
	if (!spte_is_locklessly_modifiable(spte))
		goto exit;

	/*
	 * Currently, fast page fault only works for direct mapping since
	 * the gfn is not stable for indirect shadow page.
	 * See Documentation/virtual/kvm/locking.txt to get more detail.
	 */
	ret = fast_pf_fix_direct_spte(vcpu, iterator.sptep, spte);
exit:
X
Xiao Guangrong 已提交
2779 2780
	trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
			      spte, ret);
2781 2782 2783 2784 2785
	walk_shadow_page_lockless_end(vcpu);

	return ret;
}

2786
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
2787 2788
			 gva_t gva, pfn_t *pfn, bool write, bool *writable);

2789 2790
static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
			 gfn_t gfn, bool prefault)
2791 2792
{
	int r;
2793
	int level;
2794
	int force_pt_level;
2795
	pfn_t pfn;
2796
	unsigned long mmu_seq;
2797
	bool map_writable, write = error_code & PFERR_WRITE_MASK;
2798

2799 2800 2801 2802 2803 2804 2805 2806 2807 2808
	force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
	if (likely(!force_pt_level)) {
		level = mapping_level(vcpu, gfn);
		/*
		 * This path builds a PAE pagetable - so we can map
		 * 2mb pages at maximum. Therefore check if the level
		 * is larger than that.
		 */
		if (level > PT_DIRECTORY_LEVEL)
			level = PT_DIRECTORY_LEVEL;
2809

2810 2811 2812
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
	} else
		level = PT_PAGE_TABLE_LEVEL;
M
Marcelo Tosatti 已提交
2813

2814 2815 2816
	if (fast_page_fault(vcpu, v, level, error_code))
		return 0;

2817
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
2818
	smp_rmb();
2819

2820
	if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
2821
		return 0;
2822

2823 2824
	if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
		return r;
2825

2826
	spin_lock(&vcpu->kvm->mmu_lock);
2827 2828
	if (mmu_notifier_retry(vcpu, mmu_seq))
		goto out_unlock;
2829
	kvm_mmu_free_some_pages(vcpu);
2830 2831
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
2832 2833
	r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
			 prefault);
2834 2835 2836
	spin_unlock(&vcpu->kvm->mmu_lock);


2837
	return r;
2838 2839 2840 2841 2842

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
2843 2844 2845
}


2846 2847 2848
static void mmu_free_roots(struct kvm_vcpu *vcpu)
{
	int i;
2849
	struct kvm_mmu_page *sp;
2850
	LIST_HEAD(invalid_list);
2851

2852
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
A
Avi Kivity 已提交
2853
		return;
2854
	spin_lock(&vcpu->kvm->mmu_lock);
2855 2856 2857
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
	    (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
	     vcpu->arch.mmu.direct_map)) {
2858
		hpa_t root = vcpu->arch.mmu.root_hpa;
2859

2860 2861
		sp = page_header(root);
		--sp->root_count;
2862 2863 2864 2865
		if (!sp->root_count && sp->role.invalid) {
			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
			kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
		}
2866
		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2867
		spin_unlock(&vcpu->kvm->mmu_lock);
2868 2869 2870
		return;
	}
	for (i = 0; i < 4; ++i) {
2871
		hpa_t root = vcpu->arch.mmu.pae_root[i];
2872

A
Avi Kivity 已提交
2873 2874
		if (root) {
			root &= PT64_BASE_ADDR_MASK;
2875 2876
			sp = page_header(root);
			--sp->root_count;
2877
			if (!sp->root_count && sp->role.invalid)
2878 2879
				kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
							 &invalid_list);
A
Avi Kivity 已提交
2880
		}
2881
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2882
	}
2883
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2884
	spin_unlock(&vcpu->kvm->mmu_lock);
2885
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2886 2887
}

2888 2889 2890 2891 2892
static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
{
	int ret = 0;

	if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
2893
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2894 2895 2896 2897 2898 2899
		ret = 1;
	}

	return ret;
}

2900 2901 2902
static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_page *sp;
2903
	unsigned i;
2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919

	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		spin_lock(&vcpu->kvm->mmu_lock);
		kvm_mmu_free_some_pages(vcpu);
		sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
				      1, ACC_ALL, NULL);
		++sp->root_count;
		spin_unlock(&vcpu->kvm->mmu_lock);
		vcpu->arch.mmu.root_hpa = __pa(sp->spt);
	} else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
		for (i = 0; i < 4; ++i) {
			hpa_t root = vcpu->arch.mmu.pae_root[i];

			ASSERT(!VALID_PAGE(root));
			spin_lock(&vcpu->kvm->mmu_lock);
			kvm_mmu_free_some_pages(vcpu);
2920 2921
			sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
					      i << 30,
2922 2923 2924 2925 2926 2927 2928
					      PT32_ROOT_LEVEL, 1, ACC_ALL,
					      NULL);
			root = __pa(sp->spt);
			++sp->root_count;
			spin_unlock(&vcpu->kvm->mmu_lock);
			vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
		}
2929
		vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
2930 2931 2932 2933 2934 2935 2936
	} else
		BUG();

	return 0;
}

static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
2937
{
2938
	struct kvm_mmu_page *sp;
2939 2940 2941
	u64 pdptr, pm_mask;
	gfn_t root_gfn;
	int i;
2942

2943
	root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
2944

2945 2946 2947 2948 2949 2950 2951 2952
	if (mmu_check_root(vcpu, root_gfn))
		return 1;

	/*
	 * Do we shadow a long mode page table? If so we need to
	 * write-protect the guests page table root.
	 */
	if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
2953
		hpa_t root = vcpu->arch.mmu.root_hpa;
2954 2955

		ASSERT(!VALID_PAGE(root));
2956

2957
		spin_lock(&vcpu->kvm->mmu_lock);
2958
		kvm_mmu_free_some_pages(vcpu);
2959 2960
		sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
				      0, ACC_ALL, NULL);
2961 2962
		root = __pa(sp->spt);
		++sp->root_count;
2963
		spin_unlock(&vcpu->kvm->mmu_lock);
2964
		vcpu->arch.mmu.root_hpa = root;
2965
		return 0;
2966
	}
2967

2968 2969
	/*
	 * We shadow a 32 bit page table. This may be a legacy 2-level
2970 2971
	 * or a PAE 3-level page table. In either case we need to be aware that
	 * the shadow page table may be a PAE or a long mode page table.
2972
	 */
2973 2974 2975 2976
	pm_mask = PT_PRESENT_MASK;
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
		pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;

2977
	for (i = 0; i < 4; ++i) {
2978
		hpa_t root = vcpu->arch.mmu.pae_root[i];
2979 2980

		ASSERT(!VALID_PAGE(root));
2981
		if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
2982
			pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
2983
			if (!is_present_gpte(pdptr)) {
2984
				vcpu->arch.mmu.pae_root[i] = 0;
A
Avi Kivity 已提交
2985 2986
				continue;
			}
A
Avi Kivity 已提交
2987
			root_gfn = pdptr >> PAGE_SHIFT;
2988 2989
			if (mmu_check_root(vcpu, root_gfn))
				return 1;
2990
		}
2991
		spin_lock(&vcpu->kvm->mmu_lock);
2992
		kvm_mmu_free_some_pages(vcpu);
2993
		sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
2994
				      PT32_ROOT_LEVEL, 0,
2995
				      ACC_ALL, NULL);
2996 2997
		root = __pa(sp->spt);
		++sp->root_count;
2998 2999
		spin_unlock(&vcpu->kvm->mmu_lock);

3000
		vcpu->arch.mmu.pae_root[i] = root | pm_mask;
3001
	}
3002
	vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028

	/*
	 * If we shadow a 32 bit page table with a long mode page
	 * table we enter this path.
	 */
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		if (vcpu->arch.mmu.lm_root == NULL) {
			/*
			 * The additional page necessary for this is only
			 * allocated on demand.
			 */

			u64 *lm_root;

			lm_root = (void*)get_zeroed_page(GFP_KERNEL);
			if (lm_root == NULL)
				return 1;

			lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;

			vcpu->arch.mmu.lm_root = lm_root;
		}

		vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
	}

3029
	return 0;
3030 3031
}

3032 3033 3034 3035 3036 3037 3038 3039
static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
{
	if (vcpu->arch.mmu.direct_map)
		return mmu_alloc_direct_roots(vcpu);
	else
		return mmu_alloc_shadow_roots(vcpu);
}

3040 3041 3042 3043 3044
static void mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	int i;
	struct kvm_mmu_page *sp;

3045 3046 3047
	if (vcpu->arch.mmu.direct_map)
		return;

3048 3049
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return;
3050

3051
	vcpu_clear_mmio_info(vcpu, ~0ul);
3052
	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3053
	if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3054 3055 3056
		hpa_t root = vcpu->arch.mmu.root_hpa;
		sp = page_header(root);
		mmu_sync_children(vcpu, sp);
3057
		kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3058 3059 3060 3061 3062
		return;
	}
	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->arch.mmu.pae_root[i];

3063
		if (root && VALID_PAGE(root)) {
3064 3065 3066 3067 3068
			root &= PT64_BASE_ADDR_MASK;
			sp = page_header(root);
			mmu_sync_children(vcpu, sp);
		}
	}
3069
	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3070 3071 3072 3073 3074 3075
}

void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	spin_lock(&vcpu->kvm->mmu_lock);
	mmu_sync_roots(vcpu);
3076
	spin_unlock(&vcpu->kvm->mmu_lock);
3077 3078
}

3079
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
3080
				  u32 access, struct x86_exception *exception)
A
Avi Kivity 已提交
3081
{
3082 3083
	if (exception)
		exception->error_code = 0;
A
Avi Kivity 已提交
3084 3085 3086
	return vaddr;
}

3087
static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
3088 3089
					 u32 access,
					 struct x86_exception *exception)
3090
{
3091 3092
	if (exception)
		exception->error_code = 0;
3093 3094 3095
	return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access);
}

3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152
static bool quickly_check_mmio_pf(struct kvm_vcpu *vcpu, u64 addr, bool direct)
{
	if (direct)
		return vcpu_match_mmio_gpa(vcpu, addr);

	return vcpu_match_mmio_gva(vcpu, addr);
}


/*
 * On direct hosts, the last spte is only allows two states
 * for mmio page fault:
 *   - It is the mmio spte
 *   - It is zapped or it is being zapped.
 *
 * This function completely checks the spte when the last spte
 * is not the mmio spte.
 */
static bool check_direct_spte_mmio_pf(u64 spte)
{
	return __check_direct_spte_mmio_pf(spte);
}

static u64 walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr)
{
	struct kvm_shadow_walk_iterator iterator;
	u64 spte = 0ull;

	walk_shadow_page_lockless_begin(vcpu);
	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
		if (!is_shadow_present_pte(spte))
			break;
	walk_shadow_page_lockless_end(vcpu);

	return spte;
}

/*
 * If it is a real mmio page fault, return 1 and emulat the instruction
 * directly, return 0 to let CPU fault again on the address, -1 is
 * returned if bug is detected.
 */
int handle_mmio_page_fault_common(struct kvm_vcpu *vcpu, u64 addr, bool direct)
{
	u64 spte;

	if (quickly_check_mmio_pf(vcpu, addr, direct))
		return 1;

	spte = walk_shadow_page_get_mmio_spte(vcpu, addr);

	if (is_mmio_spte(spte)) {
		gfn_t gfn = get_mmio_spte_gfn(spte);
		unsigned access = get_mmio_spte_access(spte);

		if (direct)
			addr = 0;
X
Xiao Guangrong 已提交
3153 3154

		trace_handle_mmio_page_fault(addr, gfn, access);
3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183
		vcpu_cache_mmio_info(vcpu, addr, gfn, access);
		return 1;
	}

	/*
	 * It's ok if the gva is remapped by other cpus on shadow guest,
	 * it's a BUG if the gfn is not a mmio page.
	 */
	if (direct && !check_direct_spte_mmio_pf(spte))
		return -1;

	/*
	 * If the page table is zapped by other cpus, let CPU fault again on
	 * the address.
	 */
	return 0;
}
EXPORT_SYMBOL_GPL(handle_mmio_page_fault_common);

static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr,
				  u32 error_code, bool direct)
{
	int ret;

	ret = handle_mmio_page_fault_common(vcpu, addr, direct);
	WARN_ON(ret < 0);
	return ret;
}

A
Avi Kivity 已提交
3184
static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
3185
				u32 error_code, bool prefault)
A
Avi Kivity 已提交
3186
{
3187
	gfn_t gfn;
3188
	int r;
A
Avi Kivity 已提交
3189

3190
	pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
3191 3192 3193 3194

	if (unlikely(error_code & PFERR_RSVD_MASK))
		return handle_mmio_page_fault(vcpu, gva, error_code, true);

3195 3196 3197
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;
3198

A
Avi Kivity 已提交
3199
	ASSERT(vcpu);
3200
	ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
3201

3202
	gfn = gva >> PAGE_SHIFT;
A
Avi Kivity 已提交
3203

3204
	return nonpaging_map(vcpu, gva & PAGE_MASK,
3205
			     error_code, gfn, prefault);
A
Avi Kivity 已提交
3206 3207
}

3208
static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
3209 3210
{
	struct kvm_arch_async_pf arch;
X
Xiao Guangrong 已提交
3211

3212
	arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3213
	arch.gfn = gfn;
3214
	arch.direct_map = vcpu->arch.mmu.direct_map;
X
Xiao Guangrong 已提交
3215
	arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228

	return kvm_setup_async_pf(vcpu, gva, gfn, &arch);
}

static bool can_do_async_pf(struct kvm_vcpu *vcpu)
{
	if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
		     kvm_event_needs_reinjection(vcpu)))
		return false;

	return kvm_x86_ops->interrupt_allowed(vcpu);
}

3229
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
3230
			 gva_t gva, pfn_t *pfn, bool write, bool *writable)
3231 3232 3233
{
	bool async;

3234
	*pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable);
3235 3236 3237 3238 3239 3240

	if (!async)
		return false; /* *pfn has correct page already */

	put_page(pfn_to_page(*pfn));

3241
	if (!prefault && can_do_async_pf(vcpu)) {
3242
		trace_kvm_try_async_get_page(gva, gfn);
3243 3244 3245 3246 3247 3248 3249 3250
		if (kvm_find_async_pf_gfn(vcpu, gfn)) {
			trace_kvm_async_pf_doublefault(gva, gfn);
			kvm_make_request(KVM_REQ_APF_HALT, vcpu);
			return true;
		} else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
			return true;
	}

3251
	*pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable);
3252 3253 3254 3255

	return false;
}

G
Gleb Natapov 已提交
3256
static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
3257
			  bool prefault)
3258
{
3259
	pfn_t pfn;
3260
	int r;
3261
	int level;
3262
	int force_pt_level;
M
Marcelo Tosatti 已提交
3263
	gfn_t gfn = gpa >> PAGE_SHIFT;
3264
	unsigned long mmu_seq;
3265 3266
	int write = error_code & PFERR_WRITE_MASK;
	bool map_writable;
3267 3268 3269 3270

	ASSERT(vcpu);
	ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));

3271 3272 3273
	if (unlikely(error_code & PFERR_RSVD_MASK))
		return handle_mmio_page_fault(vcpu, gpa, error_code, true);

3274 3275 3276 3277
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

3278 3279 3280 3281 3282 3283
	force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
	if (likely(!force_pt_level)) {
		level = mapping_level(vcpu, gfn);
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
	} else
		level = PT_PAGE_TABLE_LEVEL;
3284

3285 3286 3287
	if (fast_page_fault(vcpu, gpa, level, error_code))
		return 0;

3288
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
3289
	smp_rmb();
3290

3291
	if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
3292 3293
		return 0;

3294 3295 3296
	if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
		return r;

3297
	spin_lock(&vcpu->kvm->mmu_lock);
3298 3299
	if (mmu_notifier_retry(vcpu, mmu_seq))
		goto out_unlock;
3300
	kvm_mmu_free_some_pages(vcpu);
3301 3302
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3303
	r = __direct_map(vcpu, gpa, write, map_writable,
3304
			 level, gfn, pfn, prefault);
3305 3306 3307
	spin_unlock(&vcpu->kvm->mmu_lock);

	return r;
3308 3309 3310 3311 3312

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
3313 3314
}

A
Avi Kivity 已提交
3315 3316
static void nonpaging_free(struct kvm_vcpu *vcpu)
{
3317
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
3318 3319
}

3320 3321
static int nonpaging_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
A
Avi Kivity 已提交
3322 3323 3324 3325 3326
{
	context->new_cr3 = nonpaging_new_cr3;
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
	context->free = nonpaging_free;
3327
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
3328
	context->invlpg = nonpaging_invlpg;
3329
	context->update_pte = nonpaging_update_pte;
3330
	context->root_level = 0;
A
Avi Kivity 已提交
3331
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
3332
	context->root_hpa = INVALID_PAGE;
3333
	context->direct_map = true;
3334
	context->nx = false;
A
Avi Kivity 已提交
3335 3336 3337
	return 0;
}

3338
void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3339
{
A
Avi Kivity 已提交
3340
	++vcpu->stat.tlb_flush;
3341
	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
A
Avi Kivity 已提交
3342 3343 3344 3345
}

static void paging_new_cr3(struct kvm_vcpu *vcpu)
{
3346
	pgprintk("%s: cr3 %lx\n", __func__, kvm_read_cr3(vcpu));
3347
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
3348 3349
}

3350 3351
static unsigned long get_cr3(struct kvm_vcpu *vcpu)
{
3352
	return kvm_read_cr3(vcpu);
3353 3354
}

3355 3356
static void inject_page_fault(struct kvm_vcpu *vcpu,
			      struct x86_exception *fault)
A
Avi Kivity 已提交
3357
{
3358
	vcpu->arch.mmu.inject_page_fault(vcpu, fault);
A
Avi Kivity 已提交
3359 3360 3361 3362 3363 3364 3365
}

static void paging_free(struct kvm_vcpu *vcpu)
{
	nonpaging_free(vcpu);
}

3366
static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
3367 3368 3369 3370
{
	int bit7;

	bit7 = (gpte >> 7) & 1;
3371
	return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
3372 3373
}

3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390
static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access,
			   int *nr_present)
{
	if (unlikely(is_mmio_spte(*sptep))) {
		if (gfn != get_mmio_spte_gfn(*sptep)) {
			mmu_spte_clear_no_track(sptep);
			return true;
		}

		(*nr_present)++;
		mark_mmio_spte(sptep, gfn, access);
		return true;
	}

	return false;
}

A
Avi Kivity 已提交
3391 3392 3393 3394 3395 3396 3397 3398
#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

3399
static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3400
				  struct kvm_mmu *context)
3401 3402 3403 3404
{
	int maxphyaddr = cpuid_maxphyaddr(vcpu);
	u64 exb_bit_rsvd = 0;

3405
	if (!context->nx)
3406
		exb_bit_rsvd = rsvd_bits(63, 63);
3407
	switch (context->root_level) {
3408 3409 3410 3411
	case PT32_ROOT_LEVEL:
		/* no rsvd bits for 2 level 4K page table entries */
		context->rsvd_bits_mask[0][1] = 0;
		context->rsvd_bits_mask[0][0] = 0;
3412 3413 3414 3415 3416 3417 3418
		context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];

		if (!is_pse(vcpu)) {
			context->rsvd_bits_mask[1][1] = 0;
			break;
		}

3419 3420 3421 3422 3423 3424 3425 3426
		if (is_cpuid_PSE36())
			/* 36bits PSE 4MB page */
			context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
		else
			/* 32 bits PSE 4MB page */
			context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
		break;
	case PT32E_ROOT_LEVEL:
3427 3428 3429
		context->rsvd_bits_mask[0][2] =
			rsvd_bits(maxphyaddr, 63) |
			rsvd_bits(7, 8) | rsvd_bits(1, 2);	/* PDPTE */
3430
		context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3431
			rsvd_bits(maxphyaddr, 62);	/* PDE */
3432 3433 3434 3435 3436
		context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 62); 	/* PTE */
		context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 62) |
			rsvd_bits(13, 20);		/* large page */
3437
		context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
3438 3439 3440 3441 3442 3443 3444
		break;
	case PT64_ROOT_LEVEL:
		context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
		context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
		context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3445
			rsvd_bits(maxphyaddr, 51);
3446 3447 3448
		context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51);
		context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
3449 3450 3451
		context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51) |
			rsvd_bits(13, 29);
3452
		context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
3453 3454
			rsvd_bits(maxphyaddr, 51) |
			rsvd_bits(13, 20);		/* large page */
3455
		context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
3456 3457 3458 3459
		break;
	}
}

3460 3461 3462
static int paging64_init_context_common(struct kvm_vcpu *vcpu,
					struct kvm_mmu *context,
					int level)
A
Avi Kivity 已提交
3463
{
3464
	context->nx = is_nx(vcpu);
3465
	context->root_level = level;
3466

3467
	reset_rsvds_bits_mask(vcpu, context);
A
Avi Kivity 已提交
3468 3469 3470 3471 3472

	ASSERT(is_pae(vcpu));
	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
3473
	context->sync_page = paging64_sync_page;
M
Marcelo Tosatti 已提交
3474
	context->invlpg = paging64_invlpg;
3475
	context->update_pte = paging64_update_pte;
A
Avi Kivity 已提交
3476
	context->free = paging_free;
3477
	context->shadow_root_level = level;
A
Avi Kivity 已提交
3478
	context->root_hpa = INVALID_PAGE;
3479
	context->direct_map = false;
A
Avi Kivity 已提交
3480 3481 3482
	return 0;
}

3483 3484
static int paging64_init_context(struct kvm_vcpu *vcpu,
				 struct kvm_mmu *context)
3485
{
3486
	return paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
3487 3488
}

3489 3490
static int paging32_init_context(struct kvm_vcpu *vcpu,
				 struct kvm_mmu *context)
A
Avi Kivity 已提交
3491
{
3492
	context->nx = false;
3493
	context->root_level = PT32_ROOT_LEVEL;
3494

3495
	reset_rsvds_bits_mask(vcpu, context);
A
Avi Kivity 已提交
3496 3497 3498 3499 3500

	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
	context->free = paging_free;
3501
	context->sync_page = paging32_sync_page;
M
Marcelo Tosatti 已提交
3502
	context->invlpg = paging32_invlpg;
3503
	context->update_pte = paging32_update_pte;
A
Avi Kivity 已提交
3504
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
3505
	context->root_hpa = INVALID_PAGE;
3506
	context->direct_map = false;
A
Avi Kivity 已提交
3507 3508 3509
	return 0;
}

3510 3511
static int paging32E_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
A
Avi Kivity 已提交
3512
{
3513
	return paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
A
Avi Kivity 已提交
3514 3515
}

3516 3517
static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
{
3518
	struct kvm_mmu *context = vcpu->arch.walk_mmu;
3519

3520
	context->base_role.word = 0;
3521 3522 3523
	context->new_cr3 = nonpaging_new_cr3;
	context->page_fault = tdp_page_fault;
	context->free = nonpaging_free;
3524
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
3525
	context->invlpg = nonpaging_invlpg;
3526
	context->update_pte = nonpaging_update_pte;
3527
	context->shadow_root_level = kvm_x86_ops->get_tdp_level();
3528
	context->root_hpa = INVALID_PAGE;
3529
	context->direct_map = true;
3530
	context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
3531
	context->get_cr3 = get_cr3;
3532
	context->get_pdptr = kvm_pdptr_read;
3533
	context->inject_page_fault = kvm_inject_page_fault;
3534 3535

	if (!is_paging(vcpu)) {
3536
		context->nx = false;
3537 3538 3539
		context->gva_to_gpa = nonpaging_gva_to_gpa;
		context->root_level = 0;
	} else if (is_long_mode(vcpu)) {
3540
		context->nx = is_nx(vcpu);
3541
		context->root_level = PT64_ROOT_LEVEL;
3542 3543
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging64_gva_to_gpa;
3544
	} else if (is_pae(vcpu)) {
3545
		context->nx = is_nx(vcpu);
3546
		context->root_level = PT32E_ROOT_LEVEL;
3547 3548
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging64_gva_to_gpa;
3549
	} else {
3550
		context->nx = false;
3551
		context->root_level = PT32_ROOT_LEVEL;
3552 3553
		reset_rsvds_bits_mask(vcpu, context);
		context->gva_to_gpa = paging32_gva_to_gpa;
3554 3555 3556 3557 3558
	}

	return 0;
}

3559
int kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
A
Avi Kivity 已提交
3560
{
3561
	int r;
3562
	bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
A
Avi Kivity 已提交
3563
	ASSERT(vcpu);
3564
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
3565 3566

	if (!is_paging(vcpu))
3567
		r = nonpaging_init_context(vcpu, context);
A
Avi Kivity 已提交
3568
	else if (is_long_mode(vcpu))
3569
		r = paging64_init_context(vcpu, context);
A
Avi Kivity 已提交
3570
	else if (is_pae(vcpu))
3571
		r = paging32E_init_context(vcpu, context);
A
Avi Kivity 已提交
3572
	else
3573
		r = paging32_init_context(vcpu, context);
3574

3575
	vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
3576
	vcpu->arch.mmu.base_role.cr0_wp  = is_write_protection(vcpu);
3577 3578
	vcpu->arch.mmu.base_role.smep_andnot_wp
		= smep && !is_write_protection(vcpu);
3579 3580 3581 3582 3583 3584 3585

	return r;
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);

static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
{
3586
	int r = kvm_init_shadow_mmu(vcpu, vcpu->arch.walk_mmu);
3587

3588 3589
	vcpu->arch.walk_mmu->set_cr3           = kvm_x86_ops->set_cr3;
	vcpu->arch.walk_mmu->get_cr3           = get_cr3;
3590
	vcpu->arch.walk_mmu->get_pdptr         = kvm_pdptr_read;
3591
	vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
3592 3593

	return r;
A
Avi Kivity 已提交
3594 3595
}

3596 3597 3598 3599 3600
static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;

	g_context->get_cr3           = get_cr3;
3601
	g_context->get_pdptr         = kvm_pdptr_read;
3602 3603 3604 3605 3606 3607 3608 3609 3610
	g_context->inject_page_fault = kvm_inject_page_fault;

	/*
	 * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
	 * translation of l2_gpa to l1_gpa addresses is done using the
	 * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
	 * functions between mmu and nested_mmu are swapped.
	 */
	if (!is_paging(vcpu)) {
3611
		g_context->nx = false;
3612 3613 3614
		g_context->root_level = 0;
		g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
	} else if (is_long_mode(vcpu)) {
3615
		g_context->nx = is_nx(vcpu);
3616
		g_context->root_level = PT64_ROOT_LEVEL;
3617
		reset_rsvds_bits_mask(vcpu, g_context);
3618 3619
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else if (is_pae(vcpu)) {
3620
		g_context->nx = is_nx(vcpu);
3621
		g_context->root_level = PT32E_ROOT_LEVEL;
3622
		reset_rsvds_bits_mask(vcpu, g_context);
3623 3624
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else {
3625
		g_context->nx = false;
3626
		g_context->root_level = PT32_ROOT_LEVEL;
3627
		reset_rsvds_bits_mask(vcpu, g_context);
3628 3629 3630 3631 3632 3633
		g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
	}

	return 0;
}

3634 3635
static int init_kvm_mmu(struct kvm_vcpu *vcpu)
{
3636 3637 3638
	if (mmu_is_nested(vcpu))
		return init_kvm_nested_mmu(vcpu);
	else if (tdp_enabled)
3639 3640 3641 3642 3643
		return init_kvm_tdp_mmu(vcpu);
	else
		return init_kvm_softmmu(vcpu);
}

A
Avi Kivity 已提交
3644 3645 3646
static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
3647 3648
	if (VALID_PAGE(vcpu->arch.mmu.root_hpa))
		/* mmu.free() should set root_hpa = INVALID_PAGE */
3649
		vcpu->arch.mmu.free(vcpu);
A
Avi Kivity 已提交
3650 3651 3652
}

int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3653 3654
{
	destroy_kvm_mmu(vcpu);
3655
	return init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
3656
}
3657
EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
A
Avi Kivity 已提交
3658 3659

int kvm_mmu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3660
{
3661 3662
	int r;

3663
	r = mmu_topup_memory_caches(vcpu);
A
Avi Kivity 已提交
3664 3665
	if (r)
		goto out;
3666
	r = mmu_alloc_roots(vcpu);
3667
	spin_lock(&vcpu->kvm->mmu_lock);
3668
	mmu_sync_roots(vcpu);
3669
	spin_unlock(&vcpu->kvm->mmu_lock);
3670 3671
	if (r)
		goto out;
3672
	/* set_cr3() should ensure TLB has been flushed */
3673
	vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
3674 3675
out:
	return r;
A
Avi Kivity 已提交
3676
}
A
Avi Kivity 已提交
3677 3678 3679 3680 3681 3682
EXPORT_SYMBOL_GPL(kvm_mmu_load);

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
	mmu_free_roots(vcpu);
}
3683
EXPORT_SYMBOL_GPL(kvm_mmu_unload);
A
Avi Kivity 已提交
3684

3685
static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
3686 3687
				  struct kvm_mmu_page *sp, u64 *spte,
				  const void *new)
3688
{
3689
	if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
3690 3691
		++vcpu->kvm->stat.mmu_pde_zapped;
		return;
3692
        }
3693

A
Avi Kivity 已提交
3694
	++vcpu->kvm->stat.mmu_pte_updated;
3695
	vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
3696 3697
}

3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710
static bool need_remote_flush(u64 old, u64 new)
{
	if (!is_shadow_present_pte(old))
		return false;
	if (!is_shadow_present_pte(new))
		return true;
	if ((old ^ new) & PT64_BASE_ADDR_MASK)
		return true;
	old ^= PT64_NX_MASK;
	new ^= PT64_NX_MASK;
	return (old & ~new & PT64_PERM_MASK) != 0;
}

3711 3712
static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
				    bool remote_flush, bool local_flush)
3713
{
3714 3715 3716 3717
	if (zap_page)
		return;

	if (remote_flush)
3718
		kvm_flush_remote_tlbs(vcpu->kvm);
3719
	else if (local_flush)
3720 3721 3722
		kvm_mmu_flush_tlb(vcpu);
}

3723 3724
static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
				    const u8 *new, int *bytes)
3725
{
3726 3727
	u64 gentry;
	int r;
3728 3729 3730

	/*
	 * Assume that the pte write on a page table of the same type
3731 3732
	 * as the current vcpu paging mode since we update the sptes only
	 * when they have the same mode.
3733
	 */
3734
	if (is_pae(vcpu) && *bytes == 4) {
3735
		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */
3736 3737 3738
		*gpa &= ~(gpa_t)7;
		*bytes = 8;
		r = kvm_read_guest(vcpu->kvm, *gpa, &gentry, min(*bytes, 8));
3739 3740
		if (r)
			gentry = 0;
3741 3742 3743
		new = (const u8 *)&gentry;
	}

3744
	switch (*bytes) {
3745 3746 3747 3748 3749 3750 3751 3752 3753
	case 4:
		gentry = *(const u32 *)new;
		break;
	case 8:
		gentry = *(const u64 *)new;
		break;
	default:
		gentry = 0;
		break;
3754 3755
	}

3756 3757 3758 3759 3760 3761 3762
	return gentry;
}

/*
 * If we're seeing too many writes to a page, it may no longer be a page table,
 * or we may be forking, in which case it is better to unmap the page.
 */
3763
static bool detect_write_flooding(struct kvm_mmu_page *sp)
3764
{
3765 3766 3767 3768
	/*
	 * Skip write-flooding detected for the sp whose level is 1, because
	 * it can become unsync, then the guest page is not write-protected.
	 */
3769
	if (sp->role.level == PT_PAGE_TABLE_LEVEL)
3770
		return false;
3771

3772
	return ++sp->write_flooding_count >= 3;
3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788
}

/*
 * Misaligned accesses are too much trouble to fix up; also, they usually
 * indicate a page is not used as a page table.
 */
static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
				    int bytes)
{
	unsigned offset, pte_size, misaligned;

	pgprintk("misaligned: gpa %llx bytes %d role %x\n",
		 gpa, bytes, sp->role.word);

	offset = offset_in_page(gpa);
	pte_size = sp->role.cr4_pae ? 8 : 4;
3789 3790 3791 3792 3793 3794 3795 3796

	/*
	 * Sometimes, the OS only writes the last one bytes to update status
	 * bits, for example, in linux, andb instruction is used in clear_bit().
	 */
	if (!(offset & (pte_size - 1)) && bytes == 1)
		return false;

3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843
	misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
	misaligned |= bytes < 4;

	return misaligned;
}

static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
{
	unsigned page_offset, quadrant;
	u64 *spte;
	int level;

	page_offset = offset_in_page(gpa);
	level = sp->role.level;
	*nspte = 1;
	if (!sp->role.cr4_pae) {
		page_offset <<= 1;	/* 32->64 */
		/*
		 * A 32-bit pde maps 4MB while the shadow pdes map
		 * only 2MB.  So we need to double the offset again
		 * and zap two pdes instead of one.
		 */
		if (level == PT32_ROOT_LEVEL) {
			page_offset &= ~7; /* kill rounding error */
			page_offset <<= 1;
			*nspte = 2;
		}
		quadrant = page_offset >> PAGE_SHIFT;
		page_offset &= ~PAGE_MASK;
		if (quadrant != sp->role.quadrant)
			return NULL;
	}

	spte = &sp->spt[page_offset / sizeof(*spte)];
	return spte;
}

void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
		       const u8 *new, int bytes)
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	union kvm_mmu_page_role mask = { .word = 0 };
	struct kvm_mmu_page *sp;
	struct hlist_node *node;
	LIST_HEAD(invalid_list);
	u64 entry, gentry, *spte;
	int npte;
3844
	bool remote_flush, local_flush, zap_page;
3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867

	/*
	 * If we don't have indirect shadow pages, it means no page is
	 * write-protected, so we can exit simply.
	 */
	if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
		return;

	zap_page = remote_flush = local_flush = false;

	pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);

	gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, new, &bytes);

	/*
	 * No need to care whether allocation memory is successful
	 * or not since pte prefetch is skiped if it does not have
	 * enough objects in the cache.
	 */
	mmu_topup_memory_caches(vcpu);

	spin_lock(&vcpu->kvm->mmu_lock);
	++vcpu->kvm->stat.mmu_pte_write;
3868
	kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
3869

3870
	mask.cr0_wp = mask.cr4_pae = mask.nxe = 1;
3871
	for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn, node) {
3872
		if (detect_write_misaligned(sp, gpa, bytes) ||
3873
		      detect_write_flooding(sp)) {
3874
			zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
3875
						     &invalid_list);
A
Avi Kivity 已提交
3876
			++vcpu->kvm->stat.mmu_flooded;
3877 3878
			continue;
		}
3879 3880 3881 3882 3883

		spte = get_written_sptes(sp, gpa, &npte);
		if (!spte)
			continue;

3884
		local_flush = true;
3885
		while (npte--) {
3886
			entry = *spte;
3887
			mmu_page_zap_pte(vcpu->kvm, sp, spte);
3888 3889
			if (gentry &&
			      !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
3890
			      & mask.word) && rmap_can_add(vcpu))
3891
				mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
3892 3893
			if (!remote_flush && need_remote_flush(entry, *spte))
				remote_flush = true;
3894
			++spte;
3895 3896
		}
	}
3897
	mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
3898
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3899
	kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
3900
	spin_unlock(&vcpu->kvm->mmu_lock);
3901 3902
}

3903 3904
int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
3905 3906
	gpa_t gpa;
	int r;
3907

3908
	if (vcpu->arch.mmu.direct_map)
3909 3910
		return 0;

3911
	gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
3912 3913

	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
3914

3915
	return r;
3916
}
3917
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
3918

3919
void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3920
{
3921
	LIST_HEAD(invalid_list);
3922

3923
	while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES &&
3924
	       !list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
3925
		struct kvm_mmu_page *sp;
A
Avi Kivity 已提交
3926

3927
		sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
3928
				  struct kvm_mmu_page, link);
3929
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
A
Avi Kivity 已提交
3930
		++vcpu->kvm->stat.mmu_recycled;
A
Avi Kivity 已提交
3931
	}
3932
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
A
Avi Kivity 已提交
3933 3934
}

3935 3936 3937 3938 3939 3940 3941 3942
static bool is_mmio_page_fault(struct kvm_vcpu *vcpu, gva_t addr)
{
	if (vcpu->arch.mmu.direct_map || mmu_is_nested(vcpu))
		return vcpu_match_mmio_gpa(vcpu, addr);

	return vcpu_match_mmio_gva(vcpu, addr);
}

3943 3944
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
		       void *insn, int insn_len)
3945
{
3946
	int r, emulation_type = EMULTYPE_RETRY;
3947 3948
	enum emulation_result er;

G
Gleb Natapov 已提交
3949
	r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
3950 3951 3952 3953 3954 3955 3956 3957
	if (r < 0)
		goto out;

	if (!r) {
		r = 1;
		goto out;
	}

3958 3959 3960 3961
	if (is_mmio_page_fault(vcpu, cr2))
		emulation_type = 0;

	er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
3962 3963 3964 3965 3966 3967

	switch (er) {
	case EMULATE_DONE:
		return 1;
	case EMULATE_DO_MMIO:
		++vcpu->stat.mmio_exits;
3968
		/* fall through */
3969
	case EMULATE_FAIL:
3970
		return 0;
3971 3972 3973 3974 3975 3976 3977 3978
	default:
		BUG();
	}
out:
	return r;
}
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);

M
Marcelo Tosatti 已提交
3979 3980 3981 3982 3983 3984 3985 3986
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
	vcpu->arch.mmu.invlpg(vcpu, gva);
	kvm_mmu_flush_tlb(vcpu);
	++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);

3987 3988 3989 3990 3991 3992
void kvm_enable_tdp(void)
{
	tdp_enabled = true;
}
EXPORT_SYMBOL_GPL(kvm_enable_tdp);

3993 3994 3995 3996 3997 3998
void kvm_disable_tdp(void)
{
	tdp_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_tdp);

A
Avi Kivity 已提交
3999 4000
static void free_mmu_pages(struct kvm_vcpu *vcpu)
{
4001
	free_page((unsigned long)vcpu->arch.mmu.pae_root);
4002 4003
	if (vcpu->arch.mmu.lm_root != NULL)
		free_page((unsigned long)vcpu->arch.mmu.lm_root);
A
Avi Kivity 已提交
4004 4005 4006 4007
}

static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
{
4008
	struct page *page;
A
Avi Kivity 已提交
4009 4010 4011 4012
	int i;

	ASSERT(vcpu);

4013 4014 4015 4016 4017 4018 4019
	/*
	 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
	 * Therefore we need to allocate shadow page tables in the first
	 * 4GB of memory, which happens to fit the DMA32 zone.
	 */
	page = alloc_page(GFP_KERNEL | __GFP_DMA32);
	if (!page)
4020 4021
		return -ENOMEM;

4022
	vcpu->arch.mmu.pae_root = page_address(page);
4023
	for (i = 0; i < 4; ++i)
4024
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
4025

A
Avi Kivity 已提交
4026 4027 4028
	return 0;
}

4029
int kvm_mmu_create(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
4030 4031
{
	ASSERT(vcpu);
4032 4033 4034 4035 4036

	vcpu->arch.walk_mmu = &vcpu->arch.mmu;
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
	vcpu->arch.mmu.translate_gpa = translate_gpa;
	vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
A
Avi Kivity 已提交
4037

4038 4039
	return alloc_mmu_pages(vcpu);
}
A
Avi Kivity 已提交
4040

4041 4042 4043
int kvm_mmu_setup(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
4044
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
4045

4046
	return init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
4047 4048
}

4049
void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
A
Avi Kivity 已提交
4050
{
4051
	struct kvm_mmu_page *sp;
4052
	bool flush = false;
A
Avi Kivity 已提交
4053

4054
	list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
A
Avi Kivity 已提交
4055 4056 4057
		int i;
		u64 *pt;

4058
		if (!test_bit(slot, sp->slot_bitmap))
A
Avi Kivity 已提交
4059 4060
			continue;

4061
		pt = sp->spt;
4062
		for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
4063 4064 4065 4066
			if (!is_shadow_present_pte(pt[i]) ||
			      !is_last_spte(pt[i], sp->role.level))
				continue;

4067
			spte_write_protect(kvm, &pt[i], &flush, false);
4068
		}
A
Avi Kivity 已提交
4069
	}
4070
	kvm_flush_remote_tlbs(kvm);
A
Avi Kivity 已提交
4071
}
4072

4073
void kvm_mmu_zap_all(struct kvm *kvm)
D
Dor Laor 已提交
4074
{
4075
	struct kvm_mmu_page *sp, *node;
4076
	LIST_HEAD(invalid_list);
D
Dor Laor 已提交
4077

4078
	spin_lock(&kvm->mmu_lock);
4079
restart:
4080
	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
4081
		if (kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list))
4082 4083
			goto restart;

4084
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
4085
	spin_unlock(&kvm->mmu_lock);
D
Dor Laor 已提交
4086 4087
}

4088 4089
static void kvm_mmu_remove_some_alloc_mmu_pages(struct kvm *kvm,
						struct list_head *invalid_list)
4090 4091 4092 4093 4094
{
	struct kvm_mmu_page *page;

	page = container_of(kvm->arch.active_mmu_pages.prev,
			    struct kvm_mmu_page, link);
4095
	kvm_mmu_prepare_zap_page(kvm, page, invalid_list);
4096 4097
}

4098
static int mmu_shrink(struct shrinker *shrink, struct shrink_control *sc)
4099 4100
{
	struct kvm *kvm;
4101
	int nr_to_scan = sc->nr_to_scan;
4102 4103 4104

	if (nr_to_scan == 0)
		goto out;
4105

4106
	raw_spin_lock(&kvm_lock);
4107 4108

	list_for_each_entry(kvm, &vm_list, vm_list) {
4109
		int idx;
4110
		LIST_HEAD(invalid_list);
4111

4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123
		/*
		 * n_used_mmu_pages is accessed without holding kvm->mmu_lock
		 * here. We may skip a VM instance errorneosly, but we do not
		 * want to shrink a VM that only started to populate its MMU
		 * anyway.
		 */
		if (kvm->arch.n_used_mmu_pages > 0) {
			if (!nr_to_scan--)
				break;
			continue;
		}

4124
		idx = srcu_read_lock(&kvm->srcu);
4125 4126
		spin_lock(&kvm->mmu_lock);

4127
		kvm_mmu_remove_some_alloc_mmu_pages(kvm, &invalid_list);
4128
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
4129

4130
		spin_unlock(&kvm->mmu_lock);
4131
		srcu_read_unlock(&kvm->srcu, idx);
4132 4133 4134

		list_move_tail(&kvm->vm_list, &vm_list);
		break;
4135 4136
	}

4137
	raw_spin_unlock(&kvm_lock);
4138

4139 4140
out:
	return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
4141 4142 4143 4144 4145 4146 4147
}

static struct shrinker mmu_shrinker = {
	.shrink = mmu_shrink,
	.seeks = DEFAULT_SEEKS * 10,
};

I
Ingo Molnar 已提交
4148
static void mmu_destroy_caches(void)
4149
{
4150 4151
	if (pte_list_desc_cache)
		kmem_cache_destroy(pte_list_desc_cache);
4152 4153
	if (mmu_page_header_cache)
		kmem_cache_destroy(mmu_page_header_cache);
4154 4155 4156 4157
}

int kvm_mmu_module_init(void)
{
4158 4159
	pte_list_desc_cache = kmem_cache_create("pte_list_desc",
					    sizeof(struct pte_list_desc),
4160
					    0, 0, NULL);
4161
	if (!pte_list_desc_cache)
4162 4163
		goto nomem;

4164 4165
	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
4166
						  0, 0, NULL);
4167 4168 4169
	if (!mmu_page_header_cache)
		goto nomem;

4170 4171 4172
	if (percpu_counter_init(&kvm_total_used_mmu_pages, 0))
		goto nomem;

4173 4174
	register_shrinker(&mmu_shrinker);

4175 4176 4177
	return 0;

nomem:
4178
	mmu_destroy_caches();
4179 4180 4181
	return -ENOMEM;
}

4182 4183 4184 4185 4186 4187 4188
/*
 * Caculate mmu pages needed for kvm.
 */
unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
{
	unsigned int nr_mmu_pages;
	unsigned int  nr_pages = 0;
4189
	struct kvm_memslots *slots;
4190
	struct kvm_memory_slot *memslot;
4191

4192 4193
	slots = kvm_memslots(kvm);

4194 4195
	kvm_for_each_memslot(memslot, slots)
		nr_pages += memslot->npages;
4196 4197 4198 4199 4200 4201 4202 4203

	nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
	nr_mmu_pages = max(nr_mmu_pages,
			(unsigned int) KVM_MIN_ALLOC_MMU_PAGES);

	return nr_mmu_pages;
}

4204 4205 4206
int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
{
	struct kvm_shadow_walk_iterator iterator;
4207
	u64 spte;
4208 4209
	int nr_sptes = 0;

4210 4211 4212
	walk_shadow_page_lockless_begin(vcpu);
	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {
		sptes[iterator.level-1] = spte;
4213
		nr_sptes++;
4214
		if (!is_shadow_present_pte(spte))
4215 4216
			break;
	}
4217
	walk_shadow_page_lockless_end(vcpu);
4218 4219 4220 4221 4222

	return nr_sptes;
}
EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);

4223 4224 4225 4226 4227 4228 4229
void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);

	destroy_kvm_mmu(vcpu);
	free_mmu_pages(vcpu);
	mmu_free_memory_caches(vcpu);
4230 4231 4232 4233 4234 4235 4236
}

void kvm_mmu_module_exit(void)
{
	mmu_destroy_caches();
	percpu_counter_destroy(&kvm_total_used_mmu_pages);
	unregister_shrinker(&mmu_shrinker);
4237 4238
	mmu_audit_disable();
}