mmu.c 58.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
A
Avi Kivity 已提交
19 20

#include "vmx.h"
21
#include "mmu.h"
A
Avi Kivity 已提交
22

23
#include <linux/kvm_host.h>
24 25 26 27 28
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/module.h>
29
#include <linux/swap.h>
30
#include <linux/hugetlb.h>
31
#include <linux/compiler.h>
32

A
Avi Kivity 已提交
33 34
#include <asm/page.h>
#include <asm/cmpxchg.h>
35
#include <asm/io.h>
36

37 38 39 40 41 42 43
/*
 * When setting this variable to true it enables Two-Dimensional-Paging
 * where the hardware walks 2 page tables:
 * 1. the guest-virtual to guest-physical
 * 2. while doing 1. it walks guest-physical to host-physical
 * If the hardware supports that we don't need to do shadow paging.
 */
44
bool tdp_enabled = false;
45

46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
#undef MMU_DEBUG

#undef AUDIT

#ifdef AUDIT
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
#else
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
#endif

#ifdef MMU_DEBUG

#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)

#else

#define pgprintk(x...) do { } while (0)
#define rmap_printk(x...) do { } while (0)

#endif

#if defined(MMU_DEBUG) || defined(AUDIT)
69 70
static int dbg = 0;
module_param(dbg, bool, 0644);
71
#endif
72

73 74 75
#ifndef MMU_DEBUG
#define ASSERT(x) do { } while (0)
#else
76 77 78 79 80
#define ASSERT(x)							\
	if (!(x)) {							\
		printk(KERN_WARNING "assertion failed %s:%d: %s\n",	\
		       __FILE__, __LINE__, #x);				\
	}
81
#endif
82 83 84 85 86 87 88 89 90

#define PT_FIRST_AVAIL_BITS_SHIFT 9
#define PT64_SECOND_AVAIL_BITS_SHIFT 52

#define VALID_PAGE(x) ((x) != INVALID_PAGE)

#define PT64_LEVEL_BITS 9

#define PT64_LEVEL_SHIFT(level) \
M
Mike Day 已提交
91
		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
92 93 94 95 96 97 98 99 100 101 102

#define PT64_LEVEL_MASK(level) \
		(((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))

#define PT64_INDEX(address, level)\
	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))


#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
M
Mike Day 已提交
103
		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
104 105 106 107 108 109 110 111

#define PT32_LEVEL_MASK(level) \
		(((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


112
#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
113 114 115 116 117 118 119
#define PT64_DIR_BASE_ADDR_MASK \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))

#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))

120 121
#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
			| PT64_NX_MASK)
122 123 124 125

#define PFERR_PRESENT_MASK (1U << 0)
#define PFERR_WRITE_MASK (1U << 1)
#define PFERR_USER_MASK (1U << 2)
126
#define PFERR_FETCH_MASK (1U << 4)
127 128 129 130

#define PT_DIRECTORY_LEVEL 2
#define PT_PAGE_TABLE_LEVEL 1

131 132
#define RMAP_EXT 4

133 134 135 136 137
#define ACC_EXEC_MASK    1
#define ACC_WRITE_MASK   PT_WRITABLE_MASK
#define ACC_USER_MASK    PT_USER_MASK
#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)

138 139
#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)

140 141 142 143 144
struct kvm_rmap_desc {
	u64 *shadow_ptes[RMAP_EXT];
	struct kvm_rmap_desc *more;
};

145 146
static struct kmem_cache *pte_chain_cache;
static struct kmem_cache *rmap_desc_cache;
147
static struct kmem_cache *mmu_page_header_cache;
148

149 150
static u64 __read_mostly shadow_trap_nonpresent_pte;
static u64 __read_mostly shadow_notrap_nonpresent_pte;
S
Sheng Yang 已提交
151 152 153 154 155 156
static u64 __read_mostly shadow_base_present_pte;
static u64 __read_mostly shadow_nx_mask;
static u64 __read_mostly shadow_x_mask;	/* mutual exclusive with nx_mask */
static u64 __read_mostly shadow_user_mask;
static u64 __read_mostly shadow_accessed_mask;
static u64 __read_mostly shadow_dirty_mask;
157 158 159 160 161 162 163 164

void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
{
	shadow_trap_nonpresent_pte = trap_pte;
	shadow_notrap_nonpresent_pte = notrap_pte;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);

S
Sheng Yang 已提交
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
void kvm_mmu_set_base_ptes(u64 base_pte)
{
	shadow_base_present_pte = base_pte;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);

void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
		u64 dirty_mask, u64 nx_mask, u64 x_mask)
{
	shadow_user_mask = user_mask;
	shadow_accessed_mask = accessed_mask;
	shadow_dirty_mask = dirty_mask;
	shadow_nx_mask = nx_mask;
	shadow_x_mask = x_mask;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);

182 183
static int is_write_protection(struct kvm_vcpu *vcpu)
{
184
	return vcpu->arch.cr0 & X86_CR0_WP;
185 186 187 188 189 190 191
}

static int is_cpuid_PSE36(void)
{
	return 1;
}

192 193
static int is_nx(struct kvm_vcpu *vcpu)
{
194
	return vcpu->arch.shadow_efer & EFER_NX;
195 196
}

197 198 199 200 201
static int is_present_pte(unsigned long pte)
{
	return pte & PT_PRESENT_MASK;
}

202 203 204 205 206 207
static int is_shadow_present_pte(u64 pte)
{
	return pte != shadow_trap_nonpresent_pte
		&& pte != shadow_notrap_nonpresent_pte;
}

208 209 210 211 212
static int is_large_pte(u64 pte)
{
	return pte & PT_PAGE_SIZE_MASK;
}

213 214 215 216 217
static int is_writeble_pte(unsigned long pte)
{
	return pte & PT_WRITABLE_MASK;
}

218 219
static int is_dirty_pte(unsigned long pte)
{
S
Sheng Yang 已提交
220
	return pte & shadow_dirty_mask;
221 222
}

223 224
static int is_rmap_pte(u64 pte)
{
225
	return is_shadow_present_pte(pte);
226 227
}

228
static pfn_t spte_to_pfn(u64 pte)
229
{
230
	return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
231 232
}

233 234 235 236 237 238 239
static gfn_t pse36_gfn_delta(u32 gpte)
{
	int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;

	return (gpte & PT32_DIR_PSE36_MASK) << shift;
}

240 241 242 243 244 245 246 247 248
static void set_shadow_pte(u64 *sptep, u64 spte)
{
#ifdef CONFIG_X86_64
	set_64bit((unsigned long *)sptep, spte);
#else
	set_64bit((unsigned long long *)sptep, spte);
#endif
}

249
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
250
				  struct kmem_cache *base_cache, int min)
251 252 253 254
{
	void *obj;

	if (cache->nobjs >= min)
255
		return 0;
256
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
257
		obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
258
		if (!obj)
259
			return -ENOMEM;
260 261
		cache->objects[cache->nobjs++] = obj;
	}
262
	return 0;
263 264 265 266 267 268 269 270
}

static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
		kfree(mc->objects[--mc->nobjs]);
}

271
static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
272
				       int min)
273 274 275 276 277 278
{
	struct page *page;

	if (cache->nobjs >= min)
		return 0;
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
279
		page = alloc_page(GFP_KERNEL);
280 281 282 283 284 285 286 287 288 289 290
		if (!page)
			return -ENOMEM;
		set_page_private(page, 0);
		cache->objects[cache->nobjs++] = page_address(page);
	}
	return 0;
}

static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
291
		free_page((unsigned long)mc->objects[--mc->nobjs]);
292 293
}

294
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
295
{
296 297
	int r;

298
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
299
				   pte_chain_cache, 4);
300 301
	if (r)
		goto out;
302
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
303
				   rmap_desc_cache, 1);
304 305
	if (r)
		goto out;
306
	r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
307 308
	if (r)
		goto out;
309
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
310
				   mmu_page_header_cache, 4);
311 312
out:
	return r;
313 314 315 316
}

static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
317 318 319 320
	mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache);
	mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache);
	mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
	mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
}

static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
				    size_t size)
{
	void *p;

	BUG_ON(!mc->nobjs);
	p = mc->objects[--mc->nobjs];
	memset(p, 0, size);
	return p;
}

static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
{
336
	return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
337 338 339
				      sizeof(struct kvm_pte_chain));
}

340
static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
341
{
342
	kfree(pc);
343 344 345 346
}

static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
{
347
	return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
348 349 350
				      sizeof(struct kvm_rmap_desc));
}

351
static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
352
{
353
	kfree(rd);
354 355
}

356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
/*
 * Return the pointer to the largepage write count for a given
 * gfn, handling slots that are not large page aligned.
 */
static int *slot_largepage_idx(gfn_t gfn, struct kvm_memory_slot *slot)
{
	unsigned long idx;

	idx = (gfn / KVM_PAGES_PER_HPAGE) -
	      (slot->base_gfn / KVM_PAGES_PER_HPAGE);
	return &slot->lpage_info[idx].write_count;
}

static void account_shadowed(struct kvm *kvm, gfn_t gfn)
{
	int *write_count;

	write_count = slot_largepage_idx(gfn, gfn_to_memslot(kvm, gfn));
	*write_count += 1;
}

static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
{
	int *write_count;

	write_count = slot_largepage_idx(gfn, gfn_to_memslot(kvm, gfn));
	*write_count -= 1;
	WARN_ON(*write_count < 0);
}

static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
{
	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
	int *largepage_idx;

	if (slot) {
		largepage_idx = slot_largepage_idx(gfn, slot);
		return *largepage_idx;
	}

	return 1;
}

static int host_largepage_backed(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr;

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
		return 0;

	vma = find_vma(current->mm, addr);
	if (vma && is_vm_hugetlb_page(vma))
		return 1;

	return 0;
}

static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
{
	struct kvm_memory_slot *slot;

	if (has_wrprotected_page(vcpu->kvm, large_gfn))
		return 0;

	if (!host_largepage_backed(vcpu->kvm, large_gfn))
		return 0;

	slot = gfn_to_memslot(vcpu->kvm, large_gfn);
	if (slot && slot->dirty_bitmap)
		return 0;

	return 1;
}

432 433 434 435 436
/*
 * Take gfn and return the reverse mapping to it.
 * Note: gfn must be unaliased before this function get called
 */

437
static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
438 439
{
	struct kvm_memory_slot *slot;
440
	unsigned long idx;
441 442

	slot = gfn_to_memslot(kvm, gfn);
443 444 445 446 447 448 449
	if (!lpage)
		return &slot->rmap[gfn - slot->base_gfn];

	idx = (gfn / KVM_PAGES_PER_HPAGE) -
	      (slot->base_gfn / KVM_PAGES_PER_HPAGE);

	return &slot->lpage_info[idx].rmap_pde;
450 451
}

452 453 454
/*
 * Reverse mapping data structures:
 *
455 456
 * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
 * that points to page_address(page).
457
 *
458 459
 * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
 * containing more mappings.
460
 */
461
static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn, int lpage)
462
{
463
	struct kvm_mmu_page *sp;
464
	struct kvm_rmap_desc *desc;
465
	unsigned long *rmapp;
466 467 468 469
	int i;

	if (!is_rmap_pte(*spte))
		return;
470
	gfn = unalias_gfn(vcpu->kvm, gfn);
471 472
	sp = page_header(__pa(spte));
	sp->gfns[spte - sp->spt] = gfn;
473
	rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
474
	if (!*rmapp) {
475
		rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
476 477
		*rmapp = (unsigned long)spte;
	} else if (!(*rmapp & 1)) {
478
		rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
479
		desc = mmu_alloc_rmap_desc(vcpu);
480
		desc->shadow_ptes[0] = (u64 *)*rmapp;
481
		desc->shadow_ptes[1] = spte;
482
		*rmapp = (unsigned long)desc | 1;
483 484
	} else {
		rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
485
		desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
486 487 488
		while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
			desc = desc->more;
		if (desc->shadow_ptes[RMAP_EXT-1]) {
489
			desc->more = mmu_alloc_rmap_desc(vcpu);
490 491 492 493 494 495 496 497
			desc = desc->more;
		}
		for (i = 0; desc->shadow_ptes[i]; ++i)
			;
		desc->shadow_ptes[i] = spte;
	}
}

498
static void rmap_desc_remove_entry(unsigned long *rmapp,
499 500 501 502 503 504 505 506 507
				   struct kvm_rmap_desc *desc,
				   int i,
				   struct kvm_rmap_desc *prev_desc)
{
	int j;

	for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
		;
	desc->shadow_ptes[i] = desc->shadow_ptes[j];
A
Al Viro 已提交
508
	desc->shadow_ptes[j] = NULL;
509 510 511
	if (j != 0)
		return;
	if (!prev_desc && !desc->more)
512
		*rmapp = (unsigned long)desc->shadow_ptes[0];
513 514 515 516
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
517
			*rmapp = (unsigned long)desc->more | 1;
518
	mmu_free_rmap_desc(desc);
519 520
}

521
static void rmap_remove(struct kvm *kvm, u64 *spte)
522 523 524
{
	struct kvm_rmap_desc *desc;
	struct kvm_rmap_desc *prev_desc;
525
	struct kvm_mmu_page *sp;
526
	pfn_t pfn;
527
	unsigned long *rmapp;
528 529 530 531
	int i;

	if (!is_rmap_pte(*spte))
		return;
532
	sp = page_header(__pa(spte));
533
	pfn = spte_to_pfn(*spte);
S
Sheng Yang 已提交
534
	if (*spte & shadow_accessed_mask)
535
		kvm_set_pfn_accessed(pfn);
536
	if (is_writeble_pte(*spte))
537
		kvm_release_pfn_dirty(pfn);
538
	else
539
		kvm_release_pfn_clean(pfn);
540
	rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], is_large_pte(*spte));
541
	if (!*rmapp) {
542 543
		printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
		BUG();
544
	} else if (!(*rmapp & 1)) {
545
		rmap_printk("rmap_remove:  %p %llx 1->0\n", spte, *spte);
546
		if ((u64 *)*rmapp != spte) {
547 548 549 550
			printk(KERN_ERR "rmap_remove:  %p %llx 1->BUG\n",
			       spte, *spte);
			BUG();
		}
551
		*rmapp = 0;
552 553
	} else {
		rmap_printk("rmap_remove:  %p %llx many->many\n", spte, *spte);
554
		desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
555 556 557 558
		prev_desc = NULL;
		while (desc) {
			for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
				if (desc->shadow_ptes[i] == spte) {
559
					rmap_desc_remove_entry(rmapp,
560
							       desc, i,
561 562 563 564 565 566 567 568 569 570
							       prev_desc);
					return;
				}
			prev_desc = desc;
			desc = desc->more;
		}
		BUG();
	}
}

571
static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
572 573
{
	struct kvm_rmap_desc *desc;
574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
	struct kvm_rmap_desc *prev_desc;
	u64 *prev_spte;
	int i;

	if (!*rmapp)
		return NULL;
	else if (!(*rmapp & 1)) {
		if (!spte)
			return (u64 *)*rmapp;
		return NULL;
	}
	desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
	prev_desc = NULL;
	prev_spte = NULL;
	while (desc) {
		for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
			if (prev_spte == spte)
				return desc->shadow_ptes[i];
			prev_spte = desc->shadow_ptes[i];
		}
		desc = desc->more;
	}
	return NULL;
}

static void rmap_write_protect(struct kvm *kvm, u64 gfn)
{
601
	unsigned long *rmapp;
602
	u64 *spte;
603
	int write_protected = 0;
604

605
	gfn = unalias_gfn(kvm, gfn);
606
	rmapp = gfn_to_rmap(kvm, gfn, 0);
607

608 609
	spte = rmap_next(kvm, rmapp, NULL);
	while (spte) {
610 611 612
		BUG_ON(!spte);
		BUG_ON(!(*spte & PT_PRESENT_MASK));
		rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
613
		if (is_writeble_pte(*spte)) {
614
			set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
615 616
			write_protected = 1;
		}
617
		spte = rmap_next(kvm, rmapp, spte);
618
	}
619
	if (write_protected) {
620
		pfn_t pfn;
621 622

		spte = rmap_next(kvm, rmapp, NULL);
623 624
		pfn = spte_to_pfn(*spte);
		kvm_set_pfn_dirty(pfn);
625 626
	}

627 628 629 630 631 632 633 634 635 636 637 638
	/* check for huge page mappings */
	rmapp = gfn_to_rmap(kvm, gfn, 1);
	spte = rmap_next(kvm, rmapp, NULL);
	while (spte) {
		BUG_ON(!spte);
		BUG_ON(!(*spte & PT_PRESENT_MASK));
		BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
		pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
		if (is_writeble_pte(*spte)) {
			rmap_remove(kvm, spte);
			--kvm->stat.lpages;
			set_shadow_pte(spte, shadow_trap_nonpresent_pte);
639
			spte = NULL;
640 641 642 643 644
			write_protected = 1;
		}
		spte = rmap_next(kvm, rmapp, spte);
	}

645 646
	if (write_protected)
		kvm_flush_remote_tlbs(kvm);
647 648

	account_shadowed(kvm, gfn);
649 650
}

651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708
static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
{
	u64 *spte;
	int need_tlb_flush = 0;

	while ((spte = rmap_next(kvm, rmapp, NULL))) {
		BUG_ON(!(*spte & PT_PRESENT_MASK));
		rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
		rmap_remove(kvm, spte);
		set_shadow_pte(spte, shadow_trap_nonpresent_pte);
		need_tlb_flush = 1;
	}
	return need_tlb_flush;
}

static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
			  int (*handler)(struct kvm *kvm, unsigned long *rmapp))
{
	int i;
	int retval = 0;

	/*
	 * If mmap_sem isn't taken, we can look the memslots with only
	 * the mmu_lock by skipping over the slots with userspace_addr == 0.
	 */
	for (i = 0; i < kvm->nmemslots; i++) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];
		unsigned long start = memslot->userspace_addr;
		unsigned long end;

		/* mmu_lock protects userspace_addr */
		if (!start)
			continue;

		end = start + (memslot->npages << PAGE_SHIFT);
		if (hva >= start && hva < end) {
			gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
			retval |= handler(kvm, &memslot->rmap[gfn_offset]);
			retval |= handler(kvm,
					  &memslot->lpage_info[
						  gfn_offset /
						  KVM_PAGES_PER_HPAGE].rmap_pde);
		}
	}

	return retval;
}

int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
{
	return kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
}

static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp)
{
	u64 *spte;
	int young = 0;

709 710 711 712
	/* always return old for EPT */
	if (!shadow_accessed_mask)
		return 0;

713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
	spte = rmap_next(kvm, rmapp, NULL);
	while (spte) {
		int _young;
		u64 _spte = *spte;
		BUG_ON(!(_spte & PT_PRESENT_MASK));
		_young = _spte & PT_ACCESSED_MASK;
		if (_young) {
			young = 1;
			clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
		}
		spte = rmap_next(kvm, rmapp, spte);
	}
	return young;
}

int kvm_age_hva(struct kvm *kvm, unsigned long hva)
{
	return kvm_handle_hva(kvm, hva, kvm_age_rmapp);
}

733
#ifdef MMU_DEBUG
734
static int is_empty_shadow_page(u64 *spt)
735
{
736 737 738
	u64 *pos;
	u64 *end;

739
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
740
		if (is_shadow_present_pte(*pos)) {
741
			printk(KERN_ERR "%s: %p %llx\n", __func__,
742
			       pos, *pos);
743
			return 0;
744
		}
745 746
	return 1;
}
747
#endif
748

749
static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
750
{
751 752 753 754 755
	ASSERT(is_empty_shadow_page(sp->spt));
	list_del(&sp->link);
	__free_page(virt_to_page(sp->spt));
	__free_page(virt_to_page(sp->gfns));
	kfree(sp);
756
	++kvm->arch.n_free_mmu_pages;
757 758
}

759 760
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
761
	return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
762 763
}

764 765
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
					       u64 *parent_pte)
766
{
767
	struct kvm_mmu_page *sp;
768

769 770 771
	sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
	sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
	sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
772
	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
773
	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
774 775 776 777
	ASSERT(is_empty_shadow_page(sp->spt));
	sp->slot_bitmap = 0;
	sp->multimapped = 0;
	sp->parent_pte = parent_pte;
778
	--vcpu->kvm->arch.n_free_mmu_pages;
779
	return sp;
780 781
}

782
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
783
				    struct kvm_mmu_page *sp, u64 *parent_pte)
784 785 786 787 788 789 790
{
	struct kvm_pte_chain *pte_chain;
	struct hlist_node *node;
	int i;

	if (!parent_pte)
		return;
791 792
	if (!sp->multimapped) {
		u64 *old = sp->parent_pte;
793 794

		if (!old) {
795
			sp->parent_pte = parent_pte;
796 797
			return;
		}
798
		sp->multimapped = 1;
799
		pte_chain = mmu_alloc_pte_chain(vcpu);
800 801
		INIT_HLIST_HEAD(&sp->parent_ptes);
		hlist_add_head(&pte_chain->link, &sp->parent_ptes);
802 803
		pte_chain->parent_ptes[0] = old;
	}
804
	hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
805 806 807 808 809 810 811 812
		if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
			continue;
		for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
			if (!pte_chain->parent_ptes[i]) {
				pte_chain->parent_ptes[i] = parent_pte;
				return;
			}
	}
813
	pte_chain = mmu_alloc_pte_chain(vcpu);
814
	BUG_ON(!pte_chain);
815
	hlist_add_head(&pte_chain->link, &sp->parent_ptes);
816 817 818
	pte_chain->parent_ptes[0] = parent_pte;
}

819
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
820 821 822 823 824 825
				       u64 *parent_pte)
{
	struct kvm_pte_chain *pte_chain;
	struct hlist_node *node;
	int i;

826 827 828
	if (!sp->multimapped) {
		BUG_ON(sp->parent_pte != parent_pte);
		sp->parent_pte = NULL;
829 830
		return;
	}
831
	hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
832 833 834 835 836
		for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
			if (!pte_chain->parent_ptes[i])
				break;
			if (pte_chain->parent_ptes[i] != parent_pte)
				continue;
837 838
			while (i + 1 < NR_PTE_CHAIN_ENTRIES
				&& pte_chain->parent_ptes[i + 1]) {
839 840 841 842 843
				pte_chain->parent_ptes[i]
					= pte_chain->parent_ptes[i + 1];
				++i;
			}
			pte_chain->parent_ptes[i] = NULL;
844 845
			if (i == 0) {
				hlist_del(&pte_chain->link);
846
				mmu_free_pte_chain(pte_chain);
847 848 849
				if (hlist_empty(&sp->parent_ptes)) {
					sp->multimapped = 0;
					sp->parent_pte = NULL;
850 851
				}
			}
852 853 854 855 856
			return;
		}
	BUG();
}

857 858 859 860 861 862 863 864 865
static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *sp)
{
	int i;

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		sp->spt[i] = shadow_trap_nonpresent_pte;
}

866
static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn)
867 868 869
{
	unsigned index;
	struct hlist_head *bucket;
870
	struct kvm_mmu_page *sp;
871 872
	struct hlist_node *node;

873
	pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
874
	index = kvm_page_table_hashfn(gfn);
875
	bucket = &kvm->arch.mmu_page_hash[index];
876
	hlist_for_each_entry(sp, node, bucket, hash_link)
877 878
		if (sp->gfn == gfn && !sp->role.metaphysical
		    && !sp->role.invalid) {
879
			pgprintk("%s: found role %x\n",
880
				 __func__, sp->role.word);
881
			return sp;
882 883 884 885 886 887 888 889 890
		}
	return NULL;
}

static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
					     int metaphysical,
891
					     unsigned access,
892
					     u64 *parent_pte)
893 894 895 896 897
{
	union kvm_mmu_page_role role;
	unsigned index;
	unsigned quadrant;
	struct hlist_head *bucket;
898
	struct kvm_mmu_page *sp;
899 900 901
	struct hlist_node *node;

	role.word = 0;
902
	role.glevels = vcpu->arch.mmu.root_level;
903 904
	role.level = level;
	role.metaphysical = metaphysical;
905
	role.access = access;
906
	if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
907 908 909 910
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
911
	pgprintk("%s: looking gfn %lx role %x\n", __func__,
912
		 gfn, role.word);
913
	index = kvm_page_table_hashfn(gfn);
914
	bucket = &vcpu->kvm->arch.mmu_page_hash[index];
915 916 917
	hlist_for_each_entry(sp, node, bucket, hash_link)
		if (sp->gfn == gfn && sp->role.word == role.word) {
			mmu_page_add_parent_pte(vcpu, sp, parent_pte);
918
			pgprintk("%s: found\n", __func__);
919
			return sp;
920
		}
921
	++vcpu->kvm->stat.mmu_cache_miss;
922 923 924
	sp = kvm_mmu_alloc_page(vcpu, parent_pte);
	if (!sp)
		return sp;
925
	pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word);
926 927 928
	sp->gfn = gfn;
	sp->role = role;
	hlist_add_head(&sp->hash_link, bucket);
929
	if (!metaphysical)
930
		rmap_write_protect(vcpu->kvm, gfn);
931 932 933 934
	if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
		vcpu->arch.mmu.prefetch_page(vcpu, sp);
	else
		nonpaging_prefetch_page(vcpu, sp);
935
	return sp;
936 937
}

938
static void kvm_mmu_page_unlink_children(struct kvm *kvm,
939
					 struct kvm_mmu_page *sp)
940
{
941 942 943 944
	unsigned i;
	u64 *pt;
	u64 ent;

945
	pt = sp->spt;
946

947
	if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
948
		for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
949
			if (is_shadow_present_pte(pt[i]))
950
				rmap_remove(kvm, &pt[i]);
951
			pt[i] = shadow_trap_nonpresent_pte;
952 953 954 955 956 957 958
		}
		return;
	}

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
		ent = pt[i];

959 960 961 962 963 964 965 966 967 968
		if (is_shadow_present_pte(ent)) {
			if (!is_large_pte(ent)) {
				ent &= PT64_BASE_ADDR_MASK;
				mmu_page_remove_parent_pte(page_header(ent),
							   &pt[i]);
			} else {
				--kvm->stat.lpages;
				rmap_remove(kvm, &pt[i]);
			}
		}
969
		pt[i] = shadow_trap_nonpresent_pte;
970
	}
971 972
}

973
static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
974
{
975
	mmu_page_remove_parent_pte(sp, parent_pte);
976 977
}

978 979 980 981 982 983
static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
{
	int i;

	for (i = 0; i < KVM_MAX_VCPUS; ++i)
		if (kvm->vcpus[i])
984
			kvm->vcpus[i]->arch.last_pte_updated = NULL;
985 986
}

987
static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
988 989 990
{
	u64 *parent_pte;

991 992 993
	while (sp->multimapped || sp->parent_pte) {
		if (!sp->multimapped)
			parent_pte = sp->parent_pte;
994 995 996
		else {
			struct kvm_pte_chain *chain;

997
			chain = container_of(sp->parent_ptes.first,
998 999 1000
					     struct kvm_pte_chain, link);
			parent_pte = chain->parent_ptes[0];
		}
1001
		BUG_ON(!parent_pte);
1002
		kvm_mmu_put_page(sp, parent_pte);
1003
		set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
1004
	}
1005 1006 1007 1008 1009
}

static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	++kvm->stat.mmu_shadow_zapped;
1010
	kvm_mmu_page_unlink_children(kvm, sp);
1011
	kvm_mmu_unlink_parents(kvm, sp);
1012 1013 1014
	kvm_flush_remote_tlbs(kvm);
	if (!sp->role.invalid && !sp->role.metaphysical)
		unaccount_shadowed(kvm, sp->gfn);
1015 1016 1017
	if (!sp->root_count) {
		hlist_del(&sp->hash_link);
		kvm_mmu_free_page(kvm, sp);
1018 1019
	} else {
		sp->role.invalid = 1;
1020
		list_move(&sp->link, &kvm->arch.active_mmu_pages);
1021 1022
		kvm_reload_remote_mmus(kvm);
	}
1023
	kvm_mmu_reset_last_pte_updated(kvm);
1024 1025
}

1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
/*
 * Changing the number of mmu pages allocated to the vm
 * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
 */
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
{
	/*
	 * If we set the number of mmu pages to be smaller be than the
	 * number of actived pages , we must to free some mmu pages before we
	 * change the value
	 */

1038
	if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) >
1039
	    kvm_nr_mmu_pages) {
1040 1041
		int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages
				       - kvm->arch.n_free_mmu_pages;
1042 1043 1044 1045

		while (n_used_mmu_pages > kvm_nr_mmu_pages) {
			struct kvm_mmu_page *page;

1046
			page = container_of(kvm->arch.active_mmu_pages.prev,
1047 1048 1049 1050
					    struct kvm_mmu_page, link);
			kvm_mmu_zap_page(kvm, page);
			n_used_mmu_pages--;
		}
1051
		kvm->arch.n_free_mmu_pages = 0;
1052 1053
	}
	else
1054 1055
		kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages
					 - kvm->arch.n_alloc_mmu_pages;
1056

1057
	kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages;
1058 1059
}

1060
static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
1061 1062 1063
{
	unsigned index;
	struct hlist_head *bucket;
1064
	struct kvm_mmu_page *sp;
1065 1066 1067
	struct hlist_node *node, *n;
	int r;

1068
	pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
1069
	r = 0;
1070
	index = kvm_page_table_hashfn(gfn);
1071
	bucket = &kvm->arch.mmu_page_hash[index];
1072 1073
	hlist_for_each_entry_safe(sp, node, n, bucket, hash_link)
		if (sp->gfn == gfn && !sp->role.metaphysical) {
1074
			pgprintk("%s: gfn %lx role %x\n", __func__, gfn,
1075 1076
				 sp->role.word);
			kvm_mmu_zap_page(kvm, sp);
1077 1078 1079
			r = 1;
		}
	return r;
1080 1081
}

1082
static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
1083
{
1084
	struct kvm_mmu_page *sp;
1085

1086
	while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) {
1087
		pgprintk("%s: zap %lx %x\n", __func__, gfn, sp->role.word);
1088
		kvm_mmu_zap_page(kvm, sp);
1089 1090 1091
	}
}

1092
static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
1093
{
1094
	int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn));
1095
	struct kvm_mmu_page *sp = page_header(__pa(pte));
1096

1097
	__set_bit(slot, &sp->slot_bitmap);
1098 1099
}

1100 1101
struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
{
1102 1103
	struct page *page;

1104
	gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
1105 1106 1107

	if (gpa == UNMAPPED_GVA)
		return NULL;
1108 1109 1110 1111 1112 1113

	down_read(&current->mm->mmap_sem);
	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
	up_read(&current->mm->mmap_sem);

	return page;
1114 1115
}

1116 1117 1118
static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
			 unsigned pt_access, unsigned pte_access,
			 int user_fault, int write_fault, int dirty,
1119
			 int *ptwrite, int largepage, gfn_t gfn,
1120
			 pfn_t pfn, bool speculative)
1121 1122
{
	u64 spte;
1123
	int was_rmapped = 0;
1124
	int was_writeble = is_writeble_pte(*shadow_pte);
1125

1126
	pgprintk("%s: spte %llx access %x write_fault %d"
1127
		 " user_fault %d gfn %lx\n",
1128
		 __func__, *shadow_pte, pt_access,
1129 1130
		 write_fault, user_fault, gfn);

1131
	if (is_rmap_pte(*shadow_pte)) {
1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
		/*
		 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
		 * the parent of the now unreachable PTE.
		 */
		if (largepage && !is_large_pte(*shadow_pte)) {
			struct kvm_mmu_page *child;
			u64 pte = *shadow_pte;

			child = page_header(pte & PT64_BASE_ADDR_MASK);
			mmu_page_remove_parent_pte(child, shadow_pte);
1142
		} else if (pfn != spte_to_pfn(*shadow_pte)) {
1143
			pgprintk("hfn old %lx new %lx\n",
1144
				 spte_to_pfn(*shadow_pte), pfn);
1145
			rmap_remove(vcpu->kvm, shadow_pte);
1146 1147 1148 1149 1150
		} else {
			if (largepage)
				was_rmapped = is_large_pte(*shadow_pte);
			else
				was_rmapped = 1;
1151 1152 1153
		}
	}

1154 1155 1156 1157 1158
	/*
	 * We don't set the accessed bit, since we sometimes want to see
	 * whether the guest actually used the pte (in order to detect
	 * demand paging).
	 */
S
Sheng Yang 已提交
1159
	spte = shadow_base_present_pte | shadow_dirty_mask;
1160 1161
	if (!speculative)
		pte_access |= PT_ACCESSED_MASK;
1162 1163
	if (!dirty)
		pte_access &= ~ACC_WRITE_MASK;
S
Sheng Yang 已提交
1164 1165 1166 1167
	if (pte_access & ACC_EXEC_MASK)
		spte |= shadow_x_mask;
	else
		spte |= shadow_nx_mask;
1168
	if (pte_access & ACC_USER_MASK)
S
Sheng Yang 已提交
1169
		spte |= shadow_user_mask;
1170 1171
	if (largepage)
		spte |= PT_PAGE_SIZE_MASK;
1172

1173
	spte |= (u64)pfn << PAGE_SHIFT;
1174 1175 1176 1177 1178 1179 1180 1181

	if ((pte_access & ACC_WRITE_MASK)
	    || (write_fault && !is_write_protection(vcpu) && !user_fault)) {
		struct kvm_mmu_page *shadow;

		spte |= PT_WRITABLE_MASK;

		shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn);
1182 1183
		if (shadow ||
		   (largepage && has_wrprotected_page(vcpu->kvm, gfn))) {
1184
			pgprintk("%s: found shadow page for %lx, marking ro\n",
1185
				 __func__, gfn);
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
			pte_access &= ~ACC_WRITE_MASK;
			if (is_writeble_pte(spte)) {
				spte &= ~PT_WRITABLE_MASK;
				kvm_x86_ops->tlb_flush(vcpu);
			}
			if (write_fault)
				*ptwrite = 1;
		}
	}

	if (pte_access & ACC_WRITE_MASK)
		mark_page_dirty(vcpu->kvm, gfn);

1199
	pgprintk("%s: setting spte %llx\n", __func__, spte);
A
Avi Kivity 已提交
1200
	pgprintk("instantiating %s PTE (%s) at %ld (%llx) addr %p\n",
1201 1202
		 (spte&PT_PAGE_SIZE_MASK)? "2MB" : "4kB",
		 (spte&PT_WRITABLE_MASK)?"RW":"R", gfn, spte, shadow_pte);
1203
	set_shadow_pte(shadow_pte, spte);
1204 1205 1206 1207
	if (!was_rmapped && (spte & PT_PAGE_SIZE_MASK)
	    && (spte & PT_PRESENT_MASK))
		++vcpu->kvm->stat.lpages;

1208 1209
	page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
	if (!was_rmapped) {
1210
		rmap_add(vcpu, shadow_pte, gfn, largepage);
1211
		if (!is_rmap_pte(*shadow_pte))
1212
			kvm_release_pfn_clean(pfn);
1213 1214
	} else {
		if (was_writeble)
1215
			kvm_release_pfn_dirty(pfn);
1216
		else
1217
			kvm_release_pfn_clean(pfn);
1218
	}
1219
	if (speculative) {
1220
		vcpu->arch.last_pte_updated = shadow_pte;
1221 1222
		vcpu->arch.last_pte_gfn = gfn;
	}
1223 1224
}

1225 1226 1227 1228
static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
{
}

1229
static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
1230
			   int largepage, gfn_t gfn, pfn_t pfn,
1231
			   int level)
1232
{
1233
	hpa_t table_addr = vcpu->arch.mmu.root_hpa;
1234
	int pt_write = 0;
1235 1236 1237 1238 1239 1240 1241 1242

	for (; ; level--) {
		u32 index = PT64_INDEX(v, level);
		u64 *table;

		ASSERT(VALID_PAGE(table_addr));
		table = __va(table_addr);

1243
		if (level == 1 || (largepage && level == 2)) {
1244
			mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL,
1245 1246
				     0, write, 1, &pt_write, largepage,
				     gfn, pfn, false);
1247
			return pt_write;
1248 1249
		}

1250
		if (table[index] == shadow_trap_nonpresent_pte) {
1251
			struct kvm_mmu_page *new_table;
1252
			gfn_t pseudo_gfn;
1253

1254 1255 1256 1257
			pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
				>> PAGE_SHIFT;
			new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
						     v, level - 1,
1258
						     1, ACC_ALL, &table[index]);
1259
			if (!new_table) {
1260
				pgprintk("nonpaging_map: ENOMEM\n");
1261
				kvm_release_pfn_clean(pfn);
1262 1263 1264
				return -ENOMEM;
			}

1265 1266 1267 1268
			set_shadow_pte(&table[index],
				       __pa(new_table->spt)
				       | PT_PRESENT_MASK | PT_WRITABLE_MASK
				       | shadow_user_mask | shadow_x_mask);
1269 1270 1271 1272 1273
		}
		table_addr = table[index] & PT64_BASE_ADDR_MASK;
	}
}

1274 1275 1276
static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
{
	int r;
1277
	int largepage = 0;
1278
	pfn_t pfn;
1279
	unsigned long mmu_seq;
1280 1281

	down_read(&current->mm->mmap_sem);
1282 1283 1284 1285 1286
	if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
		gfn &= ~(KVM_PAGES_PER_HPAGE-1);
		largepage = 1;
	}

1287 1288
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
	/* implicit mb(), we'll read before PT lock is unlocked */
1289
	pfn = gfn_to_pfn(vcpu->kvm, gfn);
1290
	up_read(&current->mm->mmap_sem);
1291

1292
	/* mmio */
1293 1294
	if (is_error_pfn(pfn)) {
		kvm_release_pfn_clean(pfn);
1295 1296 1297
		return 1;
	}

1298
	spin_lock(&vcpu->kvm->mmu_lock);
1299 1300
	if (mmu_notifier_retry(vcpu, mmu_seq))
		goto out_unlock;
1301
	kvm_mmu_free_some_pages(vcpu);
1302
	r = __direct_map(vcpu, v, write, largepage, gfn, pfn,
1303
			 PT32E_ROOT_LEVEL);
1304 1305 1306
	spin_unlock(&vcpu->kvm->mmu_lock);


1307
	return r;
1308 1309 1310 1311 1312

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
1313 1314 1315
}


1316 1317 1318
static void mmu_free_roots(struct kvm_vcpu *vcpu)
{
	int i;
1319
	struct kvm_mmu_page *sp;
1320

1321
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
1322
		return;
1323
	spin_lock(&vcpu->kvm->mmu_lock);
1324 1325
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		hpa_t root = vcpu->arch.mmu.root_hpa;
1326

1327 1328
		sp = page_header(root);
		--sp->root_count;
1329 1330
		if (!sp->root_count && sp->role.invalid)
			kvm_mmu_zap_page(vcpu->kvm, sp);
1331
		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1332
		spin_unlock(&vcpu->kvm->mmu_lock);
1333 1334 1335
		return;
	}
	for (i = 0; i < 4; ++i) {
1336
		hpa_t root = vcpu->arch.mmu.pae_root[i];
1337

1338 1339
		if (root) {
			root &= PT64_BASE_ADDR_MASK;
1340 1341
			sp = page_header(root);
			--sp->root_count;
1342 1343
			if (!sp->root_count && sp->role.invalid)
				kvm_mmu_zap_page(vcpu->kvm, sp);
1344
		}
1345
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
1346
	}
1347
	spin_unlock(&vcpu->kvm->mmu_lock);
1348
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1349 1350 1351 1352 1353
}

static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
{
	int i;
1354
	gfn_t root_gfn;
1355
	struct kvm_mmu_page *sp;
1356
	int metaphysical = 0;
1357

1358
	root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
1359

1360 1361
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		hpa_t root = vcpu->arch.mmu.root_hpa;
1362 1363

		ASSERT(!VALID_PAGE(root));
1364 1365
		if (tdp_enabled)
			metaphysical = 1;
1366
		sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
1367 1368
				      PT64_ROOT_LEVEL, metaphysical,
				      ACC_ALL, NULL);
1369 1370
		root = __pa(sp->spt);
		++sp->root_count;
1371
		vcpu->arch.mmu.root_hpa = root;
1372 1373
		return;
	}
1374 1375 1376
	metaphysical = !is_paging(vcpu);
	if (tdp_enabled)
		metaphysical = 1;
1377
	for (i = 0; i < 4; ++i) {
1378
		hpa_t root = vcpu->arch.mmu.pae_root[i];
1379 1380

		ASSERT(!VALID_PAGE(root));
1381 1382 1383
		if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
			if (!is_present_pte(vcpu->arch.pdptrs[i])) {
				vcpu->arch.mmu.pae_root[i] = 0;
1384 1385
				continue;
			}
1386 1387
			root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
		} else if (vcpu->arch.mmu.root_level == 0)
1388
			root_gfn = 0;
1389
		sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
1390
				      PT32_ROOT_LEVEL, metaphysical,
1391
				      ACC_ALL, NULL);
1392 1393
		root = __pa(sp->spt);
		++sp->root_count;
1394
		vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
1395
	}
1396
	vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
1397 1398
}

1399 1400 1401 1402 1403 1404
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
{
	return vaddr;
}

static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
1405
				u32 error_code)
1406
{
1407
	gfn_t gfn;
1408
	int r;
1409

1410
	pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
1411 1412 1413
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;
1414

1415
	ASSERT(vcpu);
1416
	ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
1417

1418
	gfn = gva >> PAGE_SHIFT;
1419

1420 1421
	return nonpaging_map(vcpu, gva & PAGE_MASK,
			     error_code & PFERR_WRITE_MASK, gfn);
1422 1423
}

1424 1425 1426
static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
				u32 error_code)
{
1427
	pfn_t pfn;
1428
	int r;
1429 1430
	int largepage = 0;
	gfn_t gfn = gpa >> PAGE_SHIFT;
1431
	unsigned long mmu_seq;
1432 1433 1434 1435 1436 1437 1438 1439 1440

	ASSERT(vcpu);
	ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));

	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

	down_read(&current->mm->mmap_sem);
1441 1442 1443 1444
	if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
		gfn &= ~(KVM_PAGES_PER_HPAGE-1);
		largepage = 1;
	}
1445 1446
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
	/* implicit mb(), we'll read before PT lock is unlocked */
1447
	pfn = gfn_to_pfn(vcpu->kvm, gfn);
1448
	up_read(&current->mm->mmap_sem);
1449 1450
	if (is_error_pfn(pfn)) {
		kvm_release_pfn_clean(pfn);
1451 1452 1453
		return 1;
	}
	spin_lock(&vcpu->kvm->mmu_lock);
1454 1455
	if (mmu_notifier_retry(vcpu, mmu_seq))
		goto out_unlock;
1456 1457
	kvm_mmu_free_some_pages(vcpu);
	r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
1458
			 largepage, gfn, pfn, kvm_x86_ops->get_tdp_level());
1459 1460 1461
	spin_unlock(&vcpu->kvm->mmu_lock);

	return r;
1462 1463 1464 1465 1466

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
1467 1468
}

1469 1470
static void nonpaging_free(struct kvm_vcpu *vcpu)
{
1471
	mmu_free_roots(vcpu);
1472 1473 1474 1475
}

static int nonpaging_init_context(struct kvm_vcpu *vcpu)
{
1476
	struct kvm_mmu *context = &vcpu->arch.mmu;
1477 1478 1479 1480 1481

	context->new_cr3 = nonpaging_new_cr3;
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
	context->free = nonpaging_free;
1482
	context->prefetch_page = nonpaging_prefetch_page;
1483
	context->root_level = 0;
1484
	context->shadow_root_level = PT32E_ROOT_LEVEL;
1485
	context->root_hpa = INVALID_PAGE;
1486 1487 1488
	return 0;
}

1489
void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
1490
{
A
Avi Kivity 已提交
1491
	++vcpu->stat.tlb_flush;
1492
	kvm_x86_ops->tlb_flush(vcpu);
1493 1494 1495 1496
}

static void paging_new_cr3(struct kvm_vcpu *vcpu)
{
1497
	pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
1498
	mmu_free_roots(vcpu);
1499 1500 1501 1502 1503 1504
}

static void inject_page_fault(struct kvm_vcpu *vcpu,
			      u64 addr,
			      u32 err_code)
{
1505
	kvm_inject_page_fault(vcpu, addr, err_code);
1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
}

static void paging_free(struct kvm_vcpu *vcpu)
{
	nonpaging_free(vcpu);
}

#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

1521
static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
1522
{
1523
	struct kvm_mmu *context = &vcpu->arch.mmu;
1524 1525 1526 1527 1528

	ASSERT(is_pae(vcpu));
	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
1529
	context->prefetch_page = paging64_prefetch_page;
1530
	context->free = paging_free;
1531 1532
	context->root_level = level;
	context->shadow_root_level = level;
1533
	context->root_hpa = INVALID_PAGE;
1534 1535 1536
	return 0;
}

1537 1538 1539 1540 1541
static int paging64_init_context(struct kvm_vcpu *vcpu)
{
	return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
}

1542 1543
static int paging32_init_context(struct kvm_vcpu *vcpu)
{
1544
	struct kvm_mmu *context = &vcpu->arch.mmu;
1545 1546 1547 1548 1549

	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
	context->free = paging_free;
1550
	context->prefetch_page = paging32_prefetch_page;
1551 1552
	context->root_level = PT32_ROOT_LEVEL;
	context->shadow_root_level = PT32E_ROOT_LEVEL;
1553
	context->root_hpa = INVALID_PAGE;
1554 1555 1556 1557 1558
	return 0;
}

static int paging32E_init_context(struct kvm_vcpu *vcpu)
{
1559
	return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
1560 1561
}

1562 1563 1564 1565 1566 1567 1568 1569
static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu *context = &vcpu->arch.mmu;

	context->new_cr3 = nonpaging_new_cr3;
	context->page_fault = tdp_page_fault;
	context->free = nonpaging_free;
	context->prefetch_page = nonpaging_prefetch_page;
1570
	context->shadow_root_level = kvm_x86_ops->get_tdp_level();
1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590
	context->root_hpa = INVALID_PAGE;

	if (!is_paging(vcpu)) {
		context->gva_to_gpa = nonpaging_gva_to_gpa;
		context->root_level = 0;
	} else if (is_long_mode(vcpu)) {
		context->gva_to_gpa = paging64_gva_to_gpa;
		context->root_level = PT64_ROOT_LEVEL;
	} else if (is_pae(vcpu)) {
		context->gva_to_gpa = paging64_gva_to_gpa;
		context->root_level = PT32E_ROOT_LEVEL;
	} else {
		context->gva_to_gpa = paging32_gva_to_gpa;
		context->root_level = PT32_ROOT_LEVEL;
	}

	return 0;
}

static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
1591 1592
{
	ASSERT(vcpu);
1593
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
1594 1595 1596

	if (!is_paging(vcpu))
		return nonpaging_init_context(vcpu);
1597
	else if (is_long_mode(vcpu))
1598 1599 1600 1601 1602 1603 1604
		return paging64_init_context(vcpu);
	else if (is_pae(vcpu))
		return paging32E_init_context(vcpu);
	else
		return paging32_init_context(vcpu);
}

1605 1606
static int init_kvm_mmu(struct kvm_vcpu *vcpu)
{
1607 1608
	vcpu->arch.update_pte.pfn = bad_pfn;

1609 1610 1611 1612 1613 1614
	if (tdp_enabled)
		return init_kvm_tdp_mmu(vcpu);
	else
		return init_kvm_softmmu(vcpu);
}

1615 1616 1617
static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
1618 1619 1620
	if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) {
		vcpu->arch.mmu.free(vcpu);
		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1621 1622 1623 1624
	}
}

int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
1625 1626 1627 1628
{
	destroy_kvm_mmu(vcpu);
	return init_kvm_mmu(vcpu);
}
1629
EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
1630 1631

int kvm_mmu_load(struct kvm_vcpu *vcpu)
1632
{
1633 1634
	int r;

1635
	r = mmu_topup_memory_caches(vcpu);
1636 1637
	if (r)
		goto out;
1638
	spin_lock(&vcpu->kvm->mmu_lock);
1639
	kvm_mmu_free_some_pages(vcpu);
1640
	mmu_alloc_roots(vcpu);
1641
	spin_unlock(&vcpu->kvm->mmu_lock);
1642
	kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
1643
	kvm_mmu_flush_tlb(vcpu);
1644 1645
out:
	return r;
1646
}
1647 1648 1649 1650 1651 1652
EXPORT_SYMBOL_GPL(kvm_mmu_load);

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
	mmu_free_roots(vcpu);
}
1653

1654
static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
1655
				  struct kvm_mmu_page *sp,
1656 1657 1658 1659 1660 1661
				  u64 *spte)
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
1662
	if (is_shadow_present_pte(pte)) {
1663 1664
		if (sp->role.level == PT_PAGE_TABLE_LEVEL ||
		    is_large_pte(pte))
1665
			rmap_remove(vcpu->kvm, spte);
1666 1667
		else {
			child = page_header(pte & PT64_BASE_ADDR_MASK);
1668
			mmu_page_remove_parent_pte(child, spte);
1669 1670
		}
	}
1671
	set_shadow_pte(spte, shadow_trap_nonpresent_pte);
1672 1673
	if (is_large_pte(pte))
		--vcpu->kvm->stat.lpages;
1674 1675
}

1676
static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
1677
				  struct kvm_mmu_page *sp,
1678
				  u64 *spte,
1679
				  const void *new)
1680
{
1681 1682 1683 1684 1685 1686 1687
	if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
		if (!vcpu->arch.update_pte.largepage ||
		    sp->role.glevels == PT32_ROOT_LEVEL) {
			++vcpu->kvm->stat.mmu_pde_zapped;
			return;
		}
        }
1688

1689
	++vcpu->kvm->stat.mmu_pte_updated;
1690
	if (sp->role.glevels == PT32_ROOT_LEVEL)
1691
		paging32_update_pte(vcpu, sp, spte, new);
1692
	else
1693
		paging64_update_pte(vcpu, sp, spte, new);
1694 1695
}

1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716
static bool need_remote_flush(u64 old, u64 new)
{
	if (!is_shadow_present_pte(old))
		return false;
	if (!is_shadow_present_pte(new))
		return true;
	if ((old ^ new) & PT64_BASE_ADDR_MASK)
		return true;
	old ^= PT64_NX_MASK;
	new ^= PT64_NX_MASK;
	return (old & ~new & PT64_PERM_MASK) != 0;
}

static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new)
{
	if (need_remote_flush(old, new))
		kvm_flush_remote_tlbs(vcpu->kvm);
	else
		kvm_mmu_flush_tlb(vcpu);
}

1717 1718
static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
{
1719
	u64 *spte = vcpu->arch.last_pte_updated;
1720

S
Sheng Yang 已提交
1721
	return !!(spte && (*spte & shadow_accessed_mask));
1722 1723
}

1724 1725 1726 1727 1728 1729
static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
					  const u8 *new, int bytes)
{
	gfn_t gfn;
	int r;
	u64 gpte = 0;
1730
	pfn_t pfn;
1731

1732 1733
	vcpu->arch.update_pte.largepage = 0;

1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759
	if (bytes != 4 && bytes != 8)
		return;

	/*
	 * Assume that the pte write on a page table of the same type
	 * as the current vcpu paging mode.  This is nearly always true
	 * (might be false while changing modes).  Note it is verified later
	 * by update_pte().
	 */
	if (is_pae(vcpu)) {
		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */
		if ((bytes == 4) && (gpa % 4 == 0)) {
			r = kvm_read_guest(vcpu->kvm, gpa & ~(u64)7, &gpte, 8);
			if (r)
				return;
			memcpy((void *)&gpte + (gpa % 8), new, 4);
		} else if ((bytes == 8) && (gpa % 8 == 0)) {
			memcpy((void *)&gpte, new, 8);
		}
	} else {
		if ((bytes == 4) && (gpa % 4 == 0))
			memcpy((void *)&gpte, new, 4);
	}
	if (!is_present_pte(gpte))
		return;
	gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
1760

1761 1762 1763 1764 1765
	down_read(&current->mm->mmap_sem);
	if (is_large_pte(gpte) && is_largepage_backed(vcpu, gfn)) {
		gfn &= ~(KVM_PAGES_PER_HPAGE-1);
		vcpu->arch.update_pte.largepage = 1;
	}
1766 1767
	vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
	/* implicit mb(), we'll read before PT lock is unlocked */
1768
	pfn = gfn_to_pfn(vcpu->kvm, gfn);
1769
	up_read(&current->mm->mmap_sem);
1770

1771 1772
	if (is_error_pfn(pfn)) {
		kvm_release_pfn_clean(pfn);
1773 1774
		return;
	}
1775
	vcpu->arch.update_pte.gfn = gfn;
1776
	vcpu->arch.update_pte.pfn = pfn;
1777 1778
}

1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790
static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u64 *spte = vcpu->arch.last_pte_updated;

	if (spte
	    && vcpu->arch.last_pte_gfn == gfn
	    && shadow_accessed_mask
	    && !(*spte & shadow_accessed_mask)
	    && is_shadow_present_pte(*spte))
		set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
}

1791
void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1792
		       const u8 *new, int bytes)
1793
{
1794
	gfn_t gfn = gpa >> PAGE_SHIFT;
1795
	struct kvm_mmu_page *sp;
1796
	struct hlist_node *node, *n;
1797 1798
	struct hlist_head *bucket;
	unsigned index;
1799
	u64 entry, gentry;
1800 1801
	u64 *spte;
	unsigned offset = offset_in_page(gpa);
1802
	unsigned pte_size;
1803
	unsigned page_offset;
1804
	unsigned misaligned;
1805
	unsigned quadrant;
1806
	int level;
1807
	int flooded = 0;
1808
	int npte;
1809
	int r;
1810

1811
	pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
1812
	mmu_guess_page_from_pte_write(vcpu, gpa, new, bytes);
1813
	spin_lock(&vcpu->kvm->mmu_lock);
1814
	kvm_mmu_access_page(vcpu, gfn);
1815
	kvm_mmu_free_some_pages(vcpu);
1816
	++vcpu->kvm->stat.mmu_pte_write;
1817
	kvm_mmu_audit(vcpu, "pre pte write");
1818
	if (gfn == vcpu->arch.last_pt_write_gfn
1819
	    && !last_updated_pte_accessed(vcpu)) {
1820 1821
		++vcpu->arch.last_pt_write_count;
		if (vcpu->arch.last_pt_write_count >= 3)
1822 1823
			flooded = 1;
	} else {
1824 1825 1826
		vcpu->arch.last_pt_write_gfn = gfn;
		vcpu->arch.last_pt_write_count = 1;
		vcpu->arch.last_pte_updated = NULL;
1827
	}
1828
	index = kvm_page_table_hashfn(gfn);
1829
	bucket = &vcpu->kvm->arch.mmu_page_hash[index];
1830
	hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) {
1831
		if (sp->gfn != gfn || sp->role.metaphysical || sp->role.invalid)
1832
			continue;
1833
		pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
1834
		misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
1835
		misaligned |= bytes < 4;
1836
		if (misaligned || flooded) {
1837 1838 1839 1840
			/*
			 * Misaligned accesses are too much trouble to fix
			 * up; also, they usually indicate a page is not used
			 * as a page table.
1841 1842 1843 1844 1845
			 *
			 * If we're seeing too many writes to a page,
			 * it may no longer be a page table, or we may be
			 * forking, in which case it is better to unmap the
			 * page.
1846 1847
			 */
			pgprintk("misaligned: gpa %llx bytes %d role %x\n",
1848 1849
				 gpa, bytes, sp->role.word);
			kvm_mmu_zap_page(vcpu->kvm, sp);
1850
			++vcpu->kvm->stat.mmu_flooded;
1851 1852
			continue;
		}
1853
		page_offset = offset;
1854
		level = sp->role.level;
1855
		npte = 1;
1856
		if (sp->role.glevels == PT32_ROOT_LEVEL) {
1857 1858 1859 1860 1861 1862 1863
			page_offset <<= 1;	/* 32->64 */
			/*
			 * A 32-bit pde maps 4MB while the shadow pdes map
			 * only 2MB.  So we need to double the offset again
			 * and zap two pdes instead of one.
			 */
			if (level == PT32_ROOT_LEVEL) {
1864
				page_offset &= ~7; /* kill rounding error */
1865 1866 1867
				page_offset <<= 1;
				npte = 2;
			}
1868
			quadrant = page_offset >> PAGE_SHIFT;
1869
			page_offset &= ~PAGE_MASK;
1870
			if (quadrant != sp->role.quadrant)
1871
				continue;
1872
		}
1873
		spte = &sp->spt[page_offset / sizeof(*spte)];
1874 1875 1876 1877 1878 1879 1880 1881 1882
		if ((gpa & (pte_size - 1)) || (bytes < pte_size)) {
			gentry = 0;
			r = kvm_read_guest_atomic(vcpu->kvm,
						  gpa & ~(u64)(pte_size - 1),
						  &gentry, pte_size);
			new = (const void *)&gentry;
			if (r < 0)
				new = NULL;
		}
1883
		while (npte--) {
1884
			entry = *spte;
1885
			mmu_pte_write_zap_pte(vcpu, sp, spte);
1886 1887
			if (new)
				mmu_pte_write_new_pte(vcpu, sp, spte, new);
1888
			mmu_pte_write_flush_tlb(vcpu, entry, *spte);
1889
			++spte;
1890 1891
		}
	}
1892
	kvm_mmu_audit(vcpu, "post pte write");
1893
	spin_unlock(&vcpu->kvm->mmu_lock);
1894 1895 1896
	if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
		kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
		vcpu->arch.update_pte.pfn = bad_pfn;
1897
	}
1898 1899
}

1900 1901
int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
1902 1903
	gpa_t gpa;
	int r;
1904

1905 1906
	gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);

1907
	spin_lock(&vcpu->kvm->mmu_lock);
1908
	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1909
	spin_unlock(&vcpu->kvm->mmu_lock);
1910
	return r;
1911
}
1912
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
1913

1914
void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
1915
{
1916
	while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
1917
		struct kvm_mmu_page *sp;
1918

1919
		sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
1920 1921
				  struct kvm_mmu_page, link);
		kvm_mmu_zap_page(vcpu->kvm, sp);
1922
		++vcpu->kvm->stat.mmu_recycled;
1923 1924 1925
	}
}

1926 1927 1928 1929 1930
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
{
	int r;
	enum emulation_result er;

1931
	r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
1932 1933 1934 1935 1936 1937 1938 1939
	if (r < 0)
		goto out;

	if (!r) {
		r = 1;
		goto out;
	}

1940 1941 1942 1943
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		goto out;

1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962
	er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0);

	switch (er) {
	case EMULATE_DONE:
		return 1;
	case EMULATE_DO_MMIO:
		++vcpu->stat.mmio_exits;
		return 0;
	case EMULATE_FAIL:
		kvm_report_emulation_failure(vcpu, "pagetable");
		return 1;
	default:
		BUG();
	}
out:
	return r;
}
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);

1963 1964 1965 1966 1967 1968
void kvm_enable_tdp(void)
{
	tdp_enabled = true;
}
EXPORT_SYMBOL_GPL(kvm_enable_tdp);

1969 1970 1971 1972 1973 1974
void kvm_disable_tdp(void)
{
	tdp_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_tdp);

1975 1976
static void free_mmu_pages(struct kvm_vcpu *vcpu)
{
1977
	struct kvm_mmu_page *sp;
1978

1979 1980
	while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
		sp = container_of(vcpu->kvm->arch.active_mmu_pages.next,
1981 1982
				  struct kvm_mmu_page, link);
		kvm_mmu_zap_page(vcpu->kvm, sp);
1983
		cond_resched();
1984
	}
1985
	free_page((unsigned long)vcpu->arch.mmu.pae_root);
1986 1987 1988 1989
}

static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
{
1990
	struct page *page;
1991 1992 1993 1994
	int i;

	ASSERT(vcpu);

1995 1996 1997
	if (vcpu->kvm->arch.n_requested_mmu_pages)
		vcpu->kvm->arch.n_free_mmu_pages =
					vcpu->kvm->arch.n_requested_mmu_pages;
1998
	else
1999 2000
		vcpu->kvm->arch.n_free_mmu_pages =
					vcpu->kvm->arch.n_alloc_mmu_pages;
2001 2002 2003 2004 2005 2006 2007 2008
	/*
	 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
	 * Therefore we need to allocate shadow page tables in the first
	 * 4GB of memory, which happens to fit the DMA32 zone.
	 */
	page = alloc_page(GFP_KERNEL | __GFP_DMA32);
	if (!page)
		goto error_1;
2009
	vcpu->arch.mmu.pae_root = page_address(page);
2010
	for (i = 0; i < 4; ++i)
2011
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2012

2013 2014 2015 2016 2017 2018 2019
	return 0;

error_1:
	free_mmu_pages(vcpu);
	return -ENOMEM;
}

2020
int kvm_mmu_create(struct kvm_vcpu *vcpu)
2021 2022
{
	ASSERT(vcpu);
2023
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2024

2025 2026
	return alloc_mmu_pages(vcpu);
}
2027

2028 2029 2030
int kvm_mmu_setup(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
2031
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2032

2033
	return init_kvm_mmu(vcpu);
2034 2035 2036 2037 2038 2039 2040 2041
}

void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);

	destroy_kvm_mmu(vcpu);
	free_mmu_pages(vcpu);
2042
	mmu_free_memory_caches(vcpu);
2043 2044
}

2045
void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
2046
{
2047
	struct kvm_mmu_page *sp;
2048

2049
	list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
2050 2051 2052
		int i;
		u64 *pt;

2053
		if (!test_bit(slot, &sp->slot_bitmap))
2054 2055
			continue;

2056
		pt = sp->spt;
2057 2058
		for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
			/* avoid RMW */
2059
			if (pt[i] & PT_WRITABLE_MASK)
2060 2061 2062
				pt[i] &= ~PT_WRITABLE_MASK;
	}
}
2063

2064
void kvm_mmu_zap_all(struct kvm *kvm)
2065
{
2066
	struct kvm_mmu_page *sp, *node;
2067

2068
	spin_lock(&kvm->mmu_lock);
2069
	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
2070
		kvm_mmu_zap_page(kvm, sp);
2071
	spin_unlock(&kvm->mmu_lock);
2072

2073
	kvm_flush_remote_tlbs(kvm);
2074 2075
}

2076
static void kvm_mmu_remove_one_alloc_mmu_page(struct kvm *kvm)
2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095
{
	struct kvm_mmu_page *page;

	page = container_of(kvm->arch.active_mmu_pages.prev,
			    struct kvm_mmu_page, link);
	kvm_mmu_zap_page(kvm, page);
}

static int mmu_shrink(int nr_to_scan, gfp_t gfp_mask)
{
	struct kvm *kvm;
	struct kvm *kvm_freed = NULL;
	int cache_count = 0;

	spin_lock(&kvm_lock);

	list_for_each_entry(kvm, &vm_list, vm_list) {
		int npages;

2096 2097
		if (!down_read_trylock(&kvm->slots_lock))
			continue;
2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109
		spin_lock(&kvm->mmu_lock);
		npages = kvm->arch.n_alloc_mmu_pages -
			 kvm->arch.n_free_mmu_pages;
		cache_count += npages;
		if (!kvm_freed && nr_to_scan > 0 && npages > 0) {
			kvm_mmu_remove_one_alloc_mmu_page(kvm);
			cache_count--;
			kvm_freed = kvm;
		}
		nr_to_scan--;

		spin_unlock(&kvm->mmu_lock);
2110
		up_read(&kvm->slots_lock);
2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124
	}
	if (kvm_freed)
		list_move_tail(&kvm_freed->vm_list, &vm_list);

	spin_unlock(&kvm_lock);

	return cache_count;
}

static struct shrinker mmu_shrinker = {
	.shrink = mmu_shrink,
	.seeks = DEFAULT_SEEKS * 10,
};

2125
static void mmu_destroy_caches(void)
2126 2127 2128 2129 2130
{
	if (pte_chain_cache)
		kmem_cache_destroy(pte_chain_cache);
	if (rmap_desc_cache)
		kmem_cache_destroy(rmap_desc_cache);
2131 2132
	if (mmu_page_header_cache)
		kmem_cache_destroy(mmu_page_header_cache);
2133 2134
}

2135 2136 2137 2138 2139 2140
void kvm_mmu_module_exit(void)
{
	mmu_destroy_caches();
	unregister_shrinker(&mmu_shrinker);
}

2141 2142 2143 2144
int kvm_mmu_module_init(void)
{
	pte_chain_cache = kmem_cache_create("kvm_pte_chain",
					    sizeof(struct kvm_pte_chain),
2145
					    0, 0, NULL);
2146 2147 2148 2149
	if (!pte_chain_cache)
		goto nomem;
	rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
					    sizeof(struct kvm_rmap_desc),
2150
					    0, 0, NULL);
2151 2152 2153
	if (!rmap_desc_cache)
		goto nomem;

2154 2155
	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
2156
						  0, 0, NULL);
2157 2158 2159
	if (!mmu_page_header_cache)
		goto nomem;

2160 2161
	register_shrinker(&mmu_shrinker);

2162 2163 2164
	return 0;

nomem:
2165
	mmu_destroy_caches();
2166 2167 2168
	return -ENOMEM;
}

2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187
/*
 * Caculate mmu pages needed for kvm.
 */
unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
{
	int i;
	unsigned int nr_mmu_pages;
	unsigned int  nr_pages = 0;

	for (i = 0; i < kvm->nmemslots; i++)
		nr_pages += kvm->memslots[i].npages;

	nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
	nr_mmu_pages = max(nr_mmu_pages,
			(unsigned int) KVM_MIN_ALLOC_MMU_PAGES);

	return nr_mmu_pages;
}

2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222
static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
				unsigned len)
{
	if (len > buffer->len)
		return NULL;
	return buffer->ptr;
}

static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
				unsigned len)
{
	void *ret;

	ret = pv_mmu_peek_buffer(buffer, len);
	if (!ret)
		return ret;
	buffer->ptr += len;
	buffer->len -= len;
	buffer->processed += len;
	return ret;
}

static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
			     gpa_t addr, gpa_t value)
{
	int bytes = 8;
	int r;

	if (!is_long_mode(vcpu) && !is_pae(vcpu))
		bytes = 4;

	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

2223
	if (!emulator_write_phys(vcpu, addr, &value, bytes))
2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284
		return -EFAULT;

	return 1;
}

static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
{
	kvm_x86_ops->tlb_flush(vcpu);
	return 1;
}

static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
{
	spin_lock(&vcpu->kvm->mmu_lock);
	mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
	spin_unlock(&vcpu->kvm->mmu_lock);
	return 1;
}

static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
			     struct kvm_pv_mmu_op_buffer *buffer)
{
	struct kvm_mmu_op_header *header;

	header = pv_mmu_peek_buffer(buffer, sizeof *header);
	if (!header)
		return 0;
	switch (header->op) {
	case KVM_MMU_OP_WRITE_PTE: {
		struct kvm_mmu_op_write_pte *wpte;

		wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
		if (!wpte)
			return 0;
		return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
					wpte->pte_val);
	}
	case KVM_MMU_OP_FLUSH_TLB: {
		struct kvm_mmu_op_flush_tlb *ftlb;

		ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
		if (!ftlb)
			return 0;
		return kvm_pv_mmu_flush_tlb(vcpu);
	}
	case KVM_MMU_OP_RELEASE_PT: {
		struct kvm_mmu_op_release_pt *rpt;

		rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
		if (!rpt)
			return 0;
		return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
	}
	default: return 0;
	}
}

int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
		  gpa_t addr, unsigned long *ret)
{
	int r;
2285
	struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
2286

2287 2288 2289
	buffer->ptr = buffer->buf;
	buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
	buffer->processed = 0;
2290

2291
	r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
2292 2293 2294
	if (r)
		goto out;

2295 2296
	while (buffer->len) {
		r = kvm_pv_mmu_op_one(vcpu, buffer);
2297 2298 2299 2300 2301 2302 2303 2304
		if (r < 0)
			goto out;
		if (r == 0)
			break;
	}

	r = 1;
out:
2305
	*ret = buffer->processed;
2306 2307 2308
	return r;
}

2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330
#ifdef AUDIT

static const char *audit_msg;

static gva_t canonicalize(gva_t gva)
{
#ifdef CONFIG_X86_64
	gva = (long long)(gva << 16) >> 16;
#endif
	return gva;
}

static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
				gva_t va, int level)
{
	u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
	int i;
	gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
		u64 ent = pt[i];

2331
		if (ent == shadow_trap_nonpresent_pte)
2332 2333 2334
			continue;

		va = canonicalize(va);
2335 2336 2337 2338 2339
		if (level > 1) {
			if (ent == shadow_notrap_nonpresent_pte)
				printk(KERN_ERR "audit: (%s) nontrapping pte"
				       " in nonleaf level: levels %d gva %lx"
				       " level %d pte %llx\n", audit_msg,
2340
				       vcpu->arch.mmu.root_level, va, level, ent);
2341

2342
			audit_mappings_page(vcpu, ent, va, level - 1);
2343
		} else {
2344
			gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
2345
			hpa_t hpa = (hpa_t)gpa_to_pfn(vcpu, gpa) << PAGE_SHIFT;
2346

2347
			if (is_shadow_present_pte(ent)
2348
			    && (ent & PT64_BASE_ADDR_MASK) != hpa)
2349 2350
				printk(KERN_ERR "xx audit error: (%s) levels %d"
				       " gva %lx gpa %llx hpa %llx ent %llx %d\n",
2351
				       audit_msg, vcpu->arch.mmu.root_level,
M
Mike Day 已提交
2352 2353
				       va, gpa, hpa, ent,
				       is_shadow_present_pte(ent));
2354 2355 2356 2357
			else if (ent == shadow_notrap_nonpresent_pte
				 && !is_error_hpa(hpa))
				printk(KERN_ERR "audit: (%s) notrap shadow,"
				       " valid guest gva %lx\n", audit_msg, va);
2358
			kvm_release_pfn_clean(pfn);
2359

2360 2361 2362 2363 2364 2365
		}
	}
}

static void audit_mappings(struct kvm_vcpu *vcpu)
{
2366
	unsigned i;
2367

2368 2369
	if (vcpu->arch.mmu.root_level == 4)
		audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4);
2370 2371
	else
		for (i = 0; i < 4; ++i)
2372
			if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK)
2373
				audit_mappings_page(vcpu,
2374
						    vcpu->arch.mmu.pae_root[i],
2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388
						    i << 30,
						    2);
}

static int count_rmaps(struct kvm_vcpu *vcpu)
{
	int nmaps = 0;
	int i, j, k;

	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
		struct kvm_rmap_desc *d;

		for (j = 0; j < m->npages; ++j) {
2389
			unsigned long *rmapp = &m->rmap[j];
2390

2391
			if (!*rmapp)
2392
				continue;
2393
			if (!(*rmapp & 1)) {
2394 2395 2396
				++nmaps;
				continue;
			}
2397
			d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413
			while (d) {
				for (k = 0; k < RMAP_EXT; ++k)
					if (d->shadow_ptes[k])
						++nmaps;
					else
						break;
				d = d->more;
			}
		}
	}
	return nmaps;
}

static int count_writable_mappings(struct kvm_vcpu *vcpu)
{
	int nmaps = 0;
2414
	struct kvm_mmu_page *sp;
2415 2416
	int i;

2417
	list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
2418
		u64 *pt = sp->spt;
2419

2420
		if (sp->role.level != PT_PAGE_TABLE_LEVEL)
2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
			continue;

		for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
			u64 ent = pt[i];

			if (!(ent & PT_PRESENT_MASK))
				continue;
			if (!(ent & PT_WRITABLE_MASK))
				continue;
			++nmaps;
		}
	}
	return nmaps;
}

static void audit_rmap(struct kvm_vcpu *vcpu)
{
	int n_rmap = count_rmaps(vcpu);
	int n_actual = count_writable_mappings(vcpu);

	if (n_rmap != n_actual)
		printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
2443
		       __func__, audit_msg, n_rmap, n_actual);
2444 2445 2446 2447
}

static void audit_write_protection(struct kvm_vcpu *vcpu)
{
2448
	struct kvm_mmu_page *sp;
2449 2450 2451
	struct kvm_memory_slot *slot;
	unsigned long *rmapp;
	gfn_t gfn;
2452

2453
	list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
2454
		if (sp->role.metaphysical)
2455 2456
			continue;

2457 2458
		slot = gfn_to_memslot(vcpu->kvm, sp->gfn);
		gfn = unalias_gfn(vcpu->kvm, sp->gfn);
2459 2460
		rmapp = &slot->rmap[gfn - slot->base_gfn];
		if (*rmapp)
2461 2462
			printk(KERN_ERR "%s: (%s) shadow page has writable"
			       " mappings: gfn %lx role %x\n",
2463
			       __func__, audit_msg, sp->gfn,
2464
			       sp->role.word);
2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480
	}
}

static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
{
	int olddbg = dbg;

	dbg = 0;
	audit_msg = msg;
	audit_rmap(vcpu);
	audit_write_protection(vcpu);
	audit_mappings(vcpu);
	dbg = olddbg;
}

#endif
新手
引导
客服 返回
顶部