mmu.c 34.9 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
A
Avi Kivity 已提交
19 20 21 22

#include "vmx.h"
#include "kvm.h"

A
Avi Kivity 已提交
23 24 25 26 27 28
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/module.h>

A
Avi Kivity 已提交
29 30
#include <asm/page.h>
#include <asm/cmpxchg.h>
A
Avi Kivity 已提交
31

32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
#undef MMU_DEBUG

#undef AUDIT

#ifdef AUDIT
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
#else
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
#endif

#ifdef MMU_DEBUG

#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)

#else

#define pgprintk(x...) do { } while (0)
#define rmap_printk(x...) do { } while (0)

#endif

#if defined(MMU_DEBUG) || defined(AUDIT)
static int dbg = 1;
#endif
A
Avi Kivity 已提交
57

58 59 60
#ifndef MMU_DEBUG
#define ASSERT(x) do { } while (0)
#else
A
Avi Kivity 已提交
61 62 63 64 65
#define ASSERT(x)							\
	if (!(x)) {							\
		printk(KERN_WARNING "assertion failed %s:%d: %s\n",	\
		       __FILE__, __LINE__, #x);				\
	}
66
#endif
A
Avi Kivity 已提交
67

68 69 70 71
#define PT64_PT_BITS 9
#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
#define PT32_PT_BITS 10
#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
A
Avi Kivity 已提交
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126

#define PT_WRITABLE_SHIFT 1

#define PT_PRESENT_MASK (1ULL << 0)
#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
#define PT_USER_MASK (1ULL << 2)
#define PT_PWT_MASK (1ULL << 3)
#define PT_PCD_MASK (1ULL << 4)
#define PT_ACCESSED_MASK (1ULL << 5)
#define PT_DIRTY_MASK (1ULL << 6)
#define PT_PAGE_SIZE_MASK (1ULL << 7)
#define PT_PAT_MASK (1ULL << 7)
#define PT_GLOBAL_MASK (1ULL << 8)
#define PT64_NX_MASK (1ULL << 63)

#define PT_PAT_SHIFT 7
#define PT_DIR_PAT_SHIFT 12
#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)

#define PT32_DIR_PSE36_SIZE 4
#define PT32_DIR_PSE36_SHIFT 13
#define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)


#define PT_FIRST_AVAIL_BITS_SHIFT 9
#define PT64_SECOND_AVAIL_BITS_SHIFT 52

#define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)

#define VALID_PAGE(x) ((x) != INVALID_PAGE)

#define PT64_LEVEL_BITS 9

#define PT64_LEVEL_SHIFT(level) \
		( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )

#define PT64_LEVEL_MASK(level) \
		(((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))

#define PT64_INDEX(address, level)\
	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))


#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
		( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )

#define PT32_LEVEL_MASK(level) \
		(((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


127
#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
A
Avi Kivity 已提交
128 129 130 131 132 133 134 135 136 137 138
#define PT64_DIR_BASE_ADDR_MASK \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))

#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))


#define PFERR_PRESENT_MASK (1U << 0)
#define PFERR_WRITE_MASK (1U << 1)
#define PFERR_USER_MASK (1U << 2)
139
#define PFERR_FETCH_MASK (1U << 4)
A
Avi Kivity 已提交
140 141 142 143 144 145 146 147

#define PT64_ROOT_LEVEL 4
#define PT32_ROOT_LEVEL 2
#define PT32E_ROOT_LEVEL 3

#define PT_DIRECTORY_LEVEL 2
#define PT_PAGE_TABLE_LEVEL 1

148 149 150 151 152 153 154
#define RMAP_EXT 4

struct kvm_rmap_desc {
	u64 *shadow_ptes[RMAP_EXT];
	struct kvm_rmap_desc *more;
};

155 156
static struct kmem_cache *pte_chain_cache;
static struct kmem_cache *rmap_desc_cache;
157
static struct kmem_cache *mmu_page_header_cache;
158

A
Avi Kivity 已提交
159 160
static int is_write_protection(struct kvm_vcpu *vcpu)
{
161
	return vcpu->cr0 & X86_CR0_WP;
A
Avi Kivity 已提交
162 163 164 165 166 167 168
}

static int is_cpuid_PSE36(void)
{
	return 1;
}

169 170 171 172 173
static int is_nx(struct kvm_vcpu *vcpu)
{
	return vcpu->shadow_efer & EFER_NX;
}

A
Avi Kivity 已提交
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
static int is_present_pte(unsigned long pte)
{
	return pte & PT_PRESENT_MASK;
}

static int is_writeble_pte(unsigned long pte)
{
	return pte & PT_WRITABLE_MASK;
}

static int is_io_pte(unsigned long pte)
{
	return pte & PT_SHADOW_IO_MARK;
}

189 190 191 192 193 194
static int is_rmap_pte(u64 pte)
{
	return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK))
		== (PT_WRITABLE_MASK | PT_PRESENT_MASK);
}

195 196 197 198 199 200 201 202 203
static void set_shadow_pte(u64 *sptep, u64 spte)
{
#ifdef CONFIG_X86_64
	set_64bit((unsigned long *)sptep, spte);
#else
	set_64bit((unsigned long long *)sptep, spte);
#endif
}

204
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
205 206
				  struct kmem_cache *base_cache, int min,
				  gfp_t gfp_flags)
207 208 209 210
{
	void *obj;

	if (cache->nobjs >= min)
211
		return 0;
212
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
213
		obj = kmem_cache_zalloc(base_cache, gfp_flags);
214
		if (!obj)
215
			return -ENOMEM;
216 217
		cache->objects[cache->nobjs++] = obj;
	}
218
	return 0;
219 220 221 222 223 224 225 226
}

static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
		kfree(mc->objects[--mc->nobjs]);
}

A
Avi Kivity 已提交
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
				       int min, gfp_t gfp_flags)
{
	struct page *page;

	if (cache->nobjs >= min)
		return 0;
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
		page = alloc_page(gfp_flags);
		if (!page)
			return -ENOMEM;
		set_page_private(page, 0);
		cache->objects[cache->nobjs++] = page_address(page);
	}
	return 0;
}

static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
247
		free_page((unsigned long)mc->objects[--mc->nobjs]);
A
Avi Kivity 已提交
248 249
}

250
static int __mmu_topup_memory_caches(struct kvm_vcpu *vcpu, gfp_t gfp_flags)
251
{
252 253 254
	int r;

	r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache,
255
				   pte_chain_cache, 4, gfp_flags);
256 257 258
	if (r)
		goto out;
	r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
259
				   rmap_desc_cache, 1, gfp_flags);
260 261
	if (r)
		goto out;
A
Avi Kivity 已提交
262
	r = mmu_topup_memory_cache_page(&vcpu->mmu_page_cache, 4, gfp_flags);
263 264 265 266
	if (r)
		goto out;
	r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache,
				   mmu_page_header_cache, 4, gfp_flags);
267 268
out:
	return r;
269 270
}

271 272 273 274 275
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
{
	int r;

	r = __mmu_topup_memory_caches(vcpu, GFP_NOWAIT);
276
	kvm_mmu_free_some_pages(vcpu);
277
	if (r < 0) {
S
Shaohua Li 已提交
278
		mutex_unlock(&vcpu->kvm->lock);
279
		r = __mmu_topup_memory_caches(vcpu, GFP_KERNEL);
S
Shaohua Li 已提交
280
		mutex_lock(&vcpu->kvm->lock);
281 282 283 284
	}
	return r;
}

285 286 287 288
static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
	mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
	mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
A
Avi Kivity 已提交
289
	mmu_free_memory_cache_page(&vcpu->mmu_page_cache);
290
	mmu_free_memory_cache(&vcpu->mmu_page_header_cache);
291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
}

static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
				    size_t size)
{
	void *p;

	BUG_ON(!mc->nobjs);
	p = mc->objects[--mc->nobjs];
	memset(p, 0, size);
	return p;
}

static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
{
	return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache,
				      sizeof(struct kvm_pte_chain));
}

310
static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
311
{
312
	kfree(pc);
313 314 315 316 317 318 319 320
}

static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
{
	return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache,
				      sizeof(struct kvm_rmap_desc));
}

321
static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
322
{
323
	kfree(rd);
324 325
}

326 327 328 329 330 331 332 333 334
/*
 * Reverse mapping data structures:
 *
 * If page->private bit zero is zero, then page->private points to the
 * shadow page table entry that points to page_address(page).
 *
 * If page->private bit zero is one, (then page->private & ~1) points
 * to a struct kvm_rmap_desc containing more mappings.
 */
335
static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte)
336 337 338 339 340 341 342 343
{
	struct page *page;
	struct kvm_rmap_desc *desc;
	int i;

	if (!is_rmap_pte(*spte))
		return;
	page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
344
	if (!page_private(page)) {
345
		rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
346 347
		set_page_private(page,(unsigned long)spte);
	} else if (!(page_private(page) & 1)) {
348
		rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
349
		desc = mmu_alloc_rmap_desc(vcpu);
350
		desc->shadow_ptes[0] = (u64 *)page_private(page);
351
		desc->shadow_ptes[1] = spte;
352
		set_page_private(page,(unsigned long)desc | 1);
353 354
	} else {
		rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
355
		desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
356 357 358
		while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
			desc = desc->more;
		if (desc->shadow_ptes[RMAP_EXT-1]) {
359
			desc->more = mmu_alloc_rmap_desc(vcpu);
360 361 362 363 364 365 366 367
			desc = desc->more;
		}
		for (i = 0; desc->shadow_ptes[i]; ++i)
			;
		desc->shadow_ptes[i] = spte;
	}
}

368
static void rmap_desc_remove_entry(struct page *page,
369 370 371 372 373 374 375 376 377
				   struct kvm_rmap_desc *desc,
				   int i,
				   struct kvm_rmap_desc *prev_desc)
{
	int j;

	for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
		;
	desc->shadow_ptes[i] = desc->shadow_ptes[j];
A
Al Viro 已提交
378
	desc->shadow_ptes[j] = NULL;
379 380 381
	if (j != 0)
		return;
	if (!prev_desc && !desc->more)
382
		set_page_private(page,(unsigned long)desc->shadow_ptes[0]);
383 384 385 386
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
387
			set_page_private(page,(unsigned long)desc->more | 1);
388
	mmu_free_rmap_desc(desc);
389 390
}

391
static void rmap_remove(u64 *spte)
392 393 394 395 396 397 398 399 400
{
	struct page *page;
	struct kvm_rmap_desc *desc;
	struct kvm_rmap_desc *prev_desc;
	int i;

	if (!is_rmap_pte(*spte))
		return;
	page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
401
	if (!page_private(page)) {
402 403
		printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
		BUG();
404
	} else if (!(page_private(page) & 1)) {
405
		rmap_printk("rmap_remove:  %p %llx 1->0\n", spte, *spte);
406
		if ((u64 *)page_private(page) != spte) {
407 408 409 410
			printk(KERN_ERR "rmap_remove:  %p %llx 1->BUG\n",
			       spte, *spte);
			BUG();
		}
411
		set_page_private(page,0);
412 413
	} else {
		rmap_printk("rmap_remove:  %p %llx many->many\n", spte, *spte);
414
		desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
415 416 417 418
		prev_desc = NULL;
		while (desc) {
			for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
				if (desc->shadow_ptes[i] == spte) {
419
					rmap_desc_remove_entry(page,
420
							       desc, i,
421 422 423 424 425 426 427 428 429 430
							       prev_desc);
					return;
				}
			prev_desc = desc;
			desc = desc->more;
		}
		BUG();
	}
}

431
static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
432
{
433
	struct kvm *kvm = vcpu->kvm;
434 435 436 437
	struct page *page;
	struct kvm_rmap_desc *desc;
	u64 *spte;

A
Avi Kivity 已提交
438 439
	page = gfn_to_page(kvm, gfn);
	BUG_ON(!page);
440

441 442 443
	while (page_private(page)) {
		if (!(page_private(page) & 1))
			spte = (u64 *)page_private(page);
444
		else {
445
			desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
446 447 448
			spte = desc->shadow_ptes[0];
		}
		BUG_ON(!spte);
449 450
		BUG_ON((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT
		       != page_to_pfn(page));
451 452 453
		BUG_ON(!(*spte & PT_PRESENT_MASK));
		BUG_ON(!(*spte & PT_WRITABLE_MASK));
		rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
454
		rmap_remove(spte);
455
		set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
S
Shaohua Li 已提交
456
		kvm_flush_remote_tlbs(vcpu->kvm);
457 458 459
	}
}

460
#ifdef MMU_DEBUG
461
static int is_empty_shadow_page(u64 *spt)
A
Avi Kivity 已提交
462
{
463 464 465
	u64 *pos;
	u64 *end;

466
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
467 468 469
		if (*pos != 0) {
			printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
			       pos, *pos);
A
Avi Kivity 已提交
470
			return 0;
471
		}
A
Avi Kivity 已提交
472 473
	return 1;
}
474
#endif
A
Avi Kivity 已提交
475

476
static void kvm_mmu_free_page(struct kvm *kvm,
477
			      struct kvm_mmu_page *page_head)
478
{
479
	ASSERT(is_empty_shadow_page(page_head->spt));
480
	list_del(&page_head->link);
A
Avi Kivity 已提交
481
	__free_page(virt_to_page(page_head->spt));
482 483
	kfree(page_head);
	++kvm->n_free_mmu_pages;
484 485
}

486 487 488 489 490
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
	return gfn;
}

491 492
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
					       u64 *parent_pte)
A
Avi Kivity 已提交
493 494 495
{
	struct kvm_mmu_page *page;

496
	if (!vcpu->kvm->n_free_mmu_pages)
497
		return NULL;
A
Avi Kivity 已提交
498

499 500 501 502 503
	page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache,
				      sizeof *page);
	page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
	set_page_private(virt_to_page(page->spt), (unsigned long)page);
	list_add(&page->link, &vcpu->kvm->active_mmu_pages);
504
	ASSERT(is_empty_shadow_page(page->spt));
A
Avi Kivity 已提交
505
	page->slot_bitmap = 0;
506
	page->multimapped = 0;
A
Avi Kivity 已提交
507
	page->parent_pte = parent_pte;
A
Avi Kivity 已提交
508
	--vcpu->kvm->n_free_mmu_pages;
509
	return page;
A
Avi Kivity 已提交
510 511
}

512 513
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *page, u64 *parent_pte)
514 515 516 517 518 519 520 521 522 523 524 525 526 527 528
{
	struct kvm_pte_chain *pte_chain;
	struct hlist_node *node;
	int i;

	if (!parent_pte)
		return;
	if (!page->multimapped) {
		u64 *old = page->parent_pte;

		if (!old) {
			page->parent_pte = parent_pte;
			return;
		}
		page->multimapped = 1;
529
		pte_chain = mmu_alloc_pte_chain(vcpu);
530 531 532 533 534 535 536 537 538 539 540 541 542
		INIT_HLIST_HEAD(&page->parent_ptes);
		hlist_add_head(&pte_chain->link, &page->parent_ptes);
		pte_chain->parent_ptes[0] = old;
	}
	hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) {
		if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
			continue;
		for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
			if (!pte_chain->parent_ptes[i]) {
				pte_chain->parent_ptes[i] = parent_pte;
				return;
			}
	}
543
	pte_chain = mmu_alloc_pte_chain(vcpu);
544 545 546 547 548
	BUG_ON(!pte_chain);
	hlist_add_head(&pte_chain->link, &page->parent_ptes);
	pte_chain->parent_ptes[0] = parent_pte;
}

549
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page,
550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566
				       u64 *parent_pte)
{
	struct kvm_pte_chain *pte_chain;
	struct hlist_node *node;
	int i;

	if (!page->multimapped) {
		BUG_ON(page->parent_pte != parent_pte);
		page->parent_pte = NULL;
		return;
	}
	hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link)
		for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
			if (!pte_chain->parent_ptes[i])
				break;
			if (pte_chain->parent_ptes[i] != parent_pte)
				continue;
567 568
			while (i + 1 < NR_PTE_CHAIN_ENTRIES
				&& pte_chain->parent_ptes[i + 1]) {
569 570 571 572 573
				pte_chain->parent_ptes[i]
					= pte_chain->parent_ptes[i + 1];
				++i;
			}
			pte_chain->parent_ptes[i] = NULL;
574 575
			if (i == 0) {
				hlist_del(&pte_chain->link);
576
				mmu_free_pte_chain(pte_chain);
577 578 579 580 581
				if (hlist_empty(&page->parent_ptes)) {
					page->multimapped = 0;
					page->parent_pte = NULL;
				}
			}
582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611
			return;
		}
	BUG();
}

static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu,
						gfn_t gfn)
{
	unsigned index;
	struct hlist_head *bucket;
	struct kvm_mmu_page *page;
	struct hlist_node *node;

	pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
	index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
	bucket = &vcpu->kvm->mmu_page_hash[index];
	hlist_for_each_entry(page, node, bucket, hash_link)
		if (page->gfn == gfn && !page->role.metaphysical) {
			pgprintk("%s: found role %x\n",
				 __FUNCTION__, page->role.word);
			return page;
		}
	return NULL;
}

static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
					     int metaphysical,
612
					     unsigned hugepage_access,
613 614 615 616 617 618 619 620 621 622 623 624 625
					     u64 *parent_pte)
{
	union kvm_mmu_page_role role;
	unsigned index;
	unsigned quadrant;
	struct hlist_head *bucket;
	struct kvm_mmu_page *page;
	struct hlist_node *node;

	role.word = 0;
	role.glevels = vcpu->mmu.root_level;
	role.level = level;
	role.metaphysical = metaphysical;
626
	role.hugepage_access = hugepage_access;
627 628 629 630 631 632 633 634 635 636 637
	if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
	pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
		 gfn, role.word);
	index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
	bucket = &vcpu->kvm->mmu_page_hash[index];
	hlist_for_each_entry(page, node, bucket, hash_link)
		if (page->gfn == gfn && page->role.word == role.word) {
638
			mmu_page_add_parent_pte(vcpu, page, parent_pte);
639 640 641 642 643 644 645 646 647 648
			pgprintk("%s: found\n", __FUNCTION__);
			return page;
		}
	page = kvm_mmu_alloc_page(vcpu, parent_pte);
	if (!page)
		return page;
	pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
	page->gfn = gfn;
	page->role = role;
	hlist_add_head(&page->hash_link, bucket);
649
	if (!metaphysical)
650
		rmap_write_protect(vcpu, gfn);
651 652 653
	return page;
}

654
static void kvm_mmu_page_unlink_children(struct kvm *kvm,
655 656
					 struct kvm_mmu_page *page)
{
657 658 659 660
	unsigned i;
	u64 *pt;
	u64 ent;

661
	pt = page->spt;
662 663 664 665

	if (page->role.level == PT_PAGE_TABLE_LEVEL) {
		for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
			if (pt[i] & PT_PRESENT_MASK)
666
				rmap_remove(&pt[i]);
667 668
			pt[i] = 0;
		}
669
		kvm_flush_remote_tlbs(kvm);
670 671 672 673 674 675 676 677 678 679
		return;
	}

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
		ent = pt[i];

		pt[i] = 0;
		if (!(ent & PT_PRESENT_MASK))
			continue;
		ent &= PT64_BASE_ADDR_MASK;
680
		mmu_page_remove_parent_pte(page_header(ent), &pt[i]);
681
	}
682
	kvm_flush_remote_tlbs(kvm);
683 684
}

685
static void kvm_mmu_put_page(struct kvm_mmu_page *page,
686 687
			     u64 *parent_pte)
{
688
	mmu_page_remove_parent_pte(page, parent_pte);
689 690
}

691
static void kvm_mmu_zap_page(struct kvm *kvm,
692 693 694 695 696 697 698 699 700 701 702 703 704 705
			     struct kvm_mmu_page *page)
{
	u64 *parent_pte;

	while (page->multimapped || page->parent_pte) {
		if (!page->multimapped)
			parent_pte = page->parent_pte;
		else {
			struct kvm_pte_chain *chain;

			chain = container_of(page->parent_ptes.first,
					     struct kvm_pte_chain, link);
			parent_pte = chain->parent_ptes[0];
		}
706
		BUG_ON(!parent_pte);
707
		kvm_mmu_put_page(page, parent_pte);
708
		set_shadow_pte(parent_pte, 0);
709
	}
710
	kvm_mmu_page_unlink_children(kvm, page);
711 712
	if (!page->root_count) {
		hlist_del(&page->hash_link);
713
		kvm_mmu_free_page(kvm, page);
A
Avi Kivity 已提交
714
	} else
715
		list_move(&page->link, &kvm->active_mmu_pages);
716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731
}

static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	unsigned index;
	struct hlist_head *bucket;
	struct kvm_mmu_page *page;
	struct hlist_node *node, *n;
	int r;

	pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
	r = 0;
	index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
	bucket = &vcpu->kvm->mmu_page_hash[index];
	hlist_for_each_entry_safe(page, node, n, bucket, hash_link)
		if (page->gfn == gfn && !page->role.metaphysical) {
732 733
			pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn,
				 page->role.word);
734
			kvm_mmu_zap_page(vcpu->kvm, page);
735 736 737
			r = 1;
		}
	return r;
738 739
}

740 741 742 743 744 745 746
static void mmu_unshadow(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	struct kvm_mmu_page *page;

	while ((page = kvm_mmu_lookup_page(vcpu, gfn)) != NULL) {
		pgprintk("%s: zap %lx %x\n",
			 __FUNCTION__, gfn, page->role.word);
747
		kvm_mmu_zap_page(vcpu->kvm, page);
748 749 750
	}
}

A
Avi Kivity 已提交
751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770
static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
{
	int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
	struct kvm_mmu_page *page_head = page_header(__pa(pte));

	__set_bit(slot, &page_head->slot_bitmap);
}

hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
{
	hpa_t hpa = gpa_to_hpa(vcpu, gpa);

	return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
}

hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
{
	struct page *page;

	ASSERT((gpa & HPA_ERR_MASK) == 0);
A
Avi Kivity 已提交
771 772
	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
	if (!page)
A
Avi Kivity 已提交
773 774 775 776 777 778 779 780 781 782 783 784 785 786
		return gpa | HPA_ERR_MASK;
	return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
		| (gpa & (PAGE_SIZE-1));
}

hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
{
	gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);

	if (gpa == UNMAPPED_GVA)
		return UNMAPPED_GVA;
	return gpa_to_hpa(vcpu, gpa);
}

787 788 789 790 791 792 793 794 795
struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
{
	gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);

	if (gpa == UNMAPPED_GVA)
		return NULL;
	return pfn_to_page(gpa_to_hpa(vcpu, gpa) >> PAGE_SHIFT);
}

A
Avi Kivity 已提交
796 797 798 799 800 801 802 803 804 805 806 807
static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
{
}

static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
{
	int level = PT32E_ROOT_LEVEL;
	hpa_t table_addr = vcpu->mmu.root_hpa;

	for (; ; level--) {
		u32 index = PT64_INDEX(v, level);
		u64 *table;
808
		u64 pte;
A
Avi Kivity 已提交
809 810 811 812 813

		ASSERT(VALID_PAGE(table_addr));
		table = __va(table_addr);

		if (level == 1) {
814 815 816
			pte = table[index];
			if (is_present_pte(pte) && is_writeble_pte(pte))
				return 0;
A
Avi Kivity 已提交
817 818 819 820
			mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
			page_header_update_slot(vcpu->kvm, table, v);
			table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
								PT_USER_MASK;
821
			rmap_add(vcpu, &table[index]);
A
Avi Kivity 已提交
822 823 824 825
			return 0;
		}

		if (table[index] == 0) {
826
			struct kvm_mmu_page *new_table;
827
			gfn_t pseudo_gfn;
A
Avi Kivity 已提交
828

829 830 831 832
			pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
				>> PAGE_SHIFT;
			new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
						     v, level - 1,
833
						     1, 0, &table[index]);
834
			if (!new_table) {
A
Avi Kivity 已提交
835 836 837 838
				pgprintk("nonpaging_map: ENOMEM\n");
				return -ENOMEM;
			}

839
			table[index] = __pa(new_table->spt) | PT_PRESENT_MASK
840
				| PT_WRITABLE_MASK | PT_USER_MASK;
A
Avi Kivity 已提交
841 842 843 844 845
		}
		table_addr = table[index] & PT64_BASE_ADDR_MASK;
	}
}

846 847 848
static void mmu_free_roots(struct kvm_vcpu *vcpu)
{
	int i;
849
	struct kvm_mmu_page *page;
850

A
Avi Kivity 已提交
851 852
	if (!VALID_PAGE(vcpu->mmu.root_hpa))
		return;
853 854 855 856
#ifdef CONFIG_X86_64
	if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		hpa_t root = vcpu->mmu.root_hpa;

857 858
		page = page_header(root);
		--page->root_count;
859 860 861 862 863 864 865
		vcpu->mmu.root_hpa = INVALID_PAGE;
		return;
	}
#endif
	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->mmu.pae_root[i];

A
Avi Kivity 已提交
866 867 868 869 870
		if (root) {
			root &= PT64_BASE_ADDR_MASK;
			page = page_header(root);
			--page->root_count;
		}
871 872 873 874 875 876 877 878
		vcpu->mmu.pae_root[i] = INVALID_PAGE;
	}
	vcpu->mmu.root_hpa = INVALID_PAGE;
}

static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
{
	int i;
879
	gfn_t root_gfn;
880 881
	struct kvm_mmu_page *page;

882
	root_gfn = vcpu->cr3 >> PAGE_SHIFT;
883 884 885 886 887 888

#ifdef CONFIG_X86_64
	if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		hpa_t root = vcpu->mmu.root_hpa;

		ASSERT(!VALID_PAGE(root));
889
		page = kvm_mmu_get_page(vcpu, root_gfn, 0,
890
					PT64_ROOT_LEVEL, 0, 0, NULL);
891
		root = __pa(page->spt);
892
		++page->root_count;
893 894 895 896 897 898 899 900
		vcpu->mmu.root_hpa = root;
		return;
	}
#endif
	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->mmu.pae_root[i];

		ASSERT(!VALID_PAGE(root));
A
Avi Kivity 已提交
901 902 903 904 905
		if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) {
			if (!is_present_pte(vcpu->pdptrs[i])) {
				vcpu->mmu.pae_root[i] = 0;
				continue;
			}
906
			root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
A
Avi Kivity 已提交
907
		} else if (vcpu->mmu.root_level == 0)
908
			root_gfn = 0;
909
		page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
910
					PT32_ROOT_LEVEL, !is_paging(vcpu),
911
					0, NULL);
912
		root = __pa(page->spt);
913
		++page->root_count;
914 915 916 917 918
		vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
	}
	vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
}

A
Avi Kivity 已提交
919 920 921 922 923 924 925 926 927
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
{
	return vaddr;
}

static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
			       u32 error_code)
{
	gpa_t addr = gva;
A
Avi Kivity 已提交
928
	hpa_t paddr;
929
	int r;
A
Avi Kivity 已提交
930

931 932 933
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;
934

A
Avi Kivity 已提交
935 936 937 938
	ASSERT(vcpu);
	ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));


A
Avi Kivity 已提交
939
	paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
A
Avi Kivity 已提交
940

A
Avi Kivity 已提交
941 942
	if (is_error_hpa(paddr))
		return 1;
A
Avi Kivity 已提交
943

A
Avi Kivity 已提交
944
	return nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
A
Avi Kivity 已提交
945 946 947 948
}

static void nonpaging_free(struct kvm_vcpu *vcpu)
{
949
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
950 951 952 953 954 955 956 957 958 959
}

static int nonpaging_init_context(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu *context = &vcpu->mmu;

	context->new_cr3 = nonpaging_new_cr3;
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
	context->free = nonpaging_free;
960
	context->root_level = 0;
A
Avi Kivity 已提交
961
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
962
	context->root_hpa = INVALID_PAGE;
A
Avi Kivity 已提交
963 964 965 966 967
	return 0;
}

static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
968
	++vcpu->stat.tlb_flush;
969
	kvm_x86_ops->tlb_flush(vcpu);
A
Avi Kivity 已提交
970 971 972 973
}

static void paging_new_cr3(struct kvm_vcpu *vcpu)
{
974
	pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
975
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
976 977 978 979 980 981
}

static void inject_page_fault(struct kvm_vcpu *vcpu,
			      u64 addr,
			      u32 err_code)
{
982
	kvm_x86_ops->inject_page_fault(vcpu, addr, err_code);
A
Avi Kivity 已提交
983 984 985 986 987 988 989 990 991 992 993 994 995 996 997
}

static void paging_free(struct kvm_vcpu *vcpu)
{
	nonpaging_free(vcpu);
}

#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

998
static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
A
Avi Kivity 已提交
999 1000 1001 1002 1003 1004 1005 1006
{
	struct kvm_mmu *context = &vcpu->mmu;

	ASSERT(is_pae(vcpu));
	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
	context->free = paging_free;
1007 1008
	context->root_level = level;
	context->shadow_root_level = level;
A
Avi Kivity 已提交
1009
	context->root_hpa = INVALID_PAGE;
A
Avi Kivity 已提交
1010 1011 1012
	return 0;
}

1013 1014 1015 1016 1017
static int paging64_init_context(struct kvm_vcpu *vcpu)
{
	return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
}

A
Avi Kivity 已提交
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027
static int paging32_init_context(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu *context = &vcpu->mmu;

	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
	context->free = paging_free;
	context->root_level = PT32_ROOT_LEVEL;
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
1028
	context->root_hpa = INVALID_PAGE;
A
Avi Kivity 已提交
1029 1030 1031 1032 1033
	return 0;
}

static int paging32E_init_context(struct kvm_vcpu *vcpu)
{
1034
	return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
A
Avi Kivity 已提交
1035 1036 1037 1038 1039 1040 1041 1042 1043
}

static int init_kvm_mmu(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
	ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));

	if (!is_paging(vcpu))
		return nonpaging_init_context(vcpu);
A
Avi Kivity 已提交
1044
	else if (is_long_mode(vcpu))
A
Avi Kivity 已提交
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
		return paging64_init_context(vcpu);
	else if (is_pae(vcpu))
		return paging32E_init_context(vcpu);
	else
		return paging32_init_context(vcpu);
}

static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
	if (VALID_PAGE(vcpu->mmu.root_hpa)) {
		vcpu->mmu.free(vcpu);
		vcpu->mmu.root_hpa = INVALID_PAGE;
	}
}

int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1062 1063 1064 1065 1066 1067
{
	destroy_kvm_mmu(vcpu);
	return init_kvm_mmu(vcpu);
}

int kvm_mmu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1068
{
1069 1070
	int r;

S
Shaohua Li 已提交
1071
	mutex_lock(&vcpu->kvm->lock);
1072
	r = mmu_topup_memory_caches(vcpu);
A
Avi Kivity 已提交
1073 1074 1075
	if (r)
		goto out;
	mmu_alloc_roots(vcpu);
1076
	kvm_x86_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
A
Avi Kivity 已提交
1077
	kvm_mmu_flush_tlb(vcpu);
1078
out:
S
Shaohua Li 已提交
1079
	mutex_unlock(&vcpu->kvm->lock);
1080
	return r;
A
Avi Kivity 已提交
1081
}
A
Avi Kivity 已提交
1082 1083 1084 1085 1086 1087
EXPORT_SYMBOL_GPL(kvm_mmu_load);

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
	mmu_free_roots(vcpu);
}
A
Avi Kivity 已提交
1088

1089
static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
1090 1091 1092 1093 1094 1095 1096 1097 1098
				  struct kvm_mmu_page *page,
				  u64 *spte)
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
	if (is_present_pte(pte)) {
		if (page->role.level == PT_PAGE_TABLE_LEVEL)
1099
			rmap_remove(spte);
1100 1101
		else {
			child = page_header(pte & PT64_BASE_ADDR_MASK);
1102
			mmu_page_remove_parent_pte(child, spte);
1103 1104 1105
		}
	}
	*spte = 0;
1106
	kvm_flush_remote_tlbs(vcpu->kvm);
1107 1108
}

1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
				  struct kvm_mmu_page *page,
				  u64 *spte,
				  const void *new, int bytes)
{
	if (page->role.level != PT_PAGE_TABLE_LEVEL)
		return;

	if (page->role.glevels == PT32_ROOT_LEVEL)
		paging32_update_pte(vcpu, page, spte, new, bytes);
	else
		paging64_update_pte(vcpu, page, spte, new, bytes);
}

1123
void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1124
		       const u8 *new, int bytes)
1125
{
1126 1127
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_mmu_page *page;
1128
	struct hlist_node *node, *n;
1129 1130 1131 1132
	struct hlist_head *bucket;
	unsigned index;
	u64 *spte;
	unsigned offset = offset_in_page(gpa);
1133
	unsigned pte_size;
1134
	unsigned page_offset;
1135
	unsigned misaligned;
1136
	unsigned quadrant;
1137
	int level;
1138
	int flooded = 0;
1139
	int npte;
1140

1141
	pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
1142 1143 1144 1145 1146 1147 1148 1149
	if (gfn == vcpu->last_pt_write_gfn) {
		++vcpu->last_pt_write_count;
		if (vcpu->last_pt_write_count >= 3)
			flooded = 1;
	} else {
		vcpu->last_pt_write_gfn = gfn;
		vcpu->last_pt_write_count = 1;
	}
1150 1151
	index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
	bucket = &vcpu->kvm->mmu_page_hash[index];
1152
	hlist_for_each_entry_safe(page, node, n, bucket, hash_link) {
1153 1154
		if (page->gfn != gfn || page->role.metaphysical)
			continue;
1155 1156
		pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
		misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
1157
		misaligned |= bytes < 4;
1158
		if (misaligned || flooded) {
1159 1160 1161 1162
			/*
			 * Misaligned accesses are too much trouble to fix
			 * up; also, they usually indicate a page is not used
			 * as a page table.
1163 1164 1165 1166 1167
			 *
			 * If we're seeing too many writes to a page,
			 * it may no longer be a page table, or we may be
			 * forking, in which case it is better to unmap the
			 * page.
1168 1169 1170
			 */
			pgprintk("misaligned: gpa %llx bytes %d role %x\n",
				 gpa, bytes, page->role.word);
1171
			kvm_mmu_zap_page(vcpu->kvm, page);
1172 1173
			continue;
		}
1174 1175
		page_offset = offset;
		level = page->role.level;
1176
		npte = 1;
1177
		if (page->role.glevels == PT32_ROOT_LEVEL) {
1178 1179 1180 1181 1182 1183 1184
			page_offset <<= 1;	/* 32->64 */
			/*
			 * A 32-bit pde maps 4MB while the shadow pdes map
			 * only 2MB.  So we need to double the offset again
			 * and zap two pdes instead of one.
			 */
			if (level == PT32_ROOT_LEVEL) {
1185
				page_offset &= ~7; /* kill rounding error */
1186 1187 1188
				page_offset <<= 1;
				npte = 2;
			}
1189
			quadrant = page_offset >> PAGE_SHIFT;
1190
			page_offset &= ~PAGE_MASK;
1191 1192
			if (quadrant != page->role.quadrant)
				continue;
1193
		}
1194
		spte = &page->spt[page_offset / sizeof(*spte)];
1195
		while (npte--) {
1196
			mmu_pte_write_zap_pte(vcpu, page, spte);
1197
			mmu_pte_write_new_pte(vcpu, page, spte, new, bytes);
1198
			++spte;
1199 1200
		}
	}
1201 1202
}

1203 1204 1205 1206 1207 1208 1209
int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
	gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);

	return kvm_mmu_unprotect_page(vcpu, gpa >> PAGE_SHIFT);
}

1210
void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1211 1212 1213 1214 1215 1216
{
	while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) {
		struct kvm_mmu_page *page;

		page = container_of(vcpu->kvm->active_mmu_pages.prev,
				    struct kvm_mmu_page, link);
1217
		kvm_mmu_zap_page(vcpu->kvm, page);
A
Avi Kivity 已提交
1218 1219 1220
	}
}

A
Avi Kivity 已提交
1221 1222
static void free_mmu_pages(struct kvm_vcpu *vcpu)
{
1223
	struct kvm_mmu_page *page;
A
Avi Kivity 已提交
1224

1225 1226 1227
	while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
		page = container_of(vcpu->kvm->active_mmu_pages.next,
				    struct kvm_mmu_page, link);
1228
		kvm_mmu_zap_page(vcpu->kvm, page);
1229
	}
1230
	free_page((unsigned long)vcpu->mmu.pae_root);
A
Avi Kivity 已提交
1231 1232 1233 1234
}

static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
{
1235
	struct page *page;
A
Avi Kivity 已提交
1236 1237 1238 1239
	int i;

	ASSERT(vcpu);

1240
	vcpu->kvm->n_free_mmu_pages = KVM_NUM_MMU_PAGES;
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253

	/*
	 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
	 * Therefore we need to allocate shadow page tables in the first
	 * 4GB of memory, which happens to fit the DMA32 zone.
	 */
	page = alloc_page(GFP_KERNEL | __GFP_DMA32);
	if (!page)
		goto error_1;
	vcpu->mmu.pae_root = page_address(page);
	for (i = 0; i < 4; ++i)
		vcpu->mmu.pae_root[i] = INVALID_PAGE;

A
Avi Kivity 已提交
1254 1255 1256 1257 1258 1259 1260
	return 0;

error_1:
	free_mmu_pages(vcpu);
	return -ENOMEM;
}

1261
int kvm_mmu_create(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1262 1263 1264 1265
{
	ASSERT(vcpu);
	ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));

1266 1267
	return alloc_mmu_pages(vcpu);
}
A
Avi Kivity 已提交
1268

1269 1270 1271 1272
int kvm_mmu_setup(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
	ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1273

1274
	return init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
1275 1276 1277 1278 1279 1280 1281 1282
}

void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);

	destroy_kvm_mmu(vcpu);
	free_mmu_pages(vcpu);
1283
	mmu_free_memory_caches(vcpu);
A
Avi Kivity 已提交
1284 1285
}

1286
void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
A
Avi Kivity 已提交
1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
{
	struct kvm_mmu_page *page;

	list_for_each_entry(page, &kvm->active_mmu_pages, link) {
		int i;
		u64 *pt;

		if (!test_bit(slot, &page->slot_bitmap))
			continue;

1297
		pt = page->spt;
A
Avi Kivity 已提交
1298 1299
		for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
			/* avoid RMW */
1300
			if (pt[i] & PT_WRITABLE_MASK) {
1301
				rmap_remove(&pt[i]);
A
Avi Kivity 已提交
1302
				pt[i] &= ~PT_WRITABLE_MASK;
1303
			}
A
Avi Kivity 已提交
1304 1305
	}
}
1306

1307
void kvm_mmu_zap_all(struct kvm *kvm)
D
Dor Laor 已提交
1308
{
1309
	struct kvm_mmu_page *page, *node;
D
Dor Laor 已提交
1310

1311 1312
	list_for_each_entry_safe(page, node, &kvm->active_mmu_pages, link)
		kvm_mmu_zap_page(kvm, page);
D
Dor Laor 已提交
1313

1314
	kvm_flush_remote_tlbs(kvm);
D
Dor Laor 已提交
1315 1316
}

1317 1318 1319 1320 1321 1322
void kvm_mmu_module_exit(void)
{
	if (pte_chain_cache)
		kmem_cache_destroy(pte_chain_cache);
	if (rmap_desc_cache)
		kmem_cache_destroy(rmap_desc_cache);
1323 1324
	if (mmu_page_header_cache)
		kmem_cache_destroy(mmu_page_header_cache);
1325 1326 1327 1328 1329 1330
}

int kvm_mmu_module_init(void)
{
	pte_chain_cache = kmem_cache_create("kvm_pte_chain",
					    sizeof(struct kvm_pte_chain),
1331
					    0, 0, NULL);
1332 1333 1334 1335
	if (!pte_chain_cache)
		goto nomem;
	rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
					    sizeof(struct kvm_rmap_desc),
1336
					    0, 0, NULL);
1337 1338 1339
	if (!rmap_desc_cache)
		goto nomem;

1340 1341
	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
1342
						  0, 0, NULL);
1343 1344 1345
	if (!mmu_page_header_cache)
		goto nomem;

1346 1347 1348 1349 1350 1351 1352
	return 0;

nomem:
	kvm_mmu_module_exit();
	return -ENOMEM;
}

1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374
#ifdef AUDIT

static const char *audit_msg;

static gva_t canonicalize(gva_t gva)
{
#ifdef CONFIG_X86_64
	gva = (long long)(gva << 16) >> 16;
#endif
	return gva;
}

static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
				gva_t va, int level)
{
	u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
	int i;
	gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
		u64 ent = pt[i];

A
Adrian Bunk 已提交
1375
		if (!(ent & PT_PRESENT_MASK))
1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396
			continue;

		va = canonicalize(va);
		if (level > 1)
			audit_mappings_page(vcpu, ent, va, level - 1);
		else {
			gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va);
			hpa_t hpa = gpa_to_hpa(vcpu, gpa);

			if ((ent & PT_PRESENT_MASK)
			    && (ent & PT64_BASE_ADDR_MASK) != hpa)
				printk(KERN_ERR "audit error: (%s) levels %d"
				       " gva %lx gpa %llx hpa %llx ent %llx\n",
				       audit_msg, vcpu->mmu.root_level,
				       va, gpa, hpa, ent);
		}
	}
}

static void audit_mappings(struct kvm_vcpu *vcpu)
{
1397
	unsigned i;
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448

	if (vcpu->mmu.root_level == 4)
		audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4);
	else
		for (i = 0; i < 4; ++i)
			if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK)
				audit_mappings_page(vcpu,
						    vcpu->mmu.pae_root[i],
						    i << 30,
						    2);
}

static int count_rmaps(struct kvm_vcpu *vcpu)
{
	int nmaps = 0;
	int i, j, k;

	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
		struct kvm_rmap_desc *d;

		for (j = 0; j < m->npages; ++j) {
			struct page *page = m->phys_mem[j];

			if (!page->private)
				continue;
			if (!(page->private & 1)) {
				++nmaps;
				continue;
			}
			d = (struct kvm_rmap_desc *)(page->private & ~1ul);
			while (d) {
				for (k = 0; k < RMAP_EXT; ++k)
					if (d->shadow_ptes[k])
						++nmaps;
					else
						break;
				d = d->more;
			}
		}
	}
	return nmaps;
}

static int count_writable_mappings(struct kvm_vcpu *vcpu)
{
	int nmaps = 0;
	struct kvm_mmu_page *page;
	int i;

	list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
1449
		u64 *pt = page->spt;
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511

		if (page->role.level != PT_PAGE_TABLE_LEVEL)
			continue;

		for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
			u64 ent = pt[i];

			if (!(ent & PT_PRESENT_MASK))
				continue;
			if (!(ent & PT_WRITABLE_MASK))
				continue;
			++nmaps;
		}
	}
	return nmaps;
}

static void audit_rmap(struct kvm_vcpu *vcpu)
{
	int n_rmap = count_rmaps(vcpu);
	int n_actual = count_writable_mappings(vcpu);

	if (n_rmap != n_actual)
		printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
		       __FUNCTION__, audit_msg, n_rmap, n_actual);
}

static void audit_write_protection(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_page *page;

	list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
		hfn_t hfn;
		struct page *pg;

		if (page->role.metaphysical)
			continue;

		hfn = gpa_to_hpa(vcpu, (gpa_t)page->gfn << PAGE_SHIFT)
			>> PAGE_SHIFT;
		pg = pfn_to_page(hfn);
		if (pg->private)
			printk(KERN_ERR "%s: (%s) shadow page has writable"
			       " mappings: gfn %lx role %x\n",
			       __FUNCTION__, audit_msg, page->gfn,
			       page->role.word);
	}
}

static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
{
	int olddbg = dbg;

	dbg = 0;
	audit_msg = msg;
	audit_rmap(vcpu);
	audit_write_protection(vcpu);
	audit_mappings(vcpu);
	dbg = olddbg;
}

#endif