mmu.c 34.7 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
A
Avi Kivity 已提交
19 20 21 22

#include "vmx.h"
#include "kvm.h"

A
Avi Kivity 已提交
23 24 25 26 27 28
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/module.h>

A
Avi Kivity 已提交
29 30
#include <asm/page.h>
#include <asm/cmpxchg.h>
A
Avi Kivity 已提交
31

32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
#undef MMU_DEBUG

#undef AUDIT

#ifdef AUDIT
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
#else
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
#endif

#ifdef MMU_DEBUG

#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)

#else

#define pgprintk(x...) do { } while (0)
#define rmap_printk(x...) do { } while (0)

#endif

#if defined(MMU_DEBUG) || defined(AUDIT)
static int dbg = 1;
#endif
A
Avi Kivity 已提交
57

58 59 60
#ifndef MMU_DEBUG
#define ASSERT(x) do { } while (0)
#else
A
Avi Kivity 已提交
61 62 63 64 65
#define ASSERT(x)							\
	if (!(x)) {							\
		printk(KERN_WARNING "assertion failed %s:%d: %s\n",	\
		       __FILE__, __LINE__, #x);				\
	}
66
#endif
A
Avi Kivity 已提交
67

68 69 70 71
#define PT64_PT_BITS 9
#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
#define PT32_PT_BITS 10
#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
A
Avi Kivity 已提交
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126

#define PT_WRITABLE_SHIFT 1

#define PT_PRESENT_MASK (1ULL << 0)
#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
#define PT_USER_MASK (1ULL << 2)
#define PT_PWT_MASK (1ULL << 3)
#define PT_PCD_MASK (1ULL << 4)
#define PT_ACCESSED_MASK (1ULL << 5)
#define PT_DIRTY_MASK (1ULL << 6)
#define PT_PAGE_SIZE_MASK (1ULL << 7)
#define PT_PAT_MASK (1ULL << 7)
#define PT_GLOBAL_MASK (1ULL << 8)
#define PT64_NX_MASK (1ULL << 63)

#define PT_PAT_SHIFT 7
#define PT_DIR_PAT_SHIFT 12
#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)

#define PT32_DIR_PSE36_SIZE 4
#define PT32_DIR_PSE36_SHIFT 13
#define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)


#define PT_FIRST_AVAIL_BITS_SHIFT 9
#define PT64_SECOND_AVAIL_BITS_SHIFT 52

#define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)

#define VALID_PAGE(x) ((x) != INVALID_PAGE)

#define PT64_LEVEL_BITS 9

#define PT64_LEVEL_SHIFT(level) \
		( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )

#define PT64_LEVEL_MASK(level) \
		(((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))

#define PT64_INDEX(address, level)\
	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))


#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
		( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )

#define PT32_LEVEL_MASK(level) \
		(((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


127
#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
A
Avi Kivity 已提交
128 129 130 131 132 133 134 135 136 137 138
#define PT64_DIR_BASE_ADDR_MASK \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))

#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))


#define PFERR_PRESENT_MASK (1U << 0)
#define PFERR_WRITE_MASK (1U << 1)
#define PFERR_USER_MASK (1U << 2)
139
#define PFERR_FETCH_MASK (1U << 4)
A
Avi Kivity 已提交
140 141 142 143 144 145 146 147

#define PT64_ROOT_LEVEL 4
#define PT32_ROOT_LEVEL 2
#define PT32E_ROOT_LEVEL 3

#define PT_DIRECTORY_LEVEL 2
#define PT_PAGE_TABLE_LEVEL 1

148 149 150 151 152 153 154
#define RMAP_EXT 4

struct kvm_rmap_desc {
	u64 *shadow_ptes[RMAP_EXT];
	struct kvm_rmap_desc *more;
};

155 156
static struct kmem_cache *pte_chain_cache;
static struct kmem_cache *rmap_desc_cache;
157 158
static struct kmem_cache *mmu_page_cache;
static struct kmem_cache *mmu_page_header_cache;
159

A
Avi Kivity 已提交
160 161 162 163 164 165 166 167 168 169
static int is_write_protection(struct kvm_vcpu *vcpu)
{
	return vcpu->cr0 & CR0_WP_MASK;
}

static int is_cpuid_PSE36(void)
{
	return 1;
}

170 171 172 173 174
static int is_nx(struct kvm_vcpu *vcpu)
{
	return vcpu->shadow_efer & EFER_NX;
}

A
Avi Kivity 已提交
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
static int is_present_pte(unsigned long pte)
{
	return pte & PT_PRESENT_MASK;
}

static int is_writeble_pte(unsigned long pte)
{
	return pte & PT_WRITABLE_MASK;
}

static int is_io_pte(unsigned long pte)
{
	return pte & PT_SHADOW_IO_MARK;
}

190 191 192 193 194 195
static int is_rmap_pte(u64 pte)
{
	return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK))
		== (PT_WRITABLE_MASK | PT_PRESENT_MASK);
}

196 197 198 199 200 201 202 203 204
static void set_shadow_pte(u64 *sptep, u64 spte)
{
#ifdef CONFIG_X86_64
	set_64bit((unsigned long *)sptep, spte);
#else
	set_64bit((unsigned long long *)sptep, spte);
#endif
}

205
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
206 207
				  struct kmem_cache *base_cache, int min,
				  gfp_t gfp_flags)
208 209 210 211
{
	void *obj;

	if (cache->nobjs >= min)
212
		return 0;
213
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
214
		obj = kmem_cache_zalloc(base_cache, gfp_flags);
215
		if (!obj)
216
			return -ENOMEM;
217 218
		cache->objects[cache->nobjs++] = obj;
	}
219
	return 0;
220 221 222 223 224 225 226 227
}

static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
		kfree(mc->objects[--mc->nobjs]);
}

228
static int __mmu_topup_memory_caches(struct kvm_vcpu *vcpu, gfp_t gfp_flags)
229
{
230 231 232
	int r;

	r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache,
233
				   pte_chain_cache, 4, gfp_flags);
234 235 236
	if (r)
		goto out;
	r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
237
				   rmap_desc_cache, 1, gfp_flags);
238 239 240 241 242 243 244 245
	if (r)
		goto out;
	r = mmu_topup_memory_cache(&vcpu->mmu_page_cache,
				   mmu_page_cache, 4, gfp_flags);
	if (r)
		goto out;
	r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache,
				   mmu_page_header_cache, 4, gfp_flags);
246 247
out:
	return r;
248 249
}

250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
{
	int r;

	r = __mmu_topup_memory_caches(vcpu, GFP_NOWAIT);
	if (r < 0) {
		spin_unlock(&vcpu->kvm->lock);
		kvm_arch_ops->vcpu_put(vcpu);
		r = __mmu_topup_memory_caches(vcpu, GFP_KERNEL);
		kvm_arch_ops->vcpu_load(vcpu);
		spin_lock(&vcpu->kvm->lock);
	}
	return r;
}

265 266 267 268
static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
	mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
	mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
269 270
	mmu_free_memory_cache(&vcpu->mmu_page_cache);
	mmu_free_memory_cache(&vcpu->mmu_page_header_cache);
271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289
}

static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
				    size_t size)
{
	void *p;

	BUG_ON(!mc->nobjs);
	p = mc->objects[--mc->nobjs];
	memset(p, 0, size);
	return p;
}

static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
{
	return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache,
				      sizeof(struct kvm_pte_chain));
}

290
static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
291
{
292
	kfree(pc);
293 294 295 296 297 298 299 300
}

static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
{
	return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache,
				      sizeof(struct kvm_rmap_desc));
}

301
static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
302
{
303
	kfree(rd);
304 305
}

306 307 308 309 310 311 312 313 314
/*
 * Reverse mapping data structures:
 *
 * If page->private bit zero is zero, then page->private points to the
 * shadow page table entry that points to page_address(page).
 *
 * If page->private bit zero is one, (then page->private & ~1) points
 * to a struct kvm_rmap_desc containing more mappings.
 */
315
static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte)
316 317 318 319 320 321 322 323
{
	struct page *page;
	struct kvm_rmap_desc *desc;
	int i;

	if (!is_rmap_pte(*spte))
		return;
	page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
324
	if (!page_private(page)) {
325
		rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
326 327
		set_page_private(page,(unsigned long)spte);
	} else if (!(page_private(page) & 1)) {
328
		rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
329
		desc = mmu_alloc_rmap_desc(vcpu);
330
		desc->shadow_ptes[0] = (u64 *)page_private(page);
331
		desc->shadow_ptes[1] = spte;
332
		set_page_private(page,(unsigned long)desc | 1);
333 334
	} else {
		rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
335
		desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
336 337 338
		while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
			desc = desc->more;
		if (desc->shadow_ptes[RMAP_EXT-1]) {
339
			desc->more = mmu_alloc_rmap_desc(vcpu);
340 341 342 343 344 345 346 347
			desc = desc->more;
		}
		for (i = 0; desc->shadow_ptes[i]; ++i)
			;
		desc->shadow_ptes[i] = spte;
	}
}

348
static void rmap_desc_remove_entry(struct page *page,
349 350 351 352 353 354 355 356 357
				   struct kvm_rmap_desc *desc,
				   int i,
				   struct kvm_rmap_desc *prev_desc)
{
	int j;

	for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
		;
	desc->shadow_ptes[i] = desc->shadow_ptes[j];
A
Al Viro 已提交
358
	desc->shadow_ptes[j] = NULL;
359 360 361
	if (j != 0)
		return;
	if (!prev_desc && !desc->more)
362
		set_page_private(page,(unsigned long)desc->shadow_ptes[0]);
363 364 365 366
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
367
			set_page_private(page,(unsigned long)desc->more | 1);
368
	mmu_free_rmap_desc(desc);
369 370
}

371
static void rmap_remove(u64 *spte)
372 373 374 375 376 377 378 379 380
{
	struct page *page;
	struct kvm_rmap_desc *desc;
	struct kvm_rmap_desc *prev_desc;
	int i;

	if (!is_rmap_pte(*spte))
		return;
	page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
381
	if (!page_private(page)) {
382 383
		printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
		BUG();
384
	} else if (!(page_private(page) & 1)) {
385
		rmap_printk("rmap_remove:  %p %llx 1->0\n", spte, *spte);
386
		if ((u64 *)page_private(page) != spte) {
387 388 389 390
			printk(KERN_ERR "rmap_remove:  %p %llx 1->BUG\n",
			       spte, *spte);
			BUG();
		}
391
		set_page_private(page,0);
392 393
	} else {
		rmap_printk("rmap_remove:  %p %llx many->many\n", spte, *spte);
394
		desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
395 396 397 398
		prev_desc = NULL;
		while (desc) {
			for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
				if (desc->shadow_ptes[i] == spte) {
399
					rmap_desc_remove_entry(page,
400
							       desc, i,
401 402 403 404 405 406 407 408 409 410
							       prev_desc);
					return;
				}
			prev_desc = desc;
			desc = desc->more;
		}
		BUG();
	}
}

411
static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
412
{
413
	struct kvm *kvm = vcpu->kvm;
414 415 416 417
	struct page *page;
	struct kvm_rmap_desc *desc;
	u64 *spte;

A
Avi Kivity 已提交
418 419
	page = gfn_to_page(kvm, gfn);
	BUG_ON(!page);
420

421 422 423
	while (page_private(page)) {
		if (!(page_private(page) & 1))
			spte = (u64 *)page_private(page);
424
		else {
425
			desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
426 427 428
			spte = desc->shadow_ptes[0];
		}
		BUG_ON(!spte);
429 430
		BUG_ON((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT
		       != page_to_pfn(page));
431 432 433
		BUG_ON(!(*spte & PT_PRESENT_MASK));
		BUG_ON(!(*spte & PT_WRITABLE_MASK));
		rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
434
		rmap_remove(spte);
435
		set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
S
Shaohua Li 已提交
436
		kvm_flush_remote_tlbs(vcpu->kvm);
437 438 439
	}
}

440
#ifdef MMU_DEBUG
441
static int is_empty_shadow_page(u64 *spt)
A
Avi Kivity 已提交
442
{
443 444 445
	u64 *pos;
	u64 *end;

446
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
447 448 449
		if (*pos != 0) {
			printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
			       pos, *pos);
A
Avi Kivity 已提交
450
			return 0;
451
		}
A
Avi Kivity 已提交
452 453
	return 1;
}
454
#endif
A
Avi Kivity 已提交
455

456
static void kvm_mmu_free_page(struct kvm *kvm,
457
			      struct kvm_mmu_page *page_head)
458
{
459
	ASSERT(is_empty_shadow_page(page_head->spt));
460
	list_del(&page_head->link);
461 462 463
	kfree(page_head->spt);
	kfree(page_head);
	++kvm->n_free_mmu_pages;
464 465
}

466 467 468 469 470
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
	return gfn;
}

471 472
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
					       u64 *parent_pte)
A
Avi Kivity 已提交
473 474 475
{
	struct kvm_mmu_page *page;

476
	if (!vcpu->kvm->n_free_mmu_pages)
477
		return NULL;
A
Avi Kivity 已提交
478

479 480 481 482 483
	page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache,
				      sizeof *page);
	page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
	set_page_private(virt_to_page(page->spt), (unsigned long)page);
	list_add(&page->link, &vcpu->kvm->active_mmu_pages);
484
	ASSERT(is_empty_shadow_page(page->spt));
A
Avi Kivity 已提交
485
	page->slot_bitmap = 0;
486
	page->multimapped = 0;
A
Avi Kivity 已提交
487
	page->parent_pte = parent_pte;
A
Avi Kivity 已提交
488
	--vcpu->kvm->n_free_mmu_pages;
489
	return page;
A
Avi Kivity 已提交
490 491
}

492 493
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *page, u64 *parent_pte)
494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
{
	struct kvm_pte_chain *pte_chain;
	struct hlist_node *node;
	int i;

	if (!parent_pte)
		return;
	if (!page->multimapped) {
		u64 *old = page->parent_pte;

		if (!old) {
			page->parent_pte = parent_pte;
			return;
		}
		page->multimapped = 1;
509
		pte_chain = mmu_alloc_pte_chain(vcpu);
510 511 512 513 514 515 516 517 518 519 520 521 522
		INIT_HLIST_HEAD(&page->parent_ptes);
		hlist_add_head(&pte_chain->link, &page->parent_ptes);
		pte_chain->parent_ptes[0] = old;
	}
	hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) {
		if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
			continue;
		for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
			if (!pte_chain->parent_ptes[i]) {
				pte_chain->parent_ptes[i] = parent_pte;
				return;
			}
	}
523
	pte_chain = mmu_alloc_pte_chain(vcpu);
524 525 526 527 528
	BUG_ON(!pte_chain);
	hlist_add_head(&pte_chain->link, &page->parent_ptes);
	pte_chain->parent_ptes[0] = parent_pte;
}

529
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page,
530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546
				       u64 *parent_pte)
{
	struct kvm_pte_chain *pte_chain;
	struct hlist_node *node;
	int i;

	if (!page->multimapped) {
		BUG_ON(page->parent_pte != parent_pte);
		page->parent_pte = NULL;
		return;
	}
	hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link)
		for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
			if (!pte_chain->parent_ptes[i])
				break;
			if (pte_chain->parent_ptes[i] != parent_pte)
				continue;
547 548
			while (i + 1 < NR_PTE_CHAIN_ENTRIES
				&& pte_chain->parent_ptes[i + 1]) {
549 550 551 552 553
				pte_chain->parent_ptes[i]
					= pte_chain->parent_ptes[i + 1];
				++i;
			}
			pte_chain->parent_ptes[i] = NULL;
554 555
			if (i == 0) {
				hlist_del(&pte_chain->link);
556
				mmu_free_pte_chain(pte_chain);
557 558 559 560 561
				if (hlist_empty(&page->parent_ptes)) {
					page->multimapped = 0;
					page->parent_pte = NULL;
				}
			}
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591
			return;
		}
	BUG();
}

static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu,
						gfn_t gfn)
{
	unsigned index;
	struct hlist_head *bucket;
	struct kvm_mmu_page *page;
	struct hlist_node *node;

	pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
	index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
	bucket = &vcpu->kvm->mmu_page_hash[index];
	hlist_for_each_entry(page, node, bucket, hash_link)
		if (page->gfn == gfn && !page->role.metaphysical) {
			pgprintk("%s: found role %x\n",
				 __FUNCTION__, page->role.word);
			return page;
		}
	return NULL;
}

static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
					     int metaphysical,
592
					     unsigned hugepage_access,
593 594 595 596 597 598 599 600 601 602 603 604 605
					     u64 *parent_pte)
{
	union kvm_mmu_page_role role;
	unsigned index;
	unsigned quadrant;
	struct hlist_head *bucket;
	struct kvm_mmu_page *page;
	struct hlist_node *node;

	role.word = 0;
	role.glevels = vcpu->mmu.root_level;
	role.level = level;
	role.metaphysical = metaphysical;
606
	role.hugepage_access = hugepage_access;
607 608 609 610 611 612 613 614 615 616 617
	if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
	pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
		 gfn, role.word);
	index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
	bucket = &vcpu->kvm->mmu_page_hash[index];
	hlist_for_each_entry(page, node, bucket, hash_link)
		if (page->gfn == gfn && page->role.word == role.word) {
618
			mmu_page_add_parent_pte(vcpu, page, parent_pte);
619 620 621 622 623 624 625 626 627 628
			pgprintk("%s: found\n", __FUNCTION__);
			return page;
		}
	page = kvm_mmu_alloc_page(vcpu, parent_pte);
	if (!page)
		return page;
	pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
	page->gfn = gfn;
	page->role = role;
	hlist_add_head(&page->hash_link, bucket);
629
	if (!metaphysical)
630
		rmap_write_protect(vcpu, gfn);
631 632 633
	return page;
}

634
static void kvm_mmu_page_unlink_children(struct kvm *kvm,
635 636
					 struct kvm_mmu_page *page)
{
637 638 639 640
	unsigned i;
	u64 *pt;
	u64 ent;

641
	pt = page->spt;
642 643 644 645

	if (page->role.level == PT_PAGE_TABLE_LEVEL) {
		for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
			if (pt[i] & PT_PRESENT_MASK)
646
				rmap_remove(&pt[i]);
647 648
			pt[i] = 0;
		}
649
		kvm_flush_remote_tlbs(kvm);
650 651 652 653 654 655 656 657 658 659
		return;
	}

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
		ent = pt[i];

		pt[i] = 0;
		if (!(ent & PT_PRESENT_MASK))
			continue;
		ent &= PT64_BASE_ADDR_MASK;
660
		mmu_page_remove_parent_pte(page_header(ent), &pt[i]);
661
	}
662
	kvm_flush_remote_tlbs(kvm);
663 664
}

665
static void kvm_mmu_put_page(struct kvm_mmu_page *page,
666 667
			     u64 *parent_pte)
{
668
	mmu_page_remove_parent_pte(page, parent_pte);
669 670
}

671
static void kvm_mmu_zap_page(struct kvm *kvm,
672 673 674 675 676 677 678 679 680 681 682 683 684 685
			     struct kvm_mmu_page *page)
{
	u64 *parent_pte;

	while (page->multimapped || page->parent_pte) {
		if (!page->multimapped)
			parent_pte = page->parent_pte;
		else {
			struct kvm_pte_chain *chain;

			chain = container_of(page->parent_ptes.first,
					     struct kvm_pte_chain, link);
			parent_pte = chain->parent_ptes[0];
		}
686
		BUG_ON(!parent_pte);
687
		kvm_mmu_put_page(page, parent_pte);
688
		set_shadow_pte(parent_pte, 0);
689
	}
690
	kvm_mmu_page_unlink_children(kvm, page);
691 692
	if (!page->root_count) {
		hlist_del(&page->hash_link);
693
		kvm_mmu_free_page(kvm, page);
A
Avi Kivity 已提交
694
	} else
695
		list_move(&page->link, &kvm->active_mmu_pages);
696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
}

static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	unsigned index;
	struct hlist_head *bucket;
	struct kvm_mmu_page *page;
	struct hlist_node *node, *n;
	int r;

	pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
	r = 0;
	index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
	bucket = &vcpu->kvm->mmu_page_hash[index];
	hlist_for_each_entry_safe(page, node, n, bucket, hash_link)
		if (page->gfn == gfn && !page->role.metaphysical) {
712 713
			pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn,
				 page->role.word);
714
			kvm_mmu_zap_page(vcpu->kvm, page);
715 716 717
			r = 1;
		}
	return r;
718 719
}

720 721 722 723 724 725 726
static void mmu_unshadow(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	struct kvm_mmu_page *page;

	while ((page = kvm_mmu_lookup_page(vcpu, gfn)) != NULL) {
		pgprintk("%s: zap %lx %x\n",
			 __FUNCTION__, gfn, page->role.word);
727
		kvm_mmu_zap_page(vcpu->kvm, page);
728 729 730
	}
}

A
Avi Kivity 已提交
731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750
static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
{
	int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
	struct kvm_mmu_page *page_head = page_header(__pa(pte));

	__set_bit(slot, &page_head->slot_bitmap);
}

hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
{
	hpa_t hpa = gpa_to_hpa(vcpu, gpa);

	return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
}

hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
{
	struct page *page;

	ASSERT((gpa & HPA_ERR_MASK) == 0);
A
Avi Kivity 已提交
751 752
	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
	if (!page)
A
Avi Kivity 已提交
753 754 755 756 757 758 759 760 761 762 763 764 765 766
		return gpa | HPA_ERR_MASK;
	return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
		| (gpa & (PAGE_SIZE-1));
}

hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
{
	gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);

	if (gpa == UNMAPPED_GVA)
		return UNMAPPED_GVA;
	return gpa_to_hpa(vcpu, gpa);
}

767 768 769 770 771 772 773 774 775
struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
{
	gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);

	if (gpa == UNMAPPED_GVA)
		return NULL;
	return pfn_to_page(gpa_to_hpa(vcpu, gpa) >> PAGE_SHIFT);
}

A
Avi Kivity 已提交
776 777 778 779 780 781 782 783 784 785 786 787
static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
{
}

static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
{
	int level = PT32E_ROOT_LEVEL;
	hpa_t table_addr = vcpu->mmu.root_hpa;

	for (; ; level--) {
		u32 index = PT64_INDEX(v, level);
		u64 *table;
788
		u64 pte;
A
Avi Kivity 已提交
789 790 791 792 793

		ASSERT(VALID_PAGE(table_addr));
		table = __va(table_addr);

		if (level == 1) {
794 795 796
			pte = table[index];
			if (is_present_pte(pte) && is_writeble_pte(pte))
				return 0;
A
Avi Kivity 已提交
797 798 799 800
			mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
			page_header_update_slot(vcpu->kvm, table, v);
			table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
								PT_USER_MASK;
801
			rmap_add(vcpu, &table[index]);
A
Avi Kivity 已提交
802 803 804 805
			return 0;
		}

		if (table[index] == 0) {
806
			struct kvm_mmu_page *new_table;
807
			gfn_t pseudo_gfn;
A
Avi Kivity 已提交
808

809 810 811 812
			pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
				>> PAGE_SHIFT;
			new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
						     v, level - 1,
813
						     1, 0, &table[index]);
814
			if (!new_table) {
A
Avi Kivity 已提交
815 816 817 818
				pgprintk("nonpaging_map: ENOMEM\n");
				return -ENOMEM;
			}

819
			table[index] = __pa(new_table->spt) | PT_PRESENT_MASK
820
				| PT_WRITABLE_MASK | PT_USER_MASK;
A
Avi Kivity 已提交
821 822 823 824 825
		}
		table_addr = table[index] & PT64_BASE_ADDR_MASK;
	}
}

826 827 828
static void mmu_free_roots(struct kvm_vcpu *vcpu)
{
	int i;
829
	struct kvm_mmu_page *page;
830

A
Avi Kivity 已提交
831 832
	if (!VALID_PAGE(vcpu->mmu.root_hpa))
		return;
833 834 835 836
#ifdef CONFIG_X86_64
	if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		hpa_t root = vcpu->mmu.root_hpa;

837 838
		page = page_header(root);
		--page->root_count;
839 840 841 842 843 844 845
		vcpu->mmu.root_hpa = INVALID_PAGE;
		return;
	}
#endif
	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->mmu.pae_root[i];

A
Avi Kivity 已提交
846 847 848 849 850
		if (root) {
			root &= PT64_BASE_ADDR_MASK;
			page = page_header(root);
			--page->root_count;
		}
851 852 853 854 855 856 857 858
		vcpu->mmu.pae_root[i] = INVALID_PAGE;
	}
	vcpu->mmu.root_hpa = INVALID_PAGE;
}

static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
{
	int i;
859
	gfn_t root_gfn;
860 861
	struct kvm_mmu_page *page;

862
	root_gfn = vcpu->cr3 >> PAGE_SHIFT;
863 864 865 866 867 868

#ifdef CONFIG_X86_64
	if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		hpa_t root = vcpu->mmu.root_hpa;

		ASSERT(!VALID_PAGE(root));
869
		page = kvm_mmu_get_page(vcpu, root_gfn, 0,
870
					PT64_ROOT_LEVEL, 0, 0, NULL);
871
		root = __pa(page->spt);
872
		++page->root_count;
873 874 875 876 877 878 879 880
		vcpu->mmu.root_hpa = root;
		return;
	}
#endif
	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->mmu.pae_root[i];

		ASSERT(!VALID_PAGE(root));
A
Avi Kivity 已提交
881 882 883 884 885
		if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) {
			if (!is_present_pte(vcpu->pdptrs[i])) {
				vcpu->mmu.pae_root[i] = 0;
				continue;
			}
886
			root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
A
Avi Kivity 已提交
887
		} else if (vcpu->mmu.root_level == 0)
888
			root_gfn = 0;
889
		page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
890
					PT32_ROOT_LEVEL, !is_paging(vcpu),
891
					0, NULL);
892
		root = __pa(page->spt);
893
		++page->root_count;
894 895 896 897 898
		vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
	}
	vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
}

A
Avi Kivity 已提交
899 900 901 902 903 904 905 906 907
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
{
	return vaddr;
}

static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
			       u32 error_code)
{
	gpa_t addr = gva;
A
Avi Kivity 已提交
908
	hpa_t paddr;
909
	int r;
A
Avi Kivity 已提交
910

911 912 913
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;
914

A
Avi Kivity 已提交
915 916 917 918
	ASSERT(vcpu);
	ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));


A
Avi Kivity 已提交
919
	paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
A
Avi Kivity 已提交
920

A
Avi Kivity 已提交
921 922
	if (is_error_hpa(paddr))
		return 1;
A
Avi Kivity 已提交
923

A
Avi Kivity 已提交
924
	return nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
A
Avi Kivity 已提交
925 926 927 928
}

static void nonpaging_free(struct kvm_vcpu *vcpu)
{
929
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
930 931 932 933 934 935 936 937 938 939
}

static int nonpaging_init_context(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu *context = &vcpu->mmu;

	context->new_cr3 = nonpaging_new_cr3;
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
	context->free = nonpaging_free;
940
	context->root_level = 0;
A
Avi Kivity 已提交
941
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
942
	context->root_hpa = INVALID_PAGE;
A
Avi Kivity 已提交
943 944 945 946 947
	return 0;
}

static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
{
A
Avi Kivity 已提交
948
	++vcpu->stat.tlb_flush;
A
Avi Kivity 已提交
949 950 951 952 953
	kvm_arch_ops->tlb_flush(vcpu);
}

static void paging_new_cr3(struct kvm_vcpu *vcpu)
{
954
	pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
955
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977
}

static void inject_page_fault(struct kvm_vcpu *vcpu,
			      u64 addr,
			      u32 err_code)
{
	kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
}

static void paging_free(struct kvm_vcpu *vcpu)
{
	nonpaging_free(vcpu);
}

#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

978
static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
A
Avi Kivity 已提交
979 980 981 982 983 984 985 986
{
	struct kvm_mmu *context = &vcpu->mmu;

	ASSERT(is_pae(vcpu));
	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
	context->free = paging_free;
987 988
	context->root_level = level;
	context->shadow_root_level = level;
A
Avi Kivity 已提交
989
	context->root_hpa = INVALID_PAGE;
A
Avi Kivity 已提交
990 991 992
	return 0;
}

993 994 995 996 997
static int paging64_init_context(struct kvm_vcpu *vcpu)
{
	return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
}

A
Avi Kivity 已提交
998 999 1000 1001 1002 1003 1004 1005 1006 1007
static int paging32_init_context(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu *context = &vcpu->mmu;

	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
	context->free = paging_free;
	context->root_level = PT32_ROOT_LEVEL;
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
1008
	context->root_hpa = INVALID_PAGE;
A
Avi Kivity 已提交
1009 1010 1011 1012 1013
	return 0;
}

static int paging32E_init_context(struct kvm_vcpu *vcpu)
{
1014
	return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
A
Avi Kivity 已提交
1015 1016 1017 1018 1019 1020 1021 1022 1023
}

static int init_kvm_mmu(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
	ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));

	if (!is_paging(vcpu))
		return nonpaging_init_context(vcpu);
A
Avi Kivity 已提交
1024
	else if (is_long_mode(vcpu))
A
Avi Kivity 已提交
1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
		return paging64_init_context(vcpu);
	else if (is_pae(vcpu))
		return paging32E_init_context(vcpu);
	else
		return paging32_init_context(vcpu);
}

static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
	if (VALID_PAGE(vcpu->mmu.root_hpa)) {
		vcpu->mmu.free(vcpu);
		vcpu->mmu.root_hpa = INVALID_PAGE;
	}
}

int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1042 1043 1044 1045 1046 1047
{
	destroy_kvm_mmu(vcpu);
	return init_kvm_mmu(vcpu);
}

int kvm_mmu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1048
{
1049 1050
	int r;

A
Avi Kivity 已提交
1051
	spin_lock(&vcpu->kvm->lock);
1052
	r = mmu_topup_memory_caches(vcpu);
A
Avi Kivity 已提交
1053 1054 1055 1056 1057
	if (r)
		goto out;
	mmu_alloc_roots(vcpu);
	kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
	kvm_mmu_flush_tlb(vcpu);
1058
out:
A
Avi Kivity 已提交
1059
	spin_unlock(&vcpu->kvm->lock);
1060
	return r;
A
Avi Kivity 已提交
1061
}
A
Avi Kivity 已提交
1062 1063 1064 1065 1066 1067
EXPORT_SYMBOL_GPL(kvm_mmu_load);

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
	mmu_free_roots(vcpu);
}
A
Avi Kivity 已提交
1068

1069
static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
1070 1071 1072 1073 1074 1075 1076 1077 1078
				  struct kvm_mmu_page *page,
				  u64 *spte)
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
	if (is_present_pte(pte)) {
		if (page->role.level == PT_PAGE_TABLE_LEVEL)
1079
			rmap_remove(spte);
1080 1081
		else {
			child = page_header(pte & PT64_BASE_ADDR_MASK);
1082
			mmu_page_remove_parent_pte(child, spte);
1083 1084 1085
		}
	}
	*spte = 0;
1086
	kvm_flush_remote_tlbs(vcpu->kvm);
1087 1088
}

1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
				  struct kvm_mmu_page *page,
				  u64 *spte,
				  const void *new, int bytes)
{
	if (page->role.level != PT_PAGE_TABLE_LEVEL)
		return;

	if (page->role.glevels == PT32_ROOT_LEVEL)
		paging32_update_pte(vcpu, page, spte, new, bytes);
	else
		paging64_update_pte(vcpu, page, spte, new, bytes);
}

1103 1104
void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
		       const u8 *old, const u8 *new, int bytes)
1105
{
1106 1107
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_mmu_page *page;
1108
	struct hlist_node *node, *n;
1109 1110 1111 1112
	struct hlist_head *bucket;
	unsigned index;
	u64 *spte;
	unsigned offset = offset_in_page(gpa);
1113
	unsigned pte_size;
1114
	unsigned page_offset;
1115
	unsigned misaligned;
1116
	unsigned quadrant;
1117
	int level;
1118
	int flooded = 0;
1119
	int npte;
1120

1121
	pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
1122 1123 1124 1125 1126 1127 1128 1129
	if (gfn == vcpu->last_pt_write_gfn) {
		++vcpu->last_pt_write_count;
		if (vcpu->last_pt_write_count >= 3)
			flooded = 1;
	} else {
		vcpu->last_pt_write_gfn = gfn;
		vcpu->last_pt_write_count = 1;
	}
1130 1131
	index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
	bucket = &vcpu->kvm->mmu_page_hash[index];
1132
	hlist_for_each_entry_safe(page, node, n, bucket, hash_link) {
1133 1134
		if (page->gfn != gfn || page->role.metaphysical)
			continue;
1135 1136
		pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
		misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
1137
		misaligned |= bytes < 4;
1138
		if (misaligned || flooded) {
1139 1140 1141 1142
			/*
			 * Misaligned accesses are too much trouble to fix
			 * up; also, they usually indicate a page is not used
			 * as a page table.
1143 1144 1145 1146 1147
			 *
			 * If we're seeing too many writes to a page,
			 * it may no longer be a page table, or we may be
			 * forking, in which case it is better to unmap the
			 * page.
1148 1149 1150
			 */
			pgprintk("misaligned: gpa %llx bytes %d role %x\n",
				 gpa, bytes, page->role.word);
1151
			kvm_mmu_zap_page(vcpu->kvm, page);
1152 1153
			continue;
		}
1154 1155
		page_offset = offset;
		level = page->role.level;
1156
		npte = 1;
1157
		if (page->role.glevels == PT32_ROOT_LEVEL) {
1158 1159 1160 1161 1162 1163 1164
			page_offset <<= 1;	/* 32->64 */
			/*
			 * A 32-bit pde maps 4MB while the shadow pdes map
			 * only 2MB.  So we need to double the offset again
			 * and zap two pdes instead of one.
			 */
			if (level == PT32_ROOT_LEVEL) {
1165
				page_offset &= ~7; /* kill rounding error */
1166 1167 1168
				page_offset <<= 1;
				npte = 2;
			}
1169
			quadrant = page_offset >> PAGE_SHIFT;
1170
			page_offset &= ~PAGE_MASK;
1171 1172
			if (quadrant != page->role.quadrant)
				continue;
1173
		}
1174
		spte = &page->spt[page_offset / sizeof(*spte)];
1175
		while (npte--) {
1176
			mmu_pte_write_zap_pte(vcpu, page, spte);
1177
			mmu_pte_write_new_pte(vcpu, page, spte, new, bytes);
1178
			++spte;
1179 1180
		}
	}
1181 1182
}

1183 1184 1185 1186 1187 1188 1189
int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
	gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);

	return kvm_mmu_unprotect_page(vcpu, gpa >> PAGE_SHIFT);
}

A
Avi Kivity 已提交
1190 1191 1192 1193 1194 1195 1196
void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
{
	while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) {
		struct kvm_mmu_page *page;

		page = container_of(vcpu->kvm->active_mmu_pages.prev,
				    struct kvm_mmu_page, link);
1197
		kvm_mmu_zap_page(vcpu->kvm, page);
A
Avi Kivity 已提交
1198 1199 1200 1201
	}
}
EXPORT_SYMBOL_GPL(kvm_mmu_free_some_pages);

A
Avi Kivity 已提交
1202 1203
static void free_mmu_pages(struct kvm_vcpu *vcpu)
{
1204
	struct kvm_mmu_page *page;
A
Avi Kivity 已提交
1205

1206 1207 1208
	while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
		page = container_of(vcpu->kvm->active_mmu_pages.next,
				    struct kvm_mmu_page, link);
1209
		kvm_mmu_zap_page(vcpu->kvm, page);
1210
	}
1211
	free_page((unsigned long)vcpu->mmu.pae_root);
A
Avi Kivity 已提交
1212 1213 1214 1215
}

static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
{
1216
	struct page *page;
A
Avi Kivity 已提交
1217 1218 1219 1220
	int i;

	ASSERT(vcpu);

1221
	vcpu->kvm->n_free_mmu_pages = KVM_NUM_MMU_PAGES;
1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234

	/*
	 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
	 * Therefore we need to allocate shadow page tables in the first
	 * 4GB of memory, which happens to fit the DMA32 zone.
	 */
	page = alloc_page(GFP_KERNEL | __GFP_DMA32);
	if (!page)
		goto error_1;
	vcpu->mmu.pae_root = page_address(page);
	for (i = 0; i < 4; ++i)
		vcpu->mmu.pae_root[i] = INVALID_PAGE;

A
Avi Kivity 已提交
1235 1236 1237 1238 1239 1240 1241
	return 0;

error_1:
	free_mmu_pages(vcpu);
	return -ENOMEM;
}

1242
int kvm_mmu_create(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
1243 1244 1245 1246
{
	ASSERT(vcpu);
	ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));

1247 1248
	return alloc_mmu_pages(vcpu);
}
A
Avi Kivity 已提交
1249

1250 1251 1252 1253
int kvm_mmu_setup(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
	ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1254

1255
	return init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
1256 1257 1258 1259 1260 1261 1262 1263
}

void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);

	destroy_kvm_mmu(vcpu);
	free_mmu_pages(vcpu);
1264
	mmu_free_memory_caches(vcpu);
A
Avi Kivity 已提交
1265 1266
}

1267
void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
A
Avi Kivity 已提交
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
{
	struct kvm_mmu_page *page;

	list_for_each_entry(page, &kvm->active_mmu_pages, link) {
		int i;
		u64 *pt;

		if (!test_bit(slot, &page->slot_bitmap))
			continue;

1278
		pt = page->spt;
A
Avi Kivity 已提交
1279 1280
		for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
			/* avoid RMW */
1281
			if (pt[i] & PT_WRITABLE_MASK) {
1282
				rmap_remove(&pt[i]);
A
Avi Kivity 已提交
1283
				pt[i] &= ~PT_WRITABLE_MASK;
1284
			}
A
Avi Kivity 已提交
1285 1286
	}
}
1287

1288
void kvm_mmu_zap_all(struct kvm *kvm)
D
Dor Laor 已提交
1289
{
1290
	struct kvm_mmu_page *page, *node;
D
Dor Laor 已提交
1291

1292 1293
	list_for_each_entry_safe(page, node, &kvm->active_mmu_pages, link)
		kvm_mmu_zap_page(kvm, page);
D
Dor Laor 已提交
1294

1295
	kvm_flush_remote_tlbs(kvm);
D
Dor Laor 已提交
1296 1297
}

1298 1299 1300 1301 1302 1303
void kvm_mmu_module_exit(void)
{
	if (pte_chain_cache)
		kmem_cache_destroy(pte_chain_cache);
	if (rmap_desc_cache)
		kmem_cache_destroy(rmap_desc_cache);
1304 1305 1306 1307
	if (mmu_page_cache)
		kmem_cache_destroy(mmu_page_cache);
	if (mmu_page_header_cache)
		kmem_cache_destroy(mmu_page_header_cache);
1308 1309 1310 1311 1312 1313
}

int kvm_mmu_module_init(void)
{
	pte_chain_cache = kmem_cache_create("kvm_pte_chain",
					    sizeof(struct kvm_pte_chain),
1314
					    0, 0, NULL);
1315 1316 1317 1318
	if (!pte_chain_cache)
		goto nomem;
	rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
					    sizeof(struct kvm_rmap_desc),
1319
					    0, 0, NULL);
1320 1321 1322
	if (!rmap_desc_cache)
		goto nomem;

1323 1324
	mmu_page_cache = kmem_cache_create("kvm_mmu_page",
					   PAGE_SIZE,
1325
					   PAGE_SIZE, 0, NULL);
1326 1327 1328 1329 1330
	if (!mmu_page_cache)
		goto nomem;

	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
1331
						  0, 0, NULL);
1332 1333 1334
	if (!mmu_page_header_cache)
		goto nomem;

1335 1336 1337 1338 1339 1340 1341
	return 0;

nomem:
	kvm_mmu_module_exit();
	return -ENOMEM;
}

1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
#ifdef AUDIT

static const char *audit_msg;

static gva_t canonicalize(gva_t gva)
{
#ifdef CONFIG_X86_64
	gva = (long long)(gva << 16) >> 16;
#endif
	return gva;
}

static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
				gva_t va, int level)
{
	u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
	int i;
	gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
		u64 ent = pt[i];

A
Adrian Bunk 已提交
1364
		if (!(ent & PT_PRESENT_MASK))
1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385
			continue;

		va = canonicalize(va);
		if (level > 1)
			audit_mappings_page(vcpu, ent, va, level - 1);
		else {
			gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va);
			hpa_t hpa = gpa_to_hpa(vcpu, gpa);

			if ((ent & PT_PRESENT_MASK)
			    && (ent & PT64_BASE_ADDR_MASK) != hpa)
				printk(KERN_ERR "audit error: (%s) levels %d"
				       " gva %lx gpa %llx hpa %llx ent %llx\n",
				       audit_msg, vcpu->mmu.root_level,
				       va, gpa, hpa, ent);
		}
	}
}

static void audit_mappings(struct kvm_vcpu *vcpu)
{
1386
	unsigned i;
1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437

	if (vcpu->mmu.root_level == 4)
		audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4);
	else
		for (i = 0; i < 4; ++i)
			if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK)
				audit_mappings_page(vcpu,
						    vcpu->mmu.pae_root[i],
						    i << 30,
						    2);
}

static int count_rmaps(struct kvm_vcpu *vcpu)
{
	int nmaps = 0;
	int i, j, k;

	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
		struct kvm_rmap_desc *d;

		for (j = 0; j < m->npages; ++j) {
			struct page *page = m->phys_mem[j];

			if (!page->private)
				continue;
			if (!(page->private & 1)) {
				++nmaps;
				continue;
			}
			d = (struct kvm_rmap_desc *)(page->private & ~1ul);
			while (d) {
				for (k = 0; k < RMAP_EXT; ++k)
					if (d->shadow_ptes[k])
						++nmaps;
					else
						break;
				d = d->more;
			}
		}
	}
	return nmaps;
}

static int count_writable_mappings(struct kvm_vcpu *vcpu)
{
	int nmaps = 0;
	struct kvm_mmu_page *page;
	int i;

	list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
1438
		u64 *pt = page->spt;
1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500

		if (page->role.level != PT_PAGE_TABLE_LEVEL)
			continue;

		for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
			u64 ent = pt[i];

			if (!(ent & PT_PRESENT_MASK))
				continue;
			if (!(ent & PT_WRITABLE_MASK))
				continue;
			++nmaps;
		}
	}
	return nmaps;
}

static void audit_rmap(struct kvm_vcpu *vcpu)
{
	int n_rmap = count_rmaps(vcpu);
	int n_actual = count_writable_mappings(vcpu);

	if (n_rmap != n_actual)
		printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
		       __FUNCTION__, audit_msg, n_rmap, n_actual);
}

static void audit_write_protection(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_page *page;

	list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
		hfn_t hfn;
		struct page *pg;

		if (page->role.metaphysical)
			continue;

		hfn = gpa_to_hpa(vcpu, (gpa_t)page->gfn << PAGE_SHIFT)
			>> PAGE_SHIFT;
		pg = pfn_to_page(hfn);
		if (pg->private)
			printk(KERN_ERR "%s: (%s) shadow page has writable"
			       " mappings: gfn %lx role %x\n",
			       __FUNCTION__, audit_msg, page->gfn,
			       page->role.word);
	}
}

static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
{
	int olddbg = dbg;

	dbg = 0;
	audit_msg = msg;
	audit_rmap(vcpu);
	audit_write_protection(vcpu);
	audit_mappings(vcpu);
	dbg = olddbg;
}

#endif