mmu.c 73.8 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
A
Avi Kivity 已提交
19

20
#include "mmu.h"
A
Avi Kivity 已提交
21

22
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
23 24 25 26 27
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/module.h>
28
#include <linux/swap.h>
M
Marcelo Tosatti 已提交
29
#include <linux/hugetlb.h>
30
#include <linux/compiler.h>
A
Avi Kivity 已提交
31

A
Avi Kivity 已提交
32 33
#include <asm/page.h>
#include <asm/cmpxchg.h>
34
#include <asm/io.h>
35
#include <asm/vmx.h>
A
Avi Kivity 已提交
36

37 38 39 40 41 42 43
/*
 * When setting this variable to true it enables Two-Dimensional-Paging
 * where the hardware walks 2 page tables:
 * 1. the guest-virtual to guest-physical
 * 2. while doing 1. it walks guest-physical to host-physical
 * If the hardware supports that we don't need to do shadow paging.
 */
44
bool tdp_enabled = false;
45

46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
#undef MMU_DEBUG

#undef AUDIT

#ifdef AUDIT
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
#else
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
#endif

#ifdef MMU_DEBUG

#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)

#else

#define pgprintk(x...) do { } while (0)
#define rmap_printk(x...) do { } while (0)

#endif

#if defined(MMU_DEBUG) || defined(AUDIT)
69 70
static int dbg = 0;
module_param(dbg, bool, 0644);
71
#endif
A
Avi Kivity 已提交
72

73 74 75
static int oos_shadow = 1;
module_param(oos_shadow, bool, 0644);

76 77 78
#ifndef MMU_DEBUG
#define ASSERT(x) do { } while (0)
#else
A
Avi Kivity 已提交
79 80 81 82 83
#define ASSERT(x)							\
	if (!(x)) {							\
		printk(KERN_WARNING "assertion failed %s:%d: %s\n",	\
		       __FILE__, __LINE__, #x);				\
	}
84
#endif
A
Avi Kivity 已提交
85 86 87 88 89 90 91 92 93

#define PT_FIRST_AVAIL_BITS_SHIFT 9
#define PT64_SECOND_AVAIL_BITS_SHIFT 52

#define VALID_PAGE(x) ((x) != INVALID_PAGE)

#define PT64_LEVEL_BITS 9

#define PT64_LEVEL_SHIFT(level) \
M
Mike Day 已提交
94
		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
A
Avi Kivity 已提交
95 96 97 98 99 100 101 102 103 104 105

#define PT64_LEVEL_MASK(level) \
		(((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))

#define PT64_INDEX(address, level)\
	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))


#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
M
Mike Day 已提交
106
		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
A
Avi Kivity 已提交
107 108 109 110 111 112 113 114

#define PT32_LEVEL_MASK(level) \
		(((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


115
#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
A
Avi Kivity 已提交
116 117 118 119 120 121 122
#define PT64_DIR_BASE_ADDR_MASK \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))

#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))

123 124
#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
			| PT64_NX_MASK)
A
Avi Kivity 已提交
125 126 127 128

#define PFERR_PRESENT_MASK (1U << 0)
#define PFERR_WRITE_MASK (1U << 1)
#define PFERR_USER_MASK (1U << 2)
129
#define PFERR_FETCH_MASK (1U << 4)
A
Avi Kivity 已提交
130 131 132 133

#define PT_DIRECTORY_LEVEL 2
#define PT_PAGE_TABLE_LEVEL 1

134 135
#define RMAP_EXT 4

136 137 138 139 140
#define ACC_EXEC_MASK    1
#define ACC_WRITE_MASK   PT_WRITABLE_MASK
#define ACC_USER_MASK    PT_USER_MASK
#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)

141 142
#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)

143 144 145 146 147
struct kvm_rmap_desc {
	u64 *shadow_ptes[RMAP_EXT];
	struct kvm_rmap_desc *more;
};

A
Avi Kivity 已提交
148 149
struct kvm_shadow_walk {
	int (*entry)(struct kvm_shadow_walk *walk, struct kvm_vcpu *vcpu,
150
		     u64 addr, u64 *spte, int level);
A
Avi Kivity 已提交
151 152
};

153 154 155 156
struct kvm_unsync_walk {
	int (*entry) (struct kvm_mmu_page *sp, struct kvm_unsync_walk *walk);
};

M
Marcelo Tosatti 已提交
157 158
typedef int (*mmu_parent_walk_fn) (struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp);

159 160
static struct kmem_cache *pte_chain_cache;
static struct kmem_cache *rmap_desc_cache;
161
static struct kmem_cache *mmu_page_header_cache;
162

163 164
static u64 __read_mostly shadow_trap_nonpresent_pte;
static u64 __read_mostly shadow_notrap_nonpresent_pte;
S
Sheng Yang 已提交
165 166 167 168 169 170
static u64 __read_mostly shadow_base_present_pte;
static u64 __read_mostly shadow_nx_mask;
static u64 __read_mostly shadow_x_mask;	/* mutual exclusive with nx_mask */
static u64 __read_mostly shadow_user_mask;
static u64 __read_mostly shadow_accessed_mask;
static u64 __read_mostly shadow_dirty_mask;
S
Sheng Yang 已提交
171
static u64 __read_mostly shadow_mt_mask;
172 173 174 175 176 177 178 179

void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
{
	shadow_trap_nonpresent_pte = trap_pte;
	shadow_notrap_nonpresent_pte = notrap_pte;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);

S
Sheng Yang 已提交
180 181 182 183 184 185 186
void kvm_mmu_set_base_ptes(u64 base_pte)
{
	shadow_base_present_pte = base_pte;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);

void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
S
Sheng Yang 已提交
187
		u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 mt_mask)
S
Sheng Yang 已提交
188 189 190 191 192 193
{
	shadow_user_mask = user_mask;
	shadow_accessed_mask = accessed_mask;
	shadow_dirty_mask = dirty_mask;
	shadow_nx_mask = nx_mask;
	shadow_x_mask = x_mask;
S
Sheng Yang 已提交
194
	shadow_mt_mask = mt_mask;
S
Sheng Yang 已提交
195 196 197
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);

A
Avi Kivity 已提交
198 199
static int is_write_protection(struct kvm_vcpu *vcpu)
{
200
	return vcpu->arch.cr0 & X86_CR0_WP;
A
Avi Kivity 已提交
201 202 203 204 205 206 207
}

static int is_cpuid_PSE36(void)
{
	return 1;
}

208 209
static int is_nx(struct kvm_vcpu *vcpu)
{
210
	return vcpu->arch.shadow_efer & EFER_NX;
211 212
}

A
Avi Kivity 已提交
213 214 215 216 217
static int is_present_pte(unsigned long pte)
{
	return pte & PT_PRESENT_MASK;
}

218 219 220 221 222 223
static int is_shadow_present_pte(u64 pte)
{
	return pte != shadow_trap_nonpresent_pte
		&& pte != shadow_notrap_nonpresent_pte;
}

M
Marcelo Tosatti 已提交
224 225 226 227 228
static int is_large_pte(u64 pte)
{
	return pte & PT_PAGE_SIZE_MASK;
}

A
Avi Kivity 已提交
229 230 231 232 233
static int is_writeble_pte(unsigned long pte)
{
	return pte & PT_WRITABLE_MASK;
}

234 235
static int is_dirty_pte(unsigned long pte)
{
S
Sheng Yang 已提交
236
	return pte & shadow_dirty_mask;
237 238
}

239 240
static int is_rmap_pte(u64 pte)
{
241
	return is_shadow_present_pte(pte);
242 243
}

244
static pfn_t spte_to_pfn(u64 pte)
245
{
246
	return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
247 248
}

249 250 251 252 253 254 255
static gfn_t pse36_gfn_delta(u32 gpte)
{
	int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;

	return (gpte & PT32_DIR_PSE36_MASK) << shift;
}

256 257 258 259 260 261 262 263 264
static void set_shadow_pte(u64 *sptep, u64 spte)
{
#ifdef CONFIG_X86_64
	set_64bit((unsigned long *)sptep, spte);
#else
	set_64bit((unsigned long long *)sptep, spte);
#endif
}

265
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
266
				  struct kmem_cache *base_cache, int min)
267 268 269 270
{
	void *obj;

	if (cache->nobjs >= min)
271
		return 0;
272
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
273
		obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
274
		if (!obj)
275
			return -ENOMEM;
276 277
		cache->objects[cache->nobjs++] = obj;
	}
278
	return 0;
279 280 281 282 283 284 285 286
}

static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
		kfree(mc->objects[--mc->nobjs]);
}

A
Avi Kivity 已提交
287
static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
288
				       int min)
A
Avi Kivity 已提交
289 290 291 292 293 294
{
	struct page *page;

	if (cache->nobjs >= min)
		return 0;
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
295
		page = alloc_page(GFP_KERNEL);
A
Avi Kivity 已提交
296 297 298 299 300 301 302 303 304 305 306
		if (!page)
			return -ENOMEM;
		set_page_private(page, 0);
		cache->objects[cache->nobjs++] = page_address(page);
	}
	return 0;
}

static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
307
		free_page((unsigned long)mc->objects[--mc->nobjs]);
A
Avi Kivity 已提交
308 309
}

310
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
311
{
312 313
	int r;

314
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
315
				   pte_chain_cache, 4);
316 317
	if (r)
		goto out;
318
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
319
				   rmap_desc_cache, 4);
320 321
	if (r)
		goto out;
322
	r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
323 324
	if (r)
		goto out;
325
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
326
				   mmu_page_header_cache, 4);
327 328
out:
	return r;
329 330 331 332
}

static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
333 334 335 336
	mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache);
	mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache);
	mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
	mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
}

static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
				    size_t size)
{
	void *p;

	BUG_ON(!mc->nobjs);
	p = mc->objects[--mc->nobjs];
	memset(p, 0, size);
	return p;
}

static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
{
352
	return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
353 354 355
				      sizeof(struct kvm_pte_chain));
}

356
static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
357
{
358
	kfree(pc);
359 360 361 362
}

static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
{
363
	return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
364 365 366
				      sizeof(struct kvm_rmap_desc));
}

367
static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
368
{
369
	kfree(rd);
370 371
}

M
Marcelo Tosatti 已提交
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
/*
 * Return the pointer to the largepage write count for a given
 * gfn, handling slots that are not large page aligned.
 */
static int *slot_largepage_idx(gfn_t gfn, struct kvm_memory_slot *slot)
{
	unsigned long idx;

	idx = (gfn / KVM_PAGES_PER_HPAGE) -
	      (slot->base_gfn / KVM_PAGES_PER_HPAGE);
	return &slot->lpage_info[idx].write_count;
}

static void account_shadowed(struct kvm *kvm, gfn_t gfn)
{
	int *write_count;

389 390 391
	gfn = unalias_gfn(kvm, gfn);
	write_count = slot_largepage_idx(gfn,
					 gfn_to_memslot_unaliased(kvm, gfn));
M
Marcelo Tosatti 已提交
392 393 394 395 396 397 398
	*write_count += 1;
}

static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
{
	int *write_count;

399 400 401
	gfn = unalias_gfn(kvm, gfn);
	write_count = slot_largepage_idx(gfn,
					 gfn_to_memslot_unaliased(kvm, gfn));
M
Marcelo Tosatti 已提交
402 403 404 405 406 407
	*write_count -= 1;
	WARN_ON(*write_count < 0);
}

static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
{
408
	struct kvm_memory_slot *slot;
M
Marcelo Tosatti 已提交
409 410
	int *largepage_idx;

411 412
	gfn = unalias_gfn(kvm, gfn);
	slot = gfn_to_memslot_unaliased(kvm, gfn);
M
Marcelo Tosatti 已提交
413 414 415 416 417 418 419 420 421 422 423 424
	if (slot) {
		largepage_idx = slot_largepage_idx(gfn, slot);
		return *largepage_idx;
	}

	return 1;
}

static int host_largepage_backed(struct kvm *kvm, gfn_t gfn)
{
	struct vm_area_struct *vma;
	unsigned long addr;
425
	int ret = 0;
M
Marcelo Tosatti 已提交
426 427 428

	addr = gfn_to_hva(kvm, gfn);
	if (kvm_is_error_hva(addr))
429
		return ret;
M
Marcelo Tosatti 已提交
430

431
	down_read(&current->mm->mmap_sem);
M
Marcelo Tosatti 已提交
432 433
	vma = find_vma(current->mm, addr);
	if (vma && is_vm_hugetlb_page(vma))
434 435
		ret = 1;
	up_read(&current->mm->mmap_sem);
M
Marcelo Tosatti 已提交
436

437
	return ret;
M
Marcelo Tosatti 已提交
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
}

static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
{
	struct kvm_memory_slot *slot;

	if (has_wrprotected_page(vcpu->kvm, large_gfn))
		return 0;

	if (!host_largepage_backed(vcpu->kvm, large_gfn))
		return 0;

	slot = gfn_to_memslot(vcpu->kvm, large_gfn);
	if (slot && slot->dirty_bitmap)
		return 0;

	return 1;
}

457 458 459 460 461
/*
 * Take gfn and return the reverse mapping to it.
 * Note: gfn must be unaliased before this function get called
 */

M
Marcelo Tosatti 已提交
462
static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
463 464
{
	struct kvm_memory_slot *slot;
M
Marcelo Tosatti 已提交
465
	unsigned long idx;
466 467

	slot = gfn_to_memslot(kvm, gfn);
M
Marcelo Tosatti 已提交
468 469 470 471 472 473 474
	if (!lpage)
		return &slot->rmap[gfn - slot->base_gfn];

	idx = (gfn / KVM_PAGES_PER_HPAGE) -
	      (slot->base_gfn / KVM_PAGES_PER_HPAGE);

	return &slot->lpage_info[idx].rmap_pde;
475 476
}

477 478 479
/*
 * Reverse mapping data structures:
 *
480 481
 * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
 * that points to page_address(page).
482
 *
483 484
 * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
 * containing more mappings.
485
 */
M
Marcelo Tosatti 已提交
486
static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn, int lpage)
487
{
488
	struct kvm_mmu_page *sp;
489
	struct kvm_rmap_desc *desc;
490
	unsigned long *rmapp;
491 492 493 494
	int i;

	if (!is_rmap_pte(*spte))
		return;
495
	gfn = unalias_gfn(vcpu->kvm, gfn);
496 497
	sp = page_header(__pa(spte));
	sp->gfns[spte - sp->spt] = gfn;
M
Marcelo Tosatti 已提交
498
	rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
499
	if (!*rmapp) {
500
		rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
501 502
		*rmapp = (unsigned long)spte;
	} else if (!(*rmapp & 1)) {
503
		rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
504
		desc = mmu_alloc_rmap_desc(vcpu);
505
		desc->shadow_ptes[0] = (u64 *)*rmapp;
506
		desc->shadow_ptes[1] = spte;
507
		*rmapp = (unsigned long)desc | 1;
508 509
	} else {
		rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
510
		desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
511 512 513
		while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
			desc = desc->more;
		if (desc->shadow_ptes[RMAP_EXT-1]) {
514
			desc->more = mmu_alloc_rmap_desc(vcpu);
515 516 517 518 519 520 521 522
			desc = desc->more;
		}
		for (i = 0; desc->shadow_ptes[i]; ++i)
			;
		desc->shadow_ptes[i] = spte;
	}
}

523
static void rmap_desc_remove_entry(unsigned long *rmapp,
524 525 526 527 528 529 530 531 532
				   struct kvm_rmap_desc *desc,
				   int i,
				   struct kvm_rmap_desc *prev_desc)
{
	int j;

	for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
		;
	desc->shadow_ptes[i] = desc->shadow_ptes[j];
A
Al Viro 已提交
533
	desc->shadow_ptes[j] = NULL;
534 535 536
	if (j != 0)
		return;
	if (!prev_desc && !desc->more)
537
		*rmapp = (unsigned long)desc->shadow_ptes[0];
538 539 540 541
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
542
			*rmapp = (unsigned long)desc->more | 1;
543
	mmu_free_rmap_desc(desc);
544 545
}

546
static void rmap_remove(struct kvm *kvm, u64 *spte)
547 548 549
{
	struct kvm_rmap_desc *desc;
	struct kvm_rmap_desc *prev_desc;
550
	struct kvm_mmu_page *sp;
551
	pfn_t pfn;
552
	unsigned long *rmapp;
553 554 555 556
	int i;

	if (!is_rmap_pte(*spte))
		return;
557
	sp = page_header(__pa(spte));
558
	pfn = spte_to_pfn(*spte);
S
Sheng Yang 已提交
559
	if (*spte & shadow_accessed_mask)
560
		kvm_set_pfn_accessed(pfn);
561
	if (is_writeble_pte(*spte))
562
		kvm_release_pfn_dirty(pfn);
563
	else
564
		kvm_release_pfn_clean(pfn);
M
Marcelo Tosatti 已提交
565
	rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], is_large_pte(*spte));
566
	if (!*rmapp) {
567 568
		printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
		BUG();
569
	} else if (!(*rmapp & 1)) {
570
		rmap_printk("rmap_remove:  %p %llx 1->0\n", spte, *spte);
571
		if ((u64 *)*rmapp != spte) {
572 573 574 575
			printk(KERN_ERR "rmap_remove:  %p %llx 1->BUG\n",
			       spte, *spte);
			BUG();
		}
576
		*rmapp = 0;
577 578
	} else {
		rmap_printk("rmap_remove:  %p %llx many->many\n", spte, *spte);
579
		desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
580 581 582 583
		prev_desc = NULL;
		while (desc) {
			for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
				if (desc->shadow_ptes[i] == spte) {
584
					rmap_desc_remove_entry(rmapp,
585
							       desc, i,
586 587 588 589 590 591 592 593 594 595
							       prev_desc);
					return;
				}
			prev_desc = desc;
			desc = desc->more;
		}
		BUG();
	}
}

596
static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
597 598
{
	struct kvm_rmap_desc *desc;
599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623
	struct kvm_rmap_desc *prev_desc;
	u64 *prev_spte;
	int i;

	if (!*rmapp)
		return NULL;
	else if (!(*rmapp & 1)) {
		if (!spte)
			return (u64 *)*rmapp;
		return NULL;
	}
	desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
	prev_desc = NULL;
	prev_spte = NULL;
	while (desc) {
		for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
			if (prev_spte == spte)
				return desc->shadow_ptes[i];
			prev_spte = desc->shadow_ptes[i];
		}
		desc = desc->more;
	}
	return NULL;
}

624
static int rmap_write_protect(struct kvm *kvm, u64 gfn)
625
{
626
	unsigned long *rmapp;
627
	u64 *spte;
628
	int write_protected = 0;
629

630
	gfn = unalias_gfn(kvm, gfn);
M
Marcelo Tosatti 已提交
631
	rmapp = gfn_to_rmap(kvm, gfn, 0);
632

633 634
	spte = rmap_next(kvm, rmapp, NULL);
	while (spte) {
635 636 637
		BUG_ON(!spte);
		BUG_ON(!(*spte & PT_PRESENT_MASK));
		rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
638
		if (is_writeble_pte(*spte)) {
639
			set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
640 641
			write_protected = 1;
		}
642
		spte = rmap_next(kvm, rmapp, spte);
643
	}
644
	if (write_protected) {
645
		pfn_t pfn;
646 647

		spte = rmap_next(kvm, rmapp, NULL);
648 649
		pfn = spte_to_pfn(*spte);
		kvm_set_pfn_dirty(pfn);
650 651
	}

M
Marcelo Tosatti 已提交
652 653 654 655 656 657 658 659 660 661 662 663
	/* check for huge page mappings */
	rmapp = gfn_to_rmap(kvm, gfn, 1);
	spte = rmap_next(kvm, rmapp, NULL);
	while (spte) {
		BUG_ON(!spte);
		BUG_ON(!(*spte & PT_PRESENT_MASK));
		BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
		pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
		if (is_writeble_pte(*spte)) {
			rmap_remove(kvm, spte);
			--kvm->stat.lpages;
			set_shadow_pte(spte, shadow_trap_nonpresent_pte);
664
			spte = NULL;
M
Marcelo Tosatti 已提交
665 666 667 668 669
			write_protected = 1;
		}
		spte = rmap_next(kvm, rmapp, spte);
	}

670
	return write_protected;
671 672
}

673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730
static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
{
	u64 *spte;
	int need_tlb_flush = 0;

	while ((spte = rmap_next(kvm, rmapp, NULL))) {
		BUG_ON(!(*spte & PT_PRESENT_MASK));
		rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
		rmap_remove(kvm, spte);
		set_shadow_pte(spte, shadow_trap_nonpresent_pte);
		need_tlb_flush = 1;
	}
	return need_tlb_flush;
}

static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
			  int (*handler)(struct kvm *kvm, unsigned long *rmapp))
{
	int i;
	int retval = 0;

	/*
	 * If mmap_sem isn't taken, we can look the memslots with only
	 * the mmu_lock by skipping over the slots with userspace_addr == 0.
	 */
	for (i = 0; i < kvm->nmemslots; i++) {
		struct kvm_memory_slot *memslot = &kvm->memslots[i];
		unsigned long start = memslot->userspace_addr;
		unsigned long end;

		/* mmu_lock protects userspace_addr */
		if (!start)
			continue;

		end = start + (memslot->npages << PAGE_SHIFT);
		if (hva >= start && hva < end) {
			gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
			retval |= handler(kvm, &memslot->rmap[gfn_offset]);
			retval |= handler(kvm,
					  &memslot->lpage_info[
						  gfn_offset /
						  KVM_PAGES_PER_HPAGE].rmap_pde);
		}
	}

	return retval;
}

int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
{
	return kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
}

static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp)
{
	u64 *spte;
	int young = 0;

731 732 733 734
	/* always return old for EPT */
	if (!shadow_accessed_mask)
		return 0;

735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754
	spte = rmap_next(kvm, rmapp, NULL);
	while (spte) {
		int _young;
		u64 _spte = *spte;
		BUG_ON(!(_spte & PT_PRESENT_MASK));
		_young = _spte & PT_ACCESSED_MASK;
		if (_young) {
			young = 1;
			clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
		}
		spte = rmap_next(kvm, rmapp, spte);
	}
	return young;
}

int kvm_age_hva(struct kvm *kvm, unsigned long hva)
{
	return kvm_handle_hva(kvm, hva, kvm_age_rmapp);
}

755
#ifdef MMU_DEBUG
756
static int is_empty_shadow_page(u64 *spt)
A
Avi Kivity 已提交
757
{
758 759 760
	u64 *pos;
	u64 *end;

761
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
762
		if (is_shadow_present_pte(*pos)) {
763
			printk(KERN_ERR "%s: %p %llx\n", __func__,
764
			       pos, *pos);
A
Avi Kivity 已提交
765
			return 0;
766
		}
A
Avi Kivity 已提交
767 768
	return 1;
}
769
#endif
A
Avi Kivity 已提交
770

771
static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
772
{
773 774 775 776 777
	ASSERT(is_empty_shadow_page(sp->spt));
	list_del(&sp->link);
	__free_page(virt_to_page(sp->spt));
	__free_page(virt_to_page(sp->gfns));
	kfree(sp);
778
	++kvm->arch.n_free_mmu_pages;
779 780
}

781 782
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
783
	return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
784 785
}

786 787
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
					       u64 *parent_pte)
A
Avi Kivity 已提交
788
{
789
	struct kvm_mmu_page *sp;
A
Avi Kivity 已提交
790

791 792 793
	sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
	sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
	sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
794
	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
795
	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
796
	INIT_LIST_HEAD(&sp->oos_link);
797
	ASSERT(is_empty_shadow_page(sp->spt));
798
	bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
799
	sp->multimapped = 0;
800
	sp->global = 1;
801
	sp->parent_pte = parent_pte;
802
	--vcpu->kvm->arch.n_free_mmu_pages;
803
	return sp;
A
Avi Kivity 已提交
804 805
}

806
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
807
				    struct kvm_mmu_page *sp, u64 *parent_pte)
808 809 810 811 812 813 814
{
	struct kvm_pte_chain *pte_chain;
	struct hlist_node *node;
	int i;

	if (!parent_pte)
		return;
815 816
	if (!sp->multimapped) {
		u64 *old = sp->parent_pte;
817 818

		if (!old) {
819
			sp->parent_pte = parent_pte;
820 821
			return;
		}
822
		sp->multimapped = 1;
823
		pte_chain = mmu_alloc_pte_chain(vcpu);
824 825
		INIT_HLIST_HEAD(&sp->parent_ptes);
		hlist_add_head(&pte_chain->link, &sp->parent_ptes);
826 827
		pte_chain->parent_ptes[0] = old;
	}
828
	hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
829 830 831 832 833 834 835 836
		if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
			continue;
		for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
			if (!pte_chain->parent_ptes[i]) {
				pte_chain->parent_ptes[i] = parent_pte;
				return;
			}
	}
837
	pte_chain = mmu_alloc_pte_chain(vcpu);
838
	BUG_ON(!pte_chain);
839
	hlist_add_head(&pte_chain->link, &sp->parent_ptes);
840 841 842
	pte_chain->parent_ptes[0] = parent_pte;
}

843
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
844 845 846 847 848 849
				       u64 *parent_pte)
{
	struct kvm_pte_chain *pte_chain;
	struct hlist_node *node;
	int i;

850 851 852
	if (!sp->multimapped) {
		BUG_ON(sp->parent_pte != parent_pte);
		sp->parent_pte = NULL;
853 854
		return;
	}
855
	hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
856 857 858 859 860
		for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
			if (!pte_chain->parent_ptes[i])
				break;
			if (pte_chain->parent_ptes[i] != parent_pte)
				continue;
861 862
			while (i + 1 < NR_PTE_CHAIN_ENTRIES
				&& pte_chain->parent_ptes[i + 1]) {
863 864 865 866 867
				pte_chain->parent_ptes[i]
					= pte_chain->parent_ptes[i + 1];
				++i;
			}
			pte_chain->parent_ptes[i] = NULL;
868 869
			if (i == 0) {
				hlist_del(&pte_chain->link);
870
				mmu_free_pte_chain(pte_chain);
871 872 873
				if (hlist_empty(&sp->parent_ptes)) {
					sp->multimapped = 0;
					sp->parent_pte = NULL;
874 875
				}
			}
876 877 878 879 880
			return;
		}
	BUG();
}

M
Marcelo Tosatti 已提交
881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905

static void mmu_parent_walk(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
			    mmu_parent_walk_fn fn)
{
	struct kvm_pte_chain *pte_chain;
	struct hlist_node *node;
	struct kvm_mmu_page *parent_sp;
	int i;

	if (!sp->multimapped && sp->parent_pte) {
		parent_sp = page_header(__pa(sp->parent_pte));
		fn(vcpu, parent_sp);
		mmu_parent_walk(vcpu, parent_sp, fn);
		return;
	}
	hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
		for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
			if (!pte_chain->parent_ptes[i])
				break;
			parent_sp = page_header(__pa(pte_chain->parent_ptes[i]));
			fn(vcpu, parent_sp);
			mmu_parent_walk(vcpu, parent_sp, fn);
		}
}

906 907 908 909 910 911
static void kvm_mmu_update_unsync_bitmap(u64 *spte)
{
	unsigned int index;
	struct kvm_mmu_page *sp = page_header(__pa(spte));

	index = spte - sp->spt;
912 913 914
	if (!__test_and_set_bit(index, sp->unsync_child_bitmap))
		sp->unsync_children++;
	WARN_ON(!sp->unsync_children);
915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
}

static void kvm_mmu_update_parents_unsync(struct kvm_mmu_page *sp)
{
	struct kvm_pte_chain *pte_chain;
	struct hlist_node *node;
	int i;

	if (!sp->parent_pte)
		return;

	if (!sp->multimapped) {
		kvm_mmu_update_unsync_bitmap(sp->parent_pte);
		return;
	}

	hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
		for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
			if (!pte_chain->parent_ptes[i])
				break;
			kvm_mmu_update_unsync_bitmap(pte_chain->parent_ptes[i]);
		}
}

static int unsync_walk_fn(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
{
	kvm_mmu_update_parents_unsync(sp);
	return 1;
}

static void kvm_mmu_mark_parents_unsync(struct kvm_vcpu *vcpu,
					struct kvm_mmu_page *sp)
{
	mmu_parent_walk(vcpu, sp, unsync_walk_fn);
	kvm_mmu_update_parents_unsync(sp);
}

952 953 954 955 956 957 958 959 960
static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *sp)
{
	int i;

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		sp->spt[i] = shadow_trap_nonpresent_pte;
}

961 962 963 964 965 966
static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
			       struct kvm_mmu_page *sp)
{
	return 1;
}

M
Marcelo Tosatti 已提交
967 968 969 970
static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
}

971 972 973 974 975 976 977 978 979 980
#define KVM_PAGE_ARRAY_NR 16

struct kvm_mmu_pages {
	struct mmu_page_and_offset {
		struct kvm_mmu_page *sp;
		unsigned int idx;
	} page[KVM_PAGE_ARRAY_NR];
	unsigned int nr;
};

981 982 983 984 985
#define for_each_unsync_children(bitmap, idx)		\
	for (idx = find_first_bit(bitmap, 512);		\
	     idx < 512;					\
	     idx = find_next_bit(bitmap, 512, idx+1))

986 987
int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
		   int idx)
988
{
989
	int i;
990

991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005
	if (sp->unsync)
		for (i=0; i < pvec->nr; i++)
			if (pvec->page[i].sp == sp)
				return 0;

	pvec->page[pvec->nr].sp = sp;
	pvec->page[pvec->nr].idx = idx;
	pvec->nr++;
	return (pvec->nr == KVM_PAGE_ARRAY_NR);
}

static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	int i, ret, nr_unsync_leaf = 0;
1006

1007
	for_each_unsync_children(sp->unsync_child_bitmap, i) {
1008 1009
		u64 ent = sp->spt[i];

1010
		if (is_shadow_present_pte(ent) && !is_large_pte(ent)) {
1011 1012 1013 1014
			struct kvm_mmu_page *child;
			child = page_header(ent & PT64_BASE_ADDR_MASK);

			if (child->unsync_children) {
1015 1016 1017 1018 1019 1020 1021 1022 1023
				if (mmu_pages_add(pvec, child, i))
					return -ENOSPC;

				ret = __mmu_unsync_walk(child, pvec);
				if (!ret)
					__clear_bit(i, sp->unsync_child_bitmap);
				else if (ret > 0)
					nr_unsync_leaf += ret;
				else
1024 1025 1026 1027
					return ret;
			}

			if (child->unsync) {
1028 1029 1030
				nr_unsync_leaf++;
				if (mmu_pages_add(pvec, child, i))
					return -ENOSPC;
1031 1032 1033 1034
			}
		}
	}

1035
	if (find_first_bit(sp->unsync_child_bitmap, 512) == 512)
1036 1037
		sp->unsync_children = 0;

1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
	return nr_unsync_leaf;
}

static int mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	if (!sp->unsync_children)
		return 0;

	mmu_pages_add(pvec, sp, 0);
	return __mmu_unsync_walk(sp, pvec);
1049 1050
}

1051
static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn)
1052 1053 1054
{
	unsigned index;
	struct hlist_head *bucket;
1055
	struct kvm_mmu_page *sp;
1056 1057
	struct hlist_node *node;

1058
	pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
1059
	index = kvm_page_table_hashfn(gfn);
1060
	bucket = &kvm->arch.mmu_page_hash[index];
1061
	hlist_for_each_entry(sp, node, bucket, hash_link)
1062 1063
		if (sp->gfn == gfn && !sp->role.metaphysical
		    && !sp->role.invalid) {
1064
			pgprintk("%s: found role %x\n",
1065
				 __func__, sp->role.word);
1066
			return sp;
1067 1068 1069 1070
		}
	return NULL;
}

1071 1072 1073 1074 1075 1076
static void kvm_unlink_unsync_global(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	list_del(&sp->oos_link);
	--kvm->stat.mmu_unsync_global;
}

1077 1078 1079 1080
static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	WARN_ON(!sp->unsync);
	sp->unsync = 0;
1081 1082
	if (sp->global)
		kvm_unlink_unsync_global(kvm, sp);
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
	--kvm->stat.mmu_unsync;
}

static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp);

static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
{
	if (sp->role.glevels != vcpu->arch.mmu.root_level) {
		kvm_mmu_zap_page(vcpu->kvm, sp);
		return 1;
	}

1095 1096
	if (rmap_write_protect(vcpu->kvm, sp->gfn))
		kvm_flush_remote_tlbs(vcpu->kvm);
1097
	kvm_unlink_unsync_page(vcpu->kvm, sp);
1098 1099 1100 1101 1102 1103 1104 1105 1106
	if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
		kvm_mmu_zap_page(vcpu->kvm, sp);
		return 1;
	}

	kvm_mmu_flush_tlb(vcpu);
	return 0;
}

1107 1108 1109
struct mmu_page_path {
	struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
	unsigned int idx[PT64_ROOT_LEVEL-1];
1110 1111
};

1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
#define for_each_sp(pvec, sp, parents, i)			\
		for (i = mmu_pages_next(&pvec, &parents, -1),	\
			sp = pvec.page[i].sp;			\
			i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});	\
			i = mmu_pages_next(&pvec, &parents, i))

int mmu_pages_next(struct kvm_mmu_pages *pvec, struct mmu_page_path *parents,
		   int i)
{
	int n;

	for (n = i+1; n < pvec->nr; n++) {
		struct kvm_mmu_page *sp = pvec->page[n].sp;

		if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
			parents->idx[0] = pvec->page[n].idx;
			return n;
		}

		parents->parent[sp->role.level-2] = sp;
		parents->idx[sp->role.level-1] = pvec->page[n].idx;
	}

	return n;
}

void mmu_pages_clear_parents(struct mmu_page_path *parents)
1139
{
1140 1141 1142 1143 1144
	struct kvm_mmu_page *sp;
	unsigned int level = 0;

	do {
		unsigned int idx = parents->idx[level];
1145

1146 1147 1148 1149 1150 1151 1152 1153 1154
		sp = parents->parent[level];
		if (!sp)
			return;

		--sp->unsync_children;
		WARN_ON((int)sp->unsync_children < 0);
		__clear_bit(idx, sp->unsync_child_bitmap);
		level++;
	} while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1155 1156
}

1157 1158 1159
static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
			       struct mmu_page_path *parents,
			       struct kvm_mmu_pages *pvec)
1160
{
1161 1162 1163
	parents->parent[parent->role.level-1] = NULL;
	pvec->nr = 0;
}
1164

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
static void mmu_sync_children(struct kvm_vcpu *vcpu,
			      struct kvm_mmu_page *parent)
{
	int i;
	struct kvm_mmu_page *sp;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;

	kvm_mmu_pages_init(parent, &parents, &pages);
	while (mmu_unsync_walk(parent, &pages)) {
1175 1176 1177 1178 1179 1180 1181 1182
		int protected = 0;

		for_each_sp(pages, sp, parents, i)
			protected |= rmap_write_protect(vcpu->kvm, sp->gfn);

		if (protected)
			kvm_flush_remote_tlbs(vcpu->kvm);

1183 1184 1185 1186
		for_each_sp(pages, sp, parents, i) {
			kvm_sync_page(vcpu, sp);
			mmu_pages_clear_parents(&parents);
		}
1187
		cond_resched_lock(&vcpu->kvm->mmu_lock);
1188 1189
		kvm_mmu_pages_init(parent, &parents, &pages);
	}
1190 1191
}

1192 1193 1194 1195 1196
static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
					     int metaphysical,
1197
					     unsigned access,
1198
					     u64 *parent_pte)
1199 1200 1201 1202 1203
{
	union kvm_mmu_page_role role;
	unsigned index;
	unsigned quadrant;
	struct hlist_head *bucket;
1204
	struct kvm_mmu_page *sp;
1205
	struct hlist_node *node, *tmp;
1206

1207
	role = vcpu->arch.mmu.base_role;
1208 1209
	role.level = level;
	role.metaphysical = metaphysical;
1210
	role.access = access;
1211
	if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
1212 1213 1214 1215
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
1216
	pgprintk("%s: looking gfn %lx role %x\n", __func__,
1217
		 gfn, role.word);
1218
	index = kvm_page_table_hashfn(gfn);
1219
	bucket = &vcpu->kvm->arch.mmu_page_hash[index];
1220 1221 1222 1223 1224 1225 1226 1227 1228
	hlist_for_each_entry_safe(sp, node, tmp, bucket, hash_link)
		if (sp->gfn == gfn) {
			if (sp->unsync)
				if (kvm_sync_page(vcpu, sp))
					continue;

			if (sp->role.word != role.word)
				continue;

1229
			mmu_page_add_parent_pte(vcpu, sp, parent_pte);
1230 1231 1232 1233
			if (sp->unsync_children) {
				set_bit(KVM_REQ_MMU_SYNC, &vcpu->requests);
				kvm_mmu_mark_parents_unsync(vcpu, sp);
			}
1234
			pgprintk("%s: found\n", __func__);
1235
			return sp;
1236
		}
A
Avi Kivity 已提交
1237
	++vcpu->kvm->stat.mmu_cache_miss;
1238 1239 1240
	sp = kvm_mmu_alloc_page(vcpu, parent_pte);
	if (!sp)
		return sp;
1241
	pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word);
1242 1243 1244
	sp->gfn = gfn;
	sp->role = role;
	hlist_add_head(&sp->hash_link, bucket);
1245
	if (!metaphysical) {
1246 1247
		if (rmap_write_protect(vcpu->kvm, gfn))
			kvm_flush_remote_tlbs(vcpu->kvm);
1248 1249
		account_shadowed(vcpu->kvm, gfn);
	}
1250 1251 1252 1253
	if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
		vcpu->arch.mmu.prefetch_page(vcpu, sp);
	else
		nonpaging_prefetch_page(vcpu, sp);
1254
	return sp;
1255 1256
}

A
Avi Kivity 已提交
1257
static int walk_shadow(struct kvm_shadow_walk *walker,
1258
		       struct kvm_vcpu *vcpu, u64 addr)
A
Avi Kivity 已提交
1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
{
	hpa_t shadow_addr;
	int level;
	int r;
	u64 *sptep;
	unsigned index;

	shadow_addr = vcpu->arch.mmu.root_hpa;
	level = vcpu->arch.mmu.shadow_root_level;
	if (level == PT32E_ROOT_LEVEL) {
		shadow_addr = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
		shadow_addr &= PT64_BASE_ADDR_MASK;
1271 1272
		if (!shadow_addr)
			return 1;
A
Avi Kivity 已提交
1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
		--level;
	}

	while (level >= PT_PAGE_TABLE_LEVEL) {
		index = SHADOW_PT_INDEX(addr, level);
		sptep = ((u64 *)__va(shadow_addr)) + index;
		r = walker->entry(walker, vcpu, addr, sptep, level);
		if (r)
			return r;
		shadow_addr = *sptep & PT64_BASE_ADDR_MASK;
		--level;
	}
	return 0;
}

1288
static void kvm_mmu_page_unlink_children(struct kvm *kvm,
1289
					 struct kvm_mmu_page *sp)
1290
{
1291 1292 1293 1294
	unsigned i;
	u64 *pt;
	u64 ent;

1295
	pt = sp->spt;
1296

1297
	if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1298
		for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1299
			if (is_shadow_present_pte(pt[i]))
1300
				rmap_remove(kvm, &pt[i]);
1301
			pt[i] = shadow_trap_nonpresent_pte;
1302 1303 1304 1305 1306 1307 1308
		}
		return;
	}

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
		ent = pt[i];

M
Marcelo Tosatti 已提交
1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
		if (is_shadow_present_pte(ent)) {
			if (!is_large_pte(ent)) {
				ent &= PT64_BASE_ADDR_MASK;
				mmu_page_remove_parent_pte(page_header(ent),
							   &pt[i]);
			} else {
				--kvm->stat.lpages;
				rmap_remove(kvm, &pt[i]);
			}
		}
1319
		pt[i] = shadow_trap_nonpresent_pte;
1320
	}
1321 1322
}

1323
static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
1324
{
1325
	mmu_page_remove_parent_pte(sp, parent_pte);
1326 1327
}

1328 1329 1330 1331 1332 1333
static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
{
	int i;

	for (i = 0; i < KVM_MAX_VCPUS; ++i)
		if (kvm->vcpus[i])
1334
			kvm->vcpus[i]->arch.last_pte_updated = NULL;
1335 1336
}

1337
static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
1338 1339 1340
{
	u64 *parent_pte;

1341 1342 1343
	while (sp->multimapped || sp->parent_pte) {
		if (!sp->multimapped)
			parent_pte = sp->parent_pte;
1344 1345 1346
		else {
			struct kvm_pte_chain *chain;

1347
			chain = container_of(sp->parent_ptes.first,
1348 1349 1350
					     struct kvm_pte_chain, link);
			parent_pte = chain->parent_ptes[0];
		}
1351
		BUG_ON(!parent_pte);
1352
		kvm_mmu_put_page(sp, parent_pte);
1353
		set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
1354
	}
1355 1356
}

1357 1358
static int mmu_zap_unsync_children(struct kvm *kvm,
				   struct kvm_mmu_page *parent)
1359
{
1360 1361 1362
	int i, zapped = 0;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
1363

1364
	if (parent->role.level == PT_PAGE_TABLE_LEVEL)
1365
		return 0;
1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379

	kvm_mmu_pages_init(parent, &parents, &pages);
	while (mmu_unsync_walk(parent, &pages)) {
		struct kvm_mmu_page *sp;

		for_each_sp(pages, sp, parents, i) {
			kvm_mmu_zap_page(kvm, sp);
			mmu_pages_clear_parents(&parents);
		}
		zapped += pages.nr;
		kvm_mmu_pages_init(parent, &parents, &pages);
	}

	return zapped;
1380 1381
}

1382
static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1383
{
1384
	int ret;
1385
	++kvm->stat.mmu_shadow_zapped;
1386
	ret = mmu_zap_unsync_children(kvm, sp);
1387
	kvm_mmu_page_unlink_children(kvm, sp);
1388
	kvm_mmu_unlink_parents(kvm, sp);
A
Avi Kivity 已提交
1389 1390 1391
	kvm_flush_remote_tlbs(kvm);
	if (!sp->role.invalid && !sp->role.metaphysical)
		unaccount_shadowed(kvm, sp->gfn);
1392 1393
	if (sp->unsync)
		kvm_unlink_unsync_page(kvm, sp);
1394 1395 1396
	if (!sp->root_count) {
		hlist_del(&sp->hash_link);
		kvm_mmu_free_page(kvm, sp);
1397 1398
	} else {
		sp->role.invalid = 1;
A
Avi Kivity 已提交
1399
		list_move(&sp->link, &kvm->arch.active_mmu_pages);
1400 1401
		kvm_reload_remote_mmus(kvm);
	}
1402
	kvm_mmu_reset_last_pte_updated(kvm);
1403
	return ret;
1404 1405
}

1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417
/*
 * Changing the number of mmu pages allocated to the vm
 * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
 */
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
{
	/*
	 * If we set the number of mmu pages to be smaller be than the
	 * number of actived pages , we must to free some mmu pages before we
	 * change the value
	 */

1418
	if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) >
1419
	    kvm_nr_mmu_pages) {
1420 1421
		int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages
				       - kvm->arch.n_free_mmu_pages;
1422 1423 1424 1425

		while (n_used_mmu_pages > kvm_nr_mmu_pages) {
			struct kvm_mmu_page *page;

1426
			page = container_of(kvm->arch.active_mmu_pages.prev,
1427 1428 1429 1430
					    struct kvm_mmu_page, link);
			kvm_mmu_zap_page(kvm, page);
			n_used_mmu_pages--;
		}
1431
		kvm->arch.n_free_mmu_pages = 0;
1432 1433
	}
	else
1434 1435
		kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages
					 - kvm->arch.n_alloc_mmu_pages;
1436

1437
	kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages;
1438 1439
}

1440
static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
1441 1442 1443
{
	unsigned index;
	struct hlist_head *bucket;
1444
	struct kvm_mmu_page *sp;
1445 1446 1447
	struct hlist_node *node, *n;
	int r;

1448
	pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
1449
	r = 0;
1450
	index = kvm_page_table_hashfn(gfn);
1451
	bucket = &kvm->arch.mmu_page_hash[index];
1452 1453
	hlist_for_each_entry_safe(sp, node, n, bucket, hash_link)
		if (sp->gfn == gfn && !sp->role.metaphysical) {
1454
			pgprintk("%s: gfn %lx role %x\n", __func__, gfn,
1455
				 sp->role.word);
1456
			r = 1;
1457 1458
			if (kvm_mmu_zap_page(kvm, sp))
				n = bucket->first;
1459 1460
		}
	return r;
1461 1462
}

1463
static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
1464
{
1465
	struct kvm_mmu_page *sp;
1466

1467
	while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) {
1468
		pgprintk("%s: zap %lx %x\n", __func__, gfn, sp->role.word);
1469
		kvm_mmu_zap_page(kvm, sp);
1470 1471 1472
	}
}

1473
static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
A
Avi Kivity 已提交
1474
{
1475
	int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn));
1476
	struct kvm_mmu_page *sp = page_header(__pa(pte));
A
Avi Kivity 已提交
1477

1478
	__set_bit(slot, sp->slot_bitmap);
A
Avi Kivity 已提交
1479 1480
}

1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494
static void mmu_convert_notrap(struct kvm_mmu_page *sp)
{
	int i;
	u64 *pt = sp->spt;

	if (shadow_trap_nonpresent_pte == shadow_notrap_nonpresent_pte)
		return;

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
		if (pt[i] == shadow_notrap_nonpresent_pte)
			set_shadow_pte(&pt[i], shadow_trap_nonpresent_pte);
	}
}

1495 1496
struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
{
1497 1498
	struct page *page;

1499
	gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
1500 1501 1502

	if (gpa == UNMAPPED_GVA)
		return NULL;
1503 1504 1505 1506

	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);

	return page;
1507 1508
}

1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
/*
 * The function is based on mtrr_type_lookup() in
 * arch/x86/kernel/cpu/mtrr/generic.c
 */
static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
			 u64 start, u64 end)
{
	int i;
	u64 base, mask;
	u8 prev_match, curr_match;
	int num_var_ranges = KVM_NR_VAR_MTRR;

	if (!mtrr_state->enabled)
		return 0xFF;

	/* Make end inclusive end, instead of exclusive */
	end--;

	/* Look in fixed ranges. Just return the type as per start */
	if (mtrr_state->have_fixed && (start < 0x100000)) {
		int idx;

		if (start < 0x80000) {
			idx = 0;
			idx += (start >> 16);
			return mtrr_state->fixed_ranges[idx];
		} else if (start < 0xC0000) {
			idx = 1 * 8;
			idx += ((start - 0x80000) >> 14);
			return mtrr_state->fixed_ranges[idx];
		} else if (start < 0x1000000) {
			idx = 3 * 8;
			idx += ((start - 0xC0000) >> 12);
			return mtrr_state->fixed_ranges[idx];
		}
	}

	/*
	 * Look in variable ranges
	 * Look of multiple ranges matching this address and pick type
	 * as per MTRR precedence
	 */
	if (!(mtrr_state->enabled & 2))
		return mtrr_state->def_type;

	prev_match = 0xFF;
	for (i = 0; i < num_var_ranges; ++i) {
		unsigned short start_state, end_state;

		if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
			continue;

		base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
		       (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
		mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
		       (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);

		start_state = ((start & mask) == (base & mask));
		end_state = ((end & mask) == (base & mask));
		if (start_state != end_state)
			return 0xFE;

		if ((start & mask) != (base & mask))
			continue;

		curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
		if (prev_match == 0xFF) {
			prev_match = curr_match;
			continue;
		}

		if (prev_match == MTRR_TYPE_UNCACHABLE ||
		    curr_match == MTRR_TYPE_UNCACHABLE)
			return MTRR_TYPE_UNCACHABLE;

		if ((prev_match == MTRR_TYPE_WRBACK &&
		     curr_match == MTRR_TYPE_WRTHROUGH) ||
		    (prev_match == MTRR_TYPE_WRTHROUGH &&
		     curr_match == MTRR_TYPE_WRBACK)) {
			prev_match = MTRR_TYPE_WRTHROUGH;
			curr_match = MTRR_TYPE_WRTHROUGH;
		}

		if (prev_match != curr_match)
			return MTRR_TYPE_UNCACHABLE;
	}

	if (prev_match != 0xFF)
		return prev_match;

	return mtrr_state->def_type;
}

static u8 get_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u8 mtrr;

	mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
			     (gfn << PAGE_SHIFT) + PAGE_SIZE);
	if (mtrr == 0xfe || mtrr == 0xff)
		mtrr = MTRR_TYPE_WRBACK;
	return mtrr;
}

1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630
static int kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
{
	unsigned index;
	struct hlist_head *bucket;
	struct kvm_mmu_page *s;
	struct hlist_node *node, *n;

	index = kvm_page_table_hashfn(sp->gfn);
	bucket = &vcpu->kvm->arch.mmu_page_hash[index];
	/* don't unsync if pagetable is shadowed with multiple roles */
	hlist_for_each_entry_safe(s, node, n, bucket, hash_link) {
		if (s->gfn != sp->gfn || s->role.metaphysical)
			continue;
		if (s->role.word != sp->role.word)
			return 1;
	}
	++vcpu->kvm->stat.mmu_unsync;
	sp->unsync = 1;
1631 1632 1633 1634 1635 1636 1637

	if (sp->global) {
		list_add(&sp->oos_link, &vcpu->kvm->arch.oos_global_pages);
		++vcpu->kvm->stat.mmu_unsync_global;
	} else
		kvm_mmu_mark_parents_unsync(vcpu, sp);

1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652
	mmu_convert_notrap(sp);
	return 0;
}

static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
				  bool can_unsync)
{
	struct kvm_mmu_page *shadow;

	shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn);
	if (shadow) {
		if (shadow->role.level != PT_PAGE_TABLE_LEVEL)
			return 1;
		if (shadow->unsync)
			return 0;
1653
		if (can_unsync && oos_shadow)
1654 1655 1656 1657 1658 1659
			return kvm_unsync_page(vcpu, shadow);
		return 1;
	}
	return 0;
}

M
Marcelo Tosatti 已提交
1660 1661 1662
static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
		    unsigned pte_access, int user_fault,
		    int write_fault, int dirty, int largepage,
1663
		    int global, gfn_t gfn, pfn_t pfn, bool speculative,
1664
		    bool can_unsync)
1665 1666
{
	u64 spte;
M
Marcelo Tosatti 已提交
1667
	int ret = 0;
S
Sheng Yang 已提交
1668
	u64 mt_mask = shadow_mt_mask;
1669 1670
	struct kvm_mmu_page *sp = page_header(__pa(shadow_pte));

1671 1672
	if (!(vcpu->arch.cr4 & X86_CR4_PGE))
		global = 0;
1673 1674 1675 1676 1677 1678 1679
	if (!global && sp->global) {
		sp->global = 0;
		if (sp->unsync) {
			kvm_unlink_unsync_global(vcpu->kvm, sp);
			kvm_mmu_mark_parents_unsync(vcpu, sp);
		}
	}
S
Sheng Yang 已提交
1680

1681 1682 1683 1684 1685
	/*
	 * We don't set the accessed bit, since we sometimes want to see
	 * whether the guest actually used the pte (in order to detect
	 * demand paging).
	 */
S
Sheng Yang 已提交
1686
	spte = shadow_base_present_pte | shadow_dirty_mask;
1687
	if (!speculative)
1688
		spte |= shadow_accessed_mask;
1689 1690
	if (!dirty)
		pte_access &= ~ACC_WRITE_MASK;
S
Sheng Yang 已提交
1691 1692 1693 1694
	if (pte_access & ACC_EXEC_MASK)
		spte |= shadow_x_mask;
	else
		spte |= shadow_nx_mask;
1695
	if (pte_access & ACC_USER_MASK)
S
Sheng Yang 已提交
1696
		spte |= shadow_user_mask;
M
Marcelo Tosatti 已提交
1697 1698
	if (largepage)
		spte |= PT_PAGE_SIZE_MASK;
S
Sheng Yang 已提交
1699
	if (mt_mask) {
1700 1701 1702 1703 1704 1705 1706
		if (!kvm_is_mmio_pfn(pfn)) {
			mt_mask = get_memory_type(vcpu, gfn) <<
				kvm_x86_ops->get_mt_mask_shift();
			mt_mask |= VMX_EPT_IGMT_BIT;
		} else
			mt_mask = MTRR_TYPE_UNCACHABLE <<
				kvm_x86_ops->get_mt_mask_shift();
S
Sheng Yang 已提交
1707 1708
		spte |= mt_mask;
	}
1709

1710
	spte |= (u64)pfn << PAGE_SHIFT;
1711 1712 1713 1714

	if ((pte_access & ACC_WRITE_MASK)
	    || (write_fault && !is_write_protection(vcpu) && !user_fault)) {

1715 1716 1717 1718 1719 1720
		if (largepage && has_wrprotected_page(vcpu->kvm, gfn)) {
			ret = 1;
			spte = shadow_trap_nonpresent_pte;
			goto set_pte;
		}

1721 1722
		spte |= PT_WRITABLE_MASK;

1723 1724 1725 1726 1727 1728 1729 1730 1731
		/*
		 * Optimization: for pte sync, if spte was writable the hash
		 * lookup is unnecessary (and expensive). Write protection
		 * is responsibility of mmu_get_page / kvm_sync_page.
		 * Same reasoning can be applied to dirty page accounting.
		 */
		if (!can_unsync && is_writeble_pte(*shadow_pte))
			goto set_pte;

1732
		if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
1733
			pgprintk("%s: found shadow page for %lx, marking ro\n",
1734
				 __func__, gfn);
M
Marcelo Tosatti 已提交
1735
			ret = 1;
1736
			pte_access &= ~ACC_WRITE_MASK;
1737
			if (is_writeble_pte(spte))
1738 1739 1740 1741 1742 1743 1744
				spte &= ~PT_WRITABLE_MASK;
		}
	}

	if (pte_access & ACC_WRITE_MASK)
		mark_page_dirty(vcpu->kvm, gfn);

1745
set_pte:
1746
	set_shadow_pte(shadow_pte, spte);
M
Marcelo Tosatti 已提交
1747 1748 1749 1750 1751 1752
	return ret;
}

static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
			 unsigned pt_access, unsigned pte_access,
			 int user_fault, int write_fault, int dirty,
1753 1754
			 int *ptwrite, int largepage, int global,
			 gfn_t gfn, pfn_t pfn, bool speculative)
M
Marcelo Tosatti 已提交
1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786
{
	int was_rmapped = 0;
	int was_writeble = is_writeble_pte(*shadow_pte);

	pgprintk("%s: spte %llx access %x write_fault %d"
		 " user_fault %d gfn %lx\n",
		 __func__, *shadow_pte, pt_access,
		 write_fault, user_fault, gfn);

	if (is_rmap_pte(*shadow_pte)) {
		/*
		 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
		 * the parent of the now unreachable PTE.
		 */
		if (largepage && !is_large_pte(*shadow_pte)) {
			struct kvm_mmu_page *child;
			u64 pte = *shadow_pte;

			child = page_header(pte & PT64_BASE_ADDR_MASK);
			mmu_page_remove_parent_pte(child, shadow_pte);
		} else if (pfn != spte_to_pfn(*shadow_pte)) {
			pgprintk("hfn old %lx new %lx\n",
				 spte_to_pfn(*shadow_pte), pfn);
			rmap_remove(vcpu->kvm, shadow_pte);
		} else {
			if (largepage)
				was_rmapped = is_large_pte(*shadow_pte);
			else
				was_rmapped = 1;
		}
	}
	if (set_spte(vcpu, shadow_pte, pte_access, user_fault, write_fault,
1787
		      dirty, largepage, global, gfn, pfn, speculative, true)) {
M
Marcelo Tosatti 已提交
1788 1789
		if (write_fault)
			*ptwrite = 1;
1790 1791
		kvm_x86_ops->tlb_flush(vcpu);
	}
M
Marcelo Tosatti 已提交
1792 1793 1794 1795 1796 1797 1798

	pgprintk("%s: setting spte %llx\n", __func__, *shadow_pte);
	pgprintk("instantiating %s PTE (%s) at %ld (%llx) addr %p\n",
		 is_large_pte(*shadow_pte)? "2MB" : "4kB",
		 is_present_pte(*shadow_pte)?"RW":"R", gfn,
		 *shadow_pte, shadow_pte);
	if (!was_rmapped && is_large_pte(*shadow_pte))
M
Marcelo Tosatti 已提交
1799 1800
		++vcpu->kvm->stat.lpages;

1801 1802
	page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
	if (!was_rmapped) {
M
Marcelo Tosatti 已提交
1803
		rmap_add(vcpu, shadow_pte, gfn, largepage);
1804
		if (!is_rmap_pte(*shadow_pte))
1805
			kvm_release_pfn_clean(pfn);
1806 1807
	} else {
		if (was_writeble)
1808
			kvm_release_pfn_dirty(pfn);
1809
		else
1810
			kvm_release_pfn_clean(pfn);
1811
	}
1812
	if (speculative) {
1813
		vcpu->arch.last_pte_updated = shadow_pte;
1814 1815
		vcpu->arch.last_pte_gfn = gfn;
	}
1816 1817
}

A
Avi Kivity 已提交
1818 1819 1820 1821
static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
{
}

1822 1823 1824 1825 1826 1827 1828
struct direct_shadow_walk {
	struct kvm_shadow_walk walker;
	pfn_t pfn;
	int write;
	int largepage;
	int pt_write;
};
A
Avi Kivity 已提交
1829

1830 1831
static int direct_map_entry(struct kvm_shadow_walk *_walk,
			    struct kvm_vcpu *vcpu,
1832
			    u64 addr, u64 *sptep, int level)
1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843
{
	struct direct_shadow_walk *walk =
		container_of(_walk, struct direct_shadow_walk, walker);
	struct kvm_mmu_page *sp;
	gfn_t pseudo_gfn;
	gfn_t gfn = addr >> PAGE_SHIFT;

	if (level == PT_PAGE_TABLE_LEVEL
	    || (walk->largepage && level == PT_DIRECTORY_LEVEL)) {
		mmu_set_spte(vcpu, sptep, ACC_ALL, ACC_ALL,
			     0, walk->write, 1, &walk->pt_write,
1844
			     walk->largepage, 0, gfn, walk->pfn, false);
1845
		++vcpu->stat.pf_fixed;
1846 1847
		return 1;
	}
A
Avi Kivity 已提交
1848

1849 1850
	if (*sptep == shadow_trap_nonpresent_pte) {
		pseudo_gfn = (addr & PT64_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT;
1851
		sp = kvm_mmu_get_page(vcpu, pseudo_gfn, (gva_t)addr, level - 1,
1852 1853 1854 1855 1856
				      1, ACC_ALL, sptep);
		if (!sp) {
			pgprintk("nonpaging_map: ENOMEM\n");
			kvm_release_pfn_clean(walk->pfn);
			return -ENOMEM;
A
Avi Kivity 已提交
1857 1858
		}

1859 1860 1861 1862
		set_shadow_pte(sptep,
			       __pa(sp->spt)
			       | PT_PRESENT_MASK | PT_WRITABLE_MASK
			       | shadow_user_mask | shadow_x_mask);
A
Avi Kivity 已提交
1863
	}
1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
	return 0;
}

static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
			int largepage, gfn_t gfn, pfn_t pfn)
{
	int r;
	struct direct_shadow_walk walker = {
		.walker = { .entry = direct_map_entry, },
		.pfn = pfn,
		.largepage = largepage,
		.write = write,
		.pt_write = 0,
	};

1879
	r = walk_shadow(&walker.walker, vcpu, gfn << PAGE_SHIFT);
1880 1881 1882
	if (r < 0)
		return r;
	return walker.pt_write;
A
Avi Kivity 已提交
1883 1884
}

1885 1886 1887
static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
{
	int r;
M
Marcelo Tosatti 已提交
1888
	int largepage = 0;
1889
	pfn_t pfn;
1890
	unsigned long mmu_seq;
1891

M
Marcelo Tosatti 已提交
1892 1893 1894 1895 1896
	if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
		gfn &= ~(KVM_PAGES_PER_HPAGE-1);
		largepage = 1;
	}

1897
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
1898
	smp_rmb();
1899
	pfn = gfn_to_pfn(vcpu->kvm, gfn);
1900

1901
	/* mmio */
1902 1903
	if (is_error_pfn(pfn)) {
		kvm_release_pfn_clean(pfn);
1904 1905 1906
		return 1;
	}

1907
	spin_lock(&vcpu->kvm->mmu_lock);
1908 1909
	if (mmu_notifier_retry(vcpu, mmu_seq))
		goto out_unlock;
1910
	kvm_mmu_free_some_pages(vcpu);
1911
	r = __direct_map(vcpu, v, write, largepage, gfn, pfn);
1912 1913 1914
	spin_unlock(&vcpu->kvm->mmu_lock);


1915
	return r;
1916 1917 1918 1919 1920

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
1921 1922 1923
}


1924 1925 1926
static void mmu_free_roots(struct kvm_vcpu *vcpu)
{
	int i;
1927
	struct kvm_mmu_page *sp;
1928

1929
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
A
Avi Kivity 已提交
1930
		return;
1931
	spin_lock(&vcpu->kvm->mmu_lock);
1932 1933
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		hpa_t root = vcpu->arch.mmu.root_hpa;
1934

1935 1936
		sp = page_header(root);
		--sp->root_count;
1937 1938
		if (!sp->root_count && sp->role.invalid)
			kvm_mmu_zap_page(vcpu->kvm, sp);
1939
		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1940
		spin_unlock(&vcpu->kvm->mmu_lock);
1941 1942 1943
		return;
	}
	for (i = 0; i < 4; ++i) {
1944
		hpa_t root = vcpu->arch.mmu.pae_root[i];
1945

A
Avi Kivity 已提交
1946 1947
		if (root) {
			root &= PT64_BASE_ADDR_MASK;
1948 1949
			sp = page_header(root);
			--sp->root_count;
1950 1951
			if (!sp->root_count && sp->role.invalid)
				kvm_mmu_zap_page(vcpu->kvm, sp);
A
Avi Kivity 已提交
1952
		}
1953
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
1954
	}
1955
	spin_unlock(&vcpu->kvm->mmu_lock);
1956
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1957 1958 1959 1960 1961
}

static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
{
	int i;
1962
	gfn_t root_gfn;
1963
	struct kvm_mmu_page *sp;
1964
	int metaphysical = 0;
1965

1966
	root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
1967

1968 1969
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		hpa_t root = vcpu->arch.mmu.root_hpa;
1970 1971

		ASSERT(!VALID_PAGE(root));
1972 1973
		if (tdp_enabled)
			metaphysical = 1;
1974
		sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
1975 1976
				      PT64_ROOT_LEVEL, metaphysical,
				      ACC_ALL, NULL);
1977 1978
		root = __pa(sp->spt);
		++sp->root_count;
1979
		vcpu->arch.mmu.root_hpa = root;
1980 1981
		return;
	}
1982 1983 1984
	metaphysical = !is_paging(vcpu);
	if (tdp_enabled)
		metaphysical = 1;
1985
	for (i = 0; i < 4; ++i) {
1986
		hpa_t root = vcpu->arch.mmu.pae_root[i];
1987 1988

		ASSERT(!VALID_PAGE(root));
1989 1990 1991
		if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
			if (!is_present_pte(vcpu->arch.pdptrs[i])) {
				vcpu->arch.mmu.pae_root[i] = 0;
A
Avi Kivity 已提交
1992 1993
				continue;
			}
1994 1995
			root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
		} else if (vcpu->arch.mmu.root_level == 0)
1996
			root_gfn = 0;
1997
		sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
1998
				      PT32_ROOT_LEVEL, metaphysical,
1999
				      ACC_ALL, NULL);
2000 2001
		root = __pa(sp->spt);
		++sp->root_count;
2002
		vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
2003
	}
2004
	vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
2005 2006
}

2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030
static void mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	int i;
	struct kvm_mmu_page *sp;

	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return;
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		hpa_t root = vcpu->arch.mmu.root_hpa;
		sp = page_header(root);
		mmu_sync_children(vcpu, sp);
		return;
	}
	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->arch.mmu.pae_root[i];

		if (root) {
			root &= PT64_BASE_ADDR_MASK;
			sp = page_header(root);
			mmu_sync_children(vcpu, sp);
		}
	}
}

2031 2032 2033 2034 2035 2036 2037 2038 2039
static void mmu_sync_global(struct kvm_vcpu *vcpu)
{
	struct kvm *kvm = vcpu->kvm;
	struct kvm_mmu_page *sp, *n;

	list_for_each_entry_safe(sp, n, &kvm->arch.oos_global_pages, oos_link)
		kvm_sync_page(vcpu, sp);
}

2040 2041 2042 2043
void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	spin_lock(&vcpu->kvm->mmu_lock);
	mmu_sync_roots(vcpu);
2044 2045 2046 2047 2048 2049 2050
	spin_unlock(&vcpu->kvm->mmu_lock);
}

void kvm_mmu_sync_global(struct kvm_vcpu *vcpu)
{
	spin_lock(&vcpu->kvm->mmu_lock);
	mmu_sync_global(vcpu);
2051 2052 2053
	spin_unlock(&vcpu->kvm->mmu_lock);
}

A
Avi Kivity 已提交
2054 2055 2056 2057 2058 2059
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
{
	return vaddr;
}

static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
A
Avi Kivity 已提交
2060
				u32 error_code)
A
Avi Kivity 已提交
2061
{
2062
	gfn_t gfn;
2063
	int r;
A
Avi Kivity 已提交
2064

2065
	pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
2066 2067 2068
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;
2069

A
Avi Kivity 已提交
2070
	ASSERT(vcpu);
2071
	ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
2072

2073
	gfn = gva >> PAGE_SHIFT;
A
Avi Kivity 已提交
2074

2075 2076
	return nonpaging_map(vcpu, gva & PAGE_MASK,
			     error_code & PFERR_WRITE_MASK, gfn);
A
Avi Kivity 已提交
2077 2078
}

2079 2080 2081
static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
				u32 error_code)
{
2082
	pfn_t pfn;
2083
	int r;
M
Marcelo Tosatti 已提交
2084 2085
	int largepage = 0;
	gfn_t gfn = gpa >> PAGE_SHIFT;
2086
	unsigned long mmu_seq;
2087 2088 2089 2090 2091 2092 2093 2094

	ASSERT(vcpu);
	ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));

	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

M
Marcelo Tosatti 已提交
2095 2096 2097 2098
	if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
		gfn &= ~(KVM_PAGES_PER_HPAGE-1);
		largepage = 1;
	}
2099
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
2100
	smp_rmb();
2101 2102 2103
	pfn = gfn_to_pfn(vcpu->kvm, gfn);
	if (is_error_pfn(pfn)) {
		kvm_release_pfn_clean(pfn);
2104 2105 2106
		return 1;
	}
	spin_lock(&vcpu->kvm->mmu_lock);
2107 2108
	if (mmu_notifier_retry(vcpu, mmu_seq))
		goto out_unlock;
2109 2110
	kvm_mmu_free_some_pages(vcpu);
	r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
2111
			 largepage, gfn, pfn);
2112 2113 2114
	spin_unlock(&vcpu->kvm->mmu_lock);

	return r;
2115 2116 2117 2118 2119

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
2120 2121
}

A
Avi Kivity 已提交
2122 2123
static void nonpaging_free(struct kvm_vcpu *vcpu)
{
2124
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
2125 2126 2127 2128
}

static int nonpaging_init_context(struct kvm_vcpu *vcpu)
{
2129
	struct kvm_mmu *context = &vcpu->arch.mmu;
A
Avi Kivity 已提交
2130 2131 2132 2133 2134

	context->new_cr3 = nonpaging_new_cr3;
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
	context->free = nonpaging_free;
2135
	context->prefetch_page = nonpaging_prefetch_page;
2136
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
2137
	context->invlpg = nonpaging_invlpg;
2138
	context->root_level = 0;
A
Avi Kivity 已提交
2139
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
2140
	context->root_hpa = INVALID_PAGE;
A
Avi Kivity 已提交
2141 2142 2143
	return 0;
}

2144
void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2145
{
A
Avi Kivity 已提交
2146
	++vcpu->stat.tlb_flush;
2147
	kvm_x86_ops->tlb_flush(vcpu);
A
Avi Kivity 已提交
2148 2149 2150 2151
}

static void paging_new_cr3(struct kvm_vcpu *vcpu)
{
2152
	pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
2153
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
2154 2155 2156 2157 2158 2159
}

static void inject_page_fault(struct kvm_vcpu *vcpu,
			      u64 addr,
			      u32 err_code)
{
2160
	kvm_inject_page_fault(vcpu, addr, err_code);
A
Avi Kivity 已提交
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175
}

static void paging_free(struct kvm_vcpu *vcpu)
{
	nonpaging_free(vcpu);
}

#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

2176
static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
A
Avi Kivity 已提交
2177
{
2178
	struct kvm_mmu *context = &vcpu->arch.mmu;
A
Avi Kivity 已提交
2179 2180 2181 2182 2183

	ASSERT(is_pae(vcpu));
	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
2184
	context->prefetch_page = paging64_prefetch_page;
2185
	context->sync_page = paging64_sync_page;
M
Marcelo Tosatti 已提交
2186
	context->invlpg = paging64_invlpg;
A
Avi Kivity 已提交
2187
	context->free = paging_free;
2188 2189
	context->root_level = level;
	context->shadow_root_level = level;
A
Avi Kivity 已提交
2190
	context->root_hpa = INVALID_PAGE;
A
Avi Kivity 已提交
2191 2192 2193
	return 0;
}

2194 2195 2196 2197 2198
static int paging64_init_context(struct kvm_vcpu *vcpu)
{
	return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
}

A
Avi Kivity 已提交
2199 2200
static int paging32_init_context(struct kvm_vcpu *vcpu)
{
2201
	struct kvm_mmu *context = &vcpu->arch.mmu;
A
Avi Kivity 已提交
2202 2203 2204 2205 2206

	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
	context->free = paging_free;
2207
	context->prefetch_page = paging32_prefetch_page;
2208
	context->sync_page = paging32_sync_page;
M
Marcelo Tosatti 已提交
2209
	context->invlpg = paging32_invlpg;
A
Avi Kivity 已提交
2210 2211
	context->root_level = PT32_ROOT_LEVEL;
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
2212
	context->root_hpa = INVALID_PAGE;
A
Avi Kivity 已提交
2213 2214 2215 2216 2217
	return 0;
}

static int paging32E_init_context(struct kvm_vcpu *vcpu)
{
2218
	return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
A
Avi Kivity 已提交
2219 2220
}

2221 2222 2223 2224 2225 2226 2227 2228
static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu *context = &vcpu->arch.mmu;

	context->new_cr3 = nonpaging_new_cr3;
	context->page_fault = tdp_page_fault;
	context->free = nonpaging_free;
	context->prefetch_page = nonpaging_prefetch_page;
2229
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
2230
	context->invlpg = nonpaging_invlpg;
2231
	context->shadow_root_level = kvm_x86_ops->get_tdp_level();
2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251
	context->root_hpa = INVALID_PAGE;

	if (!is_paging(vcpu)) {
		context->gva_to_gpa = nonpaging_gva_to_gpa;
		context->root_level = 0;
	} else if (is_long_mode(vcpu)) {
		context->gva_to_gpa = paging64_gva_to_gpa;
		context->root_level = PT64_ROOT_LEVEL;
	} else if (is_pae(vcpu)) {
		context->gva_to_gpa = paging64_gva_to_gpa;
		context->root_level = PT32E_ROOT_LEVEL;
	} else {
		context->gva_to_gpa = paging32_gva_to_gpa;
		context->root_level = PT32_ROOT_LEVEL;
	}

	return 0;
}

static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2252
{
2253 2254
	int r;

A
Avi Kivity 已提交
2255
	ASSERT(vcpu);
2256
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
2257 2258

	if (!is_paging(vcpu))
2259
		r = nonpaging_init_context(vcpu);
A
Avi Kivity 已提交
2260
	else if (is_long_mode(vcpu))
2261
		r = paging64_init_context(vcpu);
A
Avi Kivity 已提交
2262
	else if (is_pae(vcpu))
2263
		r = paging32E_init_context(vcpu);
A
Avi Kivity 已提交
2264
	else
2265 2266 2267 2268 2269
		r = paging32_init_context(vcpu);

	vcpu->arch.mmu.base_role.glevels = vcpu->arch.mmu.root_level;

	return r;
A
Avi Kivity 已提交
2270 2271
}

2272 2273
static int init_kvm_mmu(struct kvm_vcpu *vcpu)
{
2274 2275
	vcpu->arch.update_pte.pfn = bad_pfn;

2276 2277 2278 2279 2280 2281
	if (tdp_enabled)
		return init_kvm_tdp_mmu(vcpu);
	else
		return init_kvm_softmmu(vcpu);
}

A
Avi Kivity 已提交
2282 2283 2284
static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
2285 2286 2287
	if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) {
		vcpu->arch.mmu.free(vcpu);
		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
A
Avi Kivity 已提交
2288 2289 2290 2291
	}
}

int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2292 2293 2294 2295
{
	destroy_kvm_mmu(vcpu);
	return init_kvm_mmu(vcpu);
}
2296
EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
A
Avi Kivity 已提交
2297 2298

int kvm_mmu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2299
{
2300 2301
	int r;

2302
	r = mmu_topup_memory_caches(vcpu);
A
Avi Kivity 已提交
2303 2304
	if (r)
		goto out;
2305
	spin_lock(&vcpu->kvm->mmu_lock);
2306
	kvm_mmu_free_some_pages(vcpu);
A
Avi Kivity 已提交
2307
	mmu_alloc_roots(vcpu);
2308
	mmu_sync_roots(vcpu);
2309
	spin_unlock(&vcpu->kvm->mmu_lock);
2310
	kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
A
Avi Kivity 已提交
2311
	kvm_mmu_flush_tlb(vcpu);
2312 2313
out:
	return r;
A
Avi Kivity 已提交
2314
}
A
Avi Kivity 已提交
2315 2316 2317 2318 2319 2320
EXPORT_SYMBOL_GPL(kvm_mmu_load);

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
	mmu_free_roots(vcpu);
}
A
Avi Kivity 已提交
2321

2322
static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
2323
				  struct kvm_mmu_page *sp,
2324 2325 2326 2327 2328 2329
				  u64 *spte)
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
2330
	if (is_shadow_present_pte(pte)) {
M
Marcelo Tosatti 已提交
2331 2332
		if (sp->role.level == PT_PAGE_TABLE_LEVEL ||
		    is_large_pte(pte))
2333
			rmap_remove(vcpu->kvm, spte);
2334 2335
		else {
			child = page_header(pte & PT64_BASE_ADDR_MASK);
2336
			mmu_page_remove_parent_pte(child, spte);
2337 2338
		}
	}
2339
	set_shadow_pte(spte, shadow_trap_nonpresent_pte);
M
Marcelo Tosatti 已提交
2340 2341
	if (is_large_pte(pte))
		--vcpu->kvm->stat.lpages;
2342 2343
}

2344
static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
2345
				  struct kvm_mmu_page *sp,
2346
				  u64 *spte,
2347
				  const void *new)
2348
{
2349 2350 2351 2352 2353 2354 2355
	if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
		if (!vcpu->arch.update_pte.largepage ||
		    sp->role.glevels == PT32_ROOT_LEVEL) {
			++vcpu->kvm->stat.mmu_pde_zapped;
			return;
		}
        }
2356

A
Avi Kivity 已提交
2357
	++vcpu->kvm->stat.mmu_pte_updated;
2358
	if (sp->role.glevels == PT32_ROOT_LEVEL)
2359
		paging32_update_pte(vcpu, sp, spte, new);
2360
	else
2361
		paging64_update_pte(vcpu, sp, spte, new);
2362 2363
}

2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384
static bool need_remote_flush(u64 old, u64 new)
{
	if (!is_shadow_present_pte(old))
		return false;
	if (!is_shadow_present_pte(new))
		return true;
	if ((old ^ new) & PT64_BASE_ADDR_MASK)
		return true;
	old ^= PT64_NX_MASK;
	new ^= PT64_NX_MASK;
	return (old & ~new & PT64_PERM_MASK) != 0;
}

static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new)
{
	if (need_remote_flush(old, new))
		kvm_flush_remote_tlbs(vcpu->kvm);
	else
		kvm_mmu_flush_tlb(vcpu);
}

2385 2386
static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
{
2387
	u64 *spte = vcpu->arch.last_pte_updated;
2388

S
Sheng Yang 已提交
2389
	return !!(spte && (*spte & shadow_accessed_mask));
2390 2391
}

2392 2393 2394 2395 2396 2397
static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
					  const u8 *new, int bytes)
{
	gfn_t gfn;
	int r;
	u64 gpte = 0;
2398
	pfn_t pfn;
2399

M
Marcelo Tosatti 已提交
2400 2401
	vcpu->arch.update_pte.largepage = 0;

2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427
	if (bytes != 4 && bytes != 8)
		return;

	/*
	 * Assume that the pte write on a page table of the same type
	 * as the current vcpu paging mode.  This is nearly always true
	 * (might be false while changing modes).  Note it is verified later
	 * by update_pte().
	 */
	if (is_pae(vcpu)) {
		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */
		if ((bytes == 4) && (gpa % 4 == 0)) {
			r = kvm_read_guest(vcpu->kvm, gpa & ~(u64)7, &gpte, 8);
			if (r)
				return;
			memcpy((void *)&gpte + (gpa % 8), new, 4);
		} else if ((bytes == 8) && (gpa % 8 == 0)) {
			memcpy((void *)&gpte, new, 8);
		}
	} else {
		if ((bytes == 4) && (gpa % 4 == 0))
			memcpy((void *)&gpte, new, 4);
	}
	if (!is_present_pte(gpte))
		return;
	gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
2428

M
Marcelo Tosatti 已提交
2429 2430 2431 2432
	if (is_large_pte(gpte) && is_largepage_backed(vcpu, gfn)) {
		gfn &= ~(KVM_PAGES_PER_HPAGE-1);
		vcpu->arch.update_pte.largepage = 1;
	}
2433
	vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
2434
	smp_rmb();
2435
	pfn = gfn_to_pfn(vcpu->kvm, gfn);
2436

2437 2438
	if (is_error_pfn(pfn)) {
		kvm_release_pfn_clean(pfn);
2439 2440
		return;
	}
2441
	vcpu->arch.update_pte.gfn = gfn;
2442
	vcpu->arch.update_pte.pfn = pfn;
2443 2444
}

2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456
static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u64 *spte = vcpu->arch.last_pte_updated;

	if (spte
	    && vcpu->arch.last_pte_gfn == gfn
	    && shadow_accessed_mask
	    && !(*spte & shadow_accessed_mask)
	    && is_shadow_present_pte(*spte))
		set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
}

2457
void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
2458 2459
		       const u8 *new, int bytes,
		       bool guest_initiated)
2460
{
2461
	gfn_t gfn = gpa >> PAGE_SHIFT;
2462
	struct kvm_mmu_page *sp;
2463
	struct hlist_node *node, *n;
2464 2465
	struct hlist_head *bucket;
	unsigned index;
2466
	u64 entry, gentry;
2467 2468
	u64 *spte;
	unsigned offset = offset_in_page(gpa);
2469
	unsigned pte_size;
2470
	unsigned page_offset;
2471
	unsigned misaligned;
2472
	unsigned quadrant;
2473
	int level;
2474
	int flooded = 0;
2475
	int npte;
2476
	int r;
2477

2478
	pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
2479
	mmu_guess_page_from_pte_write(vcpu, gpa, new, bytes);
2480
	spin_lock(&vcpu->kvm->mmu_lock);
2481
	kvm_mmu_access_page(vcpu, gfn);
2482
	kvm_mmu_free_some_pages(vcpu);
A
Avi Kivity 已提交
2483
	++vcpu->kvm->stat.mmu_pte_write;
2484
	kvm_mmu_audit(vcpu, "pre pte write");
2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495
	if (guest_initiated) {
		if (gfn == vcpu->arch.last_pt_write_gfn
		    && !last_updated_pte_accessed(vcpu)) {
			++vcpu->arch.last_pt_write_count;
			if (vcpu->arch.last_pt_write_count >= 3)
				flooded = 1;
		} else {
			vcpu->arch.last_pt_write_gfn = gfn;
			vcpu->arch.last_pt_write_count = 1;
			vcpu->arch.last_pte_updated = NULL;
		}
2496
	}
2497
	index = kvm_page_table_hashfn(gfn);
2498
	bucket = &vcpu->kvm->arch.mmu_page_hash[index];
2499
	hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) {
A
Avi Kivity 已提交
2500
		if (sp->gfn != gfn || sp->role.metaphysical || sp->role.invalid)
2501
			continue;
2502
		pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
2503
		misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
2504
		misaligned |= bytes < 4;
2505
		if (misaligned || flooded) {
2506 2507 2508 2509
			/*
			 * Misaligned accesses are too much trouble to fix
			 * up; also, they usually indicate a page is not used
			 * as a page table.
2510 2511 2512 2513 2514
			 *
			 * If we're seeing too many writes to a page,
			 * it may no longer be a page table, or we may be
			 * forking, in which case it is better to unmap the
			 * page.
2515 2516
			 */
			pgprintk("misaligned: gpa %llx bytes %d role %x\n",
2517
				 gpa, bytes, sp->role.word);
2518 2519
			if (kvm_mmu_zap_page(vcpu->kvm, sp))
				n = bucket->first;
A
Avi Kivity 已提交
2520
			++vcpu->kvm->stat.mmu_flooded;
2521 2522
			continue;
		}
2523
		page_offset = offset;
2524
		level = sp->role.level;
2525
		npte = 1;
2526
		if (sp->role.glevels == PT32_ROOT_LEVEL) {
2527 2528 2529 2530 2531 2532 2533
			page_offset <<= 1;	/* 32->64 */
			/*
			 * A 32-bit pde maps 4MB while the shadow pdes map
			 * only 2MB.  So we need to double the offset again
			 * and zap two pdes instead of one.
			 */
			if (level == PT32_ROOT_LEVEL) {
2534
				page_offset &= ~7; /* kill rounding error */
2535 2536 2537
				page_offset <<= 1;
				npte = 2;
			}
2538
			quadrant = page_offset >> PAGE_SHIFT;
2539
			page_offset &= ~PAGE_MASK;
2540
			if (quadrant != sp->role.quadrant)
2541
				continue;
2542
		}
2543
		spte = &sp->spt[page_offset / sizeof(*spte)];
2544 2545 2546 2547 2548 2549 2550 2551 2552
		if ((gpa & (pte_size - 1)) || (bytes < pte_size)) {
			gentry = 0;
			r = kvm_read_guest_atomic(vcpu->kvm,
						  gpa & ~(u64)(pte_size - 1),
						  &gentry, pte_size);
			new = (const void *)&gentry;
			if (r < 0)
				new = NULL;
		}
2553
		while (npte--) {
2554
			entry = *spte;
2555
			mmu_pte_write_zap_pte(vcpu, sp, spte);
2556 2557
			if (new)
				mmu_pte_write_new_pte(vcpu, sp, spte, new);
2558
			mmu_pte_write_flush_tlb(vcpu, entry, *spte);
2559
			++spte;
2560 2561
		}
	}
2562
	kvm_mmu_audit(vcpu, "post pte write");
2563
	spin_unlock(&vcpu->kvm->mmu_lock);
2564 2565 2566
	if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
		kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
		vcpu->arch.update_pte.pfn = bad_pfn;
2567
	}
2568 2569
}

2570 2571
int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
2572 2573
	gpa_t gpa;
	int r;
2574

2575 2576
	gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);

2577
	spin_lock(&vcpu->kvm->mmu_lock);
2578
	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
2579
	spin_unlock(&vcpu->kvm->mmu_lock);
2580
	return r;
2581
}
2582
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
2583

2584
void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2585
{
2586
	while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
2587
		struct kvm_mmu_page *sp;
A
Avi Kivity 已提交
2588

2589
		sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
2590 2591
				  struct kvm_mmu_page, link);
		kvm_mmu_zap_page(vcpu->kvm, sp);
A
Avi Kivity 已提交
2592
		++vcpu->kvm->stat.mmu_recycled;
A
Avi Kivity 已提交
2593 2594 2595
	}
}

2596 2597 2598 2599 2600
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
{
	int r;
	enum emulation_result er;

2601
	r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
2602 2603 2604 2605 2606 2607 2608 2609
	if (r < 0)
		goto out;

	if (!r) {
		r = 1;
		goto out;
	}

2610 2611 2612 2613
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		goto out;

2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632
	er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0);

	switch (er) {
	case EMULATE_DONE:
		return 1;
	case EMULATE_DO_MMIO:
		++vcpu->stat.mmio_exits;
		return 0;
	case EMULATE_FAIL:
		kvm_report_emulation_failure(vcpu, "pagetable");
		return 1;
	default:
		BUG();
	}
out:
	return r;
}
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);

M
Marcelo Tosatti 已提交
2633 2634 2635 2636 2637 2638 2639 2640
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
	vcpu->arch.mmu.invlpg(vcpu, gva);
	kvm_mmu_flush_tlb(vcpu);
	++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);

2641 2642 2643 2644 2645 2646
void kvm_enable_tdp(void)
{
	tdp_enabled = true;
}
EXPORT_SYMBOL_GPL(kvm_enable_tdp);

2647 2648 2649 2650 2651 2652
void kvm_disable_tdp(void)
{
	tdp_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_tdp);

A
Avi Kivity 已提交
2653 2654
static void free_mmu_pages(struct kvm_vcpu *vcpu)
{
2655
	struct kvm_mmu_page *sp;
A
Avi Kivity 已提交
2656

2657 2658
	while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
		sp = container_of(vcpu->kvm->arch.active_mmu_pages.next,
2659 2660
				  struct kvm_mmu_page, link);
		kvm_mmu_zap_page(vcpu->kvm, sp);
2661
		cond_resched();
2662
	}
2663
	free_page((unsigned long)vcpu->arch.mmu.pae_root);
A
Avi Kivity 已提交
2664 2665 2666 2667
}

static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
{
2668
	struct page *page;
A
Avi Kivity 已提交
2669 2670 2671 2672
	int i;

	ASSERT(vcpu);

2673 2674 2675
	if (vcpu->kvm->arch.n_requested_mmu_pages)
		vcpu->kvm->arch.n_free_mmu_pages =
					vcpu->kvm->arch.n_requested_mmu_pages;
2676
	else
2677 2678
		vcpu->kvm->arch.n_free_mmu_pages =
					vcpu->kvm->arch.n_alloc_mmu_pages;
2679 2680 2681 2682 2683 2684 2685 2686
	/*
	 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
	 * Therefore we need to allocate shadow page tables in the first
	 * 4GB of memory, which happens to fit the DMA32 zone.
	 */
	page = alloc_page(GFP_KERNEL | __GFP_DMA32);
	if (!page)
		goto error_1;
2687
	vcpu->arch.mmu.pae_root = page_address(page);
2688
	for (i = 0; i < 4; ++i)
2689
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2690

A
Avi Kivity 已提交
2691 2692 2693 2694 2695 2696 2697
	return 0;

error_1:
	free_mmu_pages(vcpu);
	return -ENOMEM;
}

2698
int kvm_mmu_create(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2699 2700
{
	ASSERT(vcpu);
2701
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
2702

2703 2704
	return alloc_mmu_pages(vcpu);
}
A
Avi Kivity 已提交
2705

2706 2707 2708
int kvm_mmu_setup(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
2709
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2710

2711
	return init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
2712 2713 2714 2715 2716 2717 2718 2719
}

void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);

	destroy_kvm_mmu(vcpu);
	free_mmu_pages(vcpu);
2720
	mmu_free_memory_caches(vcpu);
A
Avi Kivity 已提交
2721 2722
}

2723
void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
A
Avi Kivity 已提交
2724
{
2725
	struct kvm_mmu_page *sp;
A
Avi Kivity 已提交
2726

2727
	spin_lock(&kvm->mmu_lock);
2728
	list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
A
Avi Kivity 已提交
2729 2730 2731
		int i;
		u64 *pt;

2732
		if (!test_bit(slot, sp->slot_bitmap))
A
Avi Kivity 已提交
2733 2734
			continue;

2735
		pt = sp->spt;
A
Avi Kivity 已提交
2736 2737
		for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
			/* avoid RMW */
2738
			if (pt[i] & PT_WRITABLE_MASK)
A
Avi Kivity 已提交
2739 2740
				pt[i] &= ~PT_WRITABLE_MASK;
	}
2741
	kvm_flush_remote_tlbs(kvm);
2742
	spin_unlock(&kvm->mmu_lock);
A
Avi Kivity 已提交
2743
}
2744

2745
void kvm_mmu_zap_all(struct kvm *kvm)
D
Dor Laor 已提交
2746
{
2747
	struct kvm_mmu_page *sp, *node;
D
Dor Laor 已提交
2748

2749
	spin_lock(&kvm->mmu_lock);
2750
	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
2751 2752 2753
		if (kvm_mmu_zap_page(kvm, sp))
			node = container_of(kvm->arch.active_mmu_pages.next,
					    struct kvm_mmu_page, link);
2754
	spin_unlock(&kvm->mmu_lock);
D
Dor Laor 已提交
2755

2756
	kvm_flush_remote_tlbs(kvm);
D
Dor Laor 已提交
2757 2758
}

2759
static void kvm_mmu_remove_one_alloc_mmu_page(struct kvm *kvm)
2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778
{
	struct kvm_mmu_page *page;

	page = container_of(kvm->arch.active_mmu_pages.prev,
			    struct kvm_mmu_page, link);
	kvm_mmu_zap_page(kvm, page);
}

static int mmu_shrink(int nr_to_scan, gfp_t gfp_mask)
{
	struct kvm *kvm;
	struct kvm *kvm_freed = NULL;
	int cache_count = 0;

	spin_lock(&kvm_lock);

	list_for_each_entry(kvm, &vm_list, vm_list) {
		int npages;

2779 2780
		if (!down_read_trylock(&kvm->slots_lock))
			continue;
2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792
		spin_lock(&kvm->mmu_lock);
		npages = kvm->arch.n_alloc_mmu_pages -
			 kvm->arch.n_free_mmu_pages;
		cache_count += npages;
		if (!kvm_freed && nr_to_scan > 0 && npages > 0) {
			kvm_mmu_remove_one_alloc_mmu_page(kvm);
			cache_count--;
			kvm_freed = kvm;
		}
		nr_to_scan--;

		spin_unlock(&kvm->mmu_lock);
2793
		up_read(&kvm->slots_lock);
2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807
	}
	if (kvm_freed)
		list_move_tail(&kvm_freed->vm_list, &vm_list);

	spin_unlock(&kvm_lock);

	return cache_count;
}

static struct shrinker mmu_shrinker = {
	.shrink = mmu_shrink,
	.seeks = DEFAULT_SEEKS * 10,
};

I
Ingo Molnar 已提交
2808
static void mmu_destroy_caches(void)
2809 2810 2811 2812 2813
{
	if (pte_chain_cache)
		kmem_cache_destroy(pte_chain_cache);
	if (rmap_desc_cache)
		kmem_cache_destroy(rmap_desc_cache);
2814 2815
	if (mmu_page_header_cache)
		kmem_cache_destroy(mmu_page_header_cache);
2816 2817
}

2818 2819 2820 2821 2822 2823
void kvm_mmu_module_exit(void)
{
	mmu_destroy_caches();
	unregister_shrinker(&mmu_shrinker);
}

2824 2825 2826 2827
int kvm_mmu_module_init(void)
{
	pte_chain_cache = kmem_cache_create("kvm_pte_chain",
					    sizeof(struct kvm_pte_chain),
2828
					    0, 0, NULL);
2829 2830 2831 2832
	if (!pte_chain_cache)
		goto nomem;
	rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
					    sizeof(struct kvm_rmap_desc),
2833
					    0, 0, NULL);
2834 2835 2836
	if (!rmap_desc_cache)
		goto nomem;

2837 2838
	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
2839
						  0, 0, NULL);
2840 2841 2842
	if (!mmu_page_header_cache)
		goto nomem;

2843 2844
	register_shrinker(&mmu_shrinker);

2845 2846 2847
	return 0;

nomem:
2848
	mmu_destroy_caches();
2849 2850 2851
	return -ENOMEM;
}

2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870
/*
 * Caculate mmu pages needed for kvm.
 */
unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
{
	int i;
	unsigned int nr_mmu_pages;
	unsigned int  nr_pages = 0;

	for (i = 0; i < kvm->nmemslots; i++)
		nr_pages += kvm->memslots[i].npages;

	nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
	nr_mmu_pages = max(nr_mmu_pages,
			(unsigned int) KVM_MIN_ALLOC_MMU_PAGES);

	return nr_mmu_pages;
}

2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905
static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
				unsigned len)
{
	if (len > buffer->len)
		return NULL;
	return buffer->ptr;
}

static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
				unsigned len)
{
	void *ret;

	ret = pv_mmu_peek_buffer(buffer, len);
	if (!ret)
		return ret;
	buffer->ptr += len;
	buffer->len -= len;
	buffer->processed += len;
	return ret;
}

static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
			     gpa_t addr, gpa_t value)
{
	int bytes = 8;
	int r;

	if (!is_long_mode(vcpu) && !is_pae(vcpu))
		bytes = 4;

	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

2906
	if (!emulator_write_phys(vcpu, addr, &value, bytes))
2907 2908 2909 2910 2911 2912 2913 2914
		return -EFAULT;

	return 1;
}

static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
{
	kvm_x86_ops->tlb_flush(vcpu);
2915
	set_bit(KVM_REQ_MMU_SYNC, &vcpu->requests);
2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968
	return 1;
}

static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
{
	spin_lock(&vcpu->kvm->mmu_lock);
	mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
	spin_unlock(&vcpu->kvm->mmu_lock);
	return 1;
}

static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
			     struct kvm_pv_mmu_op_buffer *buffer)
{
	struct kvm_mmu_op_header *header;

	header = pv_mmu_peek_buffer(buffer, sizeof *header);
	if (!header)
		return 0;
	switch (header->op) {
	case KVM_MMU_OP_WRITE_PTE: {
		struct kvm_mmu_op_write_pte *wpte;

		wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
		if (!wpte)
			return 0;
		return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
					wpte->pte_val);
	}
	case KVM_MMU_OP_FLUSH_TLB: {
		struct kvm_mmu_op_flush_tlb *ftlb;

		ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
		if (!ftlb)
			return 0;
		return kvm_pv_mmu_flush_tlb(vcpu);
	}
	case KVM_MMU_OP_RELEASE_PT: {
		struct kvm_mmu_op_release_pt *rpt;

		rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
		if (!rpt)
			return 0;
		return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
	}
	default: return 0;
	}
}

int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
		  gpa_t addr, unsigned long *ret)
{
	int r;
2969
	struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
2970

2971 2972 2973
	buffer->ptr = buffer->buf;
	buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
	buffer->processed = 0;
2974

2975
	r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
2976 2977 2978
	if (r)
		goto out;

2979 2980
	while (buffer->len) {
		r = kvm_pv_mmu_op_one(vcpu, buffer);
2981 2982 2983 2984 2985 2986 2987 2988
		if (r < 0)
			goto out;
		if (r == 0)
			break;
	}

	r = 1;
out:
2989
	*ret = buffer->processed;
2990 2991 2992
	return r;
}

2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014
#ifdef AUDIT

static const char *audit_msg;

static gva_t canonicalize(gva_t gva)
{
#ifdef CONFIG_X86_64
	gva = (long long)(gva << 16) >> 16;
#endif
	return gva;
}

static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
				gva_t va, int level)
{
	u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
	int i;
	gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
		u64 ent = pt[i];

3015
		if (ent == shadow_trap_nonpresent_pte)
3016 3017 3018
			continue;

		va = canonicalize(va);
3019 3020 3021 3022 3023
		if (level > 1) {
			if (ent == shadow_notrap_nonpresent_pte)
				printk(KERN_ERR "audit: (%s) nontrapping pte"
				       " in nonleaf level: levels %d gva %lx"
				       " level %d pte %llx\n", audit_msg,
3024
				       vcpu->arch.mmu.root_level, va, level, ent);
3025

3026
			audit_mappings_page(vcpu, ent, va, level - 1);
3027
		} else {
3028
			gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
3029
			hpa_t hpa = (hpa_t)gpa_to_pfn(vcpu, gpa) << PAGE_SHIFT;
3030

3031
			if (is_shadow_present_pte(ent)
3032
			    && (ent & PT64_BASE_ADDR_MASK) != hpa)
3033 3034
				printk(KERN_ERR "xx audit error: (%s) levels %d"
				       " gva %lx gpa %llx hpa %llx ent %llx %d\n",
3035
				       audit_msg, vcpu->arch.mmu.root_level,
M
Mike Day 已提交
3036 3037
				       va, gpa, hpa, ent,
				       is_shadow_present_pte(ent));
3038 3039 3040 3041
			else if (ent == shadow_notrap_nonpresent_pte
				 && !is_error_hpa(hpa))
				printk(KERN_ERR "audit: (%s) notrap shadow,"
				       " valid guest gva %lx\n", audit_msg, va);
3042
			kvm_release_pfn_clean(pfn);
3043

3044 3045 3046 3047 3048 3049
		}
	}
}

static void audit_mappings(struct kvm_vcpu *vcpu)
{
3050
	unsigned i;
3051

3052 3053
	if (vcpu->arch.mmu.root_level == 4)
		audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4);
3054 3055
	else
		for (i = 0; i < 4; ++i)
3056
			if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK)
3057
				audit_mappings_page(vcpu,
3058
						    vcpu->arch.mmu.pae_root[i],
3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072
						    i << 30,
						    2);
}

static int count_rmaps(struct kvm_vcpu *vcpu)
{
	int nmaps = 0;
	int i, j, k;

	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
		struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
		struct kvm_rmap_desc *d;

		for (j = 0; j < m->npages; ++j) {
3073
			unsigned long *rmapp = &m->rmap[j];
3074

3075
			if (!*rmapp)
3076
				continue;
3077
			if (!(*rmapp & 1)) {
3078 3079 3080
				++nmaps;
				continue;
			}
3081
			d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097
			while (d) {
				for (k = 0; k < RMAP_EXT; ++k)
					if (d->shadow_ptes[k])
						++nmaps;
					else
						break;
				d = d->more;
			}
		}
	}
	return nmaps;
}

static int count_writable_mappings(struct kvm_vcpu *vcpu)
{
	int nmaps = 0;
3098
	struct kvm_mmu_page *sp;
3099 3100
	int i;

3101
	list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
3102
		u64 *pt = sp->spt;
3103

3104
		if (sp->role.level != PT_PAGE_TABLE_LEVEL)
3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126
			continue;

		for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
			u64 ent = pt[i];

			if (!(ent & PT_PRESENT_MASK))
				continue;
			if (!(ent & PT_WRITABLE_MASK))
				continue;
			++nmaps;
		}
	}
	return nmaps;
}

static void audit_rmap(struct kvm_vcpu *vcpu)
{
	int n_rmap = count_rmaps(vcpu);
	int n_actual = count_writable_mappings(vcpu);

	if (n_rmap != n_actual)
		printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
3127
		       __func__, audit_msg, n_rmap, n_actual);
3128 3129 3130 3131
}

static void audit_write_protection(struct kvm_vcpu *vcpu)
{
3132
	struct kvm_mmu_page *sp;
3133 3134 3135
	struct kvm_memory_slot *slot;
	unsigned long *rmapp;
	gfn_t gfn;
3136

3137
	list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
3138
		if (sp->role.metaphysical)
3139 3140
			continue;

3141
		gfn = unalias_gfn(vcpu->kvm, sp->gfn);
3142
		slot = gfn_to_memslot_unaliased(vcpu->kvm, sp->gfn);
3143 3144
		rmapp = &slot->rmap[gfn - slot->base_gfn];
		if (*rmapp)
3145 3146
			printk(KERN_ERR "%s: (%s) shadow page has writable"
			       " mappings: gfn %lx role %x\n",
3147
			       __func__, audit_msg, sp->gfn,
3148
			       sp->role.word);
3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164
	}
}

static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
{
	int olddbg = dbg;

	dbg = 0;
	audit_msg = msg;
	audit_rmap(vcpu);
	audit_write_protection(vcpu);
	audit_mappings(vcpu);
	dbg = olddbg;
}

#endif