mmu.c 89.9 KB
Newer Older
A
Avi Kivity 已提交
1 2 3 4 5 6 7 8 9
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
N
Nicolas Kaiser 已提交
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
A
Avi Kivity 已提交
11 12 13 14 15 16 17 18 19
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */
A
Avi Kivity 已提交
20

21
#include "irq.h"
22
#include "mmu.h"
23
#include "x86.h"
A
Avi Kivity 已提交
24
#include "kvm_cache_regs.h"
25
#include "x86.h"
A
Avi Kivity 已提交
26

27
#include <linux/kvm_host.h>
A
Avi Kivity 已提交
28 29 30 31 32
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/module.h>
33
#include <linux/swap.h>
M
Marcelo Tosatti 已提交
34
#include <linux/hugetlb.h>
35
#include <linux/compiler.h>
36
#include <linux/srcu.h>
37
#include <linux/slab.h>
38
#include <linux/uaccess.h>
A
Avi Kivity 已提交
39

A
Avi Kivity 已提交
40 41
#include <asm/page.h>
#include <asm/cmpxchg.h>
42
#include <asm/io.h>
43
#include <asm/vmx.h>
A
Avi Kivity 已提交
44

45 46 47 48 49 50 51
/*
 * When setting this variable to true it enables Two-Dimensional-Paging
 * where the hardware walks 2 page tables:
 * 1. the guest-virtual to guest-physical
 * 2. while doing 1. it walks guest-physical to host-physical
 * If the hardware supports that we don't need to do shadow paging.
 */
52
bool tdp_enabled = false;
53

54 55 56 57
enum {
	AUDIT_PRE_PAGE_FAULT,
	AUDIT_POST_PAGE_FAULT,
	AUDIT_PRE_PTE_WRITE,
58 59 60
	AUDIT_POST_PTE_WRITE,
	AUDIT_PRE_SYNC,
	AUDIT_POST_SYNC
61
};
62

63 64 65 66
char *audit_point_name[] = {
	"pre page fault",
	"post page fault",
	"pre pte write",
67 68 69
	"post pte write",
	"pre sync",
	"post sync"
70
};
71

72
#undef MMU_DEBUG
73 74 75 76 77 78 79 80 81 82 83 84 85

#ifdef MMU_DEBUG

#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)

#else

#define pgprintk(x...) do { } while (0)
#define rmap_printk(x...) do { } while (0)

#endif

86
#ifdef MMU_DEBUG
87 88
static int dbg = 0;
module_param(dbg, bool, 0644);
89
#endif
A
Avi Kivity 已提交
90

91 92 93
static int oos_shadow = 1;
module_param(oos_shadow, bool, 0644);

94 95 96
#ifndef MMU_DEBUG
#define ASSERT(x) do { } while (0)
#else
A
Avi Kivity 已提交
97 98 99 100 101
#define ASSERT(x)							\
	if (!(x)) {							\
		printk(KERN_WARNING "assertion failed %s:%d: %s\n",	\
		       __FILE__, __LINE__, #x);				\
	}
102
#endif
A
Avi Kivity 已提交
103

104 105
#define PTE_PREFETCH_NUM		8

A
Avi Kivity 已提交
106 107 108 109 110 111
#define PT_FIRST_AVAIL_BITS_SHIFT 9
#define PT64_SECOND_AVAIL_BITS_SHIFT 52

#define PT64_LEVEL_BITS 9

#define PT64_LEVEL_SHIFT(level) \
M
Mike Day 已提交
112
		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
A
Avi Kivity 已提交
113 114 115 116 117 118 119 120

#define PT64_INDEX(address, level)\
	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))


#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
M
Mike Day 已提交
121
		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
A
Avi Kivity 已提交
122

123 124 125
#define PT32_LVL_OFFSET_MASK(level) \
	(PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT32_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
126 127 128 129 130

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


131
#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
A
Avi Kivity 已提交
132 133
#define PT64_DIR_BASE_ADDR_MASK \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
134 135 136 137 138 139
#define PT64_LVL_ADDR_MASK(level) \
	(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
#define PT64_LVL_OFFSET_MASK(level) \
	(PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT64_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
140 141 142 143

#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
144 145 146
#define PT32_LVL_ADDR_MASK(level) \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
					    * PT32_LEVEL_BITS))) - 1))
A
Avi Kivity 已提交
147

148 149
#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
			| PT64_NX_MASK)
A
Avi Kivity 已提交
150

151
#define PTE_LIST_EXT 4
152

153 154 155 156 157
#define ACC_EXEC_MASK    1
#define ACC_WRITE_MASK   PT_WRITABLE_MASK
#define ACC_USER_MASK    PT_USER_MASK
#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)

158 159
#include <trace/events/kvm.h>

160 161 162
#define CREATE_TRACE_POINTS
#include "mmutrace.h"

163 164
#define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)

165 166
#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)

167 168 169
struct pte_list_desc {
	u64 *sptes[PTE_LIST_EXT];
	struct pte_list_desc *more;
170 171
};

172 173 174 175 176 177 178 179 180 181 182 183 184
struct kvm_shadow_walk_iterator {
	u64 addr;
	hpa_t shadow_addr;
	int level;
	u64 *sptep;
	unsigned index;
};

#define for_each_shadow_entry(_vcpu, _addr, _walker)    \
	for (shadow_walk_init(&(_walker), _vcpu, _addr);	\
	     shadow_walk_okay(&(_walker));			\
	     shadow_walk_next(&(_walker)))

185
static struct kmem_cache *pte_list_desc_cache;
186
static struct kmem_cache *mmu_page_header_cache;
187
static struct percpu_counter kvm_total_used_mmu_pages;
188

189 190
static u64 __read_mostly shadow_trap_nonpresent_pte;
static u64 __read_mostly shadow_notrap_nonpresent_pte;
S
Sheng Yang 已提交
191 192 193 194 195
static u64 __read_mostly shadow_nx_mask;
static u64 __read_mostly shadow_x_mask;	/* mutual exclusive with nx_mask */
static u64 __read_mostly shadow_user_mask;
static u64 __read_mostly shadow_accessed_mask;
static u64 __read_mostly shadow_dirty_mask;
196

197 198 199 200 201
static inline u64 rsvd_bits(int s, int e)
{
	return ((1ULL << (e - s + 1)) - 1) << s;
}

202 203 204 205 206 207 208
void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
{
	shadow_trap_nonpresent_pte = trap_pte;
	shadow_notrap_nonpresent_pte = notrap_pte;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);

S
Sheng Yang 已提交
209
void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
210
		u64 dirty_mask, u64 nx_mask, u64 x_mask)
S
Sheng Yang 已提交
211 212 213 214 215 216 217 218 219
{
	shadow_user_mask = user_mask;
	shadow_accessed_mask = accessed_mask;
	shadow_dirty_mask = dirty_mask;
	shadow_nx_mask = nx_mask;
	shadow_x_mask = x_mask;
}
EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);

A
Avi Kivity 已提交
220 221 222 223 224
static int is_cpuid_PSE36(void)
{
	return 1;
}

225 226
static int is_nx(struct kvm_vcpu *vcpu)
{
227
	return vcpu->arch.efer & EFER_NX;
228 229
}

230 231 232 233 234 235
static int is_shadow_present_pte(u64 pte)
{
	return pte != shadow_trap_nonpresent_pte
		&& pte != shadow_notrap_nonpresent_pte;
}

M
Marcelo Tosatti 已提交
236 237 238 239 240
static int is_large_pte(u64 pte)
{
	return pte & PT_PAGE_SIZE_MASK;
}

241
static int is_dirty_gpte(unsigned long pte)
242
{
A
Avi Kivity 已提交
243
	return pte & PT_DIRTY_MASK;
244 245
}

246
static int is_rmap_spte(u64 pte)
247
{
248
	return is_shadow_present_pte(pte);
249 250
}

251 252 253 254
static int is_last_spte(u64 pte, int level)
{
	if (level == PT_PAGE_TABLE_LEVEL)
		return 1;
255
	if (is_large_pte(pte))
256 257 258 259
		return 1;
	return 0;
}

260
static pfn_t spte_to_pfn(u64 pte)
261
{
262
	return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
263 264
}

265 266 267 268 269 270 271
static gfn_t pse36_gfn_delta(u32 gpte)
{
	int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;

	return (gpte & PT32_DIR_PSE36_MASK) << shift;
}

A
Avi Kivity 已提交
272
static void __set_spte(u64 *sptep, u64 spte)
273
{
274
	set_64bit(sptep, spte);
275 276
}

277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
static u64 __xchg_spte(u64 *sptep, u64 new_spte)
{
#ifdef CONFIG_X86_64
	return xchg(sptep, new_spte);
#else
	u64 old_spte;

	do {
		old_spte = *sptep;
	} while (cmpxchg64(sptep, old_spte, new_spte) != old_spte);

	return old_spte;
#endif
}

292 293 294 295 296 297 298 299
static bool spte_has_volatile_bits(u64 spte)
{
	if (!shadow_accessed_mask)
		return false;

	if (!is_shadow_present_pte(spte))
		return false;

300 301
	if ((spte & shadow_accessed_mask) &&
	      (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
302 303 304 305 306
		return false;

	return true;
}

307 308 309 310 311
static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
{
	return (old_spte & bit_mask) && !(new_spte & bit_mask);
}

312 313
static void update_spte(u64 *sptep, u64 new_spte)
{
314 315 316
	u64 mask, old_spte = *sptep;

	WARN_ON(!is_rmap_spte(new_spte));
317

318 319 320 321 322 323 324
	new_spte |= old_spte & shadow_dirty_mask;

	mask = shadow_accessed_mask;
	if (is_writable_pte(old_spte))
		mask |= shadow_dirty_mask;

	if (!spte_has_volatile_bits(old_spte) || (new_spte & mask) == mask)
325
		__set_spte(sptep, new_spte);
326
	else
327
		old_spte = __xchg_spte(sptep, new_spte);
328 329 330 331 332 333 334 335

	if (!shadow_accessed_mask)
		return;

	if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
		kvm_set_pfn_accessed(spte_to_pfn(old_spte));
	if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
		kvm_set_pfn_dirty(spte_to_pfn(old_spte));
336 337
}

338
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
339
				  struct kmem_cache *base_cache, int min)
340 341 342 343
{
	void *obj;

	if (cache->nobjs >= min)
344
		return 0;
345
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
346
		obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
347
		if (!obj)
348
			return -ENOMEM;
349 350
		cache->objects[cache->nobjs++] = obj;
	}
351
	return 0;
352 353
}

354 355
static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
				  struct kmem_cache *cache)
356 357
{
	while (mc->nobjs)
358
		kmem_cache_free(cache, mc->objects[--mc->nobjs]);
359 360
}

A
Avi Kivity 已提交
361
static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
362
				       int min)
A
Avi Kivity 已提交
363
{
364
	void *page;
A
Avi Kivity 已提交
365 366 367 368

	if (cache->nobjs >= min)
		return 0;
	while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
369
		page = (void *)__get_free_page(GFP_KERNEL);
A
Avi Kivity 已提交
370 371
		if (!page)
			return -ENOMEM;
372
		cache->objects[cache->nobjs++] = page;
A
Avi Kivity 已提交
373 374 375 376 377 378 379
	}
	return 0;
}

static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
{
	while (mc->nobjs)
380
		free_page((unsigned long)mc->objects[--mc->nobjs]);
A
Avi Kivity 已提交
381 382
}

383
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
384
{
385 386
	int r;

387
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
388
				   pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
389 390
	if (r)
		goto out;
391
	r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
392 393
	if (r)
		goto out;
394
	r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
395
				   mmu_page_header_cache, 4);
396 397
out:
	return r;
398 399 400 401
}

static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
402 403
	mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
				pte_list_desc_cache);
404
	mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
405 406
	mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
				mmu_page_header_cache);
407 408 409 410 411 412 413 414 415 416 417 418
}

static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
				    size_t size)
{
	void *p;

	BUG_ON(!mc->nobjs);
	p = mc->objects[--mc->nobjs];
	return p;
}

419
static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
420
{
421 422
	return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache,
				      sizeof(struct pte_list_desc));
423 424
}

425
static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
426
{
427
	kmem_cache_free(pte_list_desc_cache, pte_list_desc);
428 429
}

430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
{
	if (!sp->role.direct)
		return sp->gfns[index];

	return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
}

static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
{
	if (sp->role.direct)
		BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
	else
		sp->gfns[index] = gfn;
}

M
Marcelo Tosatti 已提交
446
/*
447 448
 * Return the pointer to the large page information for a given gfn,
 * handling slots that are not large page aligned.
M
Marcelo Tosatti 已提交
449
 */
450 451 452
static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
					      struct kvm_memory_slot *slot,
					      int level)
M
Marcelo Tosatti 已提交
453 454 455
{
	unsigned long idx;

456 457
	idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
	      (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
458
	return &slot->lpage_info[level - 2][idx];
M
Marcelo Tosatti 已提交
459 460 461 462
}

static void account_shadowed(struct kvm *kvm, gfn_t gfn)
{
463
	struct kvm_memory_slot *slot;
464
	struct kvm_lpage_info *linfo;
465
	int i;
M
Marcelo Tosatti 已提交
466

A
Avi Kivity 已提交
467
	slot = gfn_to_memslot(kvm, gfn);
468 469
	for (i = PT_DIRECTORY_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
470 471
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->write_count += 1;
472
	}
473
	kvm->arch.indirect_shadow_pages++;
M
Marcelo Tosatti 已提交
474 475 476 477
}

static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
{
478
	struct kvm_memory_slot *slot;
479
	struct kvm_lpage_info *linfo;
480
	int i;
M
Marcelo Tosatti 已提交
481

A
Avi Kivity 已提交
482
	slot = gfn_to_memslot(kvm, gfn);
483 484
	for (i = PT_DIRECTORY_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
485 486 487
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->write_count -= 1;
		WARN_ON(linfo->write_count < 0);
488
	}
489
	kvm->arch.indirect_shadow_pages--;
M
Marcelo Tosatti 已提交
490 491
}

492 493 494
static int has_wrprotected_page(struct kvm *kvm,
				gfn_t gfn,
				int level)
M
Marcelo Tosatti 已提交
495
{
496
	struct kvm_memory_slot *slot;
497
	struct kvm_lpage_info *linfo;
M
Marcelo Tosatti 已提交
498

A
Avi Kivity 已提交
499
	slot = gfn_to_memslot(kvm, gfn);
M
Marcelo Tosatti 已提交
500
	if (slot) {
501 502
		linfo = lpage_info_slot(gfn, slot, level);
		return linfo->write_count;
M
Marcelo Tosatti 已提交
503 504 505 506 507
	}

	return 1;
}

508
static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
M
Marcelo Tosatti 已提交
509
{
J
Joerg Roedel 已提交
510
	unsigned long page_size;
511
	int i, ret = 0;
M
Marcelo Tosatti 已提交
512

J
Joerg Roedel 已提交
513
	page_size = kvm_host_page_size(kvm, gfn);
M
Marcelo Tosatti 已提交
514

515 516 517 518 519 520 521 522
	for (i = PT_PAGE_TABLE_LEVEL;
	     i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
		if (page_size >= KVM_HPAGE_SIZE(i))
			ret = i;
		else
			break;
	}

523
	return ret;
M
Marcelo Tosatti 已提交
524 525
}

526 527 528
static struct kvm_memory_slot *
gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
			    bool no_dirty_log)
M
Marcelo Tosatti 已提交
529 530
{
	struct kvm_memory_slot *slot;
531 532 533 534 535 536 537 538 539 540 541

	slot = gfn_to_memslot(vcpu->kvm, gfn);
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
	      (no_dirty_log && slot->dirty_bitmap))
		slot = NULL;

	return slot;
}

static bool mapping_level_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t large_gfn)
{
542
	return !gfn_to_memslot_dirty_bitmap(vcpu, large_gfn, true);
543 544 545 546 547
}

static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
{
	int host_level, level, max_level;
M
Marcelo Tosatti 已提交
548

549 550 551 552 553
	host_level = host_mapping_level(vcpu->kvm, large_gfn);

	if (host_level == PT_PAGE_TABLE_LEVEL)
		return host_level;

554 555 556 557
	max_level = kvm_x86_ops->get_lpage_level() < host_level ?
		kvm_x86_ops->get_lpage_level() : host_level;

	for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
558 559 560 561
		if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
			break;

	return level - 1;
M
Marcelo Tosatti 已提交
562 563
}

564
/*
565
 * Pte mapping structures:
566
 *
567
 * If pte_list bit zero is zero, then pte_list point to the spte.
568
 *
569 570
 * If pte_list bit zero is one, (then pte_list & ~1) points to a struct
 * pte_list_desc containing more mappings.
571
 *
572
 * Returns the number of pte entries before the spte was added or zero if
573 574
 * the spte was not added.
 *
575
 */
576 577
static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
			unsigned long *pte_list)
578
{
579
	struct pte_list_desc *desc;
580
	int i, count = 0;
581

582 583 584 585 586 587 588
	if (!*pte_list) {
		rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
		*pte_list = (unsigned long)spte;
	} else if (!(*pte_list & 1)) {
		rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
		desc = mmu_alloc_pte_list_desc(vcpu);
		desc->sptes[0] = (u64 *)*pte_list;
A
Avi Kivity 已提交
589
		desc->sptes[1] = spte;
590
		*pte_list = (unsigned long)desc | 1;
591
		++count;
592
	} else {
593 594 595
		rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
		desc = (struct pte_list_desc *)(*pte_list & ~1ul);
		while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
596
			desc = desc->more;
597
			count += PTE_LIST_EXT;
598
		}
599 600
		if (desc->sptes[PTE_LIST_EXT-1]) {
			desc->more = mmu_alloc_pte_list_desc(vcpu);
601 602
			desc = desc->more;
		}
A
Avi Kivity 已提交
603
		for (i = 0; desc->sptes[i]; ++i)
604
			++count;
A
Avi Kivity 已提交
605
		desc->sptes[i] = spte;
606
	}
607
	return count;
608 609
}

610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638
static u64 *pte_list_next(unsigned long *pte_list, u64 *spte)
{
	struct pte_list_desc *desc;
	u64 *prev_spte;
	int i;

	if (!*pte_list)
		return NULL;
	else if (!(*pte_list & 1)) {
		if (!spte)
			return (u64 *)*pte_list;
		return NULL;
	}
	desc = (struct pte_list_desc *)(*pte_list & ~1ul);
	prev_spte = NULL;
	while (desc) {
		for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) {
			if (prev_spte == spte)
				return desc->sptes[i];
			prev_spte = desc->sptes[i];
		}
		desc = desc->more;
	}
	return NULL;
}

static void
pte_list_desc_remove_entry(unsigned long *pte_list, struct pte_list_desc *desc,
			   int i, struct pte_list_desc *prev_desc)
639 640 641
{
	int j;

642
	for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
643
		;
A
Avi Kivity 已提交
644 645
	desc->sptes[i] = desc->sptes[j];
	desc->sptes[j] = NULL;
646 647 648
	if (j != 0)
		return;
	if (!prev_desc && !desc->more)
649
		*pte_list = (unsigned long)desc->sptes[0];
650 651 652 653
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
654 655
			*pte_list = (unsigned long)desc->more | 1;
	mmu_free_pte_list_desc(desc);
656 657
}

658
static void pte_list_remove(u64 *spte, unsigned long *pte_list)
659
{
660 661
	struct pte_list_desc *desc;
	struct pte_list_desc *prev_desc;
662 663
	int i;

664 665
	if (!*pte_list) {
		printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
666
		BUG();
667 668 669 670
	} else if (!(*pte_list & 1)) {
		rmap_printk("pte_list_remove:  %p 1->0\n", spte);
		if ((u64 *)*pte_list != spte) {
			printk(KERN_ERR "pte_list_remove:  %p 1->BUG\n", spte);
671 672
			BUG();
		}
673
		*pte_list = 0;
674
	} else {
675 676
		rmap_printk("pte_list_remove:  %p many->many\n", spte);
		desc = (struct pte_list_desc *)(*pte_list & ~1ul);
677 678
		prev_desc = NULL;
		while (desc) {
679
			for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
A
Avi Kivity 已提交
680
				if (desc->sptes[i] == spte) {
681
					pte_list_desc_remove_entry(pte_list,
682
							       desc, i,
683 684 685 686 687 688
							       prev_desc);
					return;
				}
			prev_desc = desc;
			desc = desc->more;
		}
689
		pr_err("pte_list_remove: %p many->many\n", spte);
690 691 692 693
		BUG();
	}
}

694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
typedef void (*pte_list_walk_fn) (u64 *spte);
static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
{
	struct pte_list_desc *desc;
	int i;

	if (!*pte_list)
		return;

	if (!(*pte_list & 1))
		return fn((u64 *)*pte_list);

	desc = (struct pte_list_desc *)(*pte_list & ~1ul);
	while (desc) {
		for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
			fn(desc->sptes[i]);
		desc = desc->more;
	}
}

714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758
/*
 * Take gfn and return the reverse mapping to it.
 */
static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
{
	struct kvm_memory_slot *slot;
	struct kvm_lpage_info *linfo;

	slot = gfn_to_memslot(kvm, gfn);
	if (likely(level == PT_PAGE_TABLE_LEVEL))
		return &slot->rmap[gfn - slot->base_gfn];

	linfo = lpage_info_slot(gfn, slot, level);

	return &linfo->rmap_pde;
}

static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
{
	struct kvm_mmu_page *sp;
	unsigned long *rmapp;

	sp = page_header(__pa(spte));
	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
	return pte_list_add(vcpu, spte, rmapp);
}

static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
{
	return pte_list_next(rmapp, spte);
}

static void rmap_remove(struct kvm *kvm, u64 *spte)
{
	struct kvm_mmu_page *sp;
	gfn_t gfn;
	unsigned long *rmapp;

	sp = page_header(__pa(spte));
	gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
	rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
	pte_list_remove(spte, rmapp);
}

759
static int set_spte_track_bits(u64 *sptep, u64 new_spte)
A
Avi Kivity 已提交
760
{
761
	pfn_t pfn;
762 763
	u64 old_spte = *sptep;

764
	if (!spte_has_volatile_bits(old_spte))
765
		__set_spte(sptep, new_spte);
766
	else
767
		old_spte = __xchg_spte(sptep, new_spte);
768

769
	if (!is_rmap_spte(old_spte))
770
		return 0;
771

772
	pfn = spte_to_pfn(old_spte);
773
	if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
774
		kvm_set_pfn_accessed(pfn);
775
	if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
776
		kvm_set_pfn_dirty(pfn);
777
	return 1;
778 779 780 781
}

static void drop_spte(struct kvm *kvm, u64 *sptep, u64 new_spte)
{
782 783
	if (set_spte_track_bits(sptep, new_spte))
		rmap_remove(kvm, sptep);
A
Avi Kivity 已提交
784 785
}

786
static int rmap_write_protect(struct kvm *kvm, u64 gfn)
787
{
788
	unsigned long *rmapp;
789
	u64 *spte;
790
	int i, write_protected = 0;
791

792
	rmapp = gfn_to_rmap(kvm, gfn, PT_PAGE_TABLE_LEVEL);
793

794 795
	spte = rmap_next(kvm, rmapp, NULL);
	while (spte) {
796 797 798
		BUG_ON(!spte);
		BUG_ON(!(*spte & PT_PRESENT_MASK));
		rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
799
		if (is_writable_pte(*spte)) {
800
			update_spte(spte, *spte & ~PT_WRITABLE_MASK);
801 802
			write_protected = 1;
		}
803
		spte = rmap_next(kvm, rmapp, spte);
804
	}
805

M
Marcelo Tosatti 已提交
806
	/* check for huge page mappings */
807 808 809 810 811 812 813 814 815
	for (i = PT_DIRECTORY_LEVEL;
	     i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
		rmapp = gfn_to_rmap(kvm, gfn, i);
		spte = rmap_next(kvm, rmapp, NULL);
		while (spte) {
			BUG_ON(!spte);
			BUG_ON(!(*spte & PT_PRESENT_MASK));
			BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
			pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
816
			if (is_writable_pte(*spte)) {
A
Avi Kivity 已提交
817 818
				drop_spte(kvm, spte,
					  shadow_trap_nonpresent_pte);
819 820 821 822 823
				--kvm->stat.lpages;
				spte = NULL;
				write_protected = 1;
			}
			spte = rmap_next(kvm, rmapp, spte);
M
Marcelo Tosatti 已提交
824 825 826
		}
	}

827
	return write_protected;
828 829
}

F
Frederik Deweerdt 已提交
830 831
static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
			   unsigned long data)
832 833 834 835 836 837 838
{
	u64 *spte;
	int need_tlb_flush = 0;

	while ((spte = rmap_next(kvm, rmapp, NULL))) {
		BUG_ON(!(*spte & PT_PRESENT_MASK));
		rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
A
Avi Kivity 已提交
839
		drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
840 841 842 843 844
		need_tlb_flush = 1;
	}
	return need_tlb_flush;
}

F
Frederik Deweerdt 已提交
845 846
static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
			     unsigned long data)
847 848
{
	int need_flush = 0;
849
	u64 *spte, new_spte;
850 851 852 853 854 855 856 857 858 859 860
	pte_t *ptep = (pte_t *)data;
	pfn_t new_pfn;

	WARN_ON(pte_huge(*ptep));
	new_pfn = pte_pfn(*ptep);
	spte = rmap_next(kvm, rmapp, NULL);
	while (spte) {
		BUG_ON(!is_shadow_present_pte(*spte));
		rmap_printk("kvm_set_pte_rmapp: spte %p %llx\n", spte, *spte);
		need_flush = 1;
		if (pte_write(*ptep)) {
A
Avi Kivity 已提交
861
			drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
862 863 864 865 866 867 868
			spte = rmap_next(kvm, rmapp, NULL);
		} else {
			new_spte = *spte &~ (PT64_BASE_ADDR_MASK);
			new_spte |= (u64)new_pfn << PAGE_SHIFT;

			new_spte &= ~PT_WRITABLE_MASK;
			new_spte &= ~SPTE_HOST_WRITEABLE;
869
			new_spte &= ~shadow_accessed_mask;
870
			set_spte_track_bits(spte, new_spte);
871 872 873 874 875 876 877 878 879
			spte = rmap_next(kvm, rmapp, spte);
		}
	}
	if (need_flush)
		kvm_flush_remote_tlbs(kvm);

	return 0;
}

F
Frederik Deweerdt 已提交
880 881
static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
			  unsigned long data,
882
			  int (*handler)(struct kvm *kvm, unsigned long *rmapp,
F
Frederik Deweerdt 已提交
883
					 unsigned long data))
884
{
885
	int i, j;
886
	int ret;
887
	int retval = 0;
888 889
	struct kvm_memslots *slots;

890
	slots = kvm_memslots(kvm);
891

892 893
	for (i = 0; i < slots->nmemslots; i++) {
		struct kvm_memory_slot *memslot = &slots->memslots[i];
894 895 896 897 898 899
		unsigned long start = memslot->userspace_addr;
		unsigned long end;

		end = start + (memslot->npages << PAGE_SHIFT);
		if (hva >= start && hva < end) {
			gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
900
			gfn_t gfn = memslot->base_gfn + gfn_offset;
901

902
			ret = handler(kvm, &memslot->rmap[gfn_offset], data);
903 904

			for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
905 906 907 908 909
				struct kvm_lpage_info *linfo;

				linfo = lpage_info_slot(gfn, memslot,
							PT_DIRECTORY_LEVEL + j);
				ret |= handler(kvm, &linfo->rmap_pde, data);
910
			}
911 912
			trace_kvm_age_page(hva, memslot, ret);
			retval |= ret;
913 914 915 916 917 918 919 920
		}
	}

	return retval;
}

int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
{
921 922 923 924 925
	return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
}

void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
{
F
Frederik Deweerdt 已提交
926
	kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
927 928
}

F
Frederik Deweerdt 已提交
929 930
static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
			 unsigned long data)
931 932 933 934
{
	u64 *spte;
	int young = 0;

935 936 937 938 939 940 941
	/*
	 * Emulate the accessed bit for EPT, by checking if this page has
	 * an EPT mapping, and clearing it if it does. On the next access,
	 * a new EPT mapping will be established.
	 * This has some overhead, but not as much as the cost of swapping
	 * out actively used pages or breaking up actively used hugepages.
	 */
942
	if (!shadow_accessed_mask)
943
		return kvm_unmap_rmapp(kvm, rmapp, data);
944

945 946 947 948 949 950 951 952 953 954 955 956 957 958 959
	spte = rmap_next(kvm, rmapp, NULL);
	while (spte) {
		int _young;
		u64 _spte = *spte;
		BUG_ON(!(_spte & PT_PRESENT_MASK));
		_young = _spte & PT_ACCESSED_MASK;
		if (_young) {
			young = 1;
			clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
		}
		spte = rmap_next(kvm, rmapp, spte);
	}
	return young;
}

A
Andrea Arcangeli 已提交
960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988
static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
			      unsigned long data)
{
	u64 *spte;
	int young = 0;

	/*
	 * If there's no access bit in the secondary pte set by the
	 * hardware it's up to gup-fast/gup to set the access bit in
	 * the primary pte or in the page structure.
	 */
	if (!shadow_accessed_mask)
		goto out;

	spte = rmap_next(kvm, rmapp, NULL);
	while (spte) {
		u64 _spte = *spte;
		BUG_ON(!(_spte & PT_PRESENT_MASK));
		young = _spte & PT_ACCESSED_MASK;
		if (young) {
			young = 1;
			break;
		}
		spte = rmap_next(kvm, rmapp, spte);
	}
out:
	return young;
}

989 990
#define RMAP_RECYCLE_THRESHOLD 1000

991
static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
992 993
{
	unsigned long *rmapp;
994 995 996
	struct kvm_mmu_page *sp;

	sp = page_header(__pa(spte));
997

998
	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
999

1000
	kvm_unmap_rmapp(vcpu->kvm, rmapp, 0);
1001 1002 1003
	kvm_flush_remote_tlbs(vcpu->kvm);
}

1004 1005
int kvm_age_hva(struct kvm *kvm, unsigned long hva)
{
1006
	return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp);
1007 1008
}

A
Andrea Arcangeli 已提交
1009 1010 1011 1012 1013
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
{
	return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
}

1014
#ifdef MMU_DEBUG
1015
static int is_empty_shadow_page(u64 *spt)
A
Avi Kivity 已提交
1016
{
1017 1018 1019
	u64 *pos;
	u64 *end;

1020
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1021
		if (is_shadow_present_pte(*pos)) {
1022
			printk(KERN_ERR "%s: %p %llx\n", __func__,
1023
			       pos, *pos);
A
Avi Kivity 已提交
1024
			return 0;
1025
		}
A
Avi Kivity 已提交
1026 1027
	return 1;
}
1028
#endif
A
Avi Kivity 已提交
1029

1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
/*
 * This value is the sum of all of the kvm instances's
 * kvm->arch.n_used_mmu_pages values.  We need a global,
 * aggregate version in order to make the slab shrinker
 * faster
 */
static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
{
	kvm->arch.n_used_mmu_pages += nr;
	percpu_counter_add(&kvm_total_used_mmu_pages, nr);
}

1042
static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1043
{
1044
	ASSERT(is_empty_shadow_page(sp->spt));
1045
	hlist_del(&sp->hash_link);
1046
	list_del(&sp->link);
1047
	free_page((unsigned long)sp->spt);
1048
	if (!sp->role.direct)
1049
		free_page((unsigned long)sp->gfns);
1050
	kmem_cache_free(mmu_page_header_cache, sp);
1051
	kvm_mod_used_mmu_pages(kvm, -1);
1052 1053
}

1054 1055
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
1056
	return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
1057 1058
}

1059
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1060
				    struct kvm_mmu_page *sp, u64 *parent_pte)
1061 1062 1063 1064
{
	if (!parent_pte)
		return;

1065
	pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1066 1067
}

1068
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1069 1070
				       u64 *parent_pte)
{
1071
	pte_list_remove(parent_pte, &sp->parent_ptes);
1072 1073
}

1074 1075 1076 1077 1078 1079 1080
static void drop_parent_pte(struct kvm_mmu_page *sp,
			    u64 *parent_pte)
{
	mmu_page_remove_parent_pte(sp, parent_pte);
	__set_spte(parent_pte, shadow_trap_nonpresent_pte);
}

1081 1082
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
					       u64 *parent_pte, int direct)
M
Marcelo Tosatti 已提交
1083
{
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
	struct kvm_mmu_page *sp;
	sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache,
					sizeof *sp);
	sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
	if (!direct)
		sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache,
						  PAGE_SIZE);
	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
	bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
	sp->parent_ptes = 0;
	mmu_page_add_parent_pte(vcpu, sp, parent_pte);
	kvm_mod_used_mmu_pages(vcpu->kvm, +1);
	return sp;
M
Marcelo Tosatti 已提交
1098 1099
}

1100
static void mark_unsync(u64 *spte);
1101
static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1102
{
1103
	pte_list_walk(&sp->parent_ptes, mark_unsync);
1104 1105
}

1106
static void mark_unsync(u64 *spte)
1107
{
1108
	struct kvm_mmu_page *sp;
1109
	unsigned int index;
1110

1111
	sp = page_header(__pa(spte));
1112 1113
	index = spte - sp->spt;
	if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1114
		return;
1115
	if (sp->unsync_children++)
1116
		return;
1117
	kvm_mmu_mark_parents_unsync(sp);
1118 1119
}

1120 1121 1122 1123 1124 1125 1126 1127 1128
static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *sp)
{
	int i;

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		sp->spt[i] = shadow_trap_nonpresent_pte;
}

1129
static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1130
			       struct kvm_mmu_page *sp)
1131 1132 1133 1134
{
	return 1;
}

M
Marcelo Tosatti 已提交
1135 1136 1137 1138
static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
}

1139 1140
static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
				 struct kvm_mmu_page *sp, u64 *spte,
1141
				 const void *pte)
1142 1143 1144 1145
{
	WARN_ON(1);
}

1146 1147 1148 1149 1150 1151 1152 1153 1154 1155
#define KVM_PAGE_ARRAY_NR 16

struct kvm_mmu_pages {
	struct mmu_page_and_offset {
		struct kvm_mmu_page *sp;
		unsigned int idx;
	} page[KVM_PAGE_ARRAY_NR];
	unsigned int nr;
};

1156 1157 1158 1159 1160
#define for_each_unsync_children(bitmap, idx)		\
	for (idx = find_first_bit(bitmap, 512);		\
	     idx < 512;					\
	     idx = find_next_bit(bitmap, 512, idx+1))

1161 1162
static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
			 int idx)
1163
{
1164
	int i;
1165

1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	if (sp->unsync)
		for (i=0; i < pvec->nr; i++)
			if (pvec->page[i].sp == sp)
				return 0;

	pvec->page[pvec->nr].sp = sp;
	pvec->page[pvec->nr].idx = idx;
	pvec->nr++;
	return (pvec->nr == KVM_PAGE_ARRAY_NR);
}

static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	int i, ret, nr_unsync_leaf = 0;
1181

1182
	for_each_unsync_children(sp->unsync_child_bitmap, i) {
1183
		struct kvm_mmu_page *child;
1184 1185
		u64 ent = sp->spt[i];

1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214
		if (!is_shadow_present_pte(ent) || is_large_pte(ent))
			goto clear_child_bitmap;

		child = page_header(ent & PT64_BASE_ADDR_MASK);

		if (child->unsync_children) {
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;

			ret = __mmu_unsync_walk(child, pvec);
			if (!ret)
				goto clear_child_bitmap;
			else if (ret > 0)
				nr_unsync_leaf += ret;
			else
				return ret;
		} else if (child->unsync) {
			nr_unsync_leaf++;
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;
		} else
			 goto clear_child_bitmap;

		continue;

clear_child_bitmap:
		__clear_bit(i, sp->unsync_child_bitmap);
		sp->unsync_children--;
		WARN_ON((int)sp->unsync_children < 0);
1215 1216 1217
	}


1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
	return nr_unsync_leaf;
}

static int mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	if (!sp->unsync_children)
		return 0;

	mmu_pages_add(pvec, sp, 0);
	return __mmu_unsync_walk(sp, pvec);
1229 1230 1231 1232 1233
}

static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	WARN_ON(!sp->unsync);
1234
	trace_kvm_mmu_sync_page(sp);
1235 1236 1237 1238
	sp->unsync = 0;
	--kvm->stat.mmu_unsync;
}

1239 1240 1241 1242
static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				    struct list_head *invalid_list);
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list);
1243

1244 1245
#define for_each_gfn_sp(kvm, sp, gfn, pos)				\
  hlist_for_each_entry(sp, pos,						\
1246 1247 1248
   &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)	\
	if ((sp)->gfn != (gfn)) {} else

1249 1250
#define for_each_gfn_indirect_valid_sp(kvm, sp, gfn, pos)		\
  hlist_for_each_entry(sp, pos,						\
1251 1252 1253 1254
   &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)	\
		if ((sp)->gfn != (gfn) || (sp)->role.direct ||		\
			(sp)->role.invalid) {} else

1255
/* @sp->gfn should be write-protected at the call site */
1256
static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1257
			   struct list_head *invalid_list, bool clear_unsync)
1258
{
1259
	if (sp->role.cr4_pae != !!is_pae(vcpu)) {
1260
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1261 1262 1263
		return 1;
	}

1264
	if (clear_unsync)
1265 1266
		kvm_unlink_unsync_page(vcpu->kvm, sp);

1267
	if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
1268
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1269 1270 1271 1272 1273 1274 1275
		return 1;
	}

	kvm_mmu_flush_tlb(vcpu);
	return 0;
}

1276 1277 1278
static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
				   struct kvm_mmu_page *sp)
{
1279
	LIST_HEAD(invalid_list);
1280 1281
	int ret;

1282
	ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
1283
	if (ret)
1284 1285
		kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);

1286 1287 1288
	return ret;
}

1289 1290
static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
			 struct list_head *invalid_list)
1291
{
1292
	return __kvm_sync_page(vcpu, sp, invalid_list, true);
1293 1294
}

1295 1296 1297 1298
/* @gfn should be write-protected at the call site */
static void kvm_sync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
{
	struct kvm_mmu_page *s;
1299
	struct hlist_node *node;
1300
	LIST_HEAD(invalid_list);
1301 1302
	bool flush = false;

1303
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1304
		if (!s->unsync)
1305 1306 1307
			continue;

		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1308
		kvm_unlink_unsync_page(vcpu->kvm, s);
1309
		if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
1310
			(vcpu->arch.mmu.sync_page(vcpu, s))) {
1311
			kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
1312 1313 1314 1315 1316
			continue;
		}
		flush = true;
	}

1317
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1318 1319 1320 1321
	if (flush)
		kvm_mmu_flush_tlb(vcpu);
}

1322 1323 1324
struct mmu_page_path {
	struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
	unsigned int idx[PT64_ROOT_LEVEL-1];
1325 1326
};

1327 1328 1329 1330 1331 1332
#define for_each_sp(pvec, sp, parents, i)			\
		for (i = mmu_pages_next(&pvec, &parents, -1),	\
			sp = pvec.page[i].sp;			\
			i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});	\
			i = mmu_pages_next(&pvec, &parents, i))

1333 1334 1335
static int mmu_pages_next(struct kvm_mmu_pages *pvec,
			  struct mmu_page_path *parents,
			  int i)
1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353
{
	int n;

	for (n = i+1; n < pvec->nr; n++) {
		struct kvm_mmu_page *sp = pvec->page[n].sp;

		if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
			parents->idx[0] = pvec->page[n].idx;
			return n;
		}

		parents->parent[sp->role.level-2] = sp;
		parents->idx[sp->role.level-1] = pvec->page[n].idx;
	}

	return n;
}

1354
static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1355
{
1356 1357 1358 1359 1360
	struct kvm_mmu_page *sp;
	unsigned int level = 0;

	do {
		unsigned int idx = parents->idx[level];
1361

1362 1363 1364 1365 1366 1367 1368 1369 1370
		sp = parents->parent[level];
		if (!sp)
			return;

		--sp->unsync_children;
		WARN_ON((int)sp->unsync_children < 0);
		__clear_bit(idx, sp->unsync_child_bitmap);
		level++;
	} while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1371 1372
}

1373 1374 1375
static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
			       struct mmu_page_path *parents,
			       struct kvm_mmu_pages *pvec)
1376
{
1377 1378 1379
	parents->parent[parent->role.level-1] = NULL;
	pvec->nr = 0;
}
1380

1381 1382 1383 1384 1385 1386 1387
static void mmu_sync_children(struct kvm_vcpu *vcpu,
			      struct kvm_mmu_page *parent)
{
	int i;
	struct kvm_mmu_page *sp;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
1388
	LIST_HEAD(invalid_list);
1389 1390 1391

	kvm_mmu_pages_init(parent, &parents, &pages);
	while (mmu_unsync_walk(parent, &pages)) {
1392 1393 1394 1395 1396 1397 1398 1399
		int protected = 0;

		for_each_sp(pages, sp, parents, i)
			protected |= rmap_write_protect(vcpu->kvm, sp->gfn);

		if (protected)
			kvm_flush_remote_tlbs(vcpu->kvm);

1400
		for_each_sp(pages, sp, parents, i) {
1401
			kvm_sync_page(vcpu, sp, &invalid_list);
1402 1403
			mmu_pages_clear_parents(&parents);
		}
1404
		kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1405
		cond_resched_lock(&vcpu->kvm->mmu_lock);
1406 1407
		kvm_mmu_pages_init(parent, &parents, &pages);
	}
1408 1409
}

1410 1411 1412 1413
static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
1414
					     int direct,
1415
					     unsigned access,
1416
					     u64 *parent_pte)
1417 1418 1419
{
	union kvm_mmu_page_role role;
	unsigned quadrant;
1420
	struct kvm_mmu_page *sp;
1421
	struct hlist_node *node;
1422
	bool need_sync = false;
1423

1424
	role = vcpu->arch.mmu.base_role;
1425
	role.level = level;
1426
	role.direct = direct;
1427
	if (role.direct)
1428
		role.cr4_pae = 0;
1429
	role.access = access;
1430 1431
	if (!vcpu->arch.mmu.direct_map
	    && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
1432 1433 1434 1435
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
1436
	for_each_gfn_sp(vcpu->kvm, sp, gfn, node) {
1437 1438
		if (!need_sync && sp->unsync)
			need_sync = true;
1439

1440 1441
		if (sp->role.word != role.word)
			continue;
1442

1443 1444
		if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
			break;
1445

1446 1447
		mmu_page_add_parent_pte(vcpu, sp, parent_pte);
		if (sp->unsync_children) {
1448
			kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
1449 1450 1451
			kvm_mmu_mark_parents_unsync(sp);
		} else if (sp->unsync)
			kvm_mmu_mark_parents_unsync(sp);
1452

1453 1454 1455
		trace_kvm_mmu_get_page(sp, false);
		return sp;
	}
A
Avi Kivity 已提交
1456
	++vcpu->kvm->stat.mmu_cache_miss;
1457
	sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
1458 1459 1460 1461
	if (!sp)
		return sp;
	sp->gfn = gfn;
	sp->role = role;
1462 1463
	hlist_add_head(&sp->hash_link,
		&vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
1464
	if (!direct) {
1465 1466
		if (rmap_write_protect(vcpu->kvm, gfn))
			kvm_flush_remote_tlbs(vcpu->kvm);
1467 1468 1469
		if (level > PT_PAGE_TABLE_LEVEL && need_sync)
			kvm_sync_pages(vcpu, gfn);

1470 1471
		account_shadowed(vcpu->kvm, gfn);
	}
1472 1473 1474 1475
	if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
		vcpu->arch.mmu.prefetch_page(vcpu, sp);
	else
		nonpaging_prefetch_page(vcpu, sp);
A
Avi Kivity 已提交
1476
	trace_kvm_mmu_get_page(sp, true);
1477
	return sp;
1478 1479
}

1480 1481 1482 1483 1484 1485
static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
			     struct kvm_vcpu *vcpu, u64 addr)
{
	iterator->addr = addr;
	iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
	iterator->level = vcpu->arch.mmu.shadow_root_level;
1486 1487 1488 1489 1490 1491

	if (iterator->level == PT64_ROOT_LEVEL &&
	    vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
	    !vcpu->arch.mmu.direct_map)
		--iterator->level;

1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505
	if (iterator->level == PT32E_ROOT_LEVEL) {
		iterator->shadow_addr
			= vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
		--iterator->level;
		if (!iterator->shadow_addr)
			iterator->level = 0;
	}
}

static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
{
	if (iterator->level < PT_PAGE_TABLE_LEVEL)
		return false;
1506

1507 1508 1509 1510 1511 1512 1513
	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
	return true;
}

static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
{
1514 1515 1516 1517 1518
	if (is_last_spte(*iterator->sptep, iterator->level)) {
		iterator->level = 0;
		return;
	}

1519 1520 1521 1522
	iterator->shadow_addr = *iterator->sptep & PT64_BASE_ADDR_MASK;
	--iterator->level;
}

1523 1524 1525 1526 1527 1528 1529
static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp)
{
	u64 spte;

	spte = __pa(sp->spt)
		| PT_PRESENT_MASK | PT_ACCESSED_MASK
		| PT_WRITABLE_MASK | PT_USER_MASK;
1530
	__set_spte(sptep, spte);
1531 1532
}

1533 1534 1535 1536 1537 1538 1539 1540
static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
{
	if (is_large_pte(*sptep)) {
		drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
		kvm_flush_remote_tlbs(vcpu->kvm);
	}
}

1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557
static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
				   unsigned direct_access)
{
	if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
		struct kvm_mmu_page *child;

		/*
		 * For the direct sp, if the guest pte's dirty bit
		 * changed form clean to dirty, it will corrupt the
		 * sp's access: allow writable in the read-only sp,
		 * so we should update the spte at this point to get
		 * a new sp with the correct access.
		 */
		child = page_header(*sptep & PT64_BASE_ADDR_MASK);
		if (child->role.access == direct_access)
			return;

1558
		drop_parent_pte(child, sptep);
1559 1560 1561 1562
		kvm_flush_remote_tlbs(vcpu->kvm);
	}
}

1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574
static void mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
			     u64 *spte)
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
	if (is_shadow_present_pte(pte)) {
		if (is_last_spte(pte, sp->role.level))
			drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
		else {
			child = page_header(pte & PT64_BASE_ADDR_MASK);
1575
			drop_parent_pte(child, spte);
1576 1577 1578 1579 1580 1581 1582
		}
	}
	__set_spte(spte, shadow_trap_nonpresent_pte);
	if (is_large_pte(pte))
		--kvm->stat.lpages;
}

1583
static void kvm_mmu_page_unlink_children(struct kvm *kvm,
1584
					 struct kvm_mmu_page *sp)
1585
{
1586 1587
	unsigned i;

1588 1589
	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
		mmu_page_zap_pte(kvm, sp, sp->spt + i);
1590 1591
}

1592
static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
1593
{
1594
	mmu_page_remove_parent_pte(sp, parent_pte);
1595 1596
}

1597 1598 1599
static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
{
	int i;
1600
	struct kvm_vcpu *vcpu;
1601

1602 1603
	kvm_for_each_vcpu(i, vcpu, kvm)
		vcpu->arch.last_pte_updated = NULL;
1604 1605
}

1606
static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
1607 1608 1609
{
	u64 *parent_pte;

1610 1611
	while ((parent_pte = pte_list_next(&sp->parent_ptes, NULL)))
		drop_parent_pte(sp, parent_pte);
1612 1613
}

1614
static int mmu_zap_unsync_children(struct kvm *kvm,
1615 1616
				   struct kvm_mmu_page *parent,
				   struct list_head *invalid_list)
1617
{
1618 1619 1620
	int i, zapped = 0;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
1621

1622
	if (parent->role.level == PT_PAGE_TABLE_LEVEL)
1623
		return 0;
1624 1625 1626 1627 1628 1629

	kvm_mmu_pages_init(parent, &parents, &pages);
	while (mmu_unsync_walk(parent, &pages)) {
		struct kvm_mmu_page *sp;

		for_each_sp(pages, sp, parents, i) {
1630
			kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
1631
			mmu_pages_clear_parents(&parents);
1632
			zapped++;
1633 1634 1635 1636 1637
		}
		kvm_mmu_pages_init(parent, &parents, &pages);
	}

	return zapped;
1638 1639
}

1640 1641
static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				    struct list_head *invalid_list)
1642
{
1643
	int ret;
A
Avi Kivity 已提交
1644

1645
	trace_kvm_mmu_prepare_zap_page(sp);
1646
	++kvm->stat.mmu_shadow_zapped;
1647
	ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
1648
	kvm_mmu_page_unlink_children(kvm, sp);
1649
	kvm_mmu_unlink_parents(kvm, sp);
1650
	if (!sp->role.invalid && !sp->role.direct)
A
Avi Kivity 已提交
1651
		unaccount_shadowed(kvm, sp->gfn);
1652 1653
	if (sp->unsync)
		kvm_unlink_unsync_page(kvm, sp);
1654
	if (!sp->root_count) {
1655 1656
		/* Count self */
		ret++;
1657
		list_move(&sp->link, invalid_list);
1658
	} else {
A
Avi Kivity 已提交
1659
		list_move(&sp->link, &kvm->arch.active_mmu_pages);
1660 1661
		kvm_reload_remote_mmus(kvm);
	}
1662 1663

	sp->role.invalid = 1;
1664
	kvm_mmu_reset_last_pte_updated(kvm);
1665
	return ret;
1666 1667
}

1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list)
{
	struct kvm_mmu_page *sp;

	if (list_empty(invalid_list))
		return;

	kvm_flush_remote_tlbs(kvm);

	do {
		sp = list_first_entry(invalid_list, struct kvm_mmu_page, link);
		WARN_ON(!sp->role.invalid || sp->root_count);
		kvm_mmu_free_page(kvm, sp);
	} while (!list_empty(invalid_list));

}

1686 1687
/*
 * Changing the number of mmu pages allocated to the vm
1688
 * Note: if goal_nr_mmu_pages is too small, you will get dead lock
1689
 */
1690
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
1691
{
1692
	LIST_HEAD(invalid_list);
1693 1694 1695 1696 1697 1698
	/*
	 * If we set the number of mmu pages to be smaller be than the
	 * number of actived pages , we must to free some mmu pages before we
	 * change the value
	 */

1699 1700
	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
		while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages &&
1701
			!list_empty(&kvm->arch.active_mmu_pages)) {
1702 1703
			struct kvm_mmu_page *page;

1704
			page = container_of(kvm->arch.active_mmu_pages.prev,
1705
					    struct kvm_mmu_page, link);
1706 1707
			kvm_mmu_prepare_zap_page(kvm, page, &invalid_list);
			kvm_mmu_commit_zap_page(kvm, &invalid_list);
1708
		}
1709
		goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
1710 1711
	}

1712
	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
1713 1714
}

1715
static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
1716
{
1717
	struct kvm_mmu_page *sp;
1718
	struct hlist_node *node;
1719
	LIST_HEAD(invalid_list);
1720 1721
	int r;

1722
	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
1723
	r = 0;
1724 1725

	for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
1726
		pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
1727 1728
			 sp->role.word);
		r = 1;
1729
		kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
1730
	}
1731
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
1732
	return r;
1733 1734
}

1735
static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
1736
{
1737
	struct kvm_mmu_page *sp;
1738
	struct hlist_node *node;
1739
	LIST_HEAD(invalid_list);
1740

1741
	for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
1742
		pgprintk("%s: zap %llx %x\n",
1743
			 __func__, gfn, sp->role.word);
1744
		kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
1745
	}
1746
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
1747 1748
}

1749
static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
A
Avi Kivity 已提交
1750
{
1751
	int slot = memslot_id(kvm, gfn);
1752
	struct kvm_mmu_page *sp = page_header(__pa(pte));
A
Avi Kivity 已提交
1753

1754
	__set_bit(slot, sp->slot_bitmap);
A
Avi Kivity 已提交
1755 1756
}

1757 1758 1759 1760 1761 1762 1763 1764 1765 1766
static void mmu_convert_notrap(struct kvm_mmu_page *sp)
{
	int i;
	u64 *pt = sp->spt;

	if (shadow_trap_nonpresent_pte == shadow_notrap_nonpresent_pte)
		return;

	for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
		if (pt[i] == shadow_notrap_nonpresent_pte)
A
Avi Kivity 已提交
1767
			__set_spte(&pt[i], shadow_trap_nonpresent_pte);
1768 1769 1770
	}
}

1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863
/*
 * The function is based on mtrr_type_lookup() in
 * arch/x86/kernel/cpu/mtrr/generic.c
 */
static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
			 u64 start, u64 end)
{
	int i;
	u64 base, mask;
	u8 prev_match, curr_match;
	int num_var_ranges = KVM_NR_VAR_MTRR;

	if (!mtrr_state->enabled)
		return 0xFF;

	/* Make end inclusive end, instead of exclusive */
	end--;

	/* Look in fixed ranges. Just return the type as per start */
	if (mtrr_state->have_fixed && (start < 0x100000)) {
		int idx;

		if (start < 0x80000) {
			idx = 0;
			idx += (start >> 16);
			return mtrr_state->fixed_ranges[idx];
		} else if (start < 0xC0000) {
			idx = 1 * 8;
			idx += ((start - 0x80000) >> 14);
			return mtrr_state->fixed_ranges[idx];
		} else if (start < 0x1000000) {
			idx = 3 * 8;
			idx += ((start - 0xC0000) >> 12);
			return mtrr_state->fixed_ranges[idx];
		}
	}

	/*
	 * Look in variable ranges
	 * Look of multiple ranges matching this address and pick type
	 * as per MTRR precedence
	 */
	if (!(mtrr_state->enabled & 2))
		return mtrr_state->def_type;

	prev_match = 0xFF;
	for (i = 0; i < num_var_ranges; ++i) {
		unsigned short start_state, end_state;

		if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
			continue;

		base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
		       (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
		mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
		       (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);

		start_state = ((start & mask) == (base & mask));
		end_state = ((end & mask) == (base & mask));
		if (start_state != end_state)
			return 0xFE;

		if ((start & mask) != (base & mask))
			continue;

		curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
		if (prev_match == 0xFF) {
			prev_match = curr_match;
			continue;
		}

		if (prev_match == MTRR_TYPE_UNCACHABLE ||
		    curr_match == MTRR_TYPE_UNCACHABLE)
			return MTRR_TYPE_UNCACHABLE;

		if ((prev_match == MTRR_TYPE_WRBACK &&
		     curr_match == MTRR_TYPE_WRTHROUGH) ||
		    (prev_match == MTRR_TYPE_WRTHROUGH &&
		     curr_match == MTRR_TYPE_WRBACK)) {
			prev_match = MTRR_TYPE_WRTHROUGH;
			curr_match = MTRR_TYPE_WRTHROUGH;
		}

		if (prev_match != curr_match)
			return MTRR_TYPE_UNCACHABLE;
	}

	if (prev_match != 0xFF)
		return prev_match;

	return mtrr_state->def_type;
}

1864
u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
1865 1866 1867 1868 1869 1870 1871 1872 1873
{
	u8 mtrr;

	mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
			     (gfn << PAGE_SHIFT) + PAGE_SIZE);
	if (mtrr == 0xfe || mtrr == 0xff)
		mtrr = MTRR_TYPE_WRBACK;
	return mtrr;
}
1874
EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
1875

1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886
static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
{
	trace_kvm_mmu_unsync_page(sp);
	++vcpu->kvm->stat.mmu_unsync;
	sp->unsync = 1;

	kvm_mmu_mark_parents_unsync(sp);
	mmu_convert_notrap(sp);
}

static void kvm_unsync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
1887 1888
{
	struct kvm_mmu_page *s;
1889
	struct hlist_node *node;
1890

1891
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1892
		if (s->unsync)
1893
			continue;
1894 1895
		WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
		__kvm_unsync_page(vcpu, s);
1896 1897 1898 1899 1900 1901
	}
}

static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
				  bool can_unsync)
{
1902
	struct kvm_mmu_page *s;
1903
	struct hlist_node *node;
1904 1905
	bool need_unsync = false;

1906
	for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1907 1908 1909
		if (!can_unsync)
			return 1;

1910
		if (s->role.level != PT_PAGE_TABLE_LEVEL)
1911
			return 1;
1912 1913

		if (!need_unsync && !s->unsync) {
1914
			if (!oos_shadow)
1915 1916 1917
				return 1;
			need_unsync = true;
		}
1918
	}
1919 1920
	if (need_unsync)
		kvm_unsync_pages(vcpu, gfn);
1921 1922 1923
	return 0;
}

A
Avi Kivity 已提交
1924
static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
M
Marcelo Tosatti 已提交
1925
		    unsigned pte_access, int user_fault,
1926
		    int write_fault, int level,
1927
		    gfn_t gfn, pfn_t pfn, bool speculative,
1928
		    bool can_unsync, bool host_writable)
1929
{
1930
	u64 spte, entry = *sptep;
M
Marcelo Tosatti 已提交
1931
	int ret = 0;
S
Sheng Yang 已提交
1932

1933 1934 1935 1936 1937
	/*
	 * We don't set the accessed bit, since we sometimes want to see
	 * whether the guest actually used the pte (in order to detect
	 * demand paging).
	 */
1938
	spte = PT_PRESENT_MASK;
1939
	if (!speculative)
1940
		spte |= shadow_accessed_mask;
1941

S
Sheng Yang 已提交
1942 1943 1944 1945
	if (pte_access & ACC_EXEC_MASK)
		spte |= shadow_x_mask;
	else
		spte |= shadow_nx_mask;
1946
	if (pte_access & ACC_USER_MASK)
S
Sheng Yang 已提交
1947
		spte |= shadow_user_mask;
1948
	if (level > PT_PAGE_TABLE_LEVEL)
M
Marcelo Tosatti 已提交
1949
		spte |= PT_PAGE_SIZE_MASK;
1950
	if (tdp_enabled)
1951 1952
		spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
			kvm_is_mmio_pfn(pfn));
1953

1954
	if (host_writable)
1955
		spte |= SPTE_HOST_WRITEABLE;
1956 1957
	else
		pte_access &= ~ACC_WRITE_MASK;
1958

1959
	spte |= (u64)pfn << PAGE_SHIFT;
1960 1961

	if ((pte_access & ACC_WRITE_MASK)
1962 1963
	    || (!vcpu->arch.mmu.direct_map && write_fault
		&& !is_write_protection(vcpu) && !user_fault)) {
1964

1965 1966
		if (level > PT_PAGE_TABLE_LEVEL &&
		    has_wrprotected_page(vcpu->kvm, gfn, level)) {
1967
			ret = 1;
A
Avi Kivity 已提交
1968 1969
			drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
			goto done;
1970 1971
		}

1972 1973
		spte |= PT_WRITABLE_MASK;

1974
		if (!vcpu->arch.mmu.direct_map
1975
		    && !(pte_access & ACC_WRITE_MASK)) {
1976
			spte &= ~PT_USER_MASK;
1977 1978 1979 1980 1981 1982 1983 1984 1985
			/*
			 * If we converted a user page to a kernel page,
			 * so that the kernel can write to it when cr0.wp=0,
			 * then we should prevent the kernel from executing it
			 * if SMEP is enabled.
			 */
			if (kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
				spte |= PT64_NX_MASK;
		}
1986

1987 1988 1989 1990 1991 1992
		/*
		 * Optimization: for pte sync, if spte was writable the hash
		 * lookup is unnecessary (and expensive). Write protection
		 * is responsibility of mmu_get_page / kvm_sync_page.
		 * Same reasoning can be applied to dirty page accounting.
		 */
1993
		if (!can_unsync && is_writable_pte(*sptep))
1994 1995
			goto set_pte;

1996
		if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
1997
			pgprintk("%s: found shadow page for %llx, marking ro\n",
1998
				 __func__, gfn);
M
Marcelo Tosatti 已提交
1999
			ret = 1;
2000
			pte_access &= ~ACC_WRITE_MASK;
2001
			if (is_writable_pte(spte))
2002 2003 2004 2005 2006 2007 2008
				spte &= ~PT_WRITABLE_MASK;
		}
	}

	if (pte_access & ACC_WRITE_MASK)
		mark_page_dirty(vcpu->kvm, gfn);

2009
set_pte:
2010
	update_spte(sptep, spte);
2011 2012 2013 2014 2015 2016 2017 2018
	/*
	 * If we overwrite a writable spte with a read-only one we
	 * should flush remote TLBs. Otherwise rmap_write_protect
	 * will find a read-only spte, even though the writable spte
	 * might be cached on a CPU's TLB.
	 */
	if (is_writable_pte(entry) && !is_writable_pte(*sptep))
		kvm_flush_remote_tlbs(vcpu->kvm);
A
Avi Kivity 已提交
2019
done:
M
Marcelo Tosatti 已提交
2020 2021 2022
	return ret;
}

A
Avi Kivity 已提交
2023
static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
M
Marcelo Tosatti 已提交
2024
			 unsigned pt_access, unsigned pte_access,
2025
			 int user_fault, int write_fault,
2026
			 int *emulate, int level, gfn_t gfn,
2027
			 pfn_t pfn, bool speculative,
2028
			 bool host_writable)
M
Marcelo Tosatti 已提交
2029 2030
{
	int was_rmapped = 0;
2031
	int rmap_count;
M
Marcelo Tosatti 已提交
2032 2033

	pgprintk("%s: spte %llx access %x write_fault %d"
2034
		 " user_fault %d gfn %llx\n",
A
Avi Kivity 已提交
2035
		 __func__, *sptep, pt_access,
M
Marcelo Tosatti 已提交
2036 2037
		 write_fault, user_fault, gfn);

A
Avi Kivity 已提交
2038
	if (is_rmap_spte(*sptep)) {
M
Marcelo Tosatti 已提交
2039 2040 2041 2042
		/*
		 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
		 * the parent of the now unreachable PTE.
		 */
2043 2044
		if (level > PT_PAGE_TABLE_LEVEL &&
		    !is_large_pte(*sptep)) {
M
Marcelo Tosatti 已提交
2045
			struct kvm_mmu_page *child;
A
Avi Kivity 已提交
2046
			u64 pte = *sptep;
M
Marcelo Tosatti 已提交
2047 2048

			child = page_header(pte & PT64_BASE_ADDR_MASK);
2049
			drop_parent_pte(child, sptep);
2050
			kvm_flush_remote_tlbs(vcpu->kvm);
A
Avi Kivity 已提交
2051
		} else if (pfn != spte_to_pfn(*sptep)) {
2052
			pgprintk("hfn old %llx new %llx\n",
A
Avi Kivity 已提交
2053
				 spte_to_pfn(*sptep), pfn);
A
Avi Kivity 已提交
2054
			drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
2055
			kvm_flush_remote_tlbs(vcpu->kvm);
2056 2057
		} else
			was_rmapped = 1;
M
Marcelo Tosatti 已提交
2058
	}
2059

A
Avi Kivity 已提交
2060
	if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
2061
		      level, gfn, pfn, speculative, true,
2062
		      host_writable)) {
M
Marcelo Tosatti 已提交
2063
		if (write_fault)
2064
			*emulate = 1;
2065
		kvm_mmu_flush_tlb(vcpu);
2066
	}
M
Marcelo Tosatti 已提交
2067

A
Avi Kivity 已提交
2068
	pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2069
	pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
A
Avi Kivity 已提交
2070
		 is_large_pte(*sptep)? "2MB" : "4kB",
2071 2072
		 *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
		 *sptep, sptep);
A
Avi Kivity 已提交
2073
	if (!was_rmapped && is_large_pte(*sptep))
M
Marcelo Tosatti 已提交
2074 2075
		++vcpu->kvm->stat.lpages;

2076 2077 2078 2079 2080 2081 2082
	if (is_shadow_present_pte(*sptep)) {
		page_header_update_slot(vcpu->kvm, sptep, gfn);
		if (!was_rmapped) {
			rmap_count = rmap_add(vcpu, sptep, gfn);
			if (rmap_count > RMAP_RECYCLE_THRESHOLD)
				rmap_recycle(vcpu, sptep, gfn);
		}
2083
	}
2084
	kvm_release_pfn_clean(pfn);
2085
	if (speculative) {
A
Avi Kivity 已提交
2086
		vcpu->arch.last_pte_updated = sptep;
2087 2088
		vcpu->arch.last_pte_gfn = gfn;
	}
2089 2090
}

A
Avi Kivity 已提交
2091 2092 2093 2094
static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
{
}

2095 2096 2097 2098 2099 2100
static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
				     bool no_dirty_log)
{
	struct kvm_memory_slot *slot;
	unsigned long hva;

2101
	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121
	if (!slot) {
		get_page(bad_page);
		return page_to_pfn(bad_page);
	}

	hva = gfn_to_hva_memslot(slot, gfn);

	return hva_to_pfn_atomic(vcpu->kvm, hva);
}

static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *sp,
				    u64 *start, u64 *end)
{
	struct page *pages[PTE_PREFETCH_NUM];
	unsigned access = sp->role.access;
	int i, ret;
	gfn_t gfn;

	gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2122
	if (!gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK))
2123 2124 2125 2126 2127 2128 2129 2130
		return -1;

	ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
	if (ret <= 0)
		return -1;

	for (i = 0; i < ret; i++, gfn++, start++)
		mmu_set_spte(vcpu, start, ACC_ALL,
2131
			     access, 0, 0, NULL,
2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180
			     sp->role.level, gfn,
			     page_to_pfn(pages[i]), true, true);

	return 0;
}

static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
				  struct kvm_mmu_page *sp, u64 *sptep)
{
	u64 *spte, *start = NULL;
	int i;

	WARN_ON(!sp->role.direct);

	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
	spte = sp->spt + i;

	for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
		if (*spte != shadow_trap_nonpresent_pte || spte == sptep) {
			if (!start)
				continue;
			if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
				break;
			start = NULL;
		} else if (!start)
			start = spte;
	}
}

static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
{
	struct kvm_mmu_page *sp;

	/*
	 * Since it's no accessed bit on EPT, it's no way to
	 * distinguish between actually accessed translations
	 * and prefetched, so disable pte prefetch if EPT is
	 * enabled.
	 */
	if (!shadow_accessed_mask)
		return;

	sp = page_header(__pa(sptep));
	if (sp->role.level > PT_PAGE_TABLE_LEVEL)
		return;

	__direct_pte_prefetch(vcpu, sp, sptep);
}

2181
static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
2182 2183
			int map_writable, int level, gfn_t gfn, pfn_t pfn,
			bool prefault)
2184
{
2185
	struct kvm_shadow_walk_iterator iterator;
2186
	struct kvm_mmu_page *sp;
2187
	int emulate = 0;
2188
	gfn_t pseudo_gfn;
A
Avi Kivity 已提交
2189

2190
	for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2191
		if (iterator.level == level) {
2192 2193 2194
			unsigned pte_access = ACC_ALL;

			mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, pte_access,
2195
				     0, write, &emulate,
2196
				     level, gfn, pfn, prefault, map_writable);
2197
			direct_pte_prefetch(vcpu, iterator.sptep);
2198 2199
			++vcpu->stat.pf_fixed;
			break;
A
Avi Kivity 已提交
2200 2201
		}

2202
		if (*iterator.sptep == shadow_trap_nonpresent_pte) {
2203 2204 2205 2206
			u64 base_addr = iterator.addr;

			base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
			pseudo_gfn = base_addr >> PAGE_SHIFT;
2207 2208 2209 2210 2211 2212 2213 2214
			sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
					      iterator.level - 1,
					      1, ACC_ALL, iterator.sptep);
			if (!sp) {
				pgprintk("nonpaging_map: ENOMEM\n");
				kvm_release_pfn_clean(pfn);
				return -ENOMEM;
			}
2215

A
Avi Kivity 已提交
2216 2217 2218
			__set_spte(iterator.sptep,
				   __pa(sp->spt)
				   | PT_PRESENT_MASK | PT_WRITABLE_MASK
2219 2220
				   | shadow_user_mask | shadow_x_mask
				   | shadow_accessed_mask);
2221 2222
		}
	}
2223
	return emulate;
A
Avi Kivity 已提交
2224 2225
}

H
Huang Ying 已提交
2226
static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2227
{
H
Huang Ying 已提交
2228 2229 2230 2231 2232 2233 2234
	siginfo_t info;

	info.si_signo	= SIGBUS;
	info.si_errno	= 0;
	info.si_code	= BUS_MCEERR_AR;
	info.si_addr	= (void __user *)address;
	info.si_addr_lsb = PAGE_SHIFT;
2235

H
Huang Ying 已提交
2236
	send_sig_info(SIGBUS, &info, tsk);
2237 2238
}

2239 2240
static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gva_t gva,
			       unsigned access, gfn_t gfn, pfn_t pfn)
2241 2242 2243
{
	kvm_release_pfn_clean(pfn);
	if (is_hwpoison_pfn(pfn)) {
2244
		kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current);
2245
		return 0;
2246 2247 2248
	} else if (is_fault_pfn(pfn))
		return -EFAULT;

2249
	vcpu_cache_mmio_info(vcpu, gva, gfn, access);
2250 2251 2252
	return 1;
}

2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294
static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
					gfn_t *gfnp, pfn_t *pfnp, int *levelp)
{
	pfn_t pfn = *pfnp;
	gfn_t gfn = *gfnp;
	int level = *levelp;

	/*
	 * Check if it's a transparent hugepage. If this would be an
	 * hugetlbfs page, level wouldn't be set to
	 * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
	 * here.
	 */
	if (!is_error_pfn(pfn) && !kvm_is_mmio_pfn(pfn) &&
	    level == PT_PAGE_TABLE_LEVEL &&
	    PageTransCompound(pfn_to_page(pfn)) &&
	    !has_wrprotected_page(vcpu->kvm, gfn, PT_DIRECTORY_LEVEL)) {
		unsigned long mask;
		/*
		 * mmu_notifier_retry was successful and we hold the
		 * mmu_lock here, so the pmd can't become splitting
		 * from under us, and in turn
		 * __split_huge_page_refcount() can't run from under
		 * us and we can safely transfer the refcount from
		 * PG_tail to PG_head as we switch the pfn to tail to
		 * head.
		 */
		*levelp = level = PT_DIRECTORY_LEVEL;
		mask = KVM_PAGES_PER_HPAGE(level) - 1;
		VM_BUG_ON((gfn & mask) != (pfn & mask));
		if (pfn & mask) {
			gfn &= ~mask;
			*gfnp = gfn;
			kvm_release_pfn_clean(pfn);
			pfn &= ~mask;
			if (!get_page_unless_zero(pfn_to_page(pfn)))
				BUG();
			*pfnp = pfn;
		}
	}
}

2295
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
2296 2297 2298
			 gva_t gva, pfn_t *pfn, bool write, bool *writable);

static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn,
2299
			 bool prefault)
2300 2301
{
	int r;
2302
	int level;
2303
	int force_pt_level;
2304
	pfn_t pfn;
2305
	unsigned long mmu_seq;
2306
	bool map_writable;
2307

2308 2309 2310 2311 2312 2313 2314 2315 2316 2317
	force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
	if (likely(!force_pt_level)) {
		level = mapping_level(vcpu, gfn);
		/*
		 * This path builds a PAE pagetable - so we can map
		 * 2mb pages at maximum. Therefore check if the level
		 * is larger than that.
		 */
		if (level > PT_DIRECTORY_LEVEL)
			level = PT_DIRECTORY_LEVEL;
2318

2319 2320 2321
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
	} else
		level = PT_PAGE_TABLE_LEVEL;
M
Marcelo Tosatti 已提交
2322

2323
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
2324
	smp_rmb();
2325

2326
	if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
2327
		return 0;
2328

2329
	/* mmio */
2330
	if (is_error_pfn(pfn))
2331
		return kvm_handle_bad_page(vcpu, v, ACC_ALL, gfn, pfn);
2332

2333
	spin_lock(&vcpu->kvm->mmu_lock);
2334 2335
	if (mmu_notifier_retry(vcpu, mmu_seq))
		goto out_unlock;
2336
	kvm_mmu_free_some_pages(vcpu);
2337 2338
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
2339 2340
	r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
			 prefault);
2341 2342 2343
	spin_unlock(&vcpu->kvm->mmu_lock);


2344
	return r;
2345 2346 2347 2348 2349

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
2350 2351 2352
}


2353 2354 2355
static void mmu_free_roots(struct kvm_vcpu *vcpu)
{
	int i;
2356
	struct kvm_mmu_page *sp;
2357
	LIST_HEAD(invalid_list);
2358

2359
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
A
Avi Kivity 已提交
2360
		return;
2361
	spin_lock(&vcpu->kvm->mmu_lock);
2362 2363 2364
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
	    (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
	     vcpu->arch.mmu.direct_map)) {
2365
		hpa_t root = vcpu->arch.mmu.root_hpa;
2366

2367 2368
		sp = page_header(root);
		--sp->root_count;
2369 2370 2371 2372
		if (!sp->root_count && sp->role.invalid) {
			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
			kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
		}
2373
		vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2374
		spin_unlock(&vcpu->kvm->mmu_lock);
2375 2376 2377
		return;
	}
	for (i = 0; i < 4; ++i) {
2378
		hpa_t root = vcpu->arch.mmu.pae_root[i];
2379

A
Avi Kivity 已提交
2380 2381
		if (root) {
			root &= PT64_BASE_ADDR_MASK;
2382 2383
			sp = page_header(root);
			--sp->root_count;
2384
			if (!sp->root_count && sp->role.invalid)
2385 2386
				kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
							 &invalid_list);
A
Avi Kivity 已提交
2387
		}
2388
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2389
	}
2390
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2391
	spin_unlock(&vcpu->kvm->mmu_lock);
2392
	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2393 2394
}

2395 2396 2397 2398 2399
static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
{
	int ret = 0;

	if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
2400
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2401 2402 2403 2404 2405 2406
		ret = 1;
	}

	return ret;
}

2407 2408 2409
static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_page *sp;
2410
	unsigned i;
2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426

	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		spin_lock(&vcpu->kvm->mmu_lock);
		kvm_mmu_free_some_pages(vcpu);
		sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
				      1, ACC_ALL, NULL);
		++sp->root_count;
		spin_unlock(&vcpu->kvm->mmu_lock);
		vcpu->arch.mmu.root_hpa = __pa(sp->spt);
	} else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
		for (i = 0; i < 4; ++i) {
			hpa_t root = vcpu->arch.mmu.pae_root[i];

			ASSERT(!VALID_PAGE(root));
			spin_lock(&vcpu->kvm->mmu_lock);
			kvm_mmu_free_some_pages(vcpu);
2427 2428
			sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
					      i << 30,
2429 2430 2431 2432 2433 2434 2435
					      PT32_ROOT_LEVEL, 1, ACC_ALL,
					      NULL);
			root = __pa(sp->spt);
			++sp->root_count;
			spin_unlock(&vcpu->kvm->mmu_lock);
			vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
		}
2436
		vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
2437 2438 2439 2440 2441 2442 2443
	} else
		BUG();

	return 0;
}

static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
2444
{
2445
	struct kvm_mmu_page *sp;
2446 2447 2448
	u64 pdptr, pm_mask;
	gfn_t root_gfn;
	int i;
2449

2450
	root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
2451

2452 2453 2454 2455 2456 2457 2458 2459
	if (mmu_check_root(vcpu, root_gfn))
		return 1;

	/*
	 * Do we shadow a long mode page table? If so we need to
	 * write-protect the guests page table root.
	 */
	if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
2460
		hpa_t root = vcpu->arch.mmu.root_hpa;
2461 2462

		ASSERT(!VALID_PAGE(root));
2463

2464
		spin_lock(&vcpu->kvm->mmu_lock);
2465
		kvm_mmu_free_some_pages(vcpu);
2466 2467
		sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
				      0, ACC_ALL, NULL);
2468 2469
		root = __pa(sp->spt);
		++sp->root_count;
2470
		spin_unlock(&vcpu->kvm->mmu_lock);
2471
		vcpu->arch.mmu.root_hpa = root;
2472
		return 0;
2473
	}
2474

2475 2476
	/*
	 * We shadow a 32 bit page table. This may be a legacy 2-level
2477 2478
	 * or a PAE 3-level page table. In either case we need to be aware that
	 * the shadow page table may be a PAE or a long mode page table.
2479
	 */
2480 2481 2482 2483
	pm_mask = PT_PRESENT_MASK;
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
		pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;

2484
	for (i = 0; i < 4; ++i) {
2485
		hpa_t root = vcpu->arch.mmu.pae_root[i];
2486 2487

		ASSERT(!VALID_PAGE(root));
2488
		if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
2489
			pdptr = kvm_pdptr_read_mmu(vcpu, &vcpu->arch.mmu, i);
2490
			if (!is_present_gpte(pdptr)) {
2491
				vcpu->arch.mmu.pae_root[i] = 0;
A
Avi Kivity 已提交
2492 2493
				continue;
			}
A
Avi Kivity 已提交
2494
			root_gfn = pdptr >> PAGE_SHIFT;
2495 2496
			if (mmu_check_root(vcpu, root_gfn))
				return 1;
2497
		}
2498
		spin_lock(&vcpu->kvm->mmu_lock);
2499
		kvm_mmu_free_some_pages(vcpu);
2500
		sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
2501
				      PT32_ROOT_LEVEL, 0,
2502
				      ACC_ALL, NULL);
2503 2504
		root = __pa(sp->spt);
		++sp->root_count;
2505 2506
		spin_unlock(&vcpu->kvm->mmu_lock);

2507
		vcpu->arch.mmu.pae_root[i] = root | pm_mask;
2508
	}
2509
	vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535

	/*
	 * If we shadow a 32 bit page table with a long mode page
	 * table we enter this path.
	 */
	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
		if (vcpu->arch.mmu.lm_root == NULL) {
			/*
			 * The additional page necessary for this is only
			 * allocated on demand.
			 */

			u64 *lm_root;

			lm_root = (void*)get_zeroed_page(GFP_KERNEL);
			if (lm_root == NULL)
				return 1;

			lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;

			vcpu->arch.mmu.lm_root = lm_root;
		}

		vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
	}

2536
	return 0;
2537 2538
}

2539 2540 2541 2542 2543 2544 2545 2546
static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
{
	if (vcpu->arch.mmu.direct_map)
		return mmu_alloc_direct_roots(vcpu);
	else
		return mmu_alloc_shadow_roots(vcpu);
}

2547 2548 2549 2550 2551
static void mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	int i;
	struct kvm_mmu_page *sp;

2552 2553 2554
	if (vcpu->arch.mmu.direct_map)
		return;

2555 2556
	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
		return;
2557

2558
	vcpu_clear_mmio_info(vcpu, ~0ul);
2559
	trace_kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
2560
	if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
2561 2562 2563
		hpa_t root = vcpu->arch.mmu.root_hpa;
		sp = page_header(root);
		mmu_sync_children(vcpu, sp);
2564
		trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
2565 2566 2567 2568 2569
		return;
	}
	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->arch.mmu.pae_root[i];

2570
		if (root && VALID_PAGE(root)) {
2571 2572 2573 2574 2575
			root &= PT64_BASE_ADDR_MASK;
			sp = page_header(root);
			mmu_sync_children(vcpu, sp);
		}
	}
2576
	trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
2577 2578 2579 2580 2581 2582
}

void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	spin_lock(&vcpu->kvm->mmu_lock);
	mmu_sync_roots(vcpu);
2583
	spin_unlock(&vcpu->kvm->mmu_lock);
2584 2585
}

2586
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
2587
				  u32 access, struct x86_exception *exception)
A
Avi Kivity 已提交
2588
{
2589 2590
	if (exception)
		exception->error_code = 0;
A
Avi Kivity 已提交
2591 2592 2593
	return vaddr;
}

2594
static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
2595 2596
					 u32 access,
					 struct x86_exception *exception)
2597
{
2598 2599
	if (exception)
		exception->error_code = 0;
2600 2601 2602
	return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access);
}

A
Avi Kivity 已提交
2603
static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
2604
				u32 error_code, bool prefault)
A
Avi Kivity 已提交
2605
{
2606
	gfn_t gfn;
2607
	int r;
A
Avi Kivity 已提交
2608

2609
	pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
2610 2611 2612
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;
2613

A
Avi Kivity 已提交
2614
	ASSERT(vcpu);
2615
	ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
2616

2617
	gfn = gva >> PAGE_SHIFT;
A
Avi Kivity 已提交
2618

2619
	return nonpaging_map(vcpu, gva & PAGE_MASK,
2620
			     error_code & PFERR_WRITE_MASK, gfn, prefault);
A
Avi Kivity 已提交
2621 2622
}

2623
static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
2624 2625
{
	struct kvm_arch_async_pf arch;
X
Xiao Guangrong 已提交
2626

2627
	arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
2628
	arch.gfn = gfn;
2629
	arch.direct_map = vcpu->arch.mmu.direct_map;
X
Xiao Guangrong 已提交
2630
	arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643

	return kvm_setup_async_pf(vcpu, gva, gfn, &arch);
}

static bool can_do_async_pf(struct kvm_vcpu *vcpu)
{
	if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
		     kvm_event_needs_reinjection(vcpu)))
		return false;

	return kvm_x86_ops->interrupt_allowed(vcpu);
}

2644
static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
2645
			 gva_t gva, pfn_t *pfn, bool write, bool *writable)
2646 2647 2648
{
	bool async;

2649
	*pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable);
2650 2651 2652 2653 2654 2655

	if (!async)
		return false; /* *pfn has correct page already */

	put_page(pfn_to_page(*pfn));

2656
	if (!prefault && can_do_async_pf(vcpu)) {
2657
		trace_kvm_try_async_get_page(gva, gfn);
2658 2659 2660 2661 2662 2663 2664 2665
		if (kvm_find_async_pf_gfn(vcpu, gfn)) {
			trace_kvm_async_pf_doublefault(gva, gfn);
			kvm_make_request(KVM_REQ_APF_HALT, vcpu);
			return true;
		} else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
			return true;
	}

2666
	*pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable);
2667 2668 2669 2670

	return false;
}

G
Gleb Natapov 已提交
2671
static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
2672
			  bool prefault)
2673
{
2674
	pfn_t pfn;
2675
	int r;
2676
	int level;
2677
	int force_pt_level;
M
Marcelo Tosatti 已提交
2678
	gfn_t gfn = gpa >> PAGE_SHIFT;
2679
	unsigned long mmu_seq;
2680 2681
	int write = error_code & PFERR_WRITE_MASK;
	bool map_writable;
2682 2683 2684 2685 2686 2687 2688 2689

	ASSERT(vcpu);
	ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));

	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

2690 2691 2692 2693 2694 2695
	force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
	if (likely(!force_pt_level)) {
		level = mapping_level(vcpu, gfn);
		gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
	} else
		level = PT_PAGE_TABLE_LEVEL;
2696

2697
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
2698
	smp_rmb();
2699

2700
	if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
2701 2702 2703
		return 0;

	/* mmio */
2704
	if (is_error_pfn(pfn))
2705
		return kvm_handle_bad_page(vcpu, 0, 0, gfn, pfn);
2706
	spin_lock(&vcpu->kvm->mmu_lock);
2707 2708
	if (mmu_notifier_retry(vcpu, mmu_seq))
		goto out_unlock;
2709
	kvm_mmu_free_some_pages(vcpu);
2710 2711
	if (likely(!force_pt_level))
		transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
2712
	r = __direct_map(vcpu, gpa, write, map_writable,
2713
			 level, gfn, pfn, prefault);
2714 2715 2716
	spin_unlock(&vcpu->kvm->mmu_lock);

	return r;
2717 2718 2719 2720 2721

out_unlock:
	spin_unlock(&vcpu->kvm->mmu_lock);
	kvm_release_pfn_clean(pfn);
	return 0;
2722 2723
}

A
Avi Kivity 已提交
2724 2725
static void nonpaging_free(struct kvm_vcpu *vcpu)
{
2726
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
2727 2728
}

2729 2730
static int nonpaging_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
A
Avi Kivity 已提交
2731 2732 2733 2734 2735
{
	context->new_cr3 = nonpaging_new_cr3;
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
	context->free = nonpaging_free;
2736
	context->prefetch_page = nonpaging_prefetch_page;
2737
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
2738
	context->invlpg = nonpaging_invlpg;
2739
	context->update_pte = nonpaging_update_pte;
2740
	context->root_level = 0;
A
Avi Kivity 已提交
2741
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
2742
	context->root_hpa = INVALID_PAGE;
2743
	context->direct_map = true;
2744
	context->nx = false;
A
Avi Kivity 已提交
2745 2746 2747
	return 0;
}

2748
void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
2749
{
A
Avi Kivity 已提交
2750
	++vcpu->stat.tlb_flush;
2751
	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
A
Avi Kivity 已提交
2752 2753 2754 2755
}

static void paging_new_cr3(struct kvm_vcpu *vcpu)
{
2756
	pgprintk("%s: cr3 %lx\n", __func__, kvm_read_cr3(vcpu));
2757
	mmu_free_roots(vcpu);
A
Avi Kivity 已提交
2758 2759
}

2760 2761
static unsigned long get_cr3(struct kvm_vcpu *vcpu)
{
2762
	return kvm_read_cr3(vcpu);
2763 2764
}

2765 2766
static void inject_page_fault(struct kvm_vcpu *vcpu,
			      struct x86_exception *fault)
A
Avi Kivity 已提交
2767
{
2768
	vcpu->arch.mmu.inject_page_fault(vcpu, fault);
A
Avi Kivity 已提交
2769 2770 2771 2772 2773 2774 2775
}

static void paging_free(struct kvm_vcpu *vcpu)
{
	nonpaging_free(vcpu);
}

2776
static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
2777 2778 2779 2780
{
	int bit7;

	bit7 = (gpte >> 7) & 1;
2781
	return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
2782 2783
}

A
Avi Kivity 已提交
2784 2785 2786 2787 2788 2789 2790 2791
#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

2792 2793 2794
static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context,
				  int level)
2795 2796 2797 2798
{
	int maxphyaddr = cpuid_maxphyaddr(vcpu);
	u64 exb_bit_rsvd = 0;

2799
	if (!context->nx)
2800 2801 2802 2803 2804 2805
		exb_bit_rsvd = rsvd_bits(63, 63);
	switch (level) {
	case PT32_ROOT_LEVEL:
		/* no rsvd bits for 2 level 4K page table entries */
		context->rsvd_bits_mask[0][1] = 0;
		context->rsvd_bits_mask[0][0] = 0;
2806 2807 2808 2809 2810 2811 2812
		context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];

		if (!is_pse(vcpu)) {
			context->rsvd_bits_mask[1][1] = 0;
			break;
		}

2813 2814 2815 2816 2817 2818 2819 2820
		if (is_cpuid_PSE36())
			/* 36bits PSE 4MB page */
			context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
		else
			/* 32 bits PSE 4MB page */
			context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
		break;
	case PT32E_ROOT_LEVEL:
2821 2822 2823
		context->rsvd_bits_mask[0][2] =
			rsvd_bits(maxphyaddr, 63) |
			rsvd_bits(7, 8) | rsvd_bits(1, 2);	/* PDPTE */
2824
		context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
2825
			rsvd_bits(maxphyaddr, 62);	/* PDE */
2826 2827 2828 2829 2830
		context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 62); 	/* PTE */
		context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 62) |
			rsvd_bits(13, 20);		/* large page */
2831
		context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2832 2833 2834 2835 2836 2837 2838
		break;
	case PT64_ROOT_LEVEL:
		context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
		context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
		context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
2839
			rsvd_bits(maxphyaddr, 51);
2840 2841 2842
		context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51);
		context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
2843 2844 2845
		context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
			rsvd_bits(maxphyaddr, 51) |
			rsvd_bits(13, 29);
2846
		context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
2847 2848
			rsvd_bits(maxphyaddr, 51) |
			rsvd_bits(13, 20);		/* large page */
2849
		context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2850 2851 2852 2853
		break;
	}
}

2854 2855 2856
static int paging64_init_context_common(struct kvm_vcpu *vcpu,
					struct kvm_mmu *context,
					int level)
A
Avi Kivity 已提交
2857
{
2858 2859
	context->nx = is_nx(vcpu);

2860
	reset_rsvds_bits_mask(vcpu, context, level);
A
Avi Kivity 已提交
2861 2862 2863 2864 2865

	ASSERT(is_pae(vcpu));
	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
2866
	context->prefetch_page = paging64_prefetch_page;
2867
	context->sync_page = paging64_sync_page;
M
Marcelo Tosatti 已提交
2868
	context->invlpg = paging64_invlpg;
2869
	context->update_pte = paging64_update_pte;
A
Avi Kivity 已提交
2870
	context->free = paging_free;
2871 2872
	context->root_level = level;
	context->shadow_root_level = level;
A
Avi Kivity 已提交
2873
	context->root_hpa = INVALID_PAGE;
2874
	context->direct_map = false;
A
Avi Kivity 已提交
2875 2876 2877
	return 0;
}

2878 2879
static int paging64_init_context(struct kvm_vcpu *vcpu,
				 struct kvm_mmu *context)
2880
{
2881
	return paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
2882 2883
}

2884 2885
static int paging32_init_context(struct kvm_vcpu *vcpu,
				 struct kvm_mmu *context)
A
Avi Kivity 已提交
2886
{
2887 2888
	context->nx = false;

2889
	reset_rsvds_bits_mask(vcpu, context, PT32_ROOT_LEVEL);
A
Avi Kivity 已提交
2890 2891 2892 2893 2894

	context->new_cr3 = paging_new_cr3;
	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
	context->free = paging_free;
2895
	context->prefetch_page = paging32_prefetch_page;
2896
	context->sync_page = paging32_sync_page;
M
Marcelo Tosatti 已提交
2897
	context->invlpg = paging32_invlpg;
2898
	context->update_pte = paging32_update_pte;
A
Avi Kivity 已提交
2899 2900
	context->root_level = PT32_ROOT_LEVEL;
	context->shadow_root_level = PT32E_ROOT_LEVEL;
A
Avi Kivity 已提交
2901
	context->root_hpa = INVALID_PAGE;
2902
	context->direct_map = false;
A
Avi Kivity 已提交
2903 2904 2905
	return 0;
}

2906 2907
static int paging32E_init_context(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
A
Avi Kivity 已提交
2908
{
2909
	return paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
A
Avi Kivity 已提交
2910 2911
}

2912 2913
static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
{
2914
	struct kvm_mmu *context = vcpu->arch.walk_mmu;
2915

2916
	context->base_role.word = 0;
2917 2918 2919 2920
	context->new_cr3 = nonpaging_new_cr3;
	context->page_fault = tdp_page_fault;
	context->free = nonpaging_free;
	context->prefetch_page = nonpaging_prefetch_page;
2921
	context->sync_page = nonpaging_sync_page;
M
Marcelo Tosatti 已提交
2922
	context->invlpg = nonpaging_invlpg;
2923
	context->update_pte = nonpaging_update_pte;
2924
	context->shadow_root_level = kvm_x86_ops->get_tdp_level();
2925
	context->root_hpa = INVALID_PAGE;
2926
	context->direct_map = true;
2927
	context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
2928
	context->get_cr3 = get_cr3;
2929
	context->inject_page_fault = kvm_inject_page_fault;
2930
	context->nx = is_nx(vcpu);
2931 2932

	if (!is_paging(vcpu)) {
2933
		context->nx = false;
2934 2935 2936
		context->gva_to_gpa = nonpaging_gva_to_gpa;
		context->root_level = 0;
	} else if (is_long_mode(vcpu)) {
2937
		context->nx = is_nx(vcpu);
2938
		reset_rsvds_bits_mask(vcpu, context, PT64_ROOT_LEVEL);
2939 2940 2941
		context->gva_to_gpa = paging64_gva_to_gpa;
		context->root_level = PT64_ROOT_LEVEL;
	} else if (is_pae(vcpu)) {
2942
		context->nx = is_nx(vcpu);
2943
		reset_rsvds_bits_mask(vcpu, context, PT32E_ROOT_LEVEL);
2944 2945 2946
		context->gva_to_gpa = paging64_gva_to_gpa;
		context->root_level = PT32E_ROOT_LEVEL;
	} else {
2947
		context->nx = false;
2948
		reset_rsvds_bits_mask(vcpu, context, PT32_ROOT_LEVEL);
2949 2950 2951 2952 2953 2954 2955
		context->gva_to_gpa = paging32_gva_to_gpa;
		context->root_level = PT32_ROOT_LEVEL;
	}

	return 0;
}

2956
int kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
A
Avi Kivity 已提交
2957
{
2958
	int r;
2959
	bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
A
Avi Kivity 已提交
2960
	ASSERT(vcpu);
2961
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
2962 2963

	if (!is_paging(vcpu))
2964
		r = nonpaging_init_context(vcpu, context);
A
Avi Kivity 已提交
2965
	else if (is_long_mode(vcpu))
2966
		r = paging64_init_context(vcpu, context);
A
Avi Kivity 已提交
2967
	else if (is_pae(vcpu))
2968
		r = paging32E_init_context(vcpu, context);
A
Avi Kivity 已提交
2969
	else
2970
		r = paging32_init_context(vcpu, context);
2971

2972
	vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
2973
	vcpu->arch.mmu.base_role.cr0_wp  = is_write_protection(vcpu);
2974 2975
	vcpu->arch.mmu.base_role.smep_andnot_wp
		= smep && !is_write_protection(vcpu);
2976 2977 2978 2979 2980 2981 2982

	return r;
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);

static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
{
2983
	int r = kvm_init_shadow_mmu(vcpu, vcpu->arch.walk_mmu);
2984

2985 2986 2987
	vcpu->arch.walk_mmu->set_cr3           = kvm_x86_ops->set_cr3;
	vcpu->arch.walk_mmu->get_cr3           = get_cr3;
	vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
2988 2989

	return r;
A
Avi Kivity 已提交
2990 2991
}

2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005
static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;

	g_context->get_cr3           = get_cr3;
	g_context->inject_page_fault = kvm_inject_page_fault;

	/*
	 * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
	 * translation of l2_gpa to l1_gpa addresses is done using the
	 * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
	 * functions between mmu and nested_mmu are swapped.
	 */
	if (!is_paging(vcpu)) {
3006
		g_context->nx = false;
3007 3008 3009
		g_context->root_level = 0;
		g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
	} else if (is_long_mode(vcpu)) {
3010
		g_context->nx = is_nx(vcpu);
3011 3012 3013 3014
		reset_rsvds_bits_mask(vcpu, g_context, PT64_ROOT_LEVEL);
		g_context->root_level = PT64_ROOT_LEVEL;
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else if (is_pae(vcpu)) {
3015
		g_context->nx = is_nx(vcpu);
3016 3017 3018 3019
		reset_rsvds_bits_mask(vcpu, g_context, PT32E_ROOT_LEVEL);
		g_context->root_level = PT32E_ROOT_LEVEL;
		g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
	} else {
3020
		g_context->nx = false;
3021 3022 3023 3024 3025 3026 3027 3028
		reset_rsvds_bits_mask(vcpu, g_context, PT32_ROOT_LEVEL);
		g_context->root_level = PT32_ROOT_LEVEL;
		g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
	}

	return 0;
}

3029 3030
static int init_kvm_mmu(struct kvm_vcpu *vcpu)
{
3031 3032 3033
	if (mmu_is_nested(vcpu))
		return init_kvm_nested_mmu(vcpu);
	else if (tdp_enabled)
3034 3035 3036 3037 3038
		return init_kvm_tdp_mmu(vcpu);
	else
		return init_kvm_softmmu(vcpu);
}

A
Avi Kivity 已提交
3039 3040 3041
static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
3042 3043
	if (VALID_PAGE(vcpu->arch.mmu.root_hpa))
		/* mmu.free() should set root_hpa = INVALID_PAGE */
3044
		vcpu->arch.mmu.free(vcpu);
A
Avi Kivity 已提交
3045 3046 3047
}

int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3048 3049
{
	destroy_kvm_mmu(vcpu);
3050
	return init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
3051
}
3052
EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
A
Avi Kivity 已提交
3053 3054

int kvm_mmu_load(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3055
{
3056 3057
	int r;

3058
	r = mmu_topup_memory_caches(vcpu);
A
Avi Kivity 已提交
3059 3060
	if (r)
		goto out;
3061
	r = mmu_alloc_roots(vcpu);
3062
	spin_lock(&vcpu->kvm->mmu_lock);
3063
	mmu_sync_roots(vcpu);
3064
	spin_unlock(&vcpu->kvm->mmu_lock);
3065 3066
	if (r)
		goto out;
3067
	/* set_cr3() should ensure TLB has been flushed */
3068
	vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
3069 3070
out:
	return r;
A
Avi Kivity 已提交
3071
}
A
Avi Kivity 已提交
3072 3073 3074 3075 3076 3077
EXPORT_SYMBOL_GPL(kvm_mmu_load);

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
	mmu_free_roots(vcpu);
}
3078
EXPORT_SYMBOL_GPL(kvm_mmu_unload);
A
Avi Kivity 已提交
3079

3080
static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
3081 3082
				  struct kvm_mmu_page *sp, u64 *spte,
				  const void *new)
3083
{
3084
	if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
3085 3086
		++vcpu->kvm->stat.mmu_pde_zapped;
		return;
3087
        }
3088

A
Avi Kivity 已提交
3089
	++vcpu->kvm->stat.mmu_pte_updated;
3090
	vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
3091 3092
}

3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105
static bool need_remote_flush(u64 old, u64 new)
{
	if (!is_shadow_present_pte(old))
		return false;
	if (!is_shadow_present_pte(new))
		return true;
	if ((old ^ new) & PT64_BASE_ADDR_MASK)
		return true;
	old ^= PT64_NX_MASK;
	new ^= PT64_NX_MASK;
	return (old & ~new & PT64_PERM_MASK) != 0;
}

3106 3107
static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
				    bool remote_flush, bool local_flush)
3108
{
3109 3110 3111 3112
	if (zap_page)
		return;

	if (remote_flush)
3113
		kvm_flush_remote_tlbs(vcpu->kvm);
3114
	else if (local_flush)
3115 3116 3117
		kvm_mmu_flush_tlb(vcpu);
}

3118 3119
static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
{
3120
	u64 *spte = vcpu->arch.last_pte_updated;
3121

S
Sheng Yang 已提交
3122
	return !!(spte && (*spte & shadow_accessed_mask));
3123 3124
}

3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136
static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u64 *spte = vcpu->arch.last_pte_updated;

	if (spte
	    && vcpu->arch.last_pte_gfn == gfn
	    && shadow_accessed_mask
	    && !(*spte & shadow_accessed_mask)
	    && is_shadow_present_pte(*spte))
		set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
}

3137
void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
3138 3139
		       const u8 *new, int bytes,
		       bool guest_initiated)
3140
{
3141
	gfn_t gfn = gpa >> PAGE_SHIFT;
3142
	union kvm_mmu_page_role mask = { .word = 0 };
3143
	struct kvm_mmu_page *sp;
3144
	struct hlist_node *node;
3145
	LIST_HEAD(invalid_list);
3146 3147 3148
	u64 entry, gentry, *spte;
	unsigned pte_size, page_offset, misaligned, quadrant, offset;
	int level, npte, invlpg_counter, r, flooded = 0;
3149 3150
	bool remote_flush, local_flush, zap_page;

3151 3152 3153 3154 3155 3156 3157
	/*
	 * If we don't have indirect shadow pages, it means no page is
	 * write-protected, so we can exit simply.
	 */
	if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
		return;

3158
	zap_page = remote_flush = local_flush = false;
3159
	offset = offset_in_page(gpa);
3160

3161
	pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
3162

3163
	invlpg_counter = atomic_read(&vcpu->kvm->arch.invlpg_counter);
3164 3165 3166

	/*
	 * Assume that the pte write on a page table of the same type
3167 3168
	 * as the current vcpu paging mode since we update the sptes only
	 * when they have the same mode.
3169
	 */
3170
	if ((is_pae(vcpu) && bytes == 4) || !new) {
3171
		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */
3172 3173 3174 3175 3176
		if (is_pae(vcpu)) {
			gpa &= ~(gpa_t)7;
			bytes = 8;
		}
		r = kvm_read_guest(vcpu->kvm, gpa, &gentry, min(bytes, 8));
3177 3178
		if (r)
			gentry = 0;
3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191
		new = (const u8 *)&gentry;
	}

	switch (bytes) {
	case 4:
		gentry = *(const u32 *)new;
		break;
	case 8:
		gentry = *(const u64 *)new;
		break;
	default:
		gentry = 0;
		break;
3192 3193
	}

3194
	spin_lock(&vcpu->kvm->mmu_lock);
3195 3196
	if (atomic_read(&vcpu->kvm->arch.invlpg_counter) != invlpg_counter)
		gentry = 0;
3197
	kvm_mmu_free_some_pages(vcpu);
A
Avi Kivity 已提交
3198
	++vcpu->kvm->stat.mmu_pte_write;
3199
	trace_kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
3200
	if (guest_initiated) {
3201
		kvm_mmu_access_page(vcpu, gfn);
3202 3203 3204 3205 3206 3207 3208 3209 3210 3211
		if (gfn == vcpu->arch.last_pt_write_gfn
		    && !last_updated_pte_accessed(vcpu)) {
			++vcpu->arch.last_pt_write_count;
			if (vcpu->arch.last_pt_write_count >= 3)
				flooded = 1;
		} else {
			vcpu->arch.last_pt_write_gfn = gfn;
			vcpu->arch.last_pt_write_count = 1;
			vcpu->arch.last_pte_updated = NULL;
		}
3212
	}
3213

3214
	mask.cr0_wp = mask.cr4_pae = mask.nxe = 1;
3215
	for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn, node) {
3216
		pte_size = sp->role.cr4_pae ? 8 : 4;
3217
		misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
3218
		misaligned |= bytes < 4;
3219
		if (misaligned || flooded) {
3220 3221 3222 3223
			/*
			 * Misaligned accesses are too much trouble to fix
			 * up; also, they usually indicate a page is not used
			 * as a page table.
3224 3225 3226 3227 3228
			 *
			 * If we're seeing too many writes to a page,
			 * it may no longer be a page table, or we may be
			 * forking, in which case it is better to unmap the
			 * page.
3229 3230
			 */
			pgprintk("misaligned: gpa %llx bytes %d role %x\n",
3231
				 gpa, bytes, sp->role.word);
3232
			zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
3233
						     &invalid_list);
A
Avi Kivity 已提交
3234
			++vcpu->kvm->stat.mmu_flooded;
3235 3236
			continue;
		}
3237
		page_offset = offset;
3238
		level = sp->role.level;
3239
		npte = 1;
3240
		if (!sp->role.cr4_pae) {
3241 3242 3243 3244 3245 3246 3247
			page_offset <<= 1;	/* 32->64 */
			/*
			 * A 32-bit pde maps 4MB while the shadow pdes map
			 * only 2MB.  So we need to double the offset again
			 * and zap two pdes instead of one.
			 */
			if (level == PT32_ROOT_LEVEL) {
3248
				page_offset &= ~7; /* kill rounding error */
3249 3250 3251
				page_offset <<= 1;
				npte = 2;
			}
3252
			quadrant = page_offset >> PAGE_SHIFT;
3253
			page_offset &= ~PAGE_MASK;
3254
			if (quadrant != sp->role.quadrant)
3255
				continue;
3256
		}
3257
		local_flush = true;
3258
		spte = &sp->spt[page_offset / sizeof(*spte)];
3259
		while (npte--) {
3260
			entry = *spte;
3261
			mmu_page_zap_pte(vcpu->kvm, sp, spte);
3262 3263 3264
			if (gentry &&
			      !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
			      & mask.word))
3265
				mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
3266 3267
			if (!remote_flush && need_remote_flush(entry, *spte))
				remote_flush = true;
3268
			++spte;
3269 3270
		}
	}
3271
	mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
3272
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3273
	trace_kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
3274
	spin_unlock(&vcpu->kvm->mmu_lock);
3275 3276
}

3277 3278
int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
3279 3280
	gpa_t gpa;
	int r;
3281

3282
	if (vcpu->arch.mmu.direct_map)
3283 3284
		return 0;

3285
	gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
3286

3287
	spin_lock(&vcpu->kvm->mmu_lock);
3288
	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
3289
	spin_unlock(&vcpu->kvm->mmu_lock);
3290
	return r;
3291
}
3292
EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
3293

3294
void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3295
{
3296
	LIST_HEAD(invalid_list);
3297

3298
	while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES &&
3299
	       !list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
3300
		struct kvm_mmu_page *sp;
A
Avi Kivity 已提交
3301

3302
		sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
3303
				  struct kvm_mmu_page, link);
3304
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
3305
		kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
A
Avi Kivity 已提交
3306
		++vcpu->kvm->stat.mmu_recycled;
A
Avi Kivity 已提交
3307 3308 3309
	}
}

3310 3311
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
		       void *insn, int insn_len)
3312 3313 3314 3315
{
	int r;
	enum emulation_result er;

G
Gleb Natapov 已提交
3316
	r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
3317 3318 3319 3320 3321 3322 3323 3324
	if (r < 0)
		goto out;

	if (!r) {
		r = 1;
		goto out;
	}

3325 3326 3327 3328
	r = mmu_topup_memory_caches(vcpu);
	if (r)
		goto out;

3329
	er = x86_emulate_instruction(vcpu, cr2, 0, insn, insn_len);
3330 3331 3332 3333 3334 3335

	switch (er) {
	case EMULATE_DONE:
		return 1;
	case EMULATE_DO_MMIO:
		++vcpu->stat.mmio_exits;
3336
		/* fall through */
3337
	case EMULATE_FAIL:
3338
		return 0;
3339 3340 3341 3342 3343 3344 3345 3346
	default:
		BUG();
	}
out:
	return r;
}
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);

M
Marcelo Tosatti 已提交
3347 3348 3349 3350 3351 3352 3353 3354
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
	vcpu->arch.mmu.invlpg(vcpu, gva);
	kvm_mmu_flush_tlb(vcpu);
	++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);

3355 3356 3357 3358 3359 3360
void kvm_enable_tdp(void)
{
	tdp_enabled = true;
}
EXPORT_SYMBOL_GPL(kvm_enable_tdp);

3361 3362 3363 3364 3365 3366
void kvm_disable_tdp(void)
{
	tdp_enabled = false;
}
EXPORT_SYMBOL_GPL(kvm_disable_tdp);

A
Avi Kivity 已提交
3367 3368
static void free_mmu_pages(struct kvm_vcpu *vcpu)
{
3369
	free_page((unsigned long)vcpu->arch.mmu.pae_root);
3370 3371
	if (vcpu->arch.mmu.lm_root != NULL)
		free_page((unsigned long)vcpu->arch.mmu.lm_root);
A
Avi Kivity 已提交
3372 3373 3374 3375
}

static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
{
3376
	struct page *page;
A
Avi Kivity 已提交
3377 3378 3379 3380
	int i;

	ASSERT(vcpu);

3381 3382 3383 3384 3385 3386 3387
	/*
	 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
	 * Therefore we need to allocate shadow page tables in the first
	 * 4GB of memory, which happens to fit the DMA32 zone.
	 */
	page = alloc_page(GFP_KERNEL | __GFP_DMA32);
	if (!page)
3388 3389
		return -ENOMEM;

3390
	vcpu->arch.mmu.pae_root = page_address(page);
3391
	for (i = 0; i < 4; ++i)
3392
		vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
3393

A
Avi Kivity 已提交
3394 3395 3396
	return 0;
}

3397
int kvm_mmu_create(struct kvm_vcpu *vcpu)
A
Avi Kivity 已提交
3398 3399
{
	ASSERT(vcpu);
3400
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
A
Avi Kivity 已提交
3401

3402 3403
	return alloc_mmu_pages(vcpu);
}
A
Avi Kivity 已提交
3404

3405 3406 3407
int kvm_mmu_setup(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);
3408
	ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3409

3410
	return init_kvm_mmu(vcpu);
A
Avi Kivity 已提交
3411 3412
}

3413
void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
A
Avi Kivity 已提交
3414
{
3415
	struct kvm_mmu_page *sp;
A
Avi Kivity 已提交
3416

3417
	list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
A
Avi Kivity 已提交
3418 3419 3420
		int i;
		u64 *pt;

3421
		if (!test_bit(slot, sp->slot_bitmap))
A
Avi Kivity 已提交
3422 3423
			continue;

3424
		pt = sp->spt;
3425
		for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
3426 3427 3428 3429 3430
			if (!is_shadow_present_pte(pt[i]) ||
			      !is_last_spte(pt[i], sp->role.level))
				continue;

			if (is_large_pte(pt[i])) {
3431 3432 3433
				drop_spte(kvm, &pt[i],
					  shadow_trap_nonpresent_pte);
				--kvm->stat.lpages;
3434
				continue;
3435
			}
3436

A
Avi Kivity 已提交
3437
			/* avoid RMW */
3438
			if (is_writable_pte(pt[i]))
3439
				update_spte(&pt[i], pt[i] & ~PT_WRITABLE_MASK);
3440
		}
A
Avi Kivity 已提交
3441
	}
3442
	kvm_flush_remote_tlbs(kvm);
A
Avi Kivity 已提交
3443
}
3444

3445
void kvm_mmu_zap_all(struct kvm *kvm)
D
Dor Laor 已提交
3446
{
3447
	struct kvm_mmu_page *sp, *node;
3448
	LIST_HEAD(invalid_list);
D
Dor Laor 已提交
3449

3450
	spin_lock(&kvm->mmu_lock);
3451
restart:
3452
	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
3453
		if (kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list))
3454 3455
			goto restart;

3456
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
3457
	spin_unlock(&kvm->mmu_lock);
D
Dor Laor 已提交
3458 3459
}

3460 3461
static int kvm_mmu_remove_some_alloc_mmu_pages(struct kvm *kvm,
					       struct list_head *invalid_list)
3462 3463 3464 3465 3466
{
	struct kvm_mmu_page *page;

	page = container_of(kvm->arch.active_mmu_pages.prev,
			    struct kvm_mmu_page, link);
3467
	return kvm_mmu_prepare_zap_page(kvm, page, invalid_list);
3468 3469
}

3470
static int mmu_shrink(struct shrinker *shrink, struct shrink_control *sc)
3471 3472 3473
{
	struct kvm *kvm;
	struct kvm *kvm_freed = NULL;
3474
	int nr_to_scan = sc->nr_to_scan;
3475 3476 3477

	if (nr_to_scan == 0)
		goto out;
3478

3479
	raw_spin_lock(&kvm_lock);
3480 3481

	list_for_each_entry(kvm, &vm_list, vm_list) {
3482
		int idx, freed_pages;
3483
		LIST_HEAD(invalid_list);
3484

3485
		idx = srcu_read_lock(&kvm->srcu);
3486
		spin_lock(&kvm->mmu_lock);
3487 3488
		if (!kvm_freed && nr_to_scan > 0 &&
		    kvm->arch.n_used_mmu_pages > 0) {
3489 3490
			freed_pages = kvm_mmu_remove_some_alloc_mmu_pages(kvm,
							  &invalid_list);
3491 3492 3493 3494
			kvm_freed = kvm;
		}
		nr_to_scan--;

3495
		kvm_mmu_commit_zap_page(kvm, &invalid_list);
3496
		spin_unlock(&kvm->mmu_lock);
3497
		srcu_read_unlock(&kvm->srcu, idx);
3498 3499 3500 3501
	}
	if (kvm_freed)
		list_move_tail(&kvm_freed->vm_list, &vm_list);

3502
	raw_spin_unlock(&kvm_lock);
3503

3504 3505
out:
	return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
3506 3507 3508 3509 3510 3511 3512
}

static struct shrinker mmu_shrinker = {
	.shrink = mmu_shrink,
	.seeks = DEFAULT_SEEKS * 10,
};

I
Ingo Molnar 已提交
3513
static void mmu_destroy_caches(void)
3514
{
3515 3516
	if (pte_list_desc_cache)
		kmem_cache_destroy(pte_list_desc_cache);
3517 3518
	if (mmu_page_header_cache)
		kmem_cache_destroy(mmu_page_header_cache);
3519 3520 3521 3522
}

int kvm_mmu_module_init(void)
{
3523 3524
	pte_list_desc_cache = kmem_cache_create("pte_list_desc",
					    sizeof(struct pte_list_desc),
3525
					    0, 0, NULL);
3526
	if (!pte_list_desc_cache)
3527 3528
		goto nomem;

3529 3530
	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
3531
						  0, 0, NULL);
3532 3533 3534
	if (!mmu_page_header_cache)
		goto nomem;

3535 3536 3537
	if (percpu_counter_init(&kvm_total_used_mmu_pages, 0))
		goto nomem;

3538 3539
	register_shrinker(&mmu_shrinker);

3540 3541 3542
	return 0;

nomem:
3543
	mmu_destroy_caches();
3544 3545 3546
	return -ENOMEM;
}

3547 3548 3549 3550 3551 3552 3553 3554
/*
 * Caculate mmu pages needed for kvm.
 */
unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
{
	int i;
	unsigned int nr_mmu_pages;
	unsigned int  nr_pages = 0;
3555
	struct kvm_memslots *slots;
3556

3557 3558
	slots = kvm_memslots(kvm);

3559 3560
	for (i = 0; i < slots->nmemslots; i++)
		nr_pages += slots->memslots[i].npages;
3561 3562 3563 3564 3565 3566 3567 3568

	nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
	nr_mmu_pages = max(nr_mmu_pages,
			(unsigned int) KVM_MIN_ALLOC_MMU_PAGES);

	return nr_mmu_pages;
}

3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603
static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
				unsigned len)
{
	if (len > buffer->len)
		return NULL;
	return buffer->ptr;
}

static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
				unsigned len)
{
	void *ret;

	ret = pv_mmu_peek_buffer(buffer, len);
	if (!ret)
		return ret;
	buffer->ptr += len;
	buffer->len -= len;
	buffer->processed += len;
	return ret;
}

static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
			     gpa_t addr, gpa_t value)
{
	int bytes = 8;
	int r;

	if (!is_long_mode(vcpu) && !is_pae(vcpu))
		bytes = 4;

	r = mmu_topup_memory_caches(vcpu);
	if (r)
		return r;

3604
	if (!emulator_write_phys(vcpu, addr, &value, bytes))
3605 3606 3607 3608 3609 3610 3611
		return -EFAULT;

	return 1;
}

static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
{
3612
	(void)kvm_set_cr3(vcpu, kvm_read_cr3(vcpu));
3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665
	return 1;
}

static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
{
	spin_lock(&vcpu->kvm->mmu_lock);
	mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
	spin_unlock(&vcpu->kvm->mmu_lock);
	return 1;
}

static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
			     struct kvm_pv_mmu_op_buffer *buffer)
{
	struct kvm_mmu_op_header *header;

	header = pv_mmu_peek_buffer(buffer, sizeof *header);
	if (!header)
		return 0;
	switch (header->op) {
	case KVM_MMU_OP_WRITE_PTE: {
		struct kvm_mmu_op_write_pte *wpte;

		wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
		if (!wpte)
			return 0;
		return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
					wpte->pte_val);
	}
	case KVM_MMU_OP_FLUSH_TLB: {
		struct kvm_mmu_op_flush_tlb *ftlb;

		ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
		if (!ftlb)
			return 0;
		return kvm_pv_mmu_flush_tlb(vcpu);
	}
	case KVM_MMU_OP_RELEASE_PT: {
		struct kvm_mmu_op_release_pt *rpt;

		rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
		if (!rpt)
			return 0;
		return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
	}
	default: return 0;
	}
}

int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
		  gpa_t addr, unsigned long *ret)
{
	int r;
3666
	struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
3667

3668 3669 3670
	buffer->ptr = buffer->buf;
	buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
	buffer->processed = 0;
3671

3672
	r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
3673 3674 3675
	if (r)
		goto out;

3676 3677
	while (buffer->len) {
		r = kvm_pv_mmu_op_one(vcpu, buffer);
3678 3679 3680 3681 3682 3683 3684 3685
		if (r < 0)
			goto out;
		if (r == 0)
			break;
	}

	r = 1;
out:
3686
	*ret = buffer->processed;
3687 3688 3689
	return r;
}

3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707
int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
{
	struct kvm_shadow_walk_iterator iterator;
	int nr_sptes = 0;

	spin_lock(&vcpu->kvm->mmu_lock);
	for_each_shadow_entry(vcpu, addr, iterator) {
		sptes[iterator.level-1] = *iterator.sptep;
		nr_sptes++;
		if (!is_shadow_present_pte(*iterator.sptep))
			break;
	}
	spin_unlock(&vcpu->kvm->mmu_lock);

	return nr_sptes;
}
EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);

3708 3709 3710 3711 3712 3713 3714
void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
	ASSERT(vcpu);

	destroy_kvm_mmu(vcpu);
	free_mmu_pages(vcpu);
	mmu_free_memory_caches(vcpu);
3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727
}

#ifdef CONFIG_KVM_MMU_AUDIT
#include "mmu_audit.c"
#else
static void mmu_audit_disable(void) { }
#endif

void kvm_mmu_module_exit(void)
{
	mmu_destroy_caches();
	percpu_counter_destroy(&kvm_total_used_mmu_pages);
	unregister_shrinker(&mmu_shrinker);
3728 3729
	mmu_audit_disable();
}