mmu.c 40.3 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5
/*
 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
 */
6 7 8 9

#include <linux/mman.h>
#include <linux/kvm_host.h>
#include <linux/io.h>
10
#include <linux/hugetlb.h>
11
#include <linux/sched/signal.h>
C
Christoffer Dall 已提交
12
#include <trace/events/kvm.h>
13
#include <asm/pgalloc.h>
14
#include <asm/cacheflush.h>
15 16
#include <asm/kvm_arm.h>
#include <asm/kvm_mmu.h>
17
#include <asm/kvm_pgtable.h>
18
#include <asm/kvm_ras.h>
19
#include <asm/kvm_asm.h>
20
#include <asm/kvm_emulate.h>
21
#include <asm/virt.h>
22 23

#include "trace.h"
24

25
static struct kvm_pgtable *hyp_pgtable;
26 27
static DEFINE_MUTEX(kvm_hyp_pgd_mutex);

28 29 30 31
static unsigned long hyp_idmap_start;
static unsigned long hyp_idmap_end;
static phys_addr_t hyp_idmap_vector;

32 33
static unsigned long io_map_base;

34

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
/*
 * Release kvm_mmu_lock periodically if the memory region is large. Otherwise,
 * we may see kernel panics with CONFIG_DETECT_HUNG_TASK,
 * CONFIG_LOCKUP_DETECTOR, CONFIG_LOCKDEP. Additionally, holding the lock too
 * long will also starve other vCPUs. We have to also make sure that the page
 * tables are not freed while we released the lock.
 */
static int stage2_apply_range(struct kvm *kvm, phys_addr_t addr,
			      phys_addr_t end,
			      int (*fn)(struct kvm_pgtable *, u64, u64),
			      bool resched)
{
	int ret;
	u64 next;

	do {
		struct kvm_pgtable *pgt = kvm->arch.mmu.pgt;
		if (!pgt)
			return -EINVAL;

		next = stage2_pgd_addr_end(kvm, addr, end);
		ret = fn(pgt, addr, next - addr);
		if (ret)
			break;

		if (resched && next != end)
			cond_resched_lock(&kvm->mmu_lock);
	} while (addr = next, addr != end);

	return ret;
}

67 68 69
#define stage2_apply_range_resched(kvm, addr, end, fn)			\
	stage2_apply_range(kvm, addr, end, fn, true)

70 71 72
static bool memslot_is_logging(struct kvm_memory_slot *memslot)
{
	return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY);
73 74 75 76 77 78 79 80 81 82
}

/**
 * kvm_flush_remote_tlbs() - flush all VM TLB entries for v7/8
 * @kvm:	pointer to kvm structure.
 *
 * Interface to HYP function to flush all VM TLB entries
 */
void kvm_flush_remote_tlbs(struct kvm *kvm)
{
83
	kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu);
84
}
85

86 87 88 89 90
static bool kvm_is_device_pfn(unsigned long pfn)
{
	return !pfn_valid(pfn);
}

91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107
/*
 * Unmapping vs dcache management:
 *
 * If a guest maps certain memory pages as uncached, all writes will
 * bypass the data cache and go directly to RAM.  However, the CPUs
 * can still speculate reads (not writes) and fill cache lines with
 * data.
 *
 * Those cache lines will be *clean* cache lines though, so a
 * clean+invalidate operation is equivalent to an invalidate
 * operation, because no cache lines are marked dirty.
 *
 * Those clean cache lines could be filled prior to an uncached write
 * by the guest, and the cache coherent IO subsystem would therefore
 * end up writing old data to disk.
 *
 * This is why right after unmapping a page/section and invalidating
108 109
 * the corresponding TLBs, we flush to make sure the IO subsystem will
 * never hit in the cache.
110 111 112 113
 *
 * This is all avoided on systems that have ARM64_HAS_STAGE2_FWB, as
 * we then fully enforce cacheability of RAM, no matter what the guest
 * does.
114
 */
115 116
/**
 * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
117
 * @mmu:   The KVM stage-2 MMU pointer
118 119
 * @start: The intermediate physical base address of the range to unmap
 * @size:  The size of the area to unmap
120
 * @may_block: Whether or not we are permitted to block
121 122 123 124 125 126
 *
 * Clear a range of stage-2 mappings, lowering the various ref-counts.  Must
 * be called while holding mmu_lock (unless for freeing the stage2 pgd before
 * destroying the VM), otherwise another faulting VCPU may come in and mess
 * with things behind our backs.
 */
127 128
static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size,
				 bool may_block)
129
{
130
	struct kvm *kvm = mmu->kvm;
131
	phys_addr_t end = start + size;
132

133
	assert_spin_locked(&kvm->mmu_lock);
134
	WARN_ON(size & ~PAGE_MASK);
135 136
	WARN_ON(stage2_apply_range(kvm, start, end, kvm_pgtable_stage2_unmap,
				   may_block));
137 138
}

139 140 141 142 143
static void unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size)
{
	__unmap_stage2_range(mmu, start, size, true);
}

144 145 146 147 148 149
static void stage2_flush_memslot(struct kvm *kvm,
				 struct kvm_memory_slot *memslot)
{
	phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
	phys_addr_t end = addr + PAGE_SIZE * memslot->npages;

150
	stage2_apply_range_resched(kvm, addr, end, kvm_pgtable_stage2_flush);
151 152 153 154 155 156 157 158 159
}

/**
 * stage2_flush_vm - Invalidate cache for pages mapped in stage 2
 * @kvm: The struct kvm pointer
 *
 * Go through the stage 2 page tables and invalidate any cache lines
 * backing memory already mapped to the VM.
 */
160
static void stage2_flush_vm(struct kvm *kvm)
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
	spin_lock(&kvm->mmu_lock);

	slots = kvm_memslots(kvm);
	kvm_for_each_memslot(memslot, slots)
		stage2_flush_memslot(kvm, memslot);

	spin_unlock(&kvm->mmu_lock);
	srcu_read_unlock(&kvm->srcu, idx);
}

177
/**
178
 * free_hyp_pgds - free Hyp-mode page tables
179
 */
180
void free_hyp_pgds(void)
181
{
182
	mutex_lock(&kvm_hyp_pgd_mutex);
183 184 185
	if (hyp_pgtable) {
		kvm_pgtable_hyp_destroy(hyp_pgtable);
		kfree(hyp_pgtable);
186
	}
187 188 189
	mutex_unlock(&kvm_hyp_pgd_mutex);
}

190 191
static int __create_hyp_mappings(unsigned long start, unsigned long size,
				 unsigned long phys, enum kvm_pgtable_prot prot)
192
{
193
	int err;
194 195

	mutex_lock(&kvm_hyp_pgd_mutex);
196
	err = kvm_pgtable_hyp_map(hyp_pgtable, start, size, phys, prot);
197
	mutex_unlock(&kvm_hyp_pgd_mutex);
198

199 200 201
	return err;
}

202 203 204 205 206 207 208 209 210 211 212
static phys_addr_t kvm_kaddr_to_phys(void *kaddr)
{
	if (!is_vmalloc_addr(kaddr)) {
		BUG_ON(!virt_addr_valid(kaddr));
		return __pa(kaddr);
	} else {
		return page_to_phys(vmalloc_to_page(kaddr)) +
		       offset_in_page(kaddr);
	}
}

213
/**
214
 * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
215 216
 * @from:	The virtual kernel start address of the range
 * @to:		The virtual kernel end address of the range (exclusive)
217
 * @prot:	The protection to be applied to this range
218
 *
219 220 221
 * The same virtual address as the kernel virtual address is also used
 * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying
 * physical pages.
222
 */
223
int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
224
{
225 226
	phys_addr_t phys_addr;
	unsigned long virt_addr;
M
Marc Zyngier 已提交
227 228
	unsigned long start = kern_hyp_va((unsigned long)from);
	unsigned long end = kern_hyp_va((unsigned long)to);
229

230 231 232
	if (is_kernel_in_hyp_mode())
		return 0;

233 234
	start = start & PAGE_MASK;
	end = PAGE_ALIGN(end);
235

236 237
	for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
		int err;
238

239
		phys_addr = kvm_kaddr_to_phys(from + virt_addr - start);
240
		err = __create_hyp_mappings(virt_addr, PAGE_SIZE, phys_addr,
241
					    prot);
242 243 244 245 246
		if (err)
			return err;
	}

	return 0;
247 248
}

249
static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size,
250 251
					unsigned long *haddr,
					enum kvm_pgtable_prot prot)
252
{
253 254
	unsigned long base;
	int ret = 0;
255

256
	mutex_lock(&kvm_hyp_pgd_mutex);
257

258
	/*
F
Fuad Tabba 已提交
259
	 * This assumes that we have enough space below the idmap
260 261 262 263 264 265 266 267
	 * page to allocate our VAs. If not, the check below will
	 * kick. A potential alternative would be to detect that
	 * overflow and switch to an allocation above the idmap.
	 *
	 * The allocated size is always a multiple of PAGE_SIZE.
	 */
	size = PAGE_ALIGN(size + offset_in_page(phys_addr));
	base = io_map_base - size;
268

269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
	/*
	 * Verify that BIT(VA_BITS - 1) hasn't been flipped by
	 * allocating the new area, as it would indicate we've
	 * overflowed the idmap/IO address range.
	 */
	if ((base ^ io_map_base) & BIT(VA_BITS - 1))
		ret = -ENOMEM;
	else
		io_map_base = base;

	mutex_unlock(&kvm_hyp_pgd_mutex);

	if (ret)
		goto out;

284
	ret = __create_hyp_mappings(base, size, phys_addr, prot);
285 286 287
	if (ret)
		goto out;

288
	*haddr = base + offset_in_page(phys_addr);
289
out:
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
	return ret;
}

/**
 * create_hyp_io_mappings - Map IO into both kernel and HYP
 * @phys_addr:	The physical start address which gets mapped
 * @size:	Size of the region being mapped
 * @kaddr:	Kernel VA for this mapping
 * @haddr:	HYP VA for this mapping
 */
int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,
			   void __iomem **kaddr,
			   void __iomem **haddr)
{
	unsigned long addr;
	int ret;

	*kaddr = ioremap(phys_addr, size);
	if (!*kaddr)
		return -ENOMEM;

	if (is_kernel_in_hyp_mode()) {
		*haddr = *kaddr;
		return 0;
	}

	ret = __create_hyp_private_mapping(phys_addr, size,
					   &addr, PAGE_HYP_DEVICE);
318 319 320
	if (ret) {
		iounmap(*kaddr);
		*kaddr = NULL;
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
		*haddr = NULL;
		return ret;
	}

	*haddr = (void __iomem *)addr;
	return 0;
}

/**
 * create_hyp_exec_mappings - Map an executable range into HYP
 * @phys_addr:	The physical start address which gets mapped
 * @size:	Size of the region being mapped
 * @haddr:	HYP VA for this mapping
 */
int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size,
			     void **haddr)
{
	unsigned long addr;
	int ret;

	BUG_ON(is_kernel_in_hyp_mode());

	ret = __create_hyp_private_mapping(phys_addr, size,
					   &addr, PAGE_HYP_EXEC);
	if (ret) {
		*haddr = NULL;
347 348 349
		return ret;
	}

350
	*haddr = (void *)addr;
351
	return 0;
352 353
}

354
/**
355 356 357
 * kvm_init_stage2_mmu - Initialise a S2 MMU strucrure
 * @kvm:	The pointer to the KVM structure
 * @mmu:	The pointer to the s2 MMU structure
358
 *
359
 * Allocates only the stage-2 HW PGD level table(s).
360 361 362
 * Note we don't need locking here as this is only called when the VM is
 * created, which can only be done once.
 */
363
int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
364
{
365 366
	int cpu, err;
	struct kvm_pgtable *pgt;
367

368
	if (mmu->pgt != NULL) {
369 370 371 372
		kvm_err("kvm_arch already initialized?\n");
		return -EINVAL;
	}

373 374
	pgt = kzalloc(sizeof(*pgt), GFP_KERNEL);
	if (!pgt)
375 376
		return -ENOMEM;

377 378 379
	err = kvm_pgtable_stage2_init(pgt, kvm);
	if (err)
		goto out_free_pgtable;
380

381 382
	mmu->last_vcpu_ran = alloc_percpu(typeof(*mmu->last_vcpu_ran));
	if (!mmu->last_vcpu_ran) {
383 384
		err = -ENOMEM;
		goto out_destroy_pgtable;
385 386 387 388 389 390
	}

	for_each_possible_cpu(cpu)
		*per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1;

	mmu->kvm = kvm;
391 392
	mmu->pgt = pgt;
	mmu->pgd_phys = __pa(pgt->pgd);
393
	mmu->vmid.vmid_gen = 0;
394
	return 0;
395 396 397 398 399 400

out_destroy_pgtable:
	kvm_pgtable_stage2_destroy(pgt);
out_free_pgtable:
	kfree(pgt);
	return err;
401 402
}

403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
static void stage2_unmap_memslot(struct kvm *kvm,
				 struct kvm_memory_slot *memslot)
{
	hva_t hva = memslot->userspace_addr;
	phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
	phys_addr_t size = PAGE_SIZE * memslot->npages;
	hva_t reg_end = hva + size;

	/*
	 * A memory region could potentially cover multiple VMAs, and any holes
	 * between them, so iterate over all of them to find out if we should
	 * unmap any of them.
	 *
	 *     +--------------------------------------------+
	 * +---------------+----------------+   +----------------+
	 * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
	 * +---------------+----------------+   +----------------+
	 *     |               memory region                |
	 *     +--------------------------------------------+
	 */
	do {
424
		struct vm_area_struct *vma;
425 426
		hva_t vm_start, vm_end;

427 428
		vma = find_vma_intersection(current->mm, hva, reg_end);
		if (!vma)
429 430 431 432 433 434 435 436 437 438
			break;

		/*
		 * Take the intersection of this VMA with the memory region
		 */
		vm_start = max(hva, vma->vm_start);
		vm_end = min(reg_end, vma->vm_end);

		if (!(vma->vm_flags & VM_PFNMAP)) {
			gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
439
			unmap_stage2_range(&kvm->arch.mmu, gpa, vm_end - vm_start);
440 441 442 443 444 445 446 447 448
		}
		hva = vm_end;
	} while (hva < reg_end);
}

/**
 * stage2_unmap_vm - Unmap Stage-2 RAM mappings
 * @kvm: The struct kvm pointer
 *
F
Fuad Tabba 已提交
449
 * Go through the memregions and unmap any regular RAM
450 451 452 453 454 455 456 457 458
 * backing memory already mapped to the VM.
 */
void stage2_unmap_vm(struct kvm *kvm)
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
	int idx;

	idx = srcu_read_lock(&kvm->srcu);
459
	mmap_read_lock(current->mm);
460 461 462 463 464 465 466
	spin_lock(&kvm->mmu_lock);

	slots = kvm_memslots(kvm);
	kvm_for_each_memslot(memslot, slots)
		stage2_unmap_memslot(kvm, memslot);

	spin_unlock(&kvm->mmu_lock);
467
	mmap_read_unlock(current->mm);
468 469 470
	srcu_read_unlock(&kvm->srcu, idx);
}

471
void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu)
472
{
473
	struct kvm *kvm = mmu->kvm;
474
	struct kvm_pgtable *pgt = NULL;
475

476
	spin_lock(&kvm->mmu_lock);
477 478 479 480 481
	pgt = mmu->pgt;
	if (pgt) {
		mmu->pgd_phys = 0;
		mmu->pgt = NULL;
		free_percpu(mmu->last_vcpu_ran);
482
	}
483 484
	spin_unlock(&kvm->mmu_lock);

485 486 487
	if (pgt) {
		kvm_pgtable_stage2_destroy(pgt);
		kfree(pgt);
488
	}
489 490 491 492 493 494 495 496 497
}

/**
 * kvm_phys_addr_ioremap - map a device range to guest IPA
 *
 * @kvm:	The KVM pointer
 * @guest_ipa:	The IPA at which to insert the mapping
 * @pa:		The physical address of the device
 * @size:	The size of the mapping
498
 * @writable:   Whether or not to create a writable mapping
499 500
 */
int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
501
			  phys_addr_t pa, unsigned long size, bool writable)
502
{
503
	phys_addr_t addr;
504
	int ret = 0;
505
	struct kvm_mmu_memory_cache cache = { 0, __GFP_ZERO, NULL, };
506 507 508 509
	struct kvm_pgtable *pgt = kvm->arch.mmu.pgt;
	enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_DEVICE |
				     KVM_PGTABLE_PROT_R |
				     (writable ? KVM_PGTABLE_PROT_W : 0);
510

511 512
	size += offset_in_page(guest_ipa);
	guest_ipa &= PAGE_MASK;
513

514
	for (addr = guest_ipa; addr < guest_ipa + size; addr += PAGE_SIZE) {
515 516
		ret = kvm_mmu_topup_memory_cache(&cache,
						 kvm_mmu_cache_min_pages(kvm));
517
		if (ret)
518 519
			break;

520
		spin_lock(&kvm->mmu_lock);
521 522
		ret = kvm_pgtable_stage2_map(pgt, addr, PAGE_SIZE, pa, prot,
					     &cache);
523 524
		spin_unlock(&kvm->mmu_lock);
		if (ret)
525
			break;
526

527
		pa += PAGE_SIZE;
528 529
	}

530
	kvm_mmu_free_memory_cache(&cache);
531 532 533
	return ret;
}

534 535
/**
 * stage2_wp_range() - write protect stage2 memory region range
536
 * @mmu:        The KVM stage-2 MMU pointer
537 538 539
 * @addr:	Start address of range
 * @end:	End address of range
 */
540
static void stage2_wp_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end)
541
{
542
	struct kvm *kvm = mmu->kvm;
543
	stage2_apply_range_resched(kvm, addr, end, kvm_pgtable_stage2_wrprotect);
544 545 546 547 548 549 550 551 552
}

/**
 * kvm_mmu_wp_memory_region() - write protect stage 2 entries for memory slot
 * @kvm:	The KVM pointer
 * @slot:	The memory slot to write protect
 *
 * Called to start logging dirty pages after memory region
 * KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns
553
 * all present PUD, PMD and PTEs are write protected in the memory region.
554 555 556 557 558
 * Afterwards read of dirty page log can be called.
 *
 * Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired,
 * serializing operations for VM memory regions.
 */
559
static void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
560
{
561 562
	struct kvm_memslots *slots = kvm_memslots(kvm);
	struct kvm_memory_slot *memslot = id_to_memslot(slots, slot);
563 564 565 566 567 568 569
	phys_addr_t start, end;

	if (WARN_ON_ONCE(!memslot))
		return;

	start = memslot->base_gfn << PAGE_SHIFT;
	end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
570 571

	spin_lock(&kvm->mmu_lock);
572
	stage2_wp_range(&kvm->arch.mmu, start, end);
573 574 575
	spin_unlock(&kvm->mmu_lock);
	kvm_flush_remote_tlbs(kvm);
}
576 577

/**
578
 * kvm_mmu_write_protect_pt_masked() - write protect dirty pages
579 580 581 582 583 584 585 586 587
 * @kvm:	The KVM pointer
 * @slot:	The memory slot associated with mask
 * @gfn_offset:	The gfn offset in memory slot
 * @mask:	The mask of dirty pages at offset 'gfn_offset' in this memory
 *		slot to be write protected
 *
 * Walks bits set in mask write protects the associated pte's. Caller must
 * acquire kvm_mmu_lock.
 */
588
static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
589 590 591 592 593 594 595
		struct kvm_memory_slot *slot,
		gfn_t gfn_offset, unsigned long mask)
{
	phys_addr_t base_gfn = slot->base_gfn + gfn_offset;
	phys_addr_t start = (base_gfn +  __ffs(mask)) << PAGE_SHIFT;
	phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;

596
	stage2_wp_range(&kvm->arch.mmu, start, end);
597
}
598

599 600 601 602 603 604 605 606 607 608 609 610 611 612
/*
 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
 * dirty pages.
 *
 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
 * enable dirty logging for them.
 */
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
		struct kvm_memory_slot *slot,
		gfn_t gfn_offset, unsigned long mask)
{
	kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
}

613
static void clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size)
614
{
615
	__clean_dcache_guest_page(pfn, size);
616 617
}

618
static void invalidate_icache_guest_page(kvm_pfn_t pfn, unsigned long size)
619
{
620
	__invalidate_icache_guest_page(pfn, size);
621 622
}

623
static void kvm_send_hwpoison_signal(unsigned long address, short lsb)
624
{
625
	send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current);
626 627
}

628 629 630
static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot,
					       unsigned long hva,
					       unsigned long map_size)
631
{
632
	gpa_t gpa_start;
633 634 635
	hva_t uaddr_start, uaddr_end;
	size_t size;

636 637 638 639
	/* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */
	if (map_size == PAGE_SIZE)
		return true;

640 641 642 643 644 645 646 647 648
	size = memslot->npages * PAGE_SIZE;

	gpa_start = memslot->base_gfn << PAGE_SHIFT;

	uaddr_start = memslot->userspace_addr;
	uaddr_end = uaddr_start + size;

	/*
	 * Pages belonging to memslots that don't have the same alignment
649 650
	 * within a PMD/PUD for userspace and IPA cannot be mapped with stage-2
	 * PMD/PUD entries, because we'll end up mapping the wrong pages.
651 652 653 654 655
	 *
	 * Consider a layout like the following:
	 *
	 *    memslot->userspace_addr:
	 *    +-----+--------------------+--------------------+---+
656
	 *    |abcde|fgh  Stage-1 block  |    Stage-1 block tv|xyz|
657 658
	 *    +-----+--------------------+--------------------+---+
	 *
659
	 *    memslot->base_gfn << PAGE_SHIFT:
660
	 *      +---+--------------------+--------------------+-----+
661
	 *      |abc|def  Stage-2 block  |    Stage-2 block   |tvxyz|
662 663
	 *      +---+--------------------+--------------------+-----+
	 *
664
	 * If we create those stage-2 blocks, we'll end up with this incorrect
665 666 667 668 669
	 * mapping:
	 *   d -> f
	 *   e -> g
	 *   f -> h
	 */
670
	if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1)))
671 672 673 674
		return false;

	/*
	 * Next, let's make sure we're not trying to map anything not covered
675 676
	 * by the memslot. This means we have to prohibit block size mappings
	 * for the beginning and end of a non-block aligned and non-block sized
677 678 679 680 681 682 683 684
	 * memory slot (illustrated by the head and tail parts of the
	 * userspace view above containing pages 'abcde' and 'xyz',
	 * respectively).
	 *
	 * Note that it doesn't matter if we do the check using the
	 * userspace_addr or the base_gfn, as both are equally aligned (per
	 * the check above) and equally sized.
	 */
685 686
	return (hva & ~(map_size - 1)) >= uaddr_start &&
	       (hva & ~(map_size - 1)) + map_size <= uaddr_end;
687 688
}

689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741
/*
 * Check if the given hva is backed by a transparent huge page (THP) and
 * whether it can be mapped using block mapping in stage2. If so, adjust
 * the stage2 PFN and IPA accordingly. Only PMD_SIZE THPs are currently
 * supported. This will need to be updated to support other THP sizes.
 *
 * Returns the size of the mapping.
 */
static unsigned long
transparent_hugepage_adjust(struct kvm_memory_slot *memslot,
			    unsigned long hva, kvm_pfn_t *pfnp,
			    phys_addr_t *ipap)
{
	kvm_pfn_t pfn = *pfnp;

	/*
	 * Make sure the adjustment is done only for THP pages. Also make
	 * sure that the HVA and IPA are sufficiently aligned and that the
	 * block map is contained within the memslot.
	 */
	if (kvm_is_transparent_hugepage(pfn) &&
	    fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) {
		/*
		 * The address we faulted on is backed by a transparent huge
		 * page.  However, because we map the compound huge page and
		 * not the individual tail page, we need to transfer the
		 * refcount to the head page.  We have to be careful that the
		 * THP doesn't start to split while we are adjusting the
		 * refcounts.
		 *
		 * We are sure this doesn't happen, because mmu_notifier_retry
		 * was successful and we are holding the mmu_lock, so if this
		 * THP is trying to split, it will be blocked in the mmu
		 * notifier before touching any of the pages, specifically
		 * before being able to call __split_huge_page_refcount().
		 *
		 * We can therefore safely transfer the refcount from PG_tail
		 * to PG_head and switch the pfn from a tail page to the head
		 * page accordingly.
		 */
		*ipap &= PMD_MASK;
		kvm_release_pfn_clean(pfn);
		pfn &= ~(PTRS_PER_PMD - 1);
		kvm_get_pfn(pfn);
		*pfnp = pfn;

		return PMD_SIZE;
	}

	/* Use page mapping if we cannot use block mapping. */
	return PAGE_SIZE;
}

742
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
743
			  struct kvm_memory_slot *memslot, unsigned long hva,
744 745
			  unsigned long fault_status)
{
746
	int ret = 0;
747
	bool write_fault, writable, force_pte = false;
748 749
	bool exec_fault;
	bool device = false;
750
	unsigned long mmu_seq;
751
	struct kvm *kvm = vcpu->kvm;
752
	struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
753
	struct vm_area_struct *vma;
754
	short vma_shift;
755
	gfn_t gfn;
D
Dan Williams 已提交
756
	kvm_pfn_t pfn;
757
	bool logging_active = memslot_is_logging(memslot);
758 759
	unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu);
	unsigned long vma_pagesize, fault_granule;
760 761
	enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
	struct kvm_pgtable *pgt;
762

763
	fault_granule = 1UL << ARM64_HW_PGTABLE_LEVEL_SHIFT(fault_level);
764
	write_fault = kvm_is_write_fault(vcpu);
765
	exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu);
766 767 768
	VM_BUG_ON(write_fault && exec_fault);

	if (fault_status == FSC_PERM && !write_fault && !exec_fault) {
769 770 771 772
		kvm_err("Unexpected L2 read permission error\n");
		return -EFAULT;
	}

773
	/* Let's check if we will get back a huge page backed by hugetlbfs */
774
	mmap_read_lock(current->mm);
775
	vma = find_vma_intersection(current->mm, hva, hva + 1);
776 777
	if (unlikely(!vma)) {
		kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
778
		mmap_read_unlock(current->mm);
779 780 781
		return -EFAULT;
	}

782 783 784 785 786
	if (is_vm_hugetlb_page(vma))
		vma_shift = huge_page_shift(hstate_vma(vma));
	else
		vma_shift = PAGE_SHIFT;

787
	if (logging_active ||
788
	    (vma->vm_flags & VM_PFNMAP)) {
789
		force_pte = true;
790 791 792
		vma_shift = PAGE_SHIFT;
	}

793
	switch (vma_shift) {
794
#ifndef __PAGETABLE_PMD_FOLDED
795 796 797 798
	case PUD_SHIFT:
		if (fault_supports_stage2_huge_mapping(memslot, hva, PUD_SIZE))
			break;
		fallthrough;
799
#endif
800 801 802 803 804 805 806 807
	case CONT_PMD_SHIFT:
		vma_shift = PMD_SHIFT;
		fallthrough;
	case PMD_SHIFT:
		if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE))
			break;
		fallthrough;
	case CONT_PTE_SHIFT:
808
		vma_shift = PAGE_SHIFT;
809 810 811 812 813 814
		force_pte = true;
		fallthrough;
	case PAGE_SHIFT:
		break;
	default:
		WARN_ONCE(1, "Unknown vma_shift %d", vma_shift);
815 816
	}

817
	vma_pagesize = 1UL << vma_shift;
818
	if (vma_pagesize == PMD_SIZE || vma_pagesize == PUD_SIZE)
819
		fault_ipa &= ~(vma_pagesize - 1);
820 821

	gfn = fault_ipa >> PAGE_SHIFT;
822
	mmap_read_unlock(current->mm);
823

824 825 826 827 828 829 830 831 832 833 834 835
	/*
	 * Permission faults just need to update the existing leaf entry,
	 * and so normally don't require allocations from the memcache. The
	 * only exception to this is when dirty logging is enabled at runtime
	 * and a write fault needs to collapse a block entry into a table.
	 */
	if (fault_status != FSC_PERM || (logging_active && write_fault)) {
		ret = kvm_mmu_topup_memory_cache(memcache,
						 kvm_mmu_cache_min_pages(kvm));
		if (ret)
			return ret;
	}
836 837 838 839 840 841 842 843 844 845 846 847 848

	mmu_seq = vcpu->kvm->mmu_notifier_seq;
	/*
	 * Ensure the read of mmu_notifier_seq happens before we call
	 * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk
	 * the page we just got a reference to gets unmapped before we have a
	 * chance to grab the mmu_lock, which ensure that if the page gets
	 * unmapped afterwards, the call to kvm_unmap_hva will take it away
	 * from us again properly. This smp_rmb() interacts with the smp_wmb()
	 * in kvm_mmu_notifier_invalidate_<page|range_end>.
	 */
	smp_rmb();

849
	pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable);
850
	if (pfn == KVM_PFN_ERR_HWPOISON) {
851
		kvm_send_hwpoison_signal(hva, vma_shift);
852 853
		return 0;
	}
854
	if (is_error_noslot_pfn(pfn))
855 856
		return -EFAULT;

857
	if (kvm_is_device_pfn(pfn)) {
858
		device = true;
859
		force_pte = true;
860
	} else if (logging_active && !write_fault) {
861 862 863 864
		/*
		 * Only actually map the page as writable if this was a write
		 * fault.
		 */
865
		writable = false;
866
	}
867

868
	if (exec_fault && device)
869 870
		return -ENOEXEC;

871
	spin_lock(&kvm->mmu_lock);
872
	pgt = vcpu->arch.hw_mmu->pgt;
873
	if (mmu_notifier_retry(kvm, mmu_seq))
874
		goto out_unlock;
875

876 877 878 879 880 881 882
	/*
	 * If we are not forced to use page mapping, check if we are
	 * backed by a THP and thus use block mapping if possible.
	 */
	if (vma_pagesize == PAGE_SIZE && !force_pte)
		vma_pagesize = transparent_hugepage_adjust(memslot, hva,
							   &pfn, &fault_ipa);
883
	if (writable)
884
		prot |= KVM_PGTABLE_PROT_W;
885

886
	if (fault_status != FSC_PERM && !device)
887 888
		clean_dcache_guest_page(pfn, vma_pagesize);

889 890
	if (exec_fault) {
		prot |= KVM_PGTABLE_PROT_X;
891
		invalidate_icache_guest_page(pfn, vma_pagesize);
892
	}
893

894 895 896 897
	if (device)
		prot |= KVM_PGTABLE_PROT_DEVICE;
	else if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC))
		prot |= KVM_PGTABLE_PROT_X;
898

899 900 901 902 903 904
	/*
	 * Under the premise of getting a FSC_PERM fault, we just need to relax
	 * permissions only if vma_pagesize equals fault_granule. Otherwise,
	 * kvm_pgtable_stage2_map() should be called to change block size.
	 */
	if (fault_status == FSC_PERM && vma_pagesize == fault_granule) {
905
		ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot);
906
	} else {
907 908 909
		ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize,
					     __pfn_to_phys(pfn), prot,
					     memcache);
910
	}
911

912 913 914 915 916 917
	/* Mark the page dirty only if the fault is handled successfully */
	if (writable && !ret) {
		kvm_set_pfn_dirty(pfn);
		mark_page_dirty(kvm, gfn);
	}

918
out_unlock:
919
	spin_unlock(&kvm->mmu_lock);
920
	kvm_set_pfn_accessed(pfn);
921
	kvm_release_pfn_clean(pfn);
922
	return ret != -EAGAIN ? ret : 0;
923 924
}

925
/* Resolve the access fault by making the page young again. */
926 927
static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
{
928 929 930
	pte_t pte;
	kvm_pte_t kpte;
	struct kvm_s2_mmu *mmu;
931 932 933 934

	trace_kvm_access_fault(fault_ipa);

	spin_lock(&vcpu->kvm->mmu_lock);
935 936
	mmu = vcpu->arch.hw_mmu;
	kpte = kvm_pgtable_stage2_mkyoung(mmu->pgt, fault_ipa);
937
	spin_unlock(&vcpu->kvm->mmu_lock);
938 939 940 941

	pte = __pte(kpte);
	if (pte_valid(pte))
		kvm_set_pfn_accessed(pte_pfn(pte));
942 943
}

944 945 946 947 948 949 950 951 952 953 954
/**
 * kvm_handle_guest_abort - handles all 2nd stage aborts
 * @vcpu:	the VCPU pointer
 *
 * Any abort that gets to the host is almost guaranteed to be caused by a
 * missing second stage translation table entry, which can mean that either the
 * guest simply needs more memory and we must allocate an appropriate page or it
 * can mean that the guest tried to access I/O memory, which is emulated by user
 * space. The distinction is based on the IPA causing the fault and whether this
 * memory region has been registered as standard RAM by user space.
 */
955
int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
956
{
957 958 959
	unsigned long fault_status;
	phys_addr_t fault_ipa;
	struct kvm_memory_slot *memslot;
960 961
	unsigned long hva;
	bool is_iabt, write_fault, writable;
962 963 964
	gfn_t gfn;
	int ret, idx;

965 966 967
	fault_status = kvm_vcpu_trap_get_fault_type(vcpu);

	fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
968
	is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
969

970
	/* Synchronous External Abort? */
971
	if (kvm_vcpu_abt_issea(vcpu)) {
972 973 974 975
		/*
		 * For RAS the host kernel may handle this abort.
		 * There is no need to pass the error into the guest.
		 */
976
		if (kvm_handle_guest_sea(fault_ipa, kvm_vcpu_get_esr(vcpu)))
977
			kvm_inject_vabt(vcpu);
978 979

		return 1;
980 981
	}

G
Gavin Shan 已提交
982
	trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu),
983
			      kvm_vcpu_get_hfar(vcpu), fault_ipa);
984 985

	/* Check the stage-2 fault is trans. fault or write fault */
986 987
	if (fault_status != FSC_FAULT && fault_status != FSC_PERM &&
	    fault_status != FSC_ACCESS) {
988 989 990
		kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
			kvm_vcpu_trap_get_class(vcpu),
			(unsigned long)kvm_vcpu_trap_get_fault(vcpu),
G
Gavin Shan 已提交
991
			(unsigned long)kvm_vcpu_get_esr(vcpu));
992 993 994 995 996 997
		return -EFAULT;
	}

	idx = srcu_read_lock(&vcpu->kvm->srcu);

	gfn = fault_ipa >> PAGE_SHIFT;
998 999
	memslot = gfn_to_memslot(vcpu->kvm, gfn);
	hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
1000
	write_fault = kvm_is_write_fault(vcpu);
1001
	if (kvm_is_error_hva(hva) || (write_fault && !writable)) {
1002 1003 1004 1005 1006 1007
		/*
		 * The guest has put either its instructions or its page-tables
		 * somewhere it shouldn't have. Userspace won't be able to do
		 * anything about this (there's no syndrome for a start), so
		 * re-inject the abort back into the guest.
		 */
1008
		if (is_iabt) {
1009 1010
			ret = -ENOEXEC;
			goto out;
1011 1012
		}

1013
		if (kvm_vcpu_abt_iss1tw(vcpu)) {
1014 1015 1016 1017 1018
			kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
			ret = 1;
			goto out_unlock;
		}

1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
		/*
		 * Check for a cache maintenance operation. Since we
		 * ended-up here, we know it is outside of any memory
		 * slot. But we can't find out if that is for a device,
		 * or if the guest is just being stupid. The only thing
		 * we know for sure is that this range cannot be cached.
		 *
		 * So let's assume that the guest is just being
		 * cautious, and skip the instruction.
		 */
1029
		if (kvm_is_error_hva(hva) && kvm_vcpu_dabt_is_cm(vcpu)) {
1030
			kvm_incr_pc(vcpu);
1031 1032 1033 1034
			ret = 1;
			goto out_unlock;
		}

M
Marc Zyngier 已提交
1035 1036 1037 1038 1039 1040 1041
		/*
		 * The IPA is reported as [MAX:12], so we need to
		 * complement it with the bottom 12 bits from the
		 * faulting VA. This is always 12 bits, irrespective
		 * of the page size.
		 */
		fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1);
1042
		ret = io_mem_abort(vcpu, fault_ipa);
1043 1044 1045
		goto out_unlock;
	}

1046
	/* Userspace should not be able to register out-of-bounds IPAs */
1047
	VM_BUG_ON(fault_ipa >= kvm_phys_size(vcpu->kvm));
1048

1049 1050 1051 1052 1053 1054
	if (fault_status == FSC_ACCESS) {
		handle_access_fault(vcpu, fault_ipa);
		ret = 1;
		goto out_unlock;
	}

1055
	ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
1056 1057
	if (ret == 0)
		ret = 1;
1058 1059 1060 1061 1062
out:
	if (ret == -ENOEXEC) {
		kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
		ret = 1;
	}
1063 1064 1065
out_unlock:
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	return ret;
1066 1067
}

1068 1069 1070 1071
static int handle_hva_to_gpa(struct kvm *kvm,
			     unsigned long start,
			     unsigned long end,
			     int (*handler)(struct kvm *kvm,
1072 1073
					    gpa_t gpa, u64 size,
					    void *data),
1074
			     void *data)
1075 1076 1077
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
1078
	int ret = 0;
1079 1080 1081 1082 1083 1084

	slots = kvm_memslots(kvm);

	/* we only care about the pages that the guest sees */
	kvm_for_each_memslot(memslot, slots) {
		unsigned long hva_start, hva_end;
1085
		gfn_t gpa;
1086 1087 1088 1089 1090 1091 1092

		hva_start = max(start, memslot->userspace_addr);
		hva_end = min(end, memslot->userspace_addr +
					(memslot->npages << PAGE_SHIFT));
		if (hva_start >= hva_end)
			continue;

1093 1094
		gpa = hva_to_gfn_memslot(hva_start, memslot) << PAGE_SHIFT;
		ret |= handler(kvm, gpa, (u64)(hva_end - hva_start), data);
1095
	}
1096 1097

	return ret;
1098 1099
}

1100
static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
1101
{
1102 1103 1104 1105
	unsigned flags = *(unsigned *)data;
	bool may_block = flags & MMU_NOTIFIER_RANGE_BLOCKABLE;

	__unmap_stage2_range(&kvm->arch.mmu, gpa, size, may_block);
1106
	return 0;
1107 1108 1109
}

int kvm_unmap_hva_range(struct kvm *kvm,
1110
			unsigned long start, unsigned long end, unsigned flags)
1111
{
1112
	if (!kvm->arch.mmu.pgt)
1113 1114 1115
		return 0;

	trace_kvm_unmap_hva_range(start, end);
1116
	handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, &flags);
1117 1118 1119
	return 0;
}

1120
static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
1121
{
1122
	kvm_pfn_t *pfn = (kvm_pfn_t *)data;
1123

1124
	WARN_ON(size != PAGE_SIZE);
1125

1126
	/*
1127 1128 1129 1130
	 * The MMU notifiers will have unmapped a huge PMD before calling
	 * ->change_pte() (which in turn calls kvm_set_spte_hva()) and
	 * therefore we never need to clear out a huge PMD through this
	 * calling path and a memcache is not required.
1131
	 */
1132 1133
	kvm_pgtable_stage2_map(kvm->arch.mmu.pgt, gpa, PAGE_SIZE,
			       __pfn_to_phys(*pfn), KVM_PGTABLE_PROT_R, NULL);
1134
	return 0;
1135 1136
}

1137
int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1138 1139
{
	unsigned long end = hva + PAGE_SIZE;
1140
	kvm_pfn_t pfn = pte_pfn(pte);
1141

1142
	if (!kvm->arch.mmu.pgt)
1143
		return 0;
1144 1145

	trace_kvm_set_spte_hva(hva);
1146 1147 1148 1149 1150 1151

	/*
	 * We've moved a page around, probably through CoW, so let's treat it
	 * just like a translation fault and clean the cache to the PoC.
	 */
	clean_dcache_guest_page(pfn, PAGE_SIZE);
1152
	handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &pfn);
1153
	return 0;
1154 1155
}

1156
static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
1157
{
1158 1159
	pte_t pte;
	kvm_pte_t kpte;
1160

1161
	WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
1162 1163 1164
	kpte = kvm_pgtable_stage2_mkold(kvm->arch.mmu.pgt, gpa);
	pte = __pte(kpte);
	return pte_valid(pte) && pte_young(pte);
1165 1166
}

1167
static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
1168
{
1169
	WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
1170
	return kvm_pgtable_stage2_is_young(kvm->arch.mmu.pgt, gpa);
1171 1172 1173 1174
}

int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
{
1175
	if (!kvm->arch.mmu.pgt)
1176
		return 0;
1177 1178 1179 1180 1181 1182
	trace_kvm_age_hva(start, end);
	return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
}

int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
{
1183
	if (!kvm->arch.mmu.pgt)
1184
		return 0;
1185
	trace_kvm_test_age_hva(hva);
1186 1187
	return handle_hva_to_gpa(kvm, hva, hva + PAGE_SIZE,
				 kvm_test_age_hva_handler, NULL);
1188 1189
}

1190 1191
phys_addr_t kvm_mmu_get_httbr(void)
{
1192
	return __pa(hyp_pgtable->pgd);
1193 1194
}

1195 1196 1197 1198 1199
phys_addr_t kvm_get_idmap_vector(void)
{
	return hyp_idmap_vector;
}

1200
static int kvm_map_idmap_text(void)
1201
{
1202 1203 1204
	unsigned long size = hyp_idmap_end - hyp_idmap_start;
	int err = __create_hyp_mappings(hyp_idmap_start, size, hyp_idmap_start,
					PAGE_HYP_EXEC);
1205 1206 1207 1208 1209 1210 1211
	if (err)
		kvm_err("Failed to idmap %lx-%lx\n",
			hyp_idmap_start, hyp_idmap_end);

	return err;
}

1212 1213
int kvm_mmu_init(void)
{
1214
	int err;
1215
	u32 hyp_va_bits;
1216

1217
	hyp_idmap_start = __pa_symbol(__hyp_idmap_text_start);
1218
	hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE);
1219
	hyp_idmap_end = __pa_symbol(__hyp_idmap_text_end);
1220
	hyp_idmap_end = ALIGN(hyp_idmap_end, PAGE_SIZE);
1221
	hyp_idmap_vector = __pa_symbol(__kvm_hyp_init);
1222

1223 1224 1225 1226 1227
	/*
	 * We rely on the linker script to ensure at build time that the HYP
	 * init code does not cross a page boundary.
	 */
	BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);
1228

1229 1230
	hyp_va_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET);
	kvm_debug("Using %u-bit virtual addresses at EL2\n", hyp_va_bits);
1231 1232 1233 1234
	kvm_debug("IDMAP page: %lx\n", hyp_idmap_start);
	kvm_debug("HYP VA range: %lx:%lx\n",
		  kern_hyp_va(PAGE_OFFSET),
		  kern_hyp_va((unsigned long)high_memory - 1));
1235

M
Marc Zyngier 已提交
1236
	if (hyp_idmap_start >= kern_hyp_va(PAGE_OFFSET) &&
1237
	    hyp_idmap_start <  kern_hyp_va((unsigned long)high_memory - 1) &&
1238
	    hyp_idmap_start != (unsigned long)__hyp_idmap_text_start) {
1239 1240 1241 1242 1243 1244 1245 1246 1247
		/*
		 * The idmap page is intersecting with the VA space,
		 * it is not safe to continue further.
		 */
		kvm_err("IDMAP intersecting with HYP VA, unable to continue\n");
		err = -EINVAL;
		goto out;
	}

1248 1249 1250
	hyp_pgtable = kzalloc(sizeof(*hyp_pgtable), GFP_KERNEL);
	if (!hyp_pgtable) {
		kvm_err("Hyp mode page-table not allocated\n");
1251 1252 1253 1254
		err = -ENOMEM;
		goto out;
	}

1255 1256 1257
	err = kvm_pgtable_hyp_init(hyp_pgtable, hyp_va_bits);
	if (err)
		goto out_free_pgtable;
1258

1259 1260 1261
	err = kvm_map_idmap_text();
	if (err)
		goto out_destroy_pgtable;
1262

1263
	io_map_base = hyp_idmap_start;
1264
	return 0;
1265 1266 1267 1268 1269 1270

out_destroy_pgtable:
	kvm_pgtable_hyp_destroy(hyp_pgtable);
out_free_pgtable:
	kfree(hyp_pgtable);
	hyp_pgtable = NULL;
1271 1272
out:
	return err;
1273
}
1274 1275

void kvm_arch_commit_memory_region(struct kvm *kvm,
1276
				   const struct kvm_userspace_memory_region *mem,
1277
				   struct kvm_memory_slot *old,
1278
				   const struct kvm_memory_slot *new,
1279 1280
				   enum kvm_mr_change change)
{
1281 1282
	/*
	 * At this point memslot has been committed and there is an
F
Fuad Tabba 已提交
1283
	 * allocated dirty_bitmap[], dirty pages will be tracked while the
1284 1285
	 * memory slot is write protected.
	 */
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
	if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
		/*
		 * If we're with initial-all-set, we don't need to write
		 * protect any pages because they're all reported as dirty.
		 * Huge pages and normal pages will be write protect gradually.
		 */
		if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) {
			kvm_mmu_wp_memory_region(kvm, mem->slot);
		}
	}
1296 1297 1298 1299
}

int kvm_arch_prepare_memory_region(struct kvm *kvm,
				   struct kvm_memory_slot *memslot,
1300
				   const struct kvm_userspace_memory_region *mem,
1301 1302
				   enum kvm_mr_change change)
{
1303 1304 1305 1306 1307
	hva_t hva = mem->userspace_addr;
	hva_t reg_end = hva + mem->memory_size;
	bool writable = !(mem->flags & KVM_MEM_READONLY);
	int ret = 0;

1308 1309
	if (change != KVM_MR_CREATE && change != KVM_MR_MOVE &&
			change != KVM_MR_FLAGS_ONLY)
1310 1311
		return 0;

1312 1313 1314 1315
	/*
	 * Prevent userspace from creating a memory region outside of the IPA
	 * space addressable by the KVM guest IPA space.
	 */
1316
	if ((memslot->base_gfn + memslot->npages) > (kvm_phys_size(kvm) >> PAGE_SHIFT))
1317 1318
		return -EFAULT;

1319
	mmap_read_lock(current->mm);
1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
	/*
	 * A memory region could potentially cover multiple VMAs, and any holes
	 * between them, so iterate over all of them to find out if we can map
	 * any of them right now.
	 *
	 *     +--------------------------------------------+
	 * +---------------+----------------+   +----------------+
	 * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
	 * +---------------+----------------+   +----------------+
	 *     |               memory region                |
	 *     +--------------------------------------------+
	 */
	do {
1333
		struct vm_area_struct *vma;
1334 1335
		hva_t vm_start, vm_end;

1336 1337
		vma = find_vma_intersection(current->mm, hva, reg_end);
		if (!vma)
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
			break;

		/*
		 * Take the intersection of this VMA with the memory region
		 */
		vm_start = max(hva, vma->vm_start);
		vm_end = min(reg_end, vma->vm_end);

		if (vma->vm_flags & VM_PFNMAP) {
			gpa_t gpa = mem->guest_phys_addr +
				    (vm_start - mem->userspace_addr);
1349 1350 1351 1352
			phys_addr_t pa;

			pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
			pa += vm_start - vma->vm_start;
1353

1354
			/* IO region dirty page logging not allowed */
1355 1356 1357 1358
			if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES) {
				ret = -EINVAL;
				goto out;
			}
1359

1360 1361 1362 1363 1364 1365 1366 1367 1368
			ret = kvm_phys_addr_ioremap(kvm, gpa, pa,
						    vm_end - vm_start,
						    writable);
			if (ret)
				break;
		}
		hva = vm_end;
	} while (hva < reg_end);

1369
	if (change == KVM_MR_FLAGS_ONLY)
1370
		goto out;
1371

1372 1373
	spin_lock(&kvm->mmu_lock);
	if (ret)
1374
		unmap_stage2_range(&kvm->arch.mmu, mem->guest_phys_addr, mem->memory_size);
1375
	else if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1376 1377
		stage2_flush_memslot(kvm, memslot);
	spin_unlock(&kvm->mmu_lock);
1378
out:
1379
	mmap_read_unlock(current->mm);
1380
	return ret;
1381 1382
}

1383
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
1384 1385 1386
{
}

1387
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
1388 1389 1390 1391 1392
{
}

void kvm_arch_flush_shadow_all(struct kvm *kvm)
{
1393
	kvm_free_stage2_pgd(&kvm->arch.mmu);
1394 1395 1396 1397 1398
}

void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
1399 1400 1401 1402
	gpa_t gpa = slot->base_gfn << PAGE_SHIFT;
	phys_addr_t size = slot->npages << PAGE_SHIFT;

	spin_lock(&kvm->mmu_lock);
1403
	unmap_stage2_range(&kvm->arch.mmu, gpa, size);
1404
	spin_unlock(&kvm->mmu_lock);
1405
}
1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436

/*
 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
 *
 * Main problems:
 * - S/W ops are local to a CPU (not broadcast)
 * - We have line migration behind our back (speculation)
 * - System caches don't support S/W at all (damn!)
 *
 * In the face of the above, the best we can do is to try and convert
 * S/W ops to VA ops. Because the guest is not allowed to infer the
 * S/W to PA mapping, it can only use S/W to nuke the whole cache,
 * which is a rather good thing for us.
 *
 * Also, it is only used when turning caches on/off ("The expected
 * usage of the cache maintenance instructions that operate by set/way
 * is associated with the cache maintenance instructions associated
 * with the powerdown and powerup of caches, if this is required by
 * the implementation.").
 *
 * We use the following policy:
 *
 * - If we trap a S/W operation, we enable VM trapping to detect
 *   caches being turned on/off, and do a full clean.
 *
 * - We flush the caches on both caches being turned on and off.
 *
 * - Once the caches are enabled, we stop trapping VM ops.
 */
void kvm_set_way_flush(struct kvm_vcpu *vcpu)
{
1437
	unsigned long hcr = *vcpu_hcr(vcpu);
1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451

	/*
	 * If this is the first time we do a S/W operation
	 * (i.e. HCR_TVM not set) flush the whole memory, and set the
	 * VM trapping.
	 *
	 * Otherwise, rely on the VM trapping to wait for the MMU +
	 * Caches to be turned off. At that point, we'll be able to
	 * clean the caches again.
	 */
	if (!(hcr & HCR_TVM)) {
		trace_kvm_set_way_flush(*vcpu_pc(vcpu),
					vcpu_has_cache_enabled(vcpu));
		stage2_flush_vm(vcpu->kvm);
1452
		*vcpu_hcr(vcpu) = hcr | HCR_TVM;
1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469
	}
}

void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
{
	bool now_enabled = vcpu_has_cache_enabled(vcpu);

	/*
	 * If switching the MMU+caches on, need to invalidate the caches.
	 * If switching it off, need to clean the caches.
	 * Clean + invalidate does the trick always.
	 */
	if (now_enabled != was_enabled)
		stage2_flush_vm(vcpu->kvm);

	/* Caches are now on, stop trapping VM ops (until a S/W op) */
	if (now_enabled)
1470
		*vcpu_hcr(vcpu) &= ~HCR_TVM;
1471 1472 1473

	trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
}