mmu.c 47.2 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5
/*
 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
 */
6 7 8 9

#include <linux/mman.h>
#include <linux/kvm_host.h>
#include <linux/io.h>
10
#include <linux/hugetlb.h>
11
#include <linux/sched/signal.h>
12
#include <trace/events/kvm.h>
13
#include <asm/pgalloc.h>
14
#include <asm/cacheflush.h>
15 16
#include <asm/kvm_arm.h>
#include <asm/kvm_mmu.h>
17
#include <asm/kvm_pgtable.h>
18
#include <asm/kvm_ras.h>
19
#include <asm/kvm_asm.h>
20
#include <asm/kvm_emulate.h>
21
#include <asm/virt.h>
22 23

#include "trace.h"
24

25
static struct kvm_pgtable *hyp_pgtable;
26 27
static DEFINE_MUTEX(kvm_hyp_pgd_mutex);

28 29 30 31
static unsigned long hyp_idmap_start;
static unsigned long hyp_idmap_end;
static phys_addr_t hyp_idmap_vector;

32 33
static unsigned long io_map_base;

34

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
/*
 * Release kvm_mmu_lock periodically if the memory region is large. Otherwise,
 * we may see kernel panics with CONFIG_DETECT_HUNG_TASK,
 * CONFIG_LOCKUP_DETECTOR, CONFIG_LOCKDEP. Additionally, holding the lock too
 * long will also starve other vCPUs. We have to also make sure that the page
 * tables are not freed while we released the lock.
 */
static int stage2_apply_range(struct kvm *kvm, phys_addr_t addr,
			      phys_addr_t end,
			      int (*fn)(struct kvm_pgtable *, u64, u64),
			      bool resched)
{
	int ret;
	u64 next;

	do {
		struct kvm_pgtable *pgt = kvm->arch.mmu.pgt;
		if (!pgt)
			return -EINVAL;

		next = stage2_pgd_addr_end(kvm, addr, end);
		ret = fn(pgt, addr, next - addr);
		if (ret)
			break;

		if (resched && next != end)
61
			cond_resched_rwlock_write(&kvm->mmu_lock);
62 63 64 65 66
	} while (addr = next, addr != end);

	return ret;
}

67 68 69
#define stage2_apply_range_resched(kvm, addr, end, fn)			\
	stage2_apply_range(kvm, addr, end, fn, true)

70 71 72
static bool memslot_is_logging(struct kvm_memory_slot *memslot)
{
	return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY);
73 74 75 76 77 78 79 80 81 82
}

/**
 * kvm_flush_remote_tlbs() - flush all VM TLB entries for v7/8
 * @kvm:	pointer to kvm structure.
 *
 * Interface to HYP function to flush all VM TLB entries
 */
void kvm_flush_remote_tlbs(struct kvm *kvm)
{
83
	++kvm->stat.generic.remote_tlb_flush_requests;
84
	kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu);
85
}
86

87 88
static bool kvm_is_device_pfn(unsigned long pfn)
{
89
	return !pfn_is_map_memory(pfn);
90 91
}

92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129
static void *stage2_memcache_zalloc_page(void *arg)
{
	struct kvm_mmu_memory_cache *mc = arg;

	/* Allocated with __GFP_ZERO, so no need to zero */
	return kvm_mmu_memory_cache_alloc(mc);
}

static void *kvm_host_zalloc_pages_exact(size_t size)
{
	return alloc_pages_exact(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
}

static void kvm_host_get_page(void *addr)
{
	get_page(virt_to_page(addr));
}

static void kvm_host_put_page(void *addr)
{
	put_page(virt_to_page(addr));
}

static int kvm_host_page_count(void *addr)
{
	return page_count(virt_to_page(addr));
}

static phys_addr_t kvm_host_pa(void *addr)
{
	return __pa(addr);
}

static void *kvm_host_va(phys_addr_t phys)
{
	return __va(phys);
}

130 131 132 133 134 135 136 137 138 139
static void clean_dcache_guest_page(void *va, size_t size)
{
	__clean_dcache_guest_page(va, size);
}

static void invalidate_icache_guest_page(void *va, size_t size)
{
	__invalidate_icache_guest_page(va, size);
}

140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
/*
 * Unmapping vs dcache management:
 *
 * If a guest maps certain memory pages as uncached, all writes will
 * bypass the data cache and go directly to RAM.  However, the CPUs
 * can still speculate reads (not writes) and fill cache lines with
 * data.
 *
 * Those cache lines will be *clean* cache lines though, so a
 * clean+invalidate operation is equivalent to an invalidate
 * operation, because no cache lines are marked dirty.
 *
 * Those clean cache lines could be filled prior to an uncached write
 * by the guest, and the cache coherent IO subsystem would therefore
 * end up writing old data to disk.
 *
 * This is why right after unmapping a page/section and invalidating
157 158
 * the corresponding TLBs, we flush to make sure the IO subsystem will
 * never hit in the cache.
159 160 161 162
 *
 * This is all avoided on systems that have ARM64_HAS_STAGE2_FWB, as
 * we then fully enforce cacheability of RAM, no matter what the guest
 * does.
163
 */
164 165
/**
 * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
166
 * @mmu:   The KVM stage-2 MMU pointer
167 168
 * @start: The intermediate physical base address of the range to unmap
 * @size:  The size of the area to unmap
169
 * @may_block: Whether or not we are permitted to block
170 171 172 173 174 175
 *
 * Clear a range of stage-2 mappings, lowering the various ref-counts.  Must
 * be called while holding mmu_lock (unless for freeing the stage2 pgd before
 * destroying the VM), otherwise another faulting VCPU may come in and mess
 * with things behind our backs.
 */
176 177
static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size,
				 bool may_block)
178
{
179
	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
180
	phys_addr_t end = start + size;
181

182
	lockdep_assert_held_write(&kvm->mmu_lock);
183
	WARN_ON(size & ~PAGE_MASK);
184 185
	WARN_ON(stage2_apply_range(kvm, start, end, kvm_pgtable_stage2_unmap,
				   may_block));
186 187
}

188 189 190 191 192
static void unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size)
{
	__unmap_stage2_range(mmu, start, size, true);
}

193 194 195 196 197 198
static void stage2_flush_memslot(struct kvm *kvm,
				 struct kvm_memory_slot *memslot)
{
	phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
	phys_addr_t end = addr + PAGE_SIZE * memslot->npages;

199
	stage2_apply_range_resched(kvm, addr, end, kvm_pgtable_stage2_flush);
200 201 202 203 204 205 206 207 208
}

/**
 * stage2_flush_vm - Invalidate cache for pages mapped in stage 2
 * @kvm: The struct kvm pointer
 *
 * Go through the stage 2 page tables and invalidate any cache lines
 * backing memory already mapped to the VM.
 */
209
static void stage2_flush_vm(struct kvm *kvm)
210 211 212
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
213
	int idx, bkt;
214 215

	idx = srcu_read_lock(&kvm->srcu);
216
	write_lock(&kvm->mmu_lock);
217 218

	slots = kvm_memslots(kvm);
219
	kvm_for_each_memslot(memslot, bkt, slots)
220 221
		stage2_flush_memslot(kvm, memslot);

222
	write_unlock(&kvm->mmu_lock);
223 224 225
	srcu_read_unlock(&kvm->srcu, idx);
}

226
/**
227
 * free_hyp_pgds - free Hyp-mode page tables
228
 */
229
void free_hyp_pgds(void)
230
{
231
	mutex_lock(&kvm_hyp_pgd_mutex);
232 233 234
	if (hyp_pgtable) {
		kvm_pgtable_hyp_destroy(hyp_pgtable);
		kfree(hyp_pgtable);
235
		hyp_pgtable = NULL;
236
	}
237 238 239
	mutex_unlock(&kvm_hyp_pgd_mutex);
}

240 241
static bool kvm_host_owns_hyp_mappings(void)
{
242 243 244
	if (is_kernel_in_hyp_mode())
		return false;

245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
	if (static_branch_likely(&kvm_protected_mode_initialized))
		return false;

	/*
	 * This can happen at boot time when __create_hyp_mappings() is called
	 * after the hyp protection has been enabled, but the static key has
	 * not been flipped yet.
	 */
	if (!hyp_pgtable && is_protected_kvm_enabled())
		return false;

	WARN_ON(!hyp_pgtable);

	return true;
}

261 262
static int __create_hyp_mappings(unsigned long start, unsigned long size,
				 unsigned long phys, enum kvm_pgtable_prot prot)
263
{
264
	int err;
265

266 267
	if (WARN_ON(!kvm_host_owns_hyp_mappings()))
		return -EINVAL;
268

269
	mutex_lock(&kvm_hyp_pgd_mutex);
270
	err = kvm_pgtable_hyp_map(hyp_pgtable, start, size, phys, prot);
271
	mutex_unlock(&kvm_hyp_pgd_mutex);
272

273 274 275
	return err;
}

276 277 278 279 280 281 282 283 284 285 286
static phys_addr_t kvm_kaddr_to_phys(void *kaddr)
{
	if (!is_vmalloc_addr(kaddr)) {
		BUG_ON(!virt_addr_valid(kaddr));
		return __pa(kaddr);
	} else {
		return page_to_phys(vmalloc_to_page(kaddr)) +
		       offset_in_page(kaddr);
	}
}

287 288 289 290 291 292 293 294 295 296 297
struct hyp_shared_pfn {
	u64 pfn;
	int count;
	struct rb_node node;
};

static DEFINE_MUTEX(hyp_shared_pfns_lock);
static struct rb_root hyp_shared_pfns = RB_ROOT;

static struct hyp_shared_pfn *find_shared_pfn(u64 pfn, struct rb_node ***node,
					      struct rb_node **parent)
298
{
299 300 301 302 303 304 305 306 307 308 309 310 311 312
	struct hyp_shared_pfn *this;

	*node = &hyp_shared_pfns.rb_node;
	*parent = NULL;
	while (**node) {
		this = container_of(**node, struct hyp_shared_pfn, node);
		*parent = **node;
		if (this->pfn < pfn)
			*node = &((**node)->rb_left);
		else if (this->pfn > pfn)
			*node = &((**node)->rb_right);
		else
			return this;
	}
313

314 315 316 317
	return NULL;
}

static int share_pfn_hyp(u64 pfn)
318
{
319 320 321 322 323 324 325 326 327
	struct rb_node **node, *parent;
	struct hyp_shared_pfn *this;
	int ret = 0;

	mutex_lock(&hyp_shared_pfns_lock);
	this = find_shared_pfn(pfn, &node, &parent);
	if (this) {
		this->count++;
		goto unlock;
328 329
	}

330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
	this = kzalloc(sizeof(*this), GFP_KERNEL);
	if (!this) {
		ret = -ENOMEM;
		goto unlock;
	}

	this->pfn = pfn;
	this->count = 1;
	rb_link_node(&this->node, parent, node);
	rb_insert_color(&this->node, &hyp_shared_pfns);
	ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, pfn, 1);
unlock:
	mutex_unlock(&hyp_shared_pfns_lock);

	return ret;
345 346
}

347
static int unshare_pfn_hyp(u64 pfn)
348
{
349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
	struct rb_node **node, *parent;
	struct hyp_shared_pfn *this;
	int ret = 0;

	mutex_lock(&hyp_shared_pfns_lock);
	this = find_shared_pfn(pfn, &node, &parent);
	if (WARN_ON(!this)) {
		ret = -ENOENT;
		goto unlock;
	}

	this->count--;
	if (this->count)
		goto unlock;

	rb_erase(&this->node, &hyp_shared_pfns);
	kfree(this);
	ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn, 1);
unlock:
	mutex_unlock(&hyp_shared_pfns_lock);

	return ret;
}

373 374
int kvm_share_hyp(void *from, void *to)
{
375 376
	phys_addr_t start, end, cur;
	u64 pfn;
377 378
	int ret;

379 380 381 382 383 384 385 386 387 388 389 390 391 392
	if (is_kernel_in_hyp_mode())
		return 0;

	/*
	 * The share hcall maps things in the 'fixed-offset' region of the hyp
	 * VA space, so we can only share physically contiguous data-structures
	 * for now.
	 */
	if (is_vmalloc_or_module_addr(from) || is_vmalloc_or_module_addr(to))
		return -EINVAL;

	if (kvm_host_owns_hyp_mappings())
		return create_hyp_mappings(from, to, PAGE_HYP);

393 394 395 396 397
	start = ALIGN_DOWN(__pa(from), PAGE_SIZE);
	end = PAGE_ALIGN(__pa(to));
	for (cur = start; cur < end; cur += PAGE_SIZE) {
		pfn = __phys_to_pfn(cur);
		ret = share_pfn_hyp(pfn);
398 399 400 401 402 403 404
		if (ret)
			return ret;
	}

	return 0;
}

405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
void kvm_unshare_hyp(void *from, void *to)
{
	phys_addr_t start, end, cur;
	u64 pfn;

	if (is_kernel_in_hyp_mode() || kvm_host_owns_hyp_mappings() || !from)
		return;

	start = ALIGN_DOWN(__pa(from), PAGE_SIZE);
	end = PAGE_ALIGN(__pa(to));
	for (cur = start; cur < end; cur += PAGE_SIZE) {
		pfn = __phys_to_pfn(cur);
		WARN_ON(unshare_pfn_hyp(pfn));
	}
}

421
/**
422
 * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
423 424
 * @from:	The virtual kernel start address of the range
 * @to:		The virtual kernel end address of the range (exclusive)
425
 * @prot:	The protection to be applied to this range
426
 *
427 428 429
 * The same virtual address as the kernel virtual address is also used
 * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying
 * physical pages.
430
 */
431
int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
432
{
433 434
	phys_addr_t phys_addr;
	unsigned long virt_addr;
435 436
	unsigned long start = kern_hyp_va((unsigned long)from);
	unsigned long end = kern_hyp_va((unsigned long)to);
437

438 439 440
	if (is_kernel_in_hyp_mode())
		return 0;

441 442
	if (!kvm_host_owns_hyp_mappings())
		return -EPERM;
443

444 445
	start = start & PAGE_MASK;
	end = PAGE_ALIGN(end);
446

447 448
	for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
		int err;
449

450
		phys_addr = kvm_kaddr_to_phys(from + virt_addr - start);
451
		err = __create_hyp_mappings(virt_addr, PAGE_SIZE, phys_addr,
452
					    prot);
453 454 455 456 457
		if (err)
			return err;
	}

	return 0;
458 459
}

460
static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size,
461 462
					unsigned long *haddr,
					enum kvm_pgtable_prot prot)
463
{
464 465
	unsigned long base;
	int ret = 0;
466

467 468 469 470 471 472 473 474 475 476
	if (!kvm_host_owns_hyp_mappings()) {
		base = kvm_call_hyp_nvhe(__pkvm_create_private_mapping,
					 phys_addr, size, prot);
		if (IS_ERR_OR_NULL((void *)base))
			return PTR_ERR((void *)base);
		*haddr = base;

		return 0;
	}

477
	mutex_lock(&kvm_hyp_pgd_mutex);
478

479
	/*
480
	 * This assumes that we have enough space below the idmap
481 482 483 484 485 486 487 488
	 * page to allocate our VAs. If not, the check below will
	 * kick. A potential alternative would be to detect that
	 * overflow and switch to an allocation above the idmap.
	 *
	 * The allocated size is always a multiple of PAGE_SIZE.
	 */
	size = PAGE_ALIGN(size + offset_in_page(phys_addr));
	base = io_map_base - size;
489

490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
	/*
	 * Verify that BIT(VA_BITS - 1) hasn't been flipped by
	 * allocating the new area, as it would indicate we've
	 * overflowed the idmap/IO address range.
	 */
	if ((base ^ io_map_base) & BIT(VA_BITS - 1))
		ret = -ENOMEM;
	else
		io_map_base = base;

	mutex_unlock(&kvm_hyp_pgd_mutex);

	if (ret)
		goto out;

505
	ret = __create_hyp_mappings(base, size, phys_addr, prot);
506 507 508
	if (ret)
		goto out;

509
	*haddr = base + offset_in_page(phys_addr);
510
out:
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
	return ret;
}

/**
 * create_hyp_io_mappings - Map IO into both kernel and HYP
 * @phys_addr:	The physical start address which gets mapped
 * @size:	Size of the region being mapped
 * @kaddr:	Kernel VA for this mapping
 * @haddr:	HYP VA for this mapping
 */
int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,
			   void __iomem **kaddr,
			   void __iomem **haddr)
{
	unsigned long addr;
	int ret;

528 529 530
	if (is_protected_kvm_enabled())
		return -EPERM;

531 532 533 534 535 536 537 538 539 540 541
	*kaddr = ioremap(phys_addr, size);
	if (!*kaddr)
		return -ENOMEM;

	if (is_kernel_in_hyp_mode()) {
		*haddr = *kaddr;
		return 0;
	}

	ret = __create_hyp_private_mapping(phys_addr, size,
					   &addr, PAGE_HYP_DEVICE);
542 543 544
	if (ret) {
		iounmap(*kaddr);
		*kaddr = NULL;
545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570
		*haddr = NULL;
		return ret;
	}

	*haddr = (void __iomem *)addr;
	return 0;
}

/**
 * create_hyp_exec_mappings - Map an executable range into HYP
 * @phys_addr:	The physical start address which gets mapped
 * @size:	Size of the region being mapped
 * @haddr:	HYP VA for this mapping
 */
int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size,
			     void **haddr)
{
	unsigned long addr;
	int ret;

	BUG_ON(is_kernel_in_hyp_mode());

	ret = __create_hyp_private_mapping(phys_addr, size,
					   &addr, PAGE_HYP_EXEC);
	if (ret) {
		*haddr = NULL;
571 572 573
		return ret;
	}

574
	*haddr = (void *)addr;
575
	return 0;
576 577
}

578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
static struct kvm_pgtable_mm_ops kvm_user_mm_ops = {
	/* We shouldn't need any other callback to walk the PT */
	.phys_to_virt		= kvm_host_va,
};

static int get_user_mapping_size(struct kvm *kvm, u64 addr)
{
	struct kvm_pgtable pgt = {
		.pgd		= (kvm_pte_t *)kvm->mm->pgd,
		.ia_bits	= VA_BITS,
		.start_level	= (KVM_PGTABLE_MAX_LEVELS -
				   CONFIG_PGTABLE_LEVELS),
		.mm_ops		= &kvm_user_mm_ops,
	};
	kvm_pte_t pte = 0;	/* Keep GCC quiet... */
	u32 level = ~0;
	int ret;

	ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level);
	VM_BUG_ON(ret);
	VM_BUG_ON(level >= KVM_PGTABLE_MAX_LEVELS);
	VM_BUG_ON(!(pte & PTE_VALID));

	return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level));
}

604 605 606 607 608 609 610 611 612
static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = {
	.zalloc_page		= stage2_memcache_zalloc_page,
	.zalloc_pages_exact	= kvm_host_zalloc_pages_exact,
	.free_pages_exact	= free_pages_exact,
	.get_page		= kvm_host_get_page,
	.put_page		= kvm_host_put_page,
	.page_count		= kvm_host_page_count,
	.phys_to_virt		= kvm_host_va,
	.virt_to_phys		= kvm_host_pa,
613 614
	.dcache_clean_inval_poc	= clean_dcache_guest_page,
	.icache_inval_pou	= invalidate_icache_guest_page,
615 616
};

617
/**
618
 * kvm_init_stage2_mmu - Initialise a S2 MMU structure
619 620
 * @kvm:	The pointer to the KVM structure
 * @mmu:	The pointer to the s2 MMU structure
621
 *
622
 * Allocates only the stage-2 HW PGD level table(s).
623 624 625
 * Note we don't need locking here as this is only called when the VM is
 * created, which can only be done once.
 */
626
int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
627
{
628 629
	int cpu, err;
	struct kvm_pgtable *pgt;
630

631
	if (mmu->pgt != NULL) {
632 633 634 635
		kvm_err("kvm_arch already initialized?\n");
		return -EINVAL;
	}

636
	pgt = kzalloc(sizeof(*pgt), GFP_KERNEL_ACCOUNT);
637
	if (!pgt)
638 639
		return -ENOMEM;

640 641
	mmu->arch = &kvm->arch;
	err = kvm_pgtable_stage2_init(pgt, mmu, &kvm_s2_mm_ops);
642 643
	if (err)
		goto out_free_pgtable;
644

645 646
	mmu->last_vcpu_ran = alloc_percpu(typeof(*mmu->last_vcpu_ran));
	if (!mmu->last_vcpu_ran) {
647 648
		err = -ENOMEM;
		goto out_destroy_pgtable;
649 650 651 652 653
	}

	for_each_possible_cpu(cpu)
		*per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1;

654 655
	mmu->pgt = pgt;
	mmu->pgd_phys = __pa(pgt->pgd);
656
	return 0;
657 658 659 660 661 662

out_destroy_pgtable:
	kvm_pgtable_stage2_destroy(pgt);
out_free_pgtable:
	kfree(pgt);
	return err;
663 664
}

665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685
static void stage2_unmap_memslot(struct kvm *kvm,
				 struct kvm_memory_slot *memslot)
{
	hva_t hva = memslot->userspace_addr;
	phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
	phys_addr_t size = PAGE_SIZE * memslot->npages;
	hva_t reg_end = hva + size;

	/*
	 * A memory region could potentially cover multiple VMAs, and any holes
	 * between them, so iterate over all of them to find out if we should
	 * unmap any of them.
	 *
	 *     +--------------------------------------------+
	 * +---------------+----------------+   +----------------+
	 * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
	 * +---------------+----------------+   +----------------+
	 *     |               memory region                |
	 *     +--------------------------------------------+
	 */
	do {
686
		struct vm_area_struct *vma;
687 688
		hva_t vm_start, vm_end;

689 690
		vma = find_vma_intersection(current->mm, hva, reg_end);
		if (!vma)
691 692 693 694 695 696 697 698 699 700
			break;

		/*
		 * Take the intersection of this VMA with the memory region
		 */
		vm_start = max(hva, vma->vm_start);
		vm_end = min(reg_end, vma->vm_end);

		if (!(vma->vm_flags & VM_PFNMAP)) {
			gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
701
			unmap_stage2_range(&kvm->arch.mmu, gpa, vm_end - vm_start);
702 703 704 705 706 707 708 709 710
		}
		hva = vm_end;
	} while (hva < reg_end);
}

/**
 * stage2_unmap_vm - Unmap Stage-2 RAM mappings
 * @kvm: The struct kvm pointer
 *
711
 * Go through the memregions and unmap any regular RAM
712 713 714 715 716 717
 * backing memory already mapped to the VM.
 */
void stage2_unmap_vm(struct kvm *kvm)
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
718
	int idx, bkt;
719 720

	idx = srcu_read_lock(&kvm->srcu);
721
	mmap_read_lock(current->mm);
722
	write_lock(&kvm->mmu_lock);
723 724

	slots = kvm_memslots(kvm);
725
	kvm_for_each_memslot(memslot, bkt, slots)
726 727
		stage2_unmap_memslot(kvm, memslot);

728
	write_unlock(&kvm->mmu_lock);
729
	mmap_read_unlock(current->mm);
730 731 732
	srcu_read_unlock(&kvm->srcu, idx);
}

733
void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu)
734
{
735
	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
736
	struct kvm_pgtable *pgt = NULL;
737

738
	write_lock(&kvm->mmu_lock);
739 740 741 742 743
	pgt = mmu->pgt;
	if (pgt) {
		mmu->pgd_phys = 0;
		mmu->pgt = NULL;
		free_percpu(mmu->last_vcpu_ran);
744
	}
745
	write_unlock(&kvm->mmu_lock);
746

747 748 749
	if (pgt) {
		kvm_pgtable_stage2_destroy(pgt);
		kfree(pgt);
750
	}
751 752 753 754 755 756 757 758 759
}

/**
 * kvm_phys_addr_ioremap - map a device range to guest IPA
 *
 * @kvm:	The KVM pointer
 * @guest_ipa:	The IPA at which to insert the mapping
 * @pa:		The physical address of the device
 * @size:	The size of the mapping
760
 * @writable:   Whether or not to create a writable mapping
761 762
 */
int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
763
			  phys_addr_t pa, unsigned long size, bool writable)
764
{
765
	phys_addr_t addr;
766
	int ret = 0;
767
	struct kvm_mmu_memory_cache cache = { 0, __GFP_ZERO, NULL, };
768 769 770 771
	struct kvm_pgtable *pgt = kvm->arch.mmu.pgt;
	enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_DEVICE |
				     KVM_PGTABLE_PROT_R |
				     (writable ? KVM_PGTABLE_PROT_W : 0);
772

773 774 775
	if (is_protected_kvm_enabled())
		return -EPERM;

776 777
	size += offset_in_page(guest_ipa);
	guest_ipa &= PAGE_MASK;
778

779
	for (addr = guest_ipa; addr < guest_ipa + size; addr += PAGE_SIZE) {
780 781
		ret = kvm_mmu_topup_memory_cache(&cache,
						 kvm_mmu_cache_min_pages(kvm));
782
		if (ret)
783 784
			break;

785
		write_lock(&kvm->mmu_lock);
786 787
		ret = kvm_pgtable_stage2_map(pgt, addr, PAGE_SIZE, pa, prot,
					     &cache);
788
		write_unlock(&kvm->mmu_lock);
789
		if (ret)
790
			break;
791

792
		pa += PAGE_SIZE;
793 794
	}

795
	kvm_mmu_free_memory_cache(&cache);
796 797 798
	return ret;
}

799 800
/**
 * stage2_wp_range() - write protect stage2 memory region range
801
 * @mmu:        The KVM stage-2 MMU pointer
802 803 804
 * @addr:	Start address of range
 * @end:	End address of range
 */
805
static void stage2_wp_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end)
806
{
807
	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
808
	stage2_apply_range_resched(kvm, addr, end, kvm_pgtable_stage2_wrprotect);
809 810 811 812 813 814 815 816 817
}

/**
 * kvm_mmu_wp_memory_region() - write protect stage 2 entries for memory slot
 * @kvm:	The KVM pointer
 * @slot:	The memory slot to write protect
 *
 * Called to start logging dirty pages after memory region
 * KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns
818
 * all present PUD, PMD and PTEs are write protected in the memory region.
819 820 821 822 823
 * Afterwards read of dirty page log can be called.
 *
 * Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired,
 * serializing operations for VM memory regions.
 */
824
static void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
825
{
826 827
	struct kvm_memslots *slots = kvm_memslots(kvm);
	struct kvm_memory_slot *memslot = id_to_memslot(slots, slot);
828 829 830 831 832 833 834
	phys_addr_t start, end;

	if (WARN_ON_ONCE(!memslot))
		return;

	start = memslot->base_gfn << PAGE_SHIFT;
	end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
835

836
	write_lock(&kvm->mmu_lock);
837
	stage2_wp_range(&kvm->arch.mmu, start, end);
838
	write_unlock(&kvm->mmu_lock);
839 840
	kvm_flush_remote_tlbs(kvm);
}
841 842

/**
843
 * kvm_mmu_write_protect_pt_masked() - write protect dirty pages
844 845 846 847 848 849 850 851 852
 * @kvm:	The KVM pointer
 * @slot:	The memory slot associated with mask
 * @gfn_offset:	The gfn offset in memory slot
 * @mask:	The mask of dirty pages at offset 'gfn_offset' in this memory
 *		slot to be write protected
 *
 * Walks bits set in mask write protects the associated pte's. Caller must
 * acquire kvm_mmu_lock.
 */
853
static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
854 855 856 857 858 859 860
		struct kvm_memory_slot *slot,
		gfn_t gfn_offset, unsigned long mask)
{
	phys_addr_t base_gfn = slot->base_gfn + gfn_offset;
	phys_addr_t start = (base_gfn +  __ffs(mask)) << PAGE_SHIFT;
	phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;

861
	stage2_wp_range(&kvm->arch.mmu, start, end);
862
}
863

864 865 866 867 868 869 870 871 872 873 874 875 876 877
/*
 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
 * dirty pages.
 *
 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
 * enable dirty logging for them.
 */
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
		struct kvm_memory_slot *slot,
		gfn_t gfn_offset, unsigned long mask)
{
	kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
}

878
static void kvm_send_hwpoison_signal(unsigned long address, short lsb)
879
{
880
	send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current);
881 882
}

883 884 885
static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot,
					       unsigned long hva,
					       unsigned long map_size)
886
{
887
	gpa_t gpa_start;
888 889 890
	hva_t uaddr_start, uaddr_end;
	size_t size;

891 892 893 894
	/* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */
	if (map_size == PAGE_SIZE)
		return true;

895 896 897 898 899 900 901 902 903
	size = memslot->npages * PAGE_SIZE;

	gpa_start = memslot->base_gfn << PAGE_SHIFT;

	uaddr_start = memslot->userspace_addr;
	uaddr_end = uaddr_start + size;

	/*
	 * Pages belonging to memslots that don't have the same alignment
904 905
	 * within a PMD/PUD for userspace and IPA cannot be mapped with stage-2
	 * PMD/PUD entries, because we'll end up mapping the wrong pages.
906 907 908 909 910
	 *
	 * Consider a layout like the following:
	 *
	 *    memslot->userspace_addr:
	 *    +-----+--------------------+--------------------+---+
911
	 *    |abcde|fgh  Stage-1 block  |    Stage-1 block tv|xyz|
912 913
	 *    +-----+--------------------+--------------------+---+
	 *
914
	 *    memslot->base_gfn << PAGE_SHIFT:
915
	 *      +---+--------------------+--------------------+-----+
916
	 *      |abc|def  Stage-2 block  |    Stage-2 block   |tvxyz|
917 918
	 *      +---+--------------------+--------------------+-----+
	 *
919
	 * If we create those stage-2 blocks, we'll end up with this incorrect
920 921 922 923 924
	 * mapping:
	 *   d -> f
	 *   e -> g
	 *   f -> h
	 */
925
	if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1)))
926 927 928 929
		return false;

	/*
	 * Next, let's make sure we're not trying to map anything not covered
930 931
	 * by the memslot. This means we have to prohibit block size mappings
	 * for the beginning and end of a non-block aligned and non-block sized
932 933 934 935 936 937 938 939
	 * memory slot (illustrated by the head and tail parts of the
	 * userspace view above containing pages 'abcde' and 'xyz',
	 * respectively).
	 *
	 * Note that it doesn't matter if we do the check using the
	 * userspace_addr or the base_gfn, as both are equally aligned (per
	 * the check above) and equally sized.
	 */
940 941
	return (hva & ~(map_size - 1)) >= uaddr_start &&
	       (hva & ~(map_size - 1)) + map_size <= uaddr_end;
942 943
}

944 945 946 947 948 949 950 951 952
/*
 * Check if the given hva is backed by a transparent huge page (THP) and
 * whether it can be mapped using block mapping in stage2. If so, adjust
 * the stage2 PFN and IPA accordingly. Only PMD_SIZE THPs are currently
 * supported. This will need to be updated to support other THP sizes.
 *
 * Returns the size of the mapping.
 */
static unsigned long
953
transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot,
954 955 956 957 958 959 960 961 962 963
			    unsigned long hva, kvm_pfn_t *pfnp,
			    phys_addr_t *ipap)
{
	kvm_pfn_t pfn = *pfnp;

	/*
	 * Make sure the adjustment is done only for THP pages. Also make
	 * sure that the HVA and IPA are sufficiently aligned and that the
	 * block map is contained within the memslot.
	 */
964 965
	if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE) &&
	    get_user_mapping_size(kvm, hva) >= PMD_SIZE) {
966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986
		/*
		 * The address we faulted on is backed by a transparent huge
		 * page.  However, because we map the compound huge page and
		 * not the individual tail page, we need to transfer the
		 * refcount to the head page.  We have to be careful that the
		 * THP doesn't start to split while we are adjusting the
		 * refcounts.
		 *
		 * We are sure this doesn't happen, because mmu_notifier_retry
		 * was successful and we are holding the mmu_lock, so if this
		 * THP is trying to split, it will be blocked in the mmu
		 * notifier before touching any of the pages, specifically
		 * before being able to call __split_huge_page_refcount().
		 *
		 * We can therefore safely transfer the refcount from PG_tail
		 * to PG_head and switch the pfn from a tail page to the head
		 * page accordingly.
		 */
		*ipap &= PMD_MASK;
		kvm_release_pfn_clean(pfn);
		pfn &= ~(PTRS_PER_PMD - 1);
987
		get_page(pfn_to_page(pfn));
988 989 990 991 992 993 994 995 996
		*pfnp = pfn;

		return PMD_SIZE;
	}

	/* Use page mapping if we cannot use block mapping. */
	return PAGE_SIZE;
}

997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
static int get_vma_page_shift(struct vm_area_struct *vma, unsigned long hva)
{
	unsigned long pa;

	if (is_vm_hugetlb_page(vma) && !(vma->vm_flags & VM_PFNMAP))
		return huge_page_shift(hstate_vma(vma));

	if (!(vma->vm_flags & VM_PFNMAP))
		return PAGE_SHIFT;

	VM_BUG_ON(is_vm_hugetlb_page(vma));

	pa = (vma->vm_pgoff << PAGE_SHIFT) + (hva - vma->vm_start);

#ifndef __PAGETABLE_PMD_FOLDED
	if ((hva & (PUD_SIZE - 1)) == (pa & (PUD_SIZE - 1)) &&
	    ALIGN_DOWN(hva, PUD_SIZE) >= vma->vm_start &&
	    ALIGN(hva, PUD_SIZE) <= vma->vm_end)
		return PUD_SHIFT;
#endif

	if ((hva & (PMD_SIZE - 1)) == (pa & (PMD_SIZE - 1)) &&
	    ALIGN_DOWN(hva, PMD_SIZE) >= vma->vm_start &&
	    ALIGN(hva, PMD_SIZE) <= vma->vm_end)
		return PMD_SHIFT;

	return PAGE_SHIFT;
}

1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
/*
 * The page will be mapped in stage 2 as Normal Cacheable, so the VM will be
 * able to see the page's tags and therefore they must be initialised first. If
 * PG_mte_tagged is set, tags have already been initialised.
 *
 * The race in the test/set of the PG_mte_tagged flag is handled by:
 * - preventing VM_SHARED mappings in a memslot with MTE preventing two VMs
 *   racing to santise the same page
 * - mmap_lock protects between a VM faulting a page in and the VMM performing
 *   an mprotect() to add VM_MTE
 */
static int sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,
			     unsigned long size)
{
	unsigned long i, nr_pages = size >> PAGE_SHIFT;
	struct page *page;

	if (!kvm_has_mte(kvm))
		return 0;

	/*
	 * pfn_to_online_page() is used to reject ZONE_DEVICE pages
	 * that may not support tags.
	 */
	page = pfn_to_online_page(pfn);

	if (!page)
		return -EFAULT;

	for (i = 0; i < nr_pages; i++, page++) {
		if (!test_bit(PG_mte_tagged, &page->flags)) {
			mte_clear_page_tags(page_address(page));
			set_bit(PG_mte_tagged, &page->flags);
		}
	}

	return 0;
}

1065
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
1066
			  struct kvm_memory_slot *memslot, unsigned long hva,
1067 1068
			  unsigned long fault_status)
{
1069
	int ret = 0;
1070
	bool write_fault, writable, force_pte = false;
1071 1072
	bool exec_fault;
	bool device = false;
1073
	bool shared;
1074
	unsigned long mmu_seq;
1075
	struct kvm *kvm = vcpu->kvm;
1076
	struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
1077
	struct vm_area_struct *vma;
1078
	short vma_shift;
1079
	gfn_t gfn;
1080
	kvm_pfn_t pfn;
1081
	bool logging_active = memslot_is_logging(memslot);
1082
	bool logging_perm_fault = false;
1083 1084
	unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu);
	unsigned long vma_pagesize, fault_granule;
1085 1086
	enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
	struct kvm_pgtable *pgt;
1087

1088
	fault_granule = 1UL << ARM64_HW_PGTABLE_LEVEL_SHIFT(fault_level);
1089
	write_fault = kvm_is_write_fault(vcpu);
1090
	exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu);
1091 1092 1093
	VM_BUG_ON(write_fault && exec_fault);

	if (fault_status == FSC_PERM && !write_fault && !exec_fault) {
1094 1095 1096 1097
		kvm_err("Unexpected L2 read permission error\n");
		return -EFAULT;
	}

1098 1099 1100 1101
	/*
	 * Let's check if we will get back a huge page backed by hugetlbfs, or
	 * get block mapping for device MMIO region.
	 */
1102
	mmap_read_lock(current->mm);
1103
	vma = vma_lookup(current->mm, hva);
1104 1105
	if (unlikely(!vma)) {
		kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
1106
		mmap_read_unlock(current->mm);
1107 1108 1109
		return -EFAULT;
	}

1110 1111 1112 1113 1114
	/*
	 * logging_active is guaranteed to never be true for VM_PFNMAP
	 * memslots.
	 */
	if (logging_active) {
1115
		force_pte = true;
1116
		vma_shift = PAGE_SHIFT;
1117
		logging_perm_fault = (fault_status == FSC_PERM && write_fault);
1118 1119
	} else {
		vma_shift = get_vma_page_shift(vma, hva);
1120 1121
	}

1122
	shared = (vma->vm_flags & VM_SHARED);
1123

1124
	switch (vma_shift) {
1125
#ifndef __PAGETABLE_PMD_FOLDED
1126 1127 1128 1129
	case PUD_SHIFT:
		if (fault_supports_stage2_huge_mapping(memslot, hva, PUD_SIZE))
			break;
		fallthrough;
1130
#endif
1131 1132 1133 1134 1135 1136 1137 1138
	case CONT_PMD_SHIFT:
		vma_shift = PMD_SHIFT;
		fallthrough;
	case PMD_SHIFT:
		if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE))
			break;
		fallthrough;
	case CONT_PTE_SHIFT:
1139
		vma_shift = PAGE_SHIFT;
1140 1141 1142 1143 1144 1145
		force_pte = true;
		fallthrough;
	case PAGE_SHIFT:
		break;
	default:
		WARN_ONCE(1, "Unknown vma_shift %d", vma_shift);
1146 1147
	}

1148
	vma_pagesize = 1UL << vma_shift;
1149
	if (vma_pagesize == PMD_SIZE || vma_pagesize == PUD_SIZE)
1150
		fault_ipa &= ~(vma_pagesize - 1);
1151 1152

	gfn = fault_ipa >> PAGE_SHIFT;
1153
	mmap_read_unlock(current->mm);
1154

1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
	/*
	 * Permission faults just need to update the existing leaf entry,
	 * and so normally don't require allocations from the memcache. The
	 * only exception to this is when dirty logging is enabled at runtime
	 * and a write fault needs to collapse a block entry into a table.
	 */
	if (fault_status != FSC_PERM || (logging_active && write_fault)) {
		ret = kvm_mmu_topup_memory_cache(memcache,
						 kvm_mmu_cache_min_pages(kvm));
		if (ret)
			return ret;
	}
1167 1168 1169 1170 1171 1172 1173

	mmu_seq = vcpu->kvm->mmu_notifier_seq;
	/*
	 * Ensure the read of mmu_notifier_seq happens before we call
	 * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk
	 * the page we just got a reference to gets unmapped before we have a
	 * chance to grab the mmu_lock, which ensure that if the page gets
1174
	 * unmapped afterwards, the call to kvm_unmap_gfn will take it away
1175 1176
	 * from us again properly. This smp_rmb() interacts with the smp_wmb()
	 * in kvm_mmu_notifier_invalidate_<page|range_end>.
1177 1178 1179 1180
	 *
	 * Besides, __gfn_to_pfn_memslot() instead of gfn_to_pfn_prot() is
	 * used to avoid unnecessary overhead introduced to locate the memory
	 * slot because it's always fixed even @gfn is adjusted for huge pages.
1181 1182 1183
	 */
	smp_rmb();

1184 1185
	pfn = __gfn_to_pfn_memslot(memslot, gfn, false, NULL,
				   write_fault, &writable, NULL);
1186
	if (pfn == KVM_PFN_ERR_HWPOISON) {
1187
		kvm_send_hwpoison_signal(hva, vma_shift);
1188 1189
		return 0;
	}
1190
	if (is_error_noslot_pfn(pfn))
1191 1192
		return -EFAULT;

1193
	if (kvm_is_device_pfn(pfn)) {
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203
		/*
		 * If the page was identified as device early by looking at
		 * the VMA flags, vma_pagesize is already representing the
		 * largest quantity we can map.  If instead it was mapped
		 * via gfn_to_pfn_prot(), vma_pagesize is set to PAGE_SIZE
		 * and must not be upgraded.
		 *
		 * In both cases, we don't let transparent_hugepage_adjust()
		 * change things at the last minute.
		 */
1204 1205
		device = true;
	} else if (logging_active && !write_fault) {
1206 1207 1208 1209
		/*
		 * Only actually map the page as writable if this was a write
		 * fault.
		 */
1210
		writable = false;
1211
	}
1212

1213
	if (exec_fault && device)
1214 1215
		return -ENOEXEC;

1216 1217 1218 1219 1220 1221 1222 1223 1224
	/*
	 * To reduce MMU contentions and enhance concurrency during dirty
	 * logging dirty logging, only acquire read lock for permission
	 * relaxation.
	 */
	if (logging_perm_fault)
		read_lock(&kvm->mmu_lock);
	else
		write_lock(&kvm->mmu_lock);
1225
	pgt = vcpu->arch.hw_mmu->pgt;
1226
	if (mmu_notifier_retry(kvm, mmu_seq))
1227
		goto out_unlock;
1228

1229 1230 1231 1232
	/*
	 * If we are not forced to use page mapping, check if we are
	 * backed by a THP and thus use block mapping if possible.
	 */
1233 1234 1235 1236 1237 1238 1239 1240
	if (vma_pagesize == PAGE_SIZE && !(force_pte || device)) {
		if (fault_status == FSC_PERM && fault_granule > PAGE_SIZE)
			vma_pagesize = fault_granule;
		else
			vma_pagesize = transparent_hugepage_adjust(kvm, memslot,
								   hva, &pfn,
								   &fault_ipa);
	}
1241

1242
	if (fault_status != FSC_PERM && !device && kvm_has_mte(kvm)) {
1243
		/* Check the VMM hasn't introduced a new VM_SHARED VMA */
1244 1245 1246
		if (!shared)
			ret = sanitise_mte_tags(kvm, pfn, vma_pagesize);
		else
1247 1248 1249 1250
			ret = -EFAULT;
		if (ret)
			goto out_unlock;
	}
1251

1252
	if (writable)
1253
		prot |= KVM_PGTABLE_PROT_W;
1254

1255
	if (exec_fault)
1256
		prot |= KVM_PGTABLE_PROT_X;
1257

1258 1259 1260 1261
	if (device)
		prot |= KVM_PGTABLE_PROT_DEVICE;
	else if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC))
		prot |= KVM_PGTABLE_PROT_X;
1262

1263 1264 1265 1266 1267 1268
	/*
	 * Under the premise of getting a FSC_PERM fault, we just need to relax
	 * permissions only if vma_pagesize equals fault_granule. Otherwise,
	 * kvm_pgtable_stage2_map() should be called to change block size.
	 */
	if (fault_status == FSC_PERM && vma_pagesize == fault_granule) {
1269
		ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot);
1270
	} else {
1271 1272 1273
		ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize,
					     __pfn_to_phys(pfn), prot,
					     memcache);
1274
	}
1275

1276 1277 1278
	/* Mark the page dirty only if the fault is handled successfully */
	if (writable && !ret) {
		kvm_set_pfn_dirty(pfn);
1279
		mark_page_dirty_in_slot(kvm, memslot, gfn);
1280 1281
	}

1282
out_unlock:
1283 1284 1285 1286
	if (logging_perm_fault)
		read_unlock(&kvm->mmu_lock);
	else
		write_unlock(&kvm->mmu_lock);
1287
	kvm_set_pfn_accessed(pfn);
1288
	kvm_release_pfn_clean(pfn);
1289
	return ret != -EAGAIN ? ret : 0;
1290 1291
}

1292
/* Resolve the access fault by making the page young again. */
1293 1294
static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
{
1295 1296 1297
	pte_t pte;
	kvm_pte_t kpte;
	struct kvm_s2_mmu *mmu;
1298 1299 1300

	trace_kvm_access_fault(fault_ipa);

1301
	write_lock(&vcpu->kvm->mmu_lock);
1302 1303
	mmu = vcpu->arch.hw_mmu;
	kpte = kvm_pgtable_stage2_mkyoung(mmu->pgt, fault_ipa);
1304
	write_unlock(&vcpu->kvm->mmu_lock);
1305 1306 1307 1308

	pte = __pte(kpte);
	if (pte_valid(pte))
		kvm_set_pfn_accessed(pte_pfn(pte));
1309 1310
}

1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321
/**
 * kvm_handle_guest_abort - handles all 2nd stage aborts
 * @vcpu:	the VCPU pointer
 *
 * Any abort that gets to the host is almost guaranteed to be caused by a
 * missing second stage translation table entry, which can mean that either the
 * guest simply needs more memory and we must allocate an appropriate page or it
 * can mean that the guest tried to access I/O memory, which is emulated by user
 * space. The distinction is based on the IPA causing the fault and whether this
 * memory region has been registered as standard RAM by user space.
 */
1322
int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
1323
{
1324 1325 1326
	unsigned long fault_status;
	phys_addr_t fault_ipa;
	struct kvm_memory_slot *memslot;
1327 1328
	unsigned long hva;
	bool is_iabt, write_fault, writable;
1329 1330 1331
	gfn_t gfn;
	int ret, idx;

1332 1333 1334
	fault_status = kvm_vcpu_trap_get_fault_type(vcpu);

	fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
1335
	is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
1336

1337
	/* Synchronous External Abort? */
1338
	if (kvm_vcpu_abt_issea(vcpu)) {
1339 1340 1341 1342
		/*
		 * For RAS the host kernel may handle this abort.
		 * There is no need to pass the error into the guest.
		 */
1343
		if (kvm_handle_guest_sea(fault_ipa, kvm_vcpu_get_esr(vcpu)))
1344
			kvm_inject_vabt(vcpu);
1345 1346

		return 1;
1347 1348
	}

1349
	trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu),
1350
			      kvm_vcpu_get_hfar(vcpu), fault_ipa);
1351 1352

	/* Check the stage-2 fault is trans. fault or write fault */
1353 1354
	if (fault_status != FSC_FAULT && fault_status != FSC_PERM &&
	    fault_status != FSC_ACCESS) {
1355 1356 1357
		kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
			kvm_vcpu_trap_get_class(vcpu),
			(unsigned long)kvm_vcpu_trap_get_fault(vcpu),
1358
			(unsigned long)kvm_vcpu_get_esr(vcpu));
1359 1360 1361 1362 1363 1364
		return -EFAULT;
	}

	idx = srcu_read_lock(&vcpu->kvm->srcu);

	gfn = fault_ipa >> PAGE_SHIFT;
1365 1366
	memslot = gfn_to_memslot(vcpu->kvm, gfn);
	hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
1367
	write_fault = kvm_is_write_fault(vcpu);
1368
	if (kvm_is_error_hva(hva) || (write_fault && !writable)) {
1369 1370 1371 1372 1373 1374
		/*
		 * The guest has put either its instructions or its page-tables
		 * somewhere it shouldn't have. Userspace won't be able to do
		 * anything about this (there's no syndrome for a start), so
		 * re-inject the abort back into the guest.
		 */
1375
		if (is_iabt) {
1376 1377
			ret = -ENOEXEC;
			goto out;
1378 1379
		}

1380
		if (kvm_vcpu_abt_iss1tw(vcpu)) {
1381 1382 1383 1384 1385
			kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
			ret = 1;
			goto out_unlock;
		}

1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
		/*
		 * Check for a cache maintenance operation. Since we
		 * ended-up here, we know it is outside of any memory
		 * slot. But we can't find out if that is for a device,
		 * or if the guest is just being stupid. The only thing
		 * we know for sure is that this range cannot be cached.
		 *
		 * So let's assume that the guest is just being
		 * cautious, and skip the instruction.
		 */
1396
		if (kvm_is_error_hva(hva) && kvm_vcpu_dabt_is_cm(vcpu)) {
1397
			kvm_incr_pc(vcpu);
1398 1399 1400 1401
			ret = 1;
			goto out_unlock;
		}

1402 1403 1404 1405 1406 1407 1408
		/*
		 * The IPA is reported as [MAX:12], so we need to
		 * complement it with the bottom 12 bits from the
		 * faulting VA. This is always 12 bits, irrespective
		 * of the page size.
		 */
		fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1);
1409
		ret = io_mem_abort(vcpu, fault_ipa);
1410 1411 1412
		goto out_unlock;
	}

1413
	/* Userspace should not be able to register out-of-bounds IPAs */
1414
	VM_BUG_ON(fault_ipa >= kvm_phys_size(vcpu->kvm));
1415

1416 1417 1418 1419 1420 1421
	if (fault_status == FSC_ACCESS) {
		handle_access_fault(vcpu, fault_ipa);
		ret = 1;
		goto out_unlock;
	}

1422
	ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
1423 1424
	if (ret == 0)
		ret = 1;
1425 1426 1427 1428 1429
out:
	if (ret == -ENOEXEC) {
		kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
		ret = 1;
	}
1430 1431 1432
out_unlock:
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	return ret;
1433 1434
}

1435
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
1436
{
1437
	if (!kvm->arch.mmu.pgt)
1438
		return false;
1439

1440 1441 1442
	__unmap_stage2_range(&kvm->arch.mmu, range->start << PAGE_SHIFT,
			     (range->end - range->start) << PAGE_SHIFT,
			     range->may_block);
1443

1444
	return false;
1445 1446
}

1447
bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1448
{
1449
	kvm_pfn_t pfn = pte_pfn(range->pte);
1450
	int ret;
1451

1452
	if (!kvm->arch.mmu.pgt)
1453
		return false;
1454

1455
	WARN_ON(range->end - range->start != 1);
1456

1457 1458 1459 1460
	ret = sanitise_mte_tags(kvm, pfn, PAGE_SIZE);
	if (ret)
		return false;

1461
	/*
1462 1463 1464 1465
	 * We've moved a page around, probably through CoW, so let's treat
	 * it just like a translation fault and the map handler will clean
	 * the cache to the PoC.
	 *
1466
	 * The MMU notifiers will have unmapped a huge PMD before calling
1467
	 * ->change_pte() (which in turn calls kvm_set_spte_gfn()) and
1468 1469
	 * therefore we never need to clear out a huge PMD through this
	 * calling path and a memcache is not required.
1470
	 */
1471 1472 1473 1474
	kvm_pgtable_stage2_map(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT,
			       PAGE_SIZE, __pfn_to_phys(pfn),
			       KVM_PGTABLE_PROT_R, NULL);

1475
	return false;
1476 1477
}

1478
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1479
{
1480 1481 1482
	u64 size = (range->end - range->start) << PAGE_SHIFT;
	kvm_pte_t kpte;
	pte_t pte;
1483

1484
	if (!kvm->arch.mmu.pgt)
1485
		return false;
1486

1487
	WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
1488 1489 1490

	kpte = kvm_pgtable_stage2_mkold(kvm->arch.mmu.pgt,
					range->start << PAGE_SHIFT);
1491 1492
	pte = __pte(kpte);
	return pte_valid(pte) && pte_young(pte);
1493 1494
}

1495
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1496
{
1497
	if (!kvm->arch.mmu.pgt)
1498
		return false;
1499

1500 1501
	return kvm_pgtable_stage2_is_young(kvm->arch.mmu.pgt,
					   range->start << PAGE_SHIFT);
1502 1503
}

1504 1505
phys_addr_t kvm_mmu_get_httbr(void)
{
1506
	return __pa(hyp_pgtable->pgd);
1507 1508
}

1509 1510 1511 1512 1513
phys_addr_t kvm_get_idmap_vector(void)
{
	return hyp_idmap_vector;
}

1514
static int kvm_map_idmap_text(void)
1515
{
1516 1517 1518
	unsigned long size = hyp_idmap_end - hyp_idmap_start;
	int err = __create_hyp_mappings(hyp_idmap_start, size, hyp_idmap_start,
					PAGE_HYP_EXEC);
1519 1520 1521 1522 1523 1524 1525
	if (err)
		kvm_err("Failed to idmap %lx-%lx\n",
			hyp_idmap_start, hyp_idmap_end);

	return err;
}

1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538
static void *kvm_hyp_zalloc_page(void *arg)
{
	return (void *)get_zeroed_page(GFP_KERNEL);
}

static struct kvm_pgtable_mm_ops kvm_hyp_mm_ops = {
	.zalloc_page		= kvm_hyp_zalloc_page,
	.get_page		= kvm_host_get_page,
	.put_page		= kvm_host_put_page,
	.phys_to_virt		= kvm_host_va,
	.virt_to_phys		= kvm_host_pa,
};

1539
int kvm_mmu_init(u32 *hyp_va_bits)
1540
{
1541 1542
	int err;

1543
	hyp_idmap_start = __pa_symbol(__hyp_idmap_text_start);
1544
	hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE);
1545
	hyp_idmap_end = __pa_symbol(__hyp_idmap_text_end);
1546
	hyp_idmap_end = ALIGN(hyp_idmap_end, PAGE_SIZE);
1547
	hyp_idmap_vector = __pa_symbol(__kvm_hyp_init);
1548

1549 1550 1551 1552 1553
	/*
	 * We rely on the linker script to ensure at build time that the HYP
	 * init code does not cross a page boundary.
	 */
	BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);
1554

1555 1556
	*hyp_va_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET);
	kvm_debug("Using %u-bit virtual addresses at EL2\n", *hyp_va_bits);
1557 1558 1559 1560
	kvm_debug("IDMAP page: %lx\n", hyp_idmap_start);
	kvm_debug("HYP VA range: %lx:%lx\n",
		  kern_hyp_va(PAGE_OFFSET),
		  kern_hyp_va((unsigned long)high_memory - 1));
1561

1562
	if (hyp_idmap_start >= kern_hyp_va(PAGE_OFFSET) &&
1563
	    hyp_idmap_start <  kern_hyp_va((unsigned long)high_memory - 1) &&
1564
	    hyp_idmap_start != (unsigned long)__hyp_idmap_text_start) {
1565 1566 1567 1568 1569 1570 1571 1572 1573
		/*
		 * The idmap page is intersecting with the VA space,
		 * it is not safe to continue further.
		 */
		kvm_err("IDMAP intersecting with HYP VA, unable to continue\n");
		err = -EINVAL;
		goto out;
	}

1574 1575 1576
	hyp_pgtable = kzalloc(sizeof(*hyp_pgtable), GFP_KERNEL);
	if (!hyp_pgtable) {
		kvm_err("Hyp mode page-table not allocated\n");
1577 1578 1579 1580
		err = -ENOMEM;
		goto out;
	}

1581
	err = kvm_pgtable_hyp_init(hyp_pgtable, *hyp_va_bits, &kvm_hyp_mm_ops);
1582 1583
	if (err)
		goto out_free_pgtable;
1584

1585 1586 1587
	err = kvm_map_idmap_text();
	if (err)
		goto out_destroy_pgtable;
1588

1589
	io_map_base = hyp_idmap_start;
1590
	return 0;
1591 1592 1593 1594 1595 1596

out_destroy_pgtable:
	kvm_pgtable_hyp_destroy(hyp_pgtable);
out_free_pgtable:
	kfree(hyp_pgtable);
	hyp_pgtable = NULL;
1597 1598
out:
	return err;
1599
}
1600 1601

void kvm_arch_commit_memory_region(struct kvm *kvm,
1602
				   struct kvm_memory_slot *old,
1603
				   const struct kvm_memory_slot *new,
1604 1605
				   enum kvm_mr_change change)
{
1606 1607
	/*
	 * At this point memslot has been committed and there is an
1608
	 * allocated dirty_bitmap[], dirty pages will be tracked while the
1609 1610
	 * memory slot is write protected.
	 */
1611
	if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
1612 1613 1614 1615 1616 1617
		/*
		 * If we're with initial-all-set, we don't need to write
		 * protect any pages because they're all reported as dirty.
		 * Huge pages and normal pages will be write protect gradually.
		 */
		if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) {
1618
			kvm_mmu_wp_memory_region(kvm, new->id);
1619 1620
		}
	}
1621 1622 1623
}

int kvm_arch_prepare_memory_region(struct kvm *kvm,
1624 1625
				   const struct kvm_memory_slot *old,
				   struct kvm_memory_slot *new,
1626 1627
				   enum kvm_mr_change change)
{
1628
	hva_t hva, reg_end;
1629 1630
	int ret = 0;

1631 1632
	if (change != KVM_MR_CREATE && change != KVM_MR_MOVE &&
			change != KVM_MR_FLAGS_ONLY)
1633 1634
		return 0;

1635 1636 1637 1638
	/*
	 * Prevent userspace from creating a memory region outside of the IPA
	 * space addressable by the KVM guest IPA space.
	 */
1639
	if ((new->base_gfn + new->npages) > (kvm_phys_size(kvm) >> PAGE_SHIFT))
1640 1641
		return -EFAULT;

1642 1643 1644
	hva = new->userspace_addr;
	reg_end = hva + (new->npages << PAGE_SHIFT);

1645
	mmap_read_lock(current->mm);
1646 1647
	/*
	 * A memory region could potentially cover multiple VMAs, and any holes
1648
	 * between them, so iterate over all of them.
1649 1650 1651 1652 1653 1654 1655 1656 1657
	 *
	 *     +--------------------------------------------+
	 * +---------------+----------------+   +----------------+
	 * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
	 * +---------------+----------------+   +----------------+
	 *     |               memory region                |
	 *     +--------------------------------------------+
	 */
	do {
1658
		struct vm_area_struct *vma;
1659

1660 1661
		vma = find_vma_intersection(current->mm, hva, reg_end);
		if (!vma)
1662 1663
			break;

1664 1665 1666 1667 1668
		/*
		 * VM_SHARED mappings are not allowed with MTE to avoid races
		 * when updating the PG_mte_tagged page flag, see
		 * sanitise_mte_tags for more details.
		 */
1669 1670 1671 1672
		if (kvm_has_mte(kvm) && vma->vm_flags & VM_SHARED) {
			ret = -EINVAL;
			break;
		}
1673

1674
		if (vma->vm_flags & VM_PFNMAP) {
1675
			/* IO region dirty page logging not allowed */
1676
			if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
1677
				ret = -EINVAL;
1678
				break;
1679
			}
1680
		}
1681
		hva = min(reg_end, vma->vm_end);
1682 1683
	} while (hva < reg_end);

1684
	mmap_read_unlock(current->mm);
1685
	return ret;
1686 1687
}

1688
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
1689 1690 1691
{
}

1692
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
1693 1694 1695 1696 1697
{
}

void kvm_arch_flush_shadow_all(struct kvm *kvm)
{
1698
	kvm_free_stage2_pgd(&kvm->arch.mmu);
1699 1700 1701 1702 1703
}

void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
1704 1705 1706
	gpa_t gpa = slot->base_gfn << PAGE_SHIFT;
	phys_addr_t size = slot->npages << PAGE_SHIFT;

1707
	write_lock(&kvm->mmu_lock);
1708
	unmap_stage2_range(&kvm->arch.mmu, gpa, size);
1709
	write_unlock(&kvm->mmu_lock);
1710
}
1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741

/*
 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
 *
 * Main problems:
 * - S/W ops are local to a CPU (not broadcast)
 * - We have line migration behind our back (speculation)
 * - System caches don't support S/W at all (damn!)
 *
 * In the face of the above, the best we can do is to try and convert
 * S/W ops to VA ops. Because the guest is not allowed to infer the
 * S/W to PA mapping, it can only use S/W to nuke the whole cache,
 * which is a rather good thing for us.
 *
 * Also, it is only used when turning caches on/off ("The expected
 * usage of the cache maintenance instructions that operate by set/way
 * is associated with the cache maintenance instructions associated
 * with the powerdown and powerup of caches, if this is required by
 * the implementation.").
 *
 * We use the following policy:
 *
 * - If we trap a S/W operation, we enable VM trapping to detect
 *   caches being turned on/off, and do a full clean.
 *
 * - We flush the caches on both caches being turned on and off.
 *
 * - Once the caches are enabled, we stop trapping VM ops.
 */
void kvm_set_way_flush(struct kvm_vcpu *vcpu)
{
1742
	unsigned long hcr = *vcpu_hcr(vcpu);
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756

	/*
	 * If this is the first time we do a S/W operation
	 * (i.e. HCR_TVM not set) flush the whole memory, and set the
	 * VM trapping.
	 *
	 * Otherwise, rely on the VM trapping to wait for the MMU +
	 * Caches to be turned off. At that point, we'll be able to
	 * clean the caches again.
	 */
	if (!(hcr & HCR_TVM)) {
		trace_kvm_set_way_flush(*vcpu_pc(vcpu),
					vcpu_has_cache_enabled(vcpu));
		stage2_flush_vm(vcpu->kvm);
1757
		*vcpu_hcr(vcpu) = hcr | HCR_TVM;
1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
	}
}

void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
{
	bool now_enabled = vcpu_has_cache_enabled(vcpu);

	/*
	 * If switching the MMU+caches on, need to invalidate the caches.
	 * If switching it off, need to clean the caches.
	 * Clean + invalidate does the trick always.
	 */
	if (now_enabled != was_enabled)
		stage2_flush_vm(vcpu->kvm);

	/* Caches are now on, stop trapping VM ops (until a S/W op) */
	if (now_enabled)
1775
		*vcpu_hcr(vcpu) &= ~HCR_TVM;
1776 1777 1778

	trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
}
反馈
建议
客服 返回
顶部