book3s_64_mmu_hv.c 42.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 *
 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 */

#include <linux/types.h>
#include <linux/string.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/highmem.h>
#include <linux/gfp.h>
#include <linux/slab.h>
#include <linux/hugetlb.h>
26
#include <linux/vmalloc.h>
27
#include <linux/srcu.h>
28 29
#include <linux/anon_inodes.h>
#include <linux/file.h>
30 31 32 33 34 35 36 37 38 39

#include <asm/tlbflush.h>
#include <asm/kvm_ppc.h>
#include <asm/kvm_book3s.h>
#include <asm/mmu-hash64.h>
#include <asm/hvcall.h>
#include <asm/synch.h>
#include <asm/ppc-opcode.h>
#include <asm/cputable.h>

40 41
#include "book3s_hv_cma.h"

42 43
/* POWER7 has 10-bit LPIDs, PPC970 has 6-bit LPIDs */
#define MAX_LPID_970	63
44

45 46 47
/* Power architecture requires HPT is at least 256kB */
#define PPC_MIN_HPT_ORDER	18

48 49 50
static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
				long pte_index, unsigned long pteh,
				unsigned long ptel, unsigned long *pte_idx_ret);
51
static void kvmppc_rmap_reset(struct kvm *kvm);
52

53
long kvmppc_alloc_hpt(struct kvm *kvm, u32 *htab_orderp)
54
{
55
	unsigned long hpt = 0;
56
	struct revmap_entry *rev;
57 58
	struct page *page = NULL;
	long order = KVM_DEFAULT_HPT_ORDER;
59

60 61 62 63 64 65
	if (htab_orderp) {
		order = *htab_orderp;
		if (order < PPC_MIN_HPT_ORDER)
			order = PPC_MIN_HPT_ORDER;
	}

66
	kvm->arch.hpt_cma_alloc = 0;
67 68 69 70 71
	VM_BUG_ON(order < KVM_CMA_CHUNK_ORDER);
	page = kvm_alloc_hpt(1 << (order - PAGE_SHIFT));
	if (page) {
		hpt = (unsigned long)pfn_to_kaddr(page_to_pfn(page));
		kvm->arch.hpt_cma_alloc = 1;
72
	}
73 74 75 76 77 78 79 80 81 82 83 84

	/* Lastly try successively smaller sizes from the page allocator */
	while (!hpt && order > PPC_MIN_HPT_ORDER) {
		hpt = __get_free_pages(GFP_KERNEL|__GFP_ZERO|__GFP_REPEAT|
				       __GFP_NOWARN, order - PAGE_SHIFT);
		if (!hpt)
			--order;
	}

	if (!hpt)
		return -ENOMEM;

85
	kvm->arch.hpt_virt = hpt;
86 87 88 89 90
	kvm->arch.hpt_order = order;
	/* HPTEs are 2**4 bytes long */
	kvm->arch.hpt_npte = 1ul << (order - 4);
	/* 128 (2**7) bytes in each HPTEG */
	kvm->arch.hpt_mask = (1ul << (order - 7)) - 1;
91

92
	/* Allocate reverse map array */
93
	rev = vmalloc(sizeof(struct revmap_entry) * kvm->arch.hpt_npte);
94 95 96 97 98
	if (!rev) {
		pr_err("kvmppc_alloc_hpt: Couldn't alloc reverse map array\n");
		goto out_freehpt;
	}
	kvm->arch.revmap = rev;
99
	kvm->arch.sdr1 = __pa(hpt) | (order - 18);
100

101 102
	pr_info("KVM guest htab at %lx (order %ld), LPID %x\n",
		hpt, order, kvm->arch.lpid);
103

104 105
	if (htab_orderp)
		*htab_orderp = order;
106
	return 0;
107 108

 out_freehpt:
109 110
	if (kvm->arch.hpt_cma_alloc)
		kvm_release_hpt(page, 1 << (order - PAGE_SHIFT));
111 112
	else
		free_pages(hpt, order - PAGE_SHIFT);
113
	return -ENOMEM;
114 115
}

116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
long kvmppc_alloc_reset_hpt(struct kvm *kvm, u32 *htab_orderp)
{
	long err = -EBUSY;
	long order;

	mutex_lock(&kvm->lock);
	if (kvm->arch.rma_setup_done) {
		kvm->arch.rma_setup_done = 0;
		/* order rma_setup_done vs. vcpus_running */
		smp_mb();
		if (atomic_read(&kvm->arch.vcpus_running)) {
			kvm->arch.rma_setup_done = 1;
			goto out;
		}
	}
	if (kvm->arch.hpt_virt) {
		order = kvm->arch.hpt_order;
		/* Set the entire HPT to 0, i.e. invalid HPTEs */
		memset((void *)kvm->arch.hpt_virt, 0, 1ul << order);
135 136 137 138
		/*
		 * Reset all the reverse-mapping chains for all memslots
		 */
		kvmppc_rmap_reset(kvm);
139 140
		/* Ensure that each vcpu will flush its TLB on next entry. */
		cpumask_setall(&kvm->arch.need_tlb_flush);
141 142 143 144 145 146 147 148 149 150 151
		*htab_orderp = order;
		err = 0;
	} else {
		err = kvmppc_alloc_hpt(kvm, htab_orderp);
		order = *htab_orderp;
	}
 out:
	mutex_unlock(&kvm->lock);
	return err;
}

152 153
void kvmppc_free_hpt(struct kvm *kvm)
{
154
	kvmppc_free_lpid(kvm->arch.lpid);
155
	vfree(kvm->arch.revmap);
156 157 158
	if (kvm->arch.hpt_cma_alloc)
		kvm_release_hpt(virt_to_page(kvm->arch.hpt_virt),
				1 << (kvm->arch.hpt_order - PAGE_SHIFT));
A
Alexander Graf 已提交
159
	else
160 161
		free_pages(kvm->arch.hpt_virt,
			   kvm->arch.hpt_order - PAGE_SHIFT);
162 163
}

164 165 166 167 168 169 170 171 172 173 174 175 176 177
/* Bits in first HPTE dword for pagesize 4k, 64k or 16M */
static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize)
{
	return (pgsize > 0x1000) ? HPTE_V_LARGE : 0;
}

/* Bits in second HPTE dword for pagesize 4k, 64k or 16M */
static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize)
{
	return (pgsize == 0x10000) ? 0x1000 : 0;
}

void kvmppc_map_vrma(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
		     unsigned long porder)
178 179
{
	unsigned long i;
180
	unsigned long npages;
181 182
	unsigned long hp_v, hp_r;
	unsigned long addr, hash;
183 184
	unsigned long psize;
	unsigned long hp0, hp1;
185
	unsigned long idx_ret;
186
	long ret;
187
	struct kvm *kvm = vcpu->kvm;
188

189 190
	psize = 1ul << porder;
	npages = memslot->npages >> (porder - PAGE_SHIFT);
191 192

	/* VRMA can't be > 1TB */
193 194
	if (npages > 1ul << (40 - porder))
		npages = 1ul << (40 - porder);
195
	/* Can't use more than 1 HPTE per HPTEG */
196 197
	if (npages > kvm->arch.hpt_mask + 1)
		npages = kvm->arch.hpt_mask + 1;
198

199 200 201 202 203
	hp0 = HPTE_V_1TB_SEG | (VRMA_VSID << (40 - 16)) |
		HPTE_V_BOLTED | hpte0_pgsize_encoding(psize);
	hp1 = hpte1_pgsize_encoding(psize) |
		HPTE_R_R | HPTE_R_C | HPTE_R_M | PP_RWXX;

204
	for (i = 0; i < npages; ++i) {
205
		addr = i << porder;
206
		/* can't use hpt_hash since va > 64 bits */
207
		hash = (i ^ (VRMA_VSID ^ (VRMA_VSID << 25))) & kvm->arch.hpt_mask;
208 209 210 211 212 213
		/*
		 * We assume that the hash table is empty and no
		 * vcpus are using it at this stage.  Since we create
		 * at most one HPTE per HPTEG, we just assume entry 7
		 * is available and use it.
		 */
214
		hash = (hash << 3) + 7;
215 216
		hp_v = hp0 | ((addr >> 16) & ~0x7fUL);
		hp_r = hp1 | addr;
217 218
		ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r,
						 &idx_ret);
219 220 221 222 223
		if (ret != H_SUCCESS) {
			pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
			       addr, ret);
			break;
		}
224 225 226 227 228
	}
}

int kvmppc_mmu_hv_init(void)
{
229 230 231
	unsigned long host_lpid, rsvd_lpid;

	if (!cpu_has_feature(CPU_FTR_HVMODE))
232
		return -EINVAL;
233

234
	/* POWER7 has 10-bit LPIDs, PPC970 and e500mc have 6-bit LPIDs */
235 236 237 238 239 240 241 242
	if (cpu_has_feature(CPU_FTR_ARCH_206)) {
		host_lpid = mfspr(SPRN_LPID);	/* POWER7 */
		rsvd_lpid = LPID_RSVD;
	} else {
		host_lpid = 0;			/* PPC970 */
		rsvd_lpid = MAX_LPID_970;
	}

243 244 245
	kvmppc_init_lpid(rsvd_lpid + 1);

	kvmppc_claim_lpid(host_lpid);
246
	/* rsvd_lpid is reserved for use in partition switching */
247
	kvmppc_claim_lpid(rsvd_lpid);
248 249 250 251 252 253

	return 0;
}

static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu *vcpu)
{
254 255 256 257 258 259 260 261
	unsigned long msr = vcpu->arch.intr_msr;

	/* If transactional, change to suspend mode on IRQ delivery */
	if (MSR_TM_TRANSACTIONAL(vcpu->arch.shregs.msr))
		msr |= MSR_TS_S;
	else
		msr |= vcpu->arch.shregs.msr & MSR_TS_MASK;
	kvmppc_set_msr(vcpu, msr);
262 263
}

264 265
/*
 * This is called to get a reference to a guest page if there isn't
266
 * one already in the memslot->arch.slot_phys[] array.
267 268
 */
static long kvmppc_get_guest_page(struct kvm *kvm, unsigned long gfn,
269 270
				  struct kvm_memory_slot *memslot,
				  unsigned long psize)
271 272
{
	unsigned long start;
273 274 275
	long np, err;
	struct page *page, *hpage, *pages[1];
	unsigned long s, pgsize;
276
	unsigned long *physp;
277 278
	unsigned int is_io, got, pgorder;
	struct vm_area_struct *vma;
279
	unsigned long pfn, i, npages;
280

281
	physp = memslot->arch.slot_phys;
282 283
	if (!physp)
		return -EINVAL;
284
	if (physp[gfn - memslot->base_gfn])
285 286
		return 0;

287 288
	is_io = 0;
	got = 0;
289
	page = NULL;
290
	pgsize = psize;
291
	err = -EINVAL;
292 293 294 295
	start = gfn_to_hva_memslot(memslot, gfn);

	/* Instantiate and get the page we want access to */
	np = get_user_pages_fast(start, 1, 1, pages);
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
	if (np != 1) {
		/* Look up the vma for the page */
		down_read(&current->mm->mmap_sem);
		vma = find_vma(current->mm, start);
		if (!vma || vma->vm_start > start ||
		    start + psize > vma->vm_end ||
		    !(vma->vm_flags & VM_PFNMAP))
			goto up_err;
		is_io = hpte_cache_bits(pgprot_val(vma->vm_page_prot));
		pfn = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
		/* check alignment of pfn vs. requested page size */
		if (psize > PAGE_SIZE && (pfn & ((psize >> PAGE_SHIFT) - 1)))
			goto up_err;
		up_read(&current->mm->mmap_sem);

	} else {
		page = pages[0];
		got = KVMPPC_GOT_PAGE;

		/* See if this is a large page */
		s = PAGE_SIZE;
		if (PageHuge(page)) {
			hpage = compound_head(page);
			s <<= compound_order(hpage);
			/* Get the whole large page if slot alignment is ok */
			if (s > psize && slot_is_aligned(memslot, s) &&
			    !(memslot->userspace_addr & (s - 1))) {
				start &= ~(s - 1);
				pgsize = s;
325 326
				get_page(hpage);
				put_page(page);
327 328
				page = hpage;
			}
329
		}
330 331 332
		if (s < psize)
			goto out;
		pfn = page_to_pfn(page);
333 334
	}

335 336 337
	npages = pgsize >> PAGE_SHIFT;
	pgorder = __ilog2(npages);
	physp += (gfn - memslot->base_gfn) & ~(npages - 1);
338
	spin_lock(&kvm->arch.slot_phys_lock);
339 340
	for (i = 0; i < npages; ++i) {
		if (!physp[i]) {
341 342
			physp[i] = ((pfn + i) << PAGE_SHIFT) +
				got + is_io + pgorder;
343 344 345
			got = 0;
		}
	}
346
	spin_unlock(&kvm->arch.slot_phys_lock);
347
	err = 0;
348

349
 out:
350
	if (got)
351 352
		put_page(page);
	return err;
353 354 355 356

 up_err:
	up_read(&current->mm->mmap_sem);
	return err;
357 358
}

359 360 361
long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
				long pte_index, unsigned long pteh,
				unsigned long ptel, unsigned long *pte_idx_ret)
362 363 364 365 366
{
	unsigned long psize, gpa, gfn;
	struct kvm_memory_slot *memslot;
	long ret;

367 368 369
	if (kvm->arch.using_mmu_notifiers)
		goto do_insert;

370 371 372 373
	psize = hpte_page_size(pteh, ptel);
	if (!psize)
		return H_PARAMETER;

374 375
	pteh &= ~(HPTE_V_HVLOCK | HPTE_V_ABSENT | HPTE_V_VALID);

376 377 378 379
	/* Find the memslot (if any) for this address */
	gpa = (ptel & HPTE_R_RPN) & ~(psize - 1);
	gfn = gpa >> PAGE_SHIFT;
	memslot = gfn_to_memslot(kvm, gfn);
380 381 382 383 384 385
	if (memslot && !(memslot->flags & KVM_MEMSLOT_INVALID)) {
		if (!slot_is_aligned(memslot, psize))
			return H_PARAMETER;
		if (kvmppc_get_guest_page(kvm, gfn, memslot, psize) < 0)
			return H_PARAMETER;
	}
386

387 388 389
 do_insert:
	/* Protect linux PTE lookup from page table destruction */
	rcu_read_lock_sched();	/* this disables preemption too */
390 391
	ret = kvmppc_do_h_enter(kvm, flags, pte_index, pteh, ptel,
				current->mm->pgd, false, pte_idx_ret);
392
	rcu_read_unlock_sched();
393 394 395 396 397 398 399 400 401
	if (ret == H_TOO_HARD) {
		/* this can't happen */
		pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n");
		ret = H_RESOURCE;	/* or something */
	}
	return ret;

}

402 403 404 405 406 407 408 409 410 411 412 413 414
/*
 * We come here on a H_ENTER call from the guest when we are not
 * using mmu notifiers and we don't have the requested page pinned
 * already.
 */
long kvmppc_virtmode_h_enter(struct kvm_vcpu *vcpu, unsigned long flags,
			     long pte_index, unsigned long pteh,
			     unsigned long ptel)
{
	return kvmppc_virtmode_do_h_enter(vcpu->kvm, flags, pte_index,
					  pteh, ptel, &vcpu->arch.gpr[4]);
}

415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444
static struct kvmppc_slb *kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu *vcpu,
							 gva_t eaddr)
{
	u64 mask;
	int i;

	for (i = 0; i < vcpu->arch.slb_nr; i++) {
		if (!(vcpu->arch.slb[i].orige & SLB_ESID_V))
			continue;

		if (vcpu->arch.slb[i].origv & SLB_VSID_B_1T)
			mask = ESID_MASK_1T;
		else
			mask = ESID_MASK;

		if (((vcpu->arch.slb[i].orige ^ eaddr) & mask) == 0)
			return &vcpu->arch.slb[i];
	}
	return NULL;
}

static unsigned long kvmppc_mmu_get_real_addr(unsigned long v, unsigned long r,
			unsigned long ea)
{
	unsigned long ra_mask;

	ra_mask = hpte_page_size(v, r) - 1;
	return (r & HPTE_R_RPN & ~ra_mask) | (ea & ra_mask);
}

445
static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
446
			struct kvmppc_pte *gpte, bool data, bool iswrite)
447
{
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
	struct kvm *kvm = vcpu->kvm;
	struct kvmppc_slb *slbe;
	unsigned long slb_v;
	unsigned long pp, key;
	unsigned long v, gr;
	unsigned long *hptep;
	int index;
	int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR);

	/* Get SLB entry */
	if (virtmode) {
		slbe = kvmppc_mmu_book3s_hv_find_slbe(vcpu, eaddr);
		if (!slbe)
			return -EINVAL;
		slb_v = slbe->origv;
	} else {
		/* real mode access */
		slb_v = vcpu->kvm->arch.vrma_slb_v;
	}

468
	preempt_disable();
469 470 471
	/* Find the HPTE in the hash table */
	index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v,
					 HPTE_V_VALID | HPTE_V_ABSENT);
472 473
	if (index < 0) {
		preempt_enable();
474
		return -ENOENT;
475
	}
476 477 478 479 480 481 482
	hptep = (unsigned long *)(kvm->arch.hpt_virt + (index << 4));
	v = hptep[0] & ~HPTE_V_HVLOCK;
	gr = kvm->arch.revmap[index].guest_rpte;

	/* Unlock the HPTE */
	asm volatile("lwsync" : : : "memory");
	hptep[0] = v;
483
	preempt_enable();
484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530

	gpte->eaddr = eaddr;
	gpte->vpage = ((v & HPTE_V_AVPN) << 4) | ((eaddr >> 12) & 0xfff);

	/* Get PP bits and key for permission check */
	pp = gr & (HPTE_R_PP0 | HPTE_R_PP);
	key = (vcpu->arch.shregs.msr & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS;
	key &= slb_v;

	/* Calculate permissions */
	gpte->may_read = hpte_read_permission(pp, key);
	gpte->may_write = hpte_write_permission(pp, key);
	gpte->may_execute = gpte->may_read && !(gr & (HPTE_R_N | HPTE_R_G));

	/* Storage key permission check for POWER7 */
	if (data && virtmode && cpu_has_feature(CPU_FTR_ARCH_206)) {
		int amrfield = hpte_get_skey_perm(gr, vcpu->arch.amr);
		if (amrfield & 1)
			gpte->may_read = 0;
		if (amrfield & 2)
			gpte->may_write = 0;
	}

	/* Get the guest physical address */
	gpte->raddr = kvmppc_mmu_get_real_addr(v, gr, eaddr);
	return 0;
}

/*
 * Quick test for whether an instruction is a load or a store.
 * If the instruction is a load or a store, then this will indicate
 * which it is, at least on server processors.  (Embedded processors
 * have some external PID instructions that don't follow the rule
 * embodied here.)  If the instruction isn't a load or store, then
 * this doesn't return anything useful.
 */
static int instruction_is_store(unsigned int instr)
{
	unsigned int mask;

	mask = 0x10000000;
	if ((instr & 0xfc000000) == 0x7c000000)
		mask = 0x100;		/* major opcode 31 */
	return (instr & mask) != 0;
}

static int kvmppc_hv_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu,
531
				  unsigned long gpa, gva_t ea, int is_store)
532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
{
	int ret;
	u32 last_inst;
	unsigned long srr0 = kvmppc_get_pc(vcpu);

	/* We try to load the last instruction.  We don't let
	 * emulate_instruction do it as it doesn't check what
	 * kvmppc_ld returns.
	 * If we fail, we just return to the guest and try executing it again.
	 */
	if (vcpu->arch.last_inst == KVM_INST_FETCH_FAILED) {
		ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false);
		if (ret != EMULATE_DONE || last_inst == KVM_INST_FETCH_FAILED)
			return RESUME_GUEST;
		vcpu->arch.last_inst = last_inst;
	}

	/*
	 * WARNING: We do not know for sure whether the instruction we just
	 * read from memory is the same that caused the fault in the first
	 * place.  If the instruction we read is neither an load or a store,
	 * then it can't access memory, so we don't need to worry about
	 * enforcing access permissions.  So, assuming it is a load or
	 * store, we just check that its direction (load or store) is
	 * consistent with the original fault, since that's what we
	 * checked the access permissions against.  If there is a mismatch
	 * we just return and retry the instruction.
	 */

561
	if (instruction_is_store(kvmppc_get_last_inst(vcpu)) != !!is_store)
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
		return RESUME_GUEST;

	/*
	 * Emulated accesses are emulated by looking at the hash for
	 * translation once, then performing the access later. The
	 * translation could be invalidated in the meantime in which
	 * point performing the subsequent memory access on the old
	 * physical address could possibly be a security hole for the
	 * guest (but not the host).
	 *
	 * This is less of an issue for MMIO stores since they aren't
	 * globally visible. It could be an issue for MMIO loads to
	 * a certain extent but we'll ignore it for now.
	 */

	vcpu->arch.paddr_accessed = gpa;
578
	vcpu->arch.vaddr_accessed = ea;
579 580 581 582 583 584 585
	return kvmppc_emulate_mmio(run, vcpu);
}

int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
				unsigned long ea, unsigned long dsisr)
{
	struct kvm *kvm = vcpu->kvm;
586 587
	unsigned long *hptep, hpte[3], r;
	unsigned long mmu_seq, psize, pte_size;
588
	unsigned long gpa_base, gfn_base;
589
	unsigned long gpa, gfn, hva, pfn;
590
	struct kvm_memory_slot *memslot;
591
	unsigned long *rmap;
592
	struct revmap_entry *rev;
593 594 595
	struct page *page, *pages[1];
	long index, ret, npages;
	unsigned long is_io;
596
	unsigned int writing, write_ok;
597
	struct vm_area_struct *vma;
598
	unsigned long rcbits;
599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615

	/*
	 * Real-mode code has already searched the HPT and found the
	 * entry we're interested in.  Lock the entry and check that
	 * it hasn't changed.  If it has, just return and re-execute the
	 * instruction.
	 */
	if (ea != vcpu->arch.pgfault_addr)
		return RESUME_GUEST;
	index = vcpu->arch.pgfault_index;
	hptep = (unsigned long *)(kvm->arch.hpt_virt + (index << 4));
	rev = &kvm->arch.revmap[index];
	preempt_disable();
	while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
		cpu_relax();
	hpte[0] = hptep[0] & ~HPTE_V_HVLOCK;
	hpte[1] = hptep[1];
616
	hpte[2] = r = rev->guest_rpte;
617 618 619 620 621 622 623 624 625
	asm volatile("lwsync" : : : "memory");
	hptep[0] = hpte[0];
	preempt_enable();

	if (hpte[0] != vcpu->arch.pgfault_hpte[0] ||
	    hpte[1] != vcpu->arch.pgfault_hpte[1])
		return RESUME_GUEST;

	/* Translate the logical address and get the page */
626
	psize = hpte_page_size(hpte[0], r);
627 628 629
	gpa_base = r & HPTE_R_RPN & ~(psize - 1);
	gfn_base = gpa_base >> PAGE_SHIFT;
	gpa = gpa_base | (ea & (psize - 1));
630
	gfn = gpa >> PAGE_SHIFT;
631 632 633
	memslot = gfn_to_memslot(kvm, gfn);

	/* No memslot means it's an emulated MMIO region */
634
	if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
635
		return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea,
636 637
					      dsisr & DSISR_ISSTORE);

638 639 640
	if (!kvm->arch.using_mmu_notifiers)
		return -EFAULT;		/* should never get here */

641 642 643 644 645 646 647
	/*
	 * This should never happen, because of the slot_is_aligned()
	 * check in kvmppc_do_h_enter().
	 */
	if (gfn_base < memslot->base_gfn)
		return -EFAULT;

648 649 650 651 652 653 654 655
	/* used to check for invalidations in progress */
	mmu_seq = kvm->mmu_notifier_seq;
	smp_rmb();

	is_io = 0;
	pfn = 0;
	page = NULL;
	pte_size = PAGE_SIZE;
656 657 658
	writing = (dsisr & DSISR_ISSTORE) != 0;
	/* If writing != 0, then the HPTE must allow writing, if we get here */
	write_ok = writing;
659
	hva = gfn_to_hva_memslot(memslot, gfn);
660
	npages = get_user_pages_fast(hva, 1, writing, pages);
661 662 663 664 665 666 667 668 669 670
	if (npages < 1) {
		/* Check if it's an I/O mapping */
		down_read(&current->mm->mmap_sem);
		vma = find_vma(current->mm, hva);
		if (vma && vma->vm_start <= hva && hva + psize <= vma->vm_end &&
		    (vma->vm_flags & VM_PFNMAP)) {
			pfn = vma->vm_pgoff +
				((hva - vma->vm_start) >> PAGE_SHIFT);
			pte_size = psize;
			is_io = hpte_cache_bits(pgprot_val(vma->vm_page_prot));
671
			write_ok = vma->vm_flags & VM_WRITE;
672 673 674 675 676 677
		}
		up_read(&current->mm->mmap_sem);
		if (!pfn)
			return -EFAULT;
	} else {
		page = pages[0];
678
		pfn = page_to_pfn(page);
679 680 681 682
		if (PageHuge(page)) {
			page = compound_head(page);
			pte_size <<= compound_order(page);
		}
683 684
		/* if the guest wants write access, see if that is OK */
		if (!writing && hpte_is_writable(r)) {
685
			unsigned int hugepage_shift;
686 687 688 689 690 691 692 693
			pte_t *ptep, pte;

			/*
			 * We need to protect against page table destruction
			 * while looking up and updating the pte.
			 */
			rcu_read_lock_sched();
			ptep = find_linux_pte_or_hugepte(current->mm->pgd,
694 695 696 697
							 hva, &hugepage_shift);
			if (ptep) {
				pte = kvmppc_read_update_linux_pte(ptep, 1,
							   hugepage_shift);
698 699 700 701 702
				if (pte_write(pte))
					write_ok = 1;
			}
			rcu_read_unlock_sched();
		}
703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
	}

	ret = -EFAULT;
	if (psize > pte_size)
		goto out_put;

	/* Check WIMG vs. the actual page we're accessing */
	if (!hpte_cache_flags_ok(r, is_io)) {
		if (is_io)
			return -EFAULT;
		/*
		 * Allow guest to map emulated device memory as
		 * uncacheable, but actually make it cacheable.
		 */
		r = (r & ~(HPTE_R_W|HPTE_R_I|HPTE_R_G)) | HPTE_R_M;
	}

720 721 722 723 724 725 726 727
	/*
	 * Set the HPTE to point to pfn.
	 * Since the pfn is at PAGE_SIZE granularity, make sure we
	 * don't mask out lower-order bits if psize < PAGE_SIZE.
	 */
	if (psize < PAGE_SIZE)
		psize = PAGE_SIZE;
	r = (r & ~(HPTE_R_PP0 - psize)) | ((pfn << PAGE_SHIFT) & ~(psize - 1));
728 729
	if (hpte_is_writable(r) && !write_ok)
		r = hpte_make_readonly(r);
730 731 732 733 734 735 736 737 738 739
	ret = RESUME_GUEST;
	preempt_disable();
	while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
		cpu_relax();
	if ((hptep[0] & ~HPTE_V_HVLOCK) != hpte[0] || hptep[1] != hpte[1] ||
	    rev->guest_rpte != hpte[2])
		/* HPTE has been changed under us; let the guest retry */
		goto out_unlock;
	hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID;

740 741
	/* Always put the HPTE in the rmap chain for the page base address */
	rmap = &memslot->arch.rmap[gfn_base - memslot->base_gfn];
742 743 744 745
	lock_rmap(rmap);

	/* Check if we might have been invalidated; let the guest retry if so */
	ret = RESUME_GUEST;
746
	if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) {
747 748 749
		unlock_rmap(rmap);
		goto out_unlock;
	}
750

751 752 753 754
	/* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
	rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT;
	r &= rcbits | ~(HPTE_R_R | HPTE_R_C);

755 756 757 758 759
	if (hptep[0] & HPTE_V_VALID) {
		/* HPTE was previously valid, so we need to invalidate it */
		unlock_rmap(rmap);
		hptep[0] |= HPTE_V_ABSENT;
		kvmppc_invalidate_hpte(kvm, hptep, index);
760 761
		/* don't lose previous R and C bits */
		r |= hptep[1] & (HPTE_R_R | HPTE_R_C);
762 763 764
	} else {
		kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);
	}
765 766 767 768 769 770

	hptep[1] = r;
	eieio();
	hptep[0] = hpte[0];
	asm volatile("ptesync" : : : "memory");
	preempt_enable();
771
	if (page && hpte_is_writable(r))
772 773 774
		SetPageDirty(page);

 out_put:
775 776 777 778 779 780 781 782 783
	if (page) {
		/*
		 * We drop pages[0] here, not page because page might
		 * have been set to the head page of a compound, but
		 * we have to drop the reference on the correct tail
		 * page to match the get inside gup()
		 */
		put_page(pages[0]);
	}
784 785 786 787 788 789 790 791
	return ret;

 out_unlock:
	hptep[0] &= ~HPTE_V_HVLOCK;
	preempt_enable();
	goto out_put;
}

792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810
static void kvmppc_rmap_reset(struct kvm *kvm)
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;
	int srcu_idx;

	srcu_idx = srcu_read_lock(&kvm->srcu);
	slots = kvm->memslots;
	kvm_for_each_memslot(memslot, slots) {
		/*
		 * This assumes it is acceptable to lose reference and
		 * change bits across a reset.
		 */
		memset(memslot->arch.rmap, 0,
		       memslot->npages * sizeof(*memslot->arch.rmap));
	}
	srcu_read_unlock(&kvm->srcu, srcu_idx);
}

811 812 813 814 815 816
static int kvm_handle_hva_range(struct kvm *kvm,
				unsigned long start,
				unsigned long end,
				int (*handler)(struct kvm *kvm,
					       unsigned long *rmapp,
					       unsigned long gfn))
817 818 819 820 821 822 823 824
{
	int ret;
	int retval = 0;
	struct kvm_memslots *slots;
	struct kvm_memory_slot *memslot;

	slots = kvm_memslots(kvm);
	kvm_for_each_memslot(memslot, slots) {
825 826 827 828 829 830 831 832 833 834 835 836 837 838
		unsigned long hva_start, hva_end;
		gfn_t gfn, gfn_end;

		hva_start = max(start, memslot->userspace_addr);
		hva_end = min(end, memslot->userspace_addr +
					(memslot->npages << PAGE_SHIFT));
		if (hva_start >= hva_end)
			continue;
		/*
		 * {gfn(page) | page intersects with [hva_start, hva_end)} =
		 * {gfn, gfn+1, ..., gfn_end-1}.
		 */
		gfn = hva_to_gfn_memslot(hva_start, memslot);
		gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
839

840
		for (; gfn < gfn_end; ++gfn) {
841
			gfn_t gfn_offset = gfn - memslot->base_gfn;
842

843
			ret = handler(kvm, &memslot->arch.rmap[gfn_offset], gfn);
844 845 846 847 848 849 850
			retval |= ret;
		}
	}

	return retval;
}

851 852 853 854 855 856 857
static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
			  int (*handler)(struct kvm *kvm, unsigned long *rmapp,
					 unsigned long gfn))
{
	return kvm_handle_hva_range(kvm, hva, hva + 1, handler);
}

858 859 860 861 862 863
static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
			   unsigned long gfn)
{
	struct revmap_entry *rev = kvm->arch.revmap;
	unsigned long h, i, j;
	unsigned long *hptep;
864
	unsigned long ptel, psize, rcbits;
865 866

	for (;;) {
867
		lock_rmap(rmapp);
868
		if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
869
			unlock_rmap(rmapp);
870 871 872 873 874
			break;
		}

		/*
		 * To avoid an ABBA deadlock with the HPTE lock bit,
875 876
		 * we can't spin on the HPTE lock while holding the
		 * rmap chain lock.
877 878
		 */
		i = *rmapp & KVMPPC_RMAP_INDEX;
879 880 881 882 883 884 885 886
		hptep = (unsigned long *) (kvm->arch.hpt_virt + (i << 4));
		if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
			/* unlock rmap before spinning on the HPTE lock */
			unlock_rmap(rmapp);
			while (hptep[0] & HPTE_V_HVLOCK)
				cpu_relax();
			continue;
		}
887 888 889
		j = rev[i].forw;
		if (j == i) {
			/* chain is now empty */
890
			*rmapp &= ~(KVMPPC_RMAP_PRESENT | KVMPPC_RMAP_INDEX);
891 892 893 894 895 896
		} else {
			/* remove i from chain */
			h = rev[i].back;
			rev[h].forw = j;
			rev[j].back = h;
			rev[i].forw = rev[i].back = i;
897
			*rmapp = (*rmapp & ~KVMPPC_RMAP_INDEX) | j;
898 899
		}

900
		/* Now check and modify the HPTE */
901 902 903 904
		ptel = rev[i].guest_rpte;
		psize = hpte_page_size(hptep[0], ptel);
		if ((hptep[0] & HPTE_V_VALID) &&
		    hpte_rpn(ptel, psize) == gfn) {
905 906
			if (kvm->arch.using_mmu_notifiers)
				hptep[0] |= HPTE_V_ABSENT;
907 908 909 910
			kvmppc_invalidate_hpte(kvm, hptep, i);
			/* Harvest R and C */
			rcbits = hptep[1] & (HPTE_R_R | HPTE_R_C);
			*rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT;
911 912 913 914
			if (rcbits & ~rev[i].guest_rpte) {
				rev[i].guest_rpte = ptel | rcbits;
				note_hpte_modification(kvm, &rev[i]);
			}
915
		}
916
		unlock_rmap(rmapp);
917 918 919 920 921
		hptep[0] &= ~HPTE_V_HVLOCK;
	}
	return 0;
}

922
int kvm_unmap_hva_hv(struct kvm *kvm, unsigned long hva)
923 924 925 926 927 928
{
	if (kvm->arch.using_mmu_notifiers)
		kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
	return 0;
}

929
int kvm_unmap_hva_range_hv(struct kvm *kvm, unsigned long start, unsigned long end)
930 931 932 933 934 935
{
	if (kvm->arch.using_mmu_notifiers)
		kvm_handle_hva_range(kvm, start, end, kvm_unmap_rmapp);
	return 0;
}

936 937
void kvmppc_core_flush_memslot_hv(struct kvm *kvm,
				  struct kvm_memory_slot *memslot)
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958
{
	unsigned long *rmapp;
	unsigned long gfn;
	unsigned long n;

	rmapp = memslot->arch.rmap;
	gfn = memslot->base_gfn;
	for (n = memslot->npages; n; --n) {
		/*
		 * Testing the present bit without locking is OK because
		 * the memslot has been marked invalid already, and hence
		 * no new HPTEs referencing this page can be created,
		 * thus the present bit can't go from 0 to 1.
		 */
		if (*rmapp & KVMPPC_RMAP_PRESENT)
			kvm_unmap_rmapp(kvm, rmapp, gfn);
		++rmapp;
		++gfn;
	}
}

959 960 961
static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
			 unsigned long gfn)
{
962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997
	struct revmap_entry *rev = kvm->arch.revmap;
	unsigned long head, i, j;
	unsigned long *hptep;
	int ret = 0;

 retry:
	lock_rmap(rmapp);
	if (*rmapp & KVMPPC_RMAP_REFERENCED) {
		*rmapp &= ~KVMPPC_RMAP_REFERENCED;
		ret = 1;
	}
	if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
		unlock_rmap(rmapp);
		return ret;
	}

	i = head = *rmapp & KVMPPC_RMAP_INDEX;
	do {
		hptep = (unsigned long *) (kvm->arch.hpt_virt + (i << 4));
		j = rev[i].forw;

		/* If this HPTE isn't referenced, ignore it */
		if (!(hptep[1] & HPTE_R_R))
			continue;

		if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
			/* unlock rmap before spinning on the HPTE lock */
			unlock_rmap(rmapp);
			while (hptep[0] & HPTE_V_HVLOCK)
				cpu_relax();
			goto retry;
		}

		/* Now check and modify the HPTE */
		if ((hptep[0] & HPTE_V_VALID) && (hptep[1] & HPTE_R_R)) {
			kvmppc_clear_ref_hpte(kvm, hptep, i);
998 999 1000 1001
			if (!(rev[i].guest_rpte & HPTE_R_R)) {
				rev[i].guest_rpte |= HPTE_R_R;
				note_hpte_modification(kvm, &rev[i]);
			}
1002 1003 1004 1005 1006 1007 1008
			ret = 1;
		}
		hptep[0] &= ~HPTE_V_HVLOCK;
	} while ((i = j) != head);

	unlock_rmap(rmapp);
	return ret;
1009 1010
}

1011
int kvm_age_hva_hv(struct kvm *kvm, unsigned long hva)
1012 1013 1014 1015 1016 1017 1018 1019 1020
{
	if (!kvm->arch.using_mmu_notifiers)
		return 0;
	return kvm_handle_hva(kvm, hva, kvm_age_rmapp);
}

static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
			      unsigned long gfn)
{
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
	struct revmap_entry *rev = kvm->arch.revmap;
	unsigned long head, i, j;
	unsigned long *hp;
	int ret = 1;

	if (*rmapp & KVMPPC_RMAP_REFERENCED)
		return 1;

	lock_rmap(rmapp);
	if (*rmapp & KVMPPC_RMAP_REFERENCED)
		goto out;

	if (*rmapp & KVMPPC_RMAP_PRESENT) {
		i = head = *rmapp & KVMPPC_RMAP_INDEX;
		do {
			hp = (unsigned long *)(kvm->arch.hpt_virt + (i << 4));
			j = rev[i].forw;
			if (hp[1] & HPTE_R_R)
				goto out;
		} while ((i = j) != head);
	}
	ret = 0;

 out:
	unlock_rmap(rmapp);
	return ret;
1047 1048
}

1049
int kvm_test_age_hva_hv(struct kvm *kvm, unsigned long hva)
1050 1051 1052 1053 1054 1055
{
	if (!kvm->arch.using_mmu_notifiers)
		return 0;
	return kvm_handle_hva(kvm, hva, kvm_test_age_rmapp);
}

1056
void kvm_set_spte_hva_hv(struct kvm *kvm, unsigned long hva, pte_t pte)
1057 1058 1059 1060
{
	if (!kvm->arch.using_mmu_notifiers)
		return;
	kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
1061 1062
}

1063 1064 1065 1066 1067
/*
 * Returns the number of system pages that are dirty.
 * This can be more than 1 if we find a huge-page HPTE.
 */
static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
1068 1069 1070
{
	struct revmap_entry *rev = kvm->arch.revmap;
	unsigned long head, i, j;
1071
	unsigned long n;
1072
	unsigned long *hptep;
1073
	int npages_dirty = 0;
1074 1075 1076 1077 1078

 retry:
	lock_rmap(rmapp);
	if (*rmapp & KVMPPC_RMAP_CHANGED) {
		*rmapp &= ~KVMPPC_RMAP_CHANGED;
1079
		npages_dirty = 1;
1080 1081 1082
	}
	if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
		unlock_rmap(rmapp);
1083
		return npages_dirty;
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
	}

	i = head = *rmapp & KVMPPC_RMAP_INDEX;
	do {
		hptep = (unsigned long *) (kvm->arch.hpt_virt + (i << 4));
		j = rev[i].forw;

		if (!(hptep[1] & HPTE_R_C))
			continue;

		if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
			/* unlock rmap before spinning on the HPTE lock */
			unlock_rmap(rmapp);
			while (hptep[0] & HPTE_V_HVLOCK)
				cpu_relax();
			goto retry;
		}

		/* Now check and modify the HPTE */
		if ((hptep[0] & HPTE_V_VALID) && (hptep[1] & HPTE_R_C)) {
			/* need to make it temporarily absent to clear C */
			hptep[0] |= HPTE_V_ABSENT;
			kvmppc_invalidate_hpte(kvm, hptep, i);
			hptep[1] &= ~HPTE_R_C;
			eieio();
			hptep[0] = (hptep[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID;
1110 1111 1112 1113
			if (!(rev[i].guest_rpte & HPTE_R_C)) {
				rev[i].guest_rpte |= HPTE_R_C;
				note_hpte_modification(kvm, &rev[i]);
			}
1114 1115 1116 1117
			n = hpte_page_size(hptep[0], hptep[1]);
			n = (n + PAGE_SIZE - 1) >> PAGE_SHIFT;
			if (n > npages_dirty)
				npages_dirty = n;
1118 1119 1120 1121 1122
		}
		hptep[0] &= ~HPTE_V_HVLOCK;
	} while ((i = j) != head);

	unlock_rmap(rmapp);
1123
	return npages_dirty;
1124 1125
}

1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
static void harvest_vpa_dirty(struct kvmppc_vpa *vpa,
			      struct kvm_memory_slot *memslot,
			      unsigned long *map)
{
	unsigned long gfn;

	if (!vpa->dirty || !vpa->pinned_addr)
		return;
	gfn = vpa->gpa >> PAGE_SHIFT;
	if (gfn < memslot->base_gfn ||
	    gfn >= memslot->base_gfn + memslot->npages)
		return;

	vpa->dirty = false;
	if (map)
		__set_bit_le(gfn - memslot->base_gfn, map);
}

1144 1145
long kvmppc_hv_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot,
			     unsigned long *map)
1146
{
1147
	unsigned long i, j;
1148
	unsigned long *rmapp;
1149
	struct kvm_vcpu *vcpu;
1150 1151

	preempt_disable();
1152
	rmapp = memslot->arch.rmap;
1153
	for (i = 0; i < memslot->npages; ++i) {
1154 1155 1156 1157 1158 1159 1160 1161 1162
		int npages = kvm_test_clear_dirty_npages(kvm, rmapp);
		/*
		 * Note that if npages > 0 then i must be a multiple of npages,
		 * since we always put huge-page HPTEs in the rmap chain
		 * corresponding to their page base address.
		 */
		if (npages && map)
			for (j = i; npages; ++j, --npages)
				__set_bit_le(j, map);
1163 1164
		++rmapp;
	}
1165 1166 1167 1168 1169 1170 1171 1172 1173

	/* Harvest dirty bits from VPA and DTL updates */
	/* Note: we never modify the SLB shadow buffer areas */
	kvm_for_each_vcpu(i, vcpu, kvm) {
		spin_lock(&vcpu->arch.vpa_update_lock);
		harvest_vpa_dirty(&vcpu->arch.vpa, memslot, map);
		harvest_vpa_dirty(&vcpu->arch.dtl, memslot, map);
		spin_unlock(&vcpu->arch.vpa_update_lock);
	}
1174 1175 1176 1177
	preempt_enable();
	return 0;
}

1178 1179 1180 1181 1182
void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,
			    unsigned long *nb_ret)
{
	struct kvm_memory_slot *memslot;
	unsigned long gfn = gpa >> PAGE_SHIFT;
1183 1184
	struct page *page, *pages[1];
	int npages;
1185
	unsigned long hva, offset;
1186
	unsigned long pa;
1187
	unsigned long *physp;
1188
	int srcu_idx;
1189

1190
	srcu_idx = srcu_read_lock(&kvm->srcu);
1191 1192
	memslot = gfn_to_memslot(kvm, gfn);
	if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
1193
		goto err;
1194
	if (!kvm->arch.using_mmu_notifiers) {
1195
		physp = memslot->arch.slot_phys;
1196
		if (!physp)
1197
			goto err;
1198
		physp += gfn - memslot->base_gfn;
1199
		pa = *physp;
1200 1201 1202
		if (!pa) {
			if (kvmppc_get_guest_page(kvm, gfn, memslot,
						  PAGE_SIZE) < 0)
1203
				goto err;
1204 1205 1206
			pa = *physp;
		}
		page = pfn_to_page(pa >> PAGE_SHIFT);
1207
		get_page(page);
1208 1209 1210 1211
	} else {
		hva = gfn_to_hva_memslot(memslot, gfn);
		npages = get_user_pages_fast(hva, 1, 1, pages);
		if (npages < 1)
1212
			goto err;
1213
		page = pages[0];
1214
	}
1215 1216
	srcu_read_unlock(&kvm->srcu, srcu_idx);

1217
	offset = gpa & (PAGE_SIZE - 1);
1218
	if (nb_ret)
1219
		*nb_ret = PAGE_SIZE - offset;
1220
	return page_address(page) + offset;
1221 1222 1223 1224

 err:
	srcu_read_unlock(&kvm->srcu, srcu_idx);
	return NULL;
1225 1226
}

1227 1228
void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,
			     bool dirty)
1229 1230
{
	struct page *page = virt_to_page(va);
1231 1232 1233 1234
	struct kvm_memory_slot *memslot;
	unsigned long gfn;
	unsigned long *rmap;
	int srcu_idx;
1235 1236

	put_page(page);
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251

	if (!dirty || !kvm->arch.using_mmu_notifiers)
		return;

	/* We need to mark this page dirty in the rmap chain */
	gfn = gpa >> PAGE_SHIFT;
	srcu_idx = srcu_read_lock(&kvm->srcu);
	memslot = gfn_to_memslot(kvm, gfn);
	if (memslot) {
		rmap = &memslot->arch.rmap[gfn - memslot->base_gfn];
		lock_rmap(rmap);
		*rmap |= KVMPPC_RMAP_CHANGED;
		unlock_rmap(rmap);
	}
	srcu_read_unlock(&kvm->srcu, srcu_idx);
1252 1253
}

1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
/*
 * Functions for reading and writing the hash table via reads and
 * writes on a file descriptor.
 *
 * Reads return the guest view of the hash table, which has to be
 * pieced together from the real hash table and the guest_rpte
 * values in the revmap array.
 *
 * On writes, each HPTE written is considered in turn, and if it
 * is valid, it is written to the HPT as if an H_ENTER with the
 * exact flag set was done.  When the invalid count is non-zero
 * in the header written to the stream, the kernel will make
 * sure that that many HPTEs are invalid, and invalidate them
 * if not.
 */

struct kvm_htab_ctx {
	unsigned long	index;
	unsigned long	flags;
	struct kvm	*kvm;
	int		first_pass;
};

#define HPTE_SIZE	(2 * sizeof(unsigned long))

1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297
/*
 * Returns 1 if this HPT entry has been modified or has pending
 * R/C bit changes.
 */
static int hpte_dirty(struct revmap_entry *revp, unsigned long *hptp)
{
	unsigned long rcbits_unset;

	if (revp->guest_rpte & HPTE_GR_MODIFIED)
		return 1;

	/* Also need to consider changes in reference and changed bits */
	rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
	if ((hptp[0] & HPTE_V_VALID) && (hptp[1] & rcbits_unset))
		return 1;

	return 0;
}

1298 1299 1300 1301 1302
static long record_hpte(unsigned long flags, unsigned long *hptp,
			unsigned long *hpte, struct revmap_entry *revp,
			int want_valid, int first_pass)
{
	unsigned long v, r;
1303
	unsigned long rcbits_unset;
1304 1305 1306 1307
	int ok = 1;
	int valid, dirty;

	/* Unmodified entries are uninteresting except on the first pass */
1308
	dirty = hpte_dirty(revp, hptp);
1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328
	if (!first_pass && !dirty)
		return 0;

	valid = 0;
	if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT)) {
		valid = 1;
		if ((flags & KVM_GET_HTAB_BOLTED_ONLY) &&
		    !(hptp[0] & HPTE_V_BOLTED))
			valid = 0;
	}
	if (valid != want_valid)
		return 0;

	v = r = 0;
	if (valid || dirty) {
		/* lock the HPTE so it's stable and read it */
		preempt_disable();
		while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
			cpu_relax();
		v = hptp[0];
1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341

		/* re-evaluate valid and dirty from synchronized HPTE value */
		valid = !!(v & HPTE_V_VALID);
		dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED);

		/* Harvest R and C into guest view if necessary */
		rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
		if (valid && (rcbits_unset & hptp[1])) {
			revp->guest_rpte |= (hptp[1] & (HPTE_R_R | HPTE_R_C)) |
				HPTE_GR_MODIFIED;
			dirty = 1;
		}

1342 1343 1344
		if (v & HPTE_V_ABSENT) {
			v &= ~HPTE_V_ABSENT;
			v |= HPTE_V_VALID;
1345
			valid = 1;
1346 1347 1348
		}
		if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && !(v & HPTE_V_BOLTED))
			valid = 0;
1349 1350

		r = revp->guest_rpte;
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405
		/* only clear modified if this is the right sort of entry */
		if (valid == want_valid && dirty) {
			r &= ~HPTE_GR_MODIFIED;
			revp->guest_rpte = r;
		}
		asm volatile(PPC_RELEASE_BARRIER "" : : : "memory");
		hptp[0] &= ~HPTE_V_HVLOCK;
		preempt_enable();
		if (!(valid == want_valid && (first_pass || dirty)))
			ok = 0;
	}
	hpte[0] = v;
	hpte[1] = r;
	return ok;
}

static ssize_t kvm_htab_read(struct file *file, char __user *buf,
			     size_t count, loff_t *ppos)
{
	struct kvm_htab_ctx *ctx = file->private_data;
	struct kvm *kvm = ctx->kvm;
	struct kvm_get_htab_header hdr;
	unsigned long *hptp;
	struct revmap_entry *revp;
	unsigned long i, nb, nw;
	unsigned long __user *lbuf;
	struct kvm_get_htab_header __user *hptr;
	unsigned long flags;
	int first_pass;
	unsigned long hpte[2];

	if (!access_ok(VERIFY_WRITE, buf, count))
		return -EFAULT;

	first_pass = ctx->first_pass;
	flags = ctx->flags;

	i = ctx->index;
	hptp = (unsigned long *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));
	revp = kvm->arch.revmap + i;
	lbuf = (unsigned long __user *)buf;

	nb = 0;
	while (nb + sizeof(hdr) + HPTE_SIZE < count) {
		/* Initialize header */
		hptr = (struct kvm_get_htab_header __user *)buf;
		hdr.n_valid = 0;
		hdr.n_invalid = 0;
		nw = nb;
		nb += sizeof(hdr);
		lbuf = (unsigned long __user *)(buf + sizeof(hdr));

		/* Skip uninteresting entries, i.e. clean on not-first pass */
		if (!first_pass) {
			while (i < kvm->arch.hpt_npte &&
1406
			       !hpte_dirty(revp, hptp)) {
1407 1408 1409 1410 1411
				++i;
				hptp += 2;
				++revp;
			}
		}
1412
		hdr.index = i;
1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543

		/* Grab a series of valid entries */
		while (i < kvm->arch.hpt_npte &&
		       hdr.n_valid < 0xffff &&
		       nb + HPTE_SIZE < count &&
		       record_hpte(flags, hptp, hpte, revp, 1, first_pass)) {
			/* valid entry, write it out */
			++hdr.n_valid;
			if (__put_user(hpte[0], lbuf) ||
			    __put_user(hpte[1], lbuf + 1))
				return -EFAULT;
			nb += HPTE_SIZE;
			lbuf += 2;
			++i;
			hptp += 2;
			++revp;
		}
		/* Now skip invalid entries while we can */
		while (i < kvm->arch.hpt_npte &&
		       hdr.n_invalid < 0xffff &&
		       record_hpte(flags, hptp, hpte, revp, 0, first_pass)) {
			/* found an invalid entry */
			++hdr.n_invalid;
			++i;
			hptp += 2;
			++revp;
		}

		if (hdr.n_valid || hdr.n_invalid) {
			/* write back the header */
			if (__copy_to_user(hptr, &hdr, sizeof(hdr)))
				return -EFAULT;
			nw = nb;
			buf = (char __user *)lbuf;
		} else {
			nb = nw;
		}

		/* Check if we've wrapped around the hash table */
		if (i >= kvm->arch.hpt_npte) {
			i = 0;
			ctx->first_pass = 0;
			break;
		}
	}

	ctx->index = i;

	return nb;
}

static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
			      size_t count, loff_t *ppos)
{
	struct kvm_htab_ctx *ctx = file->private_data;
	struct kvm *kvm = ctx->kvm;
	struct kvm_get_htab_header hdr;
	unsigned long i, j;
	unsigned long v, r;
	unsigned long __user *lbuf;
	unsigned long *hptp;
	unsigned long tmp[2];
	ssize_t nb;
	long int err, ret;
	int rma_setup;

	if (!access_ok(VERIFY_READ, buf, count))
		return -EFAULT;

	/* lock out vcpus from running while we're doing this */
	mutex_lock(&kvm->lock);
	rma_setup = kvm->arch.rma_setup_done;
	if (rma_setup) {
		kvm->arch.rma_setup_done = 0;	/* temporarily */
		/* order rma_setup_done vs. vcpus_running */
		smp_mb();
		if (atomic_read(&kvm->arch.vcpus_running)) {
			kvm->arch.rma_setup_done = 1;
			mutex_unlock(&kvm->lock);
			return -EBUSY;
		}
	}

	err = 0;
	for (nb = 0; nb + sizeof(hdr) <= count; ) {
		err = -EFAULT;
		if (__copy_from_user(&hdr, buf, sizeof(hdr)))
			break;

		err = 0;
		if (nb + hdr.n_valid * HPTE_SIZE > count)
			break;

		nb += sizeof(hdr);
		buf += sizeof(hdr);

		err = -EINVAL;
		i = hdr.index;
		if (i >= kvm->arch.hpt_npte ||
		    i + hdr.n_valid + hdr.n_invalid > kvm->arch.hpt_npte)
			break;

		hptp = (unsigned long *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));
		lbuf = (unsigned long __user *)buf;
		for (j = 0; j < hdr.n_valid; ++j) {
			err = -EFAULT;
			if (__get_user(v, lbuf) || __get_user(r, lbuf + 1))
				goto out;
			err = -EINVAL;
			if (!(v & HPTE_V_VALID))
				goto out;
			lbuf += 2;
			nb += HPTE_SIZE;

			if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT))
				kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
			err = -EIO;
			ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,
							 tmp);
			if (ret != H_SUCCESS) {
				pr_err("kvm_htab_write ret %ld i=%ld v=%lx "
				       "r=%lx\n", ret, i, v, r);
				goto out;
			}
			if (!rma_setup && is_vrma_hpte(v)) {
				unsigned long psize = hpte_page_size(v, r);
				unsigned long senc = slb_pgsize_encoding(psize);
				unsigned long lpcr;

				kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
					(VRMA_VSID << SLB_VSID_SHIFT_1T);
1544 1545
				lpcr = senc << (LPCR_VRMASD_SH - 4);
				kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
				rma_setup = 1;
			}
			++i;
			hptp += 2;
		}

		for (j = 0; j < hdr.n_invalid; ++j) {
			if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT))
				kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
			++i;
			hptp += 2;
		}
		err = 0;
	}

 out:
	/* Order HPTE updates vs. rma_setup_done */
	smp_wmb();
	kvm->arch.rma_setup_done = rma_setup;
	mutex_unlock(&kvm->lock);

	if (err)
		return err;
	return nb;
}

static int kvm_htab_release(struct inode *inode, struct file *filp)
{
	struct kvm_htab_ctx *ctx = filp->private_data;

	filp->private_data = NULL;
	if (!(ctx->flags & KVM_GET_HTAB_WRITE))
		atomic_dec(&ctx->kvm->arch.hpte_mod_interest);
	kvm_put_kvm(ctx->kvm);
	kfree(ctx);
	return 0;
}

1584
static const struct file_operations kvm_htab_fops = {
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609
	.read		= kvm_htab_read,
	.write		= kvm_htab_write,
	.llseek		= default_llseek,
	.release	= kvm_htab_release,
};

int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf)
{
	int ret;
	struct kvm_htab_ctx *ctx;
	int rwflag;

	/* reject flags we don't recognize */
	if (ghf->flags & ~(KVM_GET_HTAB_BOLTED_ONLY | KVM_GET_HTAB_WRITE))
		return -EINVAL;
	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		return -ENOMEM;
	kvm_get_kvm(kvm);
	ctx->kvm = kvm;
	ctx->index = ghf->start_index;
	ctx->flags = ghf->flags;
	ctx->first_pass = 1;

	rwflag = (ghf->flags & KVM_GET_HTAB_WRITE) ? O_WRONLY : O_RDONLY;
1610
	ret = anon_inode_getfd("kvm-htab", &kvm_htab_fops, ctx, rwflag | O_CLOEXEC);
1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626
	if (ret < 0) {
		kvm_put_kvm(kvm);
		return ret;
	}

	if (rwflag == O_RDONLY) {
		mutex_lock(&kvm->slots_lock);
		atomic_inc(&kvm->arch.hpte_mod_interest);
		/* make sure kvmppc_do_h_enter etc. see the increment */
		synchronize_srcu_expedited(&kvm->srcu);
		mutex_unlock(&kvm->slots_lock);
	}

	return ret;
}

1627 1628 1629 1630
void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu)
{
	struct kvmppc_mmu *mmu = &vcpu->arch.mmu;

1631 1632 1633 1634
	if (cpu_has_feature(CPU_FTR_ARCH_206))
		vcpu->arch.slb_nr = 32;		/* POWER7 */
	else
		vcpu->arch.slb_nr = 64;
1635 1636 1637 1638 1639 1640

	mmu->xlate = kvmppc_mmu_book3s_64_hv_xlate;
	mmu->reset_msr = kvmppc_mmu_book3s_64_hv_reset_msr;

	vcpu->arch.hflags |= BOOK3S_HFLAG_SLB;
}