kvm_book3s_64.h 11.0 KB
Newer Older
A
Alexander Graf 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 *
 * Copyright SUSE Linux Products GmbH 2010
 *
 * Authors: Alexander Graf <agraf@suse.de>
 */

#ifndef __ASM_KVM_BOOK3S_64_H__
#define __ASM_KVM_BOOK3S_64_H__

23
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
24
static inline struct kvmppc_book3s_shadow_vcpu *svcpu_get(struct kvm_vcpu *vcpu)
A
Alexander Graf 已提交
25
{
26
	preempt_disable();
A
Alexander Graf 已提交
27 28
	return &get_paca()->shadow_vcpu;
}
29 30 31 32 33

static inline void svcpu_put(struct kvmppc_book3s_shadow_vcpu *svcpu)
{
	preempt_enable();
}
34
#endif
A
Alexander Graf 已提交
35

36
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
37
#define KVM_DEFAULT_HPT_ORDER	24	/* 16MB HPT by default */
38 39
#endif

40 41
#define VRMA_VSID	0x1ffffffUL	/* 1TB VSID reserved for VRMA */

42 43 44 45 46 47
/*
 * We use a lock bit in HPTE dword 0 to synchronize updates and
 * accesses to each HPTE, and another bit to indicate non-present
 * HPTEs.
 */
#define HPTE_V_HVLOCK	0x40UL
48
#define HPTE_V_ABSENT	0x20UL
49

50 51 52 53 54 55 56 57 58
/*
 * We use this bit in the guest_rpte field of the revmap entry
 * to indicate a modified HPTE.
 */
#define HPTE_GR_MODIFIED	(1ul << 62)

/* These bits are reserved in the guest view of the HPTE */
#define HPTE_GR_RESERVED	HPTE_GR_MODIFIED

59
static inline long try_lock_hpte(__be64 *hpte, unsigned long bits)
60 61
{
	unsigned long tmp, old;
62 63 64 65 66 67 68 69 70
	__be64 be_lockbit, be_bits;

	/*
	 * We load/store in native endian, but the HTAB is in big endian. If
	 * we byte swap all data we apply on the PTE we're implicitly correct
	 * again.
	 */
	be_lockbit = cpu_to_be64(HPTE_V_HVLOCK);
	be_bits = cpu_to_be64(bits);
71 72 73 74

	asm volatile("	ldarx	%0,0,%2\n"
		     "	and.	%1,%0,%3\n"
		     "	bne	2f\n"
75
		     "	or	%0,%0,%4\n"
76 77
		     "  stdcx.	%0,0,%2\n"
		     "	beq+	2f\n"
78
		     "	mr	%1,%3\n"
79 80
		     "2:	isync"
		     : "=&r" (tmp), "=&r" (old)
81
		     : "r" (hpte), "r" (be_bits), "r" (be_lockbit)
82 83 84 85
		     : "cc", "memory");
	return old == 0;
}

86 87 88 89 90 91 92 93 94 95 96 97 98 99
static inline void unlock_hpte(__be64 *hpte, unsigned long hpte_v)
{
	hpte_v &= ~HPTE_V_HVLOCK;
	asm volatile(PPC_RELEASE_BARRIER "" : : : "memory");
	hpte[0] = cpu_to_be64(hpte_v);
}

/* Without barrier */
static inline void __unlock_hpte(__be64 *hpte, unsigned long hpte_v)
{
	hpte_v &= ~HPTE_V_HVLOCK;
	hpte[0] = cpu_to_be64(hpte_v);
}

100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129
static inline int __hpte_actual_psize(unsigned int lp, int psize)
{
	int i, shift;
	unsigned int mask;

	/* start from 1 ignoring MMU_PAGE_4K */
	for (i = 1; i < MMU_PAGE_COUNT; i++) {

		/* invalid penc */
		if (mmu_psize_defs[psize].penc[i] == -1)
			continue;
		/*
		 * encoding bits per actual page size
		 *        PTE LP     actual page size
		 *    rrrr rrrz		>=8KB
		 *    rrrr rrzz		>=16KB
		 *    rrrr rzzz		>=32KB
		 *    rrrr zzzz		>=64KB
		 * .......
		 */
		shift = mmu_psize_defs[i].shift - LP_SHIFT;
		if (shift > LP_BITS)
			shift = LP_BITS;
		mask = (1 << shift) - 1;
		if ((lp & mask) == mmu_psize_defs[psize].penc[i])
			return i;
	}
	return -1;
}

130 131 132
static inline unsigned long compute_tlbie_rb(unsigned long v, unsigned long r,
					     unsigned long pte_index)
{
133
	int b_psize = MMU_PAGE_4K, a_psize = MMU_PAGE_4K;
134 135 136 137
	unsigned int penc;
	unsigned long rb = 0, va_low, sllp;
	unsigned int lp = (r >> LP_SHIFT) & ((1 << LP_BITS) - 1);

138
	if (v & HPTE_V_LARGE) {
139 140 141 142 143
		for (b_psize = 0; b_psize < MMU_PAGE_COUNT; b_psize++) {

			/* valid entries have a shift value */
			if (!mmu_psize_defs[b_psize].shift)
				continue;
144

145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
			a_psize = __hpte_actual_psize(lp, b_psize);
			if (a_psize != -1)
				break;
		}
	}
	/*
	 * Ignore the top 14 bits of va
	 * v have top two bits covering segment size, hence move
	 * by 16 bits, Also clear the lower HPTE_V_AVPN_SHIFT (7) bits.
	 * AVA field in v also have the lower 23 bits ignored.
	 * For base page size 4K we need 14 .. 65 bits (so need to
	 * collect extra 11 bits)
	 * For others we need 14..14+i
	 */
	/* This covers 14..54 bits of va*/
160
	rb = (v & ~0x7fUL) << 16;		/* AVA field */
161

162
	rb |= (v >> HPTE_V_SSIZE_SHIFT) << 8;	/*  B field */
163 164 165 166
	/*
	 * AVA in v had cleared lower 23 bits. We need to derive
	 * that from pteg index
	 */
167 168 169
	va_low = pte_index >> 3;
	if (v & HPTE_V_SECONDARY)
		va_low = ~va_low;
170 171 172 173 174 175
	/*
	 * get the vpn bits from va_low using reverse of hashing.
	 * In v we have va with 23 bits dropped and then left shifted
	 * HPTE_V_AVPN_SHIFT (7) bits. Now to find vsid we need
	 * right shift it with (SID_SHIFT - (23 - 7))
	 */
176
	if (!(v & HPTE_V_1TB_SEG))
177
		va_low ^= v >> (SID_SHIFT - 16);
178
	else
179
		va_low ^= v >> (SID_SHIFT_1T - 16);
180
	va_low &= 0x7ff;
181 182 183 184 185 186 187 188 189 190 191 192

	switch (b_psize) {
	case MMU_PAGE_4K:
		sllp = ((mmu_psize_defs[a_psize].sllp & SLB_VSID_L) >> 6) |
			((mmu_psize_defs[a_psize].sllp & SLB_VSID_LP) >> 4);
		rb |= sllp << 5;	/*  AP field */
		rb |= (va_low & 0x7ff) << 12;	/* remaining 11 bits of AVA */
		break;
	default:
	{
		int aval_shift;
		/*
193
		 * remaining bits of AVA/LP fields
194 195
		 * Also contain the rr bits of LP
		 */
196
		rb |= (va_low << mmu_psize_defs[b_psize].shift) & 0x7ff000;
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
		/*
		 * Now clear not needed LP bits based on actual psize
		 */
		rb &= ~((1ul << mmu_psize_defs[a_psize].shift) - 1);
		/*
		 * AVAL field 58..77 - base_page_shift bits of va
		 * we have space for 58..64 bits, Missing bits should
		 * be zero filled. +1 is to take care of L bit shift
		 */
		aval_shift = 64 - (77 - mmu_psize_defs[b_psize].shift) + 1;
		rb |= ((va_low << aval_shift) & 0xfe);

		rb |= 1;		/* L field */
		penc = mmu_psize_defs[b_psize].penc[a_psize];
		rb |= penc << 12;	/* LP field */
		break;
	}
214 215 216 217 218
	}
	rb |= (v >> 54) & 0x300;		/* B field */
	return rb;
}

219 220
static inline unsigned long __hpte_page_size(unsigned long h, unsigned long l,
					     bool is_base_size)
221
{
222

223 224 225 226
	int size, a_psize;
	/* Look at the 8 bit LP value */
	unsigned int lp = (l >> LP_SHIFT) & ((1 << LP_BITS) - 1);

227 228
	/* only handle 4k, 64k and 16M pages for now */
	if (!(h & HPTE_V_LARGE))
229 230 231 232 233 234 235 236
		return 1ul << 12;
	else {
		for (size = 0; size < MMU_PAGE_COUNT; size++) {
			/* valid entries have a shift value */
			if (!mmu_psize_defs[size].shift)
				continue;

			a_psize = __hpte_actual_psize(lp, size);
237 238 239
			if (a_psize != -1) {
				if (is_base_size)
					return 1ul << mmu_psize_defs[size].shift;
240
				return 1ul << mmu_psize_defs[a_psize].shift;
241
			}
242 243 244 245
		}

	}
	return 0;
246 247
}

248 249 250 251 252 253 254 255 256 257
static inline unsigned long hpte_page_size(unsigned long h, unsigned long l)
{
	return __hpte_page_size(h, l, 0);
}

static inline unsigned long hpte_base_page_size(unsigned long h, unsigned long l)
{
	return __hpte_page_size(h, l, 1);
}

258 259 260 261 262
static inline unsigned long hpte_rpn(unsigned long ptel, unsigned long psize)
{
	return ((ptel & HPTE_R_RPN) & ~(psize - 1)) >> PAGE_SHIFT;
}

263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
static inline int hpte_is_writable(unsigned long ptel)
{
	unsigned long pp = ptel & (HPTE_R_PP0 | HPTE_R_PP);

	return pp != PP_RXRX && pp != PP_RXXX;
}

static inline unsigned long hpte_make_readonly(unsigned long ptel)
{
	if ((ptel & HPTE_R_PP0) || (ptel & HPTE_R_PP) == PP_RWXX)
		ptel = (ptel & ~HPTE_R_PP) | PP_RXXX;
	else
		ptel |= PP_RXRX;
	return ptel;
}

279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
static inline int hpte_cache_flags_ok(unsigned long ptel, unsigned long io_type)
{
	unsigned int wimg = ptel & HPTE_R_WIMG;

	/* Handle SAO */
	if (wimg == (HPTE_R_W | HPTE_R_I | HPTE_R_M) &&
	    cpu_has_feature(CPU_FTR_ARCH_206))
		wimg = HPTE_R_M;

	if (!io_type)
		return wimg == HPTE_R_M;

	return (wimg & (HPTE_R_W | HPTE_R_I)) == io_type;
}

294
/*
295
 * If it's present and writable, atomically set dirty and referenced bits and
296
 * return the PTE, otherwise return 0.
297
 */
298
static inline pte_t kvmppc_read_update_linux_pte(pte_t *ptep, int writing)
299
{
300 301 302
	pte_t old_pte, new_pte = __pte(0);

	while (1) {
303 304 305 306
		/*
		 * Make sure we don't reload from ptep
		 */
		old_pte = READ_ONCE(*ptep);
307 308 309
		/*
		 * wait until _PAGE_BUSY is clear then set it atomically
		 */
310
		if (unlikely(pte_val(old_pte) & _PAGE_BUSY)) {
311 312 313 314
			cpu_relax();
			continue;
		}
		/* If pte is not present return None */
315
		if (unlikely(!(pte_val(old_pte) & _PAGE_PRESENT)))
316
			return __pte(0);
317

318 319 320
		new_pte = pte_mkyoung(old_pte);
		if (writing && pte_write(old_pte))
			new_pte = pte_mkdirty(new_pte);
321

322
		if (pte_xchg(ptep, old_pte, new_pte))
323 324 325
			break;
	}
	return new_pte;
326 327
}

328

329 330 331 332 333 334 335 336 337 338 339
/* Return HPTE cache control bits corresponding to Linux pte bits */
static inline unsigned long hpte_cache_bits(unsigned long pte_val)
{
#if _PAGE_NO_CACHE == HPTE_R_I && _PAGE_WRITETHRU == HPTE_R_W
	return pte_val & (HPTE_R_W | HPTE_R_I);
#else
	return ((pte_val & _PAGE_NO_CACHE) ? HPTE_R_I : 0) +
		((pte_val & _PAGE_WRITETHRU) ? HPTE_R_W : 0);
#endif
}

340 341 342 343
static inline bool hpte_read_permission(unsigned long pp, unsigned long key)
{
	if (key)
		return PP_RWRX <= pp && pp <= PP_RXRX;
344
	return true;
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
}

static inline bool hpte_write_permission(unsigned long pp, unsigned long key)
{
	if (key)
		return pp == PP_RWRW;
	return pp <= PP_RWRW;
}

static inline int hpte_get_skey_perm(unsigned long hpte_r, unsigned long amr)
{
	unsigned long skey;

	skey = ((hpte_r & HPTE_R_KEY_HI) >> 57) |
		((hpte_r & HPTE_R_KEY_LO) >> 9);
	return (amr >> (62 - 2 * skey)) & 3;
}

363 364 365 366 367 368 369 370 371 372 373 374 375
static inline void lock_rmap(unsigned long *rmap)
{
	do {
		while (test_bit(KVMPPC_RMAP_LOCK_BIT, rmap))
			cpu_relax();
	} while (test_and_set_bit_lock(KVMPPC_RMAP_LOCK_BIT, rmap));
}

static inline void unlock_rmap(unsigned long *rmap)
{
	__clear_bit_unlock(KVMPPC_RMAP_LOCK_BIT, rmap);
}

376 377 378 379 380 381
static inline bool slot_is_aligned(struct kvm_memory_slot *memslot,
				   unsigned long pagesize)
{
	unsigned long mask = (pagesize >> PAGE_SHIFT) - 1;

	if (pagesize <= PAGE_SIZE)
382
		return true;
383 384 385
	return !(memslot->base_gfn & mask) && !(memslot->npages & mask);
}

386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
/*
 * This works for 4k, 64k and 16M pages on POWER7,
 * and 4k and 16M pages on PPC970.
 */
static inline unsigned long slb_pgsize_encoding(unsigned long psize)
{
	unsigned long senc = 0;

	if (psize > 0x1000) {
		senc = SLB_VSID_L;
		if (psize == 0x10000)
			senc |= SLB_VSID_LP_01;
	}
	return senc;
}

static inline int is_vrma_hpte(unsigned long hpte_v)
{
	return (hpte_v & ~0xffffffUL) ==
		(HPTE_V_1TB_SEG | (VRMA_VSID << (40 - 16)));
}

408
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
409 410 411 412 413 414 415 416 417 418
/*
 * Note modification of an HPTE; set the HPTE modified bit
 * if anyone is interested.
 */
static inline void note_hpte_modification(struct kvm *kvm,
					  struct revmap_entry *rev)
{
	if (atomic_read(&kvm->arch.hpte_mod_interest))
		rev->guest_rpte |= HPTE_GR_MODIFIED;
}
419 420 421 422 423 424 425 426 427

/*
 * Like kvm_memslots(), but for use in real mode when we can't do
 * any RCU stuff (since the secondary threads are offline from the
 * kernel's point of view), and we can't print anything.
 * Thus we use rcu_dereference_raw() rather than rcu_dereference_check().
 */
static inline struct kvm_memslots *kvm_memslots_raw(struct kvm *kvm)
{
428
	return rcu_dereference_raw_notrace(kvm->memslots[0]);
429 430
}

431 432
extern void kvmppc_mmu_debugfs_init(struct kvm *kvm);

433 434
extern void kvmhv_rm_send_ipi(int cpu);

435
#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
436

A
Alexander Graf 已提交
437
#endif /* __ASM_KVM_BOOK3S_64_H__ */