switch.c 12.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2015 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

18
#include <linux/types.h>
19
#include <linux/jump_label.h>
20
#include <uapi/linux/psci.h>
21

22 23
#include <kvm/arm_psci.h>

24
#include <asm/kvm_asm.h>
25
#include <asm/kvm_emulate.h>
26
#include <asm/kvm_hyp.h>
27
#include <asm/kvm_mmu.h>
28
#include <asm/fpsimd.h>
29
#include <asm/debug-monitors.h>
30

31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
static bool __hyp_text __fpsimd_enabled_nvhe(void)
{
	return !(read_sysreg(cptr_el2) & CPTR_EL2_TFP);
}

static bool __hyp_text __fpsimd_enabled_vhe(void)
{
	return !!(read_sysreg(cpacr_el1) & CPACR_EL1_FPEN);
}

static hyp_alternate_select(__fpsimd_is_enabled,
			    __fpsimd_enabled_nvhe, __fpsimd_enabled_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

bool __hyp_text __fpsimd_enabled(void)
{
	return __fpsimd_is_enabled()();
}

50 51 52 53 54 55
static void __hyp_text __activate_traps_vhe(void)
{
	u64 val;

	val = read_sysreg(cpacr_el1);
	val |= CPACR_EL1_TTA;
56
	val &= ~(CPACR_EL1_FPEN | CPACR_EL1_ZEN);
57 58
	write_sysreg(val, cpacr_el1);

59
	write_sysreg(kvm_get_hyp_vector(), vbar_el1);
60 61 62 63 64 65 66
}

static void __hyp_text __activate_traps_nvhe(void)
{
	u64 val;

	val = CPTR_EL2_DEFAULT;
67
	val |= CPTR_EL2_TTA | CPTR_EL2_TFP | CPTR_EL2_TZ;
68 69 70 71 72 73 74
	write_sysreg(val, cptr_el2);
}

static hyp_alternate_select(__activate_traps_arch,
			    __activate_traps_nvhe, __activate_traps_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

75 76
static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
{
77
	u64 hcr = vcpu->arch.hcr_el2;
78 79 80 81 82 83 84

	/*
	 * We are about to set CPTR_EL2.TFP to trap all floating point
	 * register accesses to EL2, however, the ARM ARM clearly states that
	 * traps are only taken to EL2 if the operation would not otherwise
	 * trap to EL1.  Therefore, always make sure that for 32-bit guests,
	 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
85 86
	 * If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to
	 * it will cause an exception.
87
	 */
88
	if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd()) {
89 90 91
		write_sysreg(1 << 30, fpexc32_el2);
		isb();
	}
92

93
	if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
94 95
		write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);

96 97
	write_sysreg(hcr, hcr_el2);

98 99
	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
	write_sysreg(1 << 15, hstr_el2);
100 101 102 103 104 105 106
	/*
	 * Make sure we trap PMU access from EL0 to EL2. Also sanitize
	 * PMSELR_EL0 to make sure it never contains the cycle
	 * counter, which could make a PMXEVCNTR_EL0 access UNDEF at
	 * EL1 instead of being trapped to EL2.
	 */
	write_sysreg(0, pmselr_el0);
107
	write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
108 109 110
	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
	__activate_traps_arch()();
}
111

112 113 114
static void __hyp_text __deactivate_traps_vhe(void)
{
	extern char vectors[];	/* kernel exception vectors */
115
	u64 mdcr_el2 = read_sysreg(mdcr_el2);
116

117 118 119 120 121
	mdcr_el2 &= MDCR_EL2_HPMN_MASK |
		    MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT |
		    MDCR_EL2_TPMS;

	write_sysreg(mdcr_el2, mdcr_el2);
122
	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
123
	write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
124
	write_sysreg(vectors, vbar_el1);
125 126
}

127
static void __hyp_text __deactivate_traps_nvhe(void)
128
{
129 130 131 132 133 134
	u64 mdcr_el2 = read_sysreg(mdcr_el2);

	mdcr_el2 &= MDCR_EL2_HPMN_MASK;
	mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;

	write_sysreg(mdcr_el2, mdcr_el2);
135
	write_sysreg(HCR_RW, hcr_el2);
136 137 138 139 140 141 142 143 144
	write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
}

static hyp_alternate_select(__deactivate_traps_arch,
			    __deactivate_traps_nvhe, __deactivate_traps_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
{
145 146 147 148 149 150 151 152 153
	/*
	 * If we pended a virtual abort, preserve it until it gets
	 * cleared. See D1.14.3 (Virtual Interrupts) for details, but
	 * the crucial bit is "On taking a vSError interrupt,
	 * HCR_EL2.VSE is cleared to 0."
	 */
	if (vcpu->arch.hcr_el2 & HCR_VSE)
		vcpu->arch.hcr_el2 = read_sysreg(hcr_el2);

154
	__deactivate_traps_arch()();
155
	write_sysreg(0, hstr_el2);
156
	write_sysreg(0, pmuserenr_el0);
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
}

static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	write_sysreg(kvm->arch.vttbr, vttbr_el2);
}

static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
{
	write_sysreg(0, vttbr_el2);
}

static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
{
172 173 174 175
	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
		__vgic_v3_save_state(vcpu);
	else
		__vgic_v2_save_state(vcpu);
176 177 178 179
}

static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
{
180 181 182 183
	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
		__vgic_v3_restore_state(vcpu);
	else
		__vgic_v2_restore_state(vcpu);
184 185
}

186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
static bool __hyp_text __true_value(void)
{
	return true;
}

static bool __hyp_text __false_value(void)
{
	return false;
}

static hyp_alternate_select(__check_arm_834220,
			    __false_value, __true_value,
			    ARM64_WORKAROUND_834220);

static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar)
{
	u64 par, tmp;

	/*
	 * Resolve the IPA the hard way using the guest VA.
	 *
	 * Stage-1 translation already validated the memory access
	 * rights. As such, we can use the EL1 translation regime, and
	 * don't have to distinguish between EL0 and EL1 access.
	 *
	 * We do need to save/restore PAR_EL1 though, as we haven't
	 * saved the guest context yet, and we may return early...
	 */
	par = read_sysreg(par_el1);
	asm volatile("at s1e1r, %0" : : "r" (far));
	isb();

	tmp = read_sysreg(par_el1);
	write_sysreg(par, par_el1);

	if (unlikely(tmp & 1))
		return false; /* Translation failed, back to guest */

	/* Convert PAR to HPFAR format */
	*hpfar = ((tmp >> 12) & ((1UL << 36) - 1)) << 4;
	return true;
}

static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
{
231 232
	u8 ec;
	u64 esr;
233 234
	u64 hpfar, far;

235 236
	esr = vcpu->arch.fault.esr_el2;
	ec = ESR_ELx_EC(esr);
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266

	if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
		return true;

	far = read_sysreg_el2(far);

	/*
	 * The HPFAR can be invalid if the stage 2 fault did not
	 * happen during a stage 1 page table walk (the ESR_EL2.S1PTW
	 * bit is clear) and one of the two following cases are true:
	 *   1. The fault was due to a permission fault
	 *   2. The processor carries errata 834220
	 *
	 * Therefore, for all non S1PTW faults where we either have a
	 * permission fault or the errata workaround is enabled, we
	 * resolve the IPA using the AT instruction.
	 */
	if (!(esr & ESR_ELx_S1PTW) &&
	    (__check_arm_834220()() || (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) {
		if (!__translate_far_to_hpfar(far, &hpfar))
			return false;
	} else {
		hpfar = read_sysreg(hpfar_el2);
	}

	vcpu->arch.fault.far_el2 = far;
	vcpu->arch.fault.hpfar_el2 = hpfar;
	return true;
}

267 268 269 270 271
/* Skip an instruction which has been emulated. Returns true if
 * execution can continue or false if we need to exit hyp mode because
 * single-step was in effect.
 */
static bool __hyp_text __skip_instr(struct kvm_vcpu *vcpu)
272 273 274 275 276 277 278 279 280 281 282 283
{
	*vcpu_pc(vcpu) = read_sysreg_el2(elr);

	if (vcpu_mode_is_32bit(vcpu)) {
		vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(spsr);
		kvm_skip_instr32(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
		write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, spsr);
	} else {
		*vcpu_pc(vcpu) += 4;
	}

	write_sysreg_el2(*vcpu_pc(vcpu), elr);
284 285 286 287 288 289 290 291

	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
		vcpu->arch.fault.esr_el2 =
			(ESR_ELx_EC_SOFTSTP_LOW << ESR_ELx_EC_SHIFT) | 0x22;
		return false;
	} else {
		return true;
	}
292 293
}

294
int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
295 296 297
{
	struct kvm_cpu_context *host_ctxt;
	struct kvm_cpu_context *guest_ctxt;
298
	bool fp_enabled;
299 300 301 302 303
	u64 exit_code;

	vcpu = kern_hyp_va(vcpu);

	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
304
	host_ctxt->__hyp_running_vcpu = vcpu;
305 306
	guest_ctxt = &vcpu->arch.ctxt;

307
	__sysreg_save_host_state(host_ctxt);
308 309 310 311 312

	__activate_traps(vcpu);
	__activate_vm(vcpu);

	__vgic_restore_state(vcpu);
313
	__timer_enable_traps(vcpu);
314 315 316

	/*
	 * We must restore the 32-bit state before the sysregs, thanks
317
	 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
318 319
	 */
	__sysreg32_restore_state(vcpu);
320
	__sysreg_restore_guest_state(guest_ctxt);
321
	__debug_switch_to_guest(vcpu);
322 323

	/* Jump in the fire! */
324
again:
325 326 327
	exit_code = __guest_enter(vcpu, host_ctxt);
	/* And we're baaack! */

328 329
	if (ARM_EXCEPTION_CODE(exit_code) != ARM_EXCEPTION_IRQ)
		vcpu->arch.fault.esr_el2 = read_sysreg_el2(esr);
330 331 332 333 334 335
	/*
	 * We're using the raw exception code in order to only process
	 * the trap if no SError is pending. We will come back to the
	 * same PC once the SError has been injected, and replay the
	 * trapping instruction.
	 */
336 337 338
	if (exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
		goto again;

339 340 341 342 343 344 345 346 347 348
	if (static_branch_unlikely(&vgic_v2_cpuif_trap) &&
	    exit_code == ARM_EXCEPTION_TRAP) {
		bool valid;

		valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&
			kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&
			kvm_vcpu_dabt_isvalid(vcpu) &&
			!kvm_vcpu_dabt_isextabt(vcpu) &&
			!kvm_vcpu_dabt_iss1tw(vcpu);

349 350 351 352
		if (valid) {
			int ret = __vgic_v2_perform_cpuif_access(vcpu);

			if (ret == 1) {
353 354 355 356
				if (__skip_instr(vcpu))
					goto again;
				else
					exit_code = ARM_EXCEPTION_TRAP;
357 358 359
			}

			if (ret == -1) {
360 361 362 363 364 365 366 367
				/* Promote an illegal access to an
				 * SError. If we would be returning
				 * due to single-step clear the SS
				 * bit so handle_exit knows what to
				 * do after dealing with the error.
				 */
				if (!__skip_instr(vcpu))
					*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
368 369 370 371
				exit_code = ARM_EXCEPTION_EL1_SERROR;
			}

			/* 0 falls through to be handler out of EL2 */
372 373 374
		}
	}

375 376 377 378 379 380 381
	if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
	    exit_code == ARM_EXCEPTION_TRAP &&
	    (kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 ||
	     kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) {
		int ret = __vgic_v3_perform_cpuif_access(vcpu);

		if (ret == 1) {
382 383 384 385
			if (__skip_instr(vcpu))
				goto again;
			else
				exit_code = ARM_EXCEPTION_TRAP;
386 387 388 389 390
		}

		/* 0 falls through to be handled out of EL2 */
	}

391 392 393 394
	if (cpus_have_const_cap(ARM64_HARDEN_BP_POST_GUEST_EXIT)) {
		u32 midr = read_cpuid_id();

		/* Apply BTAC predictors mitigation to all Falkor chips */
395 396
		if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) ||
		    ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1)) {
397
			__qcom_hyp_sanitize_btac_predictors();
398
		}
399 400
	}

401 402
	fp_enabled = __fpsimd_enabled();

403
	__sysreg_save_guest_state(guest_ctxt);
404
	__sysreg32_save_state(vcpu);
405
	__timer_disable_traps(vcpu);
406 407 408 409 410
	__vgic_save_state(vcpu);

	__deactivate_traps(vcpu);
	__deactivate_vm(vcpu);

411
	__sysreg_restore_host_state(host_ctxt);
412

413 414 415 416 417
	if (fp_enabled) {
		__fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
		__fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
	}

418 419 420 421
	/*
	 * This must come after restoring the host sysregs, since a non-VHE
	 * system may enable SPE here and make use of the TTBRs.
	 */
422
	__debug_switch_to_host(vcpu);
423 424 425

	return exit_code;
}
M
Marc Zyngier 已提交
426 427 428

static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";

429
static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par,
430
					     struct kvm_cpu_context *__host_ctxt)
M
Marc Zyngier 已提交
431
{
432
	struct kvm_vcpu *vcpu;
433
	unsigned long str_va;
434

435 436 437 438 439 440 441 442 443
	vcpu = __host_ctxt->__hyp_running_vcpu;

	if (read_sysreg(vttbr_el2)) {
		__timer_disable_traps(vcpu);
		__deactivate_traps(vcpu);
		__deactivate_vm(vcpu);
		__sysreg_restore_host_state(__host_ctxt);
	}

444 445 446 447 448 449 450 451
	/*
	 * Force the panic string to be loaded from the literal pool,
	 * making sure it is a kernel address and not a PC-relative
	 * reference.
	 */
	asm volatile("ldr %0, =__hyp_panic_string" : "=r" (str_va));

	__hyp_do_panic(str_va,
452 453
		       spsr,  elr,
		       read_sysreg(esr_el2),   read_sysreg_el2(far),
454
		       read_sysreg(hpfar_el2), par, vcpu);
455 456
}

457 458
static void __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par,
				 struct kvm_cpu_context *host_ctxt)
459
{
460 461 462 463 464 465
	struct kvm_vcpu *vcpu;
	vcpu = host_ctxt->__hyp_running_vcpu;

	__deactivate_traps(vcpu);
	__sysreg_restore_host_state(host_ctxt);

466 467 468
	panic(__hyp_panic_string,
	      spsr,  elr,
	      read_sysreg_el2(esr),   read_sysreg_el2(far),
469
	      read_sysreg(hpfar_el2), par, vcpu);
470 471
}

472
void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
473 474 475
{
	u64 spsr = read_sysreg_el2(spsr);
	u64 elr = read_sysreg_el2(elr);
M
Marc Zyngier 已提交
476 477
	u64 par = read_sysreg(par_el1);

478 479 480 481
	if (!has_vhe())
		__hyp_call_panic_nvhe(spsr, elr, par, host_ctxt);
	else
		__hyp_call_panic_vhe(spsr, elr, par, host_ctxt);
M
Marc Zyngier 已提交
482 483 484

	unreachable();
}