arm.c 37.6 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6
/*
 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
 */

7
#include <linux/bug.h>
8
#include <linux/cpu_pm.h>
9 10 11
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/kvm_host.h>
12
#include <linux/list.h>
13 14 15 16 17
#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/fs.h>
#include <linux/mman.h>
#include <linux/sched.h>
18
#include <linux/kvm.h>
19 20
#include <linux/kvm_irqfd.h>
#include <linux/irqbypass.h>
21
#include <linux/sched/stat.h>
22
#include <trace/events/kvm.h>
23
#include <kvm/arm_pmu.h>
24
#include <kvm/arm_psci.h>
25 26 27 28

#define CREATE_TRACE_POINTS
#include "trace.h"

29
#include <linux/uaccess.h>
30 31
#include <asm/ptrace.h>
#include <asm/mman.h>
32
#include <asm/tlbflush.h>
33
#include <asm/cacheflush.h>
34
#include <asm/cpufeature.h>
35 36 37 38
#include <asm/virt.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_mmu.h>
39
#include <asm/kvm_emulate.h>
40
#include <asm/kvm_coproc.h>
41
#include <asm/sections.h>
42 43 44 45 46

#ifdef REQUIRES_VIRT
__asm__(".arch_extension	virt");
#endif

47
DEFINE_PER_CPU(kvm_host_data_t, kvm_host_data);
48 49
static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);

50 51 52
/* Per-CPU variable containing the currently running vcpu. */
static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);

53 54
/* The VMID used in the VTTBR */
static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
55
static u32 kvm_next_vmid;
56
static DEFINE_SPINLOCK(kvm_vmid_lock);
57

58 59
static bool vgic_present;

60 61
static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);

62 63
static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
{
64
	__this_cpu_write(kvm_arm_running_vcpu, vcpu);
65 66
}

67 68
DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);

69 70 71 72 73 74
/**
 * kvm_arm_get_running_vcpu - get the vcpu running on the current CPU.
 * Must be called from non-preemptible context
 */
struct kvm_vcpu *kvm_arm_get_running_vcpu(void)
{
75
	return __this_cpu_read(kvm_arm_running_vcpu);
76 77 78 79 80
}

/**
 * kvm_arm_get_running_vcpus - get the per-CPU array of currently running vcpus.
 */
81
struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
82 83 84 85
{
	return &kvm_arm_running_vcpu;
}

86 87 88 89 90 91 92 93 94 95
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
{
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
}

int kvm_arch_hardware_setup(void)
{
	return 0;
}

96
int kvm_arch_check_processor_compat(void)
97
{
98
	return 0;
99 100 101
}


102 103 104 105
/**
 * kvm_arch_init_vm - initializes a VM data structure
 * @kvm:	pointer to the KVM struct
 */
106 107
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
{
108
	int ret, cpu;
109

110
	ret = kvm_arm_setup_stage2(kvm, type);
111 112
	if (ret)
		return ret;
113

114 115 116 117 118 119 120
	kvm->arch.last_vcpu_ran = alloc_percpu(typeof(*kvm->arch.last_vcpu_ran));
	if (!kvm->arch.last_vcpu_ran)
		return -ENOMEM;

	for_each_possible_cpu(cpu)
		*per_cpu_ptr(kvm->arch.last_vcpu_ran, cpu) = -1;

121 122 123 124
	ret = kvm_alloc_stage2_pgd(kvm);
	if (ret)
		goto out_fail_alloc;

125
	ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP);
126 127 128
	if (ret)
		goto out_free_stage2_pgd;

129
	kvm_vgic_early_init(kvm);
130

131
	/* Mark the initial VMID generation invalid */
132
	kvm->arch.vmid.vmid_gen = 0;
133

134
	/* The maximum number of VCPUs is limited by the host's GIC model */
135 136
	kvm->arch.max_vcpus = vgic_present ?
				kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
137

138 139 140 141
	return ret;
out_free_stage2_pgd:
	kvm_free_stage2_pgd(kvm);
out_fail_alloc:
142 143
	free_percpu(kvm->arch.last_vcpu_ran);
	kvm->arch.last_vcpu_ran = NULL;
144
	return ret;
145 146
}

147 148 149 150 151
int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
{
	return 0;
}

152
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
153 154 155 156 157
{
	return VM_FAULT_SIGBUS;
}


158 159 160 161
/**
 * kvm_arch_destroy_vm - destroy the VM data structure
 * @kvm:	pointer to the KVM struct
 */
162 163 164 165
void kvm_arch_destroy_vm(struct kvm *kvm)
{
	int i;

166 167
	kvm_vgic_destroy(kvm);

168 169 170
	free_percpu(kvm->arch.last_vcpu_ran);
	kvm->arch.last_vcpu_ran = NULL;

171 172 173 174 175 176
	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
		if (kvm->vcpus[i]) {
			kvm_arch_vcpu_free(kvm->vcpus[i]);
			kvm->vcpus[i] = NULL;
		}
	}
177
	atomic_set(&kvm->online_vcpus, 0);
178 179
}

180
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
181 182 183
{
	int r;
	switch (ext) {
184
	case KVM_CAP_IRQCHIP:
185 186
		r = vgic_present;
		break;
187
	case KVM_CAP_IOEVENTFD:
188
	case KVM_CAP_DEVICE_CTRL:
189 190 191 192
	case KVM_CAP_USER_MEMORY:
	case KVM_CAP_SYNC_MMU:
	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
	case KVM_CAP_ONE_REG:
193
	case KVM_CAP_ARM_PSCI:
194
	case KVM_CAP_ARM_PSCI_0_2:
195
	case KVM_CAP_READONLY_MEM:
196
	case KVM_CAP_MP_STATE:
197
	case KVM_CAP_IMMEDIATE_EXIT:
198
	case KVM_CAP_VCPU_EVENTS:
199
	case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
200 201
		r = 1;
		break;
202 203
	case KVM_CAP_ARM_SET_DEVICE_ADDR:
		r = 1;
204
		break;
205 206 207 208 209 210
	case KVM_CAP_NR_VCPUS:
		r = num_online_cpus();
		break;
	case KVM_CAP_MAX_VCPUS:
		r = KVM_MAX_VCPUS;
		break;
211 212 213
	case KVM_CAP_MAX_VCPU_ID:
		r = KVM_MAX_VCPU_ID;
		break;
V
Vladimir Murzin 已提交
214 215 216 217 218 219
	case KVM_CAP_MSI_DEVID:
		if (!kvm)
			r = -EINVAL;
		else
			r = kvm->arch.vgic.msis_require_devid;
		break;
220 221 222 223 224 225 226
	case KVM_CAP_ARM_USER_IRQ:
		/*
		 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
		 * (bump this number if adding more devices)
		 */
		r = 1;
		break;
227
	default:
228
		r = kvm_arch_vm_ioctl_check_extension(kvm, ext);
229 230 231 232 233 234 235 236 237 238 239
		break;
	}
	return r;
}

long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg)
{
	return -EINVAL;
}

240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
struct kvm *kvm_arch_alloc_vm(void)
{
	if (!has_vhe())
		return kzalloc(sizeof(struct kvm), GFP_KERNEL);

	return vzalloc(sizeof(struct kvm));
}

void kvm_arch_free_vm(struct kvm *kvm)
{
	if (!has_vhe())
		kfree(kvm);
	else
		vfree(kvm);
}
255 256 257 258 259 260

struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
{
	int err;
	struct kvm_vcpu *vcpu;

261 262 263 264 265
	if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) {
		err = -EBUSY;
		goto out;
	}

266 267 268 269 270
	if (id >= kvm->arch.max_vcpus) {
		err = -EINVAL;
		goto out;
	}

271 272 273 274 275 276 277 278 279 280
	vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
	if (!vcpu) {
		err = -ENOMEM;
		goto out;
	}

	err = kvm_vcpu_init(vcpu, kvm, id);
	if (err)
		goto free_vcpu;

281
	err = create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP);
282 283 284
	if (err)
		goto vcpu_uninit;

285
	return vcpu;
286 287
vcpu_uninit:
	kvm_vcpu_uninit(vcpu);
288 289 290 291 292 293
free_vcpu:
	kmem_cache_free(kvm_vcpu_cache, vcpu);
out:
	return ERR_PTR(err);
}

294
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
295 296 297 298 299
{
}

void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
{
300 301 302
	if (vcpu->arch.has_run_once && unlikely(!irqchip_in_kernel(vcpu->kvm)))
		static_branch_dec(&userspace_irqchip_in_use);

303
	kvm_mmu_free_memory_caches(vcpu);
304
	kvm_timer_vcpu_terminate(vcpu);
305
	kvm_pmu_vcpu_destroy(vcpu);
306
	kvm_vcpu_uninit(vcpu);
307
	kmem_cache_free(kvm_vcpu_cache, vcpu);
308 309 310 311 312 313 314 315 316
}

void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
{
	kvm_arch_vcpu_free(vcpu);
}

int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
{
317
	return kvm_timer_is_pending(vcpu);
318 319
}

320 321
void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
{
322 323 324
	/*
	 * If we're about to block (most likely because we've just hit a
	 * WFI), we need to sync back the state of the GIC CPU interface
325
	 * so that we have the latest PMR and group enables. This ensures
326 327
	 * that kvm_arch_vcpu_runnable has up-to-date data to decide
	 * whether we have pending interrupts.
328 329 330
	 *
	 * For the same reason, we want to tell GICv4 that we need
	 * doorbells to be signalled, should an interrupt become pending.
331 332 333
	 */
	preempt_disable();
	kvm_vgic_vmcr_sync(vcpu);
334
	vgic_v4_put(vcpu, true);
335
	preempt_enable();
336 337 338 339
}

void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
{
340 341 342
	preempt_disable();
	vgic_v4_load(vcpu);
	preempt_enable();
343 344
}

345 346
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
347 348
	/* Force users to call KVM_ARM_VCPU_INIT */
	vcpu->arch.target = -1;
349
	bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
350

351 352 353
	/* Set up the timer */
	kvm_timer_vcpu_init(vcpu);

354 355
	kvm_pmu_vcpu_init(vcpu);

356 357
	kvm_arm_reset_debug_ptr(vcpu);

358
	return kvm_vgic_vcpu_init(vcpu);
359 360 361 362
}

void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
363
	int *last_ran;
364
	kvm_host_data_t *cpu_data;
365 366

	last_ran = this_cpu_ptr(vcpu->kvm->arch.last_vcpu_ran);
367
	cpu_data = this_cpu_ptr(&kvm_host_data);
368 369 370 371 372 373 374 375 376 377

	/*
	 * We might get preempted before the vCPU actually runs, but
	 * over-invalidation doesn't affect correctness.
	 */
	if (*last_ran != vcpu->vcpu_id) {
		kvm_call_hyp(__kvm_tlb_flush_local_vmid, vcpu);
		*last_ran = vcpu->vcpu_id;
	}

378
	vcpu->cpu = cpu;
379
	vcpu->arch.host_cpu_context = &cpu_data->host_ctxt;
380

381
	kvm_arm_set_running_vcpu(vcpu);
382
	kvm_vgic_load(vcpu);
383
	kvm_timer_vcpu_load(vcpu);
384
	kvm_vcpu_load_sysregs(vcpu);
385
	kvm_arch_vcpu_load_fp(vcpu);
386
	kvm_vcpu_pmu_restore_guest(vcpu);
387 388 389 390 391

	if (single_task_running())
		vcpu_clear_wfe_traps(vcpu);
	else
		vcpu_set_wfe_traps(vcpu);
392 393

	vcpu_ptrauth_setup_lazy(vcpu);
394 395 396 397
}

void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
398
	kvm_arch_vcpu_put_fp(vcpu);
399
	kvm_vcpu_put_sysregs(vcpu);
400
	kvm_timer_vcpu_put(vcpu);
401
	kvm_vgic_put(vcpu);
402
	kvm_vcpu_pmu_restore_host(vcpu);
403

404 405
	vcpu->cpu = -1;

406
	kvm_arm_set_running_vcpu(NULL);
407 408
}

A
Andrew Jones 已提交
409 410 411
static void vcpu_power_off(struct kvm_vcpu *vcpu)
{
	vcpu->arch.power_off = true;
412
	kvm_make_request(KVM_REQ_SLEEP, vcpu);
A
Andrew Jones 已提交
413 414 415
	kvm_vcpu_kick(vcpu);
}

416 417 418
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
419
	if (vcpu->arch.power_off)
420 421 422 423 424
		mp_state->mp_state = KVM_MP_STATE_STOPPED;
	else
		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;

	return 0;
425 426 427 428 429
}

int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
430 431
	int ret = 0;

432 433
	switch (mp_state->mp_state) {
	case KVM_MP_STATE_RUNNABLE:
434
		vcpu->arch.power_off = false;
435 436
		break;
	case KVM_MP_STATE_STOPPED:
A
Andrew Jones 已提交
437
		vcpu_power_off(vcpu);
438 439
		break;
	default:
440
		ret = -EINVAL;
441 442
	}

443
	return ret;
444 445
}

446 447 448 449 450 451 452
/**
 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
 * @v:		The VCPU pointer
 *
 * If the guest CPU is not waiting for interrupts or an interrupt line is
 * asserted, the CPU is by definition runnable.
 */
453 454
int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
{
455 456
	bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
	return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
457
		&& !v->arch.power_off && !v->arch.pause);
458 459
}

460 461
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
{
462
	return vcpu_mode_priv(vcpu);
463 464
}

465 466 467 468 469 470 471
/* Just ensure a guest exit from a particular CPU */
static void exit_vm_noop(void *info)
{
}

void force_vm_exit(const cpumask_t *mask)
{
472
	preempt_disable();
473
	smp_call_function_many(mask, exit_vm_noop, NULL, true);
474
	preempt_enable();
475 476 477 478
}

/**
 * need_new_vmid_gen - check that the VMID is still valid
479
 * @vmid: The VMID to check
480 481 482
 *
 * return true if there is a new generation of VMIDs being used
 *
483 484 485 486 487
 * The hardware supports a limited set of values with the value zero reserved
 * for the host, so we check if an assigned value belongs to a previous
 * generation, which which requires us to assign a new value. If we're the
 * first to use a VMID for the new generation, we must flush necessary caches
 * and TLBs on all CPUs.
488
 */
489
static bool need_new_vmid_gen(struct kvm_vmid *vmid)
490
{
491 492
	u64 current_vmid_gen = atomic64_read(&kvm_vmid_gen);
	smp_rmb(); /* Orders read of kvm_vmid_gen and kvm->arch.vmid */
493
	return unlikely(READ_ONCE(vmid->vmid_gen) != current_vmid_gen);
494 495 496
}

/**
497 498 499
 * update_vmid - Update the vmid with a valid VMID for the current generation
 * @kvm: The guest that struct vmid belongs to
 * @vmid: The stage-2 VMID information struct
500
 */
501
static void update_vmid(struct kvm_vmid *vmid)
502
{
503
	if (!need_new_vmid_gen(vmid))
504 505
		return;

506
	spin_lock(&kvm_vmid_lock);
507 508 509 510 511 512

	/*
	 * We need to re-check the vmid_gen here to ensure that if another vcpu
	 * already allocated a valid vmid for this vm, then this vcpu should
	 * use the same vmid.
	 */
513
	if (!need_new_vmid_gen(vmid)) {
514
		spin_unlock(&kvm_vmid_lock);
515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
		return;
	}

	/* First user of a new VMID generation? */
	if (unlikely(kvm_next_vmid == 0)) {
		atomic64_inc(&kvm_vmid_gen);
		kvm_next_vmid = 1;

		/*
		 * On SMP we know no other CPUs can use this CPU's or each
		 * other's VMID after force_vm_exit returns since the
		 * kvm_vmid_lock blocks them from reentry to the guest.
		 */
		force_vm_exit(cpu_all_mask);
		/*
		 * Now broadcast TLB + ICACHE invalidation over the inner
		 * shareable domain to make sure all data structures are
		 * clean.
		 */
		kvm_call_hyp(__kvm_flush_vm_context);
	}

537
	vmid->vmid = kvm_next_vmid;
538
	kvm_next_vmid++;
539
	kvm_next_vmid &= (1 << kvm_get_vmid_bits()) - 1;
540

541
	smp_wmb();
542
	WRITE_ONCE(vmid->vmid_gen, atomic64_read(&kvm_vmid_gen));
543 544

	spin_unlock(&kvm_vmid_lock);
545 546 547 548
}

static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
{
549
	struct kvm *kvm = vcpu->kvm;
550
	int ret = 0;
551

552 553 554
	if (likely(vcpu->arch.has_run_once))
		return 0;

555 556 557
	if (!kvm_arm_vcpu_is_finalized(vcpu))
		return -EPERM;

558
	vcpu->arch.has_run_once = true;
559

560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
	if (likely(irqchip_in_kernel(kvm))) {
		/*
		 * Map the VGIC hardware resources before running a vcpu the
		 * first time on this VM.
		 */
		if (unlikely(!vgic_ready(kvm))) {
			ret = kvm_vgic_map_resources(kvm);
			if (ret)
				return ret;
		}
	} else {
		/*
		 * Tell the rest of the code that there are userspace irqchip
		 * VMs in the wild.
		 */
		static_branch_inc(&userspace_irqchip_in_use);
576 577
	}

578
	ret = kvm_timer_enable(vcpu);
579 580 581 582
	if (ret)
		return ret;

	ret = kvm_arm_pmu_v3_enable(vcpu);
583

584
	return ret;
585 586
}

587 588 589 590 591
bool kvm_arch_intc_initialized(struct kvm *kvm)
{
	return vgic_initialized(kvm);
}

592
void kvm_arm_halt_guest(struct kvm *kvm)
593 594 595 596 597 598
{
	int i;
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm)
		vcpu->arch.pause = true;
599
	kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
600 601
}

602
void kvm_arm_resume_guest(struct kvm *kvm)
603 604 605 606
{
	int i;
	struct kvm_vcpu *vcpu;

607 608
	kvm_for_each_vcpu(i, vcpu, kvm) {
		vcpu->arch.pause = false;
609
		swake_up_one(kvm_arch_vcpu_wq(vcpu));
610
	}
611 612
}

613
static void vcpu_req_sleep(struct kvm_vcpu *vcpu)
614
{
615
	struct swait_queue_head *wq = kvm_arch_vcpu_wq(vcpu);
616

617
	swait_event_interruptible_exclusive(*wq, ((!vcpu->arch.power_off) &&
618
				       (!vcpu->arch.pause)));
619

A
Andrew Jones 已提交
620
	if (vcpu->arch.power_off || vcpu->arch.pause) {
621
		/* Awaken to handle a signal, request we sleep again later. */
622
		kvm_make_request(KVM_REQ_SLEEP, vcpu);
623
	}
624 625 626 627 628 629 630

	/*
	 * Make sure we will observe a potential reset request if we've
	 * observed a change to the power state. Pairs with the smp_wmb() in
	 * kvm_psci_vcpu_on().
	 */
	smp_rmb();
631 632
}

633 634 635 636 637
static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
{
	return vcpu->arch.target >= 0;
}

638 639 640
static void check_vcpu_requests(struct kvm_vcpu *vcpu)
{
	if (kvm_request_pending(vcpu)) {
641 642
		if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
			vcpu_req_sleep(vcpu);
643

644 645 646
		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
			kvm_reset_vcpu(vcpu);

647 648 649 650 651
		/*
		 * Clear IRQ_PENDING requests that were made to guarantee
		 * that a VCPU sees new virtual interrupts.
		 */
		kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
652 653 654
	}
}

655 656 657 658 659 660 661 662 663 664 665
/**
 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
 * @vcpu:	The VCPU pointer
 * @run:	The kvm_run structure pointer used for userspace state exchange
 *
 * This function is called through the VCPU_RUN ioctl called from user space. It
 * will execute VM code in a loop until the time slice for the process is used
 * or some emulation is needed from user space in which case the function will
 * return with return value 0 and with the kvm_run structure filled in with the
 * required data for the requested emulation.
 */
666 667
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
668 669
	int ret;

670
	if (unlikely(!kvm_vcpu_initialized(vcpu)))
671 672 673 674
		return -ENOEXEC;

	ret = kvm_vcpu_first_run_init(vcpu);
	if (ret)
675
		return ret;
676

C
Christoffer Dall 已提交
677 678 679
	if (run->exit_reason == KVM_EXIT_MMIO) {
		ret = kvm_handle_mmio_return(vcpu, vcpu->run);
		if (ret)
680
			return ret;
C
Christoffer Dall 已提交
681 682
	}

683 684 685 686
	if (run->immediate_exit)
		return -EINTR;

	vcpu_load(vcpu);
687

688
	kvm_sigset_activate(vcpu);
689 690 691 692 693 694 695 696 697

	ret = 1;
	run->exit_reason = KVM_EXIT_UNKNOWN;
	while (ret > 0) {
		/*
		 * Check conditions before entering the guest
		 */
		cond_resched();

698
		update_vmid(&vcpu->kvm->arch.vmid);
699

700 701
		check_vcpu_requests(vcpu);

702 703 704 705 706
		/*
		 * Preparing the interrupts to be injected also
		 * involves poking the GIC, which must be done in a
		 * non-preemptible context.
		 */
707
		preempt_disable();
708

709
		kvm_pmu_flush_hwstate(vcpu);
710

711 712
		local_irq_disable();

713 714
		kvm_vgic_flush_hwstate(vcpu);

715
		/*
716 717
		 * Exit if we have a signal pending so that we can deliver the
		 * signal to user space.
718
		 */
719
		if (signal_pending(current)) {
720 721 722 723
			ret = -EINTR;
			run->exit_reason = KVM_EXIT_INTR;
		}

724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
		/*
		 * If we're using a userspace irqchip, then check if we need
		 * to tell a userspace irqchip about timer or PMU level
		 * changes and if so, exit to userspace (the actual level
		 * state gets updated in kvm_timer_update_run and
		 * kvm_pmu_update_run below).
		 */
		if (static_branch_unlikely(&userspace_irqchip_in_use)) {
			if (kvm_timer_should_notify_user(vcpu) ||
			    kvm_pmu_should_notify_user(vcpu)) {
				ret = -EINTR;
				run->exit_reason = KVM_EXIT_INTR;
			}
		}

739 740 741 742
		/*
		 * Ensure we set mode to IN_GUEST_MODE after we disable
		 * interrupts and before the final VCPU requests check.
		 * See the comment in kvm_vcpu_exiting_guest_mode() and
743
		 * Documentation/virt/kvm/vcpu-requests.rst
744 745 746
		 */
		smp_store_mb(vcpu->mode, IN_GUEST_MODE);

747
		if (ret <= 0 || need_new_vmid_gen(&vcpu->kvm->arch.vmid) ||
A
Andrew Jones 已提交
748
		    kvm_request_pending(vcpu)) {
749
			vcpu->mode = OUTSIDE_GUEST_MODE;
750
			isb(); /* Ensure work in x_flush_hwstate is committed */
751
			kvm_pmu_sync_hwstate(vcpu);
752 753
			if (static_branch_unlikely(&userspace_irqchip_in_use))
				kvm_timer_sync_hwstate(vcpu);
754
			kvm_vgic_sync_hwstate(vcpu);
755
			local_irq_enable();
756
			preempt_enable();
757 758 759
			continue;
		}

760 761
		kvm_arm_setup_debug(vcpu);

762 763 764 765
		/**************************************************************
		 * Enter the guest
		 */
		trace_kvm_entry(*vcpu_pc(vcpu));
766
		guest_enter_irqoff();
767

768 769 770
		if (has_vhe()) {
			kvm_arm_vhe_guest_enter();
			ret = kvm_vcpu_run_vhe(vcpu);
771
			kvm_arm_vhe_guest_exit();
772
		} else {
773
			ret = kvm_call_hyp_ret(__kvm_vcpu_run_nvhe, vcpu);
774 775
		}

776
		vcpu->mode = OUTSIDE_GUEST_MODE;
777
		vcpu->stat.exits++;
778 779 780 781
		/*
		 * Back from guest
		 *************************************************************/

782 783
		kvm_arm_clear_debug(vcpu);

784
		/*
785
		 * We must sync the PMU state before the vgic state so
786 787 788 789 790
		 * that the vgic can properly sample the updated state of the
		 * interrupt line.
		 */
		kvm_pmu_sync_hwstate(vcpu);

791 792 793 794 795
		/*
		 * Sync the vgic state before syncing the timer state because
		 * the timer code needs to know if the virtual timer
		 * interrupts are active.
		 */
796 797
		kvm_vgic_sync_hwstate(vcpu);

798 799 800 801 802
		/*
		 * Sync the timer hardware state before enabling interrupts as
		 * we don't want vtimer interrupts to race with syncing the
		 * timer virtual interrupt state.
		 */
803 804
		if (static_branch_unlikely(&userspace_irqchip_in_use))
			kvm_timer_sync_hwstate(vcpu);
805

806 807
		kvm_arch_vcpu_ctxsync_fp(vcpu);

808 809 810 811 812 813 814 815 816 817 818 819 820
		/*
		 * We may have taken a host interrupt in HYP mode (ie
		 * while executing the guest). This interrupt is still
		 * pending, as we haven't serviced it yet!
		 *
		 * We're now back in SVC mode, with interrupts
		 * disabled.  Enabling the interrupts now will have
		 * the effect of taking the interrupt again, in SVC
		 * mode this time.
		 */
		local_irq_enable();

		/*
821
		 * We do local_irq_enable() before calling guest_exit() so
822 823
		 * that if a timer interrupt hits while running the guest we
		 * account that tick as being spent in the guest.  We enable
824
		 * preemption after calling guest_exit() so that if we get
825 826 827
		 * preempted we make sure ticks after that is not counted as
		 * guest time.
		 */
828
		guest_exit();
829
		trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
830

831 832 833
		/* Exit types that need handling before we can be preempted */
		handle_exit_early(vcpu, run, ret);

834 835
		preempt_enable();

836 837 838
		ret = handle_exit(vcpu, run, ret);
	}

839
	/* Tell userspace about in-kernel device output levels */
840 841 842 843
	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
		kvm_timer_update_run(vcpu);
		kvm_pmu_update_run(vcpu);
	}
844

845 846
	kvm_sigset_deactivate(vcpu);

847
	vcpu_put(vcpu);
848
	return ret;
849 850
}

851 852 853 854
static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
{
	int bit_index;
	bool set;
855
	unsigned long *hcr;
856 857 858 859 860 861

	if (number == KVM_ARM_IRQ_CPU_IRQ)
		bit_index = __ffs(HCR_VI);
	else /* KVM_ARM_IRQ_CPU_FIQ */
		bit_index = __ffs(HCR_VF);

862
	hcr = vcpu_hcr(vcpu);
863
	if (level)
864
		set = test_and_set_bit(bit_index, hcr);
865
	else
866
		set = test_and_clear_bit(bit_index, hcr);
867 868 869 870 871 872 873 874 875 876 877 878

	/*
	 * If we didn't change anything, no need to wake up or kick other CPUs
	 */
	if (set == level)
		return 0;

	/*
	 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
	 * trigger a world-switch round on the running physical CPU to set the
	 * virtual IRQ/FIQ fields in the HCR appropriately.
	 */
879
	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
880 881 882 883 884
	kvm_vcpu_kick(vcpu);

	return 0;
}

885 886
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
			  bool line_status)
887 888 889 890 891 892 893 894 895
{
	u32 irq = irq_level->irq;
	unsigned int irq_type, vcpu_idx, irq_num;
	int nrcpus = atomic_read(&kvm->online_vcpus);
	struct kvm_vcpu *vcpu = NULL;
	bool level = irq_level->level;

	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
	vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
896
	vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
897 898 899 900
	irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;

	trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);

901 902 903 904
	switch (irq_type) {
	case KVM_ARM_IRQ_TYPE_CPU:
		if (irqchip_in_kernel(kvm))
			return -ENXIO;
905

906 907
		if (vcpu_idx >= nrcpus)
			return -EINVAL;
908

909 910 911
		vcpu = kvm_get_vcpu(kvm, vcpu_idx);
		if (!vcpu)
			return -EINVAL;
912

913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929
		if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
			return -EINVAL;

		return vcpu_interrupt_line(vcpu, irq_num, level);
	case KVM_ARM_IRQ_TYPE_PPI:
		if (!irqchip_in_kernel(kvm))
			return -ENXIO;

		if (vcpu_idx >= nrcpus)
			return -EINVAL;

		vcpu = kvm_get_vcpu(kvm, vcpu_idx);
		if (!vcpu)
			return -EINVAL;

		if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
			return -EINVAL;
930

931
		return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level, NULL);
932 933 934 935
	case KVM_ARM_IRQ_TYPE_SPI:
		if (!irqchip_in_kernel(kvm))
			return -ENXIO;

936
		if (irq_num < VGIC_NR_PRIVATE_IRQS)
937 938
			return -EINVAL;

939
		return kvm_vgic_inject_irq(kvm, 0, irq_num, level, NULL);
940 941 942
	}

	return -EINVAL;
943 944
}

945 946 947
static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
			       const struct kvm_vcpu_init *init)
{
948
	unsigned int i, ret;
949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982
	int phys_target = kvm_target_cpu();

	if (init->target != phys_target)
		return -EINVAL;

	/*
	 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
	 * use the same target.
	 */
	if (vcpu->arch.target != -1 && vcpu->arch.target != init->target)
		return -EINVAL;

	/* -ENOENT for unknown features, -EINVAL for invalid combinations. */
	for (i = 0; i < sizeof(init->features) * 8; i++) {
		bool set = (init->features[i / 32] & (1 << (i % 32)));

		if (set && i >= KVM_VCPU_MAX_FEATURES)
			return -ENOENT;

		/*
		 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
		 * use the same feature set.
		 */
		if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES &&
		    test_bit(i, vcpu->arch.features) != set)
			return -EINVAL;

		if (set)
			set_bit(i, vcpu->arch.features);
	}

	vcpu->arch.target = phys_target;

	/* Now we know what it is, we can reset it. */
983 984 985 986 987
	ret = kvm_reset_vcpu(vcpu);
	if (ret) {
		vcpu->arch.target = -1;
		bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
	}
988

989 990
	return ret;
}
991

992 993 994 995 996 997 998 999 1000
static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
					 struct kvm_vcpu_init *init)
{
	int ret;

	ret = kvm_vcpu_set_target(vcpu, init);
	if (ret)
		return ret;

1001 1002 1003 1004 1005 1006 1007
	/*
	 * Ensure a rebooted VM will fault in RAM pages and detect if the
	 * guest MMU is turned off and flush the caches as needed.
	 */
	if (vcpu->arch.has_run_once)
		stage2_unmap_vm(vcpu->kvm);

1008 1009
	vcpu_reset_hcr(vcpu);

1010
	/*
1011
	 * Handle the "start in power-off" case.
1012
	 */
1013
	if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))
A
Andrew Jones 已提交
1014
		vcpu_power_off(vcpu);
1015
	else
1016
		vcpu->arch.power_off = false;
1017 1018 1019 1020

	return 0;
}

1021 1022 1023 1024 1025 1026 1027
static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
				 struct kvm_device_attr *attr)
{
	int ret = -ENXIO;

	switch (attr->group) {
	default:
1028
		ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
		break;
	}

	return ret;
}

static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
				 struct kvm_device_attr *attr)
{
	int ret = -ENXIO;

	switch (attr->group) {
	default:
1042
		ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055
		break;
	}

	return ret;
}

static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
				 struct kvm_device_attr *attr)
{
	int ret = -ENXIO;

	switch (attr->group) {
	default:
1056
		ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1057 1058 1059 1060 1061 1062
		break;
	}

	return ret;
}

1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
				   struct kvm_vcpu_events *events)
{
	memset(events, 0, sizeof(*events));

	return __kvm_arm_vcpu_get_events(vcpu, events);
}

static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
				   struct kvm_vcpu_events *events)
{
	int i;

	/* check whether the reserved field is zero */
	for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
		if (events->reserved[i])
			return -EINVAL;

	/* check whether the pad field is zero */
	for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
		if (events->exception.pad[i])
			return -EINVAL;

	return __kvm_arm_vcpu_set_events(vcpu, events);
}

1089 1090 1091 1092 1093
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
1094
	struct kvm_device_attr attr;
1095 1096
	long r;

1097 1098 1099 1100
	switch (ioctl) {
	case KVM_ARM_VCPU_INIT: {
		struct kvm_vcpu_init init;

1101
		r = -EFAULT;
1102
		if (copy_from_user(&init, argp, sizeof(init)))
1103
			break;
1104

1105 1106
		r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
		break;
1107 1108 1109 1110
	}
	case KVM_SET_ONE_REG:
	case KVM_GET_ONE_REG: {
		struct kvm_one_reg reg;
1111

1112
		r = -ENOEXEC;
1113
		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1114
			break;
1115

1116
		r = -EFAULT;
1117
		if (copy_from_user(&reg, argp, sizeof(reg)))
1118 1119
			break;

1120
		if (ioctl == KVM_SET_ONE_REG)
1121
			r = kvm_arm_set_reg(vcpu, &reg);
1122
		else
1123 1124
			r = kvm_arm_get_reg(vcpu, &reg);
		break;
1125 1126 1127 1128 1129 1130
	}
	case KVM_GET_REG_LIST: {
		struct kvm_reg_list __user *user_list = argp;
		struct kvm_reg_list reg_list;
		unsigned n;

1131
		r = -ENOEXEC;
1132
		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1133
			break;
1134

1135 1136 1137 1138
		r = -EPERM;
		if (!kvm_arm_vcpu_is_finalized(vcpu))
			break;

1139
		r = -EFAULT;
1140
		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
1141
			break;
1142 1143 1144
		n = reg_list.n;
		reg_list.n = kvm_arm_num_regs(vcpu);
		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
1145 1146
			break;
		r = -E2BIG;
1147
		if (n < reg_list.n)
1148 1149 1150
			break;
		r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
		break;
1151
	}
1152
	case KVM_SET_DEVICE_ATTR: {
1153
		r = -EFAULT;
1154
		if (copy_from_user(&attr, argp, sizeof(attr)))
1155 1156 1157
			break;
		r = kvm_arm_vcpu_set_attr(vcpu, &attr);
		break;
1158 1159
	}
	case KVM_GET_DEVICE_ATTR: {
1160
		r = -EFAULT;
1161
		if (copy_from_user(&attr, argp, sizeof(attr)))
1162 1163 1164
			break;
		r = kvm_arm_vcpu_get_attr(vcpu, &attr);
		break;
1165 1166
	}
	case KVM_HAS_DEVICE_ATTR: {
1167
		r = -EFAULT;
1168
		if (copy_from_user(&attr, argp, sizeof(attr)))
1169 1170 1171
			break;
		r = kvm_arm_vcpu_has_attr(vcpu, &attr);
		break;
1172
	}
1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		if (kvm_arm_vcpu_get_events(vcpu, &events))
			return -EINVAL;

		if (copy_to_user(argp, &events, sizeof(events)))
			return -EFAULT;

		return 0;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		if (copy_from_user(&events, argp, sizeof(events)))
			return -EFAULT;

		return kvm_arm_vcpu_set_events(vcpu, &events);
	}
1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202
	case KVM_ARM_VCPU_FINALIZE: {
		int what;

		if (!kvm_vcpu_initialized(vcpu))
			return -ENOEXEC;

		if (get_user(what, (const int __user *)argp))
			return -EFAULT;

		return kvm_arm_vcpu_finalize(vcpu, what);
	}
1203
	default:
1204
		r = -EINVAL;
1205
	}
1206 1207

	return r;
1208 1209
}

1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
/**
 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
 * @kvm: kvm instance
 * @log: slot id and address to which we copy the log
 *
 * Steps 1-4 below provide general overview of dirty page logging. See
 * kvm_get_dirty_log_protect() function description for additional details.
 *
 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
 * always flush the TLB (step 4) even if previous step failed  and the dirty
 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
 * does not preclude user space subsequent dirty log read. Flushing TLB ensures
 * writes will be marked dirty for next log read.
 *
 *   1. Take a snapshot of the bit and clear it if needed.
 *   2. Write protect the corresponding page.
 *   3. Copy the snapshot to the userspace.
 *   4. Flush TLB's if needed.
 */
1229 1230
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
{
1231
	bool flush = false;
1232 1233 1234 1235
	int r;

	mutex_lock(&kvm->slots_lock);

1236
	r = kvm_get_dirty_log_protect(kvm, log, &flush);
1237

1238
	if (flush)
1239 1240 1241 1242
		kvm_flush_remote_tlbs(kvm);

	mutex_unlock(&kvm->slots_lock);
	return r;
1243 1244
}

1245 1246 1247
int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log)
{
	bool flush = false;
1248 1249 1250 1251
	int r;

	mutex_lock(&kvm->slots_lock);

1252
	r = kvm_clear_dirty_log_protect(kvm, log, &flush);
1253

1254
	if (flush)
1255 1256 1257 1258
		kvm_flush_remote_tlbs(kvm);

	mutex_unlock(&kvm->slots_lock);
	return r;
1259 1260
}

1261 1262 1263
static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
					struct kvm_arm_device_addr *dev_addr)
{
1264 1265 1266 1267 1268 1269 1270 1271 1272
	unsigned long dev_id, type;

	dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >>
		KVM_ARM_DEVICE_ID_SHIFT;
	type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >>
		KVM_ARM_DEVICE_TYPE_SHIFT;

	switch (dev_id) {
	case KVM_ARM_DEVICE_VGIC_V2:
1273 1274
		if (!vgic_present)
			return -ENXIO;
1275
		return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
1276 1277 1278
	default:
		return -ENODEV;
	}
1279 1280
}

1281 1282 1283
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg)
{
1284 1285 1286 1287
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;

	switch (ioctl) {
1288
	case KVM_CREATE_IRQCHIP: {
1289
		int ret;
1290 1291
		if (!vgic_present)
			return -ENXIO;
1292 1293 1294 1295
		mutex_lock(&kvm->lock);
		ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
		mutex_unlock(&kvm->lock);
		return ret;
1296
	}
1297 1298 1299 1300 1301 1302 1303
	case KVM_ARM_SET_DEVICE_ADDR: {
		struct kvm_arm_device_addr dev_addr;

		if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
			return -EFAULT;
		return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
	}
1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316
	case KVM_ARM_PREFERRED_TARGET: {
		int err;
		struct kvm_vcpu_init init;

		err = kvm_vcpu_preferred_target(&init);
		if (err)
			return err;

		if (copy_to_user(argp, &init, sizeof(init)))
			return -EFAULT;

		return 0;
	}
1317 1318 1319
	default:
		return -EINVAL;
	}
1320 1321
}

1322
static void cpu_init_hyp_mode(void *dummy)
1323
{
1324
	phys_addr_t pgd_ptr;
1325 1326 1327 1328 1329
	unsigned long hyp_stack_ptr;
	unsigned long stack_page;
	unsigned long vector_ptr;

	/* Switch from the HYP stub to our own HYP init vector */
1330
	__hyp_set_vectors(kvm_get_idmap_vector());
1331

1332
	pgd_ptr = kvm_mmu_get_httbr();
1333
	stack_page = __this_cpu_read(kvm_arm_hyp_stack_page);
1334
	hyp_stack_ptr = stack_page + PAGE_SIZE;
1335
	vector_ptr = (unsigned long)kvm_get_hyp_vector();
1336

M
Marc Zyngier 已提交
1337
	__cpu_init_hyp_mode(pgd_ptr, hyp_stack_ptr, vector_ptr);
1338
	__cpu_init_stage2();
1339 1340
}

1341 1342 1343 1344 1345 1346
static void cpu_hyp_reset(void)
{
	if (!is_kernel_in_hyp_mode())
		__hyp_reset_vectors();
}

1347 1348
static void cpu_hyp_reinit(void)
{
1349 1350
	kvm_init_host_cpu_context(&this_cpu_ptr(&kvm_host_data)->host_ctxt);

1351 1352
	cpu_hyp_reset();

1353
	if (is_kernel_in_hyp_mode())
1354
		kvm_timer_init_vhe();
1355
	else
1356
		cpu_init_hyp_mode(NULL);
1357

1358
	kvm_arm_init_debug();
1359 1360 1361

	if (vgic_present)
		kvm_vgic_init_cpu_hardware();
1362 1363
}

1364 1365 1366
static void _kvm_arch_hardware_enable(void *discard)
{
	if (!__this_cpu_read(kvm_arm_hardware_enabled)) {
1367
		cpu_hyp_reinit();
1368
		__this_cpu_write(kvm_arm_hardware_enabled, 1);
1369
	}
1370
}
1371

1372 1373 1374 1375
int kvm_arch_hardware_enable(void)
{
	_kvm_arch_hardware_enable(NULL);
	return 0;
1376 1377
}

1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
static void _kvm_arch_hardware_disable(void *discard)
{
	if (__this_cpu_read(kvm_arm_hardware_enabled)) {
		cpu_hyp_reset();
		__this_cpu_write(kvm_arm_hardware_enabled, 0);
	}
}

void kvm_arch_hardware_disable(void)
{
	_kvm_arch_hardware_disable(NULL);
}
1390

1391 1392 1393 1394 1395
#ifdef CONFIG_CPU_PM
static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
				    unsigned long cmd,
				    void *v)
{
1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410
	/*
	 * kvm_arm_hardware_enabled is left with its old value over
	 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
	 * re-enable hyp.
	 */
	switch (cmd) {
	case CPU_PM_ENTER:
		if (__this_cpu_read(kvm_arm_hardware_enabled))
			/*
			 * don't update kvm_arm_hardware_enabled here
			 * so that the hardware will be re-enabled
			 * when we resume. See below.
			 */
			cpu_hyp_reset();

1411
		return NOTIFY_OK;
1412
	case CPU_PM_ENTER_FAILED:
1413 1414 1415 1416
	case CPU_PM_EXIT:
		if (__this_cpu_read(kvm_arm_hardware_enabled))
			/* The hardware was enabled before suspend. */
			cpu_hyp_reinit();
1417

1418 1419 1420 1421 1422
		return NOTIFY_OK;

	default:
		return NOTIFY_DONE;
	}
1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
}

static struct notifier_block hyp_init_cpu_pm_nb = {
	.notifier_call = hyp_init_cpu_pm_notifier,
};

static void __init hyp_cpu_pm_init(void)
{
	cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
}
1433 1434 1435 1436
static void __init hyp_cpu_pm_exit(void)
{
	cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
}
1437 1438 1439 1440
#else
static inline void hyp_cpu_pm_init(void)
{
}
1441 1442 1443
static inline void hyp_cpu_pm_exit(void)
{
}
1444 1445
#endif

1446 1447
static int init_common_resources(void)
{
1448 1449
	kvm_set_ipa_limit();

1450 1451 1452 1453 1454
	return 0;
}

static int init_subsystems(void)
{
1455
	int err = 0;
1456

1457
	/*
1458
	 * Enable hardware so that subsystem initialisation can access EL2.
1459
	 */
1460
	on_each_cpu(_kvm_arch_hardware_enable, NULL, 1);
1461 1462 1463 1464 1465 1466

	/*
	 * Register CPU lower-power notifier
	 */
	hyp_cpu_pm_init();

1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477
	/*
	 * Init HYP view of VGIC
	 */
	err = kvm_vgic_hyp_init();
	switch (err) {
	case 0:
		vgic_present = true;
		break;
	case -ENODEV:
	case -ENXIO:
		vgic_present = false;
1478
		err = 0;
1479 1480
		break;
	default:
1481
		goto out;
1482 1483 1484 1485 1486
	}

	/*
	 * Init HYP architected timer support
	 */
1487
	err = kvm_timer_hyp_init(vgic_present);
1488
	if (err)
1489
		goto out;
1490 1491 1492 1493

	kvm_perf_init();
	kvm_coproc_table_init();

1494 1495 1496 1497
out:
	on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);

	return err;
1498 1499 1500 1501 1502 1503 1504 1505 1506
}

static void teardown_hyp_mode(void)
{
	int cpu;

	free_hyp_pgds();
	for_each_possible_cpu(cpu)
		free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
1507
	hyp_cpu_pm_exit();
1508 1509
}

1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533
/**
 * Inits Hyp-mode on all online CPUs
 */
static int init_hyp_mode(void)
{
	int cpu;
	int err = 0;

	/*
	 * Allocate Hyp PGD and setup Hyp identity mapping
	 */
	err = kvm_mmu_init();
	if (err)
		goto out_err;

	/*
	 * Allocate stack pages for Hypervisor-mode
	 */
	for_each_possible_cpu(cpu) {
		unsigned long stack_page;

		stack_page = __get_free_page(GFP_KERNEL);
		if (!stack_page) {
			err = -ENOMEM;
1534
			goto out_err;
1535 1536 1537 1538 1539 1540 1541 1542
		}

		per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
	}

	/*
	 * Map the Hyp-code called directly from the host
	 */
1543
	err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
1544
				  kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
1545 1546
	if (err) {
		kvm_err("Cannot map world-switch code\n");
1547
		goto out_err;
1548 1549
	}

1550
	err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
1551
				  kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
1552 1553
	if (err) {
		kvm_err("Cannot map rodata section\n");
M
Marc Zyngier 已提交
1554 1555 1556 1557 1558 1559 1560
		goto out_err;
	}

	err = create_hyp_mappings(kvm_ksym_ref(__bss_start),
				  kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
	if (err) {
		kvm_err("Cannot map bss section\n");
1561
		goto out_err;
1562 1563
	}

1564 1565 1566 1567 1568 1569
	err = kvm_map_vectors();
	if (err) {
		kvm_err("Cannot map vectors\n");
		goto out_err;
	}

1570 1571 1572 1573 1574
	/*
	 * Map the Hyp stack pages
	 */
	for_each_possible_cpu(cpu) {
		char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
1575 1576
		err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE,
					  PAGE_HYP);
1577 1578 1579

		if (err) {
			kvm_err("Cannot map hyp stack\n");
1580
			goto out_err;
1581 1582 1583 1584
		}
	}

	for_each_possible_cpu(cpu) {
1585
		kvm_host_data_t *cpu_data;
1586

1587 1588
		cpu_data = per_cpu_ptr(&kvm_host_data, cpu);
		err = create_hyp_mappings(cpu_data, cpu_data + 1, PAGE_HYP);
1589 1590

		if (err) {
1591
			kvm_err("Cannot map host CPU state: %d\n", err);
1592
			goto out_err;
1593 1594 1595
		}
	}

1596 1597
	err = hyp_map_aux_data();
	if (err)
1598
		kvm_err("Cannot map host auxiliary data: %d\n", err);
1599

1600
	return 0;
1601

1602
out_err:
1603
	teardown_hyp_mode();
1604 1605 1606 1607
	kvm_err("error initializing Hyp mode: %d\n", err);
	return err;
}

1608 1609 1610 1611 1612
static void check_kvm_target_cpu(void *ret)
{
	*(int *)ret = kvm_target_cpu();
}

1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
{
	struct kvm_vcpu *vcpu;
	int i;

	mpidr &= MPIDR_HWID_BITMASK;
	kvm_for_each_vcpu(i, vcpu, kvm) {
		if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
			return vcpu;
	}
	return NULL;
}

1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636
bool kvm_arch_has_irq_bypass(void)
{
	return true;
}

int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

1637 1638
	return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
					  &irqfd->irq_entry);
1639 1640 1641 1642 1643 1644 1645
}
void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

1646 1647
	kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq,
				     &irqfd->irq_entry);
1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665
}

void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
{
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

	kvm_arm_halt_guest(irqfd->kvm);
}

void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
{
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

	kvm_arm_resume_guest(irqfd->kvm);
}

1666 1667 1668
/**
 * Initialize Hyp-mode and memory mappings on all CPUs.
 */
1669 1670
int kvm_arch_init(void *opaque)
{
1671
	int err;
1672
	int ret, cpu;
1673
	bool in_hyp_mode;
1674 1675

	if (!is_hyp_mode_available()) {
1676
		kvm_info("HYP mode not available\n");
1677 1678 1679
		return -ENODEV;
	}

1680 1681 1682 1683
	in_hyp_mode = is_kernel_in_hyp_mode();

	if (!in_hyp_mode && kvm_arch_requires_vhe()) {
		kvm_pr_unimpl("CPU unsupported in non-VHE mode, not initializing\n");
1684 1685 1686
		return -ENODEV;
	}

1687 1688 1689 1690 1691 1692
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1);
		if (ret < 0) {
			kvm_err("Error, CPU %d not supported!\n", cpu);
			return -ENODEV;
		}
1693 1694
	}

1695
	err = init_common_resources();
1696
	if (err)
1697
		return err;
1698

1699
	err = kvm_arm_init_sve();
1700 1701 1702
	if (err)
		return err;

1703
	if (!in_hyp_mode) {
1704
		err = init_hyp_mode();
1705 1706 1707
		if (err)
			goto out_err;
	}
1708

1709 1710 1711
	err = init_subsystems();
	if (err)
		goto out_hyp;
1712

1713 1714 1715 1716 1717
	if (in_hyp_mode)
		kvm_info("VHE mode initialized successfully\n");
	else
		kvm_info("Hyp mode initialized successfully\n");

1718
	return 0;
1719 1720

out_hyp:
1721 1722
	if (!in_hyp_mode)
		teardown_hyp_mode();
1723 1724
out_err:
	return err;
1725 1726 1727 1728 1729
}

/* NOP: Compiling as a module not supported */
void kvm_arch_exit(void)
{
1730
	kvm_perf_teardown();
1731 1732 1733 1734 1735 1736 1737 1738 1739
}

static int arm_init(void)
{
	int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
	return rc;
}

module_init(arm_init);