mips.c 41.9 KB
Newer Older
1 2 3 4 5 6 7 8 9
/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * KVM/MIPS: MIPS specific KVM APIs
 *
 * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
 * Authors: Sanjay Lal <sanjayl@kymasys.com>
10
 */
11

J
James Hogan 已提交
12
#include <linux/bitops.h>
13 14
#include <linux/errno.h>
#include <linux/err.h>
15
#include <linux/kdebug.h>
16
#include <linux/module.h>
17
#include <linux/uaccess.h>
18
#include <linux/vmalloc.h>
19
#include <linux/sched/signal.h>
20 21
#include <linux/fs.h>
#include <linux/bootmem.h>
22

23
#include <asm/fpu.h>
24 25 26
#include <asm/page.h>
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>
27
#include <asm/pgalloc.h>
28
#include <asm/pgtable.h>
29 30 31

#include <linux/kvm_host.h>

32 33
#include "interrupt.h"
#include "commpage.h"
34 35 36 37 38 39 40 41

#define CREATE_TRACE_POINTS
#include "trace.h"

#ifndef VECTORSPACING
#define VECTORSPACING 0x100	/* for EI/VI mode */
#endif

42
#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x)
43
struct kvm_stats_debugfs_item debugfs_entries[] = {
44 45 46 47 48 49 50 51 52 53 54 55 56
	{ "wait",	  VCPU_STAT(wait_exits),	 KVM_STAT_VCPU },
	{ "cache",	  VCPU_STAT(cache_exits),	 KVM_STAT_VCPU },
	{ "signal",	  VCPU_STAT(signal_exits),	 KVM_STAT_VCPU },
	{ "interrupt",	  VCPU_STAT(int_exits),		 KVM_STAT_VCPU },
	{ "cop_unsuable", VCPU_STAT(cop_unusable_exits), KVM_STAT_VCPU },
	{ "tlbmod",	  VCPU_STAT(tlbmod_exits),	 KVM_STAT_VCPU },
	{ "tlbmiss_ld",	  VCPU_STAT(tlbmiss_ld_exits),	 KVM_STAT_VCPU },
	{ "tlbmiss_st",	  VCPU_STAT(tlbmiss_st_exits),	 KVM_STAT_VCPU },
	{ "addrerr_st",	  VCPU_STAT(addrerr_st_exits),	 KVM_STAT_VCPU },
	{ "addrerr_ld",	  VCPU_STAT(addrerr_ld_exits),	 KVM_STAT_VCPU },
	{ "syscall",	  VCPU_STAT(syscall_exits),	 KVM_STAT_VCPU },
	{ "resvd_inst",	  VCPU_STAT(resvd_inst_exits),	 KVM_STAT_VCPU },
	{ "break_inst",	  VCPU_STAT(break_inst_exits),	 KVM_STAT_VCPU },
57
	{ "trap_inst",	  VCPU_STAT(trap_inst_exits),	 KVM_STAT_VCPU },
58
	{ "msa_fpe",	  VCPU_STAT(msa_fpe_exits),	 KVM_STAT_VCPU },
59
	{ "fpe",	  VCPU_STAT(fpe_exits),		 KVM_STAT_VCPU },
60
	{ "msa_disabled", VCPU_STAT(msa_disabled_exits), KVM_STAT_VCPU },
61
	{ "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU },
62 63 64 65 66 67 68 69 70 71
#ifdef CONFIG_KVM_MIPS_VZ
	{ "vz_gpsi",	  VCPU_STAT(vz_gpsi_exits),	 KVM_STAT_VCPU },
	{ "vz_gsfc",	  VCPU_STAT(vz_gsfc_exits),	 KVM_STAT_VCPU },
	{ "vz_hc",	  VCPU_STAT(vz_hc_exits),	 KVM_STAT_VCPU },
	{ "vz_grr",	  VCPU_STAT(vz_grr_exits),	 KVM_STAT_VCPU },
	{ "vz_gva",	  VCPU_STAT(vz_gva_exits),	 KVM_STAT_VCPU },
	{ "vz_ghfc",	  VCPU_STAT(vz_ghfc_exits),	 KVM_STAT_VCPU },
	{ "vz_gpa",	  VCPU_STAT(vz_gpa_exits),	 KVM_STAT_VCPU },
	{ "vz_resvd",	  VCPU_STAT(vz_resvd_exits),	 KVM_STAT_VCPU },
#endif
72
	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU },
73
	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), KVM_STAT_VCPU },
74
	{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid), KVM_STAT_VCPU },
75
	{ "halt_wakeup",  VCPU_STAT(halt_wakeup),	 KVM_STAT_VCPU },
76 77 78
	{NULL}
};

79 80 81 82 83 84 85 86 87 88 89 90 91
bool kvm_trace_guest_mode_change;

int kvm_guest_mode_change_trace_reg(void)
{
	kvm_trace_guest_mode_change = 1;
	return 0;
}

void kvm_guest_mode_change_trace_unreg(void)
{
	kvm_trace_guest_mode_change = 0;
}

92 93 94
/*
 * XXXKYMA: We are simulatoring a processor that has the WII bit set in
 * Config7, so we are "runnable" if interrupts are pending
95 96 97 98 99 100 101 102 103 104 105
 */
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
	return !!(vcpu->arch.pending_exceptions);
}

int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
{
	return 1;
}

106
int kvm_arch_hardware_enable(void)
107
{
108 109 110 111 112 113
	return kvm_mips_callbacks->hardware_enable();
}

void kvm_arch_hardware_disable(void)
{
	kvm_mips_callbacks->hardware_disable();
114 115 116 117 118 119 120 121 122
}

int kvm_arch_hardware_setup(void)
{
	return 0;
}

void kvm_arch_check_processor_compat(void *rtn)
{
123
	*(int *)rtn = 0;
124 125 126 127
}

int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
{
128
	switch (type) {
J
James Hogan 已提交
129 130 131
#ifdef CONFIG_KVM_MIPS_VZ
	case KVM_VM_MIPS_VZ:
#else
132
	case KVM_VM_MIPS_TE:
J
James Hogan 已提交
133
#endif
134 135 136 137 138 139
		break;
	default:
		/* Unsupported KVM type */
		return -EINVAL;
	};

140 141 142 143 144
	/* Allocate page table to map GPA -> RPA */
	kvm->arch.gpa_mm.pgd = kvm_pgd_alloc();
	if (!kvm->arch.gpa_mm.pgd)
		return -ENOMEM;

145 146 147
	return 0;
}

148 149 150 151 152 153 154 155 156 157
bool kvm_arch_has_vcpu_debugfs(void)
{
	return false;
}

int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
{
	return 0;
}

158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
void kvm_mips_free_vcpus(struct kvm *kvm)
{
	unsigned int i;
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_arch_vcpu_free(vcpu);
	}

	mutex_lock(&kvm->lock);

	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
		kvm->vcpus[i] = NULL;

	atomic_set(&kvm->online_vcpus, 0);

	mutex_unlock(&kvm->lock);
}

177 178 179 180 181 182 183
static void kvm_mips_free_gpa_pt(struct kvm *kvm)
{
	/* It should always be safe to remove after flushing the whole range */
	WARN_ON(!kvm_mips_flush_gpa_pt(kvm, 0, ~0));
	pgd_free(NULL, kvm->arch.gpa_mm.pgd);
}

184 185 186
void kvm_arch_destroy_vm(struct kvm *kvm)
{
	kvm_mips_free_vcpus(kvm);
187
	kvm_mips_free_gpa_pt(kvm);
188 189
}

190 191
long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl,
			unsigned long arg)
192
{
193
	return -ENOIOCTLCMD;
194 195
}

196 197
int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
			    unsigned long npages)
198 199 200 201
{
	return 0;
}

202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
void kvm_arch_flush_shadow_all(struct kvm *kvm)
{
	/* Flush whole GPA */
	kvm_mips_flush_gpa_pt(kvm, 0, ~0);

	/* Let implementation do the rest */
	kvm_mips_callbacks->flush_shadow_all(kvm);
}

void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
	/*
	 * The slot has been made invalid (ready for moving or deletion), so we
	 * need to ensure that it can no longer be accessed by any guest VCPUs.
	 */

	spin_lock(&kvm->mmu_lock);
	/* Flush slot from GPA */
	kvm_mips_flush_gpa_pt(kvm, slot->base_gfn,
			      slot->base_gfn + slot->npages - 1);
	/* Let implementation do the rest */
	kvm_mips_callbacks->flush_shadow_memslot(kvm, slot);
	spin_unlock(&kvm->mmu_lock);
}

228
int kvm_arch_prepare_memory_region(struct kvm *kvm,
229
				   struct kvm_memory_slot *memslot,
230
				   const struct kvm_userspace_memory_region *mem,
231
				   enum kvm_mr_change change)
232 233 234 235 236
{
	return 0;
}

void kvm_arch_commit_memory_region(struct kvm *kvm,
237
				   const struct kvm_userspace_memory_region *mem,
238
				   const struct kvm_memory_slot *old,
239
				   const struct kvm_memory_slot *new,
240
				   enum kvm_mr_change change)
241
{
242 243
	int needs_flush;

244 245 246
	kvm_debug("%s: kvm: %p slot: %d, GPA: %llx, size: %llx, QVA: %llx\n",
		  __func__, kvm, mem->slot, mem->guest_phys_addr,
		  mem->memory_size, mem->userspace_addr);
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268

	/*
	 * If dirty page logging is enabled, write protect all pages in the slot
	 * ready for dirty logging.
	 *
	 * There is no need to do this in any of the following cases:
	 * CREATE:	No dirty mappings will already exist.
	 * MOVE/DELETE:	The old mappings will already have been cleaned up by
	 *		kvm_arch_flush_shadow_memslot()
	 */
	if (change == KVM_MR_FLAGS_ONLY &&
	    (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
	     new->flags & KVM_MEM_LOG_DIRTY_PAGES)) {
		spin_lock(&kvm->mmu_lock);
		/* Write protect GPA page table entries */
		needs_flush = kvm_mips_mkclean_gpa_pt(kvm, new->base_gfn,
					new->base_gfn + new->npages - 1);
		/* Let implementation do the rest */
		if (needs_flush)
			kvm_mips_callbacks->flush_shadow_memslot(kvm, new);
		spin_unlock(&kvm->mmu_lock);
	}
269 270
}

271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
static inline void dump_handler(const char *symbol, void *start, void *end)
{
	u32 *p;

	pr_debug("LEAF(%s)\n", symbol);

	pr_debug("\t.set push\n");
	pr_debug("\t.set noreorder\n");

	for (p = start; p < (u32 *)end; ++p)
		pr_debug("\t.word\t0x%08x\t\t# %p\n", *p, p);

	pr_debug("\t.set\tpop\n");

	pr_debug("\tEND(%s)\n", symbol);
}

288 289
struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
{
290
	int err, size;
291
	void *gebase, *p, *handler, *refill_start, *refill_end;
292 293 294 295 296 297 298 299 300 301 302 303 304 305
	int i;

	struct kvm_vcpu *vcpu = kzalloc(sizeof(struct kvm_vcpu), GFP_KERNEL);

	if (!vcpu) {
		err = -ENOMEM;
		goto out;
	}

	err = kvm_vcpu_init(vcpu, kvm, id);

	if (err)
		goto out_free_cpu;

306
	kvm_debug("kvm @ %p: create cpu %d at %p\n", kvm, id, vcpu);
307

308 309
	/*
	 * Allocate space for host mode exception handlers that handle
310 311
	 * guest mode exits
	 */
312
	if (cpu_has_veic || cpu_has_vint)
313
		size = 0x200 + VECTORSPACING * 64;
314
	else
315
		size = 0x4000;
316 317 318 319 320

	gebase = kzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL);

	if (!gebase) {
		err = -ENOMEM;
321
		goto out_uninit_cpu;
322
	}
323 324
	kvm_debug("Allocated %d bytes for KVM Exception Handlers @ %p\n",
		  ALIGN(size, PAGE_SIZE), gebase);
325

326 327 328 329 330 331 332 333 334 335 336 337
	/*
	 * Check new ebase actually fits in CP0_EBase. The lack of a write gate
	 * limits us to the low 512MB of physical address space. If the memory
	 * we allocate is out of range, just give up now.
	 */
	if (!cpu_has_ebase_wg && virt_to_phys(gebase) >= 0x20000000) {
		kvm_err("CP0_EBase.WG required for guest exception base %pK\n",
			gebase);
		err = -ENOMEM;
		goto out_free_gebase;
	}

338 339 340
	/* Save new ebase */
	vcpu->arch.guest_ebase = gebase;

341
	/* Build guest exception vectors dynamically in unmapped memory */
342
	handler = gebase + 0x2000;
343

344
	/* TLB refill (or XTLB refill on 64-bit VZ where KX=1) */
345
	refill_start = gebase;
346 347
	if (IS_ENABLED(CONFIG_KVM_MIPS_VZ) && IS_ENABLED(CONFIG_64BIT))
		refill_start += 0x080;
348
	refill_end = kvm_mips_build_tlb_refill_exception(refill_start, handler);
349 350

	/* General Exception Entry point */
351
	kvm_mips_build_exception(gebase + 0x180, handler);
352 353 354 355 356

	/* For vectored interrupts poke the exception code @ all offsets 0-7 */
	for (i = 0; i < 8; i++) {
		kvm_debug("L1 Vectored handler @ %p\n",
			  gebase + 0x200 + (i * VECTORSPACING));
357 358
		kvm_mips_build_exception(gebase + 0x200 + i * VECTORSPACING,
					 handler);
359 360
	}

361
	/* General exit handler */
362
	p = handler;
363 364 365 366 367
	p = kvm_mips_build_exit(p);

	/* Guest entry routine */
	vcpu->arch.vcpu_run = p;
	p = kvm_mips_build_vcpu_run(p);
368

369 370 371 372 373
	/* Dump the generated code */
	pr_debug("#include <asm/asm.h>\n");
	pr_debug("#include <asm/regdef.h>\n");
	pr_debug("\n");
	dump_handler("kvm_vcpu_run", vcpu->arch.vcpu_run, p);
374
	dump_handler("kvm_tlb_refill", refill_start, refill_end);
375 376 377
	dump_handler("kvm_gen_exc", gebase + 0x180, gebase + 0x200);
	dump_handler("kvm_exit", gebase + 0x2000, vcpu->arch.vcpu_run);

378
	/* Invalidate the icache for these ranges */
379 380
	flush_icache_range((unsigned long)gebase,
			   (unsigned long)gebase + ALIGN(size, PAGE_SIZE));
381

382 383 384 385
	/*
	 * Allocate comm page for guest kernel, a TLB will be reserved for
	 * mapping GVA @ 0xFFFF8000 to this page
	 */
386 387 388 389 390 391 392
	vcpu->arch.kseg0_commpage = kzalloc(PAGE_SIZE << 1, GFP_KERNEL);

	if (!vcpu->arch.kseg0_commpage) {
		err = -ENOMEM;
		goto out_free_gebase;
	}

393
	kvm_debug("Allocated COMM page @ %p\n", vcpu->arch.kseg0_commpage);
394 395 396 397
	kvm_mips_commpage_init(vcpu);

	/* Init */
	vcpu->arch.last_sched_cpu = -1;
J
James Hogan 已提交
398
	vcpu->arch.last_exec_cpu = -1;
399 400 401 402 403 404

	return vcpu;

out_free_gebase:
	kfree(gebase);

405 406 407
out_uninit_cpu:
	kvm_vcpu_uninit(vcpu);

408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
out_free_cpu:
	kfree(vcpu);

out:
	return ERR_PTR(err);
}

void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
{
	hrtimer_cancel(&vcpu->arch.comparecount_timer);

	kvm_vcpu_uninit(vcpu);

	kvm_mips_dump_stats(vcpu);

423
	kvm_mmu_free_memory_caches(vcpu);
424 425
	kfree(vcpu->arch.guest_ebase);
	kfree(vcpu->arch.kseg0_commpage);
426
	kfree(vcpu);
427 428 429 430 431 432 433
}

void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
{
	kvm_arch_vcpu_free(vcpu);
}

434 435
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg)
436
{
437
	return -ENOIOCTLCMD;
438 439 440 441
}

int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
442
	int r = -EINTR;
443 444 445 446 447 448 449 450 451 452 453
	sigset_t sigsaved;

	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);

	if (vcpu->mmio_needed) {
		if (!vcpu->mmio_is_write)
			kvm_mips_complete_mmio_load(vcpu, run);
		vcpu->mmio_needed = 0;
	}

454 455 456
	if (run->immediate_exit)
		goto out;

457 458
	lose_fpu(1);

459
	local_irq_disable();
460
	guest_enter_irqoff();
461
	trace_kvm_enter(vcpu);
462

463 464 465 466 467 468 469 470
	/*
	 * Make sure the read of VCPU requests in vcpu_run() callback is not
	 * reordered ahead of the write to vcpu->mode, or we could miss a TLB
	 * flush request while the requester sees the VCPU as outside of guest
	 * mode and not needing an IPI.
	 */
	smp_store_mb(vcpu->mode, IN_GUEST_MODE);

471
	r = kvm_mips_callbacks->vcpu_run(run, vcpu);
472

473
	trace_kvm_out(vcpu);
474
	guest_exit_irqoff();
475 476
	local_irq_enable();

477
out:
478 479 480 481 482 483
	if (vcpu->sigset_active)
		sigprocmask(SIG_SETMASK, &sigsaved, NULL);

	return r;
}

484 485
int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
			     struct kvm_mips_interrupt *irq)
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
{
	int intr = (int)irq->irq;
	struct kvm_vcpu *dvcpu = NULL;

	if (intr == 3 || intr == -3 || intr == 4 || intr == -4)
		kvm_debug("%s: CPU: %d, INTR: %d\n", __func__, irq->cpu,
			  (int)intr);

	if (irq->cpu == -1)
		dvcpu = vcpu;
	else
		dvcpu = vcpu->kvm->vcpus[irq->cpu];

	if (intr == 2 || intr == 3 || intr == 4) {
		kvm_mips_callbacks->queue_io_int(dvcpu, irq);

	} else if (intr == -2 || intr == -3 || intr == -4) {
		kvm_mips_callbacks->dequeue_io_int(dvcpu, irq);
	} else {
		kvm_err("%s: invalid interrupt ioctl (%d:%d)\n", __func__,
			irq->cpu, irq->irq);
		return -EINVAL;
	}

	dvcpu->arch.wait = 0;

512 513
	if (swait_active(&dvcpu->wq))
		swake_up(&dvcpu->wq);
514 515 516 517

	return 0;
}

518 519
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
520
{
521
	return -ENOIOCTLCMD;
522 523
}

524 525
int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
526
{
527
	return -ENOIOCTLCMD;
528 529
}

530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563
static u64 kvm_mips_get_one_regs[] = {
	KVM_REG_MIPS_R0,
	KVM_REG_MIPS_R1,
	KVM_REG_MIPS_R2,
	KVM_REG_MIPS_R3,
	KVM_REG_MIPS_R4,
	KVM_REG_MIPS_R5,
	KVM_REG_MIPS_R6,
	KVM_REG_MIPS_R7,
	KVM_REG_MIPS_R8,
	KVM_REG_MIPS_R9,
	KVM_REG_MIPS_R10,
	KVM_REG_MIPS_R11,
	KVM_REG_MIPS_R12,
	KVM_REG_MIPS_R13,
	KVM_REG_MIPS_R14,
	KVM_REG_MIPS_R15,
	KVM_REG_MIPS_R16,
	KVM_REG_MIPS_R17,
	KVM_REG_MIPS_R18,
	KVM_REG_MIPS_R19,
	KVM_REG_MIPS_R20,
	KVM_REG_MIPS_R21,
	KVM_REG_MIPS_R22,
	KVM_REG_MIPS_R23,
	KVM_REG_MIPS_R24,
	KVM_REG_MIPS_R25,
	KVM_REG_MIPS_R26,
	KVM_REG_MIPS_R27,
	KVM_REG_MIPS_R28,
	KVM_REG_MIPS_R29,
	KVM_REG_MIPS_R30,
	KVM_REG_MIPS_R31,

564
#ifndef CONFIG_CPU_MIPSR6
565 566
	KVM_REG_MIPS_HI,
	KVM_REG_MIPS_LO,
567
#endif
568 569 570
	KVM_REG_MIPS_PC,
};

J
James Hogan 已提交
571 572 573 574 575 576 577 578 579 580
static u64 kvm_mips_get_one_regs_fpu[] = {
	KVM_REG_MIPS_FCR_IR,
	KVM_REG_MIPS_FCR_CSR,
};

static u64 kvm_mips_get_one_regs_msa[] = {
	KVM_REG_MIPS_MSA_IR,
	KVM_REG_MIPS_MSA_CSR,
};

581 582 583 584 585
static unsigned long kvm_mips_num_regs(struct kvm_vcpu *vcpu)
{
	unsigned long ret;

	ret = ARRAY_SIZE(kvm_mips_get_one_regs);
J
James Hogan 已提交
586 587 588 589 590 591 592 593
	if (kvm_mips_guest_can_have_fpu(&vcpu->arch)) {
		ret += ARRAY_SIZE(kvm_mips_get_one_regs_fpu) + 48;
		/* odd doubles */
		if (boot_cpu_data.fpu_id & MIPS_FPIR_F64)
			ret += 16;
	}
	if (kvm_mips_guest_can_have_msa(&vcpu->arch))
		ret += ARRAY_SIZE(kvm_mips_get_one_regs_msa) + 32;
594 595 596 597 598 599 600
	ret += kvm_mips_callbacks->num_regs(vcpu);

	return ret;
}

static int kvm_mips_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices)
{
J
James Hogan 已提交
601 602 603
	u64 index;
	unsigned int i;

604 605 606 607 608
	if (copy_to_user(indices, kvm_mips_get_one_regs,
			 sizeof(kvm_mips_get_one_regs)))
		return -EFAULT;
	indices += ARRAY_SIZE(kvm_mips_get_one_regs);

J
James Hogan 已提交
609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
	if (kvm_mips_guest_can_have_fpu(&vcpu->arch)) {
		if (copy_to_user(indices, kvm_mips_get_one_regs_fpu,
				 sizeof(kvm_mips_get_one_regs_fpu)))
			return -EFAULT;
		indices += ARRAY_SIZE(kvm_mips_get_one_regs_fpu);

		for (i = 0; i < 32; ++i) {
			index = KVM_REG_MIPS_FPR_32(i);
			if (copy_to_user(indices, &index, sizeof(index)))
				return -EFAULT;
			++indices;

			/* skip odd doubles if no F64 */
			if (i & 1 && !(boot_cpu_data.fpu_id & MIPS_FPIR_F64))
				continue;

			index = KVM_REG_MIPS_FPR_64(i);
			if (copy_to_user(indices, &index, sizeof(index)))
				return -EFAULT;
			++indices;
		}
	}

	if (kvm_mips_guest_can_have_msa(&vcpu->arch)) {
		if (copy_to_user(indices, kvm_mips_get_one_regs_msa,
				 sizeof(kvm_mips_get_one_regs_msa)))
			return -EFAULT;
		indices += ARRAY_SIZE(kvm_mips_get_one_regs_msa);

		for (i = 0; i < 32; ++i) {
			index = KVM_REG_MIPS_VEC_128(i);
			if (copy_to_user(indices, &index, sizeof(index)))
				return -EFAULT;
			++indices;
		}
	}

646 647 648
	return kvm_mips_callbacks->copy_reg_indices(vcpu, indices);
}

649 650 651 652
static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
			    const struct kvm_one_reg *reg)
{
	struct mips_coproc *cop0 = vcpu->arch.cop0;
J
James Hogan 已提交
653
	struct mips_fpu_struct *fpu = &vcpu->arch.fpu;
654
	int ret;
655
	s64 v;
J
James Hogan 已提交
656
	s64 vs[2];
J
James Hogan 已提交
657
	unsigned int idx;
658 659

	switch (reg->id) {
J
James Hogan 已提交
660
	/* General purpose registers */
661 662 663
	case KVM_REG_MIPS_R0 ... KVM_REG_MIPS_R31:
		v = (long)vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0];
		break;
664
#ifndef CONFIG_CPU_MIPSR6
665 666 667 668 669 670
	case KVM_REG_MIPS_HI:
		v = (long)vcpu->arch.hi;
		break;
	case KVM_REG_MIPS_LO:
		v = (long)vcpu->arch.lo;
		break;
671
#endif
672 673 674 675
	case KVM_REG_MIPS_PC:
		v = (long)vcpu->arch.pc;
		break;

J
James Hogan 已提交
676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706
	/* Floating point registers */
	case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31):
		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
			return -EINVAL;
		idx = reg->id - KVM_REG_MIPS_FPR_32(0);
		/* Odd singles in top of even double when FR=0 */
		if (kvm_read_c0_guest_status(cop0) & ST0_FR)
			v = get_fpr32(&fpu->fpr[idx], 0);
		else
			v = get_fpr32(&fpu->fpr[idx & ~1], idx & 1);
		break;
	case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31):
		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
			return -EINVAL;
		idx = reg->id - KVM_REG_MIPS_FPR_64(0);
		/* Can't access odd doubles in FR=0 mode */
		if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR))
			return -EINVAL;
		v = get_fpr64(&fpu->fpr[idx], 0);
		break;
	case KVM_REG_MIPS_FCR_IR:
		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
			return -EINVAL;
		v = boot_cpu_data.fpu_id;
		break;
	case KVM_REG_MIPS_FCR_CSR:
		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
			return -EINVAL;
		v = fpu->fcr31;
		break;

J
James Hogan 已提交
707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735
	/* MIPS SIMD Architecture (MSA) registers */
	case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31):
		if (!kvm_mips_guest_has_msa(&vcpu->arch))
			return -EINVAL;
		/* Can't access MSA registers in FR=0 mode */
		if (!(kvm_read_c0_guest_status(cop0) & ST0_FR))
			return -EINVAL;
		idx = reg->id - KVM_REG_MIPS_VEC_128(0);
#ifdef CONFIG_CPU_LITTLE_ENDIAN
		/* least significant byte first */
		vs[0] = get_fpr64(&fpu->fpr[idx], 0);
		vs[1] = get_fpr64(&fpu->fpr[idx], 1);
#else
		/* most significant byte first */
		vs[0] = get_fpr64(&fpu->fpr[idx], 1);
		vs[1] = get_fpr64(&fpu->fpr[idx], 0);
#endif
		break;
	case KVM_REG_MIPS_MSA_IR:
		if (!kvm_mips_guest_has_msa(&vcpu->arch))
			return -EINVAL;
		v = boot_cpu_data.msa_id;
		break;
	case KVM_REG_MIPS_MSA_CSR:
		if (!kvm_mips_guest_has_msa(&vcpu->arch))
			return -EINVAL;
		v = fpu->msacsr;
		break;

736
	/* registers to be handled specially */
737
	default:
738 739 740 741
		ret = kvm_mips_callbacks->get_one_reg(vcpu, reg, &v);
		if (ret)
			return ret;
		break;
742
	}
743 744
	if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) {
		u64 __user *uaddr64 = (u64 __user *)(long)reg->addr;
745

746 747 748 749
		return put_user(v, uaddr64);
	} else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) {
		u32 __user *uaddr32 = (u32 __user *)(long)reg->addr;
		u32 v32 = (u32)v;
750

751
		return put_user(v32, uaddr32);
J
James Hogan 已提交
752 753 754
	} else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) {
		void __user *uaddr = (void __user *)(long)reg->addr;

755
		return copy_to_user(uaddr, vs, 16) ? -EFAULT : 0;
756 757 758
	} else {
		return -EINVAL;
	}
759 760 761 762 763 764
}

static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
			    const struct kvm_one_reg *reg)
{
	struct mips_coproc *cop0 = vcpu->arch.cop0;
J
James Hogan 已提交
765 766
	struct mips_fpu_struct *fpu = &vcpu->arch.fpu;
	s64 v;
J
James Hogan 已提交
767
	s64 vs[2];
J
James Hogan 已提交
768
	unsigned int idx;
769

770 771 772 773 774 775 776 777 778 779 780 781
	if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) {
		u64 __user *uaddr64 = (u64 __user *)(long)reg->addr;

		if (get_user(v, uaddr64) != 0)
			return -EFAULT;
	} else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) {
		u32 __user *uaddr32 = (u32 __user *)(long)reg->addr;
		s32 v32;

		if (get_user(v32, uaddr32) != 0)
			return -EFAULT;
		v = (s64)v32;
J
James Hogan 已提交
782 783 784
	} else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) {
		void __user *uaddr = (void __user *)(long)reg->addr;

785
		return copy_from_user(vs, uaddr, 16) ? -EFAULT : 0;
786 787 788
	} else {
		return -EINVAL;
	}
789 790

	switch (reg->id) {
J
James Hogan 已提交
791
	/* General purpose registers */
792 793 794 795 796 797
	case KVM_REG_MIPS_R0:
		/* Silently ignore requests to set $0 */
		break;
	case KVM_REG_MIPS_R1 ... KVM_REG_MIPS_R31:
		vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0] = v;
		break;
798
#ifndef CONFIG_CPU_MIPSR6
799 800 801 802 803 804
	case KVM_REG_MIPS_HI:
		vcpu->arch.hi = v;
		break;
	case KVM_REG_MIPS_LO:
		vcpu->arch.lo = v;
		break;
805
#endif
806 807 808 809
	case KVM_REG_MIPS_PC:
		vcpu->arch.pc = v;
		break;

J
James Hogan 已提交
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
	/* Floating point registers */
	case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31):
		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
			return -EINVAL;
		idx = reg->id - KVM_REG_MIPS_FPR_32(0);
		/* Odd singles in top of even double when FR=0 */
		if (kvm_read_c0_guest_status(cop0) & ST0_FR)
			set_fpr32(&fpu->fpr[idx], 0, v);
		else
			set_fpr32(&fpu->fpr[idx & ~1], idx & 1, v);
		break;
	case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31):
		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
			return -EINVAL;
		idx = reg->id - KVM_REG_MIPS_FPR_64(0);
		/* Can't access odd doubles in FR=0 mode */
		if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR))
			return -EINVAL;
		set_fpr64(&fpu->fpr[idx], 0, v);
		break;
	case KVM_REG_MIPS_FCR_IR:
		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
			return -EINVAL;
		/* Read-only */
		break;
	case KVM_REG_MIPS_FCR_CSR:
		if (!kvm_mips_guest_has_fpu(&vcpu->arch))
			return -EINVAL;
		fpu->fcr31 = v;
		break;

J
James Hogan 已提交
841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866
	/* MIPS SIMD Architecture (MSA) registers */
	case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31):
		if (!kvm_mips_guest_has_msa(&vcpu->arch))
			return -EINVAL;
		idx = reg->id - KVM_REG_MIPS_VEC_128(0);
#ifdef CONFIG_CPU_LITTLE_ENDIAN
		/* least significant byte first */
		set_fpr64(&fpu->fpr[idx], 0, vs[0]);
		set_fpr64(&fpu->fpr[idx], 1, vs[1]);
#else
		/* most significant byte first */
		set_fpr64(&fpu->fpr[idx], 1, vs[0]);
		set_fpr64(&fpu->fpr[idx], 0, vs[1]);
#endif
		break;
	case KVM_REG_MIPS_MSA_IR:
		if (!kvm_mips_guest_has_msa(&vcpu->arch))
			return -EINVAL;
		/* Read-only */
		break;
	case KVM_REG_MIPS_MSA_CSR:
		if (!kvm_mips_guest_has_msa(&vcpu->arch))
			return -EINVAL;
		fpu->msacsr = v;
		break;

867
	/* registers to be handled specially */
868
	default:
869
		return kvm_mips_callbacks->set_one_reg(vcpu, reg, v);
870 871 872 873
	}
	return 0;
}

J
James Hogan 已提交
874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889
static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
				     struct kvm_enable_cap *cap)
{
	int r = 0;

	if (!kvm_vm_ioctl_check_extension(vcpu->kvm, cap->cap))
		return -EINVAL;
	if (cap->flags)
		return -EINVAL;
	if (cap->args[0])
		return -EINVAL;

	switch (cap->cap) {
	case KVM_CAP_MIPS_FPU:
		vcpu->arch.fpu_enabled = true;
		break;
J
James Hogan 已提交
890 891 892
	case KVM_CAP_MIPS_MSA:
		vcpu->arch.msa_enabled = true;
		break;
J
James Hogan 已提交
893 894 895 896 897 898 899 900
	default:
		r = -EINVAL;
		break;
	}

	return r;
}

901 902
long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
			 unsigned long arg)
903 904 905 906 907 908
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {
909 910 911
	case KVM_SET_ONE_REG:
	case KVM_GET_ONE_REG: {
		struct kvm_one_reg reg;
912

913 914 915 916 917 918 919 920 921 922 923 924 925 926 927
		if (copy_from_user(&reg, argp, sizeof(reg)))
			return -EFAULT;
		if (ioctl == KVM_SET_ONE_REG)
			return kvm_mips_set_reg(vcpu, &reg);
		else
			return kvm_mips_get_reg(vcpu, &reg);
	}
	case KVM_GET_REG_LIST: {
		struct kvm_reg_list __user *user_list = argp;
		struct kvm_reg_list reg_list;
		unsigned n;

		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
			return -EFAULT;
		n = reg_list.n;
928
		reg_list.n = kvm_mips_num_regs(vcpu);
929 930 931 932
		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
			return -EFAULT;
		if (n < reg_list.n)
			return -E2BIG;
933
		return kvm_mips_copy_reg_indices(vcpu, user_list->reg);
934
	}
935 936 937
	case KVM_INTERRUPT:
		{
			struct kvm_mips_interrupt irq;
938

939
			if (copy_from_user(&irq, argp, sizeof(irq)))
940
				return -EFAULT;
941 942 943 944 945 946
			kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__,
				  irq.irq);

			r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
			break;
		}
J
James Hogan 已提交
947 948 949 950
	case KVM_ENABLE_CAP: {
		struct kvm_enable_cap cap;

		if (copy_from_user(&cap, argp, sizeof(cap)))
951
			return -EFAULT;
J
James Hogan 已提交
952 953 954
		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
		break;
	}
955
	default:
956
		r = -ENOIOCTLCMD;
957 958 959 960
	}
	return r;
}

961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979
/**
 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
 * @kvm: kvm instance
 * @log: slot id and address to which we copy the log
 *
 * Steps 1-4 below provide general overview of dirty page logging. See
 * kvm_get_dirty_log_protect() function description for additional details.
 *
 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
 * always flush the TLB (step 4) even if previous step failed  and the dirty
 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
 * does not preclude user space subsequent dirty log read. Flushing TLB ensures
 * writes will be marked dirty for next log read.
 *
 *   1. Take a snapshot of the bit and clear it if needed.
 *   2. Write protect the corresponding page.
 *   3. Copy the snapshot to the userspace.
 *   4. Flush TLB's if needed.
 */
980 981
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
{
982
	struct kvm_memslots *slots;
983
	struct kvm_memory_slot *memslot;
984
	bool is_dirty = false;
985 986 987 988
	int r;

	mutex_lock(&kvm->slots_lock);

989
	r = kvm_get_dirty_log_protect(kvm, log, &is_dirty);
990 991

	if (is_dirty) {
992 993
		slots = kvm_memslots(kvm);
		memslot = id_to_memslot(slots, log->slot);
994

995 996
		/* Let implementation handle TLB/GVA invalidation */
		kvm_mips_callbacks->flush_shadow_memslot(kvm, memslot);
997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
	}

	mutex_unlock(&kvm->slots_lock);
	return r;
}

long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
{
	long r;

	switch (ioctl) {
	default:
1009
		r = -ENOIOCTLCMD;
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
	}

	return r;
}

int kvm_arch_init(void *opaque)
{
	if (kvm_mips_callbacks) {
		kvm_err("kvm: module already exists\n");
		return -EEXIST;
	}

1022
	return kvm_mips_emulation_init(&kvm_mips_callbacks);
1023 1024 1025 1026 1027 1028 1029
}

void kvm_arch_exit(void)
{
	kvm_mips_callbacks = NULL;
}

1030 1031
int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
1032
{
1033
	return -ENOIOCTLCMD;
1034 1035
}

1036 1037
int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
1038
{
1039
	return -ENOIOCTLCMD;
1040 1041
}

1042
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1043 1044 1045 1046 1047
{
}

int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
1048
	return -ENOIOCTLCMD;
1049 1050 1051 1052
}

int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
1053
	return -ENOIOCTLCMD;
1054 1055 1056 1057 1058 1059 1060
}

int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
{
	return VM_FAULT_SIGBUS;
}

1061
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
1062 1063 1064 1065
{
	int r;

	switch (ext) {
1066
	case KVM_CAP_ONE_REG:
J
James Hogan 已提交
1067
	case KVM_CAP_ENABLE_CAP:
1068
	case KVM_CAP_READONLY_MEM:
1069
	case KVM_CAP_SYNC_MMU:
1070
	case KVM_CAP_IMMEDIATE_EXIT:
1071 1072
		r = 1;
		break;
1073 1074 1075
	case KVM_CAP_COALESCED_MMIO:
		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
		break;
1076 1077 1078 1079 1080 1081
	case KVM_CAP_NR_VCPUS:
		r = num_online_cpus();
		break;
	case KVM_CAP_MAX_VCPUS:
		r = KVM_MAX_VCPUS;
		break;
J
James Hogan 已提交
1082
	case KVM_CAP_MIPS_FPU:
1083 1084
		/* We don't handle systems with inconsistent cpu_has_fpu */
		r = !!raw_cpu_has_fpu;
J
James Hogan 已提交
1085
		break;
J
James Hogan 已提交
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
	case KVM_CAP_MIPS_MSA:
		/*
		 * We don't support MSA vector partitioning yet:
		 * 1) It would require explicit support which can't be tested
		 *    yet due to lack of support in current hardware.
		 * 2) It extends the state that would need to be saved/restored
		 *    by e.g. QEMU for migration.
		 *
		 * When vector partitioning hardware becomes available, support
		 * could be added by requiring a flag when enabling
		 * KVM_CAP_MIPS_MSA capability to indicate that userland knows
		 * to save/restore the appropriate extra state.
		 */
		r = cpu_has_msa && !(boot_cpu_data.msa_id & MSA_IR_WRPF);
		break;
1101
	default:
1102
		r = kvm_mips_callbacks->check_extension(kvm, ext);
1103 1104 1105 1106 1107 1108 1109
		break;
	}
	return r;
}

int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
{
1110 1111
	return kvm_mips_pending_timer(vcpu) ||
		kvm_read_c0_guest_cause(vcpu->arch.cop0) & C_TI;
1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
}

int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu)
{
	int i;
	struct mips_coproc *cop0;

	if (!vcpu)
		return -1;

1122 1123 1124
	kvm_debug("VCPU Register Dump:\n");
	kvm_debug("\tpc = 0x%08lx\n", vcpu->arch.pc);
	kvm_debug("\texceptions: %08lx\n", vcpu->arch.pending_exceptions);
1125 1126

	for (i = 0; i < 32; i += 4) {
1127
		kvm_debug("\tgpr%02d: %08lx %08lx %08lx %08lx\n", i,
1128 1129 1130 1131
		       vcpu->arch.gprs[i],
		       vcpu->arch.gprs[i + 1],
		       vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]);
	}
1132 1133
	kvm_debug("\thi: 0x%08lx\n", vcpu->arch.hi);
	kvm_debug("\tlo: 0x%08lx\n", vcpu->arch.lo);
1134 1135

	cop0 = vcpu->arch.cop0;
1136
	kvm_debug("\tStatus: 0x%08x, Cause: 0x%08x\n",
1137 1138
		  kvm_read_c0_guest_status(cop0),
		  kvm_read_c0_guest_cause(cop0));
1139

1140
	kvm_debug("\tEPC: 0x%08lx\n", kvm_read_c0_guest_epc(cop0));
1141 1142 1143 1144 1145 1146 1147 1148

	return 0;
}

int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	int i;

1149
	for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
1150
		vcpu->arch.gprs[i] = regs->gpr[i];
1151
	vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */
1152 1153 1154 1155
	vcpu->arch.hi = regs->hi;
	vcpu->arch.lo = regs->lo;
	vcpu->arch.pc = regs->pc;

1156
	return 0;
1157 1158 1159 1160 1161 1162
}

int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	int i;

1163
	for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
1164
		regs->gpr[i] = vcpu->arch.gprs[i];
1165 1166 1167 1168 1169

	regs->hi = vcpu->arch.hi;
	regs->lo = vcpu->arch.lo;
	regs->pc = vcpu->arch.pc;

1170
	return 0;
1171 1172
}

1173
static void kvm_mips_comparecount_func(unsigned long data)
1174 1175 1176 1177 1178 1179
{
	struct kvm_vcpu *vcpu = (struct kvm_vcpu *)data;

	kvm_mips_callbacks->queue_timer_int(vcpu);

	vcpu->arch.wait = 0;
1180 1181
	if (swait_active(&vcpu->wq))
		swake_up(&vcpu->wq);
1182 1183
}

1184
/* low level hrtimer wake routine */
1185
static enum hrtimer_restart kvm_mips_comparecount_wakeup(struct hrtimer *timer)
1186 1187 1188 1189 1190
{
	struct kvm_vcpu *vcpu;

	vcpu = container_of(timer, struct kvm_vcpu, arch.comparecount_timer);
	kvm_mips_comparecount_func((unsigned long) vcpu);
1191
	return kvm_mips_count_timeout(vcpu);
1192 1193 1194 1195
}

int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
1196 1197 1198 1199 1200 1201
	int err;

	err = kvm_mips_callbacks->vcpu_init(vcpu);
	if (err)
		return err;

1202 1203 1204 1205 1206 1207
	hrtimer_init(&vcpu->arch.comparecount_timer, CLOCK_MONOTONIC,
		     HRTIMER_MODE_REL);
	vcpu->arch.comparecount_timer.function = kvm_mips_comparecount_wakeup;
	return 0;
}

J
James Hogan 已提交
1208 1209 1210 1211 1212
void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
{
	kvm_mips_callbacks->vcpu_uninit(vcpu);
}

1213 1214
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				  struct kvm_translation *tr)
1215 1216 1217 1218 1219 1220 1221 1222 1223 1224
{
	return 0;
}

/* Initial guest state */
int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
{
	return kvm_mips_callbacks->vcpu_setup(vcpu);
}

1225
static void kvm_mips_set_c0_status(void)
1226
{
1227
	u32 status = read_c0_status();
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240

	if (cpu_has_dsp)
		status |= (ST0_MX);

	write_c0_status(status);
	ehb();
}

/*
 * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV)
 */
int kvm_mips_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
{
1241 1242 1243
	u32 cause = vcpu->arch.host_cp0_cause;
	u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
	u32 __user *opc = (u32 __user *) vcpu->arch.pc;
1244 1245
	unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
	enum emulation_result er = EMULATE_DONE;
1246
	u32 inst;
1247 1248
	int ret = RESUME_GUEST;

1249 1250
	vcpu->mode = OUTSIDE_GUEST_MODE;

1251
	/* re-enable HTW before enabling interrupts */
1252 1253
	if (!IS_ENABLED(CONFIG_KVM_MIPS_VZ))
		htw_start();
1254

1255 1256 1257 1258
	/* Set a default exit reason */
	run->exit_reason = KVM_EXIT_UNKNOWN;
	run->ready_for_interrupt_injection = 1;

1259 1260 1261 1262
	/*
	 * Set the appropriate status bits based on host CPU features,
	 * before we hit the scheduler
	 */
1263 1264 1265 1266 1267 1268
	kvm_mips_set_c0_status();

	local_irq_enable();

	kvm_debug("kvm_mips_handle_exit: cause: %#x, PC: %p, kvm_run: %p, kvm_vcpu: %p\n",
			cause, opc, run, vcpu);
1269
	trace_kvm_exit(vcpu, exccode);
1270

1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
	if (!IS_ENABLED(CONFIG_KVM_MIPS_VZ)) {
		/*
		 * Do a privilege check, if in UM most of these exit conditions
		 * end up causing an exception to be delivered to the Guest
		 * Kernel
		 */
		er = kvm_mips_check_privilege(cause, opc, run, vcpu);
		if (er == EMULATE_PRIV_FAIL) {
			goto skip_emul;
		} else if (er == EMULATE_FAIL) {
			run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
			ret = RESUME_HOST;
			goto skip_emul;
		}
1285 1286 1287
	}

	switch (exccode) {
1288 1289
	case EXCCODE_INT:
		kvm_debug("[%d]EXCCODE_INT @ %p\n", vcpu->vcpu_id, opc);
1290 1291 1292

		++vcpu->stat.int_exits;

1293
		if (need_resched())
1294 1295 1296 1297 1298
			cond_resched();

		ret = RESUME_GUEST;
		break;

1299 1300
	case EXCCODE_CPU:
		kvm_debug("EXCCODE_CPU: @ PC: %p\n", opc);
1301 1302 1303 1304

		++vcpu->stat.cop_unusable_exits;
		ret = kvm_mips_callbacks->handle_cop_unusable(vcpu);
		/* XXXKYMA: Might need to return to user space */
1305
		if (run->exit_reason == KVM_EXIT_IRQ_WINDOW_OPEN)
1306 1307 1308
			ret = RESUME_HOST;
		break;

1309
	case EXCCODE_MOD:
1310 1311 1312 1313
		++vcpu->stat.tlbmod_exits;
		ret = kvm_mips_callbacks->handle_tlb_mod(vcpu);
		break;

1314
	case EXCCODE_TLBS:
1315
		kvm_debug("TLB ST fault:  cause %#x, status %#x, PC: %p, BadVaddr: %#lx\n",
1316 1317
			  cause, kvm_read_c0_guest_status(vcpu->arch.cop0), opc,
			  badvaddr);
1318 1319 1320 1321 1322

		++vcpu->stat.tlbmiss_st_exits;
		ret = kvm_mips_callbacks->handle_tlb_st_miss(vcpu);
		break;

1323
	case EXCCODE_TLBL:
1324 1325 1326 1327 1328 1329 1330
		kvm_debug("TLB LD fault: cause %#x, PC: %p, BadVaddr: %#lx\n",
			  cause, opc, badvaddr);

		++vcpu->stat.tlbmiss_ld_exits;
		ret = kvm_mips_callbacks->handle_tlb_ld_miss(vcpu);
		break;

1331
	case EXCCODE_ADES:
1332 1333 1334 1335
		++vcpu->stat.addrerr_st_exits;
		ret = kvm_mips_callbacks->handle_addr_err_st(vcpu);
		break;

1336
	case EXCCODE_ADEL:
1337 1338 1339 1340
		++vcpu->stat.addrerr_ld_exits;
		ret = kvm_mips_callbacks->handle_addr_err_ld(vcpu);
		break;

1341
	case EXCCODE_SYS:
1342 1343 1344 1345
		++vcpu->stat.syscall_exits;
		ret = kvm_mips_callbacks->handle_syscall(vcpu);
		break;

1346
	case EXCCODE_RI:
1347 1348 1349 1350
		++vcpu->stat.resvd_inst_exits;
		ret = kvm_mips_callbacks->handle_res_inst(vcpu);
		break;

1351
	case EXCCODE_BP:
1352 1353 1354 1355
		++vcpu->stat.break_inst_exits;
		ret = kvm_mips_callbacks->handle_break(vcpu);
		break;

1356
	case EXCCODE_TR:
1357 1358 1359 1360
		++vcpu->stat.trap_inst_exits;
		ret = kvm_mips_callbacks->handle_trap(vcpu);
		break;

1361
	case EXCCODE_MSAFPE:
1362 1363 1364 1365
		++vcpu->stat.msa_fpe_exits;
		ret = kvm_mips_callbacks->handle_msa_fpe(vcpu);
		break;

1366
	case EXCCODE_FPE:
1367 1368 1369 1370
		++vcpu->stat.fpe_exits;
		ret = kvm_mips_callbacks->handle_fpe(vcpu);
		break;

1371
	case EXCCODE_MSADIS:
1372
		++vcpu->stat.msa_disabled_exits;
1373 1374 1375
		ret = kvm_mips_callbacks->handle_msa_disabled(vcpu);
		break;

1376 1377 1378 1379 1380
	case EXCCODE_GE:
		/* defer exit accounting to handler */
		ret = kvm_mips_callbacks->handle_guest_exit(vcpu);
		break;

1381
	default:
1382 1383 1384
		if (cause & CAUSEF_BD)
			opc += 1;
		inst = 0;
1385
		kvm_get_badinstr(opc, vcpu, &inst);
1386
		kvm_err("Exception Code: %d, not yet handled, @ PC: %p, inst: 0x%08x  BadVaddr: %#lx Status: %#x\n",
1387
			exccode, opc, inst, badvaddr,
1388
			kvm_read_c0_guest_status(vcpu->arch.cop0));
1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
		kvm_arch_vcpu_dump_regs(vcpu);
		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		ret = RESUME_HOST;
		break;

	}

skip_emul:
	local_irq_disable();

1399 1400 1401
	if (ret == RESUME_GUEST)
		kvm_vz_acquire_htimer(vcpu);

1402 1403 1404 1405
	if (er == EMULATE_DONE && !(ret & RESUME_HOST))
		kvm_mips_deliver_interrupts(vcpu, cause);

	if (!(ret & RESUME_HOST)) {
1406
		/* Only check for signals if not already exiting to userspace */
1407 1408 1409 1410
		if (signal_pending(current)) {
			run->exit_reason = KVM_EXIT_INTR;
			ret = (-EINTR << 2) | RESUME_HOST;
			++vcpu->stat.signal_exits;
1411
			trace_kvm_exit(vcpu, KVM_TRACE_EXIT_SIGNAL);
1412 1413 1414
		}
	}

1415
	if (ret == RESUME_GUEST) {
1416 1417
		trace_kvm_reenter(vcpu);

1418 1419 1420 1421 1422 1423 1424 1425
		/*
		 * Make sure the read of VCPU requests in vcpu_reenter()
		 * callback is not reordered ahead of the write to vcpu->mode,
		 * or we could miss a TLB flush request while the requester sees
		 * the VCPU as outside of guest mode and not needing an IPI.
		 */
		smp_store_mb(vcpu->mode, IN_GUEST_MODE);

1426
		kvm_mips_callbacks->vcpu_reenter(run, vcpu);
1427

1428
		/*
1429 1430
		 * If FPU / MSA are enabled (i.e. the guest's FPU / MSA context
		 * is live), restore FCR31 / MSACSR.
1431 1432
		 *
		 * This should be before returning to the guest exception
1433 1434
		 * vector, as it may well cause an [MSA] FP exception if there
		 * are pending exception bits unmasked. (see
1435 1436 1437 1438 1439
		 * kvm_mips_csr_die_notifier() for how that is handled).
		 */
		if (kvm_mips_guest_has_fpu(&vcpu->arch) &&
		    read_c0_status() & ST0_CU1)
			__kvm_restore_fcsr(&vcpu->arch);
1440 1441 1442 1443

		if (kvm_mips_guest_has_msa(&vcpu->arch) &&
		    read_c0_config5() & MIPS_CONF5_MSAEN)
			__kvm_restore_msacsr(&vcpu->arch);
1444 1445
	}

1446
	/* Disable HTW before returning to guest or host */
1447 1448
	if (!IS_ENABLED(CONFIG_KVM_MIPS_VZ))
		htw_stop();
1449

1450 1451 1452
	return ret;
}

1453 1454 1455 1456 1457 1458 1459 1460
/* Enable FPU for guest and restore context */
void kvm_own_fpu(struct kvm_vcpu *vcpu)
{
	struct mips_coproc *cop0 = vcpu->arch.cop0;
	unsigned int sr, cfg5;

	preempt_disable();

1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473
	sr = kvm_read_c0_guest_status(cop0);

	/*
	 * If MSA state is already live, it is undefined how it interacts with
	 * FR=0 FPU state, and we don't want to hit reserved instruction
	 * exceptions trying to save the MSA state later when CU=1 && FR=1, so
	 * play it safe and save it first.
	 *
	 * In theory we shouldn't ever hit this case since kvm_lose_fpu() should
	 * get called when guest CU1 is set, however we can't trust the guest
	 * not to clobber the status register directly via the commpage.
	 */
	if (cpu_has_msa && sr & ST0_CU1 && !(sr & ST0_FR) &&
1474
	    vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1475 1476
		kvm_lose_fpu(vcpu);

1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488
	/*
	 * Enable FPU for guest
	 * We set FR and FRE according to guest context
	 */
	change_c0_status(ST0_CU1 | ST0_FR, sr);
	if (cpu_has_fre) {
		cfg5 = kvm_read_c0_guest_config5(cop0);
		change_c0_config5(MIPS_CONF5_FRE, cfg5);
	}
	enable_fpu_hazard();

	/* If guest FPU state not active, restore it now */
1489
	if (!(vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)) {
1490
		__kvm_restore_fpu(&vcpu->arch);
1491
		vcpu->arch.aux_inuse |= KVM_MIPS_AUX_FPU;
J
James Hogan 已提交
1492 1493 1494
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU);
	} else {
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_FPU);
1495 1496 1497 1498 1499
	}

	preempt_enable();
}

1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
#ifdef CONFIG_CPU_HAS_MSA
/* Enable MSA for guest and restore context */
void kvm_own_msa(struct kvm_vcpu *vcpu)
{
	struct mips_coproc *cop0 = vcpu->arch.cop0;
	unsigned int sr, cfg5;

	preempt_disable();

	/*
	 * Enable FPU if enabled in guest, since we're restoring FPU context
	 * anyway. We set FR and FRE according to guest context.
	 */
	if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
		sr = kvm_read_c0_guest_status(cop0);

		/*
		 * If FR=0 FPU state is already live, it is undefined how it
		 * interacts with MSA state, so play it safe and save it first.
		 */
		if (!(sr & ST0_FR) &&
1521 1522
		    (vcpu->arch.aux_inuse & (KVM_MIPS_AUX_FPU |
				KVM_MIPS_AUX_MSA)) == KVM_MIPS_AUX_FPU)
1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
			kvm_lose_fpu(vcpu);

		change_c0_status(ST0_CU1 | ST0_FR, sr);
		if (sr & ST0_CU1 && cpu_has_fre) {
			cfg5 = kvm_read_c0_guest_config5(cop0);
			change_c0_config5(MIPS_CONF5_FRE, cfg5);
		}
	}

	/* Enable MSA for guest */
	set_c0_config5(MIPS_CONF5_MSAEN);
	enable_fpu_hazard();

1536 1537
	switch (vcpu->arch.aux_inuse & (KVM_MIPS_AUX_FPU | KVM_MIPS_AUX_MSA)) {
	case KVM_MIPS_AUX_FPU:
1538 1539 1540 1541
		/*
		 * Guest FPU state already loaded, only restore upper MSA state
		 */
		__kvm_restore_msa_upper(&vcpu->arch);
1542
		vcpu->arch.aux_inuse |= KVM_MIPS_AUX_MSA;
J
James Hogan 已提交
1543
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_MSA);
1544 1545 1546 1547
		break;
	case 0:
		/* Neither FPU or MSA already active, restore full MSA state */
		__kvm_restore_msa(&vcpu->arch);
1548
		vcpu->arch.aux_inuse |= KVM_MIPS_AUX_MSA;
1549
		if (kvm_mips_guest_has_fpu(&vcpu->arch))
1550
			vcpu->arch.aux_inuse |= KVM_MIPS_AUX_FPU;
J
James Hogan 已提交
1551 1552
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE,
			      KVM_TRACE_AUX_FPU_MSA);
1553 1554
		break;
	default:
J
James Hogan 已提交
1555
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_MSA);
1556 1557 1558 1559 1560 1561 1562 1563
		break;
	}

	preempt_enable();
}
#endif

/* Drop FPU & MSA without saving it */
1564 1565 1566
void kvm_drop_fpu(struct kvm_vcpu *vcpu)
{
	preempt_disable();
1567
	if (cpu_has_msa && vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) {
1568
		disable_msa();
J
James Hogan 已提交
1569
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_DISCARD, KVM_TRACE_AUX_MSA);
1570
		vcpu->arch.aux_inuse &= ~KVM_MIPS_AUX_MSA;
1571
	}
1572
	if (vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) {
1573
		clear_c0_status(ST0_CU1 | ST0_FR);
J
James Hogan 已提交
1574
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_DISCARD, KVM_TRACE_AUX_FPU);
1575
		vcpu->arch.aux_inuse &= ~KVM_MIPS_AUX_FPU;
1576 1577 1578 1579
	}
	preempt_enable();
}

1580
/* Save and disable FPU & MSA */
1581 1582 1583
void kvm_lose_fpu(struct kvm_vcpu *vcpu)
{
	/*
1584 1585 1586 1587
	 * With T&E, FPU & MSA get disabled in root context (hardware) when it
	 * is disabled in guest context (software), but the register state in
	 * the hardware may still be in use.
	 * This is why we explicitly re-enable the hardware before saving.
1588 1589 1590
	 */

	preempt_disable();
1591
	if (cpu_has_msa && vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) {
1592 1593 1594 1595
		if (!IS_ENABLED(CONFIG_KVM_MIPS_VZ)) {
			set_c0_config5(MIPS_CONF5_MSAEN);
			enable_fpu_hazard();
		}
1596 1597

		__kvm_save_msa(&vcpu->arch);
J
James Hogan 已提交
1598
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_MSA);
1599 1600 1601

		/* Disable MSA & FPU */
		disable_msa();
1602
		if (vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) {
1603
			clear_c0_status(ST0_CU1 | ST0_FR);
1604 1605
			disable_fpu_hazard();
		}
1606 1607
		vcpu->arch.aux_inuse &= ~(KVM_MIPS_AUX_FPU | KVM_MIPS_AUX_MSA);
	} else if (vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) {
1608 1609 1610 1611
		if (!IS_ENABLED(CONFIG_KVM_MIPS_VZ)) {
			set_c0_status(ST0_CU1);
			enable_fpu_hazard();
		}
1612 1613

		__kvm_save_fpu(&vcpu->arch);
1614
		vcpu->arch.aux_inuse &= ~KVM_MIPS_AUX_FPU;
J
James Hogan 已提交
1615
		trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU);
1616 1617 1618

		/* Disable FPU */
		clear_c0_status(ST0_CU1 | ST0_FR);
1619
		disable_fpu_hazard();
1620 1621 1622 1623 1624
	}
	preempt_enable();
}

/*
1625 1626 1627
 * Step over a specific ctc1 to FCSR and a specific ctcmsa to MSACSR which are
 * used to restore guest FCSR/MSACSR state and may trigger a "harmless" FP/MSAFP
 * exception if cause bits are set in the value being written.
1628 1629 1630 1631 1632 1633 1634 1635
 */
static int kvm_mips_csr_die_notify(struct notifier_block *self,
				   unsigned long cmd, void *ptr)
{
	struct die_args *args = (struct die_args *)ptr;
	struct pt_regs *regs = args->regs;
	unsigned long pc;

1636 1637
	/* Only interested in FPE and MSAFPE */
	if (cmd != DIE_FP && cmd != DIE_MSAFP)
1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653
		return NOTIFY_DONE;

	/* Return immediately if guest context isn't active */
	if (!(current->flags & PF_VCPU))
		return NOTIFY_DONE;

	/* Should never get here from user mode */
	BUG_ON(user_mode(regs));

	pc = instruction_pointer(regs);
	switch (cmd) {
	case DIE_FP:
		/* match 2nd instruction in __kvm_restore_fcsr */
		if (pc != (unsigned long)&__kvm_restore_fcsr + 4)
			return NOTIFY_DONE;
		break;
1654 1655 1656 1657 1658 1659 1660
	case DIE_MSAFP:
		/* match 2nd/3rd instruction in __kvm_restore_msacsr */
		if (!cpu_has_msa ||
		    pc < (unsigned long)&__kvm_restore_msacsr + 4 ||
		    pc > (unsigned long)&__kvm_restore_msacsr + 8)
			return NOTIFY_DONE;
		break;
1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672
	}

	/* Move PC forward a little and continue executing */
	instruction_pointer(regs) += 4;

	return NOTIFY_STOP;
}

static struct notifier_block kvm_mips_csr_die_notifier = {
	.notifier_call = kvm_mips_csr_die_notify,
};

1673
static int __init kvm_mips_init(void)
1674 1675 1676
{
	int ret;

1677 1678 1679 1680
	ret = kvm_mips_entry_setup();
	if (ret)
		return ret;

1681 1682 1683 1684 1685
	ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);

	if (ret)
		return ret;

1686 1687
	register_die_notifier(&kvm_mips_csr_die_notifier);

1688 1689 1690
	return 0;
}

1691
static void __exit kvm_mips_exit(void)
1692 1693 1694
{
	kvm_exit();

1695
	unregister_die_notifier(&kvm_mips_csr_die_notifier);
1696 1697 1698 1699 1700 1701
}

module_init(kvm_mips_init);
module_exit(kvm_mips_exit);

EXPORT_TRACEPOINT_SYMBOL(kvm_exit);