vgic.c 54.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Copyright (C) 2012 ARM Ltd.
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */

19
#include <linux/cpu.h>
20 21 22 23
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/interrupt.h>
#include <linux/io.h>
24 25 26
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
27
#include <linux/uaccess.h>
28

29
#include <asm/kvm_emulate.h>
30 31
#include <asm/kvm_arm.h>
#include <asm/kvm_mmu.h>
32
#include <trace/events/kvm.h>
33 34
#include <asm/kvm.h>
#include <kvm/iodev.h>
35

36 37 38 39
/*
 * How the whole thing works (courtesy of Christoffer Dall):
 *
 * - At any time, the dist->irq_pending_on_cpu is the oracle that knows if
40 41 42 43 44
 *   something is pending on the CPU interface.
 * - Interrupts that are pending on the distributor are stored on the
 *   vgic.irq_pending vgic bitmap (this bitmap is updated by both user land
 *   ioctls and guest mmio ops, and other in-kernel peripherals such as the
 *   arch. timers).
45 46 47 48
 * - Every time the bitmap changes, the irq_pending_on_cpu oracle is
 *   recalculated
 * - To calculate the oracle, we need info for each cpu from
 *   compute_pending_for_cpu, which considers:
49 50
 *   - PPI: dist->irq_pending & dist->irq_enable
 *   - SPI: dist->irq_pending & dist->irq_enable & dist->irq_spi_target
51
 *   - irq_spi_target is a 'formatted' version of the GICD_ITARGETSRn
52 53 54
 *     registers, stored on each vcpu. We only keep one bit of
 *     information per interrupt, making sure that only one vcpu can
 *     accept the interrupt.
55
 * - If any of the above state changes, we must recalculate the oracle.
56 57 58 59 60 61 62 63 64
 * - The same is true when injecting an interrupt, except that we only
 *   consider a single interrupt at a time. The irq_spi_cpu array
 *   contains the target CPU for each SPI.
 *
 * The handling of level interrupts adds some extra complexity. We
 * need to track when the interrupt has been EOIed, so we can sample
 * the 'line' again. This is achieved as such:
 *
 * - When a level interrupt is moved onto a vcpu, the corresponding
65
 *   bit in irq_queued is set. As long as this bit is set, the line
66 67 68 69
 *   will be ignored for further interrupts. The interrupt is injected
 *   into the vcpu with the GICH_LR_EOI bit set (generate a
 *   maintenance interrupt on EOI).
 * - When the interrupt is EOIed, the maintenance interrupt fires,
70
 *   and clears the corresponding bit in irq_queued. This allows the
71
 *   interrupt line to be sampled again.
72 73 74 75 76
 * - Note that level-triggered interrupts can also be set to pending from
 *   writes to GICD_ISPENDRn and lowering the external input line does not
 *   cause the interrupt to become inactive in such a situation.
 *   Conversely, writes to GICD_ICPENDRn do not cause the interrupt to become
 *   inactive as long as the external input line is held high.
77 78
 */

79
#include "vgic.h"
80

81
static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu);
82 83 84
static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu);
static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr);
static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr, struct vgic_lr lr_desc);
85

86 87
static const struct vgic_ops *vgic_ops;
static const struct vgic_params *vgic;
88

89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
static void add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
{
	vcpu->kvm->arch.vgic.vm_ops.add_sgi_source(vcpu, irq, source);
}

static bool queue_sgi(struct kvm_vcpu *vcpu, int irq)
{
	return vcpu->kvm->arch.vgic.vm_ops.queue_sgi(vcpu, irq);
}

int kvm_vgic_map_resources(struct kvm *kvm)
{
	return kvm->arch.vgic.vm_ops.map_resources(kvm, vgic);
}

104
/*
105 106
 * struct vgic_bitmap contains a bitmap made of unsigned longs, but
 * extracts u32s out of them.
107 108 109 110 111 112 113 114 115 116 117 118 119 120
 *
 * This does not work on 64-bit BE systems, because the bitmap access
 * will store two consecutive 32-bit words with the higher-addressed
 * register's bits at the lower index and the lower-addressed register's
 * bits at the higher index.
 *
 * Therefore, swizzle the register index when accessing the 32-bit word
 * registers to access the right register's value.
 */
#if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 64
#define REG_OFFSET_SWIZZLE	1
#else
#define REG_OFFSET_SWIZZLE	0
#endif
121

122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
static int vgic_init_bitmap(struct vgic_bitmap *b, int nr_cpus, int nr_irqs)
{
	int nr_longs;

	nr_longs = nr_cpus + BITS_TO_LONGS(nr_irqs - VGIC_NR_PRIVATE_IRQS);

	b->private = kzalloc(sizeof(unsigned long) * nr_longs, GFP_KERNEL);
	if (!b->private)
		return -ENOMEM;

	b->shared = b->private + nr_cpus;

	return 0;
}

static void vgic_free_bitmap(struct vgic_bitmap *b)
{
	kfree(b->private);
	b->private = NULL;
	b->shared = NULL;
}

144 145 146 147 148 149 150 151 152 153 154 155 156 157
/*
 * Call this function to convert a u64 value to an unsigned long * bitmask
 * in a way that works on both 32-bit and 64-bit LE and BE platforms.
 *
 * Warning: Calling this function may modify *val.
 */
static unsigned long *u64_to_bitmask(u64 *val)
{
#if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 32
	*val = (*val >> 32) | (*val << 32);
#endif
	return (unsigned long *)val;
}

158
u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, int cpuid, u32 offset)
159 160 161
{
	offset >>= 2;
	if (!offset)
162
		return (u32 *)(x->private + cpuid) + REG_OFFSET_SWIZZLE;
163
	else
164
		return (u32 *)(x->shared) + ((offset - 1) ^ REG_OFFSET_SWIZZLE);
165 166 167 168 169 170
}

static int vgic_bitmap_get_irq_val(struct vgic_bitmap *x,
				   int cpuid, int irq)
{
	if (irq < VGIC_NR_PRIVATE_IRQS)
171
		return test_bit(irq, x->private + cpuid);
172

173
	return test_bit(irq - VGIC_NR_PRIVATE_IRQS, x->shared);
174 175
}

176 177
void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
			     int irq, int val)
178 179 180 181
{
	unsigned long *reg;

	if (irq < VGIC_NR_PRIVATE_IRQS) {
182
		reg = x->private + cpuid;
183
	} else {
184
		reg = x->shared;
185 186 187 188 189 190 191 192 193 194 195
		irq -= VGIC_NR_PRIVATE_IRQS;
	}

	if (val)
		set_bit(irq, reg);
	else
		clear_bit(irq, reg);
}

static unsigned long *vgic_bitmap_get_cpu_map(struct vgic_bitmap *x, int cpuid)
{
196
	return x->private + cpuid;
197 198
}

199
unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x)
200
{
201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
	return x->shared;
}

static int vgic_init_bytemap(struct vgic_bytemap *x, int nr_cpus, int nr_irqs)
{
	int size;

	size  = nr_cpus * VGIC_NR_PRIVATE_IRQS;
	size += nr_irqs - VGIC_NR_PRIVATE_IRQS;

	x->private = kzalloc(size, GFP_KERNEL);
	if (!x->private)
		return -ENOMEM;

	x->shared = x->private + nr_cpus * VGIC_NR_PRIVATE_IRQS / sizeof(u32);
	return 0;
}

static void vgic_free_bytemap(struct vgic_bytemap *b)
{
	kfree(b->private);
	b->private = NULL;
	b->shared = NULL;
224 225
}

226
u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset)
227
{
228 229 230 231 232 233 234 235 236 237 238
	u32 *reg;

	if (offset < VGIC_NR_PRIVATE_IRQS) {
		reg = x->private;
		offset += cpuid * VGIC_NR_PRIVATE_IRQS;
	} else {
		reg = x->shared;
		offset -= VGIC_NR_PRIVATE_IRQS;
	}

	return reg + (offset / sizeof(u32));
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
}

#define VGIC_CFG_LEVEL	0
#define VGIC_CFG_EDGE	1

static bool vgic_irq_is_edge(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
	int irq_val;

	irq_val = vgic_bitmap_get_irq_val(&dist->irq_cfg, vcpu->vcpu_id, irq);
	return irq_val == VGIC_CFG_EDGE;
}

static int vgic_irq_is_enabled(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	return vgic_bitmap_get_irq_val(&dist->irq_enabled, vcpu->vcpu_id, irq);
}

260
static int vgic_irq_is_queued(struct kvm_vcpu *vcpu, int irq)
261 262 263
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

264
	return vgic_bitmap_get_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq);
265 266
}

267 268 269 270 271 272 273
static int vgic_irq_is_active(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	return vgic_bitmap_get_irq_val(&dist->irq_active, vcpu->vcpu_id, irq);
}

274
static void vgic_irq_set_queued(struct kvm_vcpu *vcpu, int irq)
275 276 277
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

278
	vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 1);
279 280
}

281
static void vgic_irq_clear_queued(struct kvm_vcpu *vcpu, int irq)
282 283 284
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

285
	vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 0);
286 287
}

288 289 290 291 292 293 294 295 296 297 298 299 300 301
static void vgic_irq_set_active(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 1);
}

static void vgic_irq_clear_active(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 0);
}

302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
static int vgic_dist_irq_get_level(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	return vgic_bitmap_get_irq_val(&dist->irq_level, vcpu->vcpu_id, irq);
}

static void vgic_dist_irq_set_level(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 1);
}

static void vgic_dist_irq_clear_level(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 0);
}

static int vgic_dist_irq_soft_pend(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	return vgic_bitmap_get_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq);
}

static void vgic_dist_irq_clear_soft_pend(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	vgic_bitmap_set_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq, 0);
}

337 338 339 340
static int vgic_dist_irq_is_pending(struct kvm_vcpu *vcpu, int irq)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

341
	return vgic_bitmap_get_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq);
342 343
}

344
void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq)
345 346 347
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

348
	vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 1);
349 350
}

351
void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq)
352 353 354
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

355
	vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 0);
356 357 358 359 360 361 362 363 364 365 366
}

static void vgic_cpu_irq_set(struct kvm_vcpu *vcpu, int irq)
{
	if (irq < VGIC_NR_PRIVATE_IRQS)
		set_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
	else
		set_bit(irq - VGIC_NR_PRIVATE_IRQS,
			vcpu->arch.vgic_cpu.pending_shared);
}

367
void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq)
368 369 370 371 372 373 374 375
{
	if (irq < VGIC_NR_PRIVATE_IRQS)
		clear_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
	else
		clear_bit(irq - VGIC_NR_PRIVATE_IRQS,
			  vcpu->arch.vgic_cpu.pending_shared);
}

376 377 378 379 380
static bool vgic_can_sample_irq(struct kvm_vcpu *vcpu, int irq)
{
	return vgic_irq_is_edge(vcpu, irq) || !vgic_irq_is_queued(vcpu, irq);
}

381 382 383 384 385 386 387 388 389 390 391
/**
 * vgic_reg_access - access vgic register
 * @mmio:   pointer to the data describing the mmio access
 * @reg:    pointer to the virtual backing of vgic distributor data
 * @offset: least significant 2 bits used for word offset
 * @mode:   ACCESS_ mode (see defines above)
 *
 * Helper to make vgic register access easier using one of the access
 * modes defined for vgic register access
 * (read,raz,write-ignored,setbit,clearbit,write)
 */
392 393
void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
		     phys_addr_t offset, int mode)
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441
{
	int word_offset = (offset & 3) * 8;
	u32 mask = (1UL << (mmio->len * 8)) - 1;
	u32 regval;

	/*
	 * Any alignment fault should have been delivered to the guest
	 * directly (ARM ARM B3.12.7 "Prioritization of aborts").
	 */

	if (reg) {
		regval = *reg;
	} else {
		BUG_ON(mode != (ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED));
		regval = 0;
	}

	if (mmio->is_write) {
		u32 data = mmio_data_read(mmio, mask) << word_offset;
		switch (ACCESS_WRITE_MASK(mode)) {
		case ACCESS_WRITE_IGNORED:
			return;

		case ACCESS_WRITE_SETBIT:
			regval |= data;
			break;

		case ACCESS_WRITE_CLEARBIT:
			regval &= ~data;
			break;

		case ACCESS_WRITE_VALUE:
			regval = (regval & ~(mask << word_offset)) | data;
			break;
		}
		*reg = regval;
	} else {
		switch (ACCESS_READ_MASK(mode)) {
		case ACCESS_READ_RAZ:
			regval = 0;
			/* fall through */

		case ACCESS_READ_VALUE:
			mmio_data_write(mmio, mask, regval >> word_offset);
		}
	}
}

442 443
bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
			phys_addr_t offset)
444 445 446 447 448 449
{
	vgic_reg_access(mmio, NULL, offset,
			ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
	return false;
}

450 451
bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
			    phys_addr_t offset, int vcpu_id, int access)
452
{
453 454 455 456 457 458
	u32 *reg;
	int mode = ACCESS_READ_VALUE | access;
	struct kvm_vcpu *target_vcpu = kvm_get_vcpu(kvm, vcpu_id);

	reg = vgic_bitmap_get_reg(&kvm->arch.vgic.irq_enabled, vcpu_id, offset);
	vgic_reg_access(mmio, reg, offset, mode);
459
	if (mmio->is_write) {
460 461 462 463 464 465
		if (access & ACCESS_WRITE_CLEARBIT) {
			if (offset < 4) /* Force SGI enabled */
				*reg |= 0xffff;
			vgic_retire_disabled_irqs(target_vcpu);
		}
		vgic_update_state(kvm);
466 467 468 469 470 471
		return true;
	}

	return false;
}

472 473 474
bool vgic_handle_set_pending_reg(struct kvm *kvm,
				 struct kvm_exit_mmio *mmio,
				 phys_addr_t offset, int vcpu_id)
475
{
476
	u32 *reg, orig;
477
	u32 level_mask;
478 479
	int mode = ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT;
	struct vgic_dist *dist = &kvm->arch.vgic;
480

481
	reg = vgic_bitmap_get_reg(&dist->irq_cfg, vcpu_id, offset);
482 483 484
	level_mask = (~(*reg));

	/* Mark both level and edge triggered irqs as pending */
485
	reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
486
	orig = *reg;
487
	vgic_reg_access(mmio, reg, offset, mode);
488

489
	if (mmio->is_write) {
490 491
		/* Set the soft-pending flag only for level-triggered irqs */
		reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
492 493
					  vcpu_id, offset);
		vgic_reg_access(mmio, reg, offset, mode);
494 495
		*reg &= level_mask;

496 497 498 499 500 501
		/* Ignore writes to SGIs */
		if (offset < 2) {
			*reg &= ~0xffff;
			*reg |= orig & 0xffff;
		}

502
		vgic_update_state(kvm);
503 504 505 506 507 508
		return true;
	}

	return false;
}

509 510 511
bool vgic_handle_clear_pending_reg(struct kvm *kvm,
				   struct kvm_exit_mmio *mmio,
				   phys_addr_t offset, int vcpu_id)
512
{
513
	u32 *level_active;
514
	u32 *reg, orig;
515 516
	int mode = ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT;
	struct vgic_dist *dist = &kvm->arch.vgic;
517

518
	reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
519
	orig = *reg;
520
	vgic_reg_access(mmio, reg, offset, mode);
521
	if (mmio->is_write) {
522 523
		/* Re-set level triggered level-active interrupts */
		level_active = vgic_bitmap_get_reg(&dist->irq_level,
524 525
					  vcpu_id, offset);
		reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
526 527
		*reg |= *level_active;

528 529 530 531 532 533
		/* Ignore writes to SGIs */
		if (offset < 2) {
			*reg &= ~0xffff;
			*reg |= orig & 0xffff;
		}

534 535
		/* Clear soft-pending flags */
		reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
536 537
					  vcpu_id, offset);
		vgic_reg_access(mmio, reg, offset, mode);
538

539
		vgic_update_state(kvm);
540 541 542 543 544
		return true;
	}
	return false;
}

545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
bool vgic_handle_set_active_reg(struct kvm *kvm,
				struct kvm_exit_mmio *mmio,
				phys_addr_t offset, int vcpu_id)
{
	u32 *reg;
	struct vgic_dist *dist = &kvm->arch.vgic;

	reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
	vgic_reg_access(mmio, reg, offset,
			ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);

	if (mmio->is_write) {
		vgic_update_state(kvm);
		return true;
	}

	return false;
}

bool vgic_handle_clear_active_reg(struct kvm *kvm,
				  struct kvm_exit_mmio *mmio,
				  phys_addr_t offset, int vcpu_id)
{
	u32 *reg;
	struct vgic_dist *dist = &kvm->arch.vgic;

	reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
	vgic_reg_access(mmio, reg, offset,
			ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);

	if (mmio->is_write) {
		vgic_update_state(kvm);
		return true;
	}

	return false;
}

583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
static u32 vgic_cfg_expand(u16 val)
{
	u32 res = 0;
	int i;

	/*
	 * Turn a 16bit value like abcd...mnop into a 32bit word
	 * a0b0c0d0...m0n0o0p0, which is what the HW cfg register is.
	 */
	for (i = 0; i < 16; i++)
		res |= ((val >> i) & VGIC_CFG_EDGE) << (2 * i + 1);

	return res;
}

static u16 vgic_cfg_compress(u32 val)
{
	u16 res = 0;
	int i;

	/*
	 * Turn a 32bit word a0b0c0d0...m0n0o0p0 into 16bit value like
	 * abcd...mnop which is what we really care about.
	 */
	for (i = 0; i < 16; i++)
		res |= ((val >> (i * 2 + 1)) & VGIC_CFG_EDGE) << i;

	return res;
}

/*
 * The distributor uses 2 bits per IRQ for the CFG register, but the
 * LSB is always 0. As such, we only keep the upper bit, and use the
 * two above functions to compress/expand the bits
 */
618 619
bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
			 phys_addr_t offset)
620 621
{
	u32 val;
622

623
	if (offset & 4)
624 625 626 627 628 629 630 631
		val = *reg >> 16;
	else
		val = *reg & 0xffff;

	val = vgic_cfg_expand(val);
	vgic_reg_access(mmio, &val, offset,
			ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
	if (mmio->is_write) {
632
		if (offset < 8) {
633 634 635 636 637
			*reg = ~0U; /* Force PPIs/SGIs to 1 */
			return false;
		}

		val = vgic_cfg_compress(val);
638
		if (offset & 4) {
639 640 641 642 643 644 645 646 647 648 649
			*reg &= 0xffff;
			*reg |= val << 16;
		} else {
			*reg &= 0xffff << 16;
			*reg |= val;
		}
	}

	return false;
}

650
/**
651
 * vgic_unqueue_irqs - move pending/active IRQs from LRs to the distributor
652 653
 * @vgic_cpu: Pointer to the vgic_cpu struct holding the LRs
 *
654
 * Move any IRQs that have already been assigned to LRs back to the
655 656 657
 * emulated distributor state so that the complete emulated state can be read
 * from the main emulation structures without investigating the LRs.
 */
658
void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
659 660
{
	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
661
	int i;
662 663

	for_each_set_bit(i, vgic_cpu->lr_used, vgic_cpu->nr_lr) {
664
		struct vgic_lr lr = vgic_get_lr(vcpu, i);
665 666 667 668 669 670 671 672

		/*
		 * There are three options for the state bits:
		 *
		 * 01: pending
		 * 10: active
		 * 11: pending and active
		 */
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
		BUG_ON(!(lr.state & LR_STATE_MASK));

		/* Reestablish SGI source for pending and active IRQs */
		if (lr.irq < VGIC_NR_SGIS)
			add_sgi_source(vcpu, lr.irq, lr.source);

		/*
		 * If the LR holds an active (10) or a pending and active (11)
		 * interrupt then move the active state to the
		 * distributor tracking bit.
		 */
		if (lr.state & LR_STATE_ACTIVE) {
			vgic_irq_set_active(vcpu, lr.irq);
			lr.state &= ~LR_STATE_ACTIVE;
		}
688 689 690 691 692 693 694

		/*
		 * Reestablish the pending state on the distributor and the
		 * CPU interface.  It may have already been pending, but that
		 * is fine, then we are only setting a few bits that were
		 * already set.
		 */
695 696 697 698 699
		if (lr.state & LR_STATE_PENDING) {
			vgic_dist_irq_set_pending(vcpu, lr.irq);
			lr.state &= ~LR_STATE_PENDING;
		}

700
		vgic_set_lr(vcpu, i, lr);
701 702

		/*
703
		 * Mark the LR as free for other use.
704
		 */
705 706 707
		BUG_ON(lr.state & LR_STATE_MASK);
		vgic_retire_lr(i, lr.irq, vcpu);
		vgic_irq_clear_queued(vcpu, lr.irq);
708 709 710 711 712 713

		/* Finally update the VGIC state. */
		vgic_update_state(vcpu->kvm);
	}
}

714
const
715
struct vgic_io_range *vgic_find_range(const struct vgic_io_range *ranges,
716
				      int len, gpa_t offset)
717
{
718 719 720 721 722
	while (ranges->len) {
		if (offset >= ranges->base &&
		    (offset + len) <= (ranges->base + ranges->len))
			return ranges;
		ranges++;
723 724 725 726 727
	}

	return NULL;
}

728
static bool vgic_validate_access(const struct vgic_dist *dist,
729
				 const struct vgic_io_range *range,
730 731 732 733 734 735 736 737 738 739 740 741 742 743
				 unsigned long offset)
{
	int irq;

	if (!range->bits_per_irq)
		return true;	/* Not an irq-based access */

	irq = offset * 8 / range->bits_per_irq;
	if (irq >= dist->nr_irqs)
		return false;

	return true;
}

744 745 746 747 748 749 750 751 752 753 754 755 756
/*
 * Call the respective handler function for the given range.
 * We split up any 64 bit accesses into two consecutive 32 bit
 * handler calls and merge the result afterwards.
 * We do this in a little endian fashion regardless of the host's
 * or guest's endianness, because the GIC is always LE and the rest of
 * the code (vgic_reg_access) also puts it in a LE fashion already.
 * At this point we have already identified the handle function, so
 * range points to that one entry and offset is relative to this.
 */
static bool call_range_handler(struct kvm_vcpu *vcpu,
			       struct kvm_exit_mmio *mmio,
			       unsigned long offset,
757
			       const struct vgic_io_range *range)
758 759 760 761 762 763 764 765 766 767 768 769 770 771
{
	struct kvm_exit_mmio mmio32;
	bool ret;

	if (likely(mmio->len <= 4))
		return range->handle_mmio(vcpu, mmio, offset);

	/*
	 * Any access bigger than 4 bytes (that we currently handle in KVM)
	 * is actually 8 bytes long, caused by a 64-bit access
	 */

	mmio32.len = 4;
	mmio32.is_write = mmio->is_write;
772
	mmio32.private = mmio->private;
773 774

	mmio32.phys_addr = mmio->phys_addr + 4;
775
	mmio32.data = &((u32 *)mmio->data)[1];
776 777 778
	ret = range->handle_mmio(vcpu, &mmio32, offset + 4);

	mmio32.phys_addr = mmio->phys_addr;
779
	mmio32.data = &((u32 *)mmio->data)[0];
780 781 782 783 784
	ret |= range->handle_mmio(vcpu, &mmio32, offset);

	return ret;
}

785
/**
786 787
 * vgic_handle_mmio_access - handle an in-kernel MMIO access
 * This is called by the read/write KVM IO device wrappers below.
788
 * @vcpu:	pointer to the vcpu performing the access
789 790 791 792 793
 * @this:	pointer to the KVM IO device in charge
 * @addr:	guest physical address of the access
 * @len:	size of the access
 * @val:	pointer to the data region
 * @is_write:	read or write access
794
 *
795
 * returns true if the MMIO access could be performed
796
 */
797 798 799
static int vgic_handle_mmio_access(struct kvm_vcpu *vcpu,
				   struct kvm_io_device *this, gpa_t addr,
				   int len, void *val, bool is_write)
800
{
801
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
802 803 804 805 806
	struct vgic_io_device *iodev = container_of(this,
						    struct vgic_io_device, dev);
	struct kvm_run *run = vcpu->run;
	const struct vgic_io_range *range;
	struct kvm_exit_mmio mmio;
807
	bool updated_state;
808
	gpa_t offset;
809

810 811
	offset = addr - iodev->addr;
	range = vgic_find_range(iodev->reg_ranges, len, offset);
812
	if (unlikely(!range || !range->handle_mmio)) {
813 814
		pr_warn("Unhandled access %d %08llx %d\n", is_write, addr, len);
		return -ENXIO;
815 816
	}

817 818 819
	mmio.phys_addr = addr;
	mmio.len = len;
	mmio.is_write = is_write;
820
	mmio.data = val;
821 822 823
	mmio.private = iodev->redist_vcpu;

	spin_lock(&dist->lock);
824
	offset -= range->base;
825
	if (vgic_validate_access(dist, range, offset)) {
826
		updated_state = call_range_handler(vcpu, &mmio, offset, range);
827
	} else {
828 829
		if (!is_write)
			memset(val, 0, len);
830 831
		updated_state = false;
	}
832
	spin_unlock(&dist->lock);
833 834 835 836 837
	run->mmio.is_write	= is_write;
	run->mmio.len		= len;
	run->mmio.phys_addr	= addr;
	memcpy(run->mmio.data, val, len);

838 839
	kvm_handle_mmio_return(vcpu, run);

840 841 842
	if (updated_state)
		vgic_kick_vcpus(vcpu->kvm);

843 844 845 846 847 848 849 850
	return 0;
}

static int vgic_handle_mmio_read(struct kvm_vcpu *vcpu,
				 struct kvm_io_device *this,
				 gpa_t addr, int len, void *val)
{
	return vgic_handle_mmio_access(vcpu, this, addr, len, val, false);
851 852
}

853 854 855 856 857 858 859 860 861 862 863 864 865
static int vgic_handle_mmio_write(struct kvm_vcpu *vcpu,
				  struct kvm_io_device *this,
				  gpa_t addr, int len, const void *val)
{
	return vgic_handle_mmio_access(vcpu, this, addr, len, (void *)val,
				       true);
}

struct kvm_io_device_ops vgic_io_ops = {
	.read	= vgic_handle_mmio_read,
	.write	= vgic_handle_mmio_write,
};

866
/**
867 868 869 870 871 872 873
 * vgic_register_kvm_io_dev - register VGIC register frame on the KVM I/O bus
 * @kvm:            The VM structure pointer
 * @base:           The (guest) base address for the register frame
 * @len:            Length of the register frame window
 * @ranges:         Describing the handler functions for each register
 * @redist_vcpu_id: The VCPU ID to pass on to the handlers on call
 * @iodev:          Points to memory to be passed on to the handler
874
 *
875 876 877 878 879 880
 * @iodev stores the parameters of this function to be usable by the handler
 * respectively the dispatcher function (since the KVM I/O bus framework lacks
 * an opaque parameter). Initialization is done in this function, but the
 * reference should be valid and unique for the whole VGIC lifetime.
 * If the register frame is not mapped for a specific VCPU, pass -1 to
 * @redist_vcpu_id.
881
 */
882 883 884 885
int vgic_register_kvm_io_dev(struct kvm *kvm, gpa_t base, int len,
			     const struct vgic_io_range *ranges,
			     int redist_vcpu_id,
			     struct vgic_io_device *iodev)
886
{
887 888
	struct kvm_vcpu *vcpu = NULL;
	int ret;
889

890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910
	if (redist_vcpu_id >= 0)
		vcpu = kvm_get_vcpu(kvm, redist_vcpu_id);

	iodev->addr		= base;
	iodev->len		= len;
	iodev->reg_ranges	= ranges;
	iodev->redist_vcpu	= vcpu;

	kvm_iodevice_init(&iodev->dev, &vgic_io_ops);

	mutex_lock(&kvm->slots_lock);

	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, base, len,
				      &iodev->dev);
	mutex_unlock(&kvm->slots_lock);

	/* Mark the iodev as invalid if registration fails. */
	if (ret)
		iodev->dev.ops = NULL;

	return ret;
911 912
}

913 914 915 916 917
static int vgic_nr_shared_irqs(struct vgic_dist *dist)
{
	return dist->nr_irqs - VGIC_NR_PRIVATE_IRQS;
}

918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947
static int compute_active_for_cpu(struct kvm_vcpu *vcpu)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
	unsigned long *active, *enabled, *act_percpu, *act_shared;
	unsigned long active_private, active_shared;
	int nr_shared = vgic_nr_shared_irqs(dist);
	int vcpu_id;

	vcpu_id = vcpu->vcpu_id;
	act_percpu = vcpu->arch.vgic_cpu.active_percpu;
	act_shared = vcpu->arch.vgic_cpu.active_shared;

	active = vgic_bitmap_get_cpu_map(&dist->irq_active, vcpu_id);
	enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
	bitmap_and(act_percpu, active, enabled, VGIC_NR_PRIVATE_IRQS);

	active = vgic_bitmap_get_shared_map(&dist->irq_active);
	enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
	bitmap_and(act_shared, active, enabled, nr_shared);
	bitmap_and(act_shared, act_shared,
		   vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
		   nr_shared);

	active_private = find_first_bit(act_percpu, VGIC_NR_PRIVATE_IRQS);
	active_shared = find_first_bit(act_shared, nr_shared);

	return (active_private < VGIC_NR_PRIVATE_IRQS ||
		active_shared < nr_shared);
}

948 949
static int compute_pending_for_cpu(struct kvm_vcpu *vcpu)
{
950 951 952
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
	unsigned long *pending, *enabled, *pend_percpu, *pend_shared;
	unsigned long pending_private, pending_shared;
953
	int nr_shared = vgic_nr_shared_irqs(dist);
954 955 956 957 958 959
	int vcpu_id;

	vcpu_id = vcpu->vcpu_id;
	pend_percpu = vcpu->arch.vgic_cpu.pending_percpu;
	pend_shared = vcpu->arch.vgic_cpu.pending_shared;

960
	pending = vgic_bitmap_get_cpu_map(&dist->irq_pending, vcpu_id);
961 962 963
	enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
	bitmap_and(pend_percpu, pending, enabled, VGIC_NR_PRIVATE_IRQS);

964
	pending = vgic_bitmap_get_shared_map(&dist->irq_pending);
965
	enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
966
	bitmap_and(pend_shared, pending, enabled, nr_shared);
967 968
	bitmap_and(pend_shared, pend_shared,
		   vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
969
		   nr_shared);
970 971

	pending_private = find_first_bit(pend_percpu, VGIC_NR_PRIVATE_IRQS);
972
	pending_shared = find_first_bit(pend_shared, nr_shared);
973
	return (pending_private < VGIC_NR_PRIVATE_IRQS ||
974
		pending_shared < vgic_nr_shared_irqs(dist));
975 976 977 978
}

/*
 * Update the interrupt state and determine which CPUs have pending
979
 * or active interrupts. Must be called with distributor lock held.
980
 */
981
void vgic_update_state(struct kvm *kvm)
982 983 984 985 986 987
{
	struct vgic_dist *dist = &kvm->arch.vgic;
	struct kvm_vcpu *vcpu;
	int c;

	if (!dist->enabled) {
988
		set_bit(0, dist->irq_pending_on_cpu);
989 990 991 992
		return;
	}

	kvm_for_each_vcpu(c, vcpu, kvm) {
993
		if (compute_pending_for_cpu(vcpu))
994
			set_bit(c, dist->irq_pending_on_cpu);
995 996 997 998 999

		if (compute_active_for_cpu(vcpu))
			set_bit(c, dist->irq_active_on_cpu);
		else
			clear_bit(c, dist->irq_active_on_cpu);
1000
	}
1001
}
1002

1003 1004
static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr)
{
1005
	return vgic_ops->get_lr(vcpu, lr);
1006 1007 1008 1009 1010
}

static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr,
			       struct vgic_lr vlr)
{
1011
	vgic_ops->set_lr(vcpu, lr, vlr);
1012 1013
}

1014 1015 1016
static void vgic_sync_lr_elrsr(struct kvm_vcpu *vcpu, int lr,
			       struct vgic_lr vlr)
{
1017
	vgic_ops->sync_lr_elrsr(vcpu, lr, vlr);
1018 1019 1020 1021
}

static inline u64 vgic_get_elrsr(struct kvm_vcpu *vcpu)
{
1022
	return vgic_ops->get_elrsr(vcpu);
1023 1024
}

1025 1026
static inline u64 vgic_get_eisr(struct kvm_vcpu *vcpu)
{
1027
	return vgic_ops->get_eisr(vcpu);
1028 1029
}

1030 1031 1032 1033 1034
static inline void vgic_clear_eisr(struct kvm_vcpu *vcpu)
{
	vgic_ops->clear_eisr(vcpu);
}

1035 1036
static inline u32 vgic_get_interrupt_status(struct kvm_vcpu *vcpu)
{
1037
	return vgic_ops->get_interrupt_status(vcpu);
1038 1039
}

1040 1041
static inline void vgic_enable_underflow(struct kvm_vcpu *vcpu)
{
1042
	vgic_ops->enable_underflow(vcpu);
1043 1044 1045 1046
}

static inline void vgic_disable_underflow(struct kvm_vcpu *vcpu)
{
1047
	vgic_ops->disable_underflow(vcpu);
1048 1049
}

1050
void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
1051
{
1052
	vgic_ops->get_vmcr(vcpu, vmcr);
1053 1054
}

1055
void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
1056
{
1057
	vgic_ops->set_vmcr(vcpu, vmcr);
1058 1059
}

1060 1061
static inline void vgic_enable(struct kvm_vcpu *vcpu)
{
1062
	vgic_ops->enable(vcpu);
1063 1064
}

1065 1066 1067 1068 1069 1070 1071 1072 1073
static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu)
{
	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
	struct vgic_lr vlr = vgic_get_lr(vcpu, lr_nr);

	vlr.state = 0;
	vgic_set_lr(vcpu, lr_nr, vlr);
	clear_bit(lr_nr, vgic_cpu->lr_used);
	vgic_cpu->vgic_irq_lr_map[irq] = LR_EMPTY;
1074
	vgic_sync_lr_elrsr(vcpu, lr_nr, vlr);
1075
}
1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090

/*
 * An interrupt may have been disabled after being made pending on the
 * CPU interface (the classic case is a timer running while we're
 * rebooting the guest - the interrupt would kick as soon as the CPU
 * interface gets enabled, with deadly consequences).
 *
 * The solution is to examine already active LRs, and check the
 * interrupt is still enabled. If not, just retire it.
 */
static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu)
{
	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
	int lr;

1091
	for_each_set_bit(lr, vgic_cpu->lr_used, vgic->nr_lr) {
1092
		struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
1093

1094 1095
		if (!vgic_irq_is_enabled(vcpu, vlr.irq)) {
			vgic_retire_lr(lr, vlr.irq, vcpu);
1096 1097
			if (vgic_irq_is_queued(vcpu, vlr.irq))
				vgic_irq_clear_queued(vcpu, vlr.irq);
1098 1099 1100 1101
		}
	}
}

1102 1103 1104
static void vgic_queue_irq_to_lr(struct kvm_vcpu *vcpu, int irq,
				 int lr_nr, struct vgic_lr vlr)
{
1105 1106 1107 1108 1109 1110
	if (vgic_irq_is_active(vcpu, irq)) {
		vlr.state |= LR_STATE_ACTIVE;
		kvm_debug("Set active, clear distributor: 0x%x\n", vlr.state);
		vgic_irq_clear_active(vcpu, irq);
		vgic_update_state(vcpu->kvm);
	} else if (vgic_dist_irq_is_pending(vcpu, irq)) {
1111 1112 1113 1114 1115 1116 1117 1118
		vlr.state |= LR_STATE_PENDING;
		kvm_debug("Set pending: 0x%x\n", vlr.state);
	}

	if (!vgic_irq_is_edge(vcpu, irq))
		vlr.state |= LR_EOI_INT;

	vgic_set_lr(vcpu, lr_nr, vlr);
1119
	vgic_sync_lr_elrsr(vcpu, lr_nr, vlr);
1120 1121
}

1122 1123 1124
/*
 * Queue an interrupt to a CPU virtual interface. Return true on success,
 * or false if it wasn't possible to queue it.
1125
 * sgi_source must be zero for any non-SGI interrupts.
1126
 */
1127
bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
1128 1129
{
	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1130
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1131
	struct vgic_lr vlr;
1132 1133 1134 1135 1136
	int lr;

	/* Sanitize the input... */
	BUG_ON(sgi_source_id & ~7);
	BUG_ON(sgi_source_id && irq >= VGIC_NR_SGIS);
1137
	BUG_ON(irq >= dist->nr_irqs);
1138 1139 1140 1141 1142 1143

	kvm_debug("Queue IRQ%d\n", irq);

	lr = vgic_cpu->vgic_irq_lr_map[irq];

	/* Do we have an active interrupt for the same CPUID? */
1144 1145 1146 1147 1148
	if (lr != LR_EMPTY) {
		vlr = vgic_get_lr(vcpu, lr);
		if (vlr.source == sgi_source_id) {
			kvm_debug("LR%d piggyback for IRQ%d\n", lr, vlr.irq);
			BUG_ON(!test_bit(lr, vgic_cpu->lr_used));
1149
			vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
1150 1151
			return true;
		}
1152 1153 1154 1155
	}

	/* Try to use another LR for this interrupt */
	lr = find_first_zero_bit((unsigned long *)vgic_cpu->lr_used,
1156 1157
			       vgic->nr_lr);
	if (lr >= vgic->nr_lr)
1158 1159 1160 1161 1162 1163
		return false;

	kvm_debug("LR%d allocated for IRQ%d %x\n", lr, irq, sgi_source_id);
	vgic_cpu->vgic_irq_lr_map[irq] = lr;
	set_bit(lr, vgic_cpu->lr_used);

1164 1165
	vlr.irq = irq;
	vlr.source = sgi_source_id;
1166 1167
	vlr.state = 0;
	vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
1168 1169 1170 1171 1172 1173

	return true;
}

static bool vgic_queue_hwirq(struct kvm_vcpu *vcpu, int irq)
{
1174
	if (!vgic_can_sample_irq(vcpu, irq))
1175 1176 1177 1178
		return true; /* level interrupt, already queued */

	if (vgic_queue_irq(vcpu, 0, irq)) {
		if (vgic_irq_is_edge(vcpu, irq)) {
1179
			vgic_dist_irq_clear_pending(vcpu, irq);
1180 1181
			vgic_cpu_irq_clear(vcpu, irq);
		} else {
1182
			vgic_irq_set_queued(vcpu, irq);
1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
		}

		return true;
	}

	return false;
}

/*
 * Fill the list registers with pending interrupts before running the
 * guest.
 */
static void __kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
{
	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1199
	unsigned long *pa_percpu, *pa_shared;
1200 1201
	int i, vcpu_id;
	int overflow = 0;
1202
	int nr_shared = vgic_nr_shared_irqs(dist);
1203 1204 1205

	vcpu_id = vcpu->vcpu_id;

1206 1207 1208 1209 1210 1211 1212
	pa_percpu = vcpu->arch.vgic_cpu.pend_act_percpu;
	pa_shared = vcpu->arch.vgic_cpu.pend_act_shared;

	bitmap_or(pa_percpu, vgic_cpu->pending_percpu, vgic_cpu->active_percpu,
		  VGIC_NR_PRIVATE_IRQS);
	bitmap_or(pa_shared, vgic_cpu->pending_shared, vgic_cpu->active_shared,
		  nr_shared);
1213 1214 1215 1216 1217
	/*
	 * We may not have any pending interrupt, or the interrupts
	 * may have been serviced from another vcpu. In all cases,
	 * move along.
	 */
1218
	if (!kvm_vgic_vcpu_pending_irq(vcpu) && !kvm_vgic_vcpu_active_irq(vcpu))
1219 1220 1221
		goto epilog;

	/* SGIs */
1222
	for_each_set_bit(i, pa_percpu, VGIC_NR_SGIS) {
1223
		if (!queue_sgi(vcpu, i))
1224 1225 1226 1227
			overflow = 1;
	}

	/* PPIs */
1228
	for_each_set_bit_from(i, pa_percpu, VGIC_NR_PRIVATE_IRQS) {
1229 1230 1231 1232 1233
		if (!vgic_queue_hwirq(vcpu, i))
			overflow = 1;
	}

	/* SPIs */
1234
	for_each_set_bit(i, pa_shared, nr_shared) {
1235 1236 1237 1238
		if (!vgic_queue_hwirq(vcpu, i + VGIC_NR_PRIVATE_IRQS))
			overflow = 1;
	}

1239 1240 1241



1242 1243
epilog:
	if (overflow) {
1244
		vgic_enable_underflow(vcpu);
1245
	} else {
1246
		vgic_disable_underflow(vcpu);
1247 1248 1249 1250 1251 1252
		/*
		 * We're about to run this VCPU, and we've consumed
		 * everything the distributor had in store for
		 * us. Claim we don't have anything pending. We'll
		 * adjust that if needed while exiting.
		 */
1253
		clear_bit(vcpu_id, dist->irq_pending_on_cpu);
1254 1255 1256 1257 1258
	}
}

static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
{
1259
	u32 status = vgic_get_interrupt_status(vcpu);
1260
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1261
	bool level_pending = false;
1262
	struct kvm *kvm = vcpu->kvm;
1263

1264
	kvm_debug("STATUS = %08x\n", status);
1265

1266
	if (status & INT_STATUS_EOI) {
1267 1268 1269 1270
		/*
		 * Some level interrupts have been EOIed. Clear their
		 * active bit.
		 */
1271
		u64 eisr = vgic_get_eisr(vcpu);
1272
		unsigned long *eisr_ptr = u64_to_bitmask(&eisr);
1273
		int lr;
1274

1275
		for_each_set_bit(lr, eisr_ptr, vgic->nr_lr) {
1276
			struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
1277
			WARN_ON(vgic_irq_is_edge(vcpu, vlr.irq));
1278

1279
			spin_lock(&dist->lock);
1280
			vgic_irq_clear_queued(vcpu, vlr.irq);
1281 1282 1283
			WARN_ON(vlr.state & LR_STATE_MASK);
			vlr.state = 0;
			vgic_set_lr(vcpu, lr, vlr);
1284

1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297
			/*
			 * If the IRQ was EOIed it was also ACKed and we we
			 * therefore assume we can clear the soft pending
			 * state (should it had been set) for this interrupt.
			 *
			 * Note: if the IRQ soft pending state was set after
			 * the IRQ was acked, it actually shouldn't be
			 * cleared, but we have no way of knowing that unless
			 * we start trapping ACKs when the soft-pending state
			 * is set.
			 */
			vgic_dist_irq_clear_soft_pend(vcpu, vlr.irq);

1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308
			/*
			 * kvm_notify_acked_irq calls kvm_set_irq()
			 * to reset the IRQ level. Need to release the
			 * lock for kvm_set_irq to grab it.
			 */
			spin_unlock(&dist->lock);

			kvm_notify_acked_irq(kvm, 0,
					     vlr.irq - VGIC_NR_PRIVATE_IRQS);
			spin_lock(&dist->lock);

1309
			/* Any additional pending interrupt? */
1310
			if (vgic_dist_irq_get_level(vcpu, vlr.irq)) {
1311
				vgic_cpu_irq_set(vcpu, vlr.irq);
1312 1313
				level_pending = true;
			} else {
1314
				vgic_dist_irq_clear_pending(vcpu, vlr.irq);
1315
				vgic_cpu_irq_clear(vcpu, vlr.irq);
1316
			}
1317

1318 1319
			spin_unlock(&dist->lock);

1320 1321 1322 1323
			/*
			 * Despite being EOIed, the LR may not have
			 * been marked as empty.
			 */
1324
			vgic_sync_lr_elrsr(vcpu, lr, vlr);
1325 1326 1327
		}
	}

1328
	if (status & INT_STATUS_UNDERFLOW)
1329
		vgic_disable_underflow(vcpu);
1330

1331 1332 1333 1334 1335 1336 1337 1338
	/*
	 * In the next iterations of the vcpu loop, if we sync the vgic state
	 * after flushing it, but before entering the guest (this happens for
	 * pending signals and vmid rollovers), then make sure we don't pick
	 * up any old maintenance interrupts here.
	 */
	vgic_clear_eisr(vcpu);

1339 1340 1341
	return level_pending;
}

1342
/* Sync back the VGIC state after a guest run */
1343 1344 1345 1346
static void __kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
{
	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1347 1348
	u64 elrsr;
	unsigned long *elrsr_ptr;
1349 1350 1351 1352
	int lr, pending;
	bool level_pending;

	level_pending = vgic_process_maintenance(vcpu);
1353
	elrsr = vgic_get_elrsr(vcpu);
1354
	elrsr_ptr = u64_to_bitmask(&elrsr);
1355 1356

	/* Clear mappings for empty LRs */
1357
	for_each_set_bit(lr, elrsr_ptr, vgic->nr_lr) {
1358
		struct vgic_lr vlr;
1359 1360 1361 1362

		if (!test_and_clear_bit(lr, vgic_cpu->lr_used))
			continue;

1363
		vlr = vgic_get_lr(vcpu, lr);
1364

1365
		BUG_ON(vlr.irq >= dist->nr_irqs);
1366
		vgic_cpu->vgic_irq_lr_map[vlr.irq] = LR_EMPTY;
1367 1368 1369
	}

	/* Check if we still have something up our sleeve... */
1370 1371
	pending = find_first_zero_bit(elrsr_ptr, vgic->nr_lr);
	if (level_pending || pending < vgic->nr_lr)
1372
		set_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401
}

void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	if (!irqchip_in_kernel(vcpu->kvm))
		return;

	spin_lock(&dist->lock);
	__kvm_vgic_flush_hwstate(vcpu);
	spin_unlock(&dist->lock);
}

void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
{
	if (!irqchip_in_kernel(vcpu->kvm))
		return;

	__kvm_vgic_sync_hwstate(vcpu);
}

int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	if (!irqchip_in_kernel(vcpu->kvm))
		return 0;

1402
	return test_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
1403 1404
}

1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415
int kvm_vgic_vcpu_active_irq(struct kvm_vcpu *vcpu)
{
	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;

	if (!irqchip_in_kernel(vcpu->kvm))
		return 0;

	return test_bit(vcpu->vcpu_id, dist->irq_active_on_cpu);
}


1416
void vgic_kick_vcpus(struct kvm *kvm)
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
{
	struct kvm_vcpu *vcpu;
	int c;

	/*
	 * We've injected an interrupt, time to find out who deserves
	 * a good kick...
	 */
	kvm_for_each_vcpu(c, vcpu, kvm) {
		if (kvm_vgic_vcpu_pending_irq(vcpu))
			kvm_vcpu_kick(vcpu);
	}
}

static int vgic_validate_injection(struct kvm_vcpu *vcpu, int irq, int level)
{
1433
	int edge_triggered = vgic_irq_is_edge(vcpu, irq);
1434 1435 1436 1437 1438 1439

	/*
	 * Only inject an interrupt if:
	 * - edge triggered and we have a rising edge
	 * - level triggered and we change level
	 */
1440 1441
	if (edge_triggered) {
		int state = vgic_dist_irq_is_pending(vcpu, irq);
1442
		return level > state;
1443 1444
	} else {
		int state = vgic_dist_irq_get_level(vcpu, irq);
1445
		return level != state;
1446
	}
1447 1448
}

1449
static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
1450 1451 1452 1453
				  unsigned int irq_num, bool level)
{
	struct vgic_dist *dist = &kvm->arch.vgic;
	struct kvm_vcpu *vcpu;
1454
	int edge_triggered, level_triggered;
1455
	int enabled;
1456
	bool ret = true, can_inject = true;
1457 1458 1459 1460

	spin_lock(&dist->lock);

	vcpu = kvm_get_vcpu(kvm, cpuid);
1461 1462
	edge_triggered = vgic_irq_is_edge(vcpu, irq_num);
	level_triggered = !edge_triggered;
1463 1464 1465 1466 1467 1468 1469 1470

	if (!vgic_validate_injection(vcpu, irq_num, level)) {
		ret = false;
		goto out;
	}

	if (irq_num >= VGIC_NR_PRIVATE_IRQS) {
		cpuid = dist->irq_spi_cpu[irq_num - VGIC_NR_PRIVATE_IRQS];
1471 1472 1473 1474 1475
		if (cpuid == VCPU_NOT_ALLOCATED) {
			/* Pretend we use CPU0, and prevent injection */
			cpuid = 0;
			can_inject = false;
		}
1476 1477 1478 1479 1480
		vcpu = kvm_get_vcpu(kvm, cpuid);
	}

	kvm_debug("Inject IRQ%d level %d CPU%d\n", irq_num, level, cpuid);

1481 1482 1483
	if (level) {
		if (level_triggered)
			vgic_dist_irq_set_level(vcpu, irq_num);
1484
		vgic_dist_irq_set_pending(vcpu, irq_num);
1485 1486 1487 1488 1489 1490
	} else {
		if (level_triggered) {
			vgic_dist_irq_clear_level(vcpu, irq_num);
			if (!vgic_dist_irq_soft_pend(vcpu, irq_num))
				vgic_dist_irq_clear_pending(vcpu, irq_num);
		}
1491 1492 1493

		ret = false;
		goto out;
1494
	}
1495 1496 1497

	enabled = vgic_irq_is_enabled(vcpu, irq_num);

1498
	if (!enabled || !can_inject) {
1499 1500 1501 1502
		ret = false;
		goto out;
	}

1503
	if (!vgic_can_sample_irq(vcpu, irq_num)) {
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
		/*
		 * Level interrupt in progress, will be picked up
		 * when EOId.
		 */
		ret = false;
		goto out;
	}

	if (level) {
		vgic_cpu_irq_set(vcpu, irq_num);
1514
		set_bit(cpuid, dist->irq_pending_on_cpu);
1515 1516 1517 1518 1519
	}

out:
	spin_unlock(&dist->lock);

1520
	return ret ? cpuid : -EINVAL;
1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
}

/**
 * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic
 * @kvm:     The VM structure pointer
 * @cpuid:   The CPU for PPIs
 * @irq_num: The IRQ number that is assigned to the device
 * @level:   Edge-triggered:  true:  to trigger the interrupt
 *			      false: to ignore the call
 *	     Level-sensitive  true:  activates an interrupt
 *			      false: deactivates an interrupt
 *
 * The GIC is not concerned with devices being active-LOW or active-HIGH for
 * level-sensitive interrupts.  You can think of the level parameter as 1
 * being HIGH and 0 being LOW and all devices being active-HIGH.
 */
int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
			bool level)
{
1540
	int ret = 0;
1541
	int vcpu_id;
1542

1543
	if (unlikely(!vgic_initialized(kvm))) {
1544 1545 1546 1547 1548 1549 1550 1551 1552 1553
		/*
		 * We only provide the automatic initialization of the VGIC
		 * for the legacy case of a GICv2. Any other type must
		 * be explicitly initialized once setup with the respective
		 * KVM device call.
		 */
		if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2) {
			ret = -EBUSY;
			goto out;
		}
1554 1555 1556 1557 1558 1559
		mutex_lock(&kvm->lock);
		ret = vgic_init(kvm);
		mutex_unlock(&kvm->lock);

		if (ret)
			goto out;
1560
	}
1561

1562
	if (irq_num >= min(kvm->arch.vgic.nr_irqs, 1020))
1563 1564
		return -EINVAL;

1565 1566 1567 1568 1569 1570 1571 1572
	vcpu_id = vgic_update_irq_pending(kvm, cpuid, irq_num, level);
	if (vcpu_id >= 0) {
		/* kick the specified vcpu */
		kvm_vcpu_kick(kvm_get_vcpu(kvm, vcpu_id));
	}

out:
	return ret;
1573 1574
}

1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585
static irqreturn_t vgic_maintenance_handler(int irq, void *data)
{
	/*
	 * We cannot rely on the vgic maintenance interrupt to be
	 * delivered synchronously. This means we can only use it to
	 * exit the VM, and we perform the handling of EOIed
	 * interrupts on the exit path (see vgic_process_maintenance).
	 */
	return IRQ_HANDLED;
}

1586 1587 1588 1589 1590
void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
{
	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;

	kfree(vgic_cpu->pending_shared);
1591 1592
	kfree(vgic_cpu->active_shared);
	kfree(vgic_cpu->pend_act_shared);
1593 1594
	kfree(vgic_cpu->vgic_irq_lr_map);
	vgic_cpu->pending_shared = NULL;
1595 1596
	vgic_cpu->active_shared = NULL;
	vgic_cpu->pend_act_shared = NULL;
1597 1598 1599 1600 1601 1602 1603 1604 1605
	vgic_cpu->vgic_irq_lr_map = NULL;
}

static int vgic_vcpu_init_maps(struct kvm_vcpu *vcpu, int nr_irqs)
{
	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;

	int sz = (nr_irqs - VGIC_NR_PRIVATE_IRQS) / 8;
	vgic_cpu->pending_shared = kzalloc(sz, GFP_KERNEL);
1606 1607
	vgic_cpu->active_shared = kzalloc(sz, GFP_KERNEL);
	vgic_cpu->pend_act_shared = kzalloc(sz, GFP_KERNEL);
1608
	vgic_cpu->vgic_irq_lr_map = kmalloc(nr_irqs, GFP_KERNEL);
1609

1610 1611 1612 1613
	if (!vgic_cpu->pending_shared
		|| !vgic_cpu->active_shared
		|| !vgic_cpu->pend_act_shared
		|| !vgic_cpu->vgic_irq_lr_map) {
1614 1615 1616 1617
		kvm_vgic_vcpu_destroy(vcpu);
		return -ENOMEM;
	}

1618
	memset(vgic_cpu->vgic_irq_lr_map, LR_EMPTY, nr_irqs);
1619 1620

	/*
1621 1622 1623
	 * Store the number of LRs per vcpu, so we don't have to go
	 * all the way to the distributor structure to find out. Only
	 * assembly code should use this one.
1624
	 */
1625
	vgic_cpu->nr_lr = vgic->nr_lr;
1626

1627
	return 0;
1628 1629
}

1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640
/**
 * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
 *
 * The host's GIC naturally limits the maximum amount of VCPUs a guest
 * can use.
 */
int kvm_vgic_get_max_vcpus(void)
{
	return vgic->max_gic_vcpus;
}

1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662
void kvm_vgic_destroy(struct kvm *kvm)
{
	struct vgic_dist *dist = &kvm->arch.vgic;
	struct kvm_vcpu *vcpu;
	int i;

	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_vgic_vcpu_destroy(vcpu);

	vgic_free_bitmap(&dist->irq_enabled);
	vgic_free_bitmap(&dist->irq_level);
	vgic_free_bitmap(&dist->irq_pending);
	vgic_free_bitmap(&dist->irq_soft_pend);
	vgic_free_bitmap(&dist->irq_queued);
	vgic_free_bitmap(&dist->irq_cfg);
	vgic_free_bytemap(&dist->irq_priority);
	if (dist->irq_spi_target) {
		for (i = 0; i < dist->nr_cpus; i++)
			vgic_free_bitmap(&dist->irq_spi_target[i]);
	}
	kfree(dist->irq_sgi_sources);
	kfree(dist->irq_spi_cpu);
1663
	kfree(dist->irq_spi_mpidr);
1664 1665
	kfree(dist->irq_spi_target);
	kfree(dist->irq_pending_on_cpu);
1666
	kfree(dist->irq_active_on_cpu);
1667 1668 1669 1670
	dist->irq_sgi_sources = NULL;
	dist->irq_spi_cpu = NULL;
	dist->irq_spi_target = NULL;
	dist->irq_pending_on_cpu = NULL;
1671
	dist->irq_active_on_cpu = NULL;
1672
	dist->nr_cpus = 0;
1673 1674 1675 1676 1677 1678
}

/*
 * Allocate and initialize the various data structures. Must be called
 * with kvm->lock held!
 */
1679
int vgic_init(struct kvm *kvm)
1680 1681 1682 1683
{
	struct vgic_dist *dist = &kvm->arch.vgic;
	struct kvm_vcpu *vcpu;
	int nr_cpus, nr_irqs;
1684
	int ret, i, vcpu_id;
1685

1686
	if (vgic_initialized(kvm))
1687 1688 1689 1690
		return 0;

	nr_cpus = dist->nr_cpus = atomic_read(&kvm->online_vcpus);
	if (!nr_cpus)		/* No vcpus? Can't be good... */
1691
		return -ENODEV;
1692

1693 1694 1695 1696
	/*
	 * If nobody configured the number of interrupts, use the
	 * legacy one.
	 */
1697 1698 1699 1700
	if (!dist->nr_irqs)
		dist->nr_irqs = VGIC_NR_IRQS_LEGACY;

	nr_irqs = dist->nr_irqs;
1701 1702 1703 1704 1705 1706

	ret  = vgic_init_bitmap(&dist->irq_enabled, nr_cpus, nr_irqs);
	ret |= vgic_init_bitmap(&dist->irq_level, nr_cpus, nr_irqs);
	ret |= vgic_init_bitmap(&dist->irq_pending, nr_cpus, nr_irqs);
	ret |= vgic_init_bitmap(&dist->irq_soft_pend, nr_cpus, nr_irqs);
	ret |= vgic_init_bitmap(&dist->irq_queued, nr_cpus, nr_irqs);
1707
	ret |= vgic_init_bitmap(&dist->irq_active, nr_cpus, nr_irqs);
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719
	ret |= vgic_init_bitmap(&dist->irq_cfg, nr_cpus, nr_irqs);
	ret |= vgic_init_bytemap(&dist->irq_priority, nr_cpus, nr_irqs);

	if (ret)
		goto out;

	dist->irq_sgi_sources = kzalloc(nr_cpus * VGIC_NR_SGIS, GFP_KERNEL);
	dist->irq_spi_cpu = kzalloc(nr_irqs - VGIC_NR_PRIVATE_IRQS, GFP_KERNEL);
	dist->irq_spi_target = kzalloc(sizeof(*dist->irq_spi_target) * nr_cpus,
				       GFP_KERNEL);
	dist->irq_pending_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
					   GFP_KERNEL);
1720 1721
	dist->irq_active_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
					   GFP_KERNEL);
1722 1723 1724
	if (!dist->irq_sgi_sources ||
	    !dist->irq_spi_cpu ||
	    !dist->irq_spi_target ||
1725 1726
	    !dist->irq_pending_on_cpu ||
	    !dist->irq_active_on_cpu) {
1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737
		ret = -ENOMEM;
		goto out;
	}

	for (i = 0; i < nr_cpus; i++)
		ret |= vgic_init_bitmap(&dist->irq_spi_target[i],
					nr_cpus, nr_irqs);

	if (ret)
		goto out;

1738 1739 1740
	ret = kvm->arch.vgic.vm_ops.init_model(kvm);
	if (ret)
		goto out;
1741 1742

	kvm_for_each_vcpu(vcpu_id, vcpu, kvm) {
1743 1744 1745 1746 1747 1748
		ret = vgic_vcpu_init_maps(vcpu, nr_irqs);
		if (ret) {
			kvm_err("VGIC: Failed to allocate vcpu memory\n");
			break;
		}

1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
		for (i = 0; i < dist->nr_irqs; i++) {
			if (i < VGIC_NR_PPIS)
				vgic_bitmap_set_irq_val(&dist->irq_enabled,
							vcpu->vcpu_id, i, 1);
			if (i < VGIC_NR_PRIVATE_IRQS)
				vgic_bitmap_set_irq_val(&dist->irq_cfg,
							vcpu->vcpu_id, i,
							VGIC_CFG_EDGE);
		}

		vgic_enable(vcpu);
	}
1761

1762 1763 1764 1765 1766 1767 1768
out:
	if (ret)
		kvm_vgic_destroy(kvm);

	return ret;
}

1769 1770 1771 1772 1773 1774
static int init_vgic_model(struct kvm *kvm, int type)
{
	switch (type) {
	case KVM_DEV_TYPE_ARM_VGIC_V2:
		vgic_v2_init_emulation(kvm);
		break;
1775 1776 1777 1778 1779
#ifdef CONFIG_ARM_GIC_V3
	case KVM_DEV_TYPE_ARM_VGIC_V3:
		vgic_v3_init_emulation(kvm);
		break;
#endif
1780 1781 1782 1783
	default:
		return -ENODEV;
	}

1784 1785 1786
	if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus)
		return -E2BIG;

1787 1788 1789
	return 0;
}

1790
int kvm_vgic_create(struct kvm *kvm, u32 type)
1791
{
1792
	int i, vcpu_lock_idx = -1, ret;
1793
	struct kvm_vcpu *vcpu;
1794 1795 1796

	mutex_lock(&kvm->lock);

1797
	if (irqchip_in_kernel(kvm)) {
1798 1799 1800 1801
		ret = -EEXIST;
		goto out;
	}

1802 1803 1804 1805 1806 1807
	/*
	 * This function is also called by the KVM_CREATE_IRQCHIP handler,
	 * which had no chance yet to check the availability of the GICv2
	 * emulation. So check this here again. KVM_CREATE_DEVICE does
	 * the proper checks already.
	 */
1808 1809 1810 1811
	if (type == KVM_DEV_TYPE_ARM_VGIC_V2 && !vgic->can_emulate_gicv2) {
		ret = -ENODEV;
		goto out;
	}
1812

1813 1814 1815 1816 1817
	/*
	 * Any time a vcpu is run, vcpu_load is called which tries to grab the
	 * vcpu->mutex.  By grabbing the vcpu->mutex of all VCPUs we ensure
	 * that no other VCPUs are run while we create the vgic.
	 */
1818
	ret = -EBUSY;
1819 1820 1821 1822 1823 1824 1825
	kvm_for_each_vcpu(i, vcpu, kvm) {
		if (!mutex_trylock(&vcpu->mutex))
			goto out_unlock;
		vcpu_lock_idx = i;
	}

	kvm_for_each_vcpu(i, vcpu, kvm) {
1826
		if (vcpu->arch.has_run_once)
1827 1828
			goto out_unlock;
	}
1829
	ret = 0;
1830

1831 1832 1833 1834
	ret = init_vgic_model(kvm, type);
	if (ret)
		goto out_unlock;

1835
	spin_lock_init(&kvm->arch.vgic.lock);
1836
	kvm->arch.vgic.in_kernel = true;
1837
	kvm->arch.vgic.vgic_model = type;
1838
	kvm->arch.vgic.vctrl_base = vgic->vctrl_base;
1839 1840
	kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
	kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
1841
	kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;
1842

1843 1844 1845 1846 1847 1848
out_unlock:
	for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
		vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
		mutex_unlock(&vcpu->mutex);
	}

1849 1850 1851 1852 1853
out:
	mutex_unlock(&kvm->lock);
	return ret;
}

1854
static int vgic_ioaddr_overlap(struct kvm *kvm)
1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871
{
	phys_addr_t dist = kvm->arch.vgic.vgic_dist_base;
	phys_addr_t cpu = kvm->arch.vgic.vgic_cpu_base;

	if (IS_VGIC_ADDR_UNDEF(dist) || IS_VGIC_ADDR_UNDEF(cpu))
		return 0;
	if ((dist <= cpu && dist + KVM_VGIC_V2_DIST_SIZE > cpu) ||
	    (cpu <= dist && cpu + KVM_VGIC_V2_CPU_SIZE > dist))
		return -EBUSY;
	return 0;
}

static int vgic_ioaddr_assign(struct kvm *kvm, phys_addr_t *ioaddr,
			      phys_addr_t addr, phys_addr_t size)
{
	int ret;

1872 1873 1874 1875 1876 1877
	if (addr & ~KVM_PHYS_MASK)
		return -E2BIG;

	if (addr & (SZ_4K - 1))
		return -EINVAL;

1878 1879 1880 1881 1882
	if (!IS_VGIC_ADDR_UNDEF(*ioaddr))
		return -EEXIST;
	if (addr + size < addr)
		return -EINVAL;

1883
	*ioaddr = addr;
1884 1885
	ret = vgic_ioaddr_overlap(kvm);
	if (ret)
1886 1887
		*ioaddr = VGIC_ADDR_UNDEF;

1888 1889 1890
	return ret;
}

1891 1892 1893
/**
 * kvm_vgic_addr - set or get vgic VM base addresses
 * @kvm:   pointer to the vm struct
1894
 * @type:  the VGIC addr type, one of KVM_VGIC_V[23]_ADDR_TYPE_XXX
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904
 * @addr:  pointer to address value
 * @write: if true set the address in the VM address space, if false read the
 *          address
 *
 * Set or get the vgic base addresses for the distributor and the virtual CPU
 * interface in the VM physical address space.  These addresses are properties
 * of the emulated core/SoC and therefore user space initially knows this
 * information.
 */
int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)
1905 1906 1907
{
	int r = 0;
	struct vgic_dist *vgic = &kvm->arch.vgic;
1908 1909
	int type_needed;
	phys_addr_t *addr_ptr, block_size;
1910
	phys_addr_t alignment;
1911 1912 1913 1914

	mutex_lock(&kvm->lock);
	switch (type) {
	case KVM_VGIC_V2_ADDR_TYPE_DIST:
1915 1916 1917
		type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
		addr_ptr = &vgic->vgic_dist_base;
		block_size = KVM_VGIC_V2_DIST_SIZE;
1918
		alignment = SZ_4K;
1919 1920
		break;
	case KVM_VGIC_V2_ADDR_TYPE_CPU:
1921 1922 1923
		type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
		addr_ptr = &vgic->vgic_cpu_base;
		block_size = KVM_VGIC_V2_CPU_SIZE;
1924
		alignment = SZ_4K;
1925
		break;
1926 1927 1928 1929 1930
#ifdef CONFIG_ARM_GIC_V3
	case KVM_VGIC_V3_ADDR_TYPE_DIST:
		type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
		addr_ptr = &vgic->vgic_dist_base;
		block_size = KVM_VGIC_V3_DIST_SIZE;
1931
		alignment = SZ_64K;
1932 1933 1934 1935 1936
		break;
	case KVM_VGIC_V3_ADDR_TYPE_REDIST:
		type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
		addr_ptr = &vgic->vgic_redist_base;
		block_size = KVM_VGIC_V3_REDIST_SIZE;
1937
		alignment = SZ_64K;
1938 1939
		break;
#endif
1940 1941
	default:
		r = -ENODEV;
1942 1943 1944 1945 1946 1947
		goto out;
	}

	if (vgic->vgic_model != type_needed) {
		r = -ENODEV;
		goto out;
1948 1949
	}

1950 1951 1952 1953 1954 1955 1956
	if (write) {
		if (!IS_ALIGNED(*addr, alignment))
			r = -EINVAL;
		else
			r = vgic_ioaddr_assign(kvm, addr_ptr, *addr,
					       block_size);
	} else {
1957
		*addr = *addr_ptr;
1958
	}
1959 1960

out:
1961 1962 1963
	mutex_unlock(&kvm->lock);
	return r;
}
1964

1965
int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1966
{
1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980
	int r;

	switch (attr->group) {
	case KVM_DEV_ARM_VGIC_GRP_ADDR: {
		u64 __user *uaddr = (u64 __user *)(long)attr->addr;
		u64 addr;
		unsigned long type = (unsigned long)attr->attr;

		if (copy_from_user(&addr, uaddr, sizeof(addr)))
			return -EFAULT;

		r = kvm_vgic_addr(dev->kvm, type, &addr, true);
		return (r == -ENODEV) ? -ENXIO : r;
	}
1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001
	case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
		u32 __user *uaddr = (u32 __user *)(long)attr->addr;
		u32 val;
		int ret = 0;

		if (get_user(val, uaddr))
			return -EFAULT;

		/*
		 * We require:
		 * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs
		 * - at most 1024 interrupts
		 * - a multiple of 32 interrupts
		 */
		if (val < (VGIC_NR_PRIVATE_IRQS + 32) ||
		    val > VGIC_MAX_IRQS ||
		    (val & 31))
			return -EINVAL;

		mutex_lock(&dev->kvm->lock);

2002
		if (vgic_ready(dev->kvm) || dev->kvm->arch.vgic.nr_irqs)
2003 2004 2005 2006 2007 2008 2009 2010
			ret = -EBUSY;
		else
			dev->kvm->arch.vgic.nr_irqs = val;

		mutex_unlock(&dev->kvm->lock);

		return ret;
	}
2011 2012 2013 2014 2015 2016 2017 2018
	case KVM_DEV_ARM_VGIC_GRP_CTRL: {
		switch (attr->attr) {
		case KVM_DEV_ARM_VGIC_CTRL_INIT:
			r = vgic_init(dev->kvm);
			return r;
		}
		break;
	}
2019 2020
	}

2021 2022 2023
	return -ENXIO;
}

2024
int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2025
{
2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039
	int r = -ENXIO;

	switch (attr->group) {
	case KVM_DEV_ARM_VGIC_GRP_ADDR: {
		u64 __user *uaddr = (u64 __user *)(long)attr->addr;
		u64 addr;
		unsigned long type = (unsigned long)attr->attr;

		r = kvm_vgic_addr(dev->kvm, type, &addr, false);
		if (r)
			return (r == -ENODEV) ? -ENXIO : r;

		if (copy_to_user(uaddr, &addr, sizeof(addr)))
			return -EFAULT;
2040 2041
		break;
	}
2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053
	case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
		u32 __user *uaddr = (u32 __user *)(long)attr->addr;

		r = put_user(dev->kvm->arch.vgic.nr_irqs, uaddr);
		break;
	}

	}

	return r;
}

2054
int vgic_has_attr_regs(const struct vgic_io_range *ranges, phys_addr_t offset)
2055
{
2056
	if (vgic_find_range(ranges, 4, offset))
2057 2058 2059 2060 2061
		return 0;
	else
		return -ENXIO;
}

2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088
static void vgic_init_maintenance_interrupt(void *info)
{
	enable_percpu_irq(vgic->maint_irq, 0);
}

static int vgic_cpu_notify(struct notifier_block *self,
			   unsigned long action, void *cpu)
{
	switch (action) {
	case CPU_STARTING:
	case CPU_STARTING_FROZEN:
		vgic_init_maintenance_interrupt(NULL);
		break;
	case CPU_DYING:
	case CPU_DYING_FROZEN:
		disable_percpu_irq(vgic->maint_irq);
		break;
	}

	return NOTIFY_OK;
}

static struct notifier_block vgic_cpu_nb = {
	.notifier_call = vgic_cpu_notify,
};

static const struct of_device_id vgic_ids[] = {
2089 2090 2091 2092
	{ .compatible = "arm,cortex-a15-gic",	.data = vgic_v2_probe, },
	{ .compatible = "arm,cortex-a7-gic",	.data = vgic_v2_probe, },
	{ .compatible = "arm,gic-400",		.data = vgic_v2_probe, },
	{ .compatible = "arm,gic-v3",		.data = vgic_v3_probe, },
2093 2094 2095 2096 2097 2098
	{},
};

int kvm_vgic_hyp_init(void)
{
	const struct of_device_id *matched_id;
2099 2100
	const int (*vgic_probe)(struct device_node *,const struct vgic_ops **,
				const struct vgic_params **);
2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133
	struct device_node *vgic_node;
	int ret;

	vgic_node = of_find_matching_node_and_match(NULL,
						    vgic_ids, &matched_id);
	if (!vgic_node) {
		kvm_err("error: no compatible GIC node found\n");
		return -ENODEV;
	}

	vgic_probe = matched_id->data;
	ret = vgic_probe(vgic_node, &vgic_ops, &vgic);
	if (ret)
		return ret;

	ret = request_percpu_irq(vgic->maint_irq, vgic_maintenance_handler,
				 "vgic", kvm_get_running_vcpus());
	if (ret) {
		kvm_err("Cannot register interrupt %d\n", vgic->maint_irq);
		return ret;
	}

	ret = __register_cpu_notifier(&vgic_cpu_nb);
	if (ret) {
		kvm_err("Cannot register vgic CPU notifier\n");
		goto out_free_irq;
	}

	/* Callback into for arch code for setup */
	vgic_arch_setup(vgic);

	on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1);

2134
	return 0;
2135 2136 2137 2138 2139

out_free_irq:
	free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus());
	return ret;
}
2140 2141 2142 2143 2144

int kvm_irq_map_gsi(struct kvm *kvm,
		    struct kvm_kernel_irq_routing_entry *entries,
		    int gsi)
{
2145
	return 0;
2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171
}

int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)
{
	return pin;
}

int kvm_set_irq(struct kvm *kvm, int irq_source_id,
		u32 irq, int level, bool line_status)
{
	unsigned int spi = irq + VGIC_NR_PRIVATE_IRQS;

	trace_kvm_set_irq(irq, level, irq_source_id);

	BUG_ON(!vgic_initialized(kvm));

	return kvm_vgic_inject_irq(kvm, 0, spi, level);
}

/* MSI not implemented yet */
int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
		struct kvm *kvm, int irq_source_id,
		int level, bool line_status)
{
	return 0;
}
新手
引导
客服 返回
顶部