interrupt.c 75.6 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * handling kvm guest interrupts
4
 *
5
 * Copyright IBM Corp. 2008, 2015
6 7 8 9
 *
 *    Author(s): Carsten Otte <cotte@de.ibm.com>
 */

10
#include <linux/interrupt.h>
11
#include <linux/kvm_host.h>
12
#include <linux/hrtimer.h>
13
#include <linux/mmu_context.h>
14
#include <linux/signal.h>
15
#include <linux/slab.h>
16
#include <linux/bitmap.h>
17
#include <linux/vmalloc.h>
18
#include <asm/asm-offsets.h>
19
#include <asm/dis.h>
20
#include <linux/uaccess.h>
21
#include <asm/sclp.h>
22
#include <asm/isc.h>
23
#include <asm/gmap.h>
24
#include <asm/switch_to.h>
25
#include <asm/nmi.h>
26 27
#include "kvm-s390.h"
#include "gaccess.h"
28
#include "trace-s390.h"
29

30
#define PFAULT_INIT 0x0600
31 32
#define PFAULT_DONE 0x0680
#define VIRTIO_PARAM 0x0d00
33

34 35 36
/* handle external calls via sigp interpretation facility */
static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
{
37 38
	int c, scn;

39
	if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
40 41
		return 0;

42
	BUG_ON(!kvm_s390_use_sca_entries());
43
	read_lock(&vcpu->kvm->arch.sca_lock);
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
	if (vcpu->kvm->arch.use_esca) {
		struct esca_block *sca = vcpu->kvm->arch.sca;
		union esca_sigp_ctrl sigp_ctrl =
			sca->cpu[vcpu->vcpu_id].sigp_ctrl;

		c = sigp_ctrl.c;
		scn = sigp_ctrl.scn;
	} else {
		struct bsca_block *sca = vcpu->kvm->arch.sca;
		union bsca_sigp_ctrl sigp_ctrl =
			sca->cpu[vcpu->vcpu_id].sigp_ctrl;

		c = sigp_ctrl.c;
		scn = sigp_ctrl.scn;
	}
59
	read_unlock(&vcpu->kvm->arch.sca_lock);
60 61

	if (src_id)
62
		*src_id = scn;
63

64
	return c;
65 66 67 68
}

static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
{
69
	int expect, rc;
70

71
	BUG_ON(!kvm_s390_use_sca_entries());
72
	read_lock(&vcpu->kvm->arch.sca_lock);
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
	if (vcpu->kvm->arch.use_esca) {
		struct esca_block *sca = vcpu->kvm->arch.sca;
		union esca_sigp_ctrl *sigp_ctrl =
			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
		union esca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl;

		new_val.scn = src_id;
		new_val.c = 1;
		old_val.c = 0;

		expect = old_val.value;
		rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
	} else {
		struct bsca_block *sca = vcpu->kvm->arch.sca;
		union bsca_sigp_ctrl *sigp_ctrl =
			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
		union bsca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl;
90

91 92 93 94 95 96 97
		new_val.scn = src_id;
		new_val.c = 1;
		old_val.c = 0;

		expect = old_val.value;
		rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
	}
98
	read_unlock(&vcpu->kvm->arch.sca_lock);
99 100

	if (rc != expect) {
101 102 103
		/* another external call is pending */
		return -EBUSY;
	}
104
	kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
105 106 107 108 109
	return 0;
}

static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
{
110
	int rc, expect;
111

112 113
	if (!kvm_s390_use_sca_entries())
		return;
114
	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
115
	read_lock(&vcpu->kvm->arch.sca_lock);
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
	if (vcpu->kvm->arch.use_esca) {
		struct esca_block *sca = vcpu->kvm->arch.sca;
		union esca_sigp_ctrl *sigp_ctrl =
			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
		union esca_sigp_ctrl old = *sigp_ctrl;

		expect = old.value;
		rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
	} else {
		struct bsca_block *sca = vcpu->kvm->arch.sca;
		union bsca_sigp_ctrl *sigp_ctrl =
			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
		union bsca_sigp_ctrl old = *sigp_ctrl;

		expect = old.value;
		rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
	}
133
	read_unlock(&vcpu->kvm->arch.sca_lock);
134
	WARN_ON(rc != expect); /* cannot clear? */
135 136
}

137
int psw_extint_disabled(struct kvm_vcpu *vcpu)
138 139 140 141
{
	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
}

142 143 144 145 146
static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
{
	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
}

147 148 149 150 151
static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
{
	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
}

152 153
static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
{
154 155 156
	return psw_extint_disabled(vcpu) &&
	       psw_ioint_disabled(vcpu) &&
	       psw_mchk_disabled(vcpu);
157 158
}

159 160 161 162 163
static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
{
	if (psw_extint_disabled(vcpu) ||
	    !(vcpu->arch.sie_block->gcr[0] & 0x800ul))
		return 0;
164 165 166
	if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
		/* No timer interrupts when single stepping */
		return 0;
167 168 169
	return 1;
}

170 171
static int ckc_irq_pending(struct kvm_vcpu *vcpu)
{
172
	if (vcpu->arch.sie_block->ckc >= kvm_s390_get_tod_clock_fast(vcpu->kvm))
173 174 175 176 177 178 179 180 181 182 183 184
		return 0;
	return ckc_interrupts_enabled(vcpu);
}

static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
{
	return !psw_extint_disabled(vcpu) &&
	       (vcpu->arch.sie_block->gcr[0] & 0x400ul);
}

static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
{
185 186 187
	if (!cpu_timer_interrupts_enabled(vcpu))
		return 0;
	return kvm_s390_get_cpu_timer(vcpu) >> 63;
188 189
}

190 191
static uint64_t isc_to_isc_bits(int isc)
{
C
Cornelia Huck 已提交
192 193 194
	return (0x80 >> isc) << 24;
}

195 196 197 198 199
static inline u32 isc_to_int_word(u8 isc)
{
	return ((u32)isc << 27) | 0x80000000;
}

200
static inline u8 int_word_to_isc(u32 int_word)
201
{
202 203 204
	return (int_word & 0x38000000) >> 27;
}

205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
/*
 * To use atomic bitmap functions, we have to provide a bitmap address
 * that is u64 aligned. However, the ipm might be u32 aligned.
 * Therefore, we logically start the bitmap at the very beginning of the
 * struct and fixup the bit number.
 */
#define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)

static inline void kvm_s390_gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
{
	set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
}

static inline u8 kvm_s390_gisa_get_ipm(struct kvm_s390_gisa *gisa)
{
	return READ_ONCE(gisa->ipm);
}

static inline void kvm_s390_gisa_clear_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
{
	clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
}

static inline int kvm_s390_gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
{
	return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
}

233
static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
234
{
235
	return vcpu->kvm->arch.float_int.pending_irqs |
236 237 238 239 240 241
		vcpu->arch.local_int.pending_irqs;
}

static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
{
	return pending_irqs_no_gisa(vcpu) |
242
		kvm_s390_gisa_get_ipm(vcpu->kvm->arch.gisa) << IRQ_PEND_IO_ISC_7;
243 244
}

245 246
static inline int isc_to_irq_type(unsigned long isc)
{
247
	return IRQ_PEND_IO_ISC_0 - isc;
248 249 250 251
}

static inline int irq_type_to_isc(unsigned long irq_type)
{
252
	return IRQ_PEND_IO_ISC_0 - irq_type;
253 254
}

255 256 257 258 259 260 261
static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
				   unsigned long active_mask)
{
	int i;

	for (i = 0; i <= MAX_ISC; i++)
		if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
262
			active_mask &= ~(1UL << (isc_to_irq_type(i)));
263 264 265 266 267

	return active_mask;
}

static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
268
{
269 270
	unsigned long active_mask;

271
	active_mask = pending_irqs(vcpu);
272 273
	if (!active_mask)
		return 0;
274 275 276

	if (psw_extint_disabled(vcpu))
		active_mask &= ~IRQ_PEND_EXT_MASK;
277 278 279 280
	if (psw_ioint_disabled(vcpu))
		active_mask &= ~IRQ_PEND_IO_MASK;
	else
		active_mask = disable_iscs(vcpu, active_mask);
281 282 283 284 285 286 287 288
	if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul))
		__clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
	if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul))
		__clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
	if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul))
		__clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
	if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul))
		__clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
289 290
	if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
		__clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
291 292
	if (psw_mchk_disabled(vcpu))
		active_mask &= ~IRQ_PEND_MCHK_MASK;
293 294 295 296
	/*
	 * Check both floating and local interrupt's cr14 because
	 * bit IRQ_PEND_MCHK_REP could be set in both cases.
	 */
297
	if (!(vcpu->arch.sie_block->gcr[14] &
298 299
	   (vcpu->kvm->arch.float_int.mchk.cr14 |
	   vcpu->arch.local_int.irq.mchk.cr14)))
300
		__clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
301

302 303 304 305 306 307
	/*
	 * STOP irqs will never be actively delivered. They are triggered via
	 * intercept requests and cleared when the stop intercept is performed.
	 */
	__clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);

308 309 310
	return active_mask;
}

311 312
static void __set_cpu_idle(struct kvm_vcpu *vcpu)
{
313
	kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
314
	set_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask);
315 316 317 318
}

static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
{
319
	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
320
	clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask);
321 322 323 324
}

static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
{
325 326
	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
				      CPUSTAT_STOP_INT);
327
	vcpu->arch.sie_block->lctl = 0x0000;
328 329 330 331 332 333 334
	vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);

	if (guestdbg_enabled(vcpu)) {
		vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
					       LCTL_CR10 | LCTL_CR11);
		vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
	}
335 336
}

337 338
static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
{
339
	if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
340 341
		return;
	else if (psw_ioint_disabled(vcpu))
342
		kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
343 344 345 346
	else
		vcpu->arch.sie_block->lctl |= LCTL_CR6;
}

347 348
static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
{
349
	if (!(pending_irqs(vcpu) & IRQ_PEND_EXT_MASK))
350 351
		return;
	if (psw_extint_disabled(vcpu))
352
		kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
353 354 355 356 357 358
	else
		vcpu->arch.sie_block->lctl |= LCTL_CR0;
}

static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
{
359
	if (!(pending_irqs(vcpu) & IRQ_PEND_MCHK_MASK))
360 361 362 363 364 365 366
		return;
	if (psw_mchk_disabled(vcpu))
		vcpu->arch.sie_block->ictl |= ICTL_LPSW;
	else
		vcpu->arch.sie_block->lctl |= LCTL_CR14;
}

367 368 369
static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
{
	if (kvm_s390_is_stop_irq_pending(vcpu))
370
		kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
371 372
}

373 374
/* Set interception request for non-deliverable interrupts */
static void set_intercept_indicators(struct kvm_vcpu *vcpu)
375
{
376
	set_intercept_indicators_io(vcpu);
377 378
	set_intercept_indicators_ext(vcpu);
	set_intercept_indicators_mchk(vcpu);
379
	set_intercept_indicators_stop(vcpu);
380 381
}

382 383
static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
{
384
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
385 386 387 388 389 390 391
	int rc;

	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
					 0, 0);

	rc  = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
			   (u16 *)__LC_EXT_INT_CODE);
392
	rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
393 394 395 396
	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
397
	clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
398
	return rc ? -EFAULT : 0;
399 400 401 402
}

static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
{
403
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
404 405 406 407 408 409 410
	int rc;

	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
					 0, 0);

	rc  = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
			   (u16 __user *)__LC_EXT_INT_CODE);
411
	rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
412 413 414 415
	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
416
	clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
417
	return rc ? -EFAULT : 0;
418 419
}

420
static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
421
{
422 423
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_ext_info ext;
424 425
	int rc;

426 427 428 429 430 431
	spin_lock(&li->lock);
	ext = li->irq.ext;
	clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
	li->irq.ext.ext_params2 = 0;
	spin_unlock(&li->lock);

432 433
	VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
		   ext.ext_params2);
434 435
	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
					 KVM_S390_INT_PFAULT_INIT,
436
					 0, ext.ext_params2);
437 438 439 440 441 442 443

	rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
	rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
444
	rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
445
	return rc ? -EFAULT : 0;
446 447
}

448 449 450 451
static int __write_machine_check(struct kvm_vcpu *vcpu,
				 struct kvm_s390_mchk_info *mchk)
{
	unsigned long ext_sa_addr;
F
Fan Zhang 已提交
452
	unsigned long lc;
453
	freg_t fprs[NUM_FPRS];
454
	union mci mci;
455 456
	int rc;

457
	mci.val = mchk->mcic;
458
	/* take care of lazy register loading */
459 460
	save_fpu_regs();
	save_access_regs(vcpu->run->s.regs.acrs);
461 462
	if (MACHINE_HAS_GS && vcpu->arch.gs_enabled)
		save_gs_cb(current->thread.gs_cb);
463

464
	/* Extended save area */
465 466
	rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr,
			   sizeof(unsigned long));
F
Fan Zhang 已提交
467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488
	/* Only bits 0 through 63-LC are used for address formation */
	lc = ext_sa_addr & MCESA_LC_MASK;
	if (test_kvm_facility(vcpu->kvm, 133)) {
		switch (lc) {
		case 0:
		case 10:
			ext_sa_addr &= ~0x3ffUL;
			break;
		case 11:
			ext_sa_addr &= ~0x7ffUL;
			break;
		case 12:
			ext_sa_addr &= ~0xfffUL;
			break;
		default:
			ext_sa_addr = 0;
			break;
		}
	} else {
		ext_sa_addr &= ~0x3ffUL;
	}

489 490 491 492 493 494 495
	if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) {
		if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs,
				    512))
			mci.vr = 0;
	} else {
		mci.vr = 0;
	}
F
Fan Zhang 已提交
496 497 498 499 500 501 502 503
	if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133)
	    && (lc == 11 || lc == 12)) {
		if (write_guest_abs(vcpu, ext_sa_addr + 1024,
				    &vcpu->run->s.regs.gscb, 32))
			mci.gs = 0;
	} else {
		mci.gs = 0;
	}
504 505

	/* General interruption information */
506
	rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID);
507 508 509 510
	rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
	rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
511
	rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE);
512 513

	/* Register-save areas */
514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
	if (MACHINE_HAS_VX) {
		convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
		rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128);
	} else {
		rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA,
				     vcpu->run->s.regs.fprs, 128);
	}
	rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA,
			     vcpu->run->s.regs.gprs, 128);
	rc |= put_guest_lc(vcpu, current->thread.fpu.fpc,
			   (u32 __user *) __LC_FP_CREG_SAVE_AREA);
	rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr,
			   (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA);
	rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu),
			   (u64 __user *) __LC_CPU_TIMER_SAVE_AREA);
	rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8,
			   (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA);
	rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA,
			     &vcpu->run->s.regs.acrs, 64);
	rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA,
			     &vcpu->arch.sie_block->gcr, 128);
535 536

	/* Extended interruption information */
537 538
	rc |= put_guest_lc(vcpu, mchk->ext_damage_code,
			   (u32 __user *) __LC_EXT_DAMAGE_CODE);
539 540 541 542 543 544 545
	rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
			   (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
	rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout,
			     sizeof(mchk->fixed_logout));
	return rc ? -EFAULT : 0;
}

546
static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
547
{
548
	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
549
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
550 551 552
	struct kvm_s390_mchk_info mchk = {};
	int deliver = 0;
	int rc = 0;
553

554
	spin_lock(&fi->lock);
555
	spin_lock(&li->lock);
556 557 558 559 560 561 562 563 564 565 566 567 568 569
	if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
	    test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
		/*
		 * If there was an exigent machine check pending, then any
		 * repressible machine checks that might have been pending
		 * are indicated along with it, so always clear bits for
		 * repressible and exigent interrupts
		 */
		mchk = li->irq.mchk;
		clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
		clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
		memset(&li->irq.mchk, 0, sizeof(mchk));
		deliver = 1;
	}
570
	/*
571 572 573 574
	 * We indicate floating repressible conditions along with
	 * other pending conditions. Channel Report Pending and Channel
	 * Subsystem damage are the only two and and are indicated by
	 * bits in mcic and masked in cr14.
575
	 */
576 577 578 579 580 581
	if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
		mchk.mcic |= fi->mchk.mcic;
		mchk.cr14 |= fi->mchk.cr14;
		memset(&fi->mchk, 0, sizeof(mchk));
		deliver = 1;
	}
582
	spin_unlock(&li->lock);
583
	spin_unlock(&fi->lock);
584

585
	if (deliver) {
586
		VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
587 588 589 590
			   mchk.mcic);
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
						 KVM_S390_MCHK,
						 mchk.cr14, mchk.mcic);
591
		rc = __write_machine_check(vcpu, &mchk);
592
	}
593
	return rc;
594 595 596 597
}

static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
{
598
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
599 600
	int rc;

601
	VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
602 603 604 605
	vcpu->stat.deliver_restart_signal++;
	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);

	rc  = write_guest_lc(vcpu,
606
			     offsetof(struct lowcore, restart_old_psw),
607
			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
608
	rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw),
609
			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
610
	clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
611
	return rc ? -EFAULT : 0;
612 613
}

614
static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
615
{
616 617 618 619 620 621 622 623
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_prefix_info prefix;

	spin_lock(&li->lock);
	prefix = li->irq.prefix;
	li->irq.prefix.address = 0;
	clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
	spin_unlock(&li->lock);
624 625 626 627

	vcpu->stat.deliver_prefix_signal++;
	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
					 KVM_S390_SIGP_SET_PREFIX,
628
					 prefix.address, 0);
629

630
	kvm_s390_set_prefix(vcpu, prefix.address);
631 632 633
	return 0;
}

634
static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
635
{
636
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
637
	int rc;
638 639 640 641 642 643 644 645
	int cpu_addr;

	spin_lock(&li->lock);
	cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS);
	clear_bit(cpu_addr, li->sigp_emerg_pending);
	if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
		clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
	spin_unlock(&li->lock);
646

647
	VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
648
	vcpu->stat.deliver_emergency_signal++;
649 650
	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
					 cpu_addr, 0);
651 652 653

	rc  = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
			   (u16 *)__LC_EXT_INT_CODE);
654
	rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
655 656 657 658
	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
659
	return rc ? -EFAULT : 0;
660 661
}

662
static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
663
{
664 665
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_extcall_info extcall;
666 667
	int rc;

668 669 670 671 672 673
	spin_lock(&li->lock);
	extcall = li->irq.extcall;
	li->irq.extcall.code = 0;
	clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
	spin_unlock(&li->lock);

674
	VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
675 676 677
	vcpu->stat.deliver_external_call++;
	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
					 KVM_S390_INT_EXTERNAL_CALL,
678
					 extcall.code, 0);
679 680 681

	rc  = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
			   (u16 *)__LC_EXT_INT_CODE);
682
	rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
683 684 685 686
	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
			    sizeof(psw_t));
687
	return rc ? -EFAULT : 0;
688 689
}

690
static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
691
{
692 693
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_pgm_info pgm_info;
694
	int rc = 0, nullifying = false;
695
	u16 ilen;
696

697 698 699 700 701 702
	spin_lock(&li->lock);
	pgm_info = li->irq.pgm;
	clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
	memset(&li->irq.pgm, 0, sizeof(pgm_info));
	spin_unlock(&li->lock);

703
	ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK;
704 705
	VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d",
		   pgm_info.code, ilen);
706 707
	vcpu->stat.deliver_program_int++;
	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
708
					 pgm_info.code, 0);
709

710
	switch (pgm_info.code & ~PGM_PER) {
711 712 713 714 715 716 717 718 719
	case PGM_AFX_TRANSLATION:
	case PGM_ASX_TRANSLATION:
	case PGM_EX_TRANSLATION:
	case PGM_LFX_TRANSLATION:
	case PGM_LSTE_SEQUENCE:
	case PGM_LSX_TRANSLATION:
	case PGM_LX_TRANSLATION:
	case PGM_PRIMARY_AUTHORITY:
	case PGM_SECONDARY_AUTHORITY:
720 721
		nullifying = true;
		/* fall through */
722
	case PGM_SPACE_SWITCH:
723
		rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
724 725 726 727 728 729 730 731
				  (u64 *)__LC_TRANS_EXC_CODE);
		break;
	case PGM_ALEN_TRANSLATION:
	case PGM_ALE_SEQUENCE:
	case PGM_ASTE_INSTANCE:
	case PGM_ASTE_SEQUENCE:
	case PGM_ASTE_VALIDITY:
	case PGM_EXTENDED_AUTHORITY:
732
		rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
733
				  (u8 *)__LC_EXC_ACCESS_ID);
734
		nullifying = true;
735 736 737 738 739 740 741
		break;
	case PGM_ASCE_TYPE:
	case PGM_PAGE_TRANSLATION:
	case PGM_REGION_FIRST_TRANS:
	case PGM_REGION_SECOND_TRANS:
	case PGM_REGION_THIRD_TRANS:
	case PGM_SEGMENT_TRANSLATION:
742
		rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
743
				  (u64 *)__LC_TRANS_EXC_CODE);
744
		rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
745
				   (u8 *)__LC_EXC_ACCESS_ID);
746
		rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
747
				   (u8 *)__LC_OP_ACCESS_ID);
748
		nullifying = true;
749 750
		break;
	case PGM_MONITOR:
751
		rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
752
				  (u16 *)__LC_MON_CLASS_NR);
753
		rc |= put_guest_lc(vcpu, pgm_info.mon_code,
754 755
				   (u64 *)__LC_MON_CODE);
		break;
E
Eric Farman 已提交
756
	case PGM_VECTOR_PROCESSING:
757
	case PGM_DATA:
758
		rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
759 760 761
				  (u32 *)__LC_DATA_EXC_CODE);
		break;
	case PGM_PROTECTION:
762
		rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
763
				  (u64 *)__LC_TRANS_EXC_CODE);
764
		rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
765 766
				   (u8 *)__LC_EXC_ACCESS_ID);
		break;
767 768 769 770 771 772 773 774 775
	case PGM_STACK_FULL:
	case PGM_STACK_EMPTY:
	case PGM_STACK_SPECIFICATION:
	case PGM_STACK_TYPE:
	case PGM_STACK_OPERATION:
	case PGM_TRACE_TABEL:
	case PGM_CRYPTO_OPERATION:
		nullifying = true;
		break;
776 777
	}

778 779
	if (pgm_info.code & PGM_PER) {
		rc |= put_guest_lc(vcpu, pgm_info.per_code,
780
				   (u8 *) __LC_PER_CODE);
781
		rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
782
				   (u8 *)__LC_PER_ATMID);
783
		rc |= put_guest_lc(vcpu, pgm_info.per_address,
784
				   (u64 *) __LC_PER_ADDRESS);
785
		rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
786 787 788
				   (u8 *) __LC_PER_ACCESS_ID);
	}

789
	if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND))
790
		kvm_s390_rewind_psw(vcpu, ilen);
791

792 793
	/* bit 1+2 of the target are the ilc, so we can directly use ilen */
	rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC);
794 795
	rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
				 (u64 *) __LC_LAST_BREAK);
796
	rc |= put_guest_lc(vcpu, pgm_info.code,
797 798 799 800 801
			   (u16 *)__LC_PGM_INT_CODE);
	rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
	rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
802
	return rc ? -EFAULT : 0;
803 804
}

805
static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
806
{
807 808 809 810 811 812 813 814 815 816 817 818 819
	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
	struct kvm_s390_ext_info ext;
	int rc = 0;

	spin_lock(&fi->lock);
	if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
		spin_unlock(&fi->lock);
		return 0;
	}
	ext = fi->srv_signal;
	memset(&fi->srv_signal, 0, sizeof(ext));
	clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
	spin_unlock(&fi->lock);
820

821
	VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
822
		   ext.ext_params);
823
	vcpu->stat.deliver_service_signal++;
824 825
	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
					 ext.ext_params, 0);
826 827

	rc  = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
828
	rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
829 830 831 832
	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
833
	rc |= put_guest_lc(vcpu, ext.ext_params,
834
			   (u32 *)__LC_EXT_PARAMS);
835

836
	return rc ? -EFAULT : 0;
837 838
}

839
static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
840
{
841 842 843
	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
	struct kvm_s390_interrupt_info *inti;
	int rc = 0;
844

845 846 847 848 849 850 851 852 853 854 855
	spin_lock(&fi->lock);
	inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
					struct kvm_s390_interrupt_info,
					list);
	if (inti) {
		list_del(&inti->list);
		fi->counters[FIRQ_CNTR_PFAULT] -= 1;
	}
	if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
		clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
	spin_unlock(&fi->lock);
856

857
	if (inti) {
858 859 860 861 862 863
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
						 KVM_S390_INT_PFAULT_DONE, 0,
						 inti->ext.ext_params2);
		VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
			   inti->ext.ext_params2);

864 865 866 867 868 869 870 871 872 873 874 875 876 877
		rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
				(u16 *)__LC_EXT_INT_CODE);
		rc |= put_guest_lc(vcpu, PFAULT_DONE,
				(u16 *)__LC_EXT_CPU_ADDR);
		rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
				&vcpu->arch.sie_block->gpsw,
				sizeof(psw_t));
		rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
				&vcpu->arch.sie_block->gpsw,
				sizeof(psw_t));
		rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
				(u64 *)__LC_EXT_PARAMS2);
		kfree(inti);
	}
878
	return rc ? -EFAULT : 0;
879 880
}

881
static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
882
{
883 884 885
	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
	struct kvm_s390_interrupt_info *inti;
	int rc = 0;
886

887 888 889 890 891 892
	spin_lock(&fi->lock);
	inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
					struct kvm_s390_interrupt_info,
					list);
	if (inti) {
		VCPU_EVENT(vcpu, 4,
893
			   "deliver: virtio parm: 0x%x,parm64: 0x%llx",
894 895 896 897 898 899 900 901 902 903 904 905
			   inti->ext.ext_params, inti->ext.ext_params2);
		vcpu->stat.deliver_virtio_interrupt++;
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
				inti->type,
				inti->ext.ext_params,
				inti->ext.ext_params2);
		list_del(&inti->list);
		fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
	}
	if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
		clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
	spin_unlock(&fi->lock);
906

907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
	if (inti) {
		rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
				(u16 *)__LC_EXT_INT_CODE);
		rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
				(u16 *)__LC_EXT_CPU_ADDR);
		rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
				&vcpu->arch.sie_block->gpsw,
				sizeof(psw_t));
		rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
				&vcpu->arch.sie_block->gpsw,
				sizeof(psw_t));
		rc |= put_guest_lc(vcpu, inti->ext.ext_params,
				(u32 *)__LC_EXT_PARAMS);
		rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
				(u64 *)__LC_EXT_PARAMS2);
		kfree(inti);
	}
924
	return rc ? -EFAULT : 0;
925 926
}

927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943
static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
{
	int rc;

	rc  = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
	rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
	rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
	rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
	rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
			     &vcpu->arch.sie_block->gpsw,
			     sizeof(psw_t));
	rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
			    &vcpu->arch.sie_block->gpsw,
			    sizeof(psw_t));
	return rc ? -EFAULT : 0;
}

944
static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
945
				     unsigned long irq_type)
946
{
947 948 949
	struct list_head *isc_list;
	struct kvm_s390_float_interrupt *fi;
	struct kvm_s390_interrupt_info *inti = NULL;
950 951
	struct kvm_s390_io_info io;
	u32 isc;
952
	int rc = 0;
953

954
	fi = &vcpu->kvm->arch.float_int;
955

956
	spin_lock(&fi->lock);
957 958
	isc = irq_type_to_isc(irq_type);
	isc_list = &fi->lists[isc];
959 960 961 962
	inti = list_first_entry_or_null(isc_list,
					struct kvm_s390_interrupt_info,
					list);
	if (inti) {
963 964 965 966 967 968 969 970
		if (inti->type & KVM_S390_INT_IO_AI_MASK)
			VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)");
		else
			VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x",
			inti->io.subchannel_id >> 8,
			inti->io.subchannel_id >> 1 & 0x3,
			inti->io.subchannel_nr);

971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
		vcpu->stat.deliver_io_int++;
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
				inti->type,
				((__u32)inti->io.subchannel_id << 16) |
				inti->io.subchannel_nr,
				((__u64)inti->io.io_int_parm << 32) |
				inti->io.io_int_word);
		list_del(&inti->list);
		fi->counters[FIRQ_CNTR_IO] -= 1;
	}
	if (list_empty(isc_list))
		clear_bit(irq_type, &fi->pending_irqs);
	spin_unlock(&fi->lock);

	if (inti) {
986
		rc = __do_deliver_io(vcpu, &(inti->io));
987
		kfree(inti);
988
		goto out;
989
	}
990

991 992 993 994 995 996 997 998
	if (vcpu->kvm->arch.gisa &&
	    kvm_s390_gisa_tac_ipm_gisc(vcpu->kvm->arch.gisa, isc)) {
		/*
		 * in case an adapter interrupt was not delivered
		 * in SIE context KVM will handle the delivery
		 */
		VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
		memset(&io, 0, sizeof(io));
999
		io.io_int_word = isc_to_int_word(isc);
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
		vcpu->stat.deliver_io_int++;
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
			KVM_S390_INT_IO(1, 0, 0, 0),
			((__u32)io.subchannel_id << 16) |
			io.subchannel_nr,
			((__u64)io.io_int_parm << 32) |
			io.io_int_word);
		rc = __do_deliver_io(vcpu, &io);
	}
out:
	return rc;
1011 1012
}

1013 1014
/* Check whether an external call is pending (deliverable or not) */
int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
1015
{
1016
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1017

1018
	if (!sclp.has_sigpif)
1019
		return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
1020

1021
	return sca_ext_call_pending(vcpu, NULL);
1022 1023
}

1024
int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
1025
{
1026 1027
	if (deliverable_irqs(vcpu))
		return 1;
1028

1029 1030
	if (kvm_cpu_has_pending_timer(vcpu))
		return 1;
1031

1032
	/* external call pending and deliverable */
1033
	if (kvm_s390_ext_call_pending(vcpu) &&
1034 1035
	    !psw_extint_disabled(vcpu) &&
	    (vcpu->arch.sie_block->gcr[0] & 0x2000ul))
1036
		return 1;
1037

1038 1039 1040
	if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
		return 1;
	return 0;
1041 1042
}

1043 1044
int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
{
1045
	return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
1046 1047
}

1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
{
	u64 now, cputm, sltime = 0;

	if (ckc_interrupts_enabled(vcpu)) {
		now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
		sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now);
		/* already expired or overflow? */
		if (!sltime || vcpu->arch.sie_block->ckc <= now)
			return 0;
		if (cpu_timer_interrupts_enabled(vcpu)) {
			cputm = kvm_s390_get_cpu_timer(vcpu);
			/* already expired? */
			if (cputm >> 63)
				return 0;
			return min(sltime, tod_to_ns(cputm));
		}
	} else if (cpu_timer_interrupts_enabled(vcpu)) {
		sltime = kvm_s390_get_cpu_timer(vcpu);
		/* already expired? */
		if (sltime >> 63)
			return 0;
	}
	return sltime;
}

1074 1075
int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
{
1076
	u64 sltime;
1077 1078 1079

	vcpu->stat.exit_wait_state++;

1080
	/* fast path */
1081
	if (kvm_arch_vcpu_runnable(vcpu))
1082
		return 0;
1083

1084 1085
	if (psw_interrupts_disabled(vcpu)) {
		VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
1086
		return -EOPNOTSUPP; /* disabled wait */
1087 1088
	}

1089 1090
	if (!ckc_interrupts_enabled(vcpu) &&
	    !cpu_timer_interrupts_enabled(vcpu)) {
1091
		VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
1092
		__set_cpu_idle(vcpu);
1093 1094 1095
		goto no_timer;
	}

1096 1097
	sltime = __calculate_sltime(vcpu);
	if (!sltime)
1098 1099 1100
		return 0;

	__set_cpu_idle(vcpu);
T
Thomas Gleixner 已提交
1101
	hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL);
1102
	VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime);
1103
no_timer:
1104
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1105
	kvm_vcpu_block(vcpu);
1106
	__unset_cpu_idle(vcpu);
1107 1108
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);

1109
	hrtimer_cancel(&vcpu->arch.ckc_timer);
1110 1111 1112
	return 0;
}

1113 1114
void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
{
1115 1116 1117 1118 1119
	/*
	 * We cannot move this into the if, as the CPU might be already
	 * in kvm_vcpu_block without having the waitqueue set (polling)
	 */
	vcpu->valid_wakeup = true;
1120 1121 1122 1123 1124 1125
	/*
	 * This is mostly to document, that the read in swait_active could
	 * be moved before other stores, leading to subtle races.
	 * All current users do not store or use an atomic like update
	 */
	smp_mb__after_atomic();
1126
	if (swait_active(&vcpu->wq)) {
1127 1128 1129 1130 1131
		/*
		 * The vcpu gave up the cpu voluntarily, mark it as a good
		 * yield-candidate.
		 */
		vcpu->preempted = true;
1132
		swake_up(&vcpu->wq);
1133
		vcpu->stat.halt_wakeup++;
1134
	}
1135 1136 1137 1138 1139
	/*
	 * The VCPU might not be sleeping but is executing the VSIE. Let's
	 * kick it, so it leaves the SIE to process the request.
	 */
	kvm_s390_vsie_kick(vcpu);
1140 1141
}

1142 1143 1144
enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
{
	struct kvm_vcpu *vcpu;
1145
	u64 sltime;
1146 1147

	vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
1148
	sltime = __calculate_sltime(vcpu);
1149

1150 1151 1152 1153
	/*
	 * If the monotonic clock runs faster than the tod clock we might be
	 * woken up too early and have to go back to sleep to avoid deadlocks.
	 */
1154
	if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime)))
1155 1156
		return HRTIMER_RESTART;
	kvm_s390_vcpu_wakeup(vcpu);
1157 1158
	return HRTIMER_NORESTART;
}
1159

1160 1161 1162 1163
void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;

1164
	spin_lock(&li->lock);
1165 1166 1167
	li->pending_irqs = 0;
	bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS);
	memset(&li->irq, 0, sizeof(li->irq));
1168
	spin_unlock(&li->lock);
1169

1170
	sca_clear_ext_call(vcpu);
1171 1172
}

1173
int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
1174
{
1175
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1176
	int rc = 0;
1177
	unsigned long irq_type;
1178
	unsigned long irqs;
1179 1180 1181

	__reset_intercept_indicators(vcpu);

1182 1183
	/* pending ckc conditions might have been invalidated */
	clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1184
	if (ckc_irq_pending(vcpu))
1185 1186
		set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);

1187 1188 1189 1190 1191
	/* pending cpu timer conditions might have been invalidated */
	clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
	if (cpu_timer_irq_pending(vcpu))
		set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);

1192
	while ((irqs = deliverable_irqs(vcpu)) && !rc) {
1193 1194
		/* bits are in the reverse order of interrupt priority */
		irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
1195 1196 1197 1198 1199 1200 1201 1202 1203
		switch (irq_type) {
		case IRQ_PEND_IO_ISC_0:
		case IRQ_PEND_IO_ISC_1:
		case IRQ_PEND_IO_ISC_2:
		case IRQ_PEND_IO_ISC_3:
		case IRQ_PEND_IO_ISC_4:
		case IRQ_PEND_IO_ISC_5:
		case IRQ_PEND_IO_ISC_6:
		case IRQ_PEND_IO_ISC_7:
1204
			rc = __deliver_io(vcpu, irq_type);
1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
			break;
		case IRQ_PEND_MCHK_EX:
		case IRQ_PEND_MCHK_REP:
			rc = __deliver_machine_check(vcpu);
			break;
		case IRQ_PEND_PROG:
			rc = __deliver_prog(vcpu);
			break;
		case IRQ_PEND_EXT_EMERGENCY:
			rc = __deliver_emergency_signal(vcpu);
			break;
		case IRQ_PEND_EXT_EXTERNAL:
			rc = __deliver_external_call(vcpu);
			break;
		case IRQ_PEND_EXT_CLOCK_COMP:
			rc = __deliver_ckc(vcpu);
			break;
		case IRQ_PEND_EXT_CPU_TIMER:
			rc = __deliver_cpu_timer(vcpu);
			break;
		case IRQ_PEND_RESTART:
			rc = __deliver_restart(vcpu);
			break;
		case IRQ_PEND_SET_PREFIX:
			rc = __deliver_set_prefix(vcpu);
			break;
		case IRQ_PEND_PFAULT_INIT:
			rc = __deliver_pfault_init(vcpu);
			break;
		case IRQ_PEND_EXT_SERVICE:
			rc = __deliver_service(vcpu);
			break;
		case IRQ_PEND_PFAULT_DONE:
			rc = __deliver_pfault_done(vcpu);
			break;
		case IRQ_PEND_VIRTIO:
			rc = __deliver_virtio(vcpu);
			break;
		default:
			WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
			clear_bit(irq_type, &li->pending_irqs);
1246
		}
1247
	}
1248

1249
	set_intercept_indicators(vcpu);
1250 1251

	return rc;
1252 1253
}

1254
static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1255 1256 1257
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;

1258 1259 1260 1261
	VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
				   irq->u.pgm.code, 0);

1262 1263 1264 1265 1266 1267 1268
	if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) {
		/* auto detection if no valid ILC was given */
		irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK;
		irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu);
		irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID;
	}

1269 1270
	if (irq->u.pgm.code == PGM_PER) {
		li->irq.pgm.code |= PGM_PER;
1271
		li->irq.pgm.flags = irq->u.pgm.flags;
1272 1273 1274 1275 1276 1277 1278 1279
		/* only modify PER related information */
		li->irq.pgm.per_address = irq->u.pgm.per_address;
		li->irq.pgm.per_code = irq->u.pgm.per_code;
		li->irq.pgm.per_atmid = irq->u.pgm.per_atmid;
		li->irq.pgm.per_access_id = irq->u.pgm.per_access_id;
	} else if (!(irq->u.pgm.code & PGM_PER)) {
		li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) |
				   irq->u.pgm.code;
1280
		li->irq.pgm.flags = irq->u.pgm.flags;
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
		/* only modify non-PER information */
		li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code;
		li->irq.pgm.mon_code = irq->u.pgm.mon_code;
		li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code;
		li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr;
		li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id;
		li->irq.pgm.op_access_id = irq->u.pgm.op_access_id;
	} else {
		li->irq.pgm = irq->u.pgm;
	}
1291
	set_bit(IRQ_PEND_PROG, &li->pending_irqs);
1292 1293 1294
	return 0;
}

1295
static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1296 1297 1298
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;

1299 1300
	VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
		   irq->u.ext.ext_params2);
1301 1302
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
				   irq->u.ext.ext_params,
1303
				   irq->u.ext.ext_params2);
1304 1305 1306

	li->irq.ext = irq->u.ext;
	set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1307
	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1308 1309 1310
	return 0;
}

1311
static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1312 1313
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1314
	struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
1315
	uint16_t src_id = irq->u.extcall.code;
1316

1317
	VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
1318
		   src_id);
1319
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
1320
				   src_id, 0);
1321 1322

	/* sending vcpu invalid */
1323
	if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
1324 1325
		return -EINVAL;

1326
	if (sclp.has_sigpif)
1327
		return sca_inject_ext_call(vcpu, src_id);
1328

1329
	if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1330
		return -EBUSY;
1331
	*extcall = irq->u.extcall;
1332
	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1333 1334 1335
	return 0;
}

1336
static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1337 1338
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1339
	struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
1340

1341
	VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
1342
		   irq->u.prefix.address);
1343
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
1344
				   irq->u.prefix.address, 0);
1345

1346 1347 1348
	if (!is_vcpu_stopped(vcpu))
		return -EBUSY;

1349 1350
	*prefix = irq->u.prefix;
	set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
1351 1352 1353
	return 0;
}

1354
#define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
1355
static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1356 1357
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1358
	struct kvm_s390_stop_info *stop = &li->irq.stop;
1359
	int rc = 0;
1360

1361
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
1362

1363 1364 1365
	if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
		return -EINVAL;

1366 1367 1368 1369 1370 1371 1372 1373 1374
	if (is_vcpu_stopped(vcpu)) {
		if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
			rc = kvm_s390_store_status_unloaded(vcpu,
						KVM_S390_STORE_STATUS_NOADDR);
		return rc;
	}

	if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
		return -EBUSY;
1375
	stop->flags = irq->u.stop.flags;
1376
	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
1377 1378 1379 1380
	return 0;
}

static int __inject_sigp_restart(struct kvm_vcpu *vcpu,
1381
				 struct kvm_s390_irq *irq)
1382 1383 1384
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;

1385
	VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
1386
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
1387 1388

	set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1389 1390 1391 1392
	return 0;
}

static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1393
				   struct kvm_s390_irq *irq)
1394 1395 1396
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;

1397
	VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
1398 1399
		   irq->u.emerg.code);
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
1400
				   irq->u.emerg.code, 0);
1401

1402 1403 1404 1405
	/* sending vcpu invalid */
	if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL)
		return -EINVAL;

1406
	set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
1407
	set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1408
	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1409 1410 1411
	return 0;
}

1412
static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1413 1414
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1415
	struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
1416

1417
	VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
1418
		   irq->u.mchk.mcic);
1419
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
1420
				   irq->u.mchk.mcic);
1421 1422

	/*
1423 1424 1425 1426 1427 1428
	 * Because repressible machine checks can be indicated along with
	 * exigent machine checks (PoP, Chapter 11, Interruption action)
	 * we need to combine cr14, mcic and external damage code.
	 * Failing storage address and the logout area should not be or'ed
	 * together, we just indicate the last occurrence of the corresponding
	 * machine check
1429
	 */
1430
	mchk->cr14 |= irq->u.mchk.cr14;
1431
	mchk->mcic |= irq->u.mchk.mcic;
1432 1433 1434 1435
	mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
	mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
	memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
	       sizeof(mchk->fixed_logout));
1436 1437 1438 1439
	if (mchk->mcic & MCHK_EX_MASK)
		set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
	else if (mchk->mcic & MCHK_REP_MASK)
		set_bit(IRQ_PEND_MCHK_REP,  &li->pending_irqs);
1440 1441 1442
	return 0;
}

1443
static int __inject_ckc(struct kvm_vcpu *vcpu)
1444 1445 1446
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;

1447
	VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
1448
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
1449
				   0, 0);
1450 1451

	set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1452
	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1453 1454 1455
	return 0;
}

1456
static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1457 1458 1459
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;

1460
	VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
1461
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
1462
				   0, 0);
1463 1464

	set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1465
	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1466 1467 1468
	return 0;
}

1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
						  int isc, u32 schid)
{
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
	struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
	struct kvm_s390_interrupt_info *iter;
	u16 id = (schid & 0xffff0000U) >> 16;
	u16 nr = schid & 0x0000ffffU;

	spin_lock(&fi->lock);
	list_for_each_entry(iter, isc_list, list) {
		if (schid && (id != iter->io.subchannel_id ||
			      nr != iter->io.subchannel_nr))
			continue;
		/* found an appropriate entry */
		list_del_init(&iter->list);
		fi->counters[FIRQ_CNTR_IO] -= 1;
		if (list_empty(isc_list))
1487
			clear_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1488 1489 1490 1491 1492 1493
		spin_unlock(&fi->lock);
		return iter;
	}
	spin_unlock(&fi->lock);
	return NULL;
}
1494

1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528
static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
						      u64 isc_mask, u32 schid)
{
	struct kvm_s390_interrupt_info *inti = NULL;
	int isc;

	for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
		if (isc_mask & isc_to_isc_bits(isc))
			inti = get_io_int(kvm, isc, schid);
	}
	return inti;
}

static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
{
	unsigned long active_mask;
	int isc;

	if (schid)
		goto out;
	if (!kvm->arch.gisa)
		goto out;

	active_mask = (isc_mask & kvm_s390_gisa_get_ipm(kvm->arch.gisa) << 24) << 32;
	while (active_mask) {
		isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
		if (kvm_s390_gisa_tac_ipm_gisc(kvm->arch.gisa, isc))
			return isc;
		clear_bit_inv(isc, &active_mask);
	}
out:
	return -EINVAL;
}

1529 1530 1531
/*
 * Dequeue and return an I/O interrupt matching any of the interruption
 * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1532 1533 1534 1535 1536 1537 1538 1539
 * Take into account the interrupts pending in the interrupt list and in GISA.
 *
 * Note that for a guest that does not enable I/O interrupts
 * but relies on TPI, a flood of classic interrupts may starve
 * out adapter interrupts on the same isc. Linux does not do
 * that, and it is possible to work around the issue by configuring
 * different iscs for classic and adapter interrupts in the guest,
 * but we may want to revisit this in the future.
1540
 */
1541
struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1542 1543
						    u64 isc_mask, u32 schid)
{
1544
	struct kvm_s390_interrupt_info *inti, *tmp_inti;
1545 1546
	int isc;

1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562
	inti = get_top_io_int(kvm, isc_mask, schid);

	isc = get_top_gisa_isc(kvm, isc_mask, schid);
	if (isc < 0)
		/* no AI in GISA */
		goto out;

	if (!inti)
		/* AI in GISA but no classical IO int */
		goto gisa_out;

	/* both types of interrupts present */
	if (int_word_to_isc(inti->io.io_int_word) <= isc) {
		/* classical IO int with higher priority */
		kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc);
		goto out;
1563
	}
1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574
gisa_out:
	tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL);
	if (tmp_inti) {
		tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
		tmp_inti->io.io_int_word = isc_to_int_word(isc);
		if (inti)
			kvm_s390_reinject_io_int(kvm, inti);
		inti = tmp_inti;
	} else
		kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc);
out:
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656
	return inti;
}

#define SCCB_MASK 0xFFFFFFF8
#define SCCB_EVENT_PENDING 0x3

static int __inject_service(struct kvm *kvm,
			     struct kvm_s390_interrupt_info *inti)
{
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;

	spin_lock(&fi->lock);
	fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
	/*
	 * Early versions of the QEMU s390 bios will inject several
	 * service interrupts after another without handling a
	 * condition code indicating busy.
	 * We will silently ignore those superfluous sccb values.
	 * A future version of QEMU will take care of serialization
	 * of servc requests
	 */
	if (fi->srv_signal.ext_params & SCCB_MASK)
		goto out;
	fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
	set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
out:
	spin_unlock(&fi->lock);
	kfree(inti);
	return 0;
}

static int __inject_virtio(struct kvm *kvm,
			    struct kvm_s390_interrupt_info *inti)
{
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;

	spin_lock(&fi->lock);
	if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
		spin_unlock(&fi->lock);
		return -EBUSY;
	}
	fi->counters[FIRQ_CNTR_VIRTIO] += 1;
	list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
	set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
	spin_unlock(&fi->lock);
	return 0;
}

static int __inject_pfault_done(struct kvm *kvm,
				 struct kvm_s390_interrupt_info *inti)
{
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;

	spin_lock(&fi->lock);
	if (fi->counters[FIRQ_CNTR_PFAULT] >=
		(ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
		spin_unlock(&fi->lock);
		return -EBUSY;
	}
	fi->counters[FIRQ_CNTR_PFAULT] += 1;
	list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
	set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
	spin_unlock(&fi->lock);
	return 0;
}

#define CR_PENDING_SUBCLASS 28
static int __inject_float_mchk(struct kvm *kvm,
				struct kvm_s390_interrupt_info *inti)
{
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;

	spin_lock(&fi->lock);
	fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
	fi->mchk.mcic |= inti->mchk.mcic;
	set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
	spin_unlock(&fi->lock);
	kfree(inti);
	return 0;
}

static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1657 1658
{
	struct kvm_s390_float_interrupt *fi;
1659 1660
	struct list_head *list;
	int isc;
1661

1662 1663 1664 1665 1666 1667 1668 1669 1670
	isc = int_word_to_isc(inti->io.io_int_word);

	if (kvm->arch.gisa && inti->type & KVM_S390_INT_IO_AI_MASK) {
		VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
		kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc);
		kfree(inti);
		return 0;
	}

1671 1672
	fi = &kvm->arch.float_int;
	spin_lock(&fi->lock);
1673 1674 1675
	if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
		spin_unlock(&fi->lock);
		return -EBUSY;
J
Jens Freimann 已提交
1676
	}
1677 1678
	fi->counters[FIRQ_CNTR_IO] += 1;

1679 1680 1681 1682 1683 1684 1685
	if (inti->type & KVM_S390_INT_IO_AI_MASK)
		VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
	else
		VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
			inti->io.subchannel_id >> 8,
			inti->io.subchannel_id >> 1 & 0x3,
			inti->io.subchannel_nr);
1686 1687
	list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
	list_add_tail(&inti->list, list);
1688
	set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1689
	spin_unlock(&fi->lock);
1690
	return 0;
1691
}
1692

1693 1694 1695 1696
/*
 * Find a destination VCPU for a floating irq and kick it.
 */
static void __floating_irq_kick(struct kvm *kvm, u64 type)
1697
{
1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
	struct kvm_vcpu *dst_vcpu;
	int sigcpu, online_vcpus, nr_tries = 0;

	online_vcpus = atomic_read(&kvm->online_vcpus);
	if (!online_vcpus)
		return;

	/* find idle VCPUs first, then round robin */
	sigcpu = find_first_bit(fi->idle_mask, online_vcpus);
	if (sigcpu == online_vcpus) {
		do {
			sigcpu = fi->next_rr_cpu;
			fi->next_rr_cpu = (fi->next_rr_cpu + 1) % online_vcpus;
			/* avoid endless loops if all vcpus are stopped */
			if (nr_tries++ >= online_vcpus)
				return;
		} while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu)));
	}
	dst_vcpu = kvm_get_vcpu(kvm, sigcpu);

	/* make the VCPU drop out of the SIE, or wake it up if sleeping */
	switch (type) {
	case KVM_S390_MCHK:
1722
		kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
1723 1724
		break;
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1725 1726
		if (!(type & KVM_S390_INT_IO_AI_MASK && kvm->arch.gisa))
			kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
1727 1728
		break;
	default:
1729
		kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
1730 1731 1732 1733 1734 1735 1736
		break;
	}
	kvm_s390_vcpu_wakeup(dst_vcpu);
}

static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
{
1737 1738
	u64 type = READ_ONCE(inti->type);
	int rc;
1739

1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
	switch (type) {
	case KVM_S390_MCHK:
		rc = __inject_float_mchk(kvm, inti);
		break;
	case KVM_S390_INT_VIRTIO:
		rc = __inject_virtio(kvm, inti);
		break;
	case KVM_S390_INT_SERVICE:
		rc = __inject_service(kvm, inti);
		break;
	case KVM_S390_INT_PFAULT_DONE:
		rc = __inject_pfault_done(kvm, inti);
		break;
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
		rc = __inject_io(kvm, inti);
		break;
	default:
J
Jens Freimann 已提交
1757
		rc = -EINVAL;
1758
	}
1759 1760 1761
	if (rc)
		return rc;

1762
	__floating_irq_kick(kvm, type);
1763
	return 0;
1764 1765 1766 1767 1768 1769
}

int kvm_s390_inject_vm(struct kvm *kvm,
		       struct kvm_s390_interrupt *s390int)
{
	struct kvm_s390_interrupt_info *inti;
1770
	int rc;
1771

1772 1773 1774 1775
	inti = kzalloc(sizeof(*inti), GFP_KERNEL);
	if (!inti)
		return -ENOMEM;

1776 1777
	inti->type = s390int->type;
	switch (inti->type) {
1778
	case KVM_S390_INT_VIRTIO:
1779
		VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
1780 1781 1782 1783 1784
			 s390int->parm, s390int->parm64);
		inti->ext.ext_params = s390int->parm;
		inti->ext.ext_params2 = s390int->parm64;
		break;
	case KVM_S390_INT_SERVICE:
1785
		VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
1786 1787
		inti->ext.ext_params = s390int->parm;
		break;
1788 1789 1790
	case KVM_S390_INT_PFAULT_DONE:
		inti->ext.ext_params2 = s390int->parm64;
		break;
1791
	case KVM_S390_MCHK:
1792
		VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
1793 1794 1795 1796
			 s390int->parm64);
		inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
		inti->mchk.mcic = s390int->parm64;
		break;
1797 1798 1799 1800 1801 1802
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
		inti->io.subchannel_id = s390int->parm >> 16;
		inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
		inti->io.io_int_parm = s390int->parm64 >> 32;
		inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
		break;
1803 1804 1805 1806
	default:
		kfree(inti);
		return -EINVAL;
	}
1807 1808
	trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
				 2);
1809

1810 1811 1812 1813
	rc = __inject_vm(kvm, inti);
	if (rc)
		kfree(inti);
	return rc;
1814 1815
}

1816
int kvm_s390_reinject_io_int(struct kvm *kvm,
1817 1818
			      struct kvm_s390_interrupt_info *inti)
{
1819
	return __inject_vm(kvm, inti);
1820 1821
}

1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834
int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
		       struct kvm_s390_irq *irq)
{
	irq->type = s390int->type;
	switch (irq->type) {
	case KVM_S390_PROGRAM_INT:
		if (s390int->parm & 0xffff0000)
			return -EINVAL;
		irq->u.pgm.code = s390int->parm;
		break;
	case KVM_S390_SIGP_SET_PREFIX:
		irq->u.prefix.address = s390int->parm;
		break;
1835 1836 1837
	case KVM_S390_SIGP_STOP:
		irq->u.stop.flags = s390int->parm;
		break;
1838
	case KVM_S390_INT_EXTERNAL_CALL:
1839
		if (s390int->parm & 0xffff0000)
1840 1841 1842 1843
			return -EINVAL;
		irq->u.extcall.code = s390int->parm;
		break;
	case KVM_S390_INT_EMERGENCY:
1844
		if (s390int->parm & 0xffff0000)
1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
			return -EINVAL;
		irq->u.emerg.code = s390int->parm;
		break;
	case KVM_S390_MCHK:
		irq->u.mchk.mcic = s390int->parm64;
		break;
	}
	return 0;
}

1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871
int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;

	return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
}

void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;

	spin_lock(&li->lock);
	li->irq.stop.flags = 0;
	clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
	spin_unlock(&li->lock);
}

1872
static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1873
{
1874
	int rc;
1875

1876
	switch (irq->type) {
1877
	case KVM_S390_PROGRAM_INT:
1878
		rc = __inject_prog(vcpu, irq);
1879
		break;
1880
	case KVM_S390_SIGP_SET_PREFIX:
1881
		rc = __inject_set_prefix(vcpu, irq);
1882
		break;
1883
	case KVM_S390_SIGP_STOP:
1884
		rc = __inject_sigp_stop(vcpu, irq);
1885
		break;
1886
	case KVM_S390_RESTART:
1887
		rc = __inject_sigp_restart(vcpu, irq);
1888
		break;
1889
	case KVM_S390_INT_CLOCK_COMP:
1890
		rc = __inject_ckc(vcpu);
1891
		break;
1892
	case KVM_S390_INT_CPU_TIMER:
1893
		rc = __inject_cpu_timer(vcpu);
1894
		break;
1895
	case KVM_S390_INT_EXTERNAL_CALL:
1896
		rc = __inject_extcall(vcpu, irq);
1897
		break;
1898
	case KVM_S390_INT_EMERGENCY:
1899
		rc = __inject_sigp_emergency(vcpu, irq);
1900
		break;
1901
	case KVM_S390_MCHK:
1902
		rc = __inject_mchk(vcpu, irq);
1903
		break;
1904
	case KVM_S390_INT_PFAULT_INIT:
1905
		rc = __inject_pfault_init(vcpu, irq);
1906
		break;
1907 1908
	case KVM_S390_INT_VIRTIO:
	case KVM_S390_INT_SERVICE:
1909
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1910
	default:
1911
		rc = -EINVAL;
1912
	}
1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923

	return rc;
}

int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	int rc;

	spin_lock(&li->lock);
	rc = do_inject_vcpu(vcpu, irq);
1924
	spin_unlock(&li->lock);
1925 1926 1927
	if (!rc)
		kvm_s390_vcpu_wakeup(vcpu);
	return rc;
1928
}
1929

1930
static inline void clear_irq_list(struct list_head *_list)
1931
{
1932
	struct kvm_s390_interrupt_info *inti, *n;
1933

1934
	list_for_each_entry_safe(inti, n, _list, list) {
1935 1936 1937 1938 1939
		list_del(&inti->list);
		kfree(inti);
	}
}

1940 1941
static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
		       struct kvm_s390_irq *irq)
1942
{
1943
	irq->type = inti->type;
1944
	switch (inti->type) {
1945 1946
	case KVM_S390_INT_PFAULT_INIT:
	case KVM_S390_INT_PFAULT_DONE:
1947
	case KVM_S390_INT_VIRTIO:
1948
		irq->u.ext = inti->ext;
1949 1950
		break;
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1951
		irq->u.io = inti->io;
1952 1953 1954 1955
		break;
	}
}

1956 1957 1958 1959 1960 1961
void kvm_s390_clear_float_irqs(struct kvm *kvm)
{
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
	int i;

	spin_lock(&fi->lock);
1962 1963 1964
	fi->pending_irqs = 0;
	memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
	memset(&fi->mchk, 0, sizeof(fi->mchk));
1965 1966 1967 1968 1969
	for (i = 0; i < FIRQ_LIST_COUNT; i++)
		clear_irq_list(&fi->lists[i]);
	for (i = 0; i < FIRQ_MAX_COUNT; i++)
		fi->counters[i] = 0;
	spin_unlock(&fi->lock);
1970
	kvm_s390_gisa_clear(kvm);
1971 1972
};

1973
static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
1974 1975 1976
{
	struct kvm_s390_interrupt_info *inti;
	struct kvm_s390_float_interrupt *fi;
1977
	struct kvm_s390_irq *buf;
1978
	struct kvm_s390_irq *irq;
1979
	int max_irqs;
1980 1981
	int ret = 0;
	int n = 0;
1982
	int i;
1983

1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997
	if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
		return -EINVAL;

	/*
	 * We are already using -ENOMEM to signal
	 * userspace it may retry with a bigger buffer,
	 * so we need to use something else for this case
	 */
	buf = vzalloc(len);
	if (!buf)
		return -ENOBUFS;

	max_irqs = len / sizeof(struct kvm_s390_irq);

1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013
	if (kvm->arch.gisa &&
	    kvm_s390_gisa_get_ipm(kvm->arch.gisa)) {
		for (i = 0; i <= MAX_ISC; i++) {
			if (n == max_irqs) {
				/* signal userspace to try again */
				ret = -ENOMEM;
				goto out_nolock;
			}
			if (kvm_s390_gisa_tac_ipm_gisc(kvm->arch.gisa, i)) {
				irq = (struct kvm_s390_irq *) &buf[n];
				irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
				irq->u.io.io_int_word = isc_to_int_word(i);
				n++;
			}
		}
	}
2014 2015
	fi = &kvm->arch.float_int;
	spin_lock(&fi->lock);
2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027
	for (i = 0; i < FIRQ_LIST_COUNT; i++) {
		list_for_each_entry(inti, &fi->lists[i], list) {
			if (n == max_irqs) {
				/* signal userspace to try again */
				ret = -ENOMEM;
				goto out;
			}
			inti_to_irq(inti, &buf[n]);
			n++;
		}
	}
	if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs)) {
2028
		if (n == max_irqs) {
2029 2030
			/* signal userspace to try again */
			ret = -ENOMEM;
2031
			goto out;
2032
		}
2033 2034 2035
		irq = (struct kvm_s390_irq *) &buf[n];
		irq->type = KVM_S390_INT_SERVICE;
		irq->u.ext = fi->srv_signal;
2036 2037
		n++;
	}
2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
	if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
		if (n == max_irqs) {
				/* signal userspace to try again */
				ret = -ENOMEM;
				goto out;
		}
		irq = (struct kvm_s390_irq *) &buf[n];
		irq->type = KVM_S390_MCHK;
		irq->u.mchk = fi->mchk;
		n++;
}

out:
2051
	spin_unlock(&fi->lock);
2052
out_nolock:
2053 2054 2055 2056 2057
	if (!ret && n > 0) {
		if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
			ret = -EFAULT;
	}
	vfree(buf);
2058 2059 2060 2061

	return ret < 0 ? ret : n;
}

Y
Yi Min Zhao 已提交
2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083
static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
{
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
	struct kvm_s390_ais_all ais;

	if (attr->attr < sizeof(ais))
		return -EINVAL;

	if (!test_kvm_facility(kvm, 72))
		return -ENOTSUPP;

	mutex_lock(&fi->ais_lock);
	ais.simm = fi->simm;
	ais.nimm = fi->nimm;
	mutex_unlock(&fi->ais_lock);

	if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais)))
		return -EFAULT;

	return 0;
}

2084 2085 2086 2087 2088 2089
static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
{
	int r;

	switch (attr->group) {
	case KVM_DEV_FLIC_GET_ALL_IRQS:
2090
		r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
2091 2092
					  attr->attr);
		break;
Y
Yi Min Zhao 已提交
2093 2094 2095
	case KVM_DEV_FLIC_AISM_ALL:
		r = flic_ais_mode_get_all(dev->kvm, attr);
		break;
2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114
	default:
		r = -EINVAL;
	}

	return r;
}

static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
				     u64 addr)
{
	struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
	void *target = NULL;
	void __user *source;
	u64 size;

	if (get_user(inti->type, (u64 __user *)addr))
		return -EFAULT;

	switch (inti->type) {
2115 2116
	case KVM_S390_INT_PFAULT_INIT:
	case KVM_S390_INT_PFAULT_DONE:
2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164
	case KVM_S390_INT_VIRTIO:
	case KVM_S390_INT_SERVICE:
		target = (void *) &inti->ext;
		source = &uptr->u.ext;
		size = sizeof(inti->ext);
		break;
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
		target = (void *) &inti->io;
		source = &uptr->u.io;
		size = sizeof(inti->io);
		break;
	case KVM_S390_MCHK:
		target = (void *) &inti->mchk;
		source = &uptr->u.mchk;
		size = sizeof(inti->mchk);
		break;
	default:
		return -EINVAL;
	}

	if (copy_from_user(target, source, size))
		return -EFAULT;

	return 0;
}

static int enqueue_floating_irq(struct kvm_device *dev,
				struct kvm_device_attr *attr)
{
	struct kvm_s390_interrupt_info *inti = NULL;
	int r = 0;
	int len = attr->attr;

	if (len % sizeof(struct kvm_s390_irq) != 0)
		return -EINVAL;
	else if (len > KVM_S390_FLIC_MAX_BUFFER)
		return -EINVAL;

	while (len >= sizeof(struct kvm_s390_irq)) {
		inti = kzalloc(sizeof(*inti), GFP_KERNEL);
		if (!inti)
			return -ENOMEM;

		r = copy_irq_from_user(inti, attr->addr);
		if (r) {
			kfree(inti);
			return r;
		}
J
Jens Freimann 已提交
2165 2166 2167 2168 2169
		r = __inject_vm(dev->kvm, inti);
		if (r) {
			kfree(inti);
			return r;
		}
2170 2171 2172 2173 2174 2175 2176
		len -= sizeof(struct kvm_s390_irq);
		attr->addr += sizeof(struct kvm_s390_irq);
	}

	return r;
}

2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
{
	if (id >= MAX_S390_IO_ADAPTERS)
		return NULL;
	return kvm->arch.adapters[id];
}

static int register_io_adapter(struct kvm_device *dev,
			       struct kvm_device_attr *attr)
{
	struct s390_io_adapter *adapter;
	struct kvm_s390_io_adapter adapter_info;

	if (copy_from_user(&adapter_info,
			   (void __user *)attr->addr, sizeof(adapter_info)))
		return -EFAULT;

	if ((adapter_info.id >= MAX_S390_IO_ADAPTERS) ||
	    (dev->kvm->arch.adapters[adapter_info.id] != NULL))
		return -EINVAL;

	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
	if (!adapter)
		return -ENOMEM;

	INIT_LIST_HEAD(&adapter->maps);
	init_rwsem(&adapter->maps_lock);
	atomic_set(&adapter->nr_maps, 0);
	adapter->id = adapter_info.id;
	adapter->isc = adapter_info.isc;
	adapter->maskable = adapter_info.maskable;
	adapter->masked = false;
	adapter->swap = adapter_info.swap;
2210 2211
	adapter->suppressible = (adapter_info.flags) &
				KVM_S390_ADAPTER_SUPPRESSIBLE;
2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244
	dev->kvm->arch.adapters[adapter->id] = adapter;

	return 0;
}

int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
{
	int ret;
	struct s390_io_adapter *adapter = get_io_adapter(kvm, id);

	if (!adapter || !adapter->maskable)
		return -EINVAL;
	ret = adapter->masked;
	adapter->masked = masked;
	return ret;
}

static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr)
{
	struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
	struct s390_map_info *map;
	int ret;

	if (!adapter || !addr)
		return -EINVAL;

	map = kzalloc(sizeof(*map), GFP_KERNEL);
	if (!map) {
		ret = -ENOMEM;
		goto out;
	}
	INIT_LIST_HEAD(&map->list);
	map->guest_addr = addr;
2245
	map->addr = gmap_translate(kvm->arch.gmap, addr);
2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343
	if (map->addr == -EFAULT) {
		ret = -EFAULT;
		goto out;
	}
	ret = get_user_pages_fast(map->addr, 1, 1, &map->page);
	if (ret < 0)
		goto out;
	BUG_ON(ret != 1);
	down_write(&adapter->maps_lock);
	if (atomic_inc_return(&adapter->nr_maps) < MAX_S390_ADAPTER_MAPS) {
		list_add_tail(&map->list, &adapter->maps);
		ret = 0;
	} else {
		put_page(map->page);
		ret = -EINVAL;
	}
	up_write(&adapter->maps_lock);
out:
	if (ret)
		kfree(map);
	return ret;
}

static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr)
{
	struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
	struct s390_map_info *map, *tmp;
	int found = 0;

	if (!adapter || !addr)
		return -EINVAL;

	down_write(&adapter->maps_lock);
	list_for_each_entry_safe(map, tmp, &adapter->maps, list) {
		if (map->guest_addr == addr) {
			found = 1;
			atomic_dec(&adapter->nr_maps);
			list_del(&map->list);
			put_page(map->page);
			kfree(map);
			break;
		}
	}
	up_write(&adapter->maps_lock);

	return found ? 0 : -EINVAL;
}

void kvm_s390_destroy_adapters(struct kvm *kvm)
{
	int i;
	struct s390_map_info *map, *tmp;

	for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) {
		if (!kvm->arch.adapters[i])
			continue;
		list_for_each_entry_safe(map, tmp,
					 &kvm->arch.adapters[i]->maps, list) {
			list_del(&map->list);
			put_page(map->page);
			kfree(map);
		}
		kfree(kvm->arch.adapters[i]);
	}
}

static int modify_io_adapter(struct kvm_device *dev,
			     struct kvm_device_attr *attr)
{
	struct kvm_s390_io_adapter_req req;
	struct s390_io_adapter *adapter;
	int ret;

	if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
		return -EFAULT;

	adapter = get_io_adapter(dev->kvm, req.id);
	if (!adapter)
		return -EINVAL;
	switch (req.type) {
	case KVM_S390_IO_ADAPTER_MASK:
		ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
		if (ret > 0)
			ret = 0;
		break;
	case KVM_S390_IO_ADAPTER_MAP:
		ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr);
		break;
	case KVM_S390_IO_ADAPTER_UNMAP:
		ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr);
		break;
	default:
		ret = -EINVAL;
	}

	return ret;
}

2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355
static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)

{
	const u64 isc_mask = 0xffUL << 24; /* all iscs set */
	u32 schid;

	if (attr->flags)
		return -EINVAL;
	if (attr->attr != sizeof(schid))
		return -EINVAL;
	if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid)))
		return -EFAULT;
2356 2357
	if (!schid)
		return -EINVAL;
2358 2359 2360 2361 2362 2363 2364 2365 2366
	kfree(kvm_s390_get_io_int(kvm, isc_mask, schid));
	/*
	 * If userspace is conforming to the architecture, we can have at most
	 * one pending I/O interrupt per subchannel, so this is effectively a
	 * clear all.
	 */
	return 0;
}

2367 2368 2369 2370 2371 2372
static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
{
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
	struct kvm_s390_ais_req req;
	int ret = 0;

2373
	if (!test_kvm_facility(kvm, 72))
2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405
		return -ENOTSUPP;

	if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
		return -EFAULT;

	if (req.isc > MAX_ISC)
		return -EINVAL;

	trace_kvm_s390_modify_ais_mode(req.isc,
				       (fi->simm & AIS_MODE_MASK(req.isc)) ?
				       (fi->nimm & AIS_MODE_MASK(req.isc)) ?
				       2 : KVM_S390_AIS_MODE_SINGLE :
				       KVM_S390_AIS_MODE_ALL, req.mode);

	mutex_lock(&fi->ais_lock);
	switch (req.mode) {
	case KVM_S390_AIS_MODE_ALL:
		fi->simm &= ~AIS_MODE_MASK(req.isc);
		fi->nimm &= ~AIS_MODE_MASK(req.isc);
		break;
	case KVM_S390_AIS_MODE_SINGLE:
		fi->simm |= AIS_MODE_MASK(req.isc);
		fi->nimm &= ~AIS_MODE_MASK(req.isc);
		break;
	default:
		ret = -EINVAL;
	}
	mutex_unlock(&fi->ais_lock);

	return ret;
}

2406 2407 2408 2409 2410 2411 2412
static int kvm_s390_inject_airq(struct kvm *kvm,
				struct s390_io_adapter *adapter)
{
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
	struct kvm_s390_interrupt s390int = {
		.type = KVM_S390_INT_IO(1, 0, 0, 0),
		.parm = 0,
2413
		.parm64 = isc_to_int_word(adapter->isc),
2414 2415 2416
	};
	int ret = 0;

2417
	if (!test_kvm_facility(kvm, 72) || !adapter->suppressible)
2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447
		return kvm_s390_inject_vm(kvm, &s390int);

	mutex_lock(&fi->ais_lock);
	if (fi->nimm & AIS_MODE_MASK(adapter->isc)) {
		trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc);
		goto out;
	}

	ret = kvm_s390_inject_vm(kvm, &s390int);
	if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) {
		fi->nimm |= AIS_MODE_MASK(adapter->isc);
		trace_kvm_s390_modify_ais_mode(adapter->isc,
					       KVM_S390_AIS_MODE_SINGLE, 2);
	}
out:
	mutex_unlock(&fi->ais_lock);
	return ret;
}

static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
{
	unsigned int id = attr->attr;
	struct s390_io_adapter *adapter = get_io_adapter(kvm, id);

	if (!adapter)
		return -EINVAL;

	return kvm_s390_inject_airq(kvm, adapter);
}

Y
Yi Min Zhao 已提交
2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466
static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
{
	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
	struct kvm_s390_ais_all ais;

	if (!test_kvm_facility(kvm, 72))
		return -ENOTSUPP;

	if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais)))
		return -EFAULT;

	mutex_lock(&fi->ais_lock);
	fi->simm = ais.simm;
	fi->nimm = ais.nimm;
	mutex_unlock(&fi->ais_lock);

	return 0;
}

2467 2468 2469
static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
{
	int r = 0;
2470 2471
	unsigned int i;
	struct kvm_vcpu *vcpu;
2472 2473 2474 2475 2476 2477

	switch (attr->group) {
	case KVM_DEV_FLIC_ENQUEUE:
		r = enqueue_floating_irq(dev, attr);
		break;
	case KVM_DEV_FLIC_CLEAR_IRQS:
2478
		kvm_s390_clear_float_irqs(dev->kvm);
2479
		break;
2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493
	case KVM_DEV_FLIC_APF_ENABLE:
		dev->kvm->arch.gmap->pfault_enabled = 1;
		break;
	case KVM_DEV_FLIC_APF_DISABLE_WAIT:
		dev->kvm->arch.gmap->pfault_enabled = 0;
		/*
		 * Make sure no async faults are in transition when
		 * clearing the queues. So we don't need to worry
		 * about late coming workers.
		 */
		synchronize_srcu(&dev->kvm->srcu);
		kvm_for_each_vcpu(i, vcpu, dev->kvm)
			kvm_clear_async_pf_completion_queue(vcpu);
		break;
2494 2495 2496 2497 2498 2499
	case KVM_DEV_FLIC_ADAPTER_REGISTER:
		r = register_io_adapter(dev, attr);
		break;
	case KVM_DEV_FLIC_ADAPTER_MODIFY:
		r = modify_io_adapter(dev, attr);
		break;
2500 2501 2502
	case KVM_DEV_FLIC_CLEAR_IO_IRQ:
		r = clear_io_irq(dev->kvm, attr);
		break;
2503 2504 2505
	case KVM_DEV_FLIC_AISM:
		r = modify_ais_mode(dev->kvm, attr);
		break;
2506 2507 2508
	case KVM_DEV_FLIC_AIRQ_INJECT:
		r = flic_inject_airq(dev->kvm, attr);
		break;
Y
Yi Min Zhao 已提交
2509 2510 2511
	case KVM_DEV_FLIC_AISM_ALL:
		r = flic_ais_mode_set_all(dev->kvm, attr);
		break;
2512 2513 2514 2515 2516 2517 2518
	default:
		r = -EINVAL;
	}

	return r;
}

2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529
static int flic_has_attr(struct kvm_device *dev,
			     struct kvm_device_attr *attr)
{
	switch (attr->group) {
	case KVM_DEV_FLIC_GET_ALL_IRQS:
	case KVM_DEV_FLIC_ENQUEUE:
	case KVM_DEV_FLIC_CLEAR_IRQS:
	case KVM_DEV_FLIC_APF_ENABLE:
	case KVM_DEV_FLIC_APF_DISABLE_WAIT:
	case KVM_DEV_FLIC_ADAPTER_REGISTER:
	case KVM_DEV_FLIC_ADAPTER_MODIFY:
2530
	case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2531
	case KVM_DEV_FLIC_AISM:
2532
	case KVM_DEV_FLIC_AIRQ_INJECT:
Y
Yi Min Zhao 已提交
2533
	case KVM_DEV_FLIC_AISM_ALL:
2534 2535 2536 2537 2538
		return 0;
	}
	return -ENXIO;
}

2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559
static int flic_create(struct kvm_device *dev, u32 type)
{
	if (!dev)
		return -EINVAL;
	if (dev->kvm->arch.flic)
		return -EINVAL;
	dev->kvm->arch.flic = dev;
	return 0;
}

static void flic_destroy(struct kvm_device *dev)
{
	dev->kvm->arch.flic = NULL;
	kfree(dev);
}

/* s390 floating irq controller (flic) */
struct kvm_device_ops kvm_flic_ops = {
	.name = "kvm-flic",
	.get_attr = flic_get_attr,
	.set_attr = flic_set_attr,
2560
	.has_attr = flic_has_attr,
2561 2562 2563
	.create = flic_create,
	.destroy = flic_destroy,
};
2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643

static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
{
	unsigned long bit;

	bit = bit_nr + (addr % PAGE_SIZE) * 8;

	return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
}

static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter,
					  u64 addr)
{
	struct s390_map_info *map;

	if (!adapter)
		return NULL;

	list_for_each_entry(map, &adapter->maps, list) {
		if (map->guest_addr == addr)
			return map;
	}
	return NULL;
}

static int adapter_indicators_set(struct kvm *kvm,
				  struct s390_io_adapter *adapter,
				  struct kvm_s390_adapter_int *adapter_int)
{
	unsigned long bit;
	int summary_set, idx;
	struct s390_map_info *info;
	void *map;

	info = get_map_info(adapter, adapter_int->ind_addr);
	if (!info)
		return -1;
	map = page_address(info->page);
	bit = get_ind_bit(info->addr, adapter_int->ind_offset, adapter->swap);
	set_bit(bit, map);
	idx = srcu_read_lock(&kvm->srcu);
	mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT);
	set_page_dirty_lock(info->page);
	info = get_map_info(adapter, adapter_int->summary_addr);
	if (!info) {
		srcu_read_unlock(&kvm->srcu, idx);
		return -1;
	}
	map = page_address(info->page);
	bit = get_ind_bit(info->addr, adapter_int->summary_offset,
			  adapter->swap);
	summary_set = test_and_set_bit(bit, map);
	mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT);
	set_page_dirty_lock(info->page);
	srcu_read_unlock(&kvm->srcu, idx);
	return summary_set ? 0 : 1;
}

/*
 * < 0 - not injected due to error
 * = 0 - coalesced, summary indicator already active
 * > 0 - injected interrupt
 */
static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
			   struct kvm *kvm, int irq_source_id, int level,
			   bool line_status)
{
	int ret;
	struct s390_io_adapter *adapter;

	/* We're only interested in the 0->1 transition. */
	if (!level)
		return 0;
	adapter = get_io_adapter(kvm, e->adapter.adapter_id);
	if (!adapter)
		return -1;
	down_read(&adapter->maps_lock);
	ret = adapter_indicators_set(kvm, adapter, &e->adapter);
	up_read(&adapter->maps_lock);
	if ((ret > 0) && !adapter->masked) {
2644
		ret = kvm_s390_inject_airq(kvm, adapter);
2645 2646 2647 2648 2649 2650
		if (ret == 0)
			ret = 1;
	}
	return ret;
}

2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661
/*
 * Inject the machine check to the guest.
 */
void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
				     struct mcck_volatile_info *mcck_info)
{
	struct kvm_s390_interrupt_info inti;
	struct kvm_s390_irq irq;
	struct kvm_s390_mchk_info *mchk;
	union mci mci;
	__u64 cr14 = 0;         /* upper bits are not used */
2662
	int rc;
2663 2664 2665

	mci.val = mcck_info->mcic;
	if (mci.sr)
2666
		cr14 |= CR14_RECOVERY_SUBMASK;
2667
	if (mci.dg)
2668
		cr14 |= CR14_DEGRADATION_SUBMASK;
2669
	if (mci.w)
2670
		cr14 |= CR14_WARNING_SUBMASK;
2671 2672 2673 2674 2675 2676 2677 2678 2679

	mchk = mci.ck ? &inti.mchk : &irq.u.mchk;
	mchk->cr14 = cr14;
	mchk->mcic = mcck_info->mcic;
	mchk->ext_damage_code = mcck_info->ext_damage_code;
	mchk->failing_storage_address = mcck_info->failing_storage_address;
	if (mci.ck) {
		/* Inject the floating machine check */
		inti.type = KVM_S390_MCHK;
2680
		rc = __inject_vm(vcpu->kvm, &inti);
2681 2682 2683
	} else {
		/* Inject the machine check to specified vcpu */
		irq.type = KVM_S390_MCHK;
2684
		rc = kvm_s390_inject_vcpu(vcpu, &irq);
2685
	}
2686
	WARN_ON_ONCE(rc);
2687 2688
}

2689 2690
int kvm_set_routing_entry(struct kvm *kvm,
			  struct kvm_kernel_irq_routing_entry *e,
2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716
			  const struct kvm_irq_routing_entry *ue)
{
	int ret;

	switch (ue->type) {
	case KVM_IRQ_ROUTING_S390_ADAPTER:
		e->set = set_adapter_int;
		e->adapter.summary_addr = ue->u.adapter.summary_addr;
		e->adapter.ind_addr = ue->u.adapter.ind_addr;
		e->adapter.summary_offset = ue->u.adapter.summary_offset;
		e->adapter.ind_offset = ue->u.adapter.ind_offset;
		e->adapter.adapter_id = ue->u.adapter.adapter_id;
		ret = 0;
		break;
	default:
		ret = -EINVAL;
	}

	return ret;
}

int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
		int irq_source_id, int level, bool line_status)
{
	return -EINVAL;
}
2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801

int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_irq *buf;
	int r = 0;
	int n;

	buf = vmalloc(len);
	if (!buf)
		return -ENOMEM;

	if (copy_from_user((void *) buf, irqstate, len)) {
		r = -EFAULT;
		goto out_free;
	}

	/*
	 * Don't allow setting the interrupt state
	 * when there are already interrupts pending
	 */
	spin_lock(&li->lock);
	if (li->pending_irqs) {
		r = -EBUSY;
		goto out_unlock;
	}

	for (n = 0; n < len / sizeof(*buf); n++) {
		r = do_inject_vcpu(vcpu, &buf[n]);
		if (r)
			break;
	}

out_unlock:
	spin_unlock(&li->lock);
out_free:
	vfree(buf);

	return r;
}

static void store_local_irq(struct kvm_s390_local_interrupt *li,
			    struct kvm_s390_irq *irq,
			    unsigned long irq_type)
{
	switch (irq_type) {
	case IRQ_PEND_MCHK_EX:
	case IRQ_PEND_MCHK_REP:
		irq->type = KVM_S390_MCHK;
		irq->u.mchk = li->irq.mchk;
		break;
	case IRQ_PEND_PROG:
		irq->type = KVM_S390_PROGRAM_INT;
		irq->u.pgm = li->irq.pgm;
		break;
	case IRQ_PEND_PFAULT_INIT:
		irq->type = KVM_S390_INT_PFAULT_INIT;
		irq->u.ext = li->irq.ext;
		break;
	case IRQ_PEND_EXT_EXTERNAL:
		irq->type = KVM_S390_INT_EXTERNAL_CALL;
		irq->u.extcall = li->irq.extcall;
		break;
	case IRQ_PEND_EXT_CLOCK_COMP:
		irq->type = KVM_S390_INT_CLOCK_COMP;
		break;
	case IRQ_PEND_EXT_CPU_TIMER:
		irq->type = KVM_S390_INT_CPU_TIMER;
		break;
	case IRQ_PEND_SIGP_STOP:
		irq->type = KVM_S390_SIGP_STOP;
		irq->u.stop = li->irq.stop;
		break;
	case IRQ_PEND_RESTART:
		irq->type = KVM_S390_RESTART;
		break;
	case IRQ_PEND_SET_PREFIX:
		irq->type = KVM_S390_SIGP_SET_PREFIX;
		irq->u.prefix = li->irq.prefix;
		break;
	}
}

int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
{
2802
	int scn;
2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841
	unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)];
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	unsigned long pending_irqs;
	struct kvm_s390_irq irq;
	unsigned long irq_type;
	int cpuaddr;
	int n = 0;

	spin_lock(&li->lock);
	pending_irqs = li->pending_irqs;
	memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
	       sizeof(sigp_emerg_pending));
	spin_unlock(&li->lock);

	for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
		memset(&irq, 0, sizeof(irq));
		if (irq_type == IRQ_PEND_EXT_EMERGENCY)
			continue;
		if (n + sizeof(irq) > len)
			return -ENOBUFS;
		store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
		if (copy_to_user(&buf[n], &irq, sizeof(irq)))
			return -EFAULT;
		n += sizeof(irq);
	}

	if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
		for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
			memset(&irq, 0, sizeof(irq));
			if (n + sizeof(irq) > len)
				return -ENOBUFS;
			irq.type = KVM_S390_INT_EMERGENCY;
			irq.u.emerg.code = cpuaddr;
			if (copy_to_user(&buf[n], &irq, sizeof(irq)))
				return -EFAULT;
			n += sizeof(irq);
		}
	}

2842
	if (sca_ext_call_pending(vcpu, &scn)) {
2843 2844 2845 2846
		if (n + sizeof(irq) > len)
			return -ENOBUFS;
		memset(&irq, 0, sizeof(irq));
		irq.type = KVM_S390_INT_EXTERNAL_CALL;
2847
		irq.u.extcall.code = scn;
2848 2849 2850 2851 2852 2853 2854
		if (copy_to_user(&buf[n], &irq, sizeof(irq)))
			return -EFAULT;
		n += sizeof(irq);
	}

	return n;
}
2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866

void kvm_s390_gisa_clear(struct kvm *kvm)
{
	if (kvm->arch.gisa) {
		memset(kvm->arch.gisa, 0, sizeof(struct kvm_s390_gisa));
		kvm->arch.gisa->next_alert = (u32)(u64)kvm->arch.gisa;
		VM_EVENT(kvm, 3, "gisa 0x%pK cleared", kvm->arch.gisa);
	}
}

void kvm_s390_gisa_init(struct kvm *kvm)
{
2867
	if (css_general_characteristics.aiv) {
2868 2869 2870 2871
		kvm->arch.gisa = &kvm->arch.sie_page2->gisa;
		VM_EVENT(kvm, 3, "gisa 0x%pK initialized", kvm->arch.gisa);
		kvm_s390_gisa_clear(kvm);
	}
2872 2873 2874 2875 2876 2877 2878 2879
}

void kvm_s390_gisa_destroy(struct kvm *kvm)
{
	if (!kvm->arch.gisa)
		return;
	kvm->arch.gisa = NULL;
}