interrupt.c 23.7 KB
Newer Older
1
/*
2
 * handling kvm guest interrupts
3 4 5 6 7 8 9 10 11 12
 *
 * Copyright IBM Corp. 2008
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License (version 2 only)
 * as published by the Free Software Foundation.
 *
 *    Author(s): Carsten Otte <cotte@de.ibm.com>
 */

13
#include <linux/interrupt.h>
14
#include <linux/kvm_host.h>
15
#include <linux/hrtimer.h>
16
#include <linux/signal.h>
17
#include <linux/slab.h>
18 19
#include <asm/asm-offsets.h>
#include <asm/uaccess.h>
20 21
#include "kvm-s390.h"
#include "gaccess.h"
22
#include "trace-s390.h"
23

24 25 26 27 28 29 30 31 32 33
#define IOINT_SCHID_MASK 0x0000ffff
#define IOINT_SSID_MASK 0x00030000
#define IOINT_CSSID_MASK 0x03fc0000
#define IOINT_AI_MASK 0x04000000

static int is_ioint(u64 type)
{
	return ((type & 0xfffe0000u) != 0xfffe0000u);
}

34 35 36 37 38
static int psw_extint_disabled(struct kvm_vcpu *vcpu)
{
	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
}

39 40 41 42 43
static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
{
	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
}

44 45 46 47 48
static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
{
	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
}

49 50 51 52 53 54 55 56 57
static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
{
	if ((vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PER) ||
	    (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO) ||
	    (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT))
		return 0;
	return 1;
}

C
Cornelia Huck 已提交
58 59 60 61 62 63 64
static u64 int_word_to_isc_bits(u32 int_word)
{
	u8 isc = (int_word & 0x38000000) >> 27;

	return (0x80 >> isc) << 24;
}

65
static int __interrupt_is_deliverable(struct kvm_vcpu *vcpu,
66
				      struct kvm_s390_interrupt_info *inti)
67 68
{
	switch (inti->type) {
69 70 71 72 73
	case KVM_S390_INT_EXTERNAL_CALL:
		if (psw_extint_disabled(vcpu))
			return 0;
		if (vcpu->arch.sie_block->gcr[0] & 0x2000ul)
			return 1;
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
	case KVM_S390_INT_EMERGENCY:
		if (psw_extint_disabled(vcpu))
			return 0;
		if (vcpu->arch.sie_block->gcr[0] & 0x4000ul)
			return 1;
		return 0;
	case KVM_S390_INT_SERVICE:
		if (psw_extint_disabled(vcpu))
			return 0;
		if (vcpu->arch.sie_block->gcr[0] & 0x200ul)
			return 1;
		return 0;
	case KVM_S390_INT_VIRTIO:
		if (psw_extint_disabled(vcpu))
			return 0;
		if (vcpu->arch.sie_block->gcr[0] & 0x200ul)
			return 1;
		return 0;
	case KVM_S390_PROGRAM_INT:
	case KVM_S390_SIGP_STOP:
	case KVM_S390_SIGP_SET_PREFIX:
	case KVM_S390_RESTART:
		return 1;
97 98 99 100 101 102
	case KVM_S390_MCHK:
		if (psw_mchk_disabled(vcpu))
			return 0;
		if (vcpu->arch.sie_block->gcr[14] & inti->mchk.cr14)
			return 1;
		return 0;
103 104 105
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
		if (psw_ioint_disabled(vcpu))
			return 0;
C
Cornelia Huck 已提交
106 107
		if (vcpu->arch.sie_block->gcr[6] &
		    int_word_to_isc_bits(inti->io.io_int_word))
108 109
			return 1;
		return 0;
110
	default:
111 112
		printk(KERN_WARNING "illegal interrupt type %llx\n",
		       inti->type);
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
		BUG();
	}
	return 0;
}

static void __set_cpu_idle(struct kvm_vcpu *vcpu)
{
	BUG_ON(vcpu->vcpu_id > KVM_MAX_VCPUS - 1);
	atomic_set_mask(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
	set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
}

static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
{
	BUG_ON(vcpu->vcpu_id > KVM_MAX_VCPUS - 1);
	atomic_clear_mask(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
	clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
}

static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
{
	atomic_clear_mask(CPUSTAT_ECALL_PEND |
		CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT,
		&vcpu->arch.sie_block->cpuflags);
	vcpu->arch.sie_block->lctl = 0x0000;
138
	vcpu->arch.sie_block->ictl &= ~ICTL_LPSW;
139 140 141 142 143 144 145 146
}

static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag)
{
	atomic_set_mask(flag, &vcpu->arch.sie_block->cpuflags);
}

static void __set_intercept_indicator(struct kvm_vcpu *vcpu,
147
				      struct kvm_s390_interrupt_info *inti)
148 149
{
	switch (inti->type) {
150
	case KVM_S390_INT_EXTERNAL_CALL:
151 152 153 154 155 156 157 158 159 160 161
	case KVM_S390_INT_EMERGENCY:
	case KVM_S390_INT_SERVICE:
	case KVM_S390_INT_VIRTIO:
		if (psw_extint_disabled(vcpu))
			__set_cpuflag(vcpu, CPUSTAT_EXT_INT);
		else
			vcpu->arch.sie_block->lctl |= LCTL_CR0;
		break;
	case KVM_S390_SIGP_STOP:
		__set_cpuflag(vcpu, CPUSTAT_STOP_INT);
		break;
162 163 164 165 166 167
	case KVM_S390_MCHK:
		if (psw_mchk_disabled(vcpu))
			vcpu->arch.sie_block->ictl |= ICTL_LPSW;
		else
			vcpu->arch.sie_block->lctl |= LCTL_CR14;
		break;
168 169 170 171 172 173
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
		if (psw_ioint_disabled(vcpu))
			__set_cpuflag(vcpu, CPUSTAT_IO_INT);
		else
			vcpu->arch.sie_block->lctl |= LCTL_CR6;
		break;
174 175 176 177 178 179
	default:
		BUG();
	}
}

static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
180
				   struct kvm_s390_interrupt_info *inti)
181 182
{
	const unsigned short table[] = { 2, 4, 4, 6 };
183
	int rc = 0;
184 185 186 187 188

	switch (inti->type) {
	case KVM_S390_INT_EMERGENCY:
		VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp emerg");
		vcpu->stat.deliver_emergency_signal++;
189 190
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->emerg.code, 0);
191
		rc  = put_guest(vcpu, 0x1201, (u16 __user *)__LC_EXT_INT_CODE);
192
		rc |= put_guest(vcpu, inti->emerg.code,
193
				(u16 __user *)__LC_EXT_CPU_ADDR);
194 195 196 197
		rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_EXT_NEW_PSW, sizeof(psw_t));
198
		break;
199 200 201
	case KVM_S390_INT_EXTERNAL_CALL:
		VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp ext call");
		vcpu->stat.deliver_external_call++;
202 203
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->extcall.code, 0);
204
		rc  = put_guest(vcpu, 0x1202, (u16 __user *)__LC_EXT_INT_CODE);
205
		rc |= put_guest(vcpu, inti->extcall.code,
206
				(u16 __user *)__LC_EXT_CPU_ADDR);
207 208 209 210
		rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_EXT_NEW_PSW, sizeof(psw_t));
211
		break;
212 213 214 215
	case KVM_S390_INT_SERVICE:
		VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x",
			   inti->ext.ext_params);
		vcpu->stat.deliver_service_signal++;
216 217
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->ext.ext_params, 0);
218
		rc  = put_guest(vcpu, 0x2401, (u16 __user *)__LC_EXT_INT_CODE);
219 220 221 222
		rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_EXT_NEW_PSW, sizeof(psw_t));
223
		rc |= put_guest(vcpu, inti->ext.ext_params,
224
				(u32 __user *)__LC_EXT_PARAMS);
225 226
		break;
	case KVM_S390_INT_VIRTIO:
227
		VCPU_EVENT(vcpu, 4, "interrupt: virtio parm:%x,parm64:%llx",
228 229
			   inti->ext.ext_params, inti->ext.ext_params2);
		vcpu->stat.deliver_virtio_interrupt++;
230 231 232
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->ext.ext_params,
						 inti->ext.ext_params2);
233 234
		rc  = put_guest(vcpu, 0x2603, (u16 __user *)__LC_EXT_INT_CODE);
		rc |= put_guest(vcpu, 0x0d00, (u16 __user *)__LC_EXT_CPU_ADDR);
235 236 237 238
		rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_EXT_NEW_PSW, sizeof(psw_t));
239
		rc |= put_guest(vcpu, inti->ext.ext_params,
240
				(u32 __user *)__LC_EXT_PARAMS);
241
		rc |= put_guest(vcpu, inti->ext.ext_params2,
242
				(u64 __user *)__LC_EXT_PARAMS2);
243 244 245 246
		break;
	case KVM_S390_SIGP_STOP:
		VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu stop");
		vcpu->stat.deliver_stop_signal++;
247 248
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 0, 0);
249 250 251 252 253 254 255
		__set_intercept_indicator(vcpu, inti);
		break;

	case KVM_S390_SIGP_SET_PREFIX:
		VCPU_EVENT(vcpu, 4, "interrupt: set prefix to %x",
			   inti->prefix.address);
		vcpu->stat.deliver_prefix_signal++;
256 257
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->prefix.address, 0);
258
		kvm_s390_set_prefix(vcpu, inti->prefix.address);
259 260 261 262 263
		break;

	case KVM_S390_RESTART:
		VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu restart");
		vcpu->stat.deliver_restart_signal++;
264 265
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 0, 0);
266 267 268 269 270 271
		rc  = copy_to_guest(vcpu,
				    offsetof(struct _lowcore, restart_old_psw),
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      offsetof(struct _lowcore, restart_psw),
				      sizeof(psw_t));
272
		atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
273 274 275 276 277 278
		break;
	case KVM_S390_PROGRAM_INT:
		VCPU_EVENT(vcpu, 4, "interrupt: pgm check code:%x, ilc:%x",
			   inti->pgm.code,
			   table[vcpu->arch.sie_block->ipa >> 14]);
		vcpu->stat.deliver_program_int++;
279 280
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->pgm.code, 0);
281
		rc  = put_guest(vcpu, inti->pgm.code, (u16 __user *)__LC_PGM_INT_CODE);
282
		rc |= put_guest(vcpu, table[vcpu->arch.sie_block->ipa >> 14],
283
				(u16 __user *)__LC_PGM_ILC);
284 285 286 287
		rc |= copy_to_guest(vcpu, __LC_PGM_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_PGM_NEW_PSW, sizeof(psw_t));
288 289
		break;

290 291 292 293 294 295
	case KVM_S390_MCHK:
		VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx",
			   inti->mchk.mcic);
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->mchk.cr14,
						 inti->mchk.mcic);
296 297
		rc  = kvm_s390_vcpu_store_status(vcpu,
						 KVM_S390_STORE_STATUS_PREFIXED);
298
		rc |= put_guest(vcpu, inti->mchk.mcic, (u64 __user *) __LC_MCCK_CODE);
299 300 301 302
		rc |= copy_to_guest(vcpu, __LC_MCK_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_MCK_NEW_PSW, sizeof(psw_t));
303 304
		break;

305 306 307 308 309 310 311 312 313 314
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
	{
		__u32 param0 = ((__u32)inti->io.subchannel_id << 16) |
			inti->io.subchannel_nr;
		__u64 param1 = ((__u64)inti->io.io_int_parm << 32) |
			inti->io.io_int_word;
		VCPU_EVENT(vcpu, 4, "interrupt: I/O %llx", inti->type);
		vcpu->stat.deliver_io_int++;
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 param0, param1);
315
		rc  = put_guest(vcpu, inti->io.subchannel_id,
316
				(u16 __user *) __LC_SUBCHANNEL_ID);
317
		rc |= put_guest(vcpu, inti->io.subchannel_nr,
318
				(u16 __user *) __LC_SUBCHANNEL_NR);
319
		rc |= put_guest(vcpu, inti->io.io_int_parm,
320
				(u32 __user *) __LC_IO_INT_PARM);
321
		rc |= put_guest(vcpu, inti->io.io_int_word,
322
				(u32 __user *) __LC_IO_INT_WORD);
323 324 325 326
		rc |= copy_to_guest(vcpu, __LC_IO_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_IO_NEW_PSW, sizeof(psw_t));
327 328
		break;
	}
329 330 331
	default:
		BUG();
	}
332
	if (rc) {
333
		printk("kvm: The guest lowcore is not mapped during interrupt "
334
		       "delivery, killing userspace\n");
335
		do_exit(SIGKILL);
336 337 338 339 340
	}
}

static int __try_deliver_ckc_interrupt(struct kvm_vcpu *vcpu)
{
341
	int rc;
342 343 344 345 346

	if (psw_extint_disabled(vcpu))
		return 0;
	if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul))
		return 0;
347
	rc  = put_guest(vcpu, 0x1004, (u16 __user *)__LC_EXT_INT_CODE);
348 349 350 351 352
	rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
	rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
			      __LC_EXT_NEW_PSW, sizeof(psw_t));
	if (rc) {
353 354 355
		printk("kvm: The guest lowcore is not mapped during interrupt "
			"delivery, killing userspace\n");
		do_exit(SIGKILL);
356 357 358 359
	}
	return 1;
}

360
static int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu)
361
{
362 363 364
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
	struct kvm_s390_interrupt_info  *inti;
365 366 367 368 369 370 371 372 373 374 375 376 377
	int rc = 0;

	if (atomic_read(&li->active)) {
		spin_lock_bh(&li->lock);
		list_for_each_entry(inti, &li->list, list)
			if (__interrupt_is_deliverable(vcpu, inti)) {
				rc = 1;
				break;
			}
		spin_unlock_bh(&li->lock);
	}

	if ((!rc) && atomic_read(&fi->active)) {
378
		spin_lock(&fi->lock);
379 380 381 382 383
		list_for_each_entry(inti, &fi->list, list)
			if (__interrupt_is_deliverable(vcpu, inti)) {
				rc = 1;
				break;
			}
384
		spin_unlock(&fi->lock);
385 386 387
	}

	if ((!rc) && (vcpu->arch.sie_block->ckc <
388
		get_tod_clock() + vcpu->arch.sie_block->epoch)) {
389 390 391 392 393 394 395 396
		if ((!psw_extint_disabled(vcpu)) &&
			(vcpu->arch.sie_block->gcr[0] & 0x800ul))
			rc = 1;
	}

	return rc;
}

397 398 399 400 401
int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
{
	return 0;
}

402 403 404 405 406 407 408 409 410
int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
{
	u64 now, sltime;
	DECLARE_WAITQUEUE(wait, current);

	vcpu->stat.exit_wait_state++;
	if (kvm_cpu_has_interrupt(vcpu))
		return 0;

411 412 413 414 415
	__set_cpu_idle(vcpu);
	spin_lock_bh(&vcpu->arch.local_int.lock);
	vcpu->arch.local_int.timer_due = 0;
	spin_unlock_bh(&vcpu->arch.local_int.lock);

416 417 418
	if (psw_interrupts_disabled(vcpu)) {
		VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
		__unset_cpu_idle(vcpu);
419
		return -EOPNOTSUPP; /* disabled wait */
420 421 422 423 424 425 426 427
	}

	if (psw_extint_disabled(vcpu) ||
	    (!(vcpu->arch.sie_block->gcr[0] & 0x800ul))) {
		VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
		goto no_timer;
	}

428
	now = get_tod_clock() + vcpu->arch.sie_block->epoch;
429 430 431 432 433
	if (vcpu->arch.sie_block->ckc < now) {
		__unset_cpu_idle(vcpu);
		return 0;
	}

434
	sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now);
435

436 437
	hrtimer_start(&vcpu->arch.ckc_timer, ktime_set (0, sltime) , HRTIMER_MODE_REL);
	VCPU_EVENT(vcpu, 5, "enabled wait via clock comparator: %llx ns", sltime);
438
no_timer:
439
	spin_lock(&vcpu->arch.local_int.float_int->lock);
440
	spin_lock_bh(&vcpu->arch.local_int.lock);
441
	add_wait_queue(&vcpu->wq, &wait);
442 443 444 445 446 447
	while (list_empty(&vcpu->arch.local_int.list) &&
		list_empty(&vcpu->arch.local_int.float_int->list) &&
		(!vcpu->arch.local_int.timer_due) &&
		!signal_pending(current)) {
		set_current_state(TASK_INTERRUPTIBLE);
		spin_unlock_bh(&vcpu->arch.local_int.lock);
448
		spin_unlock(&vcpu->arch.local_int.float_int->lock);
449
		schedule();
450
		spin_lock(&vcpu->arch.local_int.float_int->lock);
451 452 453 454
		spin_lock_bh(&vcpu->arch.local_int.lock);
	}
	__unset_cpu_idle(vcpu);
	__set_current_state(TASK_RUNNING);
455
	remove_wait_queue(&vcpu->wq, &wait);
456
	spin_unlock_bh(&vcpu->arch.local_int.lock);
457
	spin_unlock(&vcpu->arch.local_int.float_int->lock);
458
	hrtimer_try_to_cancel(&vcpu->arch.ckc_timer);
459 460 461
	return 0;
}

462
void kvm_s390_tasklet(unsigned long parm)
463
{
464
	struct kvm_vcpu *vcpu = (struct kvm_vcpu *) parm;
465

466
	spin_lock(&vcpu->arch.local_int.lock);
467
	vcpu->arch.local_int.timer_due = 1;
468 469
	if (waitqueue_active(&vcpu->wq))
		wake_up_interruptible(&vcpu->wq);
470
	spin_unlock(&vcpu->arch.local_int.lock);
471 472
}

473 474 475 476 477 478 479 480 481 482 483 484 485
/*
 * low level hrtimer wake routine. Because this runs in hardirq context
 * we schedule a tasklet to do the real work.
 */
enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
{
	struct kvm_vcpu *vcpu;

	vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
	tasklet_schedule(&vcpu->arch.tasklet);

	return HRTIMER_NORESTART;
}
486 487 488

void kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
{
489 490 491
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
	struct kvm_s390_interrupt_info  *n, *inti = NULL;
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
	int deliver;

	__reset_intercept_indicators(vcpu);
	if (atomic_read(&li->active)) {
		do {
			deliver = 0;
			spin_lock_bh(&li->lock);
			list_for_each_entry_safe(inti, n, &li->list, list) {
				if (__interrupt_is_deliverable(vcpu, inti)) {
					list_del(&inti->list);
					deliver = 1;
					break;
				}
				__set_intercept_indicator(vcpu, inti);
			}
			if (list_empty(&li->list))
				atomic_set(&li->active, 0);
			spin_unlock_bh(&li->lock);
			if (deliver) {
				__do_deliver_interrupt(vcpu, inti);
				kfree(inti);
			}
		} while (deliver);
	}

	if ((vcpu->arch.sie_block->ckc <
518
		get_tod_clock() + vcpu->arch.sie_block->epoch))
519 520 521 522 523
		__try_deliver_ckc_interrupt(vcpu);

	if (atomic_read(&fi->active)) {
		do {
			deliver = 0;
524
			spin_lock(&fi->lock);
525 526 527 528 529 530 531 532 533 534
			list_for_each_entry_safe(inti, n, &fi->list, list) {
				if (__interrupt_is_deliverable(vcpu, inti)) {
					list_del(&inti->list);
					deliver = 1;
					break;
				}
				__set_intercept_indicator(vcpu, inti);
			}
			if (list_empty(&fi->list))
				atomic_set(&fi->active, 0);
535
			spin_unlock(&fi->lock);
536 537 538 539 540 541 542 543
			if (deliver) {
				__do_deliver_interrupt(vcpu, inti);
				kfree(inti);
			}
		} while (deliver);
	}
}

544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598
void kvm_s390_deliver_pending_machine_checks(struct kvm_vcpu *vcpu)
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
	struct kvm_s390_interrupt_info  *n, *inti = NULL;
	int deliver;

	__reset_intercept_indicators(vcpu);
	if (atomic_read(&li->active)) {
		do {
			deliver = 0;
			spin_lock_bh(&li->lock);
			list_for_each_entry_safe(inti, n, &li->list, list) {
				if ((inti->type == KVM_S390_MCHK) &&
				    __interrupt_is_deliverable(vcpu, inti)) {
					list_del(&inti->list);
					deliver = 1;
					break;
				}
				__set_intercept_indicator(vcpu, inti);
			}
			if (list_empty(&li->list))
				atomic_set(&li->active, 0);
			spin_unlock_bh(&li->lock);
			if (deliver) {
				__do_deliver_interrupt(vcpu, inti);
				kfree(inti);
			}
		} while (deliver);
	}

	if (atomic_read(&fi->active)) {
		do {
			deliver = 0;
			spin_lock(&fi->lock);
			list_for_each_entry_safe(inti, n, &fi->list, list) {
				if ((inti->type == KVM_S390_MCHK) &&
				    __interrupt_is_deliverable(vcpu, inti)) {
					list_del(&inti->list);
					deliver = 1;
					break;
				}
				__set_intercept_indicator(vcpu, inti);
			}
			if (list_empty(&fi->list))
				atomic_set(&fi->active, 0);
			spin_unlock(&fi->lock);
			if (deliver) {
				__do_deliver_interrupt(vcpu, inti);
				kfree(inti);
			}
		} while (deliver);
	}
}

599 600
int kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code)
{
601 602
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_interrupt_info *inti;
603 604 605 606 607

	inti = kzalloc(sizeof(*inti), GFP_KERNEL);
	if (!inti)
		return -ENOMEM;

608
	inti->type = KVM_S390_PROGRAM_INT;
609 610 611
	inti->pgm.code = code;

	VCPU_EVENT(vcpu, 3, "inject: program check %d (from kernel)", code);
612
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, inti->type, code, 0, 1);
613 614 615
	spin_lock_bh(&li->lock);
	list_add(&inti->list, &li->list);
	atomic_set(&li->active, 1);
616
	BUG_ON(waitqueue_active(li->wq));
617 618 619 620
	spin_unlock_bh(&li->lock);
	return 0;
}

621 622 623 624 625 626 627 628 629 630 631 632 633 634 635
struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
						    u64 cr6, u64 schid)
{
	struct kvm_s390_float_interrupt *fi;
	struct kvm_s390_interrupt_info *inti, *iter;

	if ((!schid && !cr6) || (schid && cr6))
		return NULL;
	mutex_lock(&kvm->lock);
	fi = &kvm->arch.float_int;
	spin_lock(&fi->lock);
	inti = NULL;
	list_for_each_entry(iter, &fi->list, list) {
		if (!is_ioint(iter->type))
			continue;
C
Cornelia Huck 已提交
636 637
		if (cr6 &&
		    ((cr6 & int_word_to_isc_bits(iter->io.io_int_word)) == 0))
638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
			continue;
		if (schid) {
			if (((schid & 0x00000000ffff0000) >> 16) !=
			    iter->io.subchannel_id)
				continue;
			if ((schid & 0x000000000000ffff) !=
			    iter->io.subchannel_nr)
				continue;
		}
		inti = iter;
		break;
	}
	if (inti)
		list_del_init(&inti->list);
	if (list_empty(&fi->list))
		atomic_set(&fi->active, 0);
	spin_unlock(&fi->lock);
	mutex_unlock(&kvm->lock);
	return inti;
}
658 659 660 661

int kvm_s390_inject_vm(struct kvm *kvm,
		       struct kvm_s390_interrupt *s390int)
{
662 663
	struct kvm_s390_local_interrupt *li;
	struct kvm_s390_float_interrupt *fi;
664
	struct kvm_s390_interrupt_info *inti, *iter;
665 666 667 668 669 670 671 672
	int sigcpu;

	inti = kzalloc(sizeof(*inti), GFP_KERNEL);
	if (!inti)
		return -ENOMEM;

	switch (s390int->type) {
	case KVM_S390_INT_VIRTIO:
673
		VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
674 675 676 677 678 679 680 681 682 683 684 685
			 s390int->parm, s390int->parm64);
		inti->type = s390int->type;
		inti->ext.ext_params = s390int->parm;
		inti->ext.ext_params2 = s390int->parm64;
		break;
	case KVM_S390_INT_SERVICE:
		VM_EVENT(kvm, 5, "inject: sclp parm:%x", s390int->parm);
		inti->type = s390int->type;
		inti->ext.ext_params = s390int->parm;
		break;
	case KVM_S390_PROGRAM_INT:
	case KVM_S390_SIGP_STOP:
686
	case KVM_S390_INT_EXTERNAL_CALL:
687
	case KVM_S390_INT_EMERGENCY:
688 689
		kfree(inti);
		return -EINVAL;
690 691 692 693 694 695 696
	case KVM_S390_MCHK:
		VM_EVENT(kvm, 5, "inject: machine check parm64:%llx",
			 s390int->parm64);
		inti->type = s390int->type;
		inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
		inti->mchk.mcic = s390int->parm64;
		break;
697 698 699 700 701 702 703 704 705 706 707 708 709 710
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
		if (s390int->type & IOINT_AI_MASK)
			VM_EVENT(kvm, 5, "%s", "inject: I/O (AI)");
		else
			VM_EVENT(kvm, 5, "inject: I/O css %x ss %x schid %04x",
				 s390int->type & IOINT_CSSID_MASK,
				 s390int->type & IOINT_SSID_MASK,
				 s390int->type & IOINT_SCHID_MASK);
		inti->type = s390int->type;
		inti->io.subchannel_id = s390int->parm >> 16;
		inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
		inti->io.io_int_parm = s390int->parm64 >> 32;
		inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
		break;
711 712 713 714
	default:
		kfree(inti);
		return -EINVAL;
	}
715 716
	trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
				 2);
717 718 719

	mutex_lock(&kvm->lock);
	fi = &kvm->arch.float_int;
720
	spin_lock(&fi->lock);
721 722 723
	if (!is_ioint(inti->type))
		list_add_tail(&inti->list, &fi->list);
	else {
C
Cornelia Huck 已提交
724 725
		u64 isc_bits = int_word_to_isc_bits(inti->io.io_int_word);

726 727 728 729
		/* Keep I/O interrupts sorted in isc order. */
		list_for_each_entry(iter, &fi->list, list) {
			if (!is_ioint(iter->type))
				continue;
C
Cornelia Huck 已提交
730 731
			if (int_word_to_isc_bits(iter->io.io_int_word)
			    <= isc_bits)
732 733 734 735 736
				continue;
			break;
		}
		list_add_tail(&inti->list, &iter->list);
	}
737 738 739 740 741 742 743 744 745 746 747 748
	atomic_set(&fi->active, 1);
	sigcpu = find_first_bit(fi->idle_mask, KVM_MAX_VCPUS);
	if (sigcpu == KVM_MAX_VCPUS) {
		do {
			sigcpu = fi->next_rr_cpu++;
			if (sigcpu == KVM_MAX_VCPUS)
				sigcpu = fi->next_rr_cpu = 0;
		} while (fi->local_int[sigcpu] == NULL);
	}
	li = fi->local_int[sigcpu];
	spin_lock_bh(&li->lock);
	atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags);
749 750
	if (waitqueue_active(li->wq))
		wake_up_interruptible(li->wq);
751
	spin_unlock_bh(&li->lock);
752
	spin_unlock(&fi->lock);
753 754 755 756 757 758 759
	mutex_unlock(&kvm->lock);
	return 0;
}

int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu,
			 struct kvm_s390_interrupt *s390int)
{
760 761
	struct kvm_s390_local_interrupt *li;
	struct kvm_s390_interrupt_info *inti;
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777

	inti = kzalloc(sizeof(*inti), GFP_KERNEL);
	if (!inti)
		return -ENOMEM;

	switch (s390int->type) {
	case KVM_S390_PROGRAM_INT:
		if (s390int->parm & 0xffff0000) {
			kfree(inti);
			return -EINVAL;
		}
		inti->type = s390int->type;
		inti->pgm.code = s390int->parm;
		VCPU_EVENT(vcpu, 3, "inject: program check %d (from user)",
			   s390int->parm);
		break;
778 779 780 781 782 783
	case KVM_S390_SIGP_SET_PREFIX:
		inti->prefix.address = s390int->parm;
		inti->type = s390int->type;
		VCPU_EVENT(vcpu, 3, "inject: set prefix to %x (from user)",
			   s390int->parm);
		break;
784 785
	case KVM_S390_SIGP_STOP:
	case KVM_S390_RESTART:
786 787 788
		VCPU_EVENT(vcpu, 3, "inject: type %x", s390int->type);
		inti->type = s390int->type;
		break;
789
	case KVM_S390_INT_EXTERNAL_CALL:
790 791 792 793 794 795 796 797 798
		if (s390int->parm & 0xffff0000) {
			kfree(inti);
			return -EINVAL;
		}
		VCPU_EVENT(vcpu, 3, "inject: external call source-cpu:%u",
			   s390int->parm);
		inti->type = s390int->type;
		inti->extcall.code = s390int->parm;
		break;
799
	case KVM_S390_INT_EMERGENCY:
800 801 802 803 804
		if (s390int->parm & 0xffff0000) {
			kfree(inti);
			return -EINVAL;
		}
		VCPU_EVENT(vcpu, 3, "inject: emergency %u\n", s390int->parm);
805
		inti->type = s390int->type;
806
		inti->emerg.code = s390int->parm;
807
		break;
808 809 810 811 812 813
	case KVM_S390_MCHK:
		VCPU_EVENT(vcpu, 5, "inject: machine check parm64:%llx",
			   s390int->parm64);
		inti->type = s390int->type;
		inti->mchk.mcic = s390int->parm64;
		break;
814 815
	case KVM_S390_INT_VIRTIO:
	case KVM_S390_INT_SERVICE:
816
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
817 818 819 820
	default:
		kfree(inti);
		return -EINVAL;
	}
821 822
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, s390int->type, s390int->parm,
				   s390int->parm64, 2);
823 824 825 826 827 828 829 830 831 832 833 834

	mutex_lock(&vcpu->kvm->lock);
	li = &vcpu->arch.local_int;
	spin_lock_bh(&li->lock);
	if (inti->type == KVM_S390_PROGRAM_INT)
		list_add(&inti->list, &li->list);
	else
		list_add_tail(&inti->list, &li->list);
	atomic_set(&li->active, 1);
	if (inti->type == KVM_S390_SIGP_STOP)
		li->action_bits |= ACTION_STOP_ON_STOP;
	atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags);
835 836
	if (waitqueue_active(&vcpu->wq))
		wake_up_interruptible(&vcpu->wq);
837 838 839 840
	spin_unlock_bh(&li->lock);
	mutex_unlock(&vcpu->kvm->lock);
	return 0;
}