interrupt.c 23.6 KB
Newer Older
1
/*
2
 * handling kvm guest interrupts
3 4 5 6 7 8 9 10 11 12
 *
 * Copyright IBM Corp. 2008
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License (version 2 only)
 * as published by the Free Software Foundation.
 *
 *    Author(s): Carsten Otte <cotte@de.ibm.com>
 */

13
#include <linux/interrupt.h>
14
#include <linux/kvm_host.h>
15
#include <linux/hrtimer.h>
16
#include <linux/signal.h>
17
#include <linux/slab.h>
18 19
#include <asm/asm-offsets.h>
#include <asm/uaccess.h>
20 21
#include "kvm-s390.h"
#include "gaccess.h"
22
#include "trace-s390.h"
23

24 25 26 27 28 29 30 31 32 33
#define IOINT_SCHID_MASK 0x0000ffff
#define IOINT_SSID_MASK 0x00030000
#define IOINT_CSSID_MASK 0x03fc0000
#define IOINT_AI_MASK 0x04000000

static int is_ioint(u64 type)
{
	return ((type & 0xfffe0000u) != 0xfffe0000u);
}

34 35 36 37 38
static int psw_extint_disabled(struct kvm_vcpu *vcpu)
{
	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
}

39 40 41 42 43
static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
{
	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
}

44 45 46 47 48
static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
{
	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
}

49 50 51 52 53 54 55 56 57
static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
{
	if ((vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PER) ||
	    (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO) ||
	    (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT))
		return 0;
	return 1;
}

C
Cornelia Huck 已提交
58 59 60 61 62 63 64
static u64 int_word_to_isc_bits(u32 int_word)
{
	u8 isc = (int_word & 0x38000000) >> 27;

	return (0x80 >> isc) << 24;
}

65
static int __interrupt_is_deliverable(struct kvm_vcpu *vcpu,
66
				      struct kvm_s390_interrupt_info *inti)
67 68
{
	switch (inti->type) {
69 70 71 72 73
	case KVM_S390_INT_EXTERNAL_CALL:
		if (psw_extint_disabled(vcpu))
			return 0;
		if (vcpu->arch.sie_block->gcr[0] & 0x2000ul)
			return 1;
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
	case KVM_S390_INT_EMERGENCY:
		if (psw_extint_disabled(vcpu))
			return 0;
		if (vcpu->arch.sie_block->gcr[0] & 0x4000ul)
			return 1;
		return 0;
	case KVM_S390_INT_SERVICE:
		if (psw_extint_disabled(vcpu))
			return 0;
		if (vcpu->arch.sie_block->gcr[0] & 0x200ul)
			return 1;
		return 0;
	case KVM_S390_INT_VIRTIO:
		if (psw_extint_disabled(vcpu))
			return 0;
		if (vcpu->arch.sie_block->gcr[0] & 0x200ul)
			return 1;
		return 0;
	case KVM_S390_PROGRAM_INT:
	case KVM_S390_SIGP_STOP:
	case KVM_S390_SIGP_SET_PREFIX:
	case KVM_S390_RESTART:
		return 1;
97 98 99 100 101 102
	case KVM_S390_MCHK:
		if (psw_mchk_disabled(vcpu))
			return 0;
		if (vcpu->arch.sie_block->gcr[14] & inti->mchk.cr14)
			return 1;
		return 0;
103 104 105
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
		if (psw_ioint_disabled(vcpu))
			return 0;
C
Cornelia Huck 已提交
106 107
		if (vcpu->arch.sie_block->gcr[6] &
		    int_word_to_isc_bits(inti->io.io_int_word))
108 109
			return 1;
		return 0;
110
	default:
111 112
		printk(KERN_WARNING "illegal interrupt type %llx\n",
		       inti->type);
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
		BUG();
	}
	return 0;
}

static void __set_cpu_idle(struct kvm_vcpu *vcpu)
{
	BUG_ON(vcpu->vcpu_id > KVM_MAX_VCPUS - 1);
	atomic_set_mask(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
	set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
}

static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
{
	BUG_ON(vcpu->vcpu_id > KVM_MAX_VCPUS - 1);
	atomic_clear_mask(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
	clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
}

static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
{
	atomic_clear_mask(CPUSTAT_ECALL_PEND |
		CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT,
		&vcpu->arch.sie_block->cpuflags);
	vcpu->arch.sie_block->lctl = 0x0000;
138
	vcpu->arch.sie_block->ictl &= ~ICTL_LPSW;
139 140 141 142 143 144 145 146
}

static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag)
{
	atomic_set_mask(flag, &vcpu->arch.sie_block->cpuflags);
}

static void __set_intercept_indicator(struct kvm_vcpu *vcpu,
147
				      struct kvm_s390_interrupt_info *inti)
148 149
{
	switch (inti->type) {
150
	case KVM_S390_INT_EXTERNAL_CALL:
151 152 153 154 155 156 157 158 159 160 161
	case KVM_S390_INT_EMERGENCY:
	case KVM_S390_INT_SERVICE:
	case KVM_S390_INT_VIRTIO:
		if (psw_extint_disabled(vcpu))
			__set_cpuflag(vcpu, CPUSTAT_EXT_INT);
		else
			vcpu->arch.sie_block->lctl |= LCTL_CR0;
		break;
	case KVM_S390_SIGP_STOP:
		__set_cpuflag(vcpu, CPUSTAT_STOP_INT);
		break;
162 163 164 165 166 167
	case KVM_S390_MCHK:
		if (psw_mchk_disabled(vcpu))
			vcpu->arch.sie_block->ictl |= ICTL_LPSW;
		else
			vcpu->arch.sie_block->lctl |= LCTL_CR14;
		break;
168 169 170 171 172 173
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
		if (psw_ioint_disabled(vcpu))
			__set_cpuflag(vcpu, CPUSTAT_IO_INT);
		else
			vcpu->arch.sie_block->lctl |= LCTL_CR6;
		break;
174 175 176 177 178 179
	default:
		BUG();
	}
}

static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
180
				   struct kvm_s390_interrupt_info *inti)
181 182
{
	const unsigned short table[] = { 2, 4, 4, 6 };
183
	int rc = 0;
184 185 186 187 188

	switch (inti->type) {
	case KVM_S390_INT_EMERGENCY:
		VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp emerg");
		vcpu->stat.deliver_emergency_signal++;
189 190
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->emerg.code, 0);
191 192 193 194 195 196
		rc  = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x1201);
		rc |= put_guest_u16(vcpu, __LC_EXT_CPU_ADDR, inti->emerg.code);
		rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_EXT_NEW_PSW, sizeof(psw_t));
197
		break;
198 199 200
	case KVM_S390_INT_EXTERNAL_CALL:
		VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp ext call");
		vcpu->stat.deliver_external_call++;
201 202
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->extcall.code, 0);
203 204 205 206 207 208
		rc  = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x1202);
		rc |= put_guest_u16(vcpu, __LC_EXT_CPU_ADDR, inti->extcall.code);
		rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_EXT_NEW_PSW, sizeof(psw_t));
209
		break;
210 211 212 213
	case KVM_S390_INT_SERVICE:
		VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x",
			   inti->ext.ext_params);
		vcpu->stat.deliver_service_signal++;
214 215
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->ext.ext_params, 0);
216 217 218 219 220 221
		rc  = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x2401);
		rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_EXT_NEW_PSW, sizeof(psw_t));
		rc |= put_guest_u32(vcpu, __LC_EXT_PARAMS, inti->ext.ext_params);
222 223
		break;
	case KVM_S390_INT_VIRTIO:
224
		VCPU_EVENT(vcpu, 4, "interrupt: virtio parm:%x,parm64:%llx",
225 226
			   inti->ext.ext_params, inti->ext.ext_params2);
		vcpu->stat.deliver_virtio_interrupt++;
227 228 229
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->ext.ext_params,
						 inti->ext.ext_params2);
230 231 232 233 234 235 236 237 238
		rc  = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x2603);
		rc |= put_guest_u16(vcpu, __LC_EXT_CPU_ADDR, 0x0d00);
		rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_EXT_NEW_PSW, sizeof(psw_t));
		rc |= put_guest_u32(vcpu, __LC_EXT_PARAMS, inti->ext.ext_params);
		rc |= put_guest_u64(vcpu, __LC_EXT_PARAMS2,
				    inti->ext.ext_params2);
239 240 241 242
		break;
	case KVM_S390_SIGP_STOP:
		VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu stop");
		vcpu->stat.deliver_stop_signal++;
243 244
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 0, 0);
245 246 247 248 249 250 251
		__set_intercept_indicator(vcpu, inti);
		break;

	case KVM_S390_SIGP_SET_PREFIX:
		VCPU_EVENT(vcpu, 4, "interrupt: set prefix to %x",
			   inti->prefix.address);
		vcpu->stat.deliver_prefix_signal++;
252 253
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->prefix.address, 0);
254
		kvm_s390_set_prefix(vcpu, inti->prefix.address);
255 256 257 258 259
		break;

	case KVM_S390_RESTART:
		VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu restart");
		vcpu->stat.deliver_restart_signal++;
260 261
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 0, 0);
262 263 264 265 266 267
		rc  = copy_to_guest(vcpu,
				    offsetof(struct _lowcore, restart_old_psw),
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      offsetof(struct _lowcore, restart_psw),
				      sizeof(psw_t));
268
		atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
269 270 271 272 273 274
		break;
	case KVM_S390_PROGRAM_INT:
		VCPU_EVENT(vcpu, 4, "interrupt: pgm check code:%x, ilc:%x",
			   inti->pgm.code,
			   table[vcpu->arch.sie_block->ipa >> 14]);
		vcpu->stat.deliver_program_int++;
275 276
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->pgm.code, 0);
277 278 279 280 281 282 283
		rc  = put_guest_u16(vcpu, __LC_PGM_INT_CODE, inti->pgm.code);
		rc |= put_guest_u16(vcpu, __LC_PGM_ILC,
				    table[vcpu->arch.sie_block->ipa >> 14]);
		rc |= copy_to_guest(vcpu, __LC_PGM_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_PGM_NEW_PSW, sizeof(psw_t));
284 285
		break;

286 287 288 289 290 291
	case KVM_S390_MCHK:
		VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx",
			   inti->mchk.mcic);
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 inti->mchk.cr14,
						 inti->mchk.mcic);
292 293 294 295 296 297 298
		rc  = kvm_s390_vcpu_store_status(vcpu,
						 KVM_S390_STORE_STATUS_PREFIXED);
		rc |= put_guest_u64(vcpu, __LC_MCCK_CODE, inti->mchk.mcic);
		rc |= copy_to_guest(vcpu, __LC_MCK_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_MCK_NEW_PSW, sizeof(psw_t));
299 300
		break;

301 302 303 304 305 306 307 308 309 310
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
	{
		__u32 param0 = ((__u32)inti->io.subchannel_id << 16) |
			inti->io.subchannel_nr;
		__u64 param1 = ((__u64)inti->io.io_int_parm << 32) |
			inti->io.io_int_word;
		VCPU_EVENT(vcpu, 4, "interrupt: I/O %llx", inti->type);
		vcpu->stat.deliver_io_int++;
		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
						 param0, param1);
311 312 313 314 315 316 317 318 319 320 321 322
		rc  = put_guest_u16(vcpu, __LC_SUBCHANNEL_ID,
				    inti->io.subchannel_id);
		rc |= put_guest_u16(vcpu, __LC_SUBCHANNEL_NR,
				    inti->io.subchannel_nr);
		rc |= put_guest_u32(vcpu, __LC_IO_INT_PARM,
				    inti->io.io_int_parm);
		rc |= put_guest_u32(vcpu, __LC_IO_INT_WORD,
				    inti->io.io_int_word);
		rc |= copy_to_guest(vcpu, __LC_IO_OLD_PSW,
				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
		rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
				      __LC_IO_NEW_PSW, sizeof(psw_t));
323 324
		break;
	}
325 326 327
	default:
		BUG();
	}
328
	if (rc) {
329
		printk("kvm: The guest lowcore is not mapped during interrupt "
330
		       "delivery, killing userspace\n");
331
		do_exit(SIGKILL);
332 333 334 335 336
	}
}

static int __try_deliver_ckc_interrupt(struct kvm_vcpu *vcpu)
{
337
	int rc;
338 339 340 341 342

	if (psw_extint_disabled(vcpu))
		return 0;
	if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul))
		return 0;
343 344 345 346 347 348
	rc  = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x1004);
	rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
	rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
			      __LC_EXT_NEW_PSW, sizeof(psw_t));
	if (rc) {
349 350 351
		printk("kvm: The guest lowcore is not mapped during interrupt "
			"delivery, killing userspace\n");
		do_exit(SIGKILL);
352 353 354 355
	}
	return 1;
}

356
static int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu)
357
{
358 359 360
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
	struct kvm_s390_interrupt_info  *inti;
361 362 363 364 365 366 367 368 369 370 371 372 373
	int rc = 0;

	if (atomic_read(&li->active)) {
		spin_lock_bh(&li->lock);
		list_for_each_entry(inti, &li->list, list)
			if (__interrupt_is_deliverable(vcpu, inti)) {
				rc = 1;
				break;
			}
		spin_unlock_bh(&li->lock);
	}

	if ((!rc) && atomic_read(&fi->active)) {
374
		spin_lock(&fi->lock);
375 376 377 378 379
		list_for_each_entry(inti, &fi->list, list)
			if (__interrupt_is_deliverable(vcpu, inti)) {
				rc = 1;
				break;
			}
380
		spin_unlock(&fi->lock);
381 382 383
	}

	if ((!rc) && (vcpu->arch.sie_block->ckc <
384
		get_tod_clock() + vcpu->arch.sie_block->epoch)) {
385 386 387 388 389 390 391 392
		if ((!psw_extint_disabled(vcpu)) &&
			(vcpu->arch.sie_block->gcr[0] & 0x800ul))
			rc = 1;
	}

	return rc;
}

393 394 395 396 397
int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
{
	return 0;
}

398 399 400 401 402 403 404 405 406
int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
{
	u64 now, sltime;
	DECLARE_WAITQUEUE(wait, current);

	vcpu->stat.exit_wait_state++;
	if (kvm_cpu_has_interrupt(vcpu))
		return 0;

407 408 409 410 411
	__set_cpu_idle(vcpu);
	spin_lock_bh(&vcpu->arch.local_int.lock);
	vcpu->arch.local_int.timer_due = 0;
	spin_unlock_bh(&vcpu->arch.local_int.lock);

412 413 414
	if (psw_interrupts_disabled(vcpu)) {
		VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
		__unset_cpu_idle(vcpu);
415
		return -EOPNOTSUPP; /* disabled wait */
416 417 418 419 420 421 422 423
	}

	if (psw_extint_disabled(vcpu) ||
	    (!(vcpu->arch.sie_block->gcr[0] & 0x800ul))) {
		VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
		goto no_timer;
	}

424
	now = get_tod_clock() + vcpu->arch.sie_block->epoch;
425 426 427 428 429
	if (vcpu->arch.sie_block->ckc < now) {
		__unset_cpu_idle(vcpu);
		return 0;
	}

430
	sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now);
431

432 433
	hrtimer_start(&vcpu->arch.ckc_timer, ktime_set (0, sltime) , HRTIMER_MODE_REL);
	VCPU_EVENT(vcpu, 5, "enabled wait via clock comparator: %llx ns", sltime);
434
no_timer:
435
	spin_lock(&vcpu->arch.local_int.float_int->lock);
436 437 438 439 440 441 442 443
	spin_lock_bh(&vcpu->arch.local_int.lock);
	add_wait_queue(&vcpu->arch.local_int.wq, &wait);
	while (list_empty(&vcpu->arch.local_int.list) &&
		list_empty(&vcpu->arch.local_int.float_int->list) &&
		(!vcpu->arch.local_int.timer_due) &&
		!signal_pending(current)) {
		set_current_state(TASK_INTERRUPTIBLE);
		spin_unlock_bh(&vcpu->arch.local_int.lock);
444
		spin_unlock(&vcpu->arch.local_int.float_int->lock);
445
		schedule();
446
		spin_lock(&vcpu->arch.local_int.float_int->lock);
447 448 449 450
		spin_lock_bh(&vcpu->arch.local_int.lock);
	}
	__unset_cpu_idle(vcpu);
	__set_current_state(TASK_RUNNING);
451
	remove_wait_queue(&vcpu->arch.local_int.wq, &wait);
452
	spin_unlock_bh(&vcpu->arch.local_int.lock);
453
	spin_unlock(&vcpu->arch.local_int.float_int->lock);
454
	hrtimer_try_to_cancel(&vcpu->arch.ckc_timer);
455 456 457
	return 0;
}

458
void kvm_s390_tasklet(unsigned long parm)
459
{
460
	struct kvm_vcpu *vcpu = (struct kvm_vcpu *) parm;
461

462
	spin_lock(&vcpu->arch.local_int.lock);
463 464 465
	vcpu->arch.local_int.timer_due = 1;
	if (waitqueue_active(&vcpu->arch.local_int.wq))
		wake_up_interruptible(&vcpu->arch.local_int.wq);
466
	spin_unlock(&vcpu->arch.local_int.lock);
467 468
}

469 470 471 472 473 474 475 476 477 478 479 480 481
/*
 * low level hrtimer wake routine. Because this runs in hardirq context
 * we schedule a tasklet to do the real work.
 */
enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
{
	struct kvm_vcpu *vcpu;

	vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
	tasklet_schedule(&vcpu->arch.tasklet);

	return HRTIMER_NORESTART;
}
482 483 484

void kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
{
485 486 487
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
	struct kvm_s390_interrupt_info  *n, *inti = NULL;
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
	int deliver;

	__reset_intercept_indicators(vcpu);
	if (atomic_read(&li->active)) {
		do {
			deliver = 0;
			spin_lock_bh(&li->lock);
			list_for_each_entry_safe(inti, n, &li->list, list) {
				if (__interrupt_is_deliverable(vcpu, inti)) {
					list_del(&inti->list);
					deliver = 1;
					break;
				}
				__set_intercept_indicator(vcpu, inti);
			}
			if (list_empty(&li->list))
				atomic_set(&li->active, 0);
			spin_unlock_bh(&li->lock);
			if (deliver) {
				__do_deliver_interrupt(vcpu, inti);
				kfree(inti);
			}
		} while (deliver);
	}

	if ((vcpu->arch.sie_block->ckc <
514
		get_tod_clock() + vcpu->arch.sie_block->epoch))
515 516 517 518 519
		__try_deliver_ckc_interrupt(vcpu);

	if (atomic_read(&fi->active)) {
		do {
			deliver = 0;
520
			spin_lock(&fi->lock);
521 522 523 524 525 526 527 528 529 530
			list_for_each_entry_safe(inti, n, &fi->list, list) {
				if (__interrupt_is_deliverable(vcpu, inti)) {
					list_del(&inti->list);
					deliver = 1;
					break;
				}
				__set_intercept_indicator(vcpu, inti);
			}
			if (list_empty(&fi->list))
				atomic_set(&fi->active, 0);
531
			spin_unlock(&fi->lock);
532 533 534 535 536 537 538 539
			if (deliver) {
				__do_deliver_interrupt(vcpu, inti);
				kfree(inti);
			}
		} while (deliver);
	}
}

540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594
void kvm_s390_deliver_pending_machine_checks(struct kvm_vcpu *vcpu)
{
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
	struct kvm_s390_interrupt_info  *n, *inti = NULL;
	int deliver;

	__reset_intercept_indicators(vcpu);
	if (atomic_read(&li->active)) {
		do {
			deliver = 0;
			spin_lock_bh(&li->lock);
			list_for_each_entry_safe(inti, n, &li->list, list) {
				if ((inti->type == KVM_S390_MCHK) &&
				    __interrupt_is_deliverable(vcpu, inti)) {
					list_del(&inti->list);
					deliver = 1;
					break;
				}
				__set_intercept_indicator(vcpu, inti);
			}
			if (list_empty(&li->list))
				atomic_set(&li->active, 0);
			spin_unlock_bh(&li->lock);
			if (deliver) {
				__do_deliver_interrupt(vcpu, inti);
				kfree(inti);
			}
		} while (deliver);
	}

	if (atomic_read(&fi->active)) {
		do {
			deliver = 0;
			spin_lock(&fi->lock);
			list_for_each_entry_safe(inti, n, &fi->list, list) {
				if ((inti->type == KVM_S390_MCHK) &&
				    __interrupt_is_deliverable(vcpu, inti)) {
					list_del(&inti->list);
					deliver = 1;
					break;
				}
				__set_intercept_indicator(vcpu, inti);
			}
			if (list_empty(&fi->list))
				atomic_set(&fi->active, 0);
			spin_unlock(&fi->lock);
			if (deliver) {
				__do_deliver_interrupt(vcpu, inti);
				kfree(inti);
			}
		} while (deliver);
	}
}

595 596
int kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code)
{
597 598
	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
	struct kvm_s390_interrupt_info *inti;
599 600 601 602 603

	inti = kzalloc(sizeof(*inti), GFP_KERNEL);
	if (!inti)
		return -ENOMEM;

604
	inti->type = KVM_S390_PROGRAM_INT;
605 606 607
	inti->pgm.code = code;

	VCPU_EVENT(vcpu, 3, "inject: program check %d (from kernel)", code);
608
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, inti->type, code, 0, 1);
609 610 611 612 613 614 615 616
	spin_lock_bh(&li->lock);
	list_add(&inti->list, &li->list);
	atomic_set(&li->active, 1);
	BUG_ON(waitqueue_active(&li->wq));
	spin_unlock_bh(&li->lock);
	return 0;
}

617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
						    u64 cr6, u64 schid)
{
	struct kvm_s390_float_interrupt *fi;
	struct kvm_s390_interrupt_info *inti, *iter;

	if ((!schid && !cr6) || (schid && cr6))
		return NULL;
	mutex_lock(&kvm->lock);
	fi = &kvm->arch.float_int;
	spin_lock(&fi->lock);
	inti = NULL;
	list_for_each_entry(iter, &fi->list, list) {
		if (!is_ioint(iter->type))
			continue;
C
Cornelia Huck 已提交
632 633
		if (cr6 &&
		    ((cr6 & int_word_to_isc_bits(iter->io.io_int_word)) == 0))
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653
			continue;
		if (schid) {
			if (((schid & 0x00000000ffff0000) >> 16) !=
			    iter->io.subchannel_id)
				continue;
			if ((schid & 0x000000000000ffff) !=
			    iter->io.subchannel_nr)
				continue;
		}
		inti = iter;
		break;
	}
	if (inti)
		list_del_init(&inti->list);
	if (list_empty(&fi->list))
		atomic_set(&fi->active, 0);
	spin_unlock(&fi->lock);
	mutex_unlock(&kvm->lock);
	return inti;
}
654 655 656 657

int kvm_s390_inject_vm(struct kvm *kvm,
		       struct kvm_s390_interrupt *s390int)
{
658 659
	struct kvm_s390_local_interrupt *li;
	struct kvm_s390_float_interrupt *fi;
660
	struct kvm_s390_interrupt_info *inti, *iter;
661 662 663 664 665 666 667 668
	int sigcpu;

	inti = kzalloc(sizeof(*inti), GFP_KERNEL);
	if (!inti)
		return -ENOMEM;

	switch (s390int->type) {
	case KVM_S390_INT_VIRTIO:
669
		VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
670 671 672 673 674 675 676 677 678 679 680 681
			 s390int->parm, s390int->parm64);
		inti->type = s390int->type;
		inti->ext.ext_params = s390int->parm;
		inti->ext.ext_params2 = s390int->parm64;
		break;
	case KVM_S390_INT_SERVICE:
		VM_EVENT(kvm, 5, "inject: sclp parm:%x", s390int->parm);
		inti->type = s390int->type;
		inti->ext.ext_params = s390int->parm;
		break;
	case KVM_S390_PROGRAM_INT:
	case KVM_S390_SIGP_STOP:
682
	case KVM_S390_INT_EXTERNAL_CALL:
683
	case KVM_S390_INT_EMERGENCY:
684 685
		kfree(inti);
		return -EINVAL;
686 687 688 689 690 691 692
	case KVM_S390_MCHK:
		VM_EVENT(kvm, 5, "inject: machine check parm64:%llx",
			 s390int->parm64);
		inti->type = s390int->type;
		inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
		inti->mchk.mcic = s390int->parm64;
		break;
693 694 695 696 697 698 699 700 701 702 703 704 705 706
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
		if (s390int->type & IOINT_AI_MASK)
			VM_EVENT(kvm, 5, "%s", "inject: I/O (AI)");
		else
			VM_EVENT(kvm, 5, "inject: I/O css %x ss %x schid %04x",
				 s390int->type & IOINT_CSSID_MASK,
				 s390int->type & IOINT_SSID_MASK,
				 s390int->type & IOINT_SCHID_MASK);
		inti->type = s390int->type;
		inti->io.subchannel_id = s390int->parm >> 16;
		inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
		inti->io.io_int_parm = s390int->parm64 >> 32;
		inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
		break;
707 708 709 710
	default:
		kfree(inti);
		return -EINVAL;
	}
711 712
	trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
				 2);
713 714 715

	mutex_lock(&kvm->lock);
	fi = &kvm->arch.float_int;
716
	spin_lock(&fi->lock);
717 718 719
	if (!is_ioint(inti->type))
		list_add_tail(&inti->list, &fi->list);
	else {
C
Cornelia Huck 已提交
720 721
		u64 isc_bits = int_word_to_isc_bits(inti->io.io_int_word);

722 723 724 725
		/* Keep I/O interrupts sorted in isc order. */
		list_for_each_entry(iter, &fi->list, list) {
			if (!is_ioint(iter->type))
				continue;
C
Cornelia Huck 已提交
726 727
			if (int_word_to_isc_bits(iter->io.io_int_word)
			    <= isc_bits)
728 729 730 731 732
				continue;
			break;
		}
		list_add_tail(&inti->list, &iter->list);
	}
733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
	atomic_set(&fi->active, 1);
	sigcpu = find_first_bit(fi->idle_mask, KVM_MAX_VCPUS);
	if (sigcpu == KVM_MAX_VCPUS) {
		do {
			sigcpu = fi->next_rr_cpu++;
			if (sigcpu == KVM_MAX_VCPUS)
				sigcpu = fi->next_rr_cpu = 0;
		} while (fi->local_int[sigcpu] == NULL);
	}
	li = fi->local_int[sigcpu];
	spin_lock_bh(&li->lock);
	atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags);
	if (waitqueue_active(&li->wq))
		wake_up_interruptible(&li->wq);
	spin_unlock_bh(&li->lock);
748
	spin_unlock(&fi->lock);
749 750 751 752 753 754 755
	mutex_unlock(&kvm->lock);
	return 0;
}

int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu,
			 struct kvm_s390_interrupt *s390int)
{
756 757
	struct kvm_s390_local_interrupt *li;
	struct kvm_s390_interrupt_info *inti;
758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773

	inti = kzalloc(sizeof(*inti), GFP_KERNEL);
	if (!inti)
		return -ENOMEM;

	switch (s390int->type) {
	case KVM_S390_PROGRAM_INT:
		if (s390int->parm & 0xffff0000) {
			kfree(inti);
			return -EINVAL;
		}
		inti->type = s390int->type;
		inti->pgm.code = s390int->parm;
		VCPU_EVENT(vcpu, 3, "inject: program check %d (from user)",
			   s390int->parm);
		break;
774 775 776 777 778 779
	case KVM_S390_SIGP_SET_PREFIX:
		inti->prefix.address = s390int->parm;
		inti->type = s390int->type;
		VCPU_EVENT(vcpu, 3, "inject: set prefix to %x (from user)",
			   s390int->parm);
		break;
780 781
	case KVM_S390_SIGP_STOP:
	case KVM_S390_RESTART:
782 783 784
		VCPU_EVENT(vcpu, 3, "inject: type %x", s390int->type);
		inti->type = s390int->type;
		break;
785
	case KVM_S390_INT_EXTERNAL_CALL:
786 787 788 789 790 791 792 793 794
		if (s390int->parm & 0xffff0000) {
			kfree(inti);
			return -EINVAL;
		}
		VCPU_EVENT(vcpu, 3, "inject: external call source-cpu:%u",
			   s390int->parm);
		inti->type = s390int->type;
		inti->extcall.code = s390int->parm;
		break;
795
	case KVM_S390_INT_EMERGENCY:
796 797 798 799 800
		if (s390int->parm & 0xffff0000) {
			kfree(inti);
			return -EINVAL;
		}
		VCPU_EVENT(vcpu, 3, "inject: emergency %u\n", s390int->parm);
801
		inti->type = s390int->type;
802
		inti->emerg.code = s390int->parm;
803
		break;
804 805 806 807 808 809
	case KVM_S390_MCHK:
		VCPU_EVENT(vcpu, 5, "inject: machine check parm64:%llx",
			   s390int->parm64);
		inti->type = s390int->type;
		inti->mchk.mcic = s390int->parm64;
		break;
810 811
	case KVM_S390_INT_VIRTIO:
	case KVM_S390_INT_SERVICE:
812
	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
813 814 815 816
	default:
		kfree(inti);
		return -EINVAL;
	}
817 818
	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, s390int->type, s390int->parm,
				   s390int->parm64, 2);
819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836

	mutex_lock(&vcpu->kvm->lock);
	li = &vcpu->arch.local_int;
	spin_lock_bh(&li->lock);
	if (inti->type == KVM_S390_PROGRAM_INT)
		list_add(&inti->list, &li->list);
	else
		list_add_tail(&inti->list, &li->list);
	atomic_set(&li->active, 1);
	if (inti->type == KVM_S390_SIGP_STOP)
		li->action_bits |= ACTION_STOP_ON_STOP;
	atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags);
	if (waitqueue_active(&li->wq))
		wake_up_interruptible(&vcpu->arch.local_int.wq);
	spin_unlock_bh(&li->lock);
	mutex_unlock(&vcpu->kvm->lock);
	return 0;
}