lapic.c 65.1 KB
Newer Older
E
Eddie Dong 已提交
1 2 3 4 5 6 7

/*
 * Local APIC virtualization
 *
 * Copyright (C) 2006 Qumranet, Inc.
 * Copyright (C) 2007 Novell
 * Copyright (C) 2007 Intel
N
Nicolas Kaiser 已提交
8
 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
E
Eddie Dong 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *
 * Authors:
 *   Dor Laor <dor.laor@qumranet.com>
 *   Gregory Haskins <ghaskins@novell.com>
 *   Yaozu (Eddie) Dong <eddie.dong@intel.com>
 *
 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 */

21
#include <linux/kvm_host.h>
E
Eddie Dong 已提交
22 23 24 25 26 27
#include <linux/kvm.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/smp.h>
#include <linux/hrtimer.h>
#include <linux/io.h>
28
#include <linux/export.h>
R
Roman Zippel 已提交
29
#include <linux/math64.h>
30
#include <linux/slab.h>
E
Eddie Dong 已提交
31 32 33 34 35
#include <asm/processor.h>
#include <asm/msr.h>
#include <asm/page.h>
#include <asm/current.h>
#include <asm/apicdef.h>
36
#include <asm/delay.h>
A
Arun Sharma 已提交
37
#include <linux/atomic.h>
38
#include <linux/jump_label.h>
39
#include "kvm_cache_regs.h"
E
Eddie Dong 已提交
40
#include "irq.h"
41
#include "trace.h"
42
#include "x86.h"
A
Avi Kivity 已提交
43
#include "cpuid.h"
44
#include "hyperv.h"
E
Eddie Dong 已提交
45

46 47 48 49 50 51
#ifndef CONFIG_X86_64
#define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
#else
#define mod_64(x, y) ((x) % (y))
#endif

E
Eddie Dong 已提交
52 53 54 55 56 57 58 59 60
#define PRId64 "d"
#define PRIx64 "llx"
#define PRIu64 "u"
#define PRIo64 "o"

/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
#define apic_debug(fmt, arg...)

/* 14 is the version for Xeon and Pentium 8.4.8*/
61
#define APIC_VERSION			(0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
E
Eddie Dong 已提交
62 63 64 65 66 67
#define LAPIC_MMIO_LENGTH		(1 << 12)
/* followed define is not in apicdef.h */
#define APIC_SHORT_MASK			0xc0000
#define APIC_DEST_NOSHORT		0x0
#define APIC_DEST_MASK			0x800
#define MAX_APIC_VECTOR			256
68
#define APIC_VECTORS_PER_REG		32
E
Eddie Dong 已提交
69

70 71 72
#define APIC_BROADCAST			0xFF
#define X2APIC_BROADCAST		0xFFFFFFFFul

M
Michael S. Tsirkin 已提交
73 74 75 76 77
static inline int apic_test_vector(int vec, void *bitmap)
{
	return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

78 79 80 81 82 83 84 85
bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

	return apic_test_vector(vector, apic->regs + APIC_ISR) ||
		apic_test_vector(vector, apic->regs + APIC_IRR);
}

E
Eddie Dong 已提交
86 87 88 89 90
static inline void apic_clear_vector(int vec, void *bitmap)
{
	clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

M
Michael S. Tsirkin 已提交
91 92 93 94 95 96 97 98 99 100
static inline int __apic_test_and_set_vector(int vec, void *bitmap)
{
	return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
{
	return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

101
struct static_key_deferred apic_hw_disabled __read_mostly;
102 103
struct static_key_deferred apic_sw_disabled __read_mostly;

E
Eddie Dong 已提交
104 105
static inline int apic_enabled(struct kvm_lapic *apic)
{
106
	return kvm_apic_sw_enabled(apic) &&	kvm_apic_hw_enabled(apic);
107 108
}

E
Eddie Dong 已提交
109 110 111 112 113 114 115
#define LVT_MASK	\
	(APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)

#define LINT_MASK	\
	(LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
	 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)

116 117 118 119 120 121 122 123 124 125
static inline u8 kvm_xapic_id(struct kvm_lapic *apic)
{
	return kvm_lapic_get_reg(apic, APIC_ID) >> 24;
}

static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
{
	return apic->vcpu->vcpu_id;
}

126 127 128 129 130
static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
		u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
	switch (map->mode) {
	case KVM_APIC_MODE_X2APIC: {
		u32 offset = (dest_id >> 16) * 16;
R
Radim Krčmář 已提交
131
		u32 max_apic_id = map->max_apic_id;
132 133 134 135 136 137 138 139 140

		if (offset <= max_apic_id) {
			u8 cluster_size = min(max_apic_id - offset + 1, 16U);

			*cluster = &map->phys_map[offset];
			*mask = dest_id & (0xffff >> (16 - cluster_size));
		} else {
			*mask = 0;
		}
141

142 143 144 145 146 147 148
		return true;
		}
	case KVM_APIC_MODE_XAPIC_FLAT:
		*cluster = map->xapic_flat_map;
		*mask = dest_id & 0xff;
		return true;
	case KVM_APIC_MODE_XAPIC_CLUSTER:
149
		*cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
150 151 152 153 154 155
		*mask = dest_id & 0xf;
		return true;
	default:
		/* Not optimized. */
		return false;
	}
156 157
}

158
static void kvm_apic_map_free(struct rcu_head *rcu)
159
{
160
	struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
161

162
	kvfree(map);
163 164
}

165 166 167 168 169
static void recalculate_apic_map(struct kvm *kvm)
{
	struct kvm_apic_map *new, *old = NULL;
	struct kvm_vcpu *vcpu;
	int i;
170
	u32 max_id = 255; /* enough space for any xAPIC ID */
171 172 173

	mutex_lock(&kvm->arch.apic_map_lock);

R
Radim Krčmář 已提交
174 175
	kvm_for_each_vcpu(i, vcpu, kvm)
		if (kvm_apic_present(vcpu))
176
			max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
R
Radim Krčmář 已提交
177

M
Michal Hocko 已提交
178 179
	new = kvzalloc(sizeof(struct kvm_apic_map) +
	                   sizeof(struct kvm_lapic *) * ((u64)max_id + 1), GFP_KERNEL);
R
Radim Krčmář 已提交
180

181 182 183
	if (!new)
		goto out;

R
Radim Krčmář 已提交
184 185
	new->max_apic_id = max_id;

186 187
	kvm_for_each_vcpu(i, vcpu, kvm) {
		struct kvm_lapic *apic = vcpu->arch.apic;
188 189
		struct kvm_lapic **cluster;
		u16 mask;
190 191 192
		u32 ldr;
		u8 xapic_id;
		u32 x2apic_id;
193

194 195 196
		if (!kvm_apic_present(vcpu))
			continue;

197 198 199 200 201 202 203 204 205 206 207 208 209
		xapic_id = kvm_xapic_id(apic);
		x2apic_id = kvm_x2apic_id(apic);

		/* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
		if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
				x2apic_id <= new->max_apic_id)
			new->phys_map[x2apic_id] = apic;
		/*
		 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
		 * prevent them from masking VCPUs with APIC ID <= 0xff.
		 */
		if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
			new->phys_map[xapic_id] = apic;
210

211 212
		ldr = kvm_lapic_get_reg(apic, APIC_LDR);

213 214 215 216
		if (apic_x2apic_mode(apic)) {
			new->mode |= KVM_APIC_MODE_X2APIC;
		} else if (ldr) {
			ldr = GET_APIC_LOGICAL_ID(ldr);
217
			if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
218 219 220 221 222
				new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
			else
				new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
		}

223
		if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
224 225
			continue;

226 227
		if (mask)
			cluster[ffs(mask) - 1] = apic;
228 229 230 231 232 233 234 235
	}
out:
	old = rcu_dereference_protected(kvm->arch.apic_map,
			lockdep_is_held(&kvm->arch.apic_map_lock));
	rcu_assign_pointer(kvm->arch.apic_map, new);
	mutex_unlock(&kvm->arch.apic_map_lock);

	if (old)
236
		call_rcu(&old->rcu, kvm_apic_map_free);
237

238
	kvm_make_scan_ioapic_request(kvm);
239 240
}

241 242
static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
{
243
	bool enabled = val & APIC_SPIV_APIC_ENABLED;
244

245
	kvm_lapic_set_reg(apic, APIC_SPIV, val);
246 247 248 249

	if (enabled != apic->sw_enabled) {
		apic->sw_enabled = enabled;
		if (enabled) {
250 251 252 253 254 255 256
			static_key_slow_dec_deferred(&apic_sw_disabled);
			recalculate_apic_map(apic->vcpu->kvm);
		} else
			static_key_slow_inc(&apic_sw_disabled.key);
	}
}

257
static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
258
{
259
	kvm_lapic_set_reg(apic, APIC_ID, id << 24);
260 261 262 263 264
	recalculate_apic_map(apic->vcpu->kvm);
}

static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
{
265
	kvm_lapic_set_reg(apic, APIC_LDR, id);
266 267 268
	recalculate_apic_map(apic->vcpu->kvm);
}

269 270 271 272 273
static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
{
	return ((id >> 4) << 16) | (1 << (id & 0xf));
}

274
static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
275
{
276
	u32 ldr = kvm_apic_calc_x2apic_ldr(id);
277

278 279
	WARN_ON_ONCE(id != apic->vcpu->vcpu_id);

280
	kvm_lapic_set_reg(apic, APIC_ID, id);
281
	kvm_lapic_set_reg(apic, APIC_LDR, ldr);
282 283 284
	recalculate_apic_map(apic->vcpu->kvm);
}

E
Eddie Dong 已提交
285 286
static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
{
287
	return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
E
Eddie Dong 已提交
288 289 290 291
}

static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
{
292
	return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
E
Eddie Dong 已提交
293 294
}

295 296
static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
{
297
	return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
298 299
}

E
Eddie Dong 已提交
300 301
static inline int apic_lvtt_period(struct kvm_lapic *apic)
{
302
	return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
303 304 305 306
}

static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
{
307
	return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
E
Eddie Dong 已提交
308 309
}

310 311 312 313 314
static inline int apic_lvt_nmi_mode(u32 lvt_val)
{
	return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
}

315 316 317 318 319 320
void kvm_apic_set_version(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	struct kvm_cpuid_entry2 *feat;
	u32 v = APIC_VERSION;

321
	if (!lapic_in_kernel(vcpu))
322 323 324 325 326
		return;

	feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
	if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))))
		v |= APIC_LVR_DIRECTED_EOI;
327
	kvm_lapic_set_reg(apic, APIC_LVR, v);
328 329
}

330
static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
331
	LVT_MASK ,      /* part LVTT mask, timer mode mask added at runtime */
E
Eddie Dong 已提交
332 333 334 335 336 337 338 339
	LVT_MASK | APIC_MODE_MASK,	/* LVTTHMR */
	LVT_MASK | APIC_MODE_MASK,	/* LVTPC */
	LINT_MASK, LINT_MASK,	/* LVT0-1 */
	LVT_MASK		/* LVTERR */
};

static int find_highest_vector(void *bitmap)
{
340 341
	int vec;
	u32 *reg;
E
Eddie Dong 已提交
342

343 344 345 346
	for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
	     vec >= 0; vec -= APIC_VECTORS_PER_REG) {
		reg = bitmap + REG_POS(vec);
		if (*reg)
347
			return __fls(*reg) + vec;
348
	}
E
Eddie Dong 已提交
349

350
	return -1;
E
Eddie Dong 已提交
351 352
}

M
Michael S. Tsirkin 已提交
353 354
static u8 count_vectors(void *bitmap)
{
355 356
	int vec;
	u32 *reg;
M
Michael S. Tsirkin 已提交
357
	u8 count = 0;
358 359 360 361 362 363

	for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
		reg = bitmap + REG_POS(vec);
		count += hweight32(*reg);
	}

M
Michael S. Tsirkin 已提交
364 365 366
	return count;
}

367
bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
368
{
369
	u32 i, vec;
370 371 372 373 374
	u32 pir_val, irr_val, prev_irr_val;
	int max_updated_irr;

	max_updated_irr = -1;
	*max_irr = -1;
375

376
	for (i = vec = 0; i <= 7; i++, vec += 32) {
377
		pir_val = READ_ONCE(pir[i]);
378
		irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
379
		if (pir_val) {
380
			prev_irr_val = irr_val;
381 382
			irr_val |= xchg(&pir[i], 0);
			*((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
383 384 385 386
			if (prev_irr_val != irr_val) {
				max_updated_irr =
					__fls(irr_val ^ prev_irr_val) + vec;
			}
387
		}
388
		if (irr_val)
389
			*max_irr = __fls(irr_val) + vec;
390
	}
391

392 393
	return ((max_updated_irr != -1) &&
		(max_updated_irr == *max_irr));
394
}
395 396
EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);

397
bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
398 399 400
{
	struct kvm_lapic *apic = vcpu->arch.apic;

401
	return __kvm_apic_update_irr(pir, apic->regs, max_irr);
402
}
403 404
EXPORT_SYMBOL_GPL(kvm_apic_update_irr);

405
static inline int apic_search_irr(struct kvm_lapic *apic)
E
Eddie Dong 已提交
406
{
407
	return find_highest_vector(apic->regs + APIC_IRR);
E
Eddie Dong 已提交
408 409 410 411 412 413
}

static inline int apic_find_highest_irr(struct kvm_lapic *apic)
{
	int result;

414 415 416 417
	/*
	 * Note that irr_pending is just a hint. It will be always
	 * true with virtual interrupt delivery enabled.
	 */
418 419 420 421
	if (!apic->irr_pending)
		return -1;

	result = apic_search_irr(apic);
E
Eddie Dong 已提交
422 423 424 425 426
	ASSERT(result == -1 || result >= 16);

	return result;
}

427 428
static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
{
429 430 431 432
	struct kvm_vcpu *vcpu;

	vcpu = apic->vcpu;

433
	if (unlikely(vcpu->arch.apicv_active)) {
434
		/* need to update RVI */
435
		apic_clear_vector(vec, apic->regs + APIC_IRR);
436 437
		kvm_x86_ops->hwapic_irr_update(vcpu,
				apic_find_highest_irr(apic));
438 439 440 441 442
	} else {
		apic->irr_pending = false;
		apic_clear_vector(vec, apic->regs + APIC_IRR);
		if (apic_search_irr(apic) != -1)
			apic->irr_pending = true;
443
	}
444 445
}

M
Michael S. Tsirkin 已提交
446 447
static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
{
448 449 450 451 452 453
	struct kvm_vcpu *vcpu;

	if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
		return;

	vcpu = apic->vcpu;
454

M
Michael S. Tsirkin 已提交
455
	/*
456 457 458
	 * With APIC virtualization enabled, all caching is disabled
	 * because the processor can modify ISR under the hood.  Instead
	 * just set SVI.
M
Michael S. Tsirkin 已提交
459
	 */
460
	if (unlikely(vcpu->arch.apicv_active))
461
		kvm_x86_ops->hwapic_isr_update(vcpu, vec);
462 463 464 465 466 467 468 469 470 471
	else {
		++apic->isr_count;
		BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
		/*
		 * ISR (in service register) bit is set when injecting an interrupt.
		 * The highest vector is injected. Thus the latest bit set matches
		 * the highest bit in ISR.
		 */
		apic->highest_isr_cache = vec;
	}
M
Michael S. Tsirkin 已提交
472 473
}

474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
static inline int apic_find_highest_isr(struct kvm_lapic *apic)
{
	int result;

	/*
	 * Note that isr_count is always 1, and highest_isr_cache
	 * is always -1, with APIC virtualization enabled.
	 */
	if (!apic->isr_count)
		return -1;
	if (likely(apic->highest_isr_cache != -1))
		return apic->highest_isr_cache;

	result = find_highest_vector(apic->regs + APIC_ISR);
	ASSERT(result == -1 || result >= 16);

	return result;
}

M
Michael S. Tsirkin 已提交
493 494
static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
{
495 496 497 498 499 500 501 502 503 504 505 506 507
	struct kvm_vcpu *vcpu;
	if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
		return;

	vcpu = apic->vcpu;

	/*
	 * We do get here for APIC virtualization enabled if the guest
	 * uses the Hyper-V APIC enlightenment.  In this case we may need
	 * to trigger a new interrupt delivery by writing the SVI field;
	 * on the other hand isr_count and highest_isr_cache are unused
	 * and must be left alone.
	 */
508
	if (unlikely(vcpu->arch.apicv_active))
509
		kvm_x86_ops->hwapic_isr_update(vcpu,
510 511
					       apic_find_highest_isr(apic));
	else {
M
Michael S. Tsirkin 已提交
512
		--apic->isr_count;
513 514 515
		BUG_ON(apic->isr_count < 0);
		apic->highest_isr_cache = -1;
	}
M
Michael S. Tsirkin 已提交
516 517
}

518 519
int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
{
520 521 522 523 524
	/* This may race with setting of irr in __apic_accept_irq() and
	 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
	 * will cause vmexit immediately and the value will be recalculated
	 * on the next vmentry.
	 */
525
	return apic_find_highest_irr(vcpu->arch.apic);
526
}
527
EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
528

529
static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
530
			     int vector, int level, int trig_mode,
531
			     struct dest_map *dest_map);
532

533
int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
534
		     struct dest_map *dest_map)
E
Eddie Dong 已提交
535
{
536
	struct kvm_lapic *apic = vcpu->arch.apic;
537

538
	return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
539
			irq->level, irq->trig_mode, dest_map);
E
Eddie Dong 已提交
540 541
}

542 543
static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
{
544 545 546

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
				      sizeof(val));
547 548 549 550
}

static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
{
551 552 553

	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
				      sizeof(*val));
554 555 556 557 558 559 560 561 562 563 564 565
}

static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
{
	return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
}

static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
{
	u8 val;
	if (pv_eoi_get_user(vcpu, &val) < 0)
		apic_debug("Can't read EOI MSR value: 0x%llx\n",
566
			   (unsigned long long)vcpu->arch.pv_eoi.msr_val);
567 568 569 570 571 572 573
	return val & 0x1;
}

static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
{
	if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
		apic_debug("Can't set EOI MSR value: 0x%llx\n",
574
			   (unsigned long long)vcpu->arch.pv_eoi.msr_val);
575 576 577 578 579 580 581 582 583
		return;
	}
	__set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
}

static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
{
	if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
		apic_debug("Can't clear EOI MSR value: 0x%llx\n",
584
			   (unsigned long long)vcpu->arch.pv_eoi.msr_val);
585 586 587 588 589
		return;
	}
	__clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
}

590 591
static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
{
592
	int highest_irr;
593
	if (apic->vcpu->arch.apicv_active)
594 595 596
		highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
	else
		highest_irr = apic_find_highest_irr(apic);
597 598 599 600 601 602
	if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
		return -1;
	return highest_irr;
}

static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
E
Eddie Dong 已提交
603
{
604
	u32 tpr, isrv, ppr, old_ppr;
E
Eddie Dong 已提交
605 606
	int isr;

607 608
	old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
	tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
E
Eddie Dong 已提交
609 610 611 612 613 614 615 616 617 618 619
	isr = apic_find_highest_isr(apic);
	isrv = (isr != -1) ? isr : 0;

	if ((tpr & 0xf0) >= (isrv & 0xf0))
		ppr = tpr & 0xff;
	else
		ppr = isrv & 0xf0;

	apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x",
		   apic, ppr, isr, isrv);

620 621
	*new_ppr = ppr;
	if (old_ppr != ppr)
622
		kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
623 624 625 626 627 628 629 630

	return ppr < old_ppr;
}

static void apic_update_ppr(struct kvm_lapic *apic)
{
	u32 ppr;

631 632
	if (__apic_update_ppr(apic, &ppr) &&
	    apic_has_interrupt_for_ppr(apic, ppr) != -1)
633
		kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
E
Eddie Dong 已提交
634 635
}

636 637 638 639 640 641
void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
{
	apic_update_ppr(vcpu->arch.apic);
}
EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);

E
Eddie Dong 已提交
642 643
static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
{
644
	kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
E
Eddie Dong 已提交
645 646 647
	apic_update_ppr(apic);
}

648
static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
649
{
650 651
	return mda == (apic_x2apic_mode(apic) ?
			X2APIC_BROADCAST : APIC_BROADCAST);
652 653
}

654
static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
E
Eddie Dong 已提交
655
{
656 657 658 659
	if (kvm_apic_broadcast(apic, mda))
		return true;

	if (apic_x2apic_mode(apic))
660
		return mda == kvm_x2apic_id(apic);
661

662 663 664 665 666 667 668 669 670
	/*
	 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
	 * it were in x2APIC mode.  Hotplugged VCPUs start in xAPIC mode and
	 * this allows unique addressing of VCPUs with APIC ID over 0xff.
	 * The 0xff condition is needed because writeable xAPIC ID.
	 */
	if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
		return true;

671
	return mda == kvm_xapic_id(apic);
E
Eddie Dong 已提交
672 673
}

674
static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
E
Eddie Dong 已提交
675
{
G
Gleb Natapov 已提交
676 677
	u32 logical_id;

678
	if (kvm_apic_broadcast(apic, mda))
679
		return true;
680

681
	logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
E
Eddie Dong 已提交
682

683
	if (apic_x2apic_mode(apic))
684 685
		return ((logical_id >> 16) == (mda >> 16))
		       && (logical_id & mda & 0xffff) != 0;
E
Eddie Dong 已提交
686

687
	logical_id = GET_APIC_LOGICAL_ID(logical_id);
E
Eddie Dong 已提交
688

689
	switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
E
Eddie Dong 已提交
690
	case APIC_DFR_FLAT:
691
		return (logical_id & mda) != 0;
E
Eddie Dong 已提交
692
	case APIC_DFR_CLUSTER:
693 694
		return ((logical_id >> 4) == (mda >> 4))
		       && (logical_id & mda & 0xf) != 0;
E
Eddie Dong 已提交
695
	default:
696
		apic_debug("Bad DFR vcpu %d: %08x\n",
697
			   apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR));
698
		return false;
E
Eddie Dong 已提交
699 700 701
	}
}

702 703
/* The KVM local APIC implementation has two quirks:
 *
704 705 706
 *  - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
 *    in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
 *    KVM doesn't do that aliasing.
707 708 709 710 711 712 713 714 715 716
 *
 *  - in-kernel IOAPIC messages have to be delivered directly to
 *    x2APIC, because the kernel does not support interrupt remapping.
 *    In order to support broadcast without interrupt remapping, x2APIC
 *    rewrites the destination of non-IPI messages from APIC_BROADCAST
 *    to X2APIC_BROADCAST.
 *
 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
 * important when userspace wants to use x2APIC-format MSIs, because
 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
717
 */
718 719
static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
		struct kvm_lapic *source, struct kvm_lapic *target)
720 721 722
{
	bool ipi = source != NULL;

723
	if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
724
	    !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
725 726
		return X2APIC_BROADCAST;

727
	return dest_id;
728 729
}

730
bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
731
			   int short_hand, unsigned int dest, int dest_mode)
E
Eddie Dong 已提交
732
{
733
	struct kvm_lapic *target = vcpu->arch.apic;
734
	u32 mda = kvm_apic_mda(vcpu, dest, source, target);
E
Eddie Dong 已提交
735 736

	apic_debug("target %p, source %p, dest 0x%x, "
737
		   "dest_mode 0x%x, short_hand 0x%x\n",
E
Eddie Dong 已提交
738 739
		   target, source, dest, dest_mode, short_hand);

Z
Zachary Amsden 已提交
740
	ASSERT(target);
E
Eddie Dong 已提交
741 742
	switch (short_hand) {
	case APIC_DEST_NOSHORT:
743
		if (dest_mode == APIC_DEST_PHYSICAL)
744
			return kvm_apic_match_physical_addr(target, mda);
745
		else
746
			return kvm_apic_match_logical_addr(target, mda);
E
Eddie Dong 已提交
747
	case APIC_DEST_SELF:
748
		return target == source;
E
Eddie Dong 已提交
749
	case APIC_DEST_ALLINC:
750
		return true;
E
Eddie Dong 已提交
751
	case APIC_DEST_ALLBUT:
752
		return target != source;
E
Eddie Dong 已提交
753
	default:
754 755
		apic_debug("kvm: apic: Bad dest shorthand value %x\n",
			   short_hand);
756
		return false;
E
Eddie Dong 已提交
757 758
	}
}
759
EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
E
Eddie Dong 已提交
760

761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776
int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
		       const unsigned long *bitmap, u32 bitmap_size)
{
	u32 mod;
	int i, idx = -1;

	mod = vector % dest_vcpus;

	for (i = 0; i <= mod; i++) {
		idx = find_next_bit(bitmap, bitmap_size, idx + 1);
		BUG_ON(idx == bitmap_size);
	}

	return idx;
}

777 778 779 780 781 782 783 784 785
static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
{
	if (!kvm->arch.disabled_lapic_found) {
		kvm->arch.disabled_lapic_found = true;
		printk(KERN_INFO
		       "Disabled LAPIC found during irq injection\n");
	}
}

786 787
static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
		struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
788
{
789 790 791 792 793 794 795 796 797 798 799 800
	if (kvm->arch.x2apic_broadcast_quirk_disabled) {
		if ((irq->dest_id == APIC_BROADCAST &&
				map->mode != KVM_APIC_MODE_X2APIC))
			return true;
		if (irq->dest_id == X2APIC_BROADCAST)
			return true;
	} else {
		bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
		if (irq->dest_id == (x2apic_ipi ?
		                     X2APIC_BROADCAST : APIC_BROADCAST))
			return true;
	}
801

802 803
	return false;
}
804

805 806 807 808 809 810 811 812 813 814 815 816 817
/* Return true if the interrupt can be handled by using *bitmap as index mask
 * for valid destinations in *dst array.
 * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
 * Note: we may have zero kvm_lapic destinations when we return true, which
 * means that the interrupt should be dropped.  In this case, *bitmap would be
 * zero and *dst undefined.
 */
static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
		struct kvm_lapic **src, struct kvm_lapic_irq *irq,
		struct kvm_apic_map *map, struct kvm_lapic ***dst,
		unsigned long *bitmap)
{
	int i, lowest;
818

819 820 821 822 823
	if (irq->shorthand == APIC_DEST_SELF && src) {
		*dst = src;
		*bitmap = 1;
		return true;
	} else if (irq->shorthand)
824 825
		return false;

826
	if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
827 828
		return false;

829
	if (irq->dest_mode == APIC_DEST_PHYSICAL) {
R
Radim Krčmář 已提交
830
		if (irq->dest_id > map->max_apic_id) {
831 832 833 834 835
			*bitmap = 0;
		} else {
			*dst = &map->phys_map[irq->dest_id];
			*bitmap = 1;
		}
836
		return true;
837
	}
838

839 840 841
	*bitmap = 0;
	if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
				(u16 *)bitmap))
842
		return false;
843

844 845
	if (!kvm_lowest_prio_delivery(irq))
		return true;
846

847 848 849 850 851 852 853 854 855 856
	if (!kvm_vector_hashing_enabled()) {
		lowest = -1;
		for_each_set_bit(i, bitmap, 16) {
			if (!(*dst)[i])
				continue;
			if (lowest < 0)
				lowest = i;
			else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
						(*dst)[lowest]->vcpu) < 0)
				lowest = i;
857
		}
858 859 860
	} else {
		if (!*bitmap)
			return true;
861

862 863
		lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
				bitmap, 16);
864

865 866 867 868 869 870
		if (!(*dst)[lowest]) {
			kvm_apic_disabled_lapic_found(kvm);
			*bitmap = 0;
			return true;
		}
	}
871

872
	*bitmap = (lowest >= 0) ? 1 << lowest : 0;
873

874 875
	return true;
}
876

877 878 879 880 881 882 883 884
bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
		struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
{
	struct kvm_apic_map *map;
	unsigned long bitmap;
	struct kvm_lapic **dst = NULL;
	int i;
	bool ret;
885

886
	*r = -1;
887

888 889 890 891
	if (irq->shorthand == APIC_DEST_SELF) {
		*r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
		return true;
	}
892

893 894
	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);
895

896 897 898 899 900 901 902 903
	ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
	if (ret)
		for_each_set_bit(i, &bitmap, 16) {
			if (!dst[i])
				continue;
			if (*r < 0)
				*r = 0;
			*r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
904 905 906 907 908 909
		}

	rcu_read_unlock();
	return ret;
}

910 911 912 913 914 915 916 917 918 919 920 921 922 923
/*
 * This routine tries to handler interrupts in posted mode, here is how
 * it deals with different cases:
 * - For single-destination interrupts, handle it in posted mode
 * - Else if vector hashing is enabled and it is a lowest-priority
 *   interrupt, handle it in posted mode and use the following mechanism
 *   to find the destinaiton vCPU.
 *	1. For lowest-priority interrupts, store all the possible
 *	   destination vCPUs in an array.
 *	2. Use "guest vector % max number of destination vCPUs" to find
 *	   the right destination vCPU in the array for the lowest-priority
 *	   interrupt.
 * - Otherwise, use remapped mode to inject the interrupt.
 */
924 925 926 927
bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
			struct kvm_vcpu **dest_vcpu)
{
	struct kvm_apic_map *map;
928 929
	unsigned long bitmap;
	struct kvm_lapic **dst = NULL;
930 931 932 933 934 935 936 937
	bool ret = false;

	if (irq->shorthand)
		return false;

	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);

938 939 940
	if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
			hweight16(bitmap) == 1) {
		unsigned long i = find_first_bit(&bitmap, 16);
941

942 943 944
		if (dst[i]) {
			*dest_vcpu = dst[i]->vcpu;
			ret = true;
945
		}
946 947 948 949 950 951
	}

	rcu_read_unlock();
	return ret;
}

E
Eddie Dong 已提交
952 953 954 955 956
/*
 * Add a pending IRQ into lapic.
 * Return 1 if successfully added and 0 if discarded.
 */
static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
957
			     int vector, int level, int trig_mode,
958
			     struct dest_map *dest_map)
E
Eddie Dong 已提交
959
{
960
	int result = 0;
961
	struct kvm_vcpu *vcpu = apic->vcpu;
E
Eddie Dong 已提交
962

963 964
	trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
				  trig_mode, vector);
E
Eddie Dong 已提交
965 966
	switch (delivery_mode) {
	case APIC_DM_LOWEST:
967 968
		vcpu->arch.apic_arb_prio++;
	case APIC_DM_FIXED:
969 970 971
		if (unlikely(trig_mode && !level))
			break;

E
Eddie Dong 已提交
972 973 974 975
		/* FIXME add logic for vcpu on reset */
		if (unlikely(!apic_enabled(apic)))
			break;

976 977
		result = 1;

978
		if (dest_map) {
979
			__set_bit(vcpu->vcpu_id, dest_map->map);
980 981
			dest_map->vectors[vcpu->vcpu_id] = vector;
		}
982

983 984
		if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
			if (trig_mode)
985
				kvm_lapic_set_vector(vector, apic->regs + APIC_TMR);
986 987 988 989
			else
				apic_clear_vector(vector, apic->regs + APIC_TMR);
		}

990
		if (vcpu->arch.apicv_active)
991
			kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
992
		else {
993
			kvm_lapic_set_irr(vector, apic);
994 995 996 997

			kvm_make_request(KVM_REQ_EVENT, vcpu);
			kvm_vcpu_kick(vcpu);
		}
E
Eddie Dong 已提交
998 999 1000
		break;

	case APIC_DM_REMRD:
1001 1002 1003 1004
		result = 1;
		vcpu->arch.pv.pv_unhalted = 1;
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		kvm_vcpu_kick(vcpu);
E
Eddie Dong 已提交
1005 1006 1007
		break;

	case APIC_DM_SMI:
P
Paolo Bonzini 已提交
1008 1009 1010
		result = 1;
		kvm_make_request(KVM_REQ_SMI, vcpu);
		kvm_vcpu_kick(vcpu);
E
Eddie Dong 已提交
1011
		break;
1012

E
Eddie Dong 已提交
1013
	case APIC_DM_NMI:
1014
		result = 1;
1015
		kvm_inject_nmi(vcpu);
J
Jan Kiszka 已提交
1016
		kvm_vcpu_kick(vcpu);
E
Eddie Dong 已提交
1017 1018 1019
		break;

	case APIC_DM_INIT:
1020
		if (!trig_mode || level) {
1021
			result = 1;
1022 1023 1024 1025 1026
			/* assumes that there are only KVM_APIC_INIT/SIPI */
			apic->pending_events = (1UL << KVM_APIC_INIT);
			/* make sure pending_events is visible before sending
			 * the request */
			smp_wmb();
1027
			kvm_make_request(KVM_REQ_EVENT, vcpu);
1028 1029
			kvm_vcpu_kick(vcpu);
		} else {
1030 1031
			apic_debug("Ignoring de-assert INIT to vcpu %d\n",
				   vcpu->vcpu_id);
1032
		}
E
Eddie Dong 已提交
1033 1034 1035
		break;

	case APIC_DM_STARTUP:
1036 1037
		apic_debug("SIPI to vcpu %d vector 0x%02x\n",
			   vcpu->vcpu_id, vector);
1038 1039 1040 1041 1042 1043 1044
		result = 1;
		apic->sipi_vector = vector;
		/* make sure sipi_vector is visible for the receiver */
		smp_wmb();
		set_bit(KVM_APIC_SIPI, &apic->pending_events);
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		kvm_vcpu_kick(vcpu);
E
Eddie Dong 已提交
1045 1046
		break;

1047 1048 1049 1050 1051 1052 1053 1054
	case APIC_DM_EXTINT:
		/*
		 * Should only be called by kvm_apic_local_deliver() with LVT0,
		 * before NMI watchdog was enabled. Already handled by
		 * kvm_apic_accept_pic_intr().
		 */
		break;

E
Eddie Dong 已提交
1055 1056 1057 1058 1059 1060 1061 1062
	default:
		printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
		       delivery_mode);
		break;
	}
	return result;
}

1063
int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
1064
{
1065
	return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
1066 1067
}

1068 1069
static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
{
1070
	return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
1071 1072
}

1073 1074
static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
{
1075 1076 1077 1078 1079
	int trigger_mode;

	/* Eoi the ioapic only if the ioapic doesn't own the vector. */
	if (!kvm_ioapic_handles_vector(apic, vector))
		return;
1080

1081 1082 1083 1084 1085
	/* Request a KVM exit to inform the userspace IOAPIC. */
	if (irqchip_split(apic->vcpu->kvm)) {
		apic->vcpu->arch.pending_ioapic_eoi = vector;
		kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
		return;
1086
	}
1087 1088 1089 1090 1091 1092 1093

	if (apic_test_vector(vector, apic->regs + APIC_TMR))
		trigger_mode = IOAPIC_LEVEL_TRIG;
	else
		trigger_mode = IOAPIC_EDGE_TRIG;

	kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
1094 1095
}

1096
static int apic_set_eoi(struct kvm_lapic *apic)
E
Eddie Dong 已提交
1097 1098
{
	int vector = apic_find_highest_isr(apic);
1099 1100 1101

	trace_kvm_eoi(apic, vector);

E
Eddie Dong 已提交
1102 1103 1104 1105 1106
	/*
	 * Not every write EOI will has corresponding ISR,
	 * one example is when Kernel check timer on setup_IO_APIC
	 */
	if (vector == -1)
1107
		return vector;
E
Eddie Dong 已提交
1108

M
Michael S. Tsirkin 已提交
1109
	apic_clear_isr(vector, apic);
E
Eddie Dong 已提交
1110 1111
	apic_update_ppr(apic);

1112 1113 1114
	if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
		kvm_hv_synic_send_eoi(apic->vcpu, vector);

1115
	kvm_ioapic_send_eoi(apic, vector);
1116
	kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1117
	return vector;
E
Eddie Dong 已提交
1118 1119
}

1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
/*
 * this interface assumes a trap-like exit, which has already finished
 * desired side effect including vISR and vPPR update.
 */
void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

	trace_kvm_eoi(apic, vector);

	kvm_ioapic_send_eoi(apic, vector);
	kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
}
EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);

E
Eddie Dong 已提交
1135 1136
static void apic_send_ipi(struct kvm_lapic *apic)
{
1137 1138
	u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR);
	u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2);
1139
	struct kvm_lapic_irq irq;
E
Eddie Dong 已提交
1140

1141 1142 1143
	irq.vector = icr_low & APIC_VECTOR_MASK;
	irq.delivery_mode = icr_low & APIC_MODE_MASK;
	irq.dest_mode = icr_low & APIC_DEST_MASK;
1144
	irq.level = (icr_low & APIC_INT_ASSERT) != 0;
1145 1146
	irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
	irq.shorthand = icr_low & APIC_SHORT_MASK;
1147
	irq.msi_redir_hint = false;
G
Gleb Natapov 已提交
1148 1149 1150 1151
	if (apic_x2apic_mode(apic))
		irq.dest_id = icr_high;
	else
		irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
E
Eddie Dong 已提交
1152

1153 1154
	trace_kvm_apic_ipi(icr_low, irq.dest_id);

E
Eddie Dong 已提交
1155 1156
	apic_debug("icr_high 0x%x, icr_low 0x%x, "
		   "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
1157 1158
		   "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x, "
		   "msi_redir_hint 0x%x\n",
G
Glauber Costa 已提交
1159
		   icr_high, icr_low, irq.shorthand, irq.dest_id,
1160
		   irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode,
1161
		   irq.vector, irq.msi_redir_hint);
1162

1163
	kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
E
Eddie Dong 已提交
1164 1165 1166 1167
}

static u32 apic_get_tmcct(struct kvm_lapic *apic)
{
1168
	ktime_t remaining, now;
1169
	s64 ns;
1170
	u32 tmcct;
E
Eddie Dong 已提交
1171 1172 1173

	ASSERT(apic != NULL);

1174
	/* if initial count is 0, current count should also be 0 */
1175
	if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
1176
		apic->lapic_timer.period == 0)
1177 1178
		return 0;

1179
	now = ktime_get();
1180
	remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1181
	if (ktime_to_ns(remaining) < 0)
T
Thomas Gleixner 已提交
1182
		remaining = 0;
1183

1184 1185 1186
	ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
	tmcct = div64_u64(ns,
			 (APIC_BUS_CYCLE_NS * apic->divide_count));
E
Eddie Dong 已提交
1187 1188 1189 1190

	return tmcct;
}

1191 1192 1193 1194 1195
static void __report_tpr_access(struct kvm_lapic *apic, bool write)
{
	struct kvm_vcpu *vcpu = apic->vcpu;
	struct kvm_run *run = vcpu->run;

1196
	kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
1197
	run->tpr_access.rip = kvm_rip_read(vcpu);
1198 1199 1200 1201 1202 1203 1204 1205 1206
	run->tpr_access.is_write = write;
}

static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
{
	if (apic->vcpu->arch.tpr_access_reporting)
		__report_tpr_access(apic, write);
}

E
Eddie Dong 已提交
1207 1208 1209 1210 1211 1212 1213 1214 1215
static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
{
	u32 val = 0;

	if (offset >= LAPIC_MMIO_LENGTH)
		return 0;

	switch (offset) {
	case APIC_ARBPRI:
1216
		apic_debug("Access APIC ARBPRI register which is for P6\n");
E
Eddie Dong 已提交
1217 1218 1219
		break;

	case APIC_TMCCT:	/* Timer CCR */
1220 1221 1222
		if (apic_lvtt_tscdeadline(apic))
			return 0;

E
Eddie Dong 已提交
1223 1224
		val = apic_get_tmcct(apic);
		break;
1225 1226
	case APIC_PROCPRI:
		apic_update_ppr(apic);
1227
		val = kvm_lapic_get_reg(apic, offset);
1228
		break;
1229 1230 1231
	case APIC_TASKPRI:
		report_tpr_access(apic, false);
		/* fall thru */
E
Eddie Dong 已提交
1232
	default:
1233
		val = kvm_lapic_get_reg(apic, offset);
E
Eddie Dong 已提交
1234 1235 1236 1237 1238 1239
		break;
	}

	return val;
}

G
Gregory Haskins 已提交
1240 1241 1242 1243 1244
static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
{
	return container_of(dev, struct kvm_lapic, dev);
}

1245
int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
G
Gleb Natapov 已提交
1246
		void *data)
E
Eddie Dong 已提交
1247 1248 1249
{
	unsigned char alignment = offset & 0xf;
	u32 result;
G
Guo Chao 已提交
1250
	/* this bitmask has a bit cleared for each reserved register */
G
Gleb Natapov 已提交
1251
	static const u64 rmask = 0x43ff01ffffffe70cULL;
E
Eddie Dong 已提交
1252 1253

	if ((alignment + len) > 4) {
1254 1255
		apic_debug("KVM_APIC_READ: alignment error %x %d\n",
			   offset, len);
G
Gleb Natapov 已提交
1256
		return 1;
E
Eddie Dong 已提交
1257
	}
G
Gleb Natapov 已提交
1258 1259

	if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) {
1260 1261
		apic_debug("KVM_APIC_READ: read reserved register %x\n",
			   offset);
G
Gleb Natapov 已提交
1262 1263 1264
		return 1;
	}

E
Eddie Dong 已提交
1265 1266
	result = __apic_read(apic, offset & ~0xf);

1267 1268
	trace_kvm_apic_read(offset, result);

E
Eddie Dong 已提交
1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
	switch (len) {
	case 1:
	case 2:
	case 4:
		memcpy(data, (char *)&result + alignment, len);
		break;
	default:
		printk(KERN_ERR "Local APIC read with len = %x, "
		       "should be 1,2, or 4 instead\n", len);
		break;
	}
1280
	return 0;
E
Eddie Dong 已提交
1281
}
1282
EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
E
Eddie Dong 已提交
1283

G
Gleb Natapov 已提交
1284 1285
static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
{
1286
	return kvm_apic_hw_enabled(apic) &&
G
Gleb Natapov 已提交
1287 1288 1289 1290
	    addr >= apic->base_address &&
	    addr < apic->base_address + LAPIC_MMIO_LENGTH;
}

1291
static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
G
Gleb Natapov 已提交
1292 1293 1294 1295 1296 1297 1298 1299
			   gpa_t address, int len, void *data)
{
	struct kvm_lapic *apic = to_lapic(this);
	u32 offset = address - apic->base_address;

	if (!apic_mmio_in_range(apic, address))
		return -EOPNOTSUPP;

1300
	kvm_lapic_reg_read(apic, offset, len, data);
G
Gleb Natapov 已提交
1301 1302 1303 1304

	return 0;
}

E
Eddie Dong 已提交
1305 1306 1307 1308
static void update_divide_count(struct kvm_lapic *apic)
{
	u32 tmp1, tmp2, tdcr;

1309
	tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
E
Eddie Dong 已提交
1310 1311
	tmp1 = tdcr & 0xf;
	tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
1312
	apic->divide_count = 0x1 << (tmp2 & 0x7);
E
Eddie Dong 已提交
1313 1314

	apic_debug("timer divide count is 0x%x\n",
G
Glauber Costa 已提交
1315
				   apic->divide_count);
E
Eddie Dong 已提交
1316 1317
}

1318 1319 1320 1321 1322 1323 1324
static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
{
	/*
	 * Do not allow the guest to program periodic timers with small
	 * interval, since the hrtimers are not throttled by the host
	 * scheduler.
	 */
1325
	if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338
		s64 min_period = min_timer_period_us * 1000LL;

		if (apic->lapic_timer.period < min_period) {
			pr_info_ratelimited(
			    "kvm: vcpu %i: requested %lld ns "
			    "lapic timer period limited to %lld ns\n",
			    apic->vcpu->vcpu_id,
			    apic->lapic_timer.period, min_period);
			apic->lapic_timer.period = min_period;
		}
	}
}

1339 1340
static void apic_update_lvtt(struct kvm_lapic *apic)
{
1341
	u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
1342 1343 1344
			apic->lapic_timer.timer_mode_mask;

	if (apic->lapic_timer.timer_mode != timer_mode) {
1345
		if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
1346 1347
				APIC_LVT_TIMER_TSCDEADLINE)) {
			hrtimer_cancel(&apic->lapic_timer.timer);
1348 1349 1350
			kvm_lapic_set_reg(apic, APIC_TMICT, 0);
			apic->lapic_timer.period = 0;
			apic->lapic_timer.tscdeadline = 0;
1351
		}
1352
		apic->lapic_timer.timer_mode = timer_mode;
1353
		limit_periodic_timer_frequency(apic);
1354 1355 1356
	}
}

1357 1358 1359
static void apic_timer_expired(struct kvm_lapic *apic)
{
	struct kvm_vcpu *vcpu = apic->vcpu;
1360
	struct swait_queue_head *q = &vcpu->wq;
1361
	struct kvm_timer *ktimer = &apic->lapic_timer;
1362 1363 1364 1365 1366

	if (atomic_read(&apic->lapic_timer.pending))
		return;

	atomic_inc(&apic->lapic_timer.pending);
1367
	kvm_set_pending_timer(vcpu);
1368

1369 1370 1371 1372
	/*
	 * For x86, the atomic_inc() is serialized, thus
	 * using swait_active() is safe.
	 */
1373 1374
	if (swait_active(q))
		swake_up(q);
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387

	if (apic_lvtt_tscdeadline(apic))
		ktimer->expired_tscdeadline = ktimer->tscdeadline;
}

/*
 * On APICv, this test will cause a busy wait
 * during a higher-priority task.
 */

static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
1388
	u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
1389 1390 1391

	if (kvm_apic_hw_enabled(apic)) {
		int vec = reg & APIC_VECTOR_MASK;
1392
		void *bitmap = apic->regs + APIC_ISR;
1393

1394
		if (vcpu->arch.apicv_active)
1395 1396 1397 1398
			bitmap = apic->regs + APIC_IRR;

		if (apic_test_vector(vec, bitmap))
			return true;
1399 1400 1401 1402 1403 1404 1405 1406 1407
	}
	return false;
}

void wait_lapic_expire(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u64 guest_tsc, tsc_deadline;

1408
	if (!lapic_in_kernel(vcpu))
1409 1410 1411 1412 1413 1414 1415 1416 1417 1418
		return;

	if (apic->lapic_timer.expired_tscdeadline == 0)
		return;

	if (!lapic_timer_int_injected(vcpu))
		return;

	tsc_deadline = apic->lapic_timer.expired_tscdeadline;
	apic->lapic_timer.expired_tscdeadline = 0;
1419
	guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1420
	trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline);
1421 1422 1423

	/* __delay is delay_tsc whenever the hardware has TSC, thus always.  */
	if (guest_tsc < tsc_deadline)
1424 1425
		__delay(min(tsc_deadline - guest_tsc,
			nsec_to_cycles(vcpu, lapic_timer_advance_ns)));
1426 1427
}

1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442
static void start_sw_tscdeadline(struct kvm_lapic *apic)
{
	u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline;
	u64 ns = 0;
	ktime_t expire;
	struct kvm_vcpu *vcpu = apic->vcpu;
	unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
	unsigned long flags;
	ktime_t now;

	if (unlikely(!tscdeadline || !this_tsc_khz))
		return;

	local_irq_save(flags);

1443
	now = ktime_get();
1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
	guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
	if (likely(tscdeadline > guest_tsc)) {
		ns = (tscdeadline - guest_tsc) * 1000000ULL;
		do_div(ns, this_tsc_khz);
		expire = ktime_add_ns(now, ns);
		expire = ktime_sub_ns(expire, lapic_timer_advance_ns);
		hrtimer_start(&apic->lapic_timer.timer,
				expire, HRTIMER_MODE_ABS_PINNED);
	} else
		apic_timer_expired(apic);

	local_irq_restore(flags);
}

1458
static void start_sw_period(struct kvm_lapic *apic)
1459 1460 1461 1462 1463
{
	if (!apic->lapic_timer.period)
		return;

	if (apic_lvtt_oneshot(apic) &&
1464
	    ktime_after(ktime_get(),
1465 1466 1467 1468 1469 1470 1471 1472 1473 1474
			apic->lapic_timer.target_expiration)) {
		apic_timer_expired(apic);
		return;
	}

	hrtimer_start(&apic->lapic_timer.timer,
		apic->lapic_timer.target_expiration,
		HRTIMER_MODE_ABS_PINNED);
}

1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
{
	ktime_t now, remaining;
	u64 ns_remaining_old, ns_remaining_new;

	apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
		* APIC_BUS_CYCLE_NS * apic->divide_count;
	limit_periodic_timer_frequency(apic);

	now = ktime_get();
	remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
	if (ktime_to_ns(remaining) < 0)
		remaining = 0;

	ns_remaining_old = ktime_to_ns(remaining);
	ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
	                                   apic->divide_count, old_divisor);

	apic->lapic_timer.tscdeadline +=
		nsec_to_cycles(apic->vcpu, ns_remaining_new) -
		nsec_to_cycles(apic->vcpu, ns_remaining_old);
	apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
}

1499
static bool set_target_expiration(struct kvm_lapic *apic)
1500 1501
{
	ktime_t now;
1502
	u64 tscl = rdtsc();
1503

1504
	now = ktime_get();
1505
	apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
1506
		* APIC_BUS_CYCLE_NS * apic->divide_count;
1507

1508 1509
	if (!apic->lapic_timer.period) {
		apic->lapic_timer.tscdeadline = 0;
1510
		return false;
1511 1512
	}

1513
	limit_periodic_timer_frequency(apic);
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523

	apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
		   PRIx64 ", "
		   "timer initial count 0x%x, period %lldns, "
		   "expire @ 0x%016" PRIx64 ".\n", __func__,
		   APIC_BUS_CYCLE_NS, ktime_to_ns(now),
		   kvm_lapic_get_reg(apic, APIC_TMICT),
		   apic->lapic_timer.period,
		   ktime_to_ns(ktime_add_ns(now,
				apic->lapic_timer.period)));
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538

	apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
		nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
	apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);

	return true;
}

static void advance_periodic_target_expiration(struct kvm_lapic *apic)
{
	apic->lapic_timer.tscdeadline +=
		nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
	apic->lapic_timer.target_expiration =
		ktime_add_ns(apic->lapic_timer.target_expiration,
				apic->lapic_timer.period);
1539 1540
}

1541 1542
bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
{
1543 1544 1545
	if (!lapic_in_kernel(vcpu))
		return false;

1546 1547 1548 1549
	return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
}
EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);

1550
static void cancel_hv_timer(struct kvm_lapic *apic)
1551
{
1552
	WARN_ON(preemptible());
1553
	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1554 1555 1556 1557
	kvm_x86_ops->cancel_hv_timer(apic->vcpu);
	apic->lapic_timer.hv_timer_in_use = false;
}

1558
static bool start_hv_timer(struct kvm_lapic *apic)
1559
{
1560 1561
	struct kvm_timer *ktimer = &apic->lapic_timer;
	int r;
1562

1563
	WARN_ON(preemptible());
1564 1565 1566
	if (!kvm_x86_ops->set_hv_timer)
		return false;

1567 1568 1569
	if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
		return false;

1570 1571 1572
	if (!ktimer->tscdeadline)
		return false;

1573 1574 1575 1576 1577 1578
	r = kvm_x86_ops->set_hv_timer(apic->vcpu, ktimer->tscdeadline);
	if (r < 0)
		return false;

	ktimer->hv_timer_in_use = true;
	hrtimer_cancel(&ktimer->timer);
1579

1580 1581 1582 1583 1584
	/*
	 * Also recheck ktimer->pending, in case the sw timer triggered in
	 * the window.  For periodic timer, leave the hv timer running for
	 * simplicity, and the deadline will be recomputed on the next vmexit.
	 */
1585 1586 1587
	if (!apic_lvtt_period(apic) && (r || atomic_read(&ktimer->pending))) {
		if (r)
			apic_timer_expired(apic);
1588
		return false;
1589
	}
1590 1591

	trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, true);
1592 1593 1594
	return true;
}

1595
static void start_sw_timer(struct kvm_lapic *apic)
1596
{
1597
	struct kvm_timer *ktimer = &apic->lapic_timer;
1598 1599

	WARN_ON(preemptible());
1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610
	if (apic->lapic_timer.hv_timer_in_use)
		cancel_hv_timer(apic);
	if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
		return;

	if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
		start_sw_period(apic);
	else if (apic_lvtt_tscdeadline(apic))
		start_sw_tscdeadline(apic);
	trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
}
1611

1612 1613
static void restart_apic_timer(struct kvm_lapic *apic)
{
1614
	preempt_disable();
1615 1616
	if (!start_hv_timer(apic))
		start_sw_timer(apic);
1617
	preempt_enable();
1618 1619
}

1620 1621 1622 1623
void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

1624 1625 1626 1627
	preempt_disable();
	/* If the preempt notifier has already run, it also called apic_timer_expired */
	if (!apic->lapic_timer.hv_timer_in_use)
		goto out;
1628 1629 1630 1631 1632 1633
	WARN_ON(swait_active(&vcpu->wq));
	cancel_hv_timer(apic);
	apic_timer_expired(apic);

	if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
		advance_periodic_target_expiration(apic);
1634
		restart_apic_timer(apic);
1635
	}
1636 1637
out:
	preempt_enable();
1638 1639 1640
}
EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);

1641 1642
void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
{
1643
	restart_apic_timer(vcpu->arch.apic);
1644 1645 1646 1647 1648 1649 1650
}
EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);

void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

1651
	preempt_disable();
1652
	/* Possibly the TSC deadline timer is not enabled yet */
1653 1654
	if (apic->lapic_timer.hv_timer_in_use)
		start_sw_timer(apic);
1655
	preempt_enable();
1656 1657
}
EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
1658

1659 1660 1661
void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
1662

1663 1664
	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
	restart_apic_timer(apic);
1665 1666
}

E
Eddie Dong 已提交
1667 1668
static void start_apic_timer(struct kvm_lapic *apic)
{
1669
	atomic_set(&apic->lapic_timer.pending, 0);
1670

1671 1672 1673 1674 1675
	if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
	    && !set_target_expiration(apic))
		return;

	restart_apic_timer(apic);
E
Eddie Dong 已提交
1676 1677
}

1678 1679
static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
{
1680
	bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
1681

1682 1683 1684
	if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
		apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
		if (lvt0_in_nmi_mode) {
1685 1686
			apic_debug("Receive NMI setting on APIC_LVT0 "
				   "for cpu %d\n", apic->vcpu->vcpu_id);
1687
			atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1688 1689 1690
		} else
			atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
	}
1691 1692
}

1693
int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
E
Eddie Dong 已提交
1694
{
G
Gleb Natapov 已提交
1695
	int ret = 0;
E
Eddie Dong 已提交
1696

G
Gleb Natapov 已提交
1697
	trace_kvm_apic_write(reg, val);
E
Eddie Dong 已提交
1698

G
Gleb Natapov 已提交
1699
	switch (reg) {
E
Eddie Dong 已提交
1700
	case APIC_ID:		/* Local APIC ID */
G
Gleb Natapov 已提交
1701
		if (!apic_x2apic_mode(apic))
1702
			kvm_apic_set_xapic_id(apic, val >> 24);
G
Gleb Natapov 已提交
1703 1704
		else
			ret = 1;
E
Eddie Dong 已提交
1705 1706 1707
		break;

	case APIC_TASKPRI:
1708
		report_tpr_access(apic, true);
E
Eddie Dong 已提交
1709 1710 1711 1712 1713 1714 1715 1716
		apic_set_tpr(apic, val & 0xff);
		break;

	case APIC_EOI:
		apic_set_eoi(apic);
		break;

	case APIC_LDR:
G
Gleb Natapov 已提交
1717
		if (!apic_x2apic_mode(apic))
1718
			kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
G
Gleb Natapov 已提交
1719 1720
		else
			ret = 1;
E
Eddie Dong 已提交
1721 1722 1723
		break;

	case APIC_DFR:
1724
		if (!apic_x2apic_mode(apic)) {
1725
			kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1726 1727
			recalculate_apic_map(apic->vcpu->kvm);
		} else
G
Gleb Natapov 已提交
1728
			ret = 1;
E
Eddie Dong 已提交
1729 1730
		break;

1731 1732
	case APIC_SPIV: {
		u32 mask = 0x3ff;
1733
		if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
1734
			mask |= APIC_SPIV_DIRECTED_EOI;
1735
		apic_set_spiv(apic, val & mask);
E
Eddie Dong 已提交
1736 1737 1738 1739
		if (!(val & APIC_SPIV_APIC_ENABLED)) {
			int i;
			u32 lvt_val;

1740
			for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
1741
				lvt_val = kvm_lapic_get_reg(apic,
E
Eddie Dong 已提交
1742
						       APIC_LVTT + 0x10 * i);
1743
				kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
E
Eddie Dong 已提交
1744 1745
					     lvt_val | APIC_LVT_MASKED);
			}
1746
			apic_update_lvtt(apic);
1747
			atomic_set(&apic->lapic_timer.pending, 0);
E
Eddie Dong 已提交
1748 1749 1750

		}
		break;
1751
	}
E
Eddie Dong 已提交
1752 1753
	case APIC_ICR:
		/* No delay here, so we always clear the pending bit */
1754
		kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
E
Eddie Dong 已提交
1755 1756 1757 1758
		apic_send_ipi(apic);
		break;

	case APIC_ICR2:
G
Gleb Natapov 已提交
1759 1760
		if (!apic_x2apic_mode(apic))
			val &= 0xff000000;
1761
		kvm_lapic_set_reg(apic, APIC_ICR2, val);
E
Eddie Dong 已提交
1762 1763
		break;

1764
	case APIC_LVT0:
1765
		apic_manage_nmi_watchdog(apic, val);
E
Eddie Dong 已提交
1766 1767 1768 1769 1770
	case APIC_LVTTHMR:
	case APIC_LVTPC:
	case APIC_LVT1:
	case APIC_LVTERR:
		/* TODO: Check vector */
1771
		if (!kvm_apic_sw_enabled(apic))
E
Eddie Dong 已提交
1772 1773
			val |= APIC_LVT_MASKED;

G
Gleb Natapov 已提交
1774
		val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
1775
		kvm_lapic_set_reg(apic, reg, val);
E
Eddie Dong 已提交
1776 1777 1778

		break;

1779
	case APIC_LVTT:
1780
		if (!kvm_apic_sw_enabled(apic))
1781 1782
			val |= APIC_LVT_MASKED;
		val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
1783
		kvm_lapic_set_reg(apic, APIC_LVTT, val);
1784
		apic_update_lvtt(apic);
1785 1786
		break;

E
Eddie Dong 已提交
1787
	case APIC_TMICT:
1788 1789 1790
		if (apic_lvtt_tscdeadline(apic))
			break;

1791
		hrtimer_cancel(&apic->lapic_timer.timer);
1792
		kvm_lapic_set_reg(apic, APIC_TMICT, val);
E
Eddie Dong 已提交
1793
		start_apic_timer(apic);
G
Gleb Natapov 已提交
1794
		break;
E
Eddie Dong 已提交
1795

1796 1797 1798
	case APIC_TDCR: {
		uint32_t old_divisor = apic->divide_count;

E
Eddie Dong 已提交
1799
		if (val & 4)
1800
			apic_debug("KVM_WRITE:TDCR %x\n", val);
1801
		kvm_lapic_set_reg(apic, APIC_TDCR, val);
E
Eddie Dong 已提交
1802
		update_divide_count(apic);
1803 1804 1805 1806 1807 1808
		if (apic->divide_count != old_divisor &&
				apic->lapic_timer.period) {
			hrtimer_cancel(&apic->lapic_timer.timer);
			update_target_expiration(apic, old_divisor);
			restart_apic_timer(apic);
		}
E
Eddie Dong 已提交
1809
		break;
1810
	}
G
Gleb Natapov 已提交
1811 1812
	case APIC_ESR:
		if (apic_x2apic_mode(apic) && val != 0) {
1813
			apic_debug("KVM_WRITE:ESR not zero %x\n", val);
G
Gleb Natapov 已提交
1814 1815 1816 1817 1818 1819
			ret = 1;
		}
		break;

	case APIC_SELF_IPI:
		if (apic_x2apic_mode(apic)) {
1820
			kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
G
Gleb Natapov 已提交
1821 1822 1823
		} else
			ret = 1;
		break;
E
Eddie Dong 已提交
1824
	default:
G
Gleb Natapov 已提交
1825
		ret = 1;
E
Eddie Dong 已提交
1826 1827
		break;
	}
G
Gleb Natapov 已提交
1828 1829 1830 1831
	if (ret)
		apic_debug("Local APIC Write to read-only register %x\n", reg);
	return ret;
}
1832
EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
G
Gleb Natapov 已提交
1833

1834
static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
G
Gleb Natapov 已提交
1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851
			    gpa_t address, int len, const void *data)
{
	struct kvm_lapic *apic = to_lapic(this);
	unsigned int offset = address - apic->base_address;
	u32 val;

	if (!apic_mmio_in_range(apic, address))
		return -EOPNOTSUPP;

	/*
	 * APIC register must be aligned on 128-bits boundary.
	 * 32/64/128 bits registers must be accessed thru 32 bits.
	 * Refer SDM 8.4.1
	 */
	if (len != 4 || (offset & 0xf)) {
		/* Don't shout loud, $infamous_os would cause only noise. */
		apic_debug("apic write: bad size=%d %lx\n", len, (long)address);
1852
		return 0;
G
Gleb Natapov 已提交
1853 1854 1855 1856 1857 1858 1859 1860 1861
	}

	val = *(u32*)data;

	/* too common printing */
	if (offset != APIC_EOI)
		apic_debug("%s: offset 0x%x with length 0x%x, and value is "
			   "0x%x\n", __func__, offset, len, val);

1862
	kvm_lapic_reg_write(apic, offset & 0xff0, val);
G
Gleb Natapov 已提交
1863

1864
	return 0;
E
Eddie Dong 已提交
1865 1866
}

1867 1868
void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
{
1869
	kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
1870 1871 1872
}
EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);

1873 1874 1875 1876 1877 1878 1879 1880
/* emulate APIC access in a trap manner */
void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
{
	u32 val = 0;

	/* hw has done the conditional check and inst decode */
	offset &= 0xff0;

1881
	kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
1882 1883

	/* TODO: optimize to just emulate side effect w/o one more write */
1884
	kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
1885 1886 1887
}
EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);

1888
void kvm_free_lapic(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
1889
{
1890 1891
	struct kvm_lapic *apic = vcpu->arch.apic;

1892
	if (!vcpu->arch.apic)
E
Eddie Dong 已提交
1893 1894
		return;

1895
	hrtimer_cancel(&apic->lapic_timer.timer);
E
Eddie Dong 已提交
1896

1897 1898 1899
	if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
		static_key_slow_dec_deferred(&apic_hw_disabled);

1900
	if (!apic->sw_enabled)
1901
		static_key_slow_dec_deferred(&apic_sw_disabled);
E
Eddie Dong 已提交
1902

1903 1904 1905 1906
	if (apic->regs)
		free_page((unsigned long)apic->regs);

	kfree(apic);
E
Eddie Dong 已提交
1907 1908 1909 1910 1911 1912 1913
}

/*
 *----------------------------------------------------------------------
 * LAPIC interface
 *----------------------------------------------------------------------
 */
1914 1915 1916 1917
u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

1918 1919
	if (!lapic_in_kernel(vcpu) ||
		!apic_lvtt_tscdeadline(apic))
1920 1921 1922 1923 1924 1925 1926 1927 1928
		return 0;

	return apic->lapic_timer.tscdeadline;
}

void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

1929
	if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
1930
			apic_lvtt_period(apic))
1931 1932 1933 1934 1935 1936 1937
		return;

	hrtimer_cancel(&apic->lapic_timer.timer);
	apic->lapic_timer.tscdeadline = data;
	start_apic_timer(apic);
}

E
Eddie Dong 已提交
1938 1939
void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
{
1940
	struct kvm_lapic *apic = vcpu->arch.apic;
E
Eddie Dong 已提交
1941

A
Avi Kivity 已提交
1942
	apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
1943
		     | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
E
Eddie Dong 已提交
1944 1945 1946 1947 1948 1949
}

u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
{
	u64 tpr;

1950
	tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
E
Eddie Dong 已提交
1951 1952 1953 1954 1955 1956

	return (tpr & 0xf0) >> 4;
}

void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
{
1957
	u64 old_value = vcpu->arch.apic_base;
1958
	struct kvm_lapic *apic = vcpu->arch.apic;
E
Eddie Dong 已提交
1959

1960
	if (!apic)
E
Eddie Dong 已提交
1961
		value |= MSR_IA32_APICBASE_BSP;
1962

1963 1964
	vcpu->arch.apic_base = value;

1965 1966 1967 1968 1969 1970
	if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
		kvm_update_cpuid(vcpu);

	if (!apic)
		return;

1971
	/* update jump label if enable bit changes */
1972
	if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
1973 1974
		if (value & MSR_IA32_APICBASE_ENABLE) {
			kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
1975
			static_key_slow_dec_deferred(&apic_hw_disabled);
1976
		} else {
1977
			static_key_slow_inc(&apic_hw_disabled.key);
1978 1979
			recalculate_apic_map(vcpu->kvm);
		}
1980 1981
	}

1982 1983
	if ((old_value ^ value) & X2APIC_ENABLE) {
		if (value & X2APIC_ENABLE) {
1984
			kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
1985 1986 1987
			kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true);
		} else
			kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false);
G
Gleb Natapov 已提交
1988
	}
1989

1990
	apic->base_address = apic->vcpu->arch.apic_base &
E
Eddie Dong 已提交
1991 1992
			     MSR_IA32_APICBASE_BASE;

1993 1994 1995 1996
	if ((value & MSR_IA32_APICBASE_ENABLE) &&
	     apic->base_address != APIC_DEFAULT_PHYS_BASE)
		pr_warn_once("APIC base relocation is unsupported by KVM");

E
Eddie Dong 已提交
1997 1998
	/* with FSB delivery interrupt, we can restart APIC functionality */
	apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is "
1999
		   "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address);
E
Eddie Dong 已提交
2000 2001 2002

}

2003
void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
E
Eddie Dong 已提交
2004 2005 2006 2007
{
	struct kvm_lapic *apic;
	int i;

2008
	apic_debug("%s\n", __func__);
E
Eddie Dong 已提交
2009 2010

	ASSERT(vcpu);
2011
	apic = vcpu->arch.apic;
E
Eddie Dong 已提交
2012 2013 2014
	ASSERT(apic != NULL);

	/* Stop the timer in case it's a reset to an active apic */
2015
	hrtimer_cancel(&apic->lapic_timer.timer);
E
Eddie Dong 已提交
2016

2017 2018 2019
	if (!init_event) {
		kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
		                         MSR_IA32_APICBASE_ENABLE);
2020
		kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2021
	}
2022
	kvm_apic_set_version(apic->vcpu);
E
Eddie Dong 已提交
2023

2024 2025
	for (i = 0; i < KVM_APIC_LVT_NUM; i++)
		kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
2026
	apic_update_lvtt(apic);
2027 2028
	if (kvm_vcpu_is_reset_bsp(vcpu) &&
	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
2029
		kvm_lapic_set_reg(apic, APIC_LVT0,
2030
			     SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
2031
	apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
E
Eddie Dong 已提交
2032

2033
	kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
2034
	apic_set_spiv(apic, 0xff);
2035
	kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
2036 2037
	if (!apic_x2apic_mode(apic))
		kvm_apic_set_ldr(apic, 0);
2038 2039 2040 2041 2042
	kvm_lapic_set_reg(apic, APIC_ESR, 0);
	kvm_lapic_set_reg(apic, APIC_ICR, 0);
	kvm_lapic_set_reg(apic, APIC_ICR2, 0);
	kvm_lapic_set_reg(apic, APIC_TDCR, 0);
	kvm_lapic_set_reg(apic, APIC_TMICT, 0);
E
Eddie Dong 已提交
2043
	for (i = 0; i < 8; i++) {
2044 2045 2046
		kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
		kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
		kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
E
Eddie Dong 已提交
2047
	}
2048 2049
	apic->irr_pending = vcpu->arch.apicv_active;
	apic->isr_count = vcpu->arch.apicv_active ? 1 : 0;
M
Michael S. Tsirkin 已提交
2050
	apic->highest_isr_cache = -1;
2051
	update_divide_count(apic);
2052
	atomic_set(&apic->lapic_timer.pending, 0);
2053
	if (kvm_vcpu_is_bsp(vcpu))
2054 2055
		kvm_lapic_set_base(vcpu,
				vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
2056
	vcpu->arch.pv_eoi.msr_val = 0;
E
Eddie Dong 已提交
2057
	apic_update_ppr(apic);
2058 2059 2060 2061 2062
	if (vcpu->arch.apicv_active) {
		kvm_x86_ops->apicv_post_state_restore(vcpu);
		kvm_x86_ops->hwapic_irr_update(vcpu, -1);
		kvm_x86_ops->hwapic_isr_update(vcpu, -1);
	}
E
Eddie Dong 已提交
2063

2064
	vcpu->arch.apic_arb_prio = 0;
2065
	vcpu->arch.apic_attention = 0;
2066

2067
	apic_debug("%s: vcpu=%p, id=0x%x, base_msr="
2068
		   "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__,
2069
		   vcpu, kvm_lapic_get_reg(apic, APIC_ID),
2070
		   vcpu->arch.apic_base, apic->base_address);
E
Eddie Dong 已提交
2071 2072 2073 2074 2075 2076 2077
}

/*
 *----------------------------------------------------------------------
 * timer interface
 *----------------------------------------------------------------------
 */
2078

A
Avi Kivity 已提交
2079
static bool lapic_is_periodic(struct kvm_lapic *apic)
E
Eddie Dong 已提交
2080
{
2081
	return apic_lvtt_period(apic);
E
Eddie Dong 已提交
2082 2083
}

2084 2085
int apic_has_pending_timer(struct kvm_vcpu *vcpu)
{
2086
	struct kvm_lapic *apic = vcpu->arch.apic;
2087

2088
	if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
2089
		return atomic_read(&apic->lapic_timer.pending);
2090 2091 2092 2093

	return 0;
}

A
Avi Kivity 已提交
2094
int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
2095
{
2096
	u32 reg = kvm_lapic_get_reg(apic, lvt_type);
2097 2098
	int vector, mode, trig_mode;

2099
	if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
2100 2101 2102
		vector = reg & APIC_VECTOR_MASK;
		mode = reg & APIC_MODE_MASK;
		trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
2103 2104
		return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
					NULL);
2105 2106 2107
	}
	return 0;
}
2108

2109
void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
2110
{
2111 2112 2113 2114
	struct kvm_lapic *apic = vcpu->arch.apic;

	if (apic)
		kvm_apic_local_deliver(apic, APIC_LVT0);
2115 2116
}

G
Gregory Haskins 已提交
2117 2118 2119 2120 2121
static const struct kvm_io_device_ops apic_mmio_ops = {
	.read     = apic_mmio_read,
	.write    = apic_mmio_write,
};

2122 2123 2124
static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
{
	struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
A
Avi Kivity 已提交
2125
	struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
2126

2127
	apic_timer_expired(apic);
2128

A
Avi Kivity 已提交
2129
	if (lapic_is_periodic(apic)) {
2130
		advance_periodic_target_expiration(apic);
2131 2132 2133 2134 2135 2136
		hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
		return HRTIMER_RESTART;
	} else
		return HRTIMER_NORESTART;
}

E
Eddie Dong 已提交
2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147
int kvm_create_lapic(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic;

	ASSERT(vcpu != NULL);
	apic_debug("apic_init %d\n", vcpu->vcpu_id);

	apic = kzalloc(sizeof(*apic), GFP_KERNEL);
	if (!apic)
		goto nomem;

2148
	vcpu->arch.apic = apic;
E
Eddie Dong 已提交
2149

2150 2151
	apic->regs = (void *)get_zeroed_page(GFP_KERNEL);
	if (!apic->regs) {
E
Eddie Dong 已提交
2152 2153
		printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
		       vcpu->vcpu_id);
2154
		goto nomem_free_apic;
E
Eddie Dong 已提交
2155 2156 2157
	}
	apic->vcpu = vcpu;

2158
	hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2159
		     HRTIMER_MODE_ABS_PINNED);
2160
	apic->lapic_timer.timer.function = apic_timer_fn;
2161

2162 2163 2164 2165 2166
	/*
	 * APIC is created enabled. This will prevent kvm_lapic_set_base from
	 * thinking that APIC satet has changed.
	 */
	vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
2167
	static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
G
Gregory Haskins 已提交
2168
	kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
E
Eddie Dong 已提交
2169 2170

	return 0;
2171 2172
nomem_free_apic:
	kfree(apic);
E
Eddie Dong 已提交
2173 2174 2175 2176 2177 2178
nomem:
	return -ENOMEM;
}

int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
{
2179
	struct kvm_lapic *apic = vcpu->arch.apic;
2180
	u32 ppr;
E
Eddie Dong 已提交
2181

2182
	if (!apic_enabled(apic))
E
Eddie Dong 已提交
2183 2184
		return -1;

2185 2186
	__apic_update_ppr(apic, &ppr);
	return apic_has_interrupt_for_ppr(apic, ppr);
E
Eddie Dong 已提交
2187 2188
}

Q
Qing He 已提交
2189 2190
int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
{
2191
	u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
Q
Qing He 已提交
2192 2193
	int r = 0;

2194
	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
2195 2196 2197 2198
		r = 1;
	if ((lvt0 & APIC_LVT_MASKED) == 0 &&
	    GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
		r = 1;
Q
Qing He 已提交
2199 2200 2201
	return r;
}

2202 2203
void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
{
2204
	struct kvm_lapic *apic = vcpu->arch.apic;
2205

2206
	if (atomic_read(&apic->lapic_timer.pending) > 0) {
2207
		kvm_apic_local_deliver(apic, APIC_LVTT);
2208 2209
		if (apic_lvtt_tscdeadline(apic))
			apic->lapic_timer.tscdeadline = 0;
2210 2211
		if (apic_lvtt_oneshot(apic)) {
			apic->lapic_timer.tscdeadline = 0;
T
Thomas Gleixner 已提交
2212
			apic->lapic_timer.target_expiration = 0;
2213
		}
2214
		atomic_set(&apic->lapic_timer.pending, 0);
2215 2216 2217
	}
}

E
Eddie Dong 已提交
2218 2219 2220
int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
{
	int vector = kvm_apic_has_interrupt(vcpu);
2221
	struct kvm_lapic *apic = vcpu->arch.apic;
2222
	u32 ppr;
E
Eddie Dong 已提交
2223 2224 2225 2226

	if (vector == -1)
		return -1;

2227 2228 2229 2230 2231 2232 2233
	/*
	 * We get here even with APIC virtualization enabled, if doing
	 * nested virtualization and L1 runs with the "acknowledge interrupt
	 * on exit" mode.  Then we cannot inject the interrupt via RVI,
	 * because the process would deliver it through the IDT.
	 */

E
Eddie Dong 已提交
2234
	apic_clear_irr(vector, apic);
2235
	if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
2236 2237 2238 2239 2240
		/*
		 * For auto-EOI interrupts, there might be another pending
		 * interrupt above PPR, so check whether to raise another
		 * KVM_REQ_EVENT.
		 */
2241
		apic_update_ppr(apic);
2242 2243 2244 2245 2246 2247 2248 2249 2250
	} else {
		/*
		 * For normal interrupts, PPR has been raised and there cannot
		 * be a higher-priority pending interrupt---except if there was
		 * a concurrent interrupt injection, but that would have
		 * triggered KVM_REQ_EVENT already.
		 */
		apic_set_isr(vector, apic);
		__apic_update_ppr(apic, &ppr);
2251 2252
	}

E
Eddie Dong 已提交
2253 2254
	return vector;
}
2255

2256 2257 2258 2259 2260
static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
		struct kvm_lapic_state *s, bool set)
{
	if (apic_x2apic_mode(vcpu->arch.apic)) {
		u32 *id = (u32 *)(s->regs + APIC_ID);
2261
		u32 *ldr = (u32 *)(s->regs + APIC_LDR);
2262

2263 2264 2265 2266 2267 2268 2269 2270 2271
		if (vcpu->kvm->arch.x2apic_format) {
			if (*id != vcpu->vcpu_id)
				return -EINVAL;
		} else {
			if (set)
				*id >>= 24;
			else
				*id <<= 24;
		}
2272 2273 2274 2275

		/* In x2APIC mode, the LDR is fixed and based on the id */
		if (set)
			*ldr = kvm_apic_calc_x2apic_ldr(*id);
2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287
	}

	return 0;
}

int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
{
	memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
	return kvm_apic_state_fixup(vcpu, s, false);
}

int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2288
{
2289
	struct kvm_lapic *apic = vcpu->arch.apic;
2290 2291
	int r;

2292

2293
	kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
2294 2295
	/* set SPIV separately to get count of SW disabled APICs right */
	apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
2296 2297 2298 2299

	r = kvm_apic_state_fixup(vcpu, s, true);
	if (r)
		return r;
2300
	memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
2301 2302

	recalculate_apic_map(vcpu->kvm);
2303 2304
	kvm_apic_set_version(vcpu);

2305
	apic_update_ppr(apic);
2306
	hrtimer_cancel(&apic->lapic_timer.timer);
2307
	apic_update_lvtt(apic);
2308
	apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2309 2310
	update_divide_count(apic);
	start_apic_timer(apic);
2311
	apic->irr_pending = true;
2312
	apic->isr_count = vcpu->arch.apicv_active ?
2313
				1 : count_vectors(apic->regs + APIC_ISR);
M
Michael S. Tsirkin 已提交
2314
	apic->highest_isr_cache = -1;
2315
	if (vcpu->arch.apicv_active) {
2316
		kvm_x86_ops->apicv_post_state_restore(vcpu);
W
Wei Wang 已提交
2317 2318
		kvm_x86_ops->hwapic_irr_update(vcpu,
				apic_find_highest_irr(apic));
2319
		kvm_x86_ops->hwapic_isr_update(vcpu,
2320
				apic_find_highest_isr(apic));
2321
	}
2322
	kvm_make_request(KVM_REQ_EVENT, vcpu);
2323 2324
	if (ioapic_in_kernel(vcpu->kvm))
		kvm_rtc_eoi_tracking_restore_one(vcpu);
2325 2326

	vcpu->arch.apic_arb_prio = 0;
2327 2328

	return 0;
2329
}
2330

2331
void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
2332 2333 2334
{
	struct hrtimer *timer;

2335
	if (!lapic_in_kernel(vcpu))
2336 2337
		return;

2338
	timer = &vcpu->arch.apic->lapic_timer.timer;
2339
	if (hrtimer_cancel(timer))
2340
		hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
2341
}
A
Avi Kivity 已提交
2342

2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379
/*
 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
 *
 * Detect whether guest triggered PV EOI since the
 * last entry. If yes, set EOI on guests's behalf.
 * Clear PV EOI in guest memory in any case.
 */
static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
					struct kvm_lapic *apic)
{
	bool pending;
	int vector;
	/*
	 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
	 * and KVM_PV_EOI_ENABLED in guest memory as follows:
	 *
	 * KVM_APIC_PV_EOI_PENDING is unset:
	 * 	-> host disabled PV EOI.
	 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
	 * 	-> host enabled PV EOI, guest did not execute EOI yet.
	 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
	 * 	-> host enabled PV EOI, guest executed EOI.
	 */
	BUG_ON(!pv_eoi_enabled(vcpu));
	pending = pv_eoi_get_pending(vcpu);
	/*
	 * Clear pending bit in any case: it will be set again on vmentry.
	 * While this might not be ideal from performance point of view,
	 * this makes sure pv eoi is only enabled when we know it's safe.
	 */
	pv_eoi_clr_pending(vcpu);
	if (pending)
		return;
	vector = apic_set_eoi(apic);
	trace_kvm_pv_eoi(apic, vector);
}

A
Avi Kivity 已提交
2380 2381 2382 2383
void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
{
	u32 data;

2384 2385 2386
	if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
		apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);

2387
	if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
A
Avi Kivity 已提交
2388 2389
		return;

2390 2391
	if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
				  sizeof(u32)))
2392
		return;
A
Avi Kivity 已提交
2393 2394 2395 2396

	apic_set_tpr(vcpu->arch.apic, data & 0xff);
}

2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411
/*
 * apic_sync_pv_eoi_to_guest - called before vmentry
 *
 * Detect whether it's safe to enable PV EOI and
 * if yes do so.
 */
static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
					struct kvm_lapic *apic)
{
	if (!pv_eoi_enabled(vcpu) ||
	    /* IRR set or many bits in ISR: could be nested. */
	    apic->irr_pending ||
	    /* Cache not set: could be safe but we don't bother. */
	    apic->highest_isr_cache == -1 ||
	    /* Need EOI to update ioapic. */
2412
	    kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
2413 2414 2415 2416 2417 2418 2419 2420 2421 2422
		/*
		 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
		 * so we need not do anything here.
		 */
		return;
	}

	pv_eoi_set_pending(apic->vcpu);
}

A
Avi Kivity 已提交
2423 2424 2425 2426
void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
{
	u32 data, tpr;
	int max_irr, max_isr;
2427
	struct kvm_lapic *apic = vcpu->arch.apic;
A
Avi Kivity 已提交
2428

2429 2430
	apic_sync_pv_eoi_to_guest(vcpu, apic);

2431
	if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
A
Avi Kivity 已提交
2432 2433
		return;

2434
	tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
A
Avi Kivity 已提交
2435 2436 2437 2438 2439 2440 2441 2442
	max_irr = apic_find_highest_irr(apic);
	if (max_irr < 0)
		max_irr = 0;
	max_isr = apic_find_highest_isr(apic);
	if (max_isr < 0)
		max_isr = 0;
	data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);

2443 2444
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
				sizeof(u32));
A
Avi Kivity 已提交
2445 2446
}

2447
int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
A
Avi Kivity 已提交
2448
{
2449
	if (vapic_addr) {
2450
		if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2451 2452 2453
					&vcpu->arch.apic->vapic_cache,
					vapic_addr, sizeof(u32)))
			return -EINVAL;
2454
		__set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2455
	} else {
2456
		__clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2457 2458 2459 2460
	}

	vcpu->arch.apic->vapic_addr = vapic_addr;
	return 0;
A
Avi Kivity 已提交
2461
}
G
Gleb Natapov 已提交
2462 2463 2464 2465 2466 2467

int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u32 reg = (msr - APIC_BASE_MSR) << 4;

2468
	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
G
Gleb Natapov 已提交
2469 2470
		return 1;

2471 2472 2473
	if (reg == APIC_ICR2)
		return 1;

G
Gleb Natapov 已提交
2474
	/* if this is ICR write vector before command */
2475
	if (reg == APIC_ICR)
2476 2477
		kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
	return kvm_lapic_reg_write(apic, reg, (u32)data);
G
Gleb Natapov 已提交
2478 2479 2480 2481 2482 2483 2484
}

int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;

2485
	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
G
Gleb Natapov 已提交
2486 2487
		return 1;

2488 2489 2490 2491 2492 2493
	if (reg == APIC_DFR || reg == APIC_ICR2) {
		apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n",
			   reg);
		return 1;
	}

2494
	if (kvm_lapic_reg_read(apic, reg, 4, &low))
G
Gleb Natapov 已提交
2495
		return 1;
2496
	if (reg == APIC_ICR)
2497
		kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
G
Gleb Natapov 已提交
2498 2499 2500 2501 2502

	*data = (((u64)high) << 32) | low;

	return 0;
}
G
Gleb Natapov 已提交
2503 2504 2505 2506 2507

int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

2508
	if (!lapic_in_kernel(vcpu))
G
Gleb Natapov 已提交
2509 2510 2511 2512
		return 1;

	/* if this is ICR write vector before command */
	if (reg == APIC_ICR)
2513 2514
		kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
	return kvm_lapic_reg_write(apic, reg, (u32)data);
G
Gleb Natapov 已提交
2515 2516 2517 2518 2519 2520 2521
}

int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u32 low, high = 0;

2522
	if (!lapic_in_kernel(vcpu))
G
Gleb Natapov 已提交
2523 2524
		return 1;

2525
	if (kvm_lapic_reg_read(apic, reg, 4, &low))
G
Gleb Natapov 已提交
2526 2527
		return 1;
	if (reg == APIC_ICR)
2528
		kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
G
Gleb Natapov 已提交
2529 2530 2531 2532 2533

	*data = (((u64)high) << 32) | low;

	return 0;
}
2534 2535 2536 2537 2538 2539 2540 2541 2542 2543

int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data)
{
	u64 addr = data & ~KVM_MSR_ENABLED;
	if (!IS_ALIGNED(addr, 4))
		return 1;

	vcpu->arch.pv_eoi.msr_val = data;
	if (!pv_eoi_enabled(vcpu))
		return 0;
2544
	return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data,
2545
					 addr, sizeof(u8));
2546
}
2547

2548 2549 2550
void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
2551
	u8 sipi_vector;
2552
	unsigned long pe;
2553

2554
	if (!lapic_in_kernel(vcpu) || !apic->pending_events)
2555 2556
		return;

2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567
	/*
	 * INITs are latched while in SMM.  Because an SMM CPU cannot
	 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs
	 * and delay processing of INIT until the next RSM.
	 */
	if (is_smm(vcpu)) {
		WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
		if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
			clear_bit(KVM_APIC_SIPI, &apic->pending_events);
		return;
	}
2568

2569
	pe = xchg(&apic->pending_events, 0);
2570
	if (test_bit(KVM_APIC_INIT, &pe)) {
2571 2572
		kvm_lapic_reset(vcpu, true);
		kvm_vcpu_reset(vcpu, true);
2573 2574 2575 2576 2577
		if (kvm_vcpu_is_bsp(apic->vcpu))
			vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
		else
			vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
	}
2578
	if (test_bit(KVM_APIC_SIPI, &pe) &&
2579 2580 2581 2582
	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
		/* evaluate pending_events before reading the vector */
		smp_rmb();
		sipi_vector = apic->sipi_vector;
N
Nadav Amit 已提交
2583
		apic_debug("vcpu %d received sipi with vector # %x\n",
2584 2585 2586 2587 2588 2589
			 vcpu->vcpu_id, sipi_vector);
		kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
	}
}

2590 2591 2592 2593
void kvm_lapic_init(void)
{
	/* do not patch jump label more than once per second */
	jump_label_rate_limit(&apic_hw_disabled, HZ);
2594
	jump_label_rate_limit(&apic_sw_disabled, HZ);
2595
}
2596 2597 2598 2599 2600 2601

void kvm_lapic_exit(void)
{
	static_key_deferred_flush(&apic_hw_disabled);
	static_key_deferred_flush(&apic_sw_disabled);
}