lapic.c 66.8 KB
Newer Older
E
Eddie Dong 已提交
1 2 3 4 5 6 7

/*
 * Local APIC virtualization
 *
 * Copyright (C) 2006 Qumranet, Inc.
 * Copyright (C) 2007 Novell
 * Copyright (C) 2007 Intel
N
Nicolas Kaiser 已提交
8
 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
E
Eddie Dong 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *
 * Authors:
 *   Dor Laor <dor.laor@qumranet.com>
 *   Gregory Haskins <ghaskins@novell.com>
 *   Yaozu (Eddie) Dong <eddie.dong@intel.com>
 *
 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 */

21
#include <linux/kvm_host.h>
E
Eddie Dong 已提交
22 23 24 25 26 27
#include <linux/kvm.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/smp.h>
#include <linux/hrtimer.h>
#include <linux/io.h>
28
#include <linux/export.h>
R
Roman Zippel 已提交
29
#include <linux/math64.h>
30
#include <linux/slab.h>
E
Eddie Dong 已提交
31 32 33 34 35
#include <asm/processor.h>
#include <asm/msr.h>
#include <asm/page.h>
#include <asm/current.h>
#include <asm/apicdef.h>
36
#include <asm/delay.h>
A
Arun Sharma 已提交
37
#include <linux/atomic.h>
38
#include <linux/jump_label.h>
39
#include "kvm_cache_regs.h"
E
Eddie Dong 已提交
40
#include "irq.h"
41
#include "trace.h"
42
#include "x86.h"
A
Avi Kivity 已提交
43
#include "cpuid.h"
44
#include "hyperv.h"
E
Eddie Dong 已提交
45

46 47 48 49 50 51
#ifndef CONFIG_X86_64
#define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
#else
#define mod_64(x, y) ((x) % (y))
#endif

E
Eddie Dong 已提交
52 53 54 55 56 57 58 59 60
#define PRId64 "d"
#define PRIx64 "llx"
#define PRIu64 "u"
#define PRIo64 "o"

/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
#define apic_debug(fmt, arg...)

/* 14 is the version for Xeon and Pentium 8.4.8*/
61
#define APIC_VERSION			(0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
E
Eddie Dong 已提交
62 63 64 65 66 67
#define LAPIC_MMIO_LENGTH		(1 << 12)
/* followed define is not in apicdef.h */
#define APIC_SHORT_MASK			0xc0000
#define APIC_DEST_NOSHORT		0x0
#define APIC_DEST_MASK			0x800
#define MAX_APIC_VECTOR			256
68
#define APIC_VECTORS_PER_REG		32
E
Eddie Dong 已提交
69

70 71 72
#define APIC_BROADCAST			0xFF
#define X2APIC_BROADCAST		0xFFFFFFFFul

M
Michael S. Tsirkin 已提交
73 74 75 76 77
static inline int apic_test_vector(int vec, void *bitmap)
{
	return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

78 79 80 81 82 83 84 85
bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

	return apic_test_vector(vector, apic->regs + APIC_ISR) ||
		apic_test_vector(vector, apic->regs + APIC_IRR);
}

E
Eddie Dong 已提交
86 87 88 89 90
static inline void apic_clear_vector(int vec, void *bitmap)
{
	clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

M
Michael S. Tsirkin 已提交
91 92 93 94 95 96 97 98 99 100
static inline int __apic_test_and_set_vector(int vec, void *bitmap)
{
	return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
{
	return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

101
struct static_key_deferred apic_hw_disabled __read_mostly;
102 103
struct static_key_deferred apic_sw_disabled __read_mostly;

E
Eddie Dong 已提交
104 105
static inline int apic_enabled(struct kvm_lapic *apic)
{
106
	return kvm_apic_sw_enabled(apic) &&	kvm_apic_hw_enabled(apic);
107 108
}

E
Eddie Dong 已提交
109 110 111 112 113 114 115
#define LVT_MASK	\
	(APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)

#define LINT_MASK	\
	(LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
	 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)

116 117 118 119 120 121 122 123 124 125
static inline u8 kvm_xapic_id(struct kvm_lapic *apic)
{
	return kvm_lapic_get_reg(apic, APIC_ID) >> 24;
}

static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
{
	return apic->vcpu->vcpu_id;
}

126 127 128 129 130
static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
		u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
	switch (map->mode) {
	case KVM_APIC_MODE_X2APIC: {
		u32 offset = (dest_id >> 16) * 16;
R
Radim Krčmář 已提交
131
		u32 max_apic_id = map->max_apic_id;
132 133 134 135 136 137 138 139 140

		if (offset <= max_apic_id) {
			u8 cluster_size = min(max_apic_id - offset + 1, 16U);

			*cluster = &map->phys_map[offset];
			*mask = dest_id & (0xffff >> (16 - cluster_size));
		} else {
			*mask = 0;
		}
141

142 143 144 145 146 147 148
		return true;
		}
	case KVM_APIC_MODE_XAPIC_FLAT:
		*cluster = map->xapic_flat_map;
		*mask = dest_id & 0xff;
		return true;
	case KVM_APIC_MODE_XAPIC_CLUSTER:
149
		*cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
150 151 152 153 154 155
		*mask = dest_id & 0xf;
		return true;
	default:
		/* Not optimized. */
		return false;
	}
156 157
}

158
static void kvm_apic_map_free(struct rcu_head *rcu)
159
{
160
	struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
161

162
	kvfree(map);
163 164
}

165 166 167 168 169
static void recalculate_apic_map(struct kvm *kvm)
{
	struct kvm_apic_map *new, *old = NULL;
	struct kvm_vcpu *vcpu;
	int i;
170
	u32 max_id = 255; /* enough space for any xAPIC ID */
171 172 173

	mutex_lock(&kvm->arch.apic_map_lock);

R
Radim Krčmář 已提交
174 175
	kvm_for_each_vcpu(i, vcpu, kvm)
		if (kvm_apic_present(vcpu))
176
			max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
R
Radim Krčmář 已提交
177

M
Michal Hocko 已提交
178 179
	new = kvzalloc(sizeof(struct kvm_apic_map) +
	                   sizeof(struct kvm_lapic *) * ((u64)max_id + 1), GFP_KERNEL);
R
Radim Krčmář 已提交
180

181 182 183
	if (!new)
		goto out;

R
Radim Krčmář 已提交
184 185
	new->max_apic_id = max_id;

186 187
	kvm_for_each_vcpu(i, vcpu, kvm) {
		struct kvm_lapic *apic = vcpu->arch.apic;
188 189
		struct kvm_lapic **cluster;
		u16 mask;
190 191 192
		u32 ldr;
		u8 xapic_id;
		u32 x2apic_id;
193

194 195 196
		if (!kvm_apic_present(vcpu))
			continue;

197 198 199 200 201 202 203 204 205 206 207 208 209
		xapic_id = kvm_xapic_id(apic);
		x2apic_id = kvm_x2apic_id(apic);

		/* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
		if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
				x2apic_id <= new->max_apic_id)
			new->phys_map[x2apic_id] = apic;
		/*
		 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
		 * prevent them from masking VCPUs with APIC ID <= 0xff.
		 */
		if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
			new->phys_map[xapic_id] = apic;
210

211 212
		ldr = kvm_lapic_get_reg(apic, APIC_LDR);

213 214 215 216
		if (apic_x2apic_mode(apic)) {
			new->mode |= KVM_APIC_MODE_X2APIC;
		} else if (ldr) {
			ldr = GET_APIC_LOGICAL_ID(ldr);
217
			if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
218 219 220 221 222
				new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
			else
				new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
		}

223
		if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
224 225
			continue;

226 227
		if (mask)
			cluster[ffs(mask) - 1] = apic;
228 229 230 231 232 233 234 235
	}
out:
	old = rcu_dereference_protected(kvm->arch.apic_map,
			lockdep_is_held(&kvm->arch.apic_map_lock));
	rcu_assign_pointer(kvm->arch.apic_map, new);
	mutex_unlock(&kvm->arch.apic_map_lock);

	if (old)
236
		call_rcu(&old->rcu, kvm_apic_map_free);
237

238
	kvm_make_scan_ioapic_request(kvm);
239 240
}

241 242
static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
{
243
	bool enabled = val & APIC_SPIV_APIC_ENABLED;
244

245
	kvm_lapic_set_reg(apic, APIC_SPIV, val);
246 247 248 249

	if (enabled != apic->sw_enabled) {
		apic->sw_enabled = enabled;
		if (enabled) {
250 251 252 253 254 255 256
			static_key_slow_dec_deferred(&apic_sw_disabled);
			recalculate_apic_map(apic->vcpu->kvm);
		} else
			static_key_slow_inc(&apic_sw_disabled.key);
	}
}

257
static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
258
{
259
	kvm_lapic_set_reg(apic, APIC_ID, id << 24);
260 261 262 263 264
	recalculate_apic_map(apic->vcpu->kvm);
}

static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
{
265
	kvm_lapic_set_reg(apic, APIC_LDR, id);
266 267 268
	recalculate_apic_map(apic->vcpu->kvm);
}

269 270 271 272 273
static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
{
	return ((id >> 4) << 16) | (1 << (id & 0xf));
}

274
static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
275
{
276
	u32 ldr = kvm_apic_calc_x2apic_ldr(id);
277

278 279
	WARN_ON_ONCE(id != apic->vcpu->vcpu_id);

280
	kvm_lapic_set_reg(apic, APIC_ID, id);
281
	kvm_lapic_set_reg(apic, APIC_LDR, ldr);
282 283 284
	recalculate_apic_map(apic->vcpu->kvm);
}

E
Eddie Dong 已提交
285 286
static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
{
287
	return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
E
Eddie Dong 已提交
288 289 290 291
}

static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
{
292
	return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
E
Eddie Dong 已提交
293 294
}

295 296
static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
{
297
	return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
298 299
}

E
Eddie Dong 已提交
300 301
static inline int apic_lvtt_period(struct kvm_lapic *apic)
{
302
	return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
303 304 305 306
}

static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
{
307
	return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
E
Eddie Dong 已提交
308 309
}

310 311 312 313 314
static inline int apic_lvt_nmi_mode(u32 lvt_val)
{
	return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
}

315 316 317 318 319 320
void kvm_apic_set_version(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	struct kvm_cpuid_entry2 *feat;
	u32 v = APIC_VERSION;

321
	if (!lapic_in_kernel(vcpu))
322 323
		return;

324 325 326 327 328 329 330
	/*
	 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation)
	 * which doesn't have EOI register; Some buggy OSes (e.g. Windows with
	 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC
	 * version first and level-triggered interrupts never get EOIed in
	 * IOAPIC.
	 */
331
	feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
332 333
	if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))) &&
	    !ioapic_in_kernel(vcpu->kvm))
334
		v |= APIC_LVR_DIRECTED_EOI;
335
	kvm_lapic_set_reg(apic, APIC_LVR, v);
336 337
}

338
static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
339
	LVT_MASK ,      /* part LVTT mask, timer mode mask added at runtime */
E
Eddie Dong 已提交
340 341 342 343 344 345 346 347
	LVT_MASK | APIC_MODE_MASK,	/* LVTTHMR */
	LVT_MASK | APIC_MODE_MASK,	/* LVTPC */
	LINT_MASK, LINT_MASK,	/* LVT0-1 */
	LVT_MASK		/* LVTERR */
};

static int find_highest_vector(void *bitmap)
{
348 349
	int vec;
	u32 *reg;
E
Eddie Dong 已提交
350

351 352 353 354
	for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
	     vec >= 0; vec -= APIC_VECTORS_PER_REG) {
		reg = bitmap + REG_POS(vec);
		if (*reg)
355
			return __fls(*reg) + vec;
356
	}
E
Eddie Dong 已提交
357

358
	return -1;
E
Eddie Dong 已提交
359 360
}

M
Michael S. Tsirkin 已提交
361 362
static u8 count_vectors(void *bitmap)
{
363 364
	int vec;
	u32 *reg;
M
Michael S. Tsirkin 已提交
365
	u8 count = 0;
366 367 368 369 370 371

	for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
		reg = bitmap + REG_POS(vec);
		count += hweight32(*reg);
	}

M
Michael S. Tsirkin 已提交
372 373 374
	return count;
}

375
bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
376
{
377
	u32 i, vec;
378 379 380 381 382
	u32 pir_val, irr_val, prev_irr_val;
	int max_updated_irr;

	max_updated_irr = -1;
	*max_irr = -1;
383

384
	for (i = vec = 0; i <= 7; i++, vec += 32) {
385
		pir_val = READ_ONCE(pir[i]);
386
		irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
387
		if (pir_val) {
388
			prev_irr_val = irr_val;
389 390
			irr_val |= xchg(&pir[i], 0);
			*((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
391 392 393 394
			if (prev_irr_val != irr_val) {
				max_updated_irr =
					__fls(irr_val ^ prev_irr_val) + vec;
			}
395
		}
396
		if (irr_val)
397
			*max_irr = __fls(irr_val) + vec;
398
	}
399

400 401
	return ((max_updated_irr != -1) &&
		(max_updated_irr == *max_irr));
402
}
403 404
EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);

405
bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
406 407 408
{
	struct kvm_lapic *apic = vcpu->arch.apic;

409
	return __kvm_apic_update_irr(pir, apic->regs, max_irr);
410
}
411 412
EXPORT_SYMBOL_GPL(kvm_apic_update_irr);

413
static inline int apic_search_irr(struct kvm_lapic *apic)
E
Eddie Dong 已提交
414
{
415
	return find_highest_vector(apic->regs + APIC_IRR);
E
Eddie Dong 已提交
416 417 418 419 420 421
}

static inline int apic_find_highest_irr(struct kvm_lapic *apic)
{
	int result;

422 423 424 425
	/*
	 * Note that irr_pending is just a hint. It will be always
	 * true with virtual interrupt delivery enabled.
	 */
426 427 428 429
	if (!apic->irr_pending)
		return -1;

	result = apic_search_irr(apic);
E
Eddie Dong 已提交
430 431 432 433 434
	ASSERT(result == -1 || result >= 16);

	return result;
}

435 436
static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
{
437 438 439 440
	struct kvm_vcpu *vcpu;

	vcpu = apic->vcpu;

441
	if (unlikely(vcpu->arch.apicv_active)) {
442
		/* need to update RVI */
443
		apic_clear_vector(vec, apic->regs + APIC_IRR);
444 445
		kvm_x86_ops->hwapic_irr_update(vcpu,
				apic_find_highest_irr(apic));
446 447 448 449 450
	} else {
		apic->irr_pending = false;
		apic_clear_vector(vec, apic->regs + APIC_IRR);
		if (apic_search_irr(apic) != -1)
			apic->irr_pending = true;
451
	}
452 453
}

M
Michael S. Tsirkin 已提交
454 455
static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
{
456 457 458 459 460 461
	struct kvm_vcpu *vcpu;

	if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
		return;

	vcpu = apic->vcpu;
462

M
Michael S. Tsirkin 已提交
463
	/*
464 465 466
	 * With APIC virtualization enabled, all caching is disabled
	 * because the processor can modify ISR under the hood.  Instead
	 * just set SVI.
M
Michael S. Tsirkin 已提交
467
	 */
468
	if (unlikely(vcpu->arch.apicv_active))
469
		kvm_x86_ops->hwapic_isr_update(vcpu, vec);
470 471 472 473 474 475 476 477 478 479
	else {
		++apic->isr_count;
		BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
		/*
		 * ISR (in service register) bit is set when injecting an interrupt.
		 * The highest vector is injected. Thus the latest bit set matches
		 * the highest bit in ISR.
		 */
		apic->highest_isr_cache = vec;
	}
M
Michael S. Tsirkin 已提交
480 481
}

482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
static inline int apic_find_highest_isr(struct kvm_lapic *apic)
{
	int result;

	/*
	 * Note that isr_count is always 1, and highest_isr_cache
	 * is always -1, with APIC virtualization enabled.
	 */
	if (!apic->isr_count)
		return -1;
	if (likely(apic->highest_isr_cache != -1))
		return apic->highest_isr_cache;

	result = find_highest_vector(apic->regs + APIC_ISR);
	ASSERT(result == -1 || result >= 16);

	return result;
}

M
Michael S. Tsirkin 已提交
501 502
static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
{
503 504 505 506 507 508 509 510 511 512 513 514 515
	struct kvm_vcpu *vcpu;
	if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
		return;

	vcpu = apic->vcpu;

	/*
	 * We do get here for APIC virtualization enabled if the guest
	 * uses the Hyper-V APIC enlightenment.  In this case we may need
	 * to trigger a new interrupt delivery by writing the SVI field;
	 * on the other hand isr_count and highest_isr_cache are unused
	 * and must be left alone.
	 */
516
	if (unlikely(vcpu->arch.apicv_active))
517
		kvm_x86_ops->hwapic_isr_update(vcpu,
518 519
					       apic_find_highest_isr(apic));
	else {
M
Michael S. Tsirkin 已提交
520
		--apic->isr_count;
521 522 523
		BUG_ON(apic->isr_count < 0);
		apic->highest_isr_cache = -1;
	}
M
Michael S. Tsirkin 已提交
524 525
}

526 527
int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
{
528 529 530 531 532
	/* This may race with setting of irr in __apic_accept_irq() and
	 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
	 * will cause vmexit immediately and the value will be recalculated
	 * on the next vmentry.
	 */
533
	return apic_find_highest_irr(vcpu->arch.apic);
534
}
535
EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
536

537
static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
538
			     int vector, int level, int trig_mode,
539
			     struct dest_map *dest_map);
540

541
int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
542
		     struct dest_map *dest_map)
E
Eddie Dong 已提交
543
{
544
	struct kvm_lapic *apic = vcpu->arch.apic;
545

546
	return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
547
			irq->level, irq->trig_mode, dest_map);
E
Eddie Dong 已提交
548 549
}

550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589
int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
    		    unsigned long ipi_bitmap_high, int min,
		    unsigned long icr, int op_64_bit)
{
	int i;
	struct kvm_apic_map *map;
	struct kvm_vcpu *vcpu;
	struct kvm_lapic_irq irq = {0};
	int cluster_size = op_64_bit ? 64 : 32;
	int count = 0;

	irq.vector = icr & APIC_VECTOR_MASK;
	irq.delivery_mode = icr & APIC_MODE_MASK;
	irq.level = (icr & APIC_INT_ASSERT) != 0;
	irq.trig_mode = icr & APIC_INT_LEVELTRIG;

	if (icr & APIC_DEST_MASK)
		return -KVM_EINVAL;
	if (icr & APIC_SHORT_MASK)
		return -KVM_EINVAL;

	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);

	/* Bits above cluster_size are masked in the caller.  */
	for_each_set_bit(i, &ipi_bitmap_low, BITS_PER_LONG) {
		vcpu = map->phys_map[min + i]->vcpu;
		count += kvm_apic_set_irq(vcpu, &irq, NULL);
	}

	min += cluster_size;
	for_each_set_bit(i, &ipi_bitmap_high, BITS_PER_LONG) {
		vcpu = map->phys_map[min + i]->vcpu;
		count += kvm_apic_set_irq(vcpu, &irq, NULL);
	}

	rcu_read_unlock();
	return count;
}

590 591
static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
{
592 593 594

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
				      sizeof(val));
595 596 597 598
}

static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
{
599 600 601

	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
				      sizeof(*val));
602 603 604 605 606 607 608 609 610 611 612 613
}

static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
{
	return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
}

static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
{
	u8 val;
	if (pv_eoi_get_user(vcpu, &val) < 0)
		apic_debug("Can't read EOI MSR value: 0x%llx\n",
614
			   (unsigned long long)vcpu->arch.pv_eoi.msr_val);
615 616 617 618 619 620 621
	return val & 0x1;
}

static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
{
	if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
		apic_debug("Can't set EOI MSR value: 0x%llx\n",
622
			   (unsigned long long)vcpu->arch.pv_eoi.msr_val);
623 624 625 626 627 628 629 630 631
		return;
	}
	__set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
}

static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
{
	if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
		apic_debug("Can't clear EOI MSR value: 0x%llx\n",
632
			   (unsigned long long)vcpu->arch.pv_eoi.msr_val);
633 634 635 636 637
		return;
	}
	__clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
}

638 639
static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
{
640
	int highest_irr;
641
	if (apic->vcpu->arch.apicv_active)
642 643 644
		highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
	else
		highest_irr = apic_find_highest_irr(apic);
645 646 647 648 649 650
	if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
		return -1;
	return highest_irr;
}

static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
E
Eddie Dong 已提交
651
{
652
	u32 tpr, isrv, ppr, old_ppr;
E
Eddie Dong 已提交
653 654
	int isr;

655 656
	old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
	tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
E
Eddie Dong 已提交
657 658 659 660 661 662 663 664 665 666 667
	isr = apic_find_highest_isr(apic);
	isrv = (isr != -1) ? isr : 0;

	if ((tpr & 0xf0) >= (isrv & 0xf0))
		ppr = tpr & 0xff;
	else
		ppr = isrv & 0xf0;

	apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x",
		   apic, ppr, isr, isrv);

668 669
	*new_ppr = ppr;
	if (old_ppr != ppr)
670
		kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
671 672 673 674 675 676 677 678

	return ppr < old_ppr;
}

static void apic_update_ppr(struct kvm_lapic *apic)
{
	u32 ppr;

679 680
	if (__apic_update_ppr(apic, &ppr) &&
	    apic_has_interrupt_for_ppr(apic, ppr) != -1)
681
		kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
E
Eddie Dong 已提交
682 683
}

684 685 686 687 688 689
void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
{
	apic_update_ppr(vcpu->arch.apic);
}
EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);

E
Eddie Dong 已提交
690 691
static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
{
692
	kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
E
Eddie Dong 已提交
693 694 695
	apic_update_ppr(apic);
}

696
static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
697
{
698 699
	return mda == (apic_x2apic_mode(apic) ?
			X2APIC_BROADCAST : APIC_BROADCAST);
700 701
}

702
static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
E
Eddie Dong 已提交
703
{
704 705 706 707
	if (kvm_apic_broadcast(apic, mda))
		return true;

	if (apic_x2apic_mode(apic))
708
		return mda == kvm_x2apic_id(apic);
709

710 711 712 713 714 715 716 717 718
	/*
	 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
	 * it were in x2APIC mode.  Hotplugged VCPUs start in xAPIC mode and
	 * this allows unique addressing of VCPUs with APIC ID over 0xff.
	 * The 0xff condition is needed because writeable xAPIC ID.
	 */
	if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
		return true;

719
	return mda == kvm_xapic_id(apic);
E
Eddie Dong 已提交
720 721
}

722
static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
E
Eddie Dong 已提交
723
{
G
Gleb Natapov 已提交
724 725
	u32 logical_id;

726
	if (kvm_apic_broadcast(apic, mda))
727
		return true;
728

729
	logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
E
Eddie Dong 已提交
730

731
	if (apic_x2apic_mode(apic))
732 733
		return ((logical_id >> 16) == (mda >> 16))
		       && (logical_id & mda & 0xffff) != 0;
E
Eddie Dong 已提交
734

735
	logical_id = GET_APIC_LOGICAL_ID(logical_id);
E
Eddie Dong 已提交
736

737
	switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
E
Eddie Dong 已提交
738
	case APIC_DFR_FLAT:
739
		return (logical_id & mda) != 0;
E
Eddie Dong 已提交
740
	case APIC_DFR_CLUSTER:
741 742
		return ((logical_id >> 4) == (mda >> 4))
		       && (logical_id & mda & 0xf) != 0;
E
Eddie Dong 已提交
743
	default:
744
		apic_debug("Bad DFR vcpu %d: %08x\n",
745
			   apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR));
746
		return false;
E
Eddie Dong 已提交
747 748 749
	}
}

750 751
/* The KVM local APIC implementation has two quirks:
 *
752 753 754
 *  - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
 *    in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
 *    KVM doesn't do that aliasing.
755 756 757 758 759 760 761 762 763 764
 *
 *  - in-kernel IOAPIC messages have to be delivered directly to
 *    x2APIC, because the kernel does not support interrupt remapping.
 *    In order to support broadcast without interrupt remapping, x2APIC
 *    rewrites the destination of non-IPI messages from APIC_BROADCAST
 *    to X2APIC_BROADCAST.
 *
 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
 * important when userspace wants to use x2APIC-format MSIs, because
 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
765
 */
766 767
static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
		struct kvm_lapic *source, struct kvm_lapic *target)
768 769 770
{
	bool ipi = source != NULL;

771
	if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
772
	    !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
773 774
		return X2APIC_BROADCAST;

775
	return dest_id;
776 777
}

778
bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
779
			   int short_hand, unsigned int dest, int dest_mode)
E
Eddie Dong 已提交
780
{
781
	struct kvm_lapic *target = vcpu->arch.apic;
782
	u32 mda = kvm_apic_mda(vcpu, dest, source, target);
E
Eddie Dong 已提交
783 784

	apic_debug("target %p, source %p, dest 0x%x, "
785
		   "dest_mode 0x%x, short_hand 0x%x\n",
E
Eddie Dong 已提交
786 787
		   target, source, dest, dest_mode, short_hand);

Z
Zachary Amsden 已提交
788
	ASSERT(target);
E
Eddie Dong 已提交
789 790
	switch (short_hand) {
	case APIC_DEST_NOSHORT:
791
		if (dest_mode == APIC_DEST_PHYSICAL)
792
			return kvm_apic_match_physical_addr(target, mda);
793
		else
794
			return kvm_apic_match_logical_addr(target, mda);
E
Eddie Dong 已提交
795
	case APIC_DEST_SELF:
796
		return target == source;
E
Eddie Dong 已提交
797
	case APIC_DEST_ALLINC:
798
		return true;
E
Eddie Dong 已提交
799
	case APIC_DEST_ALLBUT:
800
		return target != source;
E
Eddie Dong 已提交
801
	default:
802 803
		apic_debug("kvm: apic: Bad dest shorthand value %x\n",
			   short_hand);
804
		return false;
E
Eddie Dong 已提交
805 806
	}
}
807
EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
E
Eddie Dong 已提交
808

809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824
int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
		       const unsigned long *bitmap, u32 bitmap_size)
{
	u32 mod;
	int i, idx = -1;

	mod = vector % dest_vcpus;

	for (i = 0; i <= mod; i++) {
		idx = find_next_bit(bitmap, bitmap_size, idx + 1);
		BUG_ON(idx == bitmap_size);
	}

	return idx;
}

825 826 827 828 829 830 831 832 833
static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
{
	if (!kvm->arch.disabled_lapic_found) {
		kvm->arch.disabled_lapic_found = true;
		printk(KERN_INFO
		       "Disabled LAPIC found during irq injection\n");
	}
}

834 835
static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
		struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
836
{
837 838 839 840 841 842 843 844 845 846 847 848
	if (kvm->arch.x2apic_broadcast_quirk_disabled) {
		if ((irq->dest_id == APIC_BROADCAST &&
				map->mode != KVM_APIC_MODE_X2APIC))
			return true;
		if (irq->dest_id == X2APIC_BROADCAST)
			return true;
	} else {
		bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
		if (irq->dest_id == (x2apic_ipi ?
		                     X2APIC_BROADCAST : APIC_BROADCAST))
			return true;
	}
849

850 851
	return false;
}
852

853 854 855 856 857 858 859 860 861 862 863 864 865
/* Return true if the interrupt can be handled by using *bitmap as index mask
 * for valid destinations in *dst array.
 * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
 * Note: we may have zero kvm_lapic destinations when we return true, which
 * means that the interrupt should be dropped.  In this case, *bitmap would be
 * zero and *dst undefined.
 */
static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
		struct kvm_lapic **src, struct kvm_lapic_irq *irq,
		struct kvm_apic_map *map, struct kvm_lapic ***dst,
		unsigned long *bitmap)
{
	int i, lowest;
866

867 868 869 870 871
	if (irq->shorthand == APIC_DEST_SELF && src) {
		*dst = src;
		*bitmap = 1;
		return true;
	} else if (irq->shorthand)
872 873
		return false;

874
	if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
875 876
		return false;

877
	if (irq->dest_mode == APIC_DEST_PHYSICAL) {
R
Radim Krčmář 已提交
878
		if (irq->dest_id > map->max_apic_id) {
879 880 881 882 883
			*bitmap = 0;
		} else {
			*dst = &map->phys_map[irq->dest_id];
			*bitmap = 1;
		}
884
		return true;
885
	}
886

887 888 889
	*bitmap = 0;
	if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
				(u16 *)bitmap))
890
		return false;
891

892 893
	if (!kvm_lowest_prio_delivery(irq))
		return true;
894

895 896 897 898 899 900 901 902 903 904
	if (!kvm_vector_hashing_enabled()) {
		lowest = -1;
		for_each_set_bit(i, bitmap, 16) {
			if (!(*dst)[i])
				continue;
			if (lowest < 0)
				lowest = i;
			else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
						(*dst)[lowest]->vcpu) < 0)
				lowest = i;
905
		}
906 907 908
	} else {
		if (!*bitmap)
			return true;
909

910 911
		lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
				bitmap, 16);
912

913 914 915 916 917 918
		if (!(*dst)[lowest]) {
			kvm_apic_disabled_lapic_found(kvm);
			*bitmap = 0;
			return true;
		}
	}
919

920
	*bitmap = (lowest >= 0) ? 1 << lowest : 0;
921

922 923
	return true;
}
924

925 926 927 928 929 930 931 932
bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
		struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
{
	struct kvm_apic_map *map;
	unsigned long bitmap;
	struct kvm_lapic **dst = NULL;
	int i;
	bool ret;
933

934
	*r = -1;
935

936 937 938 939
	if (irq->shorthand == APIC_DEST_SELF) {
		*r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
		return true;
	}
940

941 942
	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);
943

944 945 946 947 948 949 950 951
	ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
	if (ret)
		for_each_set_bit(i, &bitmap, 16) {
			if (!dst[i])
				continue;
			if (*r < 0)
				*r = 0;
			*r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
952 953 954 955 956 957
		}

	rcu_read_unlock();
	return ret;
}

958 959 960 961 962 963 964 965 966 967 968 969 970 971
/*
 * This routine tries to handler interrupts in posted mode, here is how
 * it deals with different cases:
 * - For single-destination interrupts, handle it in posted mode
 * - Else if vector hashing is enabled and it is a lowest-priority
 *   interrupt, handle it in posted mode and use the following mechanism
 *   to find the destinaiton vCPU.
 *	1. For lowest-priority interrupts, store all the possible
 *	   destination vCPUs in an array.
 *	2. Use "guest vector % max number of destination vCPUs" to find
 *	   the right destination vCPU in the array for the lowest-priority
 *	   interrupt.
 * - Otherwise, use remapped mode to inject the interrupt.
 */
972 973 974 975
bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
			struct kvm_vcpu **dest_vcpu)
{
	struct kvm_apic_map *map;
976 977
	unsigned long bitmap;
	struct kvm_lapic **dst = NULL;
978 979 980 981 982 983 984 985
	bool ret = false;

	if (irq->shorthand)
		return false;

	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);

986 987 988
	if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
			hweight16(bitmap) == 1) {
		unsigned long i = find_first_bit(&bitmap, 16);
989

990 991 992
		if (dst[i]) {
			*dest_vcpu = dst[i]->vcpu;
			ret = true;
993
		}
994 995 996 997 998 999
	}

	rcu_read_unlock();
	return ret;
}

E
Eddie Dong 已提交
1000 1001 1002 1003 1004
/*
 * Add a pending IRQ into lapic.
 * Return 1 if successfully added and 0 if discarded.
 */
static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
1005
			     int vector, int level, int trig_mode,
1006
			     struct dest_map *dest_map)
E
Eddie Dong 已提交
1007
{
1008
	int result = 0;
1009
	struct kvm_vcpu *vcpu = apic->vcpu;
E
Eddie Dong 已提交
1010

1011 1012
	trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
				  trig_mode, vector);
E
Eddie Dong 已提交
1013 1014
	switch (delivery_mode) {
	case APIC_DM_LOWEST:
1015 1016
		vcpu->arch.apic_arb_prio++;
	case APIC_DM_FIXED:
1017 1018 1019
		if (unlikely(trig_mode && !level))
			break;

E
Eddie Dong 已提交
1020 1021 1022 1023
		/* FIXME add logic for vcpu on reset */
		if (unlikely(!apic_enabled(apic)))
			break;

1024 1025
		result = 1;

1026
		if (dest_map) {
1027
			__set_bit(vcpu->vcpu_id, dest_map->map);
1028 1029
			dest_map->vectors[vcpu->vcpu_id] = vector;
		}
1030

1031 1032
		if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
			if (trig_mode)
1033
				kvm_lapic_set_vector(vector, apic->regs + APIC_TMR);
1034 1035 1036 1037
			else
				apic_clear_vector(vector, apic->regs + APIC_TMR);
		}

1038
		if (vcpu->arch.apicv_active)
1039
			kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
1040
		else {
1041
			kvm_lapic_set_irr(vector, apic);
1042 1043 1044 1045

			kvm_make_request(KVM_REQ_EVENT, vcpu);
			kvm_vcpu_kick(vcpu);
		}
E
Eddie Dong 已提交
1046 1047 1048
		break;

	case APIC_DM_REMRD:
1049 1050 1051 1052
		result = 1;
		vcpu->arch.pv.pv_unhalted = 1;
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		kvm_vcpu_kick(vcpu);
E
Eddie Dong 已提交
1053 1054 1055
		break;

	case APIC_DM_SMI:
P
Paolo Bonzini 已提交
1056 1057 1058
		result = 1;
		kvm_make_request(KVM_REQ_SMI, vcpu);
		kvm_vcpu_kick(vcpu);
E
Eddie Dong 已提交
1059
		break;
1060

E
Eddie Dong 已提交
1061
	case APIC_DM_NMI:
1062
		result = 1;
1063
		kvm_inject_nmi(vcpu);
J
Jan Kiszka 已提交
1064
		kvm_vcpu_kick(vcpu);
E
Eddie Dong 已提交
1065 1066 1067
		break;

	case APIC_DM_INIT:
1068
		if (!trig_mode || level) {
1069
			result = 1;
1070 1071 1072 1073 1074
			/* assumes that there are only KVM_APIC_INIT/SIPI */
			apic->pending_events = (1UL << KVM_APIC_INIT);
			/* make sure pending_events is visible before sending
			 * the request */
			smp_wmb();
1075
			kvm_make_request(KVM_REQ_EVENT, vcpu);
1076 1077
			kvm_vcpu_kick(vcpu);
		} else {
1078 1079
			apic_debug("Ignoring de-assert INIT to vcpu %d\n",
				   vcpu->vcpu_id);
1080
		}
E
Eddie Dong 已提交
1081 1082 1083
		break;

	case APIC_DM_STARTUP:
1084 1085
		apic_debug("SIPI to vcpu %d vector 0x%02x\n",
			   vcpu->vcpu_id, vector);
1086 1087 1088 1089 1090 1091 1092
		result = 1;
		apic->sipi_vector = vector;
		/* make sure sipi_vector is visible for the receiver */
		smp_wmb();
		set_bit(KVM_APIC_SIPI, &apic->pending_events);
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		kvm_vcpu_kick(vcpu);
E
Eddie Dong 已提交
1093 1094
		break;

1095 1096 1097 1098 1099 1100 1101 1102
	case APIC_DM_EXTINT:
		/*
		 * Should only be called by kvm_apic_local_deliver() with LVT0,
		 * before NMI watchdog was enabled. Already handled by
		 * kvm_apic_accept_pic_intr().
		 */
		break;

E
Eddie Dong 已提交
1103 1104 1105 1106 1107 1108 1109 1110
	default:
		printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
		       delivery_mode);
		break;
	}
	return result;
}

1111
int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
1112
{
1113
	return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
1114 1115
}

1116 1117
static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
{
1118
	return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
1119 1120
}

1121 1122
static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
{
1123 1124 1125 1126 1127
	int trigger_mode;

	/* Eoi the ioapic only if the ioapic doesn't own the vector. */
	if (!kvm_ioapic_handles_vector(apic, vector))
		return;
1128

1129 1130 1131 1132 1133
	/* Request a KVM exit to inform the userspace IOAPIC. */
	if (irqchip_split(apic->vcpu->kvm)) {
		apic->vcpu->arch.pending_ioapic_eoi = vector;
		kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
		return;
1134
	}
1135 1136 1137 1138 1139 1140 1141

	if (apic_test_vector(vector, apic->regs + APIC_TMR))
		trigger_mode = IOAPIC_LEVEL_TRIG;
	else
		trigger_mode = IOAPIC_EDGE_TRIG;

	kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
1142 1143
}

1144
static int apic_set_eoi(struct kvm_lapic *apic)
E
Eddie Dong 已提交
1145 1146
{
	int vector = apic_find_highest_isr(apic);
1147 1148 1149

	trace_kvm_eoi(apic, vector);

E
Eddie Dong 已提交
1150 1151 1152 1153 1154
	/*
	 * Not every write EOI will has corresponding ISR,
	 * one example is when Kernel check timer on setup_IO_APIC
	 */
	if (vector == -1)
1155
		return vector;
E
Eddie Dong 已提交
1156

M
Michael S. Tsirkin 已提交
1157
	apic_clear_isr(vector, apic);
E
Eddie Dong 已提交
1158 1159
	apic_update_ppr(apic);

1160 1161 1162
	if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
		kvm_hv_synic_send_eoi(apic->vcpu, vector);

1163
	kvm_ioapic_send_eoi(apic, vector);
1164
	kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1165
	return vector;
E
Eddie Dong 已提交
1166 1167
}

1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
/*
 * this interface assumes a trap-like exit, which has already finished
 * desired side effect including vISR and vPPR update.
 */
void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

	trace_kvm_eoi(apic, vector);

	kvm_ioapic_send_eoi(apic, vector);
	kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
}
EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);

E
Eddie Dong 已提交
1183 1184
static void apic_send_ipi(struct kvm_lapic *apic)
{
1185 1186
	u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR);
	u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2);
1187
	struct kvm_lapic_irq irq;
E
Eddie Dong 已提交
1188

1189 1190 1191
	irq.vector = icr_low & APIC_VECTOR_MASK;
	irq.delivery_mode = icr_low & APIC_MODE_MASK;
	irq.dest_mode = icr_low & APIC_DEST_MASK;
1192
	irq.level = (icr_low & APIC_INT_ASSERT) != 0;
1193 1194
	irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
	irq.shorthand = icr_low & APIC_SHORT_MASK;
1195
	irq.msi_redir_hint = false;
G
Gleb Natapov 已提交
1196 1197 1198 1199
	if (apic_x2apic_mode(apic))
		irq.dest_id = icr_high;
	else
		irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
E
Eddie Dong 已提交
1200

1201 1202
	trace_kvm_apic_ipi(icr_low, irq.dest_id);

E
Eddie Dong 已提交
1203 1204
	apic_debug("icr_high 0x%x, icr_low 0x%x, "
		   "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
1205 1206
		   "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x, "
		   "msi_redir_hint 0x%x\n",
G
Glauber Costa 已提交
1207
		   icr_high, icr_low, irq.shorthand, irq.dest_id,
1208
		   irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode,
1209
		   irq.vector, irq.msi_redir_hint);
1210

1211
	kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
E
Eddie Dong 已提交
1212 1213 1214 1215
}

static u32 apic_get_tmcct(struct kvm_lapic *apic)
{
1216
	ktime_t remaining, now;
1217
	s64 ns;
1218
	u32 tmcct;
E
Eddie Dong 已提交
1219 1220 1221

	ASSERT(apic != NULL);

1222
	/* if initial count is 0, current count should also be 0 */
1223
	if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
1224
		apic->lapic_timer.period == 0)
1225 1226
		return 0;

1227
	now = ktime_get();
1228
	remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1229
	if (ktime_to_ns(remaining) < 0)
T
Thomas Gleixner 已提交
1230
		remaining = 0;
1231

1232 1233 1234
	ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
	tmcct = div64_u64(ns,
			 (APIC_BUS_CYCLE_NS * apic->divide_count));
E
Eddie Dong 已提交
1235 1236 1237 1238

	return tmcct;
}

1239 1240 1241 1242 1243
static void __report_tpr_access(struct kvm_lapic *apic, bool write)
{
	struct kvm_vcpu *vcpu = apic->vcpu;
	struct kvm_run *run = vcpu->run;

1244
	kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
1245
	run->tpr_access.rip = kvm_rip_read(vcpu);
1246 1247 1248 1249 1250 1251 1252 1253 1254
	run->tpr_access.is_write = write;
}

static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
{
	if (apic->vcpu->arch.tpr_access_reporting)
		__report_tpr_access(apic, write);
}

E
Eddie Dong 已提交
1255 1256 1257 1258 1259 1260 1261 1262 1263
static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
{
	u32 val = 0;

	if (offset >= LAPIC_MMIO_LENGTH)
		return 0;

	switch (offset) {
	case APIC_ARBPRI:
1264
		apic_debug("Access APIC ARBPRI register which is for P6\n");
E
Eddie Dong 已提交
1265 1266 1267
		break;

	case APIC_TMCCT:	/* Timer CCR */
1268 1269 1270
		if (apic_lvtt_tscdeadline(apic))
			return 0;

E
Eddie Dong 已提交
1271 1272
		val = apic_get_tmcct(apic);
		break;
1273 1274
	case APIC_PROCPRI:
		apic_update_ppr(apic);
1275
		val = kvm_lapic_get_reg(apic, offset);
1276
		break;
1277 1278 1279
	case APIC_TASKPRI:
		report_tpr_access(apic, false);
		/* fall thru */
E
Eddie Dong 已提交
1280
	default:
1281
		val = kvm_lapic_get_reg(apic, offset);
E
Eddie Dong 已提交
1282 1283 1284 1285 1286 1287
		break;
	}

	return val;
}

G
Gregory Haskins 已提交
1288 1289 1290 1291 1292
static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
{
	return container_of(dev, struct kvm_lapic, dev);
}

1293
int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
G
Gleb Natapov 已提交
1294
		void *data)
E
Eddie Dong 已提交
1295 1296 1297
{
	unsigned char alignment = offset & 0xf;
	u32 result;
G
Guo Chao 已提交
1298
	/* this bitmask has a bit cleared for each reserved register */
G
Gleb Natapov 已提交
1299
	static const u64 rmask = 0x43ff01ffffffe70cULL;
E
Eddie Dong 已提交
1300 1301

	if ((alignment + len) > 4) {
1302 1303
		apic_debug("KVM_APIC_READ: alignment error %x %d\n",
			   offset, len);
G
Gleb Natapov 已提交
1304
		return 1;
E
Eddie Dong 已提交
1305
	}
G
Gleb Natapov 已提交
1306 1307

	if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) {
1308 1309
		apic_debug("KVM_APIC_READ: read reserved register %x\n",
			   offset);
G
Gleb Natapov 已提交
1310 1311 1312
		return 1;
	}

E
Eddie Dong 已提交
1313 1314
	result = __apic_read(apic, offset & ~0xf);

1315 1316
	trace_kvm_apic_read(offset, result);

E
Eddie Dong 已提交
1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327
	switch (len) {
	case 1:
	case 2:
	case 4:
		memcpy(data, (char *)&result + alignment, len);
		break;
	default:
		printk(KERN_ERR "Local APIC read with len = %x, "
		       "should be 1,2, or 4 instead\n", len);
		break;
	}
1328
	return 0;
E
Eddie Dong 已提交
1329
}
1330
EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
E
Eddie Dong 已提交
1331

G
Gleb Natapov 已提交
1332 1333
static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
{
1334
	return kvm_apic_hw_enabled(apic) &&
G
Gleb Natapov 已提交
1335 1336 1337 1338
	    addr >= apic->base_address &&
	    addr < apic->base_address + LAPIC_MMIO_LENGTH;
}

1339
static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
G
Gleb Natapov 已提交
1340 1341 1342 1343 1344 1345 1346 1347
			   gpa_t address, int len, void *data)
{
	struct kvm_lapic *apic = to_lapic(this);
	u32 offset = address - apic->base_address;

	if (!apic_mmio_in_range(apic, address))
		return -EOPNOTSUPP;

1348
	kvm_lapic_reg_read(apic, offset, len, data);
G
Gleb Natapov 已提交
1349 1350 1351 1352

	return 0;
}

E
Eddie Dong 已提交
1353 1354 1355 1356
static void update_divide_count(struct kvm_lapic *apic)
{
	u32 tmp1, tmp2, tdcr;

1357
	tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
E
Eddie Dong 已提交
1358 1359
	tmp1 = tdcr & 0xf;
	tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
1360
	apic->divide_count = 0x1 << (tmp2 & 0x7);
E
Eddie Dong 已提交
1361 1362

	apic_debug("timer divide count is 0x%x\n",
G
Glauber Costa 已提交
1363
				   apic->divide_count);
E
Eddie Dong 已提交
1364 1365
}

1366 1367 1368 1369 1370 1371 1372
static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
{
	/*
	 * Do not allow the guest to program periodic timers with small
	 * interval, since the hrtimers are not throttled by the host
	 * scheduler.
	 */
1373
	if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
		s64 min_period = min_timer_period_us * 1000LL;

		if (apic->lapic_timer.period < min_period) {
			pr_info_ratelimited(
			    "kvm: vcpu %i: requested %lld ns "
			    "lapic timer period limited to %lld ns\n",
			    apic->vcpu->vcpu_id,
			    apic->lapic_timer.period, min_period);
			apic->lapic_timer.period = min_period;
		}
	}
}

1387 1388
static void apic_update_lvtt(struct kvm_lapic *apic)
{
1389
	u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
1390 1391 1392
			apic->lapic_timer.timer_mode_mask;

	if (apic->lapic_timer.timer_mode != timer_mode) {
1393
		if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
1394 1395
				APIC_LVT_TIMER_TSCDEADLINE)) {
			hrtimer_cancel(&apic->lapic_timer.timer);
1396 1397 1398
			kvm_lapic_set_reg(apic, APIC_TMICT, 0);
			apic->lapic_timer.period = 0;
			apic->lapic_timer.tscdeadline = 0;
1399
		}
1400
		apic->lapic_timer.timer_mode = timer_mode;
1401
		limit_periodic_timer_frequency(apic);
1402 1403 1404
	}
}

1405 1406 1407
static void apic_timer_expired(struct kvm_lapic *apic)
{
	struct kvm_vcpu *vcpu = apic->vcpu;
1408
	struct swait_queue_head *q = &vcpu->wq;
1409
	struct kvm_timer *ktimer = &apic->lapic_timer;
1410 1411 1412 1413 1414

	if (atomic_read(&apic->lapic_timer.pending))
		return;

	atomic_inc(&apic->lapic_timer.pending);
1415
	kvm_set_pending_timer(vcpu);
1416

1417 1418 1419 1420
	/*
	 * For x86, the atomic_inc() is serialized, thus
	 * using swait_active() is safe.
	 */
1421
	if (swait_active(q))
1422
		swake_up_one(q);
1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435

	if (apic_lvtt_tscdeadline(apic))
		ktimer->expired_tscdeadline = ktimer->tscdeadline;
}

/*
 * On APICv, this test will cause a busy wait
 * during a higher-priority task.
 */

static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
1436
	u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
1437 1438 1439

	if (kvm_apic_hw_enabled(apic)) {
		int vec = reg & APIC_VECTOR_MASK;
1440
		void *bitmap = apic->regs + APIC_ISR;
1441

1442
		if (vcpu->arch.apicv_active)
1443 1444 1445 1446
			bitmap = apic->regs + APIC_IRR;

		if (apic_test_vector(vec, bitmap))
			return true;
1447 1448 1449 1450 1451 1452 1453 1454 1455
	}
	return false;
}

void wait_lapic_expire(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u64 guest_tsc, tsc_deadline;

1456
	if (!lapic_in_kernel(vcpu))
1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
		return;

	if (apic->lapic_timer.expired_tscdeadline == 0)
		return;

	if (!lapic_timer_int_injected(vcpu))
		return;

	tsc_deadline = apic->lapic_timer.expired_tscdeadline;
	apic->lapic_timer.expired_tscdeadline = 0;
1467
	guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1468
	trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline);
1469 1470 1471

	/* __delay is delay_tsc whenever the hardware has TSC, thus always.  */
	if (guest_tsc < tsc_deadline)
1472 1473
		__delay(min(tsc_deadline - guest_tsc,
			nsec_to_cycles(vcpu, lapic_timer_advance_ns)));
1474 1475
}

1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490
static void start_sw_tscdeadline(struct kvm_lapic *apic)
{
	u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline;
	u64 ns = 0;
	ktime_t expire;
	struct kvm_vcpu *vcpu = apic->vcpu;
	unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
	unsigned long flags;
	ktime_t now;

	if (unlikely(!tscdeadline || !this_tsc_khz))
		return;

	local_irq_save(flags);

1491
	now = ktime_get();
1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505
	guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
	if (likely(tscdeadline > guest_tsc)) {
		ns = (tscdeadline - guest_tsc) * 1000000ULL;
		do_div(ns, this_tsc_khz);
		expire = ktime_add_ns(now, ns);
		expire = ktime_sub_ns(expire, lapic_timer_advance_ns);
		hrtimer_start(&apic->lapic_timer.timer,
				expire, HRTIMER_MODE_ABS_PINNED);
	} else
		apic_timer_expired(apic);

	local_irq_restore(flags);
}

1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529
static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
{
	ktime_t now, remaining;
	u64 ns_remaining_old, ns_remaining_new;

	apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
		* APIC_BUS_CYCLE_NS * apic->divide_count;
	limit_periodic_timer_frequency(apic);

	now = ktime_get();
	remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
	if (ktime_to_ns(remaining) < 0)
		remaining = 0;

	ns_remaining_old = ktime_to_ns(remaining);
	ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
	                                   apic->divide_count, old_divisor);

	apic->lapic_timer.tscdeadline +=
		nsec_to_cycles(apic->vcpu, ns_remaining_new) -
		nsec_to_cycles(apic->vcpu, ns_remaining_old);
	apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
}

1530
static bool set_target_expiration(struct kvm_lapic *apic)
1531 1532
{
	ktime_t now;
1533
	u64 tscl = rdtsc();
1534

1535
	now = ktime_get();
1536
	apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
1537
		* APIC_BUS_CYCLE_NS * apic->divide_count;
1538

1539 1540
	if (!apic->lapic_timer.period) {
		apic->lapic_timer.tscdeadline = 0;
1541
		return false;
1542 1543
	}

1544
	limit_periodic_timer_frequency(apic);
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554

	apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
		   PRIx64 ", "
		   "timer initial count 0x%x, period %lldns, "
		   "expire @ 0x%016" PRIx64 ".\n", __func__,
		   APIC_BUS_CYCLE_NS, ktime_to_ns(now),
		   kvm_lapic_get_reg(apic, APIC_TMICT),
		   apic->lapic_timer.period,
		   ktime_to_ns(ktime_add_ns(now,
				apic->lapic_timer.period)));
1555 1556 1557 1558 1559 1560 1561 1562 1563 1564

	apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
		nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
	apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);

	return true;
}

static void advance_periodic_target_expiration(struct kvm_lapic *apic)
{
1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575
	ktime_t now = ktime_get();
	u64 tscl = rdtsc();
	ktime_t delta;

	/*
	 * Synchronize both deadlines to the same time source or
	 * differences in the periods (caused by differences in the
	 * underlying clocks or numerical approximation errors) will
	 * cause the two to drift apart over time as the errors
	 * accumulate.
	 */
1576 1577 1578
	apic->lapic_timer.target_expiration =
		ktime_add_ns(apic->lapic_timer.target_expiration,
				apic->lapic_timer.period);
1579 1580 1581
	delta = ktime_sub(apic->lapic_timer.target_expiration, now);
	apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
		nsec_to_cycles(apic->vcpu, delta);
1582 1583
}

1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
static void start_sw_period(struct kvm_lapic *apic)
{
	if (!apic->lapic_timer.period)
		return;

	if (ktime_after(ktime_get(),
			apic->lapic_timer.target_expiration)) {
		apic_timer_expired(apic);

		if (apic_lvtt_oneshot(apic))
			return;

		advance_periodic_target_expiration(apic);
	}

	hrtimer_start(&apic->lapic_timer.timer,
		apic->lapic_timer.target_expiration,
		HRTIMER_MODE_ABS_PINNED);
}

1604 1605
bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
{
1606 1607 1608
	if (!lapic_in_kernel(vcpu))
		return false;

1609 1610 1611 1612
	return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
}
EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);

1613
static void cancel_hv_timer(struct kvm_lapic *apic)
1614
{
1615
	WARN_ON(preemptible());
1616
	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1617 1618 1619 1620
	kvm_x86_ops->cancel_hv_timer(apic->vcpu);
	apic->lapic_timer.hv_timer_in_use = false;
}

1621
static bool start_hv_timer(struct kvm_lapic *apic)
1622
{
1623 1624
	struct kvm_timer *ktimer = &apic->lapic_timer;
	int r;
1625

1626
	WARN_ON(preemptible());
1627 1628 1629
	if (!kvm_x86_ops->set_hv_timer)
		return false;

1630 1631 1632
	if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
		return false;

1633 1634 1635
	if (!ktimer->tscdeadline)
		return false;

1636 1637 1638 1639 1640 1641
	r = kvm_x86_ops->set_hv_timer(apic->vcpu, ktimer->tscdeadline);
	if (r < 0)
		return false;

	ktimer->hv_timer_in_use = true;
	hrtimer_cancel(&ktimer->timer);
1642

1643 1644 1645 1646 1647
	/*
	 * Also recheck ktimer->pending, in case the sw timer triggered in
	 * the window.  For periodic timer, leave the hv timer running for
	 * simplicity, and the deadline will be recomputed on the next vmexit.
	 */
1648 1649 1650
	if (!apic_lvtt_period(apic) && (r || atomic_read(&ktimer->pending))) {
		if (r)
			apic_timer_expired(apic);
1651
		return false;
1652
	}
1653 1654

	trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, true);
1655 1656 1657
	return true;
}

1658
static void start_sw_timer(struct kvm_lapic *apic)
1659
{
1660
	struct kvm_timer *ktimer = &apic->lapic_timer;
1661 1662

	WARN_ON(preemptible());
1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673
	if (apic->lapic_timer.hv_timer_in_use)
		cancel_hv_timer(apic);
	if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
		return;

	if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
		start_sw_period(apic);
	else if (apic_lvtt_tscdeadline(apic))
		start_sw_tscdeadline(apic);
	trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
}
1674

1675 1676
static void restart_apic_timer(struct kvm_lapic *apic)
{
1677
	preempt_disable();
1678 1679
	if (!start_hv_timer(apic))
		start_sw_timer(apic);
1680
	preempt_enable();
1681 1682
}

1683 1684 1685 1686
void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

1687 1688 1689 1690
	preempt_disable();
	/* If the preempt notifier has already run, it also called apic_timer_expired */
	if (!apic->lapic_timer.hv_timer_in_use)
		goto out;
1691 1692 1693 1694 1695 1696
	WARN_ON(swait_active(&vcpu->wq));
	cancel_hv_timer(apic);
	apic_timer_expired(apic);

	if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
		advance_periodic_target_expiration(apic);
1697
		restart_apic_timer(apic);
1698
	}
1699 1700
out:
	preempt_enable();
1701 1702 1703
}
EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);

1704 1705
void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
{
1706
	restart_apic_timer(vcpu->arch.apic);
1707 1708 1709 1710 1711 1712 1713
}
EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);

void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

1714
	preempt_disable();
1715
	/* Possibly the TSC deadline timer is not enabled yet */
1716 1717
	if (apic->lapic_timer.hv_timer_in_use)
		start_sw_timer(apic);
1718
	preempt_enable();
1719 1720
}
EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
1721

1722 1723 1724
void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
1725

1726 1727
	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
	restart_apic_timer(apic);
1728 1729
}

E
Eddie Dong 已提交
1730 1731
static void start_apic_timer(struct kvm_lapic *apic)
{
1732
	atomic_set(&apic->lapic_timer.pending, 0);
1733

1734 1735 1736 1737 1738
	if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
	    && !set_target_expiration(apic))
		return;

	restart_apic_timer(apic);
E
Eddie Dong 已提交
1739 1740
}

1741 1742
static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
{
1743
	bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
1744

1745 1746 1747
	if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
		apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
		if (lvt0_in_nmi_mode) {
1748 1749
			apic_debug("Receive NMI setting on APIC_LVT0 "
				   "for cpu %d\n", apic->vcpu->vcpu_id);
1750
			atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1751 1752 1753
		} else
			atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
	}
1754 1755
}

1756
int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
E
Eddie Dong 已提交
1757
{
G
Gleb Natapov 已提交
1758
	int ret = 0;
E
Eddie Dong 已提交
1759

G
Gleb Natapov 已提交
1760
	trace_kvm_apic_write(reg, val);
E
Eddie Dong 已提交
1761

G
Gleb Natapov 已提交
1762
	switch (reg) {
E
Eddie Dong 已提交
1763
	case APIC_ID:		/* Local APIC ID */
G
Gleb Natapov 已提交
1764
		if (!apic_x2apic_mode(apic))
1765
			kvm_apic_set_xapic_id(apic, val >> 24);
G
Gleb Natapov 已提交
1766 1767
		else
			ret = 1;
E
Eddie Dong 已提交
1768 1769 1770
		break;

	case APIC_TASKPRI:
1771
		report_tpr_access(apic, true);
E
Eddie Dong 已提交
1772 1773 1774 1775 1776 1777 1778 1779
		apic_set_tpr(apic, val & 0xff);
		break;

	case APIC_EOI:
		apic_set_eoi(apic);
		break;

	case APIC_LDR:
G
Gleb Natapov 已提交
1780
		if (!apic_x2apic_mode(apic))
1781
			kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
G
Gleb Natapov 已提交
1782 1783
		else
			ret = 1;
E
Eddie Dong 已提交
1784 1785 1786
		break;

	case APIC_DFR:
1787
		if (!apic_x2apic_mode(apic)) {
1788
			kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1789 1790
			recalculate_apic_map(apic->vcpu->kvm);
		} else
G
Gleb Natapov 已提交
1791
			ret = 1;
E
Eddie Dong 已提交
1792 1793
		break;

1794 1795
	case APIC_SPIV: {
		u32 mask = 0x3ff;
1796
		if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
1797
			mask |= APIC_SPIV_DIRECTED_EOI;
1798
		apic_set_spiv(apic, val & mask);
E
Eddie Dong 已提交
1799 1800 1801 1802
		if (!(val & APIC_SPIV_APIC_ENABLED)) {
			int i;
			u32 lvt_val;

1803
			for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
1804
				lvt_val = kvm_lapic_get_reg(apic,
E
Eddie Dong 已提交
1805
						       APIC_LVTT + 0x10 * i);
1806
				kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
E
Eddie Dong 已提交
1807 1808
					     lvt_val | APIC_LVT_MASKED);
			}
1809
			apic_update_lvtt(apic);
1810
			atomic_set(&apic->lapic_timer.pending, 0);
E
Eddie Dong 已提交
1811 1812 1813

		}
		break;
1814
	}
E
Eddie Dong 已提交
1815 1816
	case APIC_ICR:
		/* No delay here, so we always clear the pending bit */
1817
		kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
E
Eddie Dong 已提交
1818 1819 1820 1821
		apic_send_ipi(apic);
		break;

	case APIC_ICR2:
G
Gleb Natapov 已提交
1822 1823
		if (!apic_x2apic_mode(apic))
			val &= 0xff000000;
1824
		kvm_lapic_set_reg(apic, APIC_ICR2, val);
E
Eddie Dong 已提交
1825 1826
		break;

1827
	case APIC_LVT0:
1828
		apic_manage_nmi_watchdog(apic, val);
E
Eddie Dong 已提交
1829 1830 1831 1832 1833
	case APIC_LVTTHMR:
	case APIC_LVTPC:
	case APIC_LVT1:
	case APIC_LVTERR:
		/* TODO: Check vector */
1834
		if (!kvm_apic_sw_enabled(apic))
E
Eddie Dong 已提交
1835 1836
			val |= APIC_LVT_MASKED;

G
Gleb Natapov 已提交
1837
		val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
1838
		kvm_lapic_set_reg(apic, reg, val);
E
Eddie Dong 已提交
1839 1840 1841

		break;

1842
	case APIC_LVTT:
1843
		if (!kvm_apic_sw_enabled(apic))
1844 1845
			val |= APIC_LVT_MASKED;
		val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
1846
		kvm_lapic_set_reg(apic, APIC_LVTT, val);
1847
		apic_update_lvtt(apic);
1848 1849
		break;

E
Eddie Dong 已提交
1850
	case APIC_TMICT:
1851 1852 1853
		if (apic_lvtt_tscdeadline(apic))
			break;

1854
		hrtimer_cancel(&apic->lapic_timer.timer);
1855
		kvm_lapic_set_reg(apic, APIC_TMICT, val);
E
Eddie Dong 已提交
1856
		start_apic_timer(apic);
G
Gleb Natapov 已提交
1857
		break;
E
Eddie Dong 已提交
1858

1859 1860 1861
	case APIC_TDCR: {
		uint32_t old_divisor = apic->divide_count;

E
Eddie Dong 已提交
1862
		if (val & 4)
1863
			apic_debug("KVM_WRITE:TDCR %x\n", val);
1864
		kvm_lapic_set_reg(apic, APIC_TDCR, val);
E
Eddie Dong 已提交
1865
		update_divide_count(apic);
1866 1867 1868 1869 1870 1871
		if (apic->divide_count != old_divisor &&
				apic->lapic_timer.period) {
			hrtimer_cancel(&apic->lapic_timer.timer);
			update_target_expiration(apic, old_divisor);
			restart_apic_timer(apic);
		}
E
Eddie Dong 已提交
1872
		break;
1873
	}
G
Gleb Natapov 已提交
1874 1875
	case APIC_ESR:
		if (apic_x2apic_mode(apic) && val != 0) {
1876
			apic_debug("KVM_WRITE:ESR not zero %x\n", val);
G
Gleb Natapov 已提交
1877 1878 1879 1880 1881 1882
			ret = 1;
		}
		break;

	case APIC_SELF_IPI:
		if (apic_x2apic_mode(apic)) {
1883
			kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
G
Gleb Natapov 已提交
1884 1885 1886
		} else
			ret = 1;
		break;
E
Eddie Dong 已提交
1887
	default:
G
Gleb Natapov 已提交
1888
		ret = 1;
E
Eddie Dong 已提交
1889 1890
		break;
	}
G
Gleb Natapov 已提交
1891 1892 1893 1894
	if (ret)
		apic_debug("Local APIC Write to read-only register %x\n", reg);
	return ret;
}
1895
EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
G
Gleb Natapov 已提交
1896

1897
static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
G
Gleb Natapov 已提交
1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914
			    gpa_t address, int len, const void *data)
{
	struct kvm_lapic *apic = to_lapic(this);
	unsigned int offset = address - apic->base_address;
	u32 val;

	if (!apic_mmio_in_range(apic, address))
		return -EOPNOTSUPP;

	/*
	 * APIC register must be aligned on 128-bits boundary.
	 * 32/64/128 bits registers must be accessed thru 32 bits.
	 * Refer SDM 8.4.1
	 */
	if (len != 4 || (offset & 0xf)) {
		/* Don't shout loud, $infamous_os would cause only noise. */
		apic_debug("apic write: bad size=%d %lx\n", len, (long)address);
1915
		return 0;
G
Gleb Natapov 已提交
1916 1917 1918 1919 1920 1921 1922 1923 1924
	}

	val = *(u32*)data;

	/* too common printing */
	if (offset != APIC_EOI)
		apic_debug("%s: offset 0x%x with length 0x%x, and value is "
			   "0x%x\n", __func__, offset, len, val);

1925
	kvm_lapic_reg_write(apic, offset & 0xff0, val);
G
Gleb Natapov 已提交
1926

1927
	return 0;
E
Eddie Dong 已提交
1928 1929
}

1930 1931
void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
{
1932
	kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
1933 1934 1935
}
EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);

1936 1937 1938 1939 1940 1941 1942 1943
/* emulate APIC access in a trap manner */
void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
{
	u32 val = 0;

	/* hw has done the conditional check and inst decode */
	offset &= 0xff0;

1944
	kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
1945 1946

	/* TODO: optimize to just emulate side effect w/o one more write */
1947
	kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
1948 1949 1950
}
EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);

1951
void kvm_free_lapic(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
1952
{
1953 1954
	struct kvm_lapic *apic = vcpu->arch.apic;

1955
	if (!vcpu->arch.apic)
E
Eddie Dong 已提交
1956 1957
		return;

1958
	hrtimer_cancel(&apic->lapic_timer.timer);
E
Eddie Dong 已提交
1959

1960 1961 1962
	if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
		static_key_slow_dec_deferred(&apic_hw_disabled);

1963
	if (!apic->sw_enabled)
1964
		static_key_slow_dec_deferred(&apic_sw_disabled);
E
Eddie Dong 已提交
1965

1966 1967 1968 1969
	if (apic->regs)
		free_page((unsigned long)apic->regs);

	kfree(apic);
E
Eddie Dong 已提交
1970 1971 1972 1973 1974 1975 1976
}

/*
 *----------------------------------------------------------------------
 * LAPIC interface
 *----------------------------------------------------------------------
 */
1977 1978 1979 1980
u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

1981 1982
	if (!lapic_in_kernel(vcpu) ||
		!apic_lvtt_tscdeadline(apic))
1983 1984 1985 1986 1987 1988 1989 1990 1991
		return 0;

	return apic->lapic_timer.tscdeadline;
}

void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

1992
	if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
1993
			apic_lvtt_period(apic))
1994 1995 1996 1997 1998 1999 2000
		return;

	hrtimer_cancel(&apic->lapic_timer.timer);
	apic->lapic_timer.tscdeadline = data;
	start_apic_timer(apic);
}

E
Eddie Dong 已提交
2001 2002
void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
{
2003
	struct kvm_lapic *apic = vcpu->arch.apic;
E
Eddie Dong 已提交
2004

A
Avi Kivity 已提交
2005
	apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
2006
		     | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
E
Eddie Dong 已提交
2007 2008 2009 2010 2011 2012
}

u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
{
	u64 tpr;

2013
	tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
E
Eddie Dong 已提交
2014 2015 2016 2017 2018 2019

	return (tpr & 0xf0) >> 4;
}

void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
{
2020
	u64 old_value = vcpu->arch.apic_base;
2021
	struct kvm_lapic *apic = vcpu->arch.apic;
E
Eddie Dong 已提交
2022

2023
	if (!apic)
E
Eddie Dong 已提交
2024
		value |= MSR_IA32_APICBASE_BSP;
2025

2026 2027
	vcpu->arch.apic_base = value;

2028 2029 2030 2031 2032 2033
	if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
		kvm_update_cpuid(vcpu);

	if (!apic)
		return;

2034
	/* update jump label if enable bit changes */
2035
	if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
2036 2037
		if (value & MSR_IA32_APICBASE_ENABLE) {
			kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2038
			static_key_slow_dec_deferred(&apic_hw_disabled);
2039
		} else {
2040
			static_key_slow_inc(&apic_hw_disabled.key);
2041 2042
			recalculate_apic_map(vcpu->kvm);
		}
2043 2044
	}

2045 2046 2047 2048 2049
	if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE))
		kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);

	if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
		kvm_x86_ops->set_virtual_apic_mode(vcpu);
2050

2051
	apic->base_address = apic->vcpu->arch.apic_base &
E
Eddie Dong 已提交
2052 2053
			     MSR_IA32_APICBASE_BASE;

2054 2055 2056 2057
	if ((value & MSR_IA32_APICBASE_ENABLE) &&
	     apic->base_address != APIC_DEFAULT_PHYS_BASE)
		pr_warn_once("APIC base relocation is unsupported by KVM");

E
Eddie Dong 已提交
2058 2059
	/* with FSB delivery interrupt, we can restart APIC functionality */
	apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is "
2060
		   "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address);
E
Eddie Dong 已提交
2061 2062 2063

}

2064
void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
E
Eddie Dong 已提交
2065
{
2066
	struct kvm_lapic *apic = vcpu->arch.apic;
E
Eddie Dong 已提交
2067 2068
	int i;

2069 2070
	if (!apic)
		return;
E
Eddie Dong 已提交
2071

2072
	apic_debug("%s\n", __func__);
E
Eddie Dong 已提交
2073 2074

	/* Stop the timer in case it's a reset to an active apic */
2075
	hrtimer_cancel(&apic->lapic_timer.timer);
E
Eddie Dong 已提交
2076

2077 2078 2079
	if (!init_event) {
		kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
		                         MSR_IA32_APICBASE_ENABLE);
2080
		kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2081
	}
2082
	kvm_apic_set_version(apic->vcpu);
E
Eddie Dong 已提交
2083

2084 2085
	for (i = 0; i < KVM_APIC_LVT_NUM; i++)
		kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
2086
	apic_update_lvtt(apic);
2087 2088
	if (kvm_vcpu_is_reset_bsp(vcpu) &&
	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
2089
		kvm_lapic_set_reg(apic, APIC_LVT0,
2090
			     SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
2091
	apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
E
Eddie Dong 已提交
2092

2093
	kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
2094
	apic_set_spiv(apic, 0xff);
2095
	kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
2096 2097
	if (!apic_x2apic_mode(apic))
		kvm_apic_set_ldr(apic, 0);
2098 2099 2100 2101 2102
	kvm_lapic_set_reg(apic, APIC_ESR, 0);
	kvm_lapic_set_reg(apic, APIC_ICR, 0);
	kvm_lapic_set_reg(apic, APIC_ICR2, 0);
	kvm_lapic_set_reg(apic, APIC_TDCR, 0);
	kvm_lapic_set_reg(apic, APIC_TMICT, 0);
E
Eddie Dong 已提交
2103
	for (i = 0; i < 8; i++) {
2104 2105 2106
		kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
		kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
		kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
E
Eddie Dong 已提交
2107
	}
2108 2109
	apic->irr_pending = vcpu->arch.apicv_active;
	apic->isr_count = vcpu->arch.apicv_active ? 1 : 0;
M
Michael S. Tsirkin 已提交
2110
	apic->highest_isr_cache = -1;
2111
	update_divide_count(apic);
2112
	atomic_set(&apic->lapic_timer.pending, 0);
2113
	if (kvm_vcpu_is_bsp(vcpu))
2114 2115
		kvm_lapic_set_base(vcpu,
				vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
2116
	vcpu->arch.pv_eoi.msr_val = 0;
E
Eddie Dong 已提交
2117
	apic_update_ppr(apic);
2118 2119 2120 2121 2122
	if (vcpu->arch.apicv_active) {
		kvm_x86_ops->apicv_post_state_restore(vcpu);
		kvm_x86_ops->hwapic_irr_update(vcpu, -1);
		kvm_x86_ops->hwapic_isr_update(vcpu, -1);
	}
E
Eddie Dong 已提交
2123

2124
	vcpu->arch.apic_arb_prio = 0;
2125
	vcpu->arch.apic_attention = 0;
2126

2127
	apic_debug("%s: vcpu=%p, id=0x%x, base_msr="
2128
		   "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__,
2129
		   vcpu, kvm_lapic_get_reg(apic, APIC_ID),
2130
		   vcpu->arch.apic_base, apic->base_address);
E
Eddie Dong 已提交
2131 2132 2133 2134 2135 2136 2137
}

/*
 *----------------------------------------------------------------------
 * timer interface
 *----------------------------------------------------------------------
 */
2138

A
Avi Kivity 已提交
2139
static bool lapic_is_periodic(struct kvm_lapic *apic)
E
Eddie Dong 已提交
2140
{
2141
	return apic_lvtt_period(apic);
E
Eddie Dong 已提交
2142 2143
}

2144 2145
int apic_has_pending_timer(struct kvm_vcpu *vcpu)
{
2146
	struct kvm_lapic *apic = vcpu->arch.apic;
2147

2148
	if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
2149
		return atomic_read(&apic->lapic_timer.pending);
2150 2151 2152 2153

	return 0;
}

A
Avi Kivity 已提交
2154
int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
2155
{
2156
	u32 reg = kvm_lapic_get_reg(apic, lvt_type);
2157 2158
	int vector, mode, trig_mode;

2159
	if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
2160 2161 2162
		vector = reg & APIC_VECTOR_MASK;
		mode = reg & APIC_MODE_MASK;
		trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
2163 2164
		return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
					NULL);
2165 2166 2167
	}
	return 0;
}
2168

2169
void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
2170
{
2171 2172 2173 2174
	struct kvm_lapic *apic = vcpu->arch.apic;

	if (apic)
		kvm_apic_local_deliver(apic, APIC_LVT0);
2175 2176
}

G
Gregory Haskins 已提交
2177 2178 2179 2180 2181
static const struct kvm_io_device_ops apic_mmio_ops = {
	.read     = apic_mmio_read,
	.write    = apic_mmio_write,
};

2182 2183 2184
static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
{
	struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
A
Avi Kivity 已提交
2185
	struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
2186

2187
	apic_timer_expired(apic);
2188

A
Avi Kivity 已提交
2189
	if (lapic_is_periodic(apic)) {
2190
		advance_periodic_target_expiration(apic);
2191 2192 2193 2194 2195 2196
		hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
		return HRTIMER_RESTART;
	} else
		return HRTIMER_NORESTART;
}

E
Eddie Dong 已提交
2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207
int kvm_create_lapic(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic;

	ASSERT(vcpu != NULL);
	apic_debug("apic_init %d\n", vcpu->vcpu_id);

	apic = kzalloc(sizeof(*apic), GFP_KERNEL);
	if (!apic)
		goto nomem;

2208
	vcpu->arch.apic = apic;
E
Eddie Dong 已提交
2209

2210 2211
	apic->regs = (void *)get_zeroed_page(GFP_KERNEL);
	if (!apic->regs) {
E
Eddie Dong 已提交
2212 2213
		printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
		       vcpu->vcpu_id);
2214
		goto nomem_free_apic;
E
Eddie Dong 已提交
2215 2216 2217
	}
	apic->vcpu = vcpu;

2218
	hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2219
		     HRTIMER_MODE_ABS_PINNED);
2220
	apic->lapic_timer.timer.function = apic_timer_fn;
2221

2222 2223 2224 2225 2226
	/*
	 * APIC is created enabled. This will prevent kvm_lapic_set_base from
	 * thinking that APIC satet has changed.
	 */
	vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
2227
	static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
G
Gregory Haskins 已提交
2228
	kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
E
Eddie Dong 已提交
2229 2230

	return 0;
2231 2232
nomem_free_apic:
	kfree(apic);
E
Eddie Dong 已提交
2233 2234 2235 2236 2237 2238
nomem:
	return -ENOMEM;
}

int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
{
2239
	struct kvm_lapic *apic = vcpu->arch.apic;
2240
	u32 ppr;
E
Eddie Dong 已提交
2241

2242
	if (!apic_enabled(apic))
E
Eddie Dong 已提交
2243 2244
		return -1;

2245 2246
	__apic_update_ppr(apic, &ppr);
	return apic_has_interrupt_for_ppr(apic, ppr);
E
Eddie Dong 已提交
2247 2248
}

Q
Qing He 已提交
2249 2250
int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
{
2251
	u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
Q
Qing He 已提交
2252 2253
	int r = 0;

2254
	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
2255 2256 2257 2258
		r = 1;
	if ((lvt0 & APIC_LVT_MASKED) == 0 &&
	    GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
		r = 1;
Q
Qing He 已提交
2259 2260 2261
	return r;
}

2262 2263
void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
{
2264
	struct kvm_lapic *apic = vcpu->arch.apic;
2265

2266
	if (atomic_read(&apic->lapic_timer.pending) > 0) {
2267
		kvm_apic_local_deliver(apic, APIC_LVTT);
2268 2269
		if (apic_lvtt_tscdeadline(apic))
			apic->lapic_timer.tscdeadline = 0;
2270 2271
		if (apic_lvtt_oneshot(apic)) {
			apic->lapic_timer.tscdeadline = 0;
T
Thomas Gleixner 已提交
2272
			apic->lapic_timer.target_expiration = 0;
2273
		}
2274
		atomic_set(&apic->lapic_timer.pending, 0);
2275 2276 2277
	}
}

E
Eddie Dong 已提交
2278 2279 2280
int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
{
	int vector = kvm_apic_has_interrupt(vcpu);
2281
	struct kvm_lapic *apic = vcpu->arch.apic;
2282
	u32 ppr;
E
Eddie Dong 已提交
2283 2284 2285 2286

	if (vector == -1)
		return -1;

2287 2288 2289 2290 2291 2292 2293
	/*
	 * We get here even with APIC virtualization enabled, if doing
	 * nested virtualization and L1 runs with the "acknowledge interrupt
	 * on exit" mode.  Then we cannot inject the interrupt via RVI,
	 * because the process would deliver it through the IDT.
	 */

E
Eddie Dong 已提交
2294
	apic_clear_irr(vector, apic);
2295
	if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
2296 2297 2298 2299 2300
		/*
		 * For auto-EOI interrupts, there might be another pending
		 * interrupt above PPR, so check whether to raise another
		 * KVM_REQ_EVENT.
		 */
2301
		apic_update_ppr(apic);
2302 2303 2304 2305 2306 2307 2308 2309 2310
	} else {
		/*
		 * For normal interrupts, PPR has been raised and there cannot
		 * be a higher-priority pending interrupt---except if there was
		 * a concurrent interrupt injection, but that would have
		 * triggered KVM_REQ_EVENT already.
		 */
		apic_set_isr(vector, apic);
		__apic_update_ppr(apic, &ppr);
2311 2312
	}

E
Eddie Dong 已提交
2313 2314
	return vector;
}
2315

2316 2317 2318 2319 2320
static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
		struct kvm_lapic_state *s, bool set)
{
	if (apic_x2apic_mode(vcpu->arch.apic)) {
		u32 *id = (u32 *)(s->regs + APIC_ID);
2321
		u32 *ldr = (u32 *)(s->regs + APIC_LDR);
2322

2323 2324 2325 2326 2327 2328 2329 2330 2331
		if (vcpu->kvm->arch.x2apic_format) {
			if (*id != vcpu->vcpu_id)
				return -EINVAL;
		} else {
			if (set)
				*id >>= 24;
			else
				*id <<= 24;
		}
2332 2333 2334 2335

		/* In x2APIC mode, the LDR is fixed and based on the id */
		if (set)
			*ldr = kvm_apic_calc_x2apic_ldr(*id);
2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347
	}

	return 0;
}

int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
{
	memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
	return kvm_apic_state_fixup(vcpu, s, false);
}

int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2348
{
2349
	struct kvm_lapic *apic = vcpu->arch.apic;
2350 2351
	int r;

2352

2353
	kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
2354 2355
	/* set SPIV separately to get count of SW disabled APICs right */
	apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
2356 2357 2358 2359

	r = kvm_apic_state_fixup(vcpu, s, true);
	if (r)
		return r;
2360
	memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
2361 2362

	recalculate_apic_map(vcpu->kvm);
2363 2364
	kvm_apic_set_version(vcpu);

2365
	apic_update_ppr(apic);
2366
	hrtimer_cancel(&apic->lapic_timer.timer);
2367
	apic_update_lvtt(apic);
2368
	apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2369 2370
	update_divide_count(apic);
	start_apic_timer(apic);
2371
	apic->irr_pending = true;
2372
	apic->isr_count = vcpu->arch.apicv_active ?
2373
				1 : count_vectors(apic->regs + APIC_ISR);
M
Michael S. Tsirkin 已提交
2374
	apic->highest_isr_cache = -1;
2375
	if (vcpu->arch.apicv_active) {
2376
		kvm_x86_ops->apicv_post_state_restore(vcpu);
W
Wei Wang 已提交
2377 2378
		kvm_x86_ops->hwapic_irr_update(vcpu,
				apic_find_highest_irr(apic));
2379
		kvm_x86_ops->hwapic_isr_update(vcpu,
2380
				apic_find_highest_isr(apic));
2381
	}
2382
	kvm_make_request(KVM_REQ_EVENT, vcpu);
2383 2384
	if (ioapic_in_kernel(vcpu->kvm))
		kvm_rtc_eoi_tracking_restore_one(vcpu);
2385 2386

	vcpu->arch.apic_arb_prio = 0;
2387 2388

	return 0;
2389
}
2390

2391
void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
2392 2393 2394
{
	struct hrtimer *timer;

2395
	if (!lapic_in_kernel(vcpu))
2396 2397
		return;

2398
	timer = &vcpu->arch.apic->lapic_timer.timer;
2399
	if (hrtimer_cancel(timer))
2400
		hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
2401
}
A
Avi Kivity 已提交
2402

2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439
/*
 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
 *
 * Detect whether guest triggered PV EOI since the
 * last entry. If yes, set EOI on guests's behalf.
 * Clear PV EOI in guest memory in any case.
 */
static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
					struct kvm_lapic *apic)
{
	bool pending;
	int vector;
	/*
	 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
	 * and KVM_PV_EOI_ENABLED in guest memory as follows:
	 *
	 * KVM_APIC_PV_EOI_PENDING is unset:
	 * 	-> host disabled PV EOI.
	 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
	 * 	-> host enabled PV EOI, guest did not execute EOI yet.
	 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
	 * 	-> host enabled PV EOI, guest executed EOI.
	 */
	BUG_ON(!pv_eoi_enabled(vcpu));
	pending = pv_eoi_get_pending(vcpu);
	/*
	 * Clear pending bit in any case: it will be set again on vmentry.
	 * While this might not be ideal from performance point of view,
	 * this makes sure pv eoi is only enabled when we know it's safe.
	 */
	pv_eoi_clr_pending(vcpu);
	if (pending)
		return;
	vector = apic_set_eoi(apic);
	trace_kvm_pv_eoi(apic, vector);
}

A
Avi Kivity 已提交
2440 2441 2442 2443
void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
{
	u32 data;

2444 2445 2446
	if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
		apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);

2447
	if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
A
Avi Kivity 已提交
2448 2449
		return;

2450 2451
	if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
				  sizeof(u32)))
2452
		return;
A
Avi Kivity 已提交
2453 2454 2455 2456

	apic_set_tpr(vcpu->arch.apic, data & 0xff);
}

2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471
/*
 * apic_sync_pv_eoi_to_guest - called before vmentry
 *
 * Detect whether it's safe to enable PV EOI and
 * if yes do so.
 */
static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
					struct kvm_lapic *apic)
{
	if (!pv_eoi_enabled(vcpu) ||
	    /* IRR set or many bits in ISR: could be nested. */
	    apic->irr_pending ||
	    /* Cache not set: could be safe but we don't bother. */
	    apic->highest_isr_cache == -1 ||
	    /* Need EOI to update ioapic. */
2472
	    kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
2473 2474 2475 2476 2477 2478 2479 2480 2481 2482
		/*
		 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
		 * so we need not do anything here.
		 */
		return;
	}

	pv_eoi_set_pending(apic->vcpu);
}

A
Avi Kivity 已提交
2483 2484 2485 2486
void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
{
	u32 data, tpr;
	int max_irr, max_isr;
2487
	struct kvm_lapic *apic = vcpu->arch.apic;
A
Avi Kivity 已提交
2488

2489 2490
	apic_sync_pv_eoi_to_guest(vcpu, apic);

2491
	if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
A
Avi Kivity 已提交
2492 2493
		return;

2494
	tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
A
Avi Kivity 已提交
2495 2496 2497 2498 2499 2500 2501 2502
	max_irr = apic_find_highest_irr(apic);
	if (max_irr < 0)
		max_irr = 0;
	max_isr = apic_find_highest_isr(apic);
	if (max_isr < 0)
		max_isr = 0;
	data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);

2503 2504
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
				sizeof(u32));
A
Avi Kivity 已提交
2505 2506
}

2507
int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
A
Avi Kivity 已提交
2508
{
2509
	if (vapic_addr) {
2510
		if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2511 2512 2513
					&vcpu->arch.apic->vapic_cache,
					vapic_addr, sizeof(u32)))
			return -EINVAL;
2514
		__set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2515
	} else {
2516
		__clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2517 2518 2519 2520
	}

	vcpu->arch.apic->vapic_addr = vapic_addr;
	return 0;
A
Avi Kivity 已提交
2521
}
G
Gleb Natapov 已提交
2522 2523 2524 2525 2526 2527

int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u32 reg = (msr - APIC_BASE_MSR) << 4;

2528
	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
G
Gleb Natapov 已提交
2529 2530
		return 1;

2531 2532 2533
	if (reg == APIC_ICR2)
		return 1;

G
Gleb Natapov 已提交
2534
	/* if this is ICR write vector before command */
2535
	if (reg == APIC_ICR)
2536 2537
		kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
	return kvm_lapic_reg_write(apic, reg, (u32)data);
G
Gleb Natapov 已提交
2538 2539 2540 2541 2542 2543 2544
}

int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;

2545
	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
G
Gleb Natapov 已提交
2546 2547
		return 1;

2548 2549 2550 2551 2552 2553
	if (reg == APIC_DFR || reg == APIC_ICR2) {
		apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n",
			   reg);
		return 1;
	}

2554
	if (kvm_lapic_reg_read(apic, reg, 4, &low))
G
Gleb Natapov 已提交
2555
		return 1;
2556
	if (reg == APIC_ICR)
2557
		kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
G
Gleb Natapov 已提交
2558 2559 2560 2561 2562

	*data = (((u64)high) << 32) | low;

	return 0;
}
G
Gleb Natapov 已提交
2563 2564 2565 2566 2567

int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

2568
	if (!lapic_in_kernel(vcpu))
G
Gleb Natapov 已提交
2569 2570 2571 2572
		return 1;

	/* if this is ICR write vector before command */
	if (reg == APIC_ICR)
2573 2574
		kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
	return kvm_lapic_reg_write(apic, reg, (u32)data);
G
Gleb Natapov 已提交
2575 2576 2577 2578 2579 2580 2581
}

int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u32 low, high = 0;

2582
	if (!lapic_in_kernel(vcpu))
G
Gleb Natapov 已提交
2583 2584
		return 1;

2585
	if (kvm_lapic_reg_read(apic, reg, 4, &low))
G
Gleb Natapov 已提交
2586 2587
		return 1;
	if (reg == APIC_ICR)
2588
		kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
G
Gleb Natapov 已提交
2589 2590 2591 2592 2593

	*data = (((u64)high) << 32) | low;

	return 0;
}
2594 2595 2596 2597 2598 2599 2600 2601 2602 2603

int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data)
{
	u64 addr = data & ~KVM_MSR_ENABLED;
	if (!IS_ALIGNED(addr, 4))
		return 1;

	vcpu->arch.pv_eoi.msr_val = data;
	if (!pv_eoi_enabled(vcpu))
		return 0;
2604
	return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data,
2605
					 addr, sizeof(u8));
2606
}
2607

2608 2609 2610
void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
2611
	u8 sipi_vector;
2612
	unsigned long pe;
2613

2614
	if (!lapic_in_kernel(vcpu) || !apic->pending_events)
2615 2616
		return;

2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627
	/*
	 * INITs are latched while in SMM.  Because an SMM CPU cannot
	 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs
	 * and delay processing of INIT until the next RSM.
	 */
	if (is_smm(vcpu)) {
		WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
		if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
			clear_bit(KVM_APIC_SIPI, &apic->pending_events);
		return;
	}
2628

2629
	pe = xchg(&apic->pending_events, 0);
2630
	if (test_bit(KVM_APIC_INIT, &pe)) {
2631
		kvm_vcpu_reset(vcpu, true);
2632 2633 2634 2635 2636
		if (kvm_vcpu_is_bsp(apic->vcpu))
			vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
		else
			vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
	}
2637
	if (test_bit(KVM_APIC_SIPI, &pe) &&
2638 2639 2640 2641
	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
		/* evaluate pending_events before reading the vector */
		smp_rmb();
		sipi_vector = apic->sipi_vector;
N
Nadav Amit 已提交
2642
		apic_debug("vcpu %d received sipi with vector # %x\n",
2643 2644 2645 2646 2647 2648
			 vcpu->vcpu_id, sipi_vector);
		kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
	}
}

2649 2650 2651 2652
void kvm_lapic_init(void)
{
	/* do not patch jump label more than once per second */
	jump_label_rate_limit(&apic_hw_disabled, HZ);
2653
	jump_label_rate_limit(&apic_sw_disabled, HZ);
2654
}
2655 2656 2657 2658 2659 2660

void kvm_lapic_exit(void)
{
	static_key_deferred_flush(&apic_hw_disabled);
	static_key_deferred_flush(&apic_sw_disabled);
}