lapic.c 38.8 KB
Newer Older
E
Eddie Dong 已提交
1 2 3 4 5 6 7

/*
 * Local APIC virtualization
 *
 * Copyright (C) 2006 Qumranet, Inc.
 * Copyright (C) 2007 Novell
 * Copyright (C) 2007 Intel
N
Nicolas Kaiser 已提交
8
 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
E
Eddie Dong 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *
 * Authors:
 *   Dor Laor <dor.laor@qumranet.com>
 *   Gregory Haskins <ghaskins@novell.com>
 *   Yaozu (Eddie) Dong <eddie.dong@intel.com>
 *
 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 */

21
#include <linux/kvm_host.h>
E
Eddie Dong 已提交
22 23 24 25 26 27 28
#include <linux/kvm.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/smp.h>
#include <linux/hrtimer.h>
#include <linux/io.h>
#include <linux/module.h>
R
Roman Zippel 已提交
29
#include <linux/math64.h>
30
#include <linux/slab.h>
E
Eddie Dong 已提交
31 32 33 34 35
#include <asm/processor.h>
#include <asm/msr.h>
#include <asm/page.h>
#include <asm/current.h>
#include <asm/apicdef.h>
A
Arun Sharma 已提交
36
#include <linux/atomic.h>
37
#include "kvm_cache_regs.h"
E
Eddie Dong 已提交
38
#include "irq.h"
39
#include "trace.h"
40
#include "x86.h"
A
Avi Kivity 已提交
41
#include "cpuid.h"
E
Eddie Dong 已提交
42

43 44 45 46 47 48
#ifndef CONFIG_X86_64
#define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
#else
#define mod_64(x, y) ((x) % (y))
#endif

E
Eddie Dong 已提交
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
#define PRId64 "d"
#define PRIx64 "llx"
#define PRIu64 "u"
#define PRIo64 "o"

#define APIC_BUS_CYCLE_NS 1

/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
#define apic_debug(fmt, arg...)

#define APIC_LVT_NUM			6
/* 14 is the version for Xeon and Pentium 8.4.8*/
#define APIC_VERSION			(0x14UL | ((APIC_LVT_NUM - 1) << 16))
#define LAPIC_MMIO_LENGTH		(1 << 12)
/* followed define is not in apicdef.h */
#define APIC_SHORT_MASK			0xc0000
#define APIC_DEST_NOSHORT		0x0
#define APIC_DEST_MASK			0x800
#define MAX_APIC_VECTOR			256

#define VEC_POS(v) ((v) & (32 - 1))
#define REG_POS(v) (((v) >> 5) << 4)
71

72 73 74
static unsigned int min_timer_period_us = 500;
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

E
Eddie Dong 已提交
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
static inline u32 apic_get_reg(struct kvm_lapic *apic, int reg_off)
{
	return *((u32 *) (apic->regs + reg_off));
}

static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val)
{
	*((u32 *) (apic->regs + reg_off)) = val;
}

static inline int apic_test_and_set_vector(int vec, void *bitmap)
{
	return test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

static inline int apic_test_and_clear_vector(int vec, void *bitmap)
{
	return test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

M
Michael S. Tsirkin 已提交
95 96 97 98 99
static inline int apic_test_vector(int vec, void *bitmap)
{
	return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

E
Eddie Dong 已提交
100 101 102 103 104 105 106 107 108 109
static inline void apic_set_vector(int vec, void *bitmap)
{
	set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

static inline void apic_clear_vector(int vec, void *bitmap)
{
	clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

M
Michael S. Tsirkin 已提交
110 111 112 113 114 115 116 117 118 119
static inline int __apic_test_and_set_vector(int vec, void *bitmap)
{
	return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
{
	return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

E
Eddie Dong 已提交
120 121
static inline int apic_hw_enabled(struct kvm_lapic *apic)
{
122
	return (apic)->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE;
E
Eddie Dong 已提交
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
}

static inline int  apic_sw_enabled(struct kvm_lapic *apic)
{
	return apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED;
}

static inline int apic_enabled(struct kvm_lapic *apic)
{
	return apic_sw_enabled(apic) &&	apic_hw_enabled(apic);
}

#define LVT_MASK	\
	(APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)

#define LINT_MASK	\
	(LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
	 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)

static inline int kvm_apic_id(struct kvm_lapic *apic)
{
	return (apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
}

static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
{
	return !(apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
}

static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
{
	return apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
}

157 158 159 160 161 162
static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
{
	return ((apic_get_reg(apic, APIC_LVTT) &
		apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_ONESHOT);
}

E
Eddie Dong 已提交
163 164
static inline int apic_lvtt_period(struct kvm_lapic *apic)
{
165 166 167 168 169 170 171 172 173
	return ((apic_get_reg(apic, APIC_LVTT) &
		apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_PERIODIC);
}

static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
{
	return ((apic_get_reg(apic, APIC_LVTT) &
		apic->lapic_timer.timer_mode_mask) ==
			APIC_LVT_TIMER_TSCDEADLINE);
E
Eddie Dong 已提交
174 175
}

176 177 178 179 180
static inline int apic_lvt_nmi_mode(u32 lvt_val)
{
	return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
}

181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
void kvm_apic_set_version(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	struct kvm_cpuid_entry2 *feat;
	u32 v = APIC_VERSION;

	if (!irqchip_in_kernel(vcpu->kvm))
		return;

	feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
	if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))))
		v |= APIC_LVR_DIRECTED_EOI;
	apic_set_reg(apic, APIC_LVR, v);
}

G
Gleb Natapov 已提交
196 197 198 199 200
static inline int apic_x2apic_mode(struct kvm_lapic *apic)
{
	return apic->vcpu->arch.apic_base & X2APIC_ENABLE;
}

E
Eddie Dong 已提交
201
static unsigned int apic_lvt_mask[APIC_LVT_NUM] = {
202
	LVT_MASK ,      /* part LVTT mask, timer mode mask added at runtime */
E
Eddie Dong 已提交
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
	LVT_MASK | APIC_MODE_MASK,	/* LVTTHMR */
	LVT_MASK | APIC_MODE_MASK,	/* LVTPC */
	LINT_MASK, LINT_MASK,	/* LVT0-1 */
	LVT_MASK		/* LVTERR */
};

static int find_highest_vector(void *bitmap)
{
	u32 *word = bitmap;
	int word_offset = MAX_APIC_VECTOR >> 5;

	while ((word_offset != 0) && (word[(--word_offset) << 2] == 0))
		continue;

	if (likely(!word_offset && !word[0]))
		return -1;
	else
		return fls(word[word_offset << 2]) - 1 + (word_offset << 5);
}

M
Michael S. Tsirkin 已提交
223 224 225 226 227 228 229 230 231 232
static u8 count_vectors(void *bitmap)
{
	u32 *word = bitmap;
	int word_offset;
	u8 count = 0;
	for (word_offset = 0; word_offset < MAX_APIC_VECTOR >> 5; ++word_offset)
		count += hweight32(word[word_offset << 2]);
	return count;
}

E
Eddie Dong 已提交
233 234
static inline int apic_test_and_set_irr(int vec, struct kvm_lapic *apic)
{
235
	apic->irr_pending = true;
E
Eddie Dong 已提交
236 237 238
	return apic_test_and_set_vector(vec, apic->regs + APIC_IRR);
}

239
static inline int apic_search_irr(struct kvm_lapic *apic)
E
Eddie Dong 已提交
240
{
241
	return find_highest_vector(apic->regs + APIC_IRR);
E
Eddie Dong 已提交
242 243 244 245 246 247
}

static inline int apic_find_highest_irr(struct kvm_lapic *apic)
{
	int result;

248 249 250 251
	if (!apic->irr_pending)
		return -1;

	result = apic_search_irr(apic);
E
Eddie Dong 已提交
252 253 254 255 256
	ASSERT(result == -1 || result >= 16);

	return result;
}

257 258 259 260 261 262 263 264
static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
{
	apic->irr_pending = false;
	apic_clear_vector(vec, apic->regs + APIC_IRR);
	if (apic_search_irr(apic) != -1)
		apic->irr_pending = true;
}

M
Michael S. Tsirkin 已提交
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
{
	if (!__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
		++apic->isr_count;
	BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
	/*
	 * ISR (in service register) bit is set when injecting an interrupt.
	 * The highest vector is injected. Thus the latest bit set matches
	 * the highest bit in ISR.
	 */
	apic->highest_isr_cache = vec;
}

static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
{
	if (__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
		--apic->isr_count;
	BUG_ON(apic->isr_count < 0);
	apic->highest_isr_cache = -1;
}

286 287
int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
{
288
	struct kvm_lapic *apic = vcpu->arch.apic;
289 290
	int highest_irr;

291 292 293 294 295
	/* This may race with setting of irr in __apic_accept_irq() and
	 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
	 * will cause vmexit immediately and the value will be recalculated
	 * on the next vmentry.
	 */
296 297 298 299 300 301 302
	if (!apic)
		return 0;
	highest_irr = apic_find_highest_irr(apic);

	return highest_irr;
}

303 304 305
static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
			     int vector, int level, int trig_mode);

306
int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq)
E
Eddie Dong 已提交
307
{
308
	struct kvm_lapic *apic = vcpu->arch.apic;
309

310 311
	return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
			irq->level, irq->trig_mode);
E
Eddie Dong 已提交
312 313
}

314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
{

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
				      sizeof(val));
}

static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
{

	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
				      sizeof(*val));
}

static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
{
	return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
}

static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
{
	u8 val;
	if (pv_eoi_get_user(vcpu, &val) < 0)
		apic_debug("Can't read EOI MSR value: 0x%llx\n",
			   (unsigned long long)vcpi->arch.pv_eoi.msr_val);
	return val & 0x1;
}

static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
{
	if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
		apic_debug("Can't set EOI MSR value: 0x%llx\n",
			   (unsigned long long)vcpi->arch.pv_eoi.msr_val);
		return;
	}
	__set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
}

static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
{
	if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
		apic_debug("Can't clear EOI MSR value: 0x%llx\n",
			   (unsigned long long)vcpi->arch.pv_eoi.msr_val);
		return;
	}
	__clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
}

E
Eddie Dong 已提交
362 363 364
static inline int apic_find_highest_isr(struct kvm_lapic *apic)
{
	int result;
M
Michael S. Tsirkin 已提交
365 366 367 368
	if (!apic->isr_count)
		return -1;
	if (likely(apic->highest_isr_cache != -1))
		return apic->highest_isr_cache;
E
Eddie Dong 已提交
369 370 371 372 373 374 375 376 377

	result = find_highest_vector(apic->regs + APIC_ISR);
	ASSERT(result == -1 || result >= 16);

	return result;
}

static void apic_update_ppr(struct kvm_lapic *apic)
{
378
	u32 tpr, isrv, ppr, old_ppr;
E
Eddie Dong 已提交
379 380
	int isr;

381
	old_ppr = apic_get_reg(apic, APIC_PROCPRI);
E
Eddie Dong 已提交
382 383 384 385 386 387 388 389 390 391 392 393
	tpr = apic_get_reg(apic, APIC_TASKPRI);
	isr = apic_find_highest_isr(apic);
	isrv = (isr != -1) ? isr : 0;

	if ((tpr & 0xf0) >= (isrv & 0xf0))
		ppr = tpr & 0xff;
	else
		ppr = isrv & 0xf0;

	apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x",
		   apic, ppr, isr, isrv);

394 395
	if (old_ppr != ppr) {
		apic_set_reg(apic, APIC_PROCPRI, ppr);
396 397
		if (ppr < old_ppr)
			kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
398
	}
E
Eddie Dong 已提交
399 400 401 402 403 404 405 406 407 408
}

static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
{
	apic_set_reg(apic, APIC_TASKPRI, tpr);
	apic_update_ppr(apic);
}

int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest)
{
409
	return dest == 0xff || kvm_apic_id(apic) == dest;
E
Eddie Dong 已提交
410 411 412 413 414
}

int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
{
	int result = 0;
G
Gleb Natapov 已提交
415 416 417 418 419 420
	u32 logical_id;

	if (apic_x2apic_mode(apic)) {
		logical_id = apic_get_reg(apic, APIC_LDR);
		return logical_id & mda;
	}
E
Eddie Dong 已提交
421 422 423 424 425 426 427 428 429 430 431 432 433 434

	logical_id = GET_APIC_LOGICAL_ID(apic_get_reg(apic, APIC_LDR));

	switch (apic_get_reg(apic, APIC_DFR)) {
	case APIC_DFR_FLAT:
		if (logical_id & mda)
			result = 1;
		break;
	case APIC_DFR_CLUSTER:
		if (((logical_id >> 4) == (mda >> 0x4))
		    && (logical_id & mda & 0xf))
			result = 1;
		break;
	default:
435 436
		apic_debug("Bad DFR vcpu %d: %08x\n",
			   apic->vcpu->vcpu_id, apic_get_reg(apic, APIC_DFR));
E
Eddie Dong 已提交
437 438 439 440 441 442
		break;
	}

	return result;
}

443
int kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
E
Eddie Dong 已提交
444 445 446
			   int short_hand, int dest, int dest_mode)
{
	int result = 0;
447
	struct kvm_lapic *target = vcpu->arch.apic;
E
Eddie Dong 已提交
448 449

	apic_debug("target %p, source %p, dest 0x%x, "
450
		   "dest_mode 0x%x, short_hand 0x%x\n",
E
Eddie Dong 已提交
451 452
		   target, source, dest, dest_mode, short_hand);

Z
Zachary Amsden 已提交
453
	ASSERT(target);
E
Eddie Dong 已提交
454 455
	switch (short_hand) {
	case APIC_DEST_NOSHORT:
456
		if (dest_mode == 0)
E
Eddie Dong 已提交
457
			/* Physical mode. */
458 459
			result = kvm_apic_match_physical_addr(target, dest);
		else
E
Eddie Dong 已提交
460 461 462 463
			/* Logical mode. */
			result = kvm_apic_match_logical_addr(target, dest);
		break;
	case APIC_DEST_SELF:
464
		result = (target == source);
E
Eddie Dong 已提交
465 466 467 468 469
		break;
	case APIC_DEST_ALLINC:
		result = 1;
		break;
	case APIC_DEST_ALLBUT:
470
		result = (target != source);
E
Eddie Dong 已提交
471 472
		break;
	default:
473 474
		apic_debug("kvm: apic: Bad dest shorthand value %x\n",
			   short_hand);
E
Eddie Dong 已提交
475 476 477 478 479 480 481 482 483 484 485 486 487
		break;
	}

	return result;
}

/*
 * Add a pending IRQ into lapic.
 * Return 1 if successfully added and 0 if discarded.
 */
static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
			     int vector, int level, int trig_mode)
{
488
	int result = 0;
489
	struct kvm_vcpu *vcpu = apic->vcpu;
E
Eddie Dong 已提交
490 491 492

	switch (delivery_mode) {
	case APIC_DM_LOWEST:
493 494
		vcpu->arch.apic_arb_prio++;
	case APIC_DM_FIXED:
E
Eddie Dong 已提交
495 496 497 498
		/* FIXME add logic for vcpu on reset */
		if (unlikely(!apic_enabled(apic)))
			break;

499 500 501 502 503 504
		if (trig_mode) {
			apic_debug("level trig mode for vector %d", vector);
			apic_set_vector(vector, apic->regs + APIC_TMR);
		} else
			apic_clear_vector(vector, apic->regs + APIC_TMR);

505
		result = !apic_test_and_set_irr(vector, apic);
506
		trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
507
					  trig_mode, vector, !result);
508 509 510 511
		if (!result) {
			if (trig_mode)
				apic_debug("level trig mode repeatedly for "
						"vector %d", vector);
E
Eddie Dong 已提交
512 513 514
			break;
		}

515
		kvm_make_request(KVM_REQ_EVENT, vcpu);
516
		kvm_vcpu_kick(vcpu);
E
Eddie Dong 已提交
517 518 519
		break;

	case APIC_DM_REMRD:
520
		apic_debug("Ignoring delivery mode 3\n");
E
Eddie Dong 已提交
521 522 523
		break;

	case APIC_DM_SMI:
524
		apic_debug("Ignoring guest SMI\n");
E
Eddie Dong 已提交
525
		break;
526

E
Eddie Dong 已提交
527
	case APIC_DM_NMI:
528
		result = 1;
529
		kvm_inject_nmi(vcpu);
J
Jan Kiszka 已提交
530
		kvm_vcpu_kick(vcpu);
E
Eddie Dong 已提交
531 532 533
		break;

	case APIC_DM_INIT:
534
		if (!trig_mode || level) {
535
			result = 1;
536
			vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
537
			kvm_make_request(KVM_REQ_EVENT, vcpu);
538 539
			kvm_vcpu_kick(vcpu);
		} else {
540 541
			apic_debug("Ignoring de-assert INIT to vcpu %d\n",
				   vcpu->vcpu_id);
542
		}
E
Eddie Dong 已提交
543 544 545
		break;

	case APIC_DM_STARTUP:
546 547
		apic_debug("SIPI to vcpu %d vector 0x%02x\n",
			   vcpu->vcpu_id, vector);
548
		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
549
			result = 1;
550
			vcpu->arch.sipi_vector = vector;
551
			vcpu->arch.mp_state = KVM_MP_STATE_SIPI_RECEIVED;
552
			kvm_make_request(KVM_REQ_EVENT, vcpu);
553
			kvm_vcpu_kick(vcpu);
554
		}
E
Eddie Dong 已提交
555 556
		break;

557 558 559 560 561 562 563 564
	case APIC_DM_EXTINT:
		/*
		 * Should only be called by kvm_apic_local_deliver() with LVT0,
		 * before NMI watchdog was enabled. Already handled by
		 * kvm_apic_accept_pic_intr().
		 */
		break;

E
Eddie Dong 已提交
565 566 567 568 569 570 571 572
	default:
		printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
		       delivery_mode);
		break;
	}
	return result;
}

573
int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
574
{
575
	return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
576 577
}

578
static int apic_set_eoi(struct kvm_lapic *apic)
E
Eddie Dong 已提交
579 580
{
	int vector = apic_find_highest_isr(apic);
581 582 583

	trace_kvm_eoi(apic, vector);

E
Eddie Dong 已提交
584 585 586 587 588
	/*
	 * Not every write EOI will has corresponding ISR,
	 * one example is when Kernel check timer on setup_IO_APIC
	 */
	if (vector == -1)
589
		return vector;
E
Eddie Dong 已提交
590

M
Michael S. Tsirkin 已提交
591
	apic_clear_isr(vector, apic);
E
Eddie Dong 已提交
592 593
	apic_update_ppr(apic);

M
Michael S. Tsirkin 已提交
594 595 596 597 598 599 600
	if (!(apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) &&
	    kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) {
		int trigger_mode;
		if (apic_test_vector(vector, apic->regs + APIC_TMR))
			trigger_mode = IOAPIC_LEVEL_TRIG;
		else
			trigger_mode = IOAPIC_EDGE_TRIG;
601
		kvm_ioapic_update_eoi(apic->vcpu->kvm, vector, trigger_mode);
M
Michael S. Tsirkin 已提交
602
	}
603
	kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
604
	return vector;
E
Eddie Dong 已提交
605 606 607 608 609 610
}

static void apic_send_ipi(struct kvm_lapic *apic)
{
	u32 icr_low = apic_get_reg(apic, APIC_ICR);
	u32 icr_high = apic_get_reg(apic, APIC_ICR2);
611
	struct kvm_lapic_irq irq;
E
Eddie Dong 已提交
612

613 614 615 616 617 618
	irq.vector = icr_low & APIC_VECTOR_MASK;
	irq.delivery_mode = icr_low & APIC_MODE_MASK;
	irq.dest_mode = icr_low & APIC_DEST_MASK;
	irq.level = icr_low & APIC_INT_ASSERT;
	irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
	irq.shorthand = icr_low & APIC_SHORT_MASK;
G
Gleb Natapov 已提交
619 620 621 622
	if (apic_x2apic_mode(apic))
		irq.dest_id = icr_high;
	else
		irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
E
Eddie Dong 已提交
623

624 625
	trace_kvm_apic_ipi(icr_low, irq.dest_id);

E
Eddie Dong 已提交
626 627 628
	apic_debug("icr_high 0x%x, icr_low 0x%x, "
		   "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
		   "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x\n",
G
Glauber Costa 已提交
629
		   icr_high, icr_low, irq.shorthand, irq.dest_id,
630 631 632 633
		   irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode,
		   irq.vector);

	kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq);
E
Eddie Dong 已提交
634 635 636 637
}

static u32 apic_get_tmcct(struct kvm_lapic *apic)
{
638 639
	ktime_t remaining;
	s64 ns;
640
	u32 tmcct;
E
Eddie Dong 已提交
641 642 643

	ASSERT(apic != NULL);

644
	/* if initial count is 0, current count should also be 0 */
645
	if (apic_get_reg(apic, APIC_TMICT) == 0)
646 647
		return 0;

648
	remaining = hrtimer_get_remaining(&apic->lapic_timer.timer);
649 650 651
	if (ktime_to_ns(remaining) < 0)
		remaining = ktime_set(0, 0);

652 653 654
	ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
	tmcct = div64_u64(ns,
			 (APIC_BUS_CYCLE_NS * apic->divide_count));
E
Eddie Dong 已提交
655 656 657 658

	return tmcct;
}

659 660 661 662 663
static void __report_tpr_access(struct kvm_lapic *apic, bool write)
{
	struct kvm_vcpu *vcpu = apic->vcpu;
	struct kvm_run *run = vcpu->run;

664
	kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
665
	run->tpr_access.rip = kvm_rip_read(vcpu);
666 667 668 669 670 671 672 673 674
	run->tpr_access.is_write = write;
}

static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
{
	if (apic->vcpu->arch.tpr_access_reporting)
		__report_tpr_access(apic, write);
}

E
Eddie Dong 已提交
675 676 677 678 679 680 681 682
static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
{
	u32 val = 0;

	if (offset >= LAPIC_MMIO_LENGTH)
		return 0;

	switch (offset) {
G
Gleb Natapov 已提交
683 684 685 686 687 688
	case APIC_ID:
		if (apic_x2apic_mode(apic))
			val = kvm_apic_id(apic);
		else
			val = kvm_apic_id(apic) << 24;
		break;
E
Eddie Dong 已提交
689
	case APIC_ARBPRI:
690
		apic_debug("Access APIC ARBPRI register which is for P6\n");
E
Eddie Dong 已提交
691 692 693
		break;

	case APIC_TMCCT:	/* Timer CCR */
694 695 696
		if (apic_lvtt_tscdeadline(apic))
			return 0;

E
Eddie Dong 已提交
697 698
		val = apic_get_tmcct(apic);
		break;
699 700 701 702
	case APIC_PROCPRI:
		apic_update_ppr(apic);
		val = apic_get_reg(apic, offset);
		break;
703 704 705
	case APIC_TASKPRI:
		report_tpr_access(apic, false);
		/* fall thru */
E
Eddie Dong 已提交
706 707 708 709 710 711 712 713
	default:
		val = apic_get_reg(apic, offset);
		break;
	}

	return val;
}

G
Gregory Haskins 已提交
714 715 716 717 718
static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
{
	return container_of(dev, struct kvm_lapic, dev);
}

G
Gleb Natapov 已提交
719 720
static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
		void *data)
E
Eddie Dong 已提交
721 722 723
{
	unsigned char alignment = offset & 0xf;
	u32 result;
G
Guo Chao 已提交
724
	/* this bitmask has a bit cleared for each reserved register */
G
Gleb Natapov 已提交
725
	static const u64 rmask = 0x43ff01ffffffe70cULL;
E
Eddie Dong 已提交
726 727

	if ((alignment + len) > 4) {
728 729
		apic_debug("KVM_APIC_READ: alignment error %x %d\n",
			   offset, len);
G
Gleb Natapov 已提交
730
		return 1;
E
Eddie Dong 已提交
731
	}
G
Gleb Natapov 已提交
732 733

	if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) {
734 735
		apic_debug("KVM_APIC_READ: read reserved register %x\n",
			   offset);
G
Gleb Natapov 已提交
736 737 738
		return 1;
	}

E
Eddie Dong 已提交
739 740
	result = __apic_read(apic, offset & ~0xf);

741 742
	trace_kvm_apic_read(offset, result);

E
Eddie Dong 已提交
743 744 745 746 747 748 749 750 751 752 753
	switch (len) {
	case 1:
	case 2:
	case 4:
		memcpy(data, (char *)&result + alignment, len);
		break;
	default:
		printk(KERN_ERR "Local APIC read with len = %x, "
		       "should be 1,2, or 4 instead\n", len);
		break;
	}
754
	return 0;
E
Eddie Dong 已提交
755 756
}

G
Gleb Natapov 已提交
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
{
	return apic_hw_enabled(apic) &&
	    addr >= apic->base_address &&
	    addr < apic->base_address + LAPIC_MMIO_LENGTH;
}

static int apic_mmio_read(struct kvm_io_device *this,
			   gpa_t address, int len, void *data)
{
	struct kvm_lapic *apic = to_lapic(this);
	u32 offset = address - apic->base_address;

	if (!apic_mmio_in_range(apic, address))
		return -EOPNOTSUPP;

	apic_reg_read(apic, offset, len, data);

	return 0;
}

E
Eddie Dong 已提交
778 779 780 781 782 783 784
static void update_divide_count(struct kvm_lapic *apic)
{
	u32 tmp1, tmp2, tdcr;

	tdcr = apic_get_reg(apic, APIC_TDCR);
	tmp1 = tdcr & 0xf;
	tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
785
	apic->divide_count = 0x1 << (tmp2 & 0x7);
E
Eddie Dong 已提交
786 787

	apic_debug("timer divide count is 0x%x\n",
G
Glauber Costa 已提交
788
				   apic->divide_count);
E
Eddie Dong 已提交
789 790 791 792
}

static void start_apic_timer(struct kvm_lapic *apic)
{
793
	ktime_t now;
794
	atomic_set(&apic->lapic_timer.pending, 0);
795

796
	if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
G
Guo Chao 已提交
797
		/* lapic timer in oneshot or periodic mode */
798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
		now = apic->lapic_timer.timer.base->get_time();
		apic->lapic_timer.period = (u64)apic_get_reg(apic, APIC_TMICT)
			    * APIC_BUS_CYCLE_NS * apic->divide_count;

		if (!apic->lapic_timer.period)
			return;
		/*
		 * Do not allow the guest to program periodic timers with small
		 * interval, since the hrtimers are not throttled by the host
		 * scheduler.
		 */
		if (apic_lvtt_period(apic)) {
			s64 min_period = min_timer_period_us * 1000LL;

			if (apic->lapic_timer.period < min_period) {
				pr_info_ratelimited(
				    "kvm: vcpu %i: requested %lld ns "
				    "lapic timer period limited to %lld ns\n",
				    apic->vcpu->vcpu_id,
				    apic->lapic_timer.period, min_period);
				apic->lapic_timer.period = min_period;
			}
820
		}
821

822 823 824
		hrtimer_start(&apic->lapic_timer.timer,
			      ktime_add_ns(now, apic->lapic_timer.period),
			      HRTIMER_MODE_ABS);
E
Eddie Dong 已提交
825

826
		apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
E
Eddie Dong 已提交
827 828
			   PRIx64 ", "
			   "timer initial count 0x%x, period %lldns, "
829
			   "expire @ 0x%016" PRIx64 ".\n", __func__,
E
Eddie Dong 已提交
830 831
			   APIC_BUS_CYCLE_NS, ktime_to_ns(now),
			   apic_get_reg(apic, APIC_TMICT),
832
			   apic->lapic_timer.period,
E
Eddie Dong 已提交
833
			   ktime_to_ns(ktime_add_ns(now,
834
					apic->lapic_timer.period)));
835 836 837 838 839
	} else if (apic_lvtt_tscdeadline(apic)) {
		/* lapic timer in tsc deadline mode */
		u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline;
		u64 ns = 0;
		struct kvm_vcpu *vcpu = apic->vcpu;
840
		unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858
		unsigned long flags;

		if (unlikely(!tscdeadline || !this_tsc_khz))
			return;

		local_irq_save(flags);

		now = apic->lapic_timer.timer.base->get_time();
		guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu);
		if (likely(tscdeadline > guest_tsc)) {
			ns = (tscdeadline - guest_tsc) * 1000000ULL;
			do_div(ns, this_tsc_khz);
		}
		hrtimer_start(&apic->lapic_timer.timer,
			ktime_add_ns(now, ns), HRTIMER_MODE_ABS);

		local_irq_restore(flags);
	}
E
Eddie Dong 已提交
859 860
}

861 862 863 864 865 866 867 868 869 870 871 872 873 874
static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
{
	int nmi_wd_enabled = apic_lvt_nmi_mode(apic_get_reg(apic, APIC_LVT0));

	if (apic_lvt_nmi_mode(lvt0_val)) {
		if (!nmi_wd_enabled) {
			apic_debug("Receive NMI setting on APIC_LVT0 "
				   "for cpu %d\n", apic->vcpu->vcpu_id);
			apic->vcpu->kvm->arch.vapics_in_nmi_mode++;
		}
	} else if (nmi_wd_enabled)
		apic->vcpu->kvm->arch.vapics_in_nmi_mode--;
}

G
Gleb Natapov 已提交
875
static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
E
Eddie Dong 已提交
876
{
G
Gleb Natapov 已提交
877
	int ret = 0;
E
Eddie Dong 已提交
878

G
Gleb Natapov 已提交
879
	trace_kvm_apic_write(reg, val);
E
Eddie Dong 已提交
880

G
Gleb Natapov 已提交
881
	switch (reg) {
E
Eddie Dong 已提交
882
	case APIC_ID:		/* Local APIC ID */
G
Gleb Natapov 已提交
883 884 885 886
		if (!apic_x2apic_mode(apic))
			apic_set_reg(apic, APIC_ID, val);
		else
			ret = 1;
E
Eddie Dong 已提交
887 888 889
		break;

	case APIC_TASKPRI:
890
		report_tpr_access(apic, true);
E
Eddie Dong 已提交
891 892 893 894 895 896 897 898
		apic_set_tpr(apic, val & 0xff);
		break;

	case APIC_EOI:
		apic_set_eoi(apic);
		break;

	case APIC_LDR:
G
Gleb Natapov 已提交
899 900 901 902
		if (!apic_x2apic_mode(apic))
			apic_set_reg(apic, APIC_LDR, val & APIC_LDR_MASK);
		else
			ret = 1;
E
Eddie Dong 已提交
903 904 905
		break;

	case APIC_DFR:
G
Gleb Natapov 已提交
906 907 908 909
		if (!apic_x2apic_mode(apic))
			apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
		else
			ret = 1;
E
Eddie Dong 已提交
910 911
		break;

912 913 914 915 916
	case APIC_SPIV: {
		u32 mask = 0x3ff;
		if (apic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
			mask |= APIC_SPIV_DIRECTED_EOI;
		apic_set_reg(apic, APIC_SPIV, val & mask);
E
Eddie Dong 已提交
917 918 919 920 921 922 923 924 925 926
		if (!(val & APIC_SPIV_APIC_ENABLED)) {
			int i;
			u32 lvt_val;

			for (i = 0; i < APIC_LVT_NUM; i++) {
				lvt_val = apic_get_reg(apic,
						       APIC_LVTT + 0x10 * i);
				apic_set_reg(apic, APIC_LVTT + 0x10 * i,
					     lvt_val | APIC_LVT_MASKED);
			}
927
			atomic_set(&apic->lapic_timer.pending, 0);
E
Eddie Dong 已提交
928 929 930

		}
		break;
931
	}
E
Eddie Dong 已提交
932 933 934 935 936 937 938
	case APIC_ICR:
		/* No delay here, so we always clear the pending bit */
		apic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
		apic_send_ipi(apic);
		break;

	case APIC_ICR2:
G
Gleb Natapov 已提交
939 940 941
		if (!apic_x2apic_mode(apic))
			val &= 0xff000000;
		apic_set_reg(apic, APIC_ICR2, val);
E
Eddie Dong 已提交
942 943
		break;

944
	case APIC_LVT0:
945
		apic_manage_nmi_watchdog(apic, val);
E
Eddie Dong 已提交
946 947 948 949 950 951 952 953
	case APIC_LVTTHMR:
	case APIC_LVTPC:
	case APIC_LVT1:
	case APIC_LVTERR:
		/* TODO: Check vector */
		if (!apic_sw_enabled(apic))
			val |= APIC_LVT_MASKED;

G
Gleb Natapov 已提交
954 955
		val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
		apic_set_reg(apic, reg, val);
E
Eddie Dong 已提交
956 957 958

		break;

959 960 961 962 963 964 965 966 967 968 969 970
	case APIC_LVTT:
		if ((apic_get_reg(apic, APIC_LVTT) &
		    apic->lapic_timer.timer_mode_mask) !=
		   (val & apic->lapic_timer.timer_mode_mask))
			hrtimer_cancel(&apic->lapic_timer.timer);

		if (!apic_sw_enabled(apic))
			val |= APIC_LVT_MASKED;
		val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
		apic_set_reg(apic, APIC_LVTT, val);
		break;

E
Eddie Dong 已提交
971
	case APIC_TMICT:
972 973 974
		if (apic_lvtt_tscdeadline(apic))
			break;

975
		hrtimer_cancel(&apic->lapic_timer.timer);
E
Eddie Dong 已提交
976 977
		apic_set_reg(apic, APIC_TMICT, val);
		start_apic_timer(apic);
G
Gleb Natapov 已提交
978
		break;
E
Eddie Dong 已提交
979 980 981

	case APIC_TDCR:
		if (val & 4)
982
			apic_debug("KVM_WRITE:TDCR %x\n", val);
E
Eddie Dong 已提交
983 984 985 986
		apic_set_reg(apic, APIC_TDCR, val);
		update_divide_count(apic);
		break;

G
Gleb Natapov 已提交
987 988
	case APIC_ESR:
		if (apic_x2apic_mode(apic) && val != 0) {
989
			apic_debug("KVM_WRITE:ESR not zero %x\n", val);
G
Gleb Natapov 已提交
990 991 992 993 994 995 996 997 998 999
			ret = 1;
		}
		break;

	case APIC_SELF_IPI:
		if (apic_x2apic_mode(apic)) {
			apic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
		} else
			ret = 1;
		break;
E
Eddie Dong 已提交
1000
	default:
G
Gleb Natapov 已提交
1001
		ret = 1;
E
Eddie Dong 已提交
1002 1003
		break;
	}
G
Gleb Natapov 已提交
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026
	if (ret)
		apic_debug("Local APIC Write to read-only register %x\n", reg);
	return ret;
}

static int apic_mmio_write(struct kvm_io_device *this,
			    gpa_t address, int len, const void *data)
{
	struct kvm_lapic *apic = to_lapic(this);
	unsigned int offset = address - apic->base_address;
	u32 val;

	if (!apic_mmio_in_range(apic, address))
		return -EOPNOTSUPP;

	/*
	 * APIC register must be aligned on 128-bits boundary.
	 * 32/64/128 bits registers must be accessed thru 32 bits.
	 * Refer SDM 8.4.1
	 */
	if (len != 4 || (offset & 0xf)) {
		/* Don't shout loud, $infamous_os would cause only noise. */
		apic_debug("apic write: bad size=%d %lx\n", len, (long)address);
1027
		return 0;
G
Gleb Natapov 已提交
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
	}

	val = *(u32*)data;

	/* too common printing */
	if (offset != APIC_EOI)
		apic_debug("%s: offset 0x%x with length 0x%x, and value is "
			   "0x%x\n", __func__, offset, len, val);

	apic_reg_write(apic, offset & 0xff0, val);

1039
	return 0;
E
Eddie Dong 已提交
1040 1041
}

1042 1043 1044 1045 1046 1047 1048 1049 1050
void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

	if (apic)
		apic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
}
EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);

1051
void kvm_free_lapic(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
1052
{
1053
	if (!vcpu->arch.apic)
E
Eddie Dong 已提交
1054 1055
		return;

1056
	hrtimer_cancel(&vcpu->arch.apic->lapic_timer.timer);
E
Eddie Dong 已提交
1057

1058 1059
	if (vcpu->arch.apic->regs)
		free_page((unsigned long)vcpu->arch.apic->regs);
E
Eddie Dong 已提交
1060

1061
	kfree(vcpu->arch.apic);
E
Eddie Dong 已提交
1062 1063 1064 1065 1066 1067 1068 1069
}

/*
 *----------------------------------------------------------------------
 * LAPIC interface
 *----------------------------------------------------------------------
 */

1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	if (!apic)
		return 0;

	if (apic_lvtt_oneshot(apic) || apic_lvtt_period(apic))
		return 0;

	return apic->lapic_timer.tscdeadline;
}

void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	if (!apic)
		return;

	if (apic_lvtt_oneshot(apic) || apic_lvtt_period(apic))
		return;

	hrtimer_cancel(&apic->lapic_timer.timer);
	apic->lapic_timer.tscdeadline = data;
	start_apic_timer(apic);
}

E
Eddie Dong 已提交
1096 1097
void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
{
1098
	struct kvm_lapic *apic = vcpu->arch.apic;
E
Eddie Dong 已提交
1099 1100 1101

	if (!apic)
		return;
A
Avi Kivity 已提交
1102 1103
	apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
		     | (apic_get_reg(apic, APIC_TASKPRI) & 4));
E
Eddie Dong 已提交
1104 1105 1106 1107
}

u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
{
1108
	struct kvm_lapic *apic = vcpu->arch.apic;
E
Eddie Dong 已提交
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
	u64 tpr;

	if (!apic)
		return 0;
	tpr = (u64) apic_get_reg(apic, APIC_TASKPRI);

	return (tpr & 0xf0) >> 4;
}

void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
{
1120
	struct kvm_lapic *apic = vcpu->arch.apic;
E
Eddie Dong 已提交
1121 1122 1123

	if (!apic) {
		value |= MSR_IA32_APICBASE_BSP;
1124
		vcpu->arch.apic_base = value;
E
Eddie Dong 已提交
1125 1126
		return;
	}
1127 1128

	if (!kvm_vcpu_is_bsp(apic->vcpu))
E
Eddie Dong 已提交
1129 1130
		value &= ~MSR_IA32_APICBASE_BSP;

1131
	vcpu->arch.apic_base = value;
G
Gleb Natapov 已提交
1132 1133 1134 1135 1136
	if (apic_x2apic_mode(apic)) {
		u32 id = kvm_apic_id(apic);
		u32 ldr = ((id & ~0xf) << 16) | (1 << (id & 0xf));
		apic_set_reg(apic, APIC_LDR, ldr);
	}
1137
	apic->base_address = apic->vcpu->arch.apic_base &
E
Eddie Dong 已提交
1138 1139 1140 1141
			     MSR_IA32_APICBASE_BASE;

	/* with FSB delivery interrupt, we can restart APIC functionality */
	apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is "
1142
		   "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address);
E
Eddie Dong 已提交
1143 1144 1145

}

1146
void kvm_lapic_reset(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
1147 1148 1149 1150
{
	struct kvm_lapic *apic;
	int i;

1151
	apic_debug("%s\n", __func__);
E
Eddie Dong 已提交
1152 1153

	ASSERT(vcpu);
1154
	apic = vcpu->arch.apic;
E
Eddie Dong 已提交
1155 1156 1157
	ASSERT(apic != NULL);

	/* Stop the timer in case it's a reset to an active apic */
1158
	hrtimer_cancel(&apic->lapic_timer.timer);
E
Eddie Dong 已提交
1159 1160

	apic_set_reg(apic, APIC_ID, vcpu->vcpu_id << 24);
1161
	kvm_apic_set_version(apic->vcpu);
E
Eddie Dong 已提交
1162 1163 1164

	for (i = 0; i < APIC_LVT_NUM; i++)
		apic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
Q
Qing He 已提交
1165 1166
	apic_set_reg(apic, APIC_LVT0,
		     SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
E
Eddie Dong 已提交
1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181

	apic_set_reg(apic, APIC_DFR, 0xffffffffU);
	apic_set_reg(apic, APIC_SPIV, 0xff);
	apic_set_reg(apic, APIC_TASKPRI, 0);
	apic_set_reg(apic, APIC_LDR, 0);
	apic_set_reg(apic, APIC_ESR, 0);
	apic_set_reg(apic, APIC_ICR, 0);
	apic_set_reg(apic, APIC_ICR2, 0);
	apic_set_reg(apic, APIC_TDCR, 0);
	apic_set_reg(apic, APIC_TMICT, 0);
	for (i = 0; i < 8; i++) {
		apic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
		apic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
		apic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
	}
1182
	apic->irr_pending = false;
M
Michael S. Tsirkin 已提交
1183 1184
	apic->isr_count = 0;
	apic->highest_isr_cache = -1;
1185
	update_divide_count(apic);
1186
	atomic_set(&apic->lapic_timer.pending, 0);
1187
	if (kvm_vcpu_is_bsp(vcpu))
1188 1189
		kvm_lapic_set_base(vcpu,
				vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
1190
	vcpu->arch.pv_eoi.msr_val = 0;
E
Eddie Dong 已提交
1191 1192
	apic_update_ppr(apic);

1193
	vcpu->arch.apic_arb_prio = 0;
1194
	vcpu->arch.apic_attention = 0;
1195

E
Eddie Dong 已提交
1196
	apic_debug(KERN_INFO "%s: vcpu=%p, id=%d, base_msr="
1197
		   "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__,
E
Eddie Dong 已提交
1198
		   vcpu, kvm_apic_id(apic),
1199
		   vcpu->arch.apic_base, apic->base_address);
E
Eddie Dong 已提交
1200 1201
}

1202
bool kvm_apic_present(struct kvm_vcpu *vcpu)
E
Eddie Dong 已提交
1203
{
1204 1205
	return vcpu->arch.apic && apic_hw_enabled(vcpu->arch.apic);
}
E
Eddie Dong 已提交
1206

1207 1208 1209
int kvm_lapic_enabled(struct kvm_vcpu *vcpu)
{
	return kvm_apic_present(vcpu) && apic_sw_enabled(vcpu->arch.apic);
E
Eddie Dong 已提交
1210 1211 1212 1213 1214 1215 1216
}

/*
 *----------------------------------------------------------------------
 * timer interface
 *----------------------------------------------------------------------
 */
1217

A
Avi Kivity 已提交
1218
static bool lapic_is_periodic(struct kvm_lapic *apic)
E
Eddie Dong 已提交
1219
{
1220
	return apic_lvtt_period(apic);
E
Eddie Dong 已提交
1221 1222
}

1223 1224 1225 1226
int apic_has_pending_timer(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *lapic = vcpu->arch.apic;

1227
	if (lapic && apic_enabled(lapic) && apic_lvt_enabled(lapic, APIC_LVTT))
1228
		return atomic_read(&lapic->lapic_timer.pending);
1229 1230 1231 1232

	return 0;
}

A
Avi Kivity 已提交
1233
int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
1234
{
1235
	u32 reg = apic_get_reg(apic, lvt_type);
1236 1237
	int vector, mode, trig_mode;

1238
	if (apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
1239 1240 1241 1242 1243 1244 1245
		vector = reg & APIC_VECTOR_MASK;
		mode = reg & APIC_MODE_MASK;
		trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
		return __apic_accept_irq(apic, mode, vector, 1, trig_mode);
	}
	return 0;
}
1246

1247
void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
1248
{
1249 1250 1251 1252
	struct kvm_lapic *apic = vcpu->arch.apic;

	if (apic)
		kvm_apic_local_deliver(apic, APIC_LVT0);
1253 1254
}

G
Gregory Haskins 已提交
1255 1256 1257 1258 1259
static const struct kvm_io_device_ops apic_mmio_ops = {
	.read     = apic_mmio_read,
	.write    = apic_mmio_write,
};

1260 1261 1262
static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
{
	struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
A
Avi Kivity 已提交
1263 1264
	struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
	struct kvm_vcpu *vcpu = apic->vcpu;
1265 1266 1267 1268 1269 1270 1271 1272
	wait_queue_head_t *q = &vcpu->wq;

	/*
	 * There is a race window between reading and incrementing, but we do
	 * not care about potentially losing timer events in the !reinject
	 * case anyway. Note: KVM_REQ_PENDING_TIMER is implicitly checked
	 * in vcpu_enter_guest.
	 */
A
Avi Kivity 已提交
1273
	if (!atomic_read(&ktimer->pending)) {
1274 1275 1276 1277 1278 1279 1280 1281
		atomic_inc(&ktimer->pending);
		/* FIXME: this code should not know anything about vcpus */
		kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
	}

	if (waitqueue_active(q))
		wake_up_interruptible(q);

A
Avi Kivity 已提交
1282
	if (lapic_is_periodic(apic)) {
1283 1284 1285 1286 1287 1288
		hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
		return HRTIMER_RESTART;
	} else
		return HRTIMER_NORESTART;
}

E
Eddie Dong 已提交
1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299
int kvm_create_lapic(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic;

	ASSERT(vcpu != NULL);
	apic_debug("apic_init %d\n", vcpu->vcpu_id);

	apic = kzalloc(sizeof(*apic), GFP_KERNEL);
	if (!apic)
		goto nomem;

1300
	vcpu->arch.apic = apic;
E
Eddie Dong 已提交
1301

1302 1303
	apic->regs = (void *)get_zeroed_page(GFP_KERNEL);
	if (!apic->regs) {
E
Eddie Dong 已提交
1304 1305
		printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
		       vcpu->vcpu_id);
1306
		goto nomem_free_apic;
E
Eddie Dong 已提交
1307 1308 1309
	}
	apic->vcpu = vcpu;

1310 1311
	hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
		     HRTIMER_MODE_ABS);
1312
	apic->lapic_timer.timer.function = apic_timer_fn;
1313

1314
	kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE);
E
Eddie Dong 已提交
1315

1316
	kvm_lapic_reset(vcpu);
G
Gregory Haskins 已提交
1317
	kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
E
Eddie Dong 已提交
1318 1319

	return 0;
1320 1321
nomem_free_apic:
	kfree(apic);
E
Eddie Dong 已提交
1322 1323 1324 1325 1326 1327
nomem:
	return -ENOMEM;
}

int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
{
1328
	struct kvm_lapic *apic = vcpu->arch.apic;
E
Eddie Dong 已提交
1329 1330 1331 1332 1333
	int highest_irr;

	if (!apic || !apic_enabled(apic))
		return -1;

1334
	apic_update_ppr(apic);
E
Eddie Dong 已提交
1335 1336 1337 1338 1339 1340 1341
	highest_irr = apic_find_highest_irr(apic);
	if ((highest_irr == -1) ||
	    ((highest_irr & 0xF0) <= apic_get_reg(apic, APIC_PROCPRI)))
		return -1;
	return highest_irr;
}

Q
Qing He 已提交
1342 1343
int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
{
1344
	u32 lvt0 = apic_get_reg(vcpu->arch.apic, APIC_LVT0);
Q
Qing He 已提交
1345 1346
	int r = 0;

1347 1348 1349 1350 1351
	if (!apic_hw_enabled(vcpu->arch.apic))
		r = 1;
	if ((lvt0 & APIC_LVT_MASKED) == 0 &&
	    GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
		r = 1;
Q
Qing He 已提交
1352 1353 1354
	return r;
}

1355 1356
void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
{
1357
	struct kvm_lapic *apic = vcpu->arch.apic;
1358

1359
	if (apic && atomic_read(&apic->lapic_timer.pending) > 0) {
1360
		if (kvm_apic_local_deliver(apic, APIC_LVTT))
1361
			atomic_dec(&apic->lapic_timer.pending);
1362 1363 1364
	}
}

E
Eddie Dong 已提交
1365 1366 1367
int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
{
	int vector = kvm_apic_has_interrupt(vcpu);
1368
	struct kvm_lapic *apic = vcpu->arch.apic;
E
Eddie Dong 已提交
1369 1370 1371 1372

	if (vector == -1)
		return -1;

M
Michael S. Tsirkin 已提交
1373
	apic_set_isr(vector, apic);
E
Eddie Dong 已提交
1374 1375 1376 1377
	apic_update_ppr(apic);
	apic_clear_irr(vector, apic);
	return vector;
}
1378 1379 1380

void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu)
{
1381
	struct kvm_lapic *apic = vcpu->arch.apic;
1382

1383
	kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
1384 1385
	kvm_apic_set_version(vcpu);

1386
	apic_update_ppr(apic);
1387
	hrtimer_cancel(&apic->lapic_timer.timer);
1388 1389
	update_divide_count(apic);
	start_apic_timer(apic);
1390
	apic->irr_pending = true;
M
Michael S. Tsirkin 已提交
1391 1392
	apic->isr_count = count_vectors(apic->regs + APIC_ISR);
	apic->highest_isr_cache = -1;
1393
	kvm_make_request(KVM_REQ_EVENT, vcpu);
1394
}
1395

1396
void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
1397
{
1398
	struct kvm_lapic *apic = vcpu->arch.apic;
1399 1400 1401 1402 1403
	struct hrtimer *timer;

	if (!apic)
		return;

1404
	timer = &apic->lapic_timer.timer;
1405
	if (hrtimer_cancel(timer))
1406
		hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
1407
}
A
Avi Kivity 已提交
1408

1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
/*
 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
 *
 * Detect whether guest triggered PV EOI since the
 * last entry. If yes, set EOI on guests's behalf.
 * Clear PV EOI in guest memory in any case.
 */
static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
					struct kvm_lapic *apic)
{
	bool pending;
	int vector;
	/*
	 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
	 * and KVM_PV_EOI_ENABLED in guest memory as follows:
	 *
	 * KVM_APIC_PV_EOI_PENDING is unset:
	 * 	-> host disabled PV EOI.
	 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
	 * 	-> host enabled PV EOI, guest did not execute EOI yet.
	 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
	 * 	-> host enabled PV EOI, guest executed EOI.
	 */
	BUG_ON(!pv_eoi_enabled(vcpu));
	pending = pv_eoi_get_pending(vcpu);
	/*
	 * Clear pending bit in any case: it will be set again on vmentry.
	 * While this might not be ideal from performance point of view,
	 * this makes sure pv eoi is only enabled when we know it's safe.
	 */
	pv_eoi_clr_pending(vcpu);
	if (pending)
		return;
	vector = apic_set_eoi(apic);
	trace_kvm_pv_eoi(apic, vector);
}

A
Avi Kivity 已提交
1446 1447 1448 1449 1450
void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
{
	u32 data;
	void *vapic;

1451 1452 1453
	if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
		apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);

1454
	if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
A
Avi Kivity 已提交
1455 1456
		return;

1457
	vapic = kmap_atomic(vcpu->arch.apic->vapic_page);
A
Avi Kivity 已提交
1458
	data = *(u32 *)(vapic + offset_in_page(vcpu->arch.apic->vapic_addr));
1459
	kunmap_atomic(vapic);
A
Avi Kivity 已提交
1460 1461 1462 1463

	apic_set_tpr(vcpu->arch.apic, data & 0xff);
}

1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489
/*
 * apic_sync_pv_eoi_to_guest - called before vmentry
 *
 * Detect whether it's safe to enable PV EOI and
 * if yes do so.
 */
static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
					struct kvm_lapic *apic)
{
	if (!pv_eoi_enabled(vcpu) ||
	    /* IRR set or many bits in ISR: could be nested. */
	    apic->irr_pending ||
	    /* Cache not set: could be safe but we don't bother. */
	    apic->highest_isr_cache == -1 ||
	    /* Need EOI to update ioapic. */
	    kvm_ioapic_handles_vector(vcpu->kvm, apic->highest_isr_cache)) {
		/*
		 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
		 * so we need not do anything here.
		 */
		return;
	}

	pv_eoi_set_pending(apic->vcpu);
}

A
Avi Kivity 已提交
1490 1491 1492 1493
void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
{
	u32 data, tpr;
	int max_irr, max_isr;
1494
	struct kvm_lapic *apic = vcpu->arch.apic;
A
Avi Kivity 已提交
1495 1496
	void *vapic;

1497 1498
	apic_sync_pv_eoi_to_guest(vcpu, apic);

1499
	if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
A
Avi Kivity 已提交
1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
		return;

	tpr = apic_get_reg(apic, APIC_TASKPRI) & 0xff;
	max_irr = apic_find_highest_irr(apic);
	if (max_irr < 0)
		max_irr = 0;
	max_isr = apic_find_highest_isr(apic);
	if (max_isr < 0)
		max_isr = 0;
	data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);

1511
	vapic = kmap_atomic(vcpu->arch.apic->vapic_page);
A
Avi Kivity 已提交
1512
	*(u32 *)(vapic + offset_in_page(vcpu->arch.apic->vapic_addr)) = data;
1513
	kunmap_atomic(vapic);
A
Avi Kivity 已提交
1514 1515 1516 1517 1518
}

void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
{
	vcpu->arch.apic->vapic_addr = vapic_addr;
1519 1520 1521 1522
	if (vapic_addr)
		__set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
	else
		__clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
A
Avi Kivity 已提交
1523
}
G
Gleb Natapov 已提交
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555

int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u32 reg = (msr - APIC_BASE_MSR) << 4;

	if (!irqchip_in_kernel(vcpu->kvm) || !apic_x2apic_mode(apic))
		return 1;

	/* if this is ICR write vector before command */
	if (msr == 0x830)
		apic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
	return apic_reg_write(apic, reg, (u32)data);
}

int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;

	if (!irqchip_in_kernel(vcpu->kvm) || !apic_x2apic_mode(apic))
		return 1;

	if (apic_reg_read(apic, reg, 4, &low))
		return 1;
	if (msr == 0x830)
		apic_reg_read(apic, APIC_ICR2, 4, &high);

	*data = (((u64)high) << 32) | low;

	return 0;
}
G
Gleb Natapov 已提交
1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586

int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

	if (!irqchip_in_kernel(vcpu->kvm))
		return 1;

	/* if this is ICR write vector before command */
	if (reg == APIC_ICR)
		apic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
	return apic_reg_write(apic, reg, (u32)data);
}

int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	u32 low, high = 0;

	if (!irqchip_in_kernel(vcpu->kvm))
		return 1;

	if (apic_reg_read(apic, reg, 4, &low))
		return 1;
	if (reg == APIC_ICR)
		apic_reg_read(apic, APIC_ICR2, 4, &high);

	*data = (((u64)high) << 32) | low;

	return 0;
}
1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599

int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data)
{
	u64 addr = data & ~KVM_MSR_ENABLED;
	if (!IS_ALIGNED(addr, 4))
		return 1;

	vcpu->arch.pv_eoi.msr_val = data;
	if (!pv_eoi_enabled(vcpu))
		return 0;
	return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data,
					 addr);
}