apic.c 52.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
/*
 *	Local APIC handling, local APIC timers
 *
 *	(c) 1999, 2000 Ingo Molnar <mingo@redhat.com>
 *
 *	Fixes
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
 *					thanks to Eric Gilmore
 *					and Rolf G. Tews
 *					for testing these extensively.
 *	Maciej W. Rozycki	:	Various updates and fixes.
 *	Mikael Pettersson	:	Power Management for UP-APIC.
 *	Pavel Machek and
 *	Mikael Pettersson	:	PM converted to driver model.
 */

#include <linux/init.h>

#include <linux/mm.h>
#include <linux/delay.h>
#include <linux/bootmem.h>
#include <linux/interrupt.h>
#include <linux/mc146818rtc.h>
#include <linux/kernel_stat.h>
#include <linux/sysdev.h>
26
#include <linux/ioport.h>
Y
Yinghai Lu 已提交
27
#include <linux/cpu.h>
28
#include <linux/clockchips.h>
29
#include <linux/acpi_pmtmr.h>
H
Hiroshi Shimamoto 已提交
30
#include <linux/module.h>
Y
Yinghai Lu 已提交
31
#include <linux/dmi.h>
32
#include <linux/dmar.h>
33
#include <linux/ftrace.h>
L
Linus Torvalds 已提交
34 35 36 37 38

#include <asm/atomic.h>
#include <asm/smp.h>
#include <asm/mtrr.h>
#include <asm/mpspec.h>
Y
Yinghai Lu 已提交
39
#include <asm/desc.h>
Y
Yinghai Lu 已提交
40
#include <asm/arch_hooks.h>
H
Hiroshi Shimamoto 已提交
41
#include <asm/hpet.h>
L
Linus Torvalds 已提交
42
#include <asm/pgalloc.h>
Y
Yinghai Lu 已提交
43
#include <asm/i8253.h>
44
#include <asm/nmi.h>
A
Andi Kleen 已提交
45
#include <asm/idle.h>
46 47
#include <asm/proto.h>
#include <asm/timex.h>
48
#include <asm/apic.h>
49
#include <asm/i8259.h>
L
Linus Torvalds 已提交
50

51
#include <mach_apic.h>
Y
Yinghai Lu 已提交
52 53
#include <mach_apicdef.h>
#include <mach_ipi.h>
54

55 56 57 58 59 60 61
/*
 * Sanity check
 */
#if ((SPURIOUS_APIC_VECTOR & 0x0F) != 0x0F)
# error SPURIOUS_APIC_VECTOR definition error
#endif

Y
Yinghai Lu 已提交
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
#ifdef CONFIG_X86_32
/*
 * Knob to control our willingness to enable the local APIC.
 *
 * +1=force-enable
 */
static int force_enable_local_apic;
/*
 * APIC command line parameters
 */
static int __init parse_lapic(char *arg)
{
	force_enable_local_apic = 1;
	return 0;
}
early_param("lapic", parse_lapic);
Y
Yinghai Lu 已提交
78 79 80
/* Local APIC was disabled by the BIOS and enabled by the kernel */
static int enabled_via_apicbase;

Y
Yinghai Lu 已提交
81 82 83
#endif

#ifdef CONFIG_X86_64
84
static int apic_calibrate_pmtmr __initdata;
Y
Yinghai Lu 已提交
85 86 87 88 89 90 91 92 93
static __init int setup_apicpmtimer(char *s)
{
	apic_calibrate_pmtmr = 1;
	notsc_setup(NULL);
	return 0;
}
__setup("apicpmtimer", setup_apicpmtimer);
#endif

Y
Yinghai Lu 已提交
94 95 96 97 98
#ifdef CONFIG_X86_64
#define HAVE_X2APIC
#endif

#ifdef HAVE_X2APIC
99
int x2apic;
100 101
/* x2apic enabled before OS handover */
int x2apic_preenabled;
Y
Yinghai Lu 已提交
102 103 104 105 106 107 108 109 110
int disable_x2apic;
static __init int setup_nox2apic(char *str)
{
	disable_x2apic = 1;
	setup_clear_cpu_cap(X86_FEATURE_X2APIC);
	return 0;
}
early_param("nox2apic", setup_nox2apic);
#endif
L
Linus Torvalds 已提交
111

Y
Yinghai Lu 已提交
112 113 114 115
unsigned long mp_lapic_addr;
int disable_apic;
/* Disable local APIC timer from the kernel commandline or via dmi quirk */
static int disable_apic_timer __cpuinitdata;
H
Hiroshi Shimamoto 已提交
116
/* Local APIC timer works in C2 */
117 118 119
int local_apic_timer_c2_ok;
EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);

Y
Yinghai Lu 已提交
120 121
int first_system_vector = 0xfe;

H
Hiroshi Shimamoto 已提交
122 123 124
/*
 * Debug level, exported for io_apic.c
 */
125
unsigned int apic_verbosity;
H
Hiroshi Shimamoto 已提交
126

127 128
int pic_mode;

A
Alexey Starikovskiy 已提交
129 130 131
/* Have we found an MP table */
int smp_found_config;

132 133 134 135 136
static struct resource lapic_resource = {
	.name = "Local APIC",
	.flags = IORESOURCE_MEM | IORESOURCE_BUSY,
};

137 138
static unsigned int calibration_result;

139 140 141 142
static int lapic_next_event(unsigned long delta,
			    struct clock_event_device *evt);
static void lapic_timer_setup(enum clock_event_mode mode,
			      struct clock_event_device *evt);
143
static void lapic_timer_broadcast(const cpumask_t *mask);
144
static void apic_pm_activate(void);
145

146 147 148
/*
 * The local apic timer can be used for any function which is CPU local.
 */
149 150 151 152 153 154 155 156 157 158 159 160 161
static struct clock_event_device lapic_clockevent = {
	.name		= "lapic",
	.features	= CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT
			| CLOCK_EVT_FEAT_C3STOP | CLOCK_EVT_FEAT_DUMMY,
	.shift		= 32,
	.set_mode	= lapic_timer_setup,
	.set_next_event	= lapic_next_event,
	.broadcast	= lapic_timer_broadcast,
	.rating		= 100,
	.irq		= -1,
};
static DEFINE_PER_CPU(struct clock_event_device, lapic_events);

162 163
static unsigned long apic_phys;

164 165 166 167
/*
 * Get the LAPIC version
 */
static inline int lapic_get_version(void)
168
{
169
	return GET_APIC_VERSION(apic_read(APIC_LVR));
170 171
}

172
/*
173
 * Check, if the APIC is integrated or a separate chip
174 175
 */
static inline int lapic_is_integrated(void)
176
{
177
#ifdef CONFIG_X86_64
178
	return 1;
179 180 181
#else
	return APIC_INTEGRATED(lapic_get_version());
#endif
182 183 184
}

/*
185
 * Check, whether this is a modern or a first generation APIC
186
 */
187
static int modern_apic(void)
188
{
189 190 191 192 193
	/* AMD systems use old APIC versions, so check the CPU */
	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
	    boot_cpu_data.x86 >= 0xf)
		return 1;
	return lapic_get_version() >= 0x14;
194 195
}

196 197 198 199 200
/*
 * Paravirt kernels also might be using these below ops. So we still
 * use generic apic_read()/apic_write(), which might be pointing to different
 * ops in PARAVIRT case.
 */
201
void xapic_wait_icr_idle(void)
202 203 204 205 206
{
	while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
		cpu_relax();
}

207
u32 safe_xapic_wait_icr_idle(void)
208
{
209
	u32 send_status;
210 211 212 213 214 215 216 217 218 219 220 221 222
	int timeout;

	timeout = 0;
	do {
		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
		if (!send_status)
			break;
		udelay(100);
	} while (timeout++ < 1000);

	return send_status;
}

223 224
void xapic_icr_write(u32 low, u32 id)
{
225
	apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
226 227 228 229 230 231 232 233 234 235
	apic_write(APIC_ICR, low);
}

u64 xapic_icr_read(void)
{
	u32 icr1, icr2;

	icr2 = apic_read(APIC_ICR2);
	icr1 = apic_read(APIC_ICR);

236
	return icr1 | ((u64)icr2 << 32);
237 238 239 240 241 242 243 244 245 246 247 248 249 250
}

static struct apic_ops xapic_ops = {
	.read = native_apic_mem_read,
	.write = native_apic_mem_write,
	.icr_read = xapic_icr_read,
	.icr_write = xapic_icr_write,
	.wait_icr_idle = xapic_wait_icr_idle,
	.safe_wait_icr_idle = safe_xapic_wait_icr_idle,
};

struct apic_ops __read_mostly *apic_ops = &xapic_ops;
EXPORT_SYMBOL_GPL(apic_ops);

Y
Yinghai Lu 已提交
251
#ifdef HAVE_X2APIC
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
static void x2apic_wait_icr_idle(void)
{
	/* no need to wait for icr idle in x2apic */
	return;
}

static u32 safe_x2apic_wait_icr_idle(void)
{
	/* no need to wait for icr idle in x2apic */
	return 0;
}

void x2apic_icr_write(u32 low, u32 id)
{
	wrmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low);
}

u64 x2apic_icr_read(void)
{
	unsigned long val;

	rdmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), val);
	return val;
}

static struct apic_ops x2apic_ops = {
	.read = native_apic_msr_read,
	.write = native_apic_msr_write,
	.icr_read = x2apic_icr_read,
	.icr_write = x2apic_icr_write,
	.wait_icr_idle = x2apic_wait_icr_idle,
	.safe_wait_icr_idle = safe_x2apic_wait_icr_idle,
};
Y
Yinghai Lu 已提交
285
#endif
286

287 288 289
/**
 * enable_NMI_through_LVT0 - enable NMI through local vector table 0
 */
290
void __cpuinit enable_NMI_through_LVT0(void)
L
Linus Torvalds 已提交
291
{
292
	unsigned int v;
293 294 295

	/* unmask and set to NMI */
	v = APIC_DM_NMI;
296 297 298 299 300

	/* Level triggered for 82489DX (32bit mode) */
	if (!lapic_is_integrated())
		v |= APIC_LVT_LEVEL_TRIGGER;

301
	apic_write(APIC_LVT0, v);
L
Linus Torvalds 已提交
302 303
}

304 305 306 307 308 309 310 311 312 313
#ifdef CONFIG_X86_32
/**
 * get_physical_broadcast - Get number of physical broadcast IDs
 */
int get_physical_broadcast(void)
{
	return modern_apic() ? 0xff : 0xf;
}
#endif

314 315 316
/**
 * lapic_get_maxlvt - get the maximum number of local vector table entries
 */
317
int lapic_get_maxlvt(void)
L
Linus Torvalds 已提交
318
{
319
	unsigned int v;
L
Linus Torvalds 已提交
320 321

	v = apic_read(APIC_LVR);
322 323 324 325 326
	/*
	 * - we always have APIC integrated on 64bit mode
	 * - 82489DXs do not report # of LVT entries
	 */
	return APIC_INTEGRATED(GET_APIC_VERSION(v)) ? GET_APIC_MAXLVT(v) : 2;
L
Linus Torvalds 已提交
327 328
}

329 330 331 332
/*
 * Local APIC timer
 */

333 334
/* Clock divisor */
#define APIC_DIVISOR 16
335

336 337 338 339 340 341 342 343 344 345 346
/*
 * This function sets up the local APIC timer, with a timeout of
 * 'clocks' APIC bus clock. During calibration we actually call
 * this function twice on the boot CPU, once with a bogus timeout
 * value, second time for real. The other (noncalibrating) CPUs
 * call this function only once, with the real, calibrated value.
 *
 * We do reads before writes even if unnecessary, to get around the
 * P5 APIC double write bug.
 */
static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
L
Linus Torvalds 已提交
347
{
348
	unsigned int lvtt_value, tmp_value;
L
Linus Torvalds 已提交
349

350 351 352
	lvtt_value = LOCAL_TIMER_VECTOR;
	if (!oneshot)
		lvtt_value |= APIC_LVT_TIMER_PERIODIC;
353 354 355
	if (!lapic_is_integrated())
		lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV);

356 357
	if (!irqen)
		lvtt_value |= APIC_LVT_MASKED;
L
Linus Torvalds 已提交
358

359
	apic_write(APIC_LVTT, lvtt_value);
L
Linus Torvalds 已提交
360 361

	/*
362
	 * Divide PICLK by 16
L
Linus Torvalds 已提交
363
	 */
364
	tmp_value = apic_read(APIC_TDCR);
365 366 367
	apic_write(APIC_TDCR,
		(tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
		APIC_TDR_DIV_16);
368 369

	if (!oneshot)
370
		apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
L
Linus Torvalds 已提交
371 372
}

373
/*
374 375 376 377
 * Setup extended LVT, AMD specific (K8, family 10h)
 *
 * Vector mappings are hard coded. On K8 only offset 0 (APIC500) and
 * MCE interrupts are supported. Thus MCE offset must be set to 0.
378 379 380
 *
 * If mask=1, the LVT entry does not generate interrupts while mask=0
 * enables the vector. See also the BKDGs.
381
 */
382 383 384 385 386

#define APIC_EILVT_LVTOFF_MCE 0
#define APIC_EILVT_LVTOFF_IBS 1

static void setup_APIC_eilvt(u8 lvt_off, u8 vector, u8 msg_type, u8 mask)
L
Linus Torvalds 已提交
387
{
388
	unsigned long reg = (lvt_off << 4) + APIC_EILVT0;
389
	unsigned int  v   = (mask << 16) | (msg_type << 8) | vector;
A
Andi Kleen 已提交
390

391
	apic_write(reg, v);
L
Linus Torvalds 已提交
392 393
}

394 395 396 397 398 399 400 401 402 403 404
u8 setup_APIC_eilvt_mce(u8 vector, u8 msg_type, u8 mask)
{
	setup_APIC_eilvt(APIC_EILVT_LVTOFF_MCE, vector, msg_type, mask);
	return APIC_EILVT_LVTOFF_MCE;
}

u8 setup_APIC_eilvt_ibs(u8 vector, u8 msg_type, u8 mask)
{
	setup_APIC_eilvt(APIC_EILVT_LVTOFF_IBS, vector, msg_type, mask);
	return APIC_EILVT_LVTOFF_IBS;
}
405
EXPORT_SYMBOL_GPL(setup_APIC_eilvt_ibs);
406

407 408 409 410 411
/*
 * Program the next event, relative to now
 */
static int lapic_next_event(unsigned long delta,
			    struct clock_event_device *evt)
L
Linus Torvalds 已提交
412
{
413 414
	apic_write(APIC_TMICT, delta);
	return 0;
L
Linus Torvalds 已提交
415 416
}

417 418 419 420 421
/*
 * Setup the lapic timer in periodic or oneshot mode
 */
static void lapic_timer_setup(enum clock_event_mode mode,
			      struct clock_event_device *evt)
422 423
{
	unsigned long flags;
424
	unsigned int v;
425

426 427
	/* Lapic used as dummy for broadcast ? */
	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
428 429 430 431
		return;

	local_irq_save(flags);

432 433 434 435 436 437 438 439 440 441 442
	switch (mode) {
	case CLOCK_EVT_MODE_PERIODIC:
	case CLOCK_EVT_MODE_ONESHOT:
		__setup_APIC_LVTT(calibration_result,
				  mode != CLOCK_EVT_MODE_PERIODIC, 1);
		break;
	case CLOCK_EVT_MODE_UNUSED:
	case CLOCK_EVT_MODE_SHUTDOWN:
		v = apic_read(APIC_LVTT);
		v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
		apic_write(APIC_LVTT, v);
443
		apic_write(APIC_TMICT, 0xffffffff);
444 445 446 447 448
		break;
	case CLOCK_EVT_MODE_RESUME:
		/* Nothing to do here */
		break;
	}
449 450 451 452

	local_irq_restore(flags);
}

L
Linus Torvalds 已提交
453
/*
454
 * Local APIC timer broadcast function
L
Linus Torvalds 已提交
455
 */
456
static void lapic_timer_broadcast(const cpumask_t *mask)
L
Linus Torvalds 已提交
457
{
458
#ifdef CONFIG_SMP
459
	send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
460 461
#endif
}
L
Linus Torvalds 已提交
462

463 464 465 466
/*
 * Setup the local APIC timer for this CPU. Copy the initilized values
 * of the boot CPU and register the clock event in the framework.
 */
467
static void __cpuinit setup_APIC_timer(void)
468 469
{
	struct clock_event_device *levt = &__get_cpu_var(lapic_events);
L
Linus Torvalds 已提交
470

471
	memcpy(levt, &lapic_clockevent, sizeof(*levt));
472
	levt->cpumask = cpumask_of(smp_processor_id());
L
Linus Torvalds 已提交
473

474 475
	clockevents_register_device(levt);
}
L
Linus Torvalds 已提交
476

477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
/*
 * In this functions we calibrate APIC bus clocks to the external timer.
 *
 * We want to do the calibration only once since we want to have local timer
 * irqs syncron. CPUs connected by the same APIC bus have the very same bus
 * frequency.
 *
 * This was previously done by reading the PIT/HPET and waiting for a wrap
 * around to find out, that a tick has elapsed. I have a box, where the PIT
 * readout is broken, so it never gets out of the wait loop again. This was
 * also reported by others.
 *
 * Monitoring the jiffies value is inaccurate and the clockevents
 * infrastructure allows us to do a simple substitution of the interrupt
 * handler.
 *
 * The calibration routine also uses the pm_timer when possible, as the PIT
 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
 * back to normal later in the boot process).
 */

#define LAPIC_CAL_LOOPS		(HZ/10)

static __initdata int lapic_cal_loops = -1;
static __initdata long lapic_cal_t1, lapic_cal_t2;
static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;

/*
 * Temporary interrupt handler.
 */
static void __init lapic_cal_handler(struct clock_event_device *dev)
{
	unsigned long long tsc = 0;
	long tapic = apic_read(APIC_TMCCT);
	unsigned long pm = acpi_pm_read_early();

	if (cpu_has_tsc)
		rdtscll(tsc);

	switch (lapic_cal_loops++) {
	case 0:
		lapic_cal_t1 = tapic;
		lapic_cal_tsc1 = tsc;
		lapic_cal_pm1 = pm;
		lapic_cal_j1 = jiffies;
		break;

	case LAPIC_CAL_LOOPS:
		lapic_cal_t2 = tapic;
		lapic_cal_tsc2 = tsc;
		if (pm < lapic_cal_pm1)
			pm += ACPI_PM_OVRRUN;
		lapic_cal_pm2 = pm;
		lapic_cal_j2 = jiffies;
		break;
	}
}

537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
static int __init calibrate_by_pmtimer(long deltapm, long *delta)
{
	const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
	const long pm_thresh = pm_100ms / 100;
	unsigned long mult;
	u64 res;

#ifndef CONFIG_X86_PM_TIMER
	return -1;
#endif

	apic_printk(APIC_VERBOSE, "... PM timer delta = %ld\n", deltapm);

	/* Check, if the PM timer is available */
	if (!deltapm)
		return -1;

	mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);

	if (deltapm > (pm_100ms - pm_thresh) &&
	    deltapm < (pm_100ms + pm_thresh)) {
		apic_printk(APIC_VERBOSE, "... PM timer result ok\n");
	} else {
		res = (((u64)deltapm) *  mult) >> 22;
		do_div(res, 1000000);
562
		pr_warning("APIC calibration not consistent "
563 564 565 566 567
			"with PM Timer: %ldms instead of 100ms\n",
			(long)res);
		/* Correct the lapic counter value */
		res = (((u64)(*delta)) * pm_100ms);
		do_div(res, deltapm);
568
		pr_info("APIC delta adjusted to PM-Timer: "
569 570 571 572 573 574 575
			"%lu (%ld)\n", (unsigned long)res, *delta);
		*delta = (long)res;
	}

	return 0;
}

576 577 578 579 580
static int __init calibrate_APIC_clock(void)
{
	struct clock_event_device *levt = &__get_cpu_var(lapic_events);
	void (*real_handler)(struct clock_event_device *dev);
	unsigned long deltaj;
581
	long delta;
582 583 584 585 586 587 588 589 590
	int pm_referenced = 0;

	local_irq_disable();

	/* Replace the global interrupt handler */
	real_handler = global_clock_event->event_handler;
	global_clock_event->event_handler = lapic_cal_handler;

	/*
C
Cyrill Gorcunov 已提交
591
	 * Setup the APIC counter to maximum. There is no way the lapic
592 593
	 * can underflow in the 100ms detection time frame
	 */
C
Cyrill Gorcunov 已提交
594
	__setup_APIC_LVTT(0xffffffff, 0, 0);
595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610

	/* Let the interrupts run */
	local_irq_enable();

	while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
		cpu_relax();

	local_irq_disable();

	/* Restore the real event handler */
	global_clock_event->event_handler = real_handler;

	/* Build delta t1-t2 as apic timer counts down */
	delta = lapic_cal_t1 - lapic_cal_t2;
	apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);

611 612 613
	/* we trust the PM based calibration if possible */
	pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
					&delta);
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647

	/* Calculate the scaled math multiplication factor */
	lapic_clockevent.mult = div_sc(delta, TICK_NSEC * LAPIC_CAL_LOOPS,
				       lapic_clockevent.shift);
	lapic_clockevent.max_delta_ns =
		clockevent_delta2ns(0x7FFFFF, &lapic_clockevent);
	lapic_clockevent.min_delta_ns =
		clockevent_delta2ns(0xF, &lapic_clockevent);

	calibration_result = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;

	apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
	apic_printk(APIC_VERBOSE, "..... mult: %ld\n", lapic_clockevent.mult);
	apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
		    calibration_result);

	if (cpu_has_tsc) {
		delta = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
		apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
			    "%ld.%04ld MHz.\n",
			    (delta / LAPIC_CAL_LOOPS) / (1000000 / HZ),
			    (delta / LAPIC_CAL_LOOPS) % (1000000 / HZ));
	}

	apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
		    "%u.%04u MHz.\n",
		    calibration_result / (1000000 / HZ),
		    calibration_result % (1000000 / HZ));

	/*
	 * Do a sanity check on the APIC calibration result
	 */
	if (calibration_result < (1000000 / HZ)) {
		local_irq_enable();
648
		pr_warning("APIC frequency too slow, disabling apic timer\n");
649 650 651 652 653
		return -1;
	}

	levt->features &= ~CLOCK_EVT_FEAT_DUMMY;

654 655 656 657
	/*
	 * PM timer calibration failed or not turned on
	 * so lets try APIC timer based calibration
	 */
658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
	if (!pm_referenced) {
		apic_printk(APIC_VERBOSE, "... verify APIC timer\n");

		/*
		 * Setup the apic timer manually
		 */
		levt->event_handler = lapic_cal_handler;
		lapic_timer_setup(CLOCK_EVT_MODE_PERIODIC, levt);
		lapic_cal_loops = -1;

		/* Let the interrupts run */
		local_irq_enable();

		while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
			cpu_relax();

		/* Stop the lapic timer */
		lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, levt);

		/* Jiffies delta */
		deltaj = lapic_cal_j2 - lapic_cal_j1;
		apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);

		/* Check, if the jiffies result is consistent */
		if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
			apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
		else
			levt->features |= CLOCK_EVT_FEAT_DUMMY;
	} else
		local_irq_enable();

	if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
690
		pr_warning("APIC timer disabled due to verification failure.\n");
691 692 693 694 695 696
			return -1;
	}

	return 0;
}

H
Hiroshi Shimamoto 已提交
697 698 699 700 701
/*
 * Setup the boot APIC
 *
 * Calibrate and verify the result.
 */
702 703 704
void __init setup_boot_APIC_clock(void)
{
	/*
705 706 707 708
	 * The local apic timer can be disabled via the kernel
	 * commandline or from the CPU detection code. Register the lapic
	 * timer as a dummy clock event source on SMP systems, so the
	 * broadcast mechanism is used. On UP systems simply ignore it.
709 710
	 */
	if (disable_apic_timer) {
711
		pr_info("Disabling APIC timer\n");
712
		/* No broadcast on UP ! */
713 714
		if (num_possible_cpus() > 1) {
			lapic_clockevent.mult = 1;
715
			setup_APIC_timer();
716
		}
717 718 719
		return;
	}

720 721 722
	apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
		    "calibrating APIC timer ...\n");

723
	if (calibrate_APIC_clock()) {
724 725 726 727 728 729
		/* No broadcast on UP ! */
		if (num_possible_cpus() > 1)
			setup_APIC_timer();
		return;
	}

730 731 732 733 734 735 736 737
	/*
	 * If nmi_watchdog is set to IO_APIC, we need the
	 * PIT/HPET going.  Otherwise register lapic as a dummy
	 * device.
	 */
	if (nmi_watchdog != NMI_IO_APIC)
		lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
	else
738
		pr_warning("APIC timer registered as dummy,"
739
			" due to nmi_watchdog=%d!\n", nmi_watchdog);
740

741
	/* Setup the lapic or request the broadcast */
742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
	setup_APIC_timer();
}

void __cpuinit setup_secondary_APIC_clock(void)
{
	setup_APIC_timer();
}

/*
 * The guts of the apic timer interrupt
 */
static void local_apic_timer_interrupt(void)
{
	int cpu = smp_processor_id();
	struct clock_event_device *evt = &per_cpu(lapic_events, cpu);

	/*
	 * Normally we should not be here till LAPIC has been initialized but
	 * in some cases like kdump, its possible that there is a pending LAPIC
	 * timer interrupt from previous kernel's context and is delivered in
	 * new kernel the moment interrupts are enabled.
	 *
	 * Interrupts are enabled early and LAPIC is setup much later, hence
	 * its possible that when we get here evt->event_handler is NULL.
	 * Check for event_handler being NULL and discard the interrupt as
	 * spurious.
	 */
	if (!evt->event_handler) {
770
		pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu);
771 772 773 774 775 776 777 778
		/* Switch it off */
		lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, evt);
		return;
	}

	/*
	 * the NMI deadlock-detector uses this.
	 */
779
	inc_irq_stat(apic_timer_irqs);
780 781 782 783 784 785 786 787 788 789 790 791

	evt->event_handler(evt);
}

/*
 * Local APIC timer interrupt. This is the most natural way for doing
 * local interrupts, but local timer interrupts can be emulated by
 * broadcast interrupts too. [in case the hw doesn't support APIC timers]
 *
 * [ if a single-CPU system runs an SMP kernel then we call the local
 *   interrupt as well. Thus we cannot inline the local irq ... ]
 */
792
void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	/*
	 * NOTE! We'd better ACK the irq immediately,
	 * because timer handling can be slow.
	 */
	ack_APIC_irq();
	/*
	 * update_process_times() expects us to have done irq_enter().
	 * Besides, if we don't timer interrupts ignore the global
	 * interrupt lock, which is the WrongThing (tm) to do.
	 */
	exit_idle();
	irq_enter();
	local_apic_timer_interrupt();
	irq_exit();
810

811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
	set_irq_regs(old_regs);
}

int setup_profiling_timer(unsigned int multiplier)
{
	return -EINVAL;
}

/*
 * Local APIC start and shutdown
 */

/**
 * clear_local_APIC - shutdown the local APIC
 *
 * This is called, when a CPU is disabled and before rebooting, so the state of
 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
 * leftovers during boot.
 */
void clear_local_APIC(void)
{
832
	int maxlvt;
833 834
	u32 v;

835 836 837 838 839
	/* APIC hasn't been mapped yet */
	if (!apic_phys)
		return;

	maxlvt = lapic_get_maxlvt();
840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
	/*
	 * Masking an LVT entry can trigger a local APIC error
	 * if the vector is zero. Mask LVTERR first to prevent this.
	 */
	if (maxlvt >= 3) {
		v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
		apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
	}
	/*
	 * Careful: we have to set masks only first to deassert
	 * any level-triggered sources.
	 */
	v = apic_read(APIC_LVTT);
	apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
	v = apic_read(APIC_LVT0);
	apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
	v = apic_read(APIC_LVT1);
	apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
	if (maxlvt >= 4) {
		v = apic_read(APIC_LVTPC);
		apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
	}

863 864 865 866 867 868 869
	/* lets not touch this if we didn't frob it */
#if defined(CONFIG_X86_MCE_P4THERMAL) || defined(X86_MCE_INTEL)
	if (maxlvt >= 5) {
		v = apic_read(APIC_LVTTHMR);
		apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
	}
#endif
870 871 872 873 874 875 876 877 878 879
	/*
	 * Clean APIC state for other OSs:
	 */
	apic_write(APIC_LVTT, APIC_LVT_MASKED);
	apic_write(APIC_LVT0, APIC_LVT_MASKED);
	apic_write(APIC_LVT1, APIC_LVT_MASKED);
	if (maxlvt >= 3)
		apic_write(APIC_LVTERR, APIC_LVT_MASKED);
	if (maxlvt >= 4)
		apic_write(APIC_LVTPC, APIC_LVT_MASKED);
880 881 882 883 884 885 886 887

	/* Integrated APIC (!82489DX) ? */
	if (lapic_is_integrated()) {
		if (maxlvt > 3)
			/* Clear ESR due to Pentium errata 3AP and 11AP */
			apic_write(APIC_ESR, 0);
		apic_read(APIC_ESR);
	}
888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905
}

/**
 * disable_local_APIC - clear and disable the local APIC
 */
void disable_local_APIC(void)
{
	unsigned int value;

	clear_local_APIC();

	/*
	 * Disable APIC (implies clearing of registers
	 * for 82489DX!).
	 */
	value = apic_read(APIC_SPIV);
	value &= ~APIC_SPIV_APIC_ENABLED;
	apic_write(APIC_SPIV, value);
906 907 908 909 910 911 912 913 914 915 916 917 918 919

#ifdef CONFIG_X86_32
	/*
	 * When LAPIC was disabled by the BIOS and enabled by the kernel,
	 * restore the disabled state.
	 */
	if (enabled_via_apicbase) {
		unsigned int l, h;

		rdmsr(MSR_IA32_APICBASE, l, h);
		l &= ~MSR_IA32_APICBASE_ENABLE;
		wrmsr(MSR_IA32_APICBASE, l, h);
	}
#endif
920 921
}

922 923 924 925 926 927
/*
 * If Linux enabled the LAPIC against the BIOS default disable it down before
 * re-entering the BIOS on shutdown.  Otherwise the BIOS may get confused and
 * not power-off.  Additionally clear all LVT entries before disable_local_APIC
 * for the case where Linux didn't enable the LAPIC.
 */
928 929 930 931 932 933 934 935 936
void lapic_shutdown(void)
{
	unsigned long flags;

	if (!cpu_has_apic)
		return;

	local_irq_save(flags);

937 938 939 940 941 942 943
#ifdef CONFIG_X86_32
	if (!enabled_via_apicbase)
		clear_local_APIC();
	else
#endif
		disable_local_APIC();

944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986

	local_irq_restore(flags);
}

/*
 * This is to verify that we're looking at a real local APIC.
 * Check these against your board if the CPUs aren't getting
 * started for no apparent reason.
 */
int __init verify_local_APIC(void)
{
	unsigned int reg0, reg1;

	/*
	 * The version register is read-only in a real APIC.
	 */
	reg0 = apic_read(APIC_LVR);
	apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0);
	apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK);
	reg1 = apic_read(APIC_LVR);
	apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1);

	/*
	 * The two version reads above should print the same
	 * numbers.  If the second one is different, then we
	 * poke at a non-APIC.
	 */
	if (reg1 != reg0)
		return 0;

	/*
	 * Check if the version looks reasonably.
	 */
	reg1 = GET_APIC_VERSION(reg0);
	if (reg1 == 0x00 || reg1 == 0xff)
		return 0;
	reg1 = lapic_get_maxlvt();
	if (reg1 < 0x02 || reg1 == 0xff)
		return 0;

	/*
	 * The ID register is read/write in a real APIC.
	 */
987
	reg0 = apic_read(APIC_ID);
988 989
	apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0);
	apic_write(APIC_ID, reg0 ^ APIC_ID_MASK);
990
	reg1 = apic_read(APIC_ID);
991 992 993 994 995 996
	apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg1);
	apic_write(APIC_ID, reg0);
	if (reg1 != (reg0 ^ APIC_ID_MASK))
		return 0;

	/*
L
Linus Torvalds 已提交
997 998 999 1000 1001
	 * The next two are just to see if we have sane values.
	 * They're only really relevant if we're in Virtual Wire
	 * compatibility mode, but most boxes are anymore.
	 */
	reg0 = apic_read(APIC_LVT0);
1002
	apic_printk(APIC_DEBUG, "Getting LVT0: %x\n", reg0);
L
Linus Torvalds 已提交
1003 1004 1005 1006 1007 1008
	reg1 = apic_read(APIC_LVT1);
	apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1);

	return 1;
}

1009 1010 1011
/**
 * sync_Arb_IDs - synchronize APIC bus arbitration IDs
 */
L
Linus Torvalds 已提交
1012 1013
void __init sync_Arb_IDs(void)
{
C
Cyrill Gorcunov 已提交
1014 1015 1016 1017 1018
	/*
	 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
	 * needed on AMD.
	 */
	if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
L
Linus Torvalds 已提交
1019 1020 1021 1022 1023 1024 1025 1026
		return;

	/*
	 * Wait for idle.
	 */
	apic_wait_icr_idle();

	apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
1027 1028
	apic_write(APIC_ICR, APIC_DEST_ALLINC |
			APIC_INT_LEVELTRIG | APIC_DM_INIT);
L
Linus Torvalds 已提交
1029 1030 1031 1032 1033 1034 1035
}

/*
 * An initial setup of the virtual wire mode.
 */
void __init init_bsp_APIC(void)
{
1036
	unsigned int value;
L
Linus Torvalds 已提交
1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055

	/*
	 * Don't do the setup now if we have a SMP BIOS as the
	 * through-I/O-APIC virtual wire mode might be active.
	 */
	if (smp_found_config || !cpu_has_apic)
		return;

	/*
	 * Do not trust the local APIC being empty at bootup.
	 */
	clear_local_APIC();

	/*
	 * Enable APIC.
	 */
	value = apic_read(APIC_SPIV);
	value &= ~APIC_VECTOR_MASK;
	value |= APIC_SPIV_APIC_ENABLED;
1056 1057 1058 1059 1060 1061 1062 1063 1064

#ifdef CONFIG_X86_32
	/* This bit is reserved on P4/Xeon and should be cleared */
	if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
	    (boot_cpu_data.x86 == 15))
		value &= ~APIC_SPIV_FOCUS_DISABLED;
	else
#endif
		value |= APIC_SPIV_FOCUS_DISABLED;
L
Linus Torvalds 已提交
1065
	value |= SPURIOUS_APIC_VECTOR;
1066
	apic_write(APIC_SPIV, value);
L
Linus Torvalds 已提交
1067 1068 1069 1070

	/*
	 * Set up the virtual wire mode.
	 */
1071
	apic_write(APIC_LVT0, APIC_DM_EXTINT);
L
Linus Torvalds 已提交
1072
	value = APIC_DM_NMI;
1073 1074
	if (!lapic_is_integrated())		/* 82489DX */
		value |= APIC_LVT_LEVEL_TRIGGER;
1075
	apic_write(APIC_LVT1, value);
L
Linus Torvalds 已提交
1076 1077
}

1078 1079
static void __cpuinit lapic_setup_esr(void)
{
1080 1081 1082
	unsigned int oldvalue, value, maxlvt;

	if (!lapic_is_integrated()) {
1083
		pr_info("No ESR for 82489DX.\n");
1084 1085
		return;
	}
1086

1087
	if (esr_disable) {
1088
		/*
1089 1090 1091 1092
		 * Something untraceable is creating bad interrupts on
		 * secondary quads ... for the moment, just leave the
		 * ESR disabled - we can't do anything useful with the
		 * errors anyway - mbligh
1093
		 */
1094
		pr_info("Leaving ESR disabled.\n");
1095
		return;
1096
	}
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116

	maxlvt = lapic_get_maxlvt();
	if (maxlvt > 3)		/* Due to the Pentium erratum 3AP. */
		apic_write(APIC_ESR, 0);
	oldvalue = apic_read(APIC_ESR);

	/* enables sending errors */
	value = ERROR_APIC_VECTOR;
	apic_write(APIC_LVTERR, value);

	/*
	 * spec says clear errors after enabling vector.
	 */
	if (maxlvt > 3)
		apic_write(APIC_ESR, 0);
	value = apic_read(APIC_ESR);
	if (value != oldvalue)
		apic_printk(APIC_VERBOSE, "ESR value before enabling "
			"vector: 0x%08x  after: 0x%08x\n",
			oldvalue, value);
1117 1118 1119
}


1120 1121 1122 1123
/**
 * setup_local_APIC - setup the local APIC
 */
void __cpuinit setup_local_APIC(void)
L
Linus Torvalds 已提交
1124
{
1125
	unsigned int value;
1126
	int i, j;
L
Linus Torvalds 已提交
1127

1128 1129
#ifdef CONFIG_X86_32
	/* Pound the ESR really hard over the head with a big hammer - mbligh */
1130
	if (lapic_is_integrated() && esr_disable) {
1131 1132 1133 1134 1135 1136 1137
		apic_write(APIC_ESR, 0);
		apic_write(APIC_ESR, 0);
		apic_write(APIC_ESR, 0);
		apic_write(APIC_ESR, 0);
	}
#endif

J
Jack Steiner 已提交
1138
	preempt_disable();
L
Linus Torvalds 已提交
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159

	/*
	 * Double-check whether this APIC is really registered.
	 * This is meaningless in clustered apic mode, so we skip it.
	 */
	if (!apic_id_registered())
		BUG();

	/*
	 * Intel recommends to set DFR, LDR and TPR before enabling
	 * an APIC.  See e.g. "AP-388 82489DX User's Manual" (Intel
	 * document number 292116).  So here it goes...
	 */
	init_apic_ldr();

	/*
	 * Set Task Priority to 'accept all'. We never change this
	 * later on.
	 */
	value = apic_read(APIC_TASKPRI);
	value &= ~APIC_TPRI_MASK;
1160
	apic_write(APIC_TASKPRI, value);
L
Linus Torvalds 已提交
1161

1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	/*
	 * After a crash, we no longer service the interrupts and a pending
	 * interrupt from previous kernel might still have ISR bit set.
	 *
	 * Most probably by now CPU has serviced that pending interrupt and
	 * it might not have done the ack_APIC_irq() because it thought,
	 * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
	 * does not clear the ISR bit and cpu thinks it has already serivced
	 * the interrupt. Hence a vector might get locked. It was noticed
	 * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
	 */
	for (i = APIC_ISR_NR - 1; i >= 0; i--) {
		value = apic_read(APIC_ISR + i*0x10);
		for (j = 31; j >= 0; j--) {
			if (value & (1<<j))
				ack_APIC_irq();
		}
	}

L
Linus Torvalds 已提交
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
	/*
	 * Now that we are all set up, enable the APIC
	 */
	value = apic_read(APIC_SPIV);
	value &= ~APIC_VECTOR_MASK;
	/*
	 * Enable APIC
	 */
	value |= APIC_SPIV_APIC_ENABLED;

1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
#ifdef CONFIG_X86_32
	/*
	 * Some unknown Intel IO/APIC (or APIC) errata is biting us with
	 * certain networking cards. If high frequency interrupts are
	 * happening on a particular IOAPIC pin, plus the IOAPIC routing
	 * entry is masked/unmasked at a high rate as well then sooner or
	 * later IOAPIC line gets 'stuck', no more interrupts are received
	 * from the device. If focus CPU is disabled then the hang goes
	 * away, oh well :-(
	 *
	 * [ This bug can be reproduced easily with a level-triggered
	 *   PCI Ne2000 networking cards and PII/PIII processors, dual
	 *   BX chipset. ]
	 */
	/*
	 * Actually disabling the focus CPU check just makes the hang less
	 * frequent as it makes the interrupt distributon model be more
	 * like LRU than MRU (the short-term load is more even across CPUs).
	 * See also the comment in end_level_ioapic_irq().  --macro
	 */

	/*
	 * - enable focus processor (bit==0)
	 * - 64bit mode always use processor focus
	 *   so no need to set it
	 */
	value &= ~APIC_SPIV_FOCUS_DISABLED;
#endif
1219

L
Linus Torvalds 已提交
1220 1221 1222 1223
	/*
	 * Set spurious IRQ vector
	 */
	value |= SPURIOUS_APIC_VECTOR;
1224
	apic_write(APIC_SPIV, value);
L
Linus Torvalds 已提交
1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236

	/*
	 * Set up LVT0, LVT1:
	 *
	 * set up through-local-APIC on the BP's LINT0. This is not
	 * strictly necessary in pure symmetric-IO mode, but sometimes
	 * we delegate interrupts to the 8259A.
	 */
	/*
	 * TODO: set up through-local-APIC from through-I/O-APIC? --macro
	 */
	value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1237
	if (!smp_processor_id() && (pic_mode || !value)) {
L
Linus Torvalds 已提交
1238
		value = APIC_DM_EXTINT;
1239
		apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n",
1240
				smp_processor_id());
L
Linus Torvalds 已提交
1241 1242
	} else {
		value = APIC_DM_EXTINT | APIC_LVT_MASKED;
1243
		apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n",
1244
				smp_processor_id());
L
Linus Torvalds 已提交
1245
	}
1246
	apic_write(APIC_LVT0, value);
L
Linus Torvalds 已提交
1247 1248 1249 1250 1251 1252 1253 1254

	/*
	 * only the BP should see the LINT1 NMI signal, obviously.
	 */
	if (!smp_processor_id())
		value = APIC_DM_NMI;
	else
		value = APIC_DM_NMI | APIC_LVT_MASKED;
1255 1256
	if (!lapic_is_integrated())		/* 82489DX */
		value |= APIC_LVT_LEVEL_TRIGGER;
1257
	apic_write(APIC_LVT1, value);
1258

J
Jack Steiner 已提交
1259
	preempt_enable();
1260
}
L
Linus Torvalds 已提交
1261

1262 1263 1264
void __cpuinit end_local_APIC_setup(void)
{
	lapic_setup_esr();
1265 1266

#ifdef CONFIG_X86_32
1267 1268 1269 1270 1271 1272 1273
	{
		unsigned int value;
		/* Disable the local apic timer */
		value = apic_read(APIC_LVTT);
		value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
		apic_write(APIC_LVTT, value);
	}
1274 1275
#endif

1276
	setup_apic_nmi_watchdog(NULL);
1277
	apic_pm_activate();
L
Linus Torvalds 已提交
1278 1279
}

Y
Yinghai Lu 已提交
1280
#ifdef HAVE_X2APIC
1281 1282 1283 1284 1285 1286 1287
void check_x2apic(void)
{
	int msr, msr2;

	rdmsr(MSR_IA32_APICBASE, msr, msr2);

	if (msr & X2APIC_ENABLE) {
1288
		pr_info("x2apic enabled by BIOS, switching to x2apic ops\n");
1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299
		x2apic_preenabled = x2apic = 1;
		apic_ops = &x2apic_ops;
	}
}

void enable_x2apic(void)
{
	int msr, msr2;

	rdmsr(MSR_IA32_APICBASE, msr, msr2);
	if (!(msr & X2APIC_ENABLE)) {
1300
		pr_info("Enabling x2apic\n");
1301 1302 1303 1304
		wrmsr(MSR_IA32_APICBASE, msr | X2APIC_ENABLE, 0);
	}
}

A
Al Viro 已提交
1305
void __init enable_IR_x2apic(void)
1306 1307 1308 1309 1310 1311 1312 1313 1314
{
#ifdef CONFIG_INTR_REMAP
	int ret;
	unsigned long flags;

	if (!cpu_has_x2apic)
		return;

	if (!x2apic_preenabled && disable_x2apic) {
1315 1316
		pr_info("Skipped enabling x2apic and Interrupt-remapping "
			"because of nox2apic\n");
1317 1318 1319 1320 1321 1322 1323
		return;
	}

	if (x2apic_preenabled && disable_x2apic)
		panic("Bios already enabled x2apic, can't enforce nox2apic");

	if (!x2apic_preenabled && skip_ioapic_setup) {
1324 1325
		pr_info("Skipped enabling x2apic and Interrupt-remapping "
			"because of skipping io-apic setup\n");
1326 1327 1328 1329 1330
		return;
	}

	ret = dmar_table_init();
	if (ret) {
1331
		pr_info("dmar_table_init() failed with %d:\n", ret);
1332 1333 1334 1335

		if (x2apic_preenabled)
			panic("x2apic enabled by bios. But IR enabling failed");
		else
1336
			pr_info("Not enabling x2apic,Intr-remapping\n");
1337 1338 1339 1340 1341
		return;
	}

	local_irq_save(flags);
	mask_8259A();
1342 1343 1344

	ret = save_mask_IO_APIC_setup();
	if (ret) {
1345
		pr_info("Saving IO-APIC state failed: %d\n", ret);
1346 1347
		goto end;
	}
1348 1349 1350 1351 1352 1353 1354 1355 1356

	ret = enable_intr_remapping(1);

	if (ret && x2apic_preenabled) {
		local_irq_restore(flags);
		panic("x2apic enabled by bios. But IR enabling failed");
	}

	if (ret)
1357
		goto end_restore;
1358 1359 1360 1361 1362 1363

	if (!x2apic) {
		x2apic = 1;
		apic_ops = &x2apic_ops;
		enable_x2apic();
	}
1364 1365

end_restore:
1366 1367 1368 1369 1370 1371 1372 1373
	if (ret)
		/*
		 * IR enabling failed
		 */
		restore_IO_APIC_setup();
	else
		reinit_intr_remapped_IO_APIC(x2apic_preenabled);

1374
end:
1375 1376 1377 1378 1379
	unmask_8259A();
	local_irq_restore(flags);

	if (!ret) {
		if (!x2apic_preenabled)
1380
			pr_info("Enabled x2apic and interrupt-remapping\n");
1381
		else
1382
			pr_info("Enabled Interrupt-remapping\n");
1383
	} else
1384
		pr_err("Failed to enable Interrupt-remapping and x2apic\n");
1385 1386 1387 1388 1389 1390 1391 1392
#else
	if (!cpu_has_x2apic)
		return;

	if (x2apic_preenabled)
		panic("x2apic enabled prior OS handover,"
		      " enable CONFIG_INTR_REMAP");

1393 1394
	pr_info("Enable CONFIG_INTR_REMAP for enabling intr-remapping "
		" and x2apic\n");
1395 1396 1397 1398
#endif

	return;
}
Y
Yinghai Lu 已提交
1399
#endif /* HAVE_X2APIC */
1400

1401
#ifdef CONFIG_X86_64
L
Linus Torvalds 已提交
1402 1403 1404 1405
/*
 * Detect and enable local APICs on non-SMP boards.
 * Original code written by Keir Fraser.
 * On AMD64 we trust the BIOS - if it says no APIC it is likely
1406
 * not correctly set up (usually the APIC timer won't work etc.)
L
Linus Torvalds 已提交
1407
 */
1408
static int __init detect_init_APIC(void)
L
Linus Torvalds 已提交
1409 1410
{
	if (!cpu_has_apic) {
1411
		pr_info("No local APIC present\n");
L
Linus Torvalds 已提交
1412 1413 1414 1415
		return -1;
	}

	mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
1416
	boot_cpu_physical_apicid = 0;
L
Linus Torvalds 已提交
1417 1418
	return 0;
}
1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
#else
/*
 * Detect and initialize APIC
 */
static int __init detect_init_APIC(void)
{
	u32 h, l, features;

	/* Disabled by kernel option? */
	if (disable_apic)
		return -1;

	switch (boot_cpu_data.x86_vendor) {
	case X86_VENDOR_AMD:
		if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
		    (boot_cpu_data.x86 == 15))
			break;
		goto no_apic;
	case X86_VENDOR_INTEL:
		if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
		    (boot_cpu_data.x86 == 5 && cpu_has_apic))
			break;
		goto no_apic;
	default:
		goto no_apic;
	}

	if (!cpu_has_apic) {
		/*
		 * Over-ride BIOS and try to enable the local APIC only if
		 * "lapic" specified.
		 */
		if (!force_enable_local_apic) {
1452 1453
			pr_info("Local APIC disabled by BIOS -- "
				"you can enable it with \"lapic\"\n");
1454 1455 1456 1457 1458 1459 1460 1461 1462
			return -1;
		}
		/*
		 * Some BIOSes disable the local APIC in the APIC_BASE
		 * MSR. This can only be done in software for Intel P6 or later
		 * and AMD K7 (Model > 1) or later.
		 */
		rdmsr(MSR_IA32_APICBASE, l, h);
		if (!(l & MSR_IA32_APICBASE_ENABLE)) {
1463
			pr_info("Local APIC disabled by BIOS -- reenabling.\n");
1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475
			l &= ~MSR_IA32_APICBASE_BASE;
			l |= MSR_IA32_APICBASE_ENABLE | APIC_DEFAULT_PHYS_BASE;
			wrmsr(MSR_IA32_APICBASE, l, h);
			enabled_via_apicbase = 1;
		}
	}
	/*
	 * The APIC feature bit should now be enabled
	 * in `cpuid'
	 */
	features = cpuid_edx(1);
	if (!(features & (1 << X86_FEATURE_APIC))) {
1476
		pr_warning("Could not enable APIC!\n");
1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
		return -1;
	}
	set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
	mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;

	/* The BIOS may have set up the APIC at some other address */
	rdmsr(MSR_IA32_APICBASE, l, h);
	if (l & MSR_IA32_APICBASE_ENABLE)
		mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;

1487
	pr_info("Found and enabled local APIC!\n");
1488 1489 1490 1491 1492 1493

	apic_pm_activate();

	return 0;

no_apic:
1494
	pr_info("No local APIC present or hardware disabled\n");
1495 1496 1497
	return -1;
}
#endif
L
Linus Torvalds 已提交
1498

Y
Yinghai Lu 已提交
1499
#ifdef CONFIG_X86_64
1500 1501
void __init early_init_lapic_mapping(void)
{
1502
	unsigned long phys_addr;
1503 1504 1505 1506 1507 1508 1509 1510

	/*
	 * If no local APIC can be found then go out
	 * : it means there is no mpatable and MADT
	 */
	if (!smp_found_config)
		return;

1511
	phys_addr = mp_lapic_addr;
1512

1513
	set_fixmap_nocache(FIX_APIC_BASE, phys_addr);
1514
	apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
1515
		    APIC_BASE, phys_addr);
1516 1517 1518 1519 1520

	/*
	 * Fetch the APIC ID of the BSP in case we have a
	 * default configuration (or the MP table is broken).
	 */
1521
	boot_cpu_physical_apicid = read_apic_id();
1522
}
Y
Yinghai Lu 已提交
1523
#endif
1524

1525 1526 1527
/**
 * init_apic_mappings - initialize APIC mappings
 */
L
Linus Torvalds 已提交
1528 1529
void __init init_apic_mappings(void)
{
Y
Yinghai Lu 已提交
1530
#ifdef HAVE_X2APIC
1531
	if (x2apic) {
1532
		boot_cpu_physical_apicid = read_apic_id();
1533 1534
		return;
	}
Y
Yinghai Lu 已提交
1535
#endif
1536

L
Linus Torvalds 已提交
1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548
	/*
	 * If no local APIC can be found then set up a fake all
	 * zeroes page to simulate the local APIC and another
	 * one for the IO-APIC.
	 */
	if (!smp_found_config && detect_init_APIC()) {
		apic_phys = (unsigned long) alloc_bootmem_pages(PAGE_SIZE);
		apic_phys = __pa(apic_phys);
	} else
		apic_phys = mp_lapic_addr;

	set_fixmap_nocache(FIX_APIC_BASE, apic_phys);
1549
	apic_printk(APIC_VERBOSE, "mapped APIC to %08lx (%08lx)\n",
Y
Yinghai Lu 已提交
1550
				APIC_BASE, apic_phys);
L
Linus Torvalds 已提交
1551 1552 1553 1554 1555

	/*
	 * Fetch the APIC ID of the BSP in case we have a
	 * default configuration (or the MP table is broken).
	 */
Y
Yinghai Lu 已提交
1556 1557
	if (boot_cpu_physical_apicid == -1U)
		boot_cpu_physical_apicid = read_apic_id();
L
Linus Torvalds 已提交
1558 1559 1560
}

/*
1561 1562
 * This initializes the IO-APIC and APIC hardware if this is
 * a UP kernel.
L
Linus Torvalds 已提交
1563
 */
1564 1565
int apic_version[MAX_APICS];

1566
int __init APIC_init_uniprocessor(void)
L
Linus Torvalds 已提交
1567
{
Y
Yinghai Lu 已提交
1568
#ifdef CONFIG_X86_64
1569
	if (disable_apic) {
1570
		pr_info("Apic disabled\n");
1571 1572 1573 1574
		return -1;
	}
	if (!cpu_has_apic) {
		disable_apic = 1;
1575
		pr_info("Apic disabled by BIOS\n");
1576 1577
		return -1;
	}
Y
Yinghai Lu 已提交
1578 1579 1580 1581 1582 1583 1584 1585 1586
#else
	if (!smp_found_config && !cpu_has_apic)
		return -1;

	/*
	 * Complain if the BIOS pretends there is one.
	 */
	if (!cpu_has_apic &&
	    APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
1587 1588
		pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
			boot_cpu_physical_apicid);
Y
Yinghai Lu 已提交
1589 1590 1591 1592 1593
		clear_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
		return -1;
	}
#endif

Y
Yinghai Lu 已提交
1594
#ifdef HAVE_X2APIC
1595
	enable_IR_x2apic();
Y
Yinghai Lu 已提交
1596
#endif
Y
Yinghai Lu 已提交
1597
#ifdef CONFIG_X86_64
1598
	setup_apic_routing();
Y
Yinghai Lu 已提交
1599
#endif
1600

1601
	verify_local_APIC();
1602 1603
	connect_bsp_APIC();

Y
Yinghai Lu 已提交
1604
#ifdef CONFIG_X86_64
1605
	apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid));
Y
Yinghai Lu 已提交
1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616
#else
	/*
	 * Hack: In case of kdump, after a crash, kernel might be booting
	 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
	 * might be zero if read from MP tables. Get it from LAPIC.
	 */
# ifdef CONFIG_CRASH_DUMP
	boot_cpu_physical_apicid = read_apic_id();
# endif
#endif
	physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1617
	setup_local_APIC();
L
Linus Torvalds 已提交
1618

Y
Yinghai Lu 已提交
1619
#ifdef CONFIG_X86_64
1620 1621 1622 1623 1624 1625
	/*
	 * Now enable IO-APICs, actually call clear_IO_APIC
	 * We need clear_IO_APIC before enabling vector on BP
	 */
	if (!skip_ioapic_setup && nr_ioapics)
		enable_IO_APIC();
Y
Yinghai Lu 已提交
1626
#endif
1627

Y
Yinghai Lu 已提交
1628
#ifdef CONFIG_X86_IO_APIC
1629
	if (!smp_found_config || skip_ioapic_setup || !nr_ioapics)
Y
Yinghai Lu 已提交
1630
#endif
1631
		localise_nmi_watchdog();
1632 1633
	end_local_APIC_setup();

Y
Yinghai Lu 已提交
1634
#ifdef CONFIG_X86_IO_APIC
1635 1636
	if (smp_found_config && !skip_ioapic_setup && nr_ioapics)
		setup_IO_APIC();
Y
Yinghai Lu 已提交
1637
# ifdef CONFIG_X86_64
1638 1639
	else
		nr_ioapics = 0;
Y
Yinghai Lu 已提交
1640 1641 1642 1643
# endif
#endif

#ifdef CONFIG_X86_64
1644 1645
	setup_boot_APIC_clock();
	check_nmi_watchdog();
Y
Yinghai Lu 已提交
1646 1647 1648 1649
#else
	setup_boot_clock();
#endif

1650
	return 0;
L
Linus Torvalds 已提交
1651 1652 1653
}

/*
1654
 * Local APIC interrupts
L
Linus Torvalds 已提交
1655 1656
 */

1657 1658 1659
/*
 * This interrupt should _never_ happen with our APIC/SMP architecture
 */
1660
void smp_spurious_interrupt(struct pt_regs *regs)
L
Linus Torvalds 已提交
1661
{
1662 1663
	u32 v;

1664 1665
	exit_idle();
	irq_enter();
L
Linus Torvalds 已提交
1666
	/*
1667 1668 1669
	 * Check if this really is a spurious interrupt and ACK it
	 * if it is a vectored one.  Just in case...
	 * Spurious interrupts should not be ACKed.
L
Linus Torvalds 已提交
1670
	 */
1671 1672 1673
	v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1));
	if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
		ack_APIC_irq();
1674

1675 1676
	inc_irq_stat(irq_spurious_count);

1677
	/* see sw-dev-man vol 3, chapter 7.4.13.5 */
1678 1679
	pr_info("spurious APIC interrupt on CPU#%d, "
		"should never happen.\n", smp_processor_id());
1680 1681
	irq_exit();
}
L
Linus Torvalds 已提交
1682

1683 1684 1685
/*
 * This interrupt should never happen with our APIC/SMP architecture
 */
1686
void smp_error_interrupt(struct pt_regs *regs)
1687
{
1688
	u32 v, v1;
L
Linus Torvalds 已提交
1689

1690 1691 1692 1693 1694 1695 1696 1697
	exit_idle();
	irq_enter();
	/* First tickle the hardware, only then report what went on. -- REW */
	v = apic_read(APIC_ESR);
	apic_write(APIC_ESR, 0);
	v1 = apic_read(APIC_ESR);
	ack_APIC_irq();
	atomic_inc(&irq_err_count);
1698

1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710
	/*
	 * Here is what the APIC error bits mean:
	 * 0: Send CS error
	 * 1: Receive CS error
	 * 2: Send accept error
	 * 3: Receive accept error
	 * 4: Reserved
	 * 5: Send illegal vector
	 * 6: Received illegal vector
	 * 7: Illegal register address
	 */
	pr_debug("APIC error on CPU%d: %02x(%02x)\n",
1711 1712
		smp_processor_id(), v , v1);
	irq_exit();
L
Linus Torvalds 已提交
1713 1714
}

1715
/**
1716 1717
 * connect_bsp_APIC - attach the APIC to the interrupt system
 */
1718 1719
void __init connect_bsp_APIC(void)
{
1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735
#ifdef CONFIG_X86_32
	if (pic_mode) {
		/*
		 * Do not trust the local APIC being empty at bootup.
		 */
		clear_local_APIC();
		/*
		 * PIC mode, enable APIC mode in the IMCR, i.e.  connect BSP's
		 * local APIC to INT and NMI lines.
		 */
		apic_printk(APIC_VERBOSE, "leaving PIC mode, "
				"enabling APIC mode.\n");
		outb(0x70, 0x22);
		outb(0x01, 0x23);
	}
#endif
1736 1737 1738
	enable_apic_mode();
}

1739 1740 1741 1742 1743 1744 1745
/**
 * disconnect_bsp_APIC - detach the APIC from the interrupt system
 * @virt_wire_setup:	indicates, whether virtual wire mode is selected
 *
 * Virtual wire mode is necessary to deliver legacy interrupts even when the
 * APIC is disabled.
 */
1746
void disconnect_bsp_APIC(int virt_wire_setup)
L
Linus Torvalds 已提交
1747
{
1748 1749
	unsigned int value;

1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765
#ifdef CONFIG_X86_32
	if (pic_mode) {
		/*
		 * Put the board back into PIC mode (has an effect only on
		 * certain older boards).  Note that APIC interrupts, including
		 * IPIs, won't work beyond this point!  The only exception are
		 * INIT IPIs.
		 */
		apic_printk(APIC_VERBOSE, "disabling APIC mode, "
				"entering PIC mode.\n");
		outb(0x70, 0x22);
		outb(0x00, 0x23);
		return;
	}
#endif

1766
	/* Go back to Virtual Wire compatibility mode */
L
Linus Torvalds 已提交
1767

1768 1769 1770 1771 1772 1773
	/* For the spurious interrupt use vector F, and enable it */
	value = apic_read(APIC_SPIV);
	value &= ~APIC_VECTOR_MASK;
	value |= APIC_SPIV_APIC_ENABLED;
	value |= 0xf;
	apic_write(APIC_SPIV, value);
T
Thomas Gleixner 已提交
1774

1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790
	if (!virt_wire_setup) {
		/*
		 * For LVT0 make it edge triggered, active high,
		 * external and enabled
		 */
		value = apic_read(APIC_LVT0);
		value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
			APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
			APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
		value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
		value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
		apic_write(APIC_LVT0, value);
	} else {
		/* Disable LVT0 */
		apic_write(APIC_LVT0, APIC_LVT_MASKED);
	}
T
Thomas Gleixner 已提交
1791

1792 1793 1794 1795
	/*
	 * For LVT1 make it edge triggered, active high,
	 * nmi and enabled
	 */
1796 1797 1798 1799 1800 1801 1802
	value = apic_read(APIC_LVT1);
	value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
			APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
			APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
	value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
	value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
	apic_write(APIC_LVT1, value);
L
Linus Torvalds 已提交
1803 1804
}

1805 1806 1807 1808
void __cpuinit generic_processor_info(int apicid, int version)
{
	int cpu;

1809 1810 1811 1812
	/*
	 * Validate version
	 */
	if (version == 0x0) {
1813
		pr_warning("BIOS bug, APIC version is 0 for CPU#%d! "
1814 1815
			   "fixing up to 0x10. (tell your hw vendor)\n",
				version);
1816
		version = 0x10;
1817
	}
1818
	apic_version[apicid] = version;
1819

1820 1821 1822 1823 1824 1825 1826 1827 1828
	if (num_processors >= nr_cpu_ids) {
		int max = nr_cpu_ids;
		int thiscpu = max + disabled_cpus;

		pr_warning(
			"ACPI: NR_CPUS/possible_cpus limit of %i reached."
			"  Processor %d/0x%x ignored.\n", max, thiscpu, apicid);

		disabled_cpus++;
1829 1830 1831 1832
		return;
	}

	num_processors++;
1833
	cpu = cpumask_next_zero(-1, cpu_present_mask);
1834 1835 1836 1837 1838 1839 1840 1841 1842 1843

	physid_set(apicid, phys_cpu_present_map);
	if (apicid == boot_cpu_physical_apicid) {
		/*
		 * x86_bios_cpu_apicid is required to have processors listed
		 * in same order as logical cpu numbers. Hence the first
		 * entry is BSP, and so on.
		 */
		cpu = 0;
	}
1844 1845 1846
	if (apicid > max_physical_apicid)
		max_physical_apicid = apicid;

1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
#ifdef CONFIG_X86_32
	/*
	 * Would be preferable to switch to bigsmp when CONFIG_HOTPLUG_CPU=y
	 * but we need to work other dependencies like SMP_SUSPEND etc
	 * before this can be done without some confusion.
	 * if (CPU_HOTPLUG_ENABLED || num_processors > 8)
	 *       - Ashok Raj <ashok.raj@intel.com>
	 */
	if (max_physical_apicid >= 8) {
		switch (boot_cpu_data.x86_vendor) {
		case X86_VENDOR_INTEL:
			if (!APIC_XAPIC(version)) {
				def_to_bigsmp = 0;
				break;
			}
			/* If P4 and above fall through */
		case X86_VENDOR_AMD:
			def_to_bigsmp = 1;
		}
	}
#endif

#if defined(CONFIG_X86_SMP) || defined(CONFIG_X86_64)
1870
	/* are we being called early in kernel startup? */
1871 1872 1873
	if (early_per_cpu_ptr(x86_cpu_to_apicid)) {
		u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
		u16 *bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
1874 1875 1876 1877 1878 1879 1880

		cpu_to_apicid[cpu] = apicid;
		bios_cpu_apicid[cpu] = apicid;
	} else {
		per_cpu(x86_cpu_to_apicid, cpu) = apicid;
		per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
	}
1881
#endif
1882

1883 1884
	set_cpu_possible(cpu, true);
	set_cpu_present(cpu, true);
1885 1886
}

1887
#ifdef CONFIG_X86_64
1888 1889 1890 1891
int hard_smp_processor_id(void)
{
	return read_apic_id();
}
1892
#endif
1893

1894
/*
1895
 * Power management
1896
 */
1897 1898 1899
#ifdef CONFIG_PM

static struct {
1900 1901 1902 1903 1904
	/*
	 * 'active' is true if the local APIC was enabled by us and
	 * not the BIOS; this signifies that we are also responsible
	 * for disabling it before entering apm/acpi suspend
	 */
1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925
	int active;
	/* r/w apic fields */
	unsigned int apic_id;
	unsigned int apic_taskpri;
	unsigned int apic_ldr;
	unsigned int apic_dfr;
	unsigned int apic_spiv;
	unsigned int apic_lvtt;
	unsigned int apic_lvtpc;
	unsigned int apic_lvt0;
	unsigned int apic_lvt1;
	unsigned int apic_lvterr;
	unsigned int apic_tmict;
	unsigned int apic_tdcr;
	unsigned int apic_thmr;
} apic_pm_state;

static int lapic_suspend(struct sys_device *dev, pm_message_t state)
{
	unsigned long flags;
	int maxlvt;
1926

1927 1928
	if (!apic_pm_state.active)
		return 0;
1929

1930
	maxlvt = lapic_get_maxlvt();
1931

1932
	apic_pm_state.apic_id = apic_read(APIC_ID);
1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944
	apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
	apic_pm_state.apic_ldr = apic_read(APIC_LDR);
	apic_pm_state.apic_dfr = apic_read(APIC_DFR);
	apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
	apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
	if (maxlvt >= 4)
		apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
	apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
	apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
	apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
	apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
	apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
1945
#if defined(CONFIG_X86_MCE_P4THERMAL) || defined(CONFIG_X86_MCE_INTEL)
1946 1947 1948
	if (maxlvt >= 5)
		apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
#endif
1949

1950 1951 1952 1953
	local_irq_save(flags);
	disable_local_APIC();
	local_irq_restore(flags);
	return 0;
L
Linus Torvalds 已提交
1954 1955
}

1956
static int lapic_resume(struct sys_device *dev)
L
Linus Torvalds 已提交
1957
{
1958 1959 1960
	unsigned int l, h;
	unsigned long flags;
	int maxlvt;
L
Linus Torvalds 已提交
1961

1962 1963
	if (!apic_pm_state.active)
		return 0;
1964

1965
	maxlvt = lapic_get_maxlvt();
L
Linus Torvalds 已提交
1966

1967
	local_irq_save(flags);
C
Cyrill Gorcunov 已提交
1968

Y
Yinghai Lu 已提交
1969
#ifdef HAVE_X2APIC
C
Cyrill Gorcunov 已提交
1970 1971 1972 1973
	if (x2apic)
		enable_x2apic();
	else
#endif
1974
	{
C
Cyrill Gorcunov 已提交
1975 1976 1977 1978 1979 1980
		/*
		 * Make sure the APICBASE points to the right address
		 *
		 * FIXME! This will be wrong if we ever support suspend on
		 * SMP! We'll need to do this as part of the CPU restore!
		 */
1981 1982 1983 1984
		rdmsr(MSR_IA32_APICBASE, l, h);
		l &= ~MSR_IA32_APICBASE_BASE;
		l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
		wrmsr(MSR_IA32_APICBASE, l, h);
1985
	}
1986

1987 1988 1989 1990 1991 1992 1993 1994
	apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
	apic_write(APIC_ID, apic_pm_state.apic_id);
	apic_write(APIC_DFR, apic_pm_state.apic_dfr);
	apic_write(APIC_LDR, apic_pm_state.apic_ldr);
	apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
	apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
	apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
	apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
C
Cyrill Gorcunov 已提交
1995
#if defined(CONFIG_X86_MCE_P4THERMAL) || defined(CONFIG_X86_MCE_INTEL)
1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008
	if (maxlvt >= 5)
		apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
#endif
	if (maxlvt >= 4)
		apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
	apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
	apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
	apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
	apic_write(APIC_ESR, 0);
	apic_read(APIC_ESR);
	apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
	apic_write(APIC_ESR, 0);
	apic_read(APIC_ESR);
C
Cyrill Gorcunov 已提交
2009

2010
	local_irq_restore(flags);
C
Cyrill Gorcunov 已提交
2011

2012 2013
	return 0;
}
T
Thomas Gleixner 已提交
2014

2015 2016 2017 2018 2019
/*
 * This device has no shutdown method - fully functioning local APICs
 * are needed on every CPU up until machine_halt/restart/poweroff.
 */

2020 2021 2022 2023 2024
static struct sysdev_class lapic_sysclass = {
	.name		= "lapic",
	.resume		= lapic_resume,
	.suspend	= lapic_suspend,
};
T
Thomas Gleixner 已提交
2025

2026
static struct sys_device device_lapic = {
H
Hiroshi Shimamoto 已提交
2027 2028
	.id	= 0,
	.cls	= &lapic_sysclass,
2029
};
T
Thomas Gleixner 已提交
2030

2031 2032 2033
static void __cpuinit apic_pm_activate(void)
{
	apic_pm_state.active = 1;
L
Linus Torvalds 已提交
2034 2035
}

2036
static int __init init_lapic_sysfs(void)
L
Linus Torvalds 已提交
2037
{
2038
	int error;
H
Hiroshi Shimamoto 已提交
2039

2040 2041 2042
	if (!cpu_has_apic)
		return 0;
	/* XXX: remove suspend/resume procs if !apic_pm_state.active? */
H
Hiroshi Shimamoto 已提交
2043

2044 2045 2046 2047
	error = sysdev_class_register(&lapic_sysclass);
	if (!error)
		error = sysdev_register(&device_lapic);
	return error;
L
Linus Torvalds 已提交
2048
}
2049 2050 2051 2052 2053 2054 2055
device_initcall(init_lapic_sysfs);

#else	/* CONFIG_PM */

static void apic_pm_activate(void) { }

#endif	/* CONFIG_PM */
L
Linus Torvalds 已提交
2056

Y
Yinghai Lu 已提交
2057
#ifdef CONFIG_X86_64
L
Linus Torvalds 已提交
2058
/*
2059
 * apic_is_clustered_box() -- Check if we can expect good TSC
L
Linus Torvalds 已提交
2060 2061 2062
 *
 * Thus far, the major user of this is IBM's Summit2 series:
 *
2063
 * Clustered boxes may have unsynced TSC problems if they are
L
Linus Torvalds 已提交
2064 2065 2066
 * multi-chassis. Use available data to take a good guess.
 * If in doubt, go HPET.
 */
2067
__cpuinit int apic_is_clustered_box(void)
L
Linus Torvalds 已提交
2068 2069 2070
{
	int i, clusters, zeros;
	unsigned id;
2071
	u16 *bios_cpu_apicid;
L
Linus Torvalds 已提交
2072 2073
	DECLARE_BITMAP(clustermap, NUM_APIC_CLUSTERS);

2074 2075 2076 2077
	/*
	 * there is not this kind of box with AMD CPU yet.
	 * Some AMD box with quadcore cpu and 8 sockets apicid
	 * will be [4, 0x23] or [8, 0x27] could be thought to
Y
Yinghai Lu 已提交
2078
	 * vsmp box still need checking...
2079
	 */
2080
	if ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && !is_vsmp_box())
2081 2082
		return 0;

2083
	bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
2084
	bitmap_zero(clustermap, NUM_APIC_CLUSTERS);
L
Linus Torvalds 已提交
2085

2086
	for (i = 0; i < nr_cpu_ids; i++) {
2087
		/* are we being called early in kernel startup? */
2088 2089
		if (bios_cpu_apicid) {
			id = bios_cpu_apicid[i];
2090 2091 2092 2093 2094 2095 2096 2097 2098 2099
		}
		else if (i < nr_cpu_ids) {
			if (cpu_present(i))
				id = per_cpu(x86_bios_cpu_apicid, i);
			else
				continue;
		}
		else
			break;

L
Linus Torvalds 已提交
2100 2101 2102 2103 2104 2105
		if (id != BAD_APICID)
			__set_bit(APIC_CLUSTERID(id), clustermap);
	}

	/* Problem:  Partially populated chassis may not have CPUs in some of
	 * the APIC clusters they have been allocated.  Only present CPUs have
2106 2107 2108
	 * x86_bios_cpu_apicid entries, thus causing zeroes in the bitmap.
	 * Since clusters are allocated sequentially, count zeros only if
	 * they are bounded by ones.
L
Linus Torvalds 已提交
2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119
	 */
	clusters = 0;
	zeros = 0;
	for (i = 0; i < NUM_APIC_CLUSTERS; i++) {
		if (test_bit(i, clustermap)) {
			clusters += 1 + zeros;
			zeros = 0;
		} else
			++zeros;
	}

2120 2121 2122 2123 2124 2125
	/* ScaleMP vSMPowered boxes have one cluster per board and TSCs are
	 * not guaranteed to be synced between boards
	 */
	if (is_vsmp_box() && clusters > 1)
		return 1;

L
Linus Torvalds 已提交
2126
	/*
2127
	 * If clusters > 2, then should be multi-chassis.
L
Linus Torvalds 已提交
2128 2129 2130 2131 2132
	 * May have to revisit this when multi-core + hyperthreaded CPUs come
	 * out, but AFAIK this will work even for them.
	 */
	return (clusters > 2);
}
Y
Yinghai Lu 已提交
2133
#endif
L
Linus Torvalds 已提交
2134 2135

/*
2136
 * APIC command line parameters
L
Linus Torvalds 已提交
2137
 */
2138
static int __init setup_disableapic(char *arg)
2139
{
L
Linus Torvalds 已提交
2140
	disable_apic = 1;
2141
	setup_clear_cpu_cap(X86_FEATURE_APIC);
2142 2143 2144
	return 0;
}
early_param("disableapic", setup_disableapic);
L
Linus Torvalds 已提交
2145

2146
/* same as disableapic, for compatibility */
2147
static int __init setup_nolapic(char *arg)
2148
{
2149
	return setup_disableapic(arg);
2150
}
2151
early_param("nolapic", setup_nolapic);
L
Linus Torvalds 已提交
2152

2153 2154 2155 2156 2157 2158 2159
static int __init parse_lapic_timer_c2_ok(char *arg)
{
	local_apic_timer_c2_ok = 1;
	return 0;
}
early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);

2160
static int __init parse_disable_apic_timer(char *arg)
2161
{
L
Linus Torvalds 已提交
2162
	disable_apic_timer = 1;
2163
	return 0;
2164
}
2165 2166 2167 2168 2169 2170
early_param("noapictimer", parse_disable_apic_timer);

static int __init parse_nolapic_timer(char *arg)
{
	disable_apic_timer = 1;
	return 0;
2171
}
2172
early_param("nolapic_timer", parse_nolapic_timer);
2173

2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188
static int __init apic_set_verbosity(char *arg)
{
	if (!arg)  {
#ifdef CONFIG_X86_64
		skip_ioapic_setup = 0;
		return 0;
#endif
		return -EINVAL;
	}

	if (strcmp("debug", arg) == 0)
		apic_verbosity = APIC_DEBUG;
	else if (strcmp("verbose", arg) == 0)
		apic_verbosity = APIC_VERBOSE;
	else {
2189
		pr_warning("APIC Verbosity level %s not recognised"
2190 2191 2192 2193 2194 2195 2196 2197
			" use apic=verbose or apic=debug\n", arg);
		return -EINVAL;
	}

	return 0;
}
early_param("apic", apic_set_verbosity);

2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215
static int __init lapic_insert_resource(void)
{
	if (!apic_phys)
		return -1;

	/* Put local APIC into the resource map. */
	lapic_resource.start = apic_phys;
	lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
	insert_resource(&iomem_resource, &lapic_resource);

	return 0;
}

/*
 * need call insert after e820_reserve_resources()
 * that is using request_resource
 */
late_initcall(lapic_insert_resource);