io_apic.c 96.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 *	Intel IO-APIC support for multi-Pentium hosts.
 *
I
Ingo Molnar 已提交
4
 *	Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
 *
 *	Many thanks to Stig Venaas for trying out countless experimental
 *	patches and reporting/debugging problems patiently!
 *
 *	(c) 1999, Multiple IO-APIC support, developed by
 *	Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
 *      Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
 *	further tested and cleaned up by Zach Brown <zab@redhat.com>
 *	and Ingo Molnar <mingo@redhat.com>
 *
 *	Fixes
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
 *					thanks to Eric Gilmore
 *					and Rolf G. Tews
 *					for testing these extensively
 *	Paul Diefenbaugh	:	Added full ACPI support
 */

#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/sched.h>
28
#include <linux/pci.h>
L
Linus Torvalds 已提交
29 30 31
#include <linux/mc146818rtc.h>
#include <linux/compiler.h>
#include <linux/acpi.h>
32
#include <linux/module.h>
33
#include <linux/syscore_ops.h>
34
#include <linux/msi.h>
35
#include <linux/htirq.h>
36
#include <linux/freezer.h>
37
#include <linux/kthread.h>
38
#include <linux/jiffies.h>	/* time_after() */
39
#include <linux/slab.h>
40 41 42 43 44
#ifdef CONFIG_ACPI
#include <acpi/acpi_bus.h>
#endif
#include <linux/bootmem.h>
#include <linux/dmar.h>
45
#include <linux/hpet.h>
46

47
#include <asm/idle.h>
L
Linus Torvalds 已提交
48 49
#include <asm/io.h>
#include <asm/smp.h>
50
#include <asm/cpu.h>
L
Linus Torvalds 已提交
51
#include <asm/desc.h>
52 53 54
#include <asm/proto.h>
#include <asm/acpi.h>
#include <asm/dma.h>
L
Linus Torvalds 已提交
55
#include <asm/timer.h>
56
#include <asm/i8259.h>
57
#include <asm/msidef.h>
58
#include <asm/hypertransport.h>
59
#include <asm/setup.h>
60
#include <asm/intr_remapping.h>
61
#include <asm/hpet.h>
62
#include <asm/hw_irq.h>
L
Linus Torvalds 已提交
63

I
Ingo Molnar 已提交
64
#include <asm/apic.h>
L
Linus Torvalds 已提交
65

66
#define __apicdebuginit(type) static type __init
67

68 69
#define for_each_irq_pin(entry, head) \
	for (entry = head; entry; entry = entry->next)
70

71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
static void		__init __ioapic_init_mappings(void);

static unsigned int	__io_apic_read  (unsigned int apic, unsigned int reg);
static void		__io_apic_write (unsigned int apic, unsigned int reg, unsigned int val);
static void		__io_apic_modify(unsigned int apic, unsigned int reg, unsigned int val);

static struct io_apic_ops io_apic_ops = {
	.init	= __ioapic_init_mappings,
	.read	= __io_apic_read,
	.write	= __io_apic_write,
	.modify = __io_apic_modify,
};

void __init set_io_apic_ops(const struct io_apic_ops *ops)
{
	io_apic_ops = *ops;
}

89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
#ifdef CONFIG_IRQ_REMAP
static void irq_remap_modify_chip_defaults(struct irq_chip *chip);
static inline bool irq_remapped(struct irq_cfg *cfg)
{
	return cfg->irq_2_iommu.iommu != NULL;
}
#else
static inline bool irq_remapped(struct irq_cfg *cfg)
{
	return false;
}
static inline void irq_remap_modify_chip_defaults(struct irq_chip *chip)
{
}
#endif

L
Linus Torvalds 已提交
105
/*
106 107
 *      Is the SiS APIC rmw bug present ?
 *      -1 = don't know, 0 = no, 1 = yes
L
Linus Torvalds 已提交
108 109 110
 */
int sis_apic_bug = -1;

111 112
static DEFINE_RAW_SPINLOCK(ioapic_lock);
static DEFINE_RAW_SPINLOCK(vector_lock);
Y
Yinghai Lu 已提交
113

S
Suresh Siddha 已提交
114 115 116 117 118
static struct ioapic {
	/*
	 * # of IRQ routing registers
	 */
	int nr_registers;
119 120 121 122
	/*
	 * Saved state during suspend/resume, or while enabling intr-remap.
	 */
	struct IO_APIC_route_entry *saved_registers;
123 124
	/* I/O APIC config */
	struct mpc_ioapic mp_config;
125 126
	/* IO APIC gsi routing info */
	struct mp_ioapic_gsi  gsi_config;
127
	DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
S
Suresh Siddha 已提交
128
} ioapics[MAX_IO_APICS];
L
Linus Torvalds 已提交
129

130
#define mpc_ioapic_ver(ioapic_idx)	ioapics[ioapic_idx].mp_config.apicver
131

132
int mpc_ioapic_id(int ioapic_idx)
133
{
134
	return ioapics[ioapic_idx].mp_config.apicid;
135 136
}

137
unsigned int mpc_ioapic_addr(int ioapic_idx)
138
{
139
	return ioapics[ioapic_idx].mp_config.apicaddr;
140 141
}

142
struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic_idx)
143
{
144
	return &ioapics[ioapic_idx].gsi_config;
145
}
146

147
int nr_ioapics;
148

149 150
/* The one past the highest gsi number used */
u32 gsi_top;
151

152
/* MP IRQ source entries */
153
struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
154 155 156 157

/* # of MP IRQ source entries */
int mp_irq_entries;

158 159 160
/* GSI interrupts */
static int nr_irqs_gsi = NR_IRQS_LEGACY;

161 162 163 164 165 166
#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
int mp_bus_id_to_type[MAX_MP_BUSSES];
#endif

DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);

Y
Yinghai Lu 已提交
167 168
int skip_ioapic_setup;

169 170 171 172
/**
 * disable_ioapic_support() - disables ioapic support at runtime
 */
void disable_ioapic_support(void)
173 174 175 176 177 178 179 180
{
#ifdef CONFIG_PCI
	noioapicquirk = 1;
	noioapicreroute = -1;
#endif
	skip_ioapic_setup = 1;
}

181
static int __init parse_noapic(char *str)
Y
Yinghai Lu 已提交
182 183
{
	/* disable IO-APIC */
184
	disable_ioapic_support();
Y
Yinghai Lu 已提交
185 186 187
	return 0;
}
early_param("noapic", parse_noapic);
188

189 190
static int io_apic_setup_irq_pin(unsigned int irq, int node,
				 struct io_apic_irq_attr *attr);
191

192 193 194 195 196 197 198 199 200 201 202
/* Will be called in mpparse/acpi/sfi codes for saving IRQ info */
void mp_save_irq(struct mpc_intsrc *m)
{
	int i;

	apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x,"
		" IRQ %02x, APIC ID %x, APIC INT %02x\n",
		m->irqtype, m->irqflag & 3, (m->irqflag >> 2) & 3, m->srcbus,
		m->srcbusirq, m->dstapic, m->dstirq);

	for (i = 0; i < mp_irq_entries; i++) {
203
		if (!memcmp(&mp_irqs[i], m, sizeof(*m)))
204 205 206
			return;
	}

207
	memcpy(&mp_irqs[mp_irq_entries], m, sizeof(*m));
208 209 210 211
	if (++mp_irq_entries == MAX_IRQ_SOURCES)
		panic("Max # of irq sources exceeded!!\n");
}

212 213 214 215 216
struct irq_pin_list {
	int apic, pin;
	struct irq_pin_list *next;
};

T
Thomas Gleixner 已提交
217
static struct irq_pin_list *alloc_irq_pin_list(int node)
218
{
219
	return kzalloc_node(sizeof(struct irq_pin_list), GFP_KERNEL, node);
220 221
}

222

Y
Yinghai Lu 已提交
223
/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
224
static struct irq_cfg irq_cfgx[NR_IRQS_LEGACY];
Y
Yinghai Lu 已提交
225

226
int __init arch_early_irq_init(void)
227
{
228
	struct irq_cfg *cfg;
229
	int count, node, i;
T
Thomas Gleixner 已提交
230

231
	if (!legacy_pic->nr_legacy_irqs)
232 233
		io_apic_irqs = ~0UL;

234
	for (i = 0; i < nr_ioapics; i++) {
235
		ioapics[i].saved_registers =
236
			kzalloc(sizeof(struct IO_APIC_route_entry) *
S
Suresh Siddha 已提交
237
				ioapics[i].nr_registers, GFP_KERNEL);
238
		if (!ioapics[i].saved_registers)
239 240 241
			pr_err("IOAPIC %d: suspend/resume impossible!\n", i);
	}

242 243
	cfg = irq_cfgx;
	count = ARRAY_SIZE(irq_cfgx);
244
	node = cpu_to_node(0);
245

246 247 248
	/* Make sure the legacy interrupts are marked in the bitmap */
	irq_reserve_irqs(0, legacy_pic->nr_legacy_irqs);

249
	for (i = 0; i < count; i++) {
250
		irq_set_chip_data(i, &cfg[i]);
251 252
		zalloc_cpumask_var_node(&cfg[i].domain, GFP_KERNEL, node);
		zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_KERNEL, node);
253 254 255 256
		/*
		 * For legacy IRQ's, start with assigning irq0 to irq15 to
		 * IRQ0_VECTOR to IRQ15_VECTOR on cpu 0.
		 */
257
		if (i < legacy_pic->nr_legacy_irqs) {
258 259 260
			cfg[i].vector = IRQ0_VECTOR + i;
			cpumask_set_cpu(0, cfg[i].domain);
		}
261
	}
262 263

	return 0;
264
}
265

266
static struct irq_cfg *irq_cfg(unsigned int irq)
267
{
268
	return irq_get_chip_data(irq);
269
}
T
Thomas Gleixner 已提交
270

271
static struct irq_cfg *alloc_irq_cfg(unsigned int irq, int node)
272
{
273
	struct irq_cfg *cfg;
274

275
	cfg = kzalloc_node(sizeof(*cfg), GFP_KERNEL, node);
276 277
	if (!cfg)
		return NULL;
278
	if (!zalloc_cpumask_var_node(&cfg->domain, GFP_KERNEL, node))
279
		goto out_cfg;
280
	if (!zalloc_cpumask_var_node(&cfg->old_domain, GFP_KERNEL, node))
281
		goto out_domain;
282
	return cfg;
283 284 285 286 287
out_domain:
	free_cpumask_var(cfg->domain);
out_cfg:
	kfree(cfg);
	return NULL;
288 289
}

290
static void free_irq_cfg(unsigned int at, struct irq_cfg *cfg)
291
{
292 293
	if (!cfg)
		return;
294
	irq_set_chip_data(at, NULL);
295 296 297 298 299 300 301 302 303 304 305 306 307
	free_cpumask_var(cfg->domain);
	free_cpumask_var(cfg->old_domain);
	kfree(cfg);
}

static struct irq_cfg *alloc_irq_and_cfg_at(unsigned int at, int node)
{
	int res = irq_alloc_desc_at(at, node);
	struct irq_cfg *cfg;

	if (res < 0) {
		if (res != -EEXIST)
			return NULL;
308
		cfg = irq_get_chip_data(at);
309 310 311 312
		if (cfg)
			return cfg;
	}

313
	cfg = alloc_irq_cfg(at, node);
314
	if (cfg)
315
		irq_set_chip_data(at, cfg);
316 317 318 319 320 321 322 323 324 325 326 327
	else
		irq_free_desc(at);
	return cfg;
}

static int alloc_irq_from(unsigned int from, int node)
{
	return irq_alloc_desc_from(from, node);
}

static void free_irq_at(unsigned int at, struct irq_cfg *cfg)
{
328
	free_irq_cfg(at, cfg);
329 330 331
	irq_free_desc(at);
}

332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
{
	return io_apic_ops.read(apic, reg);
}

static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
{
	io_apic_ops.write(apic, reg, value);
}

static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
{
	io_apic_ops.modify(apic, reg, value);
}


L
Linus Torvalds 已提交
348 349 350 351
struct io_apic {
	unsigned int index;
	unsigned int unused[3];
	unsigned int data;
352 353
	unsigned int unused2[11];
	unsigned int eoi;
L
Linus Torvalds 已提交
354 355 356 357 358
};

static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
{
	return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
359
		+ (mpc_ioapic_addr(idx) & ~PAGE_MASK);
L
Linus Torvalds 已提交
360 361
}

362 363 364 365 366 367
static inline void io_apic_eoi(unsigned int apic, unsigned int vector)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(vector, &io_apic->eoi);
}

368
static unsigned int __io_apic_read(unsigned int apic, unsigned int reg)
L
Linus Torvalds 已提交
369 370 371 372 373 374
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	return readl(&io_apic->data);
}

375
static void __io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
L
Linus Torvalds 已提交
376 377
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
378

L
Linus Torvalds 已提交
379 380 381 382 383 384 385 386 387 388
	writel(reg, &io_apic->index);
	writel(value, &io_apic->data);
}

/*
 * Re-write a value: to be used for read-modify-write
 * cycles where the read already set up the index register.
 *
 * Older SiS APIC requires we rewrite the index register
 */
389
static void __io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
L
Linus Torvalds 已提交
390
{
391
	struct io_apic __iomem *io_apic = io_apic_base(apic);
T
Thomas Gleixner 已提交
392 393 394

	if (sis_apic_bug)
		writel(reg, &io_apic->index);
L
Linus Torvalds 已提交
395 396 397
	writel(value, &io_apic->data);
}

Y
Yinghai Lu 已提交
398
static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
399 400 401 402
{
	struct irq_pin_list *entry;
	unsigned long flags;

403
	raw_spin_lock_irqsave(&ioapic_lock, flags);
404
	for_each_irq_pin(entry, cfg->irq_2_pin) {
405 406 407 408 409 410 411
		unsigned int reg;
		int pin;

		pin = entry->pin;
		reg = io_apic_read(entry->apic, 0x10 + pin*2);
		/* Is the remote IRR bit set? */
		if (reg & IO_APIC_REDIR_REMOTE_IRR) {
412
			raw_spin_unlock_irqrestore(&ioapic_lock, flags);
413 414 415
			return true;
		}
	}
416
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
417 418 419 420

	return false;
}

421 422 423 424 425
union entry_union {
	struct { u32 w1, w2; };
	struct IO_APIC_route_entry entry;
};

426 427 428 429 430 431
static struct IO_APIC_route_entry __ioapic_read_entry(int apic, int pin)
{
	union entry_union eu;

	eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
	eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
432

433 434 435
	return eu.entry;
}

436 437 438 439
static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
{
	union entry_union eu;
	unsigned long flags;
440

441
	raw_spin_lock_irqsave(&ioapic_lock, flags);
442
	eu.entry = __ioapic_read_entry(apic, pin);
443
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
444

445 446 447
	return eu.entry;
}

448 449 450 451 452 453
/*
 * When we write a new IO APIC routing entry, we need to write the high
 * word first! If the mask bit in the low word is clear, we will enable
 * the interrupt, and we need to make sure the entry is fully populated
 * before that happens.
 */
454
static void __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
455
{
456 457
	union entry_union eu = {{0, 0}};

458
	eu.entry = e;
459 460
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
461 462
}

463
static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
464 465
{
	unsigned long flags;
466

467
	raw_spin_lock_irqsave(&ioapic_lock, flags);
468
	__ioapic_write_entry(apic, pin, e);
469
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
470 471 472 473 474 475 476 477 478 479 480 481
}

/*
 * When we mask an IO APIC routing entry, we need to write the low
 * word first, in order to set the mask bit before we change the
 * high bits!
 */
static void ioapic_mask_entry(int apic, int pin)
{
	unsigned long flags;
	union entry_union eu = { .entry.mask = 1 };

482
	raw_spin_lock_irqsave(&ioapic_lock, flags);
483 484
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
485
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
486 487
}

L
Linus Torvalds 已提交
488 489 490 491 492
/*
 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
 * shared ISA-space IRQs, so we have to support them. We are super
 * fast in the common case, and fast for shared ISA-space IRQs.
 */
493
static int __add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
L
Linus Torvalds 已提交
494
{
495
	struct irq_pin_list **last, *entry;
496

497 498 499
	/* don't allow duplicates */
	last = &cfg->irq_2_pin;
	for_each_irq_pin(entry, cfg->irq_2_pin) {
500
		if (entry->apic == apic && entry->pin == pin)
501
			return 0;
502
		last = &entry->next;
L
Linus Torvalds 已提交
503
	}
504

T
Thomas Gleixner 已提交
505
	entry = alloc_irq_pin_list(node);
506
	if (!entry) {
507 508 509
		printk(KERN_ERR "can not alloc irq_pin_list (%d,%d,%d)\n",
				node, apic, pin);
		return -ENOMEM;
510
	}
L
Linus Torvalds 已提交
511 512
	entry->apic = apic;
	entry->pin = pin;
513

514
	*last = entry;
515 516 517 518 519
	return 0;
}

static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
{
T
Thomas Gleixner 已提交
520
	if (__add_pin_to_irq_node(cfg, node, apic, pin))
521
		panic("IO-APIC: failed to add irq-pin. Can not proceed\n");
L
Linus Torvalds 已提交
522 523 524 525 526
}

/*
 * Reroute an IRQ to a different pin.
 */
527
static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
528 529
					   int oldapic, int oldpin,
					   int newapic, int newpin)
L
Linus Torvalds 已提交
530
{
531
	struct irq_pin_list *entry;
L
Linus Torvalds 已提交
532

533
	for_each_irq_pin(entry, cfg->irq_2_pin) {
L
Linus Torvalds 已提交
534 535 536
		if (entry->apic == oldapic && entry->pin == oldpin) {
			entry->apic = newapic;
			entry->pin = newpin;
537
			/* every one is different, right? */
538
			return;
539
		}
L
Linus Torvalds 已提交
540
	}
541

542 543
	/* old apic/pin didn't exist, so just add new ones */
	add_pin_to_irq_node(cfg, node, newapic, newpin);
L
Linus Torvalds 已提交
544 545
}

546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
static void __io_apic_modify_irq(struct irq_pin_list *entry,
				 int mask_and, int mask_or,
				 void (*final)(struct irq_pin_list *entry))
{
	unsigned int reg, pin;

	pin = entry->pin;
	reg = io_apic_read(entry->apic, 0x10 + pin * 2);
	reg &= mask_and;
	reg |= mask_or;
	io_apic_modify(entry->apic, 0x10 + pin * 2, reg);
	if (final)
		final(entry);
}

561 562 563
static void io_apic_modify_irq(struct irq_cfg *cfg,
			       int mask_and, int mask_or,
			       void (*final)(struct irq_pin_list *entry))
564 565
{
	struct irq_pin_list *entry;
566

567 568 569 570
	for_each_irq_pin(entry, cfg->irq_2_pin)
		__io_apic_modify_irq(entry, mask_and, mask_or, final);
}

571
static void io_apic_sync(struct irq_pin_list *entry)
L
Linus Torvalds 已提交
572
{
573 574 575 576 577
	/*
	 * Synchronize the IO-APIC and the CPU by doing
	 * a dummy read from the IO-APIC
	 */
	struct io_apic __iomem *io_apic;
578

579
	io_apic = io_apic_base(entry->apic);
Y
Yinghai Lu 已提交
580
	readl(&io_apic->data);
L
Linus Torvalds 已提交
581 582
}

T
Thomas Gleixner 已提交
583
static void mask_ioapic(struct irq_cfg *cfg)
584
{
T
Thomas Gleixner 已提交
585 586 587
	unsigned long flags;

	raw_spin_lock_irqsave(&ioapic_lock, flags);
Y
Yinghai Lu 已提交
588
	io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
T
Thomas Gleixner 已提交
589
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
590
}
L
Linus Torvalds 已提交
591

592
static void mask_ioapic_irq(struct irq_data *data)
L
Linus Torvalds 已提交
593
{
594
	mask_ioapic(data->chip_data);
T
Thomas Gleixner 已提交
595
}
Y
Yinghai Lu 已提交
596

T
Thomas Gleixner 已提交
597 598 599
static void __unmask_ioapic(struct irq_cfg *cfg)
{
	io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
L
Linus Torvalds 已提交
600 601
}

T
Thomas Gleixner 已提交
602
static void unmask_ioapic(struct irq_cfg *cfg)
L
Linus Torvalds 已提交
603 604 605
{
	unsigned long flags;

606
	raw_spin_lock_irqsave(&ioapic_lock, flags);
T
Thomas Gleixner 已提交
607
	__unmask_ioapic(cfg);
608
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
L
Linus Torvalds 已提交
609 610
}

611
static void unmask_ioapic_irq(struct irq_data *data)
Y
Yinghai Lu 已提交
612
{
613
	unmask_ioapic(data->chip_data);
Y
Yinghai Lu 已提交
614 615
}

616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675
/*
 * IO-APIC versions below 0x20 don't support EOI register.
 * For the record, here is the information about various versions:
 *     0Xh     82489DX
 *     1Xh     I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant
 *     2Xh     I/O(x)APIC which is PCI 2.2 Compliant
 *     30h-FFh Reserved
 *
 * Some of the Intel ICH Specs (ICH2 to ICH5) documents the io-apic
 * version as 0x2. This is an error with documentation and these ICH chips
 * use io-apic's of version 0x20.
 *
 * For IO-APIC's with EOI register, we use that to do an explicit EOI.
 * Otherwise, we simulate the EOI message manually by changing the trigger
 * mode to edge and then back to level, with RTE being masked during this.
 */
static void __eoi_ioapic_pin(int apic, int pin, int vector, struct irq_cfg *cfg)
{
	if (mpc_ioapic_ver(apic) >= 0x20) {
		/*
		 * Intr-remapping uses pin number as the virtual vector
		 * in the RTE. Actual vector is programmed in
		 * intr-remapping table entry. Hence for the io-apic
		 * EOI we use the pin number.
		 */
		if (cfg && irq_remapped(cfg))
			io_apic_eoi(apic, pin);
		else
			io_apic_eoi(apic, vector);
	} else {
		struct IO_APIC_route_entry entry, entry1;

		entry = entry1 = __ioapic_read_entry(apic, pin);

		/*
		 * Mask the entry and change the trigger mode to edge.
		 */
		entry1.mask = 1;
		entry1.trigger = IOAPIC_EDGE;

		__ioapic_write_entry(apic, pin, entry1);

		/*
		 * Restore the previous level triggered entry.
		 */
		__ioapic_write_entry(apic, pin, entry);
	}
}

static void eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
{
	struct irq_pin_list *entry;
	unsigned long flags;

	raw_spin_lock_irqsave(&ioapic_lock, flags);
	for_each_irq_pin(entry, cfg->irq_2_pin)
		__eoi_ioapic_pin(entry->apic, entry->pin, cfg->vector, cfg);
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
}

L
Linus Torvalds 已提交
676 677 678
static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
{
	struct IO_APIC_route_entry entry;
679

L
Linus Torvalds 已提交
680
	/* Check delivery_mode to be sure we're not clearing an SMI pin */
681
	entry = ioapic_read_entry(apic, pin);
L
Linus Torvalds 已提交
682 683
	if (entry.delivery_mode == dest_SMI)
		return;
684

L
Linus Torvalds 已提交
685
	/*
686 687 688 689 690 691 692 693 694 695
	 * Make sure the entry is masked and re-read the contents to check
	 * if it is a level triggered pin and if the remote-IRR is set.
	 */
	if (!entry.mask) {
		entry.mask = 1;
		ioapic_write_entry(apic, pin, entry);
		entry = ioapic_read_entry(apic, pin);
	}

	if (entry.irr) {
696 697
		unsigned long flags;

698 699 700 701 702 703 704 705 706 707
		/*
		 * Make sure the trigger mode is set to level. Explicit EOI
		 * doesn't clear the remote-IRR if the trigger mode is not
		 * set to level.
		 */
		if (!entry.trigger) {
			entry.trigger = IOAPIC_LEVEL;
			ioapic_write_entry(apic, pin, entry);
		}

708 709 710
		raw_spin_lock_irqsave(&ioapic_lock, flags);
		__eoi_ioapic_pin(apic, pin, entry.vector, NULL);
		raw_spin_unlock_irqrestore(&ioapic_lock, flags);
711 712 713 714 715
	}

	/*
	 * Clear the rest of the bits in the IO-APIC RTE except for the mask
	 * bit.
L
Linus Torvalds 已提交
716
	 */
717
	ioapic_mask_entry(apic, pin);
718 719 720 721
	entry = ioapic_read_entry(apic, pin);
	if (entry.irr)
		printk(KERN_ERR "Unable to reset IRR for apic: %d, pin :%d\n",
		       mpc_ioapic_id(apic), pin);
L
Linus Torvalds 已提交
722 723
}

724
static void clear_IO_APIC (void)
L
Linus Torvalds 已提交
725 726 727 728
{
	int apic, pin;

	for (apic = 0; apic < nr_ioapics; apic++)
S
Suresh Siddha 已提交
729
		for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
L
Linus Torvalds 已提交
730 731 732
			clear_IO_APIC_pin(apic, pin);
}

733
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
734 735 736 737 738 739
/*
 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
 * specific CPU-side IRQs.
 */

#define MAX_PIRQS 8
Y
Yinghai Lu 已提交
740 741 742
static int pirq_entries[MAX_PIRQS] = {
	[0 ... MAX_PIRQS - 1] = -1
};
L
Linus Torvalds 已提交
743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768

static int __init ioapic_pirq_setup(char *str)
{
	int i, max;
	int ints[MAX_PIRQS+1];

	get_options(str, ARRAY_SIZE(ints), ints);

	apic_printk(APIC_VERBOSE, KERN_INFO
			"PIRQ redirection, working around broken MP-BIOS.\n");
	max = MAX_PIRQS;
	if (ints[0] < MAX_PIRQS)
		max = ints[0];

	for (i = 0; i < max; i++) {
		apic_printk(APIC_VERBOSE, KERN_DEBUG
				"... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
		/*
		 * PIRQs are mapped upside down, usually.
		 */
		pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
	}
	return 1;
}

__setup("pirq=", ioapic_pirq_setup);
769 770 771
#endif /* CONFIG_X86_32 */

/*
772
 * Saves all the IO-APIC RTE's
773
 */
774
int save_ioapic_entries(void)
775 776
{
	int apic, pin;
777
	int err = 0;
778 779

	for (apic = 0; apic < nr_ioapics; apic++) {
780
		if (!ioapics[apic].saved_registers) {
781 782 783
			err = -ENOMEM;
			continue;
		}
784

S
Suresh Siddha 已提交
785
		for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
786
			ioapics[apic].saved_registers[pin] =
787
				ioapic_read_entry(apic, pin);
788
	}
789

790
	return err;
791 792
}

793 794 795
/*
 * Mask all IO APIC entries.
 */
796
void mask_ioapic_entries(void)
797 798 799 800
{
	int apic, pin;

	for (apic = 0; apic < nr_ioapics; apic++) {
801
		if (!ioapics[apic].saved_registers)
802
			continue;
803

S
Suresh Siddha 已提交
804
		for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
805 806
			struct IO_APIC_route_entry entry;

807
			entry = ioapics[apic].saved_registers[pin];
808 809 810 811 812 813 814 815
			if (!entry.mask) {
				entry.mask = 1;
				ioapic_write_entry(apic, pin, entry);
			}
		}
	}
}

816
/*
817
 * Restore IO APIC entries which was saved in the ioapic structure.
818
 */
819
int restore_ioapic_entries(void)
820 821 822
{
	int apic, pin;

823
	for (apic = 0; apic < nr_ioapics; apic++) {
824
		if (!ioapics[apic].saved_registers)
825
			continue;
826

S
Suresh Siddha 已提交
827
		for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
828
			ioapic_write_entry(apic, pin,
829
					   ioapics[apic].saved_registers[pin]);
830
	}
831
	return 0;
832 833
}

L
Linus Torvalds 已提交
834 835 836
/*
 * Find the IRQ entry number of a certain pin.
 */
837
static int find_irq_entry(int ioapic_idx, int pin, int type)
L
Linus Torvalds 已提交
838 839 840 841
{
	int i;

	for (i = 0; i < mp_irq_entries; i++)
842
		if (mp_irqs[i].irqtype == type &&
843
		    (mp_irqs[i].dstapic == mpc_ioapic_id(ioapic_idx) ||
844 845
		     mp_irqs[i].dstapic == MP_APIC_ALL) &&
		    mp_irqs[i].dstirq == pin)
L
Linus Torvalds 已提交
846 847 848 849 850 851 852 853
			return i;

	return -1;
}

/*
 * Find the pin to which IRQ[irq] (ISA) is connected
 */
854
static int __init find_isa_irq_pin(int irq, int type)
L
Linus Torvalds 已提交
855 856 857 858
{
	int i;

	for (i = 0; i < mp_irq_entries; i++) {
859
		int lbus = mp_irqs[i].srcbus;
L
Linus Torvalds 已提交
860

A
Alexey Starikovskiy 已提交
861
		if (test_bit(lbus, mp_bus_not_pci) &&
862 863
		    (mp_irqs[i].irqtype == type) &&
		    (mp_irqs[i].srcbusirq == irq))
L
Linus Torvalds 已提交
864

865
			return mp_irqs[i].dstirq;
L
Linus Torvalds 已提交
866 867 868 869
	}
	return -1;
}

870 871 872 873 874
static int __init find_isa_irq_apic(int irq, int type)
{
	int i;

	for (i = 0; i < mp_irq_entries; i++) {
875
		int lbus = mp_irqs[i].srcbus;
876

A
Alexey Starikovskiy 已提交
877
		if (test_bit(lbus, mp_bus_not_pci) &&
878 879
		    (mp_irqs[i].irqtype == type) &&
		    (mp_irqs[i].srcbusirq == irq))
880 881
			break;
	}
882

883
	if (i < mp_irq_entries) {
884 885 886 887 888
		int ioapic_idx;

		for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
			if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic)
				return ioapic_idx;
889 890 891 892 893
	}

	return -1;
}

894
#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
L
Linus Torvalds 已提交
895 896 897 898 899
/*
 * EISA Edge/Level control register, ELCR
 */
static int EISA_ELCR(unsigned int irq)
{
900
	if (irq < legacy_pic->nr_legacy_irqs) {
L
Linus Torvalds 已提交
901 902 903 904 905 906 907
		unsigned int port = 0x4d0 + (irq >> 3);
		return (inb(port) >> (irq & 7)) & 1;
	}
	apic_printk(APIC_VERBOSE, KERN_INFO
			"Broken MPtable reports ISA irq %d\n", irq);
	return 0;
}
908

909
#endif
L
Linus Torvalds 已提交
910

A
Alexey Starikovskiy 已提交
911 912 913 914 915 916
/* ISA interrupts are always polarity zero edge triggered,
 * when listed as conforming in the MP table. */

#define default_ISA_trigger(idx)	(0)
#define default_ISA_polarity(idx)	(0)

L
Linus Torvalds 已提交
917 918 919 920 921
/* EISA interrupts are always polarity zero and can be edge or level
 * trigger depending on the ELCR value.  If an interrupt is listed as
 * EISA conforming in the MP table, that means its trigger type must
 * be read in from the ELCR */

922
#define default_EISA_trigger(idx)	(EISA_ELCR(mp_irqs[idx].srcbusirq))
A
Alexey Starikovskiy 已提交
923
#define default_EISA_polarity(idx)	default_ISA_polarity(idx)
L
Linus Torvalds 已提交
924 925 926 927 928 929 930 931 932 933 934

/* PCI interrupts are always polarity one level triggered,
 * when listed as conforming in the MP table. */

#define default_PCI_trigger(idx)	(1)
#define default_PCI_polarity(idx)	(1)

/* MCA interrupts are always polarity zero level triggered,
 * when listed as conforming in the MP table. */

#define default_MCA_trigger(idx)	(1)
A
Alexey Starikovskiy 已提交
935
#define default_MCA_polarity(idx)	default_ISA_polarity(idx)
L
Linus Torvalds 已提交
936

937
static int irq_polarity(int idx)
L
Linus Torvalds 已提交
938
{
939
	int bus = mp_irqs[idx].srcbus;
L
Linus Torvalds 已提交
940 941 942 943 944
	int polarity;

	/*
	 * Determine IRQ line polarity (high active or low active):
	 */
945
	switch (mp_irqs[idx].irqflag & 3)
946
	{
947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974
		case 0: /* conforms, ie. bus-type dependent polarity */
			if (test_bit(bus, mp_bus_not_pci))
				polarity = default_ISA_polarity(idx);
			else
				polarity = default_PCI_polarity(idx);
			break;
		case 1: /* high active */
		{
			polarity = 0;
			break;
		}
		case 2: /* reserved */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			polarity = 1;
			break;
		}
		case 3: /* low active */
		{
			polarity = 1;
			break;
		}
		default: /* invalid */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			polarity = 1;
			break;
		}
L
Linus Torvalds 已提交
975 976 977 978
	}
	return polarity;
}

979
static int irq_trigger(int idx)
L
Linus Torvalds 已提交
980
{
981
	int bus = mp_irqs[idx].srcbus;
L
Linus Torvalds 已提交
982 983 984 985 986
	int trigger;

	/*
	 * Determine IRQ trigger mode (edge or level sensitive):
	 */
987
	switch ((mp_irqs[idx].irqflag>>2) & 3)
L
Linus Torvalds 已提交
988
	{
989 990 991 992 993
		case 0: /* conforms, ie. bus-type dependent */
			if (test_bit(bus, mp_bus_not_pci))
				trigger = default_ISA_trigger(idx);
			else
				trigger = default_PCI_trigger(idx);
994
#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
			switch (mp_bus_id_to_type[bus]) {
				case MP_BUS_ISA: /* ISA pin */
				{
					/* set before the switch */
					break;
				}
				case MP_BUS_EISA: /* EISA pin */
				{
					trigger = default_EISA_trigger(idx);
					break;
				}
				case MP_BUS_PCI: /* PCI pin */
				{
					/* set before the switch */
					break;
				}
				case MP_BUS_MCA: /* MCA pin */
				{
					trigger = default_MCA_trigger(idx);
					break;
				}
				default:
				{
					printk(KERN_WARNING "broken BIOS!!\n");
					trigger = 1;
					break;
				}
			}
#endif
L
Linus Torvalds 已提交
1024
			break;
1025
		case 1: /* edge */
L
Linus Torvalds 已提交
1026
		{
1027
			trigger = 0;
L
Linus Torvalds 已提交
1028 1029
			break;
		}
1030
		case 2: /* reserved */
L
Linus Torvalds 已提交
1031
		{
1032 1033
			printk(KERN_WARNING "broken BIOS!!\n");
			trigger = 1;
L
Linus Torvalds 已提交
1034 1035
			break;
		}
1036
		case 3: /* level */
L
Linus Torvalds 已提交
1037
		{
1038
			trigger = 1;
L
Linus Torvalds 已提交
1039 1040
			break;
		}
1041
		default: /* invalid */
L
Linus Torvalds 已提交
1042 1043
		{
			printk(KERN_WARNING "broken BIOS!!\n");
1044
			trigger = 0;
L
Linus Torvalds 已提交
1045 1046 1047 1048 1049 1050 1051 1052
			break;
		}
	}
	return trigger;
}

static int pin_2_irq(int idx, int apic, int pin)
{
1053
	int irq;
1054
	int bus = mp_irqs[idx].srcbus;
1055
	struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(apic);
L
Linus Torvalds 已提交
1056 1057 1058 1059

	/*
	 * Debugging check, we are in big trouble if this message pops up!
	 */
1060
	if (mp_irqs[idx].dstirq != pin)
L
Linus Torvalds 已提交
1061 1062
		printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");

1063
	if (test_bit(bus, mp_bus_not_pci)) {
1064
		irq = mp_irqs[idx].srcbusirq;
1065
	} else {
1066
		u32 gsi = gsi_cfg->gsi_base + pin;
1067 1068 1069 1070

		if (gsi >= NR_IRQS_LEGACY)
			irq = gsi;
		else
1071
			irq = gsi_top + gsi;
L
Linus Torvalds 已提交
1072 1073
	}

1074
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
	/*
	 * PCI IRQ command line redirection. Yes, limits are hardcoded.
	 */
	if ((pin >= 16) && (pin <= 23)) {
		if (pirq_entries[pin-16] != -1) {
			if (!pirq_entries[pin-16]) {
				apic_printk(APIC_VERBOSE, KERN_DEBUG
						"disabling PIRQ%d\n", pin-16);
			} else {
				irq = pirq_entries[pin-16];
				apic_printk(APIC_VERBOSE, KERN_DEBUG
						"using PIRQ%d -> IRQ %d\n",
						pin-16, irq);
			}
		}
	}
1091 1092
#endif

L
Linus Torvalds 已提交
1093 1094 1095
	return irq;
}

1096 1097 1098 1099 1100
/*
 * Find a specific PCI IRQ entry.
 * Not an __init, possibly needed by modules
 */
int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin,
1101
				struct io_apic_irq_attr *irq_attr)
1102
{
1103
	int ioapic_idx, i, best_guess = -1;
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115

	apic_printk(APIC_DEBUG,
		    "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
		    bus, slot, pin);
	if (test_bit(bus, mp_bus_not_pci)) {
		apic_printk(APIC_VERBOSE,
			    "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
		return -1;
	}
	for (i = 0; i < mp_irq_entries; i++) {
		int lbus = mp_irqs[i].srcbus;

1116 1117
		for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
			if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic ||
1118 1119 1120 1121 1122 1123 1124
			    mp_irqs[i].dstapic == MP_APIC_ALL)
				break;

		if (!test_bit(lbus, mp_bus_not_pci) &&
		    !mp_irqs[i].irqtype &&
		    (bus == lbus) &&
		    (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1125
			int irq = pin_2_irq(i, ioapic_idx, mp_irqs[i].dstirq);
1126

1127
			if (!(ioapic_idx || IO_APIC_IRQ(irq)))
1128 1129 1130
				continue;

			if (pin == (mp_irqs[i].srcbusirq & 3)) {
1131
				set_io_apic_irq_attr(irq_attr, ioapic_idx,
1132 1133 1134
						     mp_irqs[i].dstirq,
						     irq_trigger(i),
						     irq_polarity(i));
1135 1136 1137 1138 1139 1140 1141
				return irq;
			}
			/*
			 * Use the first all-but-pin matching entry as a
			 * best-guess fuzzy result for broken mptables.
			 */
			if (best_guess < 0) {
1142
				set_io_apic_irq_attr(irq_attr, ioapic_idx,
1143 1144 1145
						     mp_irqs[i].dstirq,
						     irq_trigger(i),
						     irq_polarity(i));
1146 1147 1148 1149 1150 1151 1152 1153
				best_guess = irq;
			}
		}
	}
	return best_guess;
}
EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);

1154 1155 1156 1157 1158
void lock_vector_lock(void)
{
	/* Used to the online set of cpus does not change
	 * during assign_irq_vector.
	 */
1159
	raw_spin_lock(&vector_lock);
1160
}
L
Linus Torvalds 已提交
1161

1162
void unlock_vector_lock(void)
L
Linus Torvalds 已提交
1163
{
1164
	raw_spin_unlock(&vector_lock);
1165
}
L
Linus Torvalds 已提交
1166

1167 1168
static int
__assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1169
{
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
1181
	static int current_vector = FIRST_EXTERNAL_VECTOR + VECTOR_OFFSET_START;
1182
	static int current_offset = VECTOR_OFFSET_START % 8;
1183
	unsigned int old_vector;
1184 1185
	int cpu, err;
	cpumask_var_t tmp_mask;
1186

1187
	if (cfg->move_in_progress)
1188
		return -EBUSY;
1189

1190 1191
	if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC))
		return -ENOMEM;
1192

1193 1194
	old_vector = cfg->vector;
	if (old_vector) {
1195 1196 1197 1198
		cpumask_and(tmp_mask, mask, cpu_online_mask);
		cpumask_and(tmp_mask, cfg->domain, tmp_mask);
		if (!cpumask_empty(tmp_mask)) {
			free_cpumask_var(tmp_mask);
1199
			return 0;
1200
		}
1201
	}
1202

1203
	/* Only try and allocate irqs on cpus that are present */
1204 1205
	err = -ENOSPC;
	for_each_cpu_and(cpu, mask, cpu_online_mask) {
1206 1207
		int new_cpu;
		int vector, offset;
1208

1209
		apic->vector_allocation_domain(cpu, tmp_mask);
1210

1211 1212
		vector = current_vector;
		offset = current_offset;
1213
next:
1214 1215
		vector += 8;
		if (vector >= first_system_vector) {
1216
			/* If out of vectors on large boxen, must share them. */
1217
			offset = (offset + 1) % 8;
1218
			vector = FIRST_EXTERNAL_VECTOR + offset;
1219 1220 1221
		}
		if (unlikely(current_vector == vector))
			continue;
1222 1223

		if (test_bit(vector, used_vectors))
1224
			goto next;
1225

1226
		for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1227 1228 1229 1230 1231 1232 1233
			if (per_cpu(vector_irq, new_cpu)[vector] != -1)
				goto next;
		/* Found one! */
		current_vector = vector;
		current_offset = offset;
		if (old_vector) {
			cfg->move_in_progress = 1;
1234
			cpumask_copy(cfg->old_domain, cfg->domain);
1235
		}
1236
		for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1237 1238
			per_cpu(vector_irq, new_cpu)[vector] = irq;
		cfg->vector = vector;
1239 1240 1241
		cpumask_copy(cfg->domain, tmp_mask);
		err = 0;
		break;
1242
	}
1243 1244
	free_cpumask_var(tmp_mask);
	return err;
1245 1246
}

1247
int assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1248 1249
{
	int err;
1250 1251
	unsigned long flags;

1252
	raw_spin_lock_irqsave(&vector_lock, flags);
Y
Yinghai Lu 已提交
1253
	err = __assign_irq_vector(irq, cfg, mask);
1254
	raw_spin_unlock_irqrestore(&vector_lock, flags);
1255 1256 1257
	return err;
}

Y
Yinghai Lu 已提交
1258
static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
1259 1260 1261 1262 1263 1264
{
	int cpu, vector;

	BUG_ON(!cfg->vector);

	vector = cfg->vector;
1265
	for_each_cpu_and(cpu, cfg->domain, cpu_online_mask)
1266 1267 1268
		per_cpu(vector_irq, cpu)[vector] = -1;

	cfg->vector = 0;
1269
	cpumask_clear(cfg->domain);
1270 1271 1272

	if (likely(!cfg->move_in_progress))
		return;
1273
	for_each_cpu_and(cpu, cfg->old_domain, cpu_online_mask) {
1274 1275 1276 1277 1278 1279 1280 1281 1282
		for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
								vector++) {
			if (per_cpu(vector_irq, cpu)[vector] != irq)
				continue;
			per_cpu(vector_irq, cpu)[vector] = -1;
			break;
		}
	}
	cfg->move_in_progress = 0;
1283 1284 1285 1286 1287 1288 1289 1290
}

void __setup_vector_irq(int cpu)
{
	/* Initialize vector_irq on a new cpu */
	int irq, vector;
	struct irq_cfg *cfg;

1291 1292 1293 1294 1295
	/*
	 * vector_lock will make sure that we don't run into irq vector
	 * assignments that might be happening on another cpu in parallel,
	 * while we setup our initial vector to irq mappings.
	 */
1296
	raw_spin_lock(&vector_lock);
1297
	/* Mark the inuse vectors */
T
Thomas Gleixner 已提交
1298
	for_each_active_irq(irq) {
1299
		cfg = irq_get_chip_data(irq);
T
Thomas Gleixner 已提交
1300 1301
		if (!cfg)
			continue;
1302 1303 1304 1305 1306 1307 1308
		/*
		 * If it is a legacy IRQ handled by the legacy PIC, this cpu
		 * will be part of the irq_cfg's domain.
		 */
		if (irq < legacy_pic->nr_legacy_irqs && !IO_APIC_IRQ(irq))
			cpumask_set_cpu(cpu, cfg->domain);

1309
		if (!cpumask_test_cpu(cpu, cfg->domain))
1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
			continue;
		vector = cfg->vector;
		per_cpu(vector_irq, cpu)[vector] = irq;
	}
	/* Mark the free vectors */
	for (vector = 0; vector < NR_VECTORS; ++vector) {
		irq = per_cpu(vector_irq, cpu)[vector];
		if (irq < 0)
			continue;

		cfg = irq_cfg(irq);
1321
		if (!cpumask_test_cpu(cpu, cfg->domain))
1322
			per_cpu(vector_irq, cpu)[vector] = -1;
1323
	}
1324
	raw_spin_unlock(&vector_lock);
L
Linus Torvalds 已提交
1325
}
1326

1327
static struct irq_chip ioapic_chip;
L
Linus Torvalds 已提交
1328

1329
#ifdef CONFIG_X86_32
1330 1331
static inline int IO_APIC_irq_trigger(int irq)
{
T
Thomas Gleixner 已提交
1332
	int apic, idx, pin;
1333

T
Thomas Gleixner 已提交
1334
	for (apic = 0; apic < nr_ioapics; apic++) {
S
Suresh Siddha 已提交
1335
		for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
T
Thomas Gleixner 已提交
1336 1337 1338 1339 1340 1341
			idx = find_irq_entry(apic, pin, mp_INT);
			if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
				return irq_trigger(idx);
		}
	}
	/*
1342 1343
         * nonexistent IRQs are edge default
         */
T
Thomas Gleixner 已提交
1344
	return 0;
1345
}
1346 1347 1348
#else
static inline int IO_APIC_irq_trigger(int irq)
{
1349
	return 1;
1350 1351
}
#endif
1352

1353 1354
static void ioapic_register_intr(unsigned int irq, struct irq_cfg *cfg,
				 unsigned long trigger)
L
Linus Torvalds 已提交
1355
{
1356 1357 1358
	struct irq_chip *chip = &ioapic_chip;
	irq_flow_handler_t hdl;
	bool fasteoi;
Y
Yinghai Lu 已提交
1359

1360
	if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1361
	    trigger == IOAPIC_LEVEL) {
1362
		irq_set_status_flags(irq, IRQ_LEVEL);
1363 1364
		fasteoi = true;
	} else {
1365
		irq_clear_status_flags(irq, IRQ_LEVEL);
1366 1367
		fasteoi = false;
	}
1368

1369
	if (irq_remapped(cfg)) {
1370
		irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
1371
		irq_remap_modify_chip_defaults(chip);
1372
		fasteoi = trigger != 0;
1373
	}
1374

1375 1376 1377
	hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq;
	irq_set_chip_and_handler_name(irq, chip, hdl,
				      fasteoi ? "fasteoi" : "edge");
L
Linus Torvalds 已提交
1378 1379
}

1380 1381 1382 1383
static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,
			       unsigned int destination, int vector,
			       struct io_apic_irq_attr *attr)
{
1384 1385 1386
	if (irq_remapping_enabled)
		return setup_ioapic_remapped_entry(irq, entry, destination,
						   vector, attr);
1387

1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399
	memset(entry, 0, sizeof(*entry));

	entry->delivery_mode = apic->irq_delivery_mode;
	entry->dest_mode     = apic->irq_dest_mode;
	entry->dest	     = destination;
	entry->vector	     = vector;
	entry->mask	     = 0;			/* enable IRQ */
	entry->trigger	     = attr->trigger;
	entry->polarity	     = attr->polarity;

	/*
	 * Mask level triggered irqs.
1400 1401
	 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
	 */
1402
	if (attr->trigger)
1403
		entry->mask = 1;
1404

1405 1406 1407
	return 0;
}

1408 1409
static void setup_ioapic_irq(unsigned int irq, struct irq_cfg *cfg,
				struct io_apic_irq_attr *attr)
1410
{
L
Linus Torvalds 已提交
1411
	struct IO_APIC_route_entry entry;
1412
	unsigned int dest;
1413 1414 1415

	if (!IO_APIC_IRQ(irq))
		return;
1416 1417 1418 1419 1420
	/*
	 * For legacy irqs, cfg->domain starts with cpu 0 for legacy
	 * controllers like 8259. Now that IO-APIC can handle this irq, update
	 * the cfg->domain.
	 */
1421
	if (irq < legacy_pic->nr_legacy_irqs && cpumask_test_cpu(0, cfg->domain))
1422 1423
		apic->vector_allocation_domain(0, cfg->domain);

1424
	if (assign_irq_vector(irq, cfg, apic->target_cpus()))
1425 1426
		return;

1427
	dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
1428 1429 1430

	apic_printk(APIC_VERBOSE,KERN_DEBUG
		    "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1431
		    "IRQ %d Mode:%i Active:%i Dest:%d)\n",
1432 1433
		    attr->ioapic, mpc_ioapic_id(attr->ioapic), attr->ioapic_pin,
		    cfg->vector, irq, attr->trigger, attr->polarity, dest);
1434

1435 1436 1437
	if (setup_ioapic_entry(irq, &entry, dest, cfg->vector, attr)) {
		pr_warn("Failed to setup ioapic entry for ioapic  %d, pin %d\n",
			mpc_ioapic_id(attr->ioapic), attr->ioapic_pin);
Y
Yinghai Lu 已提交
1438
		__clear_irq_vector(irq, cfg);
1439

1440 1441 1442
		return;
	}

1443
	ioapic_register_intr(irq, cfg, attr->trigger);
1444
	if (irq < legacy_pic->nr_legacy_irqs)
1445
		legacy_pic->mask(irq);
1446

1447
	ioapic_write_entry(attr->ioapic, attr->ioapic_pin, entry);
1448 1449
}

1450
static bool __init io_apic_pin_not_connected(int idx, int ioapic_idx, int pin)
1451 1452 1453 1454 1455
{
	if (idx != -1)
		return false;

	apic_printk(APIC_VERBOSE, KERN_DEBUG " apic %d pin %d not connected\n",
1456
		    mpc_ioapic_id(ioapic_idx), pin);
1457 1458 1459
	return true;
}

1460
static void __init __io_apic_setup_irqs(unsigned int ioapic_idx)
1461
{
1462
	int idx, node = cpu_to_node(0);
1463
	struct io_apic_irq_attr attr;
1464
	unsigned int pin, irq;
L
Linus Torvalds 已提交
1465

1466 1467 1468
	for (pin = 0; pin < ioapics[ioapic_idx].nr_registers; pin++) {
		idx = find_irq_entry(ioapic_idx, pin, mp_INT);
		if (io_apic_pin_not_connected(idx, ioapic_idx, pin))
1469
			continue;
1470

1471
		irq = pin_2_irq(idx, ioapic_idx, pin);
1472

1473
		if ((ioapic_idx > 0) && (irq > 16))
E
Eric W. Biederman 已提交
1474 1475
			continue;

1476 1477 1478 1479 1480
		/*
		 * Skip the timer IRQ if there's a quirk handler
		 * installed and if it returns 1:
		 */
		if (apic->multi_timer_check &&
1481
		    apic->multi_timer_check(ioapic_idx, irq))
1482
			continue;
1483

1484
		set_io_apic_irq_attr(&attr, ioapic_idx, pin, irq_trigger(idx),
1485
				     irq_polarity(idx));
1486

1487
		io_apic_setup_irq_pin(irq, node, &attr);
L
Linus Torvalds 已提交
1488 1489 1490
	}
}

1491 1492
static void __init setup_IO_APIC_irqs(void)
{
1493
	unsigned int ioapic_idx;
1494 1495 1496

	apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");

1497 1498
	for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
		__io_apic_setup_irqs(ioapic_idx);
1499 1500
}

Y
Yinghai Lu 已提交
1501 1502 1503 1504 1505 1506 1507
/*
 * for the gsit that is not in first ioapic
 * but could not use acpi_register_gsi()
 * like some special sci in IBM x3330
 */
void setup_IO_APIC_irq_extra(u32 gsi)
{
1508
	int ioapic_idx = 0, pin, idx, irq, node = cpu_to_node(0);
1509
	struct io_apic_irq_attr attr;
Y
Yinghai Lu 已提交
1510 1511 1512 1513

	/*
	 * Convert 'gsi' to 'ioapic.pin'.
	 */
1514 1515
	ioapic_idx = mp_find_ioapic(gsi);
	if (ioapic_idx < 0)
Y
Yinghai Lu 已提交
1516 1517
		return;

1518 1519
	pin = mp_find_ioapic_pin(ioapic_idx, gsi);
	idx = find_irq_entry(ioapic_idx, pin, mp_INT);
Y
Yinghai Lu 已提交
1520 1521 1522
	if (idx == -1)
		return;

1523
	irq = pin_2_irq(idx, ioapic_idx, pin);
1524 1525

	/* Only handle the non legacy irqs on secondary ioapics */
1526
	if (ioapic_idx == 0 || irq < NR_IRQS_LEGACY)
Y
Yinghai Lu 已提交
1527
		return;
1528

1529
	set_io_apic_irq_attr(&attr, ioapic_idx, pin, irq_trigger(idx),
1530 1531
			     irq_polarity(idx));

1532
	io_apic_setup_irq_pin_once(irq, node, &attr);
Y
Yinghai Lu 已提交
1533 1534
}

L
Linus Torvalds 已提交
1535
/*
1536
 * Set up the timer pin, possibly with the 8259A-master behind.
L
Linus Torvalds 已提交
1537
 */
1538 1539
static void __init setup_timer_IRQ0_pin(unsigned int ioapic_idx,
					 unsigned int pin, int vector)
L
Linus Torvalds 已提交
1540 1541 1542
{
	struct IO_APIC_route_entry entry;

1543
	if (irq_remapping_enabled)
1544 1545
		return;

1546
	memset(&entry, 0, sizeof(entry));
L
Linus Torvalds 已提交
1547 1548 1549 1550 1551

	/*
	 * We use logical delivery to get the timer IRQ
	 * to the first CPU.
	 */
1552
	entry.dest_mode = apic->irq_dest_mode;
Y
Yinghai Lu 已提交
1553
	entry.mask = 0;			/* don't mask IRQ for edge */
1554
	entry.dest = apic->cpu_mask_to_apicid(apic->target_cpus());
1555
	entry.delivery_mode = apic->irq_delivery_mode;
L
Linus Torvalds 已提交
1556 1557 1558 1559 1560 1561
	entry.polarity = 0;
	entry.trigger = 0;
	entry.vector = vector;

	/*
	 * The timer IRQ doesn't have to know that behind the
1562
	 * scene we may have a 8259A-master in AEOI mode ...
L
Linus Torvalds 已提交
1563
	 */
1564 1565
	irq_set_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq,
				      "edge");
L
Linus Torvalds 已提交
1566 1567 1568 1569

	/*
	 * Add it to the IO-APIC irq-routing table:
	 */
1570
	ioapic_write_entry(ioapic_idx, pin, entry);
L
Linus Torvalds 已提交
1571 1572
}

1573
__apicdebuginit(void) print_IO_APIC(int ioapic_idx)
L
Linus Torvalds 已提交
1574
{
1575
	int i;
L
Linus Torvalds 已提交
1576 1577 1578 1579 1580 1581
	union IO_APIC_reg_00 reg_00;
	union IO_APIC_reg_01 reg_01;
	union IO_APIC_reg_02 reg_02;
	union IO_APIC_reg_03 reg_03;
	unsigned long flags;

1582
	raw_spin_lock_irqsave(&ioapic_lock, flags);
1583 1584
	reg_00.raw = io_apic_read(ioapic_idx, 0);
	reg_01.raw = io_apic_read(ioapic_idx, 1);
L
Linus Torvalds 已提交
1585
	if (reg_01.bits.version >= 0x10)
1586
		reg_02.raw = io_apic_read(ioapic_idx, 2);
T
Thomas Gleixner 已提交
1587
	if (reg_01.bits.version >= 0x20)
1588
		reg_03.raw = io_apic_read(ioapic_idx, 3);
1589
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
L
Linus Torvalds 已提交
1590

1591
	printk("\n");
1592
	printk(KERN_DEBUG "IO APIC #%d......\n", mpc_ioapic_id(ioapic_idx));
L
Linus Torvalds 已提交
1593 1594 1595 1596 1597
	printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
	printk(KERN_DEBUG ".......    : physical APIC id: %02X\n", reg_00.bits.ID);
	printk(KERN_DEBUG ".......    : Delivery Type: %X\n", reg_00.bits.delivery_type);
	printk(KERN_DEBUG ".......    : LTS          : %X\n", reg_00.bits.LTS);

1598
	printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
1599 1600
	printk(KERN_DEBUG ".......     : max redirection entries: %02X\n",
		reg_01.bits.entries);
L
Linus Torvalds 已提交
1601 1602

	printk(KERN_DEBUG ".......     : PRQ implemented: %X\n", reg_01.bits.PRQ);
1603 1604
	printk(KERN_DEBUG ".......     : IO APIC version: %02X\n",
		reg_01.bits.version);
L
Linus Torvalds 已提交
1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628

	/*
	 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
	 * but the value of reg_02 is read as the previous read register
	 * value, so ignore it if reg_02 == reg_01.
	 */
	if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
		printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
		printk(KERN_DEBUG ".......     : arbitration: %02X\n", reg_02.bits.arbitration);
	}

	/*
	 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
	 * or reg_03, but the value of reg_0[23] is read as the previous read
	 * register value, so ignore it if reg_03 == reg_0[12].
	 */
	if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
	    reg_03.raw != reg_01.raw) {
		printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
		printk(KERN_DEBUG ".......     : Boot DT    : %X\n", reg_03.bits.boot_DT);
	}

	printk(KERN_DEBUG ".... IRQ redirection table:\n");

1629
	if (irq_remapping_enabled) {
1630 1631 1632 1633 1634 1635
		printk(KERN_DEBUG " NR Indx Fmt Mask Trig IRR"
			" Pol Stat Indx2 Zero Vect:\n");
	} else {
		printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
			" Stat Dmod Deli Vect:\n");
	}
L
Linus Torvalds 已提交
1636 1637

	for (i = 0; i <= reg_01.bits.entries; i++) {
1638
		if (irq_remapping_enabled) {
1639 1640 1641
			struct IO_APIC_route_entry entry;
			struct IR_IO_APIC_route_entry *ir_entry;

1642
			entry = ioapic_read_entry(ioapic_idx, i);
1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662
			ir_entry = (struct IR_IO_APIC_route_entry *) &entry;
			printk(KERN_DEBUG " %02x %04X ",
				i,
				ir_entry->index
			);
			printk("%1d   %1d    %1d    %1d   %1d   "
				"%1d    %1d     %X    %02X\n",
				ir_entry->format,
				ir_entry->mask,
				ir_entry->trigger,
				ir_entry->irr,
				ir_entry->polarity,
				ir_entry->delivery_status,
				ir_entry->index2,
				ir_entry->zero,
				ir_entry->vector
			);
		} else {
			struct IO_APIC_route_entry entry;

1663
			entry = ioapic_read_entry(ioapic_idx, i);
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679
			printk(KERN_DEBUG " %02x %02X  ",
				i,
				entry.dest
			);
			printk("%1d    %1d    %1d   %1d   %1d    "
				"%1d    %1d    %02X\n",
				entry.mask,
				entry.trigger,
				entry.irr,
				entry.polarity,
				entry.delivery_status,
				entry.dest_mode,
				entry.delivery_mode,
				entry.vector
			);
		}
L
Linus Torvalds 已提交
1680
	}
1681 1682 1683 1684
}

__apicdebuginit(void) print_IO_APICs(void)
{
1685
	int ioapic_idx;
1686 1687
	struct irq_cfg *cfg;
	unsigned int irq;
1688
	struct irq_chip *chip;
1689 1690

	printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1691
	for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1692
		printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1693 1694
		       mpc_ioapic_id(ioapic_idx),
		       ioapics[ioapic_idx].nr_registers);
1695 1696 1697 1698 1699 1700 1701

	/*
	 * We are a bit conservative about what we expect.  We have to
	 * know about every hardware change ASAP.
	 */
	printk(KERN_INFO "testing the IO APIC.......................\n");

1702 1703
	for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
		print_IO_APIC(ioapic_idx);
1704

L
Linus Torvalds 已提交
1705
	printk(KERN_DEBUG "IRQ to pin mappings:\n");
T
Thomas Gleixner 已提交
1706
	for_each_active_irq(irq) {
1707 1708
		struct irq_pin_list *entry;

1709 1710 1711 1712
		chip = irq_get_chip(irq);
		if (chip != &ioapic_chip)
			continue;

1713
		cfg = irq_get_chip_data(irq);
1714 1715
		if (!cfg)
			continue;
1716
		entry = cfg->irq_2_pin;
1717
		if (!entry)
L
Linus Torvalds 已提交
1718
			continue;
1719
		printk(KERN_DEBUG "IRQ%d ", irq);
1720
		for_each_irq_pin(entry, cfg->irq_2_pin)
L
Linus Torvalds 已提交
1721 1722 1723 1724 1725 1726 1727
			printk("-> %d:%d", entry->apic, entry->pin);
		printk("\n");
	}

	printk(KERN_INFO ".................................... done.\n");
}

1728
__apicdebuginit(void) print_APIC_field(int base)
L
Linus Torvalds 已提交
1729
{
1730
	int i;
L
Linus Torvalds 已提交
1731

1732 1733 1734 1735 1736 1737
	printk(KERN_DEBUG);

	for (i = 0; i < 8; i++)
		printk(KERN_CONT "%08x", apic_read(base + i*0x10));

	printk(KERN_CONT "\n");
L
Linus Torvalds 已提交
1738 1739
}

1740
__apicdebuginit(void) print_local_APIC(void *dummy)
L
Linus Torvalds 已提交
1741
{
1742
	unsigned int i, v, ver, maxlvt;
1743
	u64 icr;
L
Linus Torvalds 已提交
1744

1745
	printk(KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
L
Linus Torvalds 已提交
1746
		smp_processor_id(), hard_smp_processor_id());
1747
	v = apic_read(APIC_ID);
1748
	printk(KERN_INFO "... APIC ID:      %08x (%01x)\n", v, read_apic_id());
L
Linus Torvalds 已提交
1749 1750 1751
	v = apic_read(APIC_LVR);
	printk(KERN_INFO "... APIC VERSION: %08x\n", v);
	ver = GET_APIC_VERSION(v);
1752
	maxlvt = lapic_get_maxlvt();
L
Linus Torvalds 已提交
1753 1754 1755 1756

	v = apic_read(APIC_TASKPRI);
	printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);

1757
	if (APIC_INTEGRATED(ver)) {                     /* !82489DX */
1758 1759 1760 1761 1762
		if (!APIC_XAPIC(ver)) {
			v = apic_read(APIC_ARBPRI);
			printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
			       v & APIC_ARBPRI_MASK);
		}
L
Linus Torvalds 已提交
1763 1764 1765 1766
		v = apic_read(APIC_PROCPRI);
		printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
	}

1767 1768 1769 1770 1771 1772 1773 1774 1775
	/*
	 * Remote read supported only in the 82489DX and local APIC for
	 * Pentium processors.
	 */
	if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
		v = apic_read(APIC_RRR);
		printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
	}

L
Linus Torvalds 已提交
1776 1777
	v = apic_read(APIC_LDR);
	printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
1778 1779 1780 1781
	if (!x2apic_enabled()) {
		v = apic_read(APIC_DFR);
		printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
	}
L
Linus Torvalds 已提交
1782 1783 1784 1785
	v = apic_read(APIC_SPIV);
	printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);

	printk(KERN_DEBUG "... APIC ISR field:\n");
1786
	print_APIC_field(APIC_ISR);
L
Linus Torvalds 已提交
1787
	printk(KERN_DEBUG "... APIC TMR field:\n");
1788
	print_APIC_field(APIC_TMR);
L
Linus Torvalds 已提交
1789
	printk(KERN_DEBUG "... APIC IRR field:\n");
1790
	print_APIC_field(APIC_IRR);
L
Linus Torvalds 已提交
1791

1792 1793
	if (APIC_INTEGRATED(ver)) {             /* !82489DX */
		if (maxlvt > 3)         /* Due to the Pentium erratum 3AP. */
L
Linus Torvalds 已提交
1794
			apic_write(APIC_ESR, 0);
1795

L
Linus Torvalds 已提交
1796 1797 1798 1799
		v = apic_read(APIC_ESR);
		printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
	}

1800
	icr = apic_icr_read();
1801 1802
	printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
	printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
L
Linus Torvalds 已提交
1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826

	v = apic_read(APIC_LVTT);
	printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);

	if (maxlvt > 3) {                       /* PC is LVT#4. */
		v = apic_read(APIC_LVTPC);
		printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
	}
	v = apic_read(APIC_LVT0);
	printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
	v = apic_read(APIC_LVT1);
	printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);

	if (maxlvt > 2) {			/* ERR is LVT#3. */
		v = apic_read(APIC_LVTERR);
		printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
	}

	v = apic_read(APIC_TMICT);
	printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
	v = apic_read(APIC_TMCCT);
	printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
	v = apic_read(APIC_TDCR);
	printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838

	if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
		v = apic_read(APIC_EFEAT);
		maxlvt = (v >> 16) & 0xff;
		printk(KERN_DEBUG "... APIC EFEAT: %08x\n", v);
		v = apic_read(APIC_ECTRL);
		printk(KERN_DEBUG "... APIC ECTRL: %08x\n", v);
		for (i = 0; i < maxlvt; i++) {
			v = apic_read(APIC_EILVTn(i));
			printk(KERN_DEBUG "... APIC EILVT%d: %08x\n", i, v);
		}
	}
L
Linus Torvalds 已提交
1839 1840 1841
	printk("\n");
}

1842
__apicdebuginit(void) print_local_APICs(int maxcpu)
L
Linus Torvalds 已提交
1843
{
1844 1845
	int cpu;

1846 1847 1848
	if (!maxcpu)
		return;

1849
	preempt_disable();
1850 1851 1852
	for_each_online_cpu(cpu) {
		if (cpu >= maxcpu)
			break;
1853
		smp_call_function_single(cpu, print_local_APIC, NULL, 1);
1854
	}
1855
	preempt_enable();
L
Linus Torvalds 已提交
1856 1857
}

1858
__apicdebuginit(void) print_PIC(void)
L
Linus Torvalds 已提交
1859 1860 1861 1862
{
	unsigned int v;
	unsigned long flags;

1863
	if (!legacy_pic->nr_legacy_irqs)
L
Linus Torvalds 已提交
1864 1865 1866 1867
		return;

	printk(KERN_DEBUG "\nprinting PIC contents\n");

1868
	raw_spin_lock_irqsave(&i8259A_lock, flags);
L
Linus Torvalds 已提交
1869 1870 1871 1872 1873 1874 1875

	v = inb(0xa1) << 8 | inb(0x21);
	printk(KERN_DEBUG "... PIC  IMR: %04x\n", v);

	v = inb(0xa0) << 8 | inb(0x20);
	printk(KERN_DEBUG "... PIC  IRR: %04x\n", v);

1876 1877
	outb(0x0b,0xa0);
	outb(0x0b,0x20);
L
Linus Torvalds 已提交
1878
	v = inb(0xa0) << 8 | inb(0x20);
1879 1880
	outb(0x0a,0xa0);
	outb(0x0a,0x20);
L
Linus Torvalds 已提交
1881

1882
	raw_spin_unlock_irqrestore(&i8259A_lock, flags);
L
Linus Torvalds 已提交
1883 1884 1885 1886 1887 1888 1889

	printk(KERN_DEBUG "... PIC  ISR: %04x\n", v);

	v = inb(0x4d1) << 8 | inb(0x4d0);
	printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
}

1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907
static int __initdata show_lapic = 1;
static __init int setup_show_lapic(char *arg)
{
	int num = -1;

	if (strcmp(arg, "all") == 0) {
		show_lapic = CONFIG_NR_CPUS;
	} else {
		get_option(&arg, &num);
		if (num >= 0)
			show_lapic = num;
	}

	return 1;
}
__setup("show_lapic=", setup_show_lapic);

__apicdebuginit(int) print_ICs(void)
1908
{
1909 1910 1911
	if (apic_verbosity == APIC_QUIET)
		return 0;

1912
	print_PIC();
1913 1914

	/* don't print out if apic is not there */
1915
	if (!cpu_has_apic && !apic_from_smp_config())
1916 1917
		return 0;

1918
	print_local_APICs(show_lapic);
1919
	print_IO_APICs();
1920 1921 1922 1923

	return 0;
}

1924
late_initcall(print_ICs);
1925

L
Linus Torvalds 已提交
1926

Y
Yinghai Lu 已提交
1927 1928 1929
/* Where if anywhere is the i8259 connect in external int mode */
static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };

1930
void __init enable_IO_APIC(void)
L
Linus Torvalds 已提交
1931
{
1932
	int i8259_apic, i8259_pin;
1933
	int apic;
1934

1935
	if (!legacy_pic->nr_legacy_irqs)
1936 1937
		return;

1938
	for(apic = 0; apic < nr_ioapics; apic++) {
1939 1940
		int pin;
		/* See if any of the pins is in ExtINT mode */
S
Suresh Siddha 已提交
1941
		for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1942
			struct IO_APIC_route_entry entry;
1943
			entry = ioapic_read_entry(apic, pin);
1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973

			/* If the interrupt line is enabled and in ExtInt mode
			 * I have found the pin where the i8259 is connected.
			 */
			if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
				ioapic_i8259.apic = apic;
				ioapic_i8259.pin  = pin;
				goto found_i8259;
			}
		}
	}
 found_i8259:
	/* Look to see what if the MP table has reported the ExtINT */
	/* If we could not find the appropriate pin by looking at the ioapic
	 * the i8259 probably is not connected the ioapic but give the
	 * mptable a chance anyway.
	 */
	i8259_pin  = find_isa_irq_pin(0, mp_ExtINT);
	i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
	/* Trust the MP table if nothing is setup in the hardware */
	if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
		printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
		ioapic_i8259.pin  = i8259_pin;
		ioapic_i8259.apic = i8259_apic;
	}
	/* Complain if the MP table and the hardware disagree */
	if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
		(i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
	{
		printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
L
Linus Torvalds 已提交
1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991
	}

	/*
	 * Do not trust the IO-APIC being empty at bootup
	 */
	clear_IO_APIC();
}

/*
 * Not an __init, needed by the reboot code
 */
void disable_IO_APIC(void)
{
	/*
	 * Clear the IO-APIC before rebooting:
	 */
	clear_IO_APIC();

1992
	if (!legacy_pic->nr_legacy_irqs)
1993 1994
		return;

1995
	/*
1996
	 * If the i8259 is routed through an IOAPIC
1997
	 * Put that IOAPIC in virtual wire mode
1998
	 * so legacy interrupts can be delivered.
1999 2000 2001
	 *
	 * With interrupt-remapping, for now we will use virtual wire A mode,
	 * as virtual wire B is little complex (need to configure both
L
Lucas De Marchi 已提交
2002
	 * IOAPIC RTE as well as interrupt-remapping table entry).
2003
	 * As this gets called during crash dump, keep this simple for now.
2004
	 */
2005
	if (ioapic_i8259.pin != -1 && !irq_remapping_enabled) {
2006 2007 2008 2009 2010 2011 2012 2013 2014
		struct IO_APIC_route_entry entry;

		memset(&entry, 0, sizeof(entry));
		entry.mask            = 0; /* Enabled */
		entry.trigger         = 0; /* Edge */
		entry.irr             = 0;
		entry.polarity        = 0; /* High */
		entry.delivery_status = 0;
		entry.dest_mode       = 0; /* Physical */
2015
		entry.delivery_mode   = dest_ExtINT; /* ExtInt */
2016
		entry.vector          = 0;
2017
		entry.dest            = read_apic_id();
2018 2019 2020 2021

		/*
		 * Add it to the IO-APIC irq-routing table:
		 */
2022
		ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
2023
	}
2024

2025 2026 2027
	/*
	 * Use virtual wire A mode when interrupt remapping is enabled.
	 */
2028
	if (cpu_has_apic || apic_from_smp_config())
2029
		disconnect_bsp_APIC(!irq_remapping_enabled &&
2030
				ioapic_i8259.pin != -1);
L
Linus Torvalds 已提交
2031 2032
}

2033
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
2034 2035 2036 2037 2038 2039
/*
 * function to set the IO-APIC physical IDs based on the
 * values stored in the MPC table.
 *
 * by Matt Domsch <Matt_Domsch@dell.com>  Tue Dec 21 12:25:05 CST 1999
 */
2040
void __init setup_ioapic_ids_from_mpc_nocheck(void)
L
Linus Torvalds 已提交
2041 2042 2043
{
	union IO_APIC_reg_00 reg_00;
	physid_mask_t phys_id_present_map;
2044
	int ioapic_idx;
L
Linus Torvalds 已提交
2045 2046 2047 2048 2049 2050 2051 2052
	int i;
	unsigned char old_id;
	unsigned long flags;

	/*
	 * This is broken; anything with a real cpu count has to
	 * circumvent this idiocy regardless.
	 */
2053
	apic->ioapic_phys_id_map(&phys_cpu_present_map, &phys_id_present_map);
L
Linus Torvalds 已提交
2054 2055 2056 2057

	/*
	 * Set the IOAPIC ID to the value stored in the MPC table.
	 */
2058
	for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) {
L
Linus Torvalds 已提交
2059
		/* Read the register 0 value */
2060
		raw_spin_lock_irqsave(&ioapic_lock, flags);
2061
		reg_00.raw = io_apic_read(ioapic_idx, 0);
2062
		raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2063

2064
		old_id = mpc_ioapic_id(ioapic_idx);
L
Linus Torvalds 已提交
2065

2066
		if (mpc_ioapic_id(ioapic_idx) >= get_physical_broadcast()) {
L
Linus Torvalds 已提交
2067
			printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
2068
				ioapic_idx, mpc_ioapic_id(ioapic_idx));
L
Linus Torvalds 已提交
2069 2070
			printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
				reg_00.bits.ID);
2071
			ioapics[ioapic_idx].mp_config.apicid = reg_00.bits.ID;
L
Linus Torvalds 已提交
2072 2073 2074 2075 2076 2077 2078
		}

		/*
		 * Sanity check, is the ID really free? Every APIC in a
		 * system must have a unique ID or we get lots of nice
		 * 'stuck on smp_invalidate_needed IPI wait' messages.
		 */
2079
		if (apic->check_apicid_used(&phys_id_present_map,
2080
					    mpc_ioapic_id(ioapic_idx))) {
L
Linus Torvalds 已提交
2081
			printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
2082
				ioapic_idx, mpc_ioapic_id(ioapic_idx));
L
Linus Torvalds 已提交
2083 2084 2085 2086 2087 2088 2089 2090
			for (i = 0; i < get_physical_broadcast(); i++)
				if (!physid_isset(i, phys_id_present_map))
					break;
			if (i >= get_physical_broadcast())
				panic("Max APIC ID exceeded!\n");
			printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
				i);
			physid_set(i, phys_id_present_map);
2091
			ioapics[ioapic_idx].mp_config.apicid = i;
L
Linus Torvalds 已提交
2092 2093
		} else {
			physid_mask_t tmp;
2094
			apic->apicid_to_cpu_present(mpc_ioapic_id(ioapic_idx),
2095
						    &tmp);
L
Linus Torvalds 已提交
2096 2097
			apic_printk(APIC_VERBOSE, "Setting %d in the "
					"phys_id_present_map\n",
2098
					mpc_ioapic_id(ioapic_idx));
L
Linus Torvalds 已提交
2099 2100 2101 2102 2103 2104 2105
			physids_or(phys_id_present_map, phys_id_present_map, tmp);
		}

		/*
		 * We need to adjust the IRQ routing table
		 * if the ID changed.
		 */
2106
		if (old_id != mpc_ioapic_id(ioapic_idx))
L
Linus Torvalds 已提交
2107
			for (i = 0; i < mp_irq_entries; i++)
2108 2109
				if (mp_irqs[i].dstapic == old_id)
					mp_irqs[i].dstapic
2110
						= mpc_ioapic_id(ioapic_idx);
L
Linus Torvalds 已提交
2111 2112

		/*
2113 2114
		 * Update the ID register according to the right value
		 * from the MPC table if they are different.
2115
		 */
2116
		if (mpc_ioapic_id(ioapic_idx) == reg_00.bits.ID)
2117 2118
			continue;

L
Linus Torvalds 已提交
2119 2120
		apic_printk(APIC_VERBOSE, KERN_INFO
			"...changing IO-APIC physical APIC ID to %d ...",
2121
			mpc_ioapic_id(ioapic_idx));
L
Linus Torvalds 已提交
2122

2123
		reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
2124
		raw_spin_lock_irqsave(&ioapic_lock, flags);
2125
		io_apic_write(ioapic_idx, 0, reg_00.raw);
2126
		raw_spin_unlock_irqrestore(&ioapic_lock, flags);
L
Linus Torvalds 已提交
2127 2128 2129 2130

		/*
		 * Sanity check
		 */
2131
		raw_spin_lock_irqsave(&ioapic_lock, flags);
2132
		reg_00.raw = io_apic_read(ioapic_idx, 0);
2133
		raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2134
		if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx))
L
Linus Torvalds 已提交
2135 2136 2137 2138 2139
			printk("could not set ID!\n");
		else
			apic_printk(APIC_VERBOSE, " ok.\n");
	}
}
2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154

void __init setup_ioapic_ids_from_mpc(void)
{

	if (acpi_ioapic)
		return;
	/*
	 * Don't check I/O APIC IDs for xAPIC systems.  They have
	 * no meaning without the serial APIC bus.
	 */
	if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
		|| APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
		return;
	setup_ioapic_ids_from_mpc_nocheck();
}
2155
#endif
L
Linus Torvalds 已提交
2156

2157
int no_timer_check __initdata;
2158 2159 2160 2161 2162 2163 2164 2165

static int __init notimercheck(char *s)
{
	no_timer_check = 1;
	return 1;
}
__setup("no_timer_check", notimercheck);

L
Linus Torvalds 已提交
2166 2167 2168 2169 2170 2171 2172 2173
/*
 * There is a nasty bug in some older SMP boards, their mptable lies
 * about the timer IRQ. We do the following to work around the situation:
 *
 *	- timer IRQ defaults to IO-APIC IRQ
 *	- if this function detects that timer IRQs are defunct, then we fall
 *	  back to ISA timer IRQs
 */
2174
static int __init timer_irq_works(void)
L
Linus Torvalds 已提交
2175 2176
{
	unsigned long t1 = jiffies;
2177
	unsigned long flags;
L
Linus Torvalds 已提交
2178

2179 2180 2181
	if (no_timer_check)
		return 1;

2182
	local_save_flags(flags);
L
Linus Torvalds 已提交
2183 2184 2185
	local_irq_enable();
	/* Let ten ticks pass... */
	mdelay((10 * 1000) / HZ);
2186
	local_irq_restore(flags);
L
Linus Torvalds 已提交
2187 2188 2189 2190 2191 2192 2193 2194

	/*
	 * Expect a few ticks at least, to be sure some possible
	 * glue logic does not lock up after one or two first
	 * ticks in a non-ExtINT mode.  Also the local APIC
	 * might have cached one ExtINT interrupt.  Finally, at
	 * least one tick may be lost due to delays.
	 */
2195 2196

	/* jiffies wrap? */
2197
	if (time_after(jiffies, t1 + 4))
L
Linus Torvalds 已提交
2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223
		return 1;
	return 0;
}

/*
 * In the SMP+IOAPIC case it might happen that there are an unspecified
 * number of pending IRQ events unhandled. These cases are very rare,
 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
 * better to do it this way as thus we do not have to be aware of
 * 'pending' interrupts in the IRQ path, except at this point.
 */
/*
 * Edge triggered needs to resend any interrupt
 * that was delayed but this is now handled in the device
 * independent code.
 */

/*
 * Starting up a edge-triggered IO-APIC interrupt is
 * nasty - we need to make sure that we get the edge.
 * If it is already asserted for some reason, we need
 * return 1 to indicate that is was pending.
 *
 * This is not complete - we should be able to fake
 * an edge even if it isn't on the 8259A...
 */
2224

2225
static unsigned int startup_ioapic_irq(struct irq_data *data)
L
Linus Torvalds 已提交
2226
{
2227
	int was_pending = 0, irq = data->irq;
L
Linus Torvalds 已提交
2228 2229
	unsigned long flags;

2230
	raw_spin_lock_irqsave(&ioapic_lock, flags);
2231
	if (irq < legacy_pic->nr_legacy_irqs) {
2232
		legacy_pic->mask(irq);
2233
		if (legacy_pic->irq_pending(irq))
L
Linus Torvalds 已提交
2234 2235
			was_pending = 1;
	}
2236
	__unmask_ioapic(data->chip_data);
2237
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
L
Linus Torvalds 已提交
2238 2239 2240 2241

	return was_pending;
}

2242
static int ioapic_retrigger_irq(struct irq_data *data)
L
Linus Torvalds 已提交
2243
{
2244
	struct irq_cfg *cfg = data->chip_data;
2245 2246
	unsigned long flags;

2247
	raw_spin_lock_irqsave(&vector_lock, flags);
2248
	apic->send_IPI_mask(cpumask_of(cpumask_first(cfg->domain)), cfg->vector);
2249
	raw_spin_unlock_irqrestore(&vector_lock, flags);
2250 2251 2252

	return 1;
}
2253

2254 2255 2256 2257 2258 2259 2260 2261
/*
 * Level and edge triggered IO-APIC interrupts need different handling,
 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
 * handled with the level-triggered descriptor, but that one has slightly
 * more overhead. Level-triggered interrupts cannot be handled with the
 * edge-triggered handler, without risking IRQ storms and other ugly
 * races.
 */
2262

2263
#ifdef CONFIG_SMP
2264
void send_cleanup_vector(struct irq_cfg *cfg)
2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279
{
	cpumask_var_t cleanup_mask;

	if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
		unsigned int i;
		for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
			apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
	} else {
		cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
		apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
		free_cpumask_var(cleanup_mask);
	}
	cfg->move_in_progress = 0;
}

2280
static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
2281 2282 2283 2284 2285
{
	int apic, pin;
	struct irq_pin_list *entry;
	u8 vector = cfg->vector;

2286
	for_each_irq_pin(entry, cfg->irq_2_pin) {
2287 2288 2289 2290 2291 2292 2293 2294
		unsigned int reg;

		apic = entry->apic;
		pin = entry->pin;
		/*
		 * With interrupt-remapping, destination information comes
		 * from interrupt-remapping table entry.
		 */
2295
		if (!irq_remapped(cfg))
2296 2297 2298 2299 2300 2301 2302 2303 2304
			io_apic_write(apic, 0x11 + pin*2, dest);
		reg = io_apic_read(apic, 0x10 + pin*2);
		reg &= ~IO_APIC_REDIR_VECTOR_MASK;
		reg |= vector;
		io_apic_modify(apic, 0x10 + pin*2, reg);
	}
}

/*
2305
 * Either sets data->affinity to a valid value, and returns
2306
 * ->cpu_mask_to_apicid of that in dest_id, or returns -1 and
2307
 * leaves data->affinity untouched.
2308
 */
2309 2310
int __ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
			  unsigned int *dest_id)
2311
{
2312
	struct irq_cfg *cfg = data->chip_data;
2313 2314

	if (!cpumask_intersects(mask, cpu_online_mask))
2315
		return -1;
2316

2317
	if (assign_irq_vector(data->irq, data->chip_data, mask))
2318
		return -1;
2319

2320
	cpumask_copy(data->affinity, mask);
2321

2322
	*dest_id = apic->cpu_mask_to_apicid_and(mask, cfg->domain);
2323
	return 0;
2324 2325
}

2326
static int
2327 2328
ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
		    bool force)
2329
{
2330
	unsigned int dest, irq = data->irq;
2331
	unsigned long flags;
2332
	int ret;
2333

2334
	raw_spin_lock_irqsave(&ioapic_lock, flags);
2335
	ret = __ioapic_set_affinity(data, mask, &dest);
2336
	if (!ret) {
2337 2338
		/* Only the high 8 bits are valid. */
		dest = SET_APIC_LOGICAL_ID(dest);
2339
		__target_IO_APIC_irq(irq, dest, data->chip_data);
2340
	}
2341
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2342
	return ret;
2343 2344
}

2345 2346 2347
asmlinkage void smp_irq_move_cleanup_interrupt(void)
{
	unsigned vector, me;
2348

2349 2350
	ack_APIC_irq();
	irq_enter();
2351
	exit_idle();
2352 2353 2354 2355

	me = smp_processor_id();
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		unsigned int irq;
2356
		unsigned int irr;
2357 2358
		struct irq_desc *desc;
		struct irq_cfg *cfg;
T
Tejun Heo 已提交
2359
		irq = __this_cpu_read(vector_irq[vector]);
2360

2361 2362 2363
		if (irq == -1)
			continue;

2364 2365 2366 2367 2368
		desc = irq_to_desc(irq);
		if (!desc)
			continue;

		cfg = irq_cfg(irq);
2369
		raw_spin_lock(&desc->lock);
2370

2371 2372 2373 2374 2375 2376 2377
		/*
		 * Check if the irq migration is in progress. If so, we
		 * haven't received the cleanup request yet for this irq.
		 */
		if (cfg->move_in_progress)
			goto unlock;

2378
		if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2379 2380
			goto unlock;

2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392
		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		/*
		 * Check if the vector that needs to be cleanedup is
		 * registered at the cpu's IRR. If so, then this is not
		 * the best time to clean it up. Lets clean it up in the
		 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
		 * to myself.
		 */
		if (irr  & (1 << (vector % 32))) {
			apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
			goto unlock;
		}
T
Tejun Heo 已提交
2393
		__this_cpu_write(vector_irq[vector], -1);
2394
unlock:
2395
		raw_spin_unlock(&desc->lock);
2396 2397 2398 2399 2400
	}

	irq_exit();
}

T
Thomas Gleixner 已提交
2401
static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
2402
{
2403
	unsigned me;
2404

2405
	if (likely(!cfg->move_in_progress))
2406 2407 2408
		return;

	me = smp_processor_id();
2409

2410
	if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2411
		send_cleanup_vector(cfg);
2412
}
2413

T
Thomas Gleixner 已提交
2414
static void irq_complete_move(struct irq_cfg *cfg)
2415
{
T
Thomas Gleixner 已提交
2416
	__irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
2417 2418 2419 2420
}

void irq_force_complete_move(int irq)
{
2421
	struct irq_cfg *cfg = irq_get_chip_data(irq);
2422

2423 2424 2425
	if (!cfg)
		return;

T
Thomas Gleixner 已提交
2426
	__irq_complete_move(cfg, cfg->vector);
2427
}
2428
#else
T
Thomas Gleixner 已提交
2429
static inline void irq_complete_move(struct irq_cfg *cfg) { }
2430
#endif
Y
Yinghai Lu 已提交
2431

2432
static void ack_apic_edge(struct irq_data *data)
2433
{
2434
	irq_complete_move(data->chip_data);
2435
	irq_move_irq(data);
2436 2437 2438
	ack_APIC_irq();
}

Y
Yinghai Lu 已提交
2439 2440
atomic_t irq_mis_count;

2441
#ifdef CONFIG_GENERIC_PENDING_IRQ
2442 2443
static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg)
{
2444
	/* If we are moving the irq we need to mask it */
2445
	if (unlikely(irqd_is_setaffinity_pending(data))) {
T
Thomas Gleixner 已提交
2446
		mask_ioapic(cfg);
2447
		return true;
2448
	}
2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495
	return false;
}

static inline void ioapic_irqd_unmask(struct irq_data *data,
				      struct irq_cfg *cfg, bool masked)
{
	if (unlikely(masked)) {
		/* Only migrate the irq if the ack has been received.
		 *
		 * On rare occasions the broadcast level triggered ack gets
		 * delayed going to ioapics, and if we reprogram the
		 * vector while Remote IRR is still set the irq will never
		 * fire again.
		 *
		 * To prevent this scenario we read the Remote IRR bit
		 * of the ioapic.  This has two effects.
		 * - On any sane system the read of the ioapic will
		 *   flush writes (and acks) going to the ioapic from
		 *   this cpu.
		 * - We get to see if the ACK has actually been delivered.
		 *
		 * Based on failed experiments of reprogramming the
		 * ioapic entry from outside of irq context starting
		 * with masking the ioapic entry and then polling until
		 * Remote IRR was clear before reprogramming the
		 * ioapic I don't trust the Remote IRR bit to be
		 * completey accurate.
		 *
		 * However there appears to be no other way to plug
		 * this race, so if the Remote IRR bit is not
		 * accurate and is causing problems then it is a hardware bug
		 * and you can go talk to the chipset vendor about it.
		 */
		if (!io_apic_level_ack_pending(cfg))
			irq_move_masked_irq(data);
		unmask_ioapic(cfg);
	}
}
#else
static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg)
{
	return false;
}
static inline void ioapic_irqd_unmask(struct irq_data *data,
				      struct irq_cfg *cfg, bool masked)
{
}
2496 2497
#endif

2498 2499 2500 2501 2502 2503 2504 2505 2506 2507
static void ack_apic_level(struct irq_data *data)
{
	struct irq_cfg *cfg = data->chip_data;
	int i, irq = data->irq;
	unsigned long v;
	bool masked;

	irq_complete_move(cfg);
	masked = ioapic_irqd_mask(data, cfg);

Y
Yinghai Lu 已提交
2508
	/*
2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525
	 * It appears there is an erratum which affects at least version 0x11
	 * of I/O APIC (that's the 82093AA and cores integrated into various
	 * chipsets).  Under certain conditions a level-triggered interrupt is
	 * erroneously delivered as edge-triggered one but the respective IRR
	 * bit gets set nevertheless.  As a result the I/O unit expects an EOI
	 * message but it will never arrive and further interrupts are blocked
	 * from the source.  The exact reason is so far unknown, but the
	 * phenomenon was observed when two consecutive interrupt requests
	 * from a given source get delivered to the same CPU and the source is
	 * temporarily disabled in between.
	 *
	 * A workaround is to simulate an EOI message manually.  We achieve it
	 * by setting the trigger mode to edge and then to level when the edge
	 * trigger mode gets detected in the TMR of a local APIC for a
	 * level-triggered interrupt.  We mask the source for the time of the
	 * operation to prevent an edge-triggered interrupt escaping meanwhile.
	 * The idea is from Manfred Spraul.  --macro
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538
	 *
	 * Also in the case when cpu goes offline, fixup_irqs() will forward
	 * any unhandled interrupt on the offlined cpu to the new cpu
	 * destination that is handling the corresponding interrupt. This
	 * interrupt forwarding is done via IPI's. Hence, in this case also
	 * level-triggered io-apic interrupt will be seen as an edge
	 * interrupt in the IRR. And we can't rely on the cpu's EOI
	 * to be broadcasted to the IO-APIC's which will clear the remoteIRR
	 * corresponding to the level-triggered interrupt. Hence on IO-APIC's
	 * supporting EOI register, we do an explicit EOI to clear the
	 * remote IRR and on IO-APIC's which don't have an EOI register,
	 * we use the above logic (mask+edge followed by unmask+level) from
	 * Manfred Spraul to clear the remote IRR.
2539
	 */
Y
Yinghai Lu 已提交
2540
	i = cfg->vector;
Y
Yinghai Lu 已提交
2541 2542
	v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));

2543 2544 2545 2546 2547 2548
	/*
	 * We must acknowledge the irq before we move it or the acknowledge will
	 * not propagate properly.
	 */
	ack_APIC_irq();

2549 2550 2551 2552 2553 2554 2555
	/*
	 * Tail end of clearing remote IRR bit (either by delivering the EOI
	 * message via io-apic EOI register write or simulating it using
	 * mask+edge followed by unnask+level logic) manually when the
	 * level triggered interrupt is seen as the edge triggered interrupt
	 * at the cpu.
	 */
2556 2557 2558
	if (!(v & (1 << (i & 0x1f)))) {
		atomic_inc(&irq_mis_count);

T
Thomas Gleixner 已提交
2559
		eoi_ioapic_irq(irq, cfg);
2560 2561
	}

2562
	ioapic_irqd_unmask(data, cfg, masked);
Y
Yinghai Lu 已提交
2563
}
2564

2565
#ifdef CONFIG_IRQ_REMAP
2566
static void ir_ack_apic_edge(struct irq_data *data)
2567
{
2568
	ack_APIC_irq();
2569 2570
}

2571
static void ir_ack_apic_level(struct irq_data *data)
2572
{
2573
	ack_APIC_irq();
2574
	eoi_ioapic_irq(data->irq, data->chip_data);
2575
}
2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588

static void ir_print_prefix(struct irq_data *data, struct seq_file *p)
{
	seq_printf(p, " IR-%s", data->chip->name);
}

static void irq_remap_modify_chip_defaults(struct irq_chip *chip)
{
	chip->irq_print_chip = ir_print_prefix;
	chip->irq_ack = ir_ack_apic_edge;
	chip->irq_eoi = ir_ack_apic_level;

#ifdef CONFIG_SMP
2589
	chip->irq_set_affinity = set_remapped_irq_affinity;
2590 2591
#endif
}
2592
#endif /* CONFIG_IRQ_REMAP */
2593

2594
static struct irq_chip ioapic_chip __read_mostly = {
2595 2596 2597 2598 2599 2600
	.name			= "IO-APIC",
	.irq_startup		= startup_ioapic_irq,
	.irq_mask		= mask_ioapic_irq,
	.irq_unmask		= unmask_ioapic_irq,
	.irq_ack		= ack_apic_edge,
	.irq_eoi		= ack_apic_level,
2601
#ifdef CONFIG_SMP
2602
	.irq_set_affinity	= ioapic_set_affinity,
2603
#endif
2604
	.irq_retrigger		= ioapic_retrigger_irq,
L
Linus Torvalds 已提交
2605 2606 2607 2608
};

static inline void init_IO_APIC_traps(void)
{
2609
	struct irq_cfg *cfg;
T
Thomas Gleixner 已提交
2610
	unsigned int irq;
L
Linus Torvalds 已提交
2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622

	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
T
Thomas Gleixner 已提交
2623
	for_each_active_irq(irq) {
2624
		cfg = irq_get_chip_data(irq);
2625
		if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
L
Linus Torvalds 已提交
2626 2627 2628 2629 2630
			/*
			 * Hmm.. We don't have an entry for this,
			 * so default to an old-fashioned 8259
			 * interrupt if we can..
			 */
2631 2632
			if (irq < legacy_pic->nr_legacy_irqs)
				legacy_pic->make_irq(irq);
2633
			else
L
Linus Torvalds 已提交
2634
				/* Strange. Oh, well.. */
2635
				irq_set_chip(irq, &no_irq_chip);
L
Linus Torvalds 已提交
2636 2637 2638 2639
		}
	}
}

2640 2641 2642
/*
 * The local APIC irq-chip implementation:
 */
L
Linus Torvalds 已提交
2643

2644
static void mask_lapic_irq(struct irq_data *data)
L
Linus Torvalds 已提交
2645 2646 2647 2648
{
	unsigned long v;

	v = apic_read(APIC_LVT0);
2649
	apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
L
Linus Torvalds 已提交
2650 2651
}

2652
static void unmask_lapic_irq(struct irq_data *data)
L
Linus Torvalds 已提交
2653
{
2654
	unsigned long v;
L
Linus Torvalds 已提交
2655

2656
	v = apic_read(APIC_LVT0);
2657
	apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2658
}
L
Linus Torvalds 已提交
2659

2660
static void ack_lapic_irq(struct irq_data *data)
2661 2662 2663 2664
{
	ack_APIC_irq();
}

2665
static struct irq_chip lapic_chip __read_mostly = {
2666
	.name		= "local-APIC",
2667 2668 2669
	.irq_mask	= mask_lapic_irq,
	.irq_unmask	= unmask_lapic_irq,
	.irq_ack	= ack_lapic_irq,
L
Linus Torvalds 已提交
2670 2671
};

2672
static void lapic_register_intr(int irq)
2673
{
2674
	irq_clear_status_flags(irq, IRQ_LEVEL);
2675
	irq_set_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2676 2677 2678
				      "edge");
}

L
Linus Torvalds 已提交
2679 2680 2681 2682 2683 2684 2685
/*
 * This looks a bit hackish but it's about the only one way of sending
 * a few INTA cycles to 8259As and any associated glue logic.  ICR does
 * not support the ExtINT mode, unfortunately.  We need to send these
 * cycles as some i82489DX-based boards have glue logic that keeps the
 * 8259A interrupt line asserted until INTA.  --macro
 */
2686
static inline void __init unlock_ExtINT_logic(void)
L
Linus Torvalds 已提交
2687
{
2688
	int apic, pin, i;
L
Linus Torvalds 已提交
2689 2690 2691
	struct IO_APIC_route_entry entry0, entry1;
	unsigned char save_control, save_freq_select;

2692
	pin  = find_isa_irq_pin(8, mp_INT);
2693 2694 2695 2696
	if (pin == -1) {
		WARN_ON_ONCE(1);
		return;
	}
2697
	apic = find_isa_irq_apic(8, mp_INT);
2698 2699
	if (apic == -1) {
		WARN_ON_ONCE(1);
L
Linus Torvalds 已提交
2700
		return;
2701
	}
L
Linus Torvalds 已提交
2702

2703
	entry0 = ioapic_read_entry(apic, pin);
2704
	clear_IO_APIC_pin(apic, pin);
L
Linus Torvalds 已提交
2705 2706 2707 2708 2709

	memset(&entry1, 0, sizeof(entry1));

	entry1.dest_mode = 0;			/* physical delivery */
	entry1.mask = 0;			/* unmask IRQ now */
2710
	entry1.dest = hard_smp_processor_id();
L
Linus Torvalds 已提交
2711 2712 2713 2714 2715
	entry1.delivery_mode = dest_ExtINT;
	entry1.polarity = entry0.polarity;
	entry1.trigger = 0;
	entry1.vector = 0;

2716
	ioapic_write_entry(apic, pin, entry1);
L
Linus Torvalds 已提交
2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732

	save_control = CMOS_READ(RTC_CONTROL);
	save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
	CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
		   RTC_FREQ_SELECT);
	CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);

	i = 100;
	while (i-- > 0) {
		mdelay(10);
		if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
			i -= 10;
	}

	CMOS_WRITE(save_control, RTC_CONTROL);
	CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2733
	clear_IO_APIC_pin(apic, pin);
L
Linus Torvalds 已提交
2734

2735
	ioapic_write_entry(apic, pin, entry0);
L
Linus Torvalds 已提交
2736 2737
}

Y
Yinghai Lu 已提交
2738
static int disable_timer_pin_1 __initdata;
2739
/* Actually the next is obsolete, but keep it for paranoid reasons -AK */
2740
static int __init disable_timer_pin_setup(char *arg)
Y
Yinghai Lu 已提交
2741 2742 2743 2744
{
	disable_timer_pin_1 = 1;
	return 0;
}
2745
early_param("disable_timer_pin_1", disable_timer_pin_setup);
Y
Yinghai Lu 已提交
2746 2747 2748

int timer_through_8259 __initdata;

L
Linus Torvalds 已提交
2749 2750 2751 2752 2753
/*
 * This code may look a bit paranoid, but it's supposed to cooperate with
 * a wide range of boards and BIOS bugs.  Fortunately only the timer IRQ
 * is so screwy.  Thanks to Brian Perkins for testing/hacking this beast
 * fanatically on his truly buggy board.
2754 2755
 *
 * FIXME: really need to revamp this for all platforms.
L
Linus Torvalds 已提交
2756
 */
2757
static inline void __init check_timer(void)
L
Linus Torvalds 已提交
2758
{
2759
	struct irq_cfg *cfg = irq_get_chip_data(0);
2760
	int node = cpu_to_node(0);
2761
	int apic1, pin1, apic2, pin2;
2762
	unsigned long flags;
2763
	int no_pin1 = 0;
2764 2765

	local_irq_save(flags);
2766

L
Linus Torvalds 已提交
2767 2768 2769
	/*
	 * get/set the timer IRQ vector:
	 */
2770
	legacy_pic->mask(0);
2771
	assign_irq_vector(0, cfg, apic->target_cpus());
L
Linus Torvalds 已提交
2772 2773

	/*
2774 2775 2776 2777 2778 2779 2780
	 * As IRQ0 is to be enabled in the 8259A, the virtual
	 * wire has to be disabled in the local APIC.  Also
	 * timer interrupts need to be acknowledged manually in
	 * the 8259A for the i82489DX when using the NMI
	 * watchdog as that APIC treats NMIs as level-triggered.
	 * The AEOI mode will finish them in the 8259A
	 * automatically.
L
Linus Torvalds 已提交
2781
	 */
2782
	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2783
	legacy_pic->init(1);
L
Linus Torvalds 已提交
2784

2785 2786 2787 2788
	pin1  = find_isa_irq_pin(0, mp_INT);
	apic1 = find_isa_irq_apic(0, mp_INT);
	pin2  = ioapic_i8259.pin;
	apic2 = ioapic_i8259.apic;
L
Linus Torvalds 已提交
2789

2790 2791
	apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
		    "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2792
		    cfg->vector, apic1, pin1, apic2, pin2);
L
Linus Torvalds 已提交
2793

2794 2795 2796 2797 2798 2799 2800 2801
	/*
	 * Some BIOS writers are clueless and report the ExtINTA
	 * I/O APIC input from the cascaded 8259A as the timer
	 * interrupt input.  So just in case, if only one pin
	 * was found above, try it both directly and through the
	 * 8259A.
	 */
	if (pin1 == -1) {
2802
		if (irq_remapping_enabled)
2803
			panic("BIOS bug: timer not connected to IO-APIC");
2804 2805 2806 2807 2808 2809 2810 2811
		pin1 = pin2;
		apic1 = apic2;
		no_pin1 = 1;
	} else if (pin2 == -1) {
		pin2 = pin1;
		apic2 = apic1;
	}

L
Linus Torvalds 已提交
2812 2813 2814 2815
	if (pin1 != -1) {
		/*
		 * Ok, does IRQ0 through the IOAPIC work?
		 */
2816
		if (no_pin1) {
2817
			add_pin_to_irq_node(cfg, node, apic1, pin1);
2818
			setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
Y
Yinghai Lu 已提交
2819
		} else {
2820
			/* for edge trigger, setup_ioapic_irq already
Y
Yinghai Lu 已提交
2821 2822 2823 2824 2825 2826 2827
			 * leave it unmasked.
			 * so only need to unmask if it is level-trigger
			 * do we really have level trigger timer?
			 */
			int idx;
			idx = find_irq_entry(apic1, pin1, mp_INT);
			if (idx != -1 && irq_trigger(idx))
T
Thomas Gleixner 已提交
2828
				unmask_ioapic(cfg);
2829
		}
L
Linus Torvalds 已提交
2830
		if (timer_irq_works()) {
2831 2832
			if (disable_timer_pin_1 > 0)
				clear_IO_APIC_pin(0, pin1);
2833
			goto out;
L
Linus Torvalds 已提交
2834
		}
2835
		if (irq_remapping_enabled)
2836
			panic("timer doesn't work through Interrupt-remapped IO-APIC");
Y
Yinghai Lu 已提交
2837
		local_irq_disable();
2838
		clear_IO_APIC_pin(apic1, pin1);
2839
		if (!no_pin1)
2840 2841
			apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
				    "8254 timer not connected to IO-APIC\n");
L
Linus Torvalds 已提交
2842

2843 2844 2845 2846
		apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
			    "(IRQ0) through the 8259A ...\n");
		apic_printk(APIC_QUIET, KERN_INFO
			    "..... (found apic %d pin %d) ...\n", apic2, pin2);
L
Linus Torvalds 已提交
2847 2848 2849
		/*
		 * legacy devices should be connected to IO APIC #0
		 */
2850
		replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
2851
		setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2852
		legacy_pic->unmask(0);
L
Linus Torvalds 已提交
2853
		if (timer_irq_works()) {
2854
			apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2855
			timer_through_8259 = 1;
2856
			goto out;
L
Linus Torvalds 已提交
2857 2858 2859 2860
		}
		/*
		 * Cleanup, just in case ...
		 */
Y
Yinghai Lu 已提交
2861
		local_irq_disable();
2862
		legacy_pic->mask(0);
2863
		clear_IO_APIC_pin(apic2, pin2);
2864
		apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
L
Linus Torvalds 已提交
2865 2866
	}

2867 2868
	apic_printk(APIC_QUIET, KERN_INFO
		    "...trying to set up timer as Virtual Wire IRQ...\n");
L
Linus Torvalds 已提交
2869

2870
	lapic_register_intr(0);
2871
	apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);	/* Fixed mode */
2872
	legacy_pic->unmask(0);
L
Linus Torvalds 已提交
2873 2874

	if (timer_irq_works()) {
2875
		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2876
		goto out;
L
Linus Torvalds 已提交
2877
	}
Y
Yinghai Lu 已提交
2878
	local_irq_disable();
2879
	legacy_pic->mask(0);
2880
	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2881
	apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
L
Linus Torvalds 已提交
2882

2883 2884
	apic_printk(APIC_QUIET, KERN_INFO
		    "...trying to set up timer as ExtINT IRQ...\n");
L
Linus Torvalds 已提交
2885

2886 2887
	legacy_pic->init(0);
	legacy_pic->make_irq(0);
2888
	apic_write(APIC_LVT0, APIC_DM_EXTINT);
L
Linus Torvalds 已提交
2889 2890 2891 2892

	unlock_ExtINT_logic();

	if (timer_irq_works()) {
2893
		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2894
		goto out;
L
Linus Torvalds 已提交
2895
	}
Y
Yinghai Lu 已提交
2896
	local_irq_disable();
2897
	apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
2898 2899 2900 2901
	if (x2apic_preenabled)
		apic_printk(APIC_QUIET, KERN_INFO
			    "Perhaps problem with the pre-enabled x2apic mode\n"
			    "Try booting with x2apic and interrupt-remapping disabled in the bios.\n");
L
Linus Torvalds 已提交
2902
	panic("IO-APIC + timer doesn't work!  Boot with apic=debug and send a "
2903
		"report.  Then try booting with the 'noapic' option.\n");
2904 2905
out:
	local_irq_restore(flags);
L
Linus Torvalds 已提交
2906 2907 2908
}

/*
2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923
 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
 * to devices.  However there may be an I/O APIC pin available for
 * this interrupt regardless.  The pin may be left unconnected, but
 * typically it will be reused as an ExtINT cascade interrupt for
 * the master 8259A.  In the MPS case such a pin will normally be
 * reported as an ExtINT interrupt in the MP table.  With ACPI
 * there is no provision for ExtINT interrupts, and in the absence
 * of an override it would be treated as an ordinary ISA I/O APIC
 * interrupt, that is edge-triggered and unmasked by default.  We
 * used to do this, but it caused problems on some systems because
 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
 * the same ExtINT cascade interrupt to drive the local APIC of the
 * bootstrap processor.  Therefore we refrain from routing IRQ2 to
 * the I/O APIC in all cases now.  No actual device should request
 * it anyway.  --macro
L
Linus Torvalds 已提交
2924
 */
2925
#define PIC_IRQS	(1UL << PIC_CASCADE_IR)
L
Linus Torvalds 已提交
2926 2927 2928

void __init setup_IO_APIC(void)
{
2929 2930 2931 2932

	/*
	 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
	 */
2933
	io_apic_irqs = legacy_pic->nr_legacy_irqs ? ~PIC_IRQS : ~0UL;
L
Linus Torvalds 已提交
2934

2935
	apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
T
Thomas Gleixner 已提交
2936
	/*
2937 2938
         * Set up IO-APIC IRQ routing.
         */
2939 2940
	x86_init.mpparse.setup_ioapic_ids();

L
Linus Torvalds 已提交
2941 2942 2943
	sync_Arb_IDs();
	setup_IO_APIC_irqs();
	init_IO_APIC_traps();
2944
	if (legacy_pic->nr_legacy_irqs)
2945
		check_timer();
L
Linus Torvalds 已提交
2946 2947 2948
}

/*
L
Lucas De Marchi 已提交
2949
 *      Called after all the initialization is done. If we didn't find any
2950
 *      APIC bugs then we can allow the modify fast path
L
Linus Torvalds 已提交
2951
 */
2952

L
Linus Torvalds 已提交
2953 2954
static int __init io_apic_bug_finalize(void)
{
T
Thomas Gleixner 已提交
2955 2956 2957
	if (sis_apic_bug == -1)
		sis_apic_bug = 0;
	return 0;
L
Linus Torvalds 已提交
2958 2959 2960 2961
}

late_initcall(io_apic_bug_finalize);

2962
static void resume_ioapic_id(int ioapic_idx)
L
Linus Torvalds 已提交
2963 2964 2965
{
	unsigned long flags;
	union IO_APIC_reg_00 reg_00;
2966

2967
	raw_spin_lock_irqsave(&ioapic_lock, flags);
2968 2969 2970 2971
	reg_00.raw = io_apic_read(ioapic_idx, 0);
	if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) {
		reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
		io_apic_write(ioapic_idx, 0, reg_00.raw);
L
Linus Torvalds 已提交
2972
	}
2973
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2974
}
L
Linus Torvalds 已提交
2975

2976 2977
static void ioapic_resume(void)
{
2978
	int ioapic_idx;
2979

2980 2981
	for (ioapic_idx = nr_ioapics - 1; ioapic_idx >= 0; ioapic_idx--)
		resume_ioapic_id(ioapic_idx);
2982 2983

	restore_ioapic_entries();
L
Linus Torvalds 已提交
2984 2985
}

2986
static struct syscore_ops ioapic_syscore_ops = {
2987
	.suspend = save_ioapic_entries,
L
Linus Torvalds 已提交
2988 2989 2990
	.resume = ioapic_resume,
};

2991
static int __init ioapic_init_ops(void)
L
Linus Torvalds 已提交
2992
{
2993 2994
	register_syscore_ops(&ioapic_syscore_ops);

L
Linus Torvalds 已提交
2995 2996 2997
	return 0;
}

2998
device_initcall(ioapic_init_ops);
L
Linus Torvalds 已提交
2999

3000
/*
3001
 * Dynamic irq allocate and deallocation
3002
 */
3003
unsigned int create_irq_nr(unsigned int from, int node)
3004
{
3005
	struct irq_cfg *cfg;
3006
	unsigned long flags;
3007 3008
	unsigned int ret = 0;
	int irq;
3009

3010 3011
	if (from < nr_irqs_gsi)
		from = nr_irqs_gsi;
3012

3013 3014 3015 3016 3017 3018 3019
	irq = alloc_irq_from(from, node);
	if (irq < 0)
		return 0;
	cfg = alloc_irq_cfg(irq, node);
	if (!cfg) {
		free_irq_at(irq, NULL);
		return 0;
3020
	}
3021

3022 3023 3024 3025
	raw_spin_lock_irqsave(&vector_lock, flags);
	if (!__assign_irq_vector(irq, cfg, apic->target_cpus()))
		ret = irq;
	raw_spin_unlock_irqrestore(&vector_lock, flags);
3026

3027
	if (ret) {
3028
		irq_set_chip_data(irq, cfg);
3029 3030 3031 3032 3033
		irq_clear_status_flags(irq, IRQ_NOREQUEST);
	} else {
		free_irq_at(irq, cfg);
	}
	return ret;
3034 3035
}

Y
Yinghai Lu 已提交
3036 3037
int create_irq(void)
{
3038
	int node = cpu_to_node(0);
3039
	unsigned int irq_want;
3040 3041
	int irq;

3042
	irq_want = nr_irqs_gsi;
3043
	irq = create_irq_nr(irq_want, node);
3044 3045 3046 3047 3048

	if (irq == 0)
		irq = -1;

	return irq;
Y
Yinghai Lu 已提交
3049 3050
}

3051 3052
void destroy_irq(unsigned int irq)
{
3053
	struct irq_cfg *cfg = irq_get_chip_data(irq);
3054 3055
	unsigned long flags;

3056
	irq_set_status_flags(irq, IRQ_NOREQUEST|IRQ_NOPROBE);
3057

3058
	if (irq_remapped(cfg))
3059
		free_remapped_irq(irq);
3060
	raw_spin_lock_irqsave(&vector_lock, flags);
3061
	__clear_irq_vector(irq, cfg);
3062
	raw_spin_unlock_irqrestore(&vector_lock, flags);
3063
	free_irq_at(irq, cfg);
3064 3065
}

3066
/*
S
Simon Arlott 已提交
3067
 * MSI message composition
3068 3069
 */
#ifdef CONFIG_PCI_MSI
3070 3071
static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
			   struct msi_msg *msg, u8 hpet_id)
3072
{
3073 3074
	struct irq_cfg *cfg;
	int err;
3075 3076
	unsigned dest;

J
Jan Beulich 已提交
3077 3078 3079
	if (disable_apic)
		return -ENXIO;

Y
Yinghai Lu 已提交
3080
	cfg = irq_cfg(irq);
3081
	err = assign_irq_vector(irq, cfg, apic->target_cpus());
3082 3083
	if (err)
		return err;
3084

3085
	dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
3086

3087
	if (irq_remapped(cfg)) {
3088
		compose_remapped_msi_msg(pdev, irq, dest, msg, hpet_id);
3089 3090
		return err;
	}
3091

3092 3093 3094 3095 3096
	if (x2apic_enabled())
		msg->address_hi = MSI_ADDR_BASE_HI |
				  MSI_ADDR_EXT_DEST_ID(dest);
	else
		msg->address_hi = MSI_ADDR_BASE_HI;
3097

3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114
	msg->address_lo =
		MSI_ADDR_BASE_LO |
		((apic->irq_dest_mode == 0) ?
			MSI_ADDR_DEST_MODE_PHYSICAL:
			MSI_ADDR_DEST_MODE_LOGICAL) |
		((apic->irq_delivery_mode != dest_LowestPrio) ?
			MSI_ADDR_REDIRECTION_CPU:
			MSI_ADDR_REDIRECTION_LOWPRI) |
		MSI_ADDR_DEST_ID(dest);

	msg->data =
		MSI_DATA_TRIGGER_EDGE |
		MSI_DATA_LEVEL_ASSERT |
		((apic->irq_delivery_mode != dest_LowestPrio) ?
			MSI_DATA_DELIVERY_FIXED:
			MSI_DATA_DELIVERY_LOWPRI) |
		MSI_DATA_VECTOR(cfg->vector);
3115

3116
	return err;
3117 3118
}

3119
#ifdef CONFIG_SMP
3120 3121
static int
msi_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
3122
{
3123
	struct irq_cfg *cfg = data->chip_data;
3124 3125 3126
	struct msi_msg msg;
	unsigned int dest;

3127
	if (__ioapic_set_affinity(data, mask, &dest))
3128
		return -1;
3129

3130
	__get_cached_msi_msg(data->msi_desc, &msg);
3131 3132

	msg.data &= ~MSI_DATA_VECTOR_MASK;
3133
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
3134 3135 3136
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

3137
	__write_msi_msg(data->msi_desc, &msg);
3138 3139

	return 0;
3140
}
3141
#endif /* CONFIG_SMP */
3142

3143 3144 3145 3146 3147
/*
 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
 * which implement the MSI or MSI-X Capability Structure.
 */
static struct irq_chip msi_chip = {
3148 3149 3150 3151
	.name			= "PCI-MSI",
	.irq_unmask		= unmask_msi_irq,
	.irq_mask		= mask_msi_irq,
	.irq_ack		= ack_apic_edge,
3152
#ifdef CONFIG_SMP
3153
	.irq_set_affinity	= msi_set_affinity,
3154
#endif
3155
	.irq_retrigger		= ioapic_retrigger_irq,
3156 3157
};

Y
Yinghai Lu 已提交
3158
static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
3159
{
3160
	struct irq_chip *chip = &msi_chip;
3161
	struct msi_msg msg;
3162
	int ret;
3163

3164
	ret = msi_compose_msg(dev, irq, &msg, -1);
3165 3166 3167
	if (ret < 0)
		return ret;

3168
	irq_set_msi_desc(irq, msidesc);
3169 3170
	write_msi_msg(irq, &msg);

3171
	if (irq_remapped(irq_get_chip_data(irq))) {
3172
		irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
3173
		irq_remap_modify_chip_defaults(chip);
3174 3175 3176
	}

	irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
3177

Y
Yinghai Lu 已提交
3178 3179
	dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq);

3180 3181 3182
	return 0;
}

S
Stefano Stabellini 已提交
3183
int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3184
{
3185 3186
	int node, ret, sub_handle, index = 0;
	unsigned int irq, irq_want;
3187
	struct msi_desc *msidesc;
3188

3189 3190 3191 3192
	/* x86 doesn't support multiple MSI yet */
	if (type == PCI_CAP_ID_MSI && nvec > 1)
		return 1;

3193
	node = dev_to_node(&dev->dev);
3194
	irq_want = nr_irqs_gsi;
3195
	sub_handle = 0;
3196
	list_for_each_entry(msidesc, &dev->msi_list, list) {
3197
		irq = create_irq_nr(irq_want, node);
3198 3199
		if (irq == 0)
			return -1;
Y
Yinghai Lu 已提交
3200
		irq_want = irq + 1;
3201
		if (!irq_remapping_enabled)
3202 3203 3204 3205 3206 3207 3208
			goto no_ir;

		if (!sub_handle) {
			/*
			 * allocate the consecutive block of IRTE's
			 * for 'nvec'
			 */
3209
			index = msi_alloc_remapped_irq(dev, irq, nvec);
3210 3211 3212 3213 3214
			if (index < 0) {
				ret = index;
				goto error;
			}
		} else {
3215 3216
			ret = msi_setup_remapped_irq(dev, irq, index,
						     sub_handle);
3217
			if (ret < 0)
3218 3219 3220
				goto error;
		}
no_ir:
3221
		ret = setup_msi_irq(dev, msidesc, irq);
3222 3223 3224 3225 3226
		if (ret < 0)
			goto error;
		sub_handle++;
	}
	return 0;
3227 3228

error:
3229 3230
	destroy_irq(irq);
	return ret;
3231 3232
}

S
Stefano Stabellini 已提交
3233
void native_teardown_msi_irq(unsigned int irq)
3234
{
3235
	destroy_irq(irq);
3236 3237
}

3238
#ifdef CONFIG_DMAR_TABLE
3239
#ifdef CONFIG_SMP
3240 3241 3242
static int
dmar_msi_set_affinity(struct irq_data *data, const struct cpumask *mask,
		      bool force)
3243
{
3244 3245
	struct irq_cfg *cfg = data->chip_data;
	unsigned int dest, irq = data->irq;
3246 3247
	struct msi_msg msg;

3248
	if (__ioapic_set_affinity(data, mask, &dest))
3249
		return -1;
3250 3251 3252 3253 3254 3255 3256

	dmar_msi_read(irq, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3257
	msg.address_hi = MSI_ADDR_BASE_HI | MSI_ADDR_EXT_DEST_ID(dest);
3258 3259

	dmar_msi_write(irq, &msg);
3260 3261

	return 0;
3262
}
Y
Yinghai Lu 已提交
3263

3264 3265
#endif /* CONFIG_SMP */

3266
static struct irq_chip dmar_msi_type = {
3267 3268 3269 3270
	.name			= "DMAR_MSI",
	.irq_unmask		= dmar_msi_unmask,
	.irq_mask		= dmar_msi_mask,
	.irq_ack		= ack_apic_edge,
3271
#ifdef CONFIG_SMP
3272
	.irq_set_affinity	= dmar_msi_set_affinity,
3273
#endif
3274
	.irq_retrigger		= ioapic_retrigger_irq,
3275 3276 3277 3278 3279 3280
};

int arch_setup_dmar_msi(unsigned int irq)
{
	int ret;
	struct msi_msg msg;
3281

3282
	ret = msi_compose_msg(NULL, irq, &msg, -1);
3283 3284 3285
	if (ret < 0)
		return ret;
	dmar_msi_write(irq, &msg);
3286 3287
	irq_set_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
				      "edge");
3288 3289 3290 3291
	return 0;
}
#endif

3292 3293 3294
#ifdef CONFIG_HPET_TIMER

#ifdef CONFIG_SMP
3295 3296
static int hpet_msi_set_affinity(struct irq_data *data,
				 const struct cpumask *mask, bool force)
3297
{
3298
	struct irq_cfg *cfg = data->chip_data;
3299 3300 3301
	struct msi_msg msg;
	unsigned int dest;

3302
	if (__ioapic_set_affinity(data, mask, &dest))
3303
		return -1;
3304

3305
	hpet_msi_read(data->handler_data, &msg);
3306 3307 3308 3309 3310 3311

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

3312
	hpet_msi_write(data->handler_data, &msg);
3313 3314

	return 0;
3315
}
Y
Yinghai Lu 已提交
3316

3317 3318
#endif /* CONFIG_SMP */

3319
static struct irq_chip hpet_msi_type = {
3320
	.name = "HPET_MSI",
3321 3322
	.irq_unmask = hpet_msi_unmask,
	.irq_mask = hpet_msi_mask,
3323
	.irq_ack = ack_apic_edge,
3324
#ifdef CONFIG_SMP
3325
	.irq_set_affinity = hpet_msi_set_affinity,
3326
#endif
3327
	.irq_retrigger = ioapic_retrigger_irq,
3328 3329
};

3330
int arch_setup_hpet_msi(unsigned int irq, unsigned int id)
3331
{
3332
	struct irq_chip *chip = &hpet_msi_type;
3333
	struct msi_msg msg;
3334
	int ret;
3335

3336 3337
	if (irq_remapping_enabled) {
		if (!setup_hpet_msi_remapped(irq, id))
3338 3339 3340 3341
			return -1;
	}

	ret = msi_compose_msg(NULL, irq, &msg, id);
3342 3343 3344
	if (ret < 0)
		return ret;

3345
	hpet_msi_write(irq_get_handler_data(irq), &msg);
3346
	irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
3347
	if (irq_remapped(irq_get_chip_data(irq)))
3348
		irq_remap_modify_chip_defaults(chip);
Y
Yinghai Lu 已提交
3349

3350
	irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
3351 3352 3353 3354
	return 0;
}
#endif

3355
#endif /* CONFIG_PCI_MSI */
3356 3357 3358 3359 3360 3361 3362
/*
 * Hypertransport interrupt support
 */
#ifdef CONFIG_HT_IRQ

#ifdef CONFIG_SMP

3363
static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3364
{
3365 3366
	struct ht_irq_msg msg;
	fetch_ht_irq_msg(irq, &msg);
3367

3368
	msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
3369
	msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
3370

3371
	msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
3372
	msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
3373

3374
	write_ht_irq_msg(irq, &msg);
3375 3376
}

3377 3378
static int
ht_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
3379
{
3380
	struct irq_cfg *cfg = data->chip_data;
3381 3382
	unsigned int dest;

3383
	if (__ioapic_set_affinity(data, mask, &dest))
3384
		return -1;
3385

3386
	target_ht_irq(data->irq, dest, cfg->vector);
3387
	return 0;
3388
}
Y
Yinghai Lu 已提交
3389

3390 3391
#endif

3392
static struct irq_chip ht_irq_chip = {
3393 3394 3395 3396
	.name			= "PCI-HT",
	.irq_mask		= mask_ht_irq,
	.irq_unmask		= unmask_ht_irq,
	.irq_ack		= ack_apic_edge,
3397
#ifdef CONFIG_SMP
3398
	.irq_set_affinity	= ht_set_affinity,
3399
#endif
3400
	.irq_retrigger		= ioapic_retrigger_irq,
3401 3402 3403 3404
};

int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
{
3405 3406
	struct irq_cfg *cfg;
	int err;
3407

J
Jan Beulich 已提交
3408 3409 3410
	if (disable_apic)
		return -ENXIO;

Y
Yinghai Lu 已提交
3411
	cfg = irq_cfg(irq);
3412
	err = assign_irq_vector(irq, cfg, apic->target_cpus());
3413
	if (!err) {
3414
		struct ht_irq_msg msg;
3415 3416
		unsigned dest;

3417 3418
		dest = apic->cpu_mask_to_apicid_and(cfg->domain,
						    apic->target_cpus());
3419

3420
		msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
3421

3422 3423
		msg.address_lo =
			HT_IRQ_LOW_BASE |
3424
			HT_IRQ_LOW_DEST_ID(dest) |
3425
			HT_IRQ_LOW_VECTOR(cfg->vector) |
3426
			((apic->irq_dest_mode == 0) ?
3427 3428 3429
				HT_IRQ_LOW_DM_PHYSICAL :
				HT_IRQ_LOW_DM_LOGICAL) |
			HT_IRQ_LOW_RQEOI_EDGE |
3430
			((apic->irq_delivery_mode != dest_LowestPrio) ?
3431 3432 3433 3434
				HT_IRQ_LOW_MT_FIXED :
				HT_IRQ_LOW_MT_ARBITRATED) |
			HT_IRQ_LOW_IRQ_MASKED;

3435
		write_ht_irq_msg(irq, &msg);
3436

3437
		irq_set_chip_and_handler_name(irq, &ht_irq_chip,
3438
					      handle_edge_irq, "edge");
Y
Yinghai Lu 已提交
3439 3440

		dev_printk(KERN_DEBUG, &dev->dev, "irq %d for HT\n", irq);
3441
	}
3442
	return err;
3443 3444 3445
}
#endif /* CONFIG_HT_IRQ */

3446
static int
3447 3448 3449 3450 3451 3452 3453 3454 3455
io_apic_setup_irq_pin(unsigned int irq, int node, struct io_apic_irq_attr *attr)
{
	struct irq_cfg *cfg = alloc_irq_and_cfg_at(irq, node);
	int ret;

	if (!cfg)
		return -EINVAL;
	ret = __add_pin_to_irq_node(cfg, node, attr->ioapic, attr->ioapic_pin);
	if (!ret)
3456
		setup_ioapic_irq(irq, cfg, attr);
3457 3458 3459
	return ret;
}

3460 3461
int io_apic_setup_irq_pin_once(unsigned int irq, int node,
			       struct io_apic_irq_attr *attr)
3462
{
3463
	unsigned int ioapic_idx = attr->ioapic, pin = attr->ioapic_pin;
3464 3465 3466
	int ret;

	/* Avoid redundant programming */
3467
	if (test_bit(pin, ioapics[ioapic_idx].pin_programmed)) {
3468
		pr_debug("Pin %d-%d already programmed\n",
3469
			 mpc_ioapic_id(ioapic_idx), pin);
3470 3471 3472 3473
		return 0;
	}
	ret = io_apic_setup_irq_pin(irq, node, attr);
	if (!ret)
3474
		set_bit(pin, ioapics[ioapic_idx].pin_programmed);
3475 3476 3477
	return ret;
}

3478
static int __init io_apic_get_redir_entries(int ioapic)
3479 3480 3481 3482
{
	union IO_APIC_reg_01	reg_01;
	unsigned long flags;

3483
	raw_spin_lock_irqsave(&ioapic_lock, flags);
3484
	reg_01.raw = io_apic_read(ioapic, 1);
3485
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3486

3487 3488 3489 3490 3491
	/* The register returns the maximum index redir index
	 * supported, which is one less than the total number of redir
	 * entries.
	 */
	return reg_01.bits.entries + 1;
3492 3493
}

3494
static void __init probe_nr_irqs_gsi(void)
3495
{
3496
	int nr;
3497

3498
	nr = gsi_top + NR_IRQS_LEGACY;
3499
	if (nr > nr_irqs_gsi)
3500
		nr_irqs_gsi = nr;
3501 3502

	printk(KERN_DEBUG "nr_irqs_gsi: %d\n", nr_irqs_gsi);
3503 3504
}

3505 3506 3507 3508 3509
int get_nr_irqs_gsi(void)
{
	return nr_irqs_gsi;
}

Y
Yinghai Lu 已提交
3510 3511 3512 3513
int __init arch_probe_nr_irqs(void)
{
	int nr;

Y
Yinghai Lu 已提交
3514 3515
	if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
		nr_irqs = NR_VECTORS * nr_cpu_ids;
Y
Yinghai Lu 已提交
3516

Y
Yinghai Lu 已提交
3517 3518 3519 3520 3521 3522 3523 3524
	nr = nr_irqs_gsi + 8 * nr_cpu_ids;
#if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
	/*
	 * for MSI and HT dyn irq
	 */
	nr += nr_irqs_gsi * 16;
#endif
	if (nr < nr_irqs)
Y
Yinghai Lu 已提交
3525 3526
		nr_irqs = nr;

3527
	return NR_IRQS_LEGACY;
Y
Yinghai Lu 已提交
3528 3529
}

3530 3531
int io_apic_set_pci_routing(struct device *dev, int irq,
			    struct io_apic_irq_attr *irq_attr)
3532 3533 3534 3535 3536
{
	int node;

	if (!IO_APIC_IRQ(irq)) {
		apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
3537
			    irq_attr->ioapic);
3538 3539 3540
		return -EINVAL;
	}

3541
	node = dev ? dev_to_node(dev) : cpu_to_node(0);
3542

3543
	return io_apic_setup_irq_pin_once(irq, node, irq_attr);
3544 3545
}

3546
#ifdef CONFIG_X86_32
3547
static int __init io_apic_get_unique_id(int ioapic, int apic_id)
L
Linus Torvalds 已提交
3548 3549 3550 3551 3552 3553 3554 3555
{
	union IO_APIC_reg_00 reg_00;
	static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
	physid_mask_t tmp;
	unsigned long flags;
	int i = 0;

	/*
3556 3557
	 * The P4 platform supports up to 256 APIC IDs on two separate APIC
	 * buses (one for LAPICs, one for IOAPICs), where predecessors only
L
Linus Torvalds 已提交
3558
	 * supports up to 16 on one shared APIC bus.
3559
	 *
L
Linus Torvalds 已提交
3560 3561 3562 3563 3564
	 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
	 *      advantage of new APIC bus architecture.
	 */

	if (physids_empty(apic_id_map))
3565
		apic->ioapic_phys_id_map(&phys_cpu_present_map, &apic_id_map);
L
Linus Torvalds 已提交
3566

3567
	raw_spin_lock_irqsave(&ioapic_lock, flags);
L
Linus Torvalds 已提交
3568
	reg_00.raw = io_apic_read(ioapic, 0);
3569
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
L
Linus Torvalds 已提交
3570 3571 3572 3573 3574 3575 3576 3577

	if (apic_id >= get_physical_broadcast()) {
		printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
			"%d\n", ioapic, apic_id, reg_00.bits.ID);
		apic_id = reg_00.bits.ID;
	}

	/*
3578
	 * Every APIC in a system must have a unique ID or we get lots of nice
L
Linus Torvalds 已提交
3579 3580
	 * 'stuck on smp_invalidate_needed IPI wait' messages.
	 */
3581
	if (apic->check_apicid_used(&apic_id_map, apic_id)) {
L
Linus Torvalds 已提交
3582 3583

		for (i = 0; i < get_physical_broadcast(); i++) {
3584
			if (!apic->check_apicid_used(&apic_id_map, i))
L
Linus Torvalds 已提交
3585 3586 3587 3588 3589 3590 3591 3592 3593 3594
				break;
		}

		if (i == get_physical_broadcast())
			panic("Max apic_id exceeded!\n");

		printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
			"trying %d\n", ioapic, apic_id, i);

		apic_id = i;
3595
	}
L
Linus Torvalds 已提交
3596

3597
	apic->apicid_to_cpu_present(apic_id, &tmp);
L
Linus Torvalds 已提交
3598 3599 3600 3601 3602
	physids_or(apic_id_map, apic_id_map, tmp);

	if (reg_00.bits.ID != apic_id) {
		reg_00.bits.ID = apic_id;

3603
		raw_spin_lock_irqsave(&ioapic_lock, flags);
L
Linus Torvalds 已提交
3604 3605
		io_apic_write(ioapic, 0, reg_00.raw);
		reg_00.raw = io_apic_read(ioapic, 0);
3606
		raw_spin_unlock_irqrestore(&ioapic_lock, flags);
L
Linus Torvalds 已提交
3607 3608

		/* Sanity check */
3609 3610 3611 3612
		if (reg_00.bits.ID != apic_id) {
			printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
			return -1;
		}
L
Linus Torvalds 已提交
3613 3614 3615 3616 3617 3618 3619
	}

	apic_printk(APIC_VERBOSE, KERN_INFO
			"IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);

	return apic_id;
}
3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636

static u8 __init io_apic_unique_id(u8 id)
{
	if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
	    !APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
		return io_apic_get_unique_id(nr_ioapics, id);
	else
		return id;
}
#else
static u8 __init io_apic_unique_id(u8 id)
{
	int i;
	DECLARE_BITMAP(used, 256);

	bitmap_zero(used, 256);
	for (i = 0; i < nr_ioapics; i++) {
3637
		__set_bit(mpc_ioapic_id(i), used);
3638 3639 3640 3641 3642
	}
	if (!test_bit(id, used))
		return id;
	return find_first_zero_bit(used, 256);
}
3643
#endif
L
Linus Torvalds 已提交
3644

3645
static int __init io_apic_get_version(int ioapic)
L
Linus Torvalds 已提交
3646 3647 3648 3649
{
	union IO_APIC_reg_01	reg_01;
	unsigned long flags;

3650
	raw_spin_lock_irqsave(&ioapic_lock, flags);
L
Linus Torvalds 已提交
3651
	reg_01.raw = io_apic_read(ioapic, 1);
3652
	raw_spin_unlock_irqrestore(&ioapic_lock, flags);
L
Linus Torvalds 已提交
3653 3654 3655 3656

	return reg_01.bits.version;
}

3657
int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity)
3658
{
3659
	int ioapic, pin, idx;
3660 3661 3662 3663

	if (skip_ioapic_setup)
		return -1;

3664 3665
	ioapic = mp_find_ioapic(gsi);
	if (ioapic < 0)
3666 3667
		return -1;

3668 3669 3670 3671 3672 3673
	pin = mp_find_ioapic_pin(ioapic, gsi);
	if (pin < 0)
		return -1;

	idx = find_irq_entry(ioapic, pin, mp_INT);
	if (idx < 0)
3674 3675
		return -1;

3676 3677
	*trigger = irq_trigger(idx);
	*polarity = irq_polarity(idx);
3678 3679 3680
	return 0;
}

3681 3682 3683
/*
 * This function currently is only a helper for the i386 smp boot process where
 * we need to reprogram the ioredtbls to cater for the cpus which have come online
3684
 * so mask in all cases should simply be apic->target_cpus()
3685 3686 3687 3688
 */
#ifdef CONFIG_SMP
void __init setup_ioapic_dest(void)
{
E
Eric W. Biederman 已提交
3689
	int pin, ioapic, irq, irq_entry;
3690
	const struct cpumask *mask;
3691
	struct irq_data *idata;
3692 3693 3694 3695

	if (skip_ioapic_setup == 1)
		return;

E
Eric W. Biederman 已提交
3696
	for (ioapic = 0; ioapic < nr_ioapics; ioapic++)
S
Suresh Siddha 已提交
3697
	for (pin = 0; pin < ioapics[ioapic].nr_registers; pin++) {
3698 3699 3700 3701
		irq_entry = find_irq_entry(ioapic, pin, mp_INT);
		if (irq_entry == -1)
			continue;
		irq = pin_2_irq(irq_entry, ioapic, pin);
3702

E
Eric W. Biederman 已提交
3703 3704 3705
		if ((ioapic > 0) && (irq > 16))
			continue;

3706
		idata = irq_get_irq_data(irq);
3707

3708 3709 3710
		/*
		 * Honour affinities which have been set in early boot
		 */
3711 3712
		if (!irqd_can_balance(idata) || irqd_affinity_was_set(idata))
			mask = idata->affinity;
3713 3714
		else
			mask = apic->target_cpus();
3715

3716 3717
		if (irq_remapping_enabled)
			set_remapped_irq_affinity(idata, mask, false);
3718
		else
3719
			ioapic_set_affinity(idata, mask, false);
3720
	}
3721

3722 3723 3724
}
#endif

3725 3726 3727 3728
#define IOAPIC_RESOURCE_NAME_SIZE 11

static struct resource *ioapic_resources;

3729
static struct resource * __init ioapic_setup_resources(int nr_ioapics)
3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744
{
	unsigned long n;
	struct resource *res;
	char *mem;
	int i;

	if (nr_ioapics <= 0)
		return NULL;

	n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
	n *= nr_ioapics;

	mem = alloc_bootmem(n);
	res = (void *)mem;

3745
	mem += sizeof(struct resource) * nr_ioapics;
3746

3747 3748 3749
	for (i = 0; i < nr_ioapics; i++) {
		res[i].name = mem;
		res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
3750
		snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i);
3751
		mem += IOAPIC_RESOURCE_NAME_SIZE;
3752 3753 3754 3755 3756 3757 3758
	}

	ioapic_resources = res;

	return res;
}

3759
void __init ioapic_and_gsi_init(void)
3760 3761 3762 3763 3764
{
	io_apic_ops.init();
}

static void __init __ioapic_init_mappings(void)
3765 3766
{
	unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
3767
	struct resource *ioapic_res;
T
Thomas Gleixner 已提交
3768
	int i;
3769

3770
	ioapic_res = ioapic_setup_resources(nr_ioapics);
3771 3772
	for (i = 0; i < nr_ioapics; i++) {
		if (smp_found_config) {
3773
			ioapic_phys = mpc_ioapic_addr(i);
3774
#ifdef CONFIG_X86_32
T
Thomas Gleixner 已提交
3775 3776 3777 3778 3779 3780 3781 3782 3783
			if (!ioapic_phys) {
				printk(KERN_ERR
				       "WARNING: bogus zero IO-APIC "
				       "address found in MPTABLE, "
				       "disabling IO/APIC support!\n");
				smp_found_config = 0;
				skip_ioapic_setup = 1;
				goto fake_ioapic_page;
			}
3784
#endif
3785
		} else {
3786
#ifdef CONFIG_X86_32
3787
fake_ioapic_page:
3788
#endif
3789
			ioapic_phys = (unsigned long)alloc_bootmem_pages(PAGE_SIZE);
3790 3791 3792
			ioapic_phys = __pa(ioapic_phys);
		}
		set_fixmap_nocache(idx, ioapic_phys);
3793 3794 3795
		apic_printk(APIC_VERBOSE, "mapped IOAPIC to %08lx (%08lx)\n",
			__fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK),
			ioapic_phys);
3796
		idx++;
3797

3798
		ioapic_res->start = ioapic_phys;
3799
		ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1;
3800
		ioapic_res++;
3801
	}
3802 3803

	probe_nr_irqs_gsi();
3804 3805
}

3806
void __init ioapic_insert_resources(void)
3807 3808 3809 3810 3811
{
	int i;
	struct resource *r = ioapic_resources;

	if (!r) {
3812
		if (nr_ioapics > 0)
3813 3814
			printk(KERN_ERR
				"IO APIC resources couldn't be allocated.\n");
3815
		return;
3816 3817 3818 3819 3820 3821 3822
	}

	for (i = 0; i < nr_ioapics; i++) {
		insert_resource(&iomem_resource, r);
		r++;
	}
}
3823

3824
int mp_find_ioapic(u32 gsi)
3825 3826 3827
{
	int i = 0;

3828 3829 3830
	if (nr_ioapics == 0)
		return -1;

3831 3832
	/* Find the IOAPIC that manages this GSI. */
	for (i = 0; i < nr_ioapics; i++) {
3833 3834 3835
		struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i);
		if ((gsi >= gsi_cfg->gsi_base)
		    && (gsi <= gsi_cfg->gsi_end))
3836 3837
			return i;
	}
3838

3839 3840 3841 3842
	printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
	return -1;
}

3843
int mp_find_ioapic_pin(int ioapic, u32 gsi)
3844
{
3845 3846
	struct mp_ioapic_gsi *gsi_cfg;

3847 3848
	if (WARN_ON(ioapic == -1))
		return -1;
3849 3850 3851

	gsi_cfg = mp_ioapic_gsi_routing(ioapic);
	if (WARN_ON(gsi > gsi_cfg->gsi_end))
3852 3853
		return -1;

3854
	return gsi - gsi_cfg->gsi_base;
3855 3856
}

3857
static __init int bad_ioapic(unsigned long address)
3858 3859
{
	if (nr_ioapics >= MAX_IO_APICS) {
3860 3861
		pr_warn("WARNING: Max # of I/O APICs (%d) exceeded (found %d), skipping\n",
			MAX_IO_APICS, nr_ioapics);
3862 3863 3864
		return 1;
	}
	if (!address) {
3865
		pr_warn("WARNING: Bogus (zero) I/O APIC address found in table, skipping!\n");
3866 3867
		return 1;
	}
3868 3869 3870
	return 0;
}

3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889
static __init int bad_ioapic_register(int idx)
{
	union IO_APIC_reg_00 reg_00;
	union IO_APIC_reg_01 reg_01;
	union IO_APIC_reg_02 reg_02;

	reg_00.raw = io_apic_read(idx, 0);
	reg_01.raw = io_apic_read(idx, 1);
	reg_02.raw = io_apic_read(idx, 2);

	if (reg_00.raw == -1 && reg_01.raw == -1 && reg_02.raw == -1) {
		pr_warn("I/O APIC 0x%x registers return all ones, skipping!\n",
			mpc_ioapic_addr(idx));
		return 1;
	}

	return 0;
}

3890 3891 3892
void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
{
	int idx = 0;
3893
	int entries;
3894
	struct mp_ioapic_gsi *gsi_cfg;
3895 3896 3897 3898 3899 3900

	if (bad_ioapic(address))
		return;

	idx = nr_ioapics;

3901 3902 3903
	ioapics[idx].mp_config.type = MP_IOAPIC;
	ioapics[idx].mp_config.flags = MPC_APIC_USABLE;
	ioapics[idx].mp_config.apicaddr = address;
3904 3905

	set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
3906 3907 3908 3909 3910 3911

	if (bad_ioapic_register(idx)) {
		clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
		return;
	}

3912 3913
	ioapics[idx].mp_config.apicid = io_apic_unique_id(id);
	ioapics[idx].mp_config.apicver = io_apic_get_version(idx);
3914 3915 3916 3917 3918

	/*
	 * Build basic GSI lookup table to facilitate gsi->io_apic lookups
	 * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
	 */
3919
	entries = io_apic_get_redir_entries(idx);
3920 3921 3922
	gsi_cfg = mp_ioapic_gsi_routing(idx);
	gsi_cfg->gsi_base = gsi_base;
	gsi_cfg->gsi_end = gsi_base + entries - 1;
3923 3924 3925 3926

	/*
	 * The number of IO-APIC IRQ registers (== #pins):
	 */
S
Suresh Siddha 已提交
3927
	ioapics[idx].nr_registers = entries;
3928

3929 3930
	if (gsi_cfg->gsi_end >= gsi_top)
		gsi_top = gsi_cfg->gsi_end + 1;
3931

3932 3933 3934 3935
	pr_info("IOAPIC[%d]: apic_id %d, version %d, address 0x%x, GSI %d-%d\n",
		idx, mpc_ioapic_id(idx),
		mpc_ioapic_ver(idx), mpc_ioapic_addr(idx),
		gsi_cfg->gsi_base, gsi_cfg->gsi_end);
3936 3937 3938

	nr_ioapics++;
}
3939 3940 3941 3942

/* Enable IOAPIC early just for system timer */
void __init pre_init_apic_IRQ0(void)
{
3943
	struct io_apic_irq_attr attr = { 0, 0, 0, 0 };
3944 3945 3946

	printk(KERN_INFO "Early APIC setup for system timer0\n");
#ifndef CONFIG_SMP
3947 3948
	physid_set_mask_of_physid(boot_cpu_physical_apicid,
					 &phys_cpu_present_map);
3949 3950 3951
#endif
	setup_local_APIC();

3952
	io_apic_setup_irq_pin(0, 0, &attr);
3953 3954
	irq_set_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq,
				      "edge");
3955
}