io_apic.c 98.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 *	Intel IO-APIC support for multi-Pentium hosts.
 *
I
Ingo Molnar 已提交
4
 *	Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
 *
 *	Many thanks to Stig Venaas for trying out countless experimental
 *	patches and reporting/debugging problems patiently!
 *
 *	(c) 1999, Multiple IO-APIC support, developed by
 *	Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
 *      Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
 *	further tested and cleaned up by Zach Brown <zab@redhat.com>
 *	and Ingo Molnar <mingo@redhat.com>
 *
 *	Fixes
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
 *					thanks to Eric Gilmore
 *					and Rolf G. Tews
 *					for testing these extensively
 *	Paul Diefenbaugh	:	Added full ACPI support
 */

#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/sched.h>
28
#include <linux/pci.h>
L
Linus Torvalds 已提交
29 30 31
#include <linux/mc146818rtc.h>
#include <linux/compiler.h>
#include <linux/acpi.h>
32
#include <linux/module.h>
L
Linus Torvalds 已提交
33
#include <linux/sysdev.h>
34
#include <linux/msi.h>
35
#include <linux/htirq.h>
36
#include <linux/freezer.h>
37
#include <linux/kthread.h>
38
#include <linux/jiffies.h>	/* time_after() */
39 40 41 42 43
#ifdef CONFIG_ACPI
#include <acpi/acpi_bus.h>
#endif
#include <linux/bootmem.h>
#include <linux/dmar.h>
44
#include <linux/hpet.h>
45

46
#include <asm/idle.h>
L
Linus Torvalds 已提交
47 48
#include <asm/io.h>
#include <asm/smp.h>
49
#include <asm/cpu.h>
L
Linus Torvalds 已提交
50
#include <asm/desc.h>
51 52 53
#include <asm/proto.h>
#include <asm/acpi.h>
#include <asm/dma.h>
L
Linus Torvalds 已提交
54
#include <asm/timer.h>
55
#include <asm/i8259.h>
56
#include <asm/nmi.h>
57
#include <asm/msidef.h>
58
#include <asm/hypertransport.h>
59
#include <asm/setup.h>
60
#include <asm/irq_remapping.h>
61
#include <asm/hpet.h>
62
#include <asm/hw_irq.h>
63 64
#include <asm/uv/uv_hub.h>
#include <asm/uv/uv_irq.h>
L
Linus Torvalds 已提交
65

I
Ingo Molnar 已提交
66
#include <asm/apic.h>
L
Linus Torvalds 已提交
67

68 69
#define __apicdebuginit(type) static type __init

L
Linus Torvalds 已提交
70
/*
71 72
 *      Is the SiS APIC rmw bug present ?
 *      -1 = don't know, 0 = no, 1 = yes
L
Linus Torvalds 已提交
73 74 75
 */
int sis_apic_bug = -1;

Y
Yinghai Lu 已提交
76 77 78
static DEFINE_SPINLOCK(ioapic_lock);
static DEFINE_SPINLOCK(vector_lock);

L
Linus Torvalds 已提交
79 80 81 82 83
/*
 * # of IRQ routing registers
 */
int nr_ioapic_registers[MAX_IO_APICS];

84
/* I/O APIC entries */
85
struct mpc_ioapic mp_ioapics[MAX_IO_APICS];
86 87
int nr_ioapics;

88
/* MP IRQ source entries */
89
struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
90 91 92 93

/* # of MP IRQ source entries */
int mp_irq_entries;

94 95 96 97 98 99
#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
int mp_bus_id_to_type[MAX_MP_BUSSES];
#endif

DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);

Y
Yinghai Lu 已提交
100 101
int skip_ioapic_setup;

102 103 104 105 106 107 108 109 110
void arch_disable_smp_support(void)
{
#ifdef CONFIG_PCI
	noioapicquirk = 1;
	noioapicreroute = -1;
#endif
	skip_ioapic_setup = 1;
}

111
static int __init parse_noapic(char *str)
Y
Yinghai Lu 已提交
112 113
{
	/* disable IO-APIC */
114
	arch_disable_smp_support();
Y
Yinghai Lu 已提交
115 116 117
	return 0;
}
early_param("noapic", parse_noapic);
118

119 120 121 122 123 124 125 126 127 128 129 130
/*
 * This is performance-critical, we want to do it O(1)
 *
 * the indexing order of this array favors 1:1 mappings
 * between pins and IRQs.
 */

struct irq_pin_list {
	int apic, pin;
	struct irq_pin_list *next;
};

131
static struct irq_pin_list *get_one_free_irq_2_pin(int node)
132 133 134 135 136 137 138 139
{
	struct irq_pin_list *pin;

	pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node);

	return pin;
}

Y
Yinghai Lu 已提交
140
struct irq_cfg {
141
	struct irq_pin_list *irq_2_pin;
142 143
	cpumask_var_t domain;
	cpumask_var_t old_domain;
144
	unsigned move_cleanup_count;
Y
Yinghai Lu 已提交
145
	u8 vector;
146
	u8 move_in_progress : 1;
Y
Yinghai Lu 已提交
147 148 149
};

/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
150 151 152
#ifdef CONFIG_SPARSE_IRQ
static struct irq_cfg irq_cfgx[] = {
#else
T
Thomas Gleixner 已提交
153
static struct irq_cfg irq_cfgx[NR_IRQS] = {
154
#endif
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
	[0]  = { .vector = IRQ0_VECTOR,  },
	[1]  = { .vector = IRQ1_VECTOR,  },
	[2]  = { .vector = IRQ2_VECTOR,  },
	[3]  = { .vector = IRQ3_VECTOR,  },
	[4]  = { .vector = IRQ4_VECTOR,  },
	[5]  = { .vector = IRQ5_VECTOR,  },
	[6]  = { .vector = IRQ6_VECTOR,  },
	[7]  = { .vector = IRQ7_VECTOR,  },
	[8]  = { .vector = IRQ8_VECTOR,  },
	[9]  = { .vector = IRQ9_VECTOR,  },
	[10] = { .vector = IRQ10_VECTOR, },
	[11] = { .vector = IRQ11_VECTOR, },
	[12] = { .vector = IRQ12_VECTOR, },
	[13] = { .vector = IRQ13_VECTOR, },
	[14] = { .vector = IRQ14_VECTOR, },
	[15] = { .vector = IRQ15_VECTOR, },
Y
Yinghai Lu 已提交
171 172
};

173
int __init arch_early_irq_init(void)
174
{
175 176 177
	struct irq_cfg *cfg;
	struct irq_desc *desc;
	int count;
178
	int node;
179
	int i;
T
Thomas Gleixner 已提交
180

181 182
	cfg = irq_cfgx;
	count = ARRAY_SIZE(irq_cfgx);
183
	node= cpu_to_node(boot_cpu_id);
184

185 186 187
	for (i = 0; i < count; i++) {
		desc = irq_to_desc(i);
		desc->chip_data = &cfg[i];
188 189
		zalloc_cpumask_var_node(&cfg[i].domain, GFP_NOWAIT, node);
		zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_NOWAIT, node);
190 191
		if (i < NR_IRQS_LEGACY)
			cpumask_setall(cfg[i].domain);
192
	}
193 194

	return 0;
195
}
196

197
#ifdef CONFIG_SPARSE_IRQ
T
Thomas Gleixner 已提交
198
static struct irq_cfg *irq_cfg(unsigned int irq)
199
{
200 201
	struct irq_cfg *cfg = NULL;
	struct irq_desc *desc;
L
Linus Torvalds 已提交
202

203 204 205
	desc = irq_to_desc(irq);
	if (desc)
		cfg = desc->chip_data;
206

207
	return cfg;
208
}
T
Thomas Gleixner 已提交
209

210
static struct irq_cfg *get_one_free_irq_cfg(int node)
211
{
212
	struct irq_cfg *cfg;
213

214
	cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node);
215
	if (cfg) {
216
		if (!alloc_cpumask_var_node(&cfg->domain, GFP_ATOMIC, node)) {
217 218
			kfree(cfg);
			cfg = NULL;
219 220
		} else if (!alloc_cpumask_var_node(&cfg->old_domain,
							  GFP_ATOMIC, node)) {
221 222 223 224 225 226 227 228
			free_cpumask_var(cfg->domain);
			kfree(cfg);
			cfg = NULL;
		} else {
			cpumask_clear(cfg->domain);
			cpumask_clear(cfg->old_domain);
		}
	}
229

230
	return cfg;
231 232
}

233
int arch_init_chip_data(struct irq_desc *desc, int node)
234
{
235
	struct irq_cfg *cfg;
T
Thomas Gleixner 已提交
236

237 238
	cfg = desc->chip_data;
	if (!cfg) {
239
		desc->chip_data = get_one_free_irq_cfg(node);
240 241 242 243 244
		if (!desc->chip_data) {
			printk(KERN_ERR "can not alloc irq_cfg\n");
			BUG_ON(1);
		}
	}
L
Linus Torvalds 已提交
245

246
	return 0;
247
}
248

249
/* for move_irq_desc */
250
static void
251
init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int node)
252
{
253 254 255 256 257 258
	struct irq_pin_list *old_entry, *head, *tail, *entry;

	cfg->irq_2_pin = NULL;
	old_entry = old_cfg->irq_2_pin;
	if (!old_entry)
		return;
259

260
	entry = get_one_free_irq_2_pin(node);
261 262
	if (!entry)
		return;
263

264 265 266 267 268 269
	entry->apic	= old_entry->apic;
	entry->pin	= old_entry->pin;
	head		= entry;
	tail		= entry;
	old_entry	= old_entry->next;
	while (old_entry) {
270
		entry = get_one_free_irq_2_pin(node);
271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
		if (!entry) {
			entry = head;
			while (entry) {
				head = entry->next;
				kfree(entry);
				entry = head;
			}
			/* still use the old one */
			return;
		}
		entry->apic	= old_entry->apic;
		entry->pin	= old_entry->pin;
		tail->next	= entry;
		tail		= entry;
		old_entry	= old_entry->next;
	}
287

288 289
	tail->next = NULL;
	cfg->irq_2_pin = head;
290 291
}

292
static void free_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg)
293
{
294
	struct irq_pin_list *entry, *next;
295

296 297
	if (old_cfg->irq_2_pin == cfg->irq_2_pin)
		return;
Y
Yinghai Lu 已提交
298

299
	entry = old_cfg->irq_2_pin;
300

301 302 303 304 305 306
	while (entry) {
		next = entry->next;
		kfree(entry);
		entry = next;
	}
	old_cfg->irq_2_pin = NULL;
307 308
}

309
void arch_init_copy_chip_data(struct irq_desc *old_desc,
310
				 struct irq_desc *desc, int node)
311
{
312 313
	struct irq_cfg *cfg;
	struct irq_cfg *old_cfg;
314

315
	cfg = get_one_free_irq_cfg(node);
Y
Yinghai Lu 已提交
316

317 318 319 320 321 322 323 324 325
	if (!cfg)
		return;

	desc->chip_data = cfg;

	old_cfg = old_desc->chip_data;

	memcpy(cfg, old_cfg, sizeof(struct irq_cfg));

326
	init_copy_irq_2_pin(old_cfg, cfg, node);
327
}
L
Linus Torvalds 已提交
328

329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
static void free_irq_cfg(struct irq_cfg *old_cfg)
{
	kfree(old_cfg);
}

void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc)
{
	struct irq_cfg *old_cfg, *cfg;

	old_cfg = old_desc->chip_data;
	cfg = desc->chip_data;

	if (old_cfg == cfg)
		return;

	if (old_cfg) {
		free_irq_2_pin(old_cfg, cfg);
		free_irq_cfg(old_cfg);
		old_desc->chip_data = NULL;
	}
}
350
/* end for move_irq_desc */
351

352 353 354 355
#else
static struct irq_cfg *irq_cfg(unsigned int irq)
{
	return irq < nr_irqs ? irq_cfgx + irq : NULL;
356
}
L
Linus Torvalds 已提交
357

358 359
#endif

L
Linus Torvalds 已提交
360 361 362 363
struct io_apic {
	unsigned int index;
	unsigned int unused[3];
	unsigned int data;
364 365
	unsigned int unused2[11];
	unsigned int eoi;
L
Linus Torvalds 已提交
366 367 368 369 370
};

static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
{
	return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
371
		+ (mp_ioapics[idx].apicaddr & ~PAGE_MASK);
L
Linus Torvalds 已提交
372 373
}

374 375 376 377 378 379
static inline void io_apic_eoi(unsigned int apic, unsigned int vector)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(vector, &io_apic->eoi);
}

L
Linus Torvalds 已提交
380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401
static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	return readl(&io_apic->data);
}

static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	writel(value, &io_apic->data);
}

/*
 * Re-write a value: to be used for read-modify-write
 * cycles where the read already set up the index register.
 *
 * Older SiS APIC requires we rewrite the index register
 */
static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
{
402
	struct io_apic __iomem *io_apic = io_apic_base(apic);
T
Thomas Gleixner 已提交
403 404 405

	if (sis_apic_bug)
		writel(reg, &io_apic->index);
L
Linus Torvalds 已提交
406 407 408
	writel(value, &io_apic->data);
}

Y
Yinghai Lu 已提交
409
static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
{
	struct irq_pin_list *entry;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	entry = cfg->irq_2_pin;
	for (;;) {
		unsigned int reg;
		int pin;

		if (!entry)
			break;
		pin = entry->pin;
		reg = io_apic_read(entry->apic, 0x10 + pin*2);
		/* Is the remote IRR bit set? */
		if (reg & IO_APIC_REDIR_REMOTE_IRR) {
			spin_unlock_irqrestore(&ioapic_lock, flags);
			return true;
		}
		if (!entry->next)
			break;
		entry = entry->next;
	}
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return false;
}

438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
union entry_union {
	struct { u32 w1, w2; };
	struct IO_APIC_route_entry entry;
};

static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
{
	union entry_union eu;
	unsigned long flags;
	spin_lock_irqsave(&ioapic_lock, flags);
	eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
	eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
	spin_unlock_irqrestore(&ioapic_lock, flags);
	return eu.entry;
}

454 455 456 457 458 459
/*
 * When we write a new IO APIC routing entry, we need to write the high
 * word first! If the mask bit in the low word is clear, we will enable
 * the interrupt, and we need to make sure the entry is fully populated
 * before that happens.
 */
460 461
static void
__ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
462
{
463 464
	union entry_union eu = {{0, 0}};

465
	eu.entry = e;
466 467
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
468 469
}

470
void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
471 472 473 474
{
	unsigned long flags;
	spin_lock_irqsave(&ioapic_lock, flags);
	__ioapic_write_entry(apic, pin, e);
475 476 477 478 479 480 481 482 483 484 485 486 487
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

/*
 * When we mask an IO APIC routing entry, we need to write the low
 * word first, in order to set the mask bit before we change the
 * high bits!
 */
static void ioapic_mask_entry(int apic, int pin)
{
	unsigned long flags;
	union entry_union eu = { .entry.mask = 1 };

488 489 490 491 492 493
	spin_lock_irqsave(&ioapic_lock, flags);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

L
Linus Torvalds 已提交
494 495 496 497 498
/*
 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
 * shared ISA-space IRQs, so we have to support them. We are super
 * fast in the common case, and fast for shared ISA-space IRQs.
 */
499
static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
L
Linus Torvalds 已提交
500
{
501 502 503 504
	struct irq_pin_list *entry;

	entry = cfg->irq_2_pin;
	if (!entry) {
505
		entry = get_one_free_irq_2_pin(node);
506 507 508 509 510
		if (!entry) {
			printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n",
					apic, pin);
			return;
		}
511 512 513 514 515
		cfg->irq_2_pin = entry;
		entry->apic = apic;
		entry->pin = pin;
		return;
	}
L
Linus Torvalds 已提交
516

517 518 519 520
	while (entry->next) {
		/* not again, please */
		if (entry->apic == apic && entry->pin == pin)
			return;
L
Linus Torvalds 已提交
521

522
		entry = entry->next;
L
Linus Torvalds 已提交
523
	}
524

525
	entry->next = get_one_free_irq_2_pin(node);
526
	entry = entry->next;
L
Linus Torvalds 已提交
527 528 529 530 531 532 533
	entry->apic = apic;
	entry->pin = pin;
}

/*
 * Reroute an IRQ to a different pin.
 */
534
static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
L
Linus Torvalds 已提交
535 536 537
				      int oldapic, int oldpin,
				      int newapic, int newpin)
{
538 539
	struct irq_pin_list *entry = cfg->irq_2_pin;
	int replaced = 0;
L
Linus Torvalds 已提交
540

541
	while (entry) {
L
Linus Torvalds 已提交
542 543 544
		if (entry->apic == oldapic && entry->pin == oldpin) {
			entry->apic = newapic;
			entry->pin = newpin;
545 546
			replaced = 1;
			/* every one is different, right? */
L
Linus Torvalds 已提交
547
			break;
548 549
		}
		entry = entry->next;
L
Linus Torvalds 已提交
550
	}
551 552 553

	/* why? call replace before add? */
	if (!replaced)
554
		add_pin_to_irq_node(cfg, node, newapic, newpin);
L
Linus Torvalds 已提交
555 556
}

557 558 559
static void io_apic_modify_irq(struct irq_cfg *cfg,
			       int mask_and, int mask_or,
			       void (*final)(struct irq_pin_list *entry))
560 561 562
{
	int pin;
	struct irq_pin_list *entry;
563

564 565 566 567 568 569 570 571 572 573 574
	for (entry = cfg->irq_2_pin; entry != NULL; entry = entry->next) {
		unsigned int reg;
		pin = entry->pin;
		reg = io_apic_read(entry->apic, 0x10 + pin * 2);
		reg &= mask_and;
		reg |= mask_or;
		io_apic_modify(entry->apic, 0x10 + pin * 2, reg);
		if (final)
			final(entry);
	}
}
575

Y
Yinghai Lu 已提交
576
static void __unmask_IO_APIC_irq(struct irq_cfg *cfg)
577
{
Y
Yinghai Lu 已提交
578
	io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
579
}
580

581
static void io_apic_sync(struct irq_pin_list *entry)
L
Linus Torvalds 已提交
582
{
583 584 585 586 587 588
	/*
	 * Synchronize the IO-APIC and the CPU by doing
	 * a dummy read from the IO-APIC
	 */
	struct io_apic __iomem *io_apic;
	io_apic = io_apic_base(entry->apic);
Y
Yinghai Lu 已提交
589
	readl(&io_apic->data);
L
Linus Torvalds 已提交
590 591
}

Y
Yinghai Lu 已提交
592
static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
593
{
Y
Yinghai Lu 已提交
594
	io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
595
}
L
Linus Torvalds 已提交
596

Y
Yinghai Lu 已提交
597
static void __mask_and_edge_IO_APIC_irq(struct irq_cfg *cfg)
598
{
Y
Yinghai Lu 已提交
599
	io_apic_modify_irq(cfg, ~IO_APIC_REDIR_LEVEL_TRIGGER,
600 601
			IO_APIC_REDIR_MASKED, NULL);
}
L
Linus Torvalds 已提交
602

Y
Yinghai Lu 已提交
603
static void __unmask_and_level_IO_APIC_irq(struct irq_cfg *cfg)
604
{
Y
Yinghai Lu 已提交
605
	io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED,
606 607
			IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
}
608

Y
Yinghai Lu 已提交
609
static void mask_IO_APIC_irq_desc(struct irq_desc *desc)
L
Linus Torvalds 已提交
610
{
Y
Yinghai Lu 已提交
611
	struct irq_cfg *cfg = desc->chip_data;
L
Linus Torvalds 已提交
612 613
	unsigned long flags;

Y
Yinghai Lu 已提交
614 615
	BUG_ON(!cfg);

L
Linus Torvalds 已提交
616
	spin_lock_irqsave(&ioapic_lock, flags);
Y
Yinghai Lu 已提交
617
	__mask_IO_APIC_irq(cfg);
L
Linus Torvalds 已提交
618 619 620
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

Y
Yinghai Lu 已提交
621
static void unmask_IO_APIC_irq_desc(struct irq_desc *desc)
L
Linus Torvalds 已提交
622
{
Y
Yinghai Lu 已提交
623
	struct irq_cfg *cfg = desc->chip_data;
L
Linus Torvalds 已提交
624 625 626
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
Y
Yinghai Lu 已提交
627
	__unmask_IO_APIC_irq(cfg);
L
Linus Torvalds 已提交
628 629 630
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

Y
Yinghai Lu 已提交
631 632 633 634 635 636 637 638 639 640 641 642 643
static void mask_IO_APIC_irq(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	mask_IO_APIC_irq_desc(desc);
}
static void unmask_IO_APIC_irq(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	unmask_IO_APIC_irq_desc(desc);
}

L
Linus Torvalds 已提交
644 645 646
static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
{
	struct IO_APIC_route_entry entry;
647

L
Linus Torvalds 已提交
648
	/* Check delivery_mode to be sure we're not clearing an SMI pin */
649
	entry = ioapic_read_entry(apic, pin);
L
Linus Torvalds 已提交
650 651 652 653 654
	if (entry.delivery_mode == dest_SMI)
		return;
	/*
	 * Disable it in the IO-APIC irq-routing table:
	 */
655
	ioapic_mask_entry(apic, pin);
L
Linus Torvalds 已提交
656 657
}

658
static void clear_IO_APIC (void)
L
Linus Torvalds 已提交
659 660 661 662 663 664 665 666
{
	int apic, pin;

	for (apic = 0; apic < nr_ioapics; apic++)
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
			clear_IO_APIC_pin(apic, pin);
}

667
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
668 669 670 671 672 673
/*
 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
 * specific CPU-side IRQs.
 */

#define MAX_PIRQS 8
Y
Yinghai Lu 已提交
674 675 676
static int pirq_entries[MAX_PIRQS] = {
	[0 ... MAX_PIRQS - 1] = -1
};
L
Linus Torvalds 已提交
677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702

static int __init ioapic_pirq_setup(char *str)
{
	int i, max;
	int ints[MAX_PIRQS+1];

	get_options(str, ARRAY_SIZE(ints), ints);

	apic_printk(APIC_VERBOSE, KERN_INFO
			"PIRQ redirection, working around broken MP-BIOS.\n");
	max = MAX_PIRQS;
	if (ints[0] < MAX_PIRQS)
		max = ints[0];

	for (i = 0; i < max; i++) {
		apic_printk(APIC_VERBOSE, KERN_DEBUG
				"... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
		/*
		 * PIRQs are mapped upside down, usually.
		 */
		pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
	}
	return 1;
}

__setup("pirq=", ioapic_pirq_setup);
703 704
#endif /* CONFIG_X86_32 */

705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731
struct IO_APIC_route_entry **alloc_ioapic_entries(void)
{
	int apic;
	struct IO_APIC_route_entry **ioapic_entries;

	ioapic_entries = kzalloc(sizeof(*ioapic_entries) * nr_ioapics,
				GFP_ATOMIC);
	if (!ioapic_entries)
		return 0;

	for (apic = 0; apic < nr_ioapics; apic++) {
		ioapic_entries[apic] =
			kzalloc(sizeof(struct IO_APIC_route_entry) *
				nr_ioapic_registers[apic], GFP_ATOMIC);
		if (!ioapic_entries[apic])
			goto nomem;
	}

	return ioapic_entries;

nomem:
	while (--apic >= 0)
		kfree(ioapic_entries[apic]);
	kfree(ioapic_entries);

	return 0;
}
732 733

/*
734
 * Saves all the IO-APIC RTE's
735
 */
736
int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
737 738 739
{
	int apic, pin;

740 741
	if (!ioapic_entries)
		return -ENOMEM;
742 743

	for (apic = 0; apic < nr_ioapics; apic++) {
744 745
		if (!ioapic_entries[apic])
			return -ENOMEM;
746

747
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
748
			ioapic_entries[apic][pin] =
749
				ioapic_read_entry(apic, pin);
750
	}
751

752 753 754
	return 0;
}

755 756 757 758
/*
 * Mask all IO APIC entries.
 */
void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
759 760 761
{
	int apic, pin;

762 763 764
	if (!ioapic_entries)
		return;

765
	for (apic = 0; apic < nr_ioapics; apic++) {
766
		if (!ioapic_entries[apic])
767
			break;
768

769 770 771
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
			struct IO_APIC_route_entry entry;

772
			entry = ioapic_entries[apic][pin];
773 774 775 776 777 778 779 780
			if (!entry.mask) {
				entry.mask = 1;
				ioapic_write_entry(apic, pin, entry);
			}
		}
	}
}

781 782 783 784
/*
 * Restore IO APIC entries which was saved in ioapic_entries.
 */
int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
785 786 787
{
	int apic, pin;

788 789 790
	if (!ioapic_entries)
		return -ENOMEM;

791
	for (apic = 0; apic < nr_ioapics; apic++) {
792 793 794
		if (!ioapic_entries[apic])
			return -ENOMEM;

795 796
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
			ioapic_write_entry(apic, pin,
797
					ioapic_entries[apic][pin]);
798
	}
799
	return 0;
800 801
}

802 803 804 805 806 807 808 809
void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
{
	int apic;

	for (apic = 0; apic < nr_ioapics; apic++)
		kfree(ioapic_entries[apic]);

	kfree(ioapic_entries);
810
}
L
Linus Torvalds 已提交
811 812 813 814 815 816 817 818 819

/*
 * Find the IRQ entry number of a certain pin.
 */
static int find_irq_entry(int apic, int pin, int type)
{
	int i;

	for (i = 0; i < mp_irq_entries; i++)
820 821 822 823
		if (mp_irqs[i].irqtype == type &&
		    (mp_irqs[i].dstapic == mp_ioapics[apic].apicid ||
		     mp_irqs[i].dstapic == MP_APIC_ALL) &&
		    mp_irqs[i].dstirq == pin)
L
Linus Torvalds 已提交
824 825 826 827 828 829 830 831
			return i;

	return -1;
}

/*
 * Find the pin to which IRQ[irq] (ISA) is connected
 */
832
static int __init find_isa_irq_pin(int irq, int type)
L
Linus Torvalds 已提交
833 834 835 836
{
	int i;

	for (i = 0; i < mp_irq_entries; i++) {
837
		int lbus = mp_irqs[i].srcbus;
L
Linus Torvalds 已提交
838

A
Alexey Starikovskiy 已提交
839
		if (test_bit(lbus, mp_bus_not_pci) &&
840 841
		    (mp_irqs[i].irqtype == type) &&
		    (mp_irqs[i].srcbusirq == irq))
L
Linus Torvalds 已提交
842

843
			return mp_irqs[i].dstirq;
L
Linus Torvalds 已提交
844 845 846 847
	}
	return -1;
}

848 849 850 851 852
static int __init find_isa_irq_apic(int irq, int type)
{
	int i;

	for (i = 0; i < mp_irq_entries; i++) {
853
		int lbus = mp_irqs[i].srcbus;
854

A
Alexey Starikovskiy 已提交
855
		if (test_bit(lbus, mp_bus_not_pci) &&
856 857
		    (mp_irqs[i].irqtype == type) &&
		    (mp_irqs[i].srcbusirq == irq))
858 859 860 861
			break;
	}
	if (i < mp_irq_entries) {
		int apic;
862
		for(apic = 0; apic < nr_ioapics; apic++) {
863
			if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic)
864 865 866 867 868 869 870
				return apic;
		}
	}

	return -1;
}

871
#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
L
Linus Torvalds 已提交
872 873 874 875 876
/*
 * EISA Edge/Level control register, ELCR
 */
static int EISA_ELCR(unsigned int irq)
{
Y
Yinghai Lu 已提交
877
	if (irq < NR_IRQS_LEGACY) {
L
Linus Torvalds 已提交
878 879 880 881 882 883 884
		unsigned int port = 0x4d0 + (irq >> 3);
		return (inb(port) >> (irq & 7)) & 1;
	}
	apic_printk(APIC_VERBOSE, KERN_INFO
			"Broken MPtable reports ISA irq %d\n", irq);
	return 0;
}
885

886
#endif
L
Linus Torvalds 已提交
887

A
Alexey Starikovskiy 已提交
888 889 890 891 892 893
/* ISA interrupts are always polarity zero edge triggered,
 * when listed as conforming in the MP table. */

#define default_ISA_trigger(idx)	(0)
#define default_ISA_polarity(idx)	(0)

L
Linus Torvalds 已提交
894 895 896 897 898
/* EISA interrupts are always polarity zero and can be edge or level
 * trigger depending on the ELCR value.  If an interrupt is listed as
 * EISA conforming in the MP table, that means its trigger type must
 * be read in from the ELCR */

899
#define default_EISA_trigger(idx)	(EISA_ELCR(mp_irqs[idx].srcbusirq))
A
Alexey Starikovskiy 已提交
900
#define default_EISA_polarity(idx)	default_ISA_polarity(idx)
L
Linus Torvalds 已提交
901 902 903 904 905 906 907 908 909 910 911

/* PCI interrupts are always polarity one level triggered,
 * when listed as conforming in the MP table. */

#define default_PCI_trigger(idx)	(1)
#define default_PCI_polarity(idx)	(1)

/* MCA interrupts are always polarity zero level triggered,
 * when listed as conforming in the MP table. */

#define default_MCA_trigger(idx)	(1)
A
Alexey Starikovskiy 已提交
912
#define default_MCA_polarity(idx)	default_ISA_polarity(idx)
L
Linus Torvalds 已提交
913

914
static int MPBIOS_polarity(int idx)
L
Linus Torvalds 已提交
915
{
916
	int bus = mp_irqs[idx].srcbus;
L
Linus Torvalds 已提交
917 918 919 920 921
	int polarity;

	/*
	 * Determine IRQ line polarity (high active or low active):
	 */
922
	switch (mp_irqs[idx].irqflag & 3)
923
	{
924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
		case 0: /* conforms, ie. bus-type dependent polarity */
			if (test_bit(bus, mp_bus_not_pci))
				polarity = default_ISA_polarity(idx);
			else
				polarity = default_PCI_polarity(idx);
			break;
		case 1: /* high active */
		{
			polarity = 0;
			break;
		}
		case 2: /* reserved */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			polarity = 1;
			break;
		}
		case 3: /* low active */
		{
			polarity = 1;
			break;
		}
		default: /* invalid */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			polarity = 1;
			break;
		}
L
Linus Torvalds 已提交
952 953 954 955 956 957
	}
	return polarity;
}

static int MPBIOS_trigger(int idx)
{
958
	int bus = mp_irqs[idx].srcbus;
L
Linus Torvalds 已提交
959 960 961 962 963
	int trigger;

	/*
	 * Determine IRQ trigger mode (edge or level sensitive):
	 */
964
	switch ((mp_irqs[idx].irqflag>>2) & 3)
L
Linus Torvalds 已提交
965
	{
966 967 968 969 970
		case 0: /* conforms, ie. bus-type dependent */
			if (test_bit(bus, mp_bus_not_pci))
				trigger = default_ISA_trigger(idx);
			else
				trigger = default_PCI_trigger(idx);
971
#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000
			switch (mp_bus_id_to_type[bus]) {
				case MP_BUS_ISA: /* ISA pin */
				{
					/* set before the switch */
					break;
				}
				case MP_BUS_EISA: /* EISA pin */
				{
					trigger = default_EISA_trigger(idx);
					break;
				}
				case MP_BUS_PCI: /* PCI pin */
				{
					/* set before the switch */
					break;
				}
				case MP_BUS_MCA: /* MCA pin */
				{
					trigger = default_MCA_trigger(idx);
					break;
				}
				default:
				{
					printk(KERN_WARNING "broken BIOS!!\n");
					trigger = 1;
					break;
				}
			}
#endif
L
Linus Torvalds 已提交
1001
			break;
1002
		case 1: /* edge */
L
Linus Torvalds 已提交
1003
		{
1004
			trigger = 0;
L
Linus Torvalds 已提交
1005 1006
			break;
		}
1007
		case 2: /* reserved */
L
Linus Torvalds 已提交
1008
		{
1009 1010
			printk(KERN_WARNING "broken BIOS!!\n");
			trigger = 1;
L
Linus Torvalds 已提交
1011 1012
			break;
		}
1013
		case 3: /* level */
L
Linus Torvalds 已提交
1014
		{
1015
			trigger = 1;
L
Linus Torvalds 已提交
1016 1017
			break;
		}
1018
		default: /* invalid */
L
Linus Torvalds 已提交
1019 1020
		{
			printk(KERN_WARNING "broken BIOS!!\n");
1021
			trigger = 0;
L
Linus Torvalds 已提交
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
			break;
		}
	}
	return trigger;
}

static inline int irq_polarity(int idx)
{
	return MPBIOS_polarity(idx);
}

static inline int irq_trigger(int idx)
{
	return MPBIOS_trigger(idx);
}

Y
Yinghai Lu 已提交
1038
int (*ioapic_renumber_irq)(int ioapic, int irq);
L
Linus Torvalds 已提交
1039 1040 1041
static int pin_2_irq(int idx, int apic, int pin)
{
	int irq, i;
1042
	int bus = mp_irqs[idx].srcbus;
L
Linus Torvalds 已提交
1043 1044 1045 1046

	/*
	 * Debugging check, we are in big trouble if this message pops up!
	 */
1047
	if (mp_irqs[idx].dstirq != pin)
L
Linus Torvalds 已提交
1048 1049
		printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");

1050
	if (test_bit(bus, mp_bus_not_pci)) {
1051
		irq = mp_irqs[idx].srcbusirq;
1052
	} else {
A
Alexey Starikovskiy 已提交
1053 1054 1055 1056 1057 1058 1059
		/*
		 * PCI IRQs are mapped in order
		 */
		i = irq = 0;
		while (i < apic)
			irq += nr_ioapic_registers[i++];
		irq += pin;
T
Thomas Gleixner 已提交
1060
		/*
1061 1062
                 * For MPS mode, so far only needed by ES7000 platform
                 */
T
Thomas Gleixner 已提交
1063 1064
		if (ioapic_renumber_irq)
			irq = ioapic_renumber_irq(apic, irq);
L
Linus Torvalds 已提交
1065 1066
	}

1067
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
	/*
	 * PCI IRQ command line redirection. Yes, limits are hardcoded.
	 */
	if ((pin >= 16) && (pin <= 23)) {
		if (pirq_entries[pin-16] != -1) {
			if (!pirq_entries[pin-16]) {
				apic_printk(APIC_VERBOSE, KERN_DEBUG
						"disabling PIRQ%d\n", pin-16);
			} else {
				irq = pirq_entries[pin-16];
				apic_printk(APIC_VERBOSE, KERN_DEBUG
						"using PIRQ%d -> IRQ %d\n",
						pin-16, irq);
			}
		}
	}
1084 1085
#endif

L
Linus Torvalds 已提交
1086 1087 1088
	return irq;
}

1089 1090 1091 1092 1093
/*
 * Find a specific PCI IRQ entry.
 * Not an __init, possibly needed by modules
 */
int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin,
1094
				struct io_apic_irq_attr *irq_attr)
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
{
	int apic, i, best_guess = -1;

	apic_printk(APIC_DEBUG,
		    "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
		    bus, slot, pin);
	if (test_bit(bus, mp_bus_not_pci)) {
		apic_printk(APIC_VERBOSE,
			    "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
		return -1;
	}
	for (i = 0; i < mp_irq_entries; i++) {
		int lbus = mp_irqs[i].srcbus;

		for (apic = 0; apic < nr_ioapics; apic++)
			if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic ||
			    mp_irqs[i].dstapic == MP_APIC_ALL)
				break;

		if (!test_bit(lbus, mp_bus_not_pci) &&
		    !mp_irqs[i].irqtype &&
		    (bus == lbus) &&
		    (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
			int irq = pin_2_irq(i, apic, mp_irqs[i].dstirq);

			if (!(apic || IO_APIC_IRQ(irq)))
				continue;

			if (pin == (mp_irqs[i].srcbusirq & 3)) {
1124 1125 1126 1127
				set_io_apic_irq_attr(irq_attr, apic,
						     mp_irqs[i].dstirq,
						     irq_trigger(i),
						     irq_polarity(i));
1128 1129 1130 1131 1132 1133 1134
				return irq;
			}
			/*
			 * Use the first all-but-pin matching entry as a
			 * best-guess fuzzy result for broken mptables.
			 */
			if (best_guess < 0) {
1135 1136 1137 1138
				set_io_apic_irq_attr(irq_attr, apic,
						     mp_irqs[i].dstirq,
						     irq_trigger(i),
						     irq_polarity(i));
1139 1140 1141 1142 1143 1144 1145 1146
				best_guess = irq;
			}
		}
	}
	return best_guess;
}
EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);

1147 1148 1149 1150 1151 1152 1153
void lock_vector_lock(void)
{
	/* Used to the online set of cpus does not change
	 * during assign_irq_vector.
	 */
	spin_lock(&vector_lock);
}
L
Linus Torvalds 已提交
1154

1155
void unlock_vector_lock(void)
L
Linus Torvalds 已提交
1156
{
1157 1158
	spin_unlock(&vector_lock);
}
L
Linus Torvalds 已提交
1159

1160 1161
static int
__assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1162
{
1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
1174 1175
	static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
	unsigned int old_vector;
1176 1177
	int cpu, err;
	cpumask_var_t tmp_mask;
1178

1179 1180
	if ((cfg->move_in_progress) || cfg->move_cleanup_count)
		return -EBUSY;
1181

1182 1183
	if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC))
		return -ENOMEM;
1184

1185 1186
	old_vector = cfg->vector;
	if (old_vector) {
1187 1188 1189 1190
		cpumask_and(tmp_mask, mask, cpu_online_mask);
		cpumask_and(tmp_mask, cfg->domain, tmp_mask);
		if (!cpumask_empty(tmp_mask)) {
			free_cpumask_var(tmp_mask);
1191
			return 0;
1192
		}
1193
	}
1194

1195
	/* Only try and allocate irqs on cpus that are present */
1196 1197
	err = -ENOSPC;
	for_each_cpu_and(cpu, mask, cpu_online_mask) {
1198 1199
		int new_cpu;
		int vector, offset;
1200

1201
		apic->vector_allocation_domain(cpu, tmp_mask);
1202

1203 1204
		vector = current_vector;
		offset = current_offset;
1205
next:
1206 1207
		vector += 8;
		if (vector >= first_system_vector) {
1208
			/* If out of vectors on large boxen, must share them. */
1209 1210 1211 1212 1213
			offset = (offset + 1) % 8;
			vector = FIRST_DEVICE_VECTOR + offset;
		}
		if (unlikely(current_vector == vector))
			continue;
1214 1215

		if (test_bit(vector, used_vectors))
1216
			goto next;
1217

1218
		for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1219 1220 1221 1222 1223 1224 1225
			if (per_cpu(vector_irq, new_cpu)[vector] != -1)
				goto next;
		/* Found one! */
		current_vector = vector;
		current_offset = offset;
		if (old_vector) {
			cfg->move_in_progress = 1;
1226
			cpumask_copy(cfg->old_domain, cfg->domain);
1227
		}
1228
		for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1229 1230
			per_cpu(vector_irq, new_cpu)[vector] = irq;
		cfg->vector = vector;
1231 1232 1233
		cpumask_copy(cfg->domain, tmp_mask);
		err = 0;
		break;
1234
	}
1235 1236
	free_cpumask_var(tmp_mask);
	return err;
1237 1238
}

1239 1240
static int
assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1241 1242
{
	int err;
1243 1244 1245
	unsigned long flags;

	spin_lock_irqsave(&vector_lock, flags);
Y
Yinghai Lu 已提交
1246
	err = __assign_irq_vector(irq, cfg, mask);
1247
	spin_unlock_irqrestore(&vector_lock, flags);
1248 1249 1250
	return err;
}

Y
Yinghai Lu 已提交
1251
static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
1252 1253 1254 1255 1256 1257
{
	int cpu, vector;

	BUG_ON(!cfg->vector);

	vector = cfg->vector;
1258
	for_each_cpu_and(cpu, cfg->domain, cpu_online_mask)
1259 1260 1261
		per_cpu(vector_irq, cpu)[vector] = -1;

	cfg->vector = 0;
1262
	cpumask_clear(cfg->domain);
1263 1264 1265

	if (likely(!cfg->move_in_progress))
		return;
1266
	for_each_cpu_and(cpu, cfg->old_domain, cpu_online_mask) {
1267 1268 1269 1270 1271 1272 1273 1274 1275
		for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
								vector++) {
			if (per_cpu(vector_irq, cpu)[vector] != irq)
				continue;
			per_cpu(vector_irq, cpu)[vector] = -1;
			break;
		}
	}
	cfg->move_in_progress = 0;
1276 1277 1278 1279 1280 1281 1282 1283
}

void __setup_vector_irq(int cpu)
{
	/* Initialize vector_irq on a new cpu */
	/* This function must be called with vector_lock held */
	int irq, vector;
	struct irq_cfg *cfg;
1284
	struct irq_desc *desc;
1285 1286

	/* Mark the inuse vectors */
1287 1288
	for_each_irq_desc(irq, desc) {
		cfg = desc->chip_data;
1289
		if (!cpumask_test_cpu(cpu, cfg->domain))
1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300
			continue;
		vector = cfg->vector;
		per_cpu(vector_irq, cpu)[vector] = irq;
	}
	/* Mark the free vectors */
	for (vector = 0; vector < NR_VECTORS; ++vector) {
		irq = per_cpu(vector_irq, cpu)[vector];
		if (irq < 0)
			continue;

		cfg = irq_cfg(irq);
1301
		if (!cpumask_test_cpu(cpu, cfg->domain))
1302
			per_cpu(vector_irq, cpu)[vector] = -1;
1303
	}
L
Linus Torvalds 已提交
1304
}
1305

1306
static struct irq_chip ioapic_chip;
1307
static struct irq_chip ir_ioapic_chip;
L
Linus Torvalds 已提交
1308

1309 1310 1311
#define IOAPIC_AUTO     -1
#define IOAPIC_EDGE     0
#define IOAPIC_LEVEL    1
L
Linus Torvalds 已提交
1312

1313
#ifdef CONFIG_X86_32
1314 1315
static inline int IO_APIC_irq_trigger(int irq)
{
T
Thomas Gleixner 已提交
1316
	int apic, idx, pin;
1317

T
Thomas Gleixner 已提交
1318 1319 1320 1321 1322 1323 1324 1325
	for (apic = 0; apic < nr_ioapics; apic++) {
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
			idx = find_irq_entry(apic, pin, mp_INT);
			if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
				return irq_trigger(idx);
		}
	}
	/*
1326 1327
         * nonexistent IRQs are edge default
         */
T
Thomas Gleixner 已提交
1328
	return 0;
1329
}
1330 1331 1332
#else
static inline int IO_APIC_irq_trigger(int irq)
{
1333
	return 1;
1334 1335
}
#endif
1336

Y
Yinghai Lu 已提交
1337
static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long trigger)
L
Linus Torvalds 已提交
1338
{
Y
Yinghai Lu 已提交
1339

1340
	if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1341
	    trigger == IOAPIC_LEVEL)
1342
		desc->status |= IRQ_LEVEL;
1343 1344 1345
	else
		desc->status &= ~IRQ_LEVEL;

1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
	if (irq_remapped(irq)) {
		desc->status |= IRQ_MOVE_PCNTXT;
		if (trigger)
			set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
						      handle_fasteoi_irq,
						     "fasteoi");
		else
			set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
						      handle_edge_irq, "edge");
		return;
	}
1357

1358 1359
	if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
	    trigger == IOAPIC_LEVEL)
1360
		set_irq_chip_and_handler_name(irq, &ioapic_chip,
1361 1362
					      handle_fasteoi_irq,
					      "fasteoi");
1363
	else
1364
		set_irq_chip_and_handler_name(irq, &ioapic_chip,
1365
					      handle_edge_irq, "edge");
L
Linus Torvalds 已提交
1366 1367
}

1368 1369 1370
int setup_ioapic_entry(int apic_id, int irq,
		       struct IO_APIC_route_entry *entry,
		       unsigned int destination, int trigger,
1371
		       int polarity, int vector, int pin)
L
Linus Torvalds 已提交
1372
{
1373 1374 1375 1376 1377
	/*
	 * add it to the IO-APIC irq-routing table:
	 */
	memset(entry,0,sizeof(*entry));

1378
	if (intr_remapping_enabled) {
I
Ingo Molnar 已提交
1379
		struct intel_iommu *iommu = map_ioapic_to_ir(apic_id);
1380 1381 1382 1383 1384 1385
		struct irte irte;
		struct IR_IO_APIC_route_entry *ir_entry =
			(struct IR_IO_APIC_route_entry *) entry;
		int index;

		if (!iommu)
I
Ingo Molnar 已提交
1386
			panic("No mapping iommu for ioapic %d\n", apic_id);
1387 1388 1389

		index = alloc_irte(iommu, irq, 1);
		if (index < 0)
I
Ingo Molnar 已提交
1390
			panic("Failed to allocate IRTE for ioapic %d\n", apic_id);
1391 1392 1393 1394

		memset(&irte, 0, sizeof(irte));

		irte.present = 1;
1395
		irte.dst_mode = apic->irq_dest_mode;
1396 1397 1398 1399 1400 1401 1402 1403
		/*
		 * Trigger mode in the IRTE will always be edge, and the
		 * actual level or edge trigger will be setup in the IO-APIC
		 * RTE. This will help simplify level triggered irq migration.
		 * For more details, see the comments above explainig IO-APIC
		 * irq migration in the presence of interrupt-remapping.
		 */
		irte.trigger_mode = 0;
1404
		irte.dlvry_mode = apic->irq_delivery_mode;
1405 1406 1407
		irte.vector = vector;
		irte.dest_id = IRTE_DEST(destination);

1408 1409 1410
		/* Set source-id of interrupt request */
		set_ioapic_sid(&irte, apic_id);

1411 1412 1413 1414 1415 1416
		modify_irte(irq, &irte);

		ir_entry->index2 = (index >> 15) & 0x1;
		ir_entry->zero = 0;
		ir_entry->format = 1;
		ir_entry->index = (index & 0x7fff);
1417 1418 1419 1420 1421
		/*
		 * IO-APIC RTE will be configured with virtual vector.
		 * irq handler will do the explicit EOI to the io-apic.
		 */
		ir_entry->vector = pin;
1422
	} else {
1423 1424
		entry->delivery_mode = apic->irq_delivery_mode;
		entry->dest_mode = apic->irq_dest_mode;
1425
		entry->dest = destination;
1426
		entry->vector = vector;
1427
	}
1428

1429
	entry->mask = 0;				/* enable IRQ */
1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440
	entry->trigger = trigger;
	entry->polarity = polarity;

	/* Mask level triggered irqs.
	 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
	 */
	if (trigger)
		entry->mask = 1;
	return 0;
}

I
Ingo Molnar 已提交
1441
static void setup_IO_APIC_irq(int apic_id, int pin, unsigned int irq, struct irq_desc *desc,
1442
			      int trigger, int polarity)
1443 1444
{
	struct irq_cfg *cfg;
L
Linus Torvalds 已提交
1445
	struct IO_APIC_route_entry entry;
1446
	unsigned int dest;
1447 1448 1449 1450

	if (!IO_APIC_IRQ(irq))
		return;

Y
Yinghai Lu 已提交
1451
	cfg = desc->chip_data;
1452

1453
	if (assign_irq_vector(irq, cfg, apic->target_cpus()))
1454 1455
		return;

1456
	dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
1457 1458 1459 1460

	apic_printk(APIC_VERBOSE,KERN_DEBUG
		    "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
		    "IRQ %d Mode:%i Active:%i)\n",
I
Ingo Molnar 已提交
1461
		    apic_id, mp_ioapics[apic_id].apicid, pin, cfg->vector,
1462 1463 1464
		    irq, trigger, polarity);


I
Ingo Molnar 已提交
1465
	if (setup_ioapic_entry(mp_ioapics[apic_id].apicid, irq, &entry,
1466
			       dest, trigger, polarity, cfg->vector, pin)) {
1467
		printk("Failed to setup ioapic entry for ioapic  %d, pin %d\n",
I
Ingo Molnar 已提交
1468
		       mp_ioapics[apic_id].apicid, pin);
Y
Yinghai Lu 已提交
1469
		__clear_irq_vector(irq, cfg);
1470 1471 1472
		return;
	}

Y
Yinghai Lu 已提交
1473
	ioapic_register_intr(irq, desc, trigger);
Y
Yinghai Lu 已提交
1474
	if (irq < NR_IRQS_LEGACY)
1475 1476
		disable_8259A_irq(irq);

I
Ingo Molnar 已提交
1477
	ioapic_write_entry(apic_id, pin, entry);
1478 1479
}

1480 1481 1482 1483
static struct {
	DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
} mp_ioapic_routing[MAX_IO_APICS];

1484 1485
static void __init setup_IO_APIC_irqs(void)
{
1486
	int apic_id = 0, pin, idx, irq;
1487
	int notcon = 0;
1488
	struct irq_desc *desc;
Y
Yinghai Lu 已提交
1489
	struct irq_cfg *cfg;
1490
	int node = cpu_to_node(boot_cpu_id);
L
Linus Torvalds 已提交
1491 1492 1493

	apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");

1494 1495 1496 1497 1498 1499 1500
#ifdef CONFIG_ACPI
	if (!acpi_disabled && acpi_ioapic) {
		apic_id = mp_find_ioapic(0);
		if (apic_id < 0)
			apic_id = 0;
	}
#endif
1501

1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519
	for (pin = 0; pin < nr_ioapic_registers[apic_id]; pin++) {
		idx = find_irq_entry(apic_id, pin, mp_INT);
		if (idx == -1) {
			if (!notcon) {
				notcon = 1;
				apic_printk(APIC_VERBOSE,
					KERN_DEBUG " %d-%d",
					mp_ioapics[apic_id].apicid, pin);
			} else
				apic_printk(APIC_VERBOSE, " %d-%d",
					mp_ioapics[apic_id].apicid, pin);
			continue;
		}
		if (notcon) {
			apic_printk(APIC_VERBOSE,
				" (apicid-pin) not connected\n");
			notcon = 0;
		}
1520

1521
		irq = pin_2_irq(idx, apic_id, pin);
1522

1523 1524 1525 1526 1527 1528 1529
		/*
		 * Skip the timer IRQ if there's a quirk handler
		 * installed and if it returns 1:
		 */
		if (apic->multi_timer_check &&
				apic->multi_timer_check(apic_id, irq))
			continue;
1530

1531 1532 1533 1534
		desc = irq_to_desc_alloc_node(irq, node);
		if (!desc) {
			printk(KERN_INFO "can not get irq_desc for %d\n", irq);
			continue;
1535
		}
1536 1537
		cfg = desc->chip_data;
		add_pin_to_irq_node(cfg, node, apic_id, pin);
1538 1539 1540 1541
		/*
		 * don't mark it in pin_programmed, so later acpi could
		 * set it correctly when irq < 16
		 */
1542 1543
		setup_IO_APIC_irq(apic_id, pin, irq, desc,
				irq_trigger(idx), irq_polarity(idx));
L
Linus Torvalds 已提交
1544 1545
	}

1546 1547
	if (notcon)
		apic_printk(APIC_VERBOSE,
1548
			" (apicid-pin) not connected\n");
L
Linus Torvalds 已提交
1549 1550 1551
}

/*
1552
 * Set up the timer pin, possibly with the 8259A-master behind.
L
Linus Torvalds 已提交
1553
 */
I
Ingo Molnar 已提交
1554
static void __init setup_timer_IRQ0_pin(unsigned int apic_id, unsigned int pin,
1555
					int vector)
L
Linus Torvalds 已提交
1556 1557 1558
{
	struct IO_APIC_route_entry entry;

1559 1560 1561
	if (intr_remapping_enabled)
		return;

1562
	memset(&entry, 0, sizeof(entry));
L
Linus Torvalds 已提交
1563 1564 1565 1566 1567

	/*
	 * We use logical delivery to get the timer IRQ
	 * to the first CPU.
	 */
1568
	entry.dest_mode = apic->irq_dest_mode;
Y
Yinghai Lu 已提交
1569
	entry.mask = 0;			/* don't mask IRQ for edge */
1570
	entry.dest = apic->cpu_mask_to_apicid(apic->target_cpus());
1571
	entry.delivery_mode = apic->irq_delivery_mode;
L
Linus Torvalds 已提交
1572 1573 1574 1575 1576 1577
	entry.polarity = 0;
	entry.trigger = 0;
	entry.vector = vector;

	/*
	 * The timer IRQ doesn't have to know that behind the
1578
	 * scene we may have a 8259A-master in AEOI mode ...
L
Linus Torvalds 已提交
1579
	 */
1580
	set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
L
Linus Torvalds 已提交
1581 1582 1583 1584

	/*
	 * Add it to the IO-APIC irq-routing table:
	 */
I
Ingo Molnar 已提交
1585
	ioapic_write_entry(apic_id, pin, entry);
L
Linus Torvalds 已提交
1586 1587
}

1588 1589

__apicdebuginit(void) print_IO_APIC(void)
L
Linus Torvalds 已提交
1590 1591 1592 1593 1594 1595 1596
{
	int apic, i;
	union IO_APIC_reg_00 reg_00;
	union IO_APIC_reg_01 reg_01;
	union IO_APIC_reg_02 reg_02;
	union IO_APIC_reg_03 reg_03;
	unsigned long flags;
1597
	struct irq_cfg *cfg;
1598
	struct irq_desc *desc;
1599
	unsigned int irq;
L
Linus Torvalds 已提交
1600 1601 1602 1603

	if (apic_verbosity == APIC_QUIET)
		return;

1604
	printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
L
Linus Torvalds 已提交
1605 1606
	for (i = 0; i < nr_ioapics; i++)
		printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1607
		       mp_ioapics[i].apicid, nr_ioapic_registers[i]);
L
Linus Torvalds 已提交
1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621

	/*
	 * We are a bit conservative about what we expect.  We have to
	 * know about every hardware change ASAP.
	 */
	printk(KERN_INFO "testing the IO APIC.......................\n");

	for (apic = 0; apic < nr_ioapics; apic++) {

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(apic, 0);
	reg_01.raw = io_apic_read(apic, 1);
	if (reg_01.bits.version >= 0x10)
		reg_02.raw = io_apic_read(apic, 2);
T
Thomas Gleixner 已提交
1622 1623
	if (reg_01.bits.version >= 0x20)
		reg_03.raw = io_apic_read(apic, 3);
L
Linus Torvalds 已提交
1624 1625
	spin_unlock_irqrestore(&ioapic_lock, flags);

1626
	printk("\n");
1627
	printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].apicid);
L
Linus Torvalds 已提交
1628 1629 1630 1631 1632
	printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
	printk(KERN_DEBUG ".......    : physical APIC id: %02X\n", reg_00.bits.ID);
	printk(KERN_DEBUG ".......    : Delivery Type: %X\n", reg_00.bits.delivery_type);
	printk(KERN_DEBUG ".......    : LTS          : %X\n", reg_00.bits.LTS);

1633
	printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
L
Linus Torvalds 已提交
1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
	printk(KERN_DEBUG ".......     : max redirection entries: %04X\n", reg_01.bits.entries);

	printk(KERN_DEBUG ".......     : PRQ implemented: %X\n", reg_01.bits.PRQ);
	printk(KERN_DEBUG ".......     : IO APIC version: %04X\n", reg_01.bits.version);

	/*
	 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
	 * but the value of reg_02 is read as the previous read register
	 * value, so ignore it if reg_02 == reg_01.
	 */
	if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
		printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
		printk(KERN_DEBUG ".......     : arbitration: %02X\n", reg_02.bits.arbitration);
	}

	/*
	 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
	 * or reg_03, but the value of reg_0[23] is read as the previous read
	 * register value, so ignore it if reg_03 == reg_0[12].
	 */
	if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
	    reg_03.raw != reg_01.raw) {
		printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
		printk(KERN_DEBUG ".......     : Boot DT    : %X\n", reg_03.bits.boot_DT);
	}

	printk(KERN_DEBUG ".... IRQ redirection table:\n");

1662 1663
	printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
			  " Stat Dmod Deli Vect:   \n");
L
Linus Torvalds 已提交
1664 1665 1666 1667

	for (i = 0; i <= reg_01.bits.entries; i++) {
		struct IO_APIC_route_entry entry;

1668
		entry = ioapic_read_entry(apic, i);
L
Linus Torvalds 已提交
1669

1670 1671 1672 1673
		printk(KERN_DEBUG " %02x %03X ",
			i,
			entry.dest
		);
L
Linus Torvalds 已提交
1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687

		printk("%1d    %1d    %1d   %1d   %1d    %1d    %1d    %02X\n",
			entry.mask,
			entry.trigger,
			entry.irr,
			entry.polarity,
			entry.delivery_status,
			entry.dest_mode,
			entry.delivery_mode,
			entry.vector
		);
	}
	}
	printk(KERN_DEBUG "IRQ to pin mappings:\n");
1688 1689 1690 1691 1692
	for_each_irq_desc(irq, desc) {
		struct irq_pin_list *entry;

		cfg = desc->chip_data;
		entry = cfg->irq_2_pin;
1693
		if (!entry)
L
Linus Torvalds 已提交
1694
			continue;
1695
		printk(KERN_DEBUG "IRQ%d ", irq);
L
Linus Torvalds 已提交
1696 1697 1698 1699
		for (;;) {
			printk("-> %d:%d", entry->apic, entry->pin);
			if (!entry->next)
				break;
1700
			entry = entry->next;
L
Linus Torvalds 已提交
1701 1702 1703 1704 1705 1706 1707 1708 1709
		}
		printk("\n");
	}

	printk(KERN_INFO ".................................... done.\n");

	return;
}

1710
__apicdebuginit(void) print_APIC_field(int base)
L
Linus Torvalds 已提交
1711
{
1712
	int i;
L
Linus Torvalds 已提交
1713 1714 1715 1716

	if (apic_verbosity == APIC_QUIET)
		return;

1717 1718 1719 1720 1721 1722
	printk(KERN_DEBUG);

	for (i = 0; i < 8; i++)
		printk(KERN_CONT "%08x", apic_read(base + i*0x10));

	printk(KERN_CONT "\n");
L
Linus Torvalds 已提交
1723 1724
}

1725
__apicdebuginit(void) print_local_APIC(void *dummy)
L
Linus Torvalds 已提交
1726
{
1727
	unsigned int i, v, ver, maxlvt;
1728
	u64 icr;
L
Linus Torvalds 已提交
1729 1730 1731 1732

	if (apic_verbosity == APIC_QUIET)
		return;

1733
	printk(KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
L
Linus Torvalds 已提交
1734
		smp_processor_id(), hard_smp_processor_id());
1735
	v = apic_read(APIC_ID);
1736
	printk(KERN_INFO "... APIC ID:      %08x (%01x)\n", v, read_apic_id());
L
Linus Torvalds 已提交
1737 1738 1739
	v = apic_read(APIC_LVR);
	printk(KERN_INFO "... APIC VERSION: %08x\n", v);
	ver = GET_APIC_VERSION(v);
1740
	maxlvt = lapic_get_maxlvt();
L
Linus Torvalds 已提交
1741 1742 1743 1744

	v = apic_read(APIC_TASKPRI);
	printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);

1745
	if (APIC_INTEGRATED(ver)) {                     /* !82489DX */
1746 1747 1748 1749 1750
		if (!APIC_XAPIC(ver)) {
			v = apic_read(APIC_ARBPRI);
			printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
			       v & APIC_ARBPRI_MASK);
		}
L
Linus Torvalds 已提交
1751 1752 1753 1754
		v = apic_read(APIC_PROCPRI);
		printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
	}

1755 1756 1757 1758 1759 1760 1761 1762 1763
	/*
	 * Remote read supported only in the 82489DX and local APIC for
	 * Pentium processors.
	 */
	if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
		v = apic_read(APIC_RRR);
		printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
	}

L
Linus Torvalds 已提交
1764 1765
	v = apic_read(APIC_LDR);
	printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
1766 1767 1768 1769
	if (!x2apic_enabled()) {
		v = apic_read(APIC_DFR);
		printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
	}
L
Linus Torvalds 已提交
1770 1771 1772 1773
	v = apic_read(APIC_SPIV);
	printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);

	printk(KERN_DEBUG "... APIC ISR field:\n");
1774
	print_APIC_field(APIC_ISR);
L
Linus Torvalds 已提交
1775
	printk(KERN_DEBUG "... APIC TMR field:\n");
1776
	print_APIC_field(APIC_TMR);
L
Linus Torvalds 已提交
1777
	printk(KERN_DEBUG "... APIC IRR field:\n");
1778
	print_APIC_field(APIC_IRR);
L
Linus Torvalds 已提交
1779

1780 1781
	if (APIC_INTEGRATED(ver)) {             /* !82489DX */
		if (maxlvt > 3)         /* Due to the Pentium erratum 3AP. */
L
Linus Torvalds 已提交
1782
			apic_write(APIC_ESR, 0);
1783

L
Linus Torvalds 已提交
1784 1785 1786 1787
		v = apic_read(APIC_ESR);
		printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
	}

1788
	icr = apic_icr_read();
1789 1790
	printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
	printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
L
Linus Torvalds 已提交
1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814

	v = apic_read(APIC_LVTT);
	printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);

	if (maxlvt > 3) {                       /* PC is LVT#4. */
		v = apic_read(APIC_LVTPC);
		printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
	}
	v = apic_read(APIC_LVT0);
	printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
	v = apic_read(APIC_LVT1);
	printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);

	if (maxlvt > 2) {			/* ERR is LVT#3. */
		v = apic_read(APIC_LVTERR);
		printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
	}

	v = apic_read(APIC_TMICT);
	printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
	v = apic_read(APIC_TMCCT);
	printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
	v = apic_read(APIC_TDCR);
	printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826

	if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
		v = apic_read(APIC_EFEAT);
		maxlvt = (v >> 16) & 0xff;
		printk(KERN_DEBUG "... APIC EFEAT: %08x\n", v);
		v = apic_read(APIC_ECTRL);
		printk(KERN_DEBUG "... APIC ECTRL: %08x\n", v);
		for (i = 0; i < maxlvt; i++) {
			v = apic_read(APIC_EILVTn(i));
			printk(KERN_DEBUG "... APIC EILVT%d: %08x\n", i, v);
		}
	}
L
Linus Torvalds 已提交
1827 1828 1829
	printk("\n");
}

1830
__apicdebuginit(void) print_all_local_APICs(void)
L
Linus Torvalds 已提交
1831
{
1832 1833 1834 1835 1836 1837
	int cpu;

	preempt_disable();
	for_each_online_cpu(cpu)
		smp_call_function_single(cpu, print_local_APIC, NULL, 1);
	preempt_enable();
L
Linus Torvalds 已提交
1838 1839
}

1840
__apicdebuginit(void) print_PIC(void)
L
Linus Torvalds 已提交
1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857
{
	unsigned int v;
	unsigned long flags;

	if (apic_verbosity == APIC_QUIET)
		return;

	printk(KERN_DEBUG "\nprinting PIC contents\n");

	spin_lock_irqsave(&i8259A_lock, flags);

	v = inb(0xa1) << 8 | inb(0x21);
	printk(KERN_DEBUG "... PIC  IMR: %04x\n", v);

	v = inb(0xa0) << 8 | inb(0x20);
	printk(KERN_DEBUG "... PIC  IRR: %04x\n", v);

1858 1859
	outb(0x0b,0xa0);
	outb(0x0b,0x20);
L
Linus Torvalds 已提交
1860
	v = inb(0xa0) << 8 | inb(0x20);
1861 1862
	outb(0x0a,0xa0);
	outb(0x0a,0x20);
L
Linus Torvalds 已提交
1863 1864 1865 1866 1867 1868 1869 1870 1871

	spin_unlock_irqrestore(&i8259A_lock, flags);

	printk(KERN_DEBUG "... PIC  ISR: %04x\n", v);

	v = inb(0x4d1) << 8 | inb(0x4d0);
	printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
}

1872 1873 1874
__apicdebuginit(int) print_all_ICs(void)
{
	print_PIC();
1875 1876 1877 1878 1879

	/* don't print out if apic is not there */
	if (!cpu_has_apic || disable_apic)
		return 0;

1880 1881 1882 1883 1884 1885 1886 1887
	print_all_local_APICs();
	print_IO_APIC();

	return 0;
}

fs_initcall(print_all_ICs);

L
Linus Torvalds 已提交
1888

Y
Yinghai Lu 已提交
1889 1890 1891
/* Where if anywhere is the i8259 connect in external int mode */
static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };

1892
void __init enable_IO_APIC(void)
L
Linus Torvalds 已提交
1893 1894
{
	union IO_APIC_reg_01 reg_01;
1895
	int i8259_apic, i8259_pin;
1896
	int apic;
L
Linus Torvalds 已提交
1897 1898 1899 1900 1901
	unsigned long flags;

	/*
	 * The number of IO-APIC IRQ registers (== #pins):
	 */
1902
	for (apic = 0; apic < nr_ioapics; apic++) {
L
Linus Torvalds 已提交
1903
		spin_lock_irqsave(&ioapic_lock, flags);
1904
		reg_01.raw = io_apic_read(apic, 1);
L
Linus Torvalds 已提交
1905
		spin_unlock_irqrestore(&ioapic_lock, flags);
1906 1907
		nr_ioapic_registers[apic] = reg_01.bits.entries+1;
	}
1908
	for(apic = 0; apic < nr_ioapics; apic++) {
1909 1910
		int pin;
		/* See if any of the pins is in ExtINT mode */
1911
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
1912
			struct IO_APIC_route_entry entry;
1913
			entry = ioapic_read_entry(apic, pin);
1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943

			/* If the interrupt line is enabled and in ExtInt mode
			 * I have found the pin where the i8259 is connected.
			 */
			if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
				ioapic_i8259.apic = apic;
				ioapic_i8259.pin  = pin;
				goto found_i8259;
			}
		}
	}
 found_i8259:
	/* Look to see what if the MP table has reported the ExtINT */
	/* If we could not find the appropriate pin by looking at the ioapic
	 * the i8259 probably is not connected the ioapic but give the
	 * mptable a chance anyway.
	 */
	i8259_pin  = find_isa_irq_pin(0, mp_ExtINT);
	i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
	/* Trust the MP table if nothing is setup in the hardware */
	if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
		printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
		ioapic_i8259.pin  = i8259_pin;
		ioapic_i8259.apic = i8259_apic;
	}
	/* Complain if the MP table and the hardware disagree */
	if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
		(i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
	{
		printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
L
Linus Torvalds 已提交
1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
	}

	/*
	 * Do not trust the IO-APIC being empty at bootup
	 */
	clear_IO_APIC();
}

/*
 * Not an __init, needed by the reboot code
 */
void disable_IO_APIC(void)
{
	/*
	 * Clear the IO-APIC before rebooting:
	 */
	clear_IO_APIC();

1962
	/*
1963
	 * If the i8259 is routed through an IOAPIC
1964
	 * Put that IOAPIC in virtual wire mode
1965
	 * so legacy interrupts can be delivered.
1966 1967 1968 1969 1970
	 *
	 * With interrupt-remapping, for now we will use virtual wire A mode,
	 * as virtual wire B is little complex (need to configure both
	 * IOAPIC RTE aswell as interrupt-remapping table entry).
	 * As this gets called during crash dump, keep this simple for now.
1971
	 */
1972
	if (ioapic_i8259.pin != -1 && !intr_remapping_enabled) {
1973 1974 1975 1976 1977 1978 1979 1980 1981
		struct IO_APIC_route_entry entry;

		memset(&entry, 0, sizeof(entry));
		entry.mask            = 0; /* Enabled */
		entry.trigger         = 0; /* Edge */
		entry.irr             = 0;
		entry.polarity        = 0; /* High */
		entry.delivery_status = 0;
		entry.dest_mode       = 0; /* Physical */
1982
		entry.delivery_mode   = dest_ExtINT; /* ExtInt */
1983
		entry.vector          = 0;
1984
		entry.dest            = read_apic_id();
1985 1986 1987 1988

		/*
		 * Add it to the IO-APIC irq-routing table:
		 */
1989
		ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1990
	}
1991

1992 1993 1994
	/*
	 * Use virtual wire A mode when interrupt remapping is enabled.
	 */
1995 1996 1997
	if (cpu_has_apic)
		disconnect_bsp_APIC(!intr_remapping_enabled &&
				ioapic_i8259.pin != -1);
L
Linus Torvalds 已提交
1998 1999
}

2000
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011
/*
 * function to set the IO-APIC physical IDs based on the
 * values stored in the MPC table.
 *
 * by Matt Domsch <Matt_Domsch@dell.com>  Tue Dec 21 12:25:05 CST 1999
 */

static void __init setup_ioapic_ids_from_mpc(void)
{
	union IO_APIC_reg_00 reg_00;
	physid_mask_t phys_id_present_map;
I
Ingo Molnar 已提交
2012
	int apic_id;
L
Linus Torvalds 已提交
2013 2014 2015 2016
	int i;
	unsigned char old_id;
	unsigned long flags;

2017
	if (x86_quirks->setup_ioapic_ids && x86_quirks->setup_ioapic_ids())
2018 2019
		return;

2020 2021 2022 2023
	/*
	 * Don't check I/O APIC IDs for xAPIC systems.  They have
	 * no meaning without the serial APIC bus.
	 */
2024 2025
	if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
		|| APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
2026
		return;
L
Linus Torvalds 已提交
2027 2028 2029 2030
	/*
	 * This is broken; anything with a real cpu count has to
	 * circumvent this idiocy regardless.
	 */
2031
	phys_id_present_map = apic->ioapic_phys_id_map(phys_cpu_present_map);
L
Linus Torvalds 已提交
2032 2033 2034 2035

	/*
	 * Set the IOAPIC ID to the value stored in the MPC table.
	 */
I
Ingo Molnar 已提交
2036
	for (apic_id = 0; apic_id < nr_ioapics; apic_id++) {
L
Linus Torvalds 已提交
2037 2038 2039

		/* Read the register 0 value */
		spin_lock_irqsave(&ioapic_lock, flags);
I
Ingo Molnar 已提交
2040
		reg_00.raw = io_apic_read(apic_id, 0);
L
Linus Torvalds 已提交
2041
		spin_unlock_irqrestore(&ioapic_lock, flags);
2042

I
Ingo Molnar 已提交
2043
		old_id = mp_ioapics[apic_id].apicid;
L
Linus Torvalds 已提交
2044

I
Ingo Molnar 已提交
2045
		if (mp_ioapics[apic_id].apicid >= get_physical_broadcast()) {
L
Linus Torvalds 已提交
2046
			printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
I
Ingo Molnar 已提交
2047
				apic_id, mp_ioapics[apic_id].apicid);
L
Linus Torvalds 已提交
2048 2049
			printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
				reg_00.bits.ID);
I
Ingo Molnar 已提交
2050
			mp_ioapics[apic_id].apicid = reg_00.bits.ID;
L
Linus Torvalds 已提交
2051 2052 2053 2054 2055 2056 2057
		}

		/*
		 * Sanity check, is the ID really free? Every APIC in a
		 * system must have a unique ID or we get lots of nice
		 * 'stuck on smp_invalidate_needed IPI wait' messages.
		 */
2058
		if (apic->check_apicid_used(phys_id_present_map,
I
Ingo Molnar 已提交
2059
					mp_ioapics[apic_id].apicid)) {
L
Linus Torvalds 已提交
2060
			printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
I
Ingo Molnar 已提交
2061
				apic_id, mp_ioapics[apic_id].apicid);
L
Linus Torvalds 已提交
2062 2063 2064 2065 2066 2067 2068 2069
			for (i = 0; i < get_physical_broadcast(); i++)
				if (!physid_isset(i, phys_id_present_map))
					break;
			if (i >= get_physical_broadcast())
				panic("Max APIC ID exceeded!\n");
			printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
				i);
			physid_set(i, phys_id_present_map);
I
Ingo Molnar 已提交
2070
			mp_ioapics[apic_id].apicid = i;
L
Linus Torvalds 已提交
2071 2072
		} else {
			physid_mask_t tmp;
2073
			tmp = apic->apicid_to_cpu_present(mp_ioapics[apic_id].apicid);
L
Linus Torvalds 已提交
2074 2075
			apic_printk(APIC_VERBOSE, "Setting %d in the "
					"phys_id_present_map\n",
I
Ingo Molnar 已提交
2076
					mp_ioapics[apic_id].apicid);
L
Linus Torvalds 已提交
2077 2078 2079 2080 2081 2082 2083 2084
			physids_or(phys_id_present_map, phys_id_present_map, tmp);
		}


		/*
		 * We need to adjust the IRQ routing table
		 * if the ID changed.
		 */
I
Ingo Molnar 已提交
2085
		if (old_id != mp_ioapics[apic_id].apicid)
L
Linus Torvalds 已提交
2086
			for (i = 0; i < mp_irq_entries; i++)
2087 2088
				if (mp_irqs[i].dstapic == old_id)
					mp_irqs[i].dstapic
I
Ingo Molnar 已提交
2089
						= mp_ioapics[apic_id].apicid;
L
Linus Torvalds 已提交
2090 2091 2092 2093

		/*
		 * Read the right value from the MPC table and
		 * write it into the ID register.
2094
		 */
L
Linus Torvalds 已提交
2095 2096
		apic_printk(APIC_VERBOSE, KERN_INFO
			"...changing IO-APIC physical APIC ID to %d ...",
I
Ingo Molnar 已提交
2097
			mp_ioapics[apic_id].apicid);
L
Linus Torvalds 已提交
2098

I
Ingo Molnar 已提交
2099
		reg_00.bits.ID = mp_ioapics[apic_id].apicid;
L
Linus Torvalds 已提交
2100
		spin_lock_irqsave(&ioapic_lock, flags);
I
Ingo Molnar 已提交
2101
		io_apic_write(apic_id, 0, reg_00.raw);
2102
		spin_unlock_irqrestore(&ioapic_lock, flags);
L
Linus Torvalds 已提交
2103 2104 2105 2106 2107

		/*
		 * Sanity check
		 */
		spin_lock_irqsave(&ioapic_lock, flags);
I
Ingo Molnar 已提交
2108
		reg_00.raw = io_apic_read(apic_id, 0);
L
Linus Torvalds 已提交
2109
		spin_unlock_irqrestore(&ioapic_lock, flags);
I
Ingo Molnar 已提交
2110
		if (reg_00.bits.ID != mp_ioapics[apic_id].apicid)
L
Linus Torvalds 已提交
2111 2112 2113 2114 2115
			printk("could not set ID!\n");
		else
			apic_printk(APIC_VERBOSE, " ok.\n");
	}
}
2116
#endif
L
Linus Torvalds 已提交
2117

2118
int no_timer_check __initdata;
2119 2120 2121 2122 2123 2124 2125 2126

static int __init notimercheck(char *s)
{
	no_timer_check = 1;
	return 1;
}
__setup("no_timer_check", notimercheck);

L
Linus Torvalds 已提交
2127 2128 2129 2130 2131 2132 2133 2134
/*
 * There is a nasty bug in some older SMP boards, their mptable lies
 * about the timer IRQ. We do the following to work around the situation:
 *
 *	- timer IRQ defaults to IO-APIC IRQ
 *	- if this function detects that timer IRQs are defunct, then we fall
 *	  back to ISA timer IRQs
 */
2135
static int __init timer_irq_works(void)
L
Linus Torvalds 已提交
2136 2137
{
	unsigned long t1 = jiffies;
2138
	unsigned long flags;
L
Linus Torvalds 已提交
2139

2140 2141 2142
	if (no_timer_check)
		return 1;

2143
	local_save_flags(flags);
L
Linus Torvalds 已提交
2144 2145 2146
	local_irq_enable();
	/* Let ten ticks pass... */
	mdelay((10 * 1000) / HZ);
2147
	local_irq_restore(flags);
L
Linus Torvalds 已提交
2148 2149 2150 2151 2152 2153 2154 2155

	/*
	 * Expect a few ticks at least, to be sure some possible
	 * glue logic does not lock up after one or two first
	 * ticks in a non-ExtINT mode.  Also the local APIC
	 * might have cached one ExtINT interrupt.  Finally, at
	 * least one tick may be lost due to delays.
	 */
2156 2157

	/* jiffies wrap? */
2158
	if (time_after(jiffies, t1 + 4))
L
Linus Torvalds 已提交
2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184
		return 1;
	return 0;
}

/*
 * In the SMP+IOAPIC case it might happen that there are an unspecified
 * number of pending IRQ events unhandled. These cases are very rare,
 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
 * better to do it this way as thus we do not have to be aware of
 * 'pending' interrupts in the IRQ path, except at this point.
 */
/*
 * Edge triggered needs to resend any interrupt
 * that was delayed but this is now handled in the device
 * independent code.
 */

/*
 * Starting up a edge-triggered IO-APIC interrupt is
 * nasty - we need to make sure that we get the edge.
 * If it is already asserted for some reason, we need
 * return 1 to indicate that is was pending.
 *
 * This is not complete - we should be able to fake
 * an edge even if it isn't on the 8259A...
 */
2185

2186
static unsigned int startup_ioapic_irq(unsigned int irq)
L
Linus Torvalds 已提交
2187 2188 2189
{
	int was_pending = 0;
	unsigned long flags;
2190
	struct irq_cfg *cfg;
L
Linus Torvalds 已提交
2191 2192

	spin_lock_irqsave(&ioapic_lock, flags);
Y
Yinghai Lu 已提交
2193
	if (irq < NR_IRQS_LEGACY) {
L
Linus Torvalds 已提交
2194 2195 2196 2197
		disable_8259A_irq(irq);
		if (i8259A_irq_pending(irq))
			was_pending = 1;
	}
2198
	cfg = irq_cfg(irq);
Y
Yinghai Lu 已提交
2199
	__unmask_IO_APIC_irq(cfg);
L
Linus Torvalds 已提交
2200 2201 2202 2203 2204
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return was_pending;
}

2205
#ifdef CONFIG_X86_64
2206
static int ioapic_retrigger_irq(unsigned int irq)
L
Linus Torvalds 已提交
2207
{
2208 2209 2210 2211 2212

	struct irq_cfg *cfg = irq_cfg(irq);
	unsigned long flags;

	spin_lock_irqsave(&vector_lock, flags);
2213
	apic->send_IPI_mask(cpumask_of(cpumask_first(cfg->domain)), cfg->vector);
2214
	spin_unlock_irqrestore(&vector_lock, flags);
2215 2216 2217

	return 1;
}
2218 2219
#else
static int ioapic_retrigger_irq(unsigned int irq)
2220
{
2221
	apic->send_IPI_self(irq_cfg(irq)->vector);
2222

T
Thomas Gleixner 已提交
2223
	return 1;
2224 2225
}
#endif
2226

2227 2228 2229 2230 2231 2232 2233 2234
/*
 * Level and edge triggered IO-APIC interrupts need different handling,
 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
 * handled with the level-triggered descriptor, but that one has slightly
 * more overhead. Level-triggered interrupts cannot be handled with the
 * edge-triggered handler, without risking IRQ storms and other ugly
 * races.
 */
2235

2236
#ifdef CONFIG_SMP
2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
static void send_cleanup_vector(struct irq_cfg *cfg)
{
	cpumask_var_t cleanup_mask;

	if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
		unsigned int i;
		cfg->move_cleanup_count = 0;
		for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
			cfg->move_cleanup_count++;
		for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
			apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
	} else {
		cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
		cfg->move_cleanup_count = cpumask_weight(cleanup_mask);
		apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
		free_cpumask_var(cleanup_mask);
	}
	cfg->move_in_progress = 0;
}

2257
static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287
{
	int apic, pin;
	struct irq_pin_list *entry;
	u8 vector = cfg->vector;

	entry = cfg->irq_2_pin;
	for (;;) {
		unsigned int reg;

		if (!entry)
			break;

		apic = entry->apic;
		pin = entry->pin;
		/*
		 * With interrupt-remapping, destination information comes
		 * from interrupt-remapping table entry.
		 */
		if (!irq_remapped(irq))
			io_apic_write(apic, 0x11 + pin*2, dest);
		reg = io_apic_read(apic, 0x10 + pin*2);
		reg &= ~IO_APIC_REDIR_VECTOR_MASK;
		reg |= vector;
		io_apic_modify(apic, 0x10 + pin*2, reg);
		if (!entry->next)
			break;
		entry = entry->next;
	}
}

2288 2289 2290
static int
assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask);

2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
/*
 * Either sets desc->affinity to a valid value, and returns
 * ->cpu_mask_to_apicid of that, or returns BAD_APICID and
 * leaves desc->affinity untouched.
 */
static unsigned int
set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask)
{
	struct irq_cfg *cfg;
	unsigned int irq;

	if (!cpumask_intersects(mask, cpu_online_mask))
		return BAD_APICID;

	irq = desc->irq;
	cfg = desc->chip_data;
	if (assign_irq_vector(irq, cfg, mask))
		return BAD_APICID;

	cpumask_copy(desc->affinity, mask);

	return apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain);
}

2315
static int
2316 2317 2318 2319 2320 2321
set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
{
	struct irq_cfg *cfg;
	unsigned long flags;
	unsigned int dest;
	unsigned int irq;
2322
	int ret = -1;
2323 2324 2325 2326 2327 2328 2329 2330 2331 2332

	irq = desc->irq;
	cfg = desc->chip_data;

	spin_lock_irqsave(&ioapic_lock, flags);
	dest = set_desc_affinity(desc, mask);
	if (dest != BAD_APICID) {
		/* Only the high 8 bits are valid. */
		dest = SET_APIC_LOGICAL_ID(dest);
		__target_IO_APIC_irq(irq, dest, cfg);
2333
		ret = 0;
2334 2335
	}
	spin_unlock_irqrestore(&ioapic_lock, flags);
2336 2337

	return ret;
2338 2339
}

2340
static int
2341 2342 2343 2344 2345 2346
set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
{
	struct irq_desc *desc;

	desc = irq_to_desc(irq);

2347
	return set_ioapic_affinity_irq_desc(desc, mask);
2348
}
2349

2350
#ifdef CONFIG_INTR_REMAP
2351

2352 2353 2354
/*
 * Migrate the IO-APIC irq in the presence of intr-remapping.
 *
2355 2356
 * For both level and edge triggered, irq migration is a simple atomic
 * update(of vector and cpu destination) of IRTE and flush the hardware cache.
2357
 *
2358 2359 2360 2361
 * For level triggered, we eliminate the io-apic RTE modification (with the
 * updated vector information), by using a virtual vector (io-apic pin number).
 * Real vector that is used for interrupting cpu will be coming from
 * the interrupt-remapping table entry.
2362
 */
2363
static int
2364
migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
2365
{
2366 2367 2368
	struct irq_cfg *cfg;
	struct irte irte;
	unsigned int dest;
Y
Yinghai Lu 已提交
2369
	unsigned int irq;
2370
	int ret = -1;
2371

2372
	if (!cpumask_intersects(mask, cpu_online_mask))
2373
		return ret;
2374

Y
Yinghai Lu 已提交
2375
	irq = desc->irq;
2376
	if (get_irte(irq, &irte))
2377
		return ret;
2378

Y
Yinghai Lu 已提交
2379 2380
	cfg = desc->chip_data;
	if (assign_irq_vector(irq, cfg, mask))
2381
		return ret;
2382

2383
	dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask);
2384 2385 2386 2387 2388 2389 2390 2391 2392

	irte.vector = cfg->vector;
	irte.dest_id = IRTE_DEST(dest);

	/*
	 * Modified the IRTE and flushes the Interrupt entry cache.
	 */
	modify_irte(irq, &irte);

2393 2394
	if (cfg->move_in_progress)
		send_cleanup_vector(cfg);
2395

2396
	cpumask_copy(desc->affinity, mask);
2397 2398

	return 0;
2399 2400 2401 2402 2403
}

/*
 * Migrates the IRQ destination in the process context.
 */
2404
static int set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
R
Rusty Russell 已提交
2405
					    const struct cpumask *mask)
2406
{
2407
	return migrate_ioapic_irq_desc(desc, mask);
Y
Yinghai Lu 已提交
2408
}
2409
static int set_ir_ioapic_affinity_irq(unsigned int irq,
R
Rusty Russell 已提交
2410
				       const struct cpumask *mask)
Y
Yinghai Lu 已提交
2411 2412 2413
{
	struct irq_desc *desc = irq_to_desc(irq);

2414
	return set_ir_ioapic_affinity_irq_desc(desc, mask);
2415
}
2416
#else
2417
static inline int set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
2418 2419
						   const struct cpumask *mask)
{
2420
	return 0;
2421
}
2422 2423 2424 2425 2426
#endif

asmlinkage void smp_irq_move_cleanup_interrupt(void)
{
	unsigned vector, me;
2427

2428 2429 2430 2431 2432 2433 2434
	ack_APIC_irq();
	exit_idle();
	irq_enter();

	me = smp_processor_id();
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		unsigned int irq;
2435
		unsigned int irr;
2436 2437 2438 2439
		struct irq_desc *desc;
		struct irq_cfg *cfg;
		irq = __get_cpu_var(vector_irq)[vector];

2440 2441 2442
		if (irq == -1)
			continue;

2443 2444 2445 2446 2447 2448 2449 2450 2451
		desc = irq_to_desc(irq);
		if (!desc)
			continue;

		cfg = irq_cfg(irq);
		spin_lock(&desc->lock);
		if (!cfg->move_cleanup_count)
			goto unlock;

2452
		if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2453 2454
			goto unlock;

2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466
		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		/*
		 * Check if the vector that needs to be cleanedup is
		 * registered at the cpu's IRR. If so, then this is not
		 * the best time to clean it up. Lets clean it up in the
		 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
		 * to myself.
		 */
		if (irr  & (1 << (vector % 32))) {
			apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
			goto unlock;
		}
2467 2468 2469 2470 2471 2472 2473 2474 2475
		__get_cpu_var(vector_irq)[vector] = -1;
		cfg->move_cleanup_count--;
unlock:
		spin_unlock(&desc->lock);
	}

	irq_exit();
}

Y
Yinghai Lu 已提交
2476
static void irq_complete_move(struct irq_desc **descp)
2477
{
Y
Yinghai Lu 已提交
2478 2479
	struct irq_desc *desc = *descp;
	struct irq_cfg *cfg = desc->chip_data;
2480 2481
	unsigned vector, me;

2482
	if (likely(!cfg->move_in_progress))
2483 2484 2485 2486
		return;

	vector = ~get_irq_regs()->orig_ax;
	me = smp_processor_id();
2487

2488
	if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2489
		send_cleanup_vector(cfg);
2490 2491
}
#else
Y
Yinghai Lu 已提交
2492
static inline void irq_complete_move(struct irq_desc **descp) {}
2493
#endif
Y
Yinghai Lu 已提交
2494

2495 2496
static void ack_apic_edge(unsigned int irq)
{
Y
Yinghai Lu 已提交
2497 2498 2499
	struct irq_desc *desc = irq_to_desc(irq);

	irq_complete_move(&desc);
2500 2501 2502 2503
	move_native_irq(irq);
	ack_APIC_irq();
}

Y
Yinghai Lu 已提交
2504 2505
atomic_t irq_mis_count;

2506 2507
static void ack_apic_level(unsigned int irq)
{
Y
Yinghai Lu 已提交
2508
	struct irq_desc *desc = irq_to_desc(irq);
Y
Yinghai Lu 已提交
2509 2510
	unsigned long v;
	int i;
Y
Yinghai Lu 已提交
2511
	struct irq_cfg *cfg;
2512
	int do_unmask_irq = 0;
2513

Y
Yinghai Lu 已提交
2514
	irq_complete_move(&desc);
2515
#ifdef CONFIG_GENERIC_PENDING_IRQ
2516
	/* If we are moving the irq we need to mask it */
Y
Yinghai Lu 已提交
2517
	if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
2518
		do_unmask_irq = 1;
Y
Yinghai Lu 已提交
2519
		mask_IO_APIC_irq_desc(desc);
2520
	}
2521 2522
#endif

Y
Yinghai Lu 已提交
2523
	/*
2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541
	 * It appears there is an erratum which affects at least version 0x11
	 * of I/O APIC (that's the 82093AA and cores integrated into various
	 * chipsets).  Under certain conditions a level-triggered interrupt is
	 * erroneously delivered as edge-triggered one but the respective IRR
	 * bit gets set nevertheless.  As a result the I/O unit expects an EOI
	 * message but it will never arrive and further interrupts are blocked
	 * from the source.  The exact reason is so far unknown, but the
	 * phenomenon was observed when two consecutive interrupt requests
	 * from a given source get delivered to the same CPU and the source is
	 * temporarily disabled in between.
	 *
	 * A workaround is to simulate an EOI message manually.  We achieve it
	 * by setting the trigger mode to edge and then to level when the edge
	 * trigger mode gets detected in the TMR of a local APIC for a
	 * level-triggered interrupt.  We mask the source for the time of the
	 * operation to prevent an edge-triggered interrupt escaping meanwhile.
	 * The idea is from Manfred Spraul.  --macro
	 */
Y
Yinghai Lu 已提交
2542 2543
	cfg = desc->chip_data;
	i = cfg->vector;
Y
Yinghai Lu 已提交
2544 2545
	v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));

2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579
	/*
	 * We must acknowledge the irq before we move it or the acknowledge will
	 * not propagate properly.
	 */
	ack_APIC_irq();

	/* Now we can move and renable the irq */
	if (unlikely(do_unmask_irq)) {
		/* Only migrate the irq if the ack has been received.
		 *
		 * On rare occasions the broadcast level triggered ack gets
		 * delayed going to ioapics, and if we reprogram the
		 * vector while Remote IRR is still set the irq will never
		 * fire again.
		 *
		 * To prevent this scenario we read the Remote IRR bit
		 * of the ioapic.  This has two effects.
		 * - On any sane system the read of the ioapic will
		 *   flush writes (and acks) going to the ioapic from
		 *   this cpu.
		 * - We get to see if the ACK has actually been delivered.
		 *
		 * Based on failed experiments of reprogramming the
		 * ioapic entry from outside of irq context starting
		 * with masking the ioapic entry and then polling until
		 * Remote IRR was clear before reprogramming the
		 * ioapic I don't trust the Remote IRR bit to be
		 * completey accurate.
		 *
		 * However there appears to be no other way to plug
		 * this race, so if the Remote IRR bit is not
		 * accurate and is causing problems then it is a hardware bug
		 * and you can go talk to the chipset vendor about it.
		 */
Y
Yinghai Lu 已提交
2580 2581
		cfg = desc->chip_data;
		if (!io_apic_level_ack_pending(cfg))
2582
			move_masked_irq(irq);
Y
Yinghai Lu 已提交
2583
		unmask_IO_APIC_irq_desc(desc);
2584
	}
2585

2586
	/* Tail end of version 0x11 I/O APIC bug workaround */
2587 2588 2589
	if (!(v & (1 << (i & 0x1f)))) {
		atomic_inc(&irq_mis_count);
		spin_lock(&ioapic_lock);
Y
Yinghai Lu 已提交
2590 2591
		__mask_and_edge_IO_APIC_irq(cfg);
		__unmask_and_level_IO_APIC_irq(cfg);
2592 2593
		spin_unlock(&ioapic_lock);
	}
Y
Yinghai Lu 已提交
2594
}
2595

2596
#ifdef CONFIG_INTR_REMAP
2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629
static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
{
	int apic, pin;
	struct irq_pin_list *entry;

	entry = cfg->irq_2_pin;
	for (;;) {

		if (!entry)
			break;

		apic = entry->apic;
		pin = entry->pin;
		io_apic_eoi(apic, pin);
		entry = entry->next;
	}
}

static void
eoi_ioapic_irq(struct irq_desc *desc)
{
	struct irq_cfg *cfg;
	unsigned long flags;
	unsigned int irq;

	irq = desc->irq;
	cfg = desc->chip_data;

	spin_lock_irqsave(&ioapic_lock, flags);
	__eoi_ioapic_irq(irq, cfg);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

2630 2631
static void ir_ack_apic_edge(unsigned int irq)
{
2632
	ack_APIC_irq();
2633 2634 2635 2636
}

static void ir_ack_apic_level(unsigned int irq)
{
2637 2638 2639 2640
	struct irq_desc *desc = irq_to_desc(irq);

	ack_APIC_irq();
	eoi_ioapic_irq(desc);
2641 2642 2643
}
#endif /* CONFIG_INTR_REMAP */

2644
static struct irq_chip ioapic_chip __read_mostly = {
T
Thomas Gleixner 已提交
2645 2646 2647 2648 2649 2650
	.name		= "IO-APIC",
	.startup	= startup_ioapic_irq,
	.mask		= mask_IO_APIC_irq,
	.unmask		= unmask_IO_APIC_irq,
	.ack		= ack_apic_edge,
	.eoi		= ack_apic_level,
2651
#ifdef CONFIG_SMP
T
Thomas Gleixner 已提交
2652
	.set_affinity	= set_ioapic_affinity_irq,
2653
#endif
2654
	.retrigger	= ioapic_retrigger_irq,
L
Linus Torvalds 已提交
2655 2656
};

2657
static struct irq_chip ir_ioapic_chip __read_mostly = {
T
Thomas Gleixner 已提交
2658 2659 2660 2661
	.name		= "IR-IO-APIC",
	.startup	= startup_ioapic_irq,
	.mask		= mask_IO_APIC_irq,
	.unmask		= unmask_IO_APIC_irq,
2662
#ifdef CONFIG_INTR_REMAP
2663 2664
	.ack		= ir_ack_apic_edge,
	.eoi		= ir_ack_apic_level,
2665
#ifdef CONFIG_SMP
T
Thomas Gleixner 已提交
2666
	.set_affinity	= set_ir_ioapic_affinity_irq,
2667
#endif
2668 2669 2670
#endif
	.retrigger	= ioapic_retrigger_irq,
};
L
Linus Torvalds 已提交
2671 2672 2673 2674

static inline void init_IO_APIC_traps(void)
{
	int irq;
2675
	struct irq_desc *desc;
2676
	struct irq_cfg *cfg;
L
Linus Torvalds 已提交
2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688

	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
2689 2690 2691
	for_each_irq_desc(irq, desc) {
		cfg = desc->chip_data;
		if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
L
Linus Torvalds 已提交
2692 2693 2694 2695 2696
			/*
			 * Hmm.. We don't have an entry for this,
			 * so default to an old-fashioned 8259
			 * interrupt if we can..
			 */
Y
Yinghai Lu 已提交
2697
			if (irq < NR_IRQS_LEGACY)
L
Linus Torvalds 已提交
2698
				make_8259A_irq(irq);
2699
			else
L
Linus Torvalds 已提交
2700
				/* Strange. Oh, well.. */
2701
				desc->chip = &no_irq_chip;
L
Linus Torvalds 已提交
2702 2703 2704 2705
		}
	}
}

2706 2707 2708
/*
 * The local APIC irq-chip implementation:
 */
L
Linus Torvalds 已提交
2709

2710
static void mask_lapic_irq(unsigned int irq)
L
Linus Torvalds 已提交
2711 2712 2713 2714
{
	unsigned long v;

	v = apic_read(APIC_LVT0);
2715
	apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
L
Linus Torvalds 已提交
2716 2717
}

2718
static void unmask_lapic_irq(unsigned int irq)
L
Linus Torvalds 已提交
2719
{
2720
	unsigned long v;
L
Linus Torvalds 已提交
2721

2722
	v = apic_read(APIC_LVT0);
2723
	apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2724
}
L
Linus Torvalds 已提交
2725

Y
Yinghai Lu 已提交
2726
static void ack_lapic_irq(unsigned int irq)
2727 2728 2729 2730
{
	ack_APIC_irq();
}

2731
static struct irq_chip lapic_chip __read_mostly = {
2732
	.name		= "local-APIC",
2733 2734
	.mask		= mask_lapic_irq,
	.unmask		= unmask_lapic_irq,
2735
	.ack		= ack_lapic_irq,
L
Linus Torvalds 已提交
2736 2737
};

Y
Yinghai Lu 已提交
2738
static void lapic_register_intr(int irq, struct irq_desc *desc)
2739
{
2740
	desc->status &= ~IRQ_LEVEL;
2741 2742 2743 2744
	set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
				      "edge");
}

2745
static void __init setup_nmi(void)
L
Linus Torvalds 已提交
2746 2747
{
	/*
2748
	 * Dirty trick to enable the NMI watchdog ...
L
Linus Torvalds 已提交
2749 2750 2751 2752 2753 2754
	 * We put the 8259A master into AEOI mode and
	 * unmask on all local APICs LVT0 as NMI.
	 *
	 * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire')
	 * is from Maciej W. Rozycki - so we do not have to EOI from
	 * the NMI handler or the timer interrupt.
2755
	 */
L
Linus Torvalds 已提交
2756 2757
	apic_printk(APIC_VERBOSE, KERN_INFO "activating NMI Watchdog ...");

2758
	enable_NMI_through_LVT0();
L
Linus Torvalds 已提交
2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769

	apic_printk(APIC_VERBOSE, " done.\n");
}

/*
 * This looks a bit hackish but it's about the only one way of sending
 * a few INTA cycles to 8259As and any associated glue logic.  ICR does
 * not support the ExtINT mode, unfortunately.  We need to send these
 * cycles as some i82489DX-based boards have glue logic that keeps the
 * 8259A interrupt line asserted until INTA.  --macro
 */
2770
static inline void __init unlock_ExtINT_logic(void)
L
Linus Torvalds 已提交
2771
{
2772
	int apic, pin, i;
L
Linus Torvalds 已提交
2773 2774 2775
	struct IO_APIC_route_entry entry0, entry1;
	unsigned char save_control, save_freq_select;

2776
	pin  = find_isa_irq_pin(8, mp_INT);
2777 2778 2779 2780
	if (pin == -1) {
		WARN_ON_ONCE(1);
		return;
	}
2781
	apic = find_isa_irq_apic(8, mp_INT);
2782 2783
	if (apic == -1) {
		WARN_ON_ONCE(1);
L
Linus Torvalds 已提交
2784
		return;
2785
	}
L
Linus Torvalds 已提交
2786

2787
	entry0 = ioapic_read_entry(apic, pin);
2788
	clear_IO_APIC_pin(apic, pin);
L
Linus Torvalds 已提交
2789 2790 2791 2792 2793

	memset(&entry1, 0, sizeof(entry1));

	entry1.dest_mode = 0;			/* physical delivery */
	entry1.mask = 0;			/* unmask IRQ now */
2794
	entry1.dest = hard_smp_processor_id();
L
Linus Torvalds 已提交
2795 2796 2797 2798 2799
	entry1.delivery_mode = dest_ExtINT;
	entry1.polarity = entry0.polarity;
	entry1.trigger = 0;
	entry1.vector = 0;

2800
	ioapic_write_entry(apic, pin, entry1);
L
Linus Torvalds 已提交
2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816

	save_control = CMOS_READ(RTC_CONTROL);
	save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
	CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
		   RTC_FREQ_SELECT);
	CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);

	i = 100;
	while (i-- > 0) {
		mdelay(10);
		if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
			i -= 10;
	}

	CMOS_WRITE(save_control, RTC_CONTROL);
	CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2817
	clear_IO_APIC_pin(apic, pin);
L
Linus Torvalds 已提交
2818

2819
	ioapic_write_entry(apic, pin, entry0);
L
Linus Torvalds 已提交
2820 2821
}

Y
Yinghai Lu 已提交
2822
static int disable_timer_pin_1 __initdata;
2823
/* Actually the next is obsolete, but keep it for paranoid reasons -AK */
2824
static int __init disable_timer_pin_setup(char *arg)
Y
Yinghai Lu 已提交
2825 2826 2827 2828
{
	disable_timer_pin_1 = 1;
	return 0;
}
2829
early_param("disable_timer_pin_1", disable_timer_pin_setup);
Y
Yinghai Lu 已提交
2830 2831 2832

int timer_through_8259 __initdata;

L
Linus Torvalds 已提交
2833 2834 2835 2836 2837
/*
 * This code may look a bit paranoid, but it's supposed to cooperate with
 * a wide range of boards and BIOS bugs.  Fortunately only the timer IRQ
 * is so screwy.  Thanks to Brian Perkins for testing/hacking this beast
 * fanatically on his truly buggy board.
2838 2839
 *
 * FIXME: really need to revamp this for all platforms.
L
Linus Torvalds 已提交
2840
 */
2841
static inline void __init check_timer(void)
L
Linus Torvalds 已提交
2842
{
Y
Yinghai Lu 已提交
2843 2844
	struct irq_desc *desc = irq_to_desc(0);
	struct irq_cfg *cfg = desc->chip_data;
2845
	int node = cpu_to_node(boot_cpu_id);
2846
	int apic1, pin1, apic2, pin2;
2847
	unsigned long flags;
2848
	int no_pin1 = 0;
2849 2850

	local_irq_save(flags);
2851

L
Linus Torvalds 已提交
2852 2853 2854 2855
	/*
	 * get/set the timer IRQ vector:
	 */
	disable_8259A_irq(0);
2856
	assign_irq_vector(0, cfg, apic->target_cpus());
L
Linus Torvalds 已提交
2857 2858

	/*
2859 2860 2861 2862 2863 2864 2865
	 * As IRQ0 is to be enabled in the 8259A, the virtual
	 * wire has to be disabled in the local APIC.  Also
	 * timer interrupts need to be acknowledged manually in
	 * the 8259A for the i82489DX when using the NMI
	 * watchdog as that APIC treats NMIs as level-triggered.
	 * The AEOI mode will finish them in the 8259A
	 * automatically.
L
Linus Torvalds 已提交
2866
	 */
2867
	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
L
Linus Torvalds 已提交
2868
	init_8259A(1);
2869
#ifdef CONFIG_X86_32
Y
Yinghai Lu 已提交
2870 2871 2872 2873 2874 2875 2876
	{
		unsigned int ver;

		ver = apic_read(APIC_LVR);
		ver = GET_APIC_VERSION(ver);
		timer_ack = (nmi_watchdog == NMI_IO_APIC && !APIC_INTEGRATED(ver));
	}
2877
#endif
L
Linus Torvalds 已提交
2878

2879 2880 2881 2882
	pin1  = find_isa_irq_pin(0, mp_INT);
	apic1 = find_isa_irq_apic(0, mp_INT);
	pin2  = ioapic_i8259.pin;
	apic2 = ioapic_i8259.apic;
L
Linus Torvalds 已提交
2883

2884 2885
	apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
		    "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2886
		    cfg->vector, apic1, pin1, apic2, pin2);
L
Linus Torvalds 已提交
2887

2888 2889 2890 2891 2892 2893 2894 2895
	/*
	 * Some BIOS writers are clueless and report the ExtINTA
	 * I/O APIC input from the cascaded 8259A as the timer
	 * interrupt input.  So just in case, if only one pin
	 * was found above, try it both directly and through the
	 * 8259A.
	 */
	if (pin1 == -1) {
2896 2897
		if (intr_remapping_enabled)
			panic("BIOS bug: timer not connected to IO-APIC");
2898 2899 2900 2901 2902 2903 2904 2905
		pin1 = pin2;
		apic1 = apic2;
		no_pin1 = 1;
	} else if (pin2 == -1) {
		pin2 = pin1;
		apic2 = apic1;
	}

L
Linus Torvalds 已提交
2906 2907 2908 2909
	if (pin1 != -1) {
		/*
		 * Ok, does IRQ0 through the IOAPIC work?
		 */
2910
		if (no_pin1) {
2911
			add_pin_to_irq_node(cfg, node, apic1, pin1);
2912
			setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
Y
Yinghai Lu 已提交
2913 2914 2915 2916 2917 2918 2919 2920 2921 2922
		} else {
			/* for edge trigger, setup_IO_APIC_irq already
			 * leave it unmasked.
			 * so only need to unmask if it is level-trigger
			 * do we really have level trigger timer?
			 */
			int idx;
			idx = find_irq_entry(apic1, pin1, mp_INT);
			if (idx != -1 && irq_trigger(idx))
				unmask_IO_APIC_irq_desc(desc);
2923
		}
L
Linus Torvalds 已提交
2924 2925 2926 2927 2928
		if (timer_irq_works()) {
			if (nmi_watchdog == NMI_IO_APIC) {
				setup_nmi();
				enable_8259A_irq(0);
			}
2929 2930
			if (disable_timer_pin_1 > 0)
				clear_IO_APIC_pin(0, pin1);
2931
			goto out;
L
Linus Torvalds 已提交
2932
		}
2933 2934
		if (intr_remapping_enabled)
			panic("timer doesn't work through Interrupt-remapped IO-APIC");
Y
Yinghai Lu 已提交
2935
		local_irq_disable();
2936
		clear_IO_APIC_pin(apic1, pin1);
2937
		if (!no_pin1)
2938 2939
			apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
				    "8254 timer not connected to IO-APIC\n");
L
Linus Torvalds 已提交
2940

2941 2942 2943 2944
		apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
			    "(IRQ0) through the 8259A ...\n");
		apic_printk(APIC_QUIET, KERN_INFO
			    "..... (found apic %d pin %d) ...\n", apic2, pin2);
L
Linus Torvalds 已提交
2945 2946 2947
		/*
		 * legacy devices should be connected to IO APIC #0
		 */
2948
		replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
2949
		setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2950
		enable_8259A_irq(0);
L
Linus Torvalds 已提交
2951
		if (timer_irq_works()) {
2952
			apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2953
			timer_through_8259 = 1;
L
Linus Torvalds 已提交
2954
			if (nmi_watchdog == NMI_IO_APIC) {
2955
				disable_8259A_irq(0);
L
Linus Torvalds 已提交
2956
				setup_nmi();
2957
				enable_8259A_irq(0);
L
Linus Torvalds 已提交
2958
			}
2959
			goto out;
L
Linus Torvalds 已提交
2960 2961 2962 2963
		}
		/*
		 * Cleanup, just in case ...
		 */
Y
Yinghai Lu 已提交
2964
		local_irq_disable();
2965
		disable_8259A_irq(0);
2966
		clear_IO_APIC_pin(apic2, pin2);
2967
		apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
L
Linus Torvalds 已提交
2968 2969 2970
	}

	if (nmi_watchdog == NMI_IO_APIC) {
2971 2972
		apic_printk(APIC_QUIET, KERN_WARNING "timer doesn't work "
			    "through the IO-APIC - disabling NMI Watchdog!\n");
2973
		nmi_watchdog = NMI_NONE;
L
Linus Torvalds 已提交
2974
	}
2975
#ifdef CONFIG_X86_32
2976
	timer_ack = 0;
2977
#endif
L
Linus Torvalds 已提交
2978

2979 2980
	apic_printk(APIC_QUIET, KERN_INFO
		    "...trying to set up timer as Virtual Wire IRQ...\n");
L
Linus Torvalds 已提交
2981

Y
Yinghai Lu 已提交
2982
	lapic_register_intr(0, desc);
2983
	apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);	/* Fixed mode */
L
Linus Torvalds 已提交
2984 2985 2986
	enable_8259A_irq(0);

	if (timer_irq_works()) {
2987
		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2988
		goto out;
L
Linus Torvalds 已提交
2989
	}
Y
Yinghai Lu 已提交
2990
	local_irq_disable();
2991
	disable_8259A_irq(0);
2992
	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2993
	apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
L
Linus Torvalds 已提交
2994

2995 2996
	apic_printk(APIC_QUIET, KERN_INFO
		    "...trying to set up timer as ExtINT IRQ...\n");
L
Linus Torvalds 已提交
2997 2998 2999

	init_8259A(0);
	make_8259A_irq(0);
3000
	apic_write(APIC_LVT0, APIC_DM_EXTINT);
L
Linus Torvalds 已提交
3001 3002 3003 3004

	unlock_ExtINT_logic();

	if (timer_irq_works()) {
3005
		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
3006
		goto out;
L
Linus Torvalds 已提交
3007
	}
Y
Yinghai Lu 已提交
3008
	local_irq_disable();
3009
	apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
L
Linus Torvalds 已提交
3010
	panic("IO-APIC + timer doesn't work!  Boot with apic=debug and send a "
3011
		"report.  Then try booting with the 'noapic' option.\n");
3012 3013
out:
	local_irq_restore(flags);
L
Linus Torvalds 已提交
3014 3015 3016
}

/*
3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031
 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
 * to devices.  However there may be an I/O APIC pin available for
 * this interrupt regardless.  The pin may be left unconnected, but
 * typically it will be reused as an ExtINT cascade interrupt for
 * the master 8259A.  In the MPS case such a pin will normally be
 * reported as an ExtINT interrupt in the MP table.  With ACPI
 * there is no provision for ExtINT interrupts, and in the absence
 * of an override it would be treated as an ordinary ISA I/O APIC
 * interrupt, that is edge-triggered and unmasked by default.  We
 * used to do this, but it caused problems on some systems because
 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
 * the same ExtINT cascade interrupt to drive the local APIC of the
 * bootstrap processor.  Therefore we refrain from routing IRQ2 to
 * the I/O APIC in all cases now.  No actual device should request
 * it anyway.  --macro
L
Linus Torvalds 已提交
3032 3033 3034 3035 3036
 */
#define PIC_IRQS	(1 << PIC_CASCADE_IR)

void __init setup_IO_APIC(void)
{
3037 3038 3039 3040

	/*
	 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
	 */
L
Linus Torvalds 已提交
3041

3042
	io_apic_irqs = ~PIC_IRQS;
L
Linus Torvalds 已提交
3043

3044
	apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
T
Thomas Gleixner 已提交
3045
	/*
3046 3047 3048
         * Set up IO-APIC IRQ routing.
         */
#ifdef CONFIG_X86_32
T
Thomas Gleixner 已提交
3049 3050
	if (!acpi_ioapic)
		setup_ioapic_ids_from_mpc();
3051
#endif
L
Linus Torvalds 已提交
3052 3053 3054
	sync_Arb_IDs();
	setup_IO_APIC_irqs();
	init_IO_APIC_traps();
3055
	check_timer();
L
Linus Torvalds 已提交
3056 3057 3058
}

/*
3059 3060
 *      Called after all the initialization is done. If we didnt find any
 *      APIC bugs then we can allow the modify fast path
L
Linus Torvalds 已提交
3061
 */
3062

L
Linus Torvalds 已提交
3063 3064
static int __init io_apic_bug_finalize(void)
{
T
Thomas Gleixner 已提交
3065 3066 3067
	if (sis_apic_bug == -1)
		sis_apic_bug = 0;
	return 0;
L
Linus Torvalds 已提交
3068 3069 3070 3071 3072 3073 3074 3075
}

late_initcall(io_apic_bug_finalize);

struct sysfs_ioapic_data {
	struct sys_device dev;
	struct IO_APIC_route_entry entry[0];
};
3076
static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS];
L
Linus Torvalds 已提交
3077

3078
static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
L
Linus Torvalds 已提交
3079 3080 3081 3082
{
	struct IO_APIC_route_entry *entry;
	struct sysfs_ioapic_data *data;
	int i;
3083

L
Linus Torvalds 已提交
3084 3085
	data = container_of(dev, struct sysfs_ioapic_data, dev);
	entry = data->entry;
3086 3087
	for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ )
		*entry = ioapic_read_entry(dev->id, i);
L
Linus Torvalds 已提交
3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098

	return 0;
}

static int ioapic_resume(struct sys_device *dev)
{
	struct IO_APIC_route_entry *entry;
	struct sysfs_ioapic_data *data;
	unsigned long flags;
	union IO_APIC_reg_00 reg_00;
	int i;
3099

L
Linus Torvalds 已提交
3100 3101 3102 3103 3104
	data = container_of(dev, struct sysfs_ioapic_data, dev);
	entry = data->entry;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(dev->id, 0);
3105 3106
	if (reg_00.bits.ID != mp_ioapics[dev->id].apicid) {
		reg_00.bits.ID = mp_ioapics[dev->id].apicid;
L
Linus Torvalds 已提交
3107 3108 3109
		io_apic_write(dev->id, 0, reg_00.raw);
	}
	spin_unlock_irqrestore(&ioapic_lock, flags);
3110
	for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
3111
		ioapic_write_entry(dev->id, i, entry[i]);
L
Linus Torvalds 已提交
3112 3113 3114 3115 3116

	return 0;
}

static struct sysdev_class ioapic_sysdev_class = {
3117
	.name = "ioapic",
L
Linus Torvalds 已提交
3118 3119 3120 3121 3122 3123
	.suspend = ioapic_suspend,
	.resume = ioapic_resume,
};

static int __init ioapic_init_sysfs(void)
{
3124 3125
	struct sys_device * dev;
	int i, size, error;
L
Linus Torvalds 已提交
3126 3127 3128 3129 3130

	error = sysdev_class_register(&ioapic_sysdev_class);
	if (error)
		return error;

3131
	for (i = 0; i < nr_ioapics; i++ ) {
3132
		size = sizeof(struct sys_device) + nr_ioapic_registers[i]
L
Linus Torvalds 已提交
3133
			* sizeof(struct IO_APIC_route_entry);
3134
		mp_ioapic_data[i] = kzalloc(size, GFP_KERNEL);
L
Linus Torvalds 已提交
3135 3136 3137 3138 3139
		if (!mp_ioapic_data[i]) {
			printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
			continue;
		}
		dev = &mp_ioapic_data[i]->dev;
3140
		dev->id = i;
L
Linus Torvalds 已提交
3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155
		dev->cls = &ioapic_sysdev_class;
		error = sysdev_register(dev);
		if (error) {
			kfree(mp_ioapic_data[i]);
			mp_ioapic_data[i] = NULL;
			printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
			continue;
		}
	}

	return 0;
}

device_initcall(ioapic_init_sysfs);

3156
static int nr_irqs_gsi = NR_IRQS_LEGACY;
3157
/*
3158
 * Dynamic irq allocate and deallocation
3159
 */
3160
unsigned int create_irq_nr(unsigned int irq_want, int node)
3161
{
3162
	/* Allocate an unused irq */
3163 3164
	unsigned int irq;
	unsigned int new;
3165
	unsigned long flags;
3166 3167
	struct irq_cfg *cfg_new = NULL;
	struct irq_desc *desc_new = NULL;
Y
Yinghai Lu 已提交
3168 3169

	irq = 0;
3170 3171 3172
	if (irq_want < nr_irqs_gsi)
		irq_want = nr_irqs_gsi;

3173
	spin_lock_irqsave(&vector_lock, flags);
3174
	for (new = irq_want; new < nr_irqs; new++) {
3175
		desc_new = irq_to_desc_alloc_node(new, node);
3176 3177
		if (!desc_new) {
			printk(KERN_INFO "can not get irq_desc for %d\n", new);
3178
			continue;
3179 3180 3181 3182
		}
		cfg_new = desc_new->chip_data;

		if (cfg_new->vector != 0)
3183
			continue;
3184

3185
		desc_new = move_irq_desc(desc_new, node);
3186

3187
		if (__assign_irq_vector(new, cfg_new, apic->target_cpus()) == 0)
3188 3189 3190 3191
			irq = new;
		break;
	}
	spin_unlock_irqrestore(&vector_lock, flags);
3192

Y
Yinghai Lu 已提交
3193
	if (irq > 0) {
3194
		dynamic_irq_init(irq);
3195 3196 3197
		/* restore it, in case dynamic_irq_init clear it */
		if (desc_new)
			desc_new->chip_data = cfg_new;
3198 3199 3200 3201
	}
	return irq;
}

Y
Yinghai Lu 已提交
3202 3203
int create_irq(void)
{
3204
	int node = cpu_to_node(boot_cpu_id);
3205
	unsigned int irq_want;
3206 3207
	int irq;

3208
	irq_want = nr_irqs_gsi;
3209
	irq = create_irq_nr(irq_want, node);
3210 3211 3212 3213 3214

	if (irq == 0)
		irq = -1;

	return irq;
Y
Yinghai Lu 已提交
3215 3216
}

3217 3218 3219
void destroy_irq(unsigned int irq)
{
	unsigned long flags;
3220 3221
	struct irq_cfg *cfg;
	struct irq_desc *desc;
3222

3223 3224 3225
	/* store it, in case dynamic_irq_cleanup clear it */
	desc = irq_to_desc(irq);
	cfg = desc->chip_data;
3226
	dynamic_irq_cleanup(irq);
3227 3228 3229
	/* connect back irq_cfg */
	if (desc)
		desc->chip_data = cfg;
3230

3231
	free_irte(irq);
3232
	spin_lock_irqsave(&vector_lock, flags);
Y
Yinghai Lu 已提交
3233
	__clear_irq_vector(irq, cfg);
3234 3235 3236
	spin_unlock_irqrestore(&vector_lock, flags);
}

3237
/*
S
Simon Arlott 已提交
3238
 * MSI message composition
3239 3240
 */
#ifdef CONFIG_PCI_MSI
3241
static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
3242
{
3243 3244
	struct irq_cfg *cfg;
	int err;
3245 3246
	unsigned dest;

J
Jan Beulich 已提交
3247 3248 3249
	if (disable_apic)
		return -ENXIO;

Y
Yinghai Lu 已提交
3250
	cfg = irq_cfg(irq);
3251
	err = assign_irq_vector(irq, cfg, apic->target_cpus());
3252 3253
	if (err)
		return err;
3254

3255
	dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
3256

3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267
	if (irq_remapped(irq)) {
		struct irte irte;
		int ir_index;
		u16 sub_handle;

		ir_index = map_irq_to_irte_handle(irq, &sub_handle);
		BUG_ON(ir_index == -1);

		memset (&irte, 0, sizeof(irte));

		irte.present = 1;
3268
		irte.dst_mode = apic->irq_dest_mode;
3269
		irte.trigger_mode = 0; /* edge */
3270
		irte.dlvry_mode = apic->irq_delivery_mode;
3271 3272 3273
		irte.vector = cfg->vector;
		irte.dest_id = IRTE_DEST(dest);

3274 3275 3276
		/* Set source-id of interrupt request */
		set_msi_sid(&irte, pdev);

3277 3278 3279 3280 3281 3282 3283 3284
		modify_irte(irq, &irte);

		msg->address_hi = MSI_ADDR_BASE_HI;
		msg->data = sub_handle;
		msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
				  MSI_ADDR_IR_SHV |
				  MSI_ADDR_IR_INDEX1(ir_index) |
				  MSI_ADDR_IR_INDEX2(ir_index);
3285
	} else {
3286 3287 3288 3289 3290 3291
		if (x2apic_enabled())
			msg->address_hi = MSI_ADDR_BASE_HI |
					  MSI_ADDR_EXT_DEST_ID(dest);
		else
			msg->address_hi = MSI_ADDR_BASE_HI;

3292 3293
		msg->address_lo =
			MSI_ADDR_BASE_LO |
3294
			((apic->irq_dest_mode == 0) ?
3295 3296
				MSI_ADDR_DEST_MODE_PHYSICAL:
				MSI_ADDR_DEST_MODE_LOGICAL) |
3297
			((apic->irq_delivery_mode != dest_LowestPrio) ?
3298 3299 3300
				MSI_ADDR_REDIRECTION_CPU:
				MSI_ADDR_REDIRECTION_LOWPRI) |
			MSI_ADDR_DEST_ID(dest);
3301

3302 3303 3304
		msg->data =
			MSI_DATA_TRIGGER_EDGE |
			MSI_DATA_LEVEL_ASSERT |
3305
			((apic->irq_delivery_mode != dest_LowestPrio) ?
3306 3307 3308 3309
				MSI_DATA_DELIVERY_FIXED:
				MSI_DATA_DELIVERY_LOWPRI) |
			MSI_DATA_VECTOR(cfg->vector);
	}
3310
	return err;
3311 3312
}

3313
#ifdef CONFIG_SMP
3314
static int set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3315
{
Y
Yinghai Lu 已提交
3316
	struct irq_desc *desc = irq_to_desc(irq);
3317
	struct irq_cfg *cfg;
3318 3319 3320
	struct msi_msg msg;
	unsigned int dest;

3321 3322
	dest = set_desc_affinity(desc, mask);
	if (dest == BAD_APICID)
3323
		return -1;
3324

Y
Yinghai Lu 已提交
3325
	cfg = desc->chip_data;
3326

Y
Yinghai Lu 已提交
3327
	read_msi_msg_desc(desc, &msg);
3328 3329

	msg.data &= ~MSI_DATA_VECTOR_MASK;
3330
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
3331 3332 3333
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

Y
Yinghai Lu 已提交
3334
	write_msi_msg_desc(desc, &msg);
3335 3336

	return 0;
3337
}
3338 3339 3340 3341 3342
#ifdef CONFIG_INTR_REMAP
/*
 * Migrate the MSI irq to another cpumask. This migration is
 * done in the process context using interrupt-remapping hardware.
 */
3343
static int
3344
ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3345
{
Y
Yinghai Lu 已提交
3346
	struct irq_desc *desc = irq_to_desc(irq);
3347
	struct irq_cfg *cfg = desc->chip_data;
3348 3349 3350 3351
	unsigned int dest;
	struct irte irte;

	if (get_irte(irq, &irte))
3352
		return -1;
3353

3354 3355
	dest = set_desc_affinity(desc, mask);
	if (dest == BAD_APICID)
3356
		return -1;
3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370

	irte.vector = cfg->vector;
	irte.dest_id = IRTE_DEST(dest);

	/*
	 * atomically update the IRTE with the new destination and vector.
	 */
	modify_irte(irq, &irte);

	/*
	 * After this point, all the interrupts will start arriving
	 * at the new destination. So, time to cleanup the previous
	 * vector allocation.
	 */
3371 3372
	if (cfg->move_in_progress)
		send_cleanup_vector(cfg);
3373 3374

	return 0;
3375
}
Y
Yinghai Lu 已提交
3376

3377
#endif
3378
#endif /* CONFIG_SMP */
3379

3380 3381 3382 3383 3384 3385 3386 3387
/*
 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
 * which implement the MSI or MSI-X Capability Structure.
 */
static struct irq_chip msi_chip = {
	.name		= "PCI-MSI",
	.unmask		= unmask_msi_irq,
	.mask		= mask_msi_irq,
3388
	.ack		= ack_apic_edge,
3389 3390 3391 3392
#ifdef CONFIG_SMP
	.set_affinity	= set_msi_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
3393 3394
};

3395 3396 3397 3398
static struct irq_chip msi_ir_chip = {
	.name		= "IR-PCI-MSI",
	.unmask		= unmask_msi_irq,
	.mask		= mask_msi_irq,
3399
#ifdef CONFIG_INTR_REMAP
3400
	.ack		= ir_ack_apic_edge,
3401 3402
#ifdef CONFIG_SMP
	.set_affinity	= ir_set_msi_irq_affinity,
3403
#endif
3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428
#endif
	.retrigger	= ioapic_retrigger_irq,
};

/*
 * Map the PCI dev to the corresponding remapping hardware unit
 * and allocate 'nvec' consecutive interrupt-remapping table entries
 * in it.
 */
static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
{
	struct intel_iommu *iommu;
	int index;

	iommu = map_dev_to_ir(dev);
	if (!iommu) {
		printk(KERN_ERR
		       "Unable to map PCI %s to iommu\n", pci_name(dev));
		return -ENOENT;
	}

	index = alloc_irte(iommu, irq, nvec);
	if (index < 0) {
		printk(KERN_ERR
		       "Unable to allocate %d IRTE for PCI %s\n", nvec,
T
Thomas Gleixner 已提交
3429
		       pci_name(dev));
3430 3431 3432 3433
		return -ENOSPC;
	}
	return index;
}
3434

Y
Yinghai Lu 已提交
3435
static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
3436 3437 3438 3439 3440 3441 3442 3443
{
	int ret;
	struct msi_msg msg;

	ret = msi_compose_msg(dev, irq, &msg);
	if (ret < 0)
		return ret;

Y
Yinghai Lu 已提交
3444
	set_irq_msi(irq, msidesc);
3445 3446
	write_msi_msg(irq, &msg);

3447 3448 3449 3450 3451 3452 3453 3454 3455
	if (irq_remapped(irq)) {
		struct irq_desc *desc = irq_to_desc(irq);
		/*
		 * irq migration in process context
		 */
		desc->status |= IRQ_MOVE_PCNTXT;
		set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge");
	} else
		set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");
3456

Y
Yinghai Lu 已提交
3457 3458
	dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq);

3459 3460 3461
	return 0;
}

3462 3463
int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
{
3464 3465
	unsigned int irq;
	int ret, sub_handle;
3466
	struct msi_desc *msidesc;
3467
	unsigned int irq_want;
3468
	struct intel_iommu *iommu = NULL;
3469
	int index = 0;
3470
	int node;
3471

3472 3473 3474 3475
	/* x86 doesn't support multiple MSI yet */
	if (type == PCI_CAP_ID_MSI && nvec > 1)
		return 1;

3476
	node = dev_to_node(&dev->dev);
3477
	irq_want = nr_irqs_gsi;
3478
	sub_handle = 0;
3479
	list_for_each_entry(msidesc, &dev->msi_list, list) {
3480
		irq = create_irq_nr(irq_want, node);
3481 3482
		if (irq == 0)
			return -1;
Y
Yinghai Lu 已提交
3483
		irq_want = irq + 1;
3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510
		if (!intr_remapping_enabled)
			goto no_ir;

		if (!sub_handle) {
			/*
			 * allocate the consecutive block of IRTE's
			 * for 'nvec'
			 */
			index = msi_alloc_irte(dev, irq, nvec);
			if (index < 0) {
				ret = index;
				goto error;
			}
		} else {
			iommu = map_dev_to_ir(dev);
			if (!iommu) {
				ret = -ENOENT;
				goto error;
			}
			/*
			 * setup the mapping between the irq and the IRTE
			 * base index, the sub_handle pointing to the
			 * appropriate interrupt remap table entry.
			 */
			set_irte_irq(irq, iommu, index, sub_handle);
		}
no_ir:
3511
		ret = setup_msi_irq(dev, msidesc, irq);
3512 3513 3514 3515 3516
		if (ret < 0)
			goto error;
		sub_handle++;
	}
	return 0;
3517 3518

error:
3519 3520
	destroy_irq(irq);
	return ret;
3521 3522
}

3523 3524
void arch_teardown_msi_irq(unsigned int irq)
{
3525
	destroy_irq(irq);
3526 3527
}

3528
#if defined (CONFIG_DMAR) || defined (CONFIG_INTR_REMAP)
3529
#ifdef CONFIG_SMP
3530
static int dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3531
{
Y
Yinghai Lu 已提交
3532
	struct irq_desc *desc = irq_to_desc(irq);
3533 3534 3535 3536
	struct irq_cfg *cfg;
	struct msi_msg msg;
	unsigned int dest;

3537 3538
	dest = set_desc_affinity(desc, mask);
	if (dest == BAD_APICID)
3539
		return -1;
3540

Y
Yinghai Lu 已提交
3541
	cfg = desc->chip_data;
3542 3543 3544 3545 3546 3547 3548 3549 3550

	dmar_msi_read(irq, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	dmar_msi_write(irq, &msg);
3551 3552

	return 0;
3553
}
Y
Yinghai Lu 已提交
3554

3555 3556
#endif /* CONFIG_SMP */

3557
static struct irq_chip dmar_msi_type = {
3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571
	.name = "DMAR_MSI",
	.unmask = dmar_msi_unmask,
	.mask = dmar_msi_mask,
	.ack = ack_apic_edge,
#ifdef CONFIG_SMP
	.set_affinity = dmar_msi_set_affinity,
#endif
	.retrigger = ioapic_retrigger_irq,
};

int arch_setup_dmar_msi(unsigned int irq)
{
	int ret;
	struct msi_msg msg;
3572

3573 3574 3575 3576 3577 3578 3579 3580 3581 3582
	ret = msi_compose_msg(NULL, irq, &msg);
	if (ret < 0)
		return ret;
	dmar_msi_write(irq, &msg);
	set_irq_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
		"edge");
	return 0;
}
#endif

3583 3584 3585
#ifdef CONFIG_HPET_TIMER

#ifdef CONFIG_SMP
3586
static int hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3587
{
Y
Yinghai Lu 已提交
3588
	struct irq_desc *desc = irq_to_desc(irq);
3589 3590 3591 3592
	struct irq_cfg *cfg;
	struct msi_msg msg;
	unsigned int dest;

3593 3594
	dest = set_desc_affinity(desc, mask);
	if (dest == BAD_APICID)
3595
		return -1;
3596

Y
Yinghai Lu 已提交
3597
	cfg = desc->chip_data;
3598 3599 3600 3601 3602 3603 3604 3605 3606

	hpet_msi_read(irq, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	hpet_msi_write(irq, &msg);
3607 3608

	return 0;
3609
}
Y
Yinghai Lu 已提交
3610

3611 3612
#endif /* CONFIG_SMP */

3613
static struct irq_chip hpet_msi_type = {
3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627
	.name = "HPET_MSI",
	.unmask = hpet_msi_unmask,
	.mask = hpet_msi_mask,
	.ack = ack_apic_edge,
#ifdef CONFIG_SMP
	.set_affinity = hpet_msi_set_affinity,
#endif
	.retrigger = ioapic_retrigger_irq,
};

int arch_setup_hpet_msi(unsigned int irq)
{
	int ret;
	struct msi_msg msg;
3628
	struct irq_desc *desc = irq_to_desc(irq);
3629 3630 3631 3632 3633 3634

	ret = msi_compose_msg(NULL, irq, &msg);
	if (ret < 0)
		return ret;

	hpet_msi_write(irq, &msg);
3635
	desc->status |= IRQ_MOVE_PCNTXT;
3636 3637
	set_irq_chip_and_handler_name(irq, &hpet_msi_type, handle_edge_irq,
		"edge");
Y
Yinghai Lu 已提交
3638

3639 3640 3641 3642
	return 0;
}
#endif

3643
#endif /* CONFIG_PCI_MSI */
3644 3645 3646 3647 3648 3649 3650
/*
 * Hypertransport interrupt support
 */
#ifdef CONFIG_HT_IRQ

#ifdef CONFIG_SMP

3651
static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3652
{
3653 3654
	struct ht_irq_msg msg;
	fetch_ht_irq_msg(irq, &msg);
3655

3656
	msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
3657
	msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
3658

3659
	msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
3660
	msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
3661

3662
	write_ht_irq_msg(irq, &msg);
3663 3664
}

3665
static int set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask)
3666
{
Y
Yinghai Lu 已提交
3667
	struct irq_desc *desc = irq_to_desc(irq);
3668
	struct irq_cfg *cfg;
3669 3670
	unsigned int dest;

3671 3672
	dest = set_desc_affinity(desc, mask);
	if (dest == BAD_APICID)
3673
		return -1;
3674

Y
Yinghai Lu 已提交
3675
	cfg = desc->chip_data;
3676

3677
	target_ht_irq(irq, dest, cfg->vector);
3678 3679

	return 0;
3680
}
Y
Yinghai Lu 已提交
3681

3682 3683
#endif

3684
static struct irq_chip ht_irq_chip = {
3685 3686 3687
	.name		= "PCI-HT",
	.mask		= mask_ht_irq,
	.unmask		= unmask_ht_irq,
3688
	.ack		= ack_apic_edge,
3689 3690 3691 3692 3693 3694 3695 3696
#ifdef CONFIG_SMP
	.set_affinity	= set_ht_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
};

int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
{
3697 3698
	struct irq_cfg *cfg;
	int err;
3699

J
Jan Beulich 已提交
3700 3701 3702
	if (disable_apic)
		return -ENXIO;

Y
Yinghai Lu 已提交
3703
	cfg = irq_cfg(irq);
3704
	err = assign_irq_vector(irq, cfg, apic->target_cpus());
3705
	if (!err) {
3706
		struct ht_irq_msg msg;
3707 3708
		unsigned dest;

3709 3710
		dest = apic->cpu_mask_to_apicid_and(cfg->domain,
						    apic->target_cpus());
3711

3712
		msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
3713

3714 3715
		msg.address_lo =
			HT_IRQ_LOW_BASE |
3716
			HT_IRQ_LOW_DEST_ID(dest) |
3717
			HT_IRQ_LOW_VECTOR(cfg->vector) |
3718
			((apic->irq_dest_mode == 0) ?
3719 3720 3721
				HT_IRQ_LOW_DM_PHYSICAL :
				HT_IRQ_LOW_DM_LOGICAL) |
			HT_IRQ_LOW_RQEOI_EDGE |
3722
			((apic->irq_delivery_mode != dest_LowestPrio) ?
3723 3724 3725 3726
				HT_IRQ_LOW_MT_FIXED :
				HT_IRQ_LOW_MT_ARBITRATED) |
			HT_IRQ_LOW_IRQ_MASKED;

3727
		write_ht_irq_msg(irq, &msg);
3728

3729 3730
		set_irq_chip_and_handler_name(irq, &ht_irq_chip,
					      handle_edge_irq, "edge");
Y
Yinghai Lu 已提交
3731 3732

		dev_printk(KERN_DEBUG, &dev->dev, "irq %d for HT\n", irq);
3733
	}
3734
	return err;
3735 3736 3737
}
#endif /* CONFIG_HT_IRQ */

N
Nick Piggin 已提交
3738
#ifdef CONFIG_X86_UV
3739 3740 3741 3742 3743 3744 3745
/*
 * Re-target the irq to the specified CPU and enable the specified MMR located
 * on the specified blade to allow the sending of MSIs to the specified CPU.
 */
int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
		       unsigned long mmr_offset)
{
3746
	const struct cpumask *eligible_cpu = cpumask_of(cpu);
3747 3748 3749 3750 3751 3752 3753
	struct irq_cfg *cfg;
	int mmr_pnode;
	unsigned long mmr_value;
	struct uv_IO_APIC_route_entry *entry;
	unsigned long flags;
	int err;

3754 3755
	BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));

Y
Yinghai Lu 已提交
3756 3757
	cfg = irq_cfg(irq);

3758
	err = assign_irq_vector(irq, cfg, eligible_cpu);
3759 3760 3761 3762 3763 3764 3765 3766 3767 3768
	if (err != 0)
		return err;

	spin_lock_irqsave(&vector_lock, flags);
	set_irq_chip_and_handler_name(irq, &uv_irq_chip, handle_percpu_irq,
				      irq_name);
	spin_unlock_irqrestore(&vector_lock, flags);

	mmr_value = 0;
	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
3769 3770 3771 3772 3773 3774 3775
	entry->vector		= cfg->vector;
	entry->delivery_mode	= apic->irq_delivery_mode;
	entry->dest_mode	= apic->irq_dest_mode;
	entry->polarity		= 0;
	entry->trigger		= 0;
	entry->mask		= 0;
	entry->dest		= apic->cpu_mask_to_apicid(eligible_cpu);
3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792

	mmr_pnode = uv_blade_to_pnode(mmr_blade);
	uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);

	return irq;
}

/*
 * Disable the specified MMR located on the specified blade so that MSIs are
 * longer allowed to be sent.
 */
void arch_disable_uv_irq(int mmr_blade, unsigned long mmr_offset)
{
	unsigned long mmr_value;
	struct uv_IO_APIC_route_entry *entry;
	int mmr_pnode;

3793 3794
	BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));

3795 3796 3797 3798 3799 3800 3801 3802 3803
	mmr_value = 0;
	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
	entry->mask = 1;

	mmr_pnode = uv_blade_to_pnode(mmr_blade);
	uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
}
#endif /* CONFIG_X86_64 */

3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815
int __init io_apic_get_redir_entries (int ioapic)
{
	union IO_APIC_reg_01	reg_01;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_01.raw = io_apic_read(ioapic, 1);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return reg_01.bits.entries;
}

3816
void __init probe_nr_irqs_gsi(void)
3817
{
3818 3819
	int nr = 0;

3820 3821
	nr = acpi_probe_gsi();
	if (nr > nr_irqs_gsi) {
3822
		nr_irqs_gsi = nr;
3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835
	} else {
		/* for acpi=off or acpi is not compiled in */
		int idx;

		nr = 0;
		for (idx = 0; idx < nr_ioapics; idx++)
			nr += io_apic_get_redir_entries(idx) + 1;

		if (nr > nr_irqs_gsi)
			nr_irqs_gsi = nr;
	}

	printk(KERN_DEBUG "nr_irqs_gsi: %d\n", nr_irqs_gsi);
3836 3837
}

Y
Yinghai Lu 已提交
3838 3839 3840 3841 3842
#ifdef CONFIG_SPARSE_IRQ
int __init arch_probe_nr_irqs(void)
{
	int nr;

Y
Yinghai Lu 已提交
3843 3844
	if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
		nr_irqs = NR_VECTORS * nr_cpu_ids;
Y
Yinghai Lu 已提交
3845

Y
Yinghai Lu 已提交
3846 3847 3848 3849 3850 3851 3852 3853
	nr = nr_irqs_gsi + 8 * nr_cpu_ids;
#if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
	/*
	 * for MSI and HT dyn irq
	 */
	nr += nr_irqs_gsi * 16;
#endif
	if (nr < nr_irqs)
Y
Yinghai Lu 已提交
3854 3855 3856 3857 3858 3859
		nr_irqs = nr;

	return 0;
}
#endif

3860 3861
static int __io_apic_set_pci_routing(struct device *dev, int irq,
				struct io_apic_irq_attr *irq_attr)
3862 3863 3864 3865
{
	struct irq_desc *desc;
	struct irq_cfg *cfg;
	int node;
3866 3867
	int ioapic, pin;
	int trigger, polarity;
3868

3869
	ioapic = irq_attr->ioapic;
3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886
	if (!IO_APIC_IRQ(irq)) {
		apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
			ioapic);
		return -EINVAL;
	}

	if (dev)
		node = dev_to_node(dev);
	else
		node = cpu_to_node(boot_cpu_id);

	desc = irq_to_desc_alloc_node(irq, node);
	if (!desc) {
		printk(KERN_INFO "can not get irq_desc %d\n", irq);
		return 0;
	}

3887 3888 3889 3890
	pin = irq_attr->ioapic_pin;
	trigger = irq_attr->trigger;
	polarity = irq_attr->polarity;

3891 3892 3893 3894 3895 3896 3897 3898
	/*
	 * IRQs < 16 are already in the irq_2_pin[] map
	 */
	if (irq >= NR_IRQS_LEGACY) {
		cfg = desc->chip_data;
		add_pin_to_irq_node(cfg, node, ioapic, pin);
	}

3899
	setup_IO_APIC_irq(ioapic, pin, irq, desc, trigger, polarity);
3900 3901 3902 3903

	return 0;
}

3904 3905
int io_apic_set_pci_routing(struct device *dev, int irq,
				struct io_apic_irq_attr *irq_attr)
3906
{
3907
	int ioapic, pin;
3908 3909 3910 3911 3912
	/*
	 * Avoid pin reprogramming.  PRTs typically include entries
	 * with redundant pin->gsi mappings (but unique PCI devices);
	 * we only program the IOAPIC on the first.
	 */
3913 3914
	ioapic = irq_attr->ioapic;
	pin = irq_attr->ioapic_pin;
3915 3916 3917 3918 3919 3920 3921
	if (test_bit(pin, mp_ioapic_routing[ioapic].pin_programmed)) {
		pr_debug("Pin %d-%d already programmed\n",
			 mp_ioapics[ioapic].apicid, pin);
		return 0;
	}
	set_bit(pin, mp_ioapic_routing[ioapic].pin_programmed);

3922
	return __io_apic_set_pci_routing(dev, irq, irq_attr);
3923 3924
}

L
Linus Torvalds 已提交
3925
/* --------------------------------------------------------------------------
3926
                          ACPI-based IOAPIC Configuration
L
Linus Torvalds 已提交
3927 3928
   -------------------------------------------------------------------------- */

L
Len Brown 已提交
3929
#ifdef CONFIG_ACPI
L
Linus Torvalds 已提交
3930

3931
#ifdef CONFIG_X86_32
3932
int __init io_apic_get_unique_id(int ioapic, int apic_id)
L
Linus Torvalds 已提交
3933 3934 3935 3936 3937 3938 3939 3940
{
	union IO_APIC_reg_00 reg_00;
	static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
	physid_mask_t tmp;
	unsigned long flags;
	int i = 0;

	/*
3941 3942
	 * The P4 platform supports up to 256 APIC IDs on two separate APIC
	 * buses (one for LAPICs, one for IOAPICs), where predecessors only
L
Linus Torvalds 已提交
3943
	 * supports up to 16 on one shared APIC bus.
3944
	 *
L
Linus Torvalds 已提交
3945 3946 3947 3948 3949
	 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
	 *      advantage of new APIC bus architecture.
	 */

	if (physids_empty(apic_id_map))
3950
		apic_id_map = apic->ioapic_phys_id_map(phys_cpu_present_map);
L
Linus Torvalds 已提交
3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(ioapic, 0);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	if (apic_id >= get_physical_broadcast()) {
		printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
			"%d\n", ioapic, apic_id, reg_00.bits.ID);
		apic_id = reg_00.bits.ID;
	}

	/*
3963
	 * Every APIC in a system must have a unique ID or we get lots of nice
L
Linus Torvalds 已提交
3964 3965
	 * 'stuck on smp_invalidate_needed IPI wait' messages.
	 */
3966
	if (apic->check_apicid_used(apic_id_map, apic_id)) {
L
Linus Torvalds 已提交
3967 3968

		for (i = 0; i < get_physical_broadcast(); i++) {
3969
			if (!apic->check_apicid_used(apic_id_map, i))
L
Linus Torvalds 已提交
3970 3971 3972 3973 3974 3975 3976 3977 3978 3979
				break;
		}

		if (i == get_physical_broadcast())
			panic("Max apic_id exceeded!\n");

		printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
			"trying %d\n", ioapic, apic_id, i);

		apic_id = i;
3980
	}
L
Linus Torvalds 已提交
3981

3982
	tmp = apic->apicid_to_cpu_present(apic_id);
L
Linus Torvalds 已提交
3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993
	physids_or(apic_id_map, apic_id_map, tmp);

	if (reg_00.bits.ID != apic_id) {
		reg_00.bits.ID = apic_id;

		spin_lock_irqsave(&ioapic_lock, flags);
		io_apic_write(ioapic, 0, reg_00.raw);
		reg_00.raw = io_apic_read(ioapic, 0);
		spin_unlock_irqrestore(&ioapic_lock, flags);

		/* Sanity check */
3994 3995 3996 3997
		if (reg_00.bits.ID != apic_id) {
			printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
			return -1;
		}
L
Linus Torvalds 已提交
3998 3999 4000 4001 4002 4003 4004
	}

	apic_printk(APIC_VERBOSE, KERN_INFO
			"IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);

	return apic_id;
}
4005
#endif
L
Linus Torvalds 已提交
4006

4007
int __init io_apic_get_version(int ioapic)
L
Linus Torvalds 已提交
4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018
{
	union IO_APIC_reg_01	reg_01;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_01.raw = io_apic_read(ioapic, 1);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return reg_01.bits.version;
}

4019 4020 4021 4022 4023 4024 4025 4026
int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
{
	int i;

	if (skip_ioapic_setup)
		return -1;

	for (i = 0; i < mp_irq_entries; i++)
4027 4028
		if (mp_irqs[i].irqtype == mp_INT &&
		    mp_irqs[i].srcbusirq == bus_irq)
4029 4030 4031 4032 4033 4034 4035 4036 4037
			break;
	if (i >= mp_irq_entries)
		return -1;

	*trigger = irq_trigger(i);
	*polarity = irq_polarity(i);
	return 0;
}

L
Len Brown 已提交
4038
#endif /* CONFIG_ACPI */
4039

4040 4041 4042
/*
 * This function currently is only a helper for the i386 smp boot process where
 * we need to reprogram the ioredtbls to cater for the cpus which have come online
4043
 * so mask in all cases should simply be apic->target_cpus()
4044 4045 4046 4047
 */
#ifdef CONFIG_SMP
void __init setup_ioapic_dest(void)
{
4048
	int pin, ioapic = 0, irq, irq_entry;
4049
	struct irq_desc *desc;
4050
	const struct cpumask *mask;
4051 4052 4053 4054

	if (skip_ioapic_setup == 1)
		return;

4055 4056 4057 4058 4059 4060 4061
#ifdef CONFIG_ACPI
	if (!acpi_disabled && acpi_ioapic) {
		ioapic = mp_find_ioapic(0);
		if (ioapic < 0)
			ioapic = 0;
	}
#endif
4062

4063 4064 4065 4066 4067
	for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
		irq_entry = find_irq_entry(ioapic, pin, mp_INT);
		if (irq_entry == -1)
			continue;
		irq = pin_2_irq(irq_entry, ioapic, pin);
4068

4069
		desc = irq_to_desc(irq);
4070

4071 4072 4073 4074 4075 4076 4077 4078
		/*
		 * Honour affinities which have been set in early boot
		 */
		if (desc->status &
		    (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
			mask = desc->affinity;
		else
			mask = apic->target_cpus();
4079

4080 4081 4082 4083
		if (intr_remapping_enabled)
			set_ir_ioapic_affinity_irq_desc(desc, mask);
		else
			set_ioapic_affinity_irq_desc(desc, mask);
4084
	}
4085

4086 4087 4088
}
#endif

4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124
#define IOAPIC_RESOURCE_NAME_SIZE 11

static struct resource *ioapic_resources;

static struct resource * __init ioapic_setup_resources(void)
{
	unsigned long n;
	struct resource *res;
	char *mem;
	int i;

	if (nr_ioapics <= 0)
		return NULL;

	n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
	n *= nr_ioapics;

	mem = alloc_bootmem(n);
	res = (void *)mem;

	if (mem != NULL) {
		mem += sizeof(struct resource) * nr_ioapics;

		for (i = 0; i < nr_ioapics; i++) {
			res[i].name = mem;
			res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
			sprintf(mem,  "IOAPIC %u", i);
			mem += IOAPIC_RESOURCE_NAME_SIZE;
		}
	}

	ioapic_resources = res;

	return res;
}

4125 4126 4127
void __init ioapic_init_mappings(void)
{
	unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
4128
	struct resource *ioapic_res;
T
Thomas Gleixner 已提交
4129
	int i;
4130

4131
	ioapic_res = ioapic_setup_resources();
4132 4133
	for (i = 0; i < nr_ioapics; i++) {
		if (smp_found_config) {
4134
			ioapic_phys = mp_ioapics[i].apicaddr;
4135
#ifdef CONFIG_X86_32
T
Thomas Gleixner 已提交
4136 4137 4138 4139 4140 4141 4142 4143 4144
			if (!ioapic_phys) {
				printk(KERN_ERR
				       "WARNING: bogus zero IO-APIC "
				       "address found in MPTABLE, "
				       "disabling IO/APIC support!\n");
				smp_found_config = 0;
				skip_ioapic_setup = 1;
				goto fake_ioapic_page;
			}
4145
#endif
4146
		} else {
4147
#ifdef CONFIG_X86_32
4148
fake_ioapic_page:
4149
#endif
4150
			ioapic_phys = (unsigned long)
4151
				alloc_bootmem_pages(PAGE_SIZE);
4152 4153 4154
			ioapic_phys = __pa(ioapic_phys);
		}
		set_fixmap_nocache(idx, ioapic_phys);
4155 4156 4157
		apic_printk(APIC_VERBOSE,
			    "mapped IOAPIC to %08lx (%08lx)\n",
			    __fix_to_virt(idx), ioapic_phys);
4158
		idx++;
4159 4160 4161 4162 4163 4164

		if (ioapic_res != NULL) {
			ioapic_res->start = ioapic_phys;
			ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
			ioapic_res++;
		}
4165 4166 4167
	}
}

4168 4169 4170 4171 4172 4173
static int __init ioapic_insert_resources(void)
{
	int i;
	struct resource *r = ioapic_resources;

	if (!r) {
4174 4175 4176 4177 4178 4179
		if (nr_ioapics > 0) {
			printk(KERN_ERR
				"IO APIC resources couldn't be allocated.\n");
			return -1;
		}
		return 0;
4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192
	}

	for (i = 0; i < nr_ioapics; i++) {
		insert_resource(&iomem_resource, r);
		r++;
	}

	return 0;
}

/* Insert the IO APIC resources after PCI initialization has occured to handle
 * IO APICS that are mapped in on a BAR in PCI space. */
late_initcall(ioapic_insert_resources);