io_apic_64.c 77.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 *	Intel IO-APIC support for multi-Pentium hosts.
 *
 *	Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar, Hajnalka Szabo
 *
 *	Many thanks to Stig Venaas for trying out countless experimental
 *	patches and reporting/debugging problems patiently!
 *
 *	(c) 1999, Multiple IO-APIC support, developed by
 *	Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
 *      Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
 *	further tested and cleaned up by Zach Brown <zab@redhat.com>
 *	and Ingo Molnar <mingo@redhat.com>
 *
 *	Fixes
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
 *					thanks to Eric Gilmore
 *					and Rolf G. Tews
 *					for testing these extensively
 *	Paul Diefenbaugh	:	Added full ACPI support
 */

#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/sched.h>
28
#include <linux/pci.h>
L
Linus Torvalds 已提交
29
#include <linux/mc146818rtc.h>
30
#include <linux/compiler.h>
L
Linus Torvalds 已提交
31
#include <linux/acpi.h>
32
#include <linux/module.h>
L
Linus Torvalds 已提交
33
#include <linux/sysdev.h>
34
#include <linux/msi.h>
35
#include <linux/htirq.h>
36 37 38
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/jiffies.h>	/* time_after() */
39 40 41
#ifdef CONFIG_ACPI
#include <acpi/acpi_bus.h>
#endif
42
#include <linux/bootmem.h>
43
#include <linux/dmar.h>
L
Linus Torvalds 已提交
44

45
#include <asm/idle.h>
L
Linus Torvalds 已提交
46 47 48 49
#include <asm/io.h>
#include <asm/smp.h>
#include <asm/desc.h>
#include <asm/proto.h>
50
#include <asm/acpi.h>
51
#include <asm/dma.h>
52
#include <asm/timer.h>
53
#include <asm/i8259.h>
54
#include <asm/nmi.h>
55
#include <asm/msidef.h>
56
#include <asm/hypertransport.h>
57
#include <asm/setup.h>
58
#include <asm/irq_remapping.h>
L
Linus Torvalds 已提交
59

60
#include <mach_ipi.h>
61
#include <mach_apic.h>
62
#include <mach_apicdef.h>
63

64 65
#define __apicdebuginit(type) static type __init

Y
Yinghai Lu 已提交
66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
int ioapic_force;

int sis_apic_bug; /* not actually supported, dummy for compile */

static DEFINE_SPINLOCK(ioapic_lock);
static DEFINE_SPINLOCK(vector_lock);

int first_free_entry;
/*
 * Rough estimation of how many shared IRQs there are, can
 * be changed anytime.
 */
int pin_map_size;

/*
 * # of IRQ routing registers
 */
int nr_ioapic_registers[MAX_IO_APICS];

/* I/O APIC entries */
struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
int nr_ioapics;

/* MP IRQ source entries */
struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];

/* # of MP IRQ source entries */
int mp_irq_entries;

DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);

int skip_ioapic_setup;

static int __init parse_noapic(char *str)
{
	disable_ioapic_setup();
	return 0;
}
early_param("noapic", parse_noapic);


Y
Yinghai Lu 已提交
107
struct irq_cfg;
108
struct irq_pin_list;
109
struct irq_cfg {
Y
Yinghai Lu 已提交
110 111
	unsigned int irq;
	struct irq_cfg *next;
112
	struct irq_pin_list *irq_2_pin;
113
	cpumask_t domain;
114 115
	cpumask_t old_domain;
	unsigned move_cleanup_count;
116
	u8 vector;
117
	u8 move_in_progress : 1;
118 119 120
};

/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
Y
Yinghai Lu 已提交
121
static struct irq_cfg irq_cfg_legacy[] __initdata = {
Y
Yinghai Lu 已提交
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
	[0]  = { .irq =  0, .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR,  },
	[1]  = { .irq =  1, .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR,  },
	[2]  = { .irq =  2, .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR,  },
	[3]  = { .irq =  3, .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR,  },
	[4]  = { .irq =  4, .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR,  },
	[5]  = { .irq =  5, .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR,  },
	[6]  = { .irq =  6, .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR,  },
	[7]  = { .irq =  7, .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR,  },
	[8]  = { .irq =  8, .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR,  },
	[9]  = { .irq =  9, .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR,  },
	[10] = { .irq = 10, .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, },
	[11] = { .irq = 11, .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, },
	[12] = { .irq = 12, .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, },
	[13] = { .irq = 13, .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, },
	[14] = { .irq = 14, .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, },
	[15] = { .irq = 15, .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, },
138 139
};

Y
Yinghai Lu 已提交
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
static struct irq_cfg irq_cfg_init = { .irq =  -1U, };
/* need to be biger than size of irq_cfg_legacy */
static int nr_irq_cfg = 32;

static int __init parse_nr_irq_cfg(char *arg)
{
	if (arg) {
		nr_irq_cfg = simple_strtoul(arg, NULL, 0);
		if (nr_irq_cfg < 32)
			nr_irq_cfg = 32;
	}
	return 0;
}

early_param("nr_irq_cfg", parse_nr_irq_cfg);

static void init_one_irq_cfg(struct irq_cfg *cfg)
{
	memcpy(cfg, &irq_cfg_init, sizeof(struct irq_cfg));
}
Y
Yinghai Lu 已提交
160

161 162
static struct irq_cfg *irq_cfgx;
static struct irq_cfg *irq_cfgx_free;
Y
Yinghai Lu 已提交
163 164 165
static void __init init_work(void *data)
{
	struct dyn_array *da = data;
Y
Yinghai Lu 已提交
166
	struct irq_cfg *cfg;
167
	int legacy_count;
Y
Yinghai Lu 已提交
168
	int i;
Y
Yinghai Lu 已提交
169

Y
Yinghai Lu 已提交
170 171 172 173
	cfg = *da->name;

	memcpy(cfg, irq_cfg_legacy, sizeof(irq_cfg_legacy));

174 175
	legacy_count = sizeof(irq_cfg_legacy)/sizeof(irq_cfg_legacy[0]);
	for (i = legacy_count; i < *da->nr; i++)
Y
Yinghai Lu 已提交
176 177 178 179
		init_one_irq_cfg(&cfg[i]);

	for (i = 1; i < *da->nr; i++)
		cfg[i-1].next = &cfg[i];
180 181 182

	irq_cfgx_free = &irq_cfgx[legacy_count];
	irq_cfgx[legacy_count - 1].next = NULL;
Y
Yinghai Lu 已提交
183 184
}

185
#define for_each_irq_cfg(cfg)		\
186
	for (cfg = irq_cfgx; cfg; cfg = cfg->next)
187

Y
Yinghai Lu 已提交
188 189 190 191 192 193
DEFINE_DYN_ARRAY(irq_cfgx, sizeof(struct irq_cfg), nr_irq_cfg, PAGE_SIZE, init_work);

static struct irq_cfg *irq_cfg(unsigned int irq)
{
	struct irq_cfg *cfg;

194
	cfg = irq_cfgx;
Y
Yinghai Lu 已提交
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
	while (cfg) {
		if (cfg->irq == irq)
			return cfg;

		cfg = cfg->next;
	}

	return NULL;
}

static struct irq_cfg *irq_cfg_alloc(unsigned int irq)
{
	struct irq_cfg *cfg, *cfg_pri;
	int i;
	int count = 0;

211
	cfg_pri = cfg = irq_cfgx;
Y
Yinghai Lu 已提交
212 213 214 215 216 217 218 219 220
	while (cfg) {
		if (cfg->irq == irq)
			return cfg;

		cfg_pri = cfg;
		cfg = cfg->next;
		count++;
	}

221 222 223 224 225 226 227
	if (!irq_cfgx_free) {
		unsigned long phys;
		unsigned long total_bytes;
		/*
		 *  we run out of pre-allocate ones, allocate more
		 */
		printk(KERN_DEBUG "try to get more irq_cfg %d\n", nr_irq_cfg);
Y
Yinghai Lu 已提交
228

229 230 231 232 233
		total_bytes = sizeof(struct irq_cfg) * nr_irq_cfg;
		if (after_bootmem)
			cfg = kzalloc(total_bytes, GFP_ATOMIC);
		else
			cfg = __alloc_bootmem_nopanic(total_bytes, PAGE_SIZE, 0);
Y
Yinghai Lu 已提交
234

235 236
		if (!cfg)
			panic("please boot with nr_irq_cfg= %d\n", count * 2);
Y
Yinghai Lu 已提交
237

238 239
		phys = __pa(cfg);
		printk(KERN_DEBUG "irq_irq ==> [%#lx - %#lx]\n", phys, phys + total_bytes);
Y
Yinghai Lu 已提交
240

241 242
		for (i = 0; i < nr_irq_cfg; i++)
			init_one_irq_cfg(&cfg[i]);
Y
Yinghai Lu 已提交
243

244 245
		for (i = 1; i < nr_irq_cfg; i++)
			cfg[i-1].next = &cfg[i];
Y
Yinghai Lu 已提交
246

247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
		irq_cfgx_free = cfg;
	}

	cfg = irq_cfgx_free;
	irq_cfgx_free = irq_cfgx_free->next;
	cfg->next = NULL;
	if (cfg_pri)
		cfg_pri->next = cfg;
	else
		irq_cfgx = cfg;
	cfg->irq = irq;
	printk(KERN_DEBUG "found new irq_cfg for irq %d\n", cfg->irq);
#ifdef CONFIG_HAVE_SPARSE_IRQ_DEBUG
	{
		/* dump the results */
		struct irq_cfg *cfg;
		unsigned long phys;
		unsigned long bytes = sizeof(struct irq_cfg);

		printk(KERN_DEBUG "=========================== %d\n", irq);
		printk(KERN_DEBUG "irq_cfg dump after get that for %d\n", irq);
		for_each_irq_cfg(cfg) {
			phys = __pa(cfg);
			printk(KERN_DEBUG "irq_cfg %d ==> [%#lx - %#lx]\n", cfg->irq, phys, phys + bytes);
		}
		printk(KERN_DEBUG "===========================\n");
	}
#endif
Y
Yinghai Lu 已提交
275 276
	return cfg;
}
Y
Yinghai Lu 已提交
277

L
Linus Torvalds 已提交
278 279 280 281 282 283 284
/*
 * This is performance-critical, we want to do it O(1)
 *
 * the indexing order of this array favors 1:1 mappings
 * between pins and IRQs.
 */

285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
struct irq_pin_list {
	int apic, pin;
	struct irq_pin_list *next;
};

static struct irq_pin_list *irq_2_pin_head;
/* fill one page ? */
static int nr_irq_2_pin = 0x100;
static struct irq_pin_list *irq_2_pin_ptr;
static void __init irq_2_pin_init_work(void *data)
{
	struct dyn_array *da = data;
	struct irq_pin_list *pin;
	int i;

	pin = *da->name;

	for (i = 1; i < *da->nr; i++)
		pin[i-1].next = &pin[i];

	irq_2_pin_ptr = &pin[0];
}
DEFINE_DYN_ARRAY(irq_2_pin_head, sizeof(struct irq_pin_list), nr_irq_2_pin, PAGE_SIZE, irq_2_pin_init_work);

static struct irq_pin_list *get_one_free_irq_2_pin(void)
{
	struct irq_pin_list *pin;
	int i;

	pin = irq_2_pin_ptr;

	if (pin) {
		irq_2_pin_ptr = pin->next;
		pin->next = NULL;
		return pin;
	}

	/*
	 *  we run out of pre-allocate ones, allocate more
	 */
	printk(KERN_DEBUG "try to get more irq_2_pin %d\n", nr_irq_2_pin);

	if (after_bootmem)
		pin = kzalloc(sizeof(struct irq_pin_list)*nr_irq_2_pin,
				 GFP_ATOMIC);
	else
		pin = __alloc_bootmem_nopanic(sizeof(struct irq_pin_list) *
				nr_irq_2_pin, PAGE_SIZE, 0);

	if (!pin)
		panic("can not get more irq_2_pin\n");
Y
Yinghai Lu 已提交
336

337 338
	for (i = 1; i < nr_irq_2_pin; i++)
		pin[i-1].next = &pin[i];
Y
Yinghai Lu 已提交
339

340 341 342 343 344
	irq_2_pin_ptr = pin->next;
	pin->next = NULL;

	return pin;
}
L
Linus Torvalds 已提交
345

L
Linus Torvalds 已提交
346 347 348 349 350 351 352 353 354
struct io_apic {
	unsigned int index;
	unsigned int unused[3];
	unsigned int data;
};

static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
{
	return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
355
		+ (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
L
Linus Torvalds 已提交
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
}

static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	return readl(&io_apic->data);
}

static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	writel(value, &io_apic->data);
}

/*
 * Re-write a value: to be used for read-modify-write
 * cycles where the read already set up the index register.
 */
static inline void io_apic_modify(unsigned int apic, unsigned int value)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(value, &io_apic->data);
}

382
static bool io_apic_level_ack_pending(unsigned int irq)
383 384 385
{
	struct irq_pin_list *entry;
	unsigned long flags;
386
	struct irq_cfg *cfg = irq_cfg(irq);
387 388

	spin_lock_irqsave(&ioapic_lock, flags);
389
	entry = cfg->irq_2_pin;
390 391 392 393
	for (;;) {
		unsigned int reg;
		int pin;

394
		if (!entry)
395
			break;
396
		pin = entry->pin;
397 398
		reg = io_apic_read(entry->apic, 0x10 + pin*2);
		/* Is the remote IRR bit set? */
399
		if (reg & IO_APIC_REDIR_REMOTE_IRR) {
400 401 402
			spin_unlock_irqrestore(&ioapic_lock, flags);
			return true;
		}
403 404
		if (!entry->next)
			break;
405
		entry = entry->next;
406 407
	}
	spin_unlock_irqrestore(&ioapic_lock, flags);
408 409

	return false;
410 411
}

412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427
union entry_union {
	struct { u32 w1, w2; };
	struct IO_APIC_route_entry entry;
};

static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
{
	union entry_union eu;
	unsigned long flags;
	spin_lock_irqsave(&ioapic_lock, flags);
	eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
	eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
	spin_unlock_irqrestore(&ioapic_lock, flags);
	return eu.entry;
}

428 429 430 431 432 433
/*
 * When we write a new IO APIC routing entry, we need to write the high
 * word first! If the mask bit in the low word is clear, we will enable
 * the interrupt, and we need to make sure the entry is fully populated
 * before that happens.
 */
434 435
static void
__ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
436 437 438
{
	union entry_union eu;
	eu.entry = e;
439 440
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
441 442 443 444 445 446 447
}

static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
{
	unsigned long flags;
	spin_lock_irqsave(&ioapic_lock, flags);
	__ioapic_write_entry(apic, pin, e);
448 449 450 451 452 453 454 455 456 457 458 459 460
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

/*
 * When we mask an IO APIC routing entry, we need to write the low
 * word first, in order to set the mask bit before we change the
 * high bits!
 */
static void ioapic_mask_entry(int apic, int pin)
{
	unsigned long flags;
	union entry_union eu = { .entry.mask = 1 };

461 462 463 464 465 466
	spin_lock_irqsave(&ioapic_lock, flags);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

467
#ifdef CONFIG_SMP
468 469 470
static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector)
{
	int apic, pin;
471 472
	struct irq_cfg *cfg;
	struct irq_pin_list *entry;
473

474 475
	cfg = irq_cfg(irq);
	entry = cfg->irq_2_pin;
476 477
	for (;;) {
		unsigned int reg;
478 479 480 481

		if (!entry)
			break;

482 483
		apic = entry->apic;
		pin = entry->pin;
Y
Yinghai Lu 已提交
484
#ifdef CONFIG_INTR_REMAP
485 486 487 488 489 490
		/*
		 * With interrupt-remapping, destination information comes
		 * from interrupt-remapping table entry.
		 */
		if (!irq_remapped(irq))
			io_apic_write(apic, 0x11 + pin*2, dest);
Y
Yinghai Lu 已提交
491 492 493
#else
		io_apic_write(apic, 0x11 + pin*2, dest);
#endif
494
		reg = io_apic_read(apic, 0x10 + pin*2);
495
		reg &= ~IO_APIC_REDIR_VECTOR_MASK;
496 497 498 499
		reg |= vector;
		io_apic_modify(apic, reg);
		if (!entry->next)
			break;
500
		entry = entry->next;
501 502 503
	}
}

Y
Yinghai Lu 已提交
504 505
static int assign_irq_vector(int irq, cpumask_t mask);

506 507
static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
{
Y
Yinghai Lu 已提交
508
	struct irq_cfg *cfg = irq_cfg(irq);
509 510 511
	unsigned long flags;
	unsigned int dest;
	cpumask_t tmp;
512
	struct irq_desc *desc;
513 514 515

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
516
		return;
517

518
	if (assign_irq_vector(irq, mask))
519 520
		return;

521
	cpus_and(tmp, cfg->domain, mask);
522
	dest = cpu_mask_to_apicid(tmp);
523 524 525 526 527 528

	/*
	 * Only the high 8 bits are valid.
	 */
	dest = SET_APIC_LOGICAL_ID(dest);

529
	desc = irq_to_desc(irq);
530
	spin_lock_irqsave(&ioapic_lock, flags);
531
	__target_IO_APIC_irq(irq, dest, cfg->vector);
532
	desc->affinity = mask;
533 534 535 536
	spin_unlock_irqrestore(&ioapic_lock, flags);
}
#endif

L
Linus Torvalds 已提交
537 538 539 540 541 542 543
/*
 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
 * shared ISA-space IRQs, so we have to support them. We are super
 * fast in the common case, and fast for shared ISA-space IRQs.
 */
static void add_pin_to_irq(unsigned int irq, int apic, int pin)
{
544 545
	struct irq_cfg *cfg;
	struct irq_pin_list *entry;
L
Linus Torvalds 已提交
546

547 548 549 550 551 552 553 554 555 556 557
	/* first time to refer irq_cfg, so with new */
	cfg = irq_cfg_alloc(irq);
	entry = cfg->irq_2_pin;
	if (!entry) {
		entry = get_one_free_irq_2_pin();
		cfg->irq_2_pin = entry;
		entry->apic = apic;
		entry->pin = pin;
		printk(KERN_DEBUG " 0 add_pin_to_irq: irq %d --> apic %d pin %d\n", irq, apic, pin);
		return;
	}
Y
Yinghai Lu 已提交
558

559 560 561 562
	while (entry->next) {
		/* not again, please */
		if (entry->apic == apic && entry->pin == pin)
			return;
L
Linus Torvalds 已提交
563

564
		entry = entry->next;
L
Linus Torvalds 已提交
565
	}
566 567 568

	entry->next = get_one_free_irq_2_pin();
	entry = entry->next;
L
Linus Torvalds 已提交
569 570
	entry->apic = apic;
	entry->pin = pin;
571
	printk(KERN_DEBUG " x add_pin_to_irq: irq %d --> apic %d pin %d\n", irq, apic, pin);
L
Linus Torvalds 已提交
572 573
}

574 575 576 577 578 579 580
/*
 * Reroute an IRQ to a different pin.
 */
static void __init replace_pin_at_irq(unsigned int irq,
				      int oldapic, int oldpin,
				      int newapic, int newpin)
{
581 582 583
	struct irq_cfg *cfg = irq_cfg(irq);
	struct irq_pin_list *entry = cfg->irq_2_pin;
	int replaced = 0;
584

585
	while (entry) {
586 587 588
		if (entry->apic == oldapic && entry->pin == oldpin) {
			entry->apic = newapic;
			entry->pin = newpin;
589 590
			replaced = 1;
			/* every one is different, right? */
591
			break;
592 593
		}
		entry = entry->next;
594
	}
595 596 597 598

	/* why? call replace before add? */
	if (!replaced)
		add_pin_to_irq(irq, newapic, newpin);
599 600
}

601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
/*
 * Synchronize the IO-APIC and the CPU by doing
 * a dummy read from the IO-APIC
 */
static inline void io_apic_sync(unsigned int apic)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	readl(&io_apic->data);
}

#define __DO_ACTION(R, ACTION, FINAL)					\
									\
{									\
	int pin;							\
	struct irq_cfg *cfg;						\
	struct irq_pin_list *entry;					\
									\
	cfg = irq_cfg(irq);						\
	entry = cfg->irq_2_pin;						\
	for (;;) {							\
		unsigned int reg;					\
		if (!entry)						\
			break;						\
		pin = entry->pin;					\
		reg = io_apic_read(entry->apic, 0x10 + R + pin*2);	\
		reg ACTION;						\
		io_apic_modify(entry->apic, reg);			\
		FINAL;							\
		if (!entry->next)					\
			break;						\
		entry = entry->next;					\
	}								\
}
L
Linus Torvalds 已提交
634 635 636 637 638 639

#define DO_ACTION(name,R,ACTION, FINAL)					\
									\
	static void name##_IO_APIC_irq (unsigned int irq)		\
	__DO_ACTION(R, ACTION, FINAL)

640 641 642 643 644
/* mask = 1 */
DO_ACTION(__mask,	0, |= IO_APIC_REDIR_MASKED, io_apic_sync(entry->apic))

/* mask = 0 */
DO_ACTION(__unmask,	0, &= ~IO_APIC_REDIR_MASKED, )
L
Linus Torvalds 已提交
645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668

static void mask_IO_APIC_irq (unsigned int irq)
{
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	__mask_IO_APIC_irq(irq);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

static void unmask_IO_APIC_irq (unsigned int irq)
{
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	__unmask_IO_APIC_irq(irq);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
{
	struct IO_APIC_route_entry entry;

	/* Check delivery_mode to be sure we're not clearing an SMI pin */
669
	entry = ioapic_read_entry(apic, pin);
L
Linus Torvalds 已提交
670 671 672 673 674
	if (entry.delivery_mode == dest_SMI)
		return;
	/*
	 * Disable it in the IO-APIC irq-routing table:
	 */
675
	ioapic_mask_entry(apic, pin);
L
Linus Torvalds 已提交
676 677 678 679 680 681 682 683 684 685 686
}

static void clear_IO_APIC (void)
{
	int apic, pin;

	for (apic = 0; apic < nr_ioapics; apic++)
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
			clear_IO_APIC_pin(apic, pin);
}

Y
Yinghai Lu 已提交
687 688 689 690
#ifdef CONFIG_INTR_REMAP
/* I/O APIC RTE contents at the OS boot up */
static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS];

691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
/*
 * Saves and masks all the unmasked IO-APIC RTE's
 */
int save_mask_IO_APIC_setup(void)
{
	union IO_APIC_reg_01 reg_01;
	unsigned long flags;
	int apic, pin;

	/*
	 * The number of IO-APIC IRQ registers (== #pins):
	 */
	for (apic = 0; apic < nr_ioapics; apic++) {
		spin_lock_irqsave(&ioapic_lock, flags);
		reg_01.raw = io_apic_read(apic, 1);
		spin_unlock_irqrestore(&ioapic_lock, flags);
		nr_ioapic_registers[apic] = reg_01.bits.entries+1;
	}

	for (apic = 0; apic < nr_ioapics; apic++) {
		early_ioapic_entries[apic] =
			kzalloc(sizeof(struct IO_APIC_route_entry) *
				nr_ioapic_registers[apic], GFP_KERNEL);
		if (!early_ioapic_entries[apic])
			return -ENOMEM;
	}

	for (apic = 0; apic < nr_ioapics; apic++)
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
			struct IO_APIC_route_entry entry;

			entry = early_ioapic_entries[apic][pin] =
				ioapic_read_entry(apic, pin);
			if (!entry.mask) {
				entry.mask = 1;
				ioapic_write_entry(apic, pin, entry);
			}
		}
	return 0;
}

void restore_IO_APIC_setup(void)
{
	int apic, pin;

	for (apic = 0; apic < nr_ioapics; apic++)
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
			ioapic_write_entry(apic, pin,
					   early_ioapic_entries[apic][pin]);
}

void reinit_intr_remapped_IO_APIC(int intr_remapping)
{
	/*
	 * for now plain restore of previous settings.
	 * TBD: In the case of OS enabling interrupt-remapping,
	 * IO-APIC RTE's need to be setup to point to interrupt-remapping
	 * table entries. for now, do a plain restore, and wait for
	 * the setup_IO_APIC_irqs() to do proper initialization.
	 */
	restore_IO_APIC_setup();
}
Y
Yinghai Lu 已提交
753
#endif
754

L
Linus Torvalds 已提交
755 756 757 758 759 760 761 762
/*
 * Find the IRQ entry number of a certain pin.
 */
static int find_irq_entry(int apic, int pin, int type)
{
	int i;

	for (i = 0; i < mp_irq_entries; i++)
763 764 765 766
		if (mp_irqs[i].mp_irqtype == type &&
		    (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
		     mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
		    mp_irqs[i].mp_dstirq == pin)
L
Linus Torvalds 已提交
767 768 769 770 771 772 773 774
			return i;

	return -1;
}

/*
 * Find the pin to which IRQ[irq] (ISA) is connected
 */
775
static int __init find_isa_irq_pin(int irq, int type)
L
Linus Torvalds 已提交
776 777 778 779
{
	int i;

	for (i = 0; i < mp_irq_entries; i++) {
780
		int lbus = mp_irqs[i].mp_srcbus;
L
Linus Torvalds 已提交
781

782
		if (test_bit(lbus, mp_bus_not_pci) &&
783 784
		    (mp_irqs[i].mp_irqtype == type) &&
		    (mp_irqs[i].mp_srcbusirq == irq))
L
Linus Torvalds 已提交
785

786
			return mp_irqs[i].mp_dstirq;
L
Linus Torvalds 已提交
787 788 789 790
	}
	return -1;
}

791 792 793 794 795
static int __init find_isa_irq_apic(int irq, int type)
{
	int i;

	for (i = 0; i < mp_irq_entries; i++) {
796
		int lbus = mp_irqs[i].mp_srcbus;
797

798
		if (test_bit(lbus, mp_bus_not_pci) &&
799 800
		    (mp_irqs[i].mp_irqtype == type) &&
		    (mp_irqs[i].mp_srcbusirq == irq))
801 802 803 804 805
			break;
	}
	if (i < mp_irq_entries) {
		int apic;
		for(apic = 0; apic < nr_ioapics; apic++) {
806
			if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
807 808 809 810 811 812 813
				return apic;
		}
	}

	return -1;
}

L
Linus Torvalds 已提交
814 815 816 817 818 819 820 821 822 823 824 825
/*
 * Find a specific PCI IRQ entry.
 * Not an __init, possibly needed by modules
 */
static int pin_2_irq(int idx, int apic, int pin);

int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
{
	int apic, i, best_guess = -1;

	apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
		bus, slot, pin);
826
	if (test_bit(bus, mp_bus_not_pci)) {
L
Linus Torvalds 已提交
827 828 829 830
		apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
		return -1;
	}
	for (i = 0; i < mp_irq_entries; i++) {
831
		int lbus = mp_irqs[i].mp_srcbus;
L
Linus Torvalds 已提交
832 833

		for (apic = 0; apic < nr_ioapics; apic++)
834 835
			if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
			    mp_irqs[i].mp_dstapic == MP_APIC_ALL)
L
Linus Torvalds 已提交
836 837
				break;

838
		if (!test_bit(lbus, mp_bus_not_pci) &&
839
		    !mp_irqs[i].mp_irqtype &&
L
Linus Torvalds 已提交
840
		    (bus == lbus) &&
841 842
		    (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
			int irq = pin_2_irq(i,apic,mp_irqs[i].mp_dstirq);
L
Linus Torvalds 已提交
843 844 845 846

			if (!(apic || IO_APIC_IRQ(irq)))
				continue;

847
			if (pin == (mp_irqs[i].mp_srcbusirq & 3))
L
Linus Torvalds 已提交
848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871
				return irq;
			/*
			 * Use the first all-but-pin matching entry as a
			 * best-guess fuzzy result for broken mptables.
			 */
			if (best_guess < 0)
				best_guess = irq;
		}
	}
	return best_guess;
}

/* ISA interrupts are always polarity zero edge triggered,
 * when listed as conforming in the MP table. */

#define default_ISA_trigger(idx)	(0)
#define default_ISA_polarity(idx)	(0)

/* PCI interrupts are always polarity one level triggered,
 * when listed as conforming in the MP table. */

#define default_PCI_trigger(idx)	(1)
#define default_PCI_polarity(idx)	(1)

872
static int MPBIOS_polarity(int idx)
L
Linus Torvalds 已提交
873
{
874
	int bus = mp_irqs[idx].mp_srcbus;
L
Linus Torvalds 已提交
875 876 877 878 879
	int polarity;

	/*
	 * Determine IRQ line polarity (high active or low active):
	 */
880
	switch (mp_irqs[idx].mp_irqflag & 3)
L
Linus Torvalds 已提交
881 882
	{
		case 0: /* conforms, ie. bus-type dependent polarity */
883 884 885 886
			if (test_bit(bus, mp_bus_not_pci))
				polarity = default_ISA_polarity(idx);
			else
				polarity = default_PCI_polarity(idx);
L
Linus Torvalds 已提交
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915
			break;
		case 1: /* high active */
		{
			polarity = 0;
			break;
		}
		case 2: /* reserved */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			polarity = 1;
			break;
		}
		case 3: /* low active */
		{
			polarity = 1;
			break;
		}
		default: /* invalid */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			polarity = 1;
			break;
		}
	}
	return polarity;
}

static int MPBIOS_trigger(int idx)
{
916
	int bus = mp_irqs[idx].mp_srcbus;
L
Linus Torvalds 已提交
917 918 919 920 921
	int trigger;

	/*
	 * Determine IRQ trigger mode (edge or level sensitive):
	 */
922
	switch ((mp_irqs[idx].mp_irqflag>>2) & 3)
L
Linus Torvalds 已提交
923 924
	{
		case 0: /* conforms, ie. bus-type dependent */
925 926 927 928
			if (test_bit(bus, mp_bus_not_pci))
				trigger = default_ISA_trigger(idx);
			else
				trigger = default_PCI_trigger(idx);
L
Linus Torvalds 已提交
929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968
			break;
		case 1: /* edge */
		{
			trigger = 0;
			break;
		}
		case 2: /* reserved */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			trigger = 1;
			break;
		}
		case 3: /* level */
		{
			trigger = 1;
			break;
		}
		default: /* invalid */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			trigger = 0;
			break;
		}
	}
	return trigger;
}

static inline int irq_polarity(int idx)
{
	return MPBIOS_polarity(idx);
}

static inline int irq_trigger(int idx)
{
	return MPBIOS_trigger(idx);
}

static int pin_2_irq(int idx, int apic, int pin)
{
	int irq, i;
969
	int bus = mp_irqs[idx].mp_srcbus;
L
Linus Torvalds 已提交
970 971 972 973

	/*
	 * Debugging check, we are in big trouble if this message pops up!
	 */
974
	if (mp_irqs[idx].mp_dstirq != pin)
L
Linus Torvalds 已提交
975 976
		printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");

977
	if (test_bit(bus, mp_bus_not_pci)) {
978
		irq = mp_irqs[idx].mp_srcbusirq;
979 980 981 982 983 984 985 986
	} else {
		/*
		 * PCI IRQs are mapped in order
		 */
		i = irq = 0;
		while (i < apic)
			irq += nr_ioapic_registers[i++];
		irq += pin;
L
Linus Torvalds 已提交
987 988 989 990
	}
	return irq;
}

991 992 993 994 995 996 997 998 999 1000 1001 1002 1003
void lock_vector_lock(void)
{
	/* Used to the online set of cpus does not change
	 * during assign_irq_vector.
	 */
	spin_lock(&vector_lock);
}

void unlock_vector_lock(void)
{
	spin_unlock(&vector_lock);
}

1004
static int __assign_irq_vector(int irq, cpumask_t mask)
L
Linus Torvalds 已提交
1005
{
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
1017
	static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
1018
	unsigned int old_vector;
1019
	int cpu;
1020
	struct irq_cfg *cfg;
L
Linus Torvalds 已提交
1021

Y
Yinghai Lu 已提交
1022
	cfg = irq_cfg(irq);
1023

1024 1025 1026
	/* Only try and allocate irqs on cpus that are present */
	cpus_and(mask, mask, cpu_online_map);

1027 1028 1029
	if ((cfg->move_in_progress) || cfg->move_cleanup_count)
		return -EBUSY;

1030 1031 1032 1033 1034 1035
	old_vector = cfg->vector;
	if (old_vector) {
		cpumask_t tmp;
		cpus_and(tmp, cfg->domain, mask);
		if (!cpus_empty(tmp))
			return 0;
1036
	}
1037

1038
	for_each_cpu_mask_nr(cpu, mask) {
1039
		cpumask_t domain, new_mask;
1040
		int new_cpu;
1041
		int vector, offset;
1042 1043

		domain = vector_allocation_domain(cpu);
1044
		cpus_and(new_mask, domain, cpu_online_map);
1045

1046 1047
		vector = current_vector;
		offset = current_offset;
L
Linus Torvalds 已提交
1048
next:
1049
		vector += 8;
1050
		if (vector >= first_system_vector) {
1051 1052 1053 1054
			/* If we run out of vectors on large boxen, must share them. */
			offset = (offset + 1) % 8;
			vector = FIRST_DEVICE_VECTOR + offset;
		}
1055
		if (unlikely(current_vector == vector))
1056 1057 1058
			continue;
		if (vector == IA32_SYSCALL_VECTOR)
			goto next;
1059
		for_each_cpu_mask_nr(new_cpu, new_mask)
1060
			if (per_cpu(vector_irq, new_cpu)[vector] != -1)
1061
				goto next;
1062
		/* Found one! */
1063 1064
		current_vector = vector;
		current_offset = offset;
1065 1066 1067 1068
		if (old_vector) {
			cfg->move_in_progress = 1;
			cfg->old_domain = cfg->domain;
		}
1069
		for_each_cpu_mask_nr(new_cpu, new_mask)
1070
			per_cpu(vector_irq, new_cpu)[vector] = irq;
1071 1072
		cfg->vector = vector;
		cfg->domain = domain;
1073
		return 0;
L
Linus Torvalds 已提交
1074
	}
1075
	return -ENOSPC;
1076 1077
}

1078
static int assign_irq_vector(int irq, cpumask_t mask)
1079
{
1080
	int err;
1081
	unsigned long flags;
1082

1083
	spin_lock_irqsave(&vector_lock, flags);
1084
	err = __assign_irq_vector(irq, mask);
1085
	spin_unlock_irqrestore(&vector_lock, flags);
1086
	return err;
L
Linus Torvalds 已提交
1087 1088
}

1089 1090
static void __clear_irq_vector(int irq)
{
1091
	struct irq_cfg *cfg;
1092 1093 1094
	cpumask_t mask;
	int cpu, vector;

Y
Yinghai Lu 已提交
1095
	cfg = irq_cfg(irq);
1096
	BUG_ON(!cfg->vector);
1097

1098 1099
	vector = cfg->vector;
	cpus_and(mask, cfg->domain, cpu_online_map);
1100
	for_each_cpu_mask_nr(cpu, mask)
1101 1102
		per_cpu(vector_irq, cpu)[vector] = -1;

1103
	cfg->vector = 0;
1104
	cpus_clear(cfg->domain);
1105 1106
}

1107
void __setup_vector_irq(int cpu)
1108 1109 1110 1111
{
	/* Initialize vector_irq on a new cpu */
	/* This function must be called with vector_lock held */
	int irq, vector;
1112
	struct irq_cfg *cfg;
1113 1114

	/* Mark the inuse vectors */
1115
	for_each_irq_cfg(cfg) {
Y
Yinghai Lu 已提交
1116
		if (!cpu_isset(cpu, cfg->domain))
1117
			continue;
Y
Yinghai Lu 已提交
1118
		vector = cfg->vector;
1119
		irq = cfg->irq;
1120 1121 1122 1123 1124 1125 1126
		per_cpu(vector_irq, cpu)[vector] = irq;
	}
	/* Mark the free vectors */
	for (vector = 0; vector < NR_VECTORS; ++vector) {
		irq = per_cpu(vector_irq, cpu)[vector];
		if (irq < 0)
			continue;
Y
Yinghai Lu 已提交
1127 1128 1129

		cfg = irq_cfg(irq);
		if (!cpu_isset(cpu, cfg->domain))
1130 1131 1132 1133
			per_cpu(vector_irq, cpu)[vector] = -1;
	}
}

1134
static struct irq_chip ioapic_chip;
1135 1136 1137
#ifdef CONFIG_INTR_REMAP
static struct irq_chip ir_ioapic_chip;
#endif
L
Linus Torvalds 已提交
1138

1139
static void ioapic_register_intr(int irq, unsigned long trigger)
L
Linus Torvalds 已提交
1140
{
1141 1142
	struct irq_desc *desc;

1143 1144 1145 1146 1147 1148
	/* first time to use this irq_desc */
	if (irq < 16)
		desc = irq_to_desc(irq);
	else
		desc = irq_to_desc_alloc(irq);

1149
	if (trigger)
1150
		desc->status |= IRQ_LEVEL;
1151
	else
1152
		desc->status &= ~IRQ_LEVEL;
1153 1154 1155

#ifdef CONFIG_INTR_REMAP
	if (irq_remapped(irq)) {
1156
		desc->status |= IRQ_MOVE_PCNTXT;
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
		if (trigger)
			set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
						      handle_fasteoi_irq,
						     "fasteoi");
		else
			set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
						      handle_edge_irq, "edge");
		return;
	}
#endif
	if (trigger)
		set_irq_chip_and_handler_name(irq, &ioapic_chip,
					      handle_fasteoi_irq,
					      "fasteoi");
	else
1172 1173
		set_irq_chip_and_handler_name(irq, &ioapic_chip,
					      handle_edge_irq, "edge");
1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
}

static int setup_ioapic_entry(int apic, int irq,
			      struct IO_APIC_route_entry *entry,
			      unsigned int destination, int trigger,
			      int polarity, int vector)
{
	/*
	 * add it to the IO-APIC irq-routing table:
	 */
	memset(entry,0,sizeof(*entry));

#ifdef CONFIG_INTR_REMAP
	if (intr_remapping_enabled) {
		struct intel_iommu *iommu = map_ioapic_to_ir(apic);
		struct irte irte;
		struct IR_IO_APIC_route_entry *ir_entry =
			(struct IR_IO_APIC_route_entry *) entry;
		int index;

		if (!iommu)
			panic("No mapping iommu for ioapic %d\n", apic);

		index = alloc_irte(iommu, irq, 1);
		if (index < 0)
			panic("Failed to allocate IRTE for ioapic %d\n", apic);

		memset(&irte, 0, sizeof(irte));

		irte.present = 1;
		irte.dst_mode = INT_DEST_MODE;
		irte.trigger_mode = trigger;
		irte.dlvry_mode = INT_DELIVERY_MODE;
		irte.vector = vector;
		irte.dest_id = IRTE_DEST(destination);

		modify_irte(irq, &irte);

		ir_entry->index2 = (index >> 15) & 0x1;
		ir_entry->zero = 0;
		ir_entry->format = 1;
		ir_entry->index = (index & 0x7fff);
	} else
#endif
	{
		entry->delivery_mode = INT_DELIVERY_MODE;
		entry->dest_mode = INT_DEST_MODE;
		entry->dest = destination;
1222
	}
1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234

	entry->mask = 0;				/* enable IRQ */
	entry->trigger = trigger;
	entry->polarity = polarity;
	entry->vector = vector;

	/* Mask level triggered irqs.
	 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
	 */
	if (trigger)
		entry->mask = 1;
	return 0;
L
Linus Torvalds 已提交
1235
}
1236 1237 1238

static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq,
			      int trigger, int polarity)
L
Linus Torvalds 已提交
1239
{
Y
Yinghai Lu 已提交
1240
	struct irq_cfg *cfg;
L
Linus Torvalds 已提交
1241
	struct IO_APIC_route_entry entry;
1242
	cpumask_t mask;
L
Linus Torvalds 已提交
1243

1244 1245 1246
	if (!IO_APIC_IRQ(irq))
		return;

Y
Yinghai Lu 已提交
1247 1248
	cfg = irq_cfg(irq);

1249 1250
	mask = TARGET_CPUS;
	if (assign_irq_vector(irq, mask))
1251 1252
		return;

1253 1254
	cpus_and(mask, cfg->domain, mask);

1255 1256 1257
	apic_printk(APIC_VERBOSE,KERN_DEBUG
		    "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
		    "IRQ %d Mode:%i Active:%i)\n",
1258
		    apic, mp_ioapics[apic].mp_apicid, pin, cfg->vector,
1259
		    irq, trigger, polarity);
L
Linus Torvalds 已提交
1260 1261


1262 1263 1264 1265 1266 1267 1268 1269
	if (setup_ioapic_entry(mp_ioapics[apic].mp_apicid, irq, &entry,
			       cpu_mask_to_apicid(mask), trigger, polarity,
			       cfg->vector)) {
		printk("Failed to setup ioapic entry for ioapic  %d, pin %d\n",
		       mp_ioapics[apic].mp_apicid, pin);
		__clear_irq_vector(irq);
		return;
	}
1270

1271 1272 1273
	ioapic_register_intr(irq, trigger);
	if (irq < 16)
		disable_8259A_irq(irq);
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285

	ioapic_write_entry(apic, pin, entry);
}

static void __init setup_IO_APIC_irqs(void)
{
	int apic, pin, idx, irq, first_notcon = 1;

	apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");

	for (apic = 0; apic < nr_ioapics; apic++) {
	for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
L
Linus Torvalds 已提交
1286 1287 1288 1289

		idx = find_irq_entry(apic,pin,mp_INT);
		if (idx == -1) {
			if (first_notcon) {
1290
				apic_printk(APIC_VERBOSE, KERN_DEBUG " IO-APIC (apicid-pin) %d-%d", mp_ioapics[apic].mp_apicid, pin);
L
Linus Torvalds 已提交
1291 1292
				first_notcon = 0;
			} else
1293
				apic_printk(APIC_VERBOSE, ", %d-%d", mp_ioapics[apic].mp_apicid, pin);
L
Linus Torvalds 已提交
1294 1295
			continue;
		}
1296 1297 1298 1299
		if (!first_notcon) {
			apic_printk(APIC_VERBOSE, " not connected.\n");
			first_notcon = 1;
		}
L
Linus Torvalds 已提交
1300 1301 1302 1303

		irq = pin_2_irq(idx, apic, pin);
		add_pin_to_irq(irq, apic, pin);

1304 1305
		setup_IO_APIC_irq(apic, pin, irq,
				  irq_trigger(idx), irq_polarity(idx));
L
Linus Torvalds 已提交
1306 1307 1308 1309
	}
	}

	if (!first_notcon)
1310
		apic_printk(APIC_VERBOSE, " not connected.\n");
L
Linus Torvalds 已提交
1311 1312 1313
}

/*
1314
 * Set up the timer pin, possibly with the 8259A-master behind.
L
Linus Torvalds 已提交
1315
 */
1316 1317
static void __init setup_timer_IRQ0_pin(unsigned int apic, unsigned int pin,
					int vector)
L
Linus Torvalds 已提交
1318 1319 1320
{
	struct IO_APIC_route_entry entry;

Y
Yinghai Lu 已提交
1321
#ifdef CONFIG_INTR_REMAP
1322 1323
	if (intr_remapping_enabled)
		return;
Y
Yinghai Lu 已提交
1324
#endif
1325

1326
	memset(&entry, 0, sizeof(entry));
L
Linus Torvalds 已提交
1327 1328 1329 1330 1331 1332

	/*
	 * We use logical delivery to get the timer IRQ
	 * to the first CPU.
	 */
	entry.dest_mode = INT_DEST_MODE;
1333
	entry.mask = 1;					/* mask IRQ now */
1334
	entry.dest = cpu_mask_to_apicid(TARGET_CPUS);
L
Linus Torvalds 已提交
1335 1336 1337 1338 1339 1340 1341
	entry.delivery_mode = INT_DELIVERY_MODE;
	entry.polarity = 0;
	entry.trigger = 0;
	entry.vector = vector;

	/*
	 * The timer IRQ doesn't have to know that behind the
1342
	 * scene we may have a 8259A-master in AEOI mode ...
L
Linus Torvalds 已提交
1343
	 */
1344
	set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
L
Linus Torvalds 已提交
1345 1346 1347 1348

	/*
	 * Add it to the IO-APIC irq-routing table:
	 */
1349
	ioapic_write_entry(apic, pin, entry);
L
Linus Torvalds 已提交
1350 1351
}

1352 1353

__apicdebuginit(void) print_IO_APIC(void)
L
Linus Torvalds 已提交
1354 1355 1356 1357 1358 1359
{
	int apic, i;
	union IO_APIC_reg_00 reg_00;
	union IO_APIC_reg_01 reg_01;
	union IO_APIC_reg_02 reg_02;
	unsigned long flags;
1360
	struct irq_cfg *cfg;
L
Linus Torvalds 已提交
1361 1362 1363 1364 1365 1366 1367

	if (apic_verbosity == APIC_QUIET)
		return;

	printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
	for (i = 0; i < nr_ioapics; i++)
		printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1368
		       mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);
L
Linus Torvalds 已提交
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385

	/*
	 * We are a bit conservative about what we expect.  We have to
	 * know about every hardware change ASAP.
	 */
	printk(KERN_INFO "testing the IO APIC.......................\n");

	for (apic = 0; apic < nr_ioapics; apic++) {

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(apic, 0);
	reg_01.raw = io_apic_read(apic, 1);
	if (reg_01.bits.version >= 0x10)
		reg_02.raw = io_apic_read(apic, 2);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	printk("\n");
1386
	printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
L
Linus Torvalds 已提交
1387 1388
	printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
	printk(KERN_DEBUG ".......    : physical APIC id: %02X\n", reg_00.bits.ID);
1389 1390
	printk(KERN_DEBUG ".......    : Delivery Type: %X\n", reg_00.bits.delivery_type);
	printk(KERN_DEBUG ".......    : LTS          : %X\n", reg_00.bits.LTS);
L
Linus Torvalds 已提交
1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404

	printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
	printk(KERN_DEBUG ".......     : max redirection entries: %04X\n", reg_01.bits.entries);

	printk(KERN_DEBUG ".......     : PRQ implemented: %X\n", reg_01.bits.PRQ);
	printk(KERN_DEBUG ".......     : IO APIC version: %04X\n", reg_01.bits.version);

	if (reg_01.bits.version >= 0x10) {
		printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
		printk(KERN_DEBUG ".......     : arbitration: %02X\n", reg_02.bits.arbitration);
	}

	printk(KERN_DEBUG ".... IRQ redirection table:\n");

1405 1406
	printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
			  " Stat Dmod Deli Vect:   \n");
L
Linus Torvalds 已提交
1407 1408 1409 1410

	for (i = 0; i <= reg_01.bits.entries; i++) {
		struct IO_APIC_route_entry entry;

1411
		entry = ioapic_read_entry(apic, i);
L
Linus Torvalds 已提交
1412

1413
		printk(KERN_DEBUG " %02x %03X ",
L
Linus Torvalds 已提交
1414
			i,
1415
			entry.dest
L
Linus Torvalds 已提交
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
		);

		printk("%1d    %1d    %1d   %1d   %1d    %1d    %1d    %02X\n",
			entry.mask,
			entry.trigger,
			entry.irr,
			entry.polarity,
			entry.delivery_status,
			entry.dest_mode,
			entry.delivery_mode,
			entry.vector
		);
	}
	}
	printk(KERN_DEBUG "IRQ to pin mappings:\n");
1431
	for_each_irq_cfg(cfg) {
1432 1433
		struct irq_pin_list *entry = cfg->irq_2_pin;
		if (!entry)
L
Linus Torvalds 已提交
1434
			continue;
1435
		printk(KERN_DEBUG "IRQ%d ", cfg->irq);
L
Linus Torvalds 已提交
1436 1437 1438 1439
		for (;;) {
			printk("-> %d:%d", entry->apic, entry->pin);
			if (!entry->next)
				break;
1440
			entry = entry->next;
L
Linus Torvalds 已提交
1441 1442 1443 1444 1445 1446 1447 1448 1449
		}
		printk("\n");
	}

	printk(KERN_INFO ".................................... done.\n");

	return;
}

1450
__apicdebuginit(void) print_APIC_bitfield(int base)
L
Linus Torvalds 已提交
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
{
	unsigned int v;
	int i, j;

	if (apic_verbosity == APIC_QUIET)
		return;

	printk(KERN_DEBUG "0123456789abcdef0123456789abcdef\n" KERN_DEBUG);
	for (i = 0; i < 8; i++) {
		v = apic_read(base + i*0x10);
		for (j = 0; j < 32; j++) {
			if (v & (1<<j))
				printk("1");
			else
				printk("0");
		}
		printk("\n");
	}
}

1471
__apicdebuginit(void) print_local_APIC(void *dummy)
L
Linus Torvalds 已提交
1472 1473
{
	unsigned int v, ver, maxlvt;
1474
	unsigned long icr;
L
Linus Torvalds 已提交
1475 1476 1477 1478 1479 1480

	if (apic_verbosity == APIC_QUIET)
		return;

	printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
		smp_processor_id(), hard_smp_processor_id());
1481
	v = apic_read(APIC_ID);
1482
	printk(KERN_INFO "... APIC ID:      %08x (%01x)\n", v, read_apic_id());
L
Linus Torvalds 已提交
1483 1484 1485
	v = apic_read(APIC_LVR);
	printk(KERN_INFO "... APIC VERSION: %08x\n", v);
	ver = GET_APIC_VERSION(v);
1486
	maxlvt = lapic_get_maxlvt();
L
Linus Torvalds 已提交
1487 1488 1489 1490

	v = apic_read(APIC_TASKPRI);
	printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);

1491 1492 1493 1494 1495
	v = apic_read(APIC_ARBPRI);
	printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
		v & APIC_ARBPRI_MASK);
	v = apic_read(APIC_PROCPRI);
	printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
L
Linus Torvalds 已提交
1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514

	v = apic_read(APIC_EOI);
	printk(KERN_DEBUG "... APIC EOI: %08x\n", v);
	v = apic_read(APIC_RRR);
	printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
	v = apic_read(APIC_LDR);
	printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
	v = apic_read(APIC_DFR);
	printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
	v = apic_read(APIC_SPIV);
	printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);

	printk(KERN_DEBUG "... APIC ISR field:\n");
	print_APIC_bitfield(APIC_ISR);
	printk(KERN_DEBUG "... APIC TMR field:\n");
	print_APIC_bitfield(APIC_TMR);
	printk(KERN_DEBUG "... APIC IRR field:\n");
	print_APIC_bitfield(APIC_IRR);

1515 1516
	v = apic_read(APIC_ESR);
	printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
L
Linus Torvalds 已提交
1517

1518
	icr = apic_icr_read();
1519 1520
	printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
	printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
L
Linus Torvalds 已提交
1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547

	v = apic_read(APIC_LVTT);
	printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);

	if (maxlvt > 3) {                       /* PC is LVT#4. */
		v = apic_read(APIC_LVTPC);
		printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
	}
	v = apic_read(APIC_LVT0);
	printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
	v = apic_read(APIC_LVT1);
	printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);

	if (maxlvt > 2) {			/* ERR is LVT#3. */
		v = apic_read(APIC_LVTERR);
		printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
	}

	v = apic_read(APIC_TMICT);
	printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
	v = apic_read(APIC_TMCCT);
	printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
	v = apic_read(APIC_TDCR);
	printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
	printk("\n");
}

1548
__apicdebuginit(void) print_all_local_APICs(void)
L
Linus Torvalds 已提交
1549
{
1550
	on_each_cpu(print_local_APIC, NULL, 1);
L
Linus Torvalds 已提交
1551 1552
}

1553
__apicdebuginit(void) print_PIC(void)
L
Linus Torvalds 已提交
1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584
{
	unsigned int v;
	unsigned long flags;

	if (apic_verbosity == APIC_QUIET)
		return;

	printk(KERN_DEBUG "\nprinting PIC contents\n");

	spin_lock_irqsave(&i8259A_lock, flags);

	v = inb(0xa1) << 8 | inb(0x21);
	printk(KERN_DEBUG "... PIC  IMR: %04x\n", v);

	v = inb(0xa0) << 8 | inb(0x20);
	printk(KERN_DEBUG "... PIC  IRR: %04x\n", v);

	outb(0x0b,0xa0);
	outb(0x0b,0x20);
	v = inb(0xa0) << 8 | inb(0x20);
	outb(0x0a,0xa0);
	outb(0x0a,0x20);

	spin_unlock_irqrestore(&i8259A_lock, flags);

	printk(KERN_DEBUG "... PIC  ISR: %04x\n", v);

	v = inb(0x4d1) << 8 | inb(0x4d0);
	printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
}

1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
__apicdebuginit(int) print_all_ICs(void)
{
	print_PIC();
	print_all_local_APICs();
	print_IO_APIC();

	return 0;
}

fs_initcall(print_all_ICs);

L
Linus Torvalds 已提交
1596

Y
Yinghai Lu 已提交
1597 1598 1599
/* Where if anywhere is the i8259 connect in external int mode */
static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };

1600
void __init enable_IO_APIC(void)
L
Linus Torvalds 已提交
1601 1602
{
	union IO_APIC_reg_01 reg_01;
1603
	int i8259_apic, i8259_pin;
1604
	int apic;
L
Linus Torvalds 已提交
1605 1606 1607 1608 1609
	unsigned long flags;

	/*
	 * The number of IO-APIC IRQ registers (== #pins):
	 */
1610
	for (apic = 0; apic < nr_ioapics; apic++) {
L
Linus Torvalds 已提交
1611
		spin_lock_irqsave(&ioapic_lock, flags);
1612
		reg_01.raw = io_apic_read(apic, 1);
L
Linus Torvalds 已提交
1613
		spin_unlock_irqrestore(&ioapic_lock, flags);
1614 1615 1616 1617 1618 1619 1620
		nr_ioapic_registers[apic] = reg_01.bits.entries+1;
	}
	for(apic = 0; apic < nr_ioapics; apic++) {
		int pin;
		/* See if any of the pins is in ExtINT mode */
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
			struct IO_APIC_route_entry entry;
1621
			entry = ioapic_read_entry(apic, pin);
1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647

			/* If the interrupt line is enabled and in ExtInt mode
			 * I have found the pin where the i8259 is connected.
			 */
			if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
				ioapic_i8259.apic = apic;
				ioapic_i8259.pin  = pin;
				goto found_i8259;
			}
		}
	}
 found_i8259:
	/* Look to see what if the MP table has reported the ExtINT */
	i8259_pin  = find_isa_irq_pin(0, mp_ExtINT);
	i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
	/* Trust the MP table if nothing is setup in the hardware */
	if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
		printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
		ioapic_i8259.pin  = i8259_pin;
		ioapic_i8259.apic = i8259_apic;
	}
	/* Complain if the MP table and the hardware disagree */
	if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
		(i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
	{
		printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
L
Linus Torvalds 已提交
1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665
	}

	/*
	 * Do not trust the IO-APIC being empty at bootup
	 */
	clear_IO_APIC();
}

/*
 * Not an __init, needed by the reboot code
 */
void disable_IO_APIC(void)
{
	/*
	 * Clear the IO-APIC before rebooting:
	 */
	clear_IO_APIC();

1666
	/*
1667
	 * If the i8259 is routed through an IOAPIC
1668
	 * Put that IOAPIC in virtual wire mode
1669
	 * so legacy interrupts can be delivered.
1670
	 */
1671
	if (ioapic_i8259.pin != -1) {
1672 1673 1674 1675 1676 1677 1678 1679 1680
		struct IO_APIC_route_entry entry;

		memset(&entry, 0, sizeof(entry));
		entry.mask            = 0; /* Enabled */
		entry.trigger         = 0; /* Edge */
		entry.irr             = 0;
		entry.polarity        = 0; /* High */
		entry.delivery_status = 0;
		entry.dest_mode       = 0; /* Physical */
1681
		entry.delivery_mode   = dest_ExtINT; /* ExtInt */
1682
		entry.vector          = 0;
1683
		entry.dest            = read_apic_id();
1684 1685 1686 1687

		/*
		 * Add it to the IO-APIC irq-routing table:
		 */
1688
		ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1689 1690
	}

1691
	disconnect_bsp_APIC(ioapic_i8259.pin != -1);
L
Linus Torvalds 已提交
1692 1693
}

1694
int no_timer_check __initdata;
Y
Yinghai Lu 已提交
1695 1696 1697 1698 1699 1700 1701 1702

static int __init notimercheck(char *s)
{
	no_timer_check = 1;
	return 1;
}
__setup("no_timer_check", notimercheck);

L
Linus Torvalds 已提交
1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713
/*
 * There is a nasty bug in some older SMP boards, their mptable lies
 * about the timer IRQ. We do the following to work around the situation:
 *
 *	- timer IRQ defaults to IO-APIC IRQ
 *	- if this function detects that timer IRQs are defunct, then we fall
 *	  back to ISA timer IRQs
 */
static int __init timer_irq_works(void)
{
	unsigned long t1 = jiffies;
1714
	unsigned long flags;
L
Linus Torvalds 已提交
1715

Y
Yinghai Lu 已提交
1716 1717 1718
	if (no_timer_check)
		return 1;

1719
	local_save_flags(flags);
L
Linus Torvalds 已提交
1720 1721 1722
	local_irq_enable();
	/* Let ten ticks pass... */
	mdelay((10 * 1000) / HZ);
1723
	local_irq_restore(flags);
L
Linus Torvalds 已提交
1724 1725 1726 1727 1728 1729 1730 1731 1732 1733

	/*
	 * Expect a few ticks at least, to be sure some possible
	 * glue logic does not lock up after one or two first
	 * ticks in a non-ExtINT mode.  Also the local APIC
	 * might have cached one ExtINT interrupt.  Finally, at
	 * least one tick may be lost due to delays.
	 */

	/* jiffies wrap? */
1734
	if (time_after(jiffies, t1 + 4))
L
Linus Torvalds 已提交
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761
		return 1;
	return 0;
}

/*
 * In the SMP+IOAPIC case it might happen that there are an unspecified
 * number of pending IRQ events unhandled. These cases are very rare,
 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
 * better to do it this way as thus we do not have to be aware of
 * 'pending' interrupts in the IRQ path, except at this point.
 */
/*
 * Edge triggered needs to resend any interrupt
 * that was delayed but this is now handled in the device
 * independent code.
 */

/*
 * Starting up a edge-triggered IO-APIC interrupt is
 * nasty - we need to make sure that we get the edge.
 * If it is already asserted for some reason, we need
 * return 1 to indicate that is was pending.
 *
 * This is not complete - we should be able to fake
 * an edge even if it isn't on the 8259A...
 */

1762
static unsigned int startup_ioapic_irq(unsigned int irq)
L
Linus Torvalds 已提交
1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778
{
	int was_pending = 0;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	if (irq < 16) {
		disable_8259A_irq(irq);
		if (i8259A_irq_pending(irq))
			was_pending = 1;
	}
	__unmask_IO_APIC_irq(irq);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return was_pending;
}

1779
static int ioapic_retrigger_irq(unsigned int irq)
1780
{
Y
Yinghai Lu 已提交
1781
	struct irq_cfg *cfg = irq_cfg(irq);
1782
	unsigned long flags;
1783

1784
	spin_lock_irqsave(&vector_lock, flags);
1785
	send_IPI_mask(cpumask_of_cpu(first_cpu(cfg->domain)), cfg->vector);
1786
	spin_unlock_irqrestore(&vector_lock, flags);
1787 1788 1789 1790

	return 1;
}

L
Linus Torvalds 已提交
1791 1792 1793 1794 1795 1796 1797 1798 1799
/*
 * Level and edge triggered IO-APIC interrupts need different handling,
 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
 * handled with the level-triggered descriptor, but that one has slightly
 * more overhead. Level-triggered interrupts cannot be handled with the
 * edge-triggered handler, without risking IRQ storms and other ugly
 * races.
 */

1800
#ifdef CONFIG_SMP
1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827

#ifdef CONFIG_INTR_REMAP
static void ir_irq_migration(struct work_struct *work);

static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration);

/*
 * Migrate the IO-APIC irq in the presence of intr-remapping.
 *
 * For edge triggered, irq migration is a simple atomic update(of vector
 * and cpu destination) of IRTE and flush the hardware cache.
 *
 * For level triggered, we need to modify the io-apic RTE aswell with the update
 * vector information, along with modifying IRTE with vector and destination.
 * So irq migration for level triggered is little  bit more complex compared to
 * edge triggered migration. But the good news is, we use the same algorithm
 * for level triggered migration as we have today, only difference being,
 * we now initiate the irq migration from process context instead of the
 * interrupt context.
 *
 * In future, when we do a directed EOI (combined with cpu EOI broadcast
 * suppression) to the IO-APIC, level triggered irq migration will also be
 * as simple as edge triggered migration and we can do the irq migration
 * with a simple atomic update to IO-APIC RTE.
 */
static void migrate_ioapic_irq(int irq, cpumask_t mask)
{
Y
Yinghai Lu 已提交
1828
	struct irq_cfg *cfg;
1829
	struct irq_desc *desc;
1830 1831
	cpumask_t tmp, cleanup_mask;
	struct irte irte;
1832
	int modify_ioapic_rte;
1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845
	unsigned int dest;
	unsigned long flags;

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	if (get_irte(irq, &irte))
		return;

	if (assign_irq_vector(irq, mask))
		return;

Y
Yinghai Lu 已提交
1846
	cfg = irq_cfg(irq);
1847 1848 1849
	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

1850 1851
	desc = irq_to_desc(irq);
	modify_ioapic_rte = desc->status & IRQ_LEVEL;
1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872
	if (modify_ioapic_rte) {
		spin_lock_irqsave(&ioapic_lock, flags);
		__target_IO_APIC_irq(irq, dest, cfg->vector);
		spin_unlock_irqrestore(&ioapic_lock, flags);
	}

	irte.vector = cfg->vector;
	irte.dest_id = IRTE_DEST(dest);

	/*
	 * Modified the IRTE and flushes the Interrupt entry cache.
	 */
	modify_irte(irq, &irte);

	if (cfg->move_in_progress) {
		cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
		cfg->move_cleanup_count = cpus_weight(cleanup_mask);
		send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
		cfg->move_in_progress = 0;
	}

1873
	desc->affinity = mask;
1874 1875 1876 1877 1878
}

static int migrate_irq_remapped_level(int irq)
{
	int ret = -1;
1879
	struct irq_desc *desc = irq_to_desc(irq);
1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894

	mask_IO_APIC_irq(irq);

	if (io_apic_level_ack_pending(irq)) {
		/*
	 	 * Interrupt in progress. Migrating irq now will change the
		 * vector information in the IO-APIC RTE and that will confuse
		 * the EOI broadcast performed by cpu.
		 * So, delay the irq migration to the next instance.
		 */
		schedule_delayed_work(&ir_migration_work, 1);
		goto unmask;
	}

	/* everthing is clear. we have right of way */
1895
	migrate_ioapic_irq(irq, desc->pending_mask);
1896 1897

	ret = 0;
1898 1899
	desc->status &= ~IRQ_MOVE_PENDING;
	cpus_clear(desc->pending_mask);
1900 1901 1902 1903 1904 1905 1906 1907

unmask:
	unmask_IO_APIC_irq(irq);
	return ret;
}

static void ir_irq_migration(struct work_struct *work)
{
1908 1909
	unsigned int irq;
	struct irq_desc *desc;
1910

1911
	for_each_irq_desc(irq, desc) {
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922
		if (desc->status & IRQ_MOVE_PENDING) {
			unsigned long flags;

			spin_lock_irqsave(&desc->lock, flags);
			if (!desc->chip->set_affinity ||
			    !(desc->status & IRQ_MOVE_PENDING)) {
				desc->status &= ~IRQ_MOVE_PENDING;
				spin_unlock_irqrestore(&desc->lock, flags);
				continue;
			}

1923
			desc->chip->set_affinity(irq, desc->pending_mask);
1924 1925 1926 1927 1928 1929 1930 1931 1932 1933
			spin_unlock_irqrestore(&desc->lock, flags);
		}
	}
}

/*
 * Migrates the IRQ destination in the process context.
 */
static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
{
1934 1935 1936 1937 1938
	struct irq_desc *desc = irq_to_desc(irq);

	if (desc->status & IRQ_LEVEL) {
		desc->status |= IRQ_MOVE_PENDING;
		desc->pending_mask = mask;
1939 1940 1941 1942 1943 1944 1945 1946
		migrate_irq_remapped_level(irq);
		return;
	}

	migrate_ioapic_irq(irq, mask);
}
#endif

1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960
asmlinkage void smp_irq_move_cleanup_interrupt(void)
{
	unsigned vector, me;
	ack_APIC_irq();
	exit_idle();
	irq_enter();

	me = smp_processor_id();
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		unsigned int irq;
		struct irq_desc *desc;
		struct irq_cfg *cfg;
		irq = __get_cpu_var(vector_irq)[vector];

1961
		desc = irq_to_desc(irq);
1962 1963 1964
		if (!desc)
			continue;

Y
Yinghai Lu 已提交
1965
		cfg = irq_cfg(irq);
1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983
		spin_lock(&desc->lock);
		if (!cfg->move_cleanup_count)
			goto unlock;

		if ((vector == cfg->vector) && cpu_isset(me, cfg->domain))
			goto unlock;

		__get_cpu_var(vector_irq)[vector] = -1;
		cfg->move_cleanup_count--;
unlock:
		spin_unlock(&desc->lock);
	}

	irq_exit();
}

static void irq_complete_move(unsigned int irq)
{
Y
Yinghai Lu 已提交
1984
	struct irq_cfg *cfg = irq_cfg(irq);
1985 1986 1987 1988 1989
	unsigned vector, me;

	if (likely(!cfg->move_in_progress))
		return;

1990
	vector = ~get_irq_regs()->orig_ax;
1991
	me = smp_processor_id();
1992
	if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) {
1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003
		cpumask_t cleanup_mask;

		cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
		cfg->move_cleanup_count = cpus_weight(cleanup_mask);
		send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
		cfg->move_in_progress = 0;
	}
}
#else
static inline void irq_complete_move(unsigned int irq) {}
#endif
2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014
#ifdef CONFIG_INTR_REMAP
static void ack_x2apic_level(unsigned int irq)
{
	ack_x2APIC_irq();
}

static void ack_x2apic_edge(unsigned int irq)
{
	ack_x2APIC_irq();
}
#endif
2015

2016 2017
static void ack_apic_edge(unsigned int irq)
{
2018
	irq_complete_move(irq);
2019 2020 2021 2022 2023 2024 2025 2026
	move_native_irq(irq);
	ack_APIC_irq();
}

static void ack_apic_level(unsigned int irq)
{
	int do_unmask_irq = 0;

2027
	irq_complete_move(irq);
2028
#ifdef CONFIG_GENERIC_PENDING_IRQ
2029
	/* If we are moving the irq we need to mask it */
2030
	if (unlikely(irq_to_desc(irq)->status & IRQ_MOVE_PENDING)) {
2031 2032 2033 2034 2035 2036 2037
		do_unmask_irq = 1;
		mask_IO_APIC_irq(irq);
	}
#endif

	/*
	 * We must acknowledge the irq before we move it or the acknowledge will
2038
	 * not propagate properly.
2039 2040 2041 2042
	 */
	ack_APIC_irq();

	/* Now we can move and renable the irq */
2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071
	if (unlikely(do_unmask_irq)) {
		/* Only migrate the irq if the ack has been received.
		 *
		 * On rare occasions the broadcast level triggered ack gets
		 * delayed going to ioapics, and if we reprogram the
		 * vector while Remote IRR is still set the irq will never
		 * fire again.
		 *
		 * To prevent this scenario we read the Remote IRR bit
		 * of the ioapic.  This has two effects.
		 * - On any sane system the read of the ioapic will
		 *   flush writes (and acks) going to the ioapic from
		 *   this cpu.
		 * - We get to see if the ACK has actually been delivered.
		 *
		 * Based on failed experiments of reprogramming the
		 * ioapic entry from outside of irq context starting
		 * with masking the ioapic entry and then polling until
		 * Remote IRR was clear before reprogramming the
		 * ioapic I don't trust the Remote IRR bit to be
		 * completey accurate.
		 *
		 * However there appears to be no other way to plug
		 * this race, so if the Remote IRR bit is not
		 * accurate and is causing problems then it is a hardware bug
		 * and you can go talk to the chipset vendor about it.
		 */
		if (!io_apic_level_ack_pending(irq))
			move_masked_irq(irq);
2072
		unmask_IO_APIC_irq(irq);
2073
	}
2074 2075
}

2076 2077
static struct irq_chip ioapic_chip __read_mostly = {
	.name 		= "IO-APIC",
2078 2079 2080
	.startup 	= startup_ioapic_irq,
	.mask	 	= mask_IO_APIC_irq,
	.unmask	 	= unmask_IO_APIC_irq,
2081 2082
	.ack 		= ack_apic_edge,
	.eoi 		= ack_apic_level,
2083
#ifdef CONFIG_SMP
2084
	.set_affinity 	= set_ioapic_affinity_irq,
2085
#endif
2086
	.retrigger	= ioapic_retrigger_irq,
L
Linus Torvalds 已提交
2087 2088
};

2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103
#ifdef CONFIG_INTR_REMAP
static struct irq_chip ir_ioapic_chip __read_mostly = {
	.name 		= "IR-IO-APIC",
	.startup 	= startup_ioapic_irq,
	.mask	 	= mask_IO_APIC_irq,
	.unmask	 	= unmask_IO_APIC_irq,
	.ack 		= ack_x2apic_edge,
	.eoi 		= ack_x2apic_level,
#ifdef CONFIG_SMP
	.set_affinity 	= set_ir_ioapic_affinity_irq,
#endif
	.retrigger	= ioapic_retrigger_irq,
};
#endif

L
Linus Torvalds 已提交
2104 2105 2106
static inline void init_IO_APIC_traps(void)
{
	int irq;
2107
	struct irq_desc *desc;
2108
	struct irq_cfg *cfg;
L
Linus Torvalds 已提交
2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120

	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
2121 2122
	for_each_irq_cfg(cfg) {
		irq = cfg->irq;
Y
Yinghai Lu 已提交
2123
		if (IO_APIC_IRQ(irq) && !cfg->vector) {
L
Linus Torvalds 已提交
2124 2125 2126 2127 2128 2129 2130
			/*
			 * Hmm.. We don't have an entry for this,
			 * so default to an old-fashioned 8259
			 * interrupt if we can..
			 */
			if (irq < 16)
				make_8259A_irq(irq);
2131 2132
			else {
				desc = irq_to_desc(irq);
L
Linus Torvalds 已提交
2133
				/* Strange. Oh, well.. */
2134 2135
				desc->chip = &no_irq_chip;
			}
L
Linus Torvalds 已提交
2136 2137 2138 2139
		}
	}
}

2140
static void unmask_lapic_irq(unsigned int irq)
L
Linus Torvalds 已提交
2141 2142 2143 2144
{
	unsigned long v;

	v = apic_read(APIC_LVT0);
2145
	apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
L
Linus Torvalds 已提交
2146 2147
}

2148
static void mask_lapic_irq(unsigned int irq)
L
Linus Torvalds 已提交
2149 2150 2151 2152
{
	unsigned long v;

	v = apic_read(APIC_LVT0);
2153
	apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
L
Linus Torvalds 已提交
2154 2155 2156 2157 2158 2159 2160
}

static void ack_lapic_irq (unsigned int irq)
{
	ack_APIC_irq();
}

2161 2162 2163 2164 2165
static struct irq_chip lapic_chip __read_mostly = {
	.name		= "local-APIC",
	.mask		= mask_lapic_irq,
	.unmask		= unmask_lapic_irq,
	.ack		= ack_lapic_irq,
L
Linus Torvalds 已提交
2166 2167
};

2168 2169
static void lapic_register_intr(int irq)
{
2170 2171 2172 2173
	struct irq_desc *desc;

	desc = irq_to_desc(irq);
	desc->status &= ~IRQ_LEVEL;
2174 2175 2176 2177
	set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
				      "edge");
}

2178
static void __init setup_nmi(void)
L
Linus Torvalds 已提交
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
{
	/*
 	 * Dirty trick to enable the NMI watchdog ...
	 * We put the 8259A master into AEOI mode and
	 * unmask on all local APICs LVT0 as NMI.
	 *
	 * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire')
	 * is from Maciej W. Rozycki - so we do not have to EOI from
	 * the NMI handler or the timer interrupt.
	 */ 
	printk(KERN_INFO "activating NMI Watchdog ...");

2191
	enable_NMI_through_LVT0();
L
Linus Torvalds 已提交
2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202

	printk(" done.\n");
}

/*
 * This looks a bit hackish but it's about the only one way of sending
 * a few INTA cycles to 8259As and any associated glue logic.  ICR does
 * not support the ExtINT mode, unfortunately.  We need to send these
 * cycles as some i82489DX-based boards have glue logic that keeps the
 * 8259A interrupt line asserted until INTA.  --macro
 */
J
Jacek Luczak 已提交
2203
static inline void __init unlock_ExtINT_logic(void)
L
Linus Torvalds 已提交
2204
{
2205
	int apic, pin, i;
L
Linus Torvalds 已提交
2206 2207 2208
	struct IO_APIC_route_entry entry0, entry1;
	unsigned char save_control, save_freq_select;

2209 2210
	pin  = find_isa_irq_pin(8, mp_INT);
	apic = find_isa_irq_apic(8, mp_INT);
L
Linus Torvalds 已提交
2211 2212 2213
	if (pin == -1)
		return;

2214 2215
	entry0 = ioapic_read_entry(apic, pin);

2216
	clear_IO_APIC_pin(apic, pin);
L
Linus Torvalds 已提交
2217 2218 2219 2220 2221

	memset(&entry1, 0, sizeof(entry1));

	entry1.dest_mode = 0;			/* physical delivery */
	entry1.mask = 0;			/* unmask IRQ now */
2222
	entry1.dest = hard_smp_processor_id();
L
Linus Torvalds 已提交
2223 2224 2225 2226 2227
	entry1.delivery_mode = dest_ExtINT;
	entry1.polarity = entry0.polarity;
	entry1.trigger = 0;
	entry1.vector = 0;

2228
	ioapic_write_entry(apic, pin, entry1);
L
Linus Torvalds 已提交
2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244

	save_control = CMOS_READ(RTC_CONTROL);
	save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
	CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
		   RTC_FREQ_SELECT);
	CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);

	i = 100;
	while (i-- > 0) {
		mdelay(10);
		if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
			i -= 10;
	}

	CMOS_WRITE(save_control, RTC_CONTROL);
	CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2245
	clear_IO_APIC_pin(apic, pin);
L
Linus Torvalds 已提交
2246

2247
	ioapic_write_entry(apic, pin, entry0);
L
Linus Torvalds 已提交
2248 2249
}

Y
Yinghai Lu 已提交
2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260
static int disable_timer_pin_1 __initdata;
/* Actually the next is obsolete, but keep it for paranoid reasons -AK */
static int __init disable_timer_pin_setup(char *arg)
{
	disable_timer_pin_1 = 1;
	return 0;
}
early_param("disable_timer_pin_1", disable_timer_pin_setup);

int timer_through_8259 __initdata;

L
Linus Torvalds 已提交
2261 2262 2263 2264 2265
/*
 * This code may look a bit paranoid, but it's supposed to cooperate with
 * a wide range of boards and BIOS bugs.  Fortunately only the timer IRQ
 * is so screwy.  Thanks to Brian Perkins for testing/hacking this beast
 * fanatically on his truly buggy board.
2266 2267
 *
 * FIXME: really need to revamp this for modern platforms only.
L
Linus Torvalds 已提交
2268
 */
2269
static inline void __init check_timer(void)
L
Linus Torvalds 已提交
2270
{
Y
Yinghai Lu 已提交
2271
	struct irq_cfg *cfg = irq_cfg(0);
2272
	int apic1, pin1, apic2, pin2;
2273
	unsigned long flags;
2274
	int no_pin1 = 0;
2275 2276

	local_irq_save(flags);
L
Linus Torvalds 已提交
2277 2278 2279 2280 2281

	/*
	 * get/set the timer IRQ vector:
	 */
	disable_8259A_irq(0);
2282
	assign_irq_vector(0, TARGET_CPUS);
L
Linus Torvalds 已提交
2283 2284

	/*
2285 2286
	 * As IRQ0 is to be enabled in the 8259A, the virtual
	 * wire has to be disabled in the local APIC.
L
Linus Torvalds 已提交
2287
	 */
2288
	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
L
Linus Torvalds 已提交
2289 2290
	init_8259A(1);

2291 2292 2293 2294
	pin1  = find_isa_irq_pin(0, mp_INT);
	apic1 = find_isa_irq_apic(0, mp_INT);
	pin2  = ioapic_i8259.pin;
	apic2 = ioapic_i8259.apic;
L
Linus Torvalds 已提交
2295

2296 2297 2298
	apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
		    "apic1=%d pin1=%d apic2=%d pin2=%d\n",
		    cfg->vector, apic1, pin1, apic2, pin2);
2299

2300 2301 2302 2303 2304 2305 2306 2307
	/*
	 * Some BIOS writers are clueless and report the ExtINTA
	 * I/O APIC input from the cascaded 8259A as the timer
	 * interrupt input.  So just in case, if only one pin
	 * was found above, try it both directly and through the
	 * 8259A.
	 */
	if (pin1 == -1) {
Y
Yinghai Lu 已提交
2308
#ifdef CONFIG_INTR_REMAP
2309 2310
		if (intr_remapping_enabled)
			panic("BIOS bug: timer not connected to IO-APIC");
Y
Yinghai Lu 已提交
2311
#endif
2312 2313 2314 2315 2316 2317 2318 2319
		pin1 = pin2;
		apic1 = apic2;
		no_pin1 = 1;
	} else if (pin2 == -1) {
		pin2 = pin1;
		apic2 = apic1;
	}

2320 2321 2322 2323
	if (pin1 != -1) {
		/*
		 * Ok, does IRQ0 through the IOAPIC work?
		 */
2324 2325
		if (no_pin1) {
			add_pin_to_irq(0, apic1, pin1);
I
Ingo Molnar 已提交
2326
			setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
2327
		}
2328
		unmask_IO_APIC_irq(0);
Y
Yinghai Lu 已提交
2329
		if (timer_irq_works()) {
2330 2331 2332 2333 2334 2335
			if (nmi_watchdog == NMI_IO_APIC) {
				setup_nmi();
				enable_8259A_irq(0);
			}
			if (disable_timer_pin_1 > 0)
				clear_IO_APIC_pin(0, pin1);
2336
			goto out;
2337
		}
Y
Yinghai Lu 已提交
2338
#ifdef CONFIG_INTR_REMAP
2339 2340
		if (intr_remapping_enabled)
			panic("timer doesn't work through Interrupt-remapped IO-APIC");
Y
Yinghai Lu 已提交
2341
#endif
2342
		clear_IO_APIC_pin(apic1, pin1);
2343
		if (!no_pin1)
2344
			apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2345
				    "8254 timer not connected to IO-APIC\n");
L
Linus Torvalds 已提交
2346

2347 2348 2349 2350
		apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
			    "(IRQ0) through the 8259A ...\n");
		apic_printk(APIC_QUIET, KERN_INFO
			    "..... (found apic %d pin %d) ...\n", apic2, pin2);
2351 2352 2353
		/*
		 * legacy devices should be connected to IO APIC #0
		 */
2354
		replace_pin_at_irq(0, apic1, pin1, apic2, pin2);
2355
		setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2356
		unmask_IO_APIC_irq(0);
2357
		enable_8259A_irq(0);
2358
		if (timer_irq_works()) {
2359
			apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2360
			timer_through_8259 = 1;
2361
			if (nmi_watchdog == NMI_IO_APIC) {
2362
				disable_8259A_irq(0);
2363
				setup_nmi();
2364
				enable_8259A_irq(0);
2365
			}
2366
			goto out;
2367 2368 2369 2370
		}
		/*
		 * Cleanup, just in case ...
		 */
2371
		disable_8259A_irq(0);
2372
		clear_IO_APIC_pin(apic2, pin2);
2373
		apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
L
Linus Torvalds 已提交
2374 2375
	}

2376
	if (nmi_watchdog == NMI_IO_APIC) {
2377 2378
		apic_printk(APIC_QUIET, KERN_WARNING "timer doesn't work "
			    "through the IO-APIC - disabling NMI Watchdog!\n");
2379
		nmi_watchdog = NMI_NONE;
L
Linus Torvalds 已提交
2380 2381
	}

2382 2383
	apic_printk(APIC_QUIET, KERN_INFO
		    "...trying to set up timer as Virtual Wire IRQ...\n");
L
Linus Torvalds 已提交
2384

2385
	lapic_register_intr(0);
2386
	apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);	/* Fixed mode */
L
Linus Torvalds 已提交
2387 2388 2389
	enable_8259A_irq(0);

	if (timer_irq_works()) {
2390
		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2391
		goto out;
L
Linus Torvalds 已提交
2392
	}
2393
	disable_8259A_irq(0);
2394
	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2395
	apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
L
Linus Torvalds 已提交
2396

2397 2398
	apic_printk(APIC_QUIET, KERN_INFO
		    "...trying to set up timer as ExtINT IRQ...\n");
L
Linus Torvalds 已提交
2399 2400 2401

	init_8259A(0);
	make_8259A_irq(0);
2402
	apic_write(APIC_LVT0, APIC_DM_EXTINT);
L
Linus Torvalds 已提交
2403 2404 2405 2406

	unlock_ExtINT_logic();

	if (timer_irq_works()) {
2407
		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2408
		goto out;
L
Linus Torvalds 已提交
2409
	}
2410 2411 2412
	apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
	panic("IO-APIC + timer doesn't work!  Boot with apic=debug and send a "
		"report.  Then try booting with the 'noapic' option.\n");
2413 2414
out:
	local_irq_restore(flags);
L
Linus Torvalds 已提交
2415 2416 2417
}

/*
2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432
 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
 * to devices.  However there may be an I/O APIC pin available for
 * this interrupt regardless.  The pin may be left unconnected, but
 * typically it will be reused as an ExtINT cascade interrupt for
 * the master 8259A.  In the MPS case such a pin will normally be
 * reported as an ExtINT interrupt in the MP table.  With ACPI
 * there is no provision for ExtINT interrupts, and in the absence
 * of an override it would be treated as an ordinary ISA I/O APIC
 * interrupt, that is edge-triggered and unmasked by default.  We
 * used to do this, but it caused problems on some systems because
 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
 * the same ExtINT cascade interrupt to drive the local APIC of the
 * bootstrap processor.  Therefore we refrain from routing IRQ2 to
 * the I/O APIC in all cases now.  No actual device should request
 * it anyway.  --macro
L
Linus Torvalds 已提交
2433 2434 2435 2436 2437
 */
#define PIC_IRQS	(1<<2)

void __init setup_IO_APIC(void)
{
2438 2439 2440 2441

	/*
	 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
	 */
L
Linus Torvalds 已提交
2442

2443
	io_apic_irqs = ~PIC_IRQS;
L
Linus Torvalds 已提交
2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458

	apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");

	sync_Arb_IDs();
	setup_IO_APIC_irqs();
	init_IO_APIC_traps();
	check_timer();
}

struct sysfs_ioapic_data {
	struct sys_device dev;
	struct IO_APIC_route_entry entry[0];
};
static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS];

2459
static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
L
Linus Torvalds 已提交
2460 2461 2462 2463 2464 2465 2466
{
	struct IO_APIC_route_entry *entry;
	struct sysfs_ioapic_data *data;
	int i;

	data = container_of(dev, struct sysfs_ioapic_data, dev);
	entry = data->entry;
2467 2468
	for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ )
		*entry = ioapic_read_entry(dev->id, i);
L
Linus Torvalds 已提交
2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485

	return 0;
}

static int ioapic_resume(struct sys_device *dev)
{
	struct IO_APIC_route_entry *entry;
	struct sysfs_ioapic_data *data;
	unsigned long flags;
	union IO_APIC_reg_00 reg_00;
	int i;

	data = container_of(dev, struct sysfs_ioapic_data, dev);
	entry = data->entry;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(dev->id, 0);
2486 2487
	if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
		reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
L
Linus Torvalds 已提交
2488 2489 2490
		io_apic_write(dev->id, 0, reg_00.raw);
	}
	spin_unlock_irqrestore(&ioapic_lock, flags);
2491 2492
	for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
		ioapic_write_entry(dev->id, i, entry[i]);
L
Linus Torvalds 已提交
2493 2494 2495 2496 2497

	return 0;
}

static struct sysdev_class ioapic_sysdev_class = {
2498
	.name = "ioapic",
L
Linus Torvalds 已提交
2499 2500 2501 2502 2503 2504 2505
	.suspend = ioapic_suspend,
	.resume = ioapic_resume,
};

static int __init ioapic_init_sysfs(void)
{
	struct sys_device * dev;
2506
	int i, size, error;
L
Linus Torvalds 已提交
2507 2508 2509 2510 2511 2512 2513 2514

	error = sysdev_class_register(&ioapic_sysdev_class);
	if (error)
		return error;

	for (i = 0; i < nr_ioapics; i++ ) {
		size = sizeof(struct sys_device) + nr_ioapic_registers[i]
			* sizeof(struct IO_APIC_route_entry);
2515
		mp_ioapic_data[i] = kzalloc(size, GFP_KERNEL);
L
Linus Torvalds 已提交
2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536
		if (!mp_ioapic_data[i]) {
			printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
			continue;
		}
		dev = &mp_ioapic_data[i]->dev;
		dev->id = i;
		dev->cls = &ioapic_sysdev_class;
		error = sysdev_register(dev);
		if (error) {
			kfree(mp_ioapic_data[i]);
			mp_ioapic_data[i] = NULL;
			printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
			continue;
		}
	}

	return 0;
}

device_initcall(ioapic_init_sysfs);

2537
/*
2538
 * Dynamic irq allocate and deallocation
2539
 */
2540
unsigned int create_irq_nr(unsigned int irq_want)
2541
{
2542
	/* Allocate an unused irq */
2543 2544
	unsigned int irq;
	unsigned int new;
2545
	unsigned long flags;
Y
Yinghai Lu 已提交
2546
	struct irq_cfg *cfg_new;
2547

2548 2549 2550 2551 2552
#ifndef CONFIG_HAVE_SPARSE_IRQ
	irq_want = nr_irqs - 1;
#endif

	irq = 0;
2553
	spin_lock_irqsave(&vector_lock, flags);
2554
	for (new = irq_want; new > 0; new--) {
2555 2556
		if (platform_legacy_irq(new))
			continue;
Y
Yinghai Lu 已提交
2557 2558
		cfg_new = irq_cfg(new);
		if (cfg_new && cfg_new->vector != 0)
2559
			continue;
Y
Yinghai Lu 已提交
2560 2561 2562
		/* check if need to create one */
		if (!cfg_new)
			cfg_new = irq_cfg_alloc(new);
2563
		if (__assign_irq_vector(new, TARGET_CPUS) == 0)
2564 2565 2566 2567
			irq = new;
		break;
	}
	spin_unlock_irqrestore(&vector_lock, flags);
2568

2569
	if (irq > 0) {
2570 2571 2572 2573 2574
		dynamic_irq_init(irq);
	}
	return irq;
}

2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586
int create_irq(void)
{
	int irq;

	irq = create_irq_nr(nr_irqs - 1);

	if (irq == 0)
		irq = -1;

	return irq;
}

2587 2588 2589 2590 2591 2592
void destroy_irq(unsigned int irq)
{
	unsigned long flags;

	dynamic_irq_cleanup(irq);

2593 2594 2595
#ifdef CONFIG_INTR_REMAP
	free_irte(irq);
#endif
2596
	spin_lock_irqsave(&vector_lock, flags);
2597
	__clear_irq_vector(irq);
2598 2599 2600
	spin_unlock_irqrestore(&vector_lock, flags);
}

2601
/*
S
Simon Arlott 已提交
2602
 * MSI message composition
2603 2604
 */
#ifdef CONFIG_PCI_MSI
2605
static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
2606
{
Y
Yinghai Lu 已提交
2607
	struct irq_cfg *cfg;
2608
	int err;
2609
	unsigned dest;
2610
	cpumask_t tmp;
2611

2612 2613
	tmp = TARGET_CPUS;
	err = assign_irq_vector(irq, tmp);
2614 2615 2616
	if (err)
		return err;

Y
Yinghai Lu 已提交
2617
	cfg = irq_cfg(irq);
2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639
	cpus_and(tmp, cfg->domain, tmp);
	dest = cpu_mask_to_apicid(tmp);

#ifdef CONFIG_INTR_REMAP
	if (irq_remapped(irq)) {
		struct irte irte;
		int ir_index;
		u16 sub_handle;

		ir_index = map_irq_to_irte_handle(irq, &sub_handle);
		BUG_ON(ir_index == -1);

		memset (&irte, 0, sizeof(irte));

		irte.present = 1;
		irte.dst_mode = INT_DEST_MODE;
		irte.trigger_mode = 0; /* edge */
		irte.dlvry_mode = INT_DELIVERY_MODE;
		irte.vector = cfg->vector;
		irte.dest_id = IRTE_DEST(dest);

		modify_irte(irq, &irte);
2640

2641 2642 2643 2644 2645 2646 2647 2648 2649
		msg->address_hi = MSI_ADDR_BASE_HI;
		msg->data = sub_handle;
		msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
				  MSI_ADDR_IR_SHV |
				  MSI_ADDR_IR_INDEX1(ir_index) |
				  MSI_ADDR_IR_INDEX2(ir_index);
	} else
#endif
	{
2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666
		msg->address_hi = MSI_ADDR_BASE_HI;
		msg->address_lo =
			MSI_ADDR_BASE_LO |
			((INT_DEST_MODE == 0) ?
				MSI_ADDR_DEST_MODE_PHYSICAL:
				MSI_ADDR_DEST_MODE_LOGICAL) |
			((INT_DELIVERY_MODE != dest_LowestPrio) ?
				MSI_ADDR_REDIRECTION_CPU:
				MSI_ADDR_REDIRECTION_LOWPRI) |
			MSI_ADDR_DEST_ID(dest);

		msg->data =
			MSI_DATA_TRIGGER_EDGE |
			MSI_DATA_LEVEL_ASSERT |
			((INT_DELIVERY_MODE != dest_LowestPrio) ?
				MSI_DATA_DELIVERY_FIXED:
				MSI_DATA_DELIVERY_LOWPRI) |
2667
			MSI_DATA_VECTOR(cfg->vector);
2668
	}
2669
	return err;
2670 2671
}

2672 2673
#ifdef CONFIG_SMP
static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
2674
{
Y
Yinghai Lu 已提交
2675
	struct irq_cfg *cfg;
2676 2677 2678
	struct msi_msg msg;
	unsigned int dest;
	cpumask_t tmp;
2679
	struct irq_desc *desc;
2680 2681 2682

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
2683
		return;
2684

2685
	if (assign_irq_vector(irq, mask))
2686
		return;
2687

Y
Yinghai Lu 已提交
2688
	cfg = irq_cfg(irq);
2689
	cpus_and(tmp, cfg->domain, mask);
2690
	dest = cpu_mask_to_apicid(tmp);
2691

2692 2693 2694
	read_msi_msg(irq, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
2695
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
2696 2697 2698 2699
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	write_msi_msg(irq, &msg);
2700 2701
	desc = irq_to_desc(irq);
	desc->affinity = mask;
2702
}
2703 2704 2705 2706 2707 2708 2709 2710

#ifdef CONFIG_INTR_REMAP
/*
 * Migrate the MSI irq to another cpumask. This migration is
 * done in the process context using interrupt-remapping hardware.
 */
static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
{
Y
Yinghai Lu 已提交
2711
	struct irq_cfg *cfg;
2712 2713 2714
	unsigned int dest;
	cpumask_t tmp, cleanup_mask;
	struct irte irte;
2715
	struct irq_desc *desc;
2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	if (get_irte(irq, &irte))
		return;

	if (assign_irq_vector(irq, mask))
		return;

Y
Yinghai Lu 已提交
2727
	cfg = irq_cfg(irq);
2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750
	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	irte.vector = cfg->vector;
	irte.dest_id = IRTE_DEST(dest);

	/*
	 * atomically update the IRTE with the new destination and vector.
	 */
	modify_irte(irq, &irte);

	/*
	 * After this point, all the interrupts will start arriving
	 * at the new destination. So, time to cleanup the previous
	 * vector allocation.
	 */
	if (cfg->move_in_progress) {
		cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
		cfg->move_cleanup_count = cpus_weight(cleanup_mask);
		send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
		cfg->move_in_progress = 0;
	}

2751 2752
	desc = irq_to_desc(irq);
	desc->affinity = mask;
2753 2754
}
#endif
2755
#endif /* CONFIG_SMP */
2756

2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769
/*
 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
 * which implement the MSI or MSI-X Capability Structure.
 */
static struct irq_chip msi_chip = {
	.name		= "PCI-MSI",
	.unmask		= unmask_msi_irq,
	.mask		= mask_msi_irq,
	.ack		= ack_apic_edge,
#ifdef CONFIG_SMP
	.set_affinity	= set_msi_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
2770 2771
};

2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789
#ifdef CONFIG_INTR_REMAP
static struct irq_chip msi_ir_chip = {
	.name		= "IR-PCI-MSI",
	.unmask		= unmask_msi_irq,
	.mask		= mask_msi_irq,
	.ack		= ack_x2apic_edge,
#ifdef CONFIG_SMP
	.set_affinity	= ir_set_msi_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
};

/*
 * Map the PCI dev to the corresponding remapping hardware unit
 * and allocate 'nvec' consecutive interrupt-remapping table entries
 * in it.
 */
static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
2790
{
2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814
	struct intel_iommu *iommu;
	int index;

	iommu = map_dev_to_ir(dev);
	if (!iommu) {
		printk(KERN_ERR
		       "Unable to map PCI %s to iommu\n", pci_name(dev));
		return -ENOENT;
	}

	index = alloc_irte(iommu, irq, nvec);
	if (index < 0) {
		printk(KERN_ERR
		       "Unable to allocate %d IRTE for PCI %s\n", nvec,
		        pci_name(dev));
		return -ENOSPC;
	}
	return index;
}
#endif

static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq)
{
	int ret;
2815
	struct msi_msg msg;
2816 2817 2818 2819 2820 2821 2822 2823 2824 2825

	ret = msi_compose_msg(dev, irq, &msg);
	if (ret < 0)
		return ret;

	set_irq_msi(irq, desc);
	write_msi_msg(irq, &msg);

#ifdef CONFIG_INTR_REMAP
	if (irq_remapped(irq)) {
2826
		struct irq_desc *desc = irq_to_desc(irq);
2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838
		/*
		 * irq migration in process context
		 */
		desc->status |= IRQ_MOVE_PCNTXT;
		set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge");
	} else
#endif
		set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");

	return 0;
}

2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850
static unsigned int build_irq_for_pci_dev(struct pci_dev *dev)
{
	unsigned int irq;

	irq = dev->bus->number;
	irq <<= 8;
	irq |= dev->devfn;
	irq <<= 12;

	return irq;
}

2851 2852
int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
{
2853 2854 2855
	unsigned int irq;
	int ret;
	unsigned int irq_want;
2856

2857 2858 2859 2860 2861
	irq_want = build_irq_for_pci_dev(dev) + 0x100;

	irq = create_irq_nr(irq_want);
	if (irq == 0)
		return -1;
2862

2863 2864 2865 2866 2867 2868 2869 2870 2871 2872
#ifdef CONFIG_INTR_REMAP
	if (!intr_remapping_enabled)
		goto no_ir;

	ret = msi_alloc_irte(dev, irq, 1);
	if (ret < 0)
		goto error;
no_ir:
#endif
	ret = setup_msi_irq(dev, desc, irq);
2873 2874
	if (ret < 0) {
		destroy_irq(irq);
2875
		return ret;
2876
	}
2877
	return 0;
2878

2879 2880 2881 2882 2883 2884
#ifdef CONFIG_INTR_REMAP
error:
	destroy_irq(irq);
	return ret;
#endif
}
2885

2886 2887
int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
{
2888 2889
	unsigned int irq;
	int ret, sub_handle;
2890
	struct msi_desc *desc;
2891 2892
	unsigned int irq_want;

2893 2894 2895 2896 2897
#ifdef CONFIG_INTR_REMAP
	struct intel_iommu *iommu = 0;
	int index = 0;
#endif

2898
	irq_want = build_irq_for_pci_dev(dev) + 0x100;
2899 2900
	sub_handle = 0;
	list_for_each_entry(desc, &dev->msi_list, list) {
2901 2902 2903
		irq = create_irq_nr(irq_want--);
		if (irq == 0)
			return -1;
2904 2905 2906
#ifdef CONFIG_INTR_REMAP
		if (!intr_remapping_enabled)
			goto no_ir;
2907

2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937
		if (!sub_handle) {
			/*
			 * allocate the consecutive block of IRTE's
			 * for 'nvec'
			 */
			index = msi_alloc_irte(dev, irq, nvec);
			if (index < 0) {
				ret = index;
				goto error;
			}
		} else {
			iommu = map_dev_to_ir(dev);
			if (!iommu) {
				ret = -ENOENT;
				goto error;
			}
			/*
			 * setup the mapping between the irq and the IRTE
			 * base index, the sub_handle pointing to the
			 * appropriate interrupt remap table entry.
			 */
			set_irte_irq(irq, iommu, index, sub_handle);
		}
no_ir:
#endif
		ret = setup_msi_irq(dev, desc, irq);
		if (ret < 0)
			goto error;
		sub_handle++;
	}
2938
	return 0;
2939 2940 2941 2942

error:
	destroy_irq(irq);
	return ret;
2943 2944 2945 2946
}

void arch_teardown_msi_irq(unsigned int irq)
{
2947
	destroy_irq(irq);
2948 2949
}

2950 2951 2952 2953
#ifdef CONFIG_DMAR
#ifdef CONFIG_SMP
static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask)
{
Y
Yinghai Lu 已提交
2954
	struct irq_cfg *cfg;
2955 2956 2957
	struct msi_msg msg;
	unsigned int dest;
	cpumask_t tmp;
2958
	struct irq_desc *desc;
2959 2960 2961 2962 2963 2964 2965 2966

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	if (assign_irq_vector(irq, mask))
		return;

Y
Yinghai Lu 已提交
2967
	cfg = irq_cfg(irq);
2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978
	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	dmar_msi_read(irq, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	dmar_msi_write(irq, &msg);
2979 2980
	desc = irq_to_desc(irq);
	desc->affinity = mask;
2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008
}
#endif /* CONFIG_SMP */

struct irq_chip dmar_msi_type = {
	.name = "DMAR_MSI",
	.unmask = dmar_msi_unmask,
	.mask = dmar_msi_mask,
	.ack = ack_apic_edge,
#ifdef CONFIG_SMP
	.set_affinity = dmar_msi_set_affinity,
#endif
	.retrigger = ioapic_retrigger_irq,
};

int arch_setup_dmar_msi(unsigned int irq)
{
	int ret;
	struct msi_msg msg;

	ret = msi_compose_msg(NULL, irq, &msg);
	if (ret < 0)
		return ret;
	dmar_msi_write(irq, &msg);
	set_irq_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
		"edge");
	return 0;
}
#endif
3009

3010
#endif /* CONFIG_PCI_MSI */
3011 3012 3013 3014 3015 3016 3017 3018 3019
/*
 * Hypertransport interrupt support
 */
#ifdef CONFIG_HT_IRQ

#ifdef CONFIG_SMP

static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
{
3020 3021
	struct ht_irq_msg msg;
	fetch_ht_irq_msg(irq, &msg);
3022

3023 3024
	msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
	msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
3025

3026 3027
	msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
	msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
3028

3029
	write_ht_irq_msg(irq, &msg);
3030 3031 3032 3033
}

static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
{
Y
Yinghai Lu 已提交
3034
	struct irq_cfg *cfg;
3035 3036
	unsigned int dest;
	cpumask_t tmp;
3037
	struct irq_desc *desc;
3038 3039 3040

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
3041
		return;
3042

3043
	if (assign_irq_vector(irq, mask))
3044 3045
		return;

Y
Yinghai Lu 已提交
3046
	cfg = irq_cfg(irq);
3047
	cpus_and(tmp, cfg->domain, mask);
3048 3049
	dest = cpu_mask_to_apicid(tmp);

3050
	target_ht_irq(irq, dest, cfg->vector);
3051 3052
	desc = irq_to_desc(irq);
	desc->affinity = mask;
3053 3054 3055
}
#endif

3056
static struct irq_chip ht_irq_chip = {
3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068
	.name		= "PCI-HT",
	.mask		= mask_ht_irq,
	.unmask		= unmask_ht_irq,
	.ack		= ack_apic_edge,
#ifdef CONFIG_SMP
	.set_affinity	= set_ht_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
};

int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
{
Y
Yinghai Lu 已提交
3069
	struct irq_cfg *cfg;
3070
	int err;
3071
	cpumask_t tmp;
3072

3073 3074 3075
	tmp = TARGET_CPUS;
	err = assign_irq_vector(irq, tmp);
	if (!err) {
3076
		struct ht_irq_msg msg;
3077 3078
		unsigned dest;

Y
Yinghai Lu 已提交
3079
		cfg = irq_cfg(irq);
3080
		cpus_and(tmp, cfg->domain, tmp);
3081 3082
		dest = cpu_mask_to_apicid(tmp);

3083
		msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
3084

3085 3086
		msg.address_lo =
			HT_IRQ_LOW_BASE |
3087
			HT_IRQ_LOW_DEST_ID(dest) |
3088
			HT_IRQ_LOW_VECTOR(cfg->vector) |
3089 3090 3091 3092 3093 3094
			((INT_DEST_MODE == 0) ?
				HT_IRQ_LOW_DM_PHYSICAL :
				HT_IRQ_LOW_DM_LOGICAL) |
			HT_IRQ_LOW_RQEOI_EDGE |
			((INT_DELIVERY_MODE != dest_LowestPrio) ?
				HT_IRQ_LOW_MT_FIXED :
3095 3096
				HT_IRQ_LOW_MT_ARBITRATED) |
			HT_IRQ_LOW_IRQ_MASKED;
3097

3098
		write_ht_irq_msg(irq, &msg);
3099

3100 3101
		set_irq_chip_and_handler_name(irq, &ht_irq_chip,
					      handle_edge_irq, "edge");
3102
	}
3103
	return err;
3104 3105 3106
}
#endif /* CONFIG_HT_IRQ */

L
Linus Torvalds 已提交
3107 3108 3109 3110
/* --------------------------------------------------------------------------
                          ACPI-based IOAPIC Configuration
   -------------------------------------------------------------------------- */

L
Len Brown 已提交
3111
#ifdef CONFIG_ACPI
L
Linus Torvalds 已提交
3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127

#define IO_APIC_MAX_ID		0xFE

int __init io_apic_get_redir_entries (int ioapic)
{
	union IO_APIC_reg_01	reg_01;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_01.raw = io_apic_read(ioapic, 1);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return reg_01.bits.entries;
}


B
Bob Moore 已提交
3128
int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
L
Linus Torvalds 已提交
3129 3130 3131 3132 3133 3134 3135
{
	if (!IO_APIC_IRQ(irq)) {
		apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
			ioapic);
		return -EINVAL;
	}

3136 3137 3138 3139 3140 3141
	/*
	 * IRQs < 16 are already in the irq_2_pin[] map
	 */
	if (irq >= 16)
		add_pin_to_irq(irq, ioapic, pin);

3142
	setup_IO_APIC_irq(ioapic, pin, irq, triggering, polarity);
L
Linus Torvalds 已提交
3143 3144 3145 3146 3147

	return 0;
}


3148 3149 3150 3151 3152 3153 3154 3155
int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
{
	int i;

	if (skip_ioapic_setup)
		return -1;

	for (i = 0; i < mp_irq_entries; i++)
3156 3157
		if (mp_irqs[i].mp_irqtype == mp_INT &&
		    mp_irqs[i].mp_srcbusirq == bus_irq)
3158 3159 3160 3161 3162 3163 3164 3165 3166 3167
			break;
	if (i >= mp_irq_entries)
		return -1;

	*trigger = irq_trigger(i);
	*polarity = irq_polarity(i);
	return 0;
}

#endif /* CONFIG_ACPI */
L
Linus Torvalds 已提交
3168 3169 3170 3171 3172 3173

/*
 * This function currently is only a helper for the i386 smp boot process where
 * we need to reprogram the ioredtbls to cater for the cpus which have come online
 * so mask in all cases should simply be TARGET_CPUS
 */
3174
#ifdef CONFIG_SMP
L
Linus Torvalds 已提交
3175 3176 3177
void __init setup_ioapic_dest(void)
{
	int pin, ioapic, irq, irq_entry;
Y
Yinghai Lu 已提交
3178
	struct irq_cfg *cfg;
L
Linus Torvalds 已提交
3179 3180 3181 3182 3183 3184 3185 3186 3187 3188

	if (skip_ioapic_setup == 1)
		return;

	for (ioapic = 0; ioapic < nr_ioapics; ioapic++) {
		for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
			irq_entry = find_irq_entry(ioapic, pin, mp_INT);
			if (irq_entry == -1)
				continue;
			irq = pin_2_irq(irq_entry, ioapic, pin);
3189 3190 3191 3192 3193

			/* setup_IO_APIC_irqs could fail to get vector for some device
			 * when you have too many devices, because at that time only boot
			 * cpu is online.
			 */
Y
Yinghai Lu 已提交
3194 3195
			cfg = irq_cfg(irq);
			if (!cfg->vector)
3196 3197 3198
				setup_IO_APIC_irq(ioapic, pin, irq,
						  irq_trigger(irq_entry),
						  irq_polarity(irq_entry));
3199 3200 3201 3202
#ifdef CONFIG_INTR_REMAP
			else if (intr_remapping_enabled)
				set_ir_ioapic_affinity_irq(irq, TARGET_CPUS);
#endif
3203 3204
			else
				set_ioapic_affinity_irq(irq, TARGET_CPUS);
L
Linus Torvalds 已提交
3205 3206 3207 3208
		}

	}
}
3209
#endif
3210

3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255
#define IOAPIC_RESOURCE_NAME_SIZE 11

static struct resource *ioapic_resources;

static struct resource * __init ioapic_setup_resources(void)
{
	unsigned long n;
	struct resource *res;
	char *mem;
	int i;

	if (nr_ioapics <= 0)
		return NULL;

	n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
	n *= nr_ioapics;

	mem = alloc_bootmem(n);
	res = (void *)mem;

	if (mem != NULL) {
		mem += sizeof(struct resource) * nr_ioapics;

		for (i = 0; i < nr_ioapics; i++) {
			res[i].name = mem;
			res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
			sprintf(mem,  "IOAPIC %u", i);
			mem += IOAPIC_RESOURCE_NAME_SIZE;
		}
	}

	ioapic_resources = res;

	return res;
}

void __init ioapic_init_mappings(void)
{
	unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
	struct resource *ioapic_res;
	int i;

	ioapic_res = ioapic_setup_resources();
	for (i = 0; i < nr_ioapics; i++) {
		if (smp_found_config) {
3256
			ioapic_phys = mp_ioapics[i].mp_apicaddr;
3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298
		} else {
			ioapic_phys = (unsigned long)
				alloc_bootmem_pages(PAGE_SIZE);
			ioapic_phys = __pa(ioapic_phys);
		}
		set_fixmap_nocache(idx, ioapic_phys);
		apic_printk(APIC_VERBOSE,
			    "mapped IOAPIC to %016lx (%016lx)\n",
			    __fix_to_virt(idx), ioapic_phys);
		idx++;

		if (ioapic_res != NULL) {
			ioapic_res->start = ioapic_phys;
			ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
			ioapic_res++;
		}
	}
}

static int __init ioapic_insert_resources(void)
{
	int i;
	struct resource *r = ioapic_resources;

	if (!r) {
		printk(KERN_ERR
		       "IO APIC resources could be not be allocated.\n");
		return -1;
	}

	for (i = 0; i < nr_ioapics; i++) {
		insert_resource(&iomem_resource, r);
		r++;
	}

	return 0;
}

/* Insert the IO APIC resources after PCI initialization has occured to handle
 * IO APICS that are mapped in on a BAR in PCI space. */
late_initcall(ioapic_insert_resources);