io_apic.c 93.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 *	Intel IO-APIC support for multi-Pentium hosts.
 *
 *	Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar, Hajnalka Szabo
 *
 *	Many thanks to Stig Venaas for trying out countless experimental
 *	patches and reporting/debugging problems patiently!
 *
 *	(c) 1999, Multiple IO-APIC support, developed by
 *	Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
 *      Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
 *	further tested and cleaned up by Zach Brown <zab@redhat.com>
 *	and Ingo Molnar <mingo@redhat.com>
 *
 *	Fixes
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
 *					thanks to Eric Gilmore
 *					and Rolf G. Tews
 *					for testing these extensively
 *	Paul Diefenbaugh	:	Added full ACPI support
 */

#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/sched.h>
28
#include <linux/pci.h>
L
Linus Torvalds 已提交
29 30 31
#include <linux/mc146818rtc.h>
#include <linux/compiler.h>
#include <linux/acpi.h>
32
#include <linux/module.h>
L
Linus Torvalds 已提交
33
#include <linux/sysdev.h>
34
#include <linux/msi.h>
35
#include <linux/htirq.h>
36
#include <linux/freezer.h>
37
#include <linux/kthread.h>
38
#include <linux/jiffies.h>	/* time_after() */
39 40 41 42 43
#ifdef CONFIG_ACPI
#include <acpi/acpi_bus.h>
#endif
#include <linux/bootmem.h>
#include <linux/dmar.h>
44

45
#include <asm/idle.h>
L
Linus Torvalds 已提交
46 47 48
#include <asm/io.h>
#include <asm/smp.h>
#include <asm/desc.h>
49 50 51
#include <asm/proto.h>
#include <asm/acpi.h>
#include <asm/dma.h>
L
Linus Torvalds 已提交
52
#include <asm/timer.h>
53
#include <asm/i8259.h>
54
#include <asm/nmi.h>
55
#include <asm/msidef.h>
56
#include <asm/hypertransport.h>
57
#include <asm/setup.h>
58
#include <asm/irq_remapping.h>
L
Linus Torvalds 已提交
59

60
#include <mach_ipi.h>
L
Linus Torvalds 已提交
61
#include <mach_apic.h>
62
#include <mach_apicdef.h>
L
Linus Torvalds 已提交
63

64 65
#define __apicdebuginit(type) static type __init

L
Linus Torvalds 已提交
66
/*
67 68
 *      Is the SiS APIC rmw bug present ?
 *      -1 = don't know, 0 = no, 1 = yes
L
Linus Torvalds 已提交
69 70 71
 */
int sis_apic_bug = -1;

Y
Yinghai Lu 已提交
72 73 74
static DEFINE_SPINLOCK(ioapic_lock);
static DEFINE_SPINLOCK(vector_lock);

L
Linus Torvalds 已提交
75 76 77 78 79
/*
 * # of IRQ routing registers
 */
int nr_ioapic_registers[MAX_IO_APICS];

80
/* I/O APIC entries */
81
struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
82 83
int nr_ioapics;

84
/* MP IRQ source entries */
85
struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
86 87 88 89

/* # of MP IRQ source entries */
int mp_irq_entries;

90 91 92 93 94 95
#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
int mp_bus_id_to_type[MAX_MP_BUSSES];
#endif

DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);

Y
Yinghai Lu 已提交
96 97
int skip_ioapic_setup;

98
static int __init parse_noapic(char *str)
Y
Yinghai Lu 已提交
99 100 101 102 103 104
{
	/* disable IO-APIC */
	disable_ioapic_setup();
	return 0;
}
early_param("noapic", parse_noapic);
105

106
struct irq_cfg;
107
struct irq_pin_list;
Y
Yinghai Lu 已提交
108
struct irq_cfg {
109
	unsigned int irq;
110
#ifdef CONFIG_HAVE_SPARSE_IRQ
111
	struct irq_cfg *next;
112
#endif
113
	struct irq_pin_list *irq_2_pin;
114 115 116
	cpumask_t domain;
	cpumask_t old_domain;
	unsigned move_cleanup_count;
Y
Yinghai Lu 已提交
117
	u8 vector;
118
	u8 move_in_progress : 1;
Y
Yinghai Lu 已提交
119 120 121 122
};

/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
static struct irq_cfg irq_cfg_legacy[] __initdata = {
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
	[0]  = { .irq =  0, .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR,  },
	[1]  = { .irq =  1, .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR,  },
	[2]  = { .irq =  2, .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR,  },
	[3]  = { .irq =  3, .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR,  },
	[4]  = { .irq =  4, .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR,  },
	[5]  = { .irq =  5, .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR,  },
	[6]  = { .irq =  6, .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR,  },
	[7]  = { .irq =  7, .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR,  },
	[8]  = { .irq =  8, .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR,  },
	[9]  = { .irq =  9, .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR,  },
	[10] = { .irq = 10, .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, },
	[11] = { .irq = 11, .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, },
	[12] = { .irq = 12, .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, },
	[13] = { .irq = 13, .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, },
	[14] = { .irq = 14, .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, },
	[15] = { .irq = 15, .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, },
Y
Yinghai Lu 已提交
139 140
};

141 142 143 144 145 146 147 148
static struct irq_cfg irq_cfg_init = { .irq =  -1U, };

static void init_one_irq_cfg(struct irq_cfg *cfg)
{
	memcpy(cfg, &irq_cfg_init, sizeof(struct irq_cfg));
}

static struct irq_cfg *irq_cfgx;
149 150 151 152 153 154

/*
 * Protect the irq_cfgx_free freelist:
 */
static DEFINE_SPINLOCK(irq_cfg_lock);

155
#ifdef CONFIG_HAVE_SPARSE_IRQ
156
static struct irq_cfg *irq_cfgx_free;
157
#endif
Y
Yinghai Lu 已提交
158 159
static void __init init_work(void *data)
{
160 161 162 163 164 165
	struct dyn_array *da = data;
	struct irq_cfg *cfg;
	int legacy_count;
	int i;

	cfg = *da->name;
Y
Yinghai Lu 已提交
166

167
	memcpy(cfg, irq_cfg_legacy, sizeof(irq_cfg_legacy));
Y
Yinghai Lu 已提交
168

169
	legacy_count = ARRAY_SIZE(irq_cfg_legacy);
170 171
	for (i = legacy_count; i < *da->nr; i++)
		init_one_irq_cfg(&cfg[i]);
Y
Yinghai Lu 已提交
172

173
#ifdef CONFIG_HAVE_SPARSE_IRQ
174 175
	for (i = 1; i < *da->nr; i++)
		cfg[i-1].next = &cfg[i];
Y
Yinghai Lu 已提交
176

177 178
	irq_cfgx_free = &irq_cfgx[legacy_count];
	irq_cfgx[legacy_count - 1].next = NULL;
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
#endif
}

#ifdef CONFIG_HAVE_SPARSE_IRQ
/* need to be biger than size of irq_cfg_legacy */
static int nr_irq_cfg = 32;

static int __init parse_nr_irq_cfg(char *arg)
{
	if (arg) {
		nr_irq_cfg = simple_strtoul(arg, NULL, 0);
		if (nr_irq_cfg < 32)
			nr_irq_cfg = 32;
	}
	return 0;
Y
Yinghai Lu 已提交
194 195
}

196 197 198 199 200
early_param("nr_irq_cfg", parse_nr_irq_cfg);

#define for_each_irq_cfg(irqX, cfg)           \
        for (cfg = irq_cfgx, irqX = cfg->irq; cfg; cfg = cfg->next, irqX = cfg ? cfg->irq : -1U)

201 202

DEFINE_DYN_ARRAY(irq_cfgx, sizeof(struct irq_cfg), nr_irq_cfg, PAGE_SIZE, init_work);
Y
Yinghai Lu 已提交
203 204 205

static struct irq_cfg *irq_cfg(unsigned int irq)
{
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
	struct irq_cfg *cfg;

	cfg = irq_cfgx;
	while (cfg) {
		if (cfg->irq == irq)
			return cfg;

		cfg = cfg->next;
	}

	return NULL;
}

static struct irq_cfg *irq_cfg_alloc(unsigned int irq)
{
	struct irq_cfg *cfg, *cfg_pri;
222
	unsigned long flags;
223
	int count = 0;
224
	int i;
225 226 227 228 229 230 231 232 233 234 235

	cfg_pri = cfg = irq_cfgx;
	while (cfg) {
		if (cfg->irq == irq)
			return cfg;

		cfg_pri = cfg;
		cfg = cfg->next;
		count++;
	}

236
	spin_lock_irqsave(&irq_cfg_lock, flags);
237 238 239 240 241 242 243 244 245 246 247 248 249
	if (!irq_cfgx_free) {
		unsigned long phys;
		unsigned long total_bytes;
		/*
		 *  we run out of pre-allocate ones, allocate more
		 */
		printk(KERN_DEBUG "try to get more irq_cfg %d\n", nr_irq_cfg);

		total_bytes = sizeof(struct irq_cfg) * nr_irq_cfg;
		if (after_bootmem)
			cfg = kzalloc(total_bytes, GFP_ATOMIC);
		else
			cfg = __alloc_bootmem_nopanic(total_bytes, PAGE_SIZE, 0);
Y
Yinghai Lu 已提交
250

251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
		if (!cfg)
			panic("please boot with nr_irq_cfg= %d\n", count * 2);

		phys = __pa(cfg);
		printk(KERN_DEBUG "irq_irq ==> [%#lx - %#lx]\n", phys, phys + total_bytes);

		for (i = 0; i < nr_irq_cfg; i++)
			init_one_irq_cfg(&cfg[i]);

		for (i = 1; i < nr_irq_cfg; i++)
			cfg[i-1].next = &cfg[i];

		irq_cfgx_free = cfg;
	}

	cfg = irq_cfgx_free;
	irq_cfgx_free = irq_cfgx_free->next;
	cfg->next = NULL;
	if (cfg_pri)
		cfg_pri->next = cfg;
	else
		irq_cfgx = cfg;
	cfg->irq = irq;
274 275 276

	spin_unlock_irqrestore(&irq_cfg_lock, flags);

277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
	printk(KERN_DEBUG "found new irq_cfg for irq %d\n", cfg->irq);
#ifdef CONFIG_HAVE_SPARSE_IRQ_DEBUG
	{
		/* dump the results */
		struct irq_cfg *cfg;
		unsigned long phys;
		unsigned long bytes = sizeof(struct irq_cfg);

		printk(KERN_DEBUG "=========================== %d\n", irq);
		printk(KERN_DEBUG "irq_cfg dump after get that for %d\n", irq);
		for_each_irq_cfg(cfg) {
			phys = __pa(cfg);
			printk(KERN_DEBUG "irq_cfg %d ==> [%#lx - %#lx]\n", cfg->irq, phys, phys + bytes);
		}
		printk(KERN_DEBUG "===========================\n");
	}
#endif
	return cfg;
Y
Yinghai Lu 已提交
295
}
296 297 298 299 300 301
#else

#define for_each_irq_cfg(irq, cfg)		\
	for (irq = 0, cfg = &irq_cfgx[irq]; irq < nr_irqs; irq++, cfg = &irq_cfgx[irq])

DEFINE_DYN_ARRAY(irq_cfgx, sizeof(struct irq_cfg), nr_irqs, PAGE_SIZE, init_work);
Y
Yinghai Lu 已提交
302

303 304 305 306 307 308 309 310 311 312 313 314 315
struct irq_cfg *irq_cfg(unsigned int irq)
{
        if (irq < nr_irqs)
                return &irq_cfgx[irq];

        return NULL;
}
struct irq_cfg *irq_cfg_alloc(unsigned int irq)
{
        return irq_cfg(irq);
}

#endif
L
Linus Torvalds 已提交
316 317 318 319 320 321 322
/*
 * This is performance-critical, we want to do it O(1)
 *
 * the indexing order of this array favors 1:1 mappings
 * between pins and IRQs.
 */

323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
struct irq_pin_list {
	int apic, pin;
	struct irq_pin_list *next;
};

static struct irq_pin_list *irq_2_pin_head;
/* fill one page ? */
static int nr_irq_2_pin = 0x100;
static struct irq_pin_list *irq_2_pin_ptr;
static void __init irq_2_pin_init_work(void *data)
{
	struct dyn_array *da = data;
	struct irq_pin_list *pin;
	int i;

	pin = *da->name;

	for (i = 1; i < *da->nr; i++)
		pin[i-1].next = &pin[i];

	irq_2_pin_ptr = &pin[0];
}
DEFINE_DYN_ARRAY(irq_2_pin_head, sizeof(struct irq_pin_list), nr_irq_2_pin, PAGE_SIZE, irq_2_pin_init_work);

static struct irq_pin_list *get_one_free_irq_2_pin(void)
{
	struct irq_pin_list *pin;
	int i;

	pin = irq_2_pin_ptr;

	if (pin) {
		irq_2_pin_ptr = pin->next;
		pin->next = NULL;
		return pin;
	}

	/*
	 *  we run out of pre-allocate ones, allocate more
	 */
	printk(KERN_DEBUG "try to get more irq_2_pin %d\n", nr_irq_2_pin);

	if (after_bootmem)
		pin = kzalloc(sizeof(struct irq_pin_list)*nr_irq_2_pin,
				 GFP_ATOMIC);
	else
		pin = __alloc_bootmem_nopanic(sizeof(struct irq_pin_list) *
				nr_irq_2_pin, PAGE_SIZE, 0);

	if (!pin)
		panic("can not get more irq_2_pin\n");
Y
Yinghai Lu 已提交
374

375 376 377 378 379 380 381 382
	for (i = 1; i < nr_irq_2_pin; i++)
		pin[i-1].next = &pin[i];

	irq_2_pin_ptr = pin->next;
	pin->next = NULL;

	return pin;
}
L
Linus Torvalds 已提交
383

L
Linus Torvalds 已提交
384 385 386 387 388 389 390 391 392
struct io_apic {
	unsigned int index;
	unsigned int unused[3];
	unsigned int data;
};

static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
{
	return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
393
		+ (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
L
Linus Torvalds 已提交
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417
}

static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	return readl(&io_apic->data);
}

static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	writel(value, &io_apic->data);
}

/*
 * Re-write a value: to be used for read-modify-write
 * cycles where the read already set up the index register.
 *
 * Older SiS APIC requires we rewrite the index register
 */
static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
{
418 419 420
	struct io_apic __iomem *io_apic = io_apic_base(apic);
        if (sis_apic_bug)
                writel(reg, &io_apic->index);
L
Linus Torvalds 已提交
421 422 423
	writel(value, &io_apic->data);
}

424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
static bool io_apic_level_ack_pending(unsigned int irq)
{
	struct irq_pin_list *entry;
	unsigned long flags;
	struct irq_cfg *cfg = irq_cfg(irq);

	spin_lock_irqsave(&ioapic_lock, flags);
	entry = cfg->irq_2_pin;
	for (;;) {
		unsigned int reg;
		int pin;

		if (!entry)
			break;
		pin = entry->pin;
		reg = io_apic_read(entry->apic, 0x10 + pin*2);
		/* Is the remote IRR bit set? */
		if (reg & IO_APIC_REDIR_REMOTE_IRR) {
			spin_unlock_irqrestore(&ioapic_lock, flags);
			return true;
		}
		if (!entry->next)
			break;
		entry = entry->next;
	}
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return false;
}

454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
union entry_union {
	struct { u32 w1, w2; };
	struct IO_APIC_route_entry entry;
};

static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
{
	union entry_union eu;
	unsigned long flags;
	spin_lock_irqsave(&ioapic_lock, flags);
	eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
	eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
	spin_unlock_irqrestore(&ioapic_lock, flags);
	return eu.entry;
}

470 471 472 473 474 475
/*
 * When we write a new IO APIC routing entry, we need to write the high
 * word first! If the mask bit in the low word is clear, we will enable
 * the interrupt, and we need to make sure the entry is fully populated
 * before that happens.
 */
476 477
static void
__ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
478 479 480
{
	union entry_union eu;
	eu.entry = e;
481 482
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
483 484 485 486 487 488 489
}

static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
{
	unsigned long flags;
	spin_lock_irqsave(&ioapic_lock, flags);
	__ioapic_write_entry(apic, pin, e);
490 491 492 493 494 495 496 497 498 499 500 501 502
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

/*
 * When we mask an IO APIC routing entry, we need to write the low
 * word first, in order to set the mask bit before we change the
 * high bits!
 */
static void ioapic_mask_entry(int apic, int pin)
{
	unsigned long flags;
	union entry_union eu = { .entry.mask = 1 };

503 504 505 506 507 508
	spin_lock_irqsave(&ioapic_lock, flags);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
#ifdef CONFIG_SMP
static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector)
{
	int apic, pin;
	struct irq_cfg *cfg;
	struct irq_pin_list *entry;

	cfg = irq_cfg(irq);
	entry = cfg->irq_2_pin;
	for (;;) {
		unsigned int reg;

		if (!entry)
			break;

		apic = entry->apic;
		pin = entry->pin;
526 527 528 529 530 531 532 533
#ifdef CONFIG_INTR_REMAP
		/*
		 * With interrupt-remapping, destination information comes
		 * from interrupt-remapping table entry.
		 */
		if (!irq_remapped(irq))
			io_apic_write(apic, 0x11 + pin*2, dest);
#else
534
		io_apic_write(apic, 0x11 + pin*2, dest);
535
#endif
536 537 538
		reg = io_apic_read(apic, 0x10 + pin*2);
		reg &= ~IO_APIC_REDIR_VECTOR_MASK;
		reg |= vector;
539
		io_apic_modify(apic, 0x10 + pin*2, reg);
540 541 542 543 544
		if (!entry->next)
			break;
		entry = entry->next;
	}
}
Y
Yinghai Lu 已提交
545 546 547

static int assign_irq_vector(int irq, cpumask_t mask);

548 549 550 551 552 553
static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
{
	struct irq_cfg *cfg;
	unsigned long flags;
	unsigned int dest;
	cpumask_t tmp;
554
	struct irq_desc *desc;
555 556 557 558 559

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

560
	cfg = irq_cfg(irq);
561 562 563 564 565 566 567 568 569 570
	if (assign_irq_vector(irq, mask))
		return;

	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);
	/*
	 * Only the high 8 bits are valid.
	 */
	dest = SET_APIC_LOGICAL_ID(dest);

571
	desc = irq_to_desc(irq);
572 573
	spin_lock_irqsave(&ioapic_lock, flags);
	__target_IO_APIC_irq(irq, dest, cfg->vector);
574
	desc->affinity = mask;
575 576 577 578
	spin_unlock_irqrestore(&ioapic_lock, flags);
}
#endif /* CONFIG_SMP */

L
Linus Torvalds 已提交
579 580 581 582 583 584 585
/*
 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
 * shared ISA-space IRQs, so we have to support them. We are super
 * fast in the common case, and fast for shared ISA-space IRQs.
 */
static void add_pin_to_irq(unsigned int irq, int apic, int pin)
{
586 587 588 589 590 591 592 593 594 595 596 597 598 599
	struct irq_cfg *cfg;
	struct irq_pin_list *entry;

	/* first time to refer irq_cfg, so with new */
	cfg = irq_cfg_alloc(irq);
	entry = cfg->irq_2_pin;
	if (!entry) {
		entry = get_one_free_irq_2_pin();
		cfg->irq_2_pin = entry;
		entry->apic = apic;
		entry->pin = pin;
		printk(KERN_DEBUG " 0 add_pin_to_irq: irq %d --> apic %d pin %d\n", irq, apic, pin);
		return;
	}
L
Linus Torvalds 已提交
600

601 602 603 604
	while (entry->next) {
		/* not again, please */
		if (entry->apic == apic && entry->pin == pin)
			return;
L
Linus Torvalds 已提交
605

606
		entry = entry->next;
L
Linus Torvalds 已提交
607
	}
608 609 610

	entry->next = get_one_free_irq_2_pin();
	entry = entry->next;
L
Linus Torvalds 已提交
611 612
	entry->apic = apic;
	entry->pin = pin;
613
	printk(KERN_DEBUG " x add_pin_to_irq: irq %d --> apic %d pin %d\n", irq, apic, pin);
L
Linus Torvalds 已提交
614 615 616 617 618 619 620 621 622
}

/*
 * Reroute an IRQ to a different pin.
 */
static void __init replace_pin_at_irq(unsigned int irq,
				      int oldapic, int oldpin,
				      int newapic, int newpin)
{
623 624 625
	struct irq_cfg *cfg = irq_cfg(irq);
	struct irq_pin_list *entry = cfg->irq_2_pin;
	int replaced = 0;
L
Linus Torvalds 已提交
626

627
	while (entry) {
L
Linus Torvalds 已提交
628 629 630
		if (entry->apic == oldapic && entry->pin == oldpin) {
			entry->apic = newapic;
			entry->pin = newpin;
631 632
			replaced = 1;
			/* every one is different, right? */
L
Linus Torvalds 已提交
633
			break;
634 635
		}
		entry = entry->next;
L
Linus Torvalds 已提交
636
	}
637 638 639 640

	/* why? call replace before add? */
	if (!replaced)
		add_pin_to_irq(irq, newapic, newpin);
L
Linus Torvalds 已提交
641 642
}

Y
Yinghai Lu 已提交
643
#define __DO_ACTION(R, ACTION_ENABLE, ACTION_DISABLE, FINAL)		\
644 645 646 647 648 649 650 651 652 653 654 655 656 657
									\
{									\
	int pin;							\
	struct irq_cfg *cfg;						\
	struct irq_pin_list *entry;					\
									\
	cfg = irq_cfg(irq);						\
	entry = cfg->irq_2_pin;						\
	for (;;) {							\
		unsigned int reg;					\
		if (!entry)						\
			break;						\
		pin = entry->pin;					\
		reg = io_apic_read(entry->apic, 0x10 + R + pin*2);	\
Y
Yinghai Lu 已提交
658 659
		reg ACTION_DISABLE;					\
		reg ACTION_ENABLE;					\
660 661 662 663 664 665 666 667
		io_apic_modify(entry->apic, 0x10 + R + pin*2, reg);	\
		FINAL;							\
		if (!entry->next)					\
			break;						\
		entry = entry->next;					\
	}								\
}

Y
Yinghai Lu 已提交
668
#define DO_ACTION(name,R, ACTION_ENABLE, ACTION_DISABLE, FINAL)		\
669 670
									\
	static void name##_IO_APIC_irq (unsigned int irq)		\
Y
Yinghai Lu 已提交
671
	__DO_ACTION(R, ACTION_ENABLE, ACTION_DISABLE, FINAL)
672 673

/* mask = 0 */
Y
Yinghai Lu 已提交
674
DO_ACTION(__unmask,	0, |= 0, &= ~IO_APIC_REDIR_MASKED, )
675

Y
Yinghai Lu 已提交
676 677 678 679 680 681
#ifdef CONFIG_X86_64
/*
 * Synchronize the IO-APIC and the CPU by doing
 * a dummy read from the IO-APIC
 */
static inline void io_apic_sync(unsigned int apic)
L
Linus Torvalds 已提交
682
{
Y
Yinghai Lu 已提交
683 684
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	readl(&io_apic->data);
L
Linus Torvalds 已提交
685 686 687
}

/* mask = 1 */
Y
Yinghai Lu 已提交
688
DO_ACTION(__mask,	0, |= IO_APIC_REDIR_MASKED, &= ~0, io_apic_sync(entry->apic))
L
Linus Torvalds 已提交
689

Y
Yinghai Lu 已提交
690 691 692 693
#else

/* mask = 1 */
DO_ACTION(__mask,	0, |= IO_APIC_REDIR_MASKED, &= ~0, )
L
Linus Torvalds 已提交
694 695

/* mask = 1, trigger = 0 */
Y
Yinghai Lu 已提交
696
DO_ACTION(__mask_and_edge, 0, |= IO_APIC_REDIR_MASKED, &= ~IO_APIC_REDIR_LEVEL_TRIGGER, )
L
Linus Torvalds 已提交
697 698

/* mask = 0, trigger = 1 */
Y
Yinghai Lu 已提交
699
DO_ACTION(__unmask_and_level, 0, |= IO_APIC_REDIR_LEVEL_TRIGGER, &= ~IO_APIC_REDIR_MASKED, )
L
Linus Torvalds 已提交
700

701 702
#endif

703
static void mask_IO_APIC_irq (unsigned int irq)
L
Linus Torvalds 已提交
704 705 706 707 708 709 710 711
{
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	__mask_IO_APIC_irq(irq);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

712
static void unmask_IO_APIC_irq (unsigned int irq)
L
Linus Torvalds 已提交
713 714 715 716 717 718 719 720 721 722 723
{
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	__unmask_IO_APIC_irq(irq);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
{
	struct IO_APIC_route_entry entry;
724

L
Linus Torvalds 已提交
725
	/* Check delivery_mode to be sure we're not clearing an SMI pin */
726
	entry = ioapic_read_entry(apic, pin);
L
Linus Torvalds 已提交
727 728 729 730 731
	if (entry.delivery_mode == dest_SMI)
		return;
	/*
	 * Disable it in the IO-APIC irq-routing table:
	 */
732
	ioapic_mask_entry(apic, pin);
L
Linus Torvalds 已提交
733 734
}

735
static void clear_IO_APIC (void)
L
Linus Torvalds 已提交
736 737 738 739 740 741 742 743
{
	int apic, pin;

	for (apic = 0; apic < nr_ioapics; apic++)
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
			clear_IO_APIC_pin(apic, pin);
}

744
#if !defined(CONFIG_SMP) && defined(CONFIG_X86_32)
745
void send_IPI_self(int vector)
L
Linus Torvalds 已提交
746 747 748 749 750 751 752 753 754 755 756
{
	unsigned int cfg;

	/*
	 * Wait for idle.
	 */
	apic_wait_icr_idle();
	cfg = APIC_DM_FIXED | APIC_DEST_SELF | vector | APIC_DEST_LOGICAL;
	/*
	 * Send the IPI. The write to APIC_ICR fires this off.
	 */
757
	apic_write(APIC_ICR, cfg);
L
Linus Torvalds 已提交
758
}
759
#endif /* !CONFIG_SMP && CONFIG_X86_32*/
L
Linus Torvalds 已提交
760

761
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
/*
 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
 * specific CPU-side IRQs.
 */

#define MAX_PIRQS 8
static int pirq_entries [MAX_PIRQS];
static int pirqs_enabled;

static int __init ioapic_pirq_setup(char *str)
{
	int i, max;
	int ints[MAX_PIRQS+1];

	get_options(str, ARRAY_SIZE(ints), ints);

	for (i = 0; i < MAX_PIRQS; i++)
		pirq_entries[i] = -1;

	pirqs_enabled = 1;
	apic_printk(APIC_VERBOSE, KERN_INFO
			"PIRQ redirection, working around broken MP-BIOS.\n");
	max = MAX_PIRQS;
	if (ints[0] < MAX_PIRQS)
		max = ints[0];

	for (i = 0; i < max; i++) {
		apic_printk(APIC_VERBOSE, KERN_DEBUG
				"... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
		/*
		 * PIRQs are mapped upside down, usually.
		 */
		pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
	}
	return 1;
}

__setup("pirq=", ioapic_pirq_setup);
800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868
#endif /* CONFIG_X86_32 */

#ifdef CONFIG_INTR_REMAP
/* I/O APIC RTE contents at the OS boot up */
static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS];

/*
 * Saves and masks all the unmasked IO-APIC RTE's
 */
int save_mask_IO_APIC_setup(void)
{
	union IO_APIC_reg_01 reg_01;
	unsigned long flags;
	int apic, pin;

	/*
	 * The number of IO-APIC IRQ registers (== #pins):
	 */
	for (apic = 0; apic < nr_ioapics; apic++) {
		spin_lock_irqsave(&ioapic_lock, flags);
		reg_01.raw = io_apic_read(apic, 1);
		spin_unlock_irqrestore(&ioapic_lock, flags);
		nr_ioapic_registers[apic] = reg_01.bits.entries+1;
	}

	for (apic = 0; apic < nr_ioapics; apic++) {
		early_ioapic_entries[apic] =
			kzalloc(sizeof(struct IO_APIC_route_entry) *
				nr_ioapic_registers[apic], GFP_KERNEL);
		if (!early_ioapic_entries[apic])
			return -ENOMEM;
	}

	for (apic = 0; apic < nr_ioapics; apic++)
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
			struct IO_APIC_route_entry entry;

			entry = early_ioapic_entries[apic][pin] =
				ioapic_read_entry(apic, pin);
			if (!entry.mask) {
				entry.mask = 1;
				ioapic_write_entry(apic, pin, entry);
			}
		}
	return 0;
}

void restore_IO_APIC_setup(void)
{
	int apic, pin;

	for (apic = 0; apic < nr_ioapics; apic++)
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
			ioapic_write_entry(apic, pin,
					   early_ioapic_entries[apic][pin]);
}

void reinit_intr_remapped_IO_APIC(int intr_remapping)
{
	/*
	 * for now plain restore of previous settings.
	 * TBD: In the case of OS enabling interrupt-remapping,
	 * IO-APIC RTE's need to be setup to point to interrupt-remapping
	 * table entries. for now, do a plain restore, and wait for
	 * the setup_IO_APIC_irqs() to do proper initialization.
	 */
	restore_IO_APIC_setup();
}
#endif
L
Linus Torvalds 已提交
869 870 871 872 873 874 875 876 877

/*
 * Find the IRQ entry number of a certain pin.
 */
static int find_irq_entry(int apic, int pin, int type)
{
	int i;

	for (i = 0; i < mp_irq_entries; i++)
878 879 880 881
		if (mp_irqs[i].mp_irqtype == type &&
		    (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
		     mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
		    mp_irqs[i].mp_dstirq == pin)
L
Linus Torvalds 已提交
882 883 884 885 886 887 888 889
			return i;

	return -1;
}

/*
 * Find the pin to which IRQ[irq] (ISA) is connected
 */
890
static int __init find_isa_irq_pin(int irq, int type)
L
Linus Torvalds 已提交
891 892 893 894
{
	int i;

	for (i = 0; i < mp_irq_entries; i++) {
895
		int lbus = mp_irqs[i].mp_srcbus;
L
Linus Torvalds 已提交
896

A
Alexey Starikovskiy 已提交
897
		if (test_bit(lbus, mp_bus_not_pci) &&
898 899
		    (mp_irqs[i].mp_irqtype == type) &&
		    (mp_irqs[i].mp_srcbusirq == irq))
L
Linus Torvalds 已提交
900

901
			return mp_irqs[i].mp_dstirq;
L
Linus Torvalds 已提交
902 903 904 905
	}
	return -1;
}

906 907 908 909 910
static int __init find_isa_irq_apic(int irq, int type)
{
	int i;

	for (i = 0; i < mp_irq_entries; i++) {
911
		int lbus = mp_irqs[i].mp_srcbus;
912

A
Alexey Starikovskiy 已提交
913
		if (test_bit(lbus, mp_bus_not_pci) &&
914 915
		    (mp_irqs[i].mp_irqtype == type) &&
		    (mp_irqs[i].mp_srcbusirq == irq))
916 917 918 919
			break;
	}
	if (i < mp_irq_entries) {
		int apic;
920
		for(apic = 0; apic < nr_ioapics; apic++) {
921
			if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
922 923 924 925 926 927 928
				return apic;
		}
	}

	return -1;
}

L
Linus Torvalds 已提交
929 930 931 932 933 934 935 936 937 938
/*
 * Find a specific PCI IRQ entry.
 * Not an __init, possibly needed by modules
 */
static int pin_2_irq(int idx, int apic, int pin);

int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
{
	int apic, i, best_guess = -1;

939 940
	apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
		bus, slot, pin);
941
	if (test_bit(bus, mp_bus_not_pci)) {
942
		apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
L
Linus Torvalds 已提交
943 944 945
		return -1;
	}
	for (i = 0; i < mp_irq_entries; i++) {
946
		int lbus = mp_irqs[i].mp_srcbus;
L
Linus Torvalds 已提交
947 948

		for (apic = 0; apic < nr_ioapics; apic++)
949 950
			if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
			    mp_irqs[i].mp_dstapic == MP_APIC_ALL)
L
Linus Torvalds 已提交
951 952
				break;

A
Alexey Starikovskiy 已提交
953
		if (!test_bit(lbus, mp_bus_not_pci) &&
954
		    !mp_irqs[i].mp_irqtype &&
L
Linus Torvalds 已提交
955
		    (bus == lbus) &&
956
		    (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
957
			int irq = pin_2_irq(i,apic,mp_irqs[i].mp_dstirq);
L
Linus Torvalds 已提交
958 959 960 961

			if (!(apic || IO_APIC_IRQ(irq)))
				continue;

962
			if (pin == (mp_irqs[i].mp_srcbusirq & 3))
L
Linus Torvalds 已提交
963 964 965 966 967 968 969 970 971 972 973
				return irq;
			/*
			 * Use the first all-but-pin matching entry as a
			 * best-guess fuzzy result for broken mptables.
			 */
			if (best_guess < 0)
				best_guess = irq;
		}
	}
	return best_guess;
}
974

975
EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
L
Linus Torvalds 已提交
976

977
#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
L
Linus Torvalds 已提交
978 979 980 981 982 983 984 985 986 987 988 989 990
/*
 * EISA Edge/Level control register, ELCR
 */
static int EISA_ELCR(unsigned int irq)
{
	if (irq < 16) {
		unsigned int port = 0x4d0 + (irq >> 3);
		return (inb(port) >> (irq & 7)) & 1;
	}
	apic_printk(APIC_VERBOSE, KERN_INFO
			"Broken MPtable reports ISA irq %d\n", irq);
	return 0;
}
991

992
#endif
L
Linus Torvalds 已提交
993

A
Alexey Starikovskiy 已提交
994 995 996 997 998 999
/* ISA interrupts are always polarity zero edge triggered,
 * when listed as conforming in the MP table. */

#define default_ISA_trigger(idx)	(0)
#define default_ISA_polarity(idx)	(0)

L
Linus Torvalds 已提交
1000 1001 1002 1003 1004
/* EISA interrupts are always polarity zero and can be edge or level
 * trigger depending on the ELCR value.  If an interrupt is listed as
 * EISA conforming in the MP table, that means its trigger type must
 * be read in from the ELCR */

1005
#define default_EISA_trigger(idx)	(EISA_ELCR(mp_irqs[idx].mp_srcbusirq))
A
Alexey Starikovskiy 已提交
1006
#define default_EISA_polarity(idx)	default_ISA_polarity(idx)
L
Linus Torvalds 已提交
1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017

/* PCI interrupts are always polarity one level triggered,
 * when listed as conforming in the MP table. */

#define default_PCI_trigger(idx)	(1)
#define default_PCI_polarity(idx)	(1)

/* MCA interrupts are always polarity zero level triggered,
 * when listed as conforming in the MP table. */

#define default_MCA_trigger(idx)	(1)
A
Alexey Starikovskiy 已提交
1018
#define default_MCA_polarity(idx)	default_ISA_polarity(idx)
L
Linus Torvalds 已提交
1019

1020
static int MPBIOS_polarity(int idx)
L
Linus Torvalds 已提交
1021
{
1022
	int bus = mp_irqs[idx].mp_srcbus;
L
Linus Torvalds 已提交
1023 1024 1025 1026 1027
	int polarity;

	/*
	 * Determine IRQ line polarity (high active or low active):
	 */
1028
	switch (mp_irqs[idx].mp_irqflag & 3)
1029
	{
1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
		case 0: /* conforms, ie. bus-type dependent polarity */
			if (test_bit(bus, mp_bus_not_pci))
				polarity = default_ISA_polarity(idx);
			else
				polarity = default_PCI_polarity(idx);
			break;
		case 1: /* high active */
		{
			polarity = 0;
			break;
		}
		case 2: /* reserved */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			polarity = 1;
			break;
		}
		case 3: /* low active */
		{
			polarity = 1;
			break;
		}
		default: /* invalid */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			polarity = 1;
			break;
		}
L
Linus Torvalds 已提交
1058 1059 1060 1061 1062 1063
	}
	return polarity;
}

static int MPBIOS_trigger(int idx)
{
1064
	int bus = mp_irqs[idx].mp_srcbus;
L
Linus Torvalds 已提交
1065 1066 1067 1068 1069
	int trigger;

	/*
	 * Determine IRQ trigger mode (edge or level sensitive):
	 */
1070
	switch ((mp_irqs[idx].mp_irqflag>>2) & 3)
L
Linus Torvalds 已提交
1071
	{
1072 1073 1074 1075 1076
		case 0: /* conforms, ie. bus-type dependent */
			if (test_bit(bus, mp_bus_not_pci))
				trigger = default_ISA_trigger(idx);
			else
				trigger = default_PCI_trigger(idx);
1077
#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106
			switch (mp_bus_id_to_type[bus]) {
				case MP_BUS_ISA: /* ISA pin */
				{
					/* set before the switch */
					break;
				}
				case MP_BUS_EISA: /* EISA pin */
				{
					trigger = default_EISA_trigger(idx);
					break;
				}
				case MP_BUS_PCI: /* PCI pin */
				{
					/* set before the switch */
					break;
				}
				case MP_BUS_MCA: /* MCA pin */
				{
					trigger = default_MCA_trigger(idx);
					break;
				}
				default:
				{
					printk(KERN_WARNING "broken BIOS!!\n");
					trigger = 1;
					break;
				}
			}
#endif
L
Linus Torvalds 已提交
1107
			break;
1108
		case 1: /* edge */
L
Linus Torvalds 已提交
1109
		{
1110
			trigger = 0;
L
Linus Torvalds 已提交
1111 1112
			break;
		}
1113
		case 2: /* reserved */
L
Linus Torvalds 已提交
1114
		{
1115 1116
			printk(KERN_WARNING "broken BIOS!!\n");
			trigger = 1;
L
Linus Torvalds 已提交
1117 1118
			break;
		}
1119
		case 3: /* level */
L
Linus Torvalds 已提交
1120
		{
1121
			trigger = 1;
L
Linus Torvalds 已提交
1122 1123
			break;
		}
1124
		default: /* invalid */
L
Linus Torvalds 已提交
1125 1126
		{
			printk(KERN_WARNING "broken BIOS!!\n");
1127
			trigger = 0;
L
Linus Torvalds 已提交
1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
			break;
		}
	}
	return trigger;
}

static inline int irq_polarity(int idx)
{
	return MPBIOS_polarity(idx);
}

static inline int irq_trigger(int idx)
{
	return MPBIOS_trigger(idx);
}

Y
Yinghai Lu 已提交
1144
int (*ioapic_renumber_irq)(int ioapic, int irq);
L
Linus Torvalds 已提交
1145 1146 1147
static int pin_2_irq(int idx, int apic, int pin)
{
	int irq, i;
1148
	int bus = mp_irqs[idx].mp_srcbus;
L
Linus Torvalds 已提交
1149 1150 1151 1152

	/*
	 * Debugging check, we are in big trouble if this message pops up!
	 */
1153
	if (mp_irqs[idx].mp_dstirq != pin)
L
Linus Torvalds 已提交
1154 1155
		printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");

1156
	if (test_bit(bus, mp_bus_not_pci)) {
1157
		irq = mp_irqs[idx].mp_srcbusirq;
1158
	} else {
A
Alexey Starikovskiy 已提交
1159 1160 1161 1162 1163 1164 1165
		/*
		 * PCI IRQs are mapped in order
		 */
		i = irq = 0;
		while (i < apic)
			irq += nr_ioapic_registers[i++];
		irq += pin;
1166 1167 1168 1169 1170
                /*
                 * For MPS mode, so far only needed by ES7000 platform
                 */
                if (ioapic_renumber_irq)
                        irq = ioapic_renumber_irq(apic, irq);
L
Linus Torvalds 已提交
1171 1172
	}

1173
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189
	/*
	 * PCI IRQ command line redirection. Yes, limits are hardcoded.
	 */
	if ((pin >= 16) && (pin <= 23)) {
		if (pirq_entries[pin-16] != -1) {
			if (!pirq_entries[pin-16]) {
				apic_printk(APIC_VERBOSE, KERN_DEBUG
						"disabling PIRQ%d\n", pin-16);
			} else {
				irq = pirq_entries[pin-16];
				apic_printk(APIC_VERBOSE, KERN_DEBUG
						"using PIRQ%d -> IRQ %d\n",
						pin-16, irq);
			}
		}
	}
1190 1191
#endif

L
Linus Torvalds 已提交
1192 1193 1194
	return irq;
}

1195 1196 1197 1198 1199 1200 1201
void lock_vector_lock(void)
{
	/* Used to the online set of cpus does not change
	 * during assign_irq_vector.
	 */
	spin_lock(&vector_lock);
}
L
Linus Torvalds 已提交
1202

1203
void unlock_vector_lock(void)
L
Linus Torvalds 已提交
1204
{
1205 1206
	spin_unlock(&vector_lock);
}
L
Linus Torvalds 已提交
1207

1208 1209
static int __assign_irq_vector(int irq, cpumask_t mask)
{
1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220
	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
1221 1222 1223 1224
	static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
	unsigned int old_vector;
	int cpu;
	struct irq_cfg *cfg;
1225

1226
	cfg = irq_cfg(irq);
1227

1228 1229
	/* Only try and allocate irqs on cpus that are present */
	cpus_and(mask, mask, cpu_online_map);
1230

1231 1232
	if ((cfg->move_in_progress) || cfg->move_cleanup_count)
		return -EBUSY;
1233

1234 1235 1236 1237 1238 1239 1240
	old_vector = cfg->vector;
	if (old_vector) {
		cpumask_t tmp;
		cpus_and(tmp, cfg->domain, mask);
		if (!cpus_empty(tmp))
			return 0;
	}
1241

1242 1243 1244 1245
	for_each_cpu_mask_nr(cpu, mask) {
		cpumask_t domain, new_mask;
		int new_cpu;
		int vector, offset;
1246

1247 1248
		domain = vector_allocation_domain(cpu);
		cpus_and(new_mask, domain, cpu_online_map);
1249

1250 1251
		vector = current_vector;
		offset = current_offset;
1252
next:
1253 1254 1255 1256 1257 1258 1259 1260
		vector += 8;
		if (vector >= first_system_vector) {
			/* If we run out of vectors on large boxen, must share them. */
			offset = (offset + 1) % 8;
			vector = FIRST_DEVICE_VECTOR + offset;
		}
		if (unlikely(current_vector == vector))
			continue;
1261
#ifdef CONFIG_X86_64
1262 1263
		if (vector == IA32_SYSCALL_VECTOR)
			goto next;
1264
#else
1265 1266
		if (vector == SYSCALL_VECTOR)
			goto next;
1267
#endif
1268 1269 1270 1271 1272 1273 1274 1275 1276
		for_each_cpu_mask_nr(new_cpu, new_mask)
			if (per_cpu(vector_irq, new_cpu)[vector] != -1)
				goto next;
		/* Found one! */
		current_vector = vector;
		current_offset = offset;
		if (old_vector) {
			cfg->move_in_progress = 1;
			cfg->old_domain = cfg->domain;
1277
		}
1278 1279 1280 1281 1282 1283 1284
		for_each_cpu_mask_nr(new_cpu, new_mask)
			per_cpu(vector_irq, new_cpu)[vector] = irq;
		cfg->vector = vector;
		cfg->domain = domain;
		return 0;
	}
	return -ENOSPC;
1285 1286 1287 1288 1289
}

static int assign_irq_vector(int irq, cpumask_t mask)
{
	int err;
1290 1291 1292
	unsigned long flags;

	spin_lock_irqsave(&vector_lock, flags);
1293
	err = __assign_irq_vector(irq, mask);
1294
	spin_unlock_irqrestore(&vector_lock, flags);
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
	return err;
}

static void __clear_irq_vector(int irq)
{
	struct irq_cfg *cfg;
	cpumask_t mask;
	int cpu, vector;

	cfg = irq_cfg(irq);
	BUG_ON(!cfg->vector);

	vector = cfg->vector;
	cpus_and(mask, cfg->domain, cpu_online_map);
	for_each_cpu_mask_nr(cpu, mask)
		per_cpu(vector_irq, cpu)[vector] = -1;

	cfg->vector = 0;
	cpus_clear(cfg->domain);
}

void __setup_vector_irq(int cpu)
{
	/* Initialize vector_irq on a new cpu */
	/* This function must be called with vector_lock held */
	int irq, vector;
	struct irq_cfg *cfg;

	/* Mark the inuse vectors */
1324
	for_each_irq_cfg(irq, cfg) {
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338
		if (!cpu_isset(cpu, cfg->domain))
			continue;
		vector = cfg->vector;
		per_cpu(vector_irq, cpu)[vector] = irq;
	}
	/* Mark the free vectors */
	for (vector = 0; vector < NR_VECTORS; ++vector) {
		irq = per_cpu(vector_irq, cpu)[vector];
		if (irq < 0)
			continue;

		cfg = irq_cfg(irq);
		if (!cpu_isset(cpu, cfg->domain))
			per_cpu(vector_irq, cpu)[vector] = -1;
1339
	}
L
Linus Torvalds 已提交
1340
}
1341

1342
static struct irq_chip ioapic_chip;
1343 1344 1345
#ifdef CONFIG_INTR_REMAP
static struct irq_chip ir_ioapic_chip;
#endif
L
Linus Torvalds 已提交
1346

1347 1348 1349
#define IOAPIC_AUTO     -1
#define IOAPIC_EDGE     0
#define IOAPIC_LEVEL    1
L
Linus Torvalds 已提交
1350

1351
#ifdef CONFIG_X86_32
1352 1353
static inline int IO_APIC_irq_trigger(int irq)
{
1354
        int apic, idx, pin;
1355

1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366
        for (apic = 0; apic < nr_ioapics; apic++) {
                for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
                        idx = find_irq_entry(apic, pin, mp_INT);
                        if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
                                return irq_trigger(idx);
                }
        }
        /*
         * nonexistent IRQs are edge default
         */
        return 0;
1367
}
1368 1369 1370
#else
static inline int IO_APIC_irq_trigger(int irq)
{
1371
	return 1;
1372 1373
}
#endif
1374

1375
static void ioapic_register_intr(int irq, unsigned long trigger)
L
Linus Torvalds 已提交
1376
{
1377 1378
	struct irq_desc *desc;

Y
Yinghai Lu 已提交
1379 1380 1381 1382 1383 1384
	/* first time to use this irq_desc */
	if (irq < 16)
		desc = irq_to_desc(irq);
	else
		desc = irq_to_desc_alloc(irq);

1385
	if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1386
	    trigger == IOAPIC_LEVEL)
1387
		desc->status |= IRQ_LEVEL;
1388 1389 1390
	else
		desc->status &= ~IRQ_LEVEL;

1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
#ifdef CONFIG_INTR_REMAP
	if (irq_remapped(irq)) {
		desc->status |= IRQ_MOVE_PCNTXT;
		if (trigger)
			set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
						      handle_fasteoi_irq,
						     "fasteoi");
		else
			set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
						      handle_edge_irq, "edge");
		return;
	}
#endif
1404 1405
	if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
	    trigger == IOAPIC_LEVEL)
1406
		set_irq_chip_and_handler_name(irq, &ioapic_chip,
1407 1408
					      handle_fasteoi_irq,
					      "fasteoi");
1409
	else
1410
		set_irq_chip_and_handler_name(irq, &ioapic_chip,
1411
					      handle_edge_irq, "edge");
L
Linus Torvalds 已提交
1412 1413
}

1414 1415 1416 1417
static int setup_ioapic_entry(int apic, int irq,
			      struct IO_APIC_route_entry *entry,
			      unsigned int destination, int trigger,
			      int polarity, int vector)
L
Linus Torvalds 已提交
1418
{
1419 1420 1421 1422 1423
	/*
	 * add it to the IO-APIC irq-routing table:
	 */
	memset(entry,0,sizeof(*entry));

1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
#ifdef CONFIG_INTR_REMAP
	if (intr_remapping_enabled) {
		struct intel_iommu *iommu = map_ioapic_to_ir(apic);
		struct irte irte;
		struct IR_IO_APIC_route_entry *ir_entry =
			(struct IR_IO_APIC_route_entry *) entry;
		int index;

		if (!iommu)
			panic("No mapping iommu for ioapic %d\n", apic);

		index = alloc_irte(iommu, irq, 1);
		if (index < 0)
			panic("Failed to allocate IRTE for ioapic %d\n", apic);

		memset(&irte, 0, sizeof(irte));

		irte.present = 1;
		irte.dst_mode = INT_DEST_MODE;
		irte.trigger_mode = trigger;
		irte.dlvry_mode = INT_DELIVERY_MODE;
		irte.vector = vector;
		irte.dest_id = IRTE_DEST(destination);

		modify_irte(irq, &irte);

		ir_entry->index2 = (index >> 15) & 0x1;
		ir_entry->zero = 0;
		ir_entry->format = 1;
		ir_entry->index = (index & 0x7fff);
	} else
#endif
	{
		entry->delivery_mode = INT_DELIVERY_MODE;
		entry->dest_mode = INT_DEST_MODE;
		entry->dest = destination;
	}
1461

1462
	entry->mask = 0;				/* enable IRQ */
1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475
	entry->trigger = trigger;
	entry->polarity = polarity;
	entry->vector = vector;

	/* Mask level triggered irqs.
	 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
	 */
	if (trigger)
		entry->mask = 1;
	return 0;
}

static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq,
1476
			      int trigger, int polarity)
1477 1478
{
	struct irq_cfg *cfg;
L
Linus Torvalds 已提交
1479
	struct IO_APIC_route_entry entry;
1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518
	cpumask_t mask;

	if (!IO_APIC_IRQ(irq))
		return;

	cfg = irq_cfg(irq);

	mask = TARGET_CPUS;
	if (assign_irq_vector(irq, mask))
		return;

	cpus_and(mask, cfg->domain, mask);

	apic_printk(APIC_VERBOSE,KERN_DEBUG
		    "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
		    "IRQ %d Mode:%i Active:%i)\n",
		    apic, mp_ioapics[apic].mp_apicid, pin, cfg->vector,
		    irq, trigger, polarity);


	if (setup_ioapic_entry(mp_ioapics[apic].mp_apicid, irq, &entry,
			       cpu_mask_to_apicid(mask), trigger, polarity,
			       cfg->vector)) {
		printk("Failed to setup ioapic entry for ioapic  %d, pin %d\n",
		       mp_ioapics[apic].mp_apicid, pin);
		__clear_irq_vector(irq);
		return;
	}

	ioapic_register_intr(irq, trigger);
	if (irq < 16)
		disable_8259A_irq(irq);

	ioapic_write_entry(apic, pin, entry);
}

static void __init setup_IO_APIC_irqs(void)
{
	int apic, pin, idx, irq, first_notcon = 1;
L
Linus Torvalds 已提交
1519 1520 1521 1522 1523 1524

	apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");

	for (apic = 0; apic < nr_ioapics; apic++) {
	for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {

1525
		idx = find_irq_entry(apic,pin,mp_INT);
L
Linus Torvalds 已提交
1526 1527
		if (idx == -1) {
			if (first_notcon) {
1528
				apic_printk(APIC_VERBOSE, KERN_DEBUG " IO-APIC (apicid-pin) %d-%d", mp_ioapics[apic].mp_apicid, pin);
L
Linus Torvalds 已提交
1529 1530
				first_notcon = 0;
			} else
1531
				apic_printk(APIC_VERBOSE, ", %d-%d", mp_ioapics[apic].mp_apicid, pin);
L
Linus Torvalds 已提交
1532 1533
			continue;
		}
1534 1535 1536 1537 1538
		if (!first_notcon) {
			apic_printk(APIC_VERBOSE, " not connected.\n");
			first_notcon = 1;
		}

L
Linus Torvalds 已提交
1539
		irq = pin_2_irq(idx, apic, pin);
1540
#ifdef CONFIG_X86_32
1541 1542
                if (multi_timer_check(apic, irq))
                        continue;
1543
#endif
1544
		add_pin_to_irq(irq, apic, pin);
1545

1546 1547
		setup_IO_APIC_irq(apic, pin, irq,
				  irq_trigger(idx), irq_polarity(idx));
L
Linus Torvalds 已提交
1548 1549 1550 1551 1552 1553 1554 1555
	}
	}

	if (!first_notcon)
		apic_printk(APIC_VERBOSE, " not connected.\n");
}

/*
1556
 * Set up the timer pin, possibly with the 8259A-master behind.
L
Linus Torvalds 已提交
1557
 */
1558 1559
static void __init setup_timer_IRQ0_pin(unsigned int apic, unsigned int pin,
					int vector)
L
Linus Torvalds 已提交
1560 1561 1562
{
	struct IO_APIC_route_entry entry;

1563 1564 1565 1566 1567
#ifdef CONFIG_INTR_REMAP
	if (intr_remapping_enabled)
		return;
#endif

1568
	memset(&entry, 0, sizeof(entry));
L
Linus Torvalds 已提交
1569 1570 1571 1572 1573 1574

	/*
	 * We use logical delivery to get the timer IRQ
	 * to the first CPU.
	 */
	entry.dest_mode = INT_DEST_MODE;
1575
	entry.mask = 1;					/* mask IRQ now */
1576
	entry.dest = cpu_mask_to_apicid(TARGET_CPUS);
L
Linus Torvalds 已提交
1577 1578 1579 1580 1581 1582 1583
	entry.delivery_mode = INT_DELIVERY_MODE;
	entry.polarity = 0;
	entry.trigger = 0;
	entry.vector = vector;

	/*
	 * The timer IRQ doesn't have to know that behind the
1584
	 * scene we may have a 8259A-master in AEOI mode ...
L
Linus Torvalds 已提交
1585
	 */
1586
	set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
L
Linus Torvalds 已提交
1587 1588 1589 1590

	/*
	 * Add it to the IO-APIC irq-routing table:
	 */
1591
	ioapic_write_entry(apic, pin, entry);
L
Linus Torvalds 已提交
1592 1593
}

1594 1595

__apicdebuginit(void) print_IO_APIC(void)
L
Linus Torvalds 已提交
1596 1597 1598 1599 1600 1601 1602
{
	int apic, i;
	union IO_APIC_reg_00 reg_00;
	union IO_APIC_reg_01 reg_01;
	union IO_APIC_reg_02 reg_02;
	union IO_APIC_reg_03 reg_03;
	unsigned long flags;
1603
	struct irq_cfg *cfg;
1604
	unsigned int irq;
L
Linus Torvalds 已提交
1605 1606 1607 1608

	if (apic_verbosity == APIC_QUIET)
		return;

1609
	printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
L
Linus Torvalds 已提交
1610 1611
	for (i = 0; i < nr_ioapics; i++)
		printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1612
		       mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);
L
Linus Torvalds 已提交
1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626

	/*
	 * We are a bit conservative about what we expect.  We have to
	 * know about every hardware change ASAP.
	 */
	printk(KERN_INFO "testing the IO APIC.......................\n");

	for (apic = 0; apic < nr_ioapics; apic++) {

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(apic, 0);
	reg_01.raw = io_apic_read(apic, 1);
	if (reg_01.bits.version >= 0x10)
		reg_02.raw = io_apic_read(apic, 2);
1627 1628
        if (reg_01.bits.version >= 0x20)
                reg_03.raw = io_apic_read(apic, 3);
L
Linus Torvalds 已提交
1629 1630
	spin_unlock_irqrestore(&ioapic_lock, flags);

1631
	printk("\n");
1632
	printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
L
Linus Torvalds 已提交
1633 1634 1635 1636 1637
	printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
	printk(KERN_DEBUG ".......    : physical APIC id: %02X\n", reg_00.bits.ID);
	printk(KERN_DEBUG ".......    : Delivery Type: %X\n", reg_00.bits.delivery_type);
	printk(KERN_DEBUG ".......    : LTS          : %X\n", reg_00.bits.LTS);

1638
	printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
L
Linus Torvalds 已提交
1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
	printk(KERN_DEBUG ".......     : max redirection entries: %04X\n", reg_01.bits.entries);

	printk(KERN_DEBUG ".......     : PRQ implemented: %X\n", reg_01.bits.PRQ);
	printk(KERN_DEBUG ".......     : IO APIC version: %04X\n", reg_01.bits.version);

	/*
	 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
	 * but the value of reg_02 is read as the previous read register
	 * value, so ignore it if reg_02 == reg_01.
	 */
	if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
		printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
		printk(KERN_DEBUG ".......     : arbitration: %02X\n", reg_02.bits.arbitration);
	}

	/*
	 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
	 * or reg_03, but the value of reg_0[23] is read as the previous read
	 * register value, so ignore it if reg_03 == reg_0[12].
	 */
	if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
	    reg_03.raw != reg_01.raw) {
		printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
		printk(KERN_DEBUG ".......     : Boot DT    : %X\n", reg_03.bits.boot_DT);
	}

	printk(KERN_DEBUG ".... IRQ redirection table:\n");

1667 1668
	printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
			  " Stat Dmod Deli Vect:   \n");
L
Linus Torvalds 已提交
1669 1670 1671 1672

	for (i = 0; i <= reg_01.bits.entries; i++) {
		struct IO_APIC_route_entry entry;

1673
		entry = ioapic_read_entry(apic, i);
L
Linus Torvalds 已提交
1674

1675 1676 1677 1678
		printk(KERN_DEBUG " %02x %03X ",
			i,
			entry.dest
		);
L
Linus Torvalds 已提交
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692

		printk("%1d    %1d    %1d   %1d   %1d    %1d    %1d    %02X\n",
			entry.mask,
			entry.trigger,
			entry.irr,
			entry.polarity,
			entry.delivery_status,
			entry.dest_mode,
			entry.delivery_mode,
			entry.vector
		);
	}
	}
	printk(KERN_DEBUG "IRQ to pin mappings:\n");
1693
	for_each_irq_cfg(irq, cfg) {
1694 1695
		struct irq_pin_list *entry = cfg->irq_2_pin;
		if (!entry)
L
Linus Torvalds 已提交
1696
			continue;
1697
		printk(KERN_DEBUG "IRQ%d ", irq);
L
Linus Torvalds 已提交
1698 1699 1700 1701
		for (;;) {
			printk("-> %d:%d", entry->apic, entry->pin);
			if (!entry->next)
				break;
1702
			entry = entry->next;
L
Linus Torvalds 已提交
1703 1704 1705 1706 1707 1708 1709 1710 1711
		}
		printk("\n");
	}

	printk(KERN_INFO ".................................... done.\n");

	return;
}

1712
__apicdebuginit(void) print_APIC_bitfield(int base)
L
Linus Torvalds 已提交
1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
{
	unsigned int v;
	int i, j;

	if (apic_verbosity == APIC_QUIET)
		return;

	printk(KERN_DEBUG "0123456789abcdef0123456789abcdef\n" KERN_DEBUG);
	for (i = 0; i < 8; i++) {
		v = apic_read(base + i*0x10);
		for (j = 0; j < 32; j++) {
			if (v & (1<<j))
				printk("1");
			else
				printk("0");
		}
		printk("\n");
	}
}

1733
__apicdebuginit(void) print_local_APIC(void *dummy)
L
Linus Torvalds 已提交
1734 1735
{
	unsigned int v, ver, maxlvt;
1736
	u64 icr;
L
Linus Torvalds 已提交
1737 1738 1739 1740 1741 1742

	if (apic_verbosity == APIC_QUIET)
		return;

	printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
		smp_processor_id(), hard_smp_processor_id());
1743
	v = apic_read(APIC_ID);
1744
	printk(KERN_INFO "... APIC ID:      %08x (%01x)\n", v, read_apic_id());
L
Linus Torvalds 已提交
1745 1746 1747
	v = apic_read(APIC_LVR);
	printk(KERN_INFO "... APIC VERSION: %08x\n", v);
	ver = GET_APIC_VERSION(v);
1748
	maxlvt = lapic_get_maxlvt();
L
Linus Torvalds 已提交
1749 1750 1751 1752

	v = apic_read(APIC_TASKPRI);
	printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);

1753
	if (APIC_INTEGRATED(ver)) {                     /* !82489DX */
1754 1755 1756 1757 1758
		if (!APIC_XAPIC(ver)) {
			v = apic_read(APIC_ARBPRI);
			printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
			       v & APIC_ARBPRI_MASK);
		}
L
Linus Torvalds 已提交
1759 1760 1761 1762
		v = apic_read(APIC_PROCPRI);
		printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
	}

1763 1764 1765 1766 1767 1768 1769 1770 1771
	/*
	 * Remote read supported only in the 82489DX and local APIC for
	 * Pentium processors.
	 */
	if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
		v = apic_read(APIC_RRR);
		printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
	}

L
Linus Torvalds 已提交
1772 1773
	v = apic_read(APIC_LDR);
	printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
1774 1775 1776 1777
	if (!x2apic_enabled()) {
		v = apic_read(APIC_DFR);
		printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
	}
L
Linus Torvalds 已提交
1778 1779 1780 1781 1782 1783 1784 1785 1786 1787
	v = apic_read(APIC_SPIV);
	printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);

	printk(KERN_DEBUG "... APIC ISR field:\n");
	print_APIC_bitfield(APIC_ISR);
	printk(KERN_DEBUG "... APIC TMR field:\n");
	print_APIC_bitfield(APIC_TMR);
	printk(KERN_DEBUG "... APIC IRR field:\n");
	print_APIC_bitfield(APIC_IRR);

1788 1789
	if (APIC_INTEGRATED(ver)) {             /* !82489DX */
		if (maxlvt > 3)         /* Due to the Pentium erratum 3AP. */
L
Linus Torvalds 已提交
1790
			apic_write(APIC_ESR, 0);
1791

L
Linus Torvalds 已提交
1792 1793 1794 1795
		v = apic_read(APIC_ESR);
		printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
	}

1796
	icr = apic_icr_read();
1797 1798
	printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
	printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
L
Linus Torvalds 已提交
1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825

	v = apic_read(APIC_LVTT);
	printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);

	if (maxlvt > 3) {                       /* PC is LVT#4. */
		v = apic_read(APIC_LVTPC);
		printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
	}
	v = apic_read(APIC_LVT0);
	printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
	v = apic_read(APIC_LVT1);
	printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);

	if (maxlvt > 2) {			/* ERR is LVT#3. */
		v = apic_read(APIC_LVTERR);
		printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
	}

	v = apic_read(APIC_TMICT);
	printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
	v = apic_read(APIC_TMCCT);
	printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
	v = apic_read(APIC_TDCR);
	printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
	printk("\n");
}

1826
__apicdebuginit(void) print_all_local_APICs(void)
L
Linus Torvalds 已提交
1827
{
1828 1829 1830 1831 1832 1833
	int cpu;

	preempt_disable();
	for_each_online_cpu(cpu)
		smp_call_function_single(cpu, print_local_APIC, NULL, 1);
	preempt_enable();
L
Linus Torvalds 已提交
1834 1835
}

1836
__apicdebuginit(void) print_PIC(void)
L
Linus Torvalds 已提交
1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853
{
	unsigned int v;
	unsigned long flags;

	if (apic_verbosity == APIC_QUIET)
		return;

	printk(KERN_DEBUG "\nprinting PIC contents\n");

	spin_lock_irqsave(&i8259A_lock, flags);

	v = inb(0xa1) << 8 | inb(0x21);
	printk(KERN_DEBUG "... PIC  IMR: %04x\n", v);

	v = inb(0xa0) << 8 | inb(0x20);
	printk(KERN_DEBUG "... PIC  IRR: %04x\n", v);

1854 1855
	outb(0x0b,0xa0);
	outb(0x0b,0x20);
L
Linus Torvalds 已提交
1856
	v = inb(0xa0) << 8 | inb(0x20);
1857 1858
	outb(0x0a,0xa0);
	outb(0x0a,0x20);
L
Linus Torvalds 已提交
1859 1860 1861 1862 1863 1864 1865 1866 1867

	spin_unlock_irqrestore(&i8259A_lock, flags);

	printk(KERN_DEBUG "... PIC  ISR: %04x\n", v);

	v = inb(0x4d1) << 8 | inb(0x4d0);
	printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
}

1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
__apicdebuginit(int) print_all_ICs(void)
{
	print_PIC();
	print_all_local_APICs();
	print_IO_APIC();

	return 0;
}

fs_initcall(print_all_ICs);

L
Linus Torvalds 已提交
1879

Y
Yinghai Lu 已提交
1880 1881 1882
/* Where if anywhere is the i8259 connect in external int mode */
static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };

1883
void __init enable_IO_APIC(void)
L
Linus Torvalds 已提交
1884 1885
{
	union IO_APIC_reg_01 reg_01;
1886
	int i8259_apic, i8259_pin;
1887
	int apic;
L
Linus Torvalds 已提交
1888 1889
	unsigned long flags;

1890 1891
#ifdef CONFIG_X86_32
	int i;
L
Linus Torvalds 已提交
1892 1893 1894
	if (!pirqs_enabled)
		for (i = 0; i < MAX_PIRQS; i++)
			pirq_entries[i] = -1;
1895
#endif
L
Linus Torvalds 已提交
1896 1897 1898 1899

	/*
	 * The number of IO-APIC IRQ registers (== #pins):
	 */
1900
	for (apic = 0; apic < nr_ioapics; apic++) {
L
Linus Torvalds 已提交
1901
		spin_lock_irqsave(&ioapic_lock, flags);
1902
		reg_01.raw = io_apic_read(apic, 1);
L
Linus Torvalds 已提交
1903
		spin_unlock_irqrestore(&ioapic_lock, flags);
1904 1905
		nr_ioapic_registers[apic] = reg_01.bits.entries+1;
	}
1906
	for(apic = 0; apic < nr_ioapics; apic++) {
1907 1908
		int pin;
		/* See if any of the pins is in ExtINT mode */
1909
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
1910
			struct IO_APIC_route_entry entry;
1911
			entry = ioapic_read_entry(apic, pin);
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941

			/* If the interrupt line is enabled and in ExtInt mode
			 * I have found the pin where the i8259 is connected.
			 */
			if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
				ioapic_i8259.apic = apic;
				ioapic_i8259.pin  = pin;
				goto found_i8259;
			}
		}
	}
 found_i8259:
	/* Look to see what if the MP table has reported the ExtINT */
	/* If we could not find the appropriate pin by looking at the ioapic
	 * the i8259 probably is not connected the ioapic but give the
	 * mptable a chance anyway.
	 */
	i8259_pin  = find_isa_irq_pin(0, mp_ExtINT);
	i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
	/* Trust the MP table if nothing is setup in the hardware */
	if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
		printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
		ioapic_i8259.pin  = i8259_pin;
		ioapic_i8259.apic = i8259_apic;
	}
	/* Complain if the MP table and the hardware disagree */
	if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
		(i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
	{
		printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
L
Linus Torvalds 已提交
1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959
	}

	/*
	 * Do not trust the IO-APIC being empty at bootup
	 */
	clear_IO_APIC();
}

/*
 * Not an __init, needed by the reboot code
 */
void disable_IO_APIC(void)
{
	/*
	 * Clear the IO-APIC before rebooting:
	 */
	clear_IO_APIC();

1960
	/*
1961
	 * If the i8259 is routed through an IOAPIC
1962
	 * Put that IOAPIC in virtual wire mode
1963
	 * so legacy interrupts can be delivered.
1964
	 */
1965
	if (ioapic_i8259.pin != -1) {
1966 1967 1968 1969 1970 1971 1972 1973 1974
		struct IO_APIC_route_entry entry;

		memset(&entry, 0, sizeof(entry));
		entry.mask            = 0; /* Enabled */
		entry.trigger         = 0; /* Edge */
		entry.irr             = 0;
		entry.polarity        = 0; /* High */
		entry.delivery_status = 0;
		entry.dest_mode       = 0; /* Physical */
1975
		entry.delivery_mode   = dest_ExtINT; /* ExtInt */
1976
		entry.vector          = 0;
1977
		entry.dest            = read_apic_id();
1978 1979 1980 1981

		/*
		 * Add it to the IO-APIC irq-routing table:
		 */
1982
		ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1983
	}
1984

1985
	disconnect_bsp_APIC(ioapic_i8259.pin != -1);
L
Linus Torvalds 已提交
1986 1987
}

1988
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004
/*
 * function to set the IO-APIC physical IDs based on the
 * values stored in the MPC table.
 *
 * by Matt Domsch <Matt_Domsch@dell.com>  Tue Dec 21 12:25:05 CST 1999
 */

static void __init setup_ioapic_ids_from_mpc(void)
{
	union IO_APIC_reg_00 reg_00;
	physid_mask_t phys_id_present_map;
	int apic;
	int i;
	unsigned char old_id;
	unsigned long flags;

2005
	if (x86_quirks->setup_ioapic_ids && x86_quirks->setup_ioapic_ids())
2006 2007
		return;

2008 2009 2010 2011
	/*
	 * Don't check I/O APIC IDs for xAPIC systems.  They have
	 * no meaning without the serial APIC bus.
	 */
2012 2013
	if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
		|| APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
2014
		return;
L
Linus Torvalds 已提交
2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
	/*
	 * This is broken; anything with a real cpu count has to
	 * circumvent this idiocy regardless.
	 */
	phys_id_present_map = ioapic_phys_id_map(phys_cpu_present_map);

	/*
	 * Set the IOAPIC ID to the value stored in the MPC table.
	 */
	for (apic = 0; apic < nr_ioapics; apic++) {

		/* Read the register 0 value */
		spin_lock_irqsave(&ioapic_lock, flags);
		reg_00.raw = io_apic_read(apic, 0);
		spin_unlock_irqrestore(&ioapic_lock, flags);
2030

2031
		old_id = mp_ioapics[apic].mp_apicid;
L
Linus Torvalds 已提交
2032

2033
		if (mp_ioapics[apic].mp_apicid >= get_physical_broadcast()) {
L
Linus Torvalds 已提交
2034
			printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
2035
				apic, mp_ioapics[apic].mp_apicid);
L
Linus Torvalds 已提交
2036 2037
			printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
				reg_00.bits.ID);
2038
			mp_ioapics[apic].mp_apicid = reg_00.bits.ID;
L
Linus Torvalds 已提交
2039 2040 2041 2042 2043 2044 2045 2046
		}

		/*
		 * Sanity check, is the ID really free? Every APIC in a
		 * system must have a unique ID or we get lots of nice
		 * 'stuck on smp_invalidate_needed IPI wait' messages.
		 */
		if (check_apicid_used(phys_id_present_map,
2047
					mp_ioapics[apic].mp_apicid)) {
L
Linus Torvalds 已提交
2048
			printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
2049
				apic, mp_ioapics[apic].mp_apicid);
L
Linus Torvalds 已提交
2050 2051 2052 2053 2054 2055 2056 2057
			for (i = 0; i < get_physical_broadcast(); i++)
				if (!physid_isset(i, phys_id_present_map))
					break;
			if (i >= get_physical_broadcast())
				panic("Max APIC ID exceeded!\n");
			printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
				i);
			physid_set(i, phys_id_present_map);
2058
			mp_ioapics[apic].mp_apicid = i;
L
Linus Torvalds 已提交
2059 2060
		} else {
			physid_mask_t tmp;
2061
			tmp = apicid_to_cpu_present(mp_ioapics[apic].mp_apicid);
L
Linus Torvalds 已提交
2062 2063
			apic_printk(APIC_VERBOSE, "Setting %d in the "
					"phys_id_present_map\n",
2064
					mp_ioapics[apic].mp_apicid);
L
Linus Torvalds 已提交
2065 2066 2067 2068 2069 2070 2071 2072
			physids_or(phys_id_present_map, phys_id_present_map, tmp);
		}


		/*
		 * We need to adjust the IRQ routing table
		 * if the ID changed.
		 */
2073
		if (old_id != mp_ioapics[apic].mp_apicid)
L
Linus Torvalds 已提交
2074
			for (i = 0; i < mp_irq_entries; i++)
2075 2076
				if (mp_irqs[i].mp_dstapic == old_id)
					mp_irqs[i].mp_dstapic
2077
						= mp_ioapics[apic].mp_apicid;
L
Linus Torvalds 已提交
2078 2079 2080 2081

		/*
		 * Read the right value from the MPC table and
		 * write it into the ID register.
2082
		 */
L
Linus Torvalds 已提交
2083 2084
		apic_printk(APIC_VERBOSE, KERN_INFO
			"...changing IO-APIC physical APIC ID to %d ...",
2085
			mp_ioapics[apic].mp_apicid);
L
Linus Torvalds 已提交
2086

2087
		reg_00.bits.ID = mp_ioapics[apic].mp_apicid;
L
Linus Torvalds 已提交
2088
		spin_lock_irqsave(&ioapic_lock, flags);
2089 2090
		io_apic_write(apic, 0, reg_00.raw);
		spin_unlock_irqrestore(&ioapic_lock, flags);
L
Linus Torvalds 已提交
2091 2092 2093 2094 2095 2096 2097

		/*
		 * Sanity check
		 */
		spin_lock_irqsave(&ioapic_lock, flags);
		reg_00.raw = io_apic_read(apic, 0);
		spin_unlock_irqrestore(&ioapic_lock, flags);
2098
		if (reg_00.bits.ID != mp_ioapics[apic].mp_apicid)
L
Linus Torvalds 已提交
2099 2100 2101 2102 2103
			printk("could not set ID!\n");
		else
			apic_printk(APIC_VERBOSE, " ok.\n");
	}
}
2104
#endif
L
Linus Torvalds 已提交
2105

2106
int no_timer_check __initdata;
2107 2108 2109 2110 2111 2112 2113 2114

static int __init notimercheck(char *s)
{
	no_timer_check = 1;
	return 1;
}
__setup("no_timer_check", notimercheck);

L
Linus Torvalds 已提交
2115 2116 2117 2118 2119 2120 2121 2122
/*
 * There is a nasty bug in some older SMP boards, their mptable lies
 * about the timer IRQ. We do the following to work around the situation:
 *
 *	- timer IRQ defaults to IO-APIC IRQ
 *	- if this function detects that timer IRQs are defunct, then we fall
 *	  back to ISA timer IRQs
 */
2123
static int __init timer_irq_works(void)
L
Linus Torvalds 已提交
2124 2125
{
	unsigned long t1 = jiffies;
2126
	unsigned long flags;
L
Linus Torvalds 已提交
2127

2128 2129 2130
	if (no_timer_check)
		return 1;

2131
	local_save_flags(flags);
L
Linus Torvalds 已提交
2132 2133 2134
	local_irq_enable();
	/* Let ten ticks pass... */
	mdelay((10 * 1000) / HZ);
2135
	local_irq_restore(flags);
L
Linus Torvalds 已提交
2136 2137 2138 2139 2140 2141 2142 2143

	/*
	 * Expect a few ticks at least, to be sure some possible
	 * glue logic does not lock up after one or two first
	 * ticks in a non-ExtINT mode.  Also the local APIC
	 * might have cached one ExtINT interrupt.  Finally, at
	 * least one tick may be lost due to delays.
	 */
2144 2145

	/* jiffies wrap? */
2146
	if (time_after(jiffies, t1 + 4))
L
Linus Torvalds 已提交
2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172
		return 1;
	return 0;
}

/*
 * In the SMP+IOAPIC case it might happen that there are an unspecified
 * number of pending IRQ events unhandled. These cases are very rare,
 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
 * better to do it this way as thus we do not have to be aware of
 * 'pending' interrupts in the IRQ path, except at this point.
 */
/*
 * Edge triggered needs to resend any interrupt
 * that was delayed but this is now handled in the device
 * independent code.
 */

/*
 * Starting up a edge-triggered IO-APIC interrupt is
 * nasty - we need to make sure that we get the edge.
 * If it is already asserted for some reason, we need
 * return 1 to indicate that is was pending.
 *
 * This is not complete - we should be able to fake
 * an edge even if it isn't on the 8259A...
 */
2173

2174
static unsigned int startup_ioapic_irq(unsigned int irq)
L
Linus Torvalds 已提交
2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
{
	int was_pending = 0;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	if (irq < 16) {
		disable_8259A_irq(irq);
		if (i8259A_irq_pending(irq))
			was_pending = 1;
	}
	__unmask_IO_APIC_irq(irq);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return was_pending;
}

2191
#ifdef CONFIG_X86_64
2192
static int ioapic_retrigger_irq(unsigned int irq)
L
Linus Torvalds 已提交
2193
{
2194 2195 2196 2197 2198 2199 2200

	struct irq_cfg *cfg = irq_cfg(irq);
	unsigned long flags;

	spin_lock_irqsave(&vector_lock, flags);
	send_IPI_mask(cpumask_of_cpu(first_cpu(cfg->domain)), cfg->vector);
	spin_unlock_irqrestore(&vector_lock, flags);
2201 2202 2203

	return 1;
}
2204 2205
#else
static int ioapic_retrigger_irq(unsigned int irq)
2206
{
2207
        send_IPI_self(irq_cfg(irq)->vector);
2208

2209 2210 2211
        return 1;
}
#endif
2212

2213 2214 2215 2216 2217 2218 2219 2220
/*
 * Level and edge triggered IO-APIC interrupts need different handling,
 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
 * handled with the level-triggered descriptor, but that one has slightly
 * more overhead. Level-triggered interrupts cannot be handled with the
 * edge-triggered handler, without risking IRQ storms and other ugly
 * races.
 */
2221

2222
#ifdef CONFIG_SMP
2223

2224 2225
#ifdef CONFIG_INTR_REMAP
static void ir_irq_migration(struct work_struct *work);
2226

2227
static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration);
2228

2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248
/*
 * Migrate the IO-APIC irq in the presence of intr-remapping.
 *
 * For edge triggered, irq migration is a simple atomic update(of vector
 * and cpu destination) of IRTE and flush the hardware cache.
 *
 * For level triggered, we need to modify the io-apic RTE aswell with the update
 * vector information, along with modifying IRTE with vector and destination.
 * So irq migration for level triggered is little  bit more complex compared to
 * edge triggered migration. But the good news is, we use the same algorithm
 * for level triggered migration as we have today, only difference being,
 * we now initiate the irq migration from process context instead of the
 * interrupt context.
 *
 * In future, when we do a directed EOI (combined with cpu EOI broadcast
 * suppression) to the IO-APIC, level triggered irq migration will also be
 * as simple as edge triggered migration and we can do the irq migration
 * with a simple atomic update to IO-APIC RTE.
 */
static void migrate_ioapic_irq(int irq, cpumask_t mask)
2249
{
2250 2251 2252 2253 2254 2255 2256
	struct irq_cfg *cfg;
	struct irq_desc *desc;
	cpumask_t tmp, cleanup_mask;
	struct irte irte;
	int modify_ioapic_rte;
	unsigned int dest;
	unsigned long flags;
2257

2258 2259
	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
2260 2261
		return;

2262 2263
	if (get_irte(irq, &irte))
		return;
2264

2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421
	if (assign_irq_vector(irq, mask))
		return;

	cfg = irq_cfg(irq);
	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	desc = irq_to_desc(irq);
	modify_ioapic_rte = desc->status & IRQ_LEVEL;
	if (modify_ioapic_rte) {
		spin_lock_irqsave(&ioapic_lock, flags);
		__target_IO_APIC_irq(irq, dest, cfg->vector);
		spin_unlock_irqrestore(&ioapic_lock, flags);
	}

	irte.vector = cfg->vector;
	irte.dest_id = IRTE_DEST(dest);

	/*
	 * Modified the IRTE and flushes the Interrupt entry cache.
	 */
	modify_irte(irq, &irte);

	if (cfg->move_in_progress) {
		cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
		cfg->move_cleanup_count = cpus_weight(cleanup_mask);
		send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
		cfg->move_in_progress = 0;
	}

	desc->affinity = mask;
}

static int migrate_irq_remapped_level(int irq)
{
	int ret = -1;
	struct irq_desc *desc = irq_to_desc(irq);

	mask_IO_APIC_irq(irq);

	if (io_apic_level_ack_pending(irq)) {
		/*
	 	 * Interrupt in progress. Migrating irq now will change the
		 * vector information in the IO-APIC RTE and that will confuse
		 * the EOI broadcast performed by cpu.
		 * So, delay the irq migration to the next instance.
		 */
		schedule_delayed_work(&ir_migration_work, 1);
		goto unmask;
	}

	/* everthing is clear. we have right of way */
	migrate_ioapic_irq(irq, desc->pending_mask);

	ret = 0;
	desc->status &= ~IRQ_MOVE_PENDING;
	cpus_clear(desc->pending_mask);

unmask:
	unmask_IO_APIC_irq(irq);
	return ret;
}

static void ir_irq_migration(struct work_struct *work)
{
	unsigned int irq;
	struct irq_desc *desc;

	for_each_irq_desc(irq, desc) {
		if (desc->status & IRQ_MOVE_PENDING) {
			unsigned long flags;

			spin_lock_irqsave(&desc->lock, flags);
			if (!desc->chip->set_affinity ||
			    !(desc->status & IRQ_MOVE_PENDING)) {
				desc->status &= ~IRQ_MOVE_PENDING;
				spin_unlock_irqrestore(&desc->lock, flags);
				continue;
			}

			desc->chip->set_affinity(irq, desc->pending_mask);
			spin_unlock_irqrestore(&desc->lock, flags);
		}
	}
}

/*
 * Migrates the IRQ destination in the process context.
 */
static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
{
	struct irq_desc *desc = irq_to_desc(irq);

	if (desc->status & IRQ_LEVEL) {
		desc->status |= IRQ_MOVE_PENDING;
		desc->pending_mask = mask;
		migrate_irq_remapped_level(irq);
		return;
	}

	migrate_ioapic_irq(irq, mask);
}
#endif

asmlinkage void smp_irq_move_cleanup_interrupt(void)
{
	unsigned vector, me;
	ack_APIC_irq();
#ifdef CONFIG_X86_64
	exit_idle();
#endif
	irq_enter();

	me = smp_processor_id();
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		unsigned int irq;
		struct irq_desc *desc;
		struct irq_cfg *cfg;
		irq = __get_cpu_var(vector_irq)[vector];

		desc = irq_to_desc(irq);
		if (!desc)
			continue;

		cfg = irq_cfg(irq);
		spin_lock(&desc->lock);
		if (!cfg->move_cleanup_count)
			goto unlock;

		if ((vector == cfg->vector) && cpu_isset(me, cfg->domain))
			goto unlock;

		__get_cpu_var(vector_irq)[vector] = -1;
		cfg->move_cleanup_count--;
unlock:
		spin_unlock(&desc->lock);
	}

	irq_exit();
}

static void irq_complete_move(unsigned int irq)
{
	struct irq_cfg *cfg = irq_cfg(irq);
	unsigned vector, me;

	if (likely(!cfg->move_in_progress))
		return;

	vector = ~get_irq_regs()->orig_ax;
	me = smp_processor_id();
	if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) {
		cpumask_t cleanup_mask;

		cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
		cfg->move_cleanup_count = cpus_weight(cleanup_mask);
		send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2422 2423 2424 2425 2426 2427
		cfg->move_in_progress = 0;
	}
}
#else
static inline void irq_complete_move(unsigned int irq) {}
#endif
2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438
#ifdef CONFIG_INTR_REMAP
static void ack_x2apic_level(unsigned int irq)
{
	ack_x2APIC_irq();
}

static void ack_x2apic_edge(unsigned int irq)
{
	ack_x2APIC_irq();
}
#endif
2439

2440 2441 2442 2443 2444 2445 2446
static void ack_apic_edge(unsigned int irq)
{
	irq_complete_move(irq);
	move_native_irq(irq);
	ack_APIC_irq();
}

Y
Yinghai Lu 已提交
2447 2448 2449 2450
#ifdef CONFIG_X86_32
atomic_t irq_mis_count;
#endif

2451 2452
static void ack_apic_level(unsigned int irq)
{
Y
Yinghai Lu 已提交
2453 2454 2455 2456
#ifdef CONFIG_X86_32
	unsigned long v;
	int i;
#endif
2457
	int do_unmask_irq = 0;
2458

2459
	irq_complete_move(irq);
2460
#ifdef CONFIG_GENERIC_PENDING_IRQ
2461 2462 2463 2464 2465
	/* If we are moving the irq we need to mask it */
	if (unlikely(irq_to_desc(irq)->status & IRQ_MOVE_PENDING)) {
		do_unmask_irq = 1;
		mask_IO_APIC_irq(irq);
	}
2466 2467
#endif

Y
Yinghai Lu 已提交
2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492
#ifdef CONFIG_X86_32
	/*
	* It appears there is an erratum which affects at least version 0x11
	* of I/O APIC (that's the 82093AA and cores integrated into various
	* chipsets).  Under certain conditions a level-triggered interrupt is
	* erroneously delivered as edge-triggered one but the respective IRR
	* bit gets set nevertheless.  As a result the I/O unit expects an EOI
	* message but it will never arrive and further interrupts are blocked
	* from the source.  The exact reason is so far unknown, but the
	* phenomenon was observed when two consecutive interrupt requests
	* from a given source get delivered to the same CPU and the source is
	* temporarily disabled in between.
	*
	* A workaround is to simulate an EOI message manually.  We achieve it
	* by setting the trigger mode to edge and then to level when the edge
	* trigger mode gets detected in the TMR of a local APIC for a
	* level-triggered interrupt.  We mask the source for the time of the
	* operation to prevent an edge-triggered interrupt escaping meanwhile.
	* The idea is from Manfred Spraul.  --macro
	*/
	i = irq_cfg(irq)->vector;

	v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
#endif

2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
	/*
	 * We must acknowledge the irq before we move it or the acknowledge will
	 * not propagate properly.
	 */
	ack_APIC_irq();

	/* Now we can move and renable the irq */
	if (unlikely(do_unmask_irq)) {
		/* Only migrate the irq if the ack has been received.
		 *
		 * On rare occasions the broadcast level triggered ack gets
		 * delayed going to ioapics, and if we reprogram the
		 * vector while Remote IRR is still set the irq will never
		 * fire again.
		 *
		 * To prevent this scenario we read the Remote IRR bit
		 * of the ioapic.  This has two effects.
		 * - On any sane system the read of the ioapic will
		 *   flush writes (and acks) going to the ioapic from
		 *   this cpu.
		 * - We get to see if the ACK has actually been delivered.
		 *
		 * Based on failed experiments of reprogramming the
		 * ioapic entry from outside of irq context starting
		 * with masking the ioapic entry and then polling until
		 * Remote IRR was clear before reprogramming the
		 * ioapic I don't trust the Remote IRR bit to be
		 * completey accurate.
		 *
		 * However there appears to be no other way to plug
		 * this race, so if the Remote IRR bit is not
		 * accurate and is causing problems then it is a hardware bug
		 * and you can go talk to the chipset vendor about it.
		 */
		if (!io_apic_level_ack_pending(irq))
			move_masked_irq(irq);
		unmask_IO_APIC_irq(irq);
	}
2531

Y
Yinghai Lu 已提交
2532
#ifdef CONFIG_X86_32
2533 2534 2535 2536 2537 2538 2539
	if (!(v & (1 << (i & 0x1f)))) {
		atomic_inc(&irq_mis_count);
		spin_lock(&ioapic_lock);
		__mask_and_edge_IO_APIC_irq(irq);
		__unmask_and_level_IO_APIC_irq(irq);
		spin_unlock(&ioapic_lock);
	}
2540
#endif
Y
Yinghai Lu 已提交
2541
}
2542

2543 2544
static struct irq_chip ioapic_chip __read_mostly = {
	.name 		= "IO-APIC",
2545 2546 2547
	.startup 	= startup_ioapic_irq,
	.mask	 	= mask_IO_APIC_irq,
	.unmask	 	= unmask_IO_APIC_irq,
2548 2549
	.ack 		= ack_apic_edge,
	.eoi 		= ack_apic_level,
2550
#ifdef CONFIG_SMP
2551
	.set_affinity 	= set_ioapic_affinity_irq,
2552
#endif
2553
	.retrigger	= ioapic_retrigger_irq,
L
Linus Torvalds 已提交
2554 2555
};

2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569
#ifdef CONFIG_INTR_REMAP
static struct irq_chip ir_ioapic_chip __read_mostly = {
	.name 		= "IR-IO-APIC",
	.startup 	= startup_ioapic_irq,
	.mask	 	= mask_IO_APIC_irq,
	.unmask	 	= unmask_IO_APIC_irq,
	.ack 		= ack_x2apic_edge,
	.eoi 		= ack_x2apic_level,
#ifdef CONFIG_SMP
	.set_affinity 	= set_ir_ioapic_affinity_irq,
#endif
	.retrigger	= ioapic_retrigger_irq,
};
#endif
L
Linus Torvalds 已提交
2570 2571 2572 2573

static inline void init_IO_APIC_traps(void)
{
	int irq;
2574
	struct irq_desc *desc;
2575
	struct irq_cfg *cfg;
L
Linus Torvalds 已提交
2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587

	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
2588
	for_each_irq_cfg(irq, cfg) {
2589
		if (IO_APIC_IRQ(irq) && !cfg->vector) {
L
Linus Torvalds 已提交
2590 2591 2592 2593 2594 2595 2596
			/*
			 * Hmm.. We don't have an entry for this,
			 * so default to an old-fashioned 8259
			 * interrupt if we can..
			 */
			if (irq < 16)
				make_8259A_irq(irq);
2597 2598
			else {
				desc = irq_to_desc(irq);
L
Linus Torvalds 已提交
2599
				/* Strange. Oh, well.. */
2600 2601
				desc->chip = &no_irq_chip;
			}
L
Linus Torvalds 已提交
2602 2603 2604 2605
		}
	}
}

2606 2607 2608
/*
 * The local APIC irq-chip implementation:
 */
L
Linus Torvalds 已提交
2609

2610
static void mask_lapic_irq(unsigned int irq)
L
Linus Torvalds 已提交
2611 2612 2613 2614
{
	unsigned long v;

	v = apic_read(APIC_LVT0);
2615
	apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
L
Linus Torvalds 已提交
2616 2617
}

2618
static void unmask_lapic_irq(unsigned int irq)
L
Linus Torvalds 已提交
2619
{
2620
	unsigned long v;
L
Linus Torvalds 已提交
2621

2622
	v = apic_read(APIC_LVT0);
2623
	apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2624
}
L
Linus Torvalds 已提交
2625

2626
static void ack_lapic_irq (unsigned int irq)
2627 2628 2629 2630
{
	ack_APIC_irq();
}

2631
static struct irq_chip lapic_chip __read_mostly = {
2632
	.name		= "local-APIC",
2633 2634
	.mask		= mask_lapic_irq,
	.unmask		= unmask_lapic_irq,
2635
	.ack		= ack_lapic_irq,
L
Linus Torvalds 已提交
2636 2637
};

2638
static void lapic_register_intr(int irq)
2639
{
2640 2641 2642 2643
	struct irq_desc *desc;

	desc = irq_to_desc(irq);
	desc->status &= ~IRQ_LEVEL;
2644 2645 2646 2647
	set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
				      "edge");
}

2648
static void __init setup_nmi(void)
L
Linus Torvalds 已提交
2649 2650
{
	/*
2651
	 * Dirty trick to enable the NMI watchdog ...
L
Linus Torvalds 已提交
2652 2653 2654 2655 2656 2657
	 * We put the 8259A master into AEOI mode and
	 * unmask on all local APICs LVT0 as NMI.
	 *
	 * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire')
	 * is from Maciej W. Rozycki - so we do not have to EOI from
	 * the NMI handler or the timer interrupt.
2658
	 */
L
Linus Torvalds 已提交
2659 2660
	apic_printk(APIC_VERBOSE, KERN_INFO "activating NMI Watchdog ...");

2661
	enable_NMI_through_LVT0();
L
Linus Torvalds 已提交
2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672

	apic_printk(APIC_VERBOSE, " done.\n");
}

/*
 * This looks a bit hackish but it's about the only one way of sending
 * a few INTA cycles to 8259As and any associated glue logic.  ICR does
 * not support the ExtINT mode, unfortunately.  We need to send these
 * cycles as some i82489DX-based boards have glue logic that keeps the
 * 8259A interrupt line asserted until INTA.  --macro
 */
2673
static inline void __init unlock_ExtINT_logic(void)
L
Linus Torvalds 已提交
2674
{
2675
	int apic, pin, i;
L
Linus Torvalds 已提交
2676 2677 2678
	struct IO_APIC_route_entry entry0, entry1;
	unsigned char save_control, save_freq_select;

2679
	pin  = find_isa_irq_pin(8, mp_INT);
2680 2681 2682 2683
	if (pin == -1) {
		WARN_ON_ONCE(1);
		return;
	}
2684
	apic = find_isa_irq_apic(8, mp_INT);
2685 2686
	if (apic == -1) {
		WARN_ON_ONCE(1);
L
Linus Torvalds 已提交
2687
		return;
2688
	}
L
Linus Torvalds 已提交
2689

2690
	entry0 = ioapic_read_entry(apic, pin);
2691
	clear_IO_APIC_pin(apic, pin);
L
Linus Torvalds 已提交
2692 2693 2694 2695 2696

	memset(&entry1, 0, sizeof(entry1));

	entry1.dest_mode = 0;			/* physical delivery */
	entry1.mask = 0;			/* unmask IRQ now */
2697
	entry1.dest = hard_smp_processor_id();
L
Linus Torvalds 已提交
2698 2699 2700 2701 2702
	entry1.delivery_mode = dest_ExtINT;
	entry1.polarity = entry0.polarity;
	entry1.trigger = 0;
	entry1.vector = 0;

2703
	ioapic_write_entry(apic, pin, entry1);
L
Linus Torvalds 已提交
2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719

	save_control = CMOS_READ(RTC_CONTROL);
	save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
	CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
		   RTC_FREQ_SELECT);
	CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);

	i = 100;
	while (i-- > 0) {
		mdelay(10);
		if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
			i -= 10;
	}

	CMOS_WRITE(save_control, RTC_CONTROL);
	CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2720
	clear_IO_APIC_pin(apic, pin);
L
Linus Torvalds 已提交
2721

2722
	ioapic_write_entry(apic, pin, entry0);
L
Linus Torvalds 已提交
2723 2724
}

Y
Yinghai Lu 已提交
2725
static int disable_timer_pin_1 __initdata;
2726
/* Actually the next is obsolete, but keep it for paranoid reasons -AK */
2727
static int __init disable_timer_pin_setup(char *arg)
Y
Yinghai Lu 已提交
2728 2729 2730 2731
{
	disable_timer_pin_1 = 1;
	return 0;
}
2732
early_param("disable_timer_pin_1", disable_timer_pin_setup);
Y
Yinghai Lu 已提交
2733 2734 2735

int timer_through_8259 __initdata;

L
Linus Torvalds 已提交
2736 2737 2738 2739 2740
/*
 * This code may look a bit paranoid, but it's supposed to cooperate with
 * a wide range of boards and BIOS bugs.  Fortunately only the timer IRQ
 * is so screwy.  Thanks to Brian Perkins for testing/hacking this beast
 * fanatically on his truly buggy board.
2741 2742
 *
 * FIXME: really need to revamp this for all platforms.
L
Linus Torvalds 已提交
2743
 */
2744
static inline void __init check_timer(void)
L
Linus Torvalds 已提交
2745
{
2746
	struct irq_cfg *cfg = irq_cfg(0);
2747
	int apic1, pin1, apic2, pin2;
2748
	unsigned long flags;
2749 2750
	unsigned int ver;
	int no_pin1 = 0;
2751 2752

	local_irq_save(flags);
2753

2754 2755
        ver = apic_read(APIC_LVR);
        ver = GET_APIC_VERSION(ver);
I
Ingo Molnar 已提交
2756

L
Linus Torvalds 已提交
2757 2758 2759 2760
	/*
	 * get/set the timer IRQ vector:
	 */
	disable_8259A_irq(0);
2761
	assign_irq_vector(0, TARGET_CPUS);
L
Linus Torvalds 已提交
2762 2763

	/*
2764 2765 2766 2767 2768 2769 2770
	 * As IRQ0 is to be enabled in the 8259A, the virtual
	 * wire has to be disabled in the local APIC.  Also
	 * timer interrupts need to be acknowledged manually in
	 * the 8259A for the i82489DX when using the NMI
	 * watchdog as that APIC treats NMIs as level-triggered.
	 * The AEOI mode will finish them in the 8259A
	 * automatically.
L
Linus Torvalds 已提交
2771
	 */
2772
	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
L
Linus Torvalds 已提交
2773
	init_8259A(1);
2774
#ifdef CONFIG_X86_32
2775
	timer_ack = (nmi_watchdog == NMI_IO_APIC && !APIC_INTEGRATED(ver));
2776
#endif
L
Linus Torvalds 已提交
2777

2778 2779 2780 2781
	pin1  = find_isa_irq_pin(0, mp_INT);
	apic1 = find_isa_irq_apic(0, mp_INT);
	pin2  = ioapic_i8259.pin;
	apic2 = ioapic_i8259.apic;
L
Linus Torvalds 已提交
2782

2783 2784
	apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
		    "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2785
		    cfg->vector, apic1, pin1, apic2, pin2);
L
Linus Torvalds 已提交
2786

2787 2788 2789 2790 2791 2792 2793 2794
	/*
	 * Some BIOS writers are clueless and report the ExtINTA
	 * I/O APIC input from the cascaded 8259A as the timer
	 * interrupt input.  So just in case, if only one pin
	 * was found above, try it both directly and through the
	 * 8259A.
	 */
	if (pin1 == -1) {
2795 2796 2797 2798
#ifdef CONFIG_INTR_REMAP
		if (intr_remapping_enabled)
			panic("BIOS bug: timer not connected to IO-APIC");
#endif
2799 2800 2801 2802 2803 2804 2805 2806
		pin1 = pin2;
		apic1 = apic2;
		no_pin1 = 1;
	} else if (pin2 == -1) {
		pin2 = pin1;
		apic2 = apic1;
	}

L
Linus Torvalds 已提交
2807 2808 2809 2810
	if (pin1 != -1) {
		/*
		 * Ok, does IRQ0 through the IOAPIC work?
		 */
2811 2812
		if (no_pin1) {
			add_pin_to_irq(0, apic1, pin1);
2813
			setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
2814
		}
L
Linus Torvalds 已提交
2815 2816 2817 2818 2819 2820
		unmask_IO_APIC_irq(0);
		if (timer_irq_works()) {
			if (nmi_watchdog == NMI_IO_APIC) {
				setup_nmi();
				enable_8259A_irq(0);
			}
2821 2822
			if (disable_timer_pin_1 > 0)
				clear_IO_APIC_pin(0, pin1);
2823
			goto out;
L
Linus Torvalds 已提交
2824
		}
2825 2826 2827 2828
#ifdef CONFIG_INTR_REMAP
		if (intr_remapping_enabled)
			panic("timer doesn't work through Interrupt-remapped IO-APIC");
#endif
2829
		clear_IO_APIC_pin(apic1, pin1);
2830
		if (!no_pin1)
2831 2832
			apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
				    "8254 timer not connected to IO-APIC\n");
L
Linus Torvalds 已提交
2833

2834 2835 2836 2837
		apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
			    "(IRQ0) through the 8259A ...\n");
		apic_printk(APIC_QUIET, KERN_INFO
			    "..... (found apic %d pin %d) ...\n", apic2, pin2);
L
Linus Torvalds 已提交
2838 2839 2840
		/*
		 * legacy devices should be connected to IO APIC #0
		 */
2841
		replace_pin_at_irq(0, apic1, pin1, apic2, pin2);
2842
		setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2843
		unmask_IO_APIC_irq(0);
2844
		enable_8259A_irq(0);
L
Linus Torvalds 已提交
2845
		if (timer_irq_works()) {
2846
			apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2847
			timer_through_8259 = 1;
L
Linus Torvalds 已提交
2848
			if (nmi_watchdog == NMI_IO_APIC) {
2849
				disable_8259A_irq(0);
L
Linus Torvalds 已提交
2850
				setup_nmi();
2851
				enable_8259A_irq(0);
L
Linus Torvalds 已提交
2852
			}
2853
			goto out;
L
Linus Torvalds 已提交
2854 2855 2856 2857
		}
		/*
		 * Cleanup, just in case ...
		 */
2858
		disable_8259A_irq(0);
2859
		clear_IO_APIC_pin(apic2, pin2);
2860
		apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
L
Linus Torvalds 已提交
2861 2862 2863
	}

	if (nmi_watchdog == NMI_IO_APIC) {
2864 2865
		apic_printk(APIC_QUIET, KERN_WARNING "timer doesn't work "
			    "through the IO-APIC - disabling NMI Watchdog!\n");
2866
		nmi_watchdog = NMI_NONE;
L
Linus Torvalds 已提交
2867
	}
2868
#ifdef CONFIG_X86_32
2869
	timer_ack = 0;
2870
#endif
L
Linus Torvalds 已提交
2871

2872 2873
	apic_printk(APIC_QUIET, KERN_INFO
		    "...trying to set up timer as Virtual Wire IRQ...\n");
L
Linus Torvalds 已提交
2874

2875 2876
	lapic_register_intr(0);
	apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);	/* Fixed mode */
L
Linus Torvalds 已提交
2877 2878 2879
	enable_8259A_irq(0);

	if (timer_irq_works()) {
2880
		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2881
		goto out;
L
Linus Torvalds 已提交
2882
	}
2883
	disable_8259A_irq(0);
2884
	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2885
	apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
L
Linus Torvalds 已提交
2886

2887 2888
	apic_printk(APIC_QUIET, KERN_INFO
		    "...trying to set up timer as ExtINT IRQ...\n");
L
Linus Torvalds 已提交
2889 2890 2891

	init_8259A(0);
	make_8259A_irq(0);
2892
	apic_write(APIC_LVT0, APIC_DM_EXTINT);
L
Linus Torvalds 已提交
2893 2894 2895 2896

	unlock_ExtINT_logic();

	if (timer_irq_works()) {
2897
		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2898
		goto out;
L
Linus Torvalds 已提交
2899
	}
2900
	apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
L
Linus Torvalds 已提交
2901
	panic("IO-APIC + timer doesn't work!  Boot with apic=debug and send a "
2902
		"report.  Then try booting with the 'noapic' option.\n");
2903 2904
out:
	local_irq_restore(flags);
L
Linus Torvalds 已提交
2905 2906 2907
}

/*
2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922
 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
 * to devices.  However there may be an I/O APIC pin available for
 * this interrupt regardless.  The pin may be left unconnected, but
 * typically it will be reused as an ExtINT cascade interrupt for
 * the master 8259A.  In the MPS case such a pin will normally be
 * reported as an ExtINT interrupt in the MP table.  With ACPI
 * there is no provision for ExtINT interrupts, and in the absence
 * of an override it would be treated as an ordinary ISA I/O APIC
 * interrupt, that is edge-triggered and unmasked by default.  We
 * used to do this, but it caused problems on some systems because
 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
 * the same ExtINT cascade interrupt to drive the local APIC of the
 * bootstrap processor.  Therefore we refrain from routing IRQ2 to
 * the I/O APIC in all cases now.  No actual device should request
 * it anyway.  --macro
L
Linus Torvalds 已提交
2923 2924 2925 2926 2927
 */
#define PIC_IRQS	(1 << PIC_CASCADE_IR)

void __init setup_IO_APIC(void)
{
2928 2929

#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
2930
	enable_IO_APIC();
2931 2932 2933 2934 2935
#else
	/*
	 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
	 */
#endif
L
Linus Torvalds 已提交
2936

2937
	io_apic_irqs = ~PIC_IRQS;
L
Linus Torvalds 已提交
2938

2939 2940 2941 2942 2943 2944 2945 2946
	apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
        /*
         * Set up IO-APIC IRQ routing.
         */
#ifdef CONFIG_X86_32
        if (!acpi_ioapic)
                setup_ioapic_ids_from_mpc();
#endif
L
Linus Torvalds 已提交
2947 2948 2949
	sync_Arb_IDs();
	setup_IO_APIC_irqs();
	init_IO_APIC_traps();
2950
	check_timer();
L
Linus Torvalds 已提交
2951 2952 2953
}

/*
2954 2955
 *      Called after all the initialization is done. If we didnt find any
 *      APIC bugs then we can allow the modify fast path
L
Linus Torvalds 已提交
2956
 */
2957

L
Linus Torvalds 已提交
2958 2959
static int __init io_apic_bug_finalize(void)
{
2960 2961 2962
        if (sis_apic_bug == -1)
                sis_apic_bug = 0;
        return 0;
L
Linus Torvalds 已提交
2963 2964 2965 2966 2967 2968 2969 2970
}

late_initcall(io_apic_bug_finalize);

struct sysfs_ioapic_data {
	struct sys_device dev;
	struct IO_APIC_route_entry entry[0];
};
2971
static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS];
L
Linus Torvalds 已提交
2972

2973
static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
L
Linus Torvalds 已提交
2974 2975 2976 2977
{
	struct IO_APIC_route_entry *entry;
	struct sysfs_ioapic_data *data;
	int i;
2978

L
Linus Torvalds 已提交
2979 2980
	data = container_of(dev, struct sysfs_ioapic_data, dev);
	entry = data->entry;
2981 2982
	for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ )
		*entry = ioapic_read_entry(dev->id, i);
L
Linus Torvalds 已提交
2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993

	return 0;
}

static int ioapic_resume(struct sys_device *dev)
{
	struct IO_APIC_route_entry *entry;
	struct sysfs_ioapic_data *data;
	unsigned long flags;
	union IO_APIC_reg_00 reg_00;
	int i;
2994

L
Linus Torvalds 已提交
2995 2996 2997 2998 2999
	data = container_of(dev, struct sysfs_ioapic_data, dev);
	entry = data->entry;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(dev->id, 0);
3000 3001
	if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
		reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
L
Linus Torvalds 已提交
3002 3003 3004
		io_apic_write(dev->id, 0, reg_00.raw);
	}
	spin_unlock_irqrestore(&ioapic_lock, flags);
3005
	for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
3006
		ioapic_write_entry(dev->id, i, entry[i]);
L
Linus Torvalds 已提交
3007 3008 3009 3010 3011

	return 0;
}

static struct sysdev_class ioapic_sysdev_class = {
3012
	.name = "ioapic",
L
Linus Torvalds 已提交
3013 3014 3015 3016 3017 3018
	.suspend = ioapic_suspend,
	.resume = ioapic_resume,
};

static int __init ioapic_init_sysfs(void)
{
3019 3020
	struct sys_device * dev;
	int i, size, error;
L
Linus Torvalds 已提交
3021 3022 3023 3024 3025

	error = sysdev_class_register(&ioapic_sysdev_class);
	if (error)
		return error;

3026
	for (i = 0; i < nr_ioapics; i++ ) {
3027
		size = sizeof(struct sys_device) + nr_ioapic_registers[i]
L
Linus Torvalds 已提交
3028
			* sizeof(struct IO_APIC_route_entry);
3029
		mp_ioapic_data[i] = kzalloc(size, GFP_KERNEL);
L
Linus Torvalds 已提交
3030 3031 3032 3033 3034
		if (!mp_ioapic_data[i]) {
			printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
			continue;
		}
		dev = &mp_ioapic_data[i]->dev;
3035
		dev->id = i;
L
Linus Torvalds 已提交
3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050
		dev->cls = &ioapic_sysdev_class;
		error = sysdev_register(dev);
		if (error) {
			kfree(mp_ioapic_data[i]);
			mp_ioapic_data[i] = NULL;
			printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
			continue;
		}
	}

	return 0;
}

device_initcall(ioapic_init_sysfs);

3051
/*
3052
 * Dynamic irq allocate and deallocation
3053
 */
Y
Yinghai Lu 已提交
3054
unsigned int create_irq_nr(unsigned int irq_want)
3055
{
3056
	/* Allocate an unused irq */
3057 3058
	unsigned int irq;
	unsigned int new;
3059
	unsigned long flags;
3060
	struct irq_cfg *cfg_new;
3061

3062
#ifndef CONFIG_HAVE_SPARSE_IRQ
Y
Yinghai Lu 已提交
3063
	irq_want = nr_irqs - 1;
3064
#endif
Y
Yinghai Lu 已提交
3065 3066

	irq = 0;
3067
	spin_lock_irqsave(&vector_lock, flags);
3068
	for (new = irq_want; new > 0; new--) {
3069 3070
		if (platform_legacy_irq(new))
			continue;
3071 3072
		cfg_new = irq_cfg(new);
		if (cfg_new && cfg_new->vector != 0)
3073
			continue;
3074
		/* check if need to create one */
3075 3076
		if (!cfg_new)
			cfg_new = irq_cfg_alloc(new);
3077
		if (__assign_irq_vector(new, TARGET_CPUS) == 0)
3078 3079 3080 3081
			irq = new;
		break;
	}
	spin_unlock_irqrestore(&vector_lock, flags);
3082

Y
Yinghai Lu 已提交
3083
	if (irq > 0) {
3084 3085 3086 3087 3088
		dynamic_irq_init(irq);
	}
	return irq;
}

Y
Yinghai Lu 已提交
3089 3090
int create_irq(void)
{
3091 3092 3093 3094 3095 3096 3097 3098
	int irq;

	irq = create_irq_nr(nr_irqs - 1);

	if (irq == 0)
		irq = -1;

	return irq;
Y
Yinghai Lu 已提交
3099 3100
}

3101 3102 3103 3104 3105 3106
void destroy_irq(unsigned int irq)
{
	unsigned long flags;

	dynamic_irq_cleanup(irq);

3107 3108 3109
#ifdef CONFIG_INTR_REMAP
	free_irte(irq);
#endif
3110
	spin_lock_irqsave(&vector_lock, flags);
3111
	__clear_irq_vector(irq);
3112 3113 3114
	spin_unlock_irqrestore(&vector_lock, flags);
}

3115
/*
S
Simon Arlott 已提交
3116
 * MSI message composition
3117 3118
 */
#ifdef CONFIG_PCI_MSI
3119
static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
3120
{
3121 3122
	struct irq_cfg *cfg;
	int err;
3123
	unsigned dest;
3124
	cpumask_t tmp;
3125

3126 3127 3128 3129
	tmp = TARGET_CPUS;
	err = assign_irq_vector(irq, tmp);
	if (err)
		return err;
3130

3131 3132 3133 3134
	cfg = irq_cfg(irq);
	cpus_and(tmp, cfg->domain, tmp);
	dest = cpu_mask_to_apicid(tmp);

3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173
#ifdef CONFIG_INTR_REMAP
	if (irq_remapped(irq)) {
		struct irte irte;
		int ir_index;
		u16 sub_handle;

		ir_index = map_irq_to_irte_handle(irq, &sub_handle);
		BUG_ON(ir_index == -1);

		memset (&irte, 0, sizeof(irte));

		irte.present = 1;
		irte.dst_mode = INT_DEST_MODE;
		irte.trigger_mode = 0; /* edge */
		irte.dlvry_mode = INT_DELIVERY_MODE;
		irte.vector = cfg->vector;
		irte.dest_id = IRTE_DEST(dest);

		modify_irte(irq, &irte);

		msg->address_hi = MSI_ADDR_BASE_HI;
		msg->data = sub_handle;
		msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
				  MSI_ADDR_IR_SHV |
				  MSI_ADDR_IR_INDEX1(ir_index) |
				  MSI_ADDR_IR_INDEX2(ir_index);
	} else
#endif
	{
		msg->address_hi = MSI_ADDR_BASE_HI;
		msg->address_lo =
			MSI_ADDR_BASE_LO |
			((INT_DEST_MODE == 0) ?
				MSI_ADDR_DEST_MODE_PHYSICAL:
				MSI_ADDR_DEST_MODE_LOGICAL) |
			((INT_DELIVERY_MODE != dest_LowestPrio) ?
				MSI_ADDR_REDIRECTION_CPU:
				MSI_ADDR_REDIRECTION_LOWPRI) |
			MSI_ADDR_DEST_ID(dest);
3174

3175 3176 3177 3178 3179 3180 3181 3182
		msg->data =
			MSI_DATA_TRIGGER_EDGE |
			MSI_DATA_LEVEL_ASSERT |
			((INT_DELIVERY_MODE != dest_LowestPrio) ?
				MSI_DATA_DELIVERY_FIXED:
				MSI_DATA_DELIVERY_LOWPRI) |
			MSI_DATA_VECTOR(cfg->vector);
	}
3183
	return err;
3184 3185
}

3186 3187
#ifdef CONFIG_SMP
static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
3188
{
3189
	struct irq_cfg *cfg;
3190 3191 3192
	struct msi_msg msg;
	unsigned int dest;
	cpumask_t tmp;
3193
	struct irq_desc *desc;
3194 3195 3196

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
3197
		return;
3198

3199
	if (assign_irq_vector(irq, mask))
3200
		return;
3201

3202 3203 3204
	cfg = irq_cfg(irq);
	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);
3205 3206 3207 3208

	read_msi_msg(irq, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
3209
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
3210 3211 3212 3213
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	write_msi_msg(irq, &msg);
3214 3215
	desc = irq_to_desc(irq);
	desc->affinity = mask;
3216
}
3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268

#ifdef CONFIG_INTR_REMAP
/*
 * Migrate the MSI irq to another cpumask. This migration is
 * done in the process context using interrupt-remapping hardware.
 */
static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
{
	struct irq_cfg *cfg;
	unsigned int dest;
	cpumask_t tmp, cleanup_mask;
	struct irte irte;
	struct irq_desc *desc;

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	if (get_irte(irq, &irte))
		return;

	if (assign_irq_vector(irq, mask))
		return;

	cfg = irq_cfg(irq);
	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	irte.vector = cfg->vector;
	irte.dest_id = IRTE_DEST(dest);

	/*
	 * atomically update the IRTE with the new destination and vector.
	 */
	modify_irte(irq, &irte);

	/*
	 * After this point, all the interrupts will start arriving
	 * at the new destination. So, time to cleanup the previous
	 * vector allocation.
	 */
	if (cfg->move_in_progress) {
		cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
		cfg->move_cleanup_count = cpus_weight(cleanup_mask);
		send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
		cfg->move_in_progress = 0;
	}

	desc = irq_to_desc(irq);
	desc->affinity = mask;
}
#endif
3269
#endif /* CONFIG_SMP */
3270

3271 3272 3273 3274 3275 3276 3277 3278
/*
 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
 * which implement the MSI or MSI-X Capability Structure.
 */
static struct irq_chip msi_chip = {
	.name		= "PCI-MSI",
	.unmask		= unmask_msi_irq,
	.mask		= mask_msi_irq,
3279
	.ack		= ack_apic_edge,
3280 3281 3282 3283
#ifdef CONFIG_SMP
	.set_affinity	= set_msi_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
3284 3285
};

3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324
#ifdef CONFIG_INTR_REMAP
static struct irq_chip msi_ir_chip = {
	.name		= "IR-PCI-MSI",
	.unmask		= unmask_msi_irq,
	.mask		= mask_msi_irq,
	.ack		= ack_x2apic_edge,
#ifdef CONFIG_SMP
	.set_affinity	= ir_set_msi_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
};

/*
 * Map the PCI dev to the corresponding remapping hardware unit
 * and allocate 'nvec' consecutive interrupt-remapping table entries
 * in it.
 */
static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
{
	struct intel_iommu *iommu;
	int index;

	iommu = map_dev_to_ir(dev);
	if (!iommu) {
		printk(KERN_ERR
		       "Unable to map PCI %s to iommu\n", pci_name(dev));
		return -ENOENT;
	}

	index = alloc_irte(iommu, irq, nvec);
	if (index < 0) {
		printk(KERN_ERR
		       "Unable to allocate %d IRTE for PCI %s\n", nvec,
		        pci_name(dev));
		return -ENOSPC;
	}
	return index;
}
#endif
3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337

static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq)
{
	int ret;
	struct msi_msg msg;

	ret = msi_compose_msg(dev, irq, &msg);
	if (ret < 0)
		return ret;

	set_irq_msi(irq, desc);
	write_msi_msg(irq, &msg);

3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348
#ifdef CONFIG_INTR_REMAP
	if (irq_remapped(irq)) {
		struct irq_desc *desc = irq_to_desc(irq);
		/*
		 * irq migration in process context
		 */
		desc->status |= IRQ_MOVE_PCNTXT;
		set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge");
	} else
#endif
		set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");
3349 3350 3351 3352

	return 0;
}

Y
Yinghai Lu 已提交
3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364
static unsigned int build_irq_for_pci_dev(struct pci_dev *dev)
{
	unsigned int irq;

	irq = dev->bus->number;
	irq <<= 8;
	irq |= dev->devfn;
	irq <<= 12;

	return irq;
}

3365
int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
3366
{
3367 3368
	unsigned int irq;
	int ret;
Y
Yinghai Lu 已提交
3369 3370 3371 3372 3373 3374 3375
	unsigned int irq_want;

	irq_want = build_irq_for_pci_dev(dev) + 0x100;

	irq = create_irq_nr(irq_want);
	if (irq == 0)
		return -1;
3376

3377 3378 3379 3380 3381 3382 3383 3384 3385
#ifdef CONFIG_INTR_REMAP
	if (!intr_remapping_enabled)
		goto no_ir;

	ret = msi_alloc_irte(dev, irq, 1);
	if (ret < 0)
		goto error;
no_ir:
#endif
3386
	ret = setup_msi_irq(dev, desc, irq);
3387 3388
	if (ret < 0) {
		destroy_irq(irq);
3389
		return ret;
3390
	}
3391
	return 0;
3392 3393 3394 3395 3396 3397

#ifdef CONFIG_INTR_REMAP
error:
	destroy_irq(irq);
	return ret;
#endif
3398 3399
}

3400 3401
int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
{
3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452
	unsigned int irq;
	int ret, sub_handle;
	struct msi_desc *desc;
	unsigned int irq_want;

#ifdef CONFIG_INTR_REMAP
	struct intel_iommu *iommu = 0;
	int index = 0;
#endif

	irq_want = build_irq_for_pci_dev(dev) + 0x100;
	sub_handle = 0;
	list_for_each_entry(desc, &dev->msi_list, list) {
		irq = create_irq_nr(irq_want--);
		if (irq == 0)
			return -1;
#ifdef CONFIG_INTR_REMAP
		if (!intr_remapping_enabled)
			goto no_ir;

		if (!sub_handle) {
			/*
			 * allocate the consecutive block of IRTE's
			 * for 'nvec'
			 */
			index = msi_alloc_irte(dev, irq, nvec);
			if (index < 0) {
				ret = index;
				goto error;
			}
		} else {
			iommu = map_dev_to_ir(dev);
			if (!iommu) {
				ret = -ENOENT;
				goto error;
			}
			/*
			 * setup the mapping between the irq and the IRTE
			 * base index, the sub_handle pointing to the
			 * appropriate interrupt remap table entry.
			 */
			set_irte_irq(irq, iommu, index, sub_handle);
		}
no_ir:
#endif
		ret = setup_msi_irq(dev, desc, irq);
		if (ret < 0)
			goto error;
		sub_handle++;
	}
	return 0;
3453 3454

error:
3455 3456
	destroy_irq(irq);
	return ret;
3457 3458
}

3459 3460
void arch_teardown_msi_irq(unsigned int irq)
{
3461
	destroy_irq(irq);
3462 3463
}

3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512
#ifdef CONFIG_DMAR
#ifdef CONFIG_SMP
static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask)
{
	struct irq_cfg *cfg;
	struct msi_msg msg;
	unsigned int dest;
	cpumask_t tmp;
	struct irq_desc *desc;

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	if (assign_irq_vector(irq, mask))
		return;

	cfg = irq_cfg(irq);
	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	dmar_msi_read(irq, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	dmar_msi_write(irq, &msg);
	desc = irq_to_desc(irq);
	desc->affinity = mask;
}
#endif /* CONFIG_SMP */

struct irq_chip dmar_msi_type = {
	.name = "DMAR_MSI",
	.unmask = dmar_msi_unmask,
	.mask = dmar_msi_mask,
	.ack = ack_apic_edge,
#ifdef CONFIG_SMP
	.set_affinity = dmar_msi_set_affinity,
#endif
	.retrigger = ioapic_retrigger_irq,
};

int arch_setup_dmar_msi(unsigned int irq)
{
	int ret;
	struct msi_msg msg;
3513

3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524
	ret = msi_compose_msg(NULL, irq, &msg);
	if (ret < 0)
		return ret;
	dmar_msi_write(irq, &msg);
	set_irq_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
		"edge");
	return 0;
}
#endif

#endif /* CONFIG_PCI_MSI */
3525 3526 3527 3528 3529 3530 3531
/*
 * Hypertransport interrupt support
 */
#ifdef CONFIG_HT_IRQ

#ifdef CONFIG_SMP

3532
static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3533
{
3534 3535
	struct ht_irq_msg msg;
	fetch_ht_irq_msg(irq, &msg);
3536

3537
	msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
3538
	msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
3539

3540
	msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
3541
	msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
3542

3543
	write_ht_irq_msg(irq, &msg);
3544 3545 3546 3547
}

static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
{
3548
	struct irq_cfg *cfg;
3549 3550
	unsigned int dest;
	cpumask_t tmp;
3551
	struct irq_desc *desc;
3552 3553 3554

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
3555
		return;
3556

3557 3558
	if (assign_irq_vector(irq, mask))
		return;
3559

3560 3561 3562
	cfg = irq_cfg(irq);
	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);
3563

3564
	target_ht_irq(irq, dest, cfg->vector);
3565 3566
	desc = irq_to_desc(irq);
	desc->affinity = mask;
3567 3568 3569
}
#endif

3570
static struct irq_chip ht_irq_chip = {
3571 3572 3573
	.name		= "PCI-HT",
	.mask		= mask_ht_irq,
	.unmask		= unmask_ht_irq,
3574
	.ack		= ack_apic_edge,
3575 3576 3577 3578 3579 3580 3581 3582
#ifdef CONFIG_SMP
	.set_affinity	= set_ht_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
};

int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
{
3583 3584 3585
	struct irq_cfg *cfg;
	int err;
	cpumask_t tmp;
3586

3587 3588
	tmp = TARGET_CPUS;
	err = assign_irq_vector(irq, tmp);
3589
	if (!err) {
3590
		struct ht_irq_msg msg;
3591 3592
		unsigned dest;

3593 3594
		cfg = irq_cfg(irq);
		cpus_and(tmp, cfg->domain, tmp);
3595 3596
		dest = cpu_mask_to_apicid(tmp);

3597
		msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
3598

3599 3600
		msg.address_lo =
			HT_IRQ_LOW_BASE |
3601
			HT_IRQ_LOW_DEST_ID(dest) |
3602
			HT_IRQ_LOW_VECTOR(cfg->vector) |
3603 3604 3605 3606 3607 3608 3609 3610 3611
			((INT_DEST_MODE == 0) ?
				HT_IRQ_LOW_DM_PHYSICAL :
				HT_IRQ_LOW_DM_LOGICAL) |
			HT_IRQ_LOW_RQEOI_EDGE |
			((INT_DELIVERY_MODE != dest_LowestPrio) ?
				HT_IRQ_LOW_MT_FIXED :
				HT_IRQ_LOW_MT_ARBITRATED) |
			HT_IRQ_LOW_IRQ_MASKED;

3612
		write_ht_irq_msg(irq, &msg);
3613

3614 3615
		set_irq_chip_and_handler_name(irq, &ht_irq_chip,
					      handle_edge_irq, "edge");
3616
	}
3617
	return err;
3618 3619 3620
}
#endif /* CONFIG_HT_IRQ */

3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636
int __init io_apic_get_redir_entries (int ioapic)
{
	union IO_APIC_reg_01	reg_01;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_01.raw = io_apic_read(ioapic, 1);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return reg_01.bits.entries;
}

int __init probe_nr_irqs(void)
{
	int idx;
	int nr = 0;
Y
Yinghai Lu 已提交
3637 3638 3639 3640 3641
#ifndef CONFIG_XEN
	int nr_min = 32;
#else
	int nr_min = NR_IRQS;
#endif
3642 3643

	for (idx = 0; idx < nr_ioapics; idx++)
Y
Yinghai Lu 已提交
3644
		nr += io_apic_get_redir_entries(idx) + 1;
3645 3646 3647 3648 3649

	/* double it for hotplug and msi and nmi */
	nr <<= 1;

	/* something wrong ? */
Y
Yinghai Lu 已提交
3650 3651
	if (nr < nr_min)
		nr = nr_min;
3652 3653 3654 3655

	return nr;
}

L
Linus Torvalds 已提交
3656
/* --------------------------------------------------------------------------
3657
                          ACPI-based IOAPIC Configuration
L
Linus Torvalds 已提交
3658 3659
   -------------------------------------------------------------------------- */

L
Len Brown 已提交
3660
#ifdef CONFIG_ACPI
L
Linus Torvalds 已提交
3661

3662
#ifdef CONFIG_X86_32
3663
int __init io_apic_get_unique_id(int ioapic, int apic_id)
L
Linus Torvalds 已提交
3664 3665 3666 3667 3668 3669 3670 3671
{
	union IO_APIC_reg_00 reg_00;
	static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
	physid_mask_t tmp;
	unsigned long flags;
	int i = 0;

	/*
3672 3673
	 * The P4 platform supports up to 256 APIC IDs on two separate APIC
	 * buses (one for LAPICs, one for IOAPICs), where predecessors only
L
Linus Torvalds 已提交
3674
	 * supports up to 16 on one shared APIC bus.
3675
	 *
L
Linus Torvalds 已提交
3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693
	 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
	 *      advantage of new APIC bus architecture.
	 */

	if (physids_empty(apic_id_map))
		apic_id_map = ioapic_phys_id_map(phys_cpu_present_map);

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(ioapic, 0);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	if (apic_id >= get_physical_broadcast()) {
		printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
			"%d\n", ioapic, apic_id, reg_00.bits.ID);
		apic_id = reg_00.bits.ID;
	}

	/*
3694
	 * Every APIC in a system must have a unique ID or we get lots of nice
L
Linus Torvalds 已提交
3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710
	 * 'stuck on smp_invalidate_needed IPI wait' messages.
	 */
	if (check_apicid_used(apic_id_map, apic_id)) {

		for (i = 0; i < get_physical_broadcast(); i++) {
			if (!check_apicid_used(apic_id_map, i))
				break;
		}

		if (i == get_physical_broadcast())
			panic("Max apic_id exceeded!\n");

		printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
			"trying %d\n", ioapic, apic_id, i);

		apic_id = i;
3711
	}
L
Linus Torvalds 已提交
3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724

	tmp = apicid_to_cpu_present(apic_id);
	physids_or(apic_id_map, apic_id_map, tmp);

	if (reg_00.bits.ID != apic_id) {
		reg_00.bits.ID = apic_id;

		spin_lock_irqsave(&ioapic_lock, flags);
		io_apic_write(ioapic, 0, reg_00.raw);
		reg_00.raw = io_apic_read(ioapic, 0);
		spin_unlock_irqrestore(&ioapic_lock, flags);

		/* Sanity check */
3725 3726 3727 3728
		if (reg_00.bits.ID != apic_id) {
			printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
			return -1;
		}
L
Linus Torvalds 已提交
3729 3730 3731 3732 3733 3734 3735 3736
	}

	apic_printk(APIC_VERBOSE, KERN_INFO
			"IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);

	return apic_id;
}

3737
int __init io_apic_get_version(int ioapic)
L
Linus Torvalds 已提交
3738 3739 3740 3741 3742 3743 3744 3745 3746 3747
{
	union IO_APIC_reg_01	reg_01;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_01.raw = io_apic_read(ioapic, 1);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return reg_01.bits.version;
}
3748
#endif
L
Linus Torvalds 已提交
3749

3750
int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
L
Linus Torvalds 已提交
3751 3752
{
	if (!IO_APIC_IRQ(irq)) {
3753
		apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
L
Linus Torvalds 已提交
3754 3755 3756 3757 3758 3759 3760 3761 3762 3763
			ioapic);
		return -EINVAL;
	}

	/*
	 * IRQs < 16 are already in the irq_2_pin[] map
	 */
	if (irq >= 16)
		add_pin_to_irq(irq, ioapic, pin);

3764
	setup_IO_APIC_irq(ioapic, pin, irq, triggering, polarity);
L
Linus Torvalds 已提交
3765 3766 3767 3768

	return 0;
}

3769

3770 3771 3772 3773 3774 3775 3776 3777
int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
{
	int i;

	if (skip_ioapic_setup)
		return -1;

	for (i = 0; i < mp_irq_entries; i++)
3778 3779
		if (mp_irqs[i].mp_irqtype == mp_INT &&
		    mp_irqs[i].mp_srcbusirq == bus_irq)
3780 3781 3782 3783 3784 3785 3786 3787 3788
			break;
	if (i >= mp_irq_entries)
		return -1;

	*trigger = irq_trigger(i);
	*polarity = irq_polarity(i);
	return 0;
}

L
Len Brown 已提交
3789
#endif /* CONFIG_ACPI */
3790

3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820
/*
 * This function currently is only a helper for the i386 smp boot process where
 * we need to reprogram the ioredtbls to cater for the cpus which have come online
 * so mask in all cases should simply be TARGET_CPUS
 */
#ifdef CONFIG_SMP
void __init setup_ioapic_dest(void)
{
	int pin, ioapic, irq, irq_entry;
	struct irq_cfg *cfg;

	if (skip_ioapic_setup == 1)
		return;

	for (ioapic = 0; ioapic < nr_ioapics; ioapic++) {
		for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
			irq_entry = find_irq_entry(ioapic, pin, mp_INT);
			if (irq_entry == -1)
				continue;
			irq = pin_2_irq(irq_entry, ioapic, pin);

			/* setup_IO_APIC_irqs could fail to get vector for some device
			 * when you have too many devices, because at that time only boot
			 * cpu is online.
			 */
			cfg = irq_cfg(irq);
			if (!cfg->vector)
				setup_IO_APIC_irq(ioapic, pin, irq,
						  irq_trigger(irq_entry),
						  irq_polarity(irq_entry));
3821 3822 3823 3824 3825
#ifdef CONFIG_INTR_REMAP
			else if (intr_remapping_enabled)
				set_ir_ioapic_affinity_irq(irq, TARGET_CPUS);
#endif
			else
3826 3827 3828 3829 3830 3831 3832
				set_ioapic_affinity_irq(irq, TARGET_CPUS);
		}

	}
}
#endif

3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868
#define IOAPIC_RESOURCE_NAME_SIZE 11

static struct resource *ioapic_resources;

static struct resource * __init ioapic_setup_resources(void)
{
	unsigned long n;
	struct resource *res;
	char *mem;
	int i;

	if (nr_ioapics <= 0)
		return NULL;

	n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
	n *= nr_ioapics;

	mem = alloc_bootmem(n);
	res = (void *)mem;

	if (mem != NULL) {
		mem += sizeof(struct resource) * nr_ioapics;

		for (i = 0; i < nr_ioapics; i++) {
			res[i].name = mem;
			res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
			sprintf(mem,  "IOAPIC %u", i);
			mem += IOAPIC_RESOURCE_NAME_SIZE;
		}
	}

	ioapic_resources = res;

	return res;
}

3869 3870 3871 3872
void __init ioapic_init_mappings(void)
{
	unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
	int i;
3873
	struct resource *ioapic_res;
3874

3875
	ioapic_res = ioapic_setup_resources();
3876 3877 3878
	for (i = 0; i < nr_ioapics; i++) {
		if (smp_found_config) {
			ioapic_phys = mp_ioapics[i].mp_apicaddr;
3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889
#ifdef CONFIG_X86_32
                        if (!ioapic_phys) {
                                printk(KERN_ERR
                                       "WARNING: bogus zero IO-APIC "
                                       "address found in MPTABLE, "
                                       "disabling IO/APIC support!\n");
                                smp_found_config = 0;
                                skip_ioapic_setup = 1;
                                goto fake_ioapic_page;
                        }
#endif
3890
		} else {
3891
#ifdef CONFIG_X86_32
3892
fake_ioapic_page:
3893
#endif
3894
			ioapic_phys = (unsigned long)
3895
				alloc_bootmem_pages(PAGE_SIZE);
3896 3897 3898
			ioapic_phys = __pa(ioapic_phys);
		}
		set_fixmap_nocache(idx, ioapic_phys);
3899 3900 3901
		apic_printk(APIC_VERBOSE,
			    "mapped IOAPIC to %08lx (%08lx)\n",
			    __fix_to_virt(idx), ioapic_phys);
3902
		idx++;
3903 3904 3905 3906 3907 3908

		if (ioapic_res != NULL) {
			ioapic_res->start = ioapic_phys;
			ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
			ioapic_res++;
		}
3909 3910 3911
	}
}

3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933
static int __init ioapic_insert_resources(void)
{
	int i;
	struct resource *r = ioapic_resources;

	if (!r) {
		printk(KERN_ERR
		       "IO APIC resources could be not be allocated.\n");
		return -1;
	}

	for (i = 0; i < nr_ioapics; i++) {
		insert_resource(&iomem_resource, r);
		r++;
	}

	return 0;
}

/* Insert the IO APIC resources after PCI initialization has occured to handle
 * IO APICS that are mapped in on a BAR in PCI space. */
late_initcall(ioapic_insert_resources);