irq_32.c 10.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 *	Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar
 *
 * This file contains the lowest level x86-specific interrupt
 * entry, irq-stacks and irq statistics code. All the remaining
 * irq logic is done by the generic kernel/irq/ code and
 * by the x86-specific irq controller code. (e.g. i8259.c and
 * io_apic.c.)
 */

#include <linux/module.h>
#include <linux/seq_file.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
Z
Zwane Mwaikambo 已提交
15 16 17
#include <linux/notifier.h>
#include <linux/cpu.h>
#include <linux/delay.h>
L
Linus Torvalds 已提交
18

19 20 21
#include <asm/apic.h>
#include <asm/uaccess.h>

22
DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
L
Linus Torvalds 已提交
23 24
EXPORT_PER_CPU_SYMBOL(irq_stat);

25 26 27
DEFINE_PER_CPU(struct pt_regs *, irq_regs);
EXPORT_PER_CPU_SYMBOL(irq_regs);

L
Linus Torvalds 已提交
28 29 30 31 32 33
/*
 * 'what should we do if we get a hw irq event on an illegal vector'.
 * each architecture has to answer this themselves.
 */
void ack_bad_irq(unsigned int irq)
{
34 35 36 37 38 39 40 41 42 43 44 45 46 47
	printk(KERN_ERR "unexpected IRQ trap at vector %02x\n", irq);

#ifdef CONFIG_X86_LOCAL_APIC
	/*
	 * Currently unexpected vectors happen only on SMP and APIC.
	 * We _must_ ack these because every local APIC has only N
	 * irq slots per priority level, and a 'hanging, unacked' IRQ
	 * holds up an irq slot - in excessive cases (when multiple
	 * unexpected vectors occur) that might lock up the APIC
	 * completely.
	 * But only ack when the APIC is enabled -AK
	 */
	if (cpu_has_apic)
		ack_APIC_irq();
L
Linus Torvalds 已提交
48
#endif
49
}
L
Linus Torvalds 已提交
50

51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
#ifdef CONFIG_DEBUG_STACKOVERFLOW
/* Debugging check for stack overflow: is there less than 1KB free? */
static int check_stack_overflow(void)
{
	long sp;

	__asm__ __volatile__("andl %%esp,%0" :
			     "=r" (sp) : "0" (THREAD_SIZE - 1));

	return sp < (sizeof(struct thread_info) + STACK_WARN);
}

static void print_stack_overflow(void)
{
	printk(KERN_WARNING "low stack detected by irq handler\n");
	dump_stack();
}

#else
static inline int check_stack_overflow(void) { return 0; }
static inline void print_stack_overflow(void) { }
#endif

L
Linus Torvalds 已提交
74 75 76 77 78 79 80 81 82
#ifdef CONFIG_4KSTACKS
/*
 * per-CPU IRQ handling contexts (thread information and stack)
 */
union irq_ctx {
	struct thread_info      tinfo;
	u32                     stack[THREAD_SIZE/sizeof(u32)];
};

83 84
static union irq_ctx *hardirq_ctx[NR_CPUS] __read_mostly;
static union irq_ctx *softirq_ctx[NR_CPUS] __read_mostly;
L
Linus Torvalds 已提交
85

86 87
static char softirq_stack[NR_CPUS * THREAD_SIZE]
		__attribute__((__section__(".bss.page_aligned")));
L
Linus Torvalds 已提交
88

89 90
static char hardirq_stack[NR_CPUS * THREAD_SIZE]
		__attribute__((__section__(".bss.page_aligned")));
91

92
static void call_on_stack(void *func, void *stack)
93
{
94 95 96 97 98 99 100
	asm volatile("xchgl	%%ebx,%%esp	\n"
		     "call	*%%edi		\n"
		     "movl	%%ebx,%%esp	\n"
		     : "=b" (stack)
		     : "0" (stack),
		       "D"(func)
		     : "memory", "cc", "edx", "ecx", "eax");
101
}
L
Linus Torvalds 已提交
102

103 104 105 106
static inline int
execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
{
	union irq_ctx *curctx, *irqctx;
107
	u32 *isp, arg1, arg2;
L
Linus Torvalds 已提交
108 109 110 111 112 113 114 115 116 117

	curctx = (union irq_ctx *) current_thread_info();
	irqctx = hardirq_ctx[smp_processor_id()];

	/*
	 * this is where we switch to the IRQ stack. However, if we are
	 * already using the IRQ stack (because we interrupted a hardirq
	 * handler) we can't do that and just have to keep using the
	 * current stack (which is the irq stack already after all)
	 */
118 119
	if (unlikely(curctx == irqctx))
		return 0;
L
Linus Torvalds 已提交
120

121 122 123 124
	/* build the stack frame on the IRQ stack */
	isp = (u32 *) ((char*)irqctx + sizeof(*irqctx));
	irqctx->tinfo.task = curctx->tinfo.task;
	irqctx->tinfo.previous_esp = current_stack_pointer;
L
Linus Torvalds 已提交
125

126 127 128 129 130 131 132 133 134
	/*
	 * Copy the softirq bits in preempt_count so that the
	 * softirq checks work in the hardirq context.
	 */
	irqctx->tinfo.preempt_count =
		(irqctx->tinfo.preempt_count & ~SOFTIRQ_MASK) |
		(curctx->tinfo.preempt_count & SOFTIRQ_MASK);

	if (unlikely(overflow))
135 136 137 138 139 140 141 142 143
		call_on_stack(print_stack_overflow, isp);

	asm volatile("xchgl	%%ebx,%%esp	\n"
		     "call	*%%edi		\n"
		     "movl	%%ebx,%%esp	\n"
		     : "=a" (arg1), "=d" (arg2), "=b" (isp)
		     :  "0" (irq),   "1" (desc),  "2" (isp),
			"D" (desc->handle_irq)
		     : "memory", "cc", "ecx");
L
Linus Torvalds 已提交
144 145 146 147 148 149
	return 1;
}

/*
 * allocate per-cpu stacks for hardirq and for softirq processing
 */
150
void __cpuinit irq_ctx_init(int cpu)
L
Linus Torvalds 已提交
151 152 153 154 155 156 157
{
	union irq_ctx *irqctx;

	if (hardirq_ctx[cpu])
		return;

	irqctx = (union irq_ctx*) &hardirq_stack[cpu*THREAD_SIZE];
158 159 160 161 162
	irqctx->tinfo.task		= NULL;
	irqctx->tinfo.exec_domain	= NULL;
	irqctx->tinfo.cpu		= cpu;
	irqctx->tinfo.preempt_count	= HARDIRQ_OFFSET;
	irqctx->tinfo.addr_limit	= MAKE_MM_SEG(0);
L
Linus Torvalds 已提交
163 164 165 166

	hardirq_ctx[cpu] = irqctx;

	irqctx = (union irq_ctx*) &softirq_stack[cpu*THREAD_SIZE];
167 168 169 170 171
	irqctx->tinfo.task		= NULL;
	irqctx->tinfo.exec_domain	= NULL;
	irqctx->tinfo.cpu		= cpu;
	irqctx->tinfo.preempt_count	= 0;
	irqctx->tinfo.addr_limit	= MAKE_MM_SEG(0);
L
Linus Torvalds 已提交
172 173 174

	softirq_ctx[cpu] = irqctx;

175 176
	printk(KERN_DEBUG "CPU %u irqstacks, hard=%p soft=%p\n",
	       cpu,hardirq_ctx[cpu],softirq_ctx[cpu]);
L
Linus Torvalds 已提交
177 178
}

179 180 181 182 183
void irq_ctx_exit(int cpu)
{
	hardirq_ctx[cpu] = NULL;
}

L
Linus Torvalds 已提交
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
asmlinkage void do_softirq(void)
{
	unsigned long flags;
	struct thread_info *curctx;
	union irq_ctx *irqctx;
	u32 *isp;

	if (in_interrupt())
		return;

	local_irq_save(flags);

	if (local_softirq_pending()) {
		curctx = current_thread_info();
		irqctx = softirq_ctx[smp_processor_id()];
		irqctx->tinfo.task = curctx->task;
		irqctx->tinfo.previous_esp = current_stack_pointer;

		/* build the stack frame on the softirq stack */
		isp = (u32*) ((char*)irqctx + sizeof(*irqctx));

205
		call_on_stack(__do_softirq, isp);
206 207
		/*
		 * Shouldnt happen, we returned above if in_interrupt():
208
		 */
209
		WARN_ON_ONCE(softirq_count());
L
Linus Torvalds 已提交
210 211 212 213
	}

	local_irq_restore(flags);
}
214 215 216 217

#else
static inline int
execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq) { return 0; }
L
Linus Torvalds 已提交
218 219
#endif

220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
/*
 * do_IRQ handles all normal device IRQ's (the special
 * SMP cross-CPU interrupts have their own specific
 * handlers).
 */
unsigned int do_IRQ(struct pt_regs *regs)
{
	struct pt_regs *old_regs;
	/* high bit used in ret_from_ code */
	int overflow, irq = ~regs->orig_ax;
	struct irq_desc *desc = irq_desc + irq;

	if (unlikely((unsigned)irq >= NR_IRQS)) {
		printk(KERN_EMERG "%s: cannot handle IRQ %d\n",
					__func__, irq);
		BUG();
	}

	old_regs = set_irq_regs(regs);
	irq_enter();

	overflow = check_stack_overflow();

	if (!execute_on_irq_stack(overflow, desc, irq)) {
		if (unlikely(overflow))
			print_stack_overflow();
		desc->handle_irq(irq, desc);
	}

	irq_exit();
	set_irq_regs(old_regs);
	return 1;
}

L
Linus Torvalds 已提交
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
/*
 * Interrupt statistics:
 */

atomic_t irq_err_count;

/*
 * /proc/interrupts printing:
 */

int show_interrupts(struct seq_file *p, void *v)
{
	int i = *(loff_t *) v, j;
	struct irqaction * action;
	unsigned long flags;

	if (i == 0) {
		seq_printf(p, "           ");
272
		for_each_online_cpu(j)
273
			seq_printf(p, "CPU%-8d",j);
L
Linus Torvalds 已提交
274 275 276 277
		seq_putc(p, '\n');
	}

	if (i < NR_IRQS) {
278 279
		unsigned any_count = 0;

L
Linus Torvalds 已提交
280
		spin_lock_irqsave(&irq_desc[i].lock, flags);
281 282 283 284 285 286
#ifndef CONFIG_SMP
		any_count = kstat_irqs(i);
#else
		for_each_online_cpu(j)
			any_count |= kstat_cpu(j).irqs[i];
#endif
L
Linus Torvalds 已提交
287
		action = irq_desc[i].action;
288
		if (!action && !any_count)
L
Linus Torvalds 已提交
289 290 291 292 293
			goto skip;
		seq_printf(p, "%3d: ",i);
#ifndef CONFIG_SMP
		seq_printf(p, "%10u ", kstat_irqs(i));
#else
294
		for_each_online_cpu(j)
Z
Zwane Mwaikambo 已提交
295
			seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]);
L
Linus Torvalds 已提交
296
#endif
297
		seq_printf(p, " %8s", irq_desc[i].chip->name);
298
		seq_printf(p, "-%-8s", irq_desc[i].name);
L
Linus Torvalds 已提交
299

300 301 302 303 304
		if (action) {
			seq_printf(p, "  %s", action->name);
			while ((action = action->next) != NULL)
				seq_printf(p, ", %s", action->name);
		}
L
Linus Torvalds 已提交
305 306 307 308 309 310

		seq_putc(p, '\n');
skip:
		spin_unlock_irqrestore(&irq_desc[i].lock, flags);
	} else if (i == NR_IRQS) {
		seq_printf(p, "NMI: ");
311
		for_each_online_cpu(j)
Z
Zwane Mwaikambo 已提交
312
			seq_printf(p, "%10u ", nmi_count(j));
313
		seq_printf(p, "  Non-maskable interrupts\n");
L
Linus Torvalds 已提交
314 315
#ifdef CONFIG_X86_LOCAL_APIC
		seq_printf(p, "LOC: ");
316
		for_each_online_cpu(j)
Z
Zwane Mwaikambo 已提交
317 318
			seq_printf(p, "%10u ",
				per_cpu(irq_stat,j).apic_timer_irqs);
319
		seq_printf(p, "  Local timer interrupts\n");
L
Linus Torvalds 已提交
320
#endif
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
#ifdef CONFIG_SMP
		seq_printf(p, "RES: ");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				per_cpu(irq_stat,j).irq_resched_count);
		seq_printf(p, "  Rescheduling interrupts\n");
		seq_printf(p, "CAL: ");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				per_cpu(irq_stat,j).irq_call_count);
		seq_printf(p, "  function call interrupts\n");
		seq_printf(p, "TLB: ");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				per_cpu(irq_stat,j).irq_tlb_count);
		seq_printf(p, "  TLB shootdowns\n");
#endif
338
#ifdef CONFIG_X86_MCE
339 340 341 342 343
		seq_printf(p, "TRM: ");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				per_cpu(irq_stat,j).irq_thermal_count);
		seq_printf(p, "  Thermal event interrupts\n");
344 345
#endif
#ifdef CONFIG_X86_LOCAL_APIC
346 347 348 349 350
		seq_printf(p, "SPU: ");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				per_cpu(irq_stat,j).irq_spurious_count);
		seq_printf(p, "  Spurious interrupts\n");
351
#endif
L
Linus Torvalds 已提交
352 353 354 355 356 357 358
		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
#if defined(CONFIG_X86_IO_APIC)
		seq_printf(p, "MIS: %10u\n", atomic_read(&irq_mis_count));
#endif
	}
	return 0;
}
Z
Zwane Mwaikambo 已提交
359

360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393
/*
 * /proc/stat helpers
 */
u64 arch_irq_stat_cpu(unsigned int cpu)
{
	u64 sum = nmi_count(cpu);

#ifdef CONFIG_X86_LOCAL_APIC
	sum += per_cpu(irq_stat, cpu).apic_timer_irqs;
#endif
#ifdef CONFIG_SMP
	sum += per_cpu(irq_stat, cpu).irq_resched_count;
	sum += per_cpu(irq_stat, cpu).irq_call_count;
	sum += per_cpu(irq_stat, cpu).irq_tlb_count;
#endif
#ifdef CONFIG_X86_MCE
	sum += per_cpu(irq_stat, cpu).irq_thermal_count;
#endif
#ifdef CONFIG_X86_LOCAL_APIC
	sum += per_cpu(irq_stat, cpu).irq_spurious_count;
#endif
	return sum;
}

u64 arch_irq_stat(void)
{
	u64 sum = atomic_read(&irq_err_count);

#ifdef CONFIG_X86_IO_APIC
	sum += atomic_read(&irq_mis_count);
#endif
	return sum;
}

Z
Zwane Mwaikambo 已提交
394 395 396 397 398 399 400 401 402 403 404 405 406
#ifdef CONFIG_HOTPLUG_CPU
#include <mach_apic.h>

void fixup_irqs(cpumask_t map)
{
	unsigned int irq;
	static int warned;

	for (irq = 0; irq < NR_IRQS; irq++) {
		cpumask_t mask;
		if (irq == 2)
			continue;

407
		cpus_and(mask, irq_desc[irq].affinity, map);
Z
Zwane Mwaikambo 已提交
408 409 410 411
		if (any_online_cpu(mask) == NR_CPUS) {
			printk("Breaking affinity for irq %i\n", irq);
			mask = map;
		}
412 413
		if (irq_desc[irq].chip->set_affinity)
			irq_desc[irq].chip->set_affinity(irq, mask);
Z
Zwane Mwaikambo 已提交
414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434
		else if (irq_desc[irq].action && !(warned++))
			printk("Cannot set affinity for irq %i\n", irq);
	}

#if 0
	barrier();
	/* Ingo Molnar says: "after the IO-APIC masks have been redirected
	   [note the nop - the interrupt-enable boundary on x86 is two
	   instructions from sti] - to flush out pending hardirqs and
	   IPIs. After this point nothing is supposed to reach this CPU." */
	__asm__ __volatile__("sti; nop; cli");
	barrier();
#else
	/* That doesn't seem sufficient.  Give it 1ms. */
	local_irq_enable();
	mdelay(1);
	local_irq_disable();
#endif
}
#endif