irq.c 13.5 KB
Newer Older
1 2 3 4 5 6
/*
 * Common interrupt code for 32 and 64 bit
 */
#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
7
#include <linux/of.h>
8
#include <linux/seq_file.h>
9
#include <linux/smp.h>
J
Jeremy Fitzhardinge 已提交
10
#include <linux/ftrace.h>
11
#include <linux/delay.h>
12
#include <linux/export.h>
13

I
Ingo Molnar 已提交
14
#include <asm/apic.h>
15
#include <asm/io_apic.h>
16
#include <asm/irq.h>
17
#include <asm/mce.h>
18
#include <asm/hw_irq.h>
19
#include <asm/desc.h>
20 21

#define CREATE_TRACE_POINTS
22
#include <asm/trace/irq_vectors.h>
23

B
Brian Gerst 已提交
24 25 26 27 28 29
DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
EXPORT_PER_CPU_SYMBOL(irq_stat);

DEFINE_PER_CPU(struct pt_regs *, irq_regs);
EXPORT_PER_CPU_SYMBOL(irq_regs);

30 31
atomic_t irq_err_count;

32
/* Function pointer for generic interrupt vector handling */
33
void (*x86_platform_ipi_callback)(void) = NULL;
34

T
Thomas Gleixner 已提交
35 36 37 38 39 40
/*
 * 'what should we do if we get a hw irq event on an illegal vector'.
 * each architecture has to answer this themselves.
 */
void ack_bad_irq(unsigned int irq)
{
C
Cyrill Gorcunov 已提交
41 42
	if (printk_ratelimit())
		pr_err("unexpected IRQ trap at vector %02x\n", irq);
T
Thomas Gleixner 已提交
43 44 45 46 47 48 49 50 51 52

	/*
	 * Currently unexpected vectors happen only on SMP and APIC.
	 * We _must_ ack these because every local APIC has only N
	 * irq slots per priority level, and a 'hanging, unacked' IRQ
	 * holds up an irq slot - in excessive cases (when multiple
	 * unexpected vectors occur) that might lock up the APIC
	 * completely.
	 * But only ack when the APIC is enabled -AK
	 */
53
	ack_APIC_irq();
T
Thomas Gleixner 已提交
54 55
}

56
#define irq_stats(x)		(&per_cpu(irq_stat, x))
57
/*
58
 * /proc/interrupts printing for arch specific interrupts
59
 */
60
int arch_show_interrupts(struct seq_file *p, int prec)
61 62 63
{
	int j;

64
	seq_printf(p, "%*s: ", prec, "NMI");
65 66
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
67
	seq_puts(p, "  Non-maskable interrupts\n");
68
#ifdef CONFIG_X86_LOCAL_APIC
69
	seq_printf(p, "%*s: ", prec, "LOC");
70 71
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
72
	seq_puts(p, "  Local timer interrupts\n");
73 74 75 76

	seq_printf(p, "%*s: ", prec, "SPU");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
77
	seq_puts(p, "  Spurious interrupts\n");
78
	seq_printf(p, "%*s: ", prec, "PMI");
I
Ingo Molnar 已提交
79 80
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
81
	seq_puts(p, "  Performance monitoring interrupts\n");
82
	seq_printf(p, "%*s: ", prec, "IWI");
83
	for_each_online_cpu(j)
84
		seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
85
	seq_puts(p, "  IRQ work interrupts\n");
86 87
	seq_printf(p, "%*s: ", prec, "RTR");
	for_each_online_cpu(j)
88
		seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
89
	seq_puts(p, "  APIC ICR read retries\n");
90
#endif
91
	if (x86_platform_ipi_callback) {
92
		seq_printf(p, "%*s: ", prec, "PLT");
93
		for_each_online_cpu(j)
94
			seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
95
		seq_puts(p, "  Platform interrupts\n");
96
	}
97
#ifdef CONFIG_SMP
98
	seq_printf(p, "%*s: ", prec, "RES");
99 100
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
101
	seq_puts(p, "  Rescheduling interrupts\n");
102
	seq_printf(p, "%*s: ", prec, "CAL");
103
	for_each_online_cpu(j)
104
		seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
105
	seq_puts(p, "  Function call interrupts\n");
106
	seq_printf(p, "%*s: ", prec, "TLB");
107 108
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
109
	seq_puts(p, "  TLB shootdowns\n");
110
#endif
111
#ifdef CONFIG_X86_THERMAL_VECTOR
112
	seq_printf(p, "%*s: ", prec, "TRM");
113 114
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
115
	seq_puts(p, "  Thermal event interrupts\n");
116 117
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
118
	seq_printf(p, "%*s: ", prec, "THR");
119 120
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
121
	seq_puts(p, "  Threshold APIC interrupts\n");
122
#endif
123 124 125 126 127 128
#ifdef CONFIG_X86_MCE_AMD
	seq_printf(p, "%*s: ", prec, "DFR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
	seq_puts(p, "  Deferred Error APIC interrupts\n");
#endif
129
#ifdef CONFIG_X86_MCE
130 131 132
	seq_printf(p, "%*s: ", prec, "MCE");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
133
	seq_puts(p, "  Machine check exceptions\n");
134 135 136
	seq_printf(p, "%*s: ", prec, "MCP");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
137
	seq_puts(p, "  Machine check polls\n");
138
#endif
139
#if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN)
140 141 142 143 144 145 146
	if (test_bit(HYPERVISOR_CALLBACK_VECTOR, used_vectors)) {
		seq_printf(p, "%*s: ", prec, "HYP");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				   irq_stats(j)->irq_hv_callback_count);
		seq_puts(p, "  Hypervisor callback interrupts\n");
	}
147
#endif
148
	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
149
#if defined(CONFIG_X86_IO_APIC)
150
	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
151 152 153 154 155 156 157 158 159 160 161 162
#endif
#ifdef CONFIG_HAVE_KVM
	seq_printf(p, "%*s: ", prec, "PIN");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
	seq_puts(p, "  Posted-interrupt notification event\n");

	seq_printf(p, "%*s: ", prec, "PIW");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ",
			   irq_stats(j)->kvm_posted_intr_wakeup_ipis);
	seq_puts(p, "  Posted-interrupt wakeup event\n");
163 164 165 166 167 168 169 170 171 172 173 174 175
#endif
	return 0;
}

/*
 * /proc/stat helpers
 */
u64 arch_irq_stat_cpu(unsigned int cpu)
{
	u64 sum = irq_stats(cpu)->__nmi_count;

#ifdef CONFIG_X86_LOCAL_APIC
	sum += irq_stats(cpu)->apic_timer_irqs;
176
	sum += irq_stats(cpu)->irq_spurious_count;
I
Ingo Molnar 已提交
177
	sum += irq_stats(cpu)->apic_perf_irqs;
178
	sum += irq_stats(cpu)->apic_irq_work_irqs;
179
	sum += irq_stats(cpu)->icr_read_retry_count;
180
#endif
181 182
	if (x86_platform_ipi_callback)
		sum += irq_stats(cpu)->x86_platform_ipis;
183 184 185 186
#ifdef CONFIG_SMP
	sum += irq_stats(cpu)->irq_resched_count;
	sum += irq_stats(cpu)->irq_call_count;
#endif
187
#ifdef CONFIG_X86_THERMAL_VECTOR
188
	sum += irq_stats(cpu)->irq_thermal_count;
189 190
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
191
	sum += irq_stats(cpu)->irq_threshold_count;
H
Hidetoshi Seto 已提交
192
#endif
193
#ifdef CONFIG_X86_MCE
H
Hidetoshi Seto 已提交
194 195
	sum += per_cpu(mce_exception_count, cpu);
	sum += per_cpu(mce_poll_count, cpu);
196 197 198 199 200 201 202 203 204
#endif
	return sum;
}

u64 arch_irq_stat(void)
{
	u64 sum = atomic_read(&irq_err_count);
	return sum;
}
205

J
Jeremy Fitzhardinge 已提交
206 207 208 209 210 211

/*
 * do_IRQ handles all normal device IRQ's (the special
 * SMP cross-CPU interrupts have their own specific
 * handlers).
 */
212
__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
J
Jeremy Fitzhardinge 已提交
213 214
{
	struct pt_regs *old_regs = set_irq_regs(regs);
215
	struct irq_desc * desc;
J
Jeremy Fitzhardinge 已提交
216 217 218
	/* high bit used in ret_from_ code  */
	unsigned vector = ~regs->orig_ax;

219 220 221 222 223 224 225 226 227 228 229 230
	/*
	 * NB: Unlike exception entries, IRQ entries do not reliably
	 * handle context tracking in the low-level entry code.  This is
	 * because syscall entries execute briefly with IRQs on before
	 * updating context tracking state, so we can take an IRQ from
	 * kernel mode with CONTEXT_USER.  The low-level entry code only
	 * updates the context if we came from user mode, so we won't
	 * switch to CONTEXT_KERNEL.  We'll fix that once the syscall
	 * code is cleaned up enough that we can cleanly defer enabling
	 * IRQs.
	 */

231
	entering_irq();
J
Jeremy Fitzhardinge 已提交
232

233
	/* entering_irq() tells RCU that we're not quiescent.  Check it. */
234
	RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");
235

236
	desc = __this_cpu_read(vector_irq[vector]);
J
Jeremy Fitzhardinge 已提交
237

238
	if (!handle_irq(desc, regs)) {
239
		ack_APIC_irq();
J
Jeremy Fitzhardinge 已提交
240

241 242
		if (desc != VECTOR_RETRIGGERED) {
			pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n",
243
					     __func__, smp_processor_id(),
244
					     vector);
245
		} else {
246
			__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
247
		}
J
Jeremy Fitzhardinge 已提交
248 249
	}

250
	exiting_irq();
J
Jeremy Fitzhardinge 已提交
251 252 253 254 255

	set_irq_regs(old_regs);
	return 1;
}

256
/*
257
 * Handler for X86_PLATFORM_IPI_VECTOR.
258
 */
259
void __smp_x86_platform_ipi(void)
260
{
261
	inc_irq_stat(x86_platform_ipis);
262

263 264
	if (x86_platform_ipi_callback)
		x86_platform_ipi_callback();
265
}
266

267
__visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs)
268 269
{
	struct pt_regs *old_regs = set_irq_regs(regs);
270

271 272 273
	entering_ack_irq();
	__smp_x86_platform_ipi();
	exiting_irq();
274 275 276
	set_irq_regs(old_regs);
}

277
#ifdef CONFIG_HAVE_KVM
278 279 280 281 282 283 284 285 286 287 288 289
static void dummy_handler(void) {}
static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;

void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
{
	if (handler)
		kvm_posted_intr_wakeup_handler = handler;
	else
		kvm_posted_intr_wakeup_handler = dummy_handler;
}
EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);

290 291 292
/*
 * Handler for POSTED_INTERRUPT_VECTOR.
 */
293
__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
294 295 296
{
	struct pt_regs *old_regs = set_irq_regs(regs);

297
	entering_ack_irq();
298
	inc_irq_stat(kvm_posted_intr_ipis);
299 300 301 302 303 304 305 306 307 308 309 310 311 312
	exiting_irq();
	set_irq_regs(old_regs);
}

/*
 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
 */
__visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	inc_irq_stat(kvm_posted_intr_wakeup_ipis);
	kvm_posted_intr_wakeup_handler();
313
	exiting_irq();
314 315 316 317
	set_irq_regs(old_regs);
}
#endif

318
__visible void __irq_entry smp_trace_x86_platform_ipi(struct pt_regs *regs)
319 320 321 322 323 324 325 326 327 328 329
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	__smp_x86_platform_ipi();
	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
	exiting_irq();
	set_irq_regs(old_regs);
}

330
EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);
331 332

#ifdef CONFIG_HOTPLUG_CPU
333 334 335 336 337 338 339 340

/* These two declarations are only used in check_irq_vectors_for_cpu_disable()
 * below, which is protected by stop_machine().  Putting them on the stack
 * results in a stack frame overflow.  Dynamically allocating could result in a
 * failure so declare these two cpumasks as global.
 */
static struct cpumask affinity_new, online_new;

341 342 343 344 345 346 347 348 349 350
/*
 * This cpu is going to be removed and its vectors migrated to the remaining
 * online cpus.  Check to see if there are enough vectors in the remaining cpus.
 * This function is protected by stop_machine().
 */
int check_irq_vectors_for_cpu_disable(void)
{
	unsigned int this_cpu, vector, this_count, count;
	struct irq_desc *desc;
	struct irq_data *data;
351
	int cpu;
352 353 354

	this_cpu = smp_processor_id();
	cpumask_copy(&online_new, cpu_online_mask);
355
	cpumask_clear_cpu(this_cpu, &online_new);
356 357 358

	this_count = 0;
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
359 360
		desc = __this_cpu_read(vector_irq[vector]);
		if (IS_ERR_OR_NULL(desc))
361 362 363 364 365 366 367
			continue;
		/*
		 * Protect against concurrent action removal, affinity
		 * changes etc.
		 */
		raw_spin_lock(&desc->lock);
		data = irq_desc_get_irq_data(desc);
368 369
		cpumask_copy(&affinity_new,
			     irq_data_get_affinity_mask(data));
370
		cpumask_clear_cpu(this_cpu, &affinity_new);
371

372
		/* Do not count inactive or per-cpu irqs. */
373
		if (!irq_desc_has_action(desc) || irqd_is_per_cpu(data)) {
374
			raw_spin_unlock(&desc->lock);
375
			continue;
376
		}
377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395

		raw_spin_unlock(&desc->lock);
		/*
		 * A single irq may be mapped to multiple cpu's
		 * vector_irq[] (for example IOAPIC cluster mode).  In
		 * this case we have two possibilities:
		 *
		 * 1) the resulting affinity mask is empty; that is
		 * this the down'd cpu is the last cpu in the irq's
		 * affinity mask, or
		 *
		 * 2) the resulting affinity mask is no longer a
		 * subset of the online cpus but the affinity mask is
		 * not zero; that is the down'd cpu is the last online
		 * cpu in a user set affinity mask.
		 */
		if (cpumask_empty(&affinity_new) ||
		    !cpumask_subset(&affinity_new, &online_new))
			this_count++;
396
	}
397 398 399
	/* No need to check any further. */
	if (!this_count)
		return 0;
400 401 402 403 404

	count = 0;
	for_each_online_cpu(cpu) {
		if (cpu == this_cpu)
			continue;
405 406 407 408 409
		/*
		 * We scan from FIRST_EXTERNAL_VECTOR to first system
		 * vector. If the vector is marked in the used vectors
		 * bitmap or an irq is assigned to it, we don't count
		 * it as available.
410 411 412
		 *
		 * As this is an inaccurate snapshot anyway, we can do
		 * this w/o holding vector_lock.
413 414 415 416
		 */
		for (vector = FIRST_EXTERNAL_VECTOR;
		     vector < first_system_vector; vector++) {
			if (!test_bit(vector, used_vectors) &&
417 418 419 420
			    IS_ERR_OR_NULL(per_cpu(vector_irq, cpu)[vector])) {
				if (++count == this_count)
					return 0;
			}
421 422 423 424 425 426 427 428 429 430 431
		}
	}

	if (count < this_count) {
		pr_warn("CPU %d disable failed: CPU has %u vectors assigned and there are only %u available.\n",
			this_cpu, this_count, count);
		return -ERANGE;
	}
	return 0;
}

432 433 434
/* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
void fixup_irqs(void)
{
435
	unsigned int irr, vector;
436
	struct irq_desc *desc;
437
	struct irq_data *data;
438
	struct irq_chip *chip;
439

440
	irq_migrate_all_off_this_cpu();
441

442 443 444 445 446 447 448 449 450
	/*
	 * We can remove mdelay() and then send spuriuous interrupts to
	 * new cpu targets for all the irqs that were handled previously by
	 * this cpu. While it works, I have seen spurious interrupt messages
	 * (nothing wrong but still...).
	 *
	 * So for now, retain mdelay(1) and check the IRR and then send those
	 * interrupts to new targets as this cpu is already offlined...
	 */
451
	mdelay(1);
452

453 454 455 456 457
	/*
	 * We can walk the vector array of this cpu without holding
	 * vector_lock because the cpu is already marked !online, so
	 * nothing else will touch it.
	 */
458
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
459
		if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
460 461 462 463
			continue;

		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		if (irr  & (1 << (vector % 32))) {
464
			desc = __this_cpu_read(vector_irq[vector]);
465

466
			raw_spin_lock(&desc->lock);
467 468
			data = irq_desc_get_irq_data(desc);
			chip = irq_data_get_irq_chip(data);
469
			if (chip->irq_retrigger) {
470
				chip->irq_retrigger(data);
471 472
				__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
			}
473
			raw_spin_unlock(&desc->lock);
474
		}
475
		if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
476
			__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
477
	}
478 479
}
#endif