vector.c 26.0 KB
Newer Older
1 2 3 4 5
/*
 * Local APIC related interfaces to support IOAPIC, MSI, HT_IRQ etc.
 *
 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
 *	Moved from arch/x86/kernel/apic/io_apic.c.
6 7
 * Jiang Liu <jiang.liu@linux.intel.com>
 *	Enable support of hierarchical irqdomains
8 9 10 11 12 13
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/interrupt.h>
14
#include <linux/seq_file.h>
15 16 17
#include <linux/init.h>
#include <linux/compiler.h>
#include <linux/slab.h>
18
#include <asm/irqdomain.h>
19 20 21 22 23 24
#include <asm/hw_irq.h>
#include <asm/apic.h>
#include <asm/i8259.h>
#include <asm/desc.h>
#include <asm/irq_remapping.h>

25 26
#include <asm/trace/irq_vectors.h>

27 28
struct apic_chip_data {
	struct irq_cfg		cfg;
29 30
	unsigned int		cpu;
	unsigned int		prev_cpu;
31
	struct hlist_node	clist;
32 33 34 35 36
	cpumask_var_t		domain;
	cpumask_var_t		old_domain;
	u8			move_in_progress : 1;
};

37
struct irq_domain *x86_vector_domain;
38
EXPORT_SYMBOL_GPL(x86_vector_domain);
39
static DEFINE_RAW_SPINLOCK(vector_lock);
40
static cpumask_var_t vector_cpumask, vector_searchmask, searched_cpumask;
41
static struct irq_chip lapic_controller;
42
static struct irq_matrix *vector_matrix;
43 44 45
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct hlist_head, cleanup_list);
#endif
46 47 48 49 50 51 52 53 54 55 56 57 58 59

void lock_vector_lock(void)
{
	/* Used to the online set of cpus does not change
	 * during assign_irq_vector.
	 */
	raw_spin_lock(&vector_lock);
}

void unlock_vector_lock(void)
{
	raw_spin_unlock(&vector_lock);
}

60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
void init_irq_alloc_info(struct irq_alloc_info *info,
			 const struct cpumask *mask)
{
	memset(info, 0, sizeof(*info));
	info->mask = mask;
}

void copy_irq_alloc_info(struct irq_alloc_info *dst, struct irq_alloc_info *src)
{
	if (src)
		*dst = *src;
	else
		memset(dst, 0, sizeof(*dst));
}

75
static struct apic_chip_data *apic_chip_data(struct irq_data *irqd)
76
{
77
	if (!irqd)
78 79
		return NULL;

80 81
	while (irqd->parent_data)
		irqd = irqd->parent_data;
82

83
	return irqd->chip_data;
84 85
}

86
struct irq_cfg *irqd_cfg(struct irq_data *irqd)
87
{
88
	struct apic_chip_data *apicd = apic_chip_data(irqd);
89

90
	return apicd ? &apicd->cfg : NULL;
91
}
92
EXPORT_SYMBOL_GPL(irqd_cfg);
93 94

struct irq_cfg *irq_cfg(unsigned int irq)
95
{
96 97
	return irqd_cfg(irq_get_irq_data(irq));
}
98

99 100
static struct apic_chip_data *alloc_apic_chip_data(int node)
{
101
	struct apic_chip_data *apicd;
102

103 104
	apicd = kzalloc_node(sizeof(*apicd), GFP_KERNEL, node);
	if (!apicd)
105
		return NULL;
106
	if (!zalloc_cpumask_var_node(&apicd->domain, GFP_KERNEL, node))
107
		goto out_data;
108
	if (!zalloc_cpumask_var_node(&apicd->old_domain, GFP_KERNEL, node))
109
		goto out_domain;
110
	INIT_HLIST_NODE(&apicd->clist);
111
	return apicd;
112
out_domain:
113
	free_cpumask_var(apicd->domain);
114
out_data:
115
	kfree(apicd);
116 117 118
	return NULL;
}

119
static void free_apic_chip_data(struct apic_chip_data *apicd)
120
{
121 122 123 124
	if (apicd) {
		free_cpumask_var(apicd->domain);
		free_cpumask_var(apicd->old_domain);
		kfree(apicd);
125
	}
126 127
}

128
static int __assign_irq_vector(int irq, struct apic_chip_data *d,
129
			       const struct cpumask *mask,
130
			       struct irq_data *irqd)
131 132 133 134 135 136 137 138 139 140 141 142 143 144
{
	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
	static int current_vector = FIRST_EXTERNAL_VECTOR + VECTOR_OFFSET_START;
	static int current_offset = VECTOR_OFFSET_START % 16;
145
	int cpu, vector;
146

147 148 149 150
	/*
	 * If there is still a move in progress or the previous move has not
	 * been cleaned up completely, tell the caller to come back later.
	 */
151
	if (d->cfg.old_vector)
152 153 154
		return -EBUSY;

	/* Only try and allocate irqs on cpus that are present */
155
	cpumask_clear(d->old_domain);
156
	cpumask_clear(searched_cpumask);
157 158
	cpu = cpumask_first_and(mask, cpu_online_mask);
	while (cpu < nr_cpu_ids) {
159
		int new_cpu, offset;
160

161
		cpumask_copy(vector_cpumask, cpumask_of(cpu));
162

163 164 165
		/*
		 * Clear the offline cpus from @vector_cpumask for searching
		 * and verify whether the result overlaps with @mask. If true,
166
		 * then the call to apic->cpu_mask_to_apicid() will
167 168 169 170 171 172 173
		 * succeed as well. If not, no point in trying to find a
		 * vector in this mask.
		 */
		cpumask_and(vector_searchmask, vector_cpumask, cpu_online_mask);
		if (!cpumask_intersects(vector_searchmask, mask))
			goto next_cpu;

174 175
		if (cpumask_subset(vector_cpumask, d->domain)) {
			if (cpumask_equal(vector_cpumask, d->domain))
176
				goto success;
177
			/*
178 179
			 * Mark the cpus which are not longer in the mask for
			 * cleanup.
180
			 */
181 182 183
			cpumask_andnot(d->old_domain, d->domain, vector_cpumask);
			vector = d->cfg.vector;
			goto update;
184 185 186 187 188 189
		}

		vector = current_vector;
		offset = current_offset;
next:
		vector += 16;
190
		if (vector >= FIRST_SYSTEM_VECTOR) {
191 192 193 194
			offset = (offset + 1) % 16;
			vector = FIRST_EXTERNAL_VECTOR + offset;
		}

195 196 197
		/* If the search wrapped around, try the next cpu */
		if (unlikely(current_vector == vector))
			goto next_cpu;
198

199
		if (test_bit(vector, system_vectors))
200 201
			goto next;

202
		for_each_cpu(new_cpu, vector_searchmask) {
203
			if (!IS_ERR_OR_NULL(per_cpu(vector_irq, new_cpu)[vector]))
204 205 206 207 208
				goto next;
		}
		/* Found one! */
		current_vector = vector;
		current_offset = offset;
209 210
		/* Schedule the old vector for cleanup on all cpus */
		if (d->cfg.vector)
211
			cpumask_copy(d->old_domain, d->domain);
212
		for_each_cpu(new_cpu, vector_searchmask)
213
			per_cpu(vector_irq, new_cpu)[vector] = irq_to_desc(irq);
214
		goto update;
215 216 217 218 219 220 221 222 223 224 225 226 227

next_cpu:
		/*
		 * We exclude the current @vector_cpumask from the requested
		 * @mask and try again with the next online cpu in the
		 * result. We cannot modify @mask, so we use @vector_cpumask
		 * as a temporary buffer here as it will be reassigned when
		 * calling apic->vector_allocation_domain() above.
		 */
		cpumask_or(searched_cpumask, searched_cpumask, vector_cpumask);
		cpumask_andnot(vector_cpumask, mask, searched_cpumask);
		cpu = cpumask_first_and(vector_cpumask, cpu_online_mask);
		continue;
228
	}
229
	return -ENOSPC;
230

231
update:
232 233 234 235 236 237
	/*
	 * Exclude offline cpus from the cleanup mask and set the
	 * move_in_progress flag when the result is not empty.
	 */
	cpumask_and(d->old_domain, d->old_domain, cpu_online_mask);
	d->move_in_progress = !cpumask_empty(d->old_domain);
238
	d->cfg.old_vector = d->move_in_progress ? d->cfg.vector : 0;
239
	d->prev_cpu = d->cpu;
240 241
	d->cfg.vector = vector;
	cpumask_copy(d->domain, vector_cpumask);
242
success:
243 244 245 246
	/*
	 * Cache destination APIC IDs into cfg->dest_apicid. This cannot fail
	 * as we already established, that mask & d->domain & cpu_online_mask
	 * is not empty.
247 248 249
	 *
	 * vector_searchmask is a subset of d->domain and has the offline
	 * cpus masked out.
250
	 */
251
	cpumask_and(vector_searchmask, vector_searchmask, mask);
252
	BUG_ON(apic->cpu_mask_to_apicid(vector_searchmask, irqd,
253
					&d->cfg.dest_apicid));
254
	d->cpu = cpumask_first(vector_searchmask);
255
	return 0;
256 257
}

258
static int assign_irq_vector(int irq, struct apic_chip_data *apicd,
259
			     const struct cpumask *mask,
260
			     struct irq_data *irqd)
261 262 263 264 265
{
	int err;
	unsigned long flags;

	raw_spin_lock_irqsave(&vector_lock, flags);
266
	err = __assign_irq_vector(irq, apicd, mask, irqd);
267 268 269 270
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	return err;
}

271
static int assign_irq_vector_policy(int irq, int node,
272
				    struct apic_chip_data *apicd,
273
				    struct irq_alloc_info *info,
274
				    struct irq_data *irqd)
275
{
276
	if (info->mask)
277
		return assign_irq_vector(irq, apicd, info->mask, irqd);
278
	if (node != NUMA_NO_NODE &&
279
	    assign_irq_vector(irq, apicd, cpumask_of_node(node), irqd) == 0)
280
		return 0;
281
	return assign_irq_vector(irq, apicd, cpu_online_mask, irqd);
282 283
}

284
static void clear_irq_vector(int irq, struct apic_chip_data *apicd)
285
{
286
	unsigned int vector = apicd->cfg.vector;
287

288
	if (!vector)
289
		return;
290

291
	per_cpu(vector_irq, apicd->cpu)[vector] = VECTOR_UNUSED;
292
	apicd->cfg.vector = 0;
293

294 295 296
	/* Clean up move in progress */
	vector = apicd->cfg.old_vector;
	if (!vector)
297 298
		return;

299
	per_cpu(vector_irq, apicd->prev_cpu)[vector] = VECTOR_UNUSED;
300
	apicd->move_in_progress = 0;
301
	hlist_del_init(&apicd->clist);
302 303
}

304 305 306
static void x86_vector_free_irqs(struct irq_domain *domain,
				 unsigned int virq, unsigned int nr_irqs)
{
307 308
	struct apic_chip_data *apicd;
	struct irq_data *irqd;
309
	unsigned long flags;
310 311 312
	int i;

	for (i = 0; i < nr_irqs; i++) {
313 314
		irqd = irq_domain_get_irq_data(x86_vector_domain, virq + i);
		if (irqd && irqd->chip_data) {
315
			raw_spin_lock_irqsave(&vector_lock, flags);
316 317 318
			clear_irq_vector(virq + i, irqd->chip_data);
			apicd = irqd->chip_data;
			irq_domain_reset_irq_data(irqd);
319
			raw_spin_unlock_irqrestore(&vector_lock, flags);
320
			free_apic_chip_data(apicd);
321 322 323 324 325 326 327 328
		}
	}
}

static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq,
				 unsigned int nr_irqs, void *arg)
{
	struct irq_alloc_info *info = arg;
329 330
	struct apic_chip_data *apicd;
	struct irq_data *irqd;
331
	int i, err, node;
332 333 334 335 336 337 338 339 340

	if (disable_apic)
		return -ENXIO;

	/* Currently vector allocator can't guarantee contiguous allocations */
	if ((info->flags & X86_IRQ_ALLOC_CONTIGUOUS_VECTORS) && nr_irqs > 1)
		return -ENOSYS;

	for (i = 0; i < nr_irqs; i++) {
341 342 343
		irqd = irq_domain_get_irq_data(domain, virq + i);
		BUG_ON(!irqd);
		node = irq_data_get_node(irqd);
344 345
		WARN_ON_ONCE(irqd->chip_data);
		apicd = alloc_apic_chip_data(node);
346
		if (!apicd) {
347 348 349 350
			err = -ENOMEM;
			goto error;
		}

351 352 353 354
		irqd->chip = &lapic_controller;
		irqd->chip_data = apicd;
		irqd->hwirq = virq + i;
		irqd_set_single_target(irqd);
355 356 357 358 359 360 361 362 363 364 365
		/*
		 * Make sure, that the legacy to IOAPIC transition stays on
		 * the same vector. This is required for check_timer() to
		 * work correctly as it might switch back to legacy mode.
		 */
		if (info->flags & X86_IRQ_ALLOC_LEGACY) {
			apicd->cfg.vector = ISA_IRQ_VECTOR(virq + i);
			apicd->cpu = 0;
			cpumask_copy(apicd->domain, cpumask_of(0));
		}

366 367
		err = assign_irq_vector_policy(virq + i, node, apicd, info,
					       irqd);
368 369 370 371 372 373 374 375 376 377 378
		if (err)
			goto error;
	}

	return 0;

error:
	x86_vector_free_irqs(domain, virq, i + 1);
	return err;
}

379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
void x86_vector_debug_show(struct seq_file *m, struct irq_domain *d,
			   struct irq_data *irqd, int ind)
{
	unsigned int cpu, vec, prev_cpu, prev_vec;
	struct apic_chip_data *apicd;
	unsigned long flags;
	int irq;

	if (!irqd) {
		irq_matrix_debug_show(m, vector_matrix, ind);
		return;
	}

	irq = irqd->irq;
	if (irq < nr_legacy_irqs() && !test_bit(irq, &io_apic_irqs)) {
		seq_printf(m, "%*sVector: %5d\n", ind, "", ISA_IRQ_VECTOR(irq));
		seq_printf(m, "%*sTarget: Legacy PIC all CPUs\n", ind, "");
		return;
	}

	apicd = irqd->chip_data;
	if (!apicd) {
		seq_printf(m, "%*sVector: Not assigned\n", ind, "");
		return;
	}

	raw_spin_lock_irqsave(&vector_lock, flags);
	cpu = apicd->cpu;
	vec = apicd->cfg.vector;
	prev_cpu = apicd->prev_cpu;
	prev_vec = apicd->cfg.old_vector;
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	seq_printf(m, "%*sVector: %5u\n", ind, "", vec);
	seq_printf(m, "%*sTarget: %5u\n", ind, "", cpu);
	if (prev_vec) {
		seq_printf(m, "%*sPrevious vector: %5u\n", ind, "", prev_vec);
		seq_printf(m, "%*sPrevious target: %5u\n", ind, "", prev_cpu);
	}
}
#endif

T
Thomas Gleixner 已提交
421
static const struct irq_domain_ops x86_vector_domain_ops = {
422 423 424 425 426
	.alloc		= x86_vector_alloc_irqs,
	.free		= x86_vector_free_irqs,
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
	.debug_show	= x86_vector_debug_show,
#endif
427 428
};

429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448
int __init arch_probe_nr_irqs(void)
{
	int nr;

	if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
		nr_irqs = NR_VECTORS * nr_cpu_ids;

	nr = (gsi_top + nr_legacy_irqs()) + 8 * nr_cpu_ids;
#if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
	/*
	 * for MSI and HT dyn irq
	 */
	if (gsi_top <= NR_IRQS_LEGACY)
		nr +=  8 * nr_cpu_ids;
	else
		nr += gsi_top * 16;
#endif
	if (nr < nr_irqs)
		nr_irqs = nr;

449 450 451 452 453
	/*
	 * We don't know if PIC is present at this point so we need to do
	 * probe() to get the right number of legacy IRQs.
	 */
	return legacy_pic->probe();
454 455
}

456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
void lapic_assign_legacy_vector(unsigned int irq, bool replace)
{
	/*
	 * Use assign system here so it wont get accounted as allocated
	 * and moveable in the cpu hotplug check and it prevents managed
	 * irq reservation from touching it.
	 */
	irq_matrix_assign_system(vector_matrix, ISA_IRQ_VECTOR(irq), replace);
}

void __init lapic_assign_system_vectors(void)
{
	unsigned int i, vector = 0;

	for_each_set_bit_from(vector, system_vectors, NR_VECTORS)
		irq_matrix_assign_system(vector_matrix, vector, false);

	if (nr_legacy_irqs() > 1)
		lapic_assign_legacy_vector(PIC_CASCADE_IR, false);

	/* System vectors are reserved, online it */
	irq_matrix_online(vector_matrix);

	/* Mark the preallocated legacy interrupts */
	for (i = 0; i < nr_legacy_irqs(); i++) {
		if (i != PIC_CASCADE_IR)
			irq_matrix_assign(vector_matrix, ISA_IRQ_VECTOR(i));
	}
}

486 487
int __init arch_early_irq_init(void)
{
488 489 490 491 492 493
	struct fwnode_handle *fn;

	fn = irq_domain_alloc_named_fwnode("VECTOR");
	BUG_ON(!fn);
	x86_vector_domain = irq_domain_create_tree(fn, &x86_vector_domain_ops,
						   NULL);
494
	BUG_ON(x86_vector_domain == NULL);
495
	irq_domain_free_fwnode(fn);
496 497
	irq_set_default_host(x86_vector_domain);

498
	arch_init_msi_domain(x86_vector_domain);
499
	arch_init_htirq_domain(x86_vector_domain);
500

501
	BUG_ON(!alloc_cpumask_var(&vector_cpumask, GFP_KERNEL));
502
	BUG_ON(!alloc_cpumask_var(&vector_searchmask, GFP_KERNEL));
503
	BUG_ON(!alloc_cpumask_var(&searched_cpumask, GFP_KERNEL));
504

505 506 507 508 509 510 511 512
	/*
	 * Allocate the vector matrix allocator data structure and limit the
	 * search area.
	 */
	vector_matrix = irq_alloc_matrix(NR_VECTORS, FIRST_EXTERNAL_VECTOR,
					 FIRST_SYSTEM_VECTOR);
	BUG_ON(!vector_matrix);

513 514 515
	return arch_early_ioapic_init();
}

516 517
/* Temporary hack to keep things working */
static void vector_update_shutdown_irqs(void)
518
{
519
	struct irq_desc *desc;
520
	int irq;
521

522
	for_each_irq_desc(irq, desc) {
523 524
		struct irq_data *irqd = irq_desc_get_irq_data(desc);
		struct apic_chip_data *ad = apic_chip_data(irqd);
525

526
		if (ad && ad->cfg.vector && ad->cpu == smp_processor_id())
527
			this_cpu_write(vector_irq[ad->cfg.vector], desc);
528 529 530
	}
}

531 532 533 534 535 536 537 538 539 540 541 542 543
static struct irq_desc *__setup_vector_irq(int vector)
{
	int isairq = vector - ISA_IRQ_VECTOR(0);

	/* Check whether the irq is in the legacy space */
	if (isairq < 0 || isairq >= nr_legacy_irqs())
		return VECTOR_UNUSED;
	/* Check whether the irq is handled by the IOAPIC */
	if (test_bit(isairq, &io_apic_irqs))
		return VECTOR_UNUSED;
	return irq_to_desc(isairq);
}

544 545
/* Online the local APIC infrastructure and initialize the vectors */
void lapic_online(void)
546
{
547
	unsigned int vector;
548

549
	lockdep_assert_held(&vector_lock);
550 551 552 553

	/* Online the vector matrix array for this CPU */
	irq_matrix_online(vector_matrix);

554
	/*
555 556 557 558 559 560 561
	 * The interrupt affinity logic never targets interrupts to offline
	 * CPUs. The exception are the legacy PIC interrupts. In general
	 * they are only targeted to CPU0, but depending on the platform
	 * they can be distributed to any online CPU in hardware. The
	 * kernel has no influence on that. So all active legacy vectors
	 * must be installed on all CPUs. All non legacy interrupts can be
	 * cleared.
562
	 */
563 564
	for (vector = 0; vector < NR_VECTORS; vector++)
		this_cpu_write(vector_irq[vector], __setup_vector_irq(vector));
565

566 567 568 569 570 571
	/*
	 * Until the rewrite of the managed interrupt management is in
	 * place it's necessary to walk the irq descriptors and check for
	 * interrupts which are targeted at this CPU.
	 */
	vector_update_shutdown_irqs();
572 573
}

574 575 576 577 578 579 580
void lapic_offline(void)
{
	lock_vector_lock();
	irq_matrix_offline(vector_matrix);
	unlock_vector_lock();
}

581
static int apic_retrigger_irq(struct irq_data *irqd)
582
{
583
	struct apic_chip_data *apicd = apic_chip_data(irqd);
584 585 586
	unsigned long flags;

	raw_spin_lock_irqsave(&vector_lock, flags);
587
	apic->send_IPI(apicd->cpu, apicd->cfg.vector);
588 589 590 591 592
	raw_spin_unlock_irqrestore(&vector_lock, flags);

	return 1;
}

593
void apic_ack_edge(struct irq_data *irqd)
594
{
595 596
	irq_complete_move(irqd_cfg(irqd));
	irq_move_irq(irqd);
597 598 599
	ack_APIC_irq();
}

600
static int apic_set_affinity(struct irq_data *irqd,
601
			     const struct cpumask *dest, bool force)
602
{
603 604
	struct apic_chip_data *apicd = irqd->chip_data;
	int err, irq = irqd->irq;
605

606
	if (!IS_ENABLED(CONFIG_SMP))
607 608 609 610 611
		return -EPERM;

	if (!cpumask_intersects(dest, cpu_online_mask))
		return -EINVAL;

612
	err = assign_irq_vector(irq, apicd, dest, irqd);
613
	return err ? err : IRQ_SET_MASK_OK;
614 615 616
}

static struct irq_chip lapic_controller = {
T
Thomas Gleixner 已提交
617
	.name			= "APIC",
618
	.irq_ack		= apic_ack_edge,
619
	.irq_set_affinity	= apic_set_affinity,
620 621 622
	.irq_retrigger		= apic_retrigger_irq,
};

623
#ifdef CONFIG_SMP
624

625
asmlinkage __visible void __irq_entry smp_irq_move_cleanup_interrupt(void)
626
{
627 628 629
	struct hlist_head *clhead = this_cpu_ptr(&cleanup_list);
	struct apic_chip_data *apicd;
	struct hlist_node *tmp;
630

631
	entering_ack_irq();
632 633 634
	/* Prevent vectors vanishing under us */
	raw_spin_lock(&vector_lock);

635 636
	hlist_for_each_entry_safe(apicd, tmp, clhead, clist) {
		unsigned int irr, vector = apicd->cfg.old_vector;
637 638

		/*
639 640 641
		 * Paranoia: Check if the vector that needs to be cleaned
		 * up is registered at the APICs IRR. If so, then this is
		 * not the best time to clean it up. Clean it up in the
642
		 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
643 644 645
		 * to this CPU. IRQ_MOVE_CLEANUP_VECTOR is the lowest
		 * priority external vector, so on return from this
		 * interrupt the device interrupt will happen first.
646
		 */
647 648
		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		if (irr & (1U << (vector % 32))) {
649
			apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
650
			continue;
651
		}
652
		hlist_del_init(&apicd->clist);
653
		__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
654
		apicd->cfg.old_vector = 0;
655 656
	}

657
	raw_spin_unlock(&vector_lock);
658
	exiting_irq();
659 660
}

661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685
static void __send_cleanup_vector(struct apic_chip_data *apicd)
{
	unsigned int cpu;

	raw_spin_lock(&vector_lock);
	apicd->move_in_progress = 0;
	cpu = apicd->prev_cpu;
	if (cpu_online(cpu)) {
		hlist_add_head(&apicd->clist, per_cpu_ptr(&cleanup_list, cpu));
		apic->send_IPI(cpu, IRQ_MOVE_CLEANUP_VECTOR);
	} else {
		apicd->cfg.old_vector = 0;
	}
	raw_spin_unlock(&vector_lock);
}

void send_cleanup_vector(struct irq_cfg *cfg)
{
	struct apic_chip_data *apicd;

	apicd = container_of(cfg, struct apic_chip_data, cfg);
	if (apicd->move_in_progress)
		__send_cleanup_vector(apicd);
}

686 687
static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
{
688
	struct apic_chip_data *apicd;
689

690 691
	apicd = container_of(cfg, struct apic_chip_data, cfg);
	if (likely(!apicd->move_in_progress))
692 693
		return;

694
	if (vector == apicd->cfg.vector && apicd->cpu == smp_processor_id())
695
		__send_cleanup_vector(apicd);
696 697 698 699 700 701 702
}

void irq_complete_move(struct irq_cfg *cfg)
{
	__irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
}

703
/*
704
 * Called from fixup_irqs() with @desc->lock held and interrupts disabled.
705 706
 */
void irq_force_complete_move(struct irq_desc *desc)
707
{
708
	struct apic_chip_data *apicd;
709 710
	struct irq_data *irqd;
	unsigned int vector;
711

712 713 714 715 716 717 718 719 720
	/*
	 * The function is called for all descriptors regardless of which
	 * irqdomain they belong to. For example if an IRQ is provided by
	 * an irq_chip as part of a GPIO driver, the chip data for that
	 * descriptor is specific to the irq_chip in question.
	 *
	 * Check first that the chip_data is what we expect
	 * (apic_chip_data) before touching it any further.
	 */
721
	irqd = irq_domain_get_irq_data(x86_vector_domain,
722
				       irq_desc_get_irq(desc));
723
	if (!irqd)
724 725
		return;

726
	raw_spin_lock(&vector_lock);
727
	apicd = apic_chip_data(irqd);
728 729
	if (!apicd)
		goto unlock;
730

731 732 733 734 735 736
	/*
	 * If old_vector is empty, no action required.
	 */
	vector = apicd->cfg.old_vector;
	if (!vector)
		goto unlock;
737

738
	/*
739
	 * This is tricky. If the cleanup of the old vector has not been
740 741 742
	 * done yet, then the following setaffinity call will fail with
	 * -EBUSY. This can leave the interrupt in a stale state.
	 *
743 744
	 * All CPUs are stuck in stop machine with interrupts disabled so
	 * calling __irq_complete_move() would be completely pointless.
745
	 *
746 747 748 749 750 751 752
	 * 1) The interrupt is in move_in_progress state. That means that we
	 *    have not seen an interrupt since the io_apic was reprogrammed to
	 *    the new vector.
	 *
	 * 2) The interrupt has fired on the new vector, but the cleanup IPIs
	 *    have not been processed yet.
	 */
753
	if (apicd->move_in_progress) {
754
		/*
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
		 * In theory there is a race:
		 *
		 * set_ioapic(new_vector) <-- Interrupt is raised before update
		 *			      is effective, i.e. it's raised on
		 *			      the old vector.
		 *
		 * So if the target cpu cannot handle that interrupt before
		 * the old vector is cleaned up, we get a spurious interrupt
		 * and in the worst case the ioapic irq line becomes stale.
		 *
		 * But in case of cpu hotplug this should be a non issue
		 * because if the affinity update happens right before all
		 * cpus rendevouz in stop machine, there is no way that the
		 * interrupt can be blocked on the target cpu because all cpus
		 * loops first with interrupts enabled in stop machine, so the
		 * old vector is not yet cleaned up when the interrupt fires.
		 *
		 * So the only way to run into this issue is if the delivery
		 * of the interrupt on the apic/system bus would be delayed
		 * beyond the point where the target cpu disables interrupts
		 * in stop machine. I doubt that it can happen, but at least
		 * there is a theroretical chance. Virtualization might be
		 * able to expose this, but AFAICT the IOAPIC emulation is not
		 * as stupid as the real hardware.
		 *
		 * Anyway, there is nothing we can do about that at this point
		 * w/o refactoring the whole fixup_irq() business completely.
		 * We print at least the irq number and the old vector number,
		 * so we have the necessary information when a problem in that
		 * area arises.
785
		 */
786
		pr_warn("IRQ fixup: irq %d move in progress, old vector %d\n",
787
			irqd->irq, vector);
788
	}
789
	per_cpu(vector_irq, apicd->prev_cpu)[vector] = VECTOR_UNUSED;
790
	/* Cleanup the left overs of the (half finished) move */
791
	cpumask_clear(apicd->old_domain);
792
	apicd->cfg.old_vector = 0;
793
	apicd->move_in_progress = 0;
794 795
	hlist_del_init(&apicd->clist);
unlock:
796
	raw_spin_unlock(&vector_lock);
797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816
}
#endif

static void __init print_APIC_field(int base)
{
	int i;

	printk(KERN_DEBUG);

	for (i = 0; i < 8; i++)
		pr_cont("%08x", apic_read(base + i*0x10));

	pr_cont("\n");
}

static void __init print_local_APIC(void *dummy)
{
	unsigned int i, v, ver, maxlvt;
	u64 icr;

817 818
	pr_debug("printing local APIC contents on CPU#%d/%d:\n",
		 smp_processor_id(), hard_smp_processor_id());
819
	v = apic_read(APIC_ID);
820
	pr_info("... APIC ID:      %08x (%01x)\n", v, read_apic_id());
821
	v = apic_read(APIC_LVR);
822
	pr_info("... APIC VERSION: %08x\n", v);
823 824 825 826
	ver = GET_APIC_VERSION(v);
	maxlvt = lapic_get_maxlvt();

	v = apic_read(APIC_TASKPRI);
827
	pr_debug("... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
828 829 830 831 832

	/* !82489DX */
	if (APIC_INTEGRATED(ver)) {
		if (!APIC_XAPIC(ver)) {
			v = apic_read(APIC_ARBPRI);
833 834
			pr_debug("... APIC ARBPRI: %08x (%02x)\n",
				 v, v & APIC_ARBPRI_MASK);
835 836
		}
		v = apic_read(APIC_PROCPRI);
837
		pr_debug("... APIC PROCPRI: %08x\n", v);
838 839 840 841 842 843 844 845
	}

	/*
	 * Remote read supported only in the 82489DX and local APIC for
	 * Pentium processors.
	 */
	if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
		v = apic_read(APIC_RRR);
846
		pr_debug("... APIC RRR: %08x\n", v);
847 848 849
	}

	v = apic_read(APIC_LDR);
850
	pr_debug("... APIC LDR: %08x\n", v);
851 852
	if (!x2apic_enabled()) {
		v = apic_read(APIC_DFR);
853
		pr_debug("... APIC DFR: %08x\n", v);
854 855
	}
	v = apic_read(APIC_SPIV);
856
	pr_debug("... APIC SPIV: %08x\n", v);
857

858
	pr_debug("... APIC ISR field:\n");
859
	print_APIC_field(APIC_ISR);
860
	pr_debug("... APIC TMR field:\n");
861
	print_APIC_field(APIC_TMR);
862
	pr_debug("... APIC IRR field:\n");
863 864 865 866 867 868 869 870 871
	print_APIC_field(APIC_IRR);

	/* !82489DX */
	if (APIC_INTEGRATED(ver)) {
		/* Due to the Pentium erratum 3AP. */
		if (maxlvt > 3)
			apic_write(APIC_ESR, 0);

		v = apic_read(APIC_ESR);
872
		pr_debug("... APIC ESR: %08x\n", v);
873 874 875
	}

	icr = apic_icr_read();
876 877
	pr_debug("... APIC ICR: %08x\n", (u32)icr);
	pr_debug("... APIC ICR2: %08x\n", (u32)(icr >> 32));
878 879

	v = apic_read(APIC_LVTT);
880
	pr_debug("... APIC LVTT: %08x\n", v);
881 882 883 884

	if (maxlvt > 3) {
		/* PC is LVT#4. */
		v = apic_read(APIC_LVTPC);
885
		pr_debug("... APIC LVTPC: %08x\n", v);
886 887
	}
	v = apic_read(APIC_LVT0);
888
	pr_debug("... APIC LVT0: %08x\n", v);
889
	v = apic_read(APIC_LVT1);
890
	pr_debug("... APIC LVT1: %08x\n", v);
891 892 893 894

	if (maxlvt > 2) {
		/* ERR is LVT#3. */
		v = apic_read(APIC_LVTERR);
895
		pr_debug("... APIC LVTERR: %08x\n", v);
896 897 898
	}

	v = apic_read(APIC_TMICT);
899
	pr_debug("... APIC TMICT: %08x\n", v);
900
	v = apic_read(APIC_TMCCT);
901
	pr_debug("... APIC TMCCT: %08x\n", v);
902
	v = apic_read(APIC_TDCR);
903
	pr_debug("... APIC TDCR: %08x\n", v);
904 905 906 907

	if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
		v = apic_read(APIC_EFEAT);
		maxlvt = (v >> 16) & 0xff;
908
		pr_debug("... APIC EFEAT: %08x\n", v);
909
		v = apic_read(APIC_ECTRL);
910
		pr_debug("... APIC ECTRL: %08x\n", v);
911 912
		for (i = 0; i < maxlvt; i++) {
			v = apic_read(APIC_EILVTn(i));
913
			pr_debug("... APIC EILVT%d: %08x\n", i, v);
914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942
		}
	}
	pr_cont("\n");
}

static void __init print_local_APICs(int maxcpu)
{
	int cpu;

	if (!maxcpu)
		return;

	preempt_disable();
	for_each_online_cpu(cpu) {
		if (cpu >= maxcpu)
			break;
		smp_call_function_single(cpu, print_local_APIC, NULL, 1);
	}
	preempt_enable();
}

static void __init print_PIC(void)
{
	unsigned int v;
	unsigned long flags;

	if (!nr_legacy_irqs())
		return;

943
	pr_debug("\nprinting PIC contents\n");
944 945 946 947

	raw_spin_lock_irqsave(&i8259A_lock, flags);

	v = inb(0xa1) << 8 | inb(0x21);
948
	pr_debug("... PIC  IMR: %04x\n", v);
949 950

	v = inb(0xa0) << 8 | inb(0x20);
951
	pr_debug("... PIC  IRR: %04x\n", v);
952 953 954 955 956 957 958 959 960

	outb(0x0b, 0xa0);
	outb(0x0b, 0x20);
	v = inb(0xa0) << 8 | inb(0x20);
	outb(0x0a, 0xa0);
	outb(0x0a, 0x20);

	raw_spin_unlock_irqrestore(&i8259A_lock, flags);

961
	pr_debug("... PIC  ISR: %04x\n", v);
962 963

	v = inb(0x4d1) << 8 | inb(0x4d0);
964
	pr_debug("... PIC ELCR: %04x\n", v);
965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991
}

static int show_lapic __initdata = 1;
static __init int setup_show_lapic(char *arg)
{
	int num = -1;

	if (strcmp(arg, "all") == 0) {
		show_lapic = CONFIG_NR_CPUS;
	} else {
		get_option(&arg, &num);
		if (num >= 0)
			show_lapic = num;
	}

	return 1;
}
__setup("show_lapic=", setup_show_lapic);

static int __init print_ICs(void)
{
	if (apic_verbosity == APIC_QUIET)
		return 0;

	print_PIC();

	/* don't print out if apic is not there */
992
	if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
993 994 995 996 997 998 999 1000 1001
		return 0;

	print_local_APICs(show_lapic);
	print_IO_APICs();

	return 0;
}

late_initcall(print_ICs);