vector.c 30.7 KB
Newer Older
1 2 3 4 5
/*
 * Local APIC related interfaces to support IOAPIC, MSI, HT_IRQ etc.
 *
 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
 *	Moved from arch/x86/kernel/apic/io_apic.c.
6 7
 * Jiang Liu <jiang.liu@linux.intel.com>
 *	Enable support of hierarchical irqdomains
8 9 10 11 12 13
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/interrupt.h>
14
#include <linux/seq_file.h>
15 16 17
#include <linux/init.h>
#include <linux/compiler.h>
#include <linux/slab.h>
18
#include <asm/irqdomain.h>
19 20 21 22 23 24
#include <asm/hw_irq.h>
#include <asm/apic.h>
#include <asm/i8259.h>
#include <asm/desc.h>
#include <asm/irq_remapping.h>

25 26
#include <asm/trace/irq_vectors.h>

27
struct apic_chip_data {
28 29 30
	struct irq_cfg		hw_irq_cfg;
	unsigned int		vector;
	unsigned int		prev_vector;
31 32
	unsigned int		cpu;
	unsigned int		prev_cpu;
33
	unsigned int		irq;
34
	struct hlist_node	clist;
35
	unsigned int		move_in_progress	: 1,
36 37 38
				is_managed		: 1,
				can_reserve		: 1,
				has_reserved		: 1;
39 40
};

41
struct irq_domain *x86_vector_domain;
42
EXPORT_SYMBOL_GPL(x86_vector_domain);
43
static DEFINE_RAW_SPINLOCK(vector_lock);
44
static cpumask_var_t vector_searchmask;
45
static struct irq_chip lapic_controller;
46
static struct irq_matrix *vector_matrix;
47 48 49
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct hlist_head, cleanup_list);
#endif
50 51 52 53 54 55 56 57 58 59 60 61 62 63

void lock_vector_lock(void)
{
	/* Used to the online set of cpus does not change
	 * during assign_irq_vector.
	 */
	raw_spin_lock(&vector_lock);
}

void unlock_vector_lock(void)
{
	raw_spin_unlock(&vector_lock);
}

64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
void init_irq_alloc_info(struct irq_alloc_info *info,
			 const struct cpumask *mask)
{
	memset(info, 0, sizeof(*info));
	info->mask = mask;
}

void copy_irq_alloc_info(struct irq_alloc_info *dst, struct irq_alloc_info *src)
{
	if (src)
		*dst = *src;
	else
		memset(dst, 0, sizeof(*dst));
}

79
static struct apic_chip_data *apic_chip_data(struct irq_data *irqd)
80
{
81
	if (!irqd)
82 83
		return NULL;

84 85
	while (irqd->parent_data)
		irqd = irqd->parent_data;
86

87
	return irqd->chip_data;
88 89
}

90
struct irq_cfg *irqd_cfg(struct irq_data *irqd)
91
{
92
	struct apic_chip_data *apicd = apic_chip_data(irqd);
93

94
	return apicd ? &apicd->hw_irq_cfg : NULL;
95
}
96
EXPORT_SYMBOL_GPL(irqd_cfg);
97 98

struct irq_cfg *irq_cfg(unsigned int irq)
99
{
100 101
	return irqd_cfg(irq_get_irq_data(irq));
}
102

103 104
static struct apic_chip_data *alloc_apic_chip_data(int node)
{
105
	struct apic_chip_data *apicd;
106

107
	apicd = kzalloc_node(sizeof(*apicd), GFP_KERNEL, node);
108 109
	if (apicd)
		INIT_HLIST_NODE(&apicd->clist);
110
	return apicd;
111 112
}

113
static void free_apic_chip_data(struct apic_chip_data *apicd)
114
{
115
	kfree(apicd);
116 117
}

118 119
static void apic_update_irq_cfg(struct irq_data *irqd, unsigned int vector,
				unsigned int cpu)
120
{
121
	struct apic_chip_data *apicd = apic_chip_data(irqd);
122

123
	lockdep_assert_held(&vector_lock);
124

125 126 127 128 129
	apicd->hw_irq_cfg.vector = vector;
	apicd->hw_irq_cfg.dest_apicid = apic->calc_dest_apicid(cpu);
	irq_data_update_effective_affinity(irqd, cpumask_of(cpu));
	trace_vector_config(irqd->irq, vector, cpu,
			    apicd->hw_irq_cfg.dest_apicid);
130
}
131

132 133 134 135 136
static void apic_update_vector(struct irq_data *irqd, unsigned int newvec,
			       unsigned int newcpu)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	struct irq_desc *desc = irq_data_to_desc(irqd);
137

138
	lockdep_assert_held(&vector_lock);
139

140
	trace_vector_update(irqd->irq, newvec, newcpu, apicd->vector,
141
			    apicd->cpu);
142

143
	/* Setup the vector move, if required  */
144
	if (apicd->vector && cpu_online(apicd->cpu)) {
145
		apicd->move_in_progress = true;
146
		apicd->prev_vector = apicd->vector;
147 148
		apicd->prev_cpu = apicd->cpu;
	} else {
149
		apicd->prev_vector = 0;
150
	}
151

152
	apicd->vector = newvec;
153 154 155 156
	apicd->cpu = newcpu;
	BUG_ON(!IS_ERR_OR_NULL(per_cpu(vector_irq, newcpu)[newvec]));
	per_cpu(vector_irq, newcpu)[newvec] = desc;
}
157

158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
static void vector_assign_managed_shutdown(struct irq_data *irqd)
{
	unsigned int cpu = cpumask_first(cpu_online_mask);

	apic_update_irq_cfg(irqd, MANAGED_IRQ_SHUTDOWN_VECTOR, cpu);
}

static int reserve_managed_vector(struct irq_data *irqd)
{
	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	unsigned long flags;
	int ret;

	raw_spin_lock_irqsave(&vector_lock, flags);
	apicd->is_managed = true;
	ret = irq_matrix_reserve_managed(vector_matrix, affmsk);
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	trace_vector_reserve_managed(irqd->irq, ret);
	return ret;
}

180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
static void reserve_irq_vector_locked(struct irq_data *irqd)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);

	irq_matrix_reserve(vector_matrix);
	apicd->can_reserve = true;
	apicd->has_reserved = true;
	trace_vector_reserve(irqd->irq, 0);
	vector_assign_managed_shutdown(irqd);
}

static int reserve_irq_vector(struct irq_data *irqd)
{
	unsigned long flags;

	raw_spin_lock_irqsave(&vector_lock, flags);
	reserve_irq_vector_locked(irqd);
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	return 0;
}

201 202 203
static int allocate_vector(struct irq_data *irqd, const struct cpumask *dest)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
204
	bool resvd = apicd->has_reserved;
205
	unsigned int cpu = apicd->cpu;
206 207 208
	int vector = apicd->vector;

	lockdep_assert_held(&vector_lock);
209

210
	/*
211 212 213
	 * If the current target CPU is online and in the new requested
	 * affinity mask, there is no point in moving the interrupt from
	 * one CPU to another.
214
	 */
215 216 217
	if (vector && cpu_online(cpu) && cpumask_test_cpu(cpu, dest))
		return 0;

218
	vector = irq_matrix_alloc(vector_matrix, dest, resvd, &cpu);
219 220
	if (vector > 0)
		apic_update_vector(irqd, vector, cpu);
221
	trace_vector_alloc(irqd->irq, vector, resvd, vector);
222 223 224 225 226 227
	return vector;
}

static int assign_vector_locked(struct irq_data *irqd,
				const struct cpumask *dest)
{
228
	struct apic_chip_data *apicd = apic_chip_data(irqd);
229 230 231 232 233
	int vector = allocate_vector(irqd, dest);

	if (vector < 0)
		return vector;

234
	apic_update_irq_cfg(irqd, apicd->vector, apicd->cpu);
235
	return 0;
236 237
}

238
static int assign_irq_vector(struct irq_data *irqd, const struct cpumask *dest)
239 240
{
	unsigned long flags;
241
	int ret;
242 243

	raw_spin_lock_irqsave(&vector_lock, flags);
244 245
	cpumask_and(vector_searchmask, dest, cpu_online_mask);
	ret = assign_vector_locked(irqd, vector_searchmask);
246
	raw_spin_unlock_irqrestore(&vector_lock, flags);
247
	return ret;
248 249
}

250 251
static int assign_irq_vector_any_locked(struct irq_data *irqd)
{
252 253
	/* Get the affinity mask - either irq_default_affinity or (user) set */
	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);
254 255
	int node = irq_data_get_node(irqd);

256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
	if (node == NUMA_NO_NODE)
		goto all;
	/* Try the intersection of @affmsk and node mask */
	cpumask_and(vector_searchmask, cpumask_of_node(node), affmsk);
	if (!assign_vector_locked(irqd, vector_searchmask))
		return 0;
	/* Try the node mask */
	if (!assign_vector_locked(irqd, cpumask_of_node(node)))
		return 0;
all:
	/* Try the full affinity mask */
	cpumask_and(vector_searchmask, affmsk, cpu_online_mask);
	if (!assign_vector_locked(irqd, vector_searchmask))
		return 0;
	/* Try the full online mask */
271 272 273 274 275
	return assign_vector_locked(irqd, cpu_online_mask);
}

static int
assign_irq_vector_policy(struct irq_data *irqd, struct irq_alloc_info *info)
276
{
277 278
	if (irqd_affinity_is_managed(irqd))
		return reserve_managed_vector(irqd);
279
	if (info->mask)
280
		return assign_irq_vector(irqd, info->mask);
281 282 283 284
	/*
	 * Make only a global reservation with no guarantee. A real vector
	 * is associated at activation time.
	 */
285
	return reserve_irq_vector(irqd);
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
}

static int
assign_managed_vector(struct irq_data *irqd, const struct cpumask *dest)
{
	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	int vector, cpu;

	cpumask_and(vector_searchmask, vector_searchmask, affmsk);
	cpu = cpumask_first(vector_searchmask);
	if (cpu >= nr_cpu_ids)
		return -EINVAL;
	/* set_affinity might call here for nothing */
	if (apicd->vector && cpumask_test_cpu(apicd->cpu, vector_searchmask))
301
		return 0;
302 303 304 305 306 307 308
	vector = irq_matrix_alloc_managed(vector_matrix, cpu);
	trace_vector_alloc_managed(irqd->irq, vector, vector);
	if (vector < 0)
		return vector;
	apic_update_vector(irqd, vector, cpu);
	apic_update_irq_cfg(irqd, vector, cpu);
	return 0;
309 310
}

311
static void clear_irq_vector(struct irq_data *irqd)
312
{
313
	struct apic_chip_data *apicd = apic_chip_data(irqd);
314
	bool managed = irqd_affinity_is_managed(irqd);
315
	unsigned int vector = apicd->vector;
316

317
	lockdep_assert_held(&vector_lock);
318

319
	if (!vector)
320
		return;
321

322
	trace_vector_clear(irqd->irq, vector, apicd->cpu, apicd->prev_vector,
323 324
			   apicd->prev_cpu);

325
	per_cpu(vector_irq, apicd->cpu)[vector] = VECTOR_UNUSED;
326
	irq_matrix_free(vector_matrix, apicd->cpu, vector, managed);
327
	apicd->vector = 0;
328

329
	/* Clean up move in progress */
330
	vector = apicd->prev_vector;
331
	if (!vector)
332 333
		return;

334
	per_cpu(vector_irq, apicd->prev_cpu)[vector] = VECTOR_UNUSED;
335
	irq_matrix_free(vector_matrix, apicd->prev_cpu, vector, managed);
336
	apicd->prev_vector = 0;
337
	apicd->move_in_progress = 0;
338
	hlist_del_init(&apicd->clist);
339 340
}

341 342 343 344 345 346
static void x86_vector_deactivate(struct irq_domain *dom, struct irq_data *irqd)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	unsigned long flags;

	trace_vector_deactivate(irqd->irq, apicd->is_managed,
347
				apicd->can_reserve, false);
348

349 350 351 352 353
	/* Regular fixed assigned interrupt */
	if (!apicd->is_managed && !apicd->can_reserve)
		return;
	/* If the interrupt has a global reservation, nothing to do */
	if (apicd->has_reserved)
354 355 356 357
		return;

	raw_spin_lock_irqsave(&vector_lock, flags);
	clear_irq_vector(irqd);
358 359 360 361
	if (apicd->can_reserve)
		reserve_irq_vector_locked(irqd);
	else
		vector_assign_managed_shutdown(irqd);
362 363 364
	raw_spin_unlock_irqrestore(&vector_lock, flags);
}

365 366 367 368 369 370 371 372 373 374 375
static int activate_reserved(struct irq_data *irqd)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	int ret;

	ret = assign_irq_vector_any_locked(irqd);
	if (!ret)
		apicd->has_reserved = false;
	return ret;
}

376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
static int activate_managed(struct irq_data *irqd)
{
	const struct cpumask *dest = irq_data_get_affinity_mask(irqd);
	int ret;

	cpumask_and(vector_searchmask, dest, cpu_online_mask);
	if (WARN_ON_ONCE(cpumask_empty(vector_searchmask))) {
		/* Something in the core code broke! Survive gracefully */
		pr_err("Managed startup for irq %u, but no CPU\n", irqd->irq);
		return EINVAL;
	}

	ret = assign_managed_vector(irqd, vector_searchmask);
	/*
	 * This should not happen. The vector reservation got buggered.  Handle
	 * it gracefully.
	 */
	if (WARN_ON_ONCE(ret < 0)) {
		pr_err("Managed startup irq %u, no vector available\n",
		       irqd->irq);
	}
       return ret;
}

static int x86_vector_activate(struct irq_domain *dom, struct irq_data *irqd,
			       bool early)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	unsigned long flags;
	int ret = 0;

	trace_vector_activate(irqd->irq, apicd->is_managed,
408
			      apicd->can_reserve, early);
409

410 411
	/* Nothing to do for fixed assigned vectors */
	if (!apicd->can_reserve && !apicd->is_managed)
412 413 414 415 416
		return 0;

	raw_spin_lock_irqsave(&vector_lock, flags);
	if (early || irqd_is_managed_and_shutdown(irqd))
		vector_assign_managed_shutdown(irqd);
417
	else if (apicd->is_managed)
418
		ret = activate_managed(irqd);
419 420
	else if (apicd->has_reserved)
		ret = activate_reserved(irqd);
421 422 423 424 425 426 427 428 429
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	return ret;
}

static void vector_free_reserved_and_managed(struct irq_data *irqd)
{
	const struct cpumask *dest = irq_data_get_affinity_mask(irqd);
	struct apic_chip_data *apicd = apic_chip_data(irqd);

430 431
	trace_vector_teardown(irqd->irq, apicd->is_managed,
			      apicd->has_reserved);
432

433 434
	if (apicd->has_reserved)
		irq_matrix_remove_reserved(vector_matrix);
435 436 437 438
	if (apicd->is_managed)
		irq_matrix_remove_managed(vector_matrix, dest);
}

439 440 441
static void x86_vector_free_irqs(struct irq_domain *domain,
				 unsigned int virq, unsigned int nr_irqs)
{
442 443
	struct apic_chip_data *apicd;
	struct irq_data *irqd;
444
	unsigned long flags;
445 446 447
	int i;

	for (i = 0; i < nr_irqs; i++) {
448 449
		irqd = irq_domain_get_irq_data(x86_vector_domain, virq + i);
		if (irqd && irqd->chip_data) {
450
			raw_spin_lock_irqsave(&vector_lock, flags);
451
			clear_irq_vector(irqd);
452
			vector_free_reserved_and_managed(irqd);
453 454
			apicd = irqd->chip_data;
			irq_domain_reset_irq_data(irqd);
455
			raw_spin_unlock_irqrestore(&vector_lock, flags);
456
			free_apic_chip_data(apicd);
457 458 459 460
		}
	}
}

461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487
static bool vector_configure_legacy(unsigned int virq, struct irq_data *irqd,
				    struct apic_chip_data *apicd)
{
	unsigned long flags;
	bool realloc = false;

	apicd->vector = ISA_IRQ_VECTOR(virq);
	apicd->cpu = 0;

	raw_spin_lock_irqsave(&vector_lock, flags);
	/*
	 * If the interrupt is activated, then it must stay at this vector
	 * position. That's usually the timer interrupt (0).
	 */
	if (irqd_is_activated(irqd)) {
		trace_vector_setup(virq, true, 0);
		apic_update_irq_cfg(irqd, apicd->vector, apicd->cpu);
	} else {
		/* Release the vector */
		apicd->can_reserve = true;
		clear_irq_vector(irqd);
		realloc = true;
	}
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	return realloc;
}

488 489 490 491
static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq,
				 unsigned int nr_irqs, void *arg)
{
	struct irq_alloc_info *info = arg;
492 493
	struct apic_chip_data *apicd;
	struct irq_data *irqd;
494
	int i, err, node;
495 496 497 498 499 500 501 502 503

	if (disable_apic)
		return -ENXIO;

	/* Currently vector allocator can't guarantee contiguous allocations */
	if ((info->flags & X86_IRQ_ALLOC_CONTIGUOUS_VECTORS) && nr_irqs > 1)
		return -ENOSYS;

	for (i = 0; i < nr_irqs; i++) {
504 505 506
		irqd = irq_domain_get_irq_data(domain, virq + i);
		BUG_ON(!irqd);
		node = irq_data_get_node(irqd);
507 508
		WARN_ON_ONCE(irqd->chip_data);
		apicd = alloc_apic_chip_data(node);
509
		if (!apicd) {
510 511 512 513
			err = -ENOMEM;
			goto error;
		}

514
		apicd->irq = virq + i;
515 516 517 518
		irqd->chip = &lapic_controller;
		irqd->chip_data = apicd;
		irqd->hwirq = virq + i;
		irqd_set_single_target(irqd);
519
		/*
520 521 522 523 524
		 * Legacy vectors are already assigned when the IOAPIC
		 * takes them over. They stay on the same vector. This is
		 * required for check_timer() to work correctly as it might
		 * switch back to legacy mode. Only update the hardware
		 * config.
525 526
		 */
		if (info->flags & X86_IRQ_ALLOC_LEGACY) {
527 528
			if (!vector_configure_legacy(virq + i, irqd, apicd))
				continue;
529 530
		}

531
		err = assign_irq_vector_policy(irqd, info);
532
		trace_vector_setup(virq + i, false, err);
533 534 535 536 537 538 539 540 541 542 543
		if (err)
			goto error;
	}

	return 0;

error:
	x86_vector_free_irqs(domain, virq, i + 1);
	return err;
}

544 545 546 547
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
void x86_vector_debug_show(struct seq_file *m, struct irq_domain *d,
			   struct irq_data *irqd, int ind)
{
548
	unsigned int cpu, vector, prev_cpu, prev_vector;
549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572
	struct apic_chip_data *apicd;
	unsigned long flags;
	int irq;

	if (!irqd) {
		irq_matrix_debug_show(m, vector_matrix, ind);
		return;
	}

	irq = irqd->irq;
	if (irq < nr_legacy_irqs() && !test_bit(irq, &io_apic_irqs)) {
		seq_printf(m, "%*sVector: %5d\n", ind, "", ISA_IRQ_VECTOR(irq));
		seq_printf(m, "%*sTarget: Legacy PIC all CPUs\n", ind, "");
		return;
	}

	apicd = irqd->chip_data;
	if (!apicd) {
		seq_printf(m, "%*sVector: Not assigned\n", ind, "");
		return;
	}

	raw_spin_lock_irqsave(&vector_lock, flags);
	cpu = apicd->cpu;
573
	vector = apicd->vector;
574
	prev_cpu = apicd->prev_cpu;
575
	prev_vector = apicd->prev_vector;
576
	raw_spin_unlock_irqrestore(&vector_lock, flags);
577
	seq_printf(m, "%*sVector: %5u\n", ind, "", vector);
578
	seq_printf(m, "%*sTarget: %5u\n", ind, "", cpu);
579 580
	if (prev_vector) {
		seq_printf(m, "%*sPrevious vector: %5u\n", ind, "", prev_vector);
581 582 583 584 585
		seq_printf(m, "%*sPrevious target: %5u\n", ind, "", prev_cpu);
	}
}
#endif

T
Thomas Gleixner 已提交
586
static const struct irq_domain_ops x86_vector_domain_ops = {
587 588
	.alloc		= x86_vector_alloc_irqs,
	.free		= x86_vector_free_irqs,
589 590
	.activate	= x86_vector_activate,
	.deactivate	= x86_vector_deactivate,
591 592 593
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
	.debug_show	= x86_vector_debug_show,
#endif
594 595
};

596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
int __init arch_probe_nr_irqs(void)
{
	int nr;

	if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
		nr_irqs = NR_VECTORS * nr_cpu_ids;

	nr = (gsi_top + nr_legacy_irqs()) + 8 * nr_cpu_ids;
#if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
	/*
	 * for MSI and HT dyn irq
	 */
	if (gsi_top <= NR_IRQS_LEGACY)
		nr +=  8 * nr_cpu_ids;
	else
		nr += gsi_top * 16;
#endif
	if (nr < nr_irqs)
		nr_irqs = nr;

616 617 618 619 620
	/*
	 * We don't know if PIC is present at this point so we need to do
	 * probe() to get the right number of legacy IRQs.
	 */
	return legacy_pic->probe();
621 622
}

623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652
void lapic_assign_legacy_vector(unsigned int irq, bool replace)
{
	/*
	 * Use assign system here so it wont get accounted as allocated
	 * and moveable in the cpu hotplug check and it prevents managed
	 * irq reservation from touching it.
	 */
	irq_matrix_assign_system(vector_matrix, ISA_IRQ_VECTOR(irq), replace);
}

void __init lapic_assign_system_vectors(void)
{
	unsigned int i, vector = 0;

	for_each_set_bit_from(vector, system_vectors, NR_VECTORS)
		irq_matrix_assign_system(vector_matrix, vector, false);

	if (nr_legacy_irqs() > 1)
		lapic_assign_legacy_vector(PIC_CASCADE_IR, false);

	/* System vectors are reserved, online it */
	irq_matrix_online(vector_matrix);

	/* Mark the preallocated legacy interrupts */
	for (i = 0; i < nr_legacy_irqs(); i++) {
		if (i != PIC_CASCADE_IR)
			irq_matrix_assign(vector_matrix, ISA_IRQ_VECTOR(i));
	}
}

653 654
int __init arch_early_irq_init(void)
{
655 656 657 658 659 660
	struct fwnode_handle *fn;

	fn = irq_domain_alloc_named_fwnode("VECTOR");
	BUG_ON(!fn);
	x86_vector_domain = irq_domain_create_tree(fn, &x86_vector_domain_ops,
						   NULL);
661
	BUG_ON(x86_vector_domain == NULL);
662
	irq_domain_free_fwnode(fn);
663 664
	irq_set_default_host(x86_vector_domain);

665
	arch_init_msi_domain(x86_vector_domain);
666
	arch_init_htirq_domain(x86_vector_domain);
667

668
	BUG_ON(!alloc_cpumask_var(&vector_searchmask, GFP_KERNEL));
669

670 671 672 673 674 675 676 677
	/*
	 * Allocate the vector matrix allocator data structure and limit the
	 * search area.
	 */
	vector_matrix = irq_alloc_matrix(NR_VECTORS, FIRST_EXTERNAL_VECTOR,
					 FIRST_SYSTEM_VECTOR);
	BUG_ON(!vector_matrix);

678 679 680
	return arch_early_ioapic_init();
}

681
#ifdef CONFIG_SMP
682

683 684 685 686 687 688 689 690 691 692 693 694 695
static struct irq_desc *__setup_vector_irq(int vector)
{
	int isairq = vector - ISA_IRQ_VECTOR(0);

	/* Check whether the irq is in the legacy space */
	if (isairq < 0 || isairq >= nr_legacy_irqs())
		return VECTOR_UNUSED;
	/* Check whether the irq is handled by the IOAPIC */
	if (test_bit(isairq, &io_apic_irqs))
		return VECTOR_UNUSED;
	return irq_to_desc(isairq);
}

696 697
/* Online the local APIC infrastructure and initialize the vectors */
void lapic_online(void)
698
{
699
	unsigned int vector;
700

701
	lockdep_assert_held(&vector_lock);
702 703 704 705

	/* Online the vector matrix array for this CPU */
	irq_matrix_online(vector_matrix);

706
	/*
707 708 709 710 711 712 713
	 * The interrupt affinity logic never targets interrupts to offline
	 * CPUs. The exception are the legacy PIC interrupts. In general
	 * they are only targeted to CPU0, but depending on the platform
	 * they can be distributed to any online CPU in hardware. The
	 * kernel has no influence on that. So all active legacy vectors
	 * must be installed on all CPUs. All non legacy interrupts can be
	 * cleared.
714
	 */
715 716
	for (vector = 0; vector < NR_VECTORS; vector++)
		this_cpu_write(vector_irq[vector], __setup_vector_irq(vector));
717 718
}

719 720 721 722 723 724 725
void lapic_offline(void)
{
	lock_vector_lock();
	irq_matrix_offline(vector_matrix);
	unlock_vector_lock();
}

726 727 728 729 730
static int apic_set_affinity(struct irq_data *irqd,
			     const struct cpumask *dest, bool force)
{
	int err;

731 732 733 734 735 736 737
	raw_spin_lock(&vector_lock);
	cpumask_and(vector_searchmask, dest, cpu_online_mask);
	if (irqd_affinity_is_managed(irqd))
		err = assign_managed_vector(irqd, vector_searchmask);
	else
		err = assign_vector_locked(irqd, vector_searchmask);
	raw_spin_unlock(&vector_lock);
738 739 740 741 742 743 744
	return err ? err : IRQ_SET_MASK_OK;
}

#else
# define apic_set_affinity	NULL
#endif

745
static int apic_retrigger_irq(struct irq_data *irqd)
746
{
747
	struct apic_chip_data *apicd = apic_chip_data(irqd);
748 749 750
	unsigned long flags;

	raw_spin_lock_irqsave(&vector_lock, flags);
751
	apic->send_IPI(apicd->cpu, apicd->vector);
752 753 754 755 756
	raw_spin_unlock_irqrestore(&vector_lock, flags);

	return 1;
}

757
void apic_ack_edge(struct irq_data *irqd)
758
{
759 760
	irq_complete_move(irqd_cfg(irqd));
	irq_move_irq(irqd);
761 762 763
	ack_APIC_irq();
}

764
static struct irq_chip lapic_controller = {
T
Thomas Gleixner 已提交
765
	.name			= "APIC",
766
	.irq_ack		= apic_ack_edge,
767
	.irq_set_affinity	= apic_set_affinity,
768 769 770
	.irq_retrigger		= apic_retrigger_irq,
};

771
#ifdef CONFIG_SMP
772

773 774
static void free_moved_vector(struct apic_chip_data *apicd)
{
775
	unsigned int vector = apicd->prev_vector;
776
	unsigned int cpu = apicd->prev_cpu;
777 778 779 780 781 782 783 784 785
	bool managed = apicd->is_managed;

	/*
	 * This should never happen. Managed interrupts are not
	 * migrated except on CPU down, which does not involve the
	 * cleanup vector. But try to keep the accounting correct
	 * nevertheless.
	 */
	WARN_ON_ONCE(managed);
786

787 788
	trace_vector_free_moved(apicd->irq, vector, managed);
	irq_matrix_free(vector_matrix, cpu, vector, managed);
789 790
	__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
	hlist_del_init(&apicd->clist);
791
	apicd->prev_vector = 0;
792 793 794
	apicd->move_in_progress = 0;
}

795
asmlinkage __visible void __irq_entry smp_irq_move_cleanup_interrupt(void)
796
{
797 798 799
	struct hlist_head *clhead = this_cpu_ptr(&cleanup_list);
	struct apic_chip_data *apicd;
	struct hlist_node *tmp;
800

801
	entering_ack_irq();
802 803 804
	/* Prevent vectors vanishing under us */
	raw_spin_lock(&vector_lock);

805
	hlist_for_each_entry_safe(apicd, tmp, clhead, clist) {
806
		unsigned int irr, vector = apicd->prev_vector;
807 808

		/*
809 810 811
		 * Paranoia: Check if the vector that needs to be cleaned
		 * up is registered at the APICs IRR. If so, then this is
		 * not the best time to clean it up. Clean it up in the
812
		 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
813 814 815
		 * to this CPU. IRQ_MOVE_CLEANUP_VECTOR is the lowest
		 * priority external vector, so on return from this
		 * interrupt the device interrupt will happen first.
816
		 */
817 818
		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		if (irr & (1U << (vector % 32))) {
819
			apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
820
			continue;
821
		}
822
		free_moved_vector(apicd);
823 824
	}

825
	raw_spin_unlock(&vector_lock);
826
	exiting_irq();
827 828
}

829 830 831 832 833 834 835 836 837 838 839
static void __send_cleanup_vector(struct apic_chip_data *apicd)
{
	unsigned int cpu;

	raw_spin_lock(&vector_lock);
	apicd->move_in_progress = 0;
	cpu = apicd->prev_cpu;
	if (cpu_online(cpu)) {
		hlist_add_head(&apicd->clist, per_cpu_ptr(&cleanup_list, cpu));
		apic->send_IPI(cpu, IRQ_MOVE_CLEANUP_VECTOR);
	} else {
840
		apicd->prev_vector = 0;
841 842 843 844 845 846 847 848
	}
	raw_spin_unlock(&vector_lock);
}

void send_cleanup_vector(struct irq_cfg *cfg)
{
	struct apic_chip_data *apicd;

849
	apicd = container_of(cfg, struct apic_chip_data, hw_irq_cfg);
850 851 852 853
	if (apicd->move_in_progress)
		__send_cleanup_vector(apicd);
}

854 855
static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
{
856
	struct apic_chip_data *apicd;
857

858
	apicd = container_of(cfg, struct apic_chip_data, hw_irq_cfg);
859
	if (likely(!apicd->move_in_progress))
860 861
		return;

862
	if (vector == apicd->vector && apicd->cpu == smp_processor_id())
863
		__send_cleanup_vector(apicd);
864 865 866 867 868 869 870
}

void irq_complete_move(struct irq_cfg *cfg)
{
	__irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
}

871
/*
872
 * Called from fixup_irqs() with @desc->lock held and interrupts disabled.
873 874
 */
void irq_force_complete_move(struct irq_desc *desc)
875
{
876
	struct apic_chip_data *apicd;
877 878
	struct irq_data *irqd;
	unsigned int vector;
879

880 881 882 883 884 885 886 887 888
	/*
	 * The function is called for all descriptors regardless of which
	 * irqdomain they belong to. For example if an IRQ is provided by
	 * an irq_chip as part of a GPIO driver, the chip data for that
	 * descriptor is specific to the irq_chip in question.
	 *
	 * Check first that the chip_data is what we expect
	 * (apic_chip_data) before touching it any further.
	 */
889
	irqd = irq_domain_get_irq_data(x86_vector_domain,
890
				       irq_desc_get_irq(desc));
891
	if (!irqd)
892 893
		return;

894
	raw_spin_lock(&vector_lock);
895
	apicd = apic_chip_data(irqd);
896 897
	if (!apicd)
		goto unlock;
898

899
	/*
900
	 * If prev_vector is empty, no action required.
901
	 */
902
	vector = apicd->prev_vector;
903 904
	if (!vector)
		goto unlock;
905

906
	/*
907
	 * This is tricky. If the cleanup of the old vector has not been
908 909 910
	 * done yet, then the following setaffinity call will fail with
	 * -EBUSY. This can leave the interrupt in a stale state.
	 *
911 912
	 * All CPUs are stuck in stop machine with interrupts disabled so
	 * calling __irq_complete_move() would be completely pointless.
913
	 *
914 915 916 917 918 919 920
	 * 1) The interrupt is in move_in_progress state. That means that we
	 *    have not seen an interrupt since the io_apic was reprogrammed to
	 *    the new vector.
	 *
	 * 2) The interrupt has fired on the new vector, but the cleanup IPIs
	 *    have not been processed yet.
	 */
921
	if (apicd->move_in_progress) {
922
		/*
923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952
		 * In theory there is a race:
		 *
		 * set_ioapic(new_vector) <-- Interrupt is raised before update
		 *			      is effective, i.e. it's raised on
		 *			      the old vector.
		 *
		 * So if the target cpu cannot handle that interrupt before
		 * the old vector is cleaned up, we get a spurious interrupt
		 * and in the worst case the ioapic irq line becomes stale.
		 *
		 * But in case of cpu hotplug this should be a non issue
		 * because if the affinity update happens right before all
		 * cpus rendevouz in stop machine, there is no way that the
		 * interrupt can be blocked on the target cpu because all cpus
		 * loops first with interrupts enabled in stop machine, so the
		 * old vector is not yet cleaned up when the interrupt fires.
		 *
		 * So the only way to run into this issue is if the delivery
		 * of the interrupt on the apic/system bus would be delayed
		 * beyond the point where the target cpu disables interrupts
		 * in stop machine. I doubt that it can happen, but at least
		 * there is a theroretical chance. Virtualization might be
		 * able to expose this, but AFAICT the IOAPIC emulation is not
		 * as stupid as the real hardware.
		 *
		 * Anyway, there is nothing we can do about that at this point
		 * w/o refactoring the whole fixup_irq() business completely.
		 * We print at least the irq number and the old vector number,
		 * so we have the necessary information when a problem in that
		 * area arises.
953
		 */
954
		pr_warn("IRQ fixup: irq %d move in progress, old vector %d\n",
955
			irqd->irq, vector);
956
	}
957
	free_moved_vector(apicd);
958
unlock:
959
	raw_spin_unlock(&vector_lock);
960
}
961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991

#ifdef CONFIG_HOTPLUG_CPU
/*
 * Note, this is not accurate accounting, but at least good enough to
 * prevent that the actual interrupt move will run out of vectors.
 */
int lapic_can_unplug_cpu(void)
{
	unsigned int rsvd, avl, tomove, cpu = smp_processor_id();
	int ret = 0;

	raw_spin_lock(&vector_lock);
	tomove = irq_matrix_allocated(vector_matrix);
	avl = irq_matrix_available(vector_matrix, true);
	if (avl < tomove) {
		pr_warn("CPU %u has %u vectors, %u available. Cannot disable CPU\n",
			cpu, tomove, avl);
		ret = -ENOSPC;
		goto out;
	}
	rsvd = irq_matrix_reserved(vector_matrix);
	if (avl < rsvd) {
		pr_warn("Reserved vectors %u > available %u. IRQ request may fail\n",
			rsvd, avl);
	}
out:
	raw_spin_unlock(&vector_lock);
	return ret;
}
#endif /* HOTPLUG_CPU */
#endif /* SMP */
992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009

static void __init print_APIC_field(int base)
{
	int i;

	printk(KERN_DEBUG);

	for (i = 0; i < 8; i++)
		pr_cont("%08x", apic_read(base + i*0x10));

	pr_cont("\n");
}

static void __init print_local_APIC(void *dummy)
{
	unsigned int i, v, ver, maxlvt;
	u64 icr;

1010 1011
	pr_debug("printing local APIC contents on CPU#%d/%d:\n",
		 smp_processor_id(), hard_smp_processor_id());
1012
	v = apic_read(APIC_ID);
1013
	pr_info("... APIC ID:      %08x (%01x)\n", v, read_apic_id());
1014
	v = apic_read(APIC_LVR);
1015
	pr_info("... APIC VERSION: %08x\n", v);
1016 1017 1018 1019
	ver = GET_APIC_VERSION(v);
	maxlvt = lapic_get_maxlvt();

	v = apic_read(APIC_TASKPRI);
1020
	pr_debug("... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1021 1022 1023 1024 1025

	/* !82489DX */
	if (APIC_INTEGRATED(ver)) {
		if (!APIC_XAPIC(ver)) {
			v = apic_read(APIC_ARBPRI);
1026 1027
			pr_debug("... APIC ARBPRI: %08x (%02x)\n",
				 v, v & APIC_ARBPRI_MASK);
1028 1029
		}
		v = apic_read(APIC_PROCPRI);
1030
		pr_debug("... APIC PROCPRI: %08x\n", v);
1031 1032 1033 1034 1035 1036 1037 1038
	}

	/*
	 * Remote read supported only in the 82489DX and local APIC for
	 * Pentium processors.
	 */
	if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
		v = apic_read(APIC_RRR);
1039
		pr_debug("... APIC RRR: %08x\n", v);
1040 1041 1042
	}

	v = apic_read(APIC_LDR);
1043
	pr_debug("... APIC LDR: %08x\n", v);
1044 1045
	if (!x2apic_enabled()) {
		v = apic_read(APIC_DFR);
1046
		pr_debug("... APIC DFR: %08x\n", v);
1047 1048
	}
	v = apic_read(APIC_SPIV);
1049
	pr_debug("... APIC SPIV: %08x\n", v);
1050

1051
	pr_debug("... APIC ISR field:\n");
1052
	print_APIC_field(APIC_ISR);
1053
	pr_debug("... APIC TMR field:\n");
1054
	print_APIC_field(APIC_TMR);
1055
	pr_debug("... APIC IRR field:\n");
1056 1057 1058 1059 1060 1061 1062 1063 1064
	print_APIC_field(APIC_IRR);

	/* !82489DX */
	if (APIC_INTEGRATED(ver)) {
		/* Due to the Pentium erratum 3AP. */
		if (maxlvt > 3)
			apic_write(APIC_ESR, 0);

		v = apic_read(APIC_ESR);
1065
		pr_debug("... APIC ESR: %08x\n", v);
1066 1067 1068
	}

	icr = apic_icr_read();
1069 1070
	pr_debug("... APIC ICR: %08x\n", (u32)icr);
	pr_debug("... APIC ICR2: %08x\n", (u32)(icr >> 32));
1071 1072

	v = apic_read(APIC_LVTT);
1073
	pr_debug("... APIC LVTT: %08x\n", v);
1074 1075 1076 1077

	if (maxlvt > 3) {
		/* PC is LVT#4. */
		v = apic_read(APIC_LVTPC);
1078
		pr_debug("... APIC LVTPC: %08x\n", v);
1079 1080
	}
	v = apic_read(APIC_LVT0);
1081
	pr_debug("... APIC LVT0: %08x\n", v);
1082
	v = apic_read(APIC_LVT1);
1083
	pr_debug("... APIC LVT1: %08x\n", v);
1084 1085 1086 1087

	if (maxlvt > 2) {
		/* ERR is LVT#3. */
		v = apic_read(APIC_LVTERR);
1088
		pr_debug("... APIC LVTERR: %08x\n", v);
1089 1090 1091
	}

	v = apic_read(APIC_TMICT);
1092
	pr_debug("... APIC TMICT: %08x\n", v);
1093
	v = apic_read(APIC_TMCCT);
1094
	pr_debug("... APIC TMCCT: %08x\n", v);
1095
	v = apic_read(APIC_TDCR);
1096
	pr_debug("... APIC TDCR: %08x\n", v);
1097 1098 1099 1100

	if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
		v = apic_read(APIC_EFEAT);
		maxlvt = (v >> 16) & 0xff;
1101
		pr_debug("... APIC EFEAT: %08x\n", v);
1102
		v = apic_read(APIC_ECTRL);
1103
		pr_debug("... APIC ECTRL: %08x\n", v);
1104 1105
		for (i = 0; i < maxlvt; i++) {
			v = apic_read(APIC_EILVTn(i));
1106
			pr_debug("... APIC EILVT%d: %08x\n", i, v);
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
		}
	}
	pr_cont("\n");
}

static void __init print_local_APICs(int maxcpu)
{
	int cpu;

	if (!maxcpu)
		return;

	preempt_disable();
	for_each_online_cpu(cpu) {
		if (cpu >= maxcpu)
			break;
		smp_call_function_single(cpu, print_local_APIC, NULL, 1);
	}
	preempt_enable();
}

static void __init print_PIC(void)
{
	unsigned int v;
	unsigned long flags;

	if (!nr_legacy_irqs())
		return;

1136
	pr_debug("\nprinting PIC contents\n");
1137 1138 1139 1140

	raw_spin_lock_irqsave(&i8259A_lock, flags);

	v = inb(0xa1) << 8 | inb(0x21);
1141
	pr_debug("... PIC  IMR: %04x\n", v);
1142 1143

	v = inb(0xa0) << 8 | inb(0x20);
1144
	pr_debug("... PIC  IRR: %04x\n", v);
1145 1146 1147 1148 1149 1150 1151 1152 1153

	outb(0x0b, 0xa0);
	outb(0x0b, 0x20);
	v = inb(0xa0) << 8 | inb(0x20);
	outb(0x0a, 0xa0);
	outb(0x0a, 0x20);

	raw_spin_unlock_irqrestore(&i8259A_lock, flags);

1154
	pr_debug("... PIC  ISR: %04x\n", v);
1155 1156

	v = inb(0x4d1) << 8 | inb(0x4d0);
1157
	pr_debug("... PIC ELCR: %04x\n", v);
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184
}

static int show_lapic __initdata = 1;
static __init int setup_show_lapic(char *arg)
{
	int num = -1;

	if (strcmp(arg, "all") == 0) {
		show_lapic = CONFIG_NR_CPUS;
	} else {
		get_option(&arg, &num);
		if (num >= 0)
			show_lapic = num;
	}

	return 1;
}
__setup("show_lapic=", setup_show_lapic);

static int __init print_ICs(void)
{
	if (apic_verbosity == APIC_QUIET)
		return 0;

	print_PIC();

	/* don't print out if apic is not there */
1185
	if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1186 1187 1188 1189 1190 1191 1192 1193 1194
		return 0;

	print_local_APICs(show_lapic);
	print_IO_APICs();

	return 0;
}

late_initcall(print_ICs);