vector.c 31.6 KB
Newer Older
1
/*
2
 * Local APIC related interfaces to support IOAPIC, MSI, etc.
3 4 5
 *
 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
 *	Moved from arch/x86/kernel/apic/io_apic.c.
6 7
 * Jiang Liu <jiang.liu@linux.intel.com>
 *	Enable support of hierarchical irqdomains
8 9 10 11 12 13
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/interrupt.h>
14
#include <linux/seq_file.h>
15 16 17
#include <linux/init.h>
#include <linux/compiler.h>
#include <linux/slab.h>
18
#include <asm/irqdomain.h>
19 20 21 22 23 24
#include <asm/hw_irq.h>
#include <asm/apic.h>
#include <asm/i8259.h>
#include <asm/desc.h>
#include <asm/irq_remapping.h>

25 26
#include <asm/trace/irq_vectors.h>

27
struct apic_chip_data {
28 29 30
	struct irq_cfg		hw_irq_cfg;
	unsigned int		vector;
	unsigned int		prev_vector;
31 32
	unsigned int		cpu;
	unsigned int		prev_cpu;
33
	unsigned int		irq;
34
	struct hlist_node	clist;
35
	unsigned int		move_in_progress	: 1,
36 37 38
				is_managed		: 1,
				can_reserve		: 1,
				has_reserved		: 1;
39 40
};

41
struct irq_domain *x86_vector_domain;
42
EXPORT_SYMBOL_GPL(x86_vector_domain);
43
static DEFINE_RAW_SPINLOCK(vector_lock);
44
static cpumask_var_t vector_searchmask;
45
static struct irq_chip lapic_controller;
46
static struct irq_matrix *vector_matrix;
47 48 49
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct hlist_head, cleanup_list);
#endif
50 51 52 53 54 55 56 57 58 59 60 61 62 63

void lock_vector_lock(void)
{
	/* Used to the online set of cpus does not change
	 * during assign_irq_vector.
	 */
	raw_spin_lock(&vector_lock);
}

void unlock_vector_lock(void)
{
	raw_spin_unlock(&vector_lock);
}

64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
void init_irq_alloc_info(struct irq_alloc_info *info,
			 const struct cpumask *mask)
{
	memset(info, 0, sizeof(*info));
	info->mask = mask;
}

void copy_irq_alloc_info(struct irq_alloc_info *dst, struct irq_alloc_info *src)
{
	if (src)
		*dst = *src;
	else
		memset(dst, 0, sizeof(*dst));
}

79
static struct apic_chip_data *apic_chip_data(struct irq_data *irqd)
80
{
81
	if (!irqd)
82 83
		return NULL;

84 85
	while (irqd->parent_data)
		irqd = irqd->parent_data;
86

87
	return irqd->chip_data;
88 89
}

90
struct irq_cfg *irqd_cfg(struct irq_data *irqd)
91
{
92
	struct apic_chip_data *apicd = apic_chip_data(irqd);
93

94
	return apicd ? &apicd->hw_irq_cfg : NULL;
95
}
96
EXPORT_SYMBOL_GPL(irqd_cfg);
97 98

struct irq_cfg *irq_cfg(unsigned int irq)
99
{
100 101
	return irqd_cfg(irq_get_irq_data(irq));
}
102

103 104
static struct apic_chip_data *alloc_apic_chip_data(int node)
{
105
	struct apic_chip_data *apicd;
106

107
	apicd = kzalloc_node(sizeof(*apicd), GFP_KERNEL, node);
108 109
	if (apicd)
		INIT_HLIST_NODE(&apicd->clist);
110
	return apicd;
111 112
}

113
static void free_apic_chip_data(struct apic_chip_data *apicd)
114
{
115
	kfree(apicd);
116 117
}

118 119
static void apic_update_irq_cfg(struct irq_data *irqd, unsigned int vector,
				unsigned int cpu)
120
{
121
	struct apic_chip_data *apicd = apic_chip_data(irqd);
122

123
	lockdep_assert_held(&vector_lock);
124

125 126 127 128 129
	apicd->hw_irq_cfg.vector = vector;
	apicd->hw_irq_cfg.dest_apicid = apic->calc_dest_apicid(cpu);
	irq_data_update_effective_affinity(irqd, cpumask_of(cpu));
	trace_vector_config(irqd->irq, vector, cpu,
			    apicd->hw_irq_cfg.dest_apicid);
130
}
131

132 133 134 135 136
static void apic_update_vector(struct irq_data *irqd, unsigned int newvec,
			       unsigned int newcpu)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	struct irq_desc *desc = irq_data_to_desc(irqd);
137

138
	lockdep_assert_held(&vector_lock);
139

140
	trace_vector_update(irqd->irq, newvec, newcpu, apicd->vector,
141
			    apicd->cpu);
142

143
	/* Setup the vector move, if required  */
144
	if (apicd->vector && cpu_online(apicd->cpu)) {
145
		apicd->move_in_progress = true;
146
		apicd->prev_vector = apicd->vector;
147 148
		apicd->prev_cpu = apicd->cpu;
	} else {
149
		apicd->prev_vector = 0;
150
	}
151

152
	apicd->vector = newvec;
153 154 155 156
	apicd->cpu = newcpu;
	BUG_ON(!IS_ERR_OR_NULL(per_cpu(vector_irq, newcpu)[newvec]));
	per_cpu(vector_irq, newcpu)[newvec] = desc;
}
157

158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
static void vector_assign_managed_shutdown(struct irq_data *irqd)
{
	unsigned int cpu = cpumask_first(cpu_online_mask);

	apic_update_irq_cfg(irqd, MANAGED_IRQ_SHUTDOWN_VECTOR, cpu);
}

static int reserve_managed_vector(struct irq_data *irqd)
{
	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	unsigned long flags;
	int ret;

	raw_spin_lock_irqsave(&vector_lock, flags);
	apicd->is_managed = true;
	ret = irq_matrix_reserve_managed(vector_matrix, affmsk);
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	trace_vector_reserve_managed(irqd->irq, ret);
	return ret;
}

180 181 182 183 184 185 186
static void reserve_irq_vector_locked(struct irq_data *irqd)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);

	irq_matrix_reserve(vector_matrix);
	apicd->can_reserve = true;
	apicd->has_reserved = true;
187
	irqd_set_can_reserve(irqd);
188 189 190 191 192 193 194 195 196 197 198 199 200 201
	trace_vector_reserve(irqd->irq, 0);
	vector_assign_managed_shutdown(irqd);
}

static int reserve_irq_vector(struct irq_data *irqd)
{
	unsigned long flags;

	raw_spin_lock_irqsave(&vector_lock, flags);
	reserve_irq_vector_locked(irqd);
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	return 0;
}

202 203 204
static int allocate_vector(struct irq_data *irqd, const struct cpumask *dest)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
205
	bool resvd = apicd->has_reserved;
206
	unsigned int cpu = apicd->cpu;
207 208 209
	int vector = apicd->vector;

	lockdep_assert_held(&vector_lock);
210

211
	/*
212 213 214
	 * If the current target CPU is online and in the new requested
	 * affinity mask, there is no point in moving the interrupt from
	 * one CPU to another.
215
	 */
216 217 218
	if (vector && cpu_online(cpu) && cpumask_test_cpu(cpu, dest))
		return 0;

219
	vector = irq_matrix_alloc(vector_matrix, dest, resvd, &cpu);
220 221
	if (vector > 0)
		apic_update_vector(irqd, vector, cpu);
222
	trace_vector_alloc(irqd->irq, vector, resvd, vector);
223 224 225 226 227 228
	return vector;
}

static int assign_vector_locked(struct irq_data *irqd,
				const struct cpumask *dest)
{
229
	struct apic_chip_data *apicd = apic_chip_data(irqd);
230 231 232 233 234
	int vector = allocate_vector(irqd, dest);

	if (vector < 0)
		return vector;

235
	apic_update_irq_cfg(irqd, apicd->vector, apicd->cpu);
236
	return 0;
237 238
}

239
static int assign_irq_vector(struct irq_data *irqd, const struct cpumask *dest)
240 241
{
	unsigned long flags;
242
	int ret;
243 244

	raw_spin_lock_irqsave(&vector_lock, flags);
245 246
	cpumask_and(vector_searchmask, dest, cpu_online_mask);
	ret = assign_vector_locked(irqd, vector_searchmask);
247
	raw_spin_unlock_irqrestore(&vector_lock, flags);
248
	return ret;
249 250
}

251 252
static int assign_irq_vector_any_locked(struct irq_data *irqd)
{
253 254
	/* Get the affinity mask - either irq_default_affinity or (user) set */
	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);
255 256
	int node = irq_data_get_node(irqd);

257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
	if (node == NUMA_NO_NODE)
		goto all;
	/* Try the intersection of @affmsk and node mask */
	cpumask_and(vector_searchmask, cpumask_of_node(node), affmsk);
	if (!assign_vector_locked(irqd, vector_searchmask))
		return 0;
	/* Try the node mask */
	if (!assign_vector_locked(irqd, cpumask_of_node(node)))
		return 0;
all:
	/* Try the full affinity mask */
	cpumask_and(vector_searchmask, affmsk, cpu_online_mask);
	if (!assign_vector_locked(irqd, vector_searchmask))
		return 0;
	/* Try the full online mask */
272 273 274 275 276
	return assign_vector_locked(irqd, cpu_online_mask);
}

static int
assign_irq_vector_policy(struct irq_data *irqd, struct irq_alloc_info *info)
277
{
278 279
	if (irqd_affinity_is_managed(irqd))
		return reserve_managed_vector(irqd);
280
	if (info->mask)
281
		return assign_irq_vector(irqd, info->mask);
282 283 284 285
	/*
	 * Make only a global reservation with no guarantee. A real vector
	 * is associated at activation time.
	 */
286
	return reserve_irq_vector(irqd);
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
}

static int
assign_managed_vector(struct irq_data *irqd, const struct cpumask *dest)
{
	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	int vector, cpu;

	cpumask_and(vector_searchmask, vector_searchmask, affmsk);
	cpu = cpumask_first(vector_searchmask);
	if (cpu >= nr_cpu_ids)
		return -EINVAL;
	/* set_affinity might call here for nothing */
	if (apicd->vector && cpumask_test_cpu(apicd->cpu, vector_searchmask))
302
		return 0;
303 304 305 306 307 308 309
	vector = irq_matrix_alloc_managed(vector_matrix, cpu);
	trace_vector_alloc_managed(irqd->irq, vector, vector);
	if (vector < 0)
		return vector;
	apic_update_vector(irqd, vector, cpu);
	apic_update_irq_cfg(irqd, vector, cpu);
	return 0;
310 311
}

312
static void clear_irq_vector(struct irq_data *irqd)
313
{
314
	struct apic_chip_data *apicd = apic_chip_data(irqd);
315
	bool managed = irqd_affinity_is_managed(irqd);
316
	unsigned int vector = apicd->vector;
317

318
	lockdep_assert_held(&vector_lock);
319

320
	if (!vector)
321
		return;
322

323
	trace_vector_clear(irqd->irq, vector, apicd->cpu, apicd->prev_vector,
324 325
			   apicd->prev_cpu);

326
	per_cpu(vector_irq, apicd->cpu)[vector] = VECTOR_UNUSED;
327
	irq_matrix_free(vector_matrix, apicd->cpu, vector, managed);
328
	apicd->vector = 0;
329

330
	/* Clean up move in progress */
331
	vector = apicd->prev_vector;
332
	if (!vector)
333 334
		return;

335
	per_cpu(vector_irq, apicd->prev_cpu)[vector] = VECTOR_UNUSED;
336
	irq_matrix_free(vector_matrix, apicd->prev_cpu, vector, managed);
337
	apicd->prev_vector = 0;
338
	apicd->move_in_progress = 0;
339
	hlist_del_init(&apicd->clist);
340 341
}

342 343 344 345 346 347
static void x86_vector_deactivate(struct irq_domain *dom, struct irq_data *irqd)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	unsigned long flags;

	trace_vector_deactivate(irqd->irq, apicd->is_managed,
348
				apicd->can_reserve, false);
349

350 351 352 353 354
	/* Regular fixed assigned interrupt */
	if (!apicd->is_managed && !apicd->can_reserve)
		return;
	/* If the interrupt has a global reservation, nothing to do */
	if (apicd->has_reserved)
355 356 357 358
		return;

	raw_spin_lock_irqsave(&vector_lock, flags);
	clear_irq_vector(irqd);
359 360 361 362
	if (apicd->can_reserve)
		reserve_irq_vector_locked(irqd);
	else
		vector_assign_managed_shutdown(irqd);
363 364 365
	raw_spin_unlock_irqrestore(&vector_lock, flags);
}

366 367 368 369 370 371
static int activate_reserved(struct irq_data *irqd)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	int ret;

	ret = assign_irq_vector_any_locked(irqd);
372
	if (!ret) {
373
		apicd->has_reserved = false;
374 375 376 377 378 379 380 381 382 383
		/*
		 * Core might have disabled reservation mode after
		 * allocating the irq descriptor. Ideally this should
		 * happen before allocation time, but that would require
		 * completely convoluted ways of transporting that
		 * information.
		 */
		if (!irqd_can_reserve(irqd))
			apicd->can_reserve = false;
	}
384 385 386
	return ret;
}

387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411
static int activate_managed(struct irq_data *irqd)
{
	const struct cpumask *dest = irq_data_get_affinity_mask(irqd);
	int ret;

	cpumask_and(vector_searchmask, dest, cpu_online_mask);
	if (WARN_ON_ONCE(cpumask_empty(vector_searchmask))) {
		/* Something in the core code broke! Survive gracefully */
		pr_err("Managed startup for irq %u, but no CPU\n", irqd->irq);
		return EINVAL;
	}

	ret = assign_managed_vector(irqd, vector_searchmask);
	/*
	 * This should not happen. The vector reservation got buggered.  Handle
	 * it gracefully.
	 */
	if (WARN_ON_ONCE(ret < 0)) {
		pr_err("Managed startup irq %u, no vector available\n",
		       irqd->irq);
	}
       return ret;
}

static int x86_vector_activate(struct irq_domain *dom, struct irq_data *irqd,
412
			       bool reserve)
413 414 415 416 417 418
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	unsigned long flags;
	int ret = 0;

	trace_vector_activate(irqd->irq, apicd->is_managed,
419
			      apicd->can_reserve, reserve);
420

421 422
	/* Nothing to do for fixed assigned vectors */
	if (!apicd->can_reserve && !apicd->is_managed)
423 424 425
		return 0;

	raw_spin_lock_irqsave(&vector_lock, flags);
426
	if (reserve || irqd_is_managed_and_shutdown(irqd))
427
		vector_assign_managed_shutdown(irqd);
428
	else if (apicd->is_managed)
429
		ret = activate_managed(irqd);
430 431
	else if (apicd->has_reserved)
		ret = activate_reserved(irqd);
432 433 434 435 436 437 438 439 440
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	return ret;
}

static void vector_free_reserved_and_managed(struct irq_data *irqd)
{
	const struct cpumask *dest = irq_data_get_affinity_mask(irqd);
	struct apic_chip_data *apicd = apic_chip_data(irqd);

441 442
	trace_vector_teardown(irqd->irq, apicd->is_managed,
			      apicd->has_reserved);
443

444 445
	if (apicd->has_reserved)
		irq_matrix_remove_reserved(vector_matrix);
446 447 448 449
	if (apicd->is_managed)
		irq_matrix_remove_managed(vector_matrix, dest);
}

450 451 452
static void x86_vector_free_irqs(struct irq_domain *domain,
				 unsigned int virq, unsigned int nr_irqs)
{
453 454
	struct apic_chip_data *apicd;
	struct irq_data *irqd;
455
	unsigned long flags;
456 457 458
	int i;

	for (i = 0; i < nr_irqs; i++) {
459 460
		irqd = irq_domain_get_irq_data(x86_vector_domain, virq + i);
		if (irqd && irqd->chip_data) {
461
			raw_spin_lock_irqsave(&vector_lock, flags);
462
			clear_irq_vector(irqd);
463
			vector_free_reserved_and_managed(irqd);
464 465
			apicd = irqd->chip_data;
			irq_domain_reset_irq_data(irqd);
466
			raw_spin_unlock_irqrestore(&vector_lock, flags);
467
			free_apic_chip_data(apicd);
468 469 470 471
		}
	}
}

472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
static bool vector_configure_legacy(unsigned int virq, struct irq_data *irqd,
				    struct apic_chip_data *apicd)
{
	unsigned long flags;
	bool realloc = false;

	apicd->vector = ISA_IRQ_VECTOR(virq);
	apicd->cpu = 0;

	raw_spin_lock_irqsave(&vector_lock, flags);
	/*
	 * If the interrupt is activated, then it must stay at this vector
	 * position. That's usually the timer interrupt (0).
	 */
	if (irqd_is_activated(irqd)) {
		trace_vector_setup(virq, true, 0);
		apic_update_irq_cfg(irqd, apicd->vector, apicd->cpu);
	} else {
		/* Release the vector */
		apicd->can_reserve = true;
492
		irqd_set_can_reserve(irqd);
493 494 495 496 497 498 499
		clear_irq_vector(irqd);
		realloc = true;
	}
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	return realloc;
}

500 501 502 503
static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq,
				 unsigned int nr_irqs, void *arg)
{
	struct irq_alloc_info *info = arg;
504 505
	struct apic_chip_data *apicd;
	struct irq_data *irqd;
506
	int i, err, node;
507 508 509 510 511 512 513 514 515

	if (disable_apic)
		return -ENXIO;

	/* Currently vector allocator can't guarantee contiguous allocations */
	if ((info->flags & X86_IRQ_ALLOC_CONTIGUOUS_VECTORS) && nr_irqs > 1)
		return -ENOSYS;

	for (i = 0; i < nr_irqs; i++) {
516 517 518
		irqd = irq_domain_get_irq_data(domain, virq + i);
		BUG_ON(!irqd);
		node = irq_data_get_node(irqd);
519 520
		WARN_ON_ONCE(irqd->chip_data);
		apicd = alloc_apic_chip_data(node);
521
		if (!apicd) {
522 523 524 525
			err = -ENOMEM;
			goto error;
		}

526
		apicd->irq = virq + i;
527 528 529 530
		irqd->chip = &lapic_controller;
		irqd->chip_data = apicd;
		irqd->hwirq = virq + i;
		irqd_set_single_target(irqd);
531
		/*
532 533 534 535 536
		 * Legacy vectors are already assigned when the IOAPIC
		 * takes them over. They stay on the same vector. This is
		 * required for check_timer() to work correctly as it might
		 * switch back to legacy mode. Only update the hardware
		 * config.
537 538
		 */
		if (info->flags & X86_IRQ_ALLOC_LEGACY) {
539 540
			if (!vector_configure_legacy(virq + i, irqd, apicd))
				continue;
541 542
		}

543
		err = assign_irq_vector_policy(irqd, info);
544
		trace_vector_setup(virq + i, false, err);
545 546 547
		if (err) {
			irqd->chip_data = NULL;
			free_apic_chip_data(apicd);
548
			goto error;
549
		}
550 551 552 553 554
	}

	return 0;

error:
555
	x86_vector_free_irqs(domain, virq, i);
556 557 558
	return err;
}

559
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
560 561
static void x86_vector_debug_show(struct seq_file *m, struct irq_domain *d,
				  struct irq_data *irqd, int ind)
562
{
563
	unsigned int cpu, vector, prev_cpu, prev_vector;
564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
	struct apic_chip_data *apicd;
	unsigned long flags;
	int irq;

	if (!irqd) {
		irq_matrix_debug_show(m, vector_matrix, ind);
		return;
	}

	irq = irqd->irq;
	if (irq < nr_legacy_irqs() && !test_bit(irq, &io_apic_irqs)) {
		seq_printf(m, "%*sVector: %5d\n", ind, "", ISA_IRQ_VECTOR(irq));
		seq_printf(m, "%*sTarget: Legacy PIC all CPUs\n", ind, "");
		return;
	}

	apicd = irqd->chip_data;
	if (!apicd) {
		seq_printf(m, "%*sVector: Not assigned\n", ind, "");
		return;
	}

	raw_spin_lock_irqsave(&vector_lock, flags);
	cpu = apicd->cpu;
588
	vector = apicd->vector;
589
	prev_cpu = apicd->prev_cpu;
590
	prev_vector = apicd->prev_vector;
591
	raw_spin_unlock_irqrestore(&vector_lock, flags);
592
	seq_printf(m, "%*sVector: %5u\n", ind, "", vector);
593
	seq_printf(m, "%*sTarget: %5u\n", ind, "", cpu);
594 595
	if (prev_vector) {
		seq_printf(m, "%*sPrevious vector: %5u\n", ind, "", prev_vector);
596 597 598 599 600
		seq_printf(m, "%*sPrevious target: %5u\n", ind, "", prev_cpu);
	}
}
#endif

T
Thomas Gleixner 已提交
601
static const struct irq_domain_ops x86_vector_domain_ops = {
602 603
	.alloc		= x86_vector_alloc_irqs,
	.free		= x86_vector_free_irqs,
604 605
	.activate	= x86_vector_activate,
	.deactivate	= x86_vector_deactivate,
606 607 608
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
	.debug_show	= x86_vector_debug_show,
#endif
609 610
};

611 612 613 614 615 616 617 618
int __init arch_probe_nr_irqs(void)
{
	int nr;

	if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
		nr_irqs = NR_VECTORS * nr_cpu_ids;

	nr = (gsi_top + nr_legacy_irqs()) + 8 * nr_cpu_ids;
619
#if defined(CONFIG_PCI_MSI)
620 621 622 623 624 625 626 627 628 629 630
	/*
	 * for MSI and HT dyn irq
	 */
	if (gsi_top <= NR_IRQS_LEGACY)
		nr +=  8 * nr_cpu_ids;
	else
		nr += gsi_top * 16;
#endif
	if (nr < nr_irqs)
		nr_irqs = nr;

631 632 633 634 635
	/*
	 * We don't know if PIC is present at this point so we need to do
	 * probe() to get the right number of legacy IRQs.
	 */
	return legacy_pic->probe();
636 637
}

638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
void lapic_assign_legacy_vector(unsigned int irq, bool replace)
{
	/*
	 * Use assign system here so it wont get accounted as allocated
	 * and moveable in the cpu hotplug check and it prevents managed
	 * irq reservation from touching it.
	 */
	irq_matrix_assign_system(vector_matrix, ISA_IRQ_VECTOR(irq), replace);
}

void __init lapic_assign_system_vectors(void)
{
	unsigned int i, vector = 0;

	for_each_set_bit_from(vector, system_vectors, NR_VECTORS)
		irq_matrix_assign_system(vector_matrix, vector, false);

	if (nr_legacy_irqs() > 1)
		lapic_assign_legacy_vector(PIC_CASCADE_IR, false);

	/* System vectors are reserved, online it */
	irq_matrix_online(vector_matrix);

	/* Mark the preallocated legacy interrupts */
	for (i = 0; i < nr_legacy_irqs(); i++) {
		if (i != PIC_CASCADE_IR)
			irq_matrix_assign(vector_matrix, ISA_IRQ_VECTOR(i));
	}
}

668 669
int __init arch_early_irq_init(void)
{
670 671 672 673 674 675
	struct fwnode_handle *fn;

	fn = irq_domain_alloc_named_fwnode("VECTOR");
	BUG_ON(!fn);
	x86_vector_domain = irq_domain_create_tree(fn, &x86_vector_domain_ops,
						   NULL);
676
	BUG_ON(x86_vector_domain == NULL);
677
	irq_domain_free_fwnode(fn);
678 679
	irq_set_default_host(x86_vector_domain);

680 681
	arch_init_msi_domain(x86_vector_domain);

682
	BUG_ON(!alloc_cpumask_var(&vector_searchmask, GFP_KERNEL));
683

684 685 686 687 688 689 690 691
	/*
	 * Allocate the vector matrix allocator data structure and limit the
	 * search area.
	 */
	vector_matrix = irq_alloc_matrix(NR_VECTORS, FIRST_EXTERNAL_VECTOR,
					 FIRST_SYSTEM_VECTOR);
	BUG_ON(!vector_matrix);

692 693 694
	return arch_early_ioapic_init();
}

695
#ifdef CONFIG_SMP
696

697 698 699 700 701 702 703 704 705 706 707 708 709
static struct irq_desc *__setup_vector_irq(int vector)
{
	int isairq = vector - ISA_IRQ_VECTOR(0);

	/* Check whether the irq is in the legacy space */
	if (isairq < 0 || isairq >= nr_legacy_irqs())
		return VECTOR_UNUSED;
	/* Check whether the irq is handled by the IOAPIC */
	if (test_bit(isairq, &io_apic_irqs))
		return VECTOR_UNUSED;
	return irq_to_desc(isairq);
}

710 711
/* Online the local APIC infrastructure and initialize the vectors */
void lapic_online(void)
712
{
713
	unsigned int vector;
714

715
	lockdep_assert_held(&vector_lock);
716 717 718 719

	/* Online the vector matrix array for this CPU */
	irq_matrix_online(vector_matrix);

720
	/*
721 722 723 724 725 726 727
	 * The interrupt affinity logic never targets interrupts to offline
	 * CPUs. The exception are the legacy PIC interrupts. In general
	 * they are only targeted to CPU0, but depending on the platform
	 * they can be distributed to any online CPU in hardware. The
	 * kernel has no influence on that. So all active legacy vectors
	 * must be installed on all CPUs. All non legacy interrupts can be
	 * cleared.
728
	 */
729 730
	for (vector = 0; vector < NR_VECTORS; vector++)
		this_cpu_write(vector_irq[vector], __setup_vector_irq(vector));
731 732
}

733 734 735 736 737 738 739
void lapic_offline(void)
{
	lock_vector_lock();
	irq_matrix_offline(vector_matrix);
	unlock_vector_lock();
}

740 741 742
static int apic_set_affinity(struct irq_data *irqd,
			     const struct cpumask *dest, bool force)
{
743
	struct apic_chip_data *apicd = apic_chip_data(irqd);
744 745
	int err;

746 747 748 749 750 751 752 753 754 755 756 757
	/*
	 * Core code can call here for inactive interrupts. For inactive
	 * interrupts which use managed or reservation mode there is no
	 * point in going through the vector assignment right now as the
	 * activation will assign a vector which fits the destination
	 * cpumask. Let the core code store the destination mask and be
	 * done with it.
	 */
	if (!irqd_is_activated(irqd) &&
	    (apicd->is_managed || apicd->can_reserve))
		return IRQ_SET_MASK_OK;

758 759 760 761 762 763 764
	raw_spin_lock(&vector_lock);
	cpumask_and(vector_searchmask, dest, cpu_online_mask);
	if (irqd_affinity_is_managed(irqd))
		err = assign_managed_vector(irqd, vector_searchmask);
	else
		err = assign_vector_locked(irqd, vector_searchmask);
	raw_spin_unlock(&vector_lock);
765 766 767 768 769 770 771
	return err ? err : IRQ_SET_MASK_OK;
}

#else
# define apic_set_affinity	NULL
#endif

772
static int apic_retrigger_irq(struct irq_data *irqd)
773
{
774
	struct apic_chip_data *apicd = apic_chip_data(irqd);
775 776 777
	unsigned long flags;

	raw_spin_lock_irqsave(&vector_lock, flags);
778
	apic->send_IPI(apicd->cpu, apicd->vector);
779 780 781 782 783
	raw_spin_unlock_irqrestore(&vector_lock, flags);

	return 1;
}

784
void apic_ack_edge(struct irq_data *irqd)
785
{
786 787
	irq_complete_move(irqd_cfg(irqd));
	irq_move_irq(irqd);
788 789 790
	ack_APIC_irq();
}

791
static struct irq_chip lapic_controller = {
T
Thomas Gleixner 已提交
792
	.name			= "APIC",
793
	.irq_ack		= apic_ack_edge,
794
	.irq_set_affinity	= apic_set_affinity,
795 796 797
	.irq_retrigger		= apic_retrigger_irq,
};

798
#ifdef CONFIG_SMP
799

800 801
static void free_moved_vector(struct apic_chip_data *apicd)
{
802
	unsigned int vector = apicd->prev_vector;
803
	unsigned int cpu = apicd->prev_cpu;
804 805 806 807 808 809 810 811 812
	bool managed = apicd->is_managed;

	/*
	 * This should never happen. Managed interrupts are not
	 * migrated except on CPU down, which does not involve the
	 * cleanup vector. But try to keep the accounting correct
	 * nevertheless.
	 */
	WARN_ON_ONCE(managed);
813

814
	trace_vector_free_moved(apicd->irq, cpu, vector, managed);
815
	irq_matrix_free(vector_matrix, cpu, vector, managed);
816
	per_cpu(vector_irq, cpu)[vector] = VECTOR_UNUSED;
817
	hlist_del_init(&apicd->clist);
818
	apicd->prev_vector = 0;
819 820 821
	apicd->move_in_progress = 0;
}

822
asmlinkage __visible void __irq_entry smp_irq_move_cleanup_interrupt(void)
823
{
824 825 826
	struct hlist_head *clhead = this_cpu_ptr(&cleanup_list);
	struct apic_chip_data *apicd;
	struct hlist_node *tmp;
827

828
	entering_ack_irq();
829 830 831
	/* Prevent vectors vanishing under us */
	raw_spin_lock(&vector_lock);

832
	hlist_for_each_entry_safe(apicd, tmp, clhead, clist) {
833
		unsigned int irr, vector = apicd->prev_vector;
834 835

		/*
836 837 838
		 * Paranoia: Check if the vector that needs to be cleaned
		 * up is registered at the APICs IRR. If so, then this is
		 * not the best time to clean it up. Clean it up in the
839
		 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
840 841 842
		 * to this CPU. IRQ_MOVE_CLEANUP_VECTOR is the lowest
		 * priority external vector, so on return from this
		 * interrupt the device interrupt will happen first.
843
		 */
844 845
		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		if (irr & (1U << (vector % 32))) {
846
			apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
847
			continue;
848
		}
849
		free_moved_vector(apicd);
850 851
	}

852
	raw_spin_unlock(&vector_lock);
853
	exiting_irq();
854 855
}

856 857 858 859 860 861 862 863 864 865 866
static void __send_cleanup_vector(struct apic_chip_data *apicd)
{
	unsigned int cpu;

	raw_spin_lock(&vector_lock);
	apicd->move_in_progress = 0;
	cpu = apicd->prev_cpu;
	if (cpu_online(cpu)) {
		hlist_add_head(&apicd->clist, per_cpu_ptr(&cleanup_list, cpu));
		apic->send_IPI(cpu, IRQ_MOVE_CLEANUP_VECTOR);
	} else {
867
		apicd->prev_vector = 0;
868 869 870 871 872 873 874 875
	}
	raw_spin_unlock(&vector_lock);
}

void send_cleanup_vector(struct irq_cfg *cfg)
{
	struct apic_chip_data *apicd;

876
	apicd = container_of(cfg, struct apic_chip_data, hw_irq_cfg);
877 878 879 880
	if (apicd->move_in_progress)
		__send_cleanup_vector(apicd);
}

881 882
static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
{
883
	struct apic_chip_data *apicd;
884

885
	apicd = container_of(cfg, struct apic_chip_data, hw_irq_cfg);
886
	if (likely(!apicd->move_in_progress))
887 888
		return;

889
	if (vector == apicd->vector && apicd->cpu == smp_processor_id())
890
		__send_cleanup_vector(apicd);
891 892 893 894 895 896 897
}

void irq_complete_move(struct irq_cfg *cfg)
{
	__irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
}

898
/*
899
 * Called from fixup_irqs() with @desc->lock held and interrupts disabled.
900 901
 */
void irq_force_complete_move(struct irq_desc *desc)
902
{
903
	struct apic_chip_data *apicd;
904 905
	struct irq_data *irqd;
	unsigned int vector;
906

907 908 909 910 911 912 913 914 915
	/*
	 * The function is called for all descriptors regardless of which
	 * irqdomain they belong to. For example if an IRQ is provided by
	 * an irq_chip as part of a GPIO driver, the chip data for that
	 * descriptor is specific to the irq_chip in question.
	 *
	 * Check first that the chip_data is what we expect
	 * (apic_chip_data) before touching it any further.
	 */
916
	irqd = irq_domain_get_irq_data(x86_vector_domain,
917
				       irq_desc_get_irq(desc));
918
	if (!irqd)
919 920
		return;

921
	raw_spin_lock(&vector_lock);
922
	apicd = apic_chip_data(irqd);
923 924
	if (!apicd)
		goto unlock;
925

926
	/*
927
	 * If prev_vector is empty, no action required.
928
	 */
929
	vector = apicd->prev_vector;
930 931
	if (!vector)
		goto unlock;
932

933
	/*
934
	 * This is tricky. If the cleanup of the old vector has not been
935 936 937
	 * done yet, then the following setaffinity call will fail with
	 * -EBUSY. This can leave the interrupt in a stale state.
	 *
938 939
	 * All CPUs are stuck in stop machine with interrupts disabled so
	 * calling __irq_complete_move() would be completely pointless.
940
	 *
941 942 943 944 945 946 947
	 * 1) The interrupt is in move_in_progress state. That means that we
	 *    have not seen an interrupt since the io_apic was reprogrammed to
	 *    the new vector.
	 *
	 * 2) The interrupt has fired on the new vector, but the cleanup IPIs
	 *    have not been processed yet.
	 */
948
	if (apicd->move_in_progress) {
949
		/*
950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979
		 * In theory there is a race:
		 *
		 * set_ioapic(new_vector) <-- Interrupt is raised before update
		 *			      is effective, i.e. it's raised on
		 *			      the old vector.
		 *
		 * So if the target cpu cannot handle that interrupt before
		 * the old vector is cleaned up, we get a spurious interrupt
		 * and in the worst case the ioapic irq line becomes stale.
		 *
		 * But in case of cpu hotplug this should be a non issue
		 * because if the affinity update happens right before all
		 * cpus rendevouz in stop machine, there is no way that the
		 * interrupt can be blocked on the target cpu because all cpus
		 * loops first with interrupts enabled in stop machine, so the
		 * old vector is not yet cleaned up when the interrupt fires.
		 *
		 * So the only way to run into this issue is if the delivery
		 * of the interrupt on the apic/system bus would be delayed
		 * beyond the point where the target cpu disables interrupts
		 * in stop machine. I doubt that it can happen, but at least
		 * there is a theroretical chance. Virtualization might be
		 * able to expose this, but AFAICT the IOAPIC emulation is not
		 * as stupid as the real hardware.
		 *
		 * Anyway, there is nothing we can do about that at this point
		 * w/o refactoring the whole fixup_irq() business completely.
		 * We print at least the irq number and the old vector number,
		 * so we have the necessary information when a problem in that
		 * area arises.
980
		 */
981
		pr_warn("IRQ fixup: irq %d move in progress, old vector %d\n",
982
			irqd->irq, vector);
983
	}
984
	free_moved_vector(apicd);
985
unlock:
986
	raw_spin_unlock(&vector_lock);
987
}
988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018

#ifdef CONFIG_HOTPLUG_CPU
/*
 * Note, this is not accurate accounting, but at least good enough to
 * prevent that the actual interrupt move will run out of vectors.
 */
int lapic_can_unplug_cpu(void)
{
	unsigned int rsvd, avl, tomove, cpu = smp_processor_id();
	int ret = 0;

	raw_spin_lock(&vector_lock);
	tomove = irq_matrix_allocated(vector_matrix);
	avl = irq_matrix_available(vector_matrix, true);
	if (avl < tomove) {
		pr_warn("CPU %u has %u vectors, %u available. Cannot disable CPU\n",
			cpu, tomove, avl);
		ret = -ENOSPC;
		goto out;
	}
	rsvd = irq_matrix_reserved(vector_matrix);
	if (avl < rsvd) {
		pr_warn("Reserved vectors %u > available %u. IRQ request may fail\n",
			rsvd, avl);
	}
out:
	raw_spin_unlock(&vector_lock);
	return ret;
}
#endif /* HOTPLUG_CPU */
#endif /* SMP */
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036

static void __init print_APIC_field(int base)
{
	int i;

	printk(KERN_DEBUG);

	for (i = 0; i < 8; i++)
		pr_cont("%08x", apic_read(base + i*0x10));

	pr_cont("\n");
}

static void __init print_local_APIC(void *dummy)
{
	unsigned int i, v, ver, maxlvt;
	u64 icr;

1037 1038
	pr_debug("printing local APIC contents on CPU#%d/%d:\n",
		 smp_processor_id(), hard_smp_processor_id());
1039
	v = apic_read(APIC_ID);
1040
	pr_info("... APIC ID:      %08x (%01x)\n", v, read_apic_id());
1041
	v = apic_read(APIC_LVR);
1042
	pr_info("... APIC VERSION: %08x\n", v);
1043 1044 1045 1046
	ver = GET_APIC_VERSION(v);
	maxlvt = lapic_get_maxlvt();

	v = apic_read(APIC_TASKPRI);
1047
	pr_debug("... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1048 1049 1050 1051 1052

	/* !82489DX */
	if (APIC_INTEGRATED(ver)) {
		if (!APIC_XAPIC(ver)) {
			v = apic_read(APIC_ARBPRI);
1053 1054
			pr_debug("... APIC ARBPRI: %08x (%02x)\n",
				 v, v & APIC_ARBPRI_MASK);
1055 1056
		}
		v = apic_read(APIC_PROCPRI);
1057
		pr_debug("... APIC PROCPRI: %08x\n", v);
1058 1059 1060 1061 1062 1063 1064 1065
	}

	/*
	 * Remote read supported only in the 82489DX and local APIC for
	 * Pentium processors.
	 */
	if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
		v = apic_read(APIC_RRR);
1066
		pr_debug("... APIC RRR: %08x\n", v);
1067 1068 1069
	}

	v = apic_read(APIC_LDR);
1070
	pr_debug("... APIC LDR: %08x\n", v);
1071 1072
	if (!x2apic_enabled()) {
		v = apic_read(APIC_DFR);
1073
		pr_debug("... APIC DFR: %08x\n", v);
1074 1075
	}
	v = apic_read(APIC_SPIV);
1076
	pr_debug("... APIC SPIV: %08x\n", v);
1077

1078
	pr_debug("... APIC ISR field:\n");
1079
	print_APIC_field(APIC_ISR);
1080
	pr_debug("... APIC TMR field:\n");
1081
	print_APIC_field(APIC_TMR);
1082
	pr_debug("... APIC IRR field:\n");
1083 1084 1085 1086 1087 1088 1089 1090 1091
	print_APIC_field(APIC_IRR);

	/* !82489DX */
	if (APIC_INTEGRATED(ver)) {
		/* Due to the Pentium erratum 3AP. */
		if (maxlvt > 3)
			apic_write(APIC_ESR, 0);

		v = apic_read(APIC_ESR);
1092
		pr_debug("... APIC ESR: %08x\n", v);
1093 1094 1095
	}

	icr = apic_icr_read();
1096 1097
	pr_debug("... APIC ICR: %08x\n", (u32)icr);
	pr_debug("... APIC ICR2: %08x\n", (u32)(icr >> 32));
1098 1099

	v = apic_read(APIC_LVTT);
1100
	pr_debug("... APIC LVTT: %08x\n", v);
1101 1102 1103 1104

	if (maxlvt > 3) {
		/* PC is LVT#4. */
		v = apic_read(APIC_LVTPC);
1105
		pr_debug("... APIC LVTPC: %08x\n", v);
1106 1107
	}
	v = apic_read(APIC_LVT0);
1108
	pr_debug("... APIC LVT0: %08x\n", v);
1109
	v = apic_read(APIC_LVT1);
1110
	pr_debug("... APIC LVT1: %08x\n", v);
1111 1112 1113 1114

	if (maxlvt > 2) {
		/* ERR is LVT#3. */
		v = apic_read(APIC_LVTERR);
1115
		pr_debug("... APIC LVTERR: %08x\n", v);
1116 1117 1118
	}

	v = apic_read(APIC_TMICT);
1119
	pr_debug("... APIC TMICT: %08x\n", v);
1120
	v = apic_read(APIC_TMCCT);
1121
	pr_debug("... APIC TMCCT: %08x\n", v);
1122
	v = apic_read(APIC_TDCR);
1123
	pr_debug("... APIC TDCR: %08x\n", v);
1124 1125 1126 1127

	if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
		v = apic_read(APIC_EFEAT);
		maxlvt = (v >> 16) & 0xff;
1128
		pr_debug("... APIC EFEAT: %08x\n", v);
1129
		v = apic_read(APIC_ECTRL);
1130
		pr_debug("... APIC ECTRL: %08x\n", v);
1131 1132
		for (i = 0; i < maxlvt; i++) {
			v = apic_read(APIC_EILVTn(i));
1133
			pr_debug("... APIC EILVT%d: %08x\n", i, v);
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
		}
	}
	pr_cont("\n");
}

static void __init print_local_APICs(int maxcpu)
{
	int cpu;

	if (!maxcpu)
		return;

	preempt_disable();
	for_each_online_cpu(cpu) {
		if (cpu >= maxcpu)
			break;
		smp_call_function_single(cpu, print_local_APIC, NULL, 1);
	}
	preempt_enable();
}

static void __init print_PIC(void)
{
	unsigned int v;
	unsigned long flags;

	if (!nr_legacy_irqs())
		return;

1163
	pr_debug("\nprinting PIC contents\n");
1164 1165 1166 1167

	raw_spin_lock_irqsave(&i8259A_lock, flags);

	v = inb(0xa1) << 8 | inb(0x21);
1168
	pr_debug("... PIC  IMR: %04x\n", v);
1169 1170

	v = inb(0xa0) << 8 | inb(0x20);
1171
	pr_debug("... PIC  IRR: %04x\n", v);
1172 1173 1174 1175 1176 1177 1178 1179 1180

	outb(0x0b, 0xa0);
	outb(0x0b, 0x20);
	v = inb(0xa0) << 8 | inb(0x20);
	outb(0x0a, 0xa0);
	outb(0x0a, 0x20);

	raw_spin_unlock_irqrestore(&i8259A_lock, flags);

1181
	pr_debug("... PIC  ISR: %04x\n", v);
1182 1183

	v = inb(0x4d1) << 8 | inb(0x4d0);
1184
	pr_debug("... PIC ELCR: %04x\n", v);
1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
}

static int show_lapic __initdata = 1;
static __init int setup_show_lapic(char *arg)
{
	int num = -1;

	if (strcmp(arg, "all") == 0) {
		show_lapic = CONFIG_NR_CPUS;
	} else {
		get_option(&arg, &num);
		if (num >= 0)
			show_lapic = num;
	}

	return 1;
}
__setup("show_lapic=", setup_show_lapic);

static int __init print_ICs(void)
{
	if (apic_verbosity == APIC_QUIET)
		return 0;

	print_PIC();

	/* don't print out if apic is not there */
1212
	if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1213 1214 1215 1216 1217 1218 1219 1220 1221
		return 0;

	print_local_APICs(show_lapic);
	print_IO_APICs();

	return 0;
}

late_initcall(print_ICs);