vector.c 31.5 KB
Newer Older
1
/*
2
 * Local APIC related interfaces to support IOAPIC, MSI, etc.
3 4 5
 *
 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
 *	Moved from arch/x86/kernel/apic/io_apic.c.
6 7
 * Jiang Liu <jiang.liu@linux.intel.com>
 *	Enable support of hierarchical irqdomains
8 9 10 11 12 13
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/interrupt.h>
14
#include <linux/seq_file.h>
15 16 17
#include <linux/init.h>
#include <linux/compiler.h>
#include <linux/slab.h>
18
#include <asm/irqdomain.h>
19 20 21 22 23 24
#include <asm/hw_irq.h>
#include <asm/apic.h>
#include <asm/i8259.h>
#include <asm/desc.h>
#include <asm/irq_remapping.h>

25 26
#include <asm/trace/irq_vectors.h>

27
struct apic_chip_data {
28 29 30
	struct irq_cfg		hw_irq_cfg;
	unsigned int		vector;
	unsigned int		prev_vector;
31 32
	unsigned int		cpu;
	unsigned int		prev_cpu;
33
	unsigned int		irq;
34
	struct hlist_node	clist;
35
	unsigned int		move_in_progress	: 1,
36 37 38
				is_managed		: 1,
				can_reserve		: 1,
				has_reserved		: 1;
39 40
};

41
struct irq_domain *x86_vector_domain;
42
EXPORT_SYMBOL_GPL(x86_vector_domain);
43
static DEFINE_RAW_SPINLOCK(vector_lock);
44
static cpumask_var_t vector_searchmask;
45
static struct irq_chip lapic_controller;
46
static struct irq_matrix *vector_matrix;
47 48 49
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct hlist_head, cleanup_list);
#endif
50 51 52 53 54 55 56 57 58 59 60 61 62 63

void lock_vector_lock(void)
{
	/* Used to the online set of cpus does not change
	 * during assign_irq_vector.
	 */
	raw_spin_lock(&vector_lock);
}

void unlock_vector_lock(void)
{
	raw_spin_unlock(&vector_lock);
}

64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
void init_irq_alloc_info(struct irq_alloc_info *info,
			 const struct cpumask *mask)
{
	memset(info, 0, sizeof(*info));
	info->mask = mask;
}

void copy_irq_alloc_info(struct irq_alloc_info *dst, struct irq_alloc_info *src)
{
	if (src)
		*dst = *src;
	else
		memset(dst, 0, sizeof(*dst));
}

79
static struct apic_chip_data *apic_chip_data(struct irq_data *irqd)
80
{
81
	if (!irqd)
82 83
		return NULL;

84 85
	while (irqd->parent_data)
		irqd = irqd->parent_data;
86

87
	return irqd->chip_data;
88 89
}

90
struct irq_cfg *irqd_cfg(struct irq_data *irqd)
91
{
92
	struct apic_chip_data *apicd = apic_chip_data(irqd);
93

94
	return apicd ? &apicd->hw_irq_cfg : NULL;
95
}
96
EXPORT_SYMBOL_GPL(irqd_cfg);
97 98

struct irq_cfg *irq_cfg(unsigned int irq)
99
{
100 101
	return irqd_cfg(irq_get_irq_data(irq));
}
102

103 104
static struct apic_chip_data *alloc_apic_chip_data(int node)
{
105
	struct apic_chip_data *apicd;
106

107
	apicd = kzalloc_node(sizeof(*apicd), GFP_KERNEL, node);
108 109
	if (apicd)
		INIT_HLIST_NODE(&apicd->clist);
110
	return apicd;
111 112
}

113
static void free_apic_chip_data(struct apic_chip_data *apicd)
114
{
115
	kfree(apicd);
116 117
}

118 119
static void apic_update_irq_cfg(struct irq_data *irqd, unsigned int vector,
				unsigned int cpu)
120
{
121
	struct apic_chip_data *apicd = apic_chip_data(irqd);
122

123
	lockdep_assert_held(&vector_lock);
124

125 126 127 128 129
	apicd->hw_irq_cfg.vector = vector;
	apicd->hw_irq_cfg.dest_apicid = apic->calc_dest_apicid(cpu);
	irq_data_update_effective_affinity(irqd, cpumask_of(cpu));
	trace_vector_config(irqd->irq, vector, cpu,
			    apicd->hw_irq_cfg.dest_apicid);
130
}
131

132 133 134 135 136
static void apic_update_vector(struct irq_data *irqd, unsigned int newvec,
			       unsigned int newcpu)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	struct irq_desc *desc = irq_data_to_desc(irqd);
137

138
	lockdep_assert_held(&vector_lock);
139

140
	trace_vector_update(irqd->irq, newvec, newcpu, apicd->vector,
141
			    apicd->cpu);
142

143
	/* Setup the vector move, if required  */
144
	if (apicd->vector && cpu_online(apicd->cpu)) {
145
		apicd->move_in_progress = true;
146
		apicd->prev_vector = apicd->vector;
147 148
		apicd->prev_cpu = apicd->cpu;
	} else {
149
		apicd->prev_vector = 0;
150
	}
151

152
	apicd->vector = newvec;
153 154 155 156
	apicd->cpu = newcpu;
	BUG_ON(!IS_ERR_OR_NULL(per_cpu(vector_irq, newcpu)[newvec]));
	per_cpu(vector_irq, newcpu)[newvec] = desc;
}
157

158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
static void vector_assign_managed_shutdown(struct irq_data *irqd)
{
	unsigned int cpu = cpumask_first(cpu_online_mask);

	apic_update_irq_cfg(irqd, MANAGED_IRQ_SHUTDOWN_VECTOR, cpu);
}

static int reserve_managed_vector(struct irq_data *irqd)
{
	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	unsigned long flags;
	int ret;

	raw_spin_lock_irqsave(&vector_lock, flags);
	apicd->is_managed = true;
	ret = irq_matrix_reserve_managed(vector_matrix, affmsk);
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	trace_vector_reserve_managed(irqd->irq, ret);
	return ret;
}

180 181 182 183 184 185 186
static void reserve_irq_vector_locked(struct irq_data *irqd)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);

	irq_matrix_reserve(vector_matrix);
	apicd->can_reserve = true;
	apicd->has_reserved = true;
187
	irqd_set_can_reserve(irqd);
188 189 190 191 192 193 194 195 196 197 198 199 200 201
	trace_vector_reserve(irqd->irq, 0);
	vector_assign_managed_shutdown(irqd);
}

static int reserve_irq_vector(struct irq_data *irqd)
{
	unsigned long flags;

	raw_spin_lock_irqsave(&vector_lock, flags);
	reserve_irq_vector_locked(irqd);
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	return 0;
}

202 203 204
static int allocate_vector(struct irq_data *irqd, const struct cpumask *dest)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
205
	bool resvd = apicd->has_reserved;
206
	unsigned int cpu = apicd->cpu;
207 208 209
	int vector = apicd->vector;

	lockdep_assert_held(&vector_lock);
210

211
	/*
212 213 214
	 * If the current target CPU is online and in the new requested
	 * affinity mask, there is no point in moving the interrupt from
	 * one CPU to another.
215
	 */
216 217 218
	if (vector && cpu_online(cpu) && cpumask_test_cpu(cpu, dest))
		return 0;

219
	vector = irq_matrix_alloc(vector_matrix, dest, resvd, &cpu);
220 221
	if (vector > 0)
		apic_update_vector(irqd, vector, cpu);
222
	trace_vector_alloc(irqd->irq, vector, resvd, vector);
223 224 225 226 227 228
	return vector;
}

static int assign_vector_locked(struct irq_data *irqd,
				const struct cpumask *dest)
{
229
	struct apic_chip_data *apicd = apic_chip_data(irqd);
230 231 232 233 234
	int vector = allocate_vector(irqd, dest);

	if (vector < 0)
		return vector;

235
	apic_update_irq_cfg(irqd, apicd->vector, apicd->cpu);
236
	return 0;
237 238
}

239
static int assign_irq_vector(struct irq_data *irqd, const struct cpumask *dest)
240 241
{
	unsigned long flags;
242
	int ret;
243 244

	raw_spin_lock_irqsave(&vector_lock, flags);
245 246
	cpumask_and(vector_searchmask, dest, cpu_online_mask);
	ret = assign_vector_locked(irqd, vector_searchmask);
247
	raw_spin_unlock_irqrestore(&vector_lock, flags);
248
	return ret;
249 250
}

251 252
static int assign_irq_vector_any_locked(struct irq_data *irqd)
{
253 254
	/* Get the affinity mask - either irq_default_affinity or (user) set */
	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);
255 256
	int node = irq_data_get_node(irqd);

257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
	if (node == NUMA_NO_NODE)
		goto all;
	/* Try the intersection of @affmsk and node mask */
	cpumask_and(vector_searchmask, cpumask_of_node(node), affmsk);
	if (!assign_vector_locked(irqd, vector_searchmask))
		return 0;
	/* Try the node mask */
	if (!assign_vector_locked(irqd, cpumask_of_node(node)))
		return 0;
all:
	/* Try the full affinity mask */
	cpumask_and(vector_searchmask, affmsk, cpu_online_mask);
	if (!assign_vector_locked(irqd, vector_searchmask))
		return 0;
	/* Try the full online mask */
272 273 274 275 276
	return assign_vector_locked(irqd, cpu_online_mask);
}

static int
assign_irq_vector_policy(struct irq_data *irqd, struct irq_alloc_info *info)
277
{
278 279
	if (irqd_affinity_is_managed(irqd))
		return reserve_managed_vector(irqd);
280
	if (info->mask)
281
		return assign_irq_vector(irqd, info->mask);
282 283 284 285
	/*
	 * Make only a global reservation with no guarantee. A real vector
	 * is associated at activation time.
	 */
286
	return reserve_irq_vector(irqd);
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
}

static int
assign_managed_vector(struct irq_data *irqd, const struct cpumask *dest)
{
	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	int vector, cpu;

	cpumask_and(vector_searchmask, vector_searchmask, affmsk);
	cpu = cpumask_first(vector_searchmask);
	if (cpu >= nr_cpu_ids)
		return -EINVAL;
	/* set_affinity might call here for nothing */
	if (apicd->vector && cpumask_test_cpu(apicd->cpu, vector_searchmask))
302
		return 0;
303 304 305 306 307 308 309
	vector = irq_matrix_alloc_managed(vector_matrix, cpu);
	trace_vector_alloc_managed(irqd->irq, vector, vector);
	if (vector < 0)
		return vector;
	apic_update_vector(irqd, vector, cpu);
	apic_update_irq_cfg(irqd, vector, cpu);
	return 0;
310 311
}

312
static void clear_irq_vector(struct irq_data *irqd)
313
{
314
	struct apic_chip_data *apicd = apic_chip_data(irqd);
315
	bool managed = irqd_affinity_is_managed(irqd);
316
	unsigned int vector = apicd->vector;
317

318
	lockdep_assert_held(&vector_lock);
319

320
	if (!vector)
321
		return;
322

323
	trace_vector_clear(irqd->irq, vector, apicd->cpu, apicd->prev_vector,
324 325
			   apicd->prev_cpu);

326
	per_cpu(vector_irq, apicd->cpu)[vector] = VECTOR_UNUSED;
327
	irq_matrix_free(vector_matrix, apicd->cpu, vector, managed);
328
	apicd->vector = 0;
329

330
	/* Clean up move in progress */
331
	vector = apicd->prev_vector;
332
	if (!vector)
333 334
		return;

335
	per_cpu(vector_irq, apicd->prev_cpu)[vector] = VECTOR_UNUSED;
336
	irq_matrix_free(vector_matrix, apicd->prev_cpu, vector, managed);
337
	apicd->prev_vector = 0;
338
	apicd->move_in_progress = 0;
339
	hlist_del_init(&apicd->clist);
340 341
}

342 343 344 345 346 347
static void x86_vector_deactivate(struct irq_domain *dom, struct irq_data *irqd)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	unsigned long flags;

	trace_vector_deactivate(irqd->irq, apicd->is_managed,
348
				apicd->can_reserve, false);
349

350 351 352 353 354
	/* Regular fixed assigned interrupt */
	if (!apicd->is_managed && !apicd->can_reserve)
		return;
	/* If the interrupt has a global reservation, nothing to do */
	if (apicd->has_reserved)
355 356 357 358
		return;

	raw_spin_lock_irqsave(&vector_lock, flags);
	clear_irq_vector(irqd);
359 360 361 362
	if (apicd->can_reserve)
		reserve_irq_vector_locked(irqd);
	else
		vector_assign_managed_shutdown(irqd);
363 364 365
	raw_spin_unlock_irqrestore(&vector_lock, flags);
}

366 367 368 369 370 371
static int activate_reserved(struct irq_data *irqd)
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	int ret;

	ret = assign_irq_vector_any_locked(irqd);
372
	if (!ret) {
373
		apicd->has_reserved = false;
374 375 376 377 378 379 380 381 382 383
		/*
		 * Core might have disabled reservation mode after
		 * allocating the irq descriptor. Ideally this should
		 * happen before allocation time, but that would require
		 * completely convoluted ways of transporting that
		 * information.
		 */
		if (!irqd_can_reserve(irqd))
			apicd->can_reserve = false;
	}
384 385 386
	return ret;
}

387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411
static int activate_managed(struct irq_data *irqd)
{
	const struct cpumask *dest = irq_data_get_affinity_mask(irqd);
	int ret;

	cpumask_and(vector_searchmask, dest, cpu_online_mask);
	if (WARN_ON_ONCE(cpumask_empty(vector_searchmask))) {
		/* Something in the core code broke! Survive gracefully */
		pr_err("Managed startup for irq %u, but no CPU\n", irqd->irq);
		return EINVAL;
	}

	ret = assign_managed_vector(irqd, vector_searchmask);
	/*
	 * This should not happen. The vector reservation got buggered.  Handle
	 * it gracefully.
	 */
	if (WARN_ON_ONCE(ret < 0)) {
		pr_err("Managed startup irq %u, no vector available\n",
		       irqd->irq);
	}
       return ret;
}

static int x86_vector_activate(struct irq_domain *dom, struct irq_data *irqd,
412
			       bool reserve)
413 414 415 416 417 418
{
	struct apic_chip_data *apicd = apic_chip_data(irqd);
	unsigned long flags;
	int ret = 0;

	trace_vector_activate(irqd->irq, apicd->is_managed,
419
			      apicd->can_reserve, reserve);
420

421 422
	/* Nothing to do for fixed assigned vectors */
	if (!apicd->can_reserve && !apicd->is_managed)
423 424 425
		return 0;

	raw_spin_lock_irqsave(&vector_lock, flags);
426
	if (reserve || irqd_is_managed_and_shutdown(irqd))
427
		vector_assign_managed_shutdown(irqd);
428
	else if (apicd->is_managed)
429
		ret = activate_managed(irqd);
430 431
	else if (apicd->has_reserved)
		ret = activate_reserved(irqd);
432 433 434 435 436 437 438 439 440
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	return ret;
}

static void vector_free_reserved_and_managed(struct irq_data *irqd)
{
	const struct cpumask *dest = irq_data_get_affinity_mask(irqd);
	struct apic_chip_data *apicd = apic_chip_data(irqd);

441 442
	trace_vector_teardown(irqd->irq, apicd->is_managed,
			      apicd->has_reserved);
443

444 445
	if (apicd->has_reserved)
		irq_matrix_remove_reserved(vector_matrix);
446 447 448 449
	if (apicd->is_managed)
		irq_matrix_remove_managed(vector_matrix, dest);
}

450 451 452
static void x86_vector_free_irqs(struct irq_domain *domain,
				 unsigned int virq, unsigned int nr_irqs)
{
453 454
	struct apic_chip_data *apicd;
	struct irq_data *irqd;
455
	unsigned long flags;
456 457 458
	int i;

	for (i = 0; i < nr_irqs; i++) {
459 460
		irqd = irq_domain_get_irq_data(x86_vector_domain, virq + i);
		if (irqd && irqd->chip_data) {
461
			raw_spin_lock_irqsave(&vector_lock, flags);
462
			clear_irq_vector(irqd);
463
			vector_free_reserved_and_managed(irqd);
464 465
			apicd = irqd->chip_data;
			irq_domain_reset_irq_data(irqd);
466
			raw_spin_unlock_irqrestore(&vector_lock, flags);
467
			free_apic_chip_data(apicd);
468 469 470 471
		}
	}
}

472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
static bool vector_configure_legacy(unsigned int virq, struct irq_data *irqd,
				    struct apic_chip_data *apicd)
{
	unsigned long flags;
	bool realloc = false;

	apicd->vector = ISA_IRQ_VECTOR(virq);
	apicd->cpu = 0;

	raw_spin_lock_irqsave(&vector_lock, flags);
	/*
	 * If the interrupt is activated, then it must stay at this vector
	 * position. That's usually the timer interrupt (0).
	 */
	if (irqd_is_activated(irqd)) {
		trace_vector_setup(virq, true, 0);
		apic_update_irq_cfg(irqd, apicd->vector, apicd->cpu);
	} else {
		/* Release the vector */
		apicd->can_reserve = true;
492
		irqd_set_can_reserve(irqd);
493 494 495 496 497 498 499
		clear_irq_vector(irqd);
		realloc = true;
	}
	raw_spin_unlock_irqrestore(&vector_lock, flags);
	return realloc;
}

500 501 502 503
static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq,
				 unsigned int nr_irqs, void *arg)
{
	struct irq_alloc_info *info = arg;
504 505
	struct apic_chip_data *apicd;
	struct irq_data *irqd;
506
	int i, err, node;
507 508 509 510 511 512 513 514 515

	if (disable_apic)
		return -ENXIO;

	/* Currently vector allocator can't guarantee contiguous allocations */
	if ((info->flags & X86_IRQ_ALLOC_CONTIGUOUS_VECTORS) && nr_irqs > 1)
		return -ENOSYS;

	for (i = 0; i < nr_irqs; i++) {
516 517 518
		irqd = irq_domain_get_irq_data(domain, virq + i);
		BUG_ON(!irqd);
		node = irq_data_get_node(irqd);
519 520
		WARN_ON_ONCE(irqd->chip_data);
		apicd = alloc_apic_chip_data(node);
521
		if (!apicd) {
522 523 524 525
			err = -ENOMEM;
			goto error;
		}

526
		apicd->irq = virq + i;
527 528 529 530
		irqd->chip = &lapic_controller;
		irqd->chip_data = apicd;
		irqd->hwirq = virq + i;
		irqd_set_single_target(irqd);
531
		/*
532 533 534 535 536
		 * Legacy vectors are already assigned when the IOAPIC
		 * takes them over. They stay on the same vector. This is
		 * required for check_timer() to work correctly as it might
		 * switch back to legacy mode. Only update the hardware
		 * config.
537 538
		 */
		if (info->flags & X86_IRQ_ALLOC_LEGACY) {
539 540
			if (!vector_configure_legacy(virq + i, irqd, apicd))
				continue;
541 542
		}

543
		err = assign_irq_vector_policy(irqd, info);
544
		trace_vector_setup(virq + i, false, err);
545 546 547 548 549 550 551 552 553 554 555
		if (err)
			goto error;
	}

	return 0;

error:
	x86_vector_free_irqs(domain, virq, i + 1);
	return err;
}

556
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
557 558
static void x86_vector_debug_show(struct seq_file *m, struct irq_domain *d,
				  struct irq_data *irqd, int ind)
559
{
560
	unsigned int cpu, vector, prev_cpu, prev_vector;
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584
	struct apic_chip_data *apicd;
	unsigned long flags;
	int irq;

	if (!irqd) {
		irq_matrix_debug_show(m, vector_matrix, ind);
		return;
	}

	irq = irqd->irq;
	if (irq < nr_legacy_irqs() && !test_bit(irq, &io_apic_irqs)) {
		seq_printf(m, "%*sVector: %5d\n", ind, "", ISA_IRQ_VECTOR(irq));
		seq_printf(m, "%*sTarget: Legacy PIC all CPUs\n", ind, "");
		return;
	}

	apicd = irqd->chip_data;
	if (!apicd) {
		seq_printf(m, "%*sVector: Not assigned\n", ind, "");
		return;
	}

	raw_spin_lock_irqsave(&vector_lock, flags);
	cpu = apicd->cpu;
585
	vector = apicd->vector;
586
	prev_cpu = apicd->prev_cpu;
587
	prev_vector = apicd->prev_vector;
588
	raw_spin_unlock_irqrestore(&vector_lock, flags);
589
	seq_printf(m, "%*sVector: %5u\n", ind, "", vector);
590
	seq_printf(m, "%*sTarget: %5u\n", ind, "", cpu);
591 592
	if (prev_vector) {
		seq_printf(m, "%*sPrevious vector: %5u\n", ind, "", prev_vector);
593 594 595 596 597
		seq_printf(m, "%*sPrevious target: %5u\n", ind, "", prev_cpu);
	}
}
#endif

T
Thomas Gleixner 已提交
598
static const struct irq_domain_ops x86_vector_domain_ops = {
599 600
	.alloc		= x86_vector_alloc_irqs,
	.free		= x86_vector_free_irqs,
601 602
	.activate	= x86_vector_activate,
	.deactivate	= x86_vector_deactivate,
603 604 605
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
	.debug_show	= x86_vector_debug_show,
#endif
606 607
};

608 609 610 611 612 613 614 615
int __init arch_probe_nr_irqs(void)
{
	int nr;

	if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
		nr_irqs = NR_VECTORS * nr_cpu_ids;

	nr = (gsi_top + nr_legacy_irqs()) + 8 * nr_cpu_ids;
616
#if defined(CONFIG_PCI_MSI)
617 618 619 620 621 622 623 624 625 626 627
	/*
	 * for MSI and HT dyn irq
	 */
	if (gsi_top <= NR_IRQS_LEGACY)
		nr +=  8 * nr_cpu_ids;
	else
		nr += gsi_top * 16;
#endif
	if (nr < nr_irqs)
		nr_irqs = nr;

628 629 630 631 632
	/*
	 * We don't know if PIC is present at this point so we need to do
	 * probe() to get the right number of legacy IRQs.
	 */
	return legacy_pic->probe();
633 634
}

635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664
void lapic_assign_legacy_vector(unsigned int irq, bool replace)
{
	/*
	 * Use assign system here so it wont get accounted as allocated
	 * and moveable in the cpu hotplug check and it prevents managed
	 * irq reservation from touching it.
	 */
	irq_matrix_assign_system(vector_matrix, ISA_IRQ_VECTOR(irq), replace);
}

void __init lapic_assign_system_vectors(void)
{
	unsigned int i, vector = 0;

	for_each_set_bit_from(vector, system_vectors, NR_VECTORS)
		irq_matrix_assign_system(vector_matrix, vector, false);

	if (nr_legacy_irqs() > 1)
		lapic_assign_legacy_vector(PIC_CASCADE_IR, false);

	/* System vectors are reserved, online it */
	irq_matrix_online(vector_matrix);

	/* Mark the preallocated legacy interrupts */
	for (i = 0; i < nr_legacy_irqs(); i++) {
		if (i != PIC_CASCADE_IR)
			irq_matrix_assign(vector_matrix, ISA_IRQ_VECTOR(i));
	}
}

665 666
int __init arch_early_irq_init(void)
{
667 668 669 670 671 672
	struct fwnode_handle *fn;

	fn = irq_domain_alloc_named_fwnode("VECTOR");
	BUG_ON(!fn);
	x86_vector_domain = irq_domain_create_tree(fn, &x86_vector_domain_ops,
						   NULL);
673
	BUG_ON(x86_vector_domain == NULL);
674
	irq_domain_free_fwnode(fn);
675 676
	irq_set_default_host(x86_vector_domain);

677 678
	arch_init_msi_domain(x86_vector_domain);

679
	BUG_ON(!alloc_cpumask_var(&vector_searchmask, GFP_KERNEL));
680

681 682 683 684 685 686 687 688
	/*
	 * Allocate the vector matrix allocator data structure and limit the
	 * search area.
	 */
	vector_matrix = irq_alloc_matrix(NR_VECTORS, FIRST_EXTERNAL_VECTOR,
					 FIRST_SYSTEM_VECTOR);
	BUG_ON(!vector_matrix);

689 690 691
	return arch_early_ioapic_init();
}

692
#ifdef CONFIG_SMP
693

694 695 696 697 698 699 700 701 702 703 704 705 706
static struct irq_desc *__setup_vector_irq(int vector)
{
	int isairq = vector - ISA_IRQ_VECTOR(0);

	/* Check whether the irq is in the legacy space */
	if (isairq < 0 || isairq >= nr_legacy_irqs())
		return VECTOR_UNUSED;
	/* Check whether the irq is handled by the IOAPIC */
	if (test_bit(isairq, &io_apic_irqs))
		return VECTOR_UNUSED;
	return irq_to_desc(isairq);
}

707 708
/* Online the local APIC infrastructure and initialize the vectors */
void lapic_online(void)
709
{
710
	unsigned int vector;
711

712
	lockdep_assert_held(&vector_lock);
713 714 715 716

	/* Online the vector matrix array for this CPU */
	irq_matrix_online(vector_matrix);

717
	/*
718 719 720 721 722 723 724
	 * The interrupt affinity logic never targets interrupts to offline
	 * CPUs. The exception are the legacy PIC interrupts. In general
	 * they are only targeted to CPU0, but depending on the platform
	 * they can be distributed to any online CPU in hardware. The
	 * kernel has no influence on that. So all active legacy vectors
	 * must be installed on all CPUs. All non legacy interrupts can be
	 * cleared.
725
	 */
726 727
	for (vector = 0; vector < NR_VECTORS; vector++)
		this_cpu_write(vector_irq[vector], __setup_vector_irq(vector));
728 729
}

730 731 732 733 734 735 736
void lapic_offline(void)
{
	lock_vector_lock();
	irq_matrix_offline(vector_matrix);
	unlock_vector_lock();
}

737 738 739
static int apic_set_affinity(struct irq_data *irqd,
			     const struct cpumask *dest, bool force)
{
740
	struct apic_chip_data *apicd = apic_chip_data(irqd);
741 742
	int err;

743 744 745 746 747 748 749 750 751 752 753 754
	/*
	 * Core code can call here for inactive interrupts. For inactive
	 * interrupts which use managed or reservation mode there is no
	 * point in going through the vector assignment right now as the
	 * activation will assign a vector which fits the destination
	 * cpumask. Let the core code store the destination mask and be
	 * done with it.
	 */
	if (!irqd_is_activated(irqd) &&
	    (apicd->is_managed || apicd->can_reserve))
		return IRQ_SET_MASK_OK;

755 756 757 758 759 760 761
	raw_spin_lock(&vector_lock);
	cpumask_and(vector_searchmask, dest, cpu_online_mask);
	if (irqd_affinity_is_managed(irqd))
		err = assign_managed_vector(irqd, vector_searchmask);
	else
		err = assign_vector_locked(irqd, vector_searchmask);
	raw_spin_unlock(&vector_lock);
762 763 764 765 766 767 768
	return err ? err : IRQ_SET_MASK_OK;
}

#else
# define apic_set_affinity	NULL
#endif

769
static int apic_retrigger_irq(struct irq_data *irqd)
770
{
771
	struct apic_chip_data *apicd = apic_chip_data(irqd);
772 773 774
	unsigned long flags;

	raw_spin_lock_irqsave(&vector_lock, flags);
775
	apic->send_IPI(apicd->cpu, apicd->vector);
776 777 778 779 780
	raw_spin_unlock_irqrestore(&vector_lock, flags);

	return 1;
}

781
void apic_ack_edge(struct irq_data *irqd)
782
{
783 784
	irq_complete_move(irqd_cfg(irqd));
	irq_move_irq(irqd);
785 786 787
	ack_APIC_irq();
}

788
static struct irq_chip lapic_controller = {
T
Thomas Gleixner 已提交
789
	.name			= "APIC",
790
	.irq_ack		= apic_ack_edge,
791
	.irq_set_affinity	= apic_set_affinity,
792 793 794
	.irq_retrigger		= apic_retrigger_irq,
};

795
#ifdef CONFIG_SMP
796

797 798
static void free_moved_vector(struct apic_chip_data *apicd)
{
799
	unsigned int vector = apicd->prev_vector;
800
	unsigned int cpu = apicd->prev_cpu;
801 802 803 804 805 806 807 808 809
	bool managed = apicd->is_managed;

	/*
	 * This should never happen. Managed interrupts are not
	 * migrated except on CPU down, which does not involve the
	 * cleanup vector. But try to keep the accounting correct
	 * nevertheless.
	 */
	WARN_ON_ONCE(managed);
810

811
	trace_vector_free_moved(apicd->irq, cpu, vector, managed);
812
	irq_matrix_free(vector_matrix, cpu, vector, managed);
813
	per_cpu(vector_irq, cpu)[vector] = VECTOR_UNUSED;
814
	hlist_del_init(&apicd->clist);
815
	apicd->prev_vector = 0;
816 817 818
	apicd->move_in_progress = 0;
}

819
asmlinkage __visible void __irq_entry smp_irq_move_cleanup_interrupt(void)
820
{
821 822 823
	struct hlist_head *clhead = this_cpu_ptr(&cleanup_list);
	struct apic_chip_data *apicd;
	struct hlist_node *tmp;
824

825
	entering_ack_irq();
826 827 828
	/* Prevent vectors vanishing under us */
	raw_spin_lock(&vector_lock);

829
	hlist_for_each_entry_safe(apicd, tmp, clhead, clist) {
830
		unsigned int irr, vector = apicd->prev_vector;
831 832

		/*
833 834 835
		 * Paranoia: Check if the vector that needs to be cleaned
		 * up is registered at the APICs IRR. If so, then this is
		 * not the best time to clean it up. Clean it up in the
836
		 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
837 838 839
		 * to this CPU. IRQ_MOVE_CLEANUP_VECTOR is the lowest
		 * priority external vector, so on return from this
		 * interrupt the device interrupt will happen first.
840
		 */
841 842
		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		if (irr & (1U << (vector % 32))) {
843
			apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
844
			continue;
845
		}
846
		free_moved_vector(apicd);
847 848
	}

849
	raw_spin_unlock(&vector_lock);
850
	exiting_irq();
851 852
}

853 854 855 856 857 858 859 860 861 862 863
static void __send_cleanup_vector(struct apic_chip_data *apicd)
{
	unsigned int cpu;

	raw_spin_lock(&vector_lock);
	apicd->move_in_progress = 0;
	cpu = apicd->prev_cpu;
	if (cpu_online(cpu)) {
		hlist_add_head(&apicd->clist, per_cpu_ptr(&cleanup_list, cpu));
		apic->send_IPI(cpu, IRQ_MOVE_CLEANUP_VECTOR);
	} else {
864
		apicd->prev_vector = 0;
865 866 867 868 869 870 871 872
	}
	raw_spin_unlock(&vector_lock);
}

void send_cleanup_vector(struct irq_cfg *cfg)
{
	struct apic_chip_data *apicd;

873
	apicd = container_of(cfg, struct apic_chip_data, hw_irq_cfg);
874 875 876 877
	if (apicd->move_in_progress)
		__send_cleanup_vector(apicd);
}

878 879
static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
{
880
	struct apic_chip_data *apicd;
881

882
	apicd = container_of(cfg, struct apic_chip_data, hw_irq_cfg);
883
	if (likely(!apicd->move_in_progress))
884 885
		return;

886
	if (vector == apicd->vector && apicd->cpu == smp_processor_id())
887
		__send_cleanup_vector(apicd);
888 889 890 891 892 893 894
}

void irq_complete_move(struct irq_cfg *cfg)
{
	__irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
}

895
/*
896
 * Called from fixup_irqs() with @desc->lock held and interrupts disabled.
897 898
 */
void irq_force_complete_move(struct irq_desc *desc)
899
{
900
	struct apic_chip_data *apicd;
901 902
	struct irq_data *irqd;
	unsigned int vector;
903

904 905 906 907 908 909 910 911 912
	/*
	 * The function is called for all descriptors regardless of which
	 * irqdomain they belong to. For example if an IRQ is provided by
	 * an irq_chip as part of a GPIO driver, the chip data for that
	 * descriptor is specific to the irq_chip in question.
	 *
	 * Check first that the chip_data is what we expect
	 * (apic_chip_data) before touching it any further.
	 */
913
	irqd = irq_domain_get_irq_data(x86_vector_domain,
914
				       irq_desc_get_irq(desc));
915
	if (!irqd)
916 917
		return;

918
	raw_spin_lock(&vector_lock);
919
	apicd = apic_chip_data(irqd);
920 921
	if (!apicd)
		goto unlock;
922

923
	/*
924
	 * If prev_vector is empty, no action required.
925
	 */
926
	vector = apicd->prev_vector;
927 928
	if (!vector)
		goto unlock;
929

930
	/*
931
	 * This is tricky. If the cleanup of the old vector has not been
932 933 934
	 * done yet, then the following setaffinity call will fail with
	 * -EBUSY. This can leave the interrupt in a stale state.
	 *
935 936
	 * All CPUs are stuck in stop machine with interrupts disabled so
	 * calling __irq_complete_move() would be completely pointless.
937
	 *
938 939 940 941 942 943 944
	 * 1) The interrupt is in move_in_progress state. That means that we
	 *    have not seen an interrupt since the io_apic was reprogrammed to
	 *    the new vector.
	 *
	 * 2) The interrupt has fired on the new vector, but the cleanup IPIs
	 *    have not been processed yet.
	 */
945
	if (apicd->move_in_progress) {
946
		/*
947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
		 * In theory there is a race:
		 *
		 * set_ioapic(new_vector) <-- Interrupt is raised before update
		 *			      is effective, i.e. it's raised on
		 *			      the old vector.
		 *
		 * So if the target cpu cannot handle that interrupt before
		 * the old vector is cleaned up, we get a spurious interrupt
		 * and in the worst case the ioapic irq line becomes stale.
		 *
		 * But in case of cpu hotplug this should be a non issue
		 * because if the affinity update happens right before all
		 * cpus rendevouz in stop machine, there is no way that the
		 * interrupt can be blocked on the target cpu because all cpus
		 * loops first with interrupts enabled in stop machine, so the
		 * old vector is not yet cleaned up when the interrupt fires.
		 *
		 * So the only way to run into this issue is if the delivery
		 * of the interrupt on the apic/system bus would be delayed
		 * beyond the point where the target cpu disables interrupts
		 * in stop machine. I doubt that it can happen, but at least
		 * there is a theroretical chance. Virtualization might be
		 * able to expose this, but AFAICT the IOAPIC emulation is not
		 * as stupid as the real hardware.
		 *
		 * Anyway, there is nothing we can do about that at this point
		 * w/o refactoring the whole fixup_irq() business completely.
		 * We print at least the irq number and the old vector number,
		 * so we have the necessary information when a problem in that
		 * area arises.
977
		 */
978
		pr_warn("IRQ fixup: irq %d move in progress, old vector %d\n",
979
			irqd->irq, vector);
980
	}
981
	free_moved_vector(apicd);
982
unlock:
983
	raw_spin_unlock(&vector_lock);
984
}
985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015

#ifdef CONFIG_HOTPLUG_CPU
/*
 * Note, this is not accurate accounting, but at least good enough to
 * prevent that the actual interrupt move will run out of vectors.
 */
int lapic_can_unplug_cpu(void)
{
	unsigned int rsvd, avl, tomove, cpu = smp_processor_id();
	int ret = 0;

	raw_spin_lock(&vector_lock);
	tomove = irq_matrix_allocated(vector_matrix);
	avl = irq_matrix_available(vector_matrix, true);
	if (avl < tomove) {
		pr_warn("CPU %u has %u vectors, %u available. Cannot disable CPU\n",
			cpu, tomove, avl);
		ret = -ENOSPC;
		goto out;
	}
	rsvd = irq_matrix_reserved(vector_matrix);
	if (avl < rsvd) {
		pr_warn("Reserved vectors %u > available %u. IRQ request may fail\n",
			rsvd, avl);
	}
out:
	raw_spin_unlock(&vector_lock);
	return ret;
}
#endif /* HOTPLUG_CPU */
#endif /* SMP */
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033

static void __init print_APIC_field(int base)
{
	int i;

	printk(KERN_DEBUG);

	for (i = 0; i < 8; i++)
		pr_cont("%08x", apic_read(base + i*0x10));

	pr_cont("\n");
}

static void __init print_local_APIC(void *dummy)
{
	unsigned int i, v, ver, maxlvt;
	u64 icr;

1034 1035
	pr_debug("printing local APIC contents on CPU#%d/%d:\n",
		 smp_processor_id(), hard_smp_processor_id());
1036
	v = apic_read(APIC_ID);
1037
	pr_info("... APIC ID:      %08x (%01x)\n", v, read_apic_id());
1038
	v = apic_read(APIC_LVR);
1039
	pr_info("... APIC VERSION: %08x\n", v);
1040 1041 1042 1043
	ver = GET_APIC_VERSION(v);
	maxlvt = lapic_get_maxlvt();

	v = apic_read(APIC_TASKPRI);
1044
	pr_debug("... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1045 1046 1047 1048 1049

	/* !82489DX */
	if (APIC_INTEGRATED(ver)) {
		if (!APIC_XAPIC(ver)) {
			v = apic_read(APIC_ARBPRI);
1050 1051
			pr_debug("... APIC ARBPRI: %08x (%02x)\n",
				 v, v & APIC_ARBPRI_MASK);
1052 1053
		}
		v = apic_read(APIC_PROCPRI);
1054
		pr_debug("... APIC PROCPRI: %08x\n", v);
1055 1056 1057 1058 1059 1060 1061 1062
	}

	/*
	 * Remote read supported only in the 82489DX and local APIC for
	 * Pentium processors.
	 */
	if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
		v = apic_read(APIC_RRR);
1063
		pr_debug("... APIC RRR: %08x\n", v);
1064 1065 1066
	}

	v = apic_read(APIC_LDR);
1067
	pr_debug("... APIC LDR: %08x\n", v);
1068 1069
	if (!x2apic_enabled()) {
		v = apic_read(APIC_DFR);
1070
		pr_debug("... APIC DFR: %08x\n", v);
1071 1072
	}
	v = apic_read(APIC_SPIV);
1073
	pr_debug("... APIC SPIV: %08x\n", v);
1074

1075
	pr_debug("... APIC ISR field:\n");
1076
	print_APIC_field(APIC_ISR);
1077
	pr_debug("... APIC TMR field:\n");
1078
	print_APIC_field(APIC_TMR);
1079
	pr_debug("... APIC IRR field:\n");
1080 1081 1082 1083 1084 1085 1086 1087 1088
	print_APIC_field(APIC_IRR);

	/* !82489DX */
	if (APIC_INTEGRATED(ver)) {
		/* Due to the Pentium erratum 3AP. */
		if (maxlvt > 3)
			apic_write(APIC_ESR, 0);

		v = apic_read(APIC_ESR);
1089
		pr_debug("... APIC ESR: %08x\n", v);
1090 1091 1092
	}

	icr = apic_icr_read();
1093 1094
	pr_debug("... APIC ICR: %08x\n", (u32)icr);
	pr_debug("... APIC ICR2: %08x\n", (u32)(icr >> 32));
1095 1096

	v = apic_read(APIC_LVTT);
1097
	pr_debug("... APIC LVTT: %08x\n", v);
1098 1099 1100 1101

	if (maxlvt > 3) {
		/* PC is LVT#4. */
		v = apic_read(APIC_LVTPC);
1102
		pr_debug("... APIC LVTPC: %08x\n", v);
1103 1104
	}
	v = apic_read(APIC_LVT0);
1105
	pr_debug("... APIC LVT0: %08x\n", v);
1106
	v = apic_read(APIC_LVT1);
1107
	pr_debug("... APIC LVT1: %08x\n", v);
1108 1109 1110 1111

	if (maxlvt > 2) {
		/* ERR is LVT#3. */
		v = apic_read(APIC_LVTERR);
1112
		pr_debug("... APIC LVTERR: %08x\n", v);
1113 1114 1115
	}

	v = apic_read(APIC_TMICT);
1116
	pr_debug("... APIC TMICT: %08x\n", v);
1117
	v = apic_read(APIC_TMCCT);
1118
	pr_debug("... APIC TMCCT: %08x\n", v);
1119
	v = apic_read(APIC_TDCR);
1120
	pr_debug("... APIC TDCR: %08x\n", v);
1121 1122 1123 1124

	if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
		v = apic_read(APIC_EFEAT);
		maxlvt = (v >> 16) & 0xff;
1125
		pr_debug("... APIC EFEAT: %08x\n", v);
1126
		v = apic_read(APIC_ECTRL);
1127
		pr_debug("... APIC ECTRL: %08x\n", v);
1128 1129
		for (i = 0; i < maxlvt; i++) {
			v = apic_read(APIC_EILVTn(i));
1130
			pr_debug("... APIC EILVT%d: %08x\n", i, v);
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
		}
	}
	pr_cont("\n");
}

static void __init print_local_APICs(int maxcpu)
{
	int cpu;

	if (!maxcpu)
		return;

	preempt_disable();
	for_each_online_cpu(cpu) {
		if (cpu >= maxcpu)
			break;
		smp_call_function_single(cpu, print_local_APIC, NULL, 1);
	}
	preempt_enable();
}

static void __init print_PIC(void)
{
	unsigned int v;
	unsigned long flags;

	if (!nr_legacy_irqs())
		return;

1160
	pr_debug("\nprinting PIC contents\n");
1161 1162 1163 1164

	raw_spin_lock_irqsave(&i8259A_lock, flags);

	v = inb(0xa1) << 8 | inb(0x21);
1165
	pr_debug("... PIC  IMR: %04x\n", v);
1166 1167

	v = inb(0xa0) << 8 | inb(0x20);
1168
	pr_debug("... PIC  IRR: %04x\n", v);
1169 1170 1171 1172 1173 1174 1175 1176 1177

	outb(0x0b, 0xa0);
	outb(0x0b, 0x20);
	v = inb(0xa0) << 8 | inb(0x20);
	outb(0x0a, 0xa0);
	outb(0x0a, 0x20);

	raw_spin_unlock_irqrestore(&i8259A_lock, flags);

1178
	pr_debug("... PIC  ISR: %04x\n", v);
1179 1180

	v = inb(0x4d1) << 8 | inb(0x4d0);
1181
	pr_debug("... PIC ELCR: %04x\n", v);
1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
}

static int show_lapic __initdata = 1;
static __init int setup_show_lapic(char *arg)
{
	int num = -1;

	if (strcmp(arg, "all") == 0) {
		show_lapic = CONFIG_NR_CPUS;
	} else {
		get_option(&arg, &num);
		if (num >= 0)
			show_lapic = num;
	}

	return 1;
}
__setup("show_lapic=", setup_show_lapic);

static int __init print_ICs(void)
{
	if (apic_verbosity == APIC_QUIET)
		return 0;

	print_PIC();

	/* don't print out if apic is not there */
1209
	if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1210 1211 1212 1213 1214 1215 1216 1217 1218
		return 0;

	print_local_APICs(show_lapic);
	print_IO_APICs();

	return 0;
}

late_initcall(print_ICs);