irq-gic.c 38.9 KB
Newer Older
R
Russell King 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * Interrupt architecture for the GIC:
 *
 * o There is one Interrupt Distributor, which receives interrupts
 *   from system devices and sends them to the Interrupt Controllers.
 *
 * o There is one CPU Interface per CPU, which sends interrupts sent
 *   by the Distributor, and interrupts generated locally, to the
15 16 17
 *   associated CPU. The base address of the CPU interface is usually
 *   aliased so that the same address points to different chips depending
 *   on the CPU it is accessed from.
R
Russell King 已提交
18 19 20 21 22 23 24
 *
 * Note that IRQs 0-31 are special - they are local to each CPU.
 * As such, the enable set/clear, pending set/clear and active bit
 * registers are banked per-cpu for these sources.
 */
#include <linux/init.h>
#include <linux/kernel.h>
25
#include <linux/err.h>
26
#include <linux/module.h>
R
Russell King 已提交
27 28
#include <linux/list.h>
#include <linux/smp.h>
29
#include <linux/cpu.h>
30
#include <linux/cpu_pm.h>
31
#include <linux/cpumask.h>
32
#include <linux/io.h>
33 34 35
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
36
#include <linux/acpi.h>
R
Rob Herring 已提交
37
#include <linux/irqdomain.h>
M
Marc Zyngier 已提交
38 39 40
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/slab.h>
41
#include <linux/irqchip.h>
42
#include <linux/irqchip/chained_irq.h>
43
#include <linux/irqchip/arm-gic.h>
R
Russell King 已提交
44

45
#include <asm/cputype.h>
R
Russell King 已提交
46
#include <asm/irq.h>
47
#include <asm/exception.h>
48
#include <asm/smp_plat.h>
49
#include <asm/virt.h>
R
Russell King 已提交
50

51
#include "irq-gic-common.h"
R
Russell King 已提交
52

53 54 55 56 57
#ifdef CONFIG_ARM64
#include <asm/cpufeature.h>

static void gic_check_cpu_features(void)
{
58
	WARN_TAINT_ONCE(this_cpu_has_cap(ARM64_HAS_SYSREG_GIC_CPUIF),
59 60 61 62 63 64 65
			TAINT_CPU_OUT_OF_SPEC,
			"GICv3 system registers enabled, broken firmware!\n");
}
#else
#define gic_check_cpu_features()	do { } while(0)
#endif

66 67
union gic_base {
	void __iomem *common_base;
68
	void __percpu * __iomem *percpu_base;
69 70 71
};

struct gic_chip_data {
72
	struct irq_chip chip;
73 74
	union gic_base dist_base;
	union gic_base cpu_base;
75 76 77
	void __iomem *raw_dist_base;
	void __iomem *raw_cpu_base;
	u32 percpu_offset;
78 79
#ifdef CONFIG_CPU_PM
	u32 saved_spi_enable[DIV_ROUND_UP(1020, 32)];
80
	u32 saved_spi_active[DIV_ROUND_UP(1020, 32)];
81 82 83
	u32 saved_spi_conf[DIV_ROUND_UP(1020, 16)];
	u32 saved_spi_target[DIV_ROUND_UP(1020, 4)];
	u32 __percpu *saved_ppi_enable;
84
	u32 __percpu *saved_ppi_active;
85 86
	u32 __percpu *saved_ppi_conf;
#endif
87
	struct irq_domain *domain;
88 89 90 91 92 93
	unsigned int gic_irqs;
#ifdef CONFIG_GIC_NON_BANKED
	void __iomem *(*get_base)(union gic_base *);
#endif
};

94
static DEFINE_RAW_SPINLOCK(irq_controller_lock);
R
Russell King 已提交
95

96 97 98 99 100 101 102 103
/*
 * The GIC mapping of CPU interfaces does not necessarily match
 * the logical CPU numbering.  Let's use a mapping as returned
 * by the GIC itself.
 */
#define NR_GIC_CPU_IF 8
static u8 gic_cpu_map[NR_GIC_CPU_IF] __read_mostly;

104 105
static struct static_key supports_deactivate = STATIC_KEY_INIT_TRUE;

106
static struct gic_chip_data gic_data[CONFIG_ARM_GIC_MAX_NR] __read_mostly;
107

108 109
static struct gic_kvm_info gic_v2_kvm_info;

110 111 112
#ifdef CONFIG_GIC_NON_BANKED
static void __iomem *gic_get_percpu_base(union gic_base *base)
{
113
	return raw_cpu_read(*base->percpu_base);
114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
}

static void __iomem *gic_get_common_base(union gic_base *base)
{
	return base->common_base;
}

static inline void __iomem *gic_data_dist_base(struct gic_chip_data *data)
{
	return data->get_base(&data->dist_base);
}

static inline void __iomem *gic_data_cpu_base(struct gic_chip_data *data)
{
	return data->get_base(&data->cpu_base);
}

static inline void gic_set_base_accessor(struct gic_chip_data *data,
					 void __iomem *(*f)(union gic_base *))
{
	data->get_base = f;
}
#else
#define gic_data_dist_base(d)	((d)->dist_base.common_base)
#define gic_data_cpu_base(d)	((d)->cpu_base.common_base)
139
#define gic_set_base_accessor(d, f)
140 141
#endif

142
static inline void __iomem *gic_dist_base(struct irq_data *d)
143
{
144
	struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
145
	return gic_data_dist_base(gic_data);
146 147
}

148
static inline void __iomem *gic_cpu_base(struct irq_data *d)
149
{
150
	struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
151
	return gic_data_cpu_base(gic_data);
152 153
}

154
static inline unsigned int gic_irq(struct irq_data *d)
155
{
R
Rob Herring 已提交
156
	return d->hwirq;
157 158
}

159 160 161 162 163
static inline bool cascading_gic_irq(struct irq_data *d)
{
	void *data = irq_data_get_irq_handler_data(d);

	/*
164 165
	 * If handler_data is set, this is a cascading interrupt, and
	 * it cannot possibly be forwarded.
166
	 */
167
	return data != NULL;
168 169
}

R
Russell King 已提交
170 171 172
/*
 * Routines to acknowledge, disable and enable interrupts
 */
173 174 175 176 177 178 179
static void gic_poke_irq(struct irq_data *d, u32 offset)
{
	u32 mask = 1 << (gic_irq(d) % 32);
	writel_relaxed(mask, gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4);
}

static int gic_peek_irq(struct irq_data *d, u32 offset)
R
Russell King 已提交
180
{
R
Rob Herring 已提交
181
	u32 mask = 1 << (gic_irq(d) % 32);
182 183 184 185 186 187
	return !!(readl_relaxed(gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4) & mask);
}

static void gic_mask_irq(struct irq_data *d)
{
	gic_poke_irq(d, GIC_DIST_ENABLE_CLEAR);
R
Russell King 已提交
188 189
}

190 191 192
static void gic_eoimode1_mask_irq(struct irq_data *d)
{
	gic_mask_irq(d);
193 194 195 196 197 198 199 200
	/*
	 * When masking a forwarded interrupt, make sure it is
	 * deactivated as well.
	 *
	 * This ensures that an interrupt that is getting
	 * disabled/masked will not get "stuck", because there is
	 * noone to deactivate it (guest is being terminated).
	 */
201
	if (irqd_is_forwarded_to_vcpu(d))
202
		gic_poke_irq(d, GIC_DIST_ACTIVE_CLEAR);
203 204
}

205
static void gic_unmask_irq(struct irq_data *d)
R
Russell King 已提交
206
{
207
	gic_poke_irq(d, GIC_DIST_ENABLE_SET);
R
Russell King 已提交
208 209
}

210 211
static void gic_eoi_irq(struct irq_data *d)
{
212
	writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI);
213 214
}

215 216
static void gic_eoimode1_eoi_irq(struct irq_data *d)
{
217
	/* Do not deactivate an IRQ forwarded to a vcpu. */
218
	if (irqd_is_forwarded_to_vcpu(d))
219 220
		return;

221 222 223
	writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_DEACTIVATE);
}

224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
static int gic_irq_set_irqchip_state(struct irq_data *d,
				     enum irqchip_irq_state which, bool val)
{
	u32 reg;

	switch (which) {
	case IRQCHIP_STATE_PENDING:
		reg = val ? GIC_DIST_PENDING_SET : GIC_DIST_PENDING_CLEAR;
		break;

	case IRQCHIP_STATE_ACTIVE:
		reg = val ? GIC_DIST_ACTIVE_SET : GIC_DIST_ACTIVE_CLEAR;
		break;

	case IRQCHIP_STATE_MASKED:
		reg = val ? GIC_DIST_ENABLE_CLEAR : GIC_DIST_ENABLE_SET;
		break;

	default:
		return -EINVAL;
	}

	gic_poke_irq(d, reg);
	return 0;
}

static int gic_irq_get_irqchip_state(struct irq_data *d,
				      enum irqchip_irq_state which, bool *val)
{
	switch (which) {
	case IRQCHIP_STATE_PENDING:
		*val = gic_peek_irq(d, GIC_DIST_PENDING_SET);
		break;

	case IRQCHIP_STATE_ACTIVE:
		*val = gic_peek_irq(d, GIC_DIST_ACTIVE_SET);
		break;

	case IRQCHIP_STATE_MASKED:
		*val = !gic_peek_irq(d, GIC_DIST_ENABLE_SET);
		break;

	default:
		return -EINVAL;
	}

	return 0;
}

273
static int gic_set_type(struct irq_data *d, unsigned int type)
274
{
275 276
	void __iomem *base = gic_dist_base(d);
	unsigned int gicirq = gic_irq(d);
277 278 279 280 281

	/* Interrupt configuration for SGIs can't be changed */
	if (gicirq < 16)
		return -EINVAL;

282 283 284
	/* SPIs have restrictions on the supported types */
	if (gicirq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
			    type != IRQ_TYPE_EDGE_RISING)
285 286
		return -EINVAL;

287
	return gic_configure_irq(gicirq, type, base, NULL);
288 289
}

290 291 292 293 294 295
static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
{
	/* Only interrupts on the primary GIC can be forwarded to a vcpu. */
	if (cascading_gic_irq(d))
		return -EINVAL;

296 297 298 299
	if (vcpu)
		irqd_set_forwarded_to_vcpu(d);
	else
		irqd_clr_forwarded_to_vcpu(d);
300 301 302
	return 0;
}

303
#ifdef CONFIG_SMP
304 305
static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
			    bool force)
R
Russell King 已提交
306
{
307
	void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + (gic_irq(d) & ~3);
308
	unsigned int cpu, shift = (gic_irq(d) % 4) * 8;
309
	u32 val, mask, bit;
310
	unsigned long flags;
R
Russell King 已提交
311

312 313 314 315 316
	if (!force)
		cpu = cpumask_any_and(mask_val, cpu_online_mask);
	else
		cpu = cpumask_first(mask_val);

317
	if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids)
C
Chao Xie 已提交
318
		return -EINVAL;
319

320
	raw_spin_lock_irqsave(&irq_controller_lock, flags);
321
	mask = 0xff << shift;
322
	bit = gic_cpu_map[cpu] << shift;
323 324
	val = readl_relaxed(reg) & ~mask;
	writel_relaxed(val | bit, reg);
325
	raw_spin_unlock_irqrestore(&irq_controller_lock, flags);
326

327
	return IRQ_SET_MASK_OK_DONE;
R
Russell King 已提交
328
}
329
#endif
R
Russell King 已提交
330

331
static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
332 333 334 335 336 337 338
{
	u32 irqstat, irqnr;
	struct gic_chip_data *gic = &gic_data[0];
	void __iomem *cpu_base = gic_data_cpu_base(gic);

	do {
		irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK);
339
		irqnr = irqstat & GICC_IAR_INT_ID_MASK;
340

341
		if (likely(irqnr > 15 && irqnr < 1020)) {
342 343
			if (static_key_true(&supports_deactivate))
				writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
344
			handle_domain_irq(gic->domain, irqnr, regs);
345 346 347 348
			continue;
		}
		if (irqnr < 16) {
			writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
349 350
			if (static_key_true(&supports_deactivate))
				writel_relaxed(irqstat, cpu_base + GIC_CPU_DEACTIVATE);
351
#ifdef CONFIG_SMP
352 353 354 355 356 357 358 359
			/*
			 * Ensure any shared data written by the CPU sending
			 * the IPI is read after we've read the ACK register
			 * on the GIC.
			 *
			 * Pairs with the write barrier in gic_raise_softirq
			 */
			smp_rmb();
360 361 362 363 364 365 366 367
			handle_IPI(irqnr, regs);
#endif
			continue;
		}
		break;
	} while (1);
}

368
static void gic_handle_cascade_irq(struct irq_desc *desc)
369
{
370 371
	struct gic_chip_data *chip_data = irq_desc_get_handler_data(desc);
	struct irq_chip *chip = irq_desc_get_chip(desc);
372
	unsigned int cascade_irq, gic_irq;
373 374
	unsigned long status;

375
	chained_irq_enter(chip, desc);
376

377
	raw_spin_lock(&irq_controller_lock);
378
	status = readl_relaxed(gic_data_cpu_base(chip_data) + GIC_CPU_INTACK);
379
	raw_spin_unlock(&irq_controller_lock);
380

381 382
	gic_irq = (status & GICC_IAR_INT_ID_MASK);
	if (gic_irq == GICC_INT_SPURIOUS)
383 384
		goto out;

385 386
	cascade_irq = irq_find_mapping(chip_data->domain, gic_irq);
	if (unlikely(gic_irq < 32 || gic_irq > 1020))
387
		handle_bad_irq(desc);
388 389
	else
		generic_handle_irq(cascade_irq);
390 391

 out:
392
	chained_irq_exit(chip, desc);
393 394
}

395
static struct irq_chip gic_chip = {
396 397
	.irq_mask		= gic_mask_irq,
	.irq_unmask		= gic_unmask_irq,
398
	.irq_eoi		= gic_eoi_irq,
399
	.irq_set_type		= gic_set_type,
400 401
	.irq_get_irqchip_state	= gic_irq_get_irqchip_state,
	.irq_set_irqchip_state	= gic_irq_set_irqchip_state,
402 403 404
	.flags			= IRQCHIP_SET_TYPE_MASKED |
				  IRQCHIP_SKIP_SET_WAKE |
				  IRQCHIP_MASK_ON_SUSPEND,
R
Russell King 已提交
405 406
};

407 408
void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq)
{
409
	BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
410 411
	irq_set_chained_handler_and_data(irq, gic_handle_cascade_irq,
					 &gic_data[gic_nr]);
412 413
}

414 415 416 417 418 419 420 421 422 423 424 425 426
static u8 gic_get_cpumask(struct gic_chip_data *gic)
{
	void __iomem *base = gic_data_dist_base(gic);
	u32 mask, i;

	for (i = mask = 0; i < 32; i += 4) {
		mask = readl_relaxed(base + GIC_DIST_TARGET + i);
		mask |= mask >> 16;
		mask |= mask >> 8;
		if (mask)
			break;
	}

427
	if (!mask && num_possible_cpus() > 1)
428 429 430 431 432
		pr_crit("GIC CPU mask not found - kernel will fail to boot.\n");

	return mask;
}

433
static void gic_cpu_if_up(struct gic_chip_data *gic)
434
{
435
	void __iomem *cpu_base = gic_data_cpu_base(gic);
436
	u32 bypass = 0;
437 438
	u32 mode = 0;

439
	if (gic == &gic_data[0] && static_key_true(&supports_deactivate))
440
		mode = GIC_CPU_CTRL_EOImodeNS;
441 442 443 444 445 446 447

	/*
	* Preserve bypass disable bits to be written back later
	*/
	bypass = readl(cpu_base + GIC_CPU_CTRL);
	bypass &= GICC_DIS_BYPASS_MASK;

448
	writel_relaxed(bypass | mode | GICC_ENABLE, cpu_base + GIC_CPU_CTRL);
449 450 451
}


452
static void gic_dist_init(struct gic_chip_data *gic)
R
Russell King 已提交
453
{
454
	unsigned int i;
455
	u32 cpumask;
R
Rob Herring 已提交
456
	unsigned int gic_irqs = gic->gic_irqs;
457
	void __iomem *base = gic_data_dist_base(gic);
R
Russell King 已提交
458

459
	writel_relaxed(GICD_DISABLE, base + GIC_DIST_CTRL);
R
Russell King 已提交
460 461 462 463

	/*
	 * Set all global interrupts to this CPU only.
	 */
464 465 466
	cpumask = gic_get_cpumask(gic);
	cpumask |= cpumask << 8;
	cpumask |= cpumask << 16;
467
	for (i = 32; i < gic_irqs; i += 4)
468
		writel_relaxed(cpumask, base + GIC_DIST_TARGET + i * 4 / 4);
R
Russell King 已提交
469

470
	gic_dist_config(base, gic_irqs, NULL);
R
Russell King 已提交
471

472
	writel_relaxed(GICD_ENABLE, base + GIC_DIST_CTRL);
R
Russell King 已提交
473 474
}

475
static int gic_cpu_init(struct gic_chip_data *gic)
R
Russell King 已提交
476
{
477 478
	void __iomem *dist_base = gic_data_dist_base(gic);
	void __iomem *base = gic_data_cpu_base(gic);
479
	unsigned int cpu_mask, cpu = smp_processor_id();
480 481
	int i;

482
	/*
483 484 485
	 * Setting up the CPU map is only relevant for the primary GIC
	 * because any nested/secondary GICs do not directly interface
	 * with the CPU(s).
486
	 */
487 488 489 490
	if (gic == &gic_data[0]) {
		/*
		 * Get what the GIC says our CPU mask is.
		 */
491 492 493
		if (WARN_ON(cpu >= NR_GIC_CPU_IF))
			return -EINVAL;

494
		gic_check_cpu_features();
495 496
		cpu_mask = gic_get_cpumask(gic);
		gic_cpu_map[cpu] = cpu_mask;
497

498 499 500 501 502 503 504 505
		/*
		 * Clear our mask from the other map entries in case they're
		 * still undefined.
		 */
		for (i = 0; i < NR_GIC_CPU_IF; i++)
			if (i != cpu)
				gic_cpu_map[i] &= ~cpu_mask;
	}
506

507
	gic_cpu_config(dist_base, NULL);
508

509
	writel_relaxed(GICC_INT_PRI_THRESHOLD, base + GIC_CPU_PRIMASK);
510
	gic_cpu_if_up(gic);
511 512

	return 0;
R
Russell King 已提交
513 514
}

515
int gic_cpu_if_down(unsigned int gic_nr)
516
{
517
	void __iomem *cpu_base;
518 519
	u32 val = 0;

520
	if (gic_nr >= CONFIG_ARM_GIC_MAX_NR)
521 522 523
		return -EINVAL;

	cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
524 525 526
	val = readl(cpu_base + GIC_CPU_CTRL);
	val &= ~GICC_ENABLE;
	writel_relaxed(val, cpu_base + GIC_CPU_CTRL);
527 528

	return 0;
529 530
}

531 532 533 534 535 536 537
#ifdef CONFIG_CPU_PM
/*
 * Saves the GIC distributor registers during suspend or idle.  Must be called
 * with interrupts disabled but before powering down the GIC.  After calling
 * this function, no interrupts will be delivered by the GIC, and another
 * platform-specific wakeup source must be enabled.
 */
538
void gic_dist_save(struct gic_chip_data *gic)
539 540 541 542 543
{
	unsigned int gic_irqs;
	void __iomem *dist_base;
	int i;

544 545
	if (WARN_ON(!gic))
		return;
546

547 548
	gic_irqs = gic->gic_irqs;
	dist_base = gic_data_dist_base(gic);
549 550 551 552 553

	if (!dist_base)
		return;

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++)
554
		gic->saved_spi_conf[i] =
555 556 557
			readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4);

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
558
		gic->saved_spi_target[i] =
559 560 561
			readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4);

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++)
562
		gic->saved_spi_enable[i] =
563
			readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4);
564 565

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++)
566
		gic->saved_spi_active[i] =
567
			readl_relaxed(dist_base + GIC_DIST_ACTIVE_SET + i * 4);
568 569 570 571 572 573 574 575 576
}

/*
 * Restores the GIC distributor registers during resume or when coming out of
 * idle.  Must be called before enabling interrupts.  If a level interrupt
 * that occured while the GIC was suspended is still present, it will be
 * handled normally, but any edge interrupts that occured will not be seen by
 * the GIC and need to be handled by the platform-specific wakeup source.
 */
577
void gic_dist_restore(struct gic_chip_data *gic)
578 579 580 581 582
{
	unsigned int gic_irqs;
	unsigned int i;
	void __iomem *dist_base;

583 584
	if (WARN_ON(!gic))
		return;
585

586 587
	gic_irqs = gic->gic_irqs;
	dist_base = gic_data_dist_base(gic);
588 589 590 591

	if (!dist_base)
		return;

592
	writel_relaxed(GICD_DISABLE, dist_base + GIC_DIST_CTRL);
593 594

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++)
595
		writel_relaxed(gic->saved_spi_conf[i],
596 597 598
			dist_base + GIC_DIST_CONFIG + i * 4);

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
599
		writel_relaxed(GICD_INT_DEF_PRI_X4,
600 601 602
			dist_base + GIC_DIST_PRI + i * 4);

	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
603
		writel_relaxed(gic->saved_spi_target[i],
604 605
			dist_base + GIC_DIST_TARGET + i * 4);

606 607 608
	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) {
		writel_relaxed(GICD_INT_EN_CLR_X32,
			dist_base + GIC_DIST_ENABLE_CLEAR + i * 4);
609
		writel_relaxed(gic->saved_spi_enable[i],
610
			dist_base + GIC_DIST_ENABLE_SET + i * 4);
611
	}
612

613 614 615
	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) {
		writel_relaxed(GICD_INT_EN_CLR_X32,
			dist_base + GIC_DIST_ACTIVE_CLEAR + i * 4);
616
		writel_relaxed(gic->saved_spi_active[i],
617 618 619
			dist_base + GIC_DIST_ACTIVE_SET + i * 4);
	}

620
	writel_relaxed(GICD_ENABLE, dist_base + GIC_DIST_CTRL);
621 622
}

623
void gic_cpu_save(struct gic_chip_data *gic)
624 625 626 627 628 629
{
	int i;
	u32 *ptr;
	void __iomem *dist_base;
	void __iomem *cpu_base;

630 631
	if (WARN_ON(!gic))
		return;
632

633 634
	dist_base = gic_data_dist_base(gic);
	cpu_base = gic_data_cpu_base(gic);
635 636 637 638

	if (!dist_base || !cpu_base)
		return;

639
	ptr = raw_cpu_ptr(gic->saved_ppi_enable);
640 641 642
	for (i = 0; i < DIV_ROUND_UP(32, 32); i++)
		ptr[i] = readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4);

643
	ptr = raw_cpu_ptr(gic->saved_ppi_active);
644 645 646
	for (i = 0; i < DIV_ROUND_UP(32, 32); i++)
		ptr[i] = readl_relaxed(dist_base + GIC_DIST_ACTIVE_SET + i * 4);

647
	ptr = raw_cpu_ptr(gic->saved_ppi_conf);
648 649 650 651 652
	for (i = 0; i < DIV_ROUND_UP(32, 16); i++)
		ptr[i] = readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4);

}

653
void gic_cpu_restore(struct gic_chip_data *gic)
654 655 656 657 658 659
{
	int i;
	u32 *ptr;
	void __iomem *dist_base;
	void __iomem *cpu_base;

660 661
	if (WARN_ON(!gic))
		return;
662

663 664
	dist_base = gic_data_dist_base(gic);
	cpu_base = gic_data_cpu_base(gic);
665 666 667 668

	if (!dist_base || !cpu_base)
		return;

669
	ptr = raw_cpu_ptr(gic->saved_ppi_enable);
670 671 672
	for (i = 0; i < DIV_ROUND_UP(32, 32); i++) {
		writel_relaxed(GICD_INT_EN_CLR_X32,
			       dist_base + GIC_DIST_ENABLE_CLEAR + i * 4);
673
		writel_relaxed(ptr[i], dist_base + GIC_DIST_ENABLE_SET + i * 4);
674
	}
675

676
	ptr = raw_cpu_ptr(gic->saved_ppi_active);
677 678 679 680 681 682
	for (i = 0; i < DIV_ROUND_UP(32, 32); i++) {
		writel_relaxed(GICD_INT_EN_CLR_X32,
			       dist_base + GIC_DIST_ACTIVE_CLEAR + i * 4);
		writel_relaxed(ptr[i], dist_base + GIC_DIST_ACTIVE_SET + i * 4);
	}

683
	ptr = raw_cpu_ptr(gic->saved_ppi_conf);
684 685 686 687
	for (i = 0; i < DIV_ROUND_UP(32, 16); i++)
		writel_relaxed(ptr[i], dist_base + GIC_DIST_CONFIG + i * 4);

	for (i = 0; i < DIV_ROUND_UP(32, 4); i++)
688 689
		writel_relaxed(GICD_INT_DEF_PRI_X4,
					dist_base + GIC_DIST_PRI + i * 4);
690

691
	writel_relaxed(GICC_INT_PRI_THRESHOLD, cpu_base + GIC_CPU_PRIMASK);
692
	gic_cpu_if_up(gic);
693 694 695 696 697 698
}

static int gic_notifier(struct notifier_block *self, unsigned long cmd,	void *v)
{
	int i;

699
	for (i = 0; i < CONFIG_ARM_GIC_MAX_NR; i++) {
700 701 702 703 704
#ifdef CONFIG_GIC_NON_BANKED
		/* Skip over unused GICs */
		if (!gic_data[i].get_base)
			continue;
#endif
705 706
		switch (cmd) {
		case CPU_PM_ENTER:
707
			gic_cpu_save(&gic_data[i]);
708 709 710
			break;
		case CPU_PM_ENTER_FAILED:
		case CPU_PM_EXIT:
711
			gic_cpu_restore(&gic_data[i]);
712 713
			break;
		case CPU_CLUSTER_PM_ENTER:
714
			gic_dist_save(&gic_data[i]);
715 716 717
			break;
		case CPU_CLUSTER_PM_ENTER_FAILED:
		case CPU_CLUSTER_PM_EXIT:
718
			gic_dist_restore(&gic_data[i]);
719 720 721 722 723 724 725 726 727 728 729
			break;
		}
	}

	return NOTIFY_OK;
}

static struct notifier_block gic_notifier_block = {
	.notifier_call = gic_notifier,
};

730
static int gic_pm_init(struct gic_chip_data *gic)
731 732 733
{
	gic->saved_ppi_enable = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4,
		sizeof(u32));
734 735
	if (WARN_ON(!gic->saved_ppi_enable))
		return -ENOMEM;
736

737 738
	gic->saved_ppi_active = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4,
		sizeof(u32));
739 740
	if (WARN_ON(!gic->saved_ppi_active))
		goto free_ppi_enable;
741

742 743
	gic->saved_ppi_conf = __alloc_percpu(DIV_ROUND_UP(32, 16) * 4,
		sizeof(u32));
744 745
	if (WARN_ON(!gic->saved_ppi_conf))
		goto free_ppi_active;
746

747 748
	if (gic == &gic_data[0])
		cpu_pm_register_notifier(&gic_notifier_block);
749 750 751 752 753 754 755 756 757

	return 0;

free_ppi_active:
	free_percpu(gic->saved_ppi_active);
free_ppi_enable:
	free_percpu(gic->saved_ppi_enable);

	return -ENOMEM;
758 759
}
#else
760
static int gic_pm_init(struct gic_chip_data *gic)
761
{
762
	return 0;
763 764 765
}
#endif

766
#ifdef CONFIG_SMP
767
static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
768 769
{
	int cpu;
770 771 772
	unsigned long flags, map = 0;

	raw_spin_lock_irqsave(&irq_controller_lock, flags);
773 774 775

	/* Convert our logical CPU mask into a physical one. */
	for_each_cpu(cpu, mask)
776
		map |= gic_cpu_map[cpu];
777 778 779

	/*
	 * Ensure that stores to Normal memory are visible to the
780
	 * other CPUs before they observe us issuing the IPI.
781
	 */
782
	dmb(ishst);
783 784 785

	/* this always happens on GIC0 */
	writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
786 787 788 789 790 791

	raw_spin_unlock_irqrestore(&irq_controller_lock, flags);
}
#endif

#ifdef CONFIG_BL_SWITCHER
792 793 794 795 796 797 798 799 800 801 802 803 804 805
/*
 * gic_send_sgi - send a SGI directly to given CPU interface number
 *
 * cpu_id: the ID for the destination CPU interface
 * irq: the IPI number to send a SGI for
 */
void gic_send_sgi(unsigned int cpu_id, unsigned int irq)
{
	BUG_ON(cpu_id >= NR_GIC_CPU_IF);
	cpu_id = 1 << cpu_id;
	/* this always happens on GIC0 */
	writel_relaxed((cpu_id << 16) | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
}

806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
/*
 * gic_get_cpu_id - get the CPU interface ID for the specified CPU
 *
 * @cpu: the logical CPU number to get the GIC ID for.
 *
 * Return the CPU interface ID for the given logical CPU number,
 * or -1 if the CPU number is too large or the interface ID is
 * unknown (more than one bit set).
 */
int gic_get_cpu_id(unsigned int cpu)
{
	unsigned int cpu_bit;

	if (cpu >= NR_GIC_CPU_IF)
		return -1;
	cpu_bit = gic_cpu_map[cpu];
	if (cpu_bit & (cpu_bit - 1))
		return -1;
	return __ffs(cpu_bit);
}

827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
/*
 * gic_migrate_target - migrate IRQs to another CPU interface
 *
 * @new_cpu_id: the CPU target ID to migrate IRQs to
 *
 * Migrate all peripheral interrupts with a target matching the current CPU
 * to the interface corresponding to @new_cpu_id.  The CPU interface mapping
 * is also updated.  Targets to other CPU interfaces are unchanged.
 * This must be called with IRQs locally disabled.
 */
void gic_migrate_target(unsigned int new_cpu_id)
{
	unsigned int cur_cpu_id, gic_irqs, gic_nr = 0;
	void __iomem *dist_base;
	int i, ror_val, cpu = smp_processor_id();
	u32 val, cur_target_mask, active_mask;

844
	BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899

	dist_base = gic_data_dist_base(&gic_data[gic_nr]);
	if (!dist_base)
		return;
	gic_irqs = gic_data[gic_nr].gic_irqs;

	cur_cpu_id = __ffs(gic_cpu_map[cpu]);
	cur_target_mask = 0x01010101 << cur_cpu_id;
	ror_val = (cur_cpu_id - new_cpu_id) & 31;

	raw_spin_lock(&irq_controller_lock);

	/* Update the target interface for this logical CPU */
	gic_cpu_map[cpu] = 1 << new_cpu_id;

	/*
	 * Find all the peripheral interrupts targetting the current
	 * CPU interface and migrate them to the new CPU interface.
	 * We skip DIST_TARGET 0 to 7 as they are read-only.
	 */
	for (i = 8; i < DIV_ROUND_UP(gic_irqs, 4); i++) {
		val = readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4);
		active_mask = val & cur_target_mask;
		if (active_mask) {
			val &= ~active_mask;
			val |= ror32(active_mask, ror_val);
			writel_relaxed(val, dist_base + GIC_DIST_TARGET + i*4);
		}
	}

	raw_spin_unlock(&irq_controller_lock);

	/*
	 * Now let's migrate and clear any potential SGIs that might be
	 * pending for us (cur_cpu_id).  Since GIC_DIST_SGI_PENDING_SET
	 * is a banked register, we can only forward the SGI using
	 * GIC_DIST_SOFTINT.  The original SGI source is lost but Linux
	 * doesn't use that information anyway.
	 *
	 * For the same reason we do not adjust SGI source information
	 * for previously sent SGIs by us to other CPUs either.
	 */
	for (i = 0; i < 16; i += 4) {
		int j;
		val = readl_relaxed(dist_base + GIC_DIST_SGI_PENDING_SET + i);
		if (!val)
			continue;
		writel_relaxed(val, dist_base + GIC_DIST_SGI_PENDING_CLEAR + i);
		for (j = i; j < i + 4; j++) {
			if (val & 0xff)
				writel_relaxed((1 << (new_cpu_id + 16)) | j,
						dist_base + GIC_DIST_SOFTINT);
			val >>= 8;
		}
	}
900
}
901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927

/*
 * gic_get_sgir_physaddr - get the physical address for the SGI register
 *
 * REturn the physical address of the SGI register to be used
 * by some early assembly code when the kernel is not yet available.
 */
static unsigned long gic_dist_physaddr;

unsigned long gic_get_sgir_physaddr(void)
{
	if (!gic_dist_physaddr)
		return 0;
	return gic_dist_physaddr + GIC_DIST_SOFTINT;
}

void __init gic_init_physaddr(struct device_node *node)
{
	struct resource res;
	if (of_address_to_resource(node, 0, &res) == 0) {
		gic_dist_physaddr = res.start;
		pr_info("GIC physical location is %#lx\n", gic_dist_physaddr);
	}
}

#else
#define gic_init_physaddr(node)  do { } while (0)
928 929
#endif

930 931 932
static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
				irq_hw_number_t hw)
{
933
	struct gic_chip_data *gic = d->host_data;
934

935 936
	if (hw < 32) {
		irq_set_percpu_devid(irq);
937
		irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
938
				    handle_percpu_devid_irq, NULL, NULL);
939
		irq_set_status_flags(irq, IRQ_NOAUTOEN);
940
	} else {
941
		irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
942
				    handle_fasteoi_irq, NULL, NULL);
943
		irq_set_probe(irq);
944 945 946 947
	}
	return 0;
}

948 949 950 951
static void gic_irq_domain_unmap(struct irq_domain *d, unsigned int irq)
{
}

952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974
static int gic_irq_domain_translate(struct irq_domain *d,
				    struct irq_fwspec *fwspec,
				    unsigned long *hwirq,
				    unsigned int *type)
{
	if (is_of_node(fwspec->fwnode)) {
		if (fwspec->param_count < 3)
			return -EINVAL;

		/* Get the interrupt number and add 16 to skip over SGIs */
		*hwirq = fwspec->param[1] + 16;

		/*
		 * For SPIs, we need to add 16 more to get the GIC irq
		 * ID number
		 */
		if (!fwspec->param[0])
			*hwirq += 16;

		*type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
		return 0;
	}

975
	if (is_fwnode_irqchip(fwspec->fwnode)) {
976 977 978 979 980 981 982 983
		if(fwspec->param_count != 2)
			return -EINVAL;

		*hwirq = fwspec->param[0];
		*type = fwspec->param[1];
		return 0;
	}

984 985 986
	return -EINVAL;
}

987
#ifdef CONFIG_SMP
988 989
static int gic_secondary_init(struct notifier_block *nfb, unsigned long action,
			      void *hcpu)
990
{
991
	if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
992 993 994 995 996 997 998 999
		gic_cpu_init(&gic_data[0]);
	return NOTIFY_OK;
}

/*
 * Notifier for enabling the GIC CPU interface. Set an arbitrarily high
 * priority because the GIC needs to be up before the ARM generic timers.
 */
1000
static struct notifier_block gic_cpu_notifier = {
1001 1002 1003 1004 1005
	.notifier_call = gic_secondary_init,
	.priority = 100,
};
#endif

1006 1007 1008 1009 1010 1011
static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
				unsigned int nr_irqs, void *arg)
{
	int i, ret;
	irq_hw_number_t hwirq;
	unsigned int type = IRQ_TYPE_NONE;
1012
	struct irq_fwspec *fwspec = arg;
1013

1014
	ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
	if (ret)
		return ret;

	for (i = 0; i < nr_irqs; i++)
		gic_irq_domain_map(domain, virq + i, hwirq + i);

	return 0;
}

static const struct irq_domain_ops gic_irq_domain_hierarchy_ops = {
1025
	.translate = gic_irq_domain_translate,
1026 1027 1028 1029
	.alloc = gic_irq_domain_alloc,
	.free = irq_domain_free_irqs_top,
};

1030
static const struct irq_domain_ops gic_irq_domain_ops = {
1031
	.map = gic_irq_domain_map,
1032
	.unmap = gic_irq_domain_unmap,
R
Rob Herring 已提交
1033 1034
};

1035 1036
static void gic_init_chip(struct gic_chip_data *gic, struct device *dev,
			  const char *name, bool use_eoimode1)
1037
{
1038
	/* Initialize irq_chip */
1039
	gic->chip = gic_chip;
1040 1041
	gic->chip.name = name;
	gic->chip.parent_device = dev;
1042

1043
	if (use_eoimode1) {
1044 1045 1046
		gic->chip.irq_mask = gic_eoimode1_mask_irq;
		gic->chip.irq_eoi = gic_eoimode1_eoi_irq;
		gic->chip.irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity;
1047 1048
	}

1049
#ifdef CONFIG_SMP
1050
	if (gic == &gic_data[0])
1051 1052
		gic->chip.irq_set_affinity = gic_set_affinity;
#endif
1053 1054 1055 1056 1057 1058 1059
}

static int gic_init_bases(struct gic_chip_data *gic, int irq_start,
			  struct fwnode_handle *handle)
{
	irq_hw_number_t hwirq_base;
	int gic_irqs, irq_base, ret;
1060

1061
	if (IS_ENABLED(CONFIG_GIC_NON_BANKED) && gic->percpu_offset) {
1062
		/* Frankein-GIC without banked registers... */
1063 1064 1065 1066 1067 1068
		unsigned int cpu;

		gic->dist_base.percpu_base = alloc_percpu(void __iomem *);
		gic->cpu_base.percpu_base = alloc_percpu(void __iomem *);
		if (WARN_ON(!gic->dist_base.percpu_base ||
			    !gic->cpu_base.percpu_base)) {
1069 1070
			ret = -ENOMEM;
			goto error;
1071 1072 1073
		}

		for_each_possible_cpu(cpu) {
1074 1075
			u32 mpidr = cpu_logical_map(cpu);
			u32 core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
1076 1077 1078 1079 1080
			unsigned long offset = gic->percpu_offset * core_id;
			*per_cpu_ptr(gic->dist_base.percpu_base, cpu) =
				gic->raw_dist_base + offset;
			*per_cpu_ptr(gic->cpu_base.percpu_base, cpu) =
				gic->raw_cpu_base + offset;
1081 1082 1083
		}

		gic_set_base_accessor(gic, gic_get_percpu_base);
1084 1085
	} else {
		/* Normal, sane GIC... */
1086
		WARN(gic->percpu_offset,
1087
		     "GIC_NON_BANKED not enabled, ignoring %08x offset!",
1088 1089 1090
		     gic->percpu_offset);
		gic->dist_base.common_base = gic->raw_dist_base;
		gic->cpu_base.common_base = gic->raw_cpu_base;
1091 1092
		gic_set_base_accessor(gic, gic_get_common_base);
	}
1093

R
Rob Herring 已提交
1094 1095 1096 1097
	/*
	 * Find out how many interrupts are supported.
	 * The GIC only supports up to 1020 interrupt sources.
	 */
1098
	gic_irqs = readl_relaxed(gic_data_dist_base(gic) + GIC_DIST_CTR) & 0x1f;
R
Rob Herring 已提交
1099 1100 1101 1102 1103
	gic_irqs = (gic_irqs + 1) * 32;
	if (gic_irqs > 1020)
		gic_irqs = 1020;
	gic->gic_irqs = gic_irqs;

1104 1105 1106 1107 1108
	if (handle) {		/* DT/ACPI */
		gic->domain = irq_domain_create_linear(handle, gic_irqs,
						       &gic_irq_domain_hierarchy_ops,
						       gic);
	} else {		/* Legacy support */
1109 1110 1111 1112
		/*
		 * For primary GICs, skip over SGIs.
		 * For secondary GICs, skip over PPIs, too.
		 */
1113
		if (gic == &gic_data[0] && (irq_start & 31) > 0) {
1114 1115 1116 1117 1118 1119 1120 1121
			hwirq_base = 16;
			if (irq_start != -1)
				irq_start = (irq_start & ~31) + 16;
		} else {
			hwirq_base = 32;
		}

		gic_irqs -= hwirq_base; /* calculate # of irqs to allocate */
1122 1123 1124

		irq_base = irq_alloc_descs(irq_start, 16, gic_irqs,
					   numa_node_id());
1125
		if (irq_base < 0) {
1126 1127 1128 1129 1130
			WARN(1, "Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n",
			     irq_start);
			irq_base = irq_start;
		}

1131
		gic->domain = irq_domain_add_legacy(NULL, gic_irqs, irq_base,
1132
					hwirq_base, &gic_irq_domain_ops, gic);
1133
	}
1134

1135 1136 1137 1138
	if (WARN_ON(!gic->domain)) {
		ret = -ENODEV;
		goto error;
	}
1139

R
Rob Herring 已提交
1140
	gic_dist_init(gic);
1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
	ret = gic_cpu_init(gic);
	if (ret)
		goto error;

	ret = gic_pm_init(gic);
	if (ret)
		goto error;

	return 0;

error:
1152
	if (IS_ENABLED(CONFIG_GIC_NON_BANKED) && gic->percpu_offset) {
1153 1154 1155 1156 1157
		free_percpu(gic->dist_base.percpu_base);
		free_percpu(gic->cpu_base.percpu_base);
	}

	return ret;
1158 1159
}

1160 1161 1162 1163
static int __init __gic_init_bases(struct gic_chip_data *gic,
				   int irq_start,
				   struct fwnode_handle *handle)
{
1164 1165
	char *name;
	int i, ret;
1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186

	if (WARN_ON(!gic || gic->domain))
		return -EINVAL;

	if (gic == &gic_data[0]) {
		/*
		 * Initialize the CPU interface map to all CPUs.
		 * It will be refined as each CPU probes its ID.
		 * This is only necessary for the primary GIC.
		 */
		for (i = 0; i < NR_GIC_CPU_IF; i++)
			gic_cpu_map[i] = 0xff;
#ifdef CONFIG_SMP
		set_smp_cross_call(gic_raise_softirq);
		register_cpu_notifier(&gic_cpu_notifier);
#endif
		set_handle_irq(gic_handle_irq);
		if (static_key_true(&supports_deactivate))
			pr_info("GIC: Using split EOI/Deactivate mode\n");
	}

1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199
	if (static_key_true(&supports_deactivate) && gic == &gic_data[0]) {
		name = kasprintf(GFP_KERNEL, "GICv2");
		gic_init_chip(gic, NULL, name, true);
	} else {
		name = kasprintf(GFP_KERNEL, "GIC-%d", (int)(gic-&gic_data[0]));
		gic_init_chip(gic, NULL, name, false);
	}

	ret = gic_init_bases(gic, irq_start, handle);
	if (ret)
		kfree(name);

	return ret;
1200 1201
}

1202 1203
void __init gic_init(unsigned int gic_nr, int irq_start,
		     void __iomem *dist_base, void __iomem *cpu_base)
1204
{
1205 1206 1207 1208 1209
	struct gic_chip_data *gic;

	if (WARN_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR))
		return;

1210 1211 1212 1213 1214
	/*
	 * Non-DT/ACPI systems won't run a hypervisor, so let's not
	 * bother with these...
	 */
	static_key_slow_dec(&supports_deactivate);
1215 1216 1217 1218 1219 1220

	gic = &gic_data[gic_nr];
	gic->raw_dist_base = dist_base;
	gic->raw_cpu_base = cpu_base;

	__gic_init_bases(gic, irq_start, NULL);
1221 1222
}

1223 1224 1225 1226 1227 1228 1229 1230 1231
static void gic_teardown(struct gic_chip_data *gic)
{
	if (WARN_ON(!gic))
		return;

	if (gic->raw_dist_base)
		iounmap(gic->raw_dist_base);
	if (gic->raw_cpu_base)
		iounmap(gic->raw_cpu_base);
1232 1233
}

1234
#ifdef CONFIG_OF
1235
static int gic_cnt __initdata;
1236

1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
static bool gic_check_eoimode(struct device_node *node, void __iomem **base)
{
	struct resource cpuif_res;

	of_address_to_resource(node, 1, &cpuif_res);

	if (!is_hyp_mode_available())
		return false;
	if (resource_size(&cpuif_res) < SZ_8K)
		return false;
	if (resource_size(&cpuif_res) == SZ_128K) {
		u32 val_low, val_high;

		/*
		 * Verify that we have the first 4kB of a GIC400
		 * aliased over the first 64kB by checking the
		 * GICC_IIDR register on both ends.
		 */
		val_low = readl_relaxed(*base + GIC_CPU_IDENT);
		val_high = readl_relaxed(*base + GIC_CPU_IDENT + 0xf000);
		if ((val_low & 0xffff0fff) != 0x0202043B ||
		    val_low != val_high)
			return false;

		/*
		 * Move the base up by 60kB, so that we have a 8kB
		 * contiguous region, which allows us to use GICC_DIR
		 * at its normal offset. Please pass me that bucket.
		 */
		*base += 0xf000;
		cpuif_res.start += 0xf000;
		pr_warn("GIC: Adjusting CPU interface base to %pa",
			&cpuif_res.start);
	}

	return true;
}

1275
static int __init gic_of_setup(struct gic_chip_data *gic, struct device_node *node)
1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298
{
	if (!gic || !node)
		return -EINVAL;

	gic->raw_dist_base = of_iomap(node, 0);
	if (WARN(!gic->raw_dist_base, "unable to map gic dist registers\n"))
		goto error;

	gic->raw_cpu_base = of_iomap(node, 1);
	if (WARN(!gic->raw_cpu_base, "unable to map gic cpu registers\n"))
		goto error;

	if (of_property_read_u32(node, "cpu-offset", &gic->percpu_offset))
		gic->percpu_offset = 0;

	return 0;

error:
	gic_teardown(gic);

	return -ENOMEM;
}

1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321
static void __init gic_of_setup_kvm_info(struct device_node *node)
{
	int ret;
	struct resource *vctrl_res = &gic_v2_kvm_info.vctrl;
	struct resource *vcpu_res = &gic_v2_kvm_info.vcpu;

	gic_v2_kvm_info.type = GIC_V2;

	gic_v2_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
	if (!gic_v2_kvm_info.maint_irq)
		return;

	ret = of_address_to_resource(node, 2, vctrl_res);
	if (ret)
		return;

	ret = of_address_to_resource(node, 3, vcpu_res);
	if (ret)
		return;

	gic_set_kvm_info(&gic_v2_kvm_info);
}

1322
int __init
1323
gic_of_init(struct device_node *node, struct device_node *parent)
1324
{
1325
	struct gic_chip_data *gic;
1326
	int irq, ret;
1327 1328 1329 1330

	if (WARN_ON(!node))
		return -ENODEV;

1331 1332 1333 1334
	if (WARN_ON(gic_cnt >= CONFIG_ARM_GIC_MAX_NR))
		return -EINVAL;

	gic = &gic_data[gic_cnt];
1335

1336 1337 1338
	ret = gic_of_setup(gic, node);
	if (ret)
		return ret;
1339

1340 1341 1342 1343
	/*
	 * Disable split EOI/Deactivate if either HYP is not available
	 * or the CPU interface is too small.
	 */
1344
	if (gic_cnt == 0 && !gic_check_eoimode(node, &gic->raw_cpu_base))
1345 1346
		static_key_slow_dec(&supports_deactivate);

1347
	ret = __gic_init_bases(gic, -1, &node->fwnode);
1348
	if (ret) {
1349
		gic_teardown(gic);
1350 1351
		return ret;
	}
1352

1353
	if (!gic_cnt) {
1354
		gic_init_physaddr(node);
1355 1356
		gic_of_setup_kvm_info(node);
	}
1357 1358 1359 1360 1361

	if (parent) {
		irq = irq_of_parse_and_map(node, 0);
		gic_cascade_irq(gic_cnt, irq);
	}
1362 1363

	if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
1364
		gicv2m_init(&node->fwnode, gic_data[gic_cnt].domain);
1365

1366 1367 1368
	gic_cnt++;
	return 0;
}
1369
IRQCHIP_DECLARE(gic_400, "arm,gic-400", gic_of_init);
1370 1371
IRQCHIP_DECLARE(arm11mp_gic, "arm,arm11mp-gic", gic_of_init);
IRQCHIP_DECLARE(arm1176jzf_dc_gic, "arm,arm1176jzf-devchip-gic", gic_of_init);
1372 1373
IRQCHIP_DECLARE(cortex_a15_gic, "arm,cortex-a15-gic", gic_of_init);
IRQCHIP_DECLARE(cortex_a9_gic, "arm,cortex-a9-gic", gic_of_init);
1374
IRQCHIP_DECLARE(cortex_a7_gic, "arm,cortex-a7-gic", gic_of_init);
1375 1376
IRQCHIP_DECLARE(msm_8660_qgic, "qcom,msm-8660-qgic", gic_of_init);
IRQCHIP_DECLARE(msm_qgic2, "qcom,msm-qgic2", gic_of_init);
1377
IRQCHIP_DECLARE(pl390, "arm,pl390", gic_of_init);
1378

1379
#endif
1380 1381

#ifdef CONFIG_ACPI
1382 1383 1384
static struct
{
	phys_addr_t cpu_phys_base;
1385 1386 1387 1388
	u32 maint_irq;
	int maint_irq_mode;
	phys_addr_t vctrl_base;
	phys_addr_t vcpu_base;
1389
} acpi_data __initdata;
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400

static int __init
gic_acpi_parse_madt_cpu(struct acpi_subtable_header *header,
			const unsigned long end)
{
	struct acpi_madt_generic_interrupt *processor;
	phys_addr_t gic_cpu_base;
	static int cpu_base_assigned;

	processor = (struct acpi_madt_generic_interrupt *)header;

1401
	if (BAD_MADT_GICC_ENTRY(processor, end))
1402 1403 1404 1405 1406 1407 1408
		return -EINVAL;

	/*
	 * There is no support for non-banked GICv1/2 register in ACPI spec.
	 * All CPU interface addresses have to be the same.
	 */
	gic_cpu_base = processor->base_address;
1409
	if (cpu_base_assigned && gic_cpu_base != acpi_data.cpu_phys_base)
1410 1411
		return -EINVAL;

1412
	acpi_data.cpu_phys_base = gic_cpu_base;
1413 1414 1415 1416 1417 1418
	acpi_data.maint_irq = processor->vgic_interrupt;
	acpi_data.maint_irq_mode = (processor->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
				    ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
	acpi_data.vctrl_base = processor->gich_base_address;
	acpi_data.vcpu_base = processor->gicv_base_address;

1419 1420 1421 1422
	cpu_base_assigned = 1;
	return 0;
}

1423 1424 1425
/* The things you have to do to just *count* something... */
static int __init acpi_dummy_func(struct acpi_subtable_header *header,
				  const unsigned long end)
1426
{
1427 1428
	return 0;
}
1429

1430 1431 1432 1433 1434
static bool __init acpi_gic_redist_is_present(void)
{
	return acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
				     acpi_dummy_func, 0) > 0;
}
1435

1436 1437 1438 1439 1440
static bool __init gic_validate_dist(struct acpi_subtable_header *header,
				     struct acpi_probe_entry *ape)
{
	struct acpi_madt_generic_distributor *dist;
	dist = (struct acpi_madt_generic_distributor *)header;
1441

1442 1443 1444
	return (dist->version == ape->driver_data &&
		(dist->version != ACPI_MADT_GIC_VERSION_NONE ||
		 !acpi_gic_redist_is_present()));
1445 1446
}

1447 1448
#define ACPI_GICV2_DIST_MEM_SIZE	(SZ_4K)
#define ACPI_GIC_CPU_IF_MEM_SIZE	(SZ_8K)
1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
#define ACPI_GICV2_VCTRL_MEM_SIZE	(SZ_4K)
#define ACPI_GICV2_VCPU_MEM_SIZE	(SZ_8K)

static void __init gic_acpi_setup_kvm_info(void)
{
	int irq;
	struct resource *vctrl_res = &gic_v2_kvm_info.vctrl;
	struct resource *vcpu_res = &gic_v2_kvm_info.vcpu;

	gic_v2_kvm_info.type = GIC_V2;

	if (!acpi_data.vctrl_base)
		return;

	vctrl_res->flags = IORESOURCE_MEM;
	vctrl_res->start = acpi_data.vctrl_base;
	vctrl_res->end = vctrl_res->start + ACPI_GICV2_VCTRL_MEM_SIZE - 1;

	if (!acpi_data.vcpu_base)
		return;

	vcpu_res->flags = IORESOURCE_MEM;
	vcpu_res->start = acpi_data.vcpu_base;
	vcpu_res->end = vcpu_res->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;

	irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
				acpi_data.maint_irq_mode,
				ACPI_ACTIVE_HIGH);
	if (irq <= 0)
		return;

	gic_v2_kvm_info.maint_irq = irq;

	gic_set_kvm_info(&gic_v2_kvm_info);
}
1484 1485 1486

static int __init gic_v2_acpi_init(struct acpi_subtable_header *header,
				   const unsigned long end)
1487
{
1488
	struct acpi_madt_generic_distributor *dist;
1489
	struct fwnode_handle *domain_handle;
1490
	struct gic_chip_data *gic = &gic_data[0];
1491
	int count, ret;
1492 1493

	/* Collect CPU base addresses */
1494 1495
	count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
				      gic_acpi_parse_madt_cpu, 0);
1496 1497 1498 1499 1500
	if (count <= 0) {
		pr_err("No valid GICC entries exist\n");
		return -EINVAL;
	}

1501
	gic->raw_cpu_base = ioremap(acpi_data.cpu_phys_base, ACPI_GIC_CPU_IF_MEM_SIZE);
1502
	if (!gic->raw_cpu_base) {
1503 1504 1505 1506
		pr_err("Unable to map GICC registers\n");
		return -ENOMEM;
	}

1507
	dist = (struct acpi_madt_generic_distributor *)header;
1508 1509 1510
	gic->raw_dist_base = ioremap(dist->base_address,
				     ACPI_GICV2_DIST_MEM_SIZE);
	if (!gic->raw_dist_base) {
1511
		pr_err("Unable to map GICD registers\n");
1512
		gic_teardown(gic);
1513 1514 1515
		return -ENOMEM;
	}

1516 1517 1518 1519 1520 1521 1522 1523
	/*
	 * Disable split EOI/Deactivate if HYP is not available. ACPI
	 * guarantees that we'll always have a GICv2, so the CPU
	 * interface will always be the right size.
	 */
	if (!is_hyp_mode_available())
		static_key_slow_dec(&supports_deactivate);

1524
	/*
1525
	 * Initialize GIC instance zero (no multi-GIC support).
1526
	 */
1527
	domain_handle = irq_domain_alloc_fwnode(gic->raw_dist_base);
1528 1529
	if (!domain_handle) {
		pr_err("Unable to allocate domain handle\n");
1530
		gic_teardown(gic);
1531 1532 1533
		return -ENOMEM;
	}

1534
	ret = __gic_init_bases(gic, -1, domain_handle);
1535 1536 1537
	if (ret) {
		pr_err("Failed to initialise GIC\n");
		irq_domain_free_fwnode(domain_handle);
1538
		gic_teardown(gic);
1539 1540
		return ret;
	}
1541

1542
	acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle);
1543 1544 1545 1546

	if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
		gicv2m_init(NULL, gic_data[0].domain);

1547 1548
	gic_acpi_setup_kvm_info();

1549 1550
	return 0;
}
1551 1552 1553 1554 1555 1556
IRQCHIP_ACPI_DECLARE(gic_v2, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
		     gic_validate_dist, ACPI_MADT_GIC_VERSION_V2,
		     gic_v2_acpi_init);
IRQCHIP_ACPI_DECLARE(gic_v2_maybe, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
		     gic_validate_dist, ACPI_MADT_GIC_VERSION_NONE,
		     gic_v2_acpi_init);
1557
#endif