irq-gic-v3.c 21.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/cpu.h>
19
#include <linux/cpu_pm.h>
20 21 22 23 24 25 26 27
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/percpu.h>
#include <linux/slab.h>

28
#include <linux/irqchip.h>
29 30 31 32 33
#include <linux/irqchip/arm-gic-v3.h>

#include <asm/cputype.h>
#include <asm/exception.h>
#include <asm/smp_plat.h>
34
#include <asm/virt.h>
35 36 37

#include "irq-gic-common.h"

38 39 40 41 42
struct redist_region {
	void __iomem		*redist_base;
	phys_addr_t		phys_base;
};

43 44
struct gic_chip_data {
	void __iomem		*dist_base;
45 46
	struct redist_region	*redist_regions;
	struct rdists		rdists;
47 48
	struct irq_domain	*domain;
	u64			redist_stride;
49
	u32			nr_redist_regions;
50 51 52 53
	unsigned int		irq_nr;
};

static struct gic_chip_data gic_data __read_mostly;
54
static struct static_key supports_deactivate = STATIC_KEY_INIT_TRUE;
55

56 57
#define gic_data_rdist()		(this_cpu_ptr(gic_data.rdists.rdist))
#define gic_data_rdist_rd_base()	(gic_data_rdist()->rd_base)
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
#define gic_data_rdist_sgi_base()	(gic_data_rdist_rd_base() + SZ_64K)

/* Our default, arbitrary priority value. Linux only uses one anyway. */
#define DEFAULT_PMR_VALUE	0xf0

static inline unsigned int gic_irq(struct irq_data *d)
{
	return d->hwirq;
}

static inline int gic_irq_in_rdist(struct irq_data *d)
{
	return gic_irq(d) < 32;
}

static inline void __iomem *gic_dist_base(struct irq_data *d)
{
	if (gic_irq_in_rdist(d))	/* SGI+PPI -> SGI_base for this CPU */
		return gic_data_rdist_sgi_base();

	if (d->hwirq <= 1023)		/* SPI -> dist_base */
		return gic_data.dist_base;

	return NULL;
}

static void gic_do_wait_for_rwp(void __iomem *base)
{
	u32 count = 1000000;	/* 1s! */

	while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
		count--;
		if (!count) {
			pr_err_ratelimited("RWP timeout, gone fishing\n");
			return;
		}
		cpu_relax();
		udelay(1);
	};
}

/* Wait for completion of a distributor change */
static void gic_dist_wait_for_rwp(void)
{
	gic_do_wait_for_rwp(gic_data.dist_base);
}

/* Wait for completion of a redistributor change */
static void gic_redist_wait_for_rwp(void)
{
	gic_do_wait_for_rwp(gic_data_rdist_rd_base());
}

111
#ifdef CONFIG_ARM64
112
static DEFINE_STATIC_KEY_FALSE(is_cavium_thunderx);
113 114 115

static u64 __maybe_unused gic_read_iar(void)
{
116
	if (static_branch_unlikely(&is_cavium_thunderx))
117 118 119 120
		return gic_read_iar_cavium_thunderx();
	else
		return gic_read_iar_common();
}
121
#endif
122

123
static void gic_enable_redist(bool enable)
124 125 126 127 128 129 130 131
{
	void __iomem *rbase;
	u32 count = 1000000;	/* 1s! */
	u32 val;

	rbase = gic_data_rdist_rd_base();

	val = readl_relaxed(rbase + GICR_WAKER);
132 133 134 135 136
	if (enable)
		/* Wake up this CPU redistributor */
		val &= ~GICR_WAKER_ProcessorSleep;
	else
		val |= GICR_WAKER_ProcessorSleep;
137 138
	writel_relaxed(val, rbase + GICR_WAKER);

139 140 141 142 143 144 145 146 147 148
	if (!enable) {		/* Check that GICR_WAKER is writeable */
		val = readl_relaxed(rbase + GICR_WAKER);
		if (!(val & GICR_WAKER_ProcessorSleep))
			return;	/* No PM support in this redistributor */
	}

	while (count--) {
		val = readl_relaxed(rbase + GICR_WAKER);
		if (enable ^ (val & GICR_WAKER_ChildrenAsleep))
			break;
149 150 151
		cpu_relax();
		udelay(1);
	};
152 153 154
	if (!count)
		pr_err_ratelimited("redistributor failed to %s...\n",
				   enable ? "wakeup" : "sleep");
155 156 157 158 159
}

/*
 * Routines to disable, enable, EOI and route interrupts
 */
160 161 162 163 164 165 166 167 168 169 170 171 172
static int gic_peek_irq(struct irq_data *d, u32 offset)
{
	u32 mask = 1 << (gic_irq(d) % 32);
	void __iomem *base;

	if (gic_irq_in_rdist(d))
		base = gic_data_rdist_sgi_base();
	else
		base = gic_data.dist_base;

	return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask);
}

173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
static void gic_poke_irq(struct irq_data *d, u32 offset)
{
	u32 mask = 1 << (gic_irq(d) % 32);
	void (*rwp_wait)(void);
	void __iomem *base;

	if (gic_irq_in_rdist(d)) {
		base = gic_data_rdist_sgi_base();
		rwp_wait = gic_redist_wait_for_rwp;
	} else {
		base = gic_data.dist_base;
		rwp_wait = gic_dist_wait_for_rwp;
	}

	writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4);
	rwp_wait();
}

static void gic_mask_irq(struct irq_data *d)
{
	gic_poke_irq(d, GICD_ICENABLER);
}

196 197 198
static void gic_eoimode1_mask_irq(struct irq_data *d)
{
	gic_mask_irq(d);
199 200 201 202 203 204 205 206
	/*
	 * When masking a forwarded interrupt, make sure it is
	 * deactivated as well.
	 *
	 * This ensures that an interrupt that is getting
	 * disabled/masked will not get "stuck", because there is
	 * noone to deactivate it (guest is being terminated).
	 */
207
	if (irqd_is_forwarded_to_vcpu(d))
208
		gic_poke_irq(d, GICD_ICACTIVER);
209 210
}

211 212 213 214 215
static void gic_unmask_irq(struct irq_data *d)
{
	gic_poke_irq(d, GICD_ISENABLER);
}

216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
static int gic_irq_set_irqchip_state(struct irq_data *d,
				     enum irqchip_irq_state which, bool val)
{
	u32 reg;

	if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
		return -EINVAL;

	switch (which) {
	case IRQCHIP_STATE_PENDING:
		reg = val ? GICD_ISPENDR : GICD_ICPENDR;
		break;

	case IRQCHIP_STATE_ACTIVE:
		reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
		break;

	case IRQCHIP_STATE_MASKED:
		reg = val ? GICD_ICENABLER : GICD_ISENABLER;
		break;

	default:
		return -EINVAL;
	}

	gic_poke_irq(d, reg);
	return 0;
}

static int gic_irq_get_irqchip_state(struct irq_data *d,
				     enum irqchip_irq_state which, bool *val)
{
	if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
		return -EINVAL;

	switch (which) {
	case IRQCHIP_STATE_PENDING:
		*val = gic_peek_irq(d, GICD_ISPENDR);
		break;

	case IRQCHIP_STATE_ACTIVE:
		*val = gic_peek_irq(d, GICD_ISACTIVER);
		break;

	case IRQCHIP_STATE_MASKED:
		*val = !gic_peek_irq(d, GICD_ISENABLER);
		break;

	default:
		return -EINVAL;
	}

	return 0;
}

271 272 273 274 275
static void gic_eoi_irq(struct irq_data *d)
{
	gic_write_eoir(gic_irq(d));
}

276 277 278
static void gic_eoimode1_eoi_irq(struct irq_data *d)
{
	/*
279 280
	 * No need to deactivate an LPI, or an interrupt that
	 * is is getting forwarded to a vcpu.
281
	 */
282
	if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
283 284 285 286
		return;
	gic_write_dir(gic_irq(d));
}

287 288 289 290 291 292 293 294 295 296
static int gic_set_type(struct irq_data *d, unsigned int type)
{
	unsigned int irq = gic_irq(d);
	void (*rwp_wait)(void);
	void __iomem *base;

	/* Interrupt configuration for SGIs can't be changed */
	if (irq < 16)
		return -EINVAL;

297 298 299
	/* SPIs have restrictions on the supported types */
	if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
			 type != IRQ_TYPE_EDGE_RISING)
300 301 302 303 304 305 306 307 308 309
		return -EINVAL;

	if (gic_irq_in_rdist(d)) {
		base = gic_data_rdist_sgi_base();
		rwp_wait = gic_redist_wait_for_rwp;
	} else {
		base = gic_data.dist_base;
		rwp_wait = gic_dist_wait_for_rwp;
	}

310
	return gic_configure_irq(irq, type, base, rwp_wait);
311 312
}

313 314
static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
{
315 316 317 318
	if (vcpu)
		irqd_set_forwarded_to_vcpu(d);
	else
		irqd_clr_forwarded_to_vcpu(d);
319 320 321
	return 0;
}

322
static u64 gic_mpidr_to_affinity(unsigned long mpidr)
323 324 325
{
	u64 aff;

326
	aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
327 328 329 330 331 332 333 334 335
	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
	       MPIDR_AFFINITY_LEVEL(mpidr, 0));

	return aff;
}

static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
{
336
	u32 irqnr;
337 338 339 340

	do {
		irqnr = gic_read_iar();

341
		if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) {
342
			int err;
343 344 345 346

			if (static_key_true(&supports_deactivate))
				gic_write_eoir(irqnr);

347 348
			err = handle_domain_irq(gic_data.domain, irqnr, regs);
			if (err) {
349
				WARN_ONCE(true, "Unexpected interrupt received!\n");
350 351 352 353 354 355
				if (static_key_true(&supports_deactivate)) {
					if (irqnr < 8192)
						gic_write_dir(irqnr);
				} else {
					gic_write_eoir(irqnr);
				}
356
			}
357
			continue;
358 359 360
		}
		if (irqnr < 16) {
			gic_write_eoir(irqnr);
361 362
			if (static_key_true(&supports_deactivate))
				gic_write_dir(irqnr);
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394
#ifdef CONFIG_SMP
			handle_IPI(irqnr, regs);
#else
			WARN_ONCE(true, "Unexpected SGI received!\n");
#endif
			continue;
		}
	} while (irqnr != ICC_IAR1_EL1_SPURIOUS);
}

static void __init gic_dist_init(void)
{
	unsigned int i;
	u64 affinity;
	void __iomem *base = gic_data.dist_base;

	/* Disable the distributor */
	writel_relaxed(0, base + GICD_CTLR);
	gic_dist_wait_for_rwp();

	gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);

	/* Enable distributor with ARE, Group1 */
	writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
		       base + GICD_CTLR);

	/*
	 * Set all global interrupts to the boot CPU only. ARE must be
	 * enabled.
	 */
	affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
	for (i = 32; i < gic_data.irq_nr; i++)
395
		gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
396 397 398 399
}

static int gic_populate_rdist(void)
{
400
	unsigned long mpidr = cpu_logical_map(smp_processor_id());
401 402 403 404 405 406 407 408 409 410 411 412 413
	u64 typer;
	u32 aff;
	int i;

	/*
	 * Convert affinity to a 32bit value that can be matched to
	 * GICR_TYPER bits [63:32].
	 */
	aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
	       MPIDR_AFFINITY_LEVEL(mpidr, 0));

414 415
	for (i = 0; i < gic_data.nr_redist_regions; i++) {
		void __iomem *ptr = gic_data.redist_regions[i].redist_base;
416 417 418 419 420 421 422 423 424 425
		u32 reg;

		reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
		if (reg != GIC_PIDR2_ARCH_GICv3 &&
		    reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
			pr_warn("No redistributor present @%p\n", ptr);
			break;
		}

		do {
426
			typer = gic_read_typer(ptr + GICR_TYPER);
427
			if ((typer >> 32) == aff) {
428
				u64 offset = ptr - gic_data.redist_regions[i].redist_base;
429
				gic_data_rdist_rd_base() = ptr;
430
				gic_data_rdist()->phys_base = gic_data.redist_regions[i].phys_base + offset;
431 432 433
				pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
					smp_processor_id(), mpidr, i,
					&gic_data_rdist()->phys_base);
434 435 436 437 438 439 440 441 442 443 444 445 446 447
				return 0;
			}

			if (gic_data.redist_stride) {
				ptr += gic_data.redist_stride;
			} else {
				ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
				if (typer & GICR_TYPER_VLPIS)
					ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
			}
		} while (!(typer & GICR_TYPER_LAST));
	}

	/* We couldn't even deal with ourselves... */
448 449
	WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
	     smp_processor_id(), mpidr);
450 451 452
	return -ENODEV;
}

453 454
static void gic_cpu_sys_reg_init(void)
{
455 456 457 458 459 460 461 462 463
	/*
	 * Need to check that the SRE bit has actually been set. If
	 * not, it means that SRE is disabled at EL2. We're going to
	 * die painfully, and there is nothing we can do about it.
	 *
	 * Kindly inform the luser.
	 */
	if (!gic_enable_sre())
		pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
464 465 466 467

	/* Set priority mask register */
	gic_write_pmr(DEFAULT_PMR_VALUE);

468 469 470 471 472 473 474
	if (static_key_true(&supports_deactivate)) {
		/* EOI drops priority only (mode 1) */
		gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
	} else {
		/* EOI deactivates interrupt too (mode 0) */
		gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
	}
475 476 477 478 479

	/* ... and let's hit the road... */
	gic_write_grpen1(1);
}

480 481 482 483 484
static int gic_dist_supports_lpis(void)
{
	return !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS);
}

485 486 487 488 489 490 491 492
static void gic_cpu_init(void)
{
	void __iomem *rbase;

	/* Register ourselves with the rest of the world */
	if (gic_populate_rdist())
		return;

493
	gic_enable_redist(true);
494 495 496 497 498

	rbase = gic_data_rdist_sgi_base();

	gic_cpu_config(rbase, gic_redist_wait_for_rwp);

499 500 501 502
	/* Give LPIs a spin */
	if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
		its_cpu_init();

503 504
	/* initialise system registers */
	gic_cpu_sys_reg_init();
505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
}

#ifdef CONFIG_SMP
static int gic_secondary_init(struct notifier_block *nfb,
			      unsigned long action, void *hcpu)
{
	if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
		gic_cpu_init();
	return NOTIFY_OK;
}

/*
 * Notifier for enabling the GIC CPU interface. Set an arbitrarily high
 * priority because the GIC needs to be up before the ARM generic timers.
 */
static struct notifier_block gic_cpu_notifier = {
	.notifier_call = gic_secondary_init,
	.priority = 100,
};

static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
526
				   unsigned long cluster_id)
527 528
{
	int cpu = *base_cpu;
529
	unsigned long mpidr = cpu_logical_map(cpu);
530 531 532 533 534 535 536 537 538 539 540 541 542
	u16 tlist = 0;

	while (cpu < nr_cpu_ids) {
		/*
		 * If we ever get a cluster of more than 16 CPUs, just
		 * scream and skip that CPU.
		 */
		if (WARN_ON((mpidr & 0xff) >= 16))
			goto out;

		tlist |= 1 << (mpidr & 0xf);

		cpu = cpumask_next(cpu, mask);
543
		if (cpu >= nr_cpu_ids)
544 545 546 547 548 549 550 551 552 553 554 555 556 557
			goto out;

		mpidr = cpu_logical_map(cpu);

		if (cluster_id != (mpidr & ~0xffUL)) {
			cpu--;
			goto out;
		}
	}
out:
	*base_cpu = cpu;
	return tlist;
}

558 559 560 561
#define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
	(MPIDR_AFFINITY_LEVEL(cluster_id, level) \
		<< ICC_SGI1R_AFFINITY_## level ##_SHIFT)

562 563 564 565
static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
{
	u64 val;

566 567 568 569 570
	val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3)	|
	       MPIDR_TO_SGI_AFFINITY(cluster_id, 2)	|
	       irq << ICC_SGI1R_SGI_ID_SHIFT		|
	       MPIDR_TO_SGI_AFFINITY(cluster_id, 1)	|
	       tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588

	pr_debug("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
	gic_write_sgi1r(val);
}

static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
{
	int cpu;

	if (WARN_ON(irq >= 16))
		return;

	/*
	 * Ensure that stores to Normal memory are visible to the
	 * other CPUs before issuing the IPI.
	 */
	smp_wmb();

589
	for_each_cpu(cpu, mask) {
590
		unsigned long cluster_id = cpu_logical_map(cpu) & ~0xffUL;
591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
		u16 tlist;

		tlist = gic_compute_target_list(&cpu, mask, cluster_id);
		gic_send_sgi(cluster_id, tlist, irq);
	}

	/* Force the above writes to ICC_SGI1R_EL1 to be executed */
	isb();
}

static void gic_smp_init(void)
{
	set_smp_cross_call(gic_raise_softirq);
	register_cpu_notifier(&gic_cpu_notifier);
}

static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
			    bool force)
{
	unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask);
	void __iomem *reg;
	int enabled;
	u64 val;

	if (gic_irq_in_rdist(d))
		return -EINVAL;

	/* If interrupt was enabled, disable it first */
	enabled = gic_peek_irq(d, GICD_ISENABLER);
	if (enabled)
		gic_mask_irq(d);

	reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8);
	val = gic_mpidr_to_affinity(cpu_logical_map(cpu));

626
	gic_write_irouter(val, reg);
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643

	/*
	 * If the interrupt was enabled, enabled it again. Otherwise,
	 * just wait for the distributor to have digested our changes.
	 */
	if (enabled)
		gic_unmask_irq(d);
	else
		gic_dist_wait_for_rwp();

	return IRQ_SET_MASK_OK;
}
#else
#define gic_set_affinity	NULL
#define gic_smp_init()		do { } while(0)
#endif

644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670
#ifdef CONFIG_CPU_PM
static int gic_cpu_pm_notifier(struct notifier_block *self,
			       unsigned long cmd, void *v)
{
	if (cmd == CPU_PM_EXIT) {
		gic_enable_redist(true);
		gic_cpu_sys_reg_init();
	} else if (cmd == CPU_PM_ENTER) {
		gic_write_grpen1(0);
		gic_enable_redist(false);
	}
	return NOTIFY_OK;
}

static struct notifier_block gic_cpu_pm_notifier_block = {
	.notifier_call = gic_cpu_pm_notifier,
};

static void gic_cpu_pm_init(void)
{
	cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
}

#else
static inline void gic_cpu_pm_init(void) { }
#endif /* CONFIG_CPU_PM */

671 672 673 674 675 676 677
static struct irq_chip gic_chip = {
	.name			= "GICv3",
	.irq_mask		= gic_mask_irq,
	.irq_unmask		= gic_unmask_irq,
	.irq_eoi		= gic_eoi_irq,
	.irq_set_type		= gic_set_type,
	.irq_set_affinity	= gic_set_affinity,
678 679
	.irq_get_irqchip_state	= gic_irq_get_irqchip_state,
	.irq_set_irqchip_state	= gic_irq_set_irqchip_state,
680
	.flags			= IRQCHIP_SET_TYPE_MASKED,
681 682
};

683 684 685 686 687 688 689 690 691
static struct irq_chip gic_eoimode1_chip = {
	.name			= "GICv3",
	.irq_mask		= gic_eoimode1_mask_irq,
	.irq_unmask		= gic_unmask_irq,
	.irq_eoi		= gic_eoimode1_eoi_irq,
	.irq_set_type		= gic_set_type,
	.irq_set_affinity	= gic_set_affinity,
	.irq_get_irqchip_state	= gic_irq_get_irqchip_state,
	.irq_set_irqchip_state	= gic_irq_set_irqchip_state,
692
	.irq_set_vcpu_affinity	= gic_irq_set_vcpu_affinity,
693 694 695
	.flags			= IRQCHIP_SET_TYPE_MASKED,
};

696 697
#define GIC_ID_NR		(1U << gic_data.rdists.id_bits)

698 699 700
static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
			      irq_hw_number_t hw)
{
701 702 703 704 705
	struct irq_chip *chip = &gic_chip;

	if (static_key_true(&supports_deactivate))
		chip = &gic_eoimode1_chip;

706 707 708
	/* SGIs are private to the core kernel */
	if (hw < 16)
		return -EPERM;
709 710 711 712 713 714 715
	/* Nothing here */
	if (hw >= gic_data.irq_nr && hw < 8192)
		return -EPERM;
	/* Off limits */
	if (hw >= GIC_ID_NR)
		return -EPERM;

716 717 718
	/* PPIs */
	if (hw < 32) {
		irq_set_percpu_devid(irq);
719
		irq_domain_set_info(d, irq, hw, chip, d->host_data,
720
				    handle_percpu_devid_irq, NULL, NULL);
721
		irq_set_status_flags(irq, IRQ_NOAUTOEN);
722 723 724
	}
	/* SPIs */
	if (hw >= 32 && hw < gic_data.irq_nr) {
725
		irq_domain_set_info(d, irq, hw, chip, d->host_data,
726
				    handle_fasteoi_irq, NULL, NULL);
727
		irq_set_probe(irq);
728
	}
729 730 731 732
	/* LPIs */
	if (hw >= 8192 && hw < GIC_ID_NR) {
		if (!gic_dist_supports_lpis())
			return -EPERM;
733
		irq_domain_set_info(d, irq, hw, chip, d->host_data,
734 735 736
				    handle_fasteoi_irq, NULL, NULL);
	}

737 738 739 740 741 742 743 744
	return 0;
}

static int gic_irq_domain_xlate(struct irq_domain *d,
				struct device_node *controller,
				const u32 *intspec, unsigned int intsize,
				unsigned long *out_hwirq, unsigned int *out_type)
{
745
	if (irq_domain_get_of_node(d) != controller)
746 747 748 749 750 751 752 753 754 755 756
		return -EINVAL;
	if (intsize < 3)
		return -EINVAL;

	switch(intspec[0]) {
	case 0:			/* SPI */
		*out_hwirq = intspec[1] + 32;
		break;
	case 1:			/* PPI */
		*out_hwirq = intspec[1] + 16;
		break;
757 758 759
	case GIC_IRQ_TYPE_LPI:	/* LPI */
		*out_hwirq = intspec[1];
		break;
760 761 762 763 764 765 766 767
	default:
		return -EINVAL;
	}

	*out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
	return 0;
}

768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798
static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
				unsigned int nr_irqs, void *arg)
{
	int i, ret;
	irq_hw_number_t hwirq;
	unsigned int type = IRQ_TYPE_NONE;
	struct of_phandle_args *irq_data = arg;

	ret = gic_irq_domain_xlate(domain, irq_data->np, irq_data->args,
				   irq_data->args_count, &hwirq, &type);
	if (ret)
		return ret;

	for (i = 0; i < nr_irqs; i++)
		gic_irq_domain_map(domain, virq + i, hwirq + i);

	return 0;
}

static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
				unsigned int nr_irqs)
{
	int i;

	for (i = 0; i < nr_irqs; i++) {
		struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
		irq_set_handler(virq + i, NULL);
		irq_domain_reset_irq_data(d);
	}
}

799 800
static const struct irq_domain_ops gic_irq_domain_ops = {
	.xlate = gic_irq_domain_xlate,
801 802
	.alloc = gic_irq_domain_alloc,
	.free = gic_irq_domain_free,
803 804
};

805 806
static void gicv3_enable_quirks(void)
{
807
#ifdef CONFIG_ARM64
808
	if (cpus_have_cap(ARM64_WORKAROUND_CAVIUM_23154))
809
		static_branch_enable(&is_cavium_thunderx);
810
#endif
811 812
}

813 814 815
static int __init gic_of_init(struct device_node *node, struct device_node *parent)
{
	void __iomem *dist_base;
816
	struct redist_region *rdist_regs;
817
	u64 redist_stride;
818 819
	u32 nr_redist_regions;
	u32 typer;
820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839
	u32 reg;
	int gic_irqs;
	int err;
	int i;

	dist_base = of_iomap(node, 0);
	if (!dist_base) {
		pr_err("%s: unable to map gic dist registers\n",
			node->full_name);
		return -ENXIO;
	}

	reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
	if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) {
		pr_err("%s: no distributor detected, giving up\n",
			node->full_name);
		err = -ENODEV;
		goto out_unmap_dist;
	}

840 841
	if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
		nr_redist_regions = 1;
842

843 844
	rdist_regs = kzalloc(sizeof(*rdist_regs) * nr_redist_regions, GFP_KERNEL);
	if (!rdist_regs) {
845 846 847 848
		err = -ENOMEM;
		goto out_unmap_dist;
	}

849 850 851 852 853 854 855
	for (i = 0; i < nr_redist_regions; i++) {
		struct resource res;
		int ret;

		ret = of_address_to_resource(node, 1 + i, &res);
		rdist_regs[i].redist_base = of_iomap(node, 1 + i);
		if (ret || !rdist_regs[i].redist_base) {
856 857 858 859 860
			pr_err("%s: couldn't map region %d\n",
			       node->full_name, i);
			err = -ENODEV;
			goto out_unmap_rdist;
		}
861
		rdist_regs[i].phys_base = res.start;
862 863 864 865 866
	}

	if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
		redist_stride = 0;

867 868 869 870 871 872
	if (!is_hyp_mode_available())
		static_key_slow_dec(&supports_deactivate);

	if (static_key_true(&supports_deactivate))
		pr_info("GIC: Using split EOI/Deactivate mode\n");

873
	gic_data.dist_base = dist_base;
874 875
	gic_data.redist_regions = rdist_regs;
	gic_data.nr_redist_regions = nr_redist_regions;
876 877
	gic_data.redist_stride = redist_stride;

878 879
	gicv3_enable_quirks();

880 881 882 883
	/*
	 * Find out how many interrupts are supported.
	 * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)
	 */
884 885 886
	typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
	gic_data.rdists.id_bits = GICD_TYPER_ID_BITS(typer);
	gic_irqs = GICD_TYPER_IRQS(typer);
887 888 889 890 891 892
	if (gic_irqs > 1020)
		gic_irqs = 1020;
	gic_data.irq_nr = gic_irqs;

	gic_data.domain = irq_domain_add_tree(node, &gic_irq_domain_ops,
					      &gic_data);
893
	gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
894

895
	if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
896 897 898 899 900 901
		err = -ENOMEM;
		goto out_free;
	}

	set_handle_irq(gic_handle_irq);

902 903 904
	if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
		its_init(node, &gic_data.rdists, gic_data.domain);

905 906 907
	gic_smp_init();
	gic_dist_init();
	gic_cpu_init();
908
	gic_cpu_pm_init();
909 910 911 912 913 914

	return 0;

out_free:
	if (gic_data.domain)
		irq_domain_remove(gic_data.domain);
915
	free_percpu(gic_data.rdists.rdist);
916
out_unmap_rdist:
917 918 919 920
	for (i = 0; i < nr_redist_regions; i++)
		if (rdist_regs[i].redist_base)
			iounmap(rdist_regs[i].redist_base);
	kfree(rdist_regs);
921 922 923 924 925 926
out_unmap_dist:
	iounmap(dist_base);
	return err;
}

IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);