pcie-rcar.c 25.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * PCIe driver for Renesas R-Car SoCs
 *  Copyright (C) 2014 Renesas Electronics Europe Ltd
 *
 * Based on:
 *  arch/sh/drivers/pci/pcie-sh7786.c
 *  arch/sh/drivers/pci/ops-sh7786.c
 *  Copyright (C) 2009 - 2011  Paul Mundt
 *
 * This file is licensed under the terms of the GNU General Public
 * License version 2.  This program is licensed "as is" without any
 * warranty of any kind, whether express or implied.
 */

#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
18 19
#include <linux/irq.h>
#include <linux/irqdomain.h>
20 21
#include <linux/kernel.h>
#include <linux/module.h>
22
#include <linux/msi.h>
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/of_pci.h>
#include <linux/of_platform.h>
#include <linux/pci.h>
#include <linux/platform_device.h>
#include <linux/slab.h>

#define DRV_NAME "rcar-pcie"

#define PCIECAR			0x000010
#define PCIECCTLR		0x000018
#define  CONFIG_SEND_ENABLE	(1 << 31)
#define  TYPE0			(0 << 8)
#define  TYPE1			(1 << 8)
#define PCIECDR			0x000020
#define PCIEMSR			0x000028
#define PCIEINTXR		0x000400
41
#define PCIEMSITXR		0x000840
42 43 44 45 46 47 48 49

/* Transfer control */
#define PCIETCTLR		0x02000
#define  CFINIT			1
#define PCIETSTR		0x02004
#define  DATA_LINK_ACTIVE	1
#define PCIEERRFR		0x02020
#define  UNSUPPORTED_REQUEST	(1 << 4)
50 51 52 53 54
#define PCIEMSIFR		0x02044
#define PCIEMSIALR		0x02048
#define  MSIFE			1
#define PCIEMSIAUR		0x0204c
#define PCIEMSIIER		0x02050
55 56 57 58 59 60 61 62 63 64 65 66

/* root port address */
#define PCIEPRAR(x)		(0x02080 + ((x) * 0x4))

/* local address reg & mask */
#define PCIELAR(x)		(0x02200 + ((x) * 0x20))
#define PCIELAMR(x)		(0x02208 + ((x) * 0x20))
#define  LAM_PREFETCH		(1 << 3)
#define  LAM_64BIT		(1 << 2)
#define  LAR_ENABLE		(1 << 1)

/* PCIe address reg & mask */
67 68
#define PCIEPALR(x)		(0x03400 + ((x) * 0x20))
#define PCIEPAUR(x)		(0x03404 + ((x) * 0x20))
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
#define PCIEPAMR(x)		(0x03408 + ((x) * 0x20))
#define PCIEPTCTLR(x)		(0x0340c + ((x) * 0x20))
#define  PAR_ENABLE		(1 << 31)
#define  IO_SPACE		(1 << 8)

/* Configuration */
#define PCICONF(x)		(0x010000 + ((x) * 0x4))
#define PMCAP(x)		(0x010040 + ((x) * 0x4))
#define EXPCAP(x)		(0x010070 + ((x) * 0x4))
#define VCCAP(x)		(0x010100 + ((x) * 0x4))

/* link layer */
#define IDSETR1			0x011004
#define TLCTLR			0x011048
#define MACSR			0x011054
#define MACCTLR			0x011058
#define  SCRAMBLE_DISABLE	(1 << 27)

/* R-Car H1 PHY */
#define H1_PCIEPHYADRR		0x04000c
#define  WRITE_CMD		(1 << 16)
#define  PHY_ACK		(1 << 24)
#define  RATE_POS		12
#define  LANE_POS		8
#define  ADR_POS		0
#define H1_PCIEPHYDOUTR		0x040014
#define H1_PCIEPHYSR		0x040018

97 98
#define INT_PCI_MSI_NR	32

99 100 101 102 103 104 105 106 107
#define RCONF(x)	(PCICONF(0)+(x))
#define RPMCAP(x)	(PMCAP(0)+(x))
#define REXPCAP(x)	(EXPCAP(0)+(x))
#define RVCCAP(x)	(VCCAP(0)+(x))

#define  PCIE_CONF_BUS(b)	(((b) & 0xff) << 24)
#define  PCIE_CONF_DEV(d)	(((d) & 0x1f) << 19)
#define  PCIE_CONF_FUNC(f)	(((f) & 0x7) << 16)

108
#define RCAR_PCI_MAX_RESOURCES 4
109 110
#define MAX_NR_INBOUND_MAPS 6

111 112
static unsigned long global_io_offset;

113 114 115
struct rcar_msi {
	DECLARE_BITMAP(used, INT_PCI_MSI_NR);
	struct irq_domain *domain;
116
	struct msi_controller chip;
117 118 119 120 121 122
	unsigned long pages;
	struct mutex lock;
	int irq1;
	int irq2;
};

123
static inline struct rcar_msi *to_rcar_msi(struct msi_controller *chip)
124 125 126 127
{
	return container_of(chip, struct rcar_msi, chip);
}

128
/* Structure representing the PCIe interface */
129 130 131 132 133 134
/*
 * ARM pcibios functions expect the ARM struct pci_sys_data as the PCI
 * sysdata.  Add pci_sys_data as the first element in struct gen_pci so
 * that when we use a gen_pci pointer as sysdata, it is also a pointer to
 * a struct pci_sys_data.
 */
135
struct rcar_pcie {
136 137 138
#ifdef CONFIG_ARM
	struct pci_sys_data	sys;
#endif
139 140
	struct device		*dev;
	void __iomem		*base;
141
	struct resource		res[RCAR_PCI_MAX_RESOURCES];
142 143 144 145
	struct resource		busn;
	int			root_bus_nr;
	struct clk		*clk;
	struct clk		*bus_clk;
146
	struct			rcar_msi msi;
147 148
};

149 150
static void rcar_pci_write_reg(struct rcar_pcie *pcie, unsigned long val,
			       unsigned long reg)
151 152 153 154
{
	writel(val, pcie->base + reg);
}

155 156
static unsigned long rcar_pci_read_reg(struct rcar_pcie *pcie,
				       unsigned long reg)
157 158 159 160 161
{
	return readl(pcie->base + reg);
}

enum {
162 163
	RCAR_PCI_ACCESS_READ,
	RCAR_PCI_ACCESS_WRITE,
164 165 166 167 168
};

static void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data)
{
	int shift = 8 * (where & 3);
169
	u32 val = rcar_pci_read_reg(pcie, where & ~3);
170 171 172

	val &= ~(mask << shift);
	val |= data << shift;
173
	rcar_pci_write_reg(pcie, val, where & ~3);
174 175 176 177 178
}

static u32 rcar_read_conf(struct rcar_pcie *pcie, int where)
{
	int shift = 8 * (where & 3);
179
	u32 val = rcar_pci_read_reg(pcie, where & ~3);
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214

	return val >> shift;
}

/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */
static int rcar_pcie_config_access(struct rcar_pcie *pcie,
		unsigned char access_type, struct pci_bus *bus,
		unsigned int devfn, int where, u32 *data)
{
	int dev, func, reg, index;

	dev = PCI_SLOT(devfn);
	func = PCI_FUNC(devfn);
	reg = where & ~3;
	index = reg / 4;

	/*
	 * While each channel has its own memory-mapped extended config
	 * space, it's generally only accessible when in endpoint mode.
	 * When in root complex mode, the controller is unable to target
	 * itself with either type 0 or type 1 accesses, and indeed, any
	 * controller initiated target transfer to its own config space
	 * result in a completer abort.
	 *
	 * Each channel effectively only supports a single device, but as
	 * the same channel <-> device access works for any PCI_SLOT()
	 * value, we cheat a bit here and bind the controller's config
	 * space to devfn 0 in order to enable self-enumeration. In this
	 * case the regular ECAR/ECDR path is sidelined and the mangled
	 * config access itself is initiated as an internal bus transaction.
	 */
	if (pci_is_root_bus(bus)) {
		if (dev != 0)
			return PCIBIOS_DEVICE_NOT_FOUND;

215 216
		if (access_type == RCAR_PCI_ACCESS_READ) {
			*data = rcar_pci_read_reg(pcie, PCICONF(index));
217 218 219 220 221
		} else {
			/* Keep an eye out for changes to the root bus number */
			if (pci_is_root_bus(bus) && (reg == PCI_PRIMARY_BUS))
				pcie->root_bus_nr = *data & 0xff;

222
			rcar_pci_write_reg(pcie, *data, PCICONF(index));
223 224 225 226 227 228 229 230 231
		}

		return PCIBIOS_SUCCESSFUL;
	}

	if (pcie->root_bus_nr < 0)
		return PCIBIOS_DEVICE_NOT_FOUND;

	/* Clear errors */
232
	rcar_pci_write_reg(pcie, rcar_pci_read_reg(pcie, PCIEERRFR), PCIEERRFR);
233 234

	/* Set the PIO address */
235 236
	rcar_pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) |
		PCIE_CONF_DEV(dev) | PCIE_CONF_FUNC(func) | reg, PCIECAR);
237 238 239

	/* Enable the configuration access */
	if (bus->parent->number == pcie->root_bus_nr)
240
		rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR);
241
	else
242
		rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR);
243 244

	/* Check for errors */
245
	if (rcar_pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST)
246 247 248 249 250 251 252
		return PCIBIOS_DEVICE_NOT_FOUND;

	/* Check for master and target aborts */
	if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) &
		(PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT))
		return PCIBIOS_DEVICE_NOT_FOUND;

253 254
	if (access_type == RCAR_PCI_ACCESS_READ)
		*data = rcar_pci_read_reg(pcie, PCIECDR);
255
	else
256
		rcar_pci_write_reg(pcie, *data, PCIECDR);
257 258

	/* Disable the configuration access */
259
	rcar_pci_write_reg(pcie, 0, PCIECCTLR);
260 261 262 263 264 265 266

	return PCIBIOS_SUCCESSFUL;
}

static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn,
			       int where, int size, u32 *val)
{
267
	struct rcar_pcie *pcie = bus->sysdata;
268 269
	int ret;

270
	ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ,
271 272 273 274 275 276 277 278 279 280 281
				      bus, devfn, where, val);
	if (ret != PCIBIOS_SUCCESSFUL) {
		*val = 0xffffffff;
		return ret;
	}

	if (size == 1)
		*val = (*val >> (8 * (where & 3))) & 0xff;
	else if (size == 2)
		*val = (*val >> (8 * (where & 2))) & 0xffff;

282 283
	dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08lx\n",
		bus->number, devfn, where, size, (unsigned long)*val);
284 285 286 287 288 289 290 291

	return ret;
}

/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */
static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn,
				int where, int size, u32 val)
{
292
	struct rcar_pcie *pcie = bus->sysdata;
293 294 295
	int shift, ret;
	u32 data;

296
	ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ,
297 298 299 300
				      bus, devfn, where, &data);
	if (ret != PCIBIOS_SUCCESSFUL)
		return ret;

301 302
	dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08lx\n",
		bus->number, devfn, where, size, (unsigned long)val);
303 304 305 306 307 308 309 310 311 312 313 314

	if (size == 1) {
		shift = 8 * (where & 3);
		data &= ~(0xff << shift);
		data |= ((val & 0xff) << shift);
	} else if (size == 2) {
		shift = 8 * (where & 2);
		data &= ~(0xffff << shift);
		data |= ((val & 0xffff) << shift);
	} else
		data = val;

315
	ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_WRITE,
316 317 318 319 320 321 322 323 324 325
				      bus, devfn, where, &data);

	return ret;
}

static struct pci_ops rcar_pcie_ops = {
	.read	= rcar_pcie_read_conf,
	.write	= rcar_pcie_write_conf,
};

326
static void rcar_pcie_setup_window(int win, struct rcar_pcie *pcie)
327
{
328 329
	struct resource *res = &pcie->res[win];

330 331
	/* Setup PCIe address space mappings for each resource */
	resource_size_t size;
332
	resource_size_t res_start;
333 334
	u32 mask;

335
	rcar_pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win));
336 337 338 339 340 341 342

	/*
	 * The PAMR mask is calculated in units of 128Bytes, which
	 * keeps things pretty simple.
	 */
	size = resource_size(res);
	mask = (roundup_pow_of_two(size) / SZ_128) - 1;
343
	rcar_pci_write_reg(pcie, mask << 7, PCIEPAMR(win));
344

345 346 347 348 349
	if (res->flags & IORESOURCE_IO)
		res_start = pci_pio_to_address(res->start);
	else
		res_start = res->start;

350
	rcar_pci_write_reg(pcie, upper_32_bits(res_start), PCIEPAUR(win));
351
	rcar_pci_write_reg(pcie, lower_32_bits(res_start) & ~0x7F,
352
			   PCIEPALR(win));
353 354 355 356 357 358

	/* First resource is for IO */
	mask = PAR_ENABLE;
	if (res->flags & IORESOURCE_IO)
		mask |= IO_SPACE;

359
	rcar_pci_write_reg(pcie, mask, PCIEPTCTLR(win));
360 361
}

362
static int rcar_pcie_setup(struct list_head *resource, struct rcar_pcie *pcie)
363 364 365 366
{
	struct resource *res;
	int i;

367
	pcie->root_bus_nr = pcie->busn.start;
368 369

	/* Setup PCI resources */
370
	for (i = 0; i < RCAR_PCI_MAX_RESOURCES; i++) {
371 372 373 374 375

		res = &pcie->res[i];
		if (!res->flags)
			continue;

376
		rcar_pcie_setup_window(i, pcie);
377

378 379
		if (res->flags & IORESOURCE_IO) {
			phys_addr_t io_start = pci_pio_to_address(res->start);
380 381
			pci_ioremap_io(global_io_offset, io_start);
			global_io_offset += SZ_64K;
382 383
		}

384
		pci_add_resource(resource, res);
385
	}
386
	pci_add_resource(resource, &pcie->busn);
387 388 389 390

	return 1;
}

391
static int rcar_pcie_enable(struct rcar_pcie *pcie)
392
{
393 394
	struct pci_bus *bus, *child;
	LIST_HEAD(res);
395

396
	rcar_pcie_setup(&res, pcie);
397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418

	/* Do not reassign resources if probe only */
	if (!pci_has_flag(PCI_PROBE_ONLY))
		pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS);

	if (IS_ENABLED(CONFIG_PCI_MSI))
		bus = pci_scan_root_bus_msi(pcie->dev, pcie->root_bus_nr,
				&rcar_pcie_ops, pcie, &res, &pcie->msi.chip);
	else
		bus = pci_scan_root_bus(pcie->dev, pcie->root_bus_nr,
				&rcar_pcie_ops, pcie, &res);

	if (!bus) {
		dev_err(pcie->dev, "Scanning rootbus failed");
		return -ENODEV;
	}

	pci_fixup_irqs(pci_common_swizzle, of_irq_parse_and_map_pci);

	if (!pci_has_flag(PCI_PROBE_ONLY)) {
		pci_bus_size_bridges(bus);
		pci_bus_assign_resources(bus);
419

420 421 422 423 424 425 426
		list_for_each_entry(child, &bus->children, node)
			pcie_bus_configure_settings(child);
	}

	pci_bus_add_devices(bus);

	return 0;
427 428 429 430 431 432 433
}

static int phy_wait_for_ack(struct rcar_pcie *pcie)
{
	unsigned int timeout = 100;

	while (timeout--) {
434
		if (rcar_pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK)
435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
			return 0;

		udelay(100);
	}

	dev_err(pcie->dev, "Access to PCIe phy timed out\n");

	return -ETIMEDOUT;
}

static void phy_write_reg(struct rcar_pcie *pcie,
				 unsigned int rate, unsigned int addr,
				 unsigned int lane, unsigned int data)
{
	unsigned long phyaddr;

	phyaddr = WRITE_CMD |
		((rate & 1) << RATE_POS) |
		((lane & 0xf) << LANE_POS) |
		((addr & 0xff) << ADR_POS);

	/* Set write data */
457 458
	rcar_pci_write_reg(pcie, data, H1_PCIEPHYDOUTR);
	rcar_pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR);
459 460 461 462 463

	/* Ignore errors as they will be dealt with if the data link is down */
	phy_wait_for_ack(pcie);

	/* Clear command */
464 465
	rcar_pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR);
	rcar_pci_write_reg(pcie, 0, H1_PCIEPHYADRR);
466 467 468 469 470 471 472 473 474 475

	/* Ignore errors as they will be dealt with if the data link is down */
	phy_wait_for_ack(pcie);
}

static int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie)
{
	unsigned int timeout = 10;

	while (timeout--) {
476
		if ((rcar_pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE))
477 478 479 480 481 482 483 484 485 486 487 488 489
			return 0;

		msleep(5);
	}

	return -ETIMEDOUT;
}

static int rcar_pcie_hw_init(struct rcar_pcie *pcie)
{
	int err;

	/* Begin initialization */
490
	rcar_pci_write_reg(pcie, 0, PCIETCTLR);
491 492

	/* Set mode */
493
	rcar_pci_write_reg(pcie, 1, PCIEMSR);
494 495 496 497 498 499

	/*
	 * Initial header for port config space is type 1, set the device
	 * class to match. Hardware takes care of propagating the IDSETR
	 * settings, so there is no need to bother with a quirk.
	 */
500
	rcar_pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI << 16, IDSETR1);
501 502 503 504 505 506 507 508 509

	/*
	 * Setup Secondary Bus Number & Subordinate Bus Number, even though
	 * they aren't used, to avoid bridge being detected as broken.
	 */
	rcar_rmw32(pcie, RCONF(PCI_SECONDARY_BUS), 0xff, 1);
	rcar_rmw32(pcie, RCONF(PCI_SUBORDINATE_BUS), 0xff, 1);

	/* Initialize default capabilities. */
510
	rcar_rmw32(pcie, REXPCAP(0), 0xff, PCI_CAP_ID_EXP);
511 512 513 514 515 516
	rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS),
		PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ROOT_PORT << 4);
	rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), 0x7f,
		PCI_HEADER_TYPE_BRIDGE);

	/* Enable data link layer active state reporting */
517 518
	rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), PCI_EXP_LNKCAP_DLLLARC,
		PCI_EXP_LNKCAP_DLLLARC);
519 520 521 522 523

	/* Write out the physical slot number = 0 */
	rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0);

	/* Set the completion timer timeout to the maximum 50ms. */
524
	rcar_rmw32(pcie, TLCTLR + 1, 0x3f, 50);
525 526

	/* Terminate list of capabilities (Next Capability Offset=0) */
527
	rcar_rmw32(pcie, RVCCAP(0), 0xfff00000, 0);
528

529 530
	/* Enable MSI */
	if (IS_ENABLED(CONFIG_PCI_MSI))
531
		rcar_pci_write_reg(pcie, 0x801f0000, PCIEMSITXR);
532

533
	/* Finish initialization - establish a PCI Express link */
534
	rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR);
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571

	/* This will timeout if we don't have a link. */
	err = rcar_pcie_wait_for_dl(pcie);
	if (err)
		return err;

	/* Enable INTx interrupts */
	rcar_rmw32(pcie, PCIEINTXR, 0, 0xF << 8);

	wmb();

	return 0;
}

static int rcar_pcie_hw_init_h1(struct rcar_pcie *pcie)
{
	unsigned int timeout = 10;

	/* Initialize the phy */
	phy_write_reg(pcie, 0, 0x42, 0x1, 0x0EC34191);
	phy_write_reg(pcie, 1, 0x42, 0x1, 0x0EC34180);
	phy_write_reg(pcie, 0, 0x43, 0x1, 0x00210188);
	phy_write_reg(pcie, 1, 0x43, 0x1, 0x00210188);
	phy_write_reg(pcie, 0, 0x44, 0x1, 0x015C0014);
	phy_write_reg(pcie, 1, 0x44, 0x1, 0x015C0014);
	phy_write_reg(pcie, 1, 0x4C, 0x1, 0x786174A0);
	phy_write_reg(pcie, 1, 0x4D, 0x1, 0x048000BB);
	phy_write_reg(pcie, 0, 0x51, 0x1, 0x079EC062);
	phy_write_reg(pcie, 0, 0x52, 0x1, 0x20000000);
	phy_write_reg(pcie, 1, 0x52, 0x1, 0x20000000);
	phy_write_reg(pcie, 1, 0x56, 0x1, 0x00003806);

	phy_write_reg(pcie, 0, 0x60, 0x1, 0x004B03A5);
	phy_write_reg(pcie, 0, 0x64, 0x1, 0x3F0F1F0F);
	phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000);

	while (timeout--) {
572
		if (rcar_pci_read_reg(pcie, H1_PCIEPHYSR))
573 574 575 576 577 578 579 580
			return rcar_pcie_hw_init(pcie);

		msleep(5);
	}

	return -ETIMEDOUT;
}

581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610
static int rcar_msi_alloc(struct rcar_msi *chip)
{
	int msi;

	mutex_lock(&chip->lock);

	msi = find_first_zero_bit(chip->used, INT_PCI_MSI_NR);
	if (msi < INT_PCI_MSI_NR)
		set_bit(msi, chip->used);
	else
		msi = -ENOSPC;

	mutex_unlock(&chip->lock);

	return msi;
}

static void rcar_msi_free(struct rcar_msi *chip, unsigned long irq)
{
	mutex_lock(&chip->lock);
	clear_bit(irq, chip->used);
	mutex_unlock(&chip->lock);
}

static irqreturn_t rcar_pcie_msi_irq(int irq, void *data)
{
	struct rcar_pcie *pcie = data;
	struct rcar_msi *msi = &pcie->msi;
	unsigned long reg;

611
	reg = rcar_pci_read_reg(pcie, PCIEMSIFR);
612 613 614 615 616 617 618 619 620 621

	/* MSI & INTx share an interrupt - we only handle MSI here */
	if (!reg)
		return IRQ_NONE;

	while (reg) {
		unsigned int index = find_first_bit(&reg, 32);
		unsigned int irq;

		/* clear the interrupt */
622
		rcar_pci_write_reg(pcie, 1 << index, PCIEMSIFR);
623 624 625 626 627 628 629 630 631 632 633 634 635

		irq = irq_find_mapping(msi->domain, index);
		if (irq) {
			if (test_bit(index, msi->used))
				generic_handle_irq(irq);
			else
				dev_info(pcie->dev, "unhandled MSI\n");
		} else {
			/* Unknown MSI, just clear it */
			dev_dbg(pcie->dev, "unexpected MSI\n");
		}

		/* see if there's any more pending in this vector */
636
		reg = rcar_pci_read_reg(pcie, PCIEMSIFR);
637 638 639 640 641
	}

	return IRQ_HANDLED;
}

642
static int rcar_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662
			      struct msi_desc *desc)
{
	struct rcar_msi *msi = to_rcar_msi(chip);
	struct rcar_pcie *pcie = container_of(chip, struct rcar_pcie, msi.chip);
	struct msi_msg msg;
	unsigned int irq;
	int hwirq;

	hwirq = rcar_msi_alloc(msi);
	if (hwirq < 0)
		return hwirq;

	irq = irq_create_mapping(msi->domain, hwirq);
	if (!irq) {
		rcar_msi_free(msi, hwirq);
		return -EINVAL;
	}

	irq_set_msi_desc(irq, desc);

663 664
	msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE;
	msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR);
665 666
	msg.data = hwirq;

667
	pci_write_msi_msg(irq, &msg);
668 669 670 671

	return 0;
}

672
static void rcar_msi_teardown_irq(struct msi_controller *chip, unsigned int irq)
673 674 675 676 677 678 679 680 681
{
	struct rcar_msi *msi = to_rcar_msi(chip);
	struct irq_data *d = irq_get_irq_data(irq);

	rcar_msi_free(msi, d->hwirq);
}

static struct irq_chip rcar_msi_irq_chip = {
	.name = "R-Car PCIe MSI",
682 683 684 685
	.irq_enable = pci_msi_unmask_irq,
	.irq_disable = pci_msi_mask_irq,
	.irq_mask = pci_msi_mask_irq,
	.irq_unmask = pci_msi_unmask_irq,
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
};

static int rcar_msi_map(struct irq_domain *domain, unsigned int irq,
			irq_hw_number_t hwirq)
{
	irq_set_chip_and_handler(irq, &rcar_msi_irq_chip, handle_simple_irq);
	irq_set_chip_data(irq, domain->host_data);

	return 0;
}

static const struct irq_domain_ops msi_domain_ops = {
	.map = rcar_msi_map,
};

static int rcar_pcie_enable_msi(struct rcar_pcie *pcie)
{
	struct platform_device *pdev = to_platform_device(pcie->dev);
	struct rcar_msi *msi = &pcie->msi;
	unsigned long base;
	int err;

	mutex_init(&msi->lock);

	msi->chip.dev = pcie->dev;
	msi->chip.setup_irq = rcar_msi_setup_irq;
	msi->chip.teardown_irq = rcar_msi_teardown_irq;

	msi->domain = irq_domain_add_linear(pcie->dev->of_node, INT_PCI_MSI_NR,
					    &msi_domain_ops, &msi->chip);
	if (!msi->domain) {
		dev_err(&pdev->dev, "failed to create IRQ domain\n");
		return -ENOMEM;
	}

	/* Two irqs are for MSI, but they are also used for non-MSI irqs */
	err = devm_request_irq(&pdev->dev, msi->irq1, rcar_pcie_msi_irq,
			       IRQF_SHARED, rcar_msi_irq_chip.name, pcie);
	if (err < 0) {
		dev_err(&pdev->dev, "failed to request IRQ: %d\n", err);
		goto err;
	}

	err = devm_request_irq(&pdev->dev, msi->irq2, rcar_pcie_msi_irq,
			       IRQF_SHARED, rcar_msi_irq_chip.name, pcie);
	if (err < 0) {
		dev_err(&pdev->dev, "failed to request IRQ: %d\n", err);
		goto err;
	}

	/* setup MSI data target */
	msi->pages = __get_free_pages(GFP_KERNEL, 0);
	base = virt_to_phys((void *)msi->pages);

740 741
	rcar_pci_write_reg(pcie, base | MSIFE, PCIEMSIALR);
	rcar_pci_write_reg(pcie, 0, PCIEMSIAUR);
742 743

	/* enable all MSI interrupts */
744
	rcar_pci_write_reg(pcie, 0xffffffff, PCIEMSIIER);
745 746 747 748 749 750 751 752

	return 0;

err:
	irq_domain_remove(msi->domain);
	return err;
}

753 754 755 756
static int rcar_pcie_get_resources(struct platform_device *pdev,
				   struct rcar_pcie *pcie)
{
	struct resource res;
757
	int err, i;
758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781

	err = of_address_to_resource(pdev->dev.of_node, 0, &res);
	if (err)
		return err;

	pcie->clk = devm_clk_get(&pdev->dev, "pcie");
	if (IS_ERR(pcie->clk)) {
		dev_err(pcie->dev, "cannot get platform clock\n");
		return PTR_ERR(pcie->clk);
	}
	err = clk_prepare_enable(pcie->clk);
	if (err)
		goto fail_clk;

	pcie->bus_clk = devm_clk_get(&pdev->dev, "pcie_bus");
	if (IS_ERR(pcie->bus_clk)) {
		dev_err(pcie->dev, "cannot get pcie bus clock\n");
		err = PTR_ERR(pcie->bus_clk);
		goto fail_clk;
	}
	err = clk_prepare_enable(pcie->bus_clk);
	if (err)
		goto err_map_reg;

782
	i = irq_of_parse_and_map(pdev->dev.of_node, 0);
783
	if (!i) {
784 785 786 787 788 789 790
		dev_err(pcie->dev, "cannot get platform resources for msi interrupt\n");
		err = -ENOENT;
		goto err_map_reg;
	}
	pcie->msi.irq1 = i;

	i = irq_of_parse_and_map(pdev->dev.of_node, 1);
791
	if (!i) {
792 793 794 795 796 797
		dev_err(pcie->dev, "cannot get platform resources for msi interrupt\n");
		err = -ENOENT;
		goto err_map_reg;
	}
	pcie->msi.irq2 = i;

798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
	pcie->base = devm_ioremap_resource(&pdev->dev, &res);
	if (IS_ERR(pcie->base)) {
		err = PTR_ERR(pcie->base);
		goto err_map_reg;
	}

	return 0;

err_map_reg:
	clk_disable_unprepare(pcie->bus_clk);
fail_clk:
	clk_disable_unprepare(pcie->clk);

	return err;
}

static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie,
				    struct of_pci_range *range,
				    int *index)
{
	u64 restype = range->flags;
	u64 cpu_addr = range->cpu_addr;
	u64 cpu_end = range->cpu_addr + range->size;
	u64 pci_addr = range->pci_addr;
	u32 flags = LAM_64BIT | LAR_ENABLE;
	u64 mask;
	u64 size;
	int idx = *index;

	if (restype & IORESOURCE_PREFETCH)
		flags |= LAM_PREFETCH;

	/*
	 * If the size of the range is larger than the alignment of the start
	 * address, we have to use multiple entries to perform the mapping.
	 */
	if (cpu_addr > 0) {
		unsigned long nr_zeros = __ffs64(cpu_addr);
		u64 alignment = 1ULL << nr_zeros;
837

838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
		size = min(range->size, alignment);
	} else {
		size = range->size;
	}
	/* Hardware supports max 4GiB inbound region */
	size = min(size, 1ULL << 32);

	mask = roundup_pow_of_two(size) - 1;
	mask &= ~0xf;

	while (cpu_addr < cpu_end) {
		/*
		 * Set up 64-bit inbound regions as the range parser doesn't
		 * distinguish between 32 and 64-bit types.
		 */
853 854 855
		rcar_pci_write_reg(pcie, lower_32_bits(pci_addr), PCIEPRAR(idx));
		rcar_pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx));
		rcar_pci_write_reg(pcie, lower_32_bits(mask) | flags, PCIELAMR(idx));
856

857 858 859
		rcar_pci_write_reg(pcie, upper_32_bits(pci_addr), PCIEPRAR(idx+1));
		rcar_pci_write_reg(pcie, upper_32_bits(cpu_addr), PCIELAR(idx+1));
		rcar_pci_write_reg(pcie, 0, PCIELAMR(idx + 1));
860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960

		pci_addr += size;
		cpu_addr += size;
		idx += 2;

		if (idx > MAX_NR_INBOUND_MAPS) {
			dev_err(pcie->dev, "Failed to map inbound regions!\n");
			return -EINVAL;
		}
	}
	*index = idx;

	return 0;
}

static int pci_dma_range_parser_init(struct of_pci_range_parser *parser,
				     struct device_node *node)
{
	const int na = 3, ns = 2;
	int rlen;

	parser->node = node;
	parser->pna = of_n_addr_cells(node);
	parser->np = parser->pna + na + ns;

	parser->range = of_get_property(node, "dma-ranges", &rlen);
	if (!parser->range)
		return -ENOENT;

	parser->end = parser->range + rlen / sizeof(__be32);
	return 0;
}

static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie,
					  struct device_node *np)
{
	struct of_pci_range range;
	struct of_pci_range_parser parser;
	int index = 0;
	int err;

	if (pci_dma_range_parser_init(&parser, np))
		return -EINVAL;

	/* Get the dma-ranges from DT */
	for_each_of_pci_range(&parser, &range) {
		u64 end = range.cpu_addr + range.size - 1;
		dev_dbg(pcie->dev, "0x%08x 0x%016llx..0x%016llx -> 0x%016llx\n",
			range.flags, range.cpu_addr, end, range.pci_addr);

		err = rcar_pcie_inbound_ranges(pcie, &range, &index);
		if (err)
			return err;
	}

	return 0;
}

static const struct of_device_id rcar_pcie_of_match[] = {
	{ .compatible = "renesas,pcie-r8a7779", .data = rcar_pcie_hw_init_h1 },
	{ .compatible = "renesas,pcie-r8a7790", .data = rcar_pcie_hw_init },
	{ .compatible = "renesas,pcie-r8a7791", .data = rcar_pcie_hw_init },
	{},
};
MODULE_DEVICE_TABLE(of, rcar_pcie_of_match);

static int rcar_pcie_probe(struct platform_device *pdev)
{
	struct rcar_pcie *pcie;
	unsigned int data;
	struct of_pci_range range;
	struct of_pci_range_parser parser;
	const struct of_device_id *of_id;
	int err, win = 0;
	int (*hw_init_fn)(struct rcar_pcie *);

	pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL);
	if (!pcie)
		return -ENOMEM;

	pcie->dev = &pdev->dev;
	platform_set_drvdata(pdev, pcie);

	/* Get the bus range */
	if (of_pci_parse_bus_range(pdev->dev.of_node, &pcie->busn)) {
		dev_err(&pdev->dev, "failed to parse bus-range property\n");
		return -EINVAL;
	}

	if (of_pci_range_parser_init(&parser, pdev->dev.of_node)) {
		dev_err(&pdev->dev, "missing ranges property\n");
		return -EINVAL;
	}

	err = rcar_pcie_get_resources(pdev, pcie);
	if (err < 0) {
		dev_err(&pdev->dev, "failed to request resources: %d\n", err);
		return err;
	}

	for_each_of_pci_range(&parser, &range) {
961
		err = of_pci_range_to_resource(&range, pdev->dev.of_node,
962
						&pcie->res[win++]);
963 964
		if (err < 0)
			return err;
965

966
		if (win > RCAR_PCI_MAX_RESOURCES)
967 968 969 970 971 972 973
			break;
	}

	 err = rcar_pcie_parse_map_dma_ranges(pcie, pdev->dev.of_node);
	 if (err)
		return err;

974 975 976 977 978 979 980 981 982 983
	if (IS_ENABLED(CONFIG_PCI_MSI)) {
		err = rcar_pcie_enable_msi(pcie);
		if (err < 0) {
			dev_err(&pdev->dev,
				"failed to enable MSI support: %d\n",
				err);
			return err;
		}
	}

984 985 986 987 988 989 990 991 992 993 994 995
	of_id = of_match_device(rcar_pcie_of_match, pcie->dev);
	if (!of_id || !of_id->data)
		return -EINVAL;
	hw_init_fn = of_id->data;

	/* Failure to get a link might just be that no cards are inserted */
	err = hw_init_fn(pcie);
	if (err) {
		dev_info(&pdev->dev, "PCIe link down\n");
		return 0;
	}

996
	data = rcar_pci_read_reg(pcie, MACSR);
997 998
	dev_info(&pdev->dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f);

999
	return rcar_pcie_enable(pcie);
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
}

static struct platform_driver rcar_pcie_driver = {
	.driver = {
		.name = DRV_NAME,
		.of_match_table = rcar_pcie_of_match,
		.suppress_bind_attrs = true,
	},
	.probe = rcar_pcie_probe,
};
module_platform_driver(rcar_pcie_driver);

MODULE_AUTHOR("Phil Edworthy <phil.edworthy@renesas.com>");
MODULE_DESCRIPTION("Renesas R-Car PCIe driver");
1014
MODULE_LICENSE("GPL v2");