pci.c 22.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * Support PCI/PCIe on PowerNV platforms
 *
 * Currently supports only P5IOC2
 *
 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/irq.h>
#include <linux/io.h>
22
#include <linux/msi.h>
23
#include <linux/iommu.h>
24 25 26 27 28 29

#include <asm/sections.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/pci-bridge.h>
#include <asm/machdep.h>
30
#include <asm/msi_bitmap.h>
31 32 33 34
#include <asm/ppc-pci.h>
#include <asm/opal.h>
#include <asm/iommu.h>
#include <asm/tce.h>
35
#include <asm/firmware.h>
36 37
#include <asm/eeh_event.h>
#include <asm/eeh.h>
38 39 40 41

#include "powernv.h"
#include "pci.h"

42 43
/* Delay in usec */
#define PCI_RESET_DELAY_US	3000000
44 45 46 47

#define cfg_dbg(fmt...)	do { } while(0)
//#define cfg_dbg(fmt...)	printk(fmt)

48 49 50 51 52
#ifdef CONFIG_PCI_MSI
static int pnv_msi_check_device(struct pci_dev* pdev, int nvec, int type)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
53 54 55 56
	struct pci_dn *pdn = pci_get_pdn(pdev);

	if (pdn && pdn->force_32bit_msi && !phb->msi32_support)
		return -ENODEV;
57

58
	return (phb && phb->msi_bmp.bitmap) ? 0 : -ENODEV;
59 60 61 62 63 64 65 66
}

static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;
	struct msi_msg msg;
67 68
	int hwirq;
	unsigned int virq;
69 70 71 72 73 74 75 76 77 78 79
	int rc;

	if (WARN_ON(!phb))
		return -ENODEV;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
			pr_warn("%s: Supports only 64-bit MSIs\n",
				pci_name(pdev));
			return -ENXIO;
		}
80 81
		hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
		if (hwirq < 0) {
82 83 84 85
			pr_warn("%s: Failed to find a free MSI\n",
				pci_name(pdev));
			return -ENOSPC;
		}
86
		virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
87 88 89
		if (virq == NO_IRQ) {
			pr_warn("%s: Failed to map MSI to linux irq\n",
				pci_name(pdev));
90
			msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
91 92
			return -ENOMEM;
		}
93
		rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
94
				    virq, entry->msi_attrib.is_64, &msg);
95 96 97
		if (rc) {
			pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
			irq_dispose_mapping(virq);
98
			msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
			return rc;
		}
		irq_set_msi_desc(virq, entry);
		write_msi_msg(virq, &msg);
	}
	return 0;
}

static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;

	if (WARN_ON(!phb))
		return;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (entry->irq == NO_IRQ)
			continue;
		irq_set_msi_desc(entry->irq, NULL);
120 121
		msi_bitmap_free_hwirqs(&phb->msi_bmp,
			virq_to_hw(entry->irq) - phb->msi_base, 1);
122 123 124 125
		irq_dispose_mapping(entry->irq);
	}
}
#endif /* CONFIG_PCI_MSI */
126

127 128
static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
					 struct OpalIoPhbErrorCommon *common)
129
{
130
	struct OpalIoP7IOCPhbErrorData *data;
131 132
	int i;

133 134 135 136
	data = (struct OpalIoP7IOCPhbErrorData *)common;
	pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n\n",
		hose->global_number, common->version);

137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
	if (data->brdgCtl)
		pr_info("  brdgCtl:     %08x\n",
			data->brdgCtl);
	if (data->portStatusReg || data->rootCmplxStatus ||
	    data->busAgentStatus)
		pr_info("  UtlSts:      %08x %08x %08x\n",
			data->portStatusReg, data->rootCmplxStatus,
			data->busAgentStatus);
	if (data->deviceStatus || data->slotStatus   ||
	    data->linkStatus   || data->devCmdStatus ||
	    data->devSecStatus)
		pr_info("  RootSts:     %08x %08x %08x %08x %08x\n",
			data->deviceStatus, data->slotStatus,
			data->linkStatus, data->devCmdStatus,
			data->devSecStatus);
	if (data->rootErrorStatus   || data->uncorrErrorStatus ||
	    data->corrErrorStatus)
		pr_info("  RootErrSts:  %08x %08x %08x\n",
			data->rootErrorStatus, data->uncorrErrorStatus,
			data->corrErrorStatus);
	if (data->tlpHdr1 || data->tlpHdr2 ||
	    data->tlpHdr3 || data->tlpHdr4)
		pr_info("  RootErrLog:  %08x %08x %08x %08x\n",
			data->tlpHdr1, data->tlpHdr2,
			data->tlpHdr3, data->tlpHdr4);
	if (data->sourceId || data->errorClass ||
	    data->correlator)
		pr_info("  RootErrLog1: %08x %016llx %016llx\n",
			data->sourceId, data->errorClass,
			data->correlator);
	if (data->p7iocPlssr || data->p7iocCsr)
		pr_info("  PhbSts:      %016llx %016llx\n",
			data->p7iocPlssr, data->p7iocCsr);
	if (data->lemFir || data->lemErrorMask ||
	    data->lemWOF)
		pr_info("  Lem:         %016llx %016llx %016llx\n",
			data->lemFir, data->lemErrorMask,
			data->lemWOF);
	if (data->phbErrorStatus || data->phbFirstErrorStatus ||
	    data->phbErrorLog0   || data->phbErrorLog1)
		pr_info("  PhbErr:      %016llx %016llx %016llx %016llx\n",
			data->phbErrorStatus, data->phbFirstErrorStatus,
			data->phbErrorLog0, data->phbErrorLog1);
	if (data->mmioErrorStatus || data->mmioFirstErrorStatus ||
	    data->mmioErrorLog0   || data->mmioErrorLog1)
		pr_info("  OutErr:      %016llx %016llx %016llx %016llx\n",
			data->mmioErrorStatus, data->mmioFirstErrorStatus,
			data->mmioErrorLog0, data->mmioErrorLog1);
	if (data->dma0ErrorStatus || data->dma0FirstErrorStatus ||
	    data->dma0ErrorLog0   || data->dma0ErrorLog1)
		pr_info("  InAErr:      %016llx %016llx %016llx %016llx\n",
			data->dma0ErrorStatus, data->dma0FirstErrorStatus,
			data->dma0ErrorLog0, data->dma0ErrorLog1);
	if (data->dma1ErrorStatus || data->dma1FirstErrorStatus ||
	    data->dma1ErrorLog0   || data->dma1ErrorLog1)
		pr_info("  InBErr:      %016llx %016llx %016llx %016llx\n",
			data->dma1ErrorStatus, data->dma1FirstErrorStatus,
			data->dma1ErrorLog0, data->dma1ErrorLog1);
195 196 197 198 199

	for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
		if ((data->pestA[i] >> 63) == 0 &&
		    (data->pestB[i] >> 63) == 0)
			continue;
200

201 202
		pr_info("  PE[%3d] A/B: %016llx %016llx\n",
			i, data->pestA[i], data->pestB[i]);
203 204 205
	}
}

206 207
static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
					struct OpalIoPhbErrorCommon *common)
208
{
209 210 211 212 213 214
	struct OpalIoPhb3ErrorData *data;
	int i;

	data = (struct OpalIoPhb3ErrorData*)common;
	pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n\n",
		hose->global_number, common->version);
215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
	if (data->brdgCtl)
		pr_info("  brdgCtl:     %08x\n",
			data->brdgCtl);
	if (data->portStatusReg || data->rootCmplxStatus ||
	    data->busAgentStatus)
		pr_info("  UtlSts:      %08x %08x %08x\n",
			data->portStatusReg, data->rootCmplxStatus,
			data->busAgentStatus);
	if (data->deviceStatus || data->slotStatus   ||
	    data->linkStatus   || data->devCmdStatus ||
	    data->devSecStatus)
		pr_info("  RootSts:     %08x %08x %08x %08x %08x\n",
			data->deviceStatus, data->slotStatus,
			data->linkStatus, data->devCmdStatus,
			data->devSecStatus);
	if (data->rootErrorStatus || data->uncorrErrorStatus ||
	    data->corrErrorStatus)
		pr_info("  RootErrSts:  %08x %08x %08x\n",
			data->rootErrorStatus, data->uncorrErrorStatus,
			data->corrErrorStatus);
	if (data->tlpHdr1 || data->tlpHdr2 ||
	    data->tlpHdr3 || data->tlpHdr4)
		pr_info("  RootErrLog:  %08x %08x %08x %08x\n",
			data->tlpHdr1, data->tlpHdr2,
			data->tlpHdr3, data->tlpHdr4);
	if (data->sourceId || data->errorClass ||
	    data->correlator)
		pr_info("  RootErrLog1: %08x %016llx %016llx\n",
			data->sourceId, data->errorClass,
			data->correlator);
	if (data->nFir || data->nFirMask ||
	    data->nFirWOF)
		pr_info("  nFir:        %016llx %016llx %016llx\n",
			data->nFir, data->nFirMask,
			data->nFirWOF);
	if (data->phbPlssr || data->phbCsr)
		pr_info("  PhbSts:      %016llx %016llx\n",
			data->phbPlssr, data->phbCsr);
	if (data->lemFir || data->lemErrorMask ||
	    data->lemWOF)
		pr_info("  Lem:         %016llx %016llx %016llx\n",
			data->lemFir, data->lemErrorMask,
			data->lemWOF);
	if (data->phbErrorStatus || data->phbFirstErrorStatus ||
	    data->phbErrorLog0   || data->phbErrorLog1)
		pr_info("  PhbErr:      %016llx %016llx %016llx %016llx\n",
			data->phbErrorStatus, data->phbFirstErrorStatus,
			data->phbErrorLog0, data->phbErrorLog1);
	if (data->mmioErrorStatus || data->mmioFirstErrorStatus ||
	    data->mmioErrorLog0   || data->mmioErrorLog1)
		pr_info("  OutErr:      %016llx %016llx %016llx %016llx\n",
			data->mmioErrorStatus, data->mmioFirstErrorStatus,
			data->mmioErrorLog0, data->mmioErrorLog1);
	if (data->dma0ErrorStatus || data->dma0FirstErrorStatus ||
	    data->dma0ErrorLog0   || data->dma0ErrorLog1)
		pr_info("  InAErr:      %016llx %016llx %016llx %016llx\n",
			data->dma0ErrorStatus, data->dma0FirstErrorStatus,
			data->dma0ErrorLog0, data->dma0ErrorLog1);
	if (data->dma1ErrorStatus || data->dma1FirstErrorStatus ||
	    data->dma1ErrorLog0   || data->dma1ErrorLog1)
		pr_info("  InBErr:      %016llx %016llx %016llx %016llx\n",
			data->dma1ErrorStatus, data->dma1FirstErrorStatus,
			data->dma1ErrorLog0, data->dma1ErrorLog1);
278 279 280 281 282 283

	for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
		if ((data->pestA[i] >> 63) == 0 &&
		    (data->pestB[i] >> 63) == 0)
			continue;

284 285
		pr_info("  PE[%3d] A/B: %016llx %016llx\n",
			i, data->pestA[i], data->pestB[i]);
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
	}
}

void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
				unsigned char *log_buff)
{
	struct OpalIoPhbErrorCommon *common;

	if (!hose || !log_buff)
		return;

	common = (struct OpalIoPhbErrorCommon *)log_buff;
	switch (common->ioType) {
	case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
		pnv_pci_dump_p7ioc_diag_data(hose, common);
		break;
	case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
		pnv_pci_dump_phb3_diag_data(hose, common);
304 305
		break;
	default:
306 307
		pr_warn("%s: Unrecognized ioType %d\n",
			__func__, common->ioType);
308 309 310 311 312 313 314 315 316 317
	}
}

static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
{
	unsigned long flags, rc;
	int has_diag;

	spin_lock_irqsave(&phb->lock, flags);

318 319
	rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
					 PNV_PCI_DIAG_BUF_SIZE);
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
	has_diag = (rc == OPAL_SUCCESS);

	rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no,
				       OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
	if (rc) {
		pr_warning("PCI %d: Failed to clear EEH freeze state"
			   " for PE#%d, err %ld\n",
			   phb->hose->global_number, pe_no, rc);

		/* For now, let's only display the diag buffer when we fail to clear
		 * the EEH status. We'll do more sensible things later when we have
		 * proper EEH support. We need to make sure we don't pollute ourselves
		 * with the normal errors generated when probing empty slots
		 */
		if (has_diag)
335
			pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);
336 337 338 339 340 341 342 343
		else
			pr_warning("PCI %d: No diag data available\n",
				   phb->hose->global_number);
	}

	spin_unlock_irqrestore(&phb->lock, flags);
}

344 345
static void pnv_pci_config_check_eeh(struct pnv_phb *phb,
				     struct device_node *dn)
346 347 348
{
	s64	rc;
	u8	fstate;
349
	__be16	pcierr;
350 351
	u32	pe_no;

352 353 354
	/*
	 * Get the PE#. During the PCI probe stage, we might not
	 * setup that yet. So all ER errors should be mapped to
355
	 * reserved PE.
356 357
	 */
	pe_no = PCI_DN(dn)->pe_number;
358 359 360 361 362 363
	if (pe_no == IODA_INVALID_PE) {
		if (phb->type == PNV_PHB_P5IOC2)
			pe_no = 0;
		else
			pe_no = phb->ioda.reserved_pe;
	}
364 365 366 367 368

	/* Read freeze status */
	rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no, &fstate, &pcierr,
					NULL);
	if (rc) {
369 370 371
		pr_warning("%s: Can't read EEH status (PE#%d) for "
			   "%s, err %lld\n",
			   __func__, pe_no, dn->full_name, rc);
372 373
		return;
	}
374 375 376
	cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
		(PCI_DN(dn)->busno << 8) | (PCI_DN(dn)->devfn),
		pe_no, fstate);
377 378
	if (fstate != 0)
		pnv_pci_handle_eeh_config(phb, pe_no);
379 380
}

381 382
int pnv_pci_cfg_read(struct device_node *dn,
		     int where, int size, u32 *val)
383
{
384 385 386
	struct pci_dn *pdn = PCI_DN(dn);
	struct pnv_phb *phb = pdn->phb->private_data;
	u32 bdfn = (pdn->busno << 8) | pdn->devfn;
387 388 389
#ifdef CONFIG_EEH
	struct eeh_pe *phb_pe = NULL;
#endif
390 391 392 393 394 395 396 397 398 399
	s64 rc;

	switch (size) {
	case 1: {
		u8 v8;
		rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
		*val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
		break;
	}
	case 2: {
400
		__be16 v16;
401 402
		rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
						   &v16);
403
		*val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
404 405 406
		break;
	}
	case 4: {
407
		__be32 v32;
408
		rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
409
		*val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
410 411 412 413 414
		break;
	}
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
415 416
	cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
		__func__, pdn->busno, pdn->devfn, where, size, *val);
417

418 419 420 421 422 423 424
	/*
	 * Check if the specified PE has been put into frozen
	 * state. On the other hand, we needn't do that while
	 * the PHB has been put into frozen state because of
	 * PHB-fatal errors.
	 */
#ifdef CONFIG_EEH
425
	phb_pe = eeh_phb_pe_get(pdn->phb);
426 427 428
	if (phb_pe && (phb_pe->state & EEH_PE_ISOLATED))
		return PCIBIOS_SUCCESSFUL;

429
	if (phb->flags & PNV_PHB_FLAG_EEH) {
430 431 432
		if (*val == EEH_IO_ERROR_VALUE(size) &&
		    eeh_dev_check_failure(of_node_to_eeh_dev(dn)))
			return PCIBIOS_DEVICE_NOT_FOUND;
433
	} else {
434
		pnv_pci_config_check_eeh(phb, dn);
435 436
	}
#else
437
	pnv_pci_config_check_eeh(phb, dn);
438
#endif
439 440 441 442

	return PCIBIOS_SUCCESSFUL;
}

443 444
int pnv_pci_cfg_write(struct device_node *dn,
		      int where, int size, u32 val)
445
{
446 447 448
	struct pci_dn *pdn = PCI_DN(dn);
	struct pnv_phb *phb = pdn->phb->private_data;
	u32 bdfn = (pdn->busno << 8) | pdn->devfn;
449

450 451
	cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
		pdn->busno, pdn->devfn, where, size, val);
452 453 454 455 456 457 458 459 460 461 462 463 464
	switch (size) {
	case 1:
		opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
		break;
	case 2:
		opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
		break;
	case 4:
		opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
		break;
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
465

466
	/* Check if the PHB got frozen due to an error (no response) */
467
	if (!(phb->flags & PNV_PHB_FLAG_EEH))
468
		pnv_pci_config_check_eeh(phb, dn);
469 470 471 472

	return PCIBIOS_SUCCESSFUL;
}

473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
static int pnv_pci_read_config(struct pci_bus *bus,
			       unsigned int devfn,
			       int where, int size, u32 *val)
{
	struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
	struct pci_dn *pdn;

	for (dn = busdn->child; dn; dn = dn->sibling) {
		pdn = PCI_DN(dn);
		if (pdn && pdn->devfn == devfn)
			return pnv_pci_cfg_read(dn, where, size, val);
	}

	*val = 0xFFFFFFFF;
	return PCIBIOS_DEVICE_NOT_FOUND;

}

static int pnv_pci_write_config(struct pci_bus *bus,
				unsigned int devfn,
				int where, int size, u32 val)
{
	struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
	struct pci_dn *pdn;

	for (dn = busdn->child; dn; dn = dn->sibling) {
		pdn = PCI_DN(dn);
		if (pdn && pdn->devfn == devfn)
			return pnv_pci_cfg_write(dn, where, size, val);
	}

	return PCIBIOS_DEVICE_NOT_FOUND;
}

507
struct pci_ops pnv_pci_ops = {
508
	.read  = pnv_pci_read_config,
509 510 511 512 513
	.write = pnv_pci_write_config,
};

static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
			 unsigned long uaddr, enum dma_data_direction direction,
514
			 struct dma_attrs *attrs, bool rm)
515 516
{
	u64 proto_tce;
517
	__be64 *tcep, *tces;
518 519 520 521 522 523 524
	u64 rpn;

	proto_tce = TCE_PCI_READ; // Read allowed

	if (direction != DMA_TO_DEVICE)
		proto_tce |= TCE_PCI_WRITE;

525
	tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
526
	rpn = __pa(uaddr) >> TCE_SHIFT;
527

528
	while (npages--)
529
		*(tcep++) = cpu_to_be64(proto_tce | (rpn++ << TCE_RPN_SHIFT));
530 531 532 533 534 535

	/* Some implementations won't cache invalid TCEs and thus may not
	 * need that flush. We'll probably turn it_type into a bit mask
	 * of flags if that becomes the case
	 */
	if (tbl->it_type & TCE_PCI_SWINV_CREATE)
536
		pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
537 538 539 540

	return 0;
}

541 542 543 544 545 546 547 548 549 550 551
static int pnv_tce_build_vm(struct iommu_table *tbl, long index, long npages,
			    unsigned long uaddr,
			    enum dma_data_direction direction,
			    struct dma_attrs *attrs)
{
	return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs,
			false);
}

static void pnv_tce_free(struct iommu_table *tbl, long index, long npages,
		bool rm)
552
{
553
	__be64 *tcep, *tces;
554

555
	tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
556 557

	while (npages--)
558
		*(tcep++) = cpu_to_be64(0);
559

560
	if (tbl->it_type & TCE_PCI_SWINV_FREE)
561 562 563 564 565 566
		pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
}

static void pnv_tce_free_vm(struct iommu_table *tbl, long index, long npages)
{
	pnv_tce_free(tbl, index, npages, false);
567 568
}

569 570 571 572 573
static unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
{
	return ((u64 *)tbl->it_base)[index - tbl->it_offset];
}

574 575 576 577 578 579 580 581 582 583 584 585 586
static int pnv_tce_build_rm(struct iommu_table *tbl, long index, long npages,
			    unsigned long uaddr,
			    enum dma_data_direction direction,
			    struct dma_attrs *attrs)
{
	return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs, true);
}

static void pnv_tce_free_rm(struct iommu_table *tbl, long index, long npages)
{
	pnv_tce_free(tbl, index, npages, true);
}

587 588 589 590 591 592
void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
			       void *tce_mem, u64 tce_size,
			       u64 dma_offset)
{
	tbl->it_blocksize = 16;
	tbl->it_base = (unsigned long)tce_mem;
593 594
	tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K;
	tbl->it_offset = dma_offset >> tbl->it_page_shift;
595 596 597 598 599 600
	tbl->it_index = 0;
	tbl->it_size = tce_size >> 3;
	tbl->it_busno = 0;
	tbl->it_type = TCE_PCI;
}

601
static struct iommu_table *pnv_pci_setup_bml_iommu(struct pci_controller *hose)
602 603
{
	struct iommu_table *tbl;
604 605
	const __be64 *basep, *swinvp;
	const __be32 *sizep;
606 607 608 609

	basep = of_get_property(hose->dn, "linux,tce-base", NULL);
	sizep = of_get_property(hose->dn, "linux,tce-size", NULL);
	if (basep == NULL || sizep == NULL) {
610 611
		pr_err("PCI: %s has missing tce entries !\n",
		       hose->dn->full_name);
612 613 614 615 616 617 618 619
		return NULL;
	}
	tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node);
	if (WARN_ON(!tbl))
		return NULL;
	pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),
				  be32_to_cpup(sizep), 0);
	iommu_init_table(tbl, hose->node);
620
	iommu_register_group(tbl, pci_domain_nr(hose->bus), 0);
621 622 623 624 625

	/* Deal with SW invalidated TCEs when needed (BML way) */
	swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",
				 NULL);
	if (swinvp) {
626
		tbl->it_busno = be64_to_cpu(swinvp[1]);
627
		tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8);
628 629
		tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
	}
630 631 632
	return tbl;
}

633 634
static void pnv_pci_dma_fallback_setup(struct pci_controller *hose,
				       struct pci_dev *pdev)
635 636 637 638 639 640 641 642 643 644 645
{
	struct device_node *np = pci_bus_to_OF_node(hose->bus);
	struct pci_dn *pdn;

	if (np == NULL)
		return;
	pdn = PCI_DN(np);
	if (!pdn->iommu_table)
		pdn->iommu_table = pnv_pci_setup_bml_iommu(hose);
	if (!pdn->iommu_table)
		return;
646
	set_iommu_table_base_and_group(&pdev->dev, pdn->iommu_table);
647 648
}

649
static void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
650 651 652 653 654 655 656 657 658 659 660 661 662
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;

	/* If we have no phb structure, try to setup a fallback based on
	 * the device-tree (RTAS PCI for example)
	 */
	if (phb && phb->dma_dev_setup)
		phb->dma_dev_setup(phb, pdev);
	else
		pnv_pci_dma_fallback_setup(hose, pdev);
}

663 664 665 666 667 668 669 670 671 672
int pnv_pci_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;

	if (phb && phb->dma_set_mask)
		return phb->dma_set_mask(phb, pdev, dma_mask);
	return __dma_set_mask(&pdev->dev, dma_mask);
}

673 674 675 676 677 678 679 680 681 682 683 684
void pnv_pci_shutdown(void)
{
	struct pci_controller *hose;

	list_for_each_entry(hose, &hose_list, list_node) {
		struct pnv_phb *phb = hose->private_data;

		if (phb && phb->shutdown)
			phb->shutdown(phb);
	}
}

G
Gavin Shan 已提交
685
/* Fixup wrong class code in p7ioc and p8 root complex */
686
static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
687 688 689 690 691
{
	dev->class = PCI_CLASS_BRIDGE_PCI << 8;
}
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);

692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720
static int pnv_pci_probe_mode(struct pci_bus *bus)
{
	struct pci_controller *hose = pci_bus_to_host(bus);
	const __be64 *tstamp;
	u64 now, target;


	/* We hijack this as a way to ensure we have waited long
	 * enough since the reset was lifted on the PCI bus
	 */
	if (bus != hose->bus)
		return PCI_PROBE_NORMAL;
	tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL);
	if (!tstamp || !*tstamp)
		return PCI_PROBE_NORMAL;

	now = mftb() / tb_ticks_per_usec;
	target = (be64_to_cpup(tstamp) / tb_ticks_per_usec)
		+ PCI_RESET_DELAY_US;

	pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n",
		 hose->global_number, target, now);

	if (now < target)
		msleep((target - now + 999) / 1000);

	return PCI_PROBE_NORMAL;
}

721 722 723 724
void __init pnv_pci_init(void)
{
	struct device_node *np;

725
	pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
726 727 728 729 730 731 732

	/* OPAL absent, try POPAL first then RTAS detection of PHBs */
	if (!firmware_has_feature(FW_FEATURE_OPAL)) {
#ifdef CONFIG_PPC_POWERNV_RTAS
		init_pci_config_tokens();
		find_and_init_phbs();
#endif /* CONFIG_PPC_POWERNV_RTAS */
733 734 735 736 737 738 739 740 741 742 743 744 745
	}
	/* OPAL is here, do our normal stuff */
	else {
		int found_ioda = 0;

		/* Look for IODA IO-Hubs. We don't support mixing IODA
		 * and p5ioc2 due to the need to change some global
		 * probing flags
		 */
		for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
			pnv_pci_init_ioda_hub(np);
			found_ioda = 1;
		}
746 747

		/* Look for p5ioc2 IO-Hubs */
748 749 750
		if (!found_ioda)
			for_each_compatible_node(np, NULL, "ibm,p5ioc2")
				pnv_pci_init_p5ioc2_hub(np);
G
Gavin Shan 已提交
751 752 753 754

		/* Look for ioda2 built-in PHB3's */
		for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
			pnv_pci_init_ioda2_phb(np);
755 756 757 758 759 760 761
	}

	/* Setup the linkage between OF nodes and PHBs */
	pci_devs_phb_init();

	/* Configure IOMMU DMA hooks */
	ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup;
762 763 764 765
	ppc_md.tce_build = pnv_tce_build_vm;
	ppc_md.tce_free = pnv_tce_free_vm;
	ppc_md.tce_build_rm = pnv_tce_build_rm;
	ppc_md.tce_free_rm = pnv_tce_free_rm;
766
	ppc_md.tce_get = pnv_tce_get;
767
	ppc_md.pci_probe_mode = pnv_pci_probe_mode;
768 769
	set_pci_dma_ops(&dma_iommu_ops);

770 771 772 773 774 775
	/* Configure MSIs */
#ifdef CONFIG_PCI_MSI
	ppc_md.msi_check_device = pnv_msi_check_device;
	ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
	ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
#endif
776
}
777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805

static int tce_iommu_bus_notifier(struct notifier_block *nb,
		unsigned long action, void *data)
{
	struct device *dev = data;

	switch (action) {
	case BUS_NOTIFY_ADD_DEVICE:
		return iommu_add_device(dev);
	case BUS_NOTIFY_DEL_DEVICE:
		if (dev->iommu_group)
			iommu_del_device(dev);
		return 0;
	default:
		return 0;
	}
}

static struct notifier_block tce_iommu_bus_nb = {
	.notifier_call = tce_iommu_bus_notifier,
};

static int __init tce_iommu_bus_notifier_init(void)
{
	bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
	return 0;
}

subsys_initcall_sync(tce_iommu_bus_notifier_init);