pci.c 19.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
/*
 * Support PCI/PCIe on PowerNV platforms
 *
 * Currently supports only P5IOC2
 *
 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/irq.h>
#include <linux/io.h>
21
#include <linux/msi.h>
22
#include <linux/iommu.h>
23 24 25 26 27 28

#include <asm/sections.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/pci-bridge.h>
#include <asm/machdep.h>
29
#include <asm/msi_bitmap.h>
30 31 32 33
#include <asm/ppc-pci.h>
#include <asm/opal.h>
#include <asm/iommu.h>
#include <asm/tce.h>
34
#include <asm/firmware.h>
35 36
#include <asm/eeh_event.h>
#include <asm/eeh.h>
37 38 39 40

#include "powernv.h"
#include "pci.h"

41 42
/* Delay in usec */
#define PCI_RESET_DELAY_US	3000000
43 44 45 46

#define cfg_dbg(fmt...)	do { } while(0)
//#define cfg_dbg(fmt...)	printk(fmt)

47
#ifdef CONFIG_PCI_MSI
48
int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
49 50 51 52 53
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;
	struct msi_msg msg;
54 55
	int hwirq;
	unsigned int virq;
56 57
	int rc;

58 59 60
	if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
		return -ENODEV;

61
	if (pdev->no_64bit_msi && !phb->msi32_support)
62 63 64 65 66 67 68 69
		return -ENODEV;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
			pr_warn("%s: Supports only 64-bit MSIs\n",
				pci_name(pdev));
			return -ENXIO;
		}
70 71
		hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
		if (hwirq < 0) {
72 73 74 75
			pr_warn("%s: Failed to find a free MSI\n",
				pci_name(pdev));
			return -ENOSPC;
		}
76
		virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
77 78 79
		if (virq == NO_IRQ) {
			pr_warn("%s: Failed to map MSI to linux irq\n",
				pci_name(pdev));
80
			msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
81 82
			return -ENOMEM;
		}
83
		rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
84
				    virq, entry->msi_attrib.is_64, &msg);
85 86 87
		if (rc) {
			pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
			irq_dispose_mapping(virq);
88
			msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
89 90 91
			return rc;
		}
		irq_set_msi_desc(virq, entry);
92
		pci_write_msi_msg(virq, &msg);
93 94 95 96
	}
	return 0;
}

97
void pnv_teardown_msi_irqs(struct pci_dev *pdev)
98 99 100 101 102 103 104 105 106 107 108 109
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;

	if (WARN_ON(!phb))
		return;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (entry->irq == NO_IRQ)
			continue;
		irq_set_msi_desc(entry->irq, NULL);
110 111
		msi_bitmap_free_hwirqs(&phb->msi_bmp,
			virq_to_hw(entry->irq) - phb->msi_base, 1);
112 113 114 115
		irq_dispose_mapping(entry->irq);
	}
}
#endif /* CONFIG_PCI_MSI */
116

117 118
static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
					 struct OpalIoPhbErrorCommon *common)
119
{
120
	struct OpalIoP7IOCPhbErrorData *data;
121 122
	int i;

123
	data = (struct OpalIoP7IOCPhbErrorData *)common;
124
	pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
125
		hose->global_number, be32_to_cpu(common->version));
126

127
	if (data->brdgCtl)
128
		pr_info("brdgCtl:     %08x\n",
129
			be32_to_cpu(data->brdgCtl));
130 131
	if (data->portStatusReg || data->rootCmplxStatus ||
	    data->busAgentStatus)
132
		pr_info("UtlSts:      %08x %08x %08x\n",
133 134 135
			be32_to_cpu(data->portStatusReg),
			be32_to_cpu(data->rootCmplxStatus),
			be32_to_cpu(data->busAgentStatus));
136 137 138
	if (data->deviceStatus || data->slotStatus   ||
	    data->linkStatus   || data->devCmdStatus ||
	    data->devSecStatus)
139
		pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
140 141 142 143 144
			be32_to_cpu(data->deviceStatus),
			be32_to_cpu(data->slotStatus),
			be32_to_cpu(data->linkStatus),
			be32_to_cpu(data->devCmdStatus),
			be32_to_cpu(data->devSecStatus));
145 146
	if (data->rootErrorStatus   || data->uncorrErrorStatus ||
	    data->corrErrorStatus)
147
		pr_info("RootErrSts:  %08x %08x %08x\n",
148 149 150
			be32_to_cpu(data->rootErrorStatus),
			be32_to_cpu(data->uncorrErrorStatus),
			be32_to_cpu(data->corrErrorStatus));
151 152
	if (data->tlpHdr1 || data->tlpHdr2 ||
	    data->tlpHdr3 || data->tlpHdr4)
153
		pr_info("RootErrLog:  %08x %08x %08x %08x\n",
154 155 156 157
			be32_to_cpu(data->tlpHdr1),
			be32_to_cpu(data->tlpHdr2),
			be32_to_cpu(data->tlpHdr3),
			be32_to_cpu(data->tlpHdr4));
158 159
	if (data->sourceId || data->errorClass ||
	    data->correlator)
160
		pr_info("RootErrLog1: %08x %016llx %016llx\n",
161 162 163
			be32_to_cpu(data->sourceId),
			be64_to_cpu(data->errorClass),
			be64_to_cpu(data->correlator));
164
	if (data->p7iocPlssr || data->p7iocCsr)
165
		pr_info("PhbSts:      %016llx %016llx\n",
166 167
			be64_to_cpu(data->p7iocPlssr),
			be64_to_cpu(data->p7iocCsr));
168 169
	if (data->lemFir)
		pr_info("Lem:         %016llx %016llx %016llx\n",
170 171 172
			be64_to_cpu(data->lemFir),
			be64_to_cpu(data->lemErrorMask),
			be64_to_cpu(data->lemWOF));
173 174
	if (data->phbErrorStatus)
		pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
175 176 177 178
			be64_to_cpu(data->phbErrorStatus),
			be64_to_cpu(data->phbFirstErrorStatus),
			be64_to_cpu(data->phbErrorLog0),
			be64_to_cpu(data->phbErrorLog1));
179 180
	if (data->mmioErrorStatus)
		pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
181 182 183 184
			be64_to_cpu(data->mmioErrorStatus),
			be64_to_cpu(data->mmioFirstErrorStatus),
			be64_to_cpu(data->mmioErrorLog0),
			be64_to_cpu(data->mmioErrorLog1));
185 186
	if (data->dma0ErrorStatus)
		pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
187 188 189 190
			be64_to_cpu(data->dma0ErrorStatus),
			be64_to_cpu(data->dma0FirstErrorStatus),
			be64_to_cpu(data->dma0ErrorLog0),
			be64_to_cpu(data->dma0ErrorLog1));
191 192
	if (data->dma1ErrorStatus)
		pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
193 194 195 196
			be64_to_cpu(data->dma1ErrorStatus),
			be64_to_cpu(data->dma1FirstErrorStatus),
			be64_to_cpu(data->dma1ErrorLog0),
			be64_to_cpu(data->dma1ErrorLog1));
197 198 199 200 201

	for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
		if ((data->pestA[i] >> 63) == 0 &&
		    (data->pestB[i] >> 63) == 0)
			continue;
202

203
		pr_info("PE[%3d] A/B: %016llx %016llx\n",
204 205
			i, be64_to_cpu(data->pestA[i]),
			be64_to_cpu(data->pestB[i]));
206 207 208
	}
}

209 210
static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
					struct OpalIoPhbErrorCommon *common)
211
{
212 213 214 215
	struct OpalIoPhb3ErrorData *data;
	int i;

	data = (struct OpalIoPhb3ErrorData*)common;
216
	pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
217
		hose->global_number, be32_to_cpu(common->version));
218
	if (data->brdgCtl)
219
		pr_info("brdgCtl:     %08x\n",
220
			be32_to_cpu(data->brdgCtl));
221 222
	if (data->portStatusReg || data->rootCmplxStatus ||
	    data->busAgentStatus)
223
		pr_info("UtlSts:      %08x %08x %08x\n",
224 225 226
			be32_to_cpu(data->portStatusReg),
			be32_to_cpu(data->rootCmplxStatus),
			be32_to_cpu(data->busAgentStatus));
227 228 229
	if (data->deviceStatus || data->slotStatus   ||
	    data->linkStatus   || data->devCmdStatus ||
	    data->devSecStatus)
230
		pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
231 232 233 234 235
			be32_to_cpu(data->deviceStatus),
			be32_to_cpu(data->slotStatus),
			be32_to_cpu(data->linkStatus),
			be32_to_cpu(data->devCmdStatus),
			be32_to_cpu(data->devSecStatus));
236 237
	if (data->rootErrorStatus || data->uncorrErrorStatus ||
	    data->corrErrorStatus)
238
		pr_info("RootErrSts:  %08x %08x %08x\n",
239 240 241
			be32_to_cpu(data->rootErrorStatus),
			be32_to_cpu(data->uncorrErrorStatus),
			be32_to_cpu(data->corrErrorStatus));
242 243
	if (data->tlpHdr1 || data->tlpHdr2 ||
	    data->tlpHdr3 || data->tlpHdr4)
244
		pr_info("RootErrLog:  %08x %08x %08x %08x\n",
245 246 247 248
			be32_to_cpu(data->tlpHdr1),
			be32_to_cpu(data->tlpHdr2),
			be32_to_cpu(data->tlpHdr3),
			be32_to_cpu(data->tlpHdr4));
249 250
	if (data->sourceId || data->errorClass ||
	    data->correlator)
251
		pr_info("RootErrLog1: %08x %016llx %016llx\n",
252 253 254
			be32_to_cpu(data->sourceId),
			be64_to_cpu(data->errorClass),
			be64_to_cpu(data->correlator));
255 256
	if (data->nFir)
		pr_info("nFir:        %016llx %016llx %016llx\n",
257 258 259
			be64_to_cpu(data->nFir),
			be64_to_cpu(data->nFirMask),
			be64_to_cpu(data->nFirWOF));
260
	if (data->phbPlssr || data->phbCsr)
261
		pr_info("PhbSts:      %016llx %016llx\n",
262 263
			be64_to_cpu(data->phbPlssr),
			be64_to_cpu(data->phbCsr));
264 265
	if (data->lemFir)
		pr_info("Lem:         %016llx %016llx %016llx\n",
266 267 268
			be64_to_cpu(data->lemFir),
			be64_to_cpu(data->lemErrorMask),
			be64_to_cpu(data->lemWOF));
269 270
	if (data->phbErrorStatus)
		pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
271 272 273 274
			be64_to_cpu(data->phbErrorStatus),
			be64_to_cpu(data->phbFirstErrorStatus),
			be64_to_cpu(data->phbErrorLog0),
			be64_to_cpu(data->phbErrorLog1));
275 276
	if (data->mmioErrorStatus)
		pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
277 278 279 280
			be64_to_cpu(data->mmioErrorStatus),
			be64_to_cpu(data->mmioFirstErrorStatus),
			be64_to_cpu(data->mmioErrorLog0),
			be64_to_cpu(data->mmioErrorLog1));
281 282
	if (data->dma0ErrorStatus)
		pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
283 284 285 286
			be64_to_cpu(data->dma0ErrorStatus),
			be64_to_cpu(data->dma0FirstErrorStatus),
			be64_to_cpu(data->dma0ErrorLog0),
			be64_to_cpu(data->dma0ErrorLog1));
287 288
	if (data->dma1ErrorStatus)
		pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
289 290 291 292
			be64_to_cpu(data->dma1ErrorStatus),
			be64_to_cpu(data->dma1FirstErrorStatus),
			be64_to_cpu(data->dma1ErrorLog0),
			be64_to_cpu(data->dma1ErrorLog1));
293 294

	for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
295 296
		if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
		    (be64_to_cpu(data->pestB[i]) >> 63) == 0)
297 298
			continue;

299
		pr_info("PE[%3d] A/B: %016llx %016llx\n",
300 301
				i, be64_to_cpu(data->pestA[i]),
				be64_to_cpu(data->pestB[i]));
302 303 304 305 306 307 308 309 310 311 312 313
	}
}

void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
				unsigned char *log_buff)
{
	struct OpalIoPhbErrorCommon *common;

	if (!hose || !log_buff)
		return;

	common = (struct OpalIoPhbErrorCommon *)log_buff;
314
	switch (be32_to_cpu(common->ioType)) {
315 316 317 318 319
	case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
		pnv_pci_dump_p7ioc_diag_data(hose, common);
		break;
	case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
		pnv_pci_dump_phb3_diag_data(hose, common);
320 321
		break;
	default:
322
		pr_warn("%s: Unrecognized ioType %d\n",
323
			__func__, be32_to_cpu(common->ioType));
324 325 326 327 328 329
	}
}

static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
{
	unsigned long flags, rc;
330
	int has_diag, ret = 0;
331 332 333

	spin_lock_irqsave(&phb->lock, flags);

334
	/* Fetch PHB diag-data */
335 336
	rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
					 PNV_PCI_DIAG_BUF_SIZE);
337 338
	has_diag = (rc == OPAL_SUCCESS);

339 340 341 342
	/* If PHB supports compound PE, to handle it */
	if (phb->unfreeze_pe) {
		ret = phb->unfreeze_pe(phb,
				       pe_no,
343
				       OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
344 345 346 347 348 349 350 351 352 353 354
	} else {
		rc = opal_pci_eeh_freeze_clear(phb->opal_id,
					     pe_no,
					     OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
		if (rc) {
			pr_warn("%s: Failure %ld clearing frozen "
				"PHB#%x-PE#%x\n",
				__func__, rc, phb->hose->global_number,
				pe_no);
			ret = -EIO;
		}
355 356
	}

357 358 359 360 361 362 363 364 365
	/*
	 * For now, let's only display the diag buffer when we fail to clear
	 * the EEH status. We'll do more sensible things later when we have
	 * proper EEH support. We need to make sure we don't pollute ourselves
	 * with the normal errors generated when probing empty slots
	 */
	if (has_diag && ret)
		pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);

366 367 368
	spin_unlock_irqrestore(&phb->lock, flags);
}

369
static void pnv_pci_config_check_eeh(struct pci_dn *pdn)
370
{
371
	struct pnv_phb *phb = pdn->phb->private_data;
372
	u8	fstate;
373
	__be16	pcierr;
374 375
	int	pe_no;
	s64	rc;
376

377 378 379
	/*
	 * Get the PE#. During the PCI probe stage, we might not
	 * setup that yet. So all ER errors should be mapped to
380
	 * reserved PE.
381
	 */
382
	pe_no = pdn->pe_number;
383 384 385 386 387 388
	if (pe_no == IODA_INVALID_PE) {
		if (phb->type == PNV_PHB_P5IOC2)
			pe_no = 0;
		else
			pe_no = phb->ioda.reserved_pe;
	}
389

390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
	/*
	 * Fetch frozen state. If the PHB support compound PE,
	 * we need handle that case.
	 */
	if (phb->get_pe_state) {
		fstate = phb->get_pe_state(phb, pe_no);
	} else {
		rc = opal_pci_eeh_freeze_status(phb->opal_id,
						pe_no,
						&fstate,
						&pcierr,
						NULL);
		if (rc) {
			pr_warn("%s: Failure %lld getting PHB#%x-PE#%x state\n",
				__func__, rc, phb->hose->global_number, pe_no);
			return;
		}
407
	}
408

409
	cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
410
		(pdn->busno << 8) | (pdn->devfn), pe_no, fstate);
411 412 413 414 415 416 417 418 419 420 421 422

	/* Clear the frozen state if applicable */
	if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE ||
	    fstate == OPAL_EEH_STOPPED_DMA_FREEZE  ||
	    fstate == OPAL_EEH_STOPPED_MMIO_DMA_FREEZE) {
		/*
		 * If PHB supports compound PE, freeze it for
		 * consistency.
		 */
		if (phb->freeze_pe)
			phb->freeze_pe(phb, pe_no);

423
		pnv_pci_handle_eeh_config(phb, pe_no);
424
	}
425 426
}

427
int pnv_pci_cfg_read(struct pci_dn *pdn,
428
		     int where, int size, u32 *val)
429
{
430 431
	struct pnv_phb *phb = pdn->phb->private_data;
	u32 bdfn = (pdn->busno << 8) | pdn->devfn;
432 433 434 435 436 437 438 439 440 441
	s64 rc;

	switch (size) {
	case 1: {
		u8 v8;
		rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
		*val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
		break;
	}
	case 2: {
442
		__be16 v16;
443 444
		rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
						   &v16);
445
		*val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
446 447 448
		break;
	}
	case 4: {
449
		__be32 v32;
450
		rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
451
		*val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
452 453 454 455 456
		break;
	}
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
457

458 459
	cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
		__func__, pdn->busno, pdn->devfn, where, size, *val);
460 461 462
	return PCIBIOS_SUCCESSFUL;
}

463
int pnv_pci_cfg_write(struct pci_dn *pdn,
464
		      int where, int size, u32 val)
465
{
466 467
	struct pnv_phb *phb = pdn->phb->private_data;
	u32 bdfn = (pdn->busno << 8) | pdn->devfn;
468

469 470
	cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
		pdn->busno, pdn->devfn, where, size, val);
471 472 473 474 475 476 477 478 479 480 481 482 483
	switch (size) {
	case 1:
		opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
		break;
	case 2:
		opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
		break;
	case 4:
		opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
		break;
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
484

485 486 487 488
	return PCIBIOS_SUCCESSFUL;
}

#if CONFIG_EEH
489
static bool pnv_pci_cfg_check(struct pci_dn *pdn)
490 491
{
	struct eeh_dev *edev = NULL;
492
	struct pnv_phb *phb = pdn->phb->private_data;
493 494

	/* EEH not enabled ? */
495
	if (!(phb->flags & PNV_PHB_FLAG_EEH))
496
		return true;
497

498
	/* PE reset or device removed ? */
499
	edev = pdn->edev;
500 501
	if (edev) {
		if (edev->pe &&
502
		    (edev->pe->state & EEH_PE_CFG_BLOCKED))
503 504 505 506 507
			return false;

		if (edev->mode & EEH_DEV_REMOVED)
			return false;
	}
508 509 510 511

	return true;
}
#else
512
static inline pnv_pci_cfg_check(struct pci_dn *pdn)
513 514
{
	return true;
515
}
516
#endif /* CONFIG_EEH */
517

518 519 520 521 522
static int pnv_pci_read_config(struct pci_bus *bus,
			       unsigned int devfn,
			       int where, int size, u32 *val)
{
	struct pci_dn *pdn;
523 524
	struct pnv_phb *phb;
	int ret;
525

526
	*val = 0xFFFFFFFF;
527 528 529
	pdn = pci_get_pdn_by_devfn(bus, devfn);
	if (!pdn)
		return PCIBIOS_DEVICE_NOT_FOUND;
530

531
	if (!pnv_pci_cfg_check(pdn))
532 533
		return PCIBIOS_DEVICE_NOT_FOUND;

534 535 536
	ret = pnv_pci_cfg_read(pdn, where, size, val);
	phb = pdn->phb->private_data;
	if (phb->flags & PNV_PHB_FLAG_EEH && pdn->edev) {
537
		if (*val == EEH_IO_ERROR_VALUE(size) &&
538
		    eeh_dev_check_failure(pdn->edev))
539 540
                        return PCIBIOS_DEVICE_NOT_FOUND;
	} else {
541
		pnv_pci_config_check_eeh(pdn);
542
	}
543

544
	return ret;
545 546 547 548 549 550 551
}

static int pnv_pci_write_config(struct pci_bus *bus,
				unsigned int devfn,
				int where, int size, u32 val)
{
	struct pci_dn *pdn;
552 553
	struct pnv_phb *phb;
	int ret;
554

555 556 557
	pdn = pci_get_pdn_by_devfn(bus, devfn);
	if (!pdn)
		return PCIBIOS_DEVICE_NOT_FOUND;
558

559
	if (!pnv_pci_cfg_check(pdn))
560 561
		return PCIBIOS_DEVICE_NOT_FOUND;

562 563
	ret = pnv_pci_cfg_write(pdn, where, size, val);
	phb = pdn->phb->private_data;
564
	if (!(phb->flags & PNV_PHB_FLAG_EEH))
565
		pnv_pci_config_check_eeh(pdn);
566 567

	return ret;
568 569
}

570
struct pci_ops pnv_pci_ops = {
571
	.read  = pnv_pci_read_config,
572 573 574
	.write = pnv_pci_write_config,
};

575 576 577
int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
		unsigned long uaddr, enum dma_data_direction direction,
		struct dma_attrs *attrs)
578
{
579
	u64 proto_tce = iommu_direction_to_tce_perm(direction);
580
	__be64 *tcep;
581 582
	u64 rpn;

583
	tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
584
	rpn = __pa(uaddr) >> tbl->it_page_shift;
585

586
	while (npages--)
587 588
		*(tcep++) = cpu_to_be64(proto_tce |
				(rpn++ << tbl->it_page_shift));
589

590 591 592 593

	return 0;
}

594
void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
595
{
596
	__be64 *tcep;
597

598
	tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
599 600

	while (npages--)
601
		*(tcep++) = cpu_to_be64(0);
602 603
}

604
unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
605 606 607 608
{
	return ((u64 *)tbl->it_base)[index - tbl->it_offset];
}

609 610
void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
			       void *tce_mem, u64 tce_size,
611
			       u64 dma_offset, unsigned page_shift)
612 613 614
{
	tbl->it_blocksize = 16;
	tbl->it_base = (unsigned long)tce_mem;
615
	tbl->it_page_shift = page_shift;
616
	tbl->it_offset = dma_offset >> tbl->it_page_shift;
617 618 619 620 621 622
	tbl->it_index = 0;
	tbl->it_size = tce_size >> 3;
	tbl->it_busno = 0;
	tbl->it_type = TCE_PCI;
}

623
void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
624 625 626
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644
#ifdef CONFIG_PCI_IOV
	struct pnv_ioda_pe *pe;
	struct pci_dn *pdn;

	/* Fix the VF pdn PE number */
	if (pdev->is_virtfn) {
		pdn = pci_get_pdn(pdev);
		WARN_ON(pdn->pe_number != IODA_INVALID_PE);
		list_for_each_entry(pe, &phb->ioda.pe_list, list) {
			if (pe->rid == ((pdev->bus->number << 8) |
			    (pdev->devfn & 0xff))) {
				pdn->pe_number = pe->pe_number;
				pe->pdev = pdev;
				break;
			}
		}
	}
#endif /* CONFIG_PCI_IOV */
645 646 647 648 649

	if (phb && phb->dma_dev_setup)
		phb->dma_dev_setup(phb, pdev);
}

650 651 652 653 654 655 656 657 658 659 660
u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;

	if (phb && phb->dma_get_required_mask)
		return phb->dma_get_required_mask(phb, pdev);

	return __dma_get_required_mask(&pdev->dev);
}

661 662 663 664
void pnv_pci_shutdown(void)
{
	struct pci_controller *hose;

665 666 667
	list_for_each_entry(hose, &hose_list, list_node)
		if (hose->controller_ops.shutdown)
			hose->controller_ops.shutdown(hose);
668 669
}

G
Gavin Shan 已提交
670
/* Fixup wrong class code in p7ioc and p8 root complex */
671
static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
672 673 674 675 676
{
	dev->class = PCI_CLASS_BRIDGE_PCI << 8;
}
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);

677 678 679
void __init pnv_pci_init(void)
{
	struct device_node *np;
680
	bool found_ioda = false;
681

682
	pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
683

684 685 686
	/* If we don't have OPAL, eg. in sim, just skip PCI probe */
	if (!firmware_has_feature(FW_FEATURE_OPAL))
		return;
687

688 689 690 691 692 693 694 695
	/* Look for IODA IO-Hubs. We don't support mixing IODA
	 * and p5ioc2 due to the need to change some global
	 * probing flags
	 */
	for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
		pnv_pci_init_ioda_hub(np);
		found_ioda = true;
	}
696

697 698 699 700
	/* Look for p5ioc2 IO-Hubs */
	if (!found_ioda)
		for_each_compatible_node(np, NULL, "ibm,p5ioc2")
			pnv_pci_init_p5ioc2_hub(np);
G
Gavin Shan 已提交
701

702 703 704
	/* Look for ioda2 built-in PHB3's */
	for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
		pnv_pci_init_ioda2_phb(np);
705 706 707 708 709 710 711

	/* Setup the linkage between OF nodes and PHBs */
	pci_devs_phb_init();

	/* Configure IOMMU DMA hooks */
	set_pci_dma_ops(&dma_iommu_ops);
}
712

713
machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init);