pci.c 21.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
/*
 * Support PCI/PCIe on PowerNV platforms
 *
 * Currently supports only P5IOC2
 *
 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/irq.h>
#include <linux/io.h>
21
#include <linux/msi.h>
22
#include <linux/iommu.h>
23 24 25 26 27 28

#include <asm/sections.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/pci-bridge.h>
#include <asm/machdep.h>
29
#include <asm/msi_bitmap.h>
30 31 32 33
#include <asm/ppc-pci.h>
#include <asm/opal.h>
#include <asm/iommu.h>
#include <asm/tce.h>
34
#include <asm/firmware.h>
35 36
#include <asm/eeh_event.h>
#include <asm/eeh.h>
37 38 39 40

#include "powernv.h"
#include "pci.h"

41 42
/* Delay in usec */
#define PCI_RESET_DELAY_US	3000000
43 44 45 46

#define cfg_dbg(fmt...)	do { } while(0)
//#define cfg_dbg(fmt...)	printk(fmt)

47
#ifdef CONFIG_PCI_MSI
48
int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
49 50 51 52 53
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;
	struct msi_msg msg;
54 55
	int hwirq;
	unsigned int virq;
56 57
	int rc;

58 59 60
	if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
		return -ENODEV;

61
	if (pdev->no_64bit_msi && !phb->msi32_support)
62 63 64 65 66 67 68 69
		return -ENODEV;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
			pr_warn("%s: Supports only 64-bit MSIs\n",
				pci_name(pdev));
			return -ENXIO;
		}
70 71
		hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
		if (hwirq < 0) {
72 73 74 75
			pr_warn("%s: Failed to find a free MSI\n",
				pci_name(pdev));
			return -ENOSPC;
		}
76
		virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
77 78 79
		if (virq == NO_IRQ) {
			pr_warn("%s: Failed to map MSI to linux irq\n",
				pci_name(pdev));
80
			msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
81 82
			return -ENOMEM;
		}
83
		rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
84
				    virq, entry->msi_attrib.is_64, &msg);
85 86 87
		if (rc) {
			pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
			irq_dispose_mapping(virq);
88
			msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
89 90 91
			return rc;
		}
		irq_set_msi_desc(virq, entry);
92
		pci_write_msi_msg(virq, &msg);
93 94 95 96
	}
	return 0;
}

97
void pnv_teardown_msi_irqs(struct pci_dev *pdev)
98 99 100 101 102 103 104 105 106 107 108 109
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;

	if (WARN_ON(!phb))
		return;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (entry->irq == NO_IRQ)
			continue;
		irq_set_msi_desc(entry->irq, NULL);
110 111
		msi_bitmap_free_hwirqs(&phb->msi_bmp,
			virq_to_hw(entry->irq) - phb->msi_base, 1);
112 113 114 115
		irq_dispose_mapping(entry->irq);
	}
}
#endif /* CONFIG_PCI_MSI */
116

117 118
static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
					 struct OpalIoPhbErrorCommon *common)
119
{
120
	struct OpalIoP7IOCPhbErrorData *data;
121 122
	int i;

123
	data = (struct OpalIoP7IOCPhbErrorData *)common;
124
	pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
125
		hose->global_number, be32_to_cpu(common->version));
126

127
	if (data->brdgCtl)
128
		pr_info("brdgCtl:     %08x\n",
129
			be32_to_cpu(data->brdgCtl));
130 131
	if (data->portStatusReg || data->rootCmplxStatus ||
	    data->busAgentStatus)
132
		pr_info("UtlSts:      %08x %08x %08x\n",
133 134 135
			be32_to_cpu(data->portStatusReg),
			be32_to_cpu(data->rootCmplxStatus),
			be32_to_cpu(data->busAgentStatus));
136 137 138
	if (data->deviceStatus || data->slotStatus   ||
	    data->linkStatus   || data->devCmdStatus ||
	    data->devSecStatus)
139
		pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
140 141 142 143 144
			be32_to_cpu(data->deviceStatus),
			be32_to_cpu(data->slotStatus),
			be32_to_cpu(data->linkStatus),
			be32_to_cpu(data->devCmdStatus),
			be32_to_cpu(data->devSecStatus));
145 146
	if (data->rootErrorStatus   || data->uncorrErrorStatus ||
	    data->corrErrorStatus)
147
		pr_info("RootErrSts:  %08x %08x %08x\n",
148 149 150
			be32_to_cpu(data->rootErrorStatus),
			be32_to_cpu(data->uncorrErrorStatus),
			be32_to_cpu(data->corrErrorStatus));
151 152
	if (data->tlpHdr1 || data->tlpHdr2 ||
	    data->tlpHdr3 || data->tlpHdr4)
153
		pr_info("RootErrLog:  %08x %08x %08x %08x\n",
154 155 156 157
			be32_to_cpu(data->tlpHdr1),
			be32_to_cpu(data->tlpHdr2),
			be32_to_cpu(data->tlpHdr3),
			be32_to_cpu(data->tlpHdr4));
158 159
	if (data->sourceId || data->errorClass ||
	    data->correlator)
160
		pr_info("RootErrLog1: %08x %016llx %016llx\n",
161 162 163
			be32_to_cpu(data->sourceId),
			be64_to_cpu(data->errorClass),
			be64_to_cpu(data->correlator));
164
	if (data->p7iocPlssr || data->p7iocCsr)
165
		pr_info("PhbSts:      %016llx %016llx\n",
166 167
			be64_to_cpu(data->p7iocPlssr),
			be64_to_cpu(data->p7iocCsr));
168 169
	if (data->lemFir)
		pr_info("Lem:         %016llx %016llx %016llx\n",
170 171 172
			be64_to_cpu(data->lemFir),
			be64_to_cpu(data->lemErrorMask),
			be64_to_cpu(data->lemWOF));
173 174
	if (data->phbErrorStatus)
		pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
175 176 177 178
			be64_to_cpu(data->phbErrorStatus),
			be64_to_cpu(data->phbFirstErrorStatus),
			be64_to_cpu(data->phbErrorLog0),
			be64_to_cpu(data->phbErrorLog1));
179 180
	if (data->mmioErrorStatus)
		pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
181 182 183 184
			be64_to_cpu(data->mmioErrorStatus),
			be64_to_cpu(data->mmioFirstErrorStatus),
			be64_to_cpu(data->mmioErrorLog0),
			be64_to_cpu(data->mmioErrorLog1));
185 186
	if (data->dma0ErrorStatus)
		pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
187 188 189 190
			be64_to_cpu(data->dma0ErrorStatus),
			be64_to_cpu(data->dma0FirstErrorStatus),
			be64_to_cpu(data->dma0ErrorLog0),
			be64_to_cpu(data->dma0ErrorLog1));
191 192
	if (data->dma1ErrorStatus)
		pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
193 194 195 196
			be64_to_cpu(data->dma1ErrorStatus),
			be64_to_cpu(data->dma1FirstErrorStatus),
			be64_to_cpu(data->dma1ErrorLog0),
			be64_to_cpu(data->dma1ErrorLog1));
197 198 199 200 201

	for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
		if ((data->pestA[i] >> 63) == 0 &&
		    (data->pestB[i] >> 63) == 0)
			continue;
202

203
		pr_info("PE[%3d] A/B: %016llx %016llx\n",
204 205
			i, be64_to_cpu(data->pestA[i]),
			be64_to_cpu(data->pestB[i]));
206 207 208
	}
}

209 210
static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
					struct OpalIoPhbErrorCommon *common)
211
{
212 213 214 215
	struct OpalIoPhb3ErrorData *data;
	int i;

	data = (struct OpalIoPhb3ErrorData*)common;
216
	pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
217
		hose->global_number, be32_to_cpu(common->version));
218
	if (data->brdgCtl)
219
		pr_info("brdgCtl:     %08x\n",
220
			be32_to_cpu(data->brdgCtl));
221 222
	if (data->portStatusReg || data->rootCmplxStatus ||
	    data->busAgentStatus)
223
		pr_info("UtlSts:      %08x %08x %08x\n",
224 225 226
			be32_to_cpu(data->portStatusReg),
			be32_to_cpu(data->rootCmplxStatus),
			be32_to_cpu(data->busAgentStatus));
227 228 229
	if (data->deviceStatus || data->slotStatus   ||
	    data->linkStatus   || data->devCmdStatus ||
	    data->devSecStatus)
230
		pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
231 232 233 234 235
			be32_to_cpu(data->deviceStatus),
			be32_to_cpu(data->slotStatus),
			be32_to_cpu(data->linkStatus),
			be32_to_cpu(data->devCmdStatus),
			be32_to_cpu(data->devSecStatus));
236 237
	if (data->rootErrorStatus || data->uncorrErrorStatus ||
	    data->corrErrorStatus)
238
		pr_info("RootErrSts:  %08x %08x %08x\n",
239 240 241
			be32_to_cpu(data->rootErrorStatus),
			be32_to_cpu(data->uncorrErrorStatus),
			be32_to_cpu(data->corrErrorStatus));
242 243
	if (data->tlpHdr1 || data->tlpHdr2 ||
	    data->tlpHdr3 || data->tlpHdr4)
244
		pr_info("RootErrLog:  %08x %08x %08x %08x\n",
245 246 247 248
			be32_to_cpu(data->tlpHdr1),
			be32_to_cpu(data->tlpHdr2),
			be32_to_cpu(data->tlpHdr3),
			be32_to_cpu(data->tlpHdr4));
249 250
	if (data->sourceId || data->errorClass ||
	    data->correlator)
251
		pr_info("RootErrLog1: %08x %016llx %016llx\n",
252 253 254
			be32_to_cpu(data->sourceId),
			be64_to_cpu(data->errorClass),
			be64_to_cpu(data->correlator));
255 256
	if (data->nFir)
		pr_info("nFir:        %016llx %016llx %016llx\n",
257 258 259
			be64_to_cpu(data->nFir),
			be64_to_cpu(data->nFirMask),
			be64_to_cpu(data->nFirWOF));
260
	if (data->phbPlssr || data->phbCsr)
261
		pr_info("PhbSts:      %016llx %016llx\n",
262 263
			be64_to_cpu(data->phbPlssr),
			be64_to_cpu(data->phbCsr));
264 265
	if (data->lemFir)
		pr_info("Lem:         %016llx %016llx %016llx\n",
266 267 268
			be64_to_cpu(data->lemFir),
			be64_to_cpu(data->lemErrorMask),
			be64_to_cpu(data->lemWOF));
269 270
	if (data->phbErrorStatus)
		pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
271 272 273 274
			be64_to_cpu(data->phbErrorStatus),
			be64_to_cpu(data->phbFirstErrorStatus),
			be64_to_cpu(data->phbErrorLog0),
			be64_to_cpu(data->phbErrorLog1));
275 276
	if (data->mmioErrorStatus)
		pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
277 278 279 280
			be64_to_cpu(data->mmioErrorStatus),
			be64_to_cpu(data->mmioFirstErrorStatus),
			be64_to_cpu(data->mmioErrorLog0),
			be64_to_cpu(data->mmioErrorLog1));
281 282
	if (data->dma0ErrorStatus)
		pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
283 284 285 286
			be64_to_cpu(data->dma0ErrorStatus),
			be64_to_cpu(data->dma0FirstErrorStatus),
			be64_to_cpu(data->dma0ErrorLog0),
			be64_to_cpu(data->dma0ErrorLog1));
287 288
	if (data->dma1ErrorStatus)
		pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
289 290 291 292
			be64_to_cpu(data->dma1ErrorStatus),
			be64_to_cpu(data->dma1FirstErrorStatus),
			be64_to_cpu(data->dma1ErrorLog0),
			be64_to_cpu(data->dma1ErrorLog1));
293 294

	for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
295 296
		if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
		    (be64_to_cpu(data->pestB[i]) >> 63) == 0)
297 298
			continue;

299
		pr_info("PE[%3d] A/B: %016llx %016llx\n",
300 301
				i, be64_to_cpu(data->pestA[i]),
				be64_to_cpu(data->pestB[i]));
302 303 304 305 306 307 308 309 310 311 312 313
	}
}

void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
				unsigned char *log_buff)
{
	struct OpalIoPhbErrorCommon *common;

	if (!hose || !log_buff)
		return;

	common = (struct OpalIoPhbErrorCommon *)log_buff;
314
	switch (be32_to_cpu(common->ioType)) {
315 316 317 318 319
	case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
		pnv_pci_dump_p7ioc_diag_data(hose, common);
		break;
	case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
		pnv_pci_dump_phb3_diag_data(hose, common);
320 321
		break;
	default:
322
		pr_warn("%s: Unrecognized ioType %d\n",
323
			__func__, be32_to_cpu(common->ioType));
324 325 326 327 328 329
	}
}

static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
{
	unsigned long flags, rc;
330
	int has_diag, ret = 0;
331 332 333

	spin_lock_irqsave(&phb->lock, flags);

334
	/* Fetch PHB diag-data */
335 336
	rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
					 PNV_PCI_DIAG_BUF_SIZE);
337 338
	has_diag = (rc == OPAL_SUCCESS);

339 340 341 342
	/* If PHB supports compound PE, to handle it */
	if (phb->unfreeze_pe) {
		ret = phb->unfreeze_pe(phb,
				       pe_no,
343
				       OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
344 345 346 347 348 349 350 351 352 353 354
	} else {
		rc = opal_pci_eeh_freeze_clear(phb->opal_id,
					     pe_no,
					     OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
		if (rc) {
			pr_warn("%s: Failure %ld clearing frozen "
				"PHB#%x-PE#%x\n",
				__func__, rc, phb->hose->global_number,
				pe_no);
			ret = -EIO;
		}
355 356
	}

357 358 359 360 361 362 363 364 365
	/*
	 * For now, let's only display the diag buffer when we fail to clear
	 * the EEH status. We'll do more sensible things later when we have
	 * proper EEH support. We need to make sure we don't pollute ourselves
	 * with the normal errors generated when probing empty slots
	 */
	if (has_diag && ret)
		pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);

366 367 368
	spin_unlock_irqrestore(&phb->lock, flags);
}

369
static void pnv_pci_config_check_eeh(struct pci_dn *pdn)
370
{
371
	struct pnv_phb *phb = pdn->phb->private_data;
372
	u8	fstate;
373
	__be16	pcierr;
374 375
	int	pe_no;
	s64	rc;
376

377 378 379
	/*
	 * Get the PE#. During the PCI probe stage, we might not
	 * setup that yet. So all ER errors should be mapped to
380
	 * reserved PE.
381
	 */
382
	pe_no = pdn->pe_number;
383 384 385 386 387 388
	if (pe_no == IODA_INVALID_PE) {
		if (phb->type == PNV_PHB_P5IOC2)
			pe_no = 0;
		else
			pe_no = phb->ioda.reserved_pe;
	}
389

390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
	/*
	 * Fetch frozen state. If the PHB support compound PE,
	 * we need handle that case.
	 */
	if (phb->get_pe_state) {
		fstate = phb->get_pe_state(phb, pe_no);
	} else {
		rc = opal_pci_eeh_freeze_status(phb->opal_id,
						pe_no,
						&fstate,
						&pcierr,
						NULL);
		if (rc) {
			pr_warn("%s: Failure %lld getting PHB#%x-PE#%x state\n",
				__func__, rc, phb->hose->global_number, pe_no);
			return;
		}
407
	}
408

409
	cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
410
		(pdn->busno << 8) | (pdn->devfn), pe_no, fstate);
411 412 413 414 415 416 417 418 419 420 421 422

	/* Clear the frozen state if applicable */
	if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE ||
	    fstate == OPAL_EEH_STOPPED_DMA_FREEZE  ||
	    fstate == OPAL_EEH_STOPPED_MMIO_DMA_FREEZE) {
		/*
		 * If PHB supports compound PE, freeze it for
		 * consistency.
		 */
		if (phb->freeze_pe)
			phb->freeze_pe(phb, pe_no);

423
		pnv_pci_handle_eeh_config(phb, pe_no);
424
	}
425 426
}

427
int pnv_pci_cfg_read(struct pci_dn *pdn,
428
		     int where, int size, u32 *val)
429
{
430 431
	struct pnv_phb *phb = pdn->phb->private_data;
	u32 bdfn = (pdn->busno << 8) | pdn->devfn;
432 433 434 435 436 437 438 439 440 441
	s64 rc;

	switch (size) {
	case 1: {
		u8 v8;
		rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
		*val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
		break;
	}
	case 2: {
442
		__be16 v16;
443 444
		rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
						   &v16);
445
		*val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
446 447 448
		break;
	}
	case 4: {
449
		__be32 v32;
450
		rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
451
		*val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
452 453 454 455 456
		break;
	}
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
457

458 459
	cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
		__func__, pdn->busno, pdn->devfn, where, size, *val);
460 461 462
	return PCIBIOS_SUCCESSFUL;
}

463
int pnv_pci_cfg_write(struct pci_dn *pdn,
464
		      int where, int size, u32 val)
465
{
466 467
	struct pnv_phb *phb = pdn->phb->private_data;
	u32 bdfn = (pdn->busno << 8) | pdn->devfn;
468

469 470
	cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
		pdn->busno, pdn->devfn, where, size, val);
471 472 473 474 475 476 477 478 479 480 481 482 483
	switch (size) {
	case 1:
		opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
		break;
	case 2:
		opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
		break;
	case 4:
		opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
		break;
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
484

485 486 487 488
	return PCIBIOS_SUCCESSFUL;
}

#if CONFIG_EEH
489
static bool pnv_pci_cfg_check(struct pci_dn *pdn)
490 491
{
	struct eeh_dev *edev = NULL;
492
	struct pnv_phb *phb = pdn->phb->private_data;
493 494

	/* EEH not enabled ? */
495
	if (!(phb->flags & PNV_PHB_FLAG_EEH))
496
		return true;
497

498
	/* PE reset or device removed ? */
499
	edev = pdn->edev;
500 501
	if (edev) {
		if (edev->pe &&
502
		    (edev->pe->state & EEH_PE_CFG_BLOCKED))
503 504 505 506 507
			return false;

		if (edev->mode & EEH_DEV_REMOVED)
			return false;
	}
508 509 510 511

	return true;
}
#else
512
static inline pnv_pci_cfg_check(struct pci_dn *pdn)
513 514
{
	return true;
515
}
516
#endif /* CONFIG_EEH */
517

518 519 520 521 522
static int pnv_pci_read_config(struct pci_bus *bus,
			       unsigned int devfn,
			       int where, int size, u32 *val)
{
	struct pci_dn *pdn;
523 524
	struct pnv_phb *phb;
	int ret;
525

526
	*val = 0xFFFFFFFF;
527 528 529
	pdn = pci_get_pdn_by_devfn(bus, devfn);
	if (!pdn)
		return PCIBIOS_DEVICE_NOT_FOUND;
530

531
	if (!pnv_pci_cfg_check(pdn))
532 533
		return PCIBIOS_DEVICE_NOT_FOUND;

534 535 536
	ret = pnv_pci_cfg_read(pdn, where, size, val);
	phb = pdn->phb->private_data;
	if (phb->flags & PNV_PHB_FLAG_EEH && pdn->edev) {
537
		if (*val == EEH_IO_ERROR_VALUE(size) &&
538
		    eeh_dev_check_failure(pdn->edev))
539 540
                        return PCIBIOS_DEVICE_NOT_FOUND;
	} else {
541
		pnv_pci_config_check_eeh(pdn);
542
	}
543

544
	return ret;
545 546 547 548 549 550 551
}

static int pnv_pci_write_config(struct pci_bus *bus,
				unsigned int devfn,
				int where, int size, u32 val)
{
	struct pci_dn *pdn;
552 553
	struct pnv_phb *phb;
	int ret;
554

555 556 557
	pdn = pci_get_pdn_by_devfn(bus, devfn);
	if (!pdn)
		return PCIBIOS_DEVICE_NOT_FOUND;
558

559
	if (!pnv_pci_cfg_check(pdn))
560 561
		return PCIBIOS_DEVICE_NOT_FOUND;

562 563
	ret = pnv_pci_cfg_write(pdn, where, size, val);
	phb = pdn->phb->private_data;
564
	if (!(phb->flags & PNV_PHB_FLAG_EEH))
565
		pnv_pci_config_check_eeh(pdn);
566 567

	return ret;
568 569
}

570
struct pci_ops pnv_pci_ops = {
571
	.read  = pnv_pci_read_config,
572 573 574
	.write = pnv_pci_write_config,
};

575 576 577 578 579 580 581
static __be64 *pnv_tce(struct iommu_table *tbl, long idx)
{
	__be64 *tmp = ((__be64 *)tbl->it_base);

	return tmp + idx;
}

582 583 584
int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
		unsigned long uaddr, enum dma_data_direction direction,
		struct dma_attrs *attrs)
585
{
586
	u64 proto_tce = iommu_direction_to_tce_perm(direction);
587 588
	u64 rpn = __pa(uaddr) >> tbl->it_page_shift;
	long i;
589

590 591 592 593
	for (i = 0; i < npages; i++) {
		unsigned long newtce = proto_tce |
			((rpn + i) << tbl->it_page_shift);
		unsigned long idx = index - tbl->it_offset + i;
594

595 596
		*(pnv_tce(tbl, idx)) = cpu_to_be64(newtce);
	}
597 598 599 600

	return 0;
}

601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
#ifdef CONFIG_IOMMU_API
int pnv_tce_xchg(struct iommu_table *tbl, long index,
		unsigned long *hpa, enum dma_data_direction *direction)
{
	u64 proto_tce = iommu_direction_to_tce_perm(*direction);
	unsigned long newtce = *hpa | proto_tce, oldtce;
	unsigned long idx = index - tbl->it_offset;

	BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl));

	oldtce = xchg(pnv_tce(tbl, idx), cpu_to_be64(newtce));
	*hpa = be64_to_cpu(oldtce) & ~(TCE_PCI_READ | TCE_PCI_WRITE);
	*direction = iommu_tce_direction(oldtce);

	return 0;
}
#endif

619
void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
620
{
621
	long i;
622

623 624
	for (i = 0; i < npages; i++) {
		unsigned long idx = index - tbl->it_offset + i;
625

626 627
		*(pnv_tce(tbl, idx)) = cpu_to_be64(0);
	}
628 629
}

630
unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
631
{
632
	return *(pnv_tce(tbl, index - tbl->it_offset));
633 634
}

635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
struct iommu_table *pnv_pci_table_alloc(int nid)
{
	struct iommu_table *tbl;

	tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, nid);
	INIT_LIST_HEAD_RCU(&tbl->it_group_list);

	return tbl;
}

long pnv_pci_link_table_and_group(int node, int num,
		struct iommu_table *tbl,
		struct iommu_table_group *table_group)
{
	struct iommu_table_group_link *tgl = NULL;

	if (WARN_ON(!tbl || !table_group))
		return -EINVAL;

	tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL,
			node);
	if (!tgl)
		return -ENOMEM;

	tgl->table_group = table_group;
	list_add_rcu(&tgl->next, &tbl->it_group_list);

	table_group->tables[num] = tbl;

	return 0;
}

static void pnv_iommu_table_group_link_free(struct rcu_head *head)
{
	struct iommu_table_group_link *tgl = container_of(head,
			struct iommu_table_group_link, rcu);

	kfree(tgl);
}

void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,
		struct iommu_table_group *table_group)
{
	long i;
	bool found;
	struct iommu_table_group_link *tgl;

	if (!tbl || !table_group)
		return;

	/* Remove link to a group from table's list of attached groups */
	found = false;
	list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
		if (tgl->table_group == table_group) {
			list_del_rcu(&tgl->next);
			call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free);
			found = true;
			break;
		}
	}
	if (WARN_ON(!found))
		return;

	/* Clean a pointer to iommu_table in iommu_table_group::tables[] */
	found = false;
	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
		if (table_group->tables[i] == tbl) {
			table_group->tables[i] = NULL;
			found = true;
			break;
		}
	}
	WARN_ON(!found);
}

710 711
void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
			       void *tce_mem, u64 tce_size,
712
			       u64 dma_offset, unsigned page_shift)
713 714 715
{
	tbl->it_blocksize = 16;
	tbl->it_base = (unsigned long)tce_mem;
716
	tbl->it_page_shift = page_shift;
717
	tbl->it_offset = dma_offset >> tbl->it_page_shift;
718 719 720 721 722 723
	tbl->it_index = 0;
	tbl->it_size = tce_size >> 3;
	tbl->it_busno = 0;
	tbl->it_type = TCE_PCI;
}

724
void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
725 726 727
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745
#ifdef CONFIG_PCI_IOV
	struct pnv_ioda_pe *pe;
	struct pci_dn *pdn;

	/* Fix the VF pdn PE number */
	if (pdev->is_virtfn) {
		pdn = pci_get_pdn(pdev);
		WARN_ON(pdn->pe_number != IODA_INVALID_PE);
		list_for_each_entry(pe, &phb->ioda.pe_list, list) {
			if (pe->rid == ((pdev->bus->number << 8) |
			    (pdev->devfn & 0xff))) {
				pdn->pe_number = pe->pe_number;
				pe->pdev = pdev;
				break;
			}
		}
	}
#endif /* CONFIG_PCI_IOV */
746 747 748 749 750

	if (phb && phb->dma_dev_setup)
		phb->dma_dev_setup(phb, pdev);
}

751 752 753 754 755 756 757 758 759 760 761
u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;

	if (phb && phb->dma_get_required_mask)
		return phb->dma_get_required_mask(phb, pdev);

	return __dma_get_required_mask(&pdev->dev);
}

762 763 764 765
void pnv_pci_shutdown(void)
{
	struct pci_controller *hose;

766 767 768
	list_for_each_entry(hose, &hose_list, list_node)
		if (hose->controller_ops.shutdown)
			hose->controller_ops.shutdown(hose);
769 770
}

G
Gavin Shan 已提交
771
/* Fixup wrong class code in p7ioc and p8 root complex */
772
static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
773 774 775 776 777
{
	dev->class = PCI_CLASS_BRIDGE_PCI << 8;
}
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);

778 779 780
void __init pnv_pci_init(void)
{
	struct device_node *np;
781
	bool found_ioda = false;
782

783
	pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
784

785 786 787
	/* If we don't have OPAL, eg. in sim, just skip PCI probe */
	if (!firmware_has_feature(FW_FEATURE_OPAL))
		return;
788

789 790 791 792 793 794 795 796
	/* Look for IODA IO-Hubs. We don't support mixing IODA
	 * and p5ioc2 due to the need to change some global
	 * probing flags
	 */
	for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
		pnv_pci_init_ioda_hub(np);
		found_ioda = true;
	}
797

798 799 800 801
	/* Look for p5ioc2 IO-Hubs */
	if (!found_ioda)
		for_each_compatible_node(np, NULL, "ibm,p5ioc2")
			pnv_pci_init_p5ioc2_hub(np);
G
Gavin Shan 已提交
802

803 804 805
	/* Look for ioda2 built-in PHB3's */
	for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
		pnv_pci_init_ioda2_phb(np);
806 807 808 809 810 811 812

	/* Setup the linkage between OF nodes and PHBs */
	pci_devs_phb_init();

	/* Configure IOMMU DMA hooks */
	set_pci_dma_ops(&dma_iommu_ops);
}
813

814
machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init);