pci.c 24.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * Support PCI/PCIe on PowerNV platforms
 *
 * Currently supports only P5IOC2
 *
 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/irq.h>
#include <linux/io.h>
22
#include <linux/msi.h>
23
#include <linux/iommu.h>
24 25 26 27 28 29

#include <asm/sections.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/pci-bridge.h>
#include <asm/machdep.h>
30
#include <asm/msi_bitmap.h>
31 32 33 34
#include <asm/ppc-pci.h>
#include <asm/opal.h>
#include <asm/iommu.h>
#include <asm/tce.h>
35
#include <asm/firmware.h>
36 37
#include <asm/eeh_event.h>
#include <asm/eeh.h>
38 39 40 41

#include "powernv.h"
#include "pci.h"

42 43
/* Delay in usec */
#define PCI_RESET_DELAY_US	3000000
44 45 46 47

#define cfg_dbg(fmt...)	do { } while(0)
//#define cfg_dbg(fmt...)	printk(fmt)

48 49 50 51 52
#ifdef CONFIG_PCI_MSI
static int pnv_msi_check_device(struct pci_dev* pdev, int nvec, int type)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
53 54 55 56
	struct pci_dn *pdn = pci_get_pdn(pdev);

	if (pdn && pdn->force_32bit_msi && !phb->msi32_support)
		return -ENODEV;
57

58
	return (phb && phb->msi_bmp.bitmap) ? 0 : -ENODEV;
59 60 61 62 63 64 65 66
}

static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;
	struct msi_msg msg;
67 68
	int hwirq;
	unsigned int virq;
69 70 71 72 73 74 75 76 77 78 79
	int rc;

	if (WARN_ON(!phb))
		return -ENODEV;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
			pr_warn("%s: Supports only 64-bit MSIs\n",
				pci_name(pdev));
			return -ENXIO;
		}
80 81
		hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
		if (hwirq < 0) {
82 83 84 85
			pr_warn("%s: Failed to find a free MSI\n",
				pci_name(pdev));
			return -ENOSPC;
		}
86
		virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
87 88 89
		if (virq == NO_IRQ) {
			pr_warn("%s: Failed to map MSI to linux irq\n",
				pci_name(pdev));
90
			msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
91 92
			return -ENOMEM;
		}
93
		rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
94
				    virq, entry->msi_attrib.is_64, &msg);
95 96 97
		if (rc) {
			pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
			irq_dispose_mapping(virq);
98
			msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
			return rc;
		}
		irq_set_msi_desc(virq, entry);
		write_msi_msg(virq, &msg);
	}
	return 0;
}

static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;

	if (WARN_ON(!phb))
		return;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (entry->irq == NO_IRQ)
			continue;
		irq_set_msi_desc(entry->irq, NULL);
120 121
		msi_bitmap_free_hwirqs(&phb->msi_bmp,
			virq_to_hw(entry->irq) - phb->msi_base, 1);
122 123 124 125
		irq_dispose_mapping(entry->irq);
	}
}
#endif /* CONFIG_PCI_MSI */
126

127 128
static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
					 struct OpalIoPhbErrorCommon *common)
129
{
130
	struct OpalIoP7IOCPhbErrorData *data;
131 132
	int i;

133
	data = (struct OpalIoP7IOCPhbErrorData *)common;
134
	pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
135
		hose->global_number, be32_to_cpu(common->version));
136

137
	if (data->brdgCtl)
138
		pr_info("brdgCtl:     %08x\n",
139
			be32_to_cpu(data->brdgCtl));
140 141
	if (data->portStatusReg || data->rootCmplxStatus ||
	    data->busAgentStatus)
142
		pr_info("UtlSts:      %08x %08x %08x\n",
143 144 145
			be32_to_cpu(data->portStatusReg),
			be32_to_cpu(data->rootCmplxStatus),
			be32_to_cpu(data->busAgentStatus));
146 147 148
	if (data->deviceStatus || data->slotStatus   ||
	    data->linkStatus   || data->devCmdStatus ||
	    data->devSecStatus)
149
		pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
150 151 152 153 154
			be32_to_cpu(data->deviceStatus),
			be32_to_cpu(data->slotStatus),
			be32_to_cpu(data->linkStatus),
			be32_to_cpu(data->devCmdStatus),
			be32_to_cpu(data->devSecStatus));
155 156
	if (data->rootErrorStatus   || data->uncorrErrorStatus ||
	    data->corrErrorStatus)
157
		pr_info("RootErrSts:  %08x %08x %08x\n",
158 159 160
			be32_to_cpu(data->rootErrorStatus),
			be32_to_cpu(data->uncorrErrorStatus),
			be32_to_cpu(data->corrErrorStatus));
161 162
	if (data->tlpHdr1 || data->tlpHdr2 ||
	    data->tlpHdr3 || data->tlpHdr4)
163
		pr_info("RootErrLog:  %08x %08x %08x %08x\n",
164 165 166 167
			be32_to_cpu(data->tlpHdr1),
			be32_to_cpu(data->tlpHdr2),
			be32_to_cpu(data->tlpHdr3),
			be32_to_cpu(data->tlpHdr4));
168 169
	if (data->sourceId || data->errorClass ||
	    data->correlator)
170
		pr_info("RootErrLog1: %08x %016llx %016llx\n",
171 172 173
			be32_to_cpu(data->sourceId),
			be64_to_cpu(data->errorClass),
			be64_to_cpu(data->correlator));
174
	if (data->p7iocPlssr || data->p7iocCsr)
175
		pr_info("PhbSts:      %016llx %016llx\n",
176 177
			be64_to_cpu(data->p7iocPlssr),
			be64_to_cpu(data->p7iocCsr));
178 179
	if (data->lemFir)
		pr_info("Lem:         %016llx %016llx %016llx\n",
180 181 182
			be64_to_cpu(data->lemFir),
			be64_to_cpu(data->lemErrorMask),
			be64_to_cpu(data->lemWOF));
183 184
	if (data->phbErrorStatus)
		pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
185 186 187 188
			be64_to_cpu(data->phbErrorStatus),
			be64_to_cpu(data->phbFirstErrorStatus),
			be64_to_cpu(data->phbErrorLog0),
			be64_to_cpu(data->phbErrorLog1));
189 190
	if (data->mmioErrorStatus)
		pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
191 192 193 194
			be64_to_cpu(data->mmioErrorStatus),
			be64_to_cpu(data->mmioFirstErrorStatus),
			be64_to_cpu(data->mmioErrorLog0),
			be64_to_cpu(data->mmioErrorLog1));
195 196
	if (data->dma0ErrorStatus)
		pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
197 198 199 200
			be64_to_cpu(data->dma0ErrorStatus),
			be64_to_cpu(data->dma0FirstErrorStatus),
			be64_to_cpu(data->dma0ErrorLog0),
			be64_to_cpu(data->dma0ErrorLog1));
201 202
	if (data->dma1ErrorStatus)
		pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
203 204 205 206
			be64_to_cpu(data->dma1ErrorStatus),
			be64_to_cpu(data->dma1FirstErrorStatus),
			be64_to_cpu(data->dma1ErrorLog0),
			be64_to_cpu(data->dma1ErrorLog1));
207 208 209 210 211

	for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
		if ((data->pestA[i] >> 63) == 0 &&
		    (data->pestB[i] >> 63) == 0)
			continue;
212

213
		pr_info("PE[%3d] A/B: %016llx %016llx\n",
214 215
			i, be64_to_cpu(data->pestA[i]),
			be64_to_cpu(data->pestB[i]));
216 217 218
	}
}

219 220
static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
					struct OpalIoPhbErrorCommon *common)
221
{
222 223 224 225
	struct OpalIoPhb3ErrorData *data;
	int i;

	data = (struct OpalIoPhb3ErrorData*)common;
226
	pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
227
		hose->global_number, be32_to_cpu(common->version));
228
	if (data->brdgCtl)
229
		pr_info("brdgCtl:     %08x\n",
230
			be32_to_cpu(data->brdgCtl));
231 232
	if (data->portStatusReg || data->rootCmplxStatus ||
	    data->busAgentStatus)
233
		pr_info("UtlSts:      %08x %08x %08x\n",
234 235 236
			be32_to_cpu(data->portStatusReg),
			be32_to_cpu(data->rootCmplxStatus),
			be32_to_cpu(data->busAgentStatus));
237 238 239
	if (data->deviceStatus || data->slotStatus   ||
	    data->linkStatus   || data->devCmdStatus ||
	    data->devSecStatus)
240
		pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
241 242 243 244 245
			be32_to_cpu(data->deviceStatus),
			be32_to_cpu(data->slotStatus),
			be32_to_cpu(data->linkStatus),
			be32_to_cpu(data->devCmdStatus),
			be32_to_cpu(data->devSecStatus));
246 247
	if (data->rootErrorStatus || data->uncorrErrorStatus ||
	    data->corrErrorStatus)
248
		pr_info("RootErrSts:  %08x %08x %08x\n",
249 250 251
			be32_to_cpu(data->rootErrorStatus),
			be32_to_cpu(data->uncorrErrorStatus),
			be32_to_cpu(data->corrErrorStatus));
252 253
	if (data->tlpHdr1 || data->tlpHdr2 ||
	    data->tlpHdr3 || data->tlpHdr4)
254
		pr_info("RootErrLog:  %08x %08x %08x %08x\n",
255 256 257 258
			be32_to_cpu(data->tlpHdr1),
			be32_to_cpu(data->tlpHdr2),
			be32_to_cpu(data->tlpHdr3),
			be32_to_cpu(data->tlpHdr4));
259 260
	if (data->sourceId || data->errorClass ||
	    data->correlator)
261
		pr_info("RootErrLog1: %08x %016llx %016llx\n",
262 263 264
			be32_to_cpu(data->sourceId),
			be64_to_cpu(data->errorClass),
			be64_to_cpu(data->correlator));
265 266
	if (data->nFir)
		pr_info("nFir:        %016llx %016llx %016llx\n",
267 268 269
			be64_to_cpu(data->nFir),
			be64_to_cpu(data->nFirMask),
			be64_to_cpu(data->nFirWOF));
270
	if (data->phbPlssr || data->phbCsr)
271
		pr_info("PhbSts:      %016llx %016llx\n",
272 273
			be64_to_cpu(data->phbPlssr),
			be64_to_cpu(data->phbCsr));
274 275
	if (data->lemFir)
		pr_info("Lem:         %016llx %016llx %016llx\n",
276 277 278
			be64_to_cpu(data->lemFir),
			be64_to_cpu(data->lemErrorMask),
			be64_to_cpu(data->lemWOF));
279 280
	if (data->phbErrorStatus)
		pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
281 282 283 284
			be64_to_cpu(data->phbErrorStatus),
			be64_to_cpu(data->phbFirstErrorStatus),
			be64_to_cpu(data->phbErrorLog0),
			be64_to_cpu(data->phbErrorLog1));
285 286
	if (data->mmioErrorStatus)
		pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
287 288 289 290
			be64_to_cpu(data->mmioErrorStatus),
			be64_to_cpu(data->mmioFirstErrorStatus),
			be64_to_cpu(data->mmioErrorLog0),
			be64_to_cpu(data->mmioErrorLog1));
291 292
	if (data->dma0ErrorStatus)
		pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
293 294 295 296
			be64_to_cpu(data->dma0ErrorStatus),
			be64_to_cpu(data->dma0FirstErrorStatus),
			be64_to_cpu(data->dma0ErrorLog0),
			be64_to_cpu(data->dma0ErrorLog1));
297 298
	if (data->dma1ErrorStatus)
		pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
299 300 301 302
			be64_to_cpu(data->dma1ErrorStatus),
			be64_to_cpu(data->dma1FirstErrorStatus),
			be64_to_cpu(data->dma1ErrorLog0),
			be64_to_cpu(data->dma1ErrorLog1));
303 304

	for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
305 306
		if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
		    (be64_to_cpu(data->pestB[i]) >> 63) == 0)
307 308
			continue;

309
		pr_info("PE[%3d] A/B: %016llx %016llx\n",
310 311
				i, be64_to_cpu(data->pestA[i]),
				be64_to_cpu(data->pestB[i]));
312 313 314 315 316 317 318 319 320 321 322 323
	}
}

void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
				unsigned char *log_buff)
{
	struct OpalIoPhbErrorCommon *common;

	if (!hose || !log_buff)
		return;

	common = (struct OpalIoPhbErrorCommon *)log_buff;
324
	switch (be32_to_cpu(common->ioType)) {
325 326 327 328 329
	case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
		pnv_pci_dump_p7ioc_diag_data(hose, common);
		break;
	case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
		pnv_pci_dump_phb3_diag_data(hose, common);
330 331
		break;
	default:
332
		pr_warn("%s: Unrecognized ioType %d\n",
333
			__func__, be32_to_cpu(common->ioType));
334 335 336 337 338 339
	}
}

static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
{
	unsigned long flags, rc;
340
	int has_diag, ret = 0;
341 342 343

	spin_lock_irqsave(&phb->lock, flags);

344
	/* Fetch PHB diag-data */
345 346
	rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
					 PNV_PCI_DIAG_BUF_SIZE);
347 348
	has_diag = (rc == OPAL_SUCCESS);

349 350 351 352
	/* If PHB supports compound PE, to handle it */
	if (phb->unfreeze_pe) {
		ret = phb->unfreeze_pe(phb,
				       pe_no,
353
				       OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
354 355 356 357 358 359 360 361 362 363 364
	} else {
		rc = opal_pci_eeh_freeze_clear(phb->opal_id,
					     pe_no,
					     OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
		if (rc) {
			pr_warn("%s: Failure %ld clearing frozen "
				"PHB#%x-PE#%x\n",
				__func__, rc, phb->hose->global_number,
				pe_no);
			ret = -EIO;
		}
365 366
	}

367 368 369 370 371 372 373 374 375
	/*
	 * For now, let's only display the diag buffer when we fail to clear
	 * the EEH status. We'll do more sensible things later when we have
	 * proper EEH support. We need to make sure we don't pollute ourselves
	 * with the normal errors generated when probing empty slots
	 */
	if (has_diag && ret)
		pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);

376 377 378
	spin_unlock_irqrestore(&phb->lock, flags);
}

379 380
static void pnv_pci_config_check_eeh(struct pnv_phb *phb,
				     struct device_node *dn)
381 382
{
	u8	fstate;
383
	__be16	pcierr;
384 385
	int	pe_no;
	s64	rc;
386

387 388 389
	/*
	 * Get the PE#. During the PCI probe stage, we might not
	 * setup that yet. So all ER errors should be mapped to
390
	 * reserved PE.
391 392
	 */
	pe_no = PCI_DN(dn)->pe_number;
393 394 395 396 397 398
	if (pe_no == IODA_INVALID_PE) {
		if (phb->type == PNV_PHB_P5IOC2)
			pe_no = 0;
		else
			pe_no = phb->ioda.reserved_pe;
	}
399

400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416
	/*
	 * Fetch frozen state. If the PHB support compound PE,
	 * we need handle that case.
	 */
	if (phb->get_pe_state) {
		fstate = phb->get_pe_state(phb, pe_no);
	} else {
		rc = opal_pci_eeh_freeze_status(phb->opal_id,
						pe_no,
						&fstate,
						&pcierr,
						NULL);
		if (rc) {
			pr_warn("%s: Failure %lld getting PHB#%x-PE#%x state\n",
				__func__, rc, phb->hose->global_number, pe_no);
			return;
		}
417
	}
418

419 420 421
	cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
		(PCI_DN(dn)->busno << 8) | (PCI_DN(dn)->devfn),
		pe_no, fstate);
422 423 424 425 426 427 428 429 430 431 432 433

	/* Clear the frozen state if applicable */
	if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE ||
	    fstate == OPAL_EEH_STOPPED_DMA_FREEZE  ||
	    fstate == OPAL_EEH_STOPPED_MMIO_DMA_FREEZE) {
		/*
		 * If PHB supports compound PE, freeze it for
		 * consistency.
		 */
		if (phb->freeze_pe)
			phb->freeze_pe(phb, pe_no);

434
		pnv_pci_handle_eeh_config(phb, pe_no);
435
	}
436 437
}

438 439
int pnv_pci_cfg_read(struct device_node *dn,
		     int where, int size, u32 *val)
440
{
441 442 443
	struct pci_dn *pdn = PCI_DN(dn);
	struct pnv_phb *phb = pdn->phb->private_data;
	u32 bdfn = (pdn->busno << 8) | pdn->devfn;
444 445 446 447 448 449 450 451 452 453
	s64 rc;

	switch (size) {
	case 1: {
		u8 v8;
		rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
		*val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
		break;
	}
	case 2: {
454
		__be16 v16;
455 456
		rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
						   &v16);
457
		*val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
458 459 460
		break;
	}
	case 4: {
461
		__be32 v32;
462
		rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
463
		*val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
464 465 466 467 468
		break;
	}
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
469

470 471
	cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
		__func__, pdn->busno, pdn->devfn, where, size, *val);
472 473 474
	return PCIBIOS_SUCCESSFUL;
}

475 476
int pnv_pci_cfg_write(struct device_node *dn,
		      int where, int size, u32 val)
477
{
478 479 480
	struct pci_dn *pdn = PCI_DN(dn);
	struct pnv_phb *phb = pdn->phb->private_data;
	u32 bdfn = (pdn->busno << 8) | pdn->devfn;
481

482 483
	cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
		pdn->busno, pdn->devfn, where, size, val);
484 485 486 487 488 489 490 491 492 493 494 495 496
	switch (size) {
	case 1:
		opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
		break;
	case 2:
		opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
		break;
	case 4:
		opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
		break;
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
497

498 499 500 501 502 503 504 505 506 507 508
	return PCIBIOS_SUCCESSFUL;
}

#if CONFIG_EEH
static bool pnv_pci_cfg_check(struct pci_controller *hose,
			      struct device_node *dn)
{
	struct eeh_dev *edev = NULL;
	struct pnv_phb *phb = hose->private_data;

	/* EEH not enabled ? */
509
	if (!(phb->flags & PNV_PHB_FLAG_EEH))
510
		return true;
511

512
	/* PE reset or device removed ? */
513
	edev = of_node_to_eeh_dev(dn);
514 515 516 517 518 519 520 521
	if (edev) {
		if (edev->pe &&
		    (edev->pe->state & EEH_PE_RESET))
			return false;

		if (edev->mode & EEH_DEV_REMOVED)
			return false;
	}
522 523 524 525 526 527 528 529

	return true;
}
#else
static inline pnv_pci_cfg_check(struct pci_controller *hose,
				struct device_node *dn)
{
	return true;
530
}
531
#endif /* CONFIG_EEH */
532

533 534 535 536 537 538
static int pnv_pci_read_config(struct pci_bus *bus,
			       unsigned int devfn,
			       int where, int size, u32 *val)
{
	struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
	struct pci_dn *pdn;
539 540 541
	struct pnv_phb *phb;
	bool found = false;
	int ret;
542

543
	*val = 0xFFFFFFFF;
544 545
	for (dn = busdn->child; dn; dn = dn->sibling) {
		pdn = PCI_DN(dn);
546 547 548 549 550
		if (pdn && pdn->devfn == devfn) {
			phb = pdn->phb->private_data;
			found = true;
			break;
		}
551 552
	}

553 554 555 556 557 558 559 560 561 562 563
	if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
		return PCIBIOS_DEVICE_NOT_FOUND;

	ret = pnv_pci_cfg_read(dn, where, size, val);
	if (phb->flags & PNV_PHB_FLAG_EEH) {
		if (*val == EEH_IO_ERROR_VALUE(size) &&
		    eeh_dev_check_failure(of_node_to_eeh_dev(dn)))
                        return PCIBIOS_DEVICE_NOT_FOUND;
	} else {
		pnv_pci_config_check_eeh(phb, dn);
	}
564

565
	return ret;
566 567 568 569 570 571 572 573
}

static int pnv_pci_write_config(struct pci_bus *bus,
				unsigned int devfn,
				int where, int size, u32 val)
{
	struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
	struct pci_dn *pdn;
574 575 576
	struct pnv_phb *phb;
	bool found = false;
	int ret;
577 578 579

	for (dn = busdn->child; dn; dn = dn->sibling) {
		pdn = PCI_DN(dn);
580 581 582 583 584
		if (pdn && pdn->devfn == devfn) {
			phb = pdn->phb->private_data;
			found = true;
			break;
		}
585 586
	}

587 588 589 590 591 592 593 594
	if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
		return PCIBIOS_DEVICE_NOT_FOUND;

	ret = pnv_pci_cfg_write(dn, where, size, val);
	if (!(phb->flags & PNV_PHB_FLAG_EEH))
		pnv_pci_config_check_eeh(phb, dn);

	return ret;
595 596
}

597
struct pci_ops pnv_pci_ops = {
598
	.read  = pnv_pci_read_config,
599 600 601 602 603
	.write = pnv_pci_write_config,
};

static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
			 unsigned long uaddr, enum dma_data_direction direction,
604
			 struct dma_attrs *attrs, bool rm)
605 606
{
	u64 proto_tce;
607
	__be64 *tcep, *tces;
608 609 610 611 612 613 614
	u64 rpn;

	proto_tce = TCE_PCI_READ; // Read allowed

	if (direction != DMA_TO_DEVICE)
		proto_tce |= TCE_PCI_WRITE;

615
	tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
616
	rpn = __pa(uaddr) >> tbl->it_page_shift;
617

618
	while (npages--)
619 620
		*(tcep++) = cpu_to_be64(proto_tce |
				(rpn++ << tbl->it_page_shift));
621 622 623 624 625 626

	/* Some implementations won't cache invalid TCEs and thus may not
	 * need that flush. We'll probably turn it_type into a bit mask
	 * of flags if that becomes the case
	 */
	if (tbl->it_type & TCE_PCI_SWINV_CREATE)
627
		pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
628 629 630 631

	return 0;
}

632 633 634 635 636 637 638 639 640 641 642
static int pnv_tce_build_vm(struct iommu_table *tbl, long index, long npages,
			    unsigned long uaddr,
			    enum dma_data_direction direction,
			    struct dma_attrs *attrs)
{
	return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs,
			false);
}

static void pnv_tce_free(struct iommu_table *tbl, long index, long npages,
		bool rm)
643
{
644
	__be64 *tcep, *tces;
645

646
	tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
647 648

	while (npages--)
649
		*(tcep++) = cpu_to_be64(0);
650

651
	if (tbl->it_type & TCE_PCI_SWINV_FREE)
652 653 654 655 656 657
		pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
}

static void pnv_tce_free_vm(struct iommu_table *tbl, long index, long npages)
{
	pnv_tce_free(tbl, index, npages, false);
658 659
}

660 661 662 663 664
static unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
{
	return ((u64 *)tbl->it_base)[index - tbl->it_offset];
}

665 666 667 668 669 670 671 672 673 674 675 676 677
static int pnv_tce_build_rm(struct iommu_table *tbl, long index, long npages,
			    unsigned long uaddr,
			    enum dma_data_direction direction,
			    struct dma_attrs *attrs)
{
	return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs, true);
}

static void pnv_tce_free_rm(struct iommu_table *tbl, long index, long npages)
{
	pnv_tce_free(tbl, index, npages, true);
}

678 679
void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
			       void *tce_mem, u64 tce_size,
680
			       u64 dma_offset, unsigned page_shift)
681 682 683
{
	tbl->it_blocksize = 16;
	tbl->it_base = (unsigned long)tce_mem;
684
	tbl->it_page_shift = page_shift;
685
	tbl->it_offset = dma_offset >> tbl->it_page_shift;
686 687 688 689 690 691
	tbl->it_index = 0;
	tbl->it_size = tce_size >> 3;
	tbl->it_busno = 0;
	tbl->it_type = TCE_PCI;
}

692
static struct iommu_table *pnv_pci_setup_bml_iommu(struct pci_controller *hose)
693 694
{
	struct iommu_table *tbl;
695 696
	const __be64 *basep, *swinvp;
	const __be32 *sizep;
697 698 699 700

	basep = of_get_property(hose->dn, "linux,tce-base", NULL);
	sizep = of_get_property(hose->dn, "linux,tce-size", NULL);
	if (basep == NULL || sizep == NULL) {
701 702
		pr_err("PCI: %s has missing tce entries !\n",
		       hose->dn->full_name);
703 704 705 706 707 708
		return NULL;
	}
	tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node);
	if (WARN_ON(!tbl))
		return NULL;
	pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),
709
				  be32_to_cpup(sizep), 0, IOMMU_PAGE_SHIFT_4K);
710
	iommu_init_table(tbl, hose->node);
711
	iommu_register_group(tbl, pci_domain_nr(hose->bus), 0);
712 713 714 715 716

	/* Deal with SW invalidated TCEs when needed (BML way) */
	swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",
				 NULL);
	if (swinvp) {
717
		tbl->it_busno = be64_to_cpu(swinvp[1]);
718
		tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8);
719 720
		tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
	}
721 722 723
	return tbl;
}

724 725
static void pnv_pci_dma_fallback_setup(struct pci_controller *hose,
				       struct pci_dev *pdev)
726 727 728 729 730 731 732 733 734 735 736
{
	struct device_node *np = pci_bus_to_OF_node(hose->bus);
	struct pci_dn *pdn;

	if (np == NULL)
		return;
	pdn = PCI_DN(np);
	if (!pdn->iommu_table)
		pdn->iommu_table = pnv_pci_setup_bml_iommu(hose);
	if (!pdn->iommu_table)
		return;
737
	set_iommu_table_base_and_group(&pdev->dev, pdn->iommu_table);
738 739
}

740
static void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
741 742 743 744 745 746 747 748 749 750 751 752 753
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;

	/* If we have no phb structure, try to setup a fallback based on
	 * the device-tree (RTAS PCI for example)
	 */
	if (phb && phb->dma_dev_setup)
		phb->dma_dev_setup(phb, pdev);
	else
		pnv_pci_dma_fallback_setup(hose, pdev);
}

754 755 756 757 758 759 760 761 762 763
int pnv_pci_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;

	if (phb && phb->dma_set_mask)
		return phb->dma_set_mask(phb, pdev, dma_mask);
	return __dma_set_mask(&pdev->dev, dma_mask);
}

764 765 766 767 768 769 770 771 772 773 774 775
void pnv_pci_shutdown(void)
{
	struct pci_controller *hose;

	list_for_each_entry(hose, &hose_list, list_node) {
		struct pnv_phb *phb = hose->private_data;

		if (phb && phb->shutdown)
			phb->shutdown(phb);
	}
}

G
Gavin Shan 已提交
776
/* Fixup wrong class code in p7ioc and p8 root complex */
777
static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
778 779 780 781 782
{
	dev->class = PCI_CLASS_BRIDGE_PCI << 8;
}
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);

783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
static int pnv_pci_probe_mode(struct pci_bus *bus)
{
	struct pci_controller *hose = pci_bus_to_host(bus);
	const __be64 *tstamp;
	u64 now, target;


	/* We hijack this as a way to ensure we have waited long
	 * enough since the reset was lifted on the PCI bus
	 */
	if (bus != hose->bus)
		return PCI_PROBE_NORMAL;
	tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL);
	if (!tstamp || !*tstamp)
		return PCI_PROBE_NORMAL;

	now = mftb() / tb_ticks_per_usec;
	target = (be64_to_cpup(tstamp) / tb_ticks_per_usec)
		+ PCI_RESET_DELAY_US;

	pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n",
		 hose->global_number, target, now);

	if (now < target)
		msleep((target - now + 999) / 1000);

	return PCI_PROBE_NORMAL;
}

812 813 814 815
void __init pnv_pci_init(void)
{
	struct device_node *np;

816
	pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
817 818 819 820 821 822 823

	/* OPAL absent, try POPAL first then RTAS detection of PHBs */
	if (!firmware_has_feature(FW_FEATURE_OPAL)) {
#ifdef CONFIG_PPC_POWERNV_RTAS
		init_pci_config_tokens();
		find_and_init_phbs();
#endif /* CONFIG_PPC_POWERNV_RTAS */
824 825 826 827 828 829 830 831 832 833 834 835 836
	}
	/* OPAL is here, do our normal stuff */
	else {
		int found_ioda = 0;

		/* Look for IODA IO-Hubs. We don't support mixing IODA
		 * and p5ioc2 due to the need to change some global
		 * probing flags
		 */
		for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
			pnv_pci_init_ioda_hub(np);
			found_ioda = 1;
		}
837 838

		/* Look for p5ioc2 IO-Hubs */
839 840 841
		if (!found_ioda)
			for_each_compatible_node(np, NULL, "ibm,p5ioc2")
				pnv_pci_init_p5ioc2_hub(np);
G
Gavin Shan 已提交
842 843 844 845

		/* Look for ioda2 built-in PHB3's */
		for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
			pnv_pci_init_ioda2_phb(np);
846 847 848 849 850 851 852
	}

	/* Setup the linkage between OF nodes and PHBs */
	pci_devs_phb_init();

	/* Configure IOMMU DMA hooks */
	ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup;
853 854 855 856
	ppc_md.tce_build = pnv_tce_build_vm;
	ppc_md.tce_free = pnv_tce_free_vm;
	ppc_md.tce_build_rm = pnv_tce_build_rm;
	ppc_md.tce_free_rm = pnv_tce_free_rm;
857
	ppc_md.tce_get = pnv_tce_get;
858
	ppc_md.pci_probe_mode = pnv_pci_probe_mode;
859 860
	set_pci_dma_ops(&dma_iommu_ops);

861 862 863 864 865 866
	/* Configure MSIs */
#ifdef CONFIG_PCI_MSI
	ppc_md.msi_check_device = pnv_msi_check_device;
	ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
	ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
#endif
867
}
868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894

static int tce_iommu_bus_notifier(struct notifier_block *nb,
		unsigned long action, void *data)
{
	struct device *dev = data;

	switch (action) {
	case BUS_NOTIFY_ADD_DEVICE:
		return iommu_add_device(dev);
	case BUS_NOTIFY_DEL_DEVICE:
		if (dev->iommu_group)
			iommu_del_device(dev);
		return 0;
	default:
		return 0;
	}
}

static struct notifier_block tce_iommu_bus_nb = {
	.notifier_call = tce_iommu_bus_notifier,
};

static int __init tce_iommu_bus_notifier_init(void)
{
	bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
	return 0;
}
895
machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init);