pci.c 20.7 KB
Newer Older
J
Jan Glauber 已提交
1 2 3 4 5 6 7 8
/*
 * Copyright IBM Corp. 2012
 *
 * Author(s):
 *   Jan Glauber <jang@linux.vnet.ibm.com>
 *
 * The System z PCI code is a rewrite from a prototype by
 * the following people (Kudoz!):
9 10 11 12 13 14 15
 *   Alexander Schmidt
 *   Christoph Raisch
 *   Hannes Hering
 *   Hoang-Nam Nguyen
 *   Jan-Bernd Themann
 *   Stefan Roscher
 *   Thomas Klein
J
Jan Glauber 已提交
16 17 18 19 20 21 22 23 24 25
 */

#define COMPONENT "zPCI"
#define pr_fmt(fmt) COMPONENT ": " fmt

#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/export.h>
#include <linux/delay.h>
26 27
#include <linux/irq.h>
#include <linux/kernel_stat.h>
J
Jan Glauber 已提交
28 29 30 31
#include <linux/seq_file.h>
#include <linux/pci.h>
#include <linux/msi.h>

32 33
#include <asm/isc.h>
#include <asm/airq.h>
J
Jan Glauber 已提交
34 35
#include <asm/facility.h>
#include <asm/pci_insn.h>
J
Jan Glauber 已提交
36
#include <asm/pci_clp.h>
J
Jan Glauber 已提交
37
#include <asm/pci_dma.h>
J
Jan Glauber 已提交
38 39 40

#define DEBUG				/* enable pr_debug */

41 42 43
#define	SIC_IRQ_MODE_ALL		0
#define	SIC_IRQ_MODE_SINGLE		1

J
Jan Glauber 已提交
44 45 46 47
#define ZPCI_NR_DMA_SPACES		1
#define ZPCI_NR_DEVICES			CONFIG_PCI_NR_FUNCTIONS

/* list of all detected zpci devices */
48
static LIST_HEAD(zpci_list);
49
static DEFINE_SPINLOCK(zpci_list_lock);
J
Jan Glauber 已提交
50

51 52
static void zpci_enable_irq(struct irq_data *data);
static void zpci_disable_irq(struct irq_data *data);
J
Jan Glauber 已提交
53

54 55 56 57
static struct irq_chip zpci_irq_chip = {
	.name = "zPCI",
	.irq_unmask = zpci_enable_irq,
	.irq_mask = zpci_disable_irq,
58 59
};

60 61
static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES);
static DEFINE_SPINLOCK(zpci_domain_lock);
62

63
static struct airq_iv *zpci_aisb_iv;
64
static struct airq_iv *zpci_aibv[ZPCI_NR_DEVICES];
65

66 67 68 69 70 71 72
/* Adapter interrupt definitions */
static void zpci_irq_handler(struct airq_struct *airq);

static struct airq_struct zpci_airq = {
	.handler = zpci_irq_handler,
	.isc = PCI_ISC,
};
73

J
Jan Glauber 已提交
74 75 76 77 78 79
/* I/O Map */
static DEFINE_SPINLOCK(zpci_iomap_lock);
static DECLARE_BITMAP(zpci_iomap, ZPCI_IOMAP_MAX_ENTRIES);
struct zpci_iomap_entry *zpci_iomap_start;
EXPORT_SYMBOL_GPL(zpci_iomap_start);

80 81
static struct kmem_cache *zdev_fmb_cache;

J
Jan Glauber 已提交
82 83 84 85 86 87 88 89 90
struct zpci_dev *get_zdev(struct pci_dev *pdev)
{
	return (struct zpci_dev *) pdev->sysdata;
}

struct zpci_dev *get_zdev_by_fid(u32 fid)
{
	struct zpci_dev *tmp, *zdev = NULL;

91
	spin_lock(&zpci_list_lock);
J
Jan Glauber 已提交
92 93 94 95 96 97
	list_for_each_entry(tmp, &zpci_list, entry) {
		if (tmp->fid == fid) {
			zdev = tmp;
			break;
		}
	}
98
	spin_unlock(&zpci_list_lock);
J
Jan Glauber 已提交
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
	return zdev;
}

static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus)
{
	return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL;
}

int pci_domain_nr(struct pci_bus *bus)
{
	return ((struct zpci_dev *) bus->sysdata)->domain;
}
EXPORT_SYMBOL_GPL(pci_domain_nr);

int pci_proc_domain(struct pci_bus *bus)
{
	return pci_domain_nr(bus);
}
EXPORT_SYMBOL_GPL(pci_proc_domain);

119
/* Modify PCI: Register adapter interruptions */
120
static int zpci_set_airq(struct zpci_dev *zdev)
121 122
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT);
123
	struct zpci_fib fib = {0};
124

125 126 127 128 129 130 131
	fib.isc = PCI_ISC;
	fib.sum = 1;		/* enable summary notifications */
	fib.noi = airq_iv_end(zdev->aibv);
	fib.aibv = (unsigned long) zdev->aibv->vector;
	fib.aibvo = 0;		/* each zdev has its own interrupt vector */
	fib.aisb = (unsigned long) zpci_aisb_iv->vector + (zdev->aisb/64)*8;
	fib.aisbo = zdev->aisb & 63;
132

133
	return zpci_mod_fc(req, &fib);
134 135 136 137 138 139
}

struct mod_pci_args {
	u64 base;
	u64 limit;
	u64 iota;
140
	u64 fmb_addr;
141 142 143 144 145
};

static int mod_pci(struct zpci_dev *zdev, int fn, u8 dmaas, struct mod_pci_args *args)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, fn);
146 147 148 149 150 151 152 153
	struct zpci_fib fib = {0};

	fib.pba = args->base;
	fib.pal = args->limit;
	fib.iota = args->iota;
	fib.fmb_addr = args->fmb_addr;

	return zpci_mod_fc(req, &fib);
154 155
}

J
Jan Glauber 已提交
156 157 158 159
/* Modify PCI: Register I/O address translation parameters */
int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
		       u64 base, u64 limit, u64 iota)
{
160
	struct mod_pci_args args = { base, limit, iota, 0 };
J
Jan Glauber 已提交
161 162 163 164 165 166 167 168 169

	WARN_ON_ONCE(iota & 0x3fff);
	args.iota |= ZPCI_IOTA_RTTO_FLAG;
	return mod_pci(zdev, ZPCI_MOD_FC_REG_IOAT, dmaas, &args);
}

/* Modify PCI: Unregister I/O address translation parameters */
int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
{
170
	struct mod_pci_args args = { 0, 0, 0, 0 };
J
Jan Glauber 已提交
171 172 173 174

	return mod_pci(zdev, ZPCI_MOD_FC_DEREG_IOAT, dmaas, &args);
}

175
/* Modify PCI: Unregister adapter interruptions */
176
static int zpci_clear_airq(struct zpci_dev *zdev)
177
{
178
	struct mod_pci_args args = { 0, 0, 0, 0 };
179 180 181 182

	return mod_pci(zdev, ZPCI_MOD_FC_DEREG_INT, 0, &args);
}

183 184 185 186 187 188 189 190
/* Modify PCI: Set PCI function measurement parameters */
int zpci_fmb_enable_device(struct zpci_dev *zdev)
{
	struct mod_pci_args args = { 0, 0, 0, 0 };

	if (zdev->fmb)
		return -EINVAL;

191
	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
	if (!zdev->fmb)
		return -ENOMEM;
	WARN_ON((u64) zdev->fmb & 0xf);

	args.fmb_addr = virt_to_phys(zdev->fmb);
	return mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);
}

/* Modify PCI: Disable PCI function measurement */
int zpci_fmb_disable_device(struct zpci_dev *zdev)
{
	struct mod_pci_args args = { 0, 0, 0, 0 };
	int rc;

	if (!zdev->fmb)
		return -EINVAL;

	/* Function measurement is disabled if fmb address is zero */
	rc = mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);

	kmem_cache_free(zdev_fmb_cache, zdev->fmb);
	zdev->fmb = NULL;
	return rc;
}

J
Jan Glauber 已提交
217 218 219 220 221 222 223 224
#define ZPCI_PCIAS_CFGSPC	15

static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
	u64 data;
	int rc;

225
	rc = zpci_load(&data, req, offset);
226 227 228
	if (!rc) {
		data = data << ((8 - len) * 8);
		data = le64_to_cpu(data);
J
Jan Glauber 已提交
229
		*val = (u32) data;
230
	} else
J
Jan Glauber 已提交
231 232 233 234 235 236 237 238 239 240 241 242
		*val = 0xffffffff;
	return rc;
}

static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
	u64 data = val;
	int rc;

	data = cpu_to_le64(data);
	data = data >> ((8 - len) * 8);
243
	rc = zpci_store(data, req, offset);
J
Jan Glauber 已提交
244 245 246
	return rc;
}

247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
static int zpci_msi_set_mask_bits(struct msi_desc *msi, u32 mask, u32 flag)
{
	int offset, pos;
	u32 mask_bits;

	if (msi->msi_attrib.is_msix) {
		offset = msi->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
			PCI_MSIX_ENTRY_VECTOR_CTRL;
		msi->masked = readl(msi->mask_base + offset);
		writel(flag, msi->mask_base + offset);
	} else if (msi->msi_attrib.maskbit) {
		pos = (long) msi->mask_base;
		pci_read_config_dword(msi->dev, pos, &mask_bits);
		mask_bits &= ~(mask);
		mask_bits |= flag & mask;
		pci_write_config_dword(msi->dev, pos, mask_bits);
	} else
		return 0;

	msi->msi_attrib.maskbit = !!flag;
	return 1;
}

static void zpci_enable_irq(struct irq_data *data)
271
{
272
	struct msi_desc *msi = irq_get_msi_desc(data->irq);
273 274 275 276

	zpci_msi_set_mask_bits(msi, 1, 0);
}

277
static void zpci_disable_irq(struct irq_data *data)
278
{
279
	struct msi_desc *msi = irq_get_msi_desc(data->irq);
280 281 282 283

	zpci_msi_set_mask_bits(msi, 1, 1);
}

284
void pcibios_fixup_bus(struct pci_bus *bus)
J
Jan Glauber 已提交
285 286 287 288 289 290 291 292 293 294
{
}

resource_size_t pcibios_align_resource(void *data, const struct resource *res,
				       resource_size_t size,
				       resource_size_t align)
{
	return 0;
}

295 296 297 298 299 300
/* combine single writes by using store-block insn */
void __iowrite64_copy(void __iomem *to, const void *from, size_t count)
{
       zpci_memcpy_toio(to, from, count);
}

J
Jan Glauber 已提交
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
/* Create a virtual mapping cookie for a PCI BAR */
void __iomem *pci_iomap(struct pci_dev *pdev, int bar, unsigned long max)
{
	struct zpci_dev *zdev =	get_zdev(pdev);
	u64 addr;
	int idx;

	if ((bar & 7) != bar)
		return NULL;

	idx = zdev->bars[bar].map_idx;
	spin_lock(&zpci_iomap_lock);
	zpci_iomap_start[idx].fh = zdev->fh;
	zpci_iomap_start[idx].bar = bar;
	spin_unlock(&zpci_iomap_lock);

	addr = ZPCI_IOMAP_ADDR_BASE | ((u64) idx << 48);
	return (void __iomem *) addr;
}
EXPORT_SYMBOL_GPL(pci_iomap);

void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
{
	unsigned int idx;

	idx = (((__force u64) addr) & ~ZPCI_IOMAP_ADDR_BASE) >> 48;
	spin_lock(&zpci_iomap_lock);
	zpci_iomap_start[idx].fh = 0;
	zpci_iomap_start[idx].bar = 0;
	spin_unlock(&zpci_iomap_lock);
}
EXPORT_SYMBOL_GPL(pci_iounmap);

static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
		    int size, u32 *val)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);
338
	int ret;
J
Jan Glauber 已提交
339 340

	if (!zdev || devfn != ZPCI_DEVFN)
341 342 343 344 345
		ret = -ENODEV;
	else
		ret = zpci_cfg_load(zdev, where, val, size);

	return ret;
J
Jan Glauber 已提交
346 347 348 349 350 351
}

static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
		     int size, u32 val)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);
352
	int ret;
J
Jan Glauber 已提交
353 354

	if (!zdev || devfn != ZPCI_DEVFN)
355 356 357 358 359
		ret = -ENODEV;
	else
		ret = zpci_cfg_store(zdev, where, val, size);

	return ret;
J
Jan Glauber 已提交
360 361 362 363 364 365 366
}

static struct pci_ops pci_root_ops = {
	.read = pci_read,
	.write = pci_write,
};

367
static void zpci_irq_handler(struct airq_struct *airq)
368
{
369
	unsigned long si, ai;
370
	struct airq_iv *aibv;
371
	int irqs_on = 0;
372

373
	inc_irq_stat(IRQIO_PCI);
374 375 376 377 378 379 380 381 382 383 384 385
	for (si = 0;;) {
		/* Scan adapter summary indicator bit vector */
		si = airq_iv_scan(zpci_aisb_iv, si, airq_iv_end(zpci_aisb_iv));
		if (si == -1UL) {
			if (irqs_on++)
				/* End of second scan with interrupts on. */
				break;
			/* First scan complete, reenable interrupts. */
			zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
			si = 0;
			continue;
		}
386

387
		/* Scan the adapter interrupt vector for this device. */
388
		aibv = zpci_aibv[si];
389
		for (ai = 0;;) {
390
			ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv));
391 392
			if (ai == -1UL)
				break;
393
			inc_irq_stat(IRQIO_MSI);
394 395 396
			airq_iv_lock(aibv, ai);
			generic_handle_irq(airq_iv_get_data(aibv, ai));
			airq_iv_unlock(aibv, ai);
397 398
		}
	}
399
}
400

401
int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
402 403
{
	struct zpci_dev *zdev = get_zdev(pdev);
404
	unsigned int hwirq, irq, msi_vecs;
405
	unsigned long aisb;
406
	struct msi_desc *msi;
407
	struct msi_msg msg;
408 409
	int rc;

410 411 412
	if (type != PCI_CAP_ID_MSIX && type != PCI_CAP_ID_MSI)
		return -EINVAL;
	msi_vecs = min(nvec, ZPCI_MSI_VEC_MAX);
413
	msi_vecs = min_t(unsigned int, msi_vecs, CONFIG_PCI_NR_MSI);
414

415 416 417 418 419
	/* Allocate adapter summary indicator bit */
	rc = -EIO;
	aisb = airq_iv_alloc_bit(zpci_aisb_iv);
	if (aisb == -1UL)
		goto out;
420 421
	zdev->aisb = aisb;

422 423
	/* Create adapter interrupt vector */
	rc = -ENOMEM;
424
	zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK);
425 426
	if (!zdev->aibv)
		goto out_si;
427

428
	/* Wire up shortcut pointer */
429
	zpci_aibv[aisb] = zdev->aibv;
430

431 432
	/* Request MSI interrupts */
	hwirq = 0;
433
	list_for_each_entry(msi, &pdev->msi_list, list) {
434 435 436 437 438
		rc = -EIO;
		irq = irq_alloc_desc(0);	/* Alloc irq on node 0 */
		if (irq == NO_IRQ)
			goto out_msi;
		rc = irq_set_msi_desc(irq, msi);
439
		if (rc)
440
			goto out_msi;
441 442 443 444 445 446 447 448
		irq_set_chip_and_handler(irq, &zpci_irq_chip,
					 handle_simple_irq);
		msg.data = hwirq;
		msg.address_lo = zdev->msi_addr & 0xffffffff;
		msg.address_hi = zdev->msi_addr >> 32;
		write_msi_msg(irq, &msg);
		airq_iv_set_data(zdev->aibv, hwirq, irq);
		hwirq++;
449 450
	}

451 452 453 454 455 456 457 458 459
	/* Enable adapter interrupts */
	rc = zpci_set_airq(zdev);
	if (rc)
		goto out_msi;

	return (msi_vecs == nvec) ? 0 : msi_vecs;

out_msi:
	list_for_each_entry(msi, &pdev->msi_list, list) {
460
		if (hwirq-- == 0)
461
			break;
462 463 464 465 466 467
		irq_set_msi_desc(msi->irq, NULL);
		irq_free_desc(msi->irq);
		msi->msg.address_lo = 0;
		msi->msg.address_hi = 0;
		msi->msg.data = 0;
		msi->irq = 0;
468
	}
469
	zpci_aibv[aisb] = NULL;
470 471 472 473 474
	airq_iv_release(zdev->aibv);
out_si:
	airq_iv_free_bit(zpci_aisb_iv, aisb);
out:
	return rc;
475 476
}

477
void arch_teardown_msi_irqs(struct pci_dev *pdev)
478 479 480
{
	struct zpci_dev *zdev = get_zdev(pdev);
	struct msi_desc *msi;
481 482 483 484
	int rc;

	/* Disable adapter interrupts */
	rc = zpci_clear_airq(zdev);
S
Sebastian Ott 已提交
485
	if (rc)
486 487
		return;

488 489 490 491 492 493 494 495 496 497
	/* Release MSI interrupts */
	list_for_each_entry(msi, &pdev->msi_list, list) {
		zpci_msi_set_mask_bits(msi, 1, 1);
		irq_set_msi_desc(msi->irq, NULL);
		irq_free_desc(msi->irq);
		msi->msg.address_lo = 0;
		msi->msg.address_hi = 0;
		msi->msg.data = 0;
		msi->irq = 0;
	}
498

499
	zpci_aibv[zdev->aisb] = NULL;
500 501
	airq_iv_release(zdev->aibv);
	airq_iv_free_bit(zpci_aisb_iv, zdev->aisb);
502 503
}

J
Jan Glauber 已提交
504 505 506 507 508 509 510 511 512 513 514 515 516
static void zpci_map_resources(struct zpci_dev *zdev)
{
	struct pci_dev *pdev = zdev->pdev;
	resource_size_t len;
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		len = pci_resource_len(pdev, i);
		if (!len)
			continue;
		pdev->resource[i].start = (resource_size_t) pci_iomap(pdev, i, 0);
		pdev->resource[i].end = pdev->resource[i].start + len - 1;
	}
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531
}

static void zpci_unmap_resources(struct zpci_dev *zdev)
{
	struct pci_dev *pdev = zdev->pdev;
	resource_size_t len;
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		len = pci_resource_len(pdev, i);
		if (!len)
			continue;
		pci_iounmap(pdev, (void *) pdev->resource[i].start);
	}
}
J
Jan Glauber 已提交
532

533 534 535 536 537
int pcibios_add_platform_entries(struct pci_dev *pdev)
{
	return zpci_sysfs_add_device(&pdev->dev);
}

538 539
static int __init zpci_irq_init(void)
{
540
	int rc;
541

542 543
	rc = register_adapter_interrupt(&zpci_airq);
	if (rc)
544
		goto out;
545 546
	/* Set summary to 1 to be called every time for the ISC. */
	*zpci_airq.lsi_ptr = 1;
547

548 549 550 551
	rc = -ENOMEM;
	zpci_aisb_iv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC);
	if (!zpci_aisb_iv)
		goto out_airq;
552

553
	zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
554 555
	return 0;

556 557 558
out_airq:
	unregister_adapter_interrupt(&zpci_airq);
out:
559 560 561 562 563
	return rc;
}

static void zpci_irq_exit(void)
{
564
	airq_iv_release(zpci_aisb_iv);
565
	unregister_adapter_interrupt(&zpci_airq);
566 567
}

J
Jan Glauber 已提交
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590
static int zpci_alloc_iomap(struct zpci_dev *zdev)
{
	int entry;

	spin_lock(&zpci_iomap_lock);
	entry = find_first_zero_bit(zpci_iomap, ZPCI_IOMAP_MAX_ENTRIES);
	if (entry == ZPCI_IOMAP_MAX_ENTRIES) {
		spin_unlock(&zpci_iomap_lock);
		return -ENOSPC;
	}
	set_bit(entry, zpci_iomap);
	spin_unlock(&zpci_iomap_lock);
	return entry;
}

static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
{
	spin_lock(&zpci_iomap_lock);
	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
	clear_bit(entry, zpci_iomap);
	spin_unlock(&zpci_iomap_lock);
}

591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666
static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,
				    unsigned long size, unsigned long flags)
{
	struct resource *r;

	r = kzalloc(sizeof(*r), GFP_KERNEL);
	if (!r)
		return NULL;

	r->start = start;
	r->end = r->start + size - 1;
	r->flags = flags;
	r->name = zdev->res_name;

	if (request_resource(&iomem_resource, r)) {
		kfree(r);
		return NULL;
	}
	return r;
}

static int zpci_setup_bus_resources(struct zpci_dev *zdev,
				    struct list_head *resources)
{
	unsigned long addr, size, flags;
	struct resource *res;
	int i, entry;

	snprintf(zdev->res_name, sizeof(zdev->res_name),
		 "PCI Bus %04x:%02x", zdev->domain, ZPCI_BUS_NR);

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		if (!zdev->bars[i].size)
			continue;
		entry = zpci_alloc_iomap(zdev);
		if (entry < 0)
			return entry;
		zdev->bars[i].map_idx = entry;

		/* only MMIO is supported */
		flags = IORESOURCE_MEM;
		if (zdev->bars[i].val & 8)
			flags |= IORESOURCE_PREFETCH;
		if (zdev->bars[i].val & 4)
			flags |= IORESOURCE_MEM_64;

		addr = ZPCI_IOMAP_ADDR_BASE + ((u64) entry << 48);

		size = 1UL << zdev->bars[i].size;

		res = __alloc_res(zdev, addr, size, flags);
		if (!res) {
			zpci_free_iomap(zdev, entry);
			return -ENOMEM;
		}
		zdev->bars[i].res = res;
		pci_add_resource(resources, res);
	}

	return 0;
}

static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)
{
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		if (!zdev->bars[i].size)
			continue;

		zpci_free_iomap(zdev, zdev->bars[i].map_idx);
		release_resource(zdev->bars[i].res);
		kfree(zdev->bars[i].res);
	}
}

667 668 669
int pcibios_add_device(struct pci_dev *pdev)
{
	struct zpci_dev *zdev = get_zdev(pdev);
S
Sebastian Ott 已提交
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691
	struct resource *res;
	int i;

	zdev->pdev = pdev;
	zpci_map_resources(zdev);

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		res = &pdev->resource[i];
		if (res->parent || !res->flags)
			continue;
		pci_claim_resource(pdev, i);
	}

	return 0;
}

int pcibios_enable_device(struct pci_dev *pdev, int mask)
{
	struct zpci_dev *zdev = get_zdev(pdev);
	struct resource *res;
	u16 cmd;
	int i;
692

S
Sebastian Ott 已提交
693
	zdev->pdev = pdev;
694 695 696 697
	zpci_debug_init_device(zdev);
	zpci_fmb_enable_device(zdev);
	zpci_map_resources(zdev);

S
Sebastian Ott 已提交
698 699 700 701 702 703 704 705 706 707 708
	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
	for (i = 0; i < PCI_BAR_COUNT; i++) {
		res = &pdev->resource[i];

		if (res->flags & IORESOURCE_IO)
			return -EINVAL;

		if (res->flags & IORESOURCE_MEM)
			cmd |= PCI_COMMAND_MEMORY;
	}
	pci_write_config_word(pdev, PCI_COMMAND, cmd);
709 710 711
	return 0;
}

S
Sebastian Ott 已提交
712
void pcibios_disable_device(struct pci_dev *pdev)
713 714 715 716 717 718 719 720 721
{
	struct zpci_dev *zdev = get_zdev(pdev);

	zpci_unmap_resources(zdev);
	zpci_fmb_disable_device(zdev);
	zpci_debug_exit_device(zdev);
	zdev->pdev = NULL;
}

722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762
#ifdef CONFIG_HIBERNATE_CALLBACKS
static int zpci_restore(struct device *dev)
{
	struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));
	int ret = 0;

	if (zdev->state != ZPCI_FN_STATE_ONLINE)
		goto out;

	ret = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
	if (ret)
		goto out;

	zpci_map_resources(zdev);
	zpci_register_ioat(zdev, 0, zdev->start_dma + PAGE_OFFSET,
			   zdev->start_dma + zdev->iommu_size - 1,
			   (u64) zdev->dma_table);

out:
	return ret;
}

static int zpci_freeze(struct device *dev)
{
	struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));

	if (zdev->state != ZPCI_FN_STATE_ONLINE)
		return 0;

	zpci_unregister_ioat(zdev, 0);
	return clp_disable_fh(zdev);
}

struct dev_pm_ops pcibios_pm_ops = {
	.thaw_noirq = zpci_restore,
	.freeze_noirq = zpci_freeze,
	.restore_noirq = zpci_restore,
	.poweroff_noirq = zpci_freeze,
};
#endif /* CONFIG_HIBERNATE_CALLBACKS */

J
Jan Glauber 已提交
763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782
static int zpci_alloc_domain(struct zpci_dev *zdev)
{
	spin_lock(&zpci_domain_lock);
	zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
	if (zdev->domain == ZPCI_NR_DEVICES) {
		spin_unlock(&zpci_domain_lock);
		return -ENOSPC;
	}
	set_bit(zdev->domain, zpci_domain);
	spin_unlock(&zpci_domain_lock);
	return 0;
}

static void zpci_free_domain(struct zpci_dev *zdev)
{
	spin_lock(&zpci_domain_lock);
	clear_bit(zdev->domain, zpci_domain);
	spin_unlock(&zpci_domain_lock);
}

783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
void pcibios_remove_bus(struct pci_bus *bus)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);

	zpci_exit_slot(zdev);
	zpci_cleanup_bus_resources(zdev);
	zpci_free_domain(zdev);

	spin_lock(&zpci_list_lock);
	list_del(&zdev->entry);
	spin_unlock(&zpci_list_lock);

	kfree(zdev);
}

static int zpci_scan_bus(struct zpci_dev *zdev)
{
	LIST_HEAD(resources);
	int ret;

	ret = zpci_setup_bus_resources(zdev, &resources);
	if (ret)
		return ret;

	zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops,
				      zdev, &resources);
	if (!zdev->bus) {
		zpci_cleanup_bus_resources(zdev);
		return -EIO;
	}

	zdev->bus->max_bus_speed = zdev->max_bus_speed;
	return 0;
}

J
Jan Glauber 已提交
818 819 820 821 822 823 824
int zpci_enable_device(struct zpci_dev *zdev)
{
	int rc;

	rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
	if (rc)
		goto out;
J
Jan Glauber 已提交
825 826 827 828

	rc = zpci_dma_init_device(zdev);
	if (rc)
		goto out_dma;
S
Sebastian Ott 已提交
829 830

	zdev->state = ZPCI_FN_STATE_ONLINE;
J
Jan Glauber 已提交
831
	return 0;
J
Jan Glauber 已提交
832 833 834

out_dma:
	clp_disable_fh(zdev);
J
Jan Glauber 已提交
835 836 837 838 839
out:
	return rc;
}
EXPORT_SYMBOL_GPL(zpci_enable_device);

840 841 842 843 844 845 846
int zpci_disable_device(struct zpci_dev *zdev)
{
	zpci_dma_exit_device(zdev);
	return clp_disable_fh(zdev);
}
EXPORT_SYMBOL_GPL(zpci_disable_device);

J
Jan Glauber 已提交
847 848 849 850 851 852 853 854
int zpci_create_device(struct zpci_dev *zdev)
{
	int rc;

	rc = zpci_alloc_domain(zdev);
	if (rc)
		goto out;

S
Sebastian Ott 已提交
855 856 857 858 859 860
	if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
		rc = zpci_enable_device(zdev);
		if (rc)
			goto out_free;
	}
	rc = zpci_scan_bus(zdev);
J
Jan Glauber 已提交
861
	if (rc)
S
Sebastian Ott 已提交
862
		goto out_disable;
J
Jan Glauber 已提交
863

864
	spin_lock(&zpci_list_lock);
J
Jan Glauber 已提交
865
	list_add_tail(&zdev->entry, &zpci_list);
866
	spin_unlock(&zpci_list_lock);
J
Jan Glauber 已提交
867

868 869
	zpci_init_slot(zdev);

J
Jan Glauber 已提交
870 871
	return 0;

S
Sebastian Ott 已提交
872 873 874 875
out_disable:
	if (zdev->state == ZPCI_FN_STATE_ONLINE)
		zpci_disable_device(zdev);
out_free:
J
Jan Glauber 已提交
876 877 878 879 880 881 882
	zpci_free_domain(zdev);
out:
	return rc;
}

void zpci_stop_device(struct zpci_dev *zdev)
{
J
Jan Glauber 已提交
883
	zpci_dma_exit_device(zdev);
J
Jan Glauber 已提交
884 885 886 887 888 889 890 891 892 893 894 895 896 897
	/*
	 * Note: SCLP disables fh via set-pci-fn so don't
	 * do that here.
	 */
}
EXPORT_SYMBOL_GPL(zpci_stop_device);

static inline int barsize(u8 size)
{
	return (size) ? (1 << size) >> 10 : 0;
}

static int zpci_mem_init(void)
{
898 899 900
	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
				16, 0, NULL);
	if (!zdev_fmb_cache)
901
		goto error_zdev;
902

J
Jan Glauber 已提交
903 904 905 906
	/* TODO: use realloc */
	zpci_iomap_start = kzalloc(ZPCI_IOMAP_MAX_ENTRIES * sizeof(*zpci_iomap_start),
				   GFP_KERNEL);
	if (!zpci_iomap_start)
907
		goto error_iomap;
J
Jan Glauber 已提交
908 909
	return 0;

910
error_iomap:
911
	kmem_cache_destroy(zdev_fmb_cache);
J
Jan Glauber 已提交
912 913 914 915 916 917 918
error_zdev:
	return -ENOMEM;
}

static void zpci_mem_exit(void)
{
	kfree(zpci_iomap_start);
919
	kmem_cache_destroy(zdev_fmb_cache);
J
Jan Glauber 已提交
920 921
}

922
static unsigned int s390_pci_probe;
J
Jan Glauber 已提交
923 924 925

char * __init pcibios_setup(char *str)
{
S
Sebastian Ott 已提交
926 927
	if (!strcmp(str, "on")) {
		s390_pci_probe = 1;
J
Jan Glauber 已提交
928 929 930 931 932 933 934 935 936
		return NULL;
	}
	return str;
}

static int __init pci_base_init(void)
{
	int rc;

937
	if (!s390_pci_probe)
J
Jan Glauber 已提交
938 939 940 941 942 943
		return 0;

	if (!test_facility(2) || !test_facility(69)
	    || !test_facility(71) || !test_facility(72))
		return 0;

944 945
	rc = zpci_debug_init();
	if (rc)
946
		goto out;
947

J
Jan Glauber 已提交
948 949 950 951
	rc = zpci_mem_init();
	if (rc)
		goto out_mem;

952 953 954 955
	rc = zpci_irq_init();
	if (rc)
		goto out_irq;

J
Jan Glauber 已提交
956 957 958 959
	rc = zpci_dma_init();
	if (rc)
		goto out_dma;

S
Sebastian Ott 已提交
960
	rc = clp_scan_pci_devices();
J
Jan Glauber 已提交
961 962 963
	if (rc)
		goto out_find;

J
Jan Glauber 已提交
964 965
	return 0;

J
Jan Glauber 已提交
966
out_find:
J
Jan Glauber 已提交
967 968
	zpci_dma_exit();
out_dma:
969 970
	zpci_irq_exit();
out_irq:
J
Jan Glauber 已提交
971 972
	zpci_mem_exit();
out_mem:
973
	zpci_debug_exit();
974
out:
J
Jan Glauber 已提交
975 976
	return rc;
}
977
subsys_initcall_sync(pci_base_init);
978 979 980 981 982

void zpci_rescan(void)
{
	clp_rescan_pci_devices_simple();
}