pci.c 20.2 KB
Newer Older
J
Jan Glauber 已提交
1 2 3 4 5 6 7 8
/*
 * Copyright IBM Corp. 2012
 *
 * Author(s):
 *   Jan Glauber <jang@linux.vnet.ibm.com>
 *
 * The System z PCI code is a rewrite from a prototype by
 * the following people (Kudoz!):
9 10 11 12 13 14 15
 *   Alexander Schmidt
 *   Christoph Raisch
 *   Hannes Hering
 *   Hoang-Nam Nguyen
 *   Jan-Bernd Themann
 *   Stefan Roscher
 *   Thomas Klein
J
Jan Glauber 已提交
16 17
 */

G
Gerald Schaefer 已提交
18 19
#define KMSG_COMPONENT "zpci"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
J
Jan Glauber 已提交
20 21 22 23 24 25

#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/export.h>
#include <linux/delay.h>
26 27
#include <linux/irq.h>
#include <linux/kernel_stat.h>
J
Jan Glauber 已提交
28 29 30 31
#include <linux/seq_file.h>
#include <linux/pci.h>
#include <linux/msi.h>

32 33
#include <asm/isc.h>
#include <asm/airq.h>
J
Jan Glauber 已提交
34 35
#include <asm/facility.h>
#include <asm/pci_insn.h>
J
Jan Glauber 已提交
36
#include <asm/pci_clp.h>
J
Jan Glauber 已提交
37
#include <asm/pci_dma.h>
J
Jan Glauber 已提交
38 39 40

#define DEBUG				/* enable pr_debug */

41 42 43
#define	SIC_IRQ_MODE_ALL		0
#define	SIC_IRQ_MODE_SINGLE		1

J
Jan Glauber 已提交
44 45 46 47
#define ZPCI_NR_DMA_SPACES		1
#define ZPCI_NR_DEVICES			CONFIG_PCI_NR_FUNCTIONS

/* list of all detected zpci devices */
48
static LIST_HEAD(zpci_list);
49
static DEFINE_SPINLOCK(zpci_list_lock);
J
Jan Glauber 已提交
50

51 52
static struct irq_chip zpci_irq_chip = {
	.name = "zPCI",
53 54
	.irq_unmask = pci_msi_unmask_irq,
	.irq_mask = pci_msi_mask_irq,
55 56
};

57 58
static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES);
static DEFINE_SPINLOCK(zpci_domain_lock);
59

60
static struct airq_iv *zpci_aisb_iv;
61
static struct airq_iv *zpci_aibv[ZPCI_NR_DEVICES];
62

63 64 65 66 67 68 69
/* Adapter interrupt definitions */
static void zpci_irq_handler(struct airq_struct *airq);

static struct airq_struct zpci_airq = {
	.handler = zpci_irq_handler,
	.isc = PCI_ISC,
};
70

J
Jan Glauber 已提交
71 72 73 74 75 76
/* I/O Map */
static DEFINE_SPINLOCK(zpci_iomap_lock);
static DECLARE_BITMAP(zpci_iomap, ZPCI_IOMAP_MAX_ENTRIES);
struct zpci_iomap_entry *zpci_iomap_start;
EXPORT_SYMBOL_GPL(zpci_iomap_start);

77 78
static struct kmem_cache *zdev_fmb_cache;

J
Jan Glauber 已提交
79 80 81 82 83 84 85 86 87
struct zpci_dev *get_zdev(struct pci_dev *pdev)
{
	return (struct zpci_dev *) pdev->sysdata;
}

struct zpci_dev *get_zdev_by_fid(u32 fid)
{
	struct zpci_dev *tmp, *zdev = NULL;

88
	spin_lock(&zpci_list_lock);
J
Jan Glauber 已提交
89 90 91 92 93 94
	list_for_each_entry(tmp, &zpci_list, entry) {
		if (tmp->fid == fid) {
			zdev = tmp;
			break;
		}
	}
95
	spin_unlock(&zpci_list_lock);
J
Jan Glauber 已提交
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
	return zdev;
}

static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus)
{
	return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL;
}

int pci_domain_nr(struct pci_bus *bus)
{
	return ((struct zpci_dev *) bus->sysdata)->domain;
}
EXPORT_SYMBOL_GPL(pci_domain_nr);

int pci_proc_domain(struct pci_bus *bus)
{
	return pci_domain_nr(bus);
}
EXPORT_SYMBOL_GPL(pci_proc_domain);

116
/* Modify PCI: Register adapter interruptions */
117
static int zpci_set_airq(struct zpci_dev *zdev)
118 119
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT);
120
	struct zpci_fib fib = {0};
121

122 123 124 125 126 127 128
	fib.isc = PCI_ISC;
	fib.sum = 1;		/* enable summary notifications */
	fib.noi = airq_iv_end(zdev->aibv);
	fib.aibv = (unsigned long) zdev->aibv->vector;
	fib.aibvo = 0;		/* each zdev has its own interrupt vector */
	fib.aisb = (unsigned long) zpci_aisb_iv->vector + (zdev->aisb/64)*8;
	fib.aisbo = zdev->aisb & 63;
129

130
	return zpci_mod_fc(req, &fib);
131 132 133 134 135 136
}

struct mod_pci_args {
	u64 base;
	u64 limit;
	u64 iota;
137
	u64 fmb_addr;
138 139 140 141 142
};

static int mod_pci(struct zpci_dev *zdev, int fn, u8 dmaas, struct mod_pci_args *args)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, fn);
143 144 145 146 147 148 149 150
	struct zpci_fib fib = {0};

	fib.pba = args->base;
	fib.pal = args->limit;
	fib.iota = args->iota;
	fib.fmb_addr = args->fmb_addr;

	return zpci_mod_fc(req, &fib);
151 152
}

J
Jan Glauber 已提交
153 154 155 156
/* Modify PCI: Register I/O address translation parameters */
int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
		       u64 base, u64 limit, u64 iota)
{
157
	struct mod_pci_args args = { base, limit, iota, 0 };
J
Jan Glauber 已提交
158 159 160 161 162 163 164 165 166

	WARN_ON_ONCE(iota & 0x3fff);
	args.iota |= ZPCI_IOTA_RTTO_FLAG;
	return mod_pci(zdev, ZPCI_MOD_FC_REG_IOAT, dmaas, &args);
}

/* Modify PCI: Unregister I/O address translation parameters */
int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
{
167
	struct mod_pci_args args = { 0, 0, 0, 0 };
J
Jan Glauber 已提交
168 169 170 171

	return mod_pci(zdev, ZPCI_MOD_FC_DEREG_IOAT, dmaas, &args);
}

172
/* Modify PCI: Unregister adapter interruptions */
173
static int zpci_clear_airq(struct zpci_dev *zdev)
174
{
175
	struct mod_pci_args args = { 0, 0, 0, 0 };
176 177 178 179

	return mod_pci(zdev, ZPCI_MOD_FC_DEREG_INT, 0, &args);
}

180 181 182 183 184 185 186 187
/* Modify PCI: Set PCI function measurement parameters */
int zpci_fmb_enable_device(struct zpci_dev *zdev)
{
	struct mod_pci_args args = { 0, 0, 0, 0 };

	if (zdev->fmb)
		return -EINVAL;

188
	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
189 190 191 192
	if (!zdev->fmb)
		return -ENOMEM;
	WARN_ON((u64) zdev->fmb & 0xf);

193 194 195 196 197
	/* reset software counters */
	atomic64_set(&zdev->allocated_pages, 0);
	atomic64_set(&zdev->mapped_pages, 0);
	atomic64_set(&zdev->unmapped_pages, 0);

198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
	args.fmb_addr = virt_to_phys(zdev->fmb);
	return mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);
}

/* Modify PCI: Disable PCI function measurement */
int zpci_fmb_disable_device(struct zpci_dev *zdev)
{
	struct mod_pci_args args = { 0, 0, 0, 0 };
	int rc;

	if (!zdev->fmb)
		return -EINVAL;

	/* Function measurement is disabled if fmb address is zero */
	rc = mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);

	kmem_cache_free(zdev_fmb_cache, zdev->fmb);
	zdev->fmb = NULL;
	return rc;
}

J
Jan Glauber 已提交
219 220 221 222 223 224 225 226
#define ZPCI_PCIAS_CFGSPC	15

static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
	u64 data;
	int rc;

227
	rc = zpci_load(&data, req, offset);
228 229 230
	if (!rc) {
		data = data << ((8 - len) * 8);
		data = le64_to_cpu(data);
J
Jan Glauber 已提交
231
		*val = (u32) data;
232
	} else
J
Jan Glauber 已提交
233 234 235 236 237 238 239 240 241 242 243 244
		*val = 0xffffffff;
	return rc;
}

static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
	u64 data = val;
	int rc;

	data = cpu_to_le64(data);
	data = data >> ((8 - len) * 8);
245
	rc = zpci_store(data, req, offset);
J
Jan Glauber 已提交
246 247 248
	return rc;
}

249
void pcibios_fixup_bus(struct pci_bus *bus)
J
Jan Glauber 已提交
250 251 252 253 254 255 256 257 258 259
{
}

resource_size_t pcibios_align_resource(void *data, const struct resource *res,
				       resource_size_t size,
				       resource_size_t align)
{
	return 0;
}

260 261 262 263 264 265
/* combine single writes by using store-block insn */
void __iowrite64_copy(void __iomem *to, const void *from, size_t count)
{
       zpci_memcpy_toio(to, from, count);
}

J
Jan Glauber 已提交
266
/* Create a virtual mapping cookie for a PCI BAR */
M
Michael S. Tsirkin 已提交
267 268 269 270
void __iomem *pci_iomap_range(struct pci_dev *pdev,
			      int bar,
			      unsigned long offset,
			      unsigned long max)
J
Jan Glauber 已提交
271 272 273 274 275 276 277 278 279 280
{
	struct zpci_dev *zdev =	get_zdev(pdev);
	u64 addr;
	int idx;

	if ((bar & 7) != bar)
		return NULL;

	idx = zdev->bars[bar].map_idx;
	spin_lock(&zpci_iomap_lock);
M
Michael S. Tsirkin 已提交
281 282 283 284 285 286 287 288 289
	if (zpci_iomap_start[idx].count++) {
		BUG_ON(zpci_iomap_start[idx].fh != zdev->fh ||
		       zpci_iomap_start[idx].bar != bar);
	} else {
		zpci_iomap_start[idx].fh = zdev->fh;
		zpci_iomap_start[idx].bar = bar;
	}
	/* Detect overrun */
	BUG_ON(!zpci_iomap_start[idx].count);
J
Jan Glauber 已提交
290 291 292
	spin_unlock(&zpci_iomap_lock);

	addr = ZPCI_IOMAP_ADDR_BASE | ((u64) idx << 48);
M
Michael S. Tsirkin 已提交
293
	return (void __iomem *) addr + offset;
J
Jan Glauber 已提交
294
}
295
EXPORT_SYMBOL(pci_iomap_range);
M
Michael S. Tsirkin 已提交
296 297 298 299 300 301

void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen)
{
	return pci_iomap_range(dev, bar, 0, maxlen);
}
EXPORT_SYMBOL(pci_iomap);
J
Jan Glauber 已提交
302 303 304 305 306 307 308

void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
{
	unsigned int idx;

	idx = (((__force u64) addr) & ~ZPCI_IOMAP_ADDR_BASE) >> 48;
	spin_lock(&zpci_iomap_lock);
M
Michael S. Tsirkin 已提交
309 310 311 312 313 314
	/* Detect underrun */
	BUG_ON(!zpci_iomap_start[idx].count);
	if (!--zpci_iomap_start[idx].count) {
		zpci_iomap_start[idx].fh = 0;
		zpci_iomap_start[idx].bar = 0;
	}
J
Jan Glauber 已提交
315 316
	spin_unlock(&zpci_iomap_lock);
}
317
EXPORT_SYMBOL(pci_iounmap);
J
Jan Glauber 已提交
318 319 320 321 322

static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
		    int size, u32 *val)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);
323
	int ret;
J
Jan Glauber 已提交
324 325

	if (!zdev || devfn != ZPCI_DEVFN)
326 327 328 329 330
		ret = -ENODEV;
	else
		ret = zpci_cfg_load(zdev, where, val, size);

	return ret;
J
Jan Glauber 已提交
331 332 333 334 335 336
}

static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
		     int size, u32 val)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);
337
	int ret;
J
Jan Glauber 已提交
338 339

	if (!zdev || devfn != ZPCI_DEVFN)
340 341 342 343 344
		ret = -ENODEV;
	else
		ret = zpci_cfg_store(zdev, where, val, size);

	return ret;
J
Jan Glauber 已提交
345 346 347 348 349 350 351
}

static struct pci_ops pci_root_ops = {
	.read = pci_read,
	.write = pci_write,
};

352
static void zpci_irq_handler(struct airq_struct *airq)
353
{
354
	unsigned long si, ai;
355
	struct airq_iv *aibv;
356
	int irqs_on = 0;
357

358
	inc_irq_stat(IRQIO_PCI);
359 360 361 362 363 364 365 366 367 368 369 370
	for (si = 0;;) {
		/* Scan adapter summary indicator bit vector */
		si = airq_iv_scan(zpci_aisb_iv, si, airq_iv_end(zpci_aisb_iv));
		if (si == -1UL) {
			if (irqs_on++)
				/* End of second scan with interrupts on. */
				break;
			/* First scan complete, reenable interrupts. */
			zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
			si = 0;
			continue;
		}
371

372
		/* Scan the adapter interrupt vector for this device. */
373
		aibv = zpci_aibv[si];
374
		for (ai = 0;;) {
375
			ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv));
376 377
			if (ai == -1UL)
				break;
378
			inc_irq_stat(IRQIO_MSI);
379 380 381
			airq_iv_lock(aibv, ai);
			generic_handle_irq(airq_iv_get_data(aibv, ai));
			airq_iv_unlock(aibv, ai);
382 383
		}
	}
384
}
385

386
int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
387 388
{
	struct zpci_dev *zdev = get_zdev(pdev);
389
	unsigned int hwirq, msi_vecs;
390
	unsigned long aisb;
391
	struct msi_desc *msi;
392
	struct msi_msg msg;
393
	int rc, irq;
394

395 396
	if (type == PCI_CAP_ID_MSI && nvec > 1)
		return 1;
397
	msi_vecs = min_t(unsigned int, nvec, zdev->max_msi);
398

399 400 401 402 403
	/* Allocate adapter summary indicator bit */
	rc = -EIO;
	aisb = airq_iv_alloc_bit(zpci_aisb_iv);
	if (aisb == -1UL)
		goto out;
404 405
	zdev->aisb = aisb;

406 407
	/* Create adapter interrupt vector */
	rc = -ENOMEM;
408
	zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK);
409 410
	if (!zdev->aibv)
		goto out_si;
411

412
	/* Wire up shortcut pointer */
413
	zpci_aibv[aisb] = zdev->aibv;
414

415 416
	/* Request MSI interrupts */
	hwirq = 0;
417
	list_for_each_entry(msi, &pdev->msi_list, list) {
418 419
		rc = -EIO;
		irq = irq_alloc_desc(0);	/* Alloc irq on node 0 */
420
		if (irq < 0)
421 422
			goto out_msi;
		rc = irq_set_msi_desc(irq, msi);
423
		if (rc)
424
			goto out_msi;
425 426 427 428 429
		irq_set_chip_and_handler(irq, &zpci_irq_chip,
					 handle_simple_irq);
		msg.data = hwirq;
		msg.address_lo = zdev->msi_addr & 0xffffffff;
		msg.address_hi = zdev->msi_addr >> 32;
430
		pci_write_msi_msg(irq, &msg);
431 432
		airq_iv_set_data(zdev->aibv, hwirq, irq);
		hwirq++;
433 434
	}

435 436 437 438 439 440 441 442 443
	/* Enable adapter interrupts */
	rc = zpci_set_airq(zdev);
	if (rc)
		goto out_msi;

	return (msi_vecs == nvec) ? 0 : msi_vecs;

out_msi:
	list_for_each_entry(msi, &pdev->msi_list, list) {
444
		if (hwirq-- == 0)
445
			break;
446 447 448 449 450 451
		irq_set_msi_desc(msi->irq, NULL);
		irq_free_desc(msi->irq);
		msi->msg.address_lo = 0;
		msi->msg.address_hi = 0;
		msi->msg.data = 0;
		msi->irq = 0;
452
	}
453
	zpci_aibv[aisb] = NULL;
454 455 456 457 458
	airq_iv_release(zdev->aibv);
out_si:
	airq_iv_free_bit(zpci_aisb_iv, aisb);
out:
	return rc;
459 460
}

461
void arch_teardown_msi_irqs(struct pci_dev *pdev)
462 463 464
{
	struct zpci_dev *zdev = get_zdev(pdev);
	struct msi_desc *msi;
465 466 467 468
	int rc;

	/* Disable adapter interrupts */
	rc = zpci_clear_airq(zdev);
S
Sebastian Ott 已提交
469
	if (rc)
470 471
		return;

472 473
	/* Release MSI interrupts */
	list_for_each_entry(msi, &pdev->msi_list, list) {
474
		if (msi->msi_attrib.is_msix)
475
			__pci_msix_desc_mask_irq(msi, 1);
476
		else
477
			__pci_msi_desc_mask_irq(msi, 1, 1);
478 479 480 481 482 483 484
		irq_set_msi_desc(msi->irq, NULL);
		irq_free_desc(msi->irq);
		msi->msg.address_lo = 0;
		msi->msg.address_hi = 0;
		msi->msg.data = 0;
		msi->irq = 0;
	}
485

486
	zpci_aibv[zdev->aisb] = NULL;
487 488
	airq_iv_release(zdev->aibv);
	airq_iv_free_bit(zpci_aisb_iv, zdev->aisb);
489 490
}

491
static void zpci_map_resources(struct pci_dev *pdev)
J
Jan Glauber 已提交
492 493 494 495 496 497 498 499
{
	resource_size_t len;
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		len = pci_resource_len(pdev, i);
		if (!len)
			continue;
500 501
		pdev->resource[i].start =
			(resource_size_t __force) pci_iomap(pdev, i, 0);
J
Jan Glauber 已提交
502 503
		pdev->resource[i].end = pdev->resource[i].start + len - 1;
	}
504 505
}

506
static void zpci_unmap_resources(struct pci_dev *pdev)
507 508 509 510 511 512 513 514
{
	resource_size_t len;
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		len = pci_resource_len(pdev, i);
		if (!len)
			continue;
515 516
		pci_iounmap(pdev, (void __iomem __force *)
			    pdev->resource[i].start);
517 518
	}
}
J
Jan Glauber 已提交
519

520 521
static int __init zpci_irq_init(void)
{
522
	int rc;
523

524 525
	rc = register_adapter_interrupt(&zpci_airq);
	if (rc)
526
		goto out;
527 528
	/* Set summary to 1 to be called every time for the ISC. */
	*zpci_airq.lsi_ptr = 1;
529

530 531 532 533
	rc = -ENOMEM;
	zpci_aisb_iv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC);
	if (!zpci_aisb_iv)
		goto out_airq;
534

535
	zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
536 537
	return 0;

538 539 540
out_airq:
	unregister_adapter_interrupt(&zpci_airq);
out:
541 542 543 544 545
	return rc;
}

static void zpci_irq_exit(void)
{
546
	airq_iv_release(zpci_aisb_iv);
547
	unregister_adapter_interrupt(&zpci_airq);
548 549
}

J
Jan Glauber 已提交
550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572
static int zpci_alloc_iomap(struct zpci_dev *zdev)
{
	int entry;

	spin_lock(&zpci_iomap_lock);
	entry = find_first_zero_bit(zpci_iomap, ZPCI_IOMAP_MAX_ENTRIES);
	if (entry == ZPCI_IOMAP_MAX_ENTRIES) {
		spin_unlock(&zpci_iomap_lock);
		return -ENOSPC;
	}
	set_bit(entry, zpci_iomap);
	spin_unlock(&zpci_iomap_lock);
	return entry;
}

static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
{
	spin_lock(&zpci_iomap_lock);
	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
	clear_bit(entry, zpci_iomap);
	spin_unlock(&zpci_iomap_lock);
}

573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,
				    unsigned long size, unsigned long flags)
{
	struct resource *r;

	r = kzalloc(sizeof(*r), GFP_KERNEL);
	if (!r)
		return NULL;

	r->start = start;
	r->end = r->start + size - 1;
	r->flags = flags;
	r->name = zdev->res_name;

	if (request_resource(&iomem_resource, r)) {
		kfree(r);
		return NULL;
	}
	return r;
}

static int zpci_setup_bus_resources(struct zpci_dev *zdev,
				    struct list_head *resources)
{
	unsigned long addr, size, flags;
	struct resource *res;
	int i, entry;

	snprintf(zdev->res_name, sizeof(zdev->res_name),
		 "PCI Bus %04x:%02x", zdev->domain, ZPCI_BUS_NR);

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		if (!zdev->bars[i].size)
			continue;
		entry = zpci_alloc_iomap(zdev);
		if (entry < 0)
			return entry;
		zdev->bars[i].map_idx = entry;

		/* only MMIO is supported */
		flags = IORESOURCE_MEM;
		if (zdev->bars[i].val & 8)
			flags |= IORESOURCE_PREFETCH;
		if (zdev->bars[i].val & 4)
			flags |= IORESOURCE_MEM_64;

		addr = ZPCI_IOMAP_ADDR_BASE + ((u64) entry << 48);

		size = 1UL << zdev->bars[i].size;

		res = __alloc_res(zdev, addr, size, flags);
		if (!res) {
			zpci_free_iomap(zdev, entry);
			return -ENOMEM;
		}
		zdev->bars[i].res = res;
		pci_add_resource(resources, res);
	}

	return 0;
}

static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)
{
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		if (!zdev->bars[i].size)
			continue;

		zpci_free_iomap(zdev, zdev->bars[i].map_idx);
		release_resource(zdev->bars[i].res);
		kfree(zdev->bars[i].res);
	}
}

649 650 651
int pcibios_add_device(struct pci_dev *pdev)
{
	struct zpci_dev *zdev = get_zdev(pdev);
S
Sebastian Ott 已提交
652 653 654 655
	struct resource *res;
	int i;

	zdev->pdev = pdev;
656
	pdev->dev.groups = zpci_attr_groups;
657
	zpci_map_resources(pdev);
S
Sebastian Ott 已提交
658 659 660 661 662 663 664 665 666 667 668

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		res = &pdev->resource[i];
		if (res->parent || !res->flags)
			continue;
		pci_claim_resource(pdev, i);
	}

	return 0;
}

669 670 671 672 673
void pcibios_release_device(struct pci_dev *pdev)
{
	zpci_unmap_resources(pdev);
}

S
Sebastian Ott 已提交
674 675 676
int pcibios_enable_device(struct pci_dev *pdev, int mask)
{
	struct zpci_dev *zdev = get_zdev(pdev);
677

S
Sebastian Ott 已提交
678
	zdev->pdev = pdev;
679 680 681
	zpci_debug_init_device(zdev);
	zpci_fmb_enable_device(zdev);

682
	return pci_enable_resources(pdev, mask);
683 684
}

S
Sebastian Ott 已提交
685
void pcibios_disable_device(struct pci_dev *pdev)
686 687 688 689 690 691 692 693
{
	struct zpci_dev *zdev = get_zdev(pdev);

	zpci_fmb_disable_device(zdev);
	zpci_debug_exit_device(zdev);
	zdev->pdev = NULL;
}

694 695 696
#ifdef CONFIG_HIBERNATE_CALLBACKS
static int zpci_restore(struct device *dev)
{
697 698
	struct pci_dev *pdev = to_pci_dev(dev);
	struct zpci_dev *zdev = get_zdev(pdev);
699 700 701 702 703 704 705 706 707
	int ret = 0;

	if (zdev->state != ZPCI_FN_STATE_ONLINE)
		goto out;

	ret = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
	if (ret)
		goto out;

708
	zpci_map_resources(pdev);
709 710 711 712 713 714 715 716 717 718
	zpci_register_ioat(zdev, 0, zdev->start_dma + PAGE_OFFSET,
			   zdev->start_dma + zdev->iommu_size - 1,
			   (u64) zdev->dma_table);

out:
	return ret;
}

static int zpci_freeze(struct device *dev)
{
719 720
	struct pci_dev *pdev = to_pci_dev(dev);
	struct zpci_dev *zdev = get_zdev(pdev);
721 722 723 724 725

	if (zdev->state != ZPCI_FN_STATE_ONLINE)
		return 0;

	zpci_unregister_ioat(zdev, 0);
726
	zpci_unmap_resources(pdev);
727 728 729 730 731 732 733 734 735 736 737
	return clp_disable_fh(zdev);
}

struct dev_pm_ops pcibios_pm_ops = {
	.thaw_noirq = zpci_restore,
	.freeze_noirq = zpci_freeze,
	.restore_noirq = zpci_restore,
	.poweroff_noirq = zpci_freeze,
};
#endif /* CONFIG_HIBERNATE_CALLBACKS */

J
Jan Glauber 已提交
738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757
static int zpci_alloc_domain(struct zpci_dev *zdev)
{
	spin_lock(&zpci_domain_lock);
	zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
	if (zdev->domain == ZPCI_NR_DEVICES) {
		spin_unlock(&zpci_domain_lock);
		return -ENOSPC;
	}
	set_bit(zdev->domain, zpci_domain);
	spin_unlock(&zpci_domain_lock);
	return 0;
}

static void zpci_free_domain(struct zpci_dev *zdev)
{
	spin_lock(&zpci_domain_lock);
	clear_bit(zdev->domain, zpci_domain);
	spin_unlock(&zpci_domain_lock);
}

758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
void pcibios_remove_bus(struct pci_bus *bus)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);

	zpci_exit_slot(zdev);
	zpci_cleanup_bus_resources(zdev);
	zpci_free_domain(zdev);

	spin_lock(&zpci_list_lock);
	list_del(&zdev->entry);
	spin_unlock(&zpci_list_lock);

	kfree(zdev);
}

static int zpci_scan_bus(struct zpci_dev *zdev)
{
	LIST_HEAD(resources);
	int ret;

	ret = zpci_setup_bus_resources(zdev, &resources);
	if (ret)
		return ret;

	zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops,
				      zdev, &resources);
	if (!zdev->bus) {
		zpci_cleanup_bus_resources(zdev);
		return -EIO;
	}
	zdev->bus->max_bus_speed = zdev->max_bus_speed;
789
	pci_bus_add_devices(zdev->bus);
790 791 792
	return 0;
}

J
Jan Glauber 已提交
793 794 795 796 797 798 799
int zpci_enable_device(struct zpci_dev *zdev)
{
	int rc;

	rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
	if (rc)
		goto out;
J
Jan Glauber 已提交
800 801 802 803

	rc = zpci_dma_init_device(zdev);
	if (rc)
		goto out_dma;
S
Sebastian Ott 已提交
804 805

	zdev->state = ZPCI_FN_STATE_ONLINE;
J
Jan Glauber 已提交
806
	return 0;
J
Jan Glauber 已提交
807 808 809

out_dma:
	clp_disable_fh(zdev);
J
Jan Glauber 已提交
810 811 812 813 814
out:
	return rc;
}
EXPORT_SYMBOL_GPL(zpci_enable_device);

815 816 817 818 819 820 821
int zpci_disable_device(struct zpci_dev *zdev)
{
	zpci_dma_exit_device(zdev);
	return clp_disable_fh(zdev);
}
EXPORT_SYMBOL_GPL(zpci_disable_device);

J
Jan Glauber 已提交
822 823 824 825 826 827 828 829
int zpci_create_device(struct zpci_dev *zdev)
{
	int rc;

	rc = zpci_alloc_domain(zdev);
	if (rc)
		goto out;

S
Sebastian Ott 已提交
830 831 832 833 834 835
	if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
		rc = zpci_enable_device(zdev);
		if (rc)
			goto out_free;
	}
	rc = zpci_scan_bus(zdev);
J
Jan Glauber 已提交
836
	if (rc)
S
Sebastian Ott 已提交
837
		goto out_disable;
J
Jan Glauber 已提交
838

839
	spin_lock(&zpci_list_lock);
J
Jan Glauber 已提交
840
	list_add_tail(&zdev->entry, &zpci_list);
841
	spin_unlock(&zpci_list_lock);
J
Jan Glauber 已提交
842

843 844
	zpci_init_slot(zdev);

J
Jan Glauber 已提交
845 846
	return 0;

S
Sebastian Ott 已提交
847 848 849 850
out_disable:
	if (zdev->state == ZPCI_FN_STATE_ONLINE)
		zpci_disable_device(zdev);
out_free:
J
Jan Glauber 已提交
851 852 853 854 855 856 857
	zpci_free_domain(zdev);
out:
	return rc;
}

void zpci_stop_device(struct zpci_dev *zdev)
{
J
Jan Glauber 已提交
858
	zpci_dma_exit_device(zdev);
J
Jan Glauber 已提交
859 860 861 862 863 864 865 866 867 868 869 870 871 872
	/*
	 * Note: SCLP disables fh via set-pci-fn so don't
	 * do that here.
	 */
}
EXPORT_SYMBOL_GPL(zpci_stop_device);

static inline int barsize(u8 size)
{
	return (size) ? (1 << size) >> 10 : 0;
}

static int zpci_mem_init(void)
{
873 874 875
	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
				16, 0, NULL);
	if (!zdev_fmb_cache)
876
		goto error_zdev;
877

J
Jan Glauber 已提交
878 879 880 881
	/* TODO: use realloc */
	zpci_iomap_start = kzalloc(ZPCI_IOMAP_MAX_ENTRIES * sizeof(*zpci_iomap_start),
				   GFP_KERNEL);
	if (!zpci_iomap_start)
882
		goto error_iomap;
J
Jan Glauber 已提交
883 884
	return 0;

885
error_iomap:
886
	kmem_cache_destroy(zdev_fmb_cache);
J
Jan Glauber 已提交
887 888 889 890 891 892 893
error_zdev:
	return -ENOMEM;
}

static void zpci_mem_exit(void)
{
	kfree(zpci_iomap_start);
894
	kmem_cache_destroy(zdev_fmb_cache);
J
Jan Glauber 已提交
895 896
}

S
Sebastian Ott 已提交
897
static unsigned int s390_pci_probe = 1;
898
static unsigned int s390_pci_initialized;
J
Jan Glauber 已提交
899 900 901

char * __init pcibios_setup(char *str)
{
S
Sebastian Ott 已提交
902 903
	if (!strcmp(str, "off")) {
		s390_pci_probe = 0;
J
Jan Glauber 已提交
904 905 906 907 908
		return NULL;
	}
	return str;
}

909 910 911 912 913
bool zpci_is_enabled(void)
{
	return s390_pci_initialized;
}

J
Jan Glauber 已提交
914 915 916 917
static int __init pci_base_init(void)
{
	int rc;

918
	if (!s390_pci_probe)
J
Jan Glauber 已提交
919 920
		return 0;

921
	if (!test_facility(69) || !test_facility(71) || !test_facility(72))
J
Jan Glauber 已提交
922 923
		return 0;

924 925
	rc = zpci_debug_init();
	if (rc)
926
		goto out;
927

J
Jan Glauber 已提交
928 929 930 931
	rc = zpci_mem_init();
	if (rc)
		goto out_mem;

932 933 934 935
	rc = zpci_irq_init();
	if (rc)
		goto out_irq;

J
Jan Glauber 已提交
936 937 938 939
	rc = zpci_dma_init();
	if (rc)
		goto out_dma;

S
Sebastian Ott 已提交
940
	rc = clp_scan_pci_devices();
J
Jan Glauber 已提交
941 942 943
	if (rc)
		goto out_find;

944
	s390_pci_initialized = 1;
J
Jan Glauber 已提交
945 946
	return 0;

J
Jan Glauber 已提交
947
out_find:
J
Jan Glauber 已提交
948 949
	zpci_dma_exit();
out_dma:
950 951
	zpci_irq_exit();
out_irq:
J
Jan Glauber 已提交
952 953
	zpci_mem_exit();
out_mem:
954
	zpci_debug_exit();
955
out:
J
Jan Glauber 已提交
956 957
	return rc;
}
958
subsys_initcall_sync(pci_base_init);
959 960 961

void zpci_rescan(void)
{
962 963
	if (zpci_is_enabled())
		clp_rescan_pci_devices_simple();
964
}