pci.c 20.2 KB
Newer Older
J
Jan Glauber 已提交
1 2 3 4 5 6 7 8
/*
 * Copyright IBM Corp. 2012
 *
 * Author(s):
 *   Jan Glauber <jang@linux.vnet.ibm.com>
 *
 * The System z PCI code is a rewrite from a prototype by
 * the following people (Kudoz!):
9 10 11 12 13 14 15
 *   Alexander Schmidt
 *   Christoph Raisch
 *   Hannes Hering
 *   Hoang-Nam Nguyen
 *   Jan-Bernd Themann
 *   Stefan Roscher
 *   Thomas Klein
J
Jan Glauber 已提交
16 17
 */

G
Gerald Schaefer 已提交
18 19
#define KMSG_COMPONENT "zpci"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
J
Jan Glauber 已提交
20 21 22 23 24 25

#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/export.h>
#include <linux/delay.h>
26 27
#include <linux/irq.h>
#include <linux/kernel_stat.h>
J
Jan Glauber 已提交
28 29 30 31
#include <linux/seq_file.h>
#include <linux/pci.h>
#include <linux/msi.h>

32 33
#include <asm/isc.h>
#include <asm/airq.h>
J
Jan Glauber 已提交
34 35
#include <asm/facility.h>
#include <asm/pci_insn.h>
J
Jan Glauber 已提交
36
#include <asm/pci_clp.h>
J
Jan Glauber 已提交
37
#include <asm/pci_dma.h>
J
Jan Glauber 已提交
38 39 40

#define DEBUG				/* enable pr_debug */

41 42 43
#define	SIC_IRQ_MODE_ALL		0
#define	SIC_IRQ_MODE_SINGLE		1

J
Jan Glauber 已提交
44 45 46 47
#define ZPCI_NR_DMA_SPACES		1
#define ZPCI_NR_DEVICES			CONFIG_PCI_NR_FUNCTIONS

/* list of all detected zpci devices */
48
static LIST_HEAD(zpci_list);
49
static DEFINE_SPINLOCK(zpci_list_lock);
J
Jan Glauber 已提交
50

51 52
static struct irq_chip zpci_irq_chip = {
	.name = "zPCI",
53 54
	.irq_unmask = pci_msi_unmask_irq,
	.irq_mask = pci_msi_mask_irq,
55 56
};

57 58
static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES);
static DEFINE_SPINLOCK(zpci_domain_lock);
59

60
static struct airq_iv *zpci_aisb_iv;
61
static struct airq_iv *zpci_aibv[ZPCI_NR_DEVICES];
62

63 64 65 66 67 68 69
/* Adapter interrupt definitions */
static void zpci_irq_handler(struct airq_struct *airq);

static struct airq_struct zpci_airq = {
	.handler = zpci_irq_handler,
	.isc = PCI_ISC,
};
70

S
Sebastian Ott 已提交
71 72 73 74
#define ZPCI_IOMAP_ENTRIES						\
	min(((unsigned long) CONFIG_PCI_NR_FUNCTIONS * PCI_BAR_COUNT),	\
	    ZPCI_IOMAP_MAX_ENTRIES)

J
Jan Glauber 已提交
75
static DEFINE_SPINLOCK(zpci_iomap_lock);
S
Sebastian Ott 已提交
76
static unsigned long *zpci_iomap_bitmap;
J
Jan Glauber 已提交
77 78 79
struct zpci_iomap_entry *zpci_iomap_start;
EXPORT_SYMBOL_GPL(zpci_iomap_start);

80 81
static struct kmem_cache *zdev_fmb_cache;

J
Jan Glauber 已提交
82 83 84 85
struct zpci_dev *get_zdev_by_fid(u32 fid)
{
	struct zpci_dev *tmp, *zdev = NULL;

86
	spin_lock(&zpci_list_lock);
J
Jan Glauber 已提交
87 88 89 90 91 92
	list_for_each_entry(tmp, &zpci_list, entry) {
		if (tmp->fid == fid) {
			zdev = tmp;
			break;
		}
	}
93
	spin_unlock(&zpci_list_lock);
J
Jan Glauber 已提交
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
	return zdev;
}

static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus)
{
	return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL;
}

int pci_domain_nr(struct pci_bus *bus)
{
	return ((struct zpci_dev *) bus->sysdata)->domain;
}
EXPORT_SYMBOL_GPL(pci_domain_nr);

int pci_proc_domain(struct pci_bus *bus)
{
	return pci_domain_nr(bus);
}
EXPORT_SYMBOL_GPL(pci_proc_domain);

114
/* Modify PCI: Register adapter interruptions */
115
static int zpci_set_airq(struct zpci_dev *zdev)
116 117
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT);
118
	struct zpci_fib fib = {0};
119

120 121 122 123 124 125 126
	fib.isc = PCI_ISC;
	fib.sum = 1;		/* enable summary notifications */
	fib.noi = airq_iv_end(zdev->aibv);
	fib.aibv = (unsigned long) zdev->aibv->vector;
	fib.aibvo = 0;		/* each zdev has its own interrupt vector */
	fib.aisb = (unsigned long) zpci_aisb_iv->vector + (zdev->aisb/64)*8;
	fib.aisbo = zdev->aisb & 63;
127

128
	return zpci_mod_fc(req, &fib);
129 130 131 132 133 134
}

struct mod_pci_args {
	u64 base;
	u64 limit;
	u64 iota;
135
	u64 fmb_addr;
136 137 138 139 140
};

static int mod_pci(struct zpci_dev *zdev, int fn, u8 dmaas, struct mod_pci_args *args)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, fn);
141 142 143 144 145 146 147 148
	struct zpci_fib fib = {0};

	fib.pba = args->base;
	fib.pal = args->limit;
	fib.iota = args->iota;
	fib.fmb_addr = args->fmb_addr;

	return zpci_mod_fc(req, &fib);
149 150
}

J
Jan Glauber 已提交
151 152 153 154
/* Modify PCI: Register I/O address translation parameters */
int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
		       u64 base, u64 limit, u64 iota)
{
155
	struct mod_pci_args args = { base, limit, iota, 0 };
J
Jan Glauber 已提交
156 157 158 159 160 161 162 163 164

	WARN_ON_ONCE(iota & 0x3fff);
	args.iota |= ZPCI_IOTA_RTTO_FLAG;
	return mod_pci(zdev, ZPCI_MOD_FC_REG_IOAT, dmaas, &args);
}

/* Modify PCI: Unregister I/O address translation parameters */
int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
{
165
	struct mod_pci_args args = { 0, 0, 0, 0 };
J
Jan Glauber 已提交
166 167 168 169

	return mod_pci(zdev, ZPCI_MOD_FC_DEREG_IOAT, dmaas, &args);
}

170
/* Modify PCI: Unregister adapter interruptions */
171
static int zpci_clear_airq(struct zpci_dev *zdev)
172
{
173
	struct mod_pci_args args = { 0, 0, 0, 0 };
174 175 176 177

	return mod_pci(zdev, ZPCI_MOD_FC_DEREG_INT, 0, &args);
}

178 179 180 181 182 183 184 185
/* Modify PCI: Set PCI function measurement parameters */
int zpci_fmb_enable_device(struct zpci_dev *zdev)
{
	struct mod_pci_args args = { 0, 0, 0, 0 };

	if (zdev->fmb)
		return -EINVAL;

186
	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
187 188 189 190
	if (!zdev->fmb)
		return -ENOMEM;
	WARN_ON((u64) zdev->fmb & 0xf);

191 192 193 194 195
	/* reset software counters */
	atomic64_set(&zdev->allocated_pages, 0);
	atomic64_set(&zdev->mapped_pages, 0);
	atomic64_set(&zdev->unmapped_pages, 0);

196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
	args.fmb_addr = virt_to_phys(zdev->fmb);
	return mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);
}

/* Modify PCI: Disable PCI function measurement */
int zpci_fmb_disable_device(struct zpci_dev *zdev)
{
	struct mod_pci_args args = { 0, 0, 0, 0 };
	int rc;

	if (!zdev->fmb)
		return -EINVAL;

	/* Function measurement is disabled if fmb address is zero */
	rc = mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);

	kmem_cache_free(zdev_fmb_cache, zdev->fmb);
	zdev->fmb = NULL;
	return rc;
}

J
Jan Glauber 已提交
217 218 219 220 221 222 223 224
#define ZPCI_PCIAS_CFGSPC	15

static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
	u64 data;
	int rc;

225
	rc = zpci_load(&data, req, offset);
226 227 228
	if (!rc) {
		data = data << ((8 - len) * 8);
		data = le64_to_cpu(data);
J
Jan Glauber 已提交
229
		*val = (u32) data;
230
	} else
J
Jan Glauber 已提交
231 232 233 234 235 236 237 238 239 240 241 242
		*val = 0xffffffff;
	return rc;
}

static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
	u64 data = val;
	int rc;

	data = cpu_to_le64(data);
	data = data >> ((8 - len) * 8);
243
	rc = zpci_store(data, req, offset);
J
Jan Glauber 已提交
244 245 246
	return rc;
}

247
void pcibios_fixup_bus(struct pci_bus *bus)
J
Jan Glauber 已提交
248 249 250 251 252 253 254 255 256 257
{
}

resource_size_t pcibios_align_resource(void *data, const struct resource *res,
				       resource_size_t size,
				       resource_size_t align)
{
	return 0;
}

258 259 260 261 262 263
/* combine single writes by using store-block insn */
void __iowrite64_copy(void __iomem *to, const void *from, size_t count)
{
       zpci_memcpy_toio(to, from, count);
}

J
Jan Glauber 已提交
264
/* Create a virtual mapping cookie for a PCI BAR */
M
Michael S. Tsirkin 已提交
265 266 267 268
void __iomem *pci_iomap_range(struct pci_dev *pdev,
			      int bar,
			      unsigned long offset,
			      unsigned long max)
J
Jan Glauber 已提交
269
{
S
Sebastian Ott 已提交
270
	struct zpci_dev *zdev =	to_zpci(pdev);
J
Jan Glauber 已提交
271 272
	int idx;

S
Sebastian Ott 已提交
273
	if (!pci_resource_len(pdev, bar))
J
Jan Glauber 已提交
274 275 276 277
		return NULL;

	idx = zdev->bars[bar].map_idx;
	spin_lock(&zpci_iomap_lock);
M
Michael S. Tsirkin 已提交
278
	/* Detect overrun */
279 280 281
	WARN_ON(!++zpci_iomap_start[idx].count);
	zpci_iomap_start[idx].fh = zdev->fh;
	zpci_iomap_start[idx].bar = bar;
J
Jan Glauber 已提交
282 283
	spin_unlock(&zpci_iomap_lock);

284
	return (void __iomem *) ZPCI_ADDR(idx) + offset;
J
Jan Glauber 已提交
285
}
286
EXPORT_SYMBOL(pci_iomap_range);
M
Michael S. Tsirkin 已提交
287 288 289 290 291 292

void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen)
{
	return pci_iomap_range(dev, bar, 0, maxlen);
}
EXPORT_SYMBOL(pci_iomap);
J
Jan Glauber 已提交
293 294 295

void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
{
296
	unsigned int idx = ZPCI_IDX(addr);
J
Jan Glauber 已提交
297 298

	spin_lock(&zpci_iomap_lock);
M
Michael S. Tsirkin 已提交
299
	/* Detect underrun */
300
	WARN_ON(!zpci_iomap_start[idx].count);
M
Michael S. Tsirkin 已提交
301 302 303 304
	if (!--zpci_iomap_start[idx].count) {
		zpci_iomap_start[idx].fh = 0;
		zpci_iomap_start[idx].bar = 0;
	}
J
Jan Glauber 已提交
305 306
	spin_unlock(&zpci_iomap_lock);
}
307
EXPORT_SYMBOL(pci_iounmap);
J
Jan Glauber 已提交
308 309 310 311 312

static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
		    int size, u32 *val)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);
313
	int ret;
J
Jan Glauber 已提交
314 315

	if (!zdev || devfn != ZPCI_DEVFN)
316 317 318 319 320
		ret = -ENODEV;
	else
		ret = zpci_cfg_load(zdev, where, val, size);

	return ret;
J
Jan Glauber 已提交
321 322 323 324 325 326
}

static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
		     int size, u32 val)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);
327
	int ret;
J
Jan Glauber 已提交
328 329

	if (!zdev || devfn != ZPCI_DEVFN)
330 331 332 333 334
		ret = -ENODEV;
	else
		ret = zpci_cfg_store(zdev, where, val, size);

	return ret;
J
Jan Glauber 已提交
335 336 337 338 339 340 341
}

static struct pci_ops pci_root_ops = {
	.read = pci_read,
	.write = pci_write,
};

342
static void zpci_irq_handler(struct airq_struct *airq)
343
{
344
	unsigned long si, ai;
345
	struct airq_iv *aibv;
346
	int irqs_on = 0;
347

348
	inc_irq_stat(IRQIO_PCI);
349 350 351 352 353 354 355 356 357 358 359 360
	for (si = 0;;) {
		/* Scan adapter summary indicator bit vector */
		si = airq_iv_scan(zpci_aisb_iv, si, airq_iv_end(zpci_aisb_iv));
		if (si == -1UL) {
			if (irqs_on++)
				/* End of second scan with interrupts on. */
				break;
			/* First scan complete, reenable interrupts. */
			zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
			si = 0;
			continue;
		}
361

362
		/* Scan the adapter interrupt vector for this device. */
363
		aibv = zpci_aibv[si];
364
		for (ai = 0;;) {
365
			ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv));
366 367
			if (ai == -1UL)
				break;
368
			inc_irq_stat(IRQIO_MSI);
369 370 371
			airq_iv_lock(aibv, ai);
			generic_handle_irq(airq_iv_get_data(aibv, ai));
			airq_iv_unlock(aibv, ai);
372 373
		}
	}
374
}
375

376
int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
377
{
S
Sebastian Ott 已提交
378
	struct zpci_dev *zdev = to_zpci(pdev);
379
	unsigned int hwirq, msi_vecs;
380
	unsigned long aisb;
381
	struct msi_desc *msi;
382
	struct msi_msg msg;
383
	int rc, irq;
384

385 386
	if (type == PCI_CAP_ID_MSI && nvec > 1)
		return 1;
387
	msi_vecs = min_t(unsigned int, nvec, zdev->max_msi);
388

389 390 391 392 393
	/* Allocate adapter summary indicator bit */
	rc = -EIO;
	aisb = airq_iv_alloc_bit(zpci_aisb_iv);
	if (aisb == -1UL)
		goto out;
394 395
	zdev->aisb = aisb;

396 397
	/* Create adapter interrupt vector */
	rc = -ENOMEM;
398
	zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK);
399 400
	if (!zdev->aibv)
		goto out_si;
401

402
	/* Wire up shortcut pointer */
403
	zpci_aibv[aisb] = zdev->aibv;
404

405 406
	/* Request MSI interrupts */
	hwirq = 0;
407
	for_each_pci_msi_entry(msi, pdev) {
408 409
		rc = -EIO;
		irq = irq_alloc_desc(0);	/* Alloc irq on node 0 */
410
		if (irq < 0)
411 412
			goto out_msi;
		rc = irq_set_msi_desc(irq, msi);
413
		if (rc)
414
			goto out_msi;
415 416 417 418 419
		irq_set_chip_and_handler(irq, &zpci_irq_chip,
					 handle_simple_irq);
		msg.data = hwirq;
		msg.address_lo = zdev->msi_addr & 0xffffffff;
		msg.address_hi = zdev->msi_addr >> 32;
420
		pci_write_msi_msg(irq, &msg);
421 422
		airq_iv_set_data(zdev->aibv, hwirq, irq);
		hwirq++;
423 424
	}

425 426 427 428 429 430 431 432
	/* Enable adapter interrupts */
	rc = zpci_set_airq(zdev);
	if (rc)
		goto out_msi;

	return (msi_vecs == nvec) ? 0 : msi_vecs;

out_msi:
433
	for_each_pci_msi_entry(msi, pdev) {
434
		if (hwirq-- == 0)
435
			break;
436 437 438 439 440 441
		irq_set_msi_desc(msi->irq, NULL);
		irq_free_desc(msi->irq);
		msi->msg.address_lo = 0;
		msi->msg.address_hi = 0;
		msi->msg.data = 0;
		msi->irq = 0;
442
	}
443
	zpci_aibv[aisb] = NULL;
444 445 446 447 448
	airq_iv_release(zdev->aibv);
out_si:
	airq_iv_free_bit(zpci_aisb_iv, aisb);
out:
	return rc;
449 450
}

451
void arch_teardown_msi_irqs(struct pci_dev *pdev)
452
{
S
Sebastian Ott 已提交
453
	struct zpci_dev *zdev = to_zpci(pdev);
454
	struct msi_desc *msi;
455 456 457 458
	int rc;

	/* Disable adapter interrupts */
	rc = zpci_clear_airq(zdev);
S
Sebastian Ott 已提交
459
	if (rc)
460 461
		return;

462
	/* Release MSI interrupts */
463
	for_each_pci_msi_entry(msi, pdev) {
464
		if (msi->msi_attrib.is_msix)
465
			__pci_msix_desc_mask_irq(msi, 1);
466
		else
467
			__pci_msi_desc_mask_irq(msi, 1, 1);
468 469 470 471 472 473 474
		irq_set_msi_desc(msi->irq, NULL);
		irq_free_desc(msi->irq);
		msi->msg.address_lo = 0;
		msi->msg.address_hi = 0;
		msi->msg.data = 0;
		msi->irq = 0;
	}
475

476
	zpci_aibv[zdev->aisb] = NULL;
477 478
	airq_iv_release(zdev->aibv);
	airq_iv_free_bit(zpci_aisb_iv, zdev->aisb);
479 480
}

481
static void zpci_map_resources(struct pci_dev *pdev)
J
Jan Glauber 已提交
482 483 484 485 486 487 488 489
{
	resource_size_t len;
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		len = pci_resource_len(pdev, i);
		if (!len)
			continue;
490 491
		pdev->resource[i].start =
			(resource_size_t __force) pci_iomap(pdev, i, 0);
J
Jan Glauber 已提交
492 493
		pdev->resource[i].end = pdev->resource[i].start + len - 1;
	}
494 495
}

496
static void zpci_unmap_resources(struct pci_dev *pdev)
497 498 499 500 501 502 503 504
{
	resource_size_t len;
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		len = pci_resource_len(pdev, i);
		if (!len)
			continue;
505 506
		pci_iounmap(pdev, (void __iomem __force *)
			    pdev->resource[i].start);
507 508
	}
}
J
Jan Glauber 已提交
509

510 511
static int __init zpci_irq_init(void)
{
512
	int rc;
513

514 515
	rc = register_adapter_interrupt(&zpci_airq);
	if (rc)
516
		goto out;
517 518
	/* Set summary to 1 to be called every time for the ISC. */
	*zpci_airq.lsi_ptr = 1;
519

520 521 522 523
	rc = -ENOMEM;
	zpci_aisb_iv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC);
	if (!zpci_aisb_iv)
		goto out_airq;
524

525
	zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
526 527
	return 0;

528 529 530
out_airq:
	unregister_adapter_interrupt(&zpci_airq);
out:
531 532 533 534 535
	return rc;
}

static void zpci_irq_exit(void)
{
536
	airq_iv_release(zpci_aisb_iv);
537
	unregister_adapter_interrupt(&zpci_airq);
538 539
}

J
Jan Glauber 已提交
540 541
static int zpci_alloc_iomap(struct zpci_dev *zdev)
{
S
Sebastian Ott 已提交
542
	unsigned long entry;
J
Jan Glauber 已提交
543 544

	spin_lock(&zpci_iomap_lock);
S
Sebastian Ott 已提交
545 546
	entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES);
	if (entry == ZPCI_IOMAP_ENTRIES) {
J
Jan Glauber 已提交
547 548 549
		spin_unlock(&zpci_iomap_lock);
		return -ENOSPC;
	}
S
Sebastian Ott 已提交
550
	set_bit(entry, zpci_iomap_bitmap);
J
Jan Glauber 已提交
551 552 553 554 555 556 557 558
	spin_unlock(&zpci_iomap_lock);
	return entry;
}

static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
{
	spin_lock(&zpci_iomap_lock);
	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
S
Sebastian Ott 已提交
559
	clear_bit(entry, zpci_iomap_bitmap);
J
Jan Glauber 已提交
560 561 562
	spin_unlock(&zpci_iomap_lock);
}

563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608
static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,
				    unsigned long size, unsigned long flags)
{
	struct resource *r;

	r = kzalloc(sizeof(*r), GFP_KERNEL);
	if (!r)
		return NULL;

	r->start = start;
	r->end = r->start + size - 1;
	r->flags = flags;
	r->name = zdev->res_name;

	if (request_resource(&iomem_resource, r)) {
		kfree(r);
		return NULL;
	}
	return r;
}

static int zpci_setup_bus_resources(struct zpci_dev *zdev,
				    struct list_head *resources)
{
	unsigned long addr, size, flags;
	struct resource *res;
	int i, entry;

	snprintf(zdev->res_name, sizeof(zdev->res_name),
		 "PCI Bus %04x:%02x", zdev->domain, ZPCI_BUS_NR);

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		if (!zdev->bars[i].size)
			continue;
		entry = zpci_alloc_iomap(zdev);
		if (entry < 0)
			return entry;
		zdev->bars[i].map_idx = entry;

		/* only MMIO is supported */
		flags = IORESOURCE_MEM;
		if (zdev->bars[i].val & 8)
			flags |= IORESOURCE_PREFETCH;
		if (zdev->bars[i].val & 4)
			flags |= IORESOURCE_MEM_64;

609
		addr = ZPCI_ADDR(entry);
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628
		size = 1UL << zdev->bars[i].size;

		res = __alloc_res(zdev, addr, size, flags);
		if (!res) {
			zpci_free_iomap(zdev, entry);
			return -ENOMEM;
		}
		zdev->bars[i].res = res;
		pci_add_resource(resources, res);
	}

	return 0;
}

static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)
{
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
629
		if (!zdev->bars[i].size || !zdev->bars[i].res)
630 631 632 633 634 635 636 637
			continue;

		zpci_free_iomap(zdev, zdev->bars[i].map_idx);
		release_resource(zdev->bars[i].res);
		kfree(zdev->bars[i].res);
	}
}

638 639
int pcibios_add_device(struct pci_dev *pdev)
{
S
Sebastian Ott 已提交
640 641 642
	struct resource *res;
	int i;

643
	pdev->dev.groups = zpci_attr_groups;
644
	zpci_map_resources(pdev);
S
Sebastian Ott 已提交
645 646 647 648 649 650 651 652 653 654 655

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		res = &pdev->resource[i];
		if (res->parent || !res->flags)
			continue;
		pci_claim_resource(pdev, i);
	}

	return 0;
}

656 657 658 659 660
void pcibios_release_device(struct pci_dev *pdev)
{
	zpci_unmap_resources(pdev);
}

S
Sebastian Ott 已提交
661 662
int pcibios_enable_device(struct pci_dev *pdev, int mask)
{
S
Sebastian Ott 已提交
663
	struct zpci_dev *zdev = to_zpci(pdev);
664

665
	zpci_debug_init_device(zdev, dev_name(&pdev->dev));
666 667
	zpci_fmb_enable_device(zdev);

668
	return pci_enable_resources(pdev, mask);
669 670
}

S
Sebastian Ott 已提交
671
void pcibios_disable_device(struct pci_dev *pdev)
672
{
S
Sebastian Ott 已提交
673
	struct zpci_dev *zdev = to_zpci(pdev);
674 675 676 677 678

	zpci_fmb_disable_device(zdev);
	zpci_debug_exit_device(zdev);
}

679 680 681
#ifdef CONFIG_HIBERNATE_CALLBACKS
static int zpci_restore(struct device *dev)
{
682
	struct pci_dev *pdev = to_pci_dev(dev);
S
Sebastian Ott 已提交
683
	struct zpci_dev *zdev = to_zpci(pdev);
684 685 686 687 688 689 690 691 692
	int ret = 0;

	if (zdev->state != ZPCI_FN_STATE_ONLINE)
		goto out;

	ret = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
	if (ret)
		goto out;

693
	zpci_map_resources(pdev);
694
	zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
695 696 697 698 699 700 701 702
			   (u64) zdev->dma_table);

out:
	return ret;
}

static int zpci_freeze(struct device *dev)
{
703
	struct pci_dev *pdev = to_pci_dev(dev);
S
Sebastian Ott 已提交
704
	struct zpci_dev *zdev = to_zpci(pdev);
705 706 707 708 709

	if (zdev->state != ZPCI_FN_STATE_ONLINE)
		return 0;

	zpci_unregister_ioat(zdev, 0);
710
	zpci_unmap_resources(pdev);
711 712 713 714 715 716 717 718 719 720 721
	return clp_disable_fh(zdev);
}

struct dev_pm_ops pcibios_pm_ops = {
	.thaw_noirq = zpci_restore,
	.freeze_noirq = zpci_freeze,
	.restore_noirq = zpci_restore,
	.poweroff_noirq = zpci_freeze,
};
#endif /* CONFIG_HIBERNATE_CALLBACKS */

J
Jan Glauber 已提交
722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741
static int zpci_alloc_domain(struct zpci_dev *zdev)
{
	spin_lock(&zpci_domain_lock);
	zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
	if (zdev->domain == ZPCI_NR_DEVICES) {
		spin_unlock(&zpci_domain_lock);
		return -ENOSPC;
	}
	set_bit(zdev->domain, zpci_domain);
	spin_unlock(&zpci_domain_lock);
	return 0;
}

static void zpci_free_domain(struct zpci_dev *zdev)
{
	spin_lock(&zpci_domain_lock);
	clear_bit(zdev->domain, zpci_domain);
	spin_unlock(&zpci_domain_lock);
}

742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763
void pcibios_remove_bus(struct pci_bus *bus)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);

	zpci_exit_slot(zdev);
	zpci_cleanup_bus_resources(zdev);
	zpci_free_domain(zdev);

	spin_lock(&zpci_list_lock);
	list_del(&zdev->entry);
	spin_unlock(&zpci_list_lock);

	kfree(zdev);
}

static int zpci_scan_bus(struct zpci_dev *zdev)
{
	LIST_HEAD(resources);
	int ret;

	ret = zpci_setup_bus_resources(zdev, &resources);
	if (ret)
764
		goto error;
765 766 767 768

	zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops,
				      zdev, &resources);
	if (!zdev->bus) {
769 770
		ret = -EIO;
		goto error;
771 772
	}
	zdev->bus->max_bus_speed = zdev->max_bus_speed;
773
	pci_bus_add_devices(zdev->bus);
774
	return 0;
775 776 777 778 779

error:
	zpci_cleanup_bus_resources(zdev);
	pci_free_resource_list(&resources);
	return ret;
780 781
}

J
Jan Glauber 已提交
782 783 784 785 786 787 788
int zpci_enable_device(struct zpci_dev *zdev)
{
	int rc;

	rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
	if (rc)
		goto out;
J
Jan Glauber 已提交
789 790 791 792

	rc = zpci_dma_init_device(zdev);
	if (rc)
		goto out_dma;
S
Sebastian Ott 已提交
793 794

	zdev->state = ZPCI_FN_STATE_ONLINE;
J
Jan Glauber 已提交
795
	return 0;
J
Jan Glauber 已提交
796 797 798

out_dma:
	clp_disable_fh(zdev);
J
Jan Glauber 已提交
799 800 801 802 803
out:
	return rc;
}
EXPORT_SYMBOL_GPL(zpci_enable_device);

804 805 806 807 808 809 810
int zpci_disable_device(struct zpci_dev *zdev)
{
	zpci_dma_exit_device(zdev);
	return clp_disable_fh(zdev);
}
EXPORT_SYMBOL_GPL(zpci_disable_device);

J
Jan Glauber 已提交
811 812 813 814 815 816 817 818
int zpci_create_device(struct zpci_dev *zdev)
{
	int rc;

	rc = zpci_alloc_domain(zdev);
	if (rc)
		goto out;

819
	mutex_init(&zdev->lock);
S
Sebastian Ott 已提交
820 821 822 823 824 825
	if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
		rc = zpci_enable_device(zdev);
		if (rc)
			goto out_free;
	}
	rc = zpci_scan_bus(zdev);
J
Jan Glauber 已提交
826
	if (rc)
S
Sebastian Ott 已提交
827
		goto out_disable;
J
Jan Glauber 已提交
828

829
	spin_lock(&zpci_list_lock);
J
Jan Glauber 已提交
830
	list_add_tail(&zdev->entry, &zpci_list);
831
	spin_unlock(&zpci_list_lock);
J
Jan Glauber 已提交
832

833 834
	zpci_init_slot(zdev);

J
Jan Glauber 已提交
835 836
	return 0;

S
Sebastian Ott 已提交
837 838 839 840
out_disable:
	if (zdev->state == ZPCI_FN_STATE_ONLINE)
		zpci_disable_device(zdev);
out_free:
J
Jan Glauber 已提交
841 842 843 844 845 846 847
	zpci_free_domain(zdev);
out:
	return rc;
}

void zpci_stop_device(struct zpci_dev *zdev)
{
J
Jan Glauber 已提交
848
	zpci_dma_exit_device(zdev);
J
Jan Glauber 已提交
849 850 851 852 853 854 855 856 857 858 859 860 861 862
	/*
	 * Note: SCLP disables fh via set-pci-fn so don't
	 * do that here.
	 */
}
EXPORT_SYMBOL_GPL(zpci_stop_device);

static inline int barsize(u8 size)
{
	return (size) ? (1 << size) >> 10 : 0;
}

static int zpci_mem_init(void)
{
863 864 865
	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
				16, 0, NULL);
	if (!zdev_fmb_cache)
S
Sebastian Ott 已提交
866
		goto error_fmb;
867

S
Sebastian Ott 已提交
868 869
	zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES,
				   sizeof(*zpci_iomap_start), GFP_KERNEL);
J
Jan Glauber 已提交
870
	if (!zpci_iomap_start)
871
		goto error_iomap;
J
Jan Glauber 已提交
872

S
Sebastian Ott 已提交
873 874 875 876 877 878 879 880
	zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES),
				    sizeof(*zpci_iomap_bitmap), GFP_KERNEL);
	if (!zpci_iomap_bitmap)
		goto error_iomap_bitmap;

	return 0;
error_iomap_bitmap:
	kfree(zpci_iomap_start);
881
error_iomap:
882
	kmem_cache_destroy(zdev_fmb_cache);
S
Sebastian Ott 已提交
883
error_fmb:
J
Jan Glauber 已提交
884 885 886 887 888
	return -ENOMEM;
}

static void zpci_mem_exit(void)
{
S
Sebastian Ott 已提交
889
	kfree(zpci_iomap_bitmap);
J
Jan Glauber 已提交
890
	kfree(zpci_iomap_start);
891
	kmem_cache_destroy(zdev_fmb_cache);
J
Jan Glauber 已提交
892 893
}

S
Sebastian Ott 已提交
894
static unsigned int s390_pci_probe = 1;
895
static unsigned int s390_pci_initialized;
J
Jan Glauber 已提交
896 897 898

char * __init pcibios_setup(char *str)
{
S
Sebastian Ott 已提交
899 900
	if (!strcmp(str, "off")) {
		s390_pci_probe = 0;
J
Jan Glauber 已提交
901 902 903 904 905
		return NULL;
	}
	return str;
}

906 907 908 909 910
bool zpci_is_enabled(void)
{
	return s390_pci_initialized;
}

J
Jan Glauber 已提交
911 912 913 914
static int __init pci_base_init(void)
{
	int rc;

915
	if (!s390_pci_probe)
J
Jan Glauber 已提交
916 917
		return 0;

918
	if (!test_facility(69) || !test_facility(71) || !test_facility(72))
J
Jan Glauber 已提交
919 920
		return 0;

921 922
	rc = zpci_debug_init();
	if (rc)
923
		goto out;
924

J
Jan Glauber 已提交
925 926 927 928
	rc = zpci_mem_init();
	if (rc)
		goto out_mem;

929 930 931 932
	rc = zpci_irq_init();
	if (rc)
		goto out_irq;

J
Jan Glauber 已提交
933 934 935 936
	rc = zpci_dma_init();
	if (rc)
		goto out_dma;

S
Sebastian Ott 已提交
937
	rc = clp_scan_pci_devices();
J
Jan Glauber 已提交
938 939 940
	if (rc)
		goto out_find;

941
	s390_pci_initialized = 1;
J
Jan Glauber 已提交
942 943
	return 0;

J
Jan Glauber 已提交
944
out_find:
J
Jan Glauber 已提交
945 946
	zpci_dma_exit();
out_dma:
947 948
	zpci_irq_exit();
out_irq:
J
Jan Glauber 已提交
949 950
	zpci_mem_exit();
out_mem:
951
	zpci_debug_exit();
952
out:
J
Jan Glauber 已提交
953 954
	return rc;
}
955
subsys_initcall_sync(pci_base_init);
956 957 958

void zpci_rescan(void)
{
959 960
	if (zpci_is_enabled())
		clp_rescan_pci_devices_simple();
961
}