pci.c 20.3 KB
Newer Older
J
Jan Glauber 已提交
1 2 3 4 5 6 7 8
/*
 * Copyright IBM Corp. 2012
 *
 * Author(s):
 *   Jan Glauber <jang@linux.vnet.ibm.com>
 *
 * The System z PCI code is a rewrite from a prototype by
 * the following people (Kudoz!):
9 10 11 12 13 14 15
 *   Alexander Schmidt
 *   Christoph Raisch
 *   Hannes Hering
 *   Hoang-Nam Nguyen
 *   Jan-Bernd Themann
 *   Stefan Roscher
 *   Thomas Klein
J
Jan Glauber 已提交
16 17 18 19 20 21 22 23 24 25
 */

#define COMPONENT "zPCI"
#define pr_fmt(fmt) COMPONENT ": " fmt

#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/export.h>
#include <linux/delay.h>
26 27
#include <linux/irq.h>
#include <linux/kernel_stat.h>
J
Jan Glauber 已提交
28 29 30 31
#include <linux/seq_file.h>
#include <linux/pci.h>
#include <linux/msi.h>

32 33
#include <asm/isc.h>
#include <asm/airq.h>
J
Jan Glauber 已提交
34 35
#include <asm/facility.h>
#include <asm/pci_insn.h>
J
Jan Glauber 已提交
36
#include <asm/pci_clp.h>
J
Jan Glauber 已提交
37
#include <asm/pci_dma.h>
J
Jan Glauber 已提交
38 39 40

#define DEBUG				/* enable pr_debug */

41 42 43
#define	SIC_IRQ_MODE_ALL		0
#define	SIC_IRQ_MODE_SINGLE		1

J
Jan Glauber 已提交
44 45 46 47
#define ZPCI_NR_DMA_SPACES		1
#define ZPCI_NR_DEVICES			CONFIG_PCI_NR_FUNCTIONS

/* list of all detected zpci devices */
48 49
static LIST_HEAD(zpci_list);
static DEFINE_MUTEX(zpci_list_lock);
J
Jan Glauber 已提交
50

51 52
static void zpci_enable_irq(struct irq_data *data);
static void zpci_disable_irq(struct irq_data *data);
J
Jan Glauber 已提交
53

54 55 56 57
static struct irq_chip zpci_irq_chip = {
	.name = "zPCI",
	.irq_unmask = zpci_enable_irq,
	.irq_mask = zpci_disable_irq,
58 59
};

60 61
static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES);
static DEFINE_SPINLOCK(zpci_domain_lock);
62

63
static struct airq_iv *zpci_aisb_iv;
64
static struct airq_iv *zpci_aibv[ZPCI_NR_DEVICES];
65

66 67 68 69 70 71 72
/* Adapter interrupt definitions */
static void zpci_irq_handler(struct airq_struct *airq);

static struct airq_struct zpci_airq = {
	.handler = zpci_irq_handler,
	.isc = PCI_ISC,
};
73

J
Jan Glauber 已提交
74 75 76 77 78 79
/* I/O Map */
static DEFINE_SPINLOCK(zpci_iomap_lock);
static DECLARE_BITMAP(zpci_iomap, ZPCI_IOMAP_MAX_ENTRIES);
struct zpci_iomap_entry *zpci_iomap_start;
EXPORT_SYMBOL_GPL(zpci_iomap_start);

80 81
static struct kmem_cache *zdev_fmb_cache;

J
Jan Glauber 已提交
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
struct zpci_dev *get_zdev(struct pci_dev *pdev)
{
	return (struct zpci_dev *) pdev->sysdata;
}

struct zpci_dev *get_zdev_by_fid(u32 fid)
{
	struct zpci_dev *tmp, *zdev = NULL;

	mutex_lock(&zpci_list_lock);
	list_for_each_entry(tmp, &zpci_list, entry) {
		if (tmp->fid == fid) {
			zdev = tmp;
			break;
		}
	}
	mutex_unlock(&zpci_list_lock);
	return zdev;
}

bool zpci_fid_present(u32 fid)
{
	return (get_zdev_by_fid(fid) != NULL) ? true : false;
}

static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus)
{
	return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL;
}

int pci_domain_nr(struct pci_bus *bus)
{
	return ((struct zpci_dev *) bus->sysdata)->domain;
}
EXPORT_SYMBOL_GPL(pci_domain_nr);

int pci_proc_domain(struct pci_bus *bus)
{
	return pci_domain_nr(bus);
}
EXPORT_SYMBOL_GPL(pci_proc_domain);

124
/* Modify PCI: Register adapter interruptions */
125
static int zpci_set_airq(struct zpci_dev *zdev)
126 127 128 129 130 131 132 133 134 135 136
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT);
	struct zpci_fib *fib;
	int rc;

	fib = (void *) get_zeroed_page(GFP_KERNEL);
	if (!fib)
		return -ENOMEM;

	fib->isc = PCI_ISC;
	fib->sum = 1;		/* enable summary notifications */
137 138 139 140 141
	fib->noi = airq_iv_end(zdev->aibv);
	fib->aibv = (unsigned long) zdev->aibv->vector;
	fib->aibvo = 0;		/* each zdev has its own interrupt vector */
	fib->aisb = (unsigned long) zpci_aisb_iv->vector + (zdev->aisb/64)*8;
	fib->aisbo = zdev->aisb & 63;
142

143
	rc = zpci_mod_fc(req, fib);
144 145 146 147 148 149 150 151 152 153
	pr_debug("%s mpcifc returned noi: %d\n", __func__, fib->noi);

	free_page((unsigned long) fib);
	return rc;
}

struct mod_pci_args {
	u64 base;
	u64 limit;
	u64 iota;
154
	u64 fmb_addr;
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
};

static int mod_pci(struct zpci_dev *zdev, int fn, u8 dmaas, struct mod_pci_args *args)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, fn);
	struct zpci_fib *fib;
	int rc;

	/* The FIB must be available even if it's not used */
	fib = (void *) get_zeroed_page(GFP_KERNEL);
	if (!fib)
		return -ENOMEM;

	fib->pba = args->base;
	fib->pal = args->limit;
	fib->iota = args->iota;
171
	fib->fmb_addr = args->fmb_addr;
172

173
	rc = zpci_mod_fc(req, fib);
174 175 176 177
	free_page((unsigned long) fib);
	return rc;
}

J
Jan Glauber 已提交
178 179 180 181
/* Modify PCI: Register I/O address translation parameters */
int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
		       u64 base, u64 limit, u64 iota)
{
182
	struct mod_pci_args args = { base, limit, iota, 0 };
J
Jan Glauber 已提交
183 184 185 186 187 188 189 190 191

	WARN_ON_ONCE(iota & 0x3fff);
	args.iota |= ZPCI_IOTA_RTTO_FLAG;
	return mod_pci(zdev, ZPCI_MOD_FC_REG_IOAT, dmaas, &args);
}

/* Modify PCI: Unregister I/O address translation parameters */
int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
{
192
	struct mod_pci_args args = { 0, 0, 0, 0 };
J
Jan Glauber 已提交
193 194 195 196

	return mod_pci(zdev, ZPCI_MOD_FC_DEREG_IOAT, dmaas, &args);
}

197
/* Modify PCI: Unregister adapter interruptions */
198
static int zpci_clear_airq(struct zpci_dev *zdev)
199
{
200
	struct mod_pci_args args = { 0, 0, 0, 0 };
201 202 203 204

	return mod_pci(zdev, ZPCI_MOD_FC_DEREG_INT, 0, &args);
}

205 206 207 208 209 210 211 212
/* Modify PCI: Set PCI function measurement parameters */
int zpci_fmb_enable_device(struct zpci_dev *zdev)
{
	struct mod_pci_args args = { 0, 0, 0, 0 };

	if (zdev->fmb)
		return -EINVAL;

213
	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
	if (!zdev->fmb)
		return -ENOMEM;
	WARN_ON((u64) zdev->fmb & 0xf);

	args.fmb_addr = virt_to_phys(zdev->fmb);
	return mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);
}

/* Modify PCI: Disable PCI function measurement */
int zpci_fmb_disable_device(struct zpci_dev *zdev)
{
	struct mod_pci_args args = { 0, 0, 0, 0 };
	int rc;

	if (!zdev->fmb)
		return -EINVAL;

	/* Function measurement is disabled if fmb address is zero */
	rc = mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);

	kmem_cache_free(zdev_fmb_cache, zdev->fmb);
	zdev->fmb = NULL;
	return rc;
}

J
Jan Glauber 已提交
239 240 241 242 243 244 245 246
#define ZPCI_PCIAS_CFGSPC	15

static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
	u64 data;
	int rc;

247
	rc = zpci_load(&data, req, offset);
248 249 250
	if (!rc) {
		data = data << ((8 - len) * 8);
		data = le64_to_cpu(data);
J
Jan Glauber 已提交
251
		*val = (u32) data;
252
	} else
J
Jan Glauber 已提交
253 254 255 256 257 258 259 260 261 262 263 264
		*val = 0xffffffff;
	return rc;
}

static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
{
	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
	u64 data = val;
	int rc;

	data = cpu_to_le64(data);
	data = data >> ((8 - len) * 8);
265
	rc = zpci_store(data, req, offset);
J
Jan Glauber 已提交
266 267 268
	return rc;
}

269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
static int zpci_msi_set_mask_bits(struct msi_desc *msi, u32 mask, u32 flag)
{
	int offset, pos;
	u32 mask_bits;

	if (msi->msi_attrib.is_msix) {
		offset = msi->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
			PCI_MSIX_ENTRY_VECTOR_CTRL;
		msi->masked = readl(msi->mask_base + offset);
		writel(flag, msi->mask_base + offset);
	} else if (msi->msi_attrib.maskbit) {
		pos = (long) msi->mask_base;
		pci_read_config_dword(msi->dev, pos, &mask_bits);
		mask_bits &= ~(mask);
		mask_bits |= flag & mask;
		pci_write_config_dword(msi->dev, pos, mask_bits);
	} else
		return 0;

	msi->msi_attrib.maskbit = !!flag;
	return 1;
}

static void zpci_enable_irq(struct irq_data *data)
293
{
294
	struct msi_desc *msi = irq_get_msi_desc(data->irq);
295 296 297 298

	zpci_msi_set_mask_bits(msi, 1, 0);
}

299
static void zpci_disable_irq(struct irq_data *data)
300
{
301
	struct msi_desc *msi = irq_get_msi_desc(data->irq);
302 303 304 305

	zpci_msi_set_mask_bits(msi, 1, 1);
}

306
void pcibios_fixup_bus(struct pci_bus *bus)
J
Jan Glauber 已提交
307 308 309 310 311 312 313 314 315 316
{
}

resource_size_t pcibios_align_resource(void *data, const struct resource *res,
				       resource_size_t size,
				       resource_size_t align)
{
	return 0;
}

317 318 319 320 321 322
/* combine single writes by using store-block insn */
void __iowrite64_copy(void __iomem *to, const void *from, size_t count)
{
       zpci_memcpy_toio(to, from, count);
}

J
Jan Glauber 已提交
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
/* Create a virtual mapping cookie for a PCI BAR */
void __iomem *pci_iomap(struct pci_dev *pdev, int bar, unsigned long max)
{
	struct zpci_dev *zdev =	get_zdev(pdev);
	u64 addr;
	int idx;

	if ((bar & 7) != bar)
		return NULL;

	idx = zdev->bars[bar].map_idx;
	spin_lock(&zpci_iomap_lock);
	zpci_iomap_start[idx].fh = zdev->fh;
	zpci_iomap_start[idx].bar = bar;
	spin_unlock(&zpci_iomap_lock);

	addr = ZPCI_IOMAP_ADDR_BASE | ((u64) idx << 48);
	return (void __iomem *) addr;
}
EXPORT_SYMBOL_GPL(pci_iomap);

void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
{
	unsigned int idx;

	idx = (((__force u64) addr) & ~ZPCI_IOMAP_ADDR_BASE) >> 48;
	spin_lock(&zpci_iomap_lock);
	zpci_iomap_start[idx].fh = 0;
	zpci_iomap_start[idx].bar = 0;
	spin_unlock(&zpci_iomap_lock);
}
EXPORT_SYMBOL_GPL(pci_iounmap);

static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
		    int size, u32 *val)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);
360
	int ret;
J
Jan Glauber 已提交
361 362

	if (!zdev || devfn != ZPCI_DEVFN)
363 364 365 366 367
		ret = -ENODEV;
	else
		ret = zpci_cfg_load(zdev, where, val, size);

	return ret;
J
Jan Glauber 已提交
368 369 370 371 372 373
}

static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
		     int size, u32 val)
{
	struct zpci_dev *zdev = get_zdev_by_bus(bus);
374
	int ret;
J
Jan Glauber 已提交
375 376

	if (!zdev || devfn != ZPCI_DEVFN)
377 378 379 380 381
		ret = -ENODEV;
	else
		ret = zpci_cfg_store(zdev, where, val, size);

	return ret;
J
Jan Glauber 已提交
382 383 384 385 386 387 388
}

static struct pci_ops pci_root_ops = {
	.read = pci_read,
	.write = pci_write,
};

389
static void zpci_irq_handler(struct airq_struct *airq)
390
{
391
	unsigned long si, ai;
392
	struct airq_iv *aibv;
393
	int irqs_on = 0;
394

395
	inc_irq_stat(IRQIO_PCI);
396 397 398 399 400 401 402 403 404 405 406 407
	for (si = 0;;) {
		/* Scan adapter summary indicator bit vector */
		si = airq_iv_scan(zpci_aisb_iv, si, airq_iv_end(zpci_aisb_iv));
		if (si == -1UL) {
			if (irqs_on++)
				/* End of second scan with interrupts on. */
				break;
			/* First scan complete, reenable interrupts. */
			zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
			si = 0;
			continue;
		}
408

409
		/* Scan the adapter interrupt vector for this device. */
410
		aibv = zpci_aibv[si];
411
		for (ai = 0;;) {
412
			ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv));
413 414
			if (ai == -1UL)
				break;
415
			inc_irq_stat(IRQIO_MSI);
416 417 418
			airq_iv_lock(aibv, ai);
			generic_handle_irq(airq_iv_get_data(aibv, ai));
			airq_iv_unlock(aibv, ai);
419 420
		}
	}
421
}
422

423
int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
424 425
{
	struct zpci_dev *zdev = get_zdev(pdev);
426
	unsigned int hwirq, irq, msi_vecs;
427
	unsigned long aisb;
428
	struct msi_desc *msi;
429
	struct msi_msg msg;
430 431
	int rc;

432 433 434 435
	pr_debug("%s: requesting %d MSI-X interrupts...", __func__, nvec);
	if (type != PCI_CAP_ID_MSIX && type != PCI_CAP_ID_MSI)
		return -EINVAL;
	msi_vecs = min(nvec, ZPCI_MSI_VEC_MAX);
436
	msi_vecs = min_t(unsigned int, msi_vecs, CONFIG_PCI_NR_MSI);
437

438 439 440 441 442
	/* Allocate adapter summary indicator bit */
	rc = -EIO;
	aisb = airq_iv_alloc_bit(zpci_aisb_iv);
	if (aisb == -1UL)
		goto out;
443 444
	zdev->aisb = aisb;

445 446
	/* Create adapter interrupt vector */
	rc = -ENOMEM;
447
	zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK);
448 449
	if (!zdev->aibv)
		goto out_si;
450

451
	/* Wire up shortcut pointer */
452
	zpci_aibv[aisb] = zdev->aibv;
453

454 455
	/* Request MSI interrupts */
	hwirq = 0;
456
	list_for_each_entry(msi, &pdev->msi_list, list) {
457 458 459 460 461
		rc = -EIO;
		irq = irq_alloc_desc(0);	/* Alloc irq on node 0 */
		if (irq == NO_IRQ)
			goto out_msi;
		rc = irq_set_msi_desc(irq, msi);
462
		if (rc)
463
			goto out_msi;
464 465 466 467 468 469 470 471
		irq_set_chip_and_handler(irq, &zpci_irq_chip,
					 handle_simple_irq);
		msg.data = hwirq;
		msg.address_lo = zdev->msi_addr & 0xffffffff;
		msg.address_hi = zdev->msi_addr >> 32;
		write_msi_msg(irq, &msg);
		airq_iv_set_data(zdev->aibv, hwirq, irq);
		hwirq++;
472 473
	}

474 475 476 477 478 479 480 481 482
	/* Enable adapter interrupts */
	rc = zpci_set_airq(zdev);
	if (rc)
		goto out_msi;

	return (msi_vecs == nvec) ? 0 : msi_vecs;

out_msi:
	list_for_each_entry(msi, &pdev->msi_list, list) {
483
		if (hwirq-- == 0)
484
			break;
485 486 487 488 489 490
		irq_set_msi_desc(msi->irq, NULL);
		irq_free_desc(msi->irq);
		msi->msg.address_lo = 0;
		msi->msg.address_hi = 0;
		msi->msg.data = 0;
		msi->irq = 0;
491
	}
492
	zpci_aibv[aisb] = NULL;
493 494 495 496 497 498
	airq_iv_release(zdev->aibv);
out_si:
	airq_iv_free_bit(zpci_aisb_iv, aisb);
out:
	dev_err(&pdev->dev, "register MSI failed with: %d\n", rc);
	return rc;
499 500
}

501
void arch_teardown_msi_irqs(struct pci_dev *pdev)
502 503 504
{
	struct zpci_dev *zdev = get_zdev(pdev);
	struct msi_desc *msi;
505 506 507
	int rc;

	pr_info("%s: on pdev: %p\n", __func__, pdev);
508

509 510
	/* Disable adapter interrupts */
	rc = zpci_clear_airq(zdev);
511 512 513 514 515
	if (rc) {
		dev_err(&pdev->dev, "deregister MSI failed with: %d\n", rc);
		return;
	}

516 517 518 519 520 521 522 523 524 525
	/* Release MSI interrupts */
	list_for_each_entry(msi, &pdev->msi_list, list) {
		zpci_msi_set_mask_bits(msi, 1, 1);
		irq_set_msi_desc(msi->irq, NULL);
		irq_free_desc(msi->irq);
		msi->msg.address_lo = 0;
		msi->msg.address_hi = 0;
		msi->msg.data = 0;
		msi->irq = 0;
	}
526

527
	zpci_aibv[zdev->aisb] = NULL;
528 529
	airq_iv_release(zdev->aibv);
	airq_iv_free_bit(zpci_aisb_iv, zdev->aisb);
530 531
}

J
Jan Glauber 已提交
532 533 534 535 536 537 538 539 540 541 542 543 544
static void zpci_map_resources(struct zpci_dev *zdev)
{
	struct pci_dev *pdev = zdev->pdev;
	resource_size_t len;
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		len = pci_resource_len(pdev, i);
		if (!len)
			continue;
		pdev->resource[i].start = (resource_size_t) pci_iomap(pdev, i, 0);
		pdev->resource[i].end = pdev->resource[i].start + len - 1;
	}
545 546 547 548 549 550 551 552 553 554 555 556 557 558 559
}

static void zpci_unmap_resources(struct zpci_dev *zdev)
{
	struct pci_dev *pdev = zdev->pdev;
	resource_size_t len;
	int i;

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		len = pci_resource_len(pdev, i);
		if (!len)
			continue;
		pci_iounmap(pdev, (void *) pdev->resource[i].start);
	}
}
J
Jan Glauber 已提交
560 561 562 563 564 565 566

struct zpci_dev *zpci_alloc_device(void)
{
	struct zpci_dev *zdev;

	/* Alloc memory for our private pci device data */
	zdev = kzalloc(sizeof(*zdev), GFP_KERNEL);
567
	return zdev ? : ERR_PTR(-ENOMEM);
J
Jan Glauber 已提交
568 569 570 571 572 573 574
}

void zpci_free_device(struct zpci_dev *zdev)
{
	kfree(zdev);
}

575 576 577 578 579
int pcibios_add_platform_entries(struct pci_dev *pdev)
{
	return zpci_sysfs_add_device(&pdev->dev);
}

580 581
static int __init zpci_irq_init(void)
{
582
	int rc;
583

584 585
	rc = register_adapter_interrupt(&zpci_airq);
	if (rc)
586
		goto out;
587 588
	/* Set summary to 1 to be called every time for the ISC. */
	*zpci_airq.lsi_ptr = 1;
589

590 591 592 593
	rc = -ENOMEM;
	zpci_aisb_iv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC);
	if (!zpci_aisb_iv)
		goto out_airq;
594

595
	zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
596 597
	return 0;

598 599 600
out_airq:
	unregister_adapter_interrupt(&zpci_airq);
out:
601 602 603 604 605
	return rc;
}

static void zpci_irq_exit(void)
{
606
	airq_iv_release(zpci_aisb_iv);
607
	unregister_adapter_interrupt(&zpci_airq);
608 609
}

J
Jan Glauber 已提交
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
static struct resource *zpci_alloc_bus_resource(unsigned long start, unsigned long size,
						unsigned long flags, int domain)
{
	struct resource *r;
	char *name;
	int rc;

	r = kzalloc(sizeof(*r), GFP_KERNEL);
	if (!r)
		return ERR_PTR(-ENOMEM);
	r->start = start;
	r->end = r->start + size - 1;
	r->flags = flags;
	r->parent = &iomem_resource;
	name = kmalloc(18, GFP_KERNEL);
	if (!name) {
		kfree(r);
		return ERR_PTR(-ENOMEM);
	}
	sprintf(name, "PCI Bus: %04x:%02x", domain, ZPCI_BUS_NR);
	r->name = name;

	rc = request_resource(&iomem_resource, r);
	if (rc)
		pr_debug("request resource %pR failed\n", r);
	return r;
}

static int zpci_alloc_iomap(struct zpci_dev *zdev)
{
	int entry;

	spin_lock(&zpci_iomap_lock);
	entry = find_first_zero_bit(zpci_iomap, ZPCI_IOMAP_MAX_ENTRIES);
	if (entry == ZPCI_IOMAP_MAX_ENTRIES) {
		spin_unlock(&zpci_iomap_lock);
		return -ENOSPC;
	}
	set_bit(entry, zpci_iomap);
	spin_unlock(&zpci_iomap_lock);
	return entry;
}

static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
{
	spin_lock(&zpci_iomap_lock);
	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
	clear_bit(entry, zpci_iomap);
	spin_unlock(&zpci_iomap_lock);
}

661 662 663
int pcibios_add_device(struct pci_dev *pdev)
{
	struct zpci_dev *zdev = get_zdev(pdev);
S
Sebastian Ott 已提交
664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685
	struct resource *res;
	int i;

	zdev->pdev = pdev;
	zpci_map_resources(zdev);

	for (i = 0; i < PCI_BAR_COUNT; i++) {
		res = &pdev->resource[i];
		if (res->parent || !res->flags)
			continue;
		pci_claim_resource(pdev, i);
	}

	return 0;
}

int pcibios_enable_device(struct pci_dev *pdev, int mask)
{
	struct zpci_dev *zdev = get_zdev(pdev);
	struct resource *res;
	u16 cmd;
	int i;
686

S
Sebastian Ott 已提交
687
	zdev->pdev = pdev;
688 689 690 691
	zpci_debug_init_device(zdev);
	zpci_fmb_enable_device(zdev);
	zpci_map_resources(zdev);

S
Sebastian Ott 已提交
692 693 694 695 696 697 698 699 700 701 702
	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
	for (i = 0; i < PCI_BAR_COUNT; i++) {
		res = &pdev->resource[i];

		if (res->flags & IORESOURCE_IO)
			return -EINVAL;

		if (res->flags & IORESOURCE_MEM)
			cmd |= PCI_COMMAND_MEMORY;
	}
	pci_write_config_word(pdev, PCI_COMMAND, cmd);
703 704 705
	return 0;
}

S
Sebastian Ott 已提交
706
void pcibios_disable_device(struct pci_dev *pdev)
707 708 709 710 711 712 713 714 715
{
	struct zpci_dev *zdev = get_zdev(pdev);

	zpci_unmap_resources(zdev);
	zpci_fmb_disable_device(zdev);
	zpci_debug_exit_device(zdev);
	zdev->pdev = NULL;
}

S
Sebastian Ott 已提交
716
static int zpci_scan_bus(struct zpci_dev *zdev)
J
Jan Glauber 已提交
717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
{
	struct resource *res;
	LIST_HEAD(resources);
	int i;

	/* allocate mapping entry for each used bar */
	for (i = 0; i < PCI_BAR_COUNT; i++) {
		unsigned long addr, size, flags;
		int entry;

		if (!zdev->bars[i].size)
			continue;
		entry = zpci_alloc_iomap(zdev);
		if (entry < 0)
			return entry;
		zdev->bars[i].map_idx = entry;

		/* only MMIO is supported */
		flags = IORESOURCE_MEM;
		if (zdev->bars[i].val & 8)
			flags |= IORESOURCE_PREFETCH;
		if (zdev->bars[i].val & 4)
			flags |= IORESOURCE_MEM_64;

		addr = ZPCI_IOMAP_ADDR_BASE + ((u64) entry << 48);

		size = 1UL << zdev->bars[i].size;

		res = zpci_alloc_bus_resource(addr, size, flags, zdev->domain);
		if (IS_ERR(res)) {
			zpci_free_iomap(zdev, entry);
			return PTR_ERR(res);
		}
		pci_add_resource(&resources, res);
	}

S
Sebastian Ott 已提交
753 754
	zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops,
				      zdev, &resources);
J
Jan Glauber 已提交
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781
	if (!zdev->bus)
		return -EIO;

	zdev->bus->max_bus_speed = zdev->max_bus_speed;
	return 0;
}

static int zpci_alloc_domain(struct zpci_dev *zdev)
{
	spin_lock(&zpci_domain_lock);
	zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
	if (zdev->domain == ZPCI_NR_DEVICES) {
		spin_unlock(&zpci_domain_lock);
		return -ENOSPC;
	}
	set_bit(zdev->domain, zpci_domain);
	spin_unlock(&zpci_domain_lock);
	return 0;
}

static void zpci_free_domain(struct zpci_dev *zdev)
{
	spin_lock(&zpci_domain_lock);
	clear_bit(zdev->domain, zpci_domain);
	spin_unlock(&zpci_domain_lock);
}

J
Jan Glauber 已提交
782 783 784 785 786 787 788 789
int zpci_enable_device(struct zpci_dev *zdev)
{
	int rc;

	rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
	if (rc)
		goto out;
	pr_info("Enabled fh: 0x%x fid: 0x%x\n", zdev->fh, zdev->fid);
J
Jan Glauber 已提交
790 791 792 793

	rc = zpci_dma_init_device(zdev);
	if (rc)
		goto out_dma;
J
Jan Glauber 已提交
794
	return 0;
J
Jan Glauber 已提交
795 796 797

out_dma:
	clp_disable_fh(zdev);
J
Jan Glauber 已提交
798 799 800 801 802
out:
	return rc;
}
EXPORT_SYMBOL_GPL(zpci_enable_device);

803 804 805 806 807 808 809
int zpci_disable_device(struct zpci_dev *zdev)
{
	zpci_dma_exit_device(zdev);
	return clp_disable_fh(zdev);
}
EXPORT_SYMBOL_GPL(zpci_disable_device);

J
Jan Glauber 已提交
810 811 812 813 814 815 816 817
int zpci_create_device(struct zpci_dev *zdev)
{
	int rc;

	rc = zpci_alloc_domain(zdev);
	if (rc)
		goto out;

S
Sebastian Ott 已提交
818 819 820 821 822 823 824 825
	if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
		rc = zpci_enable_device(zdev);
		if (rc)
			goto out_free;

		zdev->state = ZPCI_FN_STATE_ONLINE;
	}
	rc = zpci_scan_bus(zdev);
J
Jan Glauber 已提交
826
	if (rc)
S
Sebastian Ott 已提交
827
		goto out_disable;
J
Jan Glauber 已提交
828 829 830 831 832

	mutex_lock(&zpci_list_lock);
	list_add_tail(&zdev->entry, &zpci_list);
	mutex_unlock(&zpci_list_lock);

833 834
	zpci_init_slot(zdev);

J
Jan Glauber 已提交
835 836
	return 0;

S
Sebastian Ott 已提交
837 838 839 840
out_disable:
	if (zdev->state == ZPCI_FN_STATE_ONLINE)
		zpci_disable_device(zdev);
out_free:
J
Jan Glauber 已提交
841 842 843 844 845 846 847
	zpci_free_domain(zdev);
out:
	return rc;
}

void zpci_stop_device(struct zpci_dev *zdev)
{
J
Jan Glauber 已提交
848
	zpci_dma_exit_device(zdev);
J
Jan Glauber 已提交
849 850 851 852 853 854 855 856 857 858 859 860 861 862
	/*
	 * Note: SCLP disables fh via set-pci-fn so don't
	 * do that here.
	 */
}
EXPORT_SYMBOL_GPL(zpci_stop_device);

static inline int barsize(u8 size)
{
	return (size) ? (1 << size) >> 10 : 0;
}

static int zpci_mem_init(void)
{
863 864 865
	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
				16, 0, NULL);
	if (!zdev_fmb_cache)
866
		goto error_zdev;
867

J
Jan Glauber 已提交
868 869 870 871
	/* TODO: use realloc */
	zpci_iomap_start = kzalloc(ZPCI_IOMAP_MAX_ENTRIES * sizeof(*zpci_iomap_start),
				   GFP_KERNEL);
	if (!zpci_iomap_start)
872
		goto error_iomap;
J
Jan Glauber 已提交
873 874
	return 0;

875
error_iomap:
876
	kmem_cache_destroy(zdev_fmb_cache);
J
Jan Glauber 已提交
877 878 879 880 881 882 883
error_zdev:
	return -ENOMEM;
}

static void zpci_mem_exit(void)
{
	kfree(zpci_iomap_start);
884
	kmem_cache_destroy(zdev_fmb_cache);
J
Jan Glauber 已提交
885 886
}

887
static unsigned int s390_pci_probe;
J
Jan Glauber 已提交
888 889 890

char * __init pcibios_setup(char *str)
{
S
Sebastian Ott 已提交
891 892
	if (!strcmp(str, "on")) {
		s390_pci_probe = 1;
J
Jan Glauber 已提交
893 894 895 896 897 898 899 900 901
		return NULL;
	}
	return str;
}

static int __init pci_base_init(void)
{
	int rc;

902
	if (!s390_pci_probe)
J
Jan Glauber 已提交
903 904 905 906 907 908 909 910 911 912
		return 0;

	if (!test_facility(2) || !test_facility(69)
	    || !test_facility(71) || !test_facility(72))
		return 0;

	pr_info("Probing PCI hardware: PCI:%d  SID:%d  AEN:%d\n",
		test_facility(69), test_facility(70),
		test_facility(71));

913 914
	rc = zpci_debug_init();
	if (rc)
915
		goto out;
916

J
Jan Glauber 已提交
917 918 919 920
	rc = zpci_mem_init();
	if (rc)
		goto out_mem;

921 922 923 924
	rc = zpci_irq_init();
	if (rc)
		goto out_irq;

J
Jan Glauber 已提交
925 926 927 928
	rc = zpci_dma_init();
	if (rc)
		goto out_dma;

J
Jan Glauber 已提交
929 930 931 932
	rc = clp_find_pci_devices();
	if (rc)
		goto out_find;

J
Jan Glauber 已提交
933 934
	return 0;

J
Jan Glauber 已提交
935
out_find:
J
Jan Glauber 已提交
936 937
	zpci_dma_exit();
out_dma:
938 939
	zpci_irq_exit();
out_irq:
J
Jan Glauber 已提交
940 941
	zpci_mem_exit();
out_mem:
942
	zpci_debug_exit();
943
out:
J
Jan Glauber 已提交
944 945
	return rc;
}
946
subsys_initcall_sync(pci_base_init);