portdrv_core.c 16.4 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
L
Linus Torvalds 已提交
2 3 4 5 6 7 8 9 10 11 12 13
/*
 * Purpose:	PCI Express Port Bus Driver's Core Functions
 *
 * Copyright (C) 2004 Intel
 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
 */

#include <linux/module.h>
#include <linux/pci.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/pm.h>
14
#include <linux/pm_runtime.h>
T
Tim Schmielau 已提交
15 16
#include <linux/string.h>
#include <linux/slab.h>
17
#include <linux/aer.h>
L
Linus Torvalds 已提交
18

19
#include "../pci.h"
L
Linus Torvalds 已提交
20 21
#include "portdrv.h"

22 23
struct portdrv_service_data {
	struct pcie_port_service_driver *drv;
24
	struct device *dev;
25 26 27
	u32 service;
};

28 29 30 31 32 33
/**
 * release_pcie_device - free PCI Express port service device structure
 * @dev: Port service device to release
 *
 * Invoked automatically when device is being removed in response to
 * device_unregister(dev).  Release all resources being claimed.
L
Linus Torvalds 已提交
34 35 36
 */
static void release_pcie_device(struct device *dev)
{
37
	kfree(to_pcie_device(dev));
L
Linus Torvalds 已提交
38 39
}

40 41 42 43
/*
 * Fill in *pme, *aer, *dpc with the relevant Interrupt Message Numbers if
 * services are enabled in "mask".  Return the number of MSI/MSI-X vectors
 * required to accommodate the largest Message Number.
44
 */
45 46
static int pcie_message_numbers(struct pci_dev *dev, int mask,
				u32 *pme, u32 *aer, u32 *dpc)
47
{
48
	u32 nvec = 0, pos;
49
	u16 reg16;
50 51

	/*
52 53 54 55
	 * The Interrupt Message Number indicates which vector is used, i.e.,
	 * the MSI-X table entry or the MSI offset between the base Message
	 * Data and the generated interrupt message.  See PCIe r3.1, sec
	 * 7.8.2, 7.10.10, 7.31.2.
56 57
	 */

58 59
	if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP |
		    PCIE_PORT_SERVICE_BWNOTIF)) {
60
		pcie_capability_read_word(dev, PCI_EXP_FLAGS, &reg16);
61 62
		*pme = (reg16 & PCI_EXP_FLAGS_IRQ) >> 9;
		nvec = *pme + 1;
63 64
	}

65
#ifdef CONFIG_PCIEAER
66
	if (mask & PCIE_PORT_SERVICE_AER) {
67 68 69
		u32 reg32;

		pos = dev->aer_cap;
70 71 72 73 74 75
		if (pos) {
			pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS,
					      &reg32);
			*aer = (reg32 & PCI_ERR_ROOT_AER_IRQ) >> 27;
			nvec = max(nvec, *aer + 1);
		}
76
	}
77
#endif
78

79 80
	if (mask & PCIE_PORT_SERVICE_DPC) {
		pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC);
81 82 83 84 85 86 87 88 89 90 91
		if (pos) {
			pci_read_config_word(dev, pos + PCI_EXP_DPC_CAP,
					     &reg16);
			*dpc = reg16 & PCI_EXP_DPC_IRQ;
			nvec = max(nvec, *dpc + 1);
		}
	}

	return nvec;
}

92
/**
93 94
 * pcie_port_enable_irq_vec - try to set up MSI-X or MSI as interrupt mode
 * for given port
95
 * @dev: PCI Express port to handle
96
 * @irqs: Array of interrupt vectors to populate
97 98 99 100
 * @mask: Bitmask of port capabilities returned by get_port_device_capability()
 *
 * Return value: 0 on success, error code on failure
 */
101
static int pcie_port_enable_irq_vec(struct pci_dev *dev, int *irqs, int mask)
102
{
103
	int nr_entries, nvec, pcie_irq;
104
	u32 pme = 0, aer = 0, dpc = 0;
105

B
Bjorn Helgaas 已提交
106
	/* Allocate the maximum possible number of MSI/MSI-X vectors */
107 108
	nr_entries = pci_alloc_irq_vectors(dev, 1, PCIE_PORT_MAX_MSI_ENTRIES,
			PCI_IRQ_MSIX | PCI_IRQ_MSI);
109 110
	if (nr_entries < 0)
		return nr_entries;
111

112 113 114 115 116
	/* See how many and which Interrupt Message Numbers we actually use */
	nvec = pcie_message_numbers(dev, mask, &pme, &aer, &dpc);
	if (nvec > nr_entries) {
		pci_free_irq_vectors(dev);
		return -EIO;
117 118
	}

119
	/*
120 121 122 123 124 125 126 127 128
	 * If we allocated more than we need, free them and reallocate fewer.
	 *
	 * Reallocating may change the specific vectors we get, so
	 * pci_irq_vector() must be done *after* the reallocation.
	 *
	 * If we're using MSI, hardware is *allowed* to change the Interrupt
	 * Message Numbers when we free and reallocate the vectors, but we
	 * assume it won't because we allocate enough vectors for the
	 * biggest Message Number we found.
129
	 */
130 131
	if (nvec != nr_entries) {
		pci_free_irq_vectors(dev);
132

133
		nr_entries = pci_alloc_irq_vectors(dev, nvec, nvec,
134
				PCI_IRQ_MSIX | PCI_IRQ_MSI);
135 136
		if (nr_entries < 0)
			return nr_entries;
137 138
	}

139 140 141 142 143 144 145
	/* PME, hotplug and bandwidth notification share an MSI/MSI-X vector */
	if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP |
		    PCIE_PORT_SERVICE_BWNOTIF)) {
		pcie_irq = pci_irq_vector(dev, pme);
		irqs[PCIE_PORT_SERVICE_PME_SHIFT] = pcie_irq;
		irqs[PCIE_PORT_SERVICE_HP_SHIFT] = pcie_irq;
		irqs[PCIE_PORT_SERVICE_BWNOTIF_SHIFT] = pcie_irq;
146 147
	}

148 149
	if (mask & PCIE_PORT_SERVICE_AER)
		irqs[PCIE_PORT_SERVICE_AER_SHIFT] = pci_irq_vector(dev, aer);
150

151 152
	if (mask & PCIE_PORT_SERVICE_DPC)
		irqs[PCIE_PORT_SERVICE_DPC_SHIFT] = pci_irq_vector(dev, dpc);
153

154
	return 0;
155 156
}

157
/**
158
 * pcie_init_service_irqs - initialize irqs for PCI Express port services
159
 * @dev: PCI Express port to handle
160
 * @irqs: Array of irqs to populate
161 162 163 164
 * @mask: Bitmask of port capabilities returned by get_port_device_capability()
 *
 * Return value: Interrupt mode associated with the port
 */
165
static int pcie_init_service_irqs(struct pci_dev *dev, int *irqs, int mask)
L
Linus Torvalds 已提交
166
{
167 168 169 170
	int ret, i;

	for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
		irqs[i] = -1;
171

172
	/*
173 174 175
	 * If we support PME but can't use MSI/MSI-X for it, we have to
	 * fall back to INTx or other interrupts, e.g., a system shared
	 * interrupt.
176
	 */
177 178 179 180 181 182
	if ((mask & PCIE_PORT_SERVICE_PME) && pcie_pme_no_msi())
		goto legacy_irq;

	/* Try to use MSI-X or MSI if supported */
	if (pcie_port_enable_irq_vec(dev, irqs, mask) == 0)
		return 0;
183

184 185 186
legacy_irq:
	/* fall back to legacy IRQ */
	ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_LEGACY);
187 188
	if (ret < 0)
		return -ENODEV;
189

190 191
	for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
		irqs[i] = pci_irq_vector(dev, 0);
L
Linus Torvalds 已提交
192

193
	return 0;
L
Linus Torvalds 已提交
194 195
}

196 197 198 199 200 201 202 203 204 205
/**
 * get_port_device_capability - discover capabilities of a PCI Express port
 * @dev: PCI Express port to examine
 *
 * The capabilities are read from the port's PCI Express configuration registers
 * as described in PCI Express Base Specification 1.0a sections 7.8.2, 7.8.9 and
 * 7.9 - 7.11.
 *
 * Return value: Bitmask of discovered port capabilities
 */
L
Linus Torvalds 已提交
206 207
static int get_port_device_capability(struct pci_dev *dev)
{
208
	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
209
	int services = 0;
210

211
	if (dev->is_hotplug_bridge &&
212
	    (pcie_ports_native || host->native_pcie_hotplug)) {
213
		services |= PCIE_PORT_SERVICE_HP;
214

215 216 217 218 219 220
		/*
		 * Disable hot-plug interrupts in case they have been enabled
		 * by the BIOS and the hot-plug service driver is not loaded.
		 */
		pcie_capability_clear_word(dev, PCI_EXP_SLTCTL,
			  PCI_EXP_SLTCTL_CCIE | PCI_EXP_SLTCTL_HPIE);
221
	}
222

223 224 225
#ifdef CONFIG_PCIEAER
	if (dev->aer_cap && pci_aer_available() &&
	    (pcie_ports_native || host->native_aer)) {
226
		services |= PCIE_PORT_SERVICE_AER;
227

228 229 230 231 232 233
		/*
		 * Disable AER on this port in case it's been enabled by the
		 * BIOS (the AER service driver will enable it when necessary).
		 */
		pci_disable_pcie_error_reporting(dev);
	}
234
#endif
235 236 237 238 239 240 241

	/*
	 * Root ports are capable of generating PME too.  Root Complex
	 * Event Collectors can also generate PMEs, but we don't handle
	 * those yet.
	 */
	if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT &&
242
	    (pcie_ports_native || host->native_pme)) {
243
		services |= PCIE_PORT_SERVICE_PME;
244

245 246 247 248 249 250 251
		/*
		 * Disable PME interrupt on this port in case it's been enabled
		 * by the BIOS (the PME service driver will enable it when
		 * necessary).
		 */
		pcie_pme_interrupt_enable(dev, false);
	}
252 253

	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC) &&
254
	    pci_aer_available() && services & PCIE_PORT_SERVICE_AER)
255
		services |= PCIE_PORT_SERVICE_DPC;
L
Linus Torvalds 已提交
256

257 258 259 260
	if (pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM ||
	    pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT)
		services |= PCIE_PORT_SERVICE_BWNOTIF;

L
Linus Torvalds 已提交
261 262 263
	return services;
}

264
/**
265 266 267
 * pcie_device_init - allocate and initialize PCI Express port service device
 * @pdev: PCI Express port to associate the service device with
 * @service: Type of service to associate with the service device
268 269
 * @irq: Interrupt vector to associate with the service device
 */
270
static int pcie_device_init(struct pci_dev *pdev, int service, int irq)
L
Linus Torvalds 已提交
271
{
272 273
	int retval;
	struct pcie_device *pcie;
L
Linus Torvalds 已提交
274 275
	struct device *device;

276 277 278 279 280 281
	pcie = kzalloc(sizeof(*pcie), GFP_KERNEL);
	if (!pcie)
		return -ENOMEM;
	pcie->port = pdev;
	pcie->irq = irq;
	pcie->service = service;
L
Linus Torvalds 已提交
282 283

	/* Initialize generic device interface */
284
	device = &pcie->device;
L
Linus Torvalds 已提交
285 286
	device->bus = &pcie_port_bus_type;
	device->release = release_pcie_device;	/* callback to free pcie dev */
287
	dev_set_name(device, "%s:pcie%03x",
288
		     pci_name(pdev),
289
		     get_descriptor_id(pci_pcie_type(pdev), service));
290
	device->parent = &pdev->dev;
291
	device_enable_async_suspend(device);
292 293

	retval = device_register(device);
B
Bjorn Helgaas 已提交
294
	if (retval) {
295
		put_device(device);
B
Bjorn Helgaas 已提交
296 297 298
		return retval;
	}

299 300
	pm_runtime_no_callbacks(device);

B
Bjorn Helgaas 已提交
301
	return 0;
L
Linus Torvalds 已提交
302 303
}

304 305 306 307 308 309 310
/**
 * pcie_port_device_register - register PCI Express port
 * @dev: PCI Express port to register
 *
 * Allocate the port extension structure and register services associated with
 * the port.
 */
L
Linus Torvalds 已提交
311 312
int pcie_port_device_register(struct pci_dev *dev)
{
313
	int status, capabilities, i, nr_service;
314
	int irqs[PCIE_PORT_DEVICE_MAXSERVICES];
L
Linus Torvalds 已提交
315

316 317 318
	/* Enable PCI Express port device */
	status = pci_enable_device(dev);
	if (status)
319
		return status;
320 321 322

	/* Get and check PCI Express port services */
	capabilities = get_port_device_capability(dev);
323
	if (!capabilities)
324 325
		return 0;

326
	pci_set_master(dev);
327 328 329
	/*
	 * Initialize service irqs. Don't use service devices that
	 * require interrupts if there is no way to generate them.
330 331 332
	 * However, some drivers may have a polling mode (e.g. pciehp_poll_mode)
	 * that can be used in the absence of irqs.  Allow them to determine
	 * if that is to be used.
333
	 */
334
	status = pcie_init_service_irqs(dev, irqs, capabilities);
335
	if (status) {
336
		capabilities &= PCIE_PORT_SERVICE_HP;
337
		if (!capabilities)
338
			goto error_disable;
339
	}
L
Linus Torvalds 已提交
340 341

	/* Allocate child services if any */
342 343 344
	status = -ENODEV;
	nr_service = 0;
	for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) {
345 346 347
		int service = 1 << i;
		if (!(capabilities & service))
			continue;
348 349
		if (!pcie_device_init(dev, service, irqs[i]))
			nr_service++;
350
	}
351
	if (!nr_service)
352
		goto error_cleanup_irqs;
353

L
Linus Torvalds 已提交
354
	return 0;
355

356
error_cleanup_irqs:
357
	pci_free_irq_vectors(dev);
358 359
error_disable:
	pci_disable_device(dev);
360
	return status;
L
Linus Torvalds 已提交
361 362 363
}

#ifdef CONFIG_PM
364 365 366
typedef int (*pcie_pm_callback_t)(struct pcie_device *);

static int pm_iter(struct device *dev, void *data)
L
Linus Torvalds 已提交
367 368
{
	struct pcie_port_service_driver *service_driver;
369 370
	size_t offset = *(size_t *)data;
	pcie_pm_callback_t cb;
371

372 373
	if ((dev->bus == &pcie_port_bus_type) && dev->driver) {
		service_driver = to_service_driver(dev->driver);
374 375 376
		cb = *(pcie_pm_callback_t *)((void *)service_driver + offset);
		if (cb)
			return cb(to_pcie_device(dev));
377
	}
378 379
	return 0;
}
L
Linus Torvalds 已提交
380

381 382 383 384
/**
 * pcie_port_device_suspend - suspend port services associated with a PCIe port
 * @dev: PCI Express port to handle
 */
385
int pcie_port_device_suspend(struct device *dev)
386
{
387 388
	size_t off = offsetof(struct pcie_port_service_driver, suspend);
	return device_for_each_child(dev, &off, pm_iter);
389
}
L
Linus Torvalds 已提交
390

391 392 393 394 395 396
int pcie_port_device_resume_noirq(struct device *dev)
{
	size_t off = offsetof(struct pcie_port_service_driver, resume_noirq);
	return device_for_each_child(dev, &off, pm_iter);
}

397
/**
398
 * pcie_port_device_resume - resume port services associated with a PCIe port
399 400
 * @dev: PCI Express port to handle
 */
401
int pcie_port_device_resume(struct device *dev)
402
{
403 404
	size_t off = offsetof(struct pcie_port_service_driver, resume);
	return device_for_each_child(dev, &off, pm_iter);
L
Linus Torvalds 已提交
405
}
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425

/**
 * pcie_port_device_runtime_suspend - runtime suspend port services
 * @dev: PCI Express port to handle
 */
int pcie_port_device_runtime_suspend(struct device *dev)
{
	size_t off = offsetof(struct pcie_port_service_driver, runtime_suspend);
	return device_for_each_child(dev, &off, pm_iter);
}

/**
 * pcie_port_device_runtime_resume - runtime resume port services
 * @dev: PCI Express port to handle
 */
int pcie_port_device_runtime_resume(struct device *dev)
{
	size_t off = offsetof(struct pcie_port_service_driver, runtime_resume);
	return device_for_each_child(dev, &off, pm_iter);
}
426
#endif /* PM */
L
Linus Torvalds 已提交
427

428
static int remove_iter(struct device *dev, void *data)
L
Linus Torvalds 已提交
429
{
430
	if (dev->bus == &pcie_port_bus_type)
431
		device_unregister(dev);
432 433 434
	return 0;
}

435 436 437 438 439 440 441 442 443 444 445 446 447
static int find_service_iter(struct device *device, void *data)
{
	struct pcie_port_service_driver *service_driver;
	struct portdrv_service_data *pdrvs;
	u32 service;

	pdrvs = (struct portdrv_service_data *) data;
	service = pdrvs->service;

	if (device->bus == &pcie_port_bus_type && device->driver) {
		service_driver = to_service_driver(device->driver);
		if (service_driver->service == service) {
			pdrvs->drv = service_driver;
448
			pdrvs->dev = device;
449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
			return 1;
		}
	}

	return 0;
}

/**
 * pcie_port_find_service - find the service driver
 * @dev: PCI Express port the service is associated with
 * @service: Service to find
 *
 * Find PCI Express port service driver associated with given service
 */
struct pcie_port_service_driver *pcie_port_find_service(struct pci_dev *dev,
							u32 service)
{
	struct pcie_port_service_driver *drv;
	struct portdrv_service_data pdrvs;

	pdrvs.drv = NULL;
	pdrvs.service = service;
	device_for_each_child(&dev->dev, &pdrvs, find_service_iter);

	drv = pdrvs.drv;
	return drv;
}

477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
/**
 * pcie_port_find_device - find the struct device
 * @dev: PCI Express port the service is associated with
 * @service: For the service to find
 *
 * Find the struct device associated with given service on a pci_dev
 */
struct device *pcie_port_find_device(struct pci_dev *dev,
				      u32 service)
{
	struct device *device;
	struct portdrv_service_data pdrvs;

	pdrvs.dev = NULL;
	pdrvs.service = service;
	device_for_each_child(&dev->dev, &pdrvs, find_service_iter);

	device = pdrvs.dev;
	return device;
}
497
EXPORT_SYMBOL_GPL(pcie_port_find_device);
498

499 500 501 502 503 504 505
/**
 * pcie_port_device_remove - unregister PCI Express port service devices
 * @dev: PCI Express port the service devices to unregister are associated with
 *
 * Remove PCI Express port service devices associated with given port and
 * disable MSI-X or MSI for the port.
 */
506 507
void pcie_port_device_remove(struct pci_dev *dev)
{
508
	device_for_each_child(&dev->dev, NULL, remove_iter);
509
	pci_free_irq_vectors(dev);
510
	pci_disable_device(dev);
L
Linus Torvalds 已提交
511 512
}

513 514 515 516 517 518 519 520
/**
 * pcie_port_probe_service - probe driver for given PCI Express port service
 * @dev: PCI Express port service device to probe against
 *
 * If PCI Express port service driver is registered with
 * pcie_port_service_register(), this function will be called by the driver core
 * whenever match is found between the driver and a port service device.
 */
521
static int pcie_port_probe_service(struct device *dev)
L
Linus Torvalds 已提交
522
{
523 524 525 526 527 528 529 530 531 532 533 534
	struct pcie_device *pciedev;
	struct pcie_port_service_driver *driver;
	int status;

	if (!dev || !dev->driver)
		return -ENODEV;

	driver = to_service_driver(dev->driver);
	if (!driver || !driver->probe)
		return -ENODEV;

	pciedev = to_pcie_device(dev);
535
	status = driver->probe(pciedev);
B
Bjorn Helgaas 已提交
536 537 538 539 540
	if (status)
		return status;

	get_device(dev);
	return 0;
L
Linus Torvalds 已提交
541 542
}

543 544 545 546 547 548 549 550 551
/**
 * pcie_port_remove_service - detach driver from given PCI Express port service
 * @dev: PCI Express port service device to handle
 *
 * If PCI Express port service driver is registered with
 * pcie_port_service_register(), this function will be called by the driver core
 * when device_unregister() is called for the port service device associated
 * with the driver.
 */
552
static int pcie_port_remove_service(struct device *dev)
L
Linus Torvalds 已提交
553
{
554 555 556 557 558 559 560 561 562 563 564 565 566
	struct pcie_device *pciedev;
	struct pcie_port_service_driver *driver;

	if (!dev || !dev->driver)
		return 0;

	pciedev = to_pcie_device(dev);
	driver = to_service_driver(dev->driver);
	if (driver && driver->remove) {
		driver->remove(pciedev);
		put_device(dev);
	}
	return 0;
L
Linus Torvalds 已提交
567 568
}

569 570 571 572 573 574 575 576 577
/**
 * pcie_port_shutdown_service - shut down given PCI Express port service
 * @dev: PCI Express port service device to handle
 *
 * If PCI Express port service driver is registered with
 * pcie_port_service_register(), this function will be called by the driver core
 * when device_shutdown() is called for the port service device associated
 * with the driver.
 */
578 579
static void pcie_port_shutdown_service(struct device *dev) {}

580 581 582 583
/**
 * pcie_port_service_register - register PCI Express port service driver
 * @new: PCI Express port service driver to register
 */
L
Linus Torvalds 已提交
584 585
int pcie_port_service_register(struct pcie_port_service_driver *new)
{
586 587 588
	if (pcie_ports_disabled)
		return -ENODEV;

589
	new->driver.name = new->name;
L
Linus Torvalds 已提交
590 591 592 593 594 595
	new->driver.bus = &pcie_port_bus_type;
	new->driver.probe = pcie_port_probe_service;
	new->driver.remove = pcie_port_remove_service;
	new->driver.shutdown = pcie_port_shutdown_service;

	return driver_register(&new->driver);
596
}
597
EXPORT_SYMBOL(pcie_port_service_register);
L
Linus Torvalds 已提交
598

599 600 601 602
/**
 * pcie_port_service_unregister - unregister PCI Express port service driver
 * @drv: PCI Express port service driver to unregister
 */
603
void pcie_port_service_unregister(struct pcie_port_service_driver *drv)
L
Linus Torvalds 已提交
604
{
605
	driver_unregister(&drv->driver);
L
Linus Torvalds 已提交
606 607
}
EXPORT_SYMBOL(pcie_port_service_unregister);