aer_inject.c 13.4 KB
Newer Older
H
Huang Ying 已提交
1
/*
2
 * PCIe AER software error injection support.
H
Huang Ying 已提交
3
 *
4
 * Debuging PCIe AER code is quite difficult because it is hard to
H
Huang Ying 已提交
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 * trigger various real hardware errors. Software based error
 * injection can fake almost all kinds of errors with the help of a
 * user space helper tool aer-inject, which can be gotten from:
 *   http://www.kernel.org/pub/linux/utils/pci/aer-inject/
 *
 * Copyright 2009 Intel Corporation.
 *     Huang Ying <ying.huang@intel.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 *
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/miscdevice.h>
#include <linux/pci.h>
24
#include <linux/slab.h>
H
Huang Ying 已提交
25
#include <linux/fs.h>
26
#include <linux/uaccess.h>
27
#include <linux/stddef.h>
28
#include <linux/device.h>
H
Huang Ying 已提交
29 30
#include "aerdrv.h"

31
/* Override the existing corrected and uncorrected error masks */
32
static bool aer_mask_override;
33 34
module_param(aer_mask_override, bool, 0);

35
struct aer_error_inj {
H
Huang Ying 已提交
36 37 38 39 40 41 42 43 44
	u8 bus;
	u8 dev;
	u8 fn;
	u32 uncor_status;
	u32 cor_status;
	u32 header_log0;
	u32 header_log1;
	u32 header_log2;
	u32 header_log3;
45
	u32 domain;
H
Huang Ying 已提交
46 47
};

48
struct aer_error {
H
Huang Ying 已提交
49
	struct list_head list;
50
	u32 domain;
H
Huang Ying 已提交
51 52 53 54 55 56 57 58 59 60 61 62 63 64
	unsigned int bus;
	unsigned int devfn;
	int pos_cap_err;

	u32 uncor_status;
	u32 cor_status;
	u32 header_log0;
	u32 header_log1;
	u32 header_log2;
	u32 header_log3;
	u32 root_status;
	u32 source_id;
};

65
struct pci_bus_ops {
H
Huang Ying 已提交
66 67 68 69 70 71 72 73 74 75 76 77
	struct list_head list;
	struct pci_bus *bus;
	struct pci_ops *ops;
};

static LIST_HEAD(einjected);

static LIST_HEAD(pci_bus_ops_list);

/* Protect einjected and pci_bus_ops_list */
static DEFINE_SPINLOCK(inject_lock);

78
static void aer_error_init(struct aer_error *err, u32 domain,
79 80
			   unsigned int bus, unsigned int devfn,
			   int pos_cap_err)
H
Huang Ying 已提交
81 82
{
	INIT_LIST_HEAD(&err->list);
83
	err->domain = domain;
H
Huang Ying 已提交
84 85 86 87 88 89
	err->bus = bus;
	err->devfn = devfn;
	err->pos_cap_err = pos_cap_err;
}

/* inject_lock must be held before calling */
90
static struct aer_error *__find_aer_error(u32 domain, unsigned int bus,
91
					  unsigned int devfn)
H
Huang Ying 已提交
92 93 94 95
{
	struct aer_error *err;

	list_for_each_entry(err, &einjected, list) {
96 97 98
		if (domain == err->domain &&
		    bus == err->bus &&
		    devfn == err->devfn)
H
Huang Ying 已提交
99 100 101 102 103 104 105 106
			return err;
	}
	return NULL;
}

/* inject_lock must be held before calling */
static struct aer_error *__find_aer_error_by_dev(struct pci_dev *dev)
{
107 108 109
	int domain = pci_domain_nr(dev->bus);
	if (domain < 0)
		return NULL;
110
	return __find_aer_error(domain, dev->bus->number, dev->devfn);
H
Huang Ying 已提交
111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
}

/* inject_lock must be held before calling */
static struct pci_ops *__find_pci_bus_ops(struct pci_bus *bus)
{
	struct pci_bus_ops *bus_ops;

	list_for_each_entry(bus_ops, &pci_bus_ops_list, list) {
		if (bus_ops->bus == bus)
			return bus_ops->ops;
	}
	return NULL;
}

static struct pci_bus_ops *pci_bus_ops_pop(void)
{
	unsigned long flags;
128
	struct pci_bus_ops *bus_ops;
H
Huang Ying 已提交
129 130

	spin_lock_irqsave(&inject_lock, flags);
131 132 133 134
	bus_ops = list_first_entry_or_null(&pci_bus_ops_list,
					   struct pci_bus_ops, list);
	if (bus_ops)
		list_del(&bus_ops->list);
H
Huang Ying 已提交
135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
	spin_unlock_irqrestore(&inject_lock, flags);
	return bus_ops;
}

static u32 *find_pci_config_dword(struct aer_error *err, int where,
				  int *prw1cs)
{
	int rw1cs = 0;
	u32 *target = NULL;

	if (err->pos_cap_err == -1)
		return NULL;

	switch (where - err->pos_cap_err) {
	case PCI_ERR_UNCOR_STATUS:
		target = &err->uncor_status;
		rw1cs = 1;
		break;
	case PCI_ERR_COR_STATUS:
		target = &err->cor_status;
		rw1cs = 1;
		break;
	case PCI_ERR_HEADER_LOG:
		target = &err->header_log0;
		break;
	case PCI_ERR_HEADER_LOG+4:
		target = &err->header_log1;
		break;
	case PCI_ERR_HEADER_LOG+8:
164
		target = &err->header_log2;
H
Huang Ying 已提交
165 166 167 168 169 170 171 172
		break;
	case PCI_ERR_HEADER_LOG+12:
		target = &err->header_log3;
		break;
	case PCI_ERR_ROOT_STATUS:
		target = &err->root_status;
		rw1cs = 1;
		break;
173
	case PCI_ERR_ROOT_ERR_SRC:
H
Huang Ying 已提交
174 175 176 177 178 179 180 181
		target = &err->source_id;
		break;
	}
	if (prw1cs)
		*prw1cs = rw1cs;
	return target;
}

182 183
static int aer_inj_read_config(struct pci_bus *bus, unsigned int devfn,
			       int where, int size, u32 *val)
H
Huang Ying 已提交
184 185 186 187 188
{
	u32 *sim;
	struct aer_error *err;
	unsigned long flags;
	struct pci_ops *ops;
189
	struct pci_ops *my_ops;
190
	int domain;
191
	int rv;
H
Huang Ying 已提交
192 193 194 195

	spin_lock_irqsave(&inject_lock, flags);
	if (size != sizeof(u32))
		goto out;
196 197 198
	domain = pci_domain_nr(bus);
	if (domain < 0)
		goto out;
199
	err = __find_aer_error(domain, bus->number, devfn);
H
Huang Ying 已提交
200 201 202 203 204 205 206 207 208 209 210
	if (!err)
		goto out;

	sim = find_pci_config_dword(err, where, NULL);
	if (sim) {
		*val = *sim;
		spin_unlock_irqrestore(&inject_lock, flags);
		return 0;
	}
out:
	ops = __find_pci_bus_ops(bus);
211 212 213 214 215 216 217 218 219 220 221
	/*
	 * pci_lock must already be held, so we can directly
	 * manipulate bus->ops.  Many config access functions,
	 * including pci_generic_config_read() require the original
	 * bus->ops be installed to function, so temporarily put them
	 * back.
	 */
	my_ops = bus->ops;
	bus->ops = ops;
	rv = ops->read(bus, devfn, where, size, val);
	bus->ops = my_ops;
H
Huang Ying 已提交
222
	spin_unlock_irqrestore(&inject_lock, flags);
223
	return rv;
H
Huang Ying 已提交
224 225
}

226 227
static int aer_inj_write_config(struct pci_bus *bus, unsigned int devfn,
				int where, int size, u32 val)
H
Huang Ying 已提交
228 229 230 231 232 233
{
	u32 *sim;
	struct aer_error *err;
	unsigned long flags;
	int rw1cs;
	struct pci_ops *ops;
234
	struct pci_ops *my_ops;
235
	int domain;
236
	int rv;
H
Huang Ying 已提交
237 238 239 240

	spin_lock_irqsave(&inject_lock, flags);
	if (size != sizeof(u32))
		goto out;
241 242 243
	domain = pci_domain_nr(bus);
	if (domain < 0)
		goto out;
244
	err = __find_aer_error(domain, bus->number, devfn);
H
Huang Ying 已提交
245 246 247 248 249 250 251 252 253 254 255 256 257 258
	if (!err)
		goto out;

	sim = find_pci_config_dword(err, where, &rw1cs);
	if (sim) {
		if (rw1cs)
			*sim ^= val;
		else
			*sim = val;
		spin_unlock_irqrestore(&inject_lock, flags);
		return 0;
	}
out:
	ops = __find_pci_bus_ops(bus);
259 260 261 262 263 264 265 266 267 268 269
	/*
	 * pci_lock must already be held, so we can directly
	 * manipulate bus->ops.  Many config access functions,
	 * including pci_generic_config_write() require the original
	 * bus->ops be installed to function, so temporarily put them
	 * back.
	 */
	my_ops = bus->ops;
	bus->ops = ops;
	rv = ops->write(bus, devfn, where, size, val);
	bus->ops = my_ops;
H
Huang Ying 已提交
270
	spin_unlock_irqrestore(&inject_lock, flags);
271
	return rv;
H
Huang Ying 已提交
272 273
}

274 275 276
static struct pci_ops aer_inj_pci_ops = {
	.read = aer_inj_read_config,
	.write = aer_inj_write_config,
H
Huang Ying 已提交
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
};

static void pci_bus_ops_init(struct pci_bus_ops *bus_ops,
			     struct pci_bus *bus,
			     struct pci_ops *ops)
{
	INIT_LIST_HEAD(&bus_ops->list);
	bus_ops->bus = bus;
	bus_ops->ops = ops;
}

static int pci_bus_set_aer_ops(struct pci_bus *bus)
{
	struct pci_ops *ops;
	struct pci_bus_ops *bus_ops;
	unsigned long flags;

	bus_ops = kmalloc(sizeof(*bus_ops), GFP_KERNEL);
	if (!bus_ops)
		return -ENOMEM;
297
	ops = pci_bus_set_ops(bus, &aer_inj_pci_ops);
H
Huang Ying 已提交
298
	spin_lock_irqsave(&inject_lock, flags);
299
	if (ops == &aer_inj_pci_ops)
H
Huang Ying 已提交
300 301 302 303 304 305
		goto out;
	pci_bus_ops_init(bus_ops, bus, ops);
	list_add(&bus_ops->list, &pci_bus_ops_list);
	bus_ops = NULL;
out:
	spin_unlock_irqrestore(&inject_lock, flags);
306
	kfree(bus_ops);
H
Huang Ying 已提交
307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338
	return 0;
}

static int find_aer_device_iter(struct device *device, void *data)
{
	struct pcie_device **result = data;
	struct pcie_device *pcie_dev;

	if (device->bus == &pcie_port_bus_type) {
		pcie_dev = to_pcie_device(device);
		if (pcie_dev->service & PCIE_PORT_SERVICE_AER) {
			*result = pcie_dev;
			return 1;
		}
	}
	return 0;
}

static int find_aer_device(struct pci_dev *dev, struct pcie_device **result)
{
	return device_for_each_child(&dev->dev, result, find_aer_device_iter);
}

static int aer_inject(struct aer_error_inj *einj)
{
	struct aer_error *err, *rperr;
	struct aer_error *err_alloc = NULL, *rperr_alloc = NULL;
	struct pci_dev *dev, *rpdev;
	struct pcie_device *edev;
	unsigned long flags;
	unsigned int devfn = PCI_DEVFN(einj->dev, einj->fn);
	int pos_cap_err, rp_pos_cap_err;
339
	u32 sever, cor_mask, uncor_mask, cor_mask_orig = 0, uncor_mask_orig = 0;
H
Huang Ying 已提交
340 341
	int ret = 0;

342
	dev = pci_get_domain_bus_and_slot(einj->domain, einj->bus, devfn);
H
Huang Ying 已提交
343
	if (!dev)
344
		return -ENODEV;
H
Huang Ying 已提交
345 346
	rpdev = pcie_find_root_port(dev);
	if (!rpdev) {
347
		pci_err(dev, "aer_inject: Root port not found\n");
348
		ret = -ENODEV;
H
Huang Ying 已提交
349 350 351 352 353
		goto out_put;
	}

	pos_cap_err = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
	if (!pos_cap_err) {
354
		pci_err(dev, "aer_inject: Device doesn't support AER\n");
355
		ret = -EPROTONOSUPPORT;
H
Huang Ying 已提交
356 357 358
		goto out_put;
	}
	pci_read_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_SEVER, &sever);
359 360 361
	pci_read_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK, &cor_mask);
	pci_read_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK,
			      &uncor_mask);
H
Huang Ying 已提交
362 363 364

	rp_pos_cap_err = pci_find_ext_capability(rpdev, PCI_EXT_CAP_ID_ERR);
	if (!rp_pos_cap_err) {
365
		pci_err(rpdev, "aer_inject: Root port doesn't support AER\n");
366
		ret = -EPROTONOSUPPORT;
H
Huang Ying 已提交
367 368 369 370 371 372 373 374 375 376 377 378 379 380
		goto out_put;
	}

	err_alloc =  kzalloc(sizeof(struct aer_error), GFP_KERNEL);
	if (!err_alloc) {
		ret = -ENOMEM;
		goto out_put;
	}
	rperr_alloc =  kzalloc(sizeof(struct aer_error), GFP_KERNEL);
	if (!rperr_alloc) {
		ret = -ENOMEM;
		goto out_put;
	}

381 382 383 384 385 386 387 388 389 390 391 392
	if (aer_mask_override) {
		cor_mask_orig = cor_mask;
		cor_mask &= !(einj->cor_status);
		pci_write_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK,
				       cor_mask);

		uncor_mask_orig = uncor_mask;
		uncor_mask &= !(einj->uncor_status);
		pci_write_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK,
				       uncor_mask);
	}

H
Huang Ying 已提交
393 394 395 396 397 398
	spin_lock_irqsave(&inject_lock, flags);

	err = __find_aer_error_by_dev(dev);
	if (!err) {
		err = err_alloc;
		err_alloc = NULL;
399 400
		aer_error_init(err, einj->domain, einj->bus, devfn,
			       pos_cap_err);
H
Huang Ying 已提交
401 402 403 404 405 406 407 408 409
		list_add(&err->list, &einjected);
	}
	err->uncor_status |= einj->uncor_status;
	err->cor_status |= einj->cor_status;
	err->header_log0 = einj->header_log0;
	err->header_log1 = einj->header_log1;
	err->header_log2 = einj->header_log2;
	err->header_log3 = einj->header_log3;

410 411
	if (!aer_mask_override && einj->cor_status &&
	    !(einj->cor_status & ~cor_mask)) {
412
		ret = -EINVAL;
413
		pci_warn(dev, "aer_inject: The correctable error(s) is masked by device\n");
414 415 416
		spin_unlock_irqrestore(&inject_lock, flags);
		goto out_put;
	}
417 418
	if (!aer_mask_override && einj->uncor_status &&
	    !(einj->uncor_status & ~uncor_mask)) {
419
		ret = -EINVAL;
420
		pci_warn(dev, "aer_inject: The uncorrectable error(s) is masked by device\n");
421 422 423 424
		spin_unlock_irqrestore(&inject_lock, flags);
		goto out_put;
	}

H
Huang Ying 已提交
425 426 427 428
	rperr = __find_aer_error_by_dev(rpdev);
	if (!rperr) {
		rperr = rperr_alloc;
		rperr_alloc = NULL;
429 430
		aer_error_init(rperr, pci_domain_nr(rpdev->bus),
			       rpdev->bus->number, rpdev->devfn,
H
Huang Ying 已提交
431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
			       rp_pos_cap_err);
		list_add(&rperr->list, &einjected);
	}
	if (einj->cor_status) {
		if (rperr->root_status & PCI_ERR_ROOT_COR_RCV)
			rperr->root_status |= PCI_ERR_ROOT_MULTI_COR_RCV;
		else
			rperr->root_status |= PCI_ERR_ROOT_COR_RCV;
		rperr->source_id &= 0xffff0000;
		rperr->source_id |= (einj->bus << 8) | devfn;
	}
	if (einj->uncor_status) {
		if (rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV)
			rperr->root_status |= PCI_ERR_ROOT_MULTI_UNCOR_RCV;
		if (sever & einj->uncor_status) {
			rperr->root_status |= PCI_ERR_ROOT_FATAL_RCV;
			if (!(rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV))
				rperr->root_status |= PCI_ERR_ROOT_FIRST_FATAL;
		} else
			rperr->root_status |= PCI_ERR_ROOT_NONFATAL_RCV;
		rperr->root_status |= PCI_ERR_ROOT_UNCOR_RCV;
		rperr->source_id &= 0x0000ffff;
		rperr->source_id |= ((einj->bus << 8) | devfn) << 16;
	}
	spin_unlock_irqrestore(&inject_lock, flags);

457 458 459 460 461 462 463
	if (aer_mask_override) {
		pci_write_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK,
				       cor_mask_orig);
		pci_write_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK,
				       uncor_mask_orig);
	}

H
Huang Ying 已提交
464 465 466 467 468 469 470
	ret = pci_bus_set_aer_ops(dev->bus);
	if (ret)
		goto out_put;
	ret = pci_bus_set_aer_ops(rpdev->bus);
	if (ret)
		goto out_put;

471 472
	if (find_aer_device(rpdev, &edev)) {
		if (!get_service_data(edev)) {
473 474
			dev_warn(&edev->device,
				 "aer_inject: AER service is not initialized\n");
475
			ret = -EPROTONOSUPPORT;
476 477
			goto out_put;
		}
478 479 480
		dev_info(&edev->device,
			 "aer_inject: Injecting errors %08x/%08x into device %s\n",
			 einj->cor_status, einj->uncor_status, pci_name(dev));
H
Huang Ying 已提交
481
		aer_irq(-1, edev);
482
	} else {
483
		pci_err(rpdev, "aer_inject: AER device not found\n");
484
		ret = -ENODEV;
485
	}
H
Huang Ying 已提交
486
out_put:
487 488
	kfree(err_alloc);
	kfree(rperr_alloc);
H
Huang Ying 已提交
489 490 491 492 493 494 495 496 497 498 499 500
	pci_dev_put(dev);
	return ret;
}

static ssize_t aer_inject_write(struct file *filp, const char __user *ubuf,
				size_t usize, loff_t *off)
{
	struct aer_error_inj einj;
	int ret;

	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;
501 502
	if (usize < offsetof(struct aer_error_inj, domain) ||
	    usize > sizeof(einj))
H
Huang Ying 已提交
503 504
		return -EINVAL;

505
	memset(&einj, 0, sizeof(einj));
H
Huang Ying 已提交
506 507 508 509 510 511 512 513 514 515
	if (copy_from_user(&einj, ubuf, usize))
		return -EFAULT;

	ret = aer_inject(&einj);
	return ret ? ret : usize;
}

static const struct file_operations aer_inject_fops = {
	.write = aer_inject_write,
	.owner = THIS_MODULE,
516
	.llseek = noop_llseek,
H
Huang Ying 已提交
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
};

static struct miscdevice aer_inject_device = {
	.minor = MISC_DYNAMIC_MINOR,
	.name = "aer_inject",
	.fops = &aer_inject_fops,
};

static int __init aer_inject_init(void)
{
	return misc_register(&aer_inject_device);
}

static void __exit aer_inject_exit(void)
{
	struct aer_error *err, *err_next;
	unsigned long flags;
	struct pci_bus_ops *bus_ops;

	misc_deregister(&aer_inject_device);

	while ((bus_ops = pci_bus_ops_pop())) {
		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
		kfree(bus_ops);
	}

	spin_lock_irqsave(&inject_lock, flags);
544
	list_for_each_entry_safe(err, err_next, &einjected, list) {
H
Huang Ying 已提交
545 546 547 548 549 550 551 552 553
		list_del(&err->list);
		kfree(err);
	}
	spin_unlock_irqrestore(&inject_lock, flags);
}

module_init(aer_inject_init);
module_exit(aer_inject_exit);

554
MODULE_DESCRIPTION("PCIe AER software error injector");
H
Huang Ying 已提交
555
MODULE_LICENSE("GPL");