intel_irq_remapping.c 24.1 KB
Newer Older
Y
Yinghai Lu 已提交
1
#include <linux/interrupt.h>
2
#include <linux/dmar.h>
3
#include <linux/spinlock.h>
4
#include <linux/slab.h>
5
#include <linux/jiffies.h>
6
#include <linux/hpet.h>
7
#include <linux/pci.h>
8
#include <linux/irq.h>
9
#include <asm/io_apic.h>
Y
Yinghai Lu 已提交
10
#include <asm/smp.h>
11
#include <asm/cpu.h>
K
Kay, Allen M 已提交
12
#include <linux/intel-iommu.h>
13
#include <acpi/acpi.h>
14
#include <asm/irq_remapping.h>
15
#include <asm/pci-direct.h>
16
#include <asm/msidef.h>
17

18
#include "irq_remapping.h"
19

20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
struct ioapic_scope {
	struct intel_iommu *iommu;
	unsigned int id;
	unsigned int bus;	/* PCI bus number */
	unsigned int devfn;	/* PCI devfn number */
};

struct hpet_scope {
	struct intel_iommu *iommu;
	u8 id;
	unsigned int bus;
	unsigned int devfn;
};

#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0)
35
#define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8)
36

37
static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
38 39
static struct hpet_scope ir_hpet[MAX_HPET_TBS];
static int ir_ioapic_num, ir_hpet_num;
40

41
static DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
42

43 44
static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
{
45
	struct irq_cfg *cfg = irq_get_chip_data(irq);
46
	return cfg ? &cfg->irq_2_iommu : NULL;
47 48
}

49 50
int get_irte(int irq, struct irte *entry)
{
51
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
52
	unsigned long flags;
53
	int index;
54

55
	if (!entry || !irq_iommu)
56 57
		return -1;

58
	raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
59

60 61
	index = irq_iommu->irte_index + irq_iommu->sub_handle;
	*entry = *(irq_iommu->iommu->ir_table->base + index);
62

63
	raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
64 65 66
	return 0;
}

67
static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
68 69
{
	struct ir_table *table = iommu->ir_table;
70
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
71 72
	u16 index, start_index;
	unsigned int mask = 0;
73
	unsigned long flags;
74 75
	int i;

76
	if (!count || !irq_iommu)
77 78
		return -1;

79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
	/*
	 * start the IRTE search from index 0.
	 */
	index = start_index = 0;

	if (count > 1) {
		count = __roundup_pow_of_two(count);
		mask = ilog2(count);
	}

	if (mask > ecap_max_handle_mask(iommu->ecap)) {
		printk(KERN_ERR
		       "Requested mask %x exceeds the max invalidation handle"
		       " mask value %Lx\n", mask,
		       ecap_max_handle_mask(iommu->ecap));
		return -1;
	}

97
	raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
98 99 100 101 102 103 104 105 106 107 108
	do {
		for (i = index; i < index + count; i++)
			if  (table->base[i].present)
				break;
		/* empty index found */
		if (i == index + count)
			break;

		index = (index + count) % INTR_REMAP_TABLE_ENTRIES;

		if (index == start_index) {
109
			raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
110 111 112 113 114 115 116 117
			printk(KERN_ERR "can't allocate an IRTE\n");
			return -1;
		}
	} while (1);

	for (i = index; i < index + count; i++)
		table->base[i].present = 1;

118 119 120 121
	irq_iommu->iommu = iommu;
	irq_iommu->irte_index =  index;
	irq_iommu->sub_handle = 0;
	irq_iommu->irte_mask = mask;
122

123
	raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
124 125 126 127

	return index;
}

128
static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
129 130 131 132 133 134 135
{
	struct qi_desc desc;

	desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
		   | QI_IEC_SELECTIVE;
	desc.high = 0;

136
	return qi_submit_sync(&desc, iommu);
137 138
}

139
static int map_irq_to_irte_handle(int irq, u16 *sub_handle)
140
{
141
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
142
	unsigned long flags;
143
	int index;
144

145
	if (!irq_iommu)
146 147
		return -1;

148
	raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
149 150
	*sub_handle = irq_iommu->sub_handle;
	index = irq_iommu->irte_index;
151
	raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
152 153 154
	return index;
}

155
static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
156
{
157
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
158
	unsigned long flags;
159

160
	if (!irq_iommu)
161
		return -1;
162

163
	raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
164

165 166 167 168
	irq_iommu->iommu = iommu;
	irq_iommu->irte_index = index;
	irq_iommu->sub_handle = subhandle;
	irq_iommu->irte_mask = 0;
169

170
	raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
171 172 173 174

	return 0;
}

175
static int modify_irte(int irq, struct irte *irte_modified)
176
{
177
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
178
	struct intel_iommu *iommu;
179
	unsigned long flags;
180 181
	struct irte *irte;
	int rc, index;
182

183
	if (!irq_iommu)
184
		return -1;
185

186
	raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
187

188
	iommu = irq_iommu->iommu;
189

190
	index = irq_iommu->irte_index + irq_iommu->sub_handle;
191 192
	irte = &iommu->ir_table->base[index];

193 194
	set_64bit(&irte->low, irte_modified->low);
	set_64bit(&irte->high, irte_modified->high);
195 196
	__iommu_flush_cache(iommu, irte, sizeof(*irte));

197
	rc = qi_flush_iec(iommu, index, 0);
198
	raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
199 200

	return rc;
201 202
}

203
static struct intel_iommu *map_hpet_to_ir(u8 hpet_id)
204 205 206 207 208 209 210 211 212
{
	int i;

	for (i = 0; i < MAX_HPET_TBS; i++)
		if (ir_hpet[i].id == hpet_id)
			return ir_hpet[i].iommu;
	return NULL;
}

213
static struct intel_iommu *map_ioapic_to_ir(int apic)
214 215 216 217 218 219 220 221 222
{
	int i;

	for (i = 0; i < MAX_IO_APICS; i++)
		if (ir_ioapic[i].id == apic)
			return ir_ioapic[i].iommu;
	return NULL;
}

223
static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev)
224 225 226 227 228 229 230 231 232 233
{
	struct dmar_drhd_unit *drhd;

	drhd = dmar_find_matched_drhd_unit(dev);
	if (!drhd)
		return NULL;

	return drhd->iommu;
}

234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
static int clear_entries(struct irq_2_iommu *irq_iommu)
{
	struct irte *start, *entry, *end;
	struct intel_iommu *iommu;
	int index;

	if (irq_iommu->sub_handle)
		return 0;

	iommu = irq_iommu->iommu;
	index = irq_iommu->irte_index + irq_iommu->sub_handle;

	start = iommu->ir_table->base + index;
	end = start + (1 << irq_iommu->irte_mask);

	for (entry = start; entry < end; entry++) {
250 251
		set_64bit(&entry->low, 0);
		set_64bit(&entry->high, 0);
252 253 254 255 256
	}

	return qi_flush_iec(iommu, index, irq_iommu->irte_mask);
}

257
static int free_irte(int irq)
258
{
259
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
260
	unsigned long flags;
261
	int rc;
262

263
	if (!irq_iommu)
264
		return -1;
265

266
	raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
267

268
	rc = clear_entries(irq_iommu);
269

270 271 272 273
	irq_iommu->iommu = NULL;
	irq_iommu->irte_index = 0;
	irq_iommu->sub_handle = 0;
	irq_iommu->irte_mask = 0;
274

275
	raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
276

277
	return rc;
278 279
}

280 281 282 283
/*
 * source validation type
 */
#define SVT_NO_VERIFY		0x0  /* no verification is required */
L
Lucas De Marchi 已提交
284
#define SVT_VERIFY_SID_SQ	0x1  /* verify using SID and SQ fields */
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
#define SVT_VERIFY_BUS		0x2  /* verify bus of request-id */

/*
 * source-id qualifier
 */
#define SQ_ALL_16	0x0  /* verify all 16 bits of request-id */
#define SQ_13_IGNORE_1	0x1  /* verify most significant 13 bits, ignore
			      * the third least significant bit
			      */
#define SQ_13_IGNORE_2	0x2  /* verify most significant 13 bits, ignore
			      * the second and third least significant bits
			      */
#define SQ_13_IGNORE_3	0x3  /* verify most significant 13 bits, ignore
			      * the least three significant bits
			      */

/*
 * set SVT, SQ and SID fields of irte to verify
 * source ids of interrupt requests
 */
static void set_irte_sid(struct irte *irte, unsigned int svt,
			 unsigned int sq, unsigned int sid)
{
308 309
	if (disable_sourceid_checking)
		svt = SVT_NO_VERIFY;
310 311 312 313 314
	irte->svt = svt;
	irte->sq = sq;
	irte->sid = sid;
}

315
static int set_ioapic_sid(struct irte *irte, int apic)
316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
{
	int i;
	u16 sid = 0;

	if (!irte)
		return -1;

	for (i = 0; i < MAX_IO_APICS; i++) {
		if (ir_ioapic[i].id == apic) {
			sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn;
			break;
		}
	}

	if (sid == 0) {
		pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic);
		return -1;
	}

	set_irte_sid(irte, 1, 0, sid);

	return 0;
}

340
static int set_hpet_sid(struct irte *irte, u8 id)
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
{
	int i;
	u16 sid = 0;

	if (!irte)
		return -1;

	for (i = 0; i < MAX_HPET_TBS; i++) {
		if (ir_hpet[i].id == id) {
			sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn;
			break;
		}
	}

	if (sid == 0) {
		pr_warning("Failed to set source-id of HPET block (%d)\n", id);
		return -1;
	}

	/*
	 * Should really use SQ_ALL_16. Some platforms are broken.
	 * While we figure out the right quirks for these broken platforms, use
	 * SQ_13_IGNORE_3 for now.
	 */
	set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid);

	return 0;
}

370
static int set_msi_sid(struct irte *irte, struct pci_dev *dev)
371 372 373 374 375 376 377
{
	struct pci_dev *bridge;

	if (!irte || !dev)
		return -1;

	/* PCIe device or Root Complex integrated PCI device */
378
	if (pci_is_pcie(dev) || !dev->bus->parent) {
379 380 381 382 383 384 385
		set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
			     (dev->bus->number << 8) | dev->devfn);
		return 0;
	}

	bridge = pci_find_upstream_pcie_bridge(dev);
	if (bridge) {
386
		if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */
387 388 389 390 391 392 393 394 395 396
			set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16,
				(bridge->bus->number << 8) | dev->bus->number);
		else /* this is a legacy PCI bridge */
			set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
				(bridge->bus->number << 8) | bridge->devfn);
	}

	return 0;
}

397
static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)
398 399
{
	u64 addr;
400
	u32 sts;
401 402 403 404
	unsigned long flags;

	addr = virt_to_phys((void *)iommu->ir_table->base);

405
	raw_spin_lock_irqsave(&iommu->register_lock, flags);
406 407 408 409 410

	dmar_writeq(iommu->reg + DMAR_IRTA_REG,
		    (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);

	/* Set interrupt-remapping table pointer */
411
	iommu->gcmd |= DMA_GCMD_SIRTP;
412
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
413 414 415

	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
		      readl, (sts & DMA_GSTS_IRTPS), sts);
416
	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
417 418 419 420 421 422 423

	/*
	 * global invalidation of interrupt entry cache before enabling
	 * interrupt-remapping.
	 */
	qi_global_iec(iommu);

424
	raw_spin_lock_irqsave(&iommu->register_lock, flags);
425 426 427

	/* Enable interrupt-remapping */
	iommu->gcmd |= DMA_GCMD_IRE;
428
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
429 430 431 432

	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
		      readl, (sts & DMA_GSTS_IRES), sts);

433
	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
434 435 436
}


437
static int intel_setup_irq_remapping(struct intel_iommu *iommu, int mode)
438 439 440 441 442
{
	struct ir_table *ir_table;
	struct page *pages;

	ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
443
					     GFP_ATOMIC);
444 445 446 447

	if (!iommu->ir_table)
		return -ENOMEM;

448 449
	pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
				 INTR_REMAP_PAGE_ORDER);
450 451 452 453 454 455 456 457 458 459

	if (!pages) {
		printk(KERN_ERR "failed to allocate pages of order %d\n",
		       INTR_REMAP_PAGE_ORDER);
		kfree(iommu->ir_table);
		return -ENOMEM;
	}

	ir_table->base = page_address(pages);

460
	iommu_set_irq_remapping(iommu, mode);
461 462 463
	return 0;
}

464 465 466
/*
 * Disable Interrupt Remapping.
 */
467
static void iommu_disable_irq_remapping(struct intel_iommu *iommu)
468 469 470 471 472 473 474
{
	unsigned long flags;
	u32 sts;

	if (!ecap_ir_support(iommu->ecap))
		return;

475 476 477 478 479 480
	/*
	 * global invalidation of interrupt entry cache before disabling
	 * interrupt-remapping.
	 */
	qi_global_iec(iommu);

481
	raw_spin_lock_irqsave(&iommu->register_lock, flags);
482 483 484 485 486 487 488 489 490 491 492 493

	sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
	if (!(sts & DMA_GSTS_IRES))
		goto end;

	iommu->gcmd &= ~DMA_GCMD_IRE;
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
		      readl, !(sts & DMA_GSTS_IRES), sts);

end:
494
	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
495 496
}

497 498 499 500 501 502 503 504 505
static int __init dmar_x2apic_optout(void)
{
	struct acpi_table_dmar *dmar;
	dmar = (struct acpi_table_dmar *)dmar_tbl;
	if (!dmar || no_x2apic_optout)
		return 0;
	return dmar->flags & DMAR_X2APIC_OPT_OUT;
}

506
static int __init intel_irq_remapping_supported(void)
507 508 509
{
	struct dmar_drhd_unit *drhd;

510
	if (disable_irq_remap)
511 512
		return 0;

513 514 515
	if (!dmar_ir_support())
		return 0;

516 517 518 519 520 521 522 523 524 525
	for_each_drhd_unit(drhd) {
		struct intel_iommu *iommu = drhd->iommu;

		if (!ecap_ir_support(iommu->ecap))
			return 0;
	}

	return 1;
}

526
static int __init intel_enable_irq_remapping(void)
527 528 529
{
	struct dmar_drhd_unit *drhd;
	int setup = 0;
530
	int eim = 0;
531

532 533 534 535 536
	if (parse_ioapics_under_ir() != 1) {
		printk(KERN_INFO "Not enable interrupt remapping\n");
		return -1;
	}

537 538 539 540 541 542 543 544
	if (x2apic_supported()) {
		eim = !dmar_x2apic_optout();
		WARN(!eim, KERN_WARNING
			   "Your BIOS is broken and requested that x2apic be disabled\n"
			   "This will leave your machine vulnerable to irq-injection attacks\n"
			   "Use 'intremap=no_x2apic_optout' to override BIOS request\n");
	}

545 546 547
	for_each_drhd_unit(drhd) {
		struct intel_iommu *iommu = drhd->iommu;

548 549 550 551 552 553 554
		/*
		 * If the queued invalidation is already initialized,
		 * shouldn't disable it.
		 */
		if (iommu->qi)
			continue;

555 556 557 558 559 560 561 562 563
		/*
		 * Clear previous faults.
		 */
		dmar_fault(-1, iommu);

		/*
		 * Disable intr remapping and queued invalidation, if already
		 * enabled prior to OS handover.
		 */
564
		iommu_disable_irq_remapping(iommu);
565 566 567 568

		dmar_disable_qi(iommu);
	}

569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609
	/*
	 * check for the Interrupt-remapping support
	 */
	for_each_drhd_unit(drhd) {
		struct intel_iommu *iommu = drhd->iommu;

		if (!ecap_ir_support(iommu->ecap))
			continue;

		if (eim && !ecap_eim_support(iommu->ecap)) {
			printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
			       " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
			return -1;
		}
	}

	/*
	 * Enable queued invalidation for all the DRHD's.
	 */
	for_each_drhd_unit(drhd) {
		int ret;
		struct intel_iommu *iommu = drhd->iommu;
		ret = dmar_enable_qi(iommu);

		if (ret) {
			printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
			       " invalidation, ecap %Lx, ret %d\n",
			       drhd->reg_base_addr, iommu->ecap, ret);
			return -1;
		}
	}

	/*
	 * Setup Interrupt-remapping for all the DRHD's now.
	 */
	for_each_drhd_unit(drhd) {
		struct intel_iommu *iommu = drhd->iommu;

		if (!ecap_ir_support(iommu->ecap))
			continue;

610
		if (intel_setup_irq_remapping(iommu, eim))
611 612 613 614 615 616 617 618
			goto error;

		setup = 1;
	}

	if (!setup)
		goto error;

619
	irq_remapping_enabled = 1;
620
	pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
621

622
	return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
623 624 625 626 627 628 629

error:
	/*
	 * handle error condition gracefully here!
	 */
	return -1;
}
630

631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658
static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope,
				      struct intel_iommu *iommu)
{
	struct acpi_dmar_pci_path *path;
	u8 bus;
	int count;

	bus = scope->bus;
	path = (struct acpi_dmar_pci_path *)(scope + 1);
	count = (scope->length - sizeof(struct acpi_dmar_device_scope))
		/ sizeof(struct acpi_dmar_pci_path);

	while (--count > 0) {
		/*
		 * Access PCI directly due to the PCI
		 * subsystem isn't initialized yet.
		 */
		bus = read_pci_config_byte(bus, path->dev, path->fn,
					   PCI_SECONDARY_BUS);
		path++;
	}
	ir_hpet[ir_hpet_num].bus   = bus;
	ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn);
	ir_hpet[ir_hpet_num].iommu = iommu;
	ir_hpet[ir_hpet_num].id    = scope->enumeration_id;
	ir_hpet_num++;
}

659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope,
				      struct intel_iommu *iommu)
{
	struct acpi_dmar_pci_path *path;
	u8 bus;
	int count;

	bus = scope->bus;
	path = (struct acpi_dmar_pci_path *)(scope + 1);
	count = (scope->length - sizeof(struct acpi_dmar_device_scope))
		/ sizeof(struct acpi_dmar_pci_path);

	while (--count > 0) {
		/*
		 * Access PCI directly due to the PCI
		 * subsystem isn't initialized yet.
		 */
		bus = read_pci_config_byte(bus, path->dev, path->fn,
					   PCI_SECONDARY_BUS);
		path++;
	}

	ir_ioapic[ir_ioapic_num].bus   = bus;
	ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn);
	ir_ioapic[ir_ioapic_num].iommu = iommu;
	ir_ioapic[ir_ioapic_num].id    = scope->enumeration_id;
	ir_ioapic_num++;
}

688 689
static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header,
				      struct intel_iommu *iommu)
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707
{
	struct acpi_dmar_hardware_unit *drhd;
	struct acpi_dmar_device_scope *scope;
	void *start, *end;

	drhd = (struct acpi_dmar_hardware_unit *)header;

	start = (void *)(drhd + 1);
	end = ((void *)drhd) + header->length;

	while (start < end) {
		scope = start;
		if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
			if (ir_ioapic_num == MAX_IO_APICS) {
				printk(KERN_WARNING "Exceeded Max IO APICS\n");
				return -1;
			}

Y
Yinghai Lu 已提交
708 709 710
			printk(KERN_INFO "IOAPIC id %d under DRHD base "
			       " 0x%Lx IOMMU %d\n", scope->enumeration_id,
			       drhd->address, iommu->seq_id);
711

712
			ir_parse_one_ioapic_scope(scope, iommu);
713 714 715 716 717 718 719 720 721 722 723
		} else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) {
			if (ir_hpet_num == MAX_HPET_TBS) {
				printk(KERN_WARNING "Exceeded Max HPET blocks\n");
				return -1;
			}

			printk(KERN_INFO "HPET id %d under DRHD base"
			       " 0x%Lx\n", scope->enumeration_id,
			       drhd->address);

			ir_parse_one_hpet_scope(scope, iommu);
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743
		}
		start += scope->length;
	}

	return 0;
}

/*
 * Finds the assocaition between IOAPIC's and its Interrupt-remapping
 * hardware unit.
 */
int __init parse_ioapics_under_ir(void)
{
	struct dmar_drhd_unit *drhd;
	int ir_supported = 0;

	for_each_drhd_unit(drhd) {
		struct intel_iommu *iommu = drhd->iommu;

		if (ecap_ir_support(iommu->ecap)) {
744
			if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
745 746 747 748 749 750 751 752 753 754 755 756 757 758
				return -1;

			ir_supported = 1;
		}
	}

	if (ir_supported && ir_ioapic_num != nr_ioapics) {
		printk(KERN_WARNING
		       "Not all IO-APIC's listed under remapping hardware\n");
		return -1;
	}

	return ir_supported;
}
759

760
int __init ir_dev_scope_init(void)
761
{
762
	if (!irq_remapping_enabled)
763 764 765 766 767 768
		return 0;

	return dmar_dev_scope_init();
}
rootfs_initcall(ir_dev_scope_init);

769
static void disable_irq_remapping(void)
770 771 772 773 774 775 776 777 778 779 780
{
	struct dmar_drhd_unit *drhd;
	struct intel_iommu *iommu = NULL;

	/*
	 * Disable Interrupt-remapping for all the DRHD's now.
	 */
	for_each_iommu(iommu, drhd) {
		if (!ecap_ir_support(iommu->ecap))
			continue;

781
		iommu_disable_irq_remapping(iommu);
782 783 784
	}
}

785
static int reenable_irq_remapping(int eim)
786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802
{
	struct dmar_drhd_unit *drhd;
	int setup = 0;
	struct intel_iommu *iommu = NULL;

	for_each_iommu(iommu, drhd)
		if (iommu->qi)
			dmar_reenable_qi(iommu);

	/*
	 * Setup Interrupt-remapping for all the DRHD's now.
	 */
	for_each_iommu(iommu, drhd) {
		if (!ecap_ir_support(iommu->ecap))
			continue;

		/* Set up interrupt remapping for iommu.*/
803
		iommu_set_irq_remapping(iommu, eim);
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
		setup = 1;
	}

	if (!setup)
		goto error;

	return 0;

error:
	/*
	 * handle error condition gracefully here!
	 */
	return -1;
}

819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904
static void prepare_irte(struct irte *irte, int vector,
			 unsigned int dest)
{
	memset(irte, 0, sizeof(*irte));

	irte->present = 1;
	irte->dst_mode = apic->irq_dest_mode;
	/*
	 * Trigger mode in the IRTE will always be edge, and for IO-APIC, the
	 * actual level or edge trigger will be setup in the IO-APIC
	 * RTE. This will help simplify level triggered irq migration.
	 * For more details, see the comments (in io_apic.c) explainig IO-APIC
	 * irq migration in the presence of interrupt-remapping.
	*/
	irte->trigger_mode = 0;
	irte->dlvry_mode = apic->irq_delivery_mode;
	irte->vector = vector;
	irte->dest_id = IRTE_DEST(dest);
	irte->redir_hint = 1;
}

static int intel_setup_ioapic_entry(int irq,
				    struct IO_APIC_route_entry *route_entry,
				    unsigned int destination, int vector,
				    struct io_apic_irq_attr *attr)
{
	int ioapic_id = mpc_ioapic_id(attr->ioapic);
	struct intel_iommu *iommu = map_ioapic_to_ir(ioapic_id);
	struct IR_IO_APIC_route_entry *entry;
	struct irte irte;
	int index;

	if (!iommu) {
		pr_warn("No mapping iommu for ioapic %d\n", ioapic_id);
		return -ENODEV;
	}

	entry = (struct IR_IO_APIC_route_entry *)route_entry;

	index = alloc_irte(iommu, irq, 1);
	if (index < 0) {
		pr_warn("Failed to allocate IRTE for ioapic %d\n", ioapic_id);
		return -ENOMEM;
	}

	prepare_irte(&irte, vector, destination);

	/* Set source-id of interrupt request */
	set_ioapic_sid(&irte, ioapic_id);

	modify_irte(irq, &irte);

	apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: "
		"Set IRTE entry (P:%d FPD:%d Dst_Mode:%d "
		"Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X "
		"Avail:%X Vector:%02X Dest:%08X "
		"SID:%04X SQ:%X SVT:%X)\n",
		attr->ioapic, irte.present, irte.fpd, irte.dst_mode,
		irte.redir_hint, irte.trigger_mode, irte.dlvry_mode,
		irte.avail, irte.vector, irte.dest_id,
		irte.sid, irte.sq, irte.svt);

	memset(entry, 0, sizeof(*entry));

	entry->index2	= (index >> 15) & 0x1;
	entry->zero	= 0;
	entry->format	= 1;
	entry->index	= (index & 0x7fff);
	/*
	 * IO-APIC RTE will be configured with virtual vector.
	 * irq handler will do the explicit EOI to the io-apic.
	 */
	entry->vector	= attr->ioapic_pin;
	entry->mask	= 0;			/* enable IRQ */
	entry->trigger	= attr->trigger;
	entry->polarity	= attr->polarity;

	/* Mask level triggered irqs.
	 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
	 */
	if (attr->trigger)
		entry->mask = 1;

	return 0;
}

S
Suresh Siddha 已提交
905
#ifdef CONFIG_SMP
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958
/*
 * Migrate the IO-APIC irq in the presence of intr-remapping.
 *
 * For both level and edge triggered, irq migration is a simple atomic
 * update(of vector and cpu destination) of IRTE and flush the hardware cache.
 *
 * For level triggered, we eliminate the io-apic RTE modification (with the
 * updated vector information), by using a virtual vector (io-apic pin number).
 * Real vector that is used for interrupting cpu will be coming from
 * the interrupt-remapping table entry.
 *
 * As the migration is a simple atomic update of IRTE, the same mechanism
 * is used to migrate MSI irq's in the presence of interrupt-remapping.
 */
static int
intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
			  bool force)
{
	struct irq_cfg *cfg = data->chip_data;
	unsigned int dest, irq = data->irq;
	struct irte irte;

	if (!cpumask_intersects(mask, cpu_online_mask))
		return -EINVAL;

	if (get_irte(irq, &irte))
		return -EBUSY;

	if (assign_irq_vector(irq, cfg, mask))
		return -EBUSY;

	dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask);

	irte.vector = cfg->vector;
	irte.dest_id = IRTE_DEST(dest);

	/*
	 * Atomically updates the IRTE with the new destination, vector
	 * and flushes the interrupt entry cache.
	 */
	modify_irte(irq, &irte);

	/*
	 * After this point, all the interrupts will start arriving
	 * at the new destination. So, time to cleanup the previous
	 * vector allocation.
	 */
	if (cfg->move_in_progress)
		send_cleanup_vector(cfg);

	cpumask_copy(data->affinity, mask);
	return 0;
}
S
Suresh Siddha 已提交
959
#endif
960

961 962 963 964 965 966
static void intel_compose_msi_msg(struct pci_dev *pdev,
				  unsigned int irq, unsigned int dest,
				  struct msi_msg *msg, u8 hpet_id)
{
	struct irq_cfg *cfg;
	struct irte irte;
967
	u16 sub_handle = 0;
968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
	int ir_index;

	cfg = irq_get_chip_data(irq);

	ir_index = map_irq_to_irte_handle(irq, &sub_handle);
	BUG_ON(ir_index == -1);

	prepare_irte(&irte, cfg->vector, dest);

	/* Set source-id of interrupt request */
	if (pdev)
		set_msi_sid(&irte, pdev);
	else
		set_hpet_sid(&irte, hpet_id);

	modify_irte(irq, &irte);

	msg->address_hi = MSI_ADDR_BASE_HI;
	msg->data = sub_handle;
	msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
			  MSI_ADDR_IR_SHV |
			  MSI_ADDR_IR_INDEX1(ir_index) |
			  MSI_ADDR_IR_INDEX2(ir_index);
}

/*
 * Map the PCI dev to the corresponding remapping hardware unit
 * and allocate 'nvec' consecutive interrupt-remapping table entries
 * in it.
 */
static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec)
{
	struct intel_iommu *iommu;
	int index;

	iommu = map_dev_to_ir(dev);
	if (!iommu) {
		printk(KERN_ERR
		       "Unable to map PCI %s to iommu\n", pci_name(dev));
		return -ENOENT;
	}

	index = alloc_irte(iommu, irq, nvec);
	if (index < 0) {
		printk(KERN_ERR
		       "Unable to allocate %d IRTE for PCI %s\n", nvec,
		       pci_name(dev));
		return -ENOSPC;
	}
	return index;
}

static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
			       int index, int sub_handle)
{
	struct intel_iommu *iommu;

	iommu = map_dev_to_ir(pdev);
	if (!iommu)
		return -ENOENT;
	/*
	 * setup the mapping between the irq and the IRTE
	 * base index, the sub_handle pointing to the
	 * appropriate interrupt remap table entry.
	 */
	set_irte_irq(irq, iommu, index, sub_handle);

	return 0;
}

static int intel_setup_hpet_msi(unsigned int irq, unsigned int id)
{
	struct intel_iommu *iommu = map_hpet_to_ir(id);
	int index;

	if (!iommu)
		return -1;

	index = alloc_irte(iommu, irq, 1);
	if (index < 0)
		return -1;

	return 0;
}

1053
struct irq_remap_ops intel_irq_remap_ops = {
1054 1055 1056 1057 1058
	.supported		= intel_irq_remapping_supported,
	.prepare		= dmar_table_init,
	.enable			= intel_enable_irq_remapping,
	.disable		= disable_irq_remapping,
	.reenable		= reenable_irq_remapping,
1059
	.enable_faulting	= enable_drhd_fault_handling,
1060
	.setup_ioapic_entry	= intel_setup_ioapic_entry,
S
Suresh Siddha 已提交
1061
#ifdef CONFIG_SMP
1062
	.set_affinity		= intel_ioapic_set_affinity,
S
Suresh Siddha 已提交
1063
#endif
1064
	.free_irq		= free_irte,
1065 1066 1067 1068
	.compose_msi_msg	= intel_compose_msi_msg,
	.msi_alloc_irq		= intel_msi_alloc_irq,
	.msi_setup_irq		= intel_msi_setup_irq,
	.setup_hpet_msi		= intel_setup_hpet_msi,
1069
};