amd_iommu.c 95.5 KB
Newer Older
1
/*
2
 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
 * Author: Joerg Roedel <joerg.roedel@amd.com>
 *         Leo Duran <leo.duran@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

20
#include <linux/ratelimit.h>
21
#include <linux/pci.h>
22
#include <linux/pci-ats.h>
A
Akinobu Mita 已提交
23
#include <linux/bitmap.h>
24
#include <linux/slab.h>
25
#include <linux/debugfs.h>
26
#include <linux/scatterlist.h>
27
#include <linux/dma-mapping.h>
28
#include <linux/iommu-helper.h>
29
#include <linux/iommu.h>
30
#include <linux/delay.h>
31
#include <linux/amd-iommu.h>
32 33
#include <linux/notifier.h>
#include <linux/export.h>
34 35 36 37 38 39
#include <linux/irq.h>
#include <linux/msi.h>
#include <asm/irq_remapping.h>
#include <asm/io_apic.h>
#include <asm/apic.h>
#include <asm/hw_irq.h>
40
#include <asm/msidef.h>
41
#include <asm/proto.h>
42
#include <asm/iommu.h>
43
#include <asm/gart.h>
44
#include <asm/dma.h>
45 46 47

#include "amd_iommu_proto.h"
#include "amd_iommu_types.h"
48
#include "irq_remapping.h"
49 50 51

#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))

52
#define LOOP_TIMEOUT	100000
53

54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
/*
 * This bitmap is used to advertise the page sizes our hardware support
 * to the IOMMU core, which will then use this information to split
 * physically contiguous memory regions it is mapping into page sizes
 * that we support.
 *
 * Traditionally the IOMMU core just handed us the mappings directly,
 * after making sure the size is an order of a 4KiB page and that the
 * mapping has natural alignment.
 *
 * To retain this behavior, we currently advertise that we support
 * all page sizes that are an order of 4KiB.
 *
 * If at some point we'd like to utilize the IOMMU core's new behavior,
 * we could change this to advertise the real page sizes we support.
 */
#define AMD_IOMMU_PGSIZES	(~0xFFFUL)

72 73
static DEFINE_RWLOCK(amd_iommu_devtable_lock);

74 75 76 77
/* A list of preallocated protection domains */
static LIST_HEAD(iommu_pd_list);
static DEFINE_SPINLOCK(iommu_pd_list_lock);

78 79 80 81
/* List of all available dev_data structures */
static LIST_HEAD(dev_data_list);
static DEFINE_SPINLOCK(dev_data_list_lock);

82 83 84
LIST_HEAD(ioapic_map);
LIST_HEAD(hpet_map);

85 86 87 88 89 90
/*
 * Domain for untranslated devices - only allocated
 * if iommu=pt passed on kernel cmd line.
 */
static struct protection_domain *pt_domain;

91 92
static struct iommu_ops amd_iommu_ops;

93
static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
94
int amd_iommu_max_glx_val = -1;
95

96 97
static struct dma_map_ops amd_iommu_dma_ops;

98 99 100
/*
 * general struct to manage commands send to an IOMMU
 */
101
struct iommu_cmd {
102 103 104
	u32 data[4];
};

105 106
struct kmem_cache *amd_iommu_irq_cache;

107
static void update_domain(struct protection_domain *domain);
108
static int __init alloc_passthrough_domain(void);
109

110 111 112 113 114 115
/****************************************************************************
 *
 * Helper functions
 *
 ****************************************************************************/

116
static struct iommu_dev_data *alloc_dev_data(u16 devid)
117 118 119 120 121 122 123 124
{
	struct iommu_dev_data *dev_data;
	unsigned long flags;

	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
	if (!dev_data)
		return NULL;

125
	dev_data->devid = devid;
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
	atomic_set(&dev_data->bind, 0);

	spin_lock_irqsave(&dev_data_list_lock, flags);
	list_add_tail(&dev_data->dev_data_list, &dev_data_list);
	spin_unlock_irqrestore(&dev_data_list_lock, flags);

	return dev_data;
}

static void free_dev_data(struct iommu_dev_data *dev_data)
{
	unsigned long flags;

	spin_lock_irqsave(&dev_data_list_lock, flags);
	list_del(&dev_data->dev_data_list);
	spin_unlock_irqrestore(&dev_data_list_lock, flags);

	kfree(dev_data);
}

146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
static struct iommu_dev_data *search_dev_data(u16 devid)
{
	struct iommu_dev_data *dev_data;
	unsigned long flags;

	spin_lock_irqsave(&dev_data_list_lock, flags);
	list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {
		if (dev_data->devid == devid)
			goto out_unlock;
	}

	dev_data = NULL;

out_unlock:
	spin_unlock_irqrestore(&dev_data_list_lock, flags);

	return dev_data;
}

static struct iommu_dev_data *find_dev_data(u16 devid)
{
	struct iommu_dev_data *dev_data;

	dev_data = search_dev_data(devid);

	if (dev_data == NULL)
		dev_data = alloc_dev_data(devid);

	return dev_data;
}

177 178 179 180 181 182 183
static inline u16 get_device_id(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);

	return calc_devid(pdev->bus->number, pdev->devfn);
}

184 185 186 187 188
static struct iommu_dev_data *get_dev_data(struct device *dev)
{
	return dev->archdata.iommu;
}

189 190 191 192
static bool pci_iommuv2_capable(struct pci_dev *pdev)
{
	static const int caps[] = {
		PCI_EXT_CAP_ID_ATS,
193 194
		PCI_EXT_CAP_ID_PRI,
		PCI_EXT_CAP_ID_PASID,
195 196 197 198 199 200 201 202 203 204 205 206
	};
	int i, pos;

	for (i = 0; i < 3; ++i) {
		pos = pci_find_ext_capability(pdev, caps[i]);
		if (pos == 0)
			return false;
	}

	return true;
}

207 208 209 210 211 212 213 214 215
static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
{
	struct iommu_dev_data *dev_data;

	dev_data = get_dev_data(&pdev->dev);

	return dev_data->errata & (1 << erratum) ? true : false;
}

216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
/*
 * In this function the list of preallocated protection domains is traversed to
 * find the domain for a specific device
 */
static struct dma_ops_domain *find_protection_domain(u16 devid)
{
	struct dma_ops_domain *entry, *ret = NULL;
	unsigned long flags;
	u16 alias = amd_iommu_alias_table[devid];

	if (list_empty(&iommu_pd_list))
		return NULL;

	spin_lock_irqsave(&iommu_pd_list_lock, flags);

	list_for_each_entry(entry, &iommu_pd_list, list) {
		if (entry->target_dev == devid ||
		    entry->target_dev == alias) {
			ret = entry;
			break;
		}
	}

	spin_unlock_irqrestore(&iommu_pd_list_lock, flags);

	return ret;
}

244 245 246 247 248 249 250 251 252 253 254 255
/*
 * This function checks if the driver got a valid device from the caller to
 * avoid dereferencing invalid pointers.
 */
static bool check_device(struct device *dev)
{
	u16 devid;

	if (!dev || !dev->dma_mask)
		return false;

	/* No device or no PCI device */
256
	if (dev->bus != &pci_bus_type)
257 258 259 260 261 262 263 264 265 266 267 268 269 270
		return false;

	devid = get_device_id(dev);

	/* Out of our scope? */
	if (devid > amd_iommu_last_bdf)
		return false;

	if (amd_iommu_rlookup_table[devid] == NULL)
		return false;

	return true;
}

271 272 273 274 275 276 277 278
static void swap_pci_ref(struct pci_dev **from, struct pci_dev *to)
{
	pci_dev_put(*from);
	*from = to;
}

#define REQ_ACS_FLAGS	(PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)

279
static int init_iommu_group(struct device *dev)
280 281
{
	struct iommu_dev_data *dev_data;
282
	struct iommu_group *group;
283
	struct pci_dev *dma_pdev = NULL;
284
	int ret;
285

286 287 288
	group = iommu_group_get(dev);
	if (group) {
		iommu_group_put(group);
289
		return 0;
290
	}
291

292
	dev_data = find_dev_data(get_device_id(dev));
293 294 295
	if (!dev_data)
		return -ENOMEM;

296 297
	if (dev_data->alias_data) {
		u16 alias;
298

299
		alias = amd_iommu_alias_table[dev_data->devid];
300
		dma_pdev = pci_get_bus_and_slot(alias >> 8, alias & 0xff);
301 302
	}

303 304
	if (!dma_pdev)
		dma_pdev = pci_dev_get(to_pci_dev(dev));
305

306
	/* Account for quirked devices */
307 308
	swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));

309 310 311 312
	/*
	 * If it's a multifunction device that does not support our
	 * required ACS flags, add to the same group as function 0.
	 */
313 314 315 316 317 318 319
	if (dma_pdev->multifunction &&
	    !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS))
		swap_pci_ref(&dma_pdev,
			     pci_get_slot(dma_pdev->bus,
					  PCI_DEVFN(PCI_SLOT(dma_pdev->devfn),
					  0)));

320 321 322 323 324
	/*
	 * Devices on the root bus go through the iommu.  If that's not us,
	 * find the next upstream device and test ACS up to the root bus.
	 * Finding the next device may require skipping virtual buses.
	 */
325
	while (!pci_is_root_bus(dma_pdev->bus)) {
326 327 328 329 330 331 332 333 334 335
		struct pci_bus *bus = dma_pdev->bus;

		while (!bus->self) {
			if (!pci_is_root_bus(bus))
				bus = bus->parent;
			else
				goto root_bus;
		}

		if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
336 337
			break;

338
		swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
339 340
	}

341
root_bus:
342 343 344 345 346 347
	group = iommu_group_get(&dma_pdev->dev);
	pci_dev_put(dma_pdev);
	if (!group) {
		group = iommu_group_alloc();
		if (IS_ERR(group))
			return PTR_ERR(group);
348
	}
349

350 351 352 353
	ret = iommu_group_add_device(group, dev);

	iommu_group_put(group);

354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
	return ret;
}

static int iommu_init_device(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct iommu_dev_data *dev_data;
	u16 alias;
	int ret;

	if (dev->archdata.iommu)
		return 0;

	dev_data = find_dev_data(get_device_id(dev));
	if (!dev_data)
		return -ENOMEM;

	alias = amd_iommu_alias_table[dev_data->devid];
	if (alias != dev_data->devid) {
		struct iommu_dev_data *alias_data;

		alias_data = find_dev_data(alias);
		if (alias_data == NULL) {
			pr_err("AMD-Vi: Warning: Unhandled device %s\n",
					dev_name(dev));
			free_dev_data(dev_data);
			return -ENOTSUPP;
		}
		dev_data->alias_data = alias_data;
	}

	ret = init_iommu_group(dev);
386 387 388
	if (ret)
		return ret;

389 390 391 392 393 394 395
	if (pci_iommuv2_capable(pdev)) {
		struct amd_iommu *iommu;

		iommu              = amd_iommu_rlookup_table[dev_data->devid];
		dev_data->iommu_v2 = iommu->is_iommu_v2;
	}

396 397 398 399 400
	dev->archdata.iommu = dev_data;

	return 0;
}

401 402 403 404 405 406 407 408 409 410 411 412 413 414
static void iommu_ignore_device(struct device *dev)
{
	u16 devid, alias;

	devid = get_device_id(dev);
	alias = amd_iommu_alias_table[devid];

	memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
	memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));

	amd_iommu_rlookup_table[devid] = NULL;
	amd_iommu_rlookup_table[alias] = NULL;
}

415 416
static void iommu_uninit_device(struct device *dev)
{
417 418
	iommu_group_remove_device(dev);

419 420 421 422 423
	/*
	 * Nothing to do here - we keep dev_data around for unplugged devices
	 * and reuse it when the device is re-plugged - not doing so would
	 * introduce a ton of races.
	 */
424
}
J
Joerg Roedel 已提交
425 426 427

void __init amd_iommu_uninit_devices(void)
{
428
	struct iommu_dev_data *dev_data, *n;
J
Joerg Roedel 已提交
429 430 431 432 433 434 435 436 437
	struct pci_dev *pdev = NULL;

	for_each_pci_dev(pdev) {

		if (!check_device(&pdev->dev))
			continue;

		iommu_uninit_device(&pdev->dev);
	}
438 439 440 441

	/* Free all of our dev_data structures */
	list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
		free_dev_data(dev_data);
J
Joerg Roedel 已提交
442 443 444 445 446 447 448 449 450 451 452 453 454
}

int __init amd_iommu_init_devices(void)
{
	struct pci_dev *pdev = NULL;
	int ret = 0;

	for_each_pci_dev(pdev) {

		if (!check_device(&pdev->dev))
			continue;

		ret = iommu_init_device(&pdev->dev);
455 456 457
		if (ret == -ENOTSUPP)
			iommu_ignore_device(&pdev->dev);
		else if (ret)
J
Joerg Roedel 已提交
458 459 460 461 462 463 464 465 466 467 468
			goto out_free;
	}

	return 0;

out_free:

	amd_iommu_uninit_devices();

	return ret;
}
469 470 471 472 473 474
#ifdef CONFIG_AMD_IOMMU_STATS

/*
 * Initialization code for statistics collection
 */

475
DECLARE_STATS_COUNTER(compl_wait);
476
DECLARE_STATS_COUNTER(cnt_map_single);
477
DECLARE_STATS_COUNTER(cnt_unmap_single);
478
DECLARE_STATS_COUNTER(cnt_map_sg);
479
DECLARE_STATS_COUNTER(cnt_unmap_sg);
480
DECLARE_STATS_COUNTER(cnt_alloc_coherent);
481
DECLARE_STATS_COUNTER(cnt_free_coherent);
482
DECLARE_STATS_COUNTER(cross_page);
483
DECLARE_STATS_COUNTER(domain_flush_single);
484
DECLARE_STATS_COUNTER(domain_flush_all);
485
DECLARE_STATS_COUNTER(alloced_io_mem);
486
DECLARE_STATS_COUNTER(total_map_requests);
487 488 489 490 491
DECLARE_STATS_COUNTER(complete_ppr);
DECLARE_STATS_COUNTER(invalidate_iotlb);
DECLARE_STATS_COUNTER(invalidate_iotlb_all);
DECLARE_STATS_COUNTER(pri_requests);

492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510
static struct dentry *stats_dir;
static struct dentry *de_fflush;

static void amd_iommu_stats_add(struct __iommu_counter *cnt)
{
	if (stats_dir == NULL)
		return;

	cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
				       &cnt->value);
}

static void amd_iommu_stats_init(void)
{
	stats_dir = debugfs_create_dir("amd-iommu", NULL);
	if (stats_dir == NULL)
		return;

	de_fflush  = debugfs_create_bool("fullflush", 0444, stats_dir,
511
					 &amd_iommu_unmap_flush);
512 513

	amd_iommu_stats_add(&compl_wait);
514
	amd_iommu_stats_add(&cnt_map_single);
515
	amd_iommu_stats_add(&cnt_unmap_single);
516
	amd_iommu_stats_add(&cnt_map_sg);
517
	amd_iommu_stats_add(&cnt_unmap_sg);
518
	amd_iommu_stats_add(&cnt_alloc_coherent);
519
	amd_iommu_stats_add(&cnt_free_coherent);
520
	amd_iommu_stats_add(&cross_page);
521
	amd_iommu_stats_add(&domain_flush_single);
522
	amd_iommu_stats_add(&domain_flush_all);
523
	amd_iommu_stats_add(&alloced_io_mem);
524
	amd_iommu_stats_add(&total_map_requests);
525 526 527 528
	amd_iommu_stats_add(&complete_ppr);
	amd_iommu_stats_add(&invalidate_iotlb);
	amd_iommu_stats_add(&invalidate_iotlb_all);
	amd_iommu_stats_add(&pri_requests);
529 530 531 532
}

#endif

533 534 535 536 537 538
/****************************************************************************
 *
 * Interrupt handling functions
 *
 ****************************************************************************/

539 540 541 542
static void dump_dte_entry(u16 devid)
{
	int i;

543 544
	for (i = 0; i < 4; ++i)
		pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
545 546 547
			amd_iommu_dev_table[devid].data[i]);
}

548 549 550 551 552 553 554 555 556
static void dump_command(unsigned long phys_addr)
{
	struct iommu_cmd *cmd = phys_to_virt(phys_addr);
	int i;

	for (i = 0; i < 4; ++i)
		pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
}

557
static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
558
{
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
	int type, devid, domid, flags;
	volatile u32 *event = __evt;
	int count = 0;
	u64 address;

retry:
	type    = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
	domid   = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
	flags   = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
	address = (u64)(((u64)event[3]) << 32) | event[2];

	if (type == 0) {
		/* Did we hit the erratum? */
		if (++count == LOOP_TIMEOUT) {
			pr_err("AMD-Vi: No event written to event log\n");
			return;
		}
		udelay(1);
		goto retry;
	}
580

581
	printk(KERN_ERR "AMD-Vi: Event logged [");
582 583 584 585 586 587 588

	switch (type) {
	case EVENT_TYPE_ILL_DEV:
		printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       address, flags);
589
		dump_dte_entry(devid);
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610
		break;
	case EVENT_TYPE_IO_FAULT:
		printk("IO_PAGE_FAULT device=%02x:%02x.%x "
		       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       domid, address, flags);
		break;
	case EVENT_TYPE_DEV_TAB_ERR:
		printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       address, flags);
		break;
	case EVENT_TYPE_PAGE_TAB_ERR:
		printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
		       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       domid, address, flags);
		break;
	case EVENT_TYPE_ILL_CMD:
		printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
611
		dump_command(address);
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
		break;
	case EVENT_TYPE_CMD_HARD_ERR:
		printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
		       "flags=0x%04x]\n", address, flags);
		break;
	case EVENT_TYPE_IOTLB_INV_TO:
		printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
		       "address=0x%016llx]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       address);
		break;
	case EVENT_TYPE_INV_DEV_REQ:
		printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       address, flags);
		break;
	default:
		printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
	}
632 633

	memset(__evt, 0, 4 * sizeof(u32));
634 635 636 637 638 639 640 641 642 643 644 645 646
}

static void iommu_poll_events(struct amd_iommu *iommu)
{
	u32 head, tail;
	unsigned long flags;

	spin_lock_irqsave(&iommu->lock, flags);

	head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
	tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

	while (head != tail) {
647
		iommu_print_event(iommu, iommu->evt_buf + head);
648 649 650 651 652 653 654 655
		head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
	}

	writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

	spin_unlock_irqrestore(&iommu->lock, flags);
}

656
static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
657 658 659
{
	struct amd_iommu_fault fault;

660 661
	INC_STATS_COUNTER(pri_requests);

662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
	if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
		pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
		return;
	}

	fault.address   = raw[1];
	fault.pasid     = PPR_PASID(raw[0]);
	fault.device_id = PPR_DEVID(raw[0]);
	fault.tag       = PPR_TAG(raw[0]);
	fault.flags     = PPR_FLAGS(raw[0]);

	atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
}

static void iommu_poll_ppr_log(struct amd_iommu *iommu)
{
	unsigned long flags;
	u32 head, tail;

	if (iommu->ppr_log == NULL)
		return;

684 685 686
	/* enable ppr interrupts again */
	writel(MMIO_STATUS_PPR_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET);

687 688 689 690 691 692
	spin_lock_irqsave(&iommu->lock, flags);

	head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
	tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

	while (head != tail) {
693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708
		volatile u64 *raw;
		u64 entry[2];
		int i;

		raw = (u64 *)(iommu->ppr_log + head);

		/*
		 * Hardware bug: Interrupt may arrive before the entry is
		 * written to memory. If this happens we need to wait for the
		 * entry to arrive.
		 */
		for (i = 0; i < LOOP_TIMEOUT; ++i) {
			if (PPR_REQ_TYPE(raw[0]) != 0)
				break;
			udelay(1);
		}
709

710 711 712
		/* Avoid memcpy function-call overhead */
		entry[0] = raw[0];
		entry[1] = raw[1];
713

714 715 716 717 718 719 720
		/*
		 * To detect the hardware bug we need to clear the entry
		 * back to zero.
		 */
		raw[0] = raw[1] = 0UL;

		/* Update head pointer of hardware ring-buffer */
721 722
		head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
		writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
723 724 725

		/*
		 * Release iommu->lock because ppr-handling might need to
F
Frank Arnold 已提交
726
		 * re-acquire it
727 728 729 730 731 732 733 734 735 736
		 */
		spin_unlock_irqrestore(&iommu->lock, flags);

		/* Handle PPR entry */
		iommu_handle_ppr_entry(iommu, entry);

		spin_lock_irqsave(&iommu->lock, flags);

		/* Refresh ring-buffer information */
		head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
737 738 739 740 741 742
		tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
	}

	spin_unlock_irqrestore(&iommu->lock, flags);
}

743
irqreturn_t amd_iommu_int_thread(int irq, void *data)
744
{
745 746
	struct amd_iommu *iommu;

747
	for_each_iommu(iommu) {
748
		iommu_poll_events(iommu);
749 750
		iommu_poll_ppr_log(iommu);
	}
751 752

	return IRQ_HANDLED;
753 754
}

755 756 757 758 759
irqreturn_t amd_iommu_int_handler(int irq, void *data)
{
	return IRQ_WAKE_THREAD;
}

760 761 762 763 764 765
/****************************************************************************
 *
 * IOMMU command queuing functions
 *
 ****************************************************************************/

766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785
static int wait_on_sem(volatile u64 *sem)
{
	int i = 0;

	while (*sem == 0 && i < LOOP_TIMEOUT) {
		udelay(1);
		i += 1;
	}

	if (i == LOOP_TIMEOUT) {
		pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
		return -EIO;
	}

	return 0;
}

static void copy_cmd_to_buffer(struct amd_iommu *iommu,
			       struct iommu_cmd *cmd,
			       u32 tail)
786 787 788
{
	u8 *target;

789
	target = iommu->cmd_buf + tail;
790 791 792 793 794 795
	tail   = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;

	/* Copy command to buffer */
	memcpy(target, cmd, sizeof(*cmd));

	/* Tell the IOMMU about it */
796
	writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
797
}
798

799
static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
800
{
801 802
	WARN_ON(address & 0x7ULL);

803
	memset(cmd, 0, sizeof(*cmd));
804 805 806
	cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
	cmd->data[1] = upper_32_bits(__pa(address));
	cmd->data[2] = 1;
807 808 809
	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
}

810 811 812 813 814 815 816
static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
{
	memset(cmd, 0, sizeof(*cmd));
	cmd->data[0] = devid;
	CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
}

817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
				  size_t size, u16 domid, int pde)
{
	u64 pages;
	int s;

	pages = iommu_num_pages(address, size, PAGE_SIZE);
	s     = 0;

	if (pages > 1) {
		/*
		 * If we have to flush more than one page, flush all
		 * TLB entries for this domain
		 */
		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
		s = 1;
	}

	address &= PAGE_MASK;

	memset(cmd, 0, sizeof(*cmd));
	cmd->data[1] |= domid;
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[3]  = upper_32_bits(address);
	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
	if (s) /* size bit - we flush more than one 4kb page */
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
F
Frank Arnold 已提交
844
	if (pde) /* PDE bit - we want to flush everything, not only the PTEs */
845 846 847
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
}

848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
				  u64 address, size_t size)
{
	u64 pages;
	int s;

	pages = iommu_num_pages(address, size, PAGE_SIZE);
	s     = 0;

	if (pages > 1) {
		/*
		 * If we have to flush more than one page, flush all
		 * TLB entries for this domain
		 */
		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
		s = 1;
	}

	address &= PAGE_MASK;

	memset(cmd, 0, sizeof(*cmd));
	cmd->data[0]  = devid;
	cmd->data[0] |= (qdep & 0xff) << 24;
	cmd->data[1]  = devid;
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[3]  = upper_32_bits(address);
	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
	if (s)
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
}

879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916
static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
				  u64 address, bool size)
{
	memset(cmd, 0, sizeof(*cmd));

	address &= ~(0xfffULL);

	cmd->data[0]  = pasid & PASID_MASK;
	cmd->data[1]  = domid;
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[3]  = upper_32_bits(address);
	cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
	cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
	if (size)
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
}

static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
				  int qdep, u64 address, bool size)
{
	memset(cmd, 0, sizeof(*cmd));

	address &= ~(0xfffULL);

	cmd->data[0]  = devid;
	cmd->data[0] |= (pasid & 0xff) << 16;
	cmd->data[0] |= (qdep  & 0xff) << 24;
	cmd->data[1]  = devid;
	cmd->data[1] |= ((pasid >> 8) & 0xfff) << 16;
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
	cmd->data[3]  = upper_32_bits(address);
	if (size)
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
}

917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932
static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
			       int status, int tag, bool gn)
{
	memset(cmd, 0, sizeof(*cmd));

	cmd->data[0]  = devid;
	if (gn) {
		cmd->data[1]  = pasid & PASID_MASK;
		cmd->data[2]  = CMD_INV_IOMMU_PAGES_GN_MASK;
	}
	cmd->data[3]  = tag & 0x1ff;
	cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;

	CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
}

933 934 935 936
static void build_inv_all(struct iommu_cmd *cmd)
{
	memset(cmd, 0, sizeof(*cmd));
	CMD_SET_TYPE(cmd, CMD_INV_ALL);
937 938
}

939 940 941 942 943 944 945
static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
{
	memset(cmd, 0, sizeof(*cmd));
	cmd->data[0] = devid;
	CMD_SET_TYPE(cmd, CMD_INV_IRT);
}

946 947
/*
 * Writes the command to the IOMMUs command buffer and informs the
948
 * hardware about the new command.
949
 */
950 951 952
static int iommu_queue_command_sync(struct amd_iommu *iommu,
				    struct iommu_cmd *cmd,
				    bool sync)
953
{
954
	u32 left, tail, head, next_tail;
955 956
	unsigned long flags;

957
	WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
958 959

again:
960 961
	spin_lock_irqsave(&iommu->lock, flags);

962 963 964 965
	head      = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
	tail      = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
	next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
	left      = (head - next_tail) % iommu->cmd_buf_size;
966

967 968 969 970
	if (left <= 2) {
		struct iommu_cmd sync_cmd;
		volatile u64 sem = 0;
		int ret;
971

972 973
		build_completion_wait(&sync_cmd, (u64)&sem);
		copy_cmd_to_buffer(iommu, &sync_cmd, tail);
974

975 976 977 978 979 980
		spin_unlock_irqrestore(&iommu->lock, flags);

		if ((ret = wait_on_sem(&sem)) != 0)
			return ret;

		goto again;
981 982
	}

983 984 985
	copy_cmd_to_buffer(iommu, cmd, tail);

	/* We need to sync now to make sure all commands are processed */
986
	iommu->need_sync = sync;
987

988
	spin_unlock_irqrestore(&iommu->lock, flags);
989

990
	return 0;
991 992
}

993 994 995 996 997
static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
{
	return iommu_queue_command_sync(iommu, cmd, true);
}

998 999 1000 1001
/*
 * This function queues a completion wait command into the command
 * buffer of an IOMMU
 */
1002
static int iommu_completion_wait(struct amd_iommu *iommu)
1003 1004
{
	struct iommu_cmd cmd;
1005
	volatile u64 sem = 0;
1006
	int ret;
1007

1008
	if (!iommu->need_sync)
1009
		return 0;
1010

1011
	build_completion_wait(&cmd, (u64)&sem);
1012

1013
	ret = iommu_queue_command_sync(iommu, &cmd, false);
1014
	if (ret)
1015
		return ret;
1016

1017
	return wait_on_sem(&sem);
1018 1019
}

1020
static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
1021
{
1022
	struct iommu_cmd cmd;
1023

1024
	build_inv_dte(&cmd, devid);
1025

1026 1027
	return iommu_queue_command(iommu, &cmd);
}
1028

1029 1030 1031
static void iommu_flush_dte_all(struct amd_iommu *iommu)
{
	u32 devid;
1032

1033 1034
	for (devid = 0; devid <= 0xffff; ++devid)
		iommu_flush_dte(iommu, devid);
1035

1036 1037
	iommu_completion_wait(iommu);
}
1038

1039 1040 1041 1042 1043 1044 1045
/*
 * This function uses heavy locking and may disable irqs for some time. But
 * this is no issue because it is only called during resume.
 */
static void iommu_flush_tlb_all(struct amd_iommu *iommu)
{
	u32 dom_id;
1046

1047 1048 1049 1050 1051 1052
	for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
		struct iommu_cmd cmd;
		build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
				      dom_id, 1);
		iommu_queue_command(iommu, &cmd);
	}
1053

1054
	iommu_completion_wait(iommu);
1055 1056
}

1057
static void iommu_flush_all(struct amd_iommu *iommu)
1058
{
1059
	struct iommu_cmd cmd;
1060

1061
	build_inv_all(&cmd);
1062

1063 1064 1065 1066
	iommu_queue_command(iommu, &cmd);
	iommu_completion_wait(iommu);
}

1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085
static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
{
	struct iommu_cmd cmd;

	build_inv_irt(&cmd, devid);

	iommu_queue_command(iommu, &cmd);
}

static void iommu_flush_irt_all(struct amd_iommu *iommu)
{
	u32 devid;

	for (devid = 0; devid <= MAX_DEV_TABLE_ENTRIES; devid++)
		iommu_flush_irt(iommu, devid);

	iommu_completion_wait(iommu);
}

1086 1087
void iommu_flush_all_caches(struct amd_iommu *iommu)
{
1088 1089 1090 1091
	if (iommu_feature(iommu, FEATURE_IA)) {
		iommu_flush_all(iommu);
	} else {
		iommu_flush_dte_all(iommu);
1092
		iommu_flush_irt_all(iommu);
1093
		iommu_flush_tlb_all(iommu);
1094 1095 1096
	}
}

1097
/*
1098
 * Command send function for flushing on-device TLB
1099
 */
1100 1101
static int device_flush_iotlb(struct iommu_dev_data *dev_data,
			      u64 address, size_t size)
1102 1103
{
	struct amd_iommu *iommu;
1104
	struct iommu_cmd cmd;
1105
	int qdep;
1106

1107 1108
	qdep     = dev_data->ats.qdep;
	iommu    = amd_iommu_rlookup_table[dev_data->devid];
1109

1110
	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
1111 1112

	return iommu_queue_command(iommu, &cmd);
1113 1114
}

1115 1116 1117
/*
 * Command send function for invalidating a device table entry
 */
1118
static int device_flush_dte(struct iommu_dev_data *dev_data)
1119
{
1120
	struct amd_iommu *iommu;
1121
	int ret;
1122

1123
	iommu = amd_iommu_rlookup_table[dev_data->devid];
1124

1125
	ret = iommu_flush_dte(iommu, dev_data->devid);
1126 1127 1128
	if (ret)
		return ret;

1129
	if (dev_data->ats.enabled)
1130
		ret = device_flush_iotlb(dev_data, 0, ~0UL);
1131 1132

	return ret;
1133 1134
}

1135 1136 1137 1138 1139
/*
 * TLB invalidation function which is called from the mapping functions.
 * It invalidates a single PTE if the range to flush is within a single
 * page. Otherwise it flushes the whole TLB of the IOMMU.
 */
1140 1141
static void __domain_flush_pages(struct protection_domain *domain,
				 u64 address, size_t size, int pde)
1142
{
1143
	struct iommu_dev_data *dev_data;
1144 1145
	struct iommu_cmd cmd;
	int ret = 0, i;
1146

1147
	build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
1148

1149 1150 1151 1152 1153 1154 1155 1156
	for (i = 0; i < amd_iommus_present; ++i) {
		if (!domain->dev_iommu[i])
			continue;

		/*
		 * Devices of this domain are behind this IOMMU
		 * We need a TLB flush
		 */
1157
		ret |= iommu_queue_command(amd_iommus[i], &cmd);
1158 1159
	}

1160 1161
	list_for_each_entry(dev_data, &domain->dev_list, list) {

1162
		if (!dev_data->ats.enabled)
1163 1164
			continue;

1165
		ret |= device_flush_iotlb(dev_data, address, size);
1166 1167
	}

1168
	WARN_ON(ret);
1169 1170
}

1171 1172
static void domain_flush_pages(struct protection_domain *domain,
			       u64 address, size_t size)
1173
{
1174
	__domain_flush_pages(domain, address, size, 0);
1175
}
1176

1177
/* Flush the whole IO/TLB for a given protection domain */
1178
static void domain_flush_tlb(struct protection_domain *domain)
1179
{
1180
	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
1181 1182
}

1183
/* Flush the whole IO/TLB for a given protection domain - including PDE */
1184
static void domain_flush_tlb_pde(struct protection_domain *domain)
1185
{
1186
	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
1187 1188
}

1189
static void domain_flush_complete(struct protection_domain *domain)
1190
{
1191
	int i;
1192

1193 1194 1195
	for (i = 0; i < amd_iommus_present; ++i) {
		if (!domain->dev_iommu[i])
			continue;
1196

1197 1198 1199 1200 1201
		/*
		 * Devices of this domain are behind this IOMMU
		 * We need to wait for completion of all commands.
		 */
		iommu_completion_wait(amd_iommus[i]);
1202
	}
1203 1204
}

1205

1206
/*
1207
 * This function flushes the DTEs for all devices in domain
1208
 */
1209
static void domain_flush_devices(struct protection_domain *domain)
1210
{
1211
	struct iommu_dev_data *dev_data;
1212

1213
	list_for_each_entry(dev_data, &domain->dev_list, list)
1214
		device_flush_dte(dev_data);
1215 1216
}

1217 1218 1219 1220 1221 1222 1223
/****************************************************************************
 *
 * The functions below are used the create the page table mappings for
 * unity mapped regions.
 *
 ****************************************************************************/

1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252
/*
 * This function is used to add another level to an IO page table. Adding
 * another level increases the size of the address space by 9 bits to a size up
 * to 64 bits.
 */
static bool increase_address_space(struct protection_domain *domain,
				   gfp_t gfp)
{
	u64 *pte;

	if (domain->mode == PAGE_MODE_6_LEVEL)
		/* address space already 64 bit large */
		return false;

	pte = (void *)get_zeroed_page(gfp);
	if (!pte)
		return false;

	*pte             = PM_LEVEL_PDE(domain->mode,
					virt_to_phys(domain->pt_root));
	domain->pt_root  = pte;
	domain->mode    += 1;
	domain->updated  = true;

	return true;
}

static u64 *alloc_pte(struct protection_domain *domain,
		      unsigned long address,
1253
		      unsigned long page_size,
1254 1255 1256
		      u64 **pte_page,
		      gfp_t gfp)
{
1257
	int level, end_lvl;
1258
	u64 *pte, *page;
1259 1260

	BUG_ON(!is_power_of_2(page_size));
1261 1262 1263 1264

	while (address > PM_LEVEL_SIZE(domain->mode))
		increase_address_space(domain, gfp);

1265 1266 1267 1268
	level   = domain->mode - 1;
	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
	address = PAGE_SIZE_ALIGN(address, page_size);
	end_lvl = PAGE_SIZE_LEVEL(page_size);
1269 1270 1271 1272 1273 1274 1275 1276 1277

	while (level > end_lvl) {
		if (!IOMMU_PTE_PRESENT(*pte)) {
			page = (u64 *)get_zeroed_page(gfp);
			if (!page)
				return NULL;
			*pte = PM_LEVEL_PDE(level, virt_to_phys(page));
		}

1278 1279 1280 1281
		/* No level skipping support yet */
		if (PM_PTE_LEVEL(*pte) != level)
			return NULL;

1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298
		level -= 1;

		pte = IOMMU_PTE_PAGE(*pte);

		if (pte_page && level == end_lvl)
			*pte_page = pte;

		pte = &pte[PM_LEVEL_INDEX(level, address)];
	}

	return pte;
}

/*
 * This function checks if there is a PTE for a given dma address. If
 * there is one, it returns the pointer to it.
 */
1299
static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
1300 1301 1302 1303
{
	int level;
	u64 *pte;

1304 1305 1306 1307 1308
	if (address > PM_LEVEL_SIZE(domain->mode))
		return NULL;

	level   =  domain->mode - 1;
	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1309

1310 1311 1312
	while (level > 0) {

		/* Not Present */
1313 1314 1315
		if (!IOMMU_PTE_PRESENT(*pte))
			return NULL;

1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334
		/* Large PTE */
		if (PM_PTE_LEVEL(*pte) == 0x07) {
			unsigned long pte_mask, __pte;

			/*
			 * If we have a series of large PTEs, make
			 * sure to return a pointer to the first one.
			 */
			pte_mask = PTE_PAGE_SIZE(*pte);
			pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
			__pte    = ((unsigned long)pte) & pte_mask;

			return (u64 *)__pte;
		}

		/* No level skipping support yet */
		if (PM_PTE_LEVEL(*pte) != level)
			return NULL;

1335 1336
		level -= 1;

1337
		/* Walk to the next level */
1338 1339 1340 1341 1342 1343 1344
		pte = IOMMU_PTE_PAGE(*pte);
		pte = &pte[PM_LEVEL_INDEX(level, address)];
	}

	return pte;
}

1345 1346 1347 1348 1349 1350 1351
/*
 * Generic mapping functions. It maps a physical address into a DMA
 * address space. It allocates the page table pages if necessary.
 * In the future it can be extended to a generic mapping function
 * supporting all features of AMD IOMMU page tables like level skipping
 * and full 64 bit address spaces.
 */
1352 1353 1354
static int iommu_map_page(struct protection_domain *dom,
			  unsigned long bus_addr,
			  unsigned long phys_addr,
1355
			  int prot,
1356
			  unsigned long page_size)
1357
{
1358
	u64 __pte, *pte;
1359
	int i, count;
1360

1361
	if (!(prot & IOMMU_PROT_MASK))
1362 1363
		return -EINVAL;

1364 1365 1366 1367 1368 1369 1370 1371
	bus_addr  = PAGE_ALIGN(bus_addr);
	phys_addr = PAGE_ALIGN(phys_addr);
	count     = PAGE_SIZE_PTE_COUNT(page_size);
	pte       = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);

	for (i = 0; i < count; ++i)
		if (IOMMU_PTE_PRESENT(pte[i]))
			return -EBUSY;
1372

1373 1374 1375 1376 1377
	if (page_size > PAGE_SIZE) {
		__pte = PAGE_SIZE_PTE(phys_addr, page_size);
		__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
	} else
		__pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
1378 1379 1380 1381 1382 1383

	if (prot & IOMMU_PROT_IR)
		__pte |= IOMMU_PTE_IR;
	if (prot & IOMMU_PROT_IW)
		__pte |= IOMMU_PTE_IW;

1384 1385
	for (i = 0; i < count; ++i)
		pte[i] = __pte;
1386

1387 1388
	update_domain(dom);

1389 1390 1391
	return 0;
}

1392 1393 1394
static unsigned long iommu_unmap_page(struct protection_domain *dom,
				      unsigned long bus_addr,
				      unsigned long page_size)
1395
{
1396 1397 1398 1399 1400 1401
	unsigned long long unmap_size, unmapped;
	u64 *pte;

	BUG_ON(!is_power_of_2(page_size));

	unmapped = 0;
1402

1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
	while (unmapped < page_size) {

		pte = fetch_pte(dom, bus_addr);

		if (!pte) {
			/*
			 * No PTE for this address
			 * move forward in 4kb steps
			 */
			unmap_size = PAGE_SIZE;
		} else if (PM_PTE_LEVEL(*pte) == 0) {
			/* 4kb PTE found for this address */
			unmap_size = PAGE_SIZE;
			*pte       = 0ULL;
		} else {
			int count, i;

			/* Large PTE found which maps this address */
			unmap_size = PTE_PAGE_SIZE(*pte);
			count      = PAGE_SIZE_PTE_COUNT(unmap_size);
			for (i = 0; i < count; i++)
				pte[i] = 0ULL;
		}

		bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;
		unmapped += unmap_size;
	}

	BUG_ON(!is_power_of_2(unmapped));
1432

1433
	return unmapped;
1434 1435
}

1436 1437 1438 1439
/*
 * This function checks if a specific unity mapping entry is needed for
 * this specific IOMMU.
 */
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453
static int iommu_for_unity_map(struct amd_iommu *iommu,
			       struct unity_map_entry *entry)
{
	u16 bdf, i;

	for (i = entry->devid_start; i <= entry->devid_end; ++i) {
		bdf = amd_iommu_alias_table[i];
		if (amd_iommu_rlookup_table[bdf] == iommu)
			return 1;
	}

	return 0;
}

1454 1455 1456 1457
/*
 * This function actually applies the mapping to the page table of the
 * dma_ops domain.
 */
1458 1459 1460 1461 1462 1463 1464 1465
static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
			     struct unity_map_entry *e)
{
	u64 addr;
	int ret;

	for (addr = e->address_start; addr < e->address_end;
	     addr += PAGE_SIZE) {
1466
		ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
1467
				     PAGE_SIZE);
1468 1469 1470 1471 1472 1473 1474
		if (ret)
			return ret;
		/*
		 * if unity mapping is in aperture range mark the page
		 * as allocated in the aperture
		 */
		if (addr < dma_dom->aperture_size)
1475
			__set_bit(addr >> PAGE_SHIFT,
1476
				  dma_dom->aperture[0]->bitmap);
1477 1478 1479 1480 1481
	}

	return 0;
}

1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
/*
 * Init the unity mappings for a specific IOMMU in the system
 *
 * Basically iterates over all unity mapping entries and applies them to
 * the default domain DMA of that IOMMU if necessary.
 */
static int iommu_init_unity_mappings(struct amd_iommu *iommu)
{
	struct unity_map_entry *entry;
	int ret;

	list_for_each_entry(entry, &amd_iommu_unity_map, list) {
		if (!iommu_for_unity_map(iommu, entry))
			continue;
		ret = dma_ops_unity_map(iommu->default_dom, entry);
		if (ret)
			return ret;
	}

	return 0;
}

1504 1505 1506
/*
 * Inits the unity mappings required for a specific device
 */
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
					  u16 devid)
{
	struct unity_map_entry *e;
	int ret;

	list_for_each_entry(e, &amd_iommu_unity_map, list) {
		if (!(devid >= e->devid_start && devid <= e->devid_end))
			continue;
		ret = dma_ops_unity_map(dma_dom, e);
		if (ret)
			return ret;
	}

	return 0;
}

1524 1525 1526 1527 1528 1529 1530 1531 1532
/****************************************************************************
 *
 * The next functions belong to the address allocator for the dma_ops
 * interface functions. They work like the allocators in the other IOMMU
 * drivers. Its basically a bitmap which marks the allocated pages in
 * the aperture. Maybe it could be enhanced in the future to a more
 * efficient allocator.
 *
 ****************************************************************************/
1533

1534
/*
1535
 * The address allocator core functions.
1536 1537 1538
 *
 * called with domain->lock held
 */
1539

1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559
/*
 * Used to reserve address ranges in the aperture (e.g. for exclusion
 * ranges.
 */
static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
				      unsigned long start_page,
				      unsigned int pages)
{
	unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;

	if (start_page + pages > last_page)
		pages = last_page - start_page;

	for (i = start_page; i < start_page + pages; ++i) {
		int index = i / APERTURE_RANGE_PAGES;
		int page  = i % APERTURE_RANGE_PAGES;
		__set_bit(page, dom->aperture[index]->bitmap);
	}
}

1560 1561 1562 1563 1564
/*
 * This function is used to add a new aperture range to an existing
 * aperture in case of dma_ops domain allocation or address allocation
 * failure.
 */
1565
static int alloc_new_range(struct dma_ops_domain *dma_dom,
1566 1567 1568
			   bool populate, gfp_t gfp)
{
	int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1569
	struct amd_iommu *iommu;
1570
	unsigned long i, old_size;
1571

1572 1573 1574 1575
#ifdef CONFIG_IOMMU_STRESS
	populate = false;
#endif

1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594
	if (index >= APERTURE_MAX_RANGES)
		return -ENOMEM;

	dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
	if (!dma_dom->aperture[index])
		return -ENOMEM;

	dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
	if (!dma_dom->aperture[index]->bitmap)
		goto out_free;

	dma_dom->aperture[index]->offset = dma_dom->aperture_size;

	if (populate) {
		unsigned long address = dma_dom->aperture_size;
		int i, num_ptes = APERTURE_RANGE_PAGES / 512;
		u64 *pte, *pte_page;

		for (i = 0; i < num_ptes; ++i) {
1595
			pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
1596 1597 1598 1599 1600 1601 1602 1603 1604 1605
					&pte_page, gfp);
			if (!pte)
				goto out_free;

			dma_dom->aperture[index]->pte_pages[i] = pte_page;

			address += APERTURE_RANGE_SIZE / 64;
		}
	}

1606
	old_size                = dma_dom->aperture_size;
1607 1608
	dma_dom->aperture_size += APERTURE_RANGE_SIZE;

1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620
	/* Reserve address range used for MSI messages */
	if (old_size < MSI_ADDR_BASE_LO &&
	    dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
		unsigned long spage;
		int pages;

		pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
		spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;

		dma_ops_reserve_addresses(dma_dom, spage, pages);
	}

1621
	/* Initialize the exclusion range if necessary */
1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632
	for_each_iommu(iommu) {
		if (iommu->exclusion_start &&
		    iommu->exclusion_start >= dma_dom->aperture[index]->offset
		    && iommu->exclusion_start < dma_dom->aperture_size) {
			unsigned long startpage;
			int pages = iommu_num_pages(iommu->exclusion_start,
						    iommu->exclusion_length,
						    PAGE_SIZE);
			startpage = iommu->exclusion_start >> PAGE_SHIFT;
			dma_ops_reserve_addresses(dma_dom, startpage, pages);
		}
1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643
	}

	/*
	 * Check for areas already mapped as present in the new aperture
	 * range and mark those pages as reserved in the allocator. Such
	 * mappings may already exist as a result of requested unity
	 * mappings for devices.
	 */
	for (i = dma_dom->aperture[index]->offset;
	     i < dma_dom->aperture_size;
	     i += PAGE_SIZE) {
1644
		u64 *pte = fetch_pte(&dma_dom->domain, i);
1645 1646 1647
		if (!pte || !IOMMU_PTE_PRESENT(*pte))
			continue;

1648
		dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);
1649 1650
	}

1651 1652
	update_domain(&dma_dom->domain);

1653 1654 1655
	return 0;

out_free:
1656 1657
	update_domain(&dma_dom->domain);

1658 1659 1660 1661 1662 1663 1664 1665
	free_page((unsigned long)dma_dom->aperture[index]->bitmap);

	kfree(dma_dom->aperture[index]);
	dma_dom->aperture[index] = NULL;

	return -ENOMEM;
}

1666 1667 1668 1669 1670 1671 1672
static unsigned long dma_ops_area_alloc(struct device *dev,
					struct dma_ops_domain *dom,
					unsigned int pages,
					unsigned long align_mask,
					u64 dma_mask,
					unsigned long start)
{
1673
	unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1674 1675 1676 1677 1678 1679
	int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
	int i = start >> APERTURE_RANGE_SHIFT;
	unsigned long boundary_size;
	unsigned long address = -1;
	unsigned long limit;

1680 1681
	next_bit >>= PAGE_SHIFT;

1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
			PAGE_SIZE) >> PAGE_SHIFT;

	for (;i < max_index; ++i) {
		unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;

		if (dom->aperture[i]->offset >= dma_mask)
			break;

		limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
					       dma_mask >> PAGE_SHIFT);

		address = iommu_area_alloc(dom->aperture[i]->bitmap,
					   limit, next_bit, pages, 0,
					    boundary_size, align_mask);
		if (address != -1) {
			address = dom->aperture[i]->offset +
				  (address << PAGE_SHIFT);
1700
			dom->next_address = address + (pages << PAGE_SHIFT);
1701 1702 1703 1704 1705 1706 1707 1708 1709
			break;
		}

		next_bit = 0;
	}

	return address;
}

1710 1711
static unsigned long dma_ops_alloc_addresses(struct device *dev,
					     struct dma_ops_domain *dom,
1712
					     unsigned int pages,
1713 1714
					     unsigned long align_mask,
					     u64 dma_mask)
1715 1716 1717
{
	unsigned long address;

1718 1719 1720 1721
#ifdef CONFIG_IOMMU_STRESS
	dom->next_address = 0;
	dom->need_flush = true;
#endif
1722

1723
	address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1724
				     dma_mask, dom->next_address);
1725

1726
	if (address == -1) {
1727
		dom->next_address = 0;
1728 1729
		address = dma_ops_area_alloc(dev, dom, pages, align_mask,
					     dma_mask, 0);
1730 1731
		dom->need_flush = true;
	}
1732

1733
	if (unlikely(address == -1))
1734
		address = DMA_ERROR_CODE;
1735 1736 1737 1738 1739 1740

	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);

	return address;
}

1741 1742 1743 1744 1745
/*
 * The address free function.
 *
 * called with domain->lock held
 */
1746 1747 1748 1749
static void dma_ops_free_addresses(struct dma_ops_domain *dom,
				   unsigned long address,
				   unsigned int pages)
{
1750 1751
	unsigned i = address >> APERTURE_RANGE_SHIFT;
	struct aperture_range *range = dom->aperture[i];
1752

1753 1754
	BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);

1755 1756 1757 1758
#ifdef CONFIG_IOMMU_STRESS
	if (i < 4)
		return;
#endif
1759

1760
	if (address >= dom->next_address)
1761
		dom->need_flush = true;
1762 1763

	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1764

A
Akinobu Mita 已提交
1765
	bitmap_clear(range->bitmap, address, pages);
1766

1767 1768
}

1769 1770 1771 1772 1773 1774 1775 1776 1777 1778
/****************************************************************************
 *
 * The next functions belong to the domain allocation. A domain is
 * allocated for every IOMMU as the default domain. If device isolation
 * is enabled, every device get its own domain. The most important thing
 * about domains is the page table mapping the DMA address space they
 * contain.
 *
 ****************************************************************************/

1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
/*
 * This function adds a protection domain to the global protection domain list
 */
static void add_domain_to_list(struct protection_domain *domain)
{
	unsigned long flags;

	spin_lock_irqsave(&amd_iommu_pd_lock, flags);
	list_add(&domain->list, &amd_iommu_pd_list);
	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
}

/*
 * This function removes a protection domain to the global
 * protection domain list
 */
static void del_domain_from_list(struct protection_domain *domain)
{
	unsigned long flags;

	spin_lock_irqsave(&amd_iommu_pd_lock, flags);
	list_del(&domain->list);
	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
}

1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820
static u16 domain_id_alloc(void)
{
	unsigned long flags;
	int id;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
	BUG_ON(id == 0);
	if (id > 0 && id < MAX_DOMAIN_ID)
		__set_bit(id, amd_iommu_pd_alloc_bitmap);
	else
		id = 0;
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

	return id;
}

1821 1822 1823 1824 1825 1826 1827 1828 1829 1830
static void domain_id_free(int id)
{
	unsigned long flags;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	if (id > 0 && id < MAX_DOMAIN_ID)
		__clear_bit(id, amd_iommu_pd_alloc_bitmap);
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}

1831
static void free_pagetable(struct protection_domain *domain)
1832 1833 1834 1835
{
	int i, j;
	u64 *p1, *p2, *p3;

1836
	p1 = domain->pt_root;
1837 1838 1839 1840 1841 1842 1843 1844 1845

	if (!p1)
		return;

	for (i = 0; i < 512; ++i) {
		if (!IOMMU_PTE_PRESENT(p1[i]))
			continue;

		p2 = IOMMU_PTE_PAGE(p1[i]);
1846
		for (j = 0; j < 512; ++j) {
1847 1848 1849 1850 1851 1852 1853 1854 1855 1856
			if (!IOMMU_PTE_PRESENT(p2[j]))
				continue;
			p3 = IOMMU_PTE_PAGE(p2[j]);
			free_page((unsigned long)p3);
		}

		free_page((unsigned long)p2);
	}

	free_page((unsigned long)p1);
1857 1858

	domain->pt_root = NULL;
1859 1860
}

1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890
static void free_gcr3_tbl_level1(u64 *tbl)
{
	u64 *ptr;
	int i;

	for (i = 0; i < 512; ++i) {
		if (!(tbl[i] & GCR3_VALID))
			continue;

		ptr = __va(tbl[i] & PAGE_MASK);

		free_page((unsigned long)ptr);
	}
}

static void free_gcr3_tbl_level2(u64 *tbl)
{
	u64 *ptr;
	int i;

	for (i = 0; i < 512; ++i) {
		if (!(tbl[i] & GCR3_VALID))
			continue;

		ptr = __va(tbl[i] & PAGE_MASK);

		free_gcr3_tbl_level1(ptr);
	}
}

1891 1892
static void free_gcr3_table(struct protection_domain *domain)
{
1893 1894 1895 1896 1897 1898 1899
	if (domain->glx == 2)
		free_gcr3_tbl_level2(domain->gcr3_tbl);
	else if (domain->glx == 1)
		free_gcr3_tbl_level1(domain->gcr3_tbl);
	else if (domain->glx != 0)
		BUG();

1900 1901 1902
	free_page((unsigned long)domain->gcr3_tbl);
}

1903 1904 1905 1906
/*
 * Free a domain, only used if something went wrong in the
 * allocation path and we need to free an already allocated page table
 */
1907 1908
static void dma_ops_domain_free(struct dma_ops_domain *dom)
{
1909 1910
	int i;

1911 1912 1913
	if (!dom)
		return;

1914 1915
	del_domain_from_list(&dom->domain);

1916
	free_pagetable(&dom->domain);
1917

1918 1919 1920 1921 1922 1923
	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
		if (!dom->aperture[i])
			continue;
		free_page((unsigned long)dom->aperture[i]->bitmap);
		kfree(dom->aperture[i]);
	}
1924 1925 1926 1927

	kfree(dom);
}

1928 1929
/*
 * Allocates a new protection domain usable for the dma_ops functions.
1930
 * It also initializes the page table and the address allocator data
1931 1932
 * structures required for the dma_ops interface
 */
1933
static struct dma_ops_domain *dma_ops_domain_alloc(void)
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945
{
	struct dma_ops_domain *dma_dom;

	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
	if (!dma_dom)
		return NULL;

	spin_lock_init(&dma_dom->domain.lock);

	dma_dom->domain.id = domain_id_alloc();
	if (dma_dom->domain.id == 0)
		goto free_dma_dom;
1946
	INIT_LIST_HEAD(&dma_dom->domain.dev_list);
1947
	dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
1948
	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
1949
	dma_dom->domain.flags = PD_DMA_OPS_MASK;
1950 1951 1952 1953
	dma_dom->domain.priv = dma_dom;
	if (!dma_dom->domain.pt_root)
		goto free_dma_dom;

1954
	dma_dom->need_flush = false;
1955
	dma_dom->target_dev = 0xffff;
1956

1957 1958
	add_domain_to_list(&dma_dom->domain);

1959
	if (alloc_new_range(dma_dom, true, GFP_KERNEL))
1960 1961
		goto free_dma_dom;

1962
	/*
1963 1964
	 * mark the first page as allocated so we never return 0 as
	 * a valid dma-address. So we can use 0 as error value
1965
	 */
1966
	dma_dom->aperture[0]->bitmap[0] = 1;
1967
	dma_dom->next_address = 0;
1968 1969 1970 1971 1972 1973 1974 1975 1976 1977


	return dma_dom;

free_dma_dom:
	dma_ops_domain_free(dma_dom);

	return NULL;
}

1978 1979 1980 1981 1982 1983 1984 1985 1986
/*
 * little helper function to check whether a given protection domain is a
 * dma_ops domain
 */
static bool dma_ops_domain(struct protection_domain *domain)
{
	return domain->flags & PD_DMA_OPS_MASK;
}

1987
static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
1988
{
1989
	u64 pte_root = 0;
1990
	u64 flags = 0;
1991

1992 1993 1994
	if (domain->mode != PAGE_MODE_NONE)
		pte_root = virt_to_phys(domain->pt_root);

1995 1996 1997
	pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
		    << DEV_ENTRY_MODE_SHIFT;
	pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
1998

1999 2000
	flags = amd_iommu_dev_table[devid].data[1];

2001 2002 2003
	if (ats)
		flags |= DTE_FLAG_IOTLB;

2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
	if (domain->flags & PD_IOMMUV2_MASK) {
		u64 gcr3 = __pa(domain->gcr3_tbl);
		u64 glx  = domain->glx;
		u64 tmp;

		pte_root |= DTE_FLAG_GV;
		pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;

		/* First mask out possible old values for GCR3 table */
		tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
		flags    &= ~tmp;

		tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
		flags    &= ~tmp;

		/* Encode GCR3 table into DTE */
		tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
		pte_root |= tmp;

		tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
		flags    |= tmp;

		tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
		flags    |= tmp;
	}

2030 2031 2032 2033 2034
	flags &= ~(0xffffUL);
	flags |= domain->id;

	amd_iommu_dev_table[devid].data[1]  = flags;
	amd_iommu_dev_table[devid].data[0]  = pte_root;
2035 2036 2037 2038 2039 2040 2041 2042 2043
}

static void clear_dte_entry(u16 devid)
{
	/* remove entry from the device table seen by the hardware */
	amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
	amd_iommu_dev_table[devid].data[1] = 0;

	amd_iommu_apply_erratum_63(devid);
2044 2045
}

2046 2047
static void do_attach(struct iommu_dev_data *dev_data,
		      struct protection_domain *domain)
2048 2049
{
	struct amd_iommu *iommu;
2050
	bool ats;
2051

2052 2053
	iommu = amd_iommu_rlookup_table[dev_data->devid];
	ats   = dev_data->ats.enabled;
2054 2055 2056 2057

	/* Update data structures */
	dev_data->domain = domain;
	list_add(&dev_data->list, &domain->dev_list);
2058
	set_dte_entry(dev_data->devid, domain, ats);
2059 2060 2061 2062 2063 2064

	/* Do reference counting */
	domain->dev_iommu[iommu->index] += 1;
	domain->dev_cnt                 += 1;

	/* Flush the DTE entry */
2065
	device_flush_dte(dev_data);
2066 2067
}

2068
static void do_detach(struct iommu_dev_data *dev_data)
2069 2070 2071
{
	struct amd_iommu *iommu;

2072
	iommu = amd_iommu_rlookup_table[dev_data->devid];
2073 2074

	/* decrease reference counters */
2075 2076 2077 2078 2079 2080
	dev_data->domain->dev_iommu[iommu->index] -= 1;
	dev_data->domain->dev_cnt                 -= 1;

	/* Update data structures */
	dev_data->domain = NULL;
	list_del(&dev_data->list);
2081
	clear_dte_entry(dev_data->devid);
2082

2083
	/* Flush the DTE entry */
2084
	device_flush_dte(dev_data);
2085 2086 2087 2088 2089 2090
}

/*
 * If a device is not yet associated with a domain, this function does
 * assigns it visible for the hardware
 */
2091
static int __attach_device(struct iommu_dev_data *dev_data,
2092
			   struct protection_domain *domain)
2093
{
2094
	int ret;
2095

2096 2097 2098
	/* lock domain */
	spin_lock(&domain->lock);

2099 2100
	if (dev_data->alias_data != NULL) {
		struct iommu_dev_data *alias_data = dev_data->alias_data;
2101

2102 2103 2104 2105 2106
		/* Some sanity checks */
		ret = -EBUSY;
		if (alias_data->domain != NULL &&
				alias_data->domain != domain)
			goto out_unlock;
2107

2108 2109 2110
		if (dev_data->domain != NULL &&
				dev_data->domain != domain)
			goto out_unlock;
2111

2112
		/* Do real assignment */
2113
		if (alias_data->domain == NULL)
2114
			do_attach(alias_data, domain);
2115 2116

		atomic_inc(&alias_data->bind);
2117
	}
2118

2119
	if (dev_data->domain == NULL)
2120
		do_attach(dev_data, domain);
2121

2122 2123
	atomic_inc(&dev_data->bind);

2124 2125 2126 2127
	ret = 0;

out_unlock:

2128 2129
	/* ready */
	spin_unlock(&domain->lock);
2130

2131
	return ret;
2132
}
2133

2134 2135 2136 2137 2138 2139 2140 2141

static void pdev_iommuv2_disable(struct pci_dev *pdev)
{
	pci_disable_ats(pdev);
	pci_disable_pri(pdev);
	pci_disable_pasid(pdev);
}

2142 2143 2144 2145 2146 2147
/* FIXME: Change generic reset-function to do the same */
static int pri_reset_while_enabled(struct pci_dev *pdev)
{
	u16 control;
	int pos;

2148
	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2149 2150 2151
	if (!pos)
		return -EINVAL;

2152 2153 2154
	pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
	control |= PCI_PRI_CTRL_RESET;
	pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
2155 2156 2157 2158

	return 0;
}

2159 2160
static int pdev_iommuv2_enable(struct pci_dev *pdev)
{
2161 2162 2163 2164 2165 2166 2167 2168
	bool reset_enable;
	int reqs, ret;

	/* FIXME: Hardcode number of outstanding requests for now */
	reqs = 32;
	if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
		reqs = 1;
	reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);
2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179

	/* Only allow access to user-accessible pages */
	ret = pci_enable_pasid(pdev, 0);
	if (ret)
		goto out_err;

	/* First reset the PRI state of the device */
	ret = pci_reset_pri(pdev);
	if (ret)
		goto out_err;

2180 2181
	/* Enable PRI */
	ret = pci_enable_pri(pdev, reqs);
2182 2183 2184
	if (ret)
		goto out_err;

2185 2186 2187 2188 2189 2190
	if (reset_enable) {
		ret = pri_reset_while_enabled(pdev);
		if (ret)
			goto out_err;
	}

2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203
	ret = pci_enable_ats(pdev, PAGE_SHIFT);
	if (ret)
		goto out_err;

	return 0;

out_err:
	pci_disable_pri(pdev);
	pci_disable_pasid(pdev);

	return ret;
}

2204
/* FIXME: Move this to PCI code */
2205
#define PCI_PRI_TLP_OFF		(1 << 15)
2206

J
Joerg Roedel 已提交
2207
static bool pci_pri_tlp_required(struct pci_dev *pdev)
2208
{
2209
	u16 status;
2210 2211
	int pos;

2212
	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2213 2214 2215
	if (!pos)
		return false;

2216
	pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
2217

2218
	return (status & PCI_PRI_TLP_OFF) ? true : false;
2219 2220
}

2221
/*
F
Frank Arnold 已提交
2222
 * If a device is not yet associated with a domain, this function
2223 2224
 * assigns it visible for the hardware
 */
2225 2226
static int attach_device(struct device *dev,
			 struct protection_domain *domain)
2227
{
2228
	struct pci_dev *pdev = to_pci_dev(dev);
2229
	struct iommu_dev_data *dev_data;
2230
	unsigned long flags;
2231
	int ret;
2232

2233 2234
	dev_data = get_dev_data(dev);

2235 2236 2237 2238 2239 2240 2241 2242 2243
	if (domain->flags & PD_IOMMUV2_MASK) {
		if (!dev_data->iommu_v2 || !dev_data->passthrough)
			return -EINVAL;

		if (pdev_iommuv2_enable(pdev) != 0)
			return -EINVAL;

		dev_data->ats.enabled = true;
		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
2244
		dev_data->pri_tlp     = pci_pri_tlp_required(pdev);
2245 2246
	} else if (amd_iommu_iotlb_sup &&
		   pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
2247 2248 2249
		dev_data->ats.enabled = true;
		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
	}
2250

2251
	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2252
	ret = __attach_device(dev_data, domain);
2253 2254
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

2255 2256 2257 2258 2259
	/*
	 * We might boot into a crash-kernel here. The crashed kernel
	 * left the caches in the IOMMU dirty. So we have to flush
	 * here to evict all dirty stuff.
	 */
2260
	domain_flush_tlb_pde(domain);
2261 2262

	return ret;
2263 2264
}

2265 2266 2267
/*
 * Removes a device from a protection domain (unlocked)
 */
2268
static void __detach_device(struct iommu_dev_data *dev_data)
2269
{
2270
	struct protection_domain *domain;
2271
	unsigned long flags;
2272

2273
	BUG_ON(!dev_data->domain);
2274

2275 2276 2277
	domain = dev_data->domain;

	spin_lock_irqsave(&domain->lock, flags);
2278

2279 2280 2281
	if (dev_data->alias_data != NULL) {
		struct iommu_dev_data *alias_data = dev_data->alias_data;

2282
		if (atomic_dec_and_test(&alias_data->bind))
2283
			do_detach(alias_data);
2284 2285
	}

2286
	if (atomic_dec_and_test(&dev_data->bind))
2287
		do_detach(dev_data);
2288

2289
	spin_unlock_irqrestore(&domain->lock, flags);
2290 2291 2292

	/*
	 * If we run in passthrough mode the device must be assigned to the
2293 2294
	 * passthrough domain if it is detached from any other domain.
	 * Make sure we can deassign from the pt_domain itself.
2295
	 */
2296
	if (dev_data->passthrough &&
2297
	    (dev_data->domain == NULL && domain != pt_domain))
2298
		__attach_device(dev_data, pt_domain);
2299 2300 2301 2302 2303
}

/*
 * Removes a device from a protection domain (with devtable_lock held)
 */
2304
static void detach_device(struct device *dev)
2305
{
2306
	struct protection_domain *domain;
2307
	struct iommu_dev_data *dev_data;
2308 2309
	unsigned long flags;

2310
	dev_data = get_dev_data(dev);
2311
	domain   = dev_data->domain;
2312

2313 2314
	/* lock device table */
	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2315
	__detach_device(dev_data);
2316
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2317

2318 2319 2320
	if (domain->flags & PD_IOMMUV2_MASK)
		pdev_iommuv2_disable(to_pci_dev(dev));
	else if (dev_data->ats.enabled)
2321
		pci_disable_ats(to_pci_dev(dev));
2322 2323

	dev_data->ats.enabled = false;
2324
}
2325

2326 2327 2328 2329 2330 2331
/*
 * Find out the protection domain structure for a given PCI device. This
 * will give us the pointer to the page table root for example.
 */
static struct protection_domain *domain_for_device(struct device *dev)
{
2332
	struct iommu_dev_data *dev_data;
2333
	struct protection_domain *dom = NULL;
2334 2335
	unsigned long flags;

2336
	dev_data   = get_dev_data(dev);
2337

2338 2339
	if (dev_data->domain)
		return dev_data->domain;
2340

2341 2342
	if (dev_data->alias_data != NULL) {
		struct iommu_dev_data *alias_data = dev_data->alias_data;
2343 2344 2345 2346 2347 2348 2349 2350

		read_lock_irqsave(&amd_iommu_devtable_lock, flags);
		if (alias_data->domain != NULL) {
			__attach_device(dev_data, alias_data->domain);
			dom = alias_data->domain;
		}
		read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
	}
2351 2352 2353 2354

	return dom;
}

2355 2356 2357 2358
static int device_change_notifier(struct notifier_block *nb,
				  unsigned long action, void *data)
{
	struct dma_ops_domain *dma_domain;
2359 2360 2361
	struct protection_domain *domain;
	struct iommu_dev_data *dev_data;
	struct device *dev = data;
2362
	struct amd_iommu *iommu;
2363
	unsigned long flags;
2364
	u16 devid;
2365

2366 2367
	if (!check_device(dev))
		return 0;
2368

2369 2370 2371
	devid    = get_device_id(dev);
	iommu    = amd_iommu_rlookup_table[devid];
	dev_data = get_dev_data(dev);
2372 2373

	switch (action) {
2374
	case BUS_NOTIFY_UNBOUND_DRIVER:
2375 2376 2377

		domain = domain_for_device(dev);

2378 2379
		if (!domain)
			goto out;
2380
		if (dev_data->passthrough)
2381
			break;
2382
		detach_device(dev);
2383 2384
		break;
	case BUS_NOTIFY_ADD_DEVICE:
2385 2386 2387

		iommu_init_device(dev);

2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399
		/*
		 * dev_data is still NULL and
		 * got initialized in iommu_init_device
		 */
		dev_data = get_dev_data(dev);

		if (iommu_pass_through || dev_data->iommu_v2) {
			dev_data->passthrough = true;
			attach_device(dev, pt_domain);
			break;
		}

2400 2401
		domain = domain_for_device(dev);

2402 2403 2404 2405
		/* allocate a protection domain if a device is added */
		dma_domain = find_protection_domain(devid);
		if (dma_domain)
			goto out;
2406
		dma_domain = dma_ops_domain_alloc();
2407 2408 2409 2410 2411 2412 2413 2414
		if (!dma_domain)
			goto out;
		dma_domain->target_dev = devid;

		spin_lock_irqsave(&iommu_pd_list_lock, flags);
		list_add_tail(&dma_domain->list, &iommu_pd_list);
		spin_unlock_irqrestore(&iommu_pd_list_lock, flags);

2415 2416
		dev_data = get_dev_data(dev);

2417
		dev->archdata.dma_ops = &amd_iommu_dma_ops;
2418

2419
		break;
2420 2421 2422 2423
	case BUS_NOTIFY_DEL_DEVICE:

		iommu_uninit_device(dev);

2424 2425 2426 2427 2428 2429 2430 2431 2432 2433
	default:
		goto out;
	}

	iommu_completion_wait(iommu);

out:
	return 0;
}

2434
static struct notifier_block device_nb = {
2435 2436
	.notifier_call = device_change_notifier,
};
2437

2438 2439 2440 2441 2442
void amd_iommu_init_notifier(void)
{
	bus_register_notifier(&pci_bus_type, &device_nb);
}

2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455
/*****************************************************************************
 *
 * The next functions belong to the dma_ops mapping/unmapping code.
 *
 *****************************************************************************/

/*
 * In the dma_ops path we only have the struct device. This function
 * finds the corresponding IOMMU, the protection domain and the
 * requestor id for a given device.
 * If the device is not yet associated with a domain this is also done
 * in this function.
 */
2456
static struct protection_domain *get_domain(struct device *dev)
2457
{
2458
	struct protection_domain *domain;
2459
	struct dma_ops_domain *dma_dom;
2460
	u16 devid = get_device_id(dev);
2461

2462
	if (!check_device(dev))
2463
		return ERR_PTR(-EINVAL);
2464

2465 2466 2467
	domain = domain_for_device(dev);
	if (domain != NULL && !dma_ops_domain(domain))
		return ERR_PTR(-EBUSY);
2468

2469 2470
	if (domain != NULL)
		return domain;
2471

F
Frank Arnold 已提交
2472
	/* Device not bound yet - bind it */
2473
	dma_dom = find_protection_domain(devid);
2474
	if (!dma_dom)
2475 2476
		dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
	attach_device(dev, &dma_dom->domain);
2477
	DUMP_printk("Using protection domain %d for device %s\n",
2478
		    dma_dom->domain.id, dev_name(dev));
2479

2480
	return &dma_dom->domain;
2481 2482
}

2483 2484
static void update_device_table(struct protection_domain *domain)
{
2485
	struct iommu_dev_data *dev_data;
2486

2487 2488
	list_for_each_entry(dev_data, &domain->dev_list, list)
		set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
2489 2490 2491 2492 2493 2494 2495 2496
}

static void update_domain(struct protection_domain *domain)
{
	if (!domain->updated)
		return;

	update_device_table(domain);
2497 2498 2499

	domain_flush_devices(domain);
	domain_flush_tlb_pde(domain);
2500 2501 2502 2503

	domain->updated = false;
}

2504 2505 2506 2507 2508 2509
/*
 * This function fetches the PTE for a given address in the aperture
 */
static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
			    unsigned long address)
{
2510
	struct aperture_range *aperture;
2511 2512
	u64 *pte, *pte_page;

2513 2514 2515 2516 2517
	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
	if (!aperture)
		return NULL;

	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2518
	if (!pte) {
2519
		pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
2520
				GFP_ATOMIC);
2521 2522
		aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
	} else
2523
		pte += PM_LEVEL_INDEX(0, address);
2524

2525
	update_domain(&dom->domain);
2526 2527 2528 2529

	return pte;
}

2530 2531 2532 2533
/*
 * This is the generic map function. It maps one 4kb page at paddr to
 * the given address in the DMA address space for the domain.
 */
2534
static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
2535 2536 2537 2538 2539 2540 2541 2542 2543 2544
				     unsigned long address,
				     phys_addr_t paddr,
				     int direction)
{
	u64 *pte, __pte;

	WARN_ON(address > dom->aperture_size);

	paddr &= PAGE_MASK;

2545
	pte  = dma_ops_get_pte(dom, address);
2546
	if (!pte)
2547
		return DMA_ERROR_CODE;
2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564

	__pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;

	if (direction == DMA_TO_DEVICE)
		__pte |= IOMMU_PTE_IR;
	else if (direction == DMA_FROM_DEVICE)
		__pte |= IOMMU_PTE_IW;
	else if (direction == DMA_BIDIRECTIONAL)
		__pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;

	WARN_ON(*pte);

	*pte = __pte;

	return (dma_addr_t)address;
}

2565 2566 2567
/*
 * The generic unmapping function for on page in the DMA address space.
 */
2568
static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
2569 2570
				 unsigned long address)
{
2571
	struct aperture_range *aperture;
2572 2573 2574 2575 2576
	u64 *pte;

	if (address >= dom->aperture_size)
		return;

2577 2578 2579 2580 2581 2582 2583
	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
	if (!aperture)
		return;

	pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
	if (!pte)
		return;
2584

2585
	pte += PM_LEVEL_INDEX(0, address);
2586 2587 2588 2589 2590 2591

	WARN_ON(!*pte);

	*pte = 0ULL;
}

2592 2593
/*
 * This function contains common code for mapping of a physically
J
Joerg Roedel 已提交
2594 2595
 * contiguous memory region into DMA address space. It is used by all
 * mapping functions provided with this IOMMU driver.
2596 2597
 * Must be called with the domain lock held.
 */
2598 2599 2600 2601
static dma_addr_t __map_single(struct device *dev,
			       struct dma_ops_domain *dma_dom,
			       phys_addr_t paddr,
			       size_t size,
2602
			       int dir,
2603 2604
			       bool align,
			       u64 dma_mask)
2605 2606
{
	dma_addr_t offset = paddr & ~PAGE_MASK;
2607
	dma_addr_t address, start, ret;
2608
	unsigned int pages;
2609
	unsigned long align_mask = 0;
2610 2611
	int i;

2612
	pages = iommu_num_pages(paddr, size, PAGE_SIZE);
2613 2614
	paddr &= PAGE_MASK;

2615 2616
	INC_STATS_COUNTER(total_map_requests);

2617 2618 2619
	if (pages > 1)
		INC_STATS_COUNTER(cross_page);

2620 2621 2622
	if (align)
		align_mask = (1UL << get_order(size)) - 1;

2623
retry:
2624 2625
	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
					  dma_mask);
2626
	if (unlikely(address == DMA_ERROR_CODE)) {
2627 2628 2629 2630 2631 2632 2633
		/*
		 * setting next_address here will let the address
		 * allocator only scan the new allocated range in the
		 * first run. This is a small optimization.
		 */
		dma_dom->next_address = dma_dom->aperture_size;

2634
		if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
2635 2636 2637
			goto out;

		/*
2638
		 * aperture was successfully enlarged by 128 MB, try
2639 2640 2641 2642
		 * allocation again
		 */
		goto retry;
	}
2643 2644 2645

	start = address;
	for (i = 0; i < pages; ++i) {
2646
		ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
2647
		if (ret == DMA_ERROR_CODE)
2648 2649
			goto out_unmap;

2650 2651 2652 2653 2654
		paddr += PAGE_SIZE;
		start += PAGE_SIZE;
	}
	address += offset;

2655 2656
	ADD_STATS_COUNTER(alloced_io_mem, size);

2657
	if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
2658
		domain_flush_tlb(&dma_dom->domain);
2659
		dma_dom->need_flush = false;
2660
	} else if (unlikely(amd_iommu_np_cache))
2661
		domain_flush_pages(&dma_dom->domain, address, size);
2662

2663 2664
out:
	return address;
2665 2666 2667 2668 2669

out_unmap:

	for (--i; i >= 0; --i) {
		start -= PAGE_SIZE;
2670
		dma_ops_domain_unmap(dma_dom, start);
2671 2672 2673 2674
	}

	dma_ops_free_addresses(dma_dom, address, pages);

2675
	return DMA_ERROR_CODE;
2676 2677
}

2678 2679 2680 2681
/*
 * Does the reverse of the __map_single function. Must be called with
 * the domain lock held too
 */
2682
static void __unmap_single(struct dma_ops_domain *dma_dom,
2683 2684 2685 2686
			   dma_addr_t dma_addr,
			   size_t size,
			   int dir)
{
2687
	dma_addr_t flush_addr;
2688 2689 2690
	dma_addr_t i, start;
	unsigned int pages;

2691
	if ((dma_addr == DMA_ERROR_CODE) ||
2692
	    (dma_addr + size > dma_dom->aperture_size))
2693 2694
		return;

2695
	flush_addr = dma_addr;
2696
	pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
2697 2698 2699 2700
	dma_addr &= PAGE_MASK;
	start = dma_addr;

	for (i = 0; i < pages; ++i) {
2701
		dma_ops_domain_unmap(dma_dom, start);
2702 2703 2704
		start += PAGE_SIZE;
	}

2705 2706
	SUB_STATS_COUNTER(alloced_io_mem, size);

2707
	dma_ops_free_addresses(dma_dom, dma_addr, pages);
2708

2709
	if (amd_iommu_unmap_flush || dma_dom->need_flush) {
2710
		domain_flush_pages(&dma_dom->domain, flush_addr, size);
2711 2712
		dma_dom->need_flush = false;
	}
2713 2714
}

2715 2716 2717
/*
 * The exported map_single function for dma_ops.
 */
2718 2719 2720 2721
static dma_addr_t map_page(struct device *dev, struct page *page,
			   unsigned long offset, size_t size,
			   enum dma_data_direction dir,
			   struct dma_attrs *attrs)
2722 2723 2724 2725
{
	unsigned long flags;
	struct protection_domain *domain;
	dma_addr_t addr;
2726
	u64 dma_mask;
2727
	phys_addr_t paddr = page_to_phys(page) + offset;
2728

2729 2730
	INC_STATS_COUNTER(cnt_map_single);

2731 2732
	domain = get_domain(dev);
	if (PTR_ERR(domain) == -EINVAL)
2733
		return (dma_addr_t)paddr;
2734 2735
	else if (IS_ERR(domain))
		return DMA_ERROR_CODE;
2736

2737 2738
	dma_mask = *dev->dma_mask;

2739
	spin_lock_irqsave(&domain->lock, flags);
2740

2741
	addr = __map_single(dev, domain->priv, paddr, size, dir, false,
2742
			    dma_mask);
2743
	if (addr == DMA_ERROR_CODE)
2744 2745
		goto out;

2746
	domain_flush_complete(domain);
2747 2748 2749 2750 2751 2752 2753

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return addr;
}

2754 2755 2756
/*
 * The exported unmap_single function for dma_ops.
 */
2757 2758
static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
		       enum dma_data_direction dir, struct dma_attrs *attrs)
2759 2760 2761 2762
{
	unsigned long flags;
	struct protection_domain *domain;

2763 2764
	INC_STATS_COUNTER(cnt_unmap_single);

2765 2766
	domain = get_domain(dev);
	if (IS_ERR(domain))
2767 2768
		return;

2769 2770
	spin_lock_irqsave(&domain->lock, flags);

2771
	__unmap_single(domain->priv, dma_addr, size, dir);
2772

2773
	domain_flush_complete(domain);
2774 2775 2776 2777

	spin_unlock_irqrestore(&domain->lock, flags);
}

2778 2779 2780 2781
/*
 * This is a special map_sg function which is used if we should map a
 * device which is not handled by an AMD IOMMU in the system.
 */
2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795
static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
			   int nelems, int dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sglist, s, nelems, i) {
		s->dma_address = (dma_addr_t)sg_phys(s);
		s->dma_length  = s->length;
	}

	return nelems;
}

2796 2797 2798 2799
/*
 * The exported map_sg function for dma_ops (handles scatter-gather
 * lists).
 */
2800
static int map_sg(struct device *dev, struct scatterlist *sglist,
2801 2802
		  int nelems, enum dma_data_direction dir,
		  struct dma_attrs *attrs)
2803 2804 2805 2806 2807 2808 2809
{
	unsigned long flags;
	struct protection_domain *domain;
	int i;
	struct scatterlist *s;
	phys_addr_t paddr;
	int mapped_elems = 0;
2810
	u64 dma_mask;
2811

2812 2813
	INC_STATS_COUNTER(cnt_map_sg);

2814 2815
	domain = get_domain(dev);
	if (PTR_ERR(domain) == -EINVAL)
2816
		return map_sg_no_iommu(dev, sglist, nelems, dir);
2817 2818
	else if (IS_ERR(domain))
		return 0;
2819

2820
	dma_mask = *dev->dma_mask;
2821 2822 2823 2824 2825 2826

	spin_lock_irqsave(&domain->lock, flags);

	for_each_sg(sglist, s, nelems, i) {
		paddr = sg_phys(s);

2827
		s->dma_address = __map_single(dev, domain->priv,
2828 2829
					      paddr, s->length, dir, false,
					      dma_mask);
2830 2831 2832 2833 2834 2835 2836 2837

		if (s->dma_address) {
			s->dma_length = s->length;
			mapped_elems++;
		} else
			goto unmap;
	}

2838
	domain_flush_complete(domain);
2839 2840 2841 2842 2843 2844 2845 2846

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return mapped_elems;
unmap:
	for_each_sg(sglist, s, mapped_elems, i) {
		if (s->dma_address)
2847
			__unmap_single(domain->priv, s->dma_address,
2848 2849 2850 2851 2852 2853 2854 2855 2856
				       s->dma_length, dir);
		s->dma_address = s->dma_length = 0;
	}

	mapped_elems = 0;

	goto out;
}

2857 2858 2859 2860
/*
 * The exported map_sg function for dma_ops (handles scatter-gather
 * lists).
 */
2861
static void unmap_sg(struct device *dev, struct scatterlist *sglist,
2862 2863
		     int nelems, enum dma_data_direction dir,
		     struct dma_attrs *attrs)
2864 2865 2866 2867 2868 2869
{
	unsigned long flags;
	struct protection_domain *domain;
	struct scatterlist *s;
	int i;

2870 2871
	INC_STATS_COUNTER(cnt_unmap_sg);

2872 2873
	domain = get_domain(dev);
	if (IS_ERR(domain))
2874 2875
		return;

2876 2877 2878
	spin_lock_irqsave(&domain->lock, flags);

	for_each_sg(sglist, s, nelems, i) {
2879
		__unmap_single(domain->priv, s->dma_address,
2880 2881 2882 2883
			       s->dma_length, dir);
		s->dma_address = s->dma_length = 0;
	}

2884
	domain_flush_complete(domain);
2885 2886 2887 2888

	spin_unlock_irqrestore(&domain->lock, flags);
}

2889 2890 2891
/*
 * The exported alloc_coherent function for dma_ops.
 */
2892
static void *alloc_coherent(struct device *dev, size_t size,
2893 2894
			    dma_addr_t *dma_addr, gfp_t flag,
			    struct dma_attrs *attrs)
2895 2896 2897 2898 2899
{
	unsigned long flags;
	void *virt_addr;
	struct protection_domain *domain;
	phys_addr_t paddr;
2900
	u64 dma_mask = dev->coherent_dma_mask;
2901

2902 2903
	INC_STATS_COUNTER(cnt_alloc_coherent);

2904 2905
	domain = get_domain(dev);
	if (PTR_ERR(domain) == -EINVAL) {
2906 2907 2908
		virt_addr = (void *)__get_free_pages(flag, get_order(size));
		*dma_addr = __pa(virt_addr);
		return virt_addr;
2909 2910
	} else if (IS_ERR(domain))
		return NULL;
2911

2912 2913 2914
	dma_mask  = dev->coherent_dma_mask;
	flag     &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
	flag     |= __GFP_ZERO;
2915 2916 2917

	virt_addr = (void *)__get_free_pages(flag, get_order(size));
	if (!virt_addr)
2918
		return NULL;
2919 2920 2921

	paddr = virt_to_phys(virt_addr);

2922 2923 2924
	if (!dma_mask)
		dma_mask = *dev->dma_mask;

2925 2926
	spin_lock_irqsave(&domain->lock, flags);

2927
	*dma_addr = __map_single(dev, domain->priv, paddr,
2928
				 size, DMA_BIDIRECTIONAL, true, dma_mask);
2929

2930
	if (*dma_addr == DMA_ERROR_CODE) {
J
Jiri Slaby 已提交
2931
		spin_unlock_irqrestore(&domain->lock, flags);
2932
		goto out_free;
J
Jiri Slaby 已提交
2933
	}
2934

2935
	domain_flush_complete(domain);
2936 2937 2938 2939

	spin_unlock_irqrestore(&domain->lock, flags);

	return virt_addr;
2940 2941 2942 2943 2944 2945

out_free:

	free_pages((unsigned long)virt_addr, get_order(size));

	return NULL;
2946 2947
}

2948 2949 2950
/*
 * The exported free_coherent function for dma_ops.
 */
2951
static void free_coherent(struct device *dev, size_t size,
2952 2953
			  void *virt_addr, dma_addr_t dma_addr,
			  struct dma_attrs *attrs)
2954 2955 2956 2957
{
	unsigned long flags;
	struct protection_domain *domain;

2958 2959
	INC_STATS_COUNTER(cnt_free_coherent);

2960 2961
	domain = get_domain(dev);
	if (IS_ERR(domain))
2962 2963
		goto free_mem;

2964 2965
	spin_lock_irqsave(&domain->lock, flags);

2966
	__unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
2967

2968
	domain_flush_complete(domain);
2969 2970 2971 2972 2973 2974 2975

	spin_unlock_irqrestore(&domain->lock, flags);

free_mem:
	free_pages((unsigned long)virt_addr, get_order(size));
}

2976 2977 2978 2979 2980 2981
/*
 * This function is called by the DMA layer to find out if we can handle a
 * particular device. It is part of the dma_ops.
 */
static int amd_iommu_dma_supported(struct device *dev, u64 mask)
{
2982
	return check_device(dev);
2983 2984
}

2985
/*
2986 2987
 * The function for pre-allocating protection domains.
 *
2988 2989 2990 2991
 * If the driver core informs the DMA layer if a driver grabs a device
 * we don't need to preallocate the protection domains anymore.
 * For now we have to.
 */
S
Steffen Persvold 已提交
2992
static void __init prealloc_protection_domains(void)
2993
{
2994
	struct iommu_dev_data *dev_data;
2995
	struct dma_ops_domain *dma_dom;
2996
	struct pci_dev *dev = NULL;
2997
	u16 devid;
2998

2999
	for_each_pci_dev(dev) {
3000 3001 3002

		/* Do we handle this device? */
		if (!check_device(&dev->dev))
3003
			continue;
3004

3005 3006 3007 3008 3009 3010
		dev_data = get_dev_data(&dev->dev);
		if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
			/* Make sure passthrough domain is allocated */
			alloc_passthrough_domain();
			dev_data->passthrough = true;
			attach_device(&dev->dev, pt_domain);
F
Frank Arnold 已提交
3011
			pr_info("AMD-Vi: Using passthrough domain for device %s\n",
3012 3013 3014
				dev_name(&dev->dev));
		}

3015
		/* Is there already any domain for it? */
3016
		if (domain_for_device(&dev->dev))
3017
			continue;
3018 3019 3020

		devid = get_device_id(&dev->dev);

3021
		dma_dom = dma_ops_domain_alloc();
3022 3023 3024
		if (!dma_dom)
			continue;
		init_unity_mappings_for_device(dma_dom, devid);
3025 3026
		dma_dom->target_dev = devid;

3027
		attach_device(&dev->dev, &dma_dom->domain);
3028

3029
		list_add_tail(&dma_dom->list, &iommu_pd_list);
3030 3031 3032
	}
}

3033
static struct dma_map_ops amd_iommu_dma_ops = {
3034 3035
	.alloc = alloc_coherent,
	.free = free_coherent,
3036 3037
	.map_page = map_page,
	.unmap_page = unmap_page,
3038 3039
	.map_sg = map_sg,
	.unmap_sg = unmap_sg,
3040
	.dma_supported = amd_iommu_dma_supported,
3041 3042
};

3043 3044
static unsigned device_dma_ops_init(void)
{
3045
	struct iommu_dev_data *dev_data;
3046 3047 3048 3049 3050
	struct pci_dev *pdev = NULL;
	unsigned unhandled = 0;

	for_each_pci_dev(pdev) {
		if (!check_device(&pdev->dev)) {
3051 3052 3053

			iommu_ignore_device(&pdev->dev);

3054 3055 3056 3057
			unhandled += 1;
			continue;
		}

3058 3059 3060 3061 3062 3063
		dev_data = get_dev_data(&pdev->dev);

		if (!dev_data->passthrough)
			pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
		else
			pdev->dev.archdata.dma_ops = &nommu_dma_ops;
3064 3065 3066 3067 3068
	}

	return unhandled;
}

3069 3070 3071
/*
 * The function which clues the AMD IOMMU driver into dma_ops.
 */
3072 3073 3074

void __init amd_iommu_init_api(void)
{
3075
	bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
3076 3077
}

3078 3079 3080
int __init amd_iommu_init_dma_ops(void)
{
	struct amd_iommu *iommu;
3081
	int ret, unhandled;
3082

3083 3084 3085 3086 3087
	/*
	 * first allocate a default protection domain for every IOMMU we
	 * found in the system. Devices not assigned to any other
	 * protection domain will be assigned to the default one.
	 */
3088
	for_each_iommu(iommu) {
3089
		iommu->default_dom = dma_ops_domain_alloc();
3090 3091
		if (iommu->default_dom == NULL)
			return -ENOMEM;
3092
		iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
3093 3094 3095 3096 3097
		ret = iommu_init_unity_mappings(iommu);
		if (ret)
			goto free_domains;
	}

3098
	/*
3099
	 * Pre-allocate the protection domains for each device.
3100
	 */
3101
	prealloc_protection_domains();
3102 3103

	iommu_detected = 1;
3104
	swiotlb = 0;
3105

3106
	/* Make the driver finally visible to the drivers */
3107 3108 3109 3110 3111
	unhandled = device_dma_ops_init();
	if (unhandled && max_pfn > MAX_DMA32_PFN) {
		/* There are unhandled devices - initialize swiotlb for them */
		swiotlb = 1;
	}
3112

3113 3114
	amd_iommu_stats_init();

3115 3116 3117 3118 3119
	if (amd_iommu_unmap_flush)
		pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");
	else
		pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");

3120 3121 3122 3123
	return 0;

free_domains:

3124
	for_each_iommu(iommu) {
3125 3126 3127 3128 3129 3130
		if (iommu->default_dom)
			dma_ops_domain_free(iommu->default_dom);
	}

	return ret;
}
3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143

/*****************************************************************************
 *
 * The following functions belong to the exported interface of AMD IOMMU
 *
 * This interface allows access to lower level functions of the IOMMU
 * like protection domain handling and assignement of devices to domains
 * which is not possible with the dma_ops interface.
 *
 *****************************************************************************/

static void cleanup_domain(struct protection_domain *domain)
{
3144
	struct iommu_dev_data *dev_data, *next;
3145 3146 3147 3148
	unsigned long flags;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);

3149
	list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) {
3150
		__detach_device(dev_data);
3151 3152
		atomic_set(&dev_data->bind, 0);
	}
3153 3154 3155 3156

	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}

3157 3158 3159 3160 3161
static void protection_domain_free(struct protection_domain *domain)
{
	if (!domain)
		return;

3162 3163
	del_domain_from_list(domain);

3164 3165 3166 3167 3168 3169 3170
	if (domain->id)
		domain_id_free(domain->id);

	kfree(domain);
}

static struct protection_domain *protection_domain_alloc(void)
3171 3172 3173 3174 3175
{
	struct protection_domain *domain;

	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
	if (!domain)
3176
		return NULL;
3177 3178

	spin_lock_init(&domain->lock);
3179
	mutex_init(&domain->api_lock);
3180 3181
	domain->id = domain_id_alloc();
	if (!domain->id)
3182
		goto out_err;
3183
	INIT_LIST_HEAD(&domain->dev_list);
3184

3185 3186
	add_domain_to_list(domain);

3187 3188 3189 3190 3191 3192 3193 3194
	return domain;

out_err:
	kfree(domain);

	return NULL;
}

3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208
static int __init alloc_passthrough_domain(void)
{
	if (pt_domain != NULL)
		return 0;

	/* allocate passthrough domain */
	pt_domain = protection_domain_alloc();
	if (!pt_domain)
		return -ENOMEM;

	pt_domain->mode = PAGE_MODE_NONE;

	return 0;
}
3209 3210 3211 3212 3213 3214
static int amd_iommu_domain_init(struct iommu_domain *dom)
{
	struct protection_domain *domain;

	domain = protection_domain_alloc();
	if (!domain)
3215
		goto out_free;
3216 3217

	domain->mode    = PAGE_MODE_3_LEVEL;
3218 3219 3220 3221
	domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
	if (!domain->pt_root)
		goto out_free;

3222 3223
	domain->iommu_domain = dom;

3224 3225
	dom->priv = domain;

3226 3227 3228 3229
	dom->geometry.aperture_start = 0;
	dom->geometry.aperture_end   = ~0ULL;
	dom->geometry.force_aperture = true;

3230 3231 3232
	return 0;

out_free:
3233
	protection_domain_free(domain);
3234 3235 3236 3237

	return -ENOMEM;
}

3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249
static void amd_iommu_domain_destroy(struct iommu_domain *dom)
{
	struct protection_domain *domain = dom->priv;

	if (!domain)
		return;

	if (domain->dev_cnt > 0)
		cleanup_domain(domain);

	BUG_ON(domain->dev_cnt != 0);

3250 3251
	if (domain->mode != PAGE_MODE_NONE)
		free_pagetable(domain);
3252

3253 3254 3255
	if (domain->flags & PD_IOMMUV2_MASK)
		free_gcr3_table(domain);

3256
	protection_domain_free(domain);
3257 3258 3259 3260

	dom->priv = NULL;
}

3261 3262 3263
static void amd_iommu_detach_device(struct iommu_domain *dom,
				    struct device *dev)
{
3264
	struct iommu_dev_data *dev_data = dev->archdata.iommu;
3265 3266 3267
	struct amd_iommu *iommu;
	u16 devid;

3268
	if (!check_device(dev))
3269 3270
		return;

3271
	devid = get_device_id(dev);
3272

3273
	if (dev_data->domain != NULL)
3274
		detach_device(dev);
3275 3276 3277 3278 3279 3280 3281 3282

	iommu = amd_iommu_rlookup_table[devid];
	if (!iommu)
		return;

	iommu_completion_wait(iommu);
}

3283 3284 3285 3286
static int amd_iommu_attach_device(struct iommu_domain *dom,
				   struct device *dev)
{
	struct protection_domain *domain = dom->priv;
3287
	struct iommu_dev_data *dev_data;
3288
	struct amd_iommu *iommu;
3289
	int ret;
3290

3291
	if (!check_device(dev))
3292 3293
		return -EINVAL;

3294 3295
	dev_data = dev->archdata.iommu;

3296
	iommu = amd_iommu_rlookup_table[dev_data->devid];
3297 3298 3299
	if (!iommu)
		return -EINVAL;

3300
	if (dev_data->domain)
3301
		detach_device(dev);
3302

3303
	ret = attach_device(dev, domain);
3304 3305 3306

	iommu_completion_wait(iommu);

3307
	return ret;
3308 3309
}

3310
static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
3311
			 phys_addr_t paddr, size_t page_size, int iommu_prot)
3312 3313 3314 3315 3316
{
	struct protection_domain *domain = dom->priv;
	int prot = 0;
	int ret;

3317 3318 3319
	if (domain->mode == PAGE_MODE_NONE)
		return -EINVAL;

3320 3321 3322 3323 3324
	if (iommu_prot & IOMMU_READ)
		prot |= IOMMU_PROT_IR;
	if (iommu_prot & IOMMU_WRITE)
		prot |= IOMMU_PROT_IW;

3325
	mutex_lock(&domain->api_lock);
3326
	ret = iommu_map_page(domain, iova, paddr, prot, page_size);
3327 3328
	mutex_unlock(&domain->api_lock);

3329
	return ret;
3330 3331
}

3332 3333
static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
			   size_t page_size)
3334 3335
{
	struct protection_domain *domain = dom->priv;
3336
	size_t unmap_size;
3337

3338 3339 3340
	if (domain->mode == PAGE_MODE_NONE)
		return -EINVAL;

3341
	mutex_lock(&domain->api_lock);
3342
	unmap_size = iommu_unmap_page(domain, iova, page_size);
3343
	mutex_unlock(&domain->api_lock);
3344

3345
	domain_flush_tlb_pde(domain);
3346

3347
	return unmap_size;
3348 3349
}

3350 3351 3352 3353
static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
					  unsigned long iova)
{
	struct protection_domain *domain = dom->priv;
3354
	unsigned long offset_mask;
3355
	phys_addr_t paddr;
3356
	u64 *pte, __pte;
3357

3358 3359 3360
	if (domain->mode == PAGE_MODE_NONE)
		return iova;

3361
	pte = fetch_pte(domain, iova);
3362

3363
	if (!pte || !IOMMU_PTE_PRESENT(*pte))
3364 3365
		return 0;

3366 3367 3368 3369 3370 3371 3372
	if (PM_PTE_LEVEL(*pte) == 0)
		offset_mask = PAGE_SIZE - 1;
	else
		offset_mask = PTE_PAGE_SIZE(*pte) - 1;

	__pte = *pte & PM_ADDR_MASK;
	paddr = (__pte & ~offset_mask) | (iova & offset_mask);
3373 3374 3375 3376

	return paddr;
}

S
Sheng Yang 已提交
3377 3378 3379
static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
				    unsigned long cap)
{
3380 3381 3382
	switch (cap) {
	case IOMMU_CAP_CACHE_COHERENCY:
		return 1;
3383 3384
	case IOMMU_CAP_INTR_REMAP:
		return irq_remapping_enabled;
3385 3386
	}

S
Sheng Yang 已提交
3387 3388 3389
	return 0;
}

3390 3391 3392 3393 3394
static struct iommu_ops amd_iommu_ops = {
	.domain_init = amd_iommu_domain_init,
	.domain_destroy = amd_iommu_domain_destroy,
	.attach_dev = amd_iommu_attach_device,
	.detach_dev = amd_iommu_detach_device,
3395 3396
	.map = amd_iommu_map,
	.unmap = amd_iommu_unmap,
3397
	.iova_to_phys = amd_iommu_iova_to_phys,
S
Sheng Yang 已提交
3398
	.domain_has_cap = amd_iommu_domain_has_cap,
3399
	.pgsize_bitmap	= AMD_IOMMU_PGSIZES,
3400 3401
};

3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413
/*****************************************************************************
 *
 * The next functions do a basic initialization of IOMMU for pass through
 * mode
 *
 * In passthrough mode the IOMMU is initialized and enabled but not used for
 * DMA-API translation.
 *
 *****************************************************************************/

int __init amd_iommu_init_passthrough(void)
{
3414
	struct iommu_dev_data *dev_data;
3415
	struct pci_dev *dev = NULL;
3416
	struct amd_iommu *iommu;
3417
	u16 devid;
3418
	int ret;
3419

3420 3421 3422
	ret = alloc_passthrough_domain();
	if (ret)
		return ret;
3423

3424
	for_each_pci_dev(dev) {
3425
		if (!check_device(&dev->dev))
3426 3427
			continue;

3428 3429 3430
		dev_data = get_dev_data(&dev->dev);
		dev_data->passthrough = true;

3431 3432
		devid = get_device_id(&dev->dev);

3433
		iommu = amd_iommu_rlookup_table[devid];
3434 3435 3436
		if (!iommu)
			continue;

3437
		attach_device(&dev->dev, pt_domain);
3438 3439
	}

J
Joerg Roedel 已提交
3440 3441
	amd_iommu_stats_init();

3442 3443 3444 3445
	pr_info("AMD-Vi: Initialized for Passthrough Mode\n");

	return 0;
}
3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458

/* IOMMUv2 specific functions */
int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
{
	return atomic_notifier_chain_register(&ppr_notifier, nb);
}
EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);

int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
{
	return atomic_notifier_chain_unregister(&ppr_notifier, nb);
}
EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479

void amd_iommu_domain_direct_map(struct iommu_domain *dom)
{
	struct protection_domain *domain = dom->priv;
	unsigned long flags;

	spin_lock_irqsave(&domain->lock, flags);

	/* Update data structure */
	domain->mode    = PAGE_MODE_NONE;
	domain->updated = true;

	/* Make changes visible to IOMMUs */
	update_domain(domain);

	/* Page-table is not visible to IOMMU anymore, so free it */
	free_pagetable(domain);

	spin_unlock_irqrestore(&domain->lock, flags);
}
EXPORT_SYMBOL(amd_iommu_domain_direct_map);
3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526

int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
{
	struct protection_domain *domain = dom->priv;
	unsigned long flags;
	int levels, ret;

	if (pasids <= 0 || pasids > (PASID_MASK + 1))
		return -EINVAL;

	/* Number of GCR3 table levels required */
	for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
		levels += 1;

	if (levels > amd_iommu_max_glx_val)
		return -EINVAL;

	spin_lock_irqsave(&domain->lock, flags);

	/*
	 * Save us all sanity checks whether devices already in the
	 * domain support IOMMUv2. Just force that the domain has no
	 * devices attached when it is switched into IOMMUv2 mode.
	 */
	ret = -EBUSY;
	if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
		goto out;

	ret = -ENOMEM;
	domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
	if (domain->gcr3_tbl == NULL)
		goto out;

	domain->glx      = levels;
	domain->flags   |= PD_IOMMUV2_MASK;
	domain->updated  = true;

	update_domain(domain);

	ret = 0;

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586

static int __flush_pasid(struct protection_domain *domain, int pasid,
			 u64 address, bool size)
{
	struct iommu_dev_data *dev_data;
	struct iommu_cmd cmd;
	int i, ret;

	if (!(domain->flags & PD_IOMMUV2_MASK))
		return -EINVAL;

	build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);

	/*
	 * IOMMU TLB needs to be flushed before Device TLB to
	 * prevent device TLB refill from IOMMU TLB
	 */
	for (i = 0; i < amd_iommus_present; ++i) {
		if (domain->dev_iommu[i] == 0)
			continue;

		ret = iommu_queue_command(amd_iommus[i], &cmd);
		if (ret != 0)
			goto out;
	}

	/* Wait until IOMMU TLB flushes are complete */
	domain_flush_complete(domain);

	/* Now flush device TLBs */
	list_for_each_entry(dev_data, &domain->dev_list, list) {
		struct amd_iommu *iommu;
		int qdep;

		BUG_ON(!dev_data->ats.enabled);

		qdep  = dev_data->ats.qdep;
		iommu = amd_iommu_rlookup_table[dev_data->devid];

		build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
				      qdep, address, size);

		ret = iommu_queue_command(iommu, &cmd);
		if (ret != 0)
			goto out;
	}

	/* Wait until all device TLBs are flushed */
	domain_flush_complete(domain);

	ret = 0;

out:

	return ret;
}

static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
				  u64 address)
{
3587 3588
	INC_STATS_COUNTER(invalidate_iotlb);

3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608
	return __flush_pasid(domain, pasid, address, false);
}

int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
			 u64 address)
{
	struct protection_domain *domain = dom->priv;
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __amd_iommu_flush_page(domain, pasid, address);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_flush_page);

static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
{
3609 3610
	INC_STATS_COUNTER(invalidate_iotlb_all);

3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628
	return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
			     true);
}

int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
{
	struct protection_domain *domain = dom->priv;
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __amd_iommu_flush_tlb(domain, pasid);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_flush_tlb);

3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721
static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
{
	int index;
	u64 *pte;

	while (true) {

		index = (pasid >> (9 * level)) & 0x1ff;
		pte   = &root[index];

		if (level == 0)
			break;

		if (!(*pte & GCR3_VALID)) {
			if (!alloc)
				return NULL;

			root = (void *)get_zeroed_page(GFP_ATOMIC);
			if (root == NULL)
				return NULL;

			*pte = __pa(root) | GCR3_VALID;
		}

		root = __va(*pte & PAGE_MASK);

		level -= 1;
	}

	return pte;
}

static int __set_gcr3(struct protection_domain *domain, int pasid,
		      unsigned long cr3)
{
	u64 *pte;

	if (domain->mode != PAGE_MODE_NONE)
		return -EINVAL;

	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
	if (pte == NULL)
		return -ENOMEM;

	*pte = (cr3 & PAGE_MASK) | GCR3_VALID;

	return __amd_iommu_flush_tlb(domain, pasid);
}

static int __clear_gcr3(struct protection_domain *domain, int pasid)
{
	u64 *pte;

	if (domain->mode != PAGE_MODE_NONE)
		return -EINVAL;

	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
	if (pte == NULL)
		return 0;

	*pte = 0;

	return __amd_iommu_flush_tlb(domain, pasid);
}

int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
			      unsigned long cr3)
{
	struct protection_domain *domain = dom->priv;
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __set_gcr3(domain, pasid, cr3);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);

int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
{
	struct protection_domain *domain = dom->priv;
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __clear_gcr3(domain, pasid);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
3722 3723 3724 3725 3726 3727 3728 3729

int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
			   int status, int tag)
{
	struct iommu_dev_data *dev_data;
	struct amd_iommu *iommu;
	struct iommu_cmd cmd;

3730 3731
	INC_STATS_COUNTER(complete_ppr);

3732 3733 3734 3735 3736 3737 3738 3739 3740
	dev_data = get_dev_data(&pdev->dev);
	iommu    = amd_iommu_rlookup_table[dev_data->devid];

	build_complete_ppr(&cmd, dev_data->devid, pasid, status,
			   tag, dev_data->pri_tlp);

	return iommu_queue_command(iommu, &cmd);
}
EXPORT_SYMBOL(amd_iommu_complete_ppr);
3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756

struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
{
	struct protection_domain *domain;

	domain = get_domain(&pdev->dev);
	if (IS_ERR(domain))
		return NULL;

	/* Only return IOMMUv2 domains */
	if (!(domain->flags & PD_IOMMUV2_MASK))
		return NULL;

	return domain->iommu_domain;
}
EXPORT_SYMBOL(amd_iommu_get_v2_domain);
3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768

void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
{
	struct iommu_dev_data *dev_data;

	if (!amd_iommu_v2_supported())
		return;

	dev_data = get_dev_data(&pdev->dev);
	dev_data->errata |= (1 << erratum);
}
EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811

int amd_iommu_device_info(struct pci_dev *pdev,
                          struct amd_iommu_device_info *info)
{
	int max_pasids;
	int pos;

	if (pdev == NULL || info == NULL)
		return -EINVAL;

	if (!amd_iommu_v2_supported())
		return -EINVAL;

	memset(info, 0, sizeof(*info));

	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
	if (pos)
		info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;

	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
	if (pos)
		info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;

	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
	if (pos) {
		int features;

		max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
		max_pasids = min(max_pasids, (1 << 20));

		info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
		info->max_pasids = min(pci_max_pasids(pdev), max_pasids);

		features = pci_pasid_features(pdev);
		if (features & PCI_PASID_CAP_EXEC)
			info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
		if (features & PCI_PASID_CAP_PRIV)
			info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
	}

	return 0;
}
EXPORT_SYMBOL(amd_iommu_device_info);
3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894

#ifdef CONFIG_IRQ_REMAP

/*****************************************************************************
 *
 * Interrupt Remapping Implementation
 *
 *****************************************************************************/

union irte {
	u32 val;
	struct {
		u32 valid	: 1,
		    no_fault	: 1,
		    int_type	: 3,
		    rq_eoi	: 1,
		    dm		: 1,
		    rsvd_1	: 1,
		    destination	: 8,
		    vector	: 8,
		    rsvd_2	: 8;
	} fields;
};

#define DTE_IRQ_PHYS_ADDR_MASK	(((1ULL << 45)-1) << 6)
#define DTE_IRQ_REMAP_INTCTL    (2ULL << 60)
#define DTE_IRQ_TABLE_LEN       (8ULL << 1)
#define DTE_IRQ_REMAP_ENABLE    1ULL

static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
{
	u64 dte;

	dte	= amd_iommu_dev_table[devid].data[2];
	dte	&= ~DTE_IRQ_PHYS_ADDR_MASK;
	dte	|= virt_to_phys(table->table);
	dte	|= DTE_IRQ_REMAP_INTCTL;
	dte	|= DTE_IRQ_TABLE_LEN;
	dte	|= DTE_IRQ_REMAP_ENABLE;

	amd_iommu_dev_table[devid].data[2] = dte;
}

#define IRTE_ALLOCATED (~1U)

static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
{
	struct irq_remap_table *table = NULL;
	struct amd_iommu *iommu;
	unsigned long flags;
	u16 alias;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);

	iommu = amd_iommu_rlookup_table[devid];
	if (!iommu)
		goto out_unlock;

	table = irq_lookup_table[devid];
	if (table)
		goto out;

	alias = amd_iommu_alias_table[devid];
	table = irq_lookup_table[alias];
	if (table) {
		irq_lookup_table[devid] = table;
		set_dte_irq_entry(devid, table);
		iommu_flush_dte(iommu, devid);
		goto out;
	}

	/* Nothing there yet, allocate new irq remapping table */
	table = kzalloc(sizeof(*table), GFP_ATOMIC);
	if (!table)
		goto out;

	if (ioapic)
		/* Keep the first 32 indexes free for IOAPIC interrupts */
		table->min_index = 32;

	table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_ATOMIC);
	if (!table->table) {
		kfree(table);
3895
		table = NULL;
3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033
		goto out;
	}

	memset(table->table, 0, MAX_IRQS_PER_TABLE * sizeof(u32));

	if (ioapic) {
		int i;

		for (i = 0; i < 32; ++i)
			table->table[i] = IRTE_ALLOCATED;
	}

	irq_lookup_table[devid] = table;
	set_dte_irq_entry(devid, table);
	iommu_flush_dte(iommu, devid);
	if (devid != alias) {
		irq_lookup_table[alias] = table;
		set_dte_irq_entry(devid, table);
		iommu_flush_dte(iommu, alias);
	}

out:
	iommu_completion_wait(iommu);

out_unlock:
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

	return table;
}

static int alloc_irq_index(struct irq_cfg *cfg, u16 devid, int count)
{
	struct irq_remap_table *table;
	unsigned long flags;
	int index, c;

	table = get_irq_table(devid, false);
	if (!table)
		return -ENODEV;

	spin_lock_irqsave(&table->lock, flags);

	/* Scan table for free entries */
	for (c = 0, index = table->min_index;
	     index < MAX_IRQS_PER_TABLE;
	     ++index) {
		if (table->table[index] == 0)
			c += 1;
		else
			c = 0;

		if (c == count)	{
			struct irq_2_iommu *irte_info;

			for (; c != 0; --c)
				table->table[index - c + 1] = IRTE_ALLOCATED;

			index -= count - 1;

			irte_info             = &cfg->irq_2_iommu;
			irte_info->sub_handle = devid;
			irte_info->irte_index = index;
			irte_info->iommu      = (void *)cfg;

			goto out;
		}
	}

	index = -ENOSPC;

out:
	spin_unlock_irqrestore(&table->lock, flags);

	return index;
}

static int get_irte(u16 devid, int index, union irte *irte)
{
	struct irq_remap_table *table;
	unsigned long flags;

	table = get_irq_table(devid, false);
	if (!table)
		return -ENOMEM;

	spin_lock_irqsave(&table->lock, flags);
	irte->val = table->table[index];
	spin_unlock_irqrestore(&table->lock, flags);

	return 0;
}

static int modify_irte(u16 devid, int index, union irte irte)
{
	struct irq_remap_table *table;
	struct amd_iommu *iommu;
	unsigned long flags;

	iommu = amd_iommu_rlookup_table[devid];
	if (iommu == NULL)
		return -EINVAL;

	table = get_irq_table(devid, false);
	if (!table)
		return -ENOMEM;

	spin_lock_irqsave(&table->lock, flags);
	table->table[index] = irte.val;
	spin_unlock_irqrestore(&table->lock, flags);

	iommu_flush_irt(iommu, devid);
	iommu_completion_wait(iommu);

	return 0;
}

static void free_irte(u16 devid, int index)
{
	struct irq_remap_table *table;
	struct amd_iommu *iommu;
	unsigned long flags;

	iommu = amd_iommu_rlookup_table[devid];
	if (iommu == NULL)
		return;

	table = get_irq_table(devid, false);
	if (!table)
		return;

	spin_lock_irqsave(&table->lock, flags);
	table->table[index] = 0;
	spin_unlock_irqrestore(&table->lock, flags);

	iommu_flush_irt(iommu, devid);
	iommu_completion_wait(iommu);
}

4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158
static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,
			      unsigned int destination, int vector,
			      struct io_apic_irq_attr *attr)
{
	struct irq_remap_table *table;
	struct irq_2_iommu *irte_info;
	struct irq_cfg *cfg;
	union irte irte;
	int ioapic_id;
	int index;
	int devid;
	int ret;

	cfg = irq_get_chip_data(irq);
	if (!cfg)
		return -EINVAL;

	irte_info = &cfg->irq_2_iommu;
	ioapic_id = mpc_ioapic_id(attr->ioapic);
	devid     = get_ioapic_devid(ioapic_id);

	if (devid < 0)
		return devid;

	table = get_irq_table(devid, true);
	if (table == NULL)
		return -ENOMEM;

	index = attr->ioapic_pin;

	/* Setup IRQ remapping info */
	irte_info->sub_handle = devid;
	irte_info->irte_index = index;
	irte_info->iommu      = (void *)cfg;

	/* Setup IRTE for IOMMU */
	irte.val		= 0;
	irte.fields.vector      = vector;
	irte.fields.int_type    = apic->irq_delivery_mode;
	irte.fields.destination = destination;
	irte.fields.dm          = apic->irq_dest_mode;
	irte.fields.valid       = 1;

	ret = modify_irte(devid, index, irte);
	if (ret)
		return ret;

	/* Setup IOAPIC entry */
	memset(entry, 0, sizeof(*entry));

	entry->vector        = index;
	entry->mask          = 0;
	entry->trigger       = attr->trigger;
	entry->polarity      = attr->polarity;

	/*
	 * Mask level triggered irqs.
	 */
	if (attr->trigger)
		entry->mask = 1;

	return 0;
}

static int set_affinity(struct irq_data *data, const struct cpumask *mask,
			bool force)
{
	struct irq_2_iommu *irte_info;
	unsigned int dest, irq;
	struct irq_cfg *cfg;
	union irte irte;
	int err;

	if (!config_enabled(CONFIG_SMP))
		return -1;

	cfg       = data->chip_data;
	irq       = data->irq;
	irte_info = &cfg->irq_2_iommu;

	if (!cpumask_intersects(mask, cpu_online_mask))
		return -EINVAL;

	if (get_irte(irte_info->sub_handle, irte_info->irte_index, &irte))
		return -EBUSY;

	if (assign_irq_vector(irq, cfg, mask))
		return -EBUSY;

	err = apic->cpu_mask_to_apicid_and(cfg->domain, mask, &dest);
	if (err) {
		if (assign_irq_vector(irq, cfg, data->affinity))
			pr_err("AMD-Vi: Failed to recover vector for irq %d\n", irq);
		return err;
	}

	irte.fields.vector      = cfg->vector;
	irte.fields.destination = dest;

	modify_irte(irte_info->sub_handle, irte_info->irte_index, irte);

	if (cfg->move_in_progress)
		send_cleanup_vector(cfg);

	cpumask_copy(data->affinity, mask);

	return 0;
}

static int free_irq(int irq)
{
	struct irq_2_iommu *irte_info;
	struct irq_cfg *cfg;

	cfg = irq_get_chip_data(irq);
	if (!cfg)
		return -EINVAL;

	irte_info = &cfg->irq_2_iommu;

	free_irte(irte_info->sub_handle, irte_info->irte_index);

	return 0;
}

4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232
static void compose_msi_msg(struct pci_dev *pdev,
			    unsigned int irq, unsigned int dest,
			    struct msi_msg *msg, u8 hpet_id)
{
	struct irq_2_iommu *irte_info;
	struct irq_cfg *cfg;
	union irte irte;

	cfg = irq_get_chip_data(irq);
	if (!cfg)
		return;

	irte_info = &cfg->irq_2_iommu;

	irte.val		= 0;
	irte.fields.vector	= cfg->vector;
	irte.fields.int_type    = apic->irq_delivery_mode;
	irte.fields.destination	= dest;
	irte.fields.dm		= apic->irq_dest_mode;
	irte.fields.valid	= 1;

	modify_irte(irte_info->sub_handle, irte_info->irte_index, irte);

	msg->address_hi = MSI_ADDR_BASE_HI;
	msg->address_lo = MSI_ADDR_BASE_LO;
	msg->data       = irte_info->irte_index;
}

static int msi_alloc_irq(struct pci_dev *pdev, int irq, int nvec)
{
	struct irq_cfg *cfg;
	int index;
	u16 devid;

	if (!pdev)
		return -EINVAL;

	cfg = irq_get_chip_data(irq);
	if (!cfg)
		return -EINVAL;

	devid = get_device_id(&pdev->dev);
	index = alloc_irq_index(cfg, devid, nvec);

	return index < 0 ? MAX_IRQS_PER_TABLE : index;
}

static int msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
			 int index, int offset)
{
	struct irq_2_iommu *irte_info;
	struct irq_cfg *cfg;
	u16 devid;

	if (!pdev)
		return -EINVAL;

	cfg = irq_get_chip_data(irq);
	if (!cfg)
		return -EINVAL;

	if (index >= MAX_IRQS_PER_TABLE)
		return 0;

	devid		= get_device_id(&pdev->dev);
	irte_info	= &cfg->irq_2_iommu;

	irte_info->sub_handle = devid;
	irte_info->irte_index = index + offset;
	irte_info->iommu      = (void *)cfg;

	return 0;
}

4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258
static int setup_hpet_msi(unsigned int irq, unsigned int id)
{
	struct irq_2_iommu *irte_info;
	struct irq_cfg *cfg;
	int index, devid;

	cfg = irq_get_chip_data(irq);
	if (!cfg)
		return -EINVAL;

	irte_info = &cfg->irq_2_iommu;
	devid     = get_hpet_devid(id);
	if (devid < 0)
		return devid;

	index = alloc_irq_index(cfg, devid, 1);
	if (index < 0)
		return index;

	irte_info->sub_handle = devid;
	irte_info->irte_index = index;
	irte_info->iommu      = (void *)cfg;

	return 0;
}

4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273
struct irq_remap_ops amd_iommu_irq_ops = {
	.supported		= amd_iommu_supported,
	.prepare		= amd_iommu_prepare,
	.enable			= amd_iommu_enable,
	.disable		= amd_iommu_disable,
	.reenable		= amd_iommu_reenable,
	.enable_faulting	= amd_iommu_enable_faulting,
	.setup_ioapic_entry	= setup_ioapic_entry,
	.set_affinity		= set_affinity,
	.free_irq		= free_irq,
	.compose_msi_msg	= compose_msi_msg,
	.msi_alloc_irq		= msi_alloc_irq,
	.msi_setup_irq		= msi_setup_irq,
	.setup_hpet_msi		= setup_hpet_msi,
};
4274
#endif