amd_iommu.c 95.9 KB
Newer Older
1
/*
2
 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
J
Joerg Roedel 已提交
3
 * Author: Joerg Roedel <jroedel@suse.de>
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
 *         Leo Duran <leo.duran@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

20
#include <linux/ratelimit.h>
21
#include <linux/pci.h>
22
#include <linux/acpi.h>
23
#include <linux/pci-ats.h>
A
Akinobu Mita 已提交
24
#include <linux/bitmap.h>
25
#include <linux/slab.h>
26
#include <linux/debugfs.h>
27
#include <linux/scatterlist.h>
28
#include <linux/dma-mapping.h>
29
#include <linux/iommu-helper.h>
30
#include <linux/iommu.h>
31
#include <linux/delay.h>
32
#include <linux/amd-iommu.h>
33 34
#include <linux/notifier.h>
#include <linux/export.h>
35 36
#include <linux/irq.h>
#include <linux/msi.h>
37
#include <linux/dma-contiguous.h>
38
#include <linux/irqdomain.h>
39
#include <linux/percpu.h>
40 41 42 43
#include <asm/irq_remapping.h>
#include <asm/io_apic.h>
#include <asm/apic.h>
#include <asm/hw_irq.h>
44
#include <asm/msidef.h>
45
#include <asm/proto.h>
46
#include <asm/iommu.h>
47
#include <asm/gart.h>
48
#include <asm/dma.h>
49 50 51

#include "amd_iommu_proto.h"
#include "amd_iommu_types.h"
52
#include "irq_remapping.h"
53 54 55

#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))

56
#define LOOP_TIMEOUT	100000
57

58 59 60 61 62 63
/*
 * This bitmap is used to advertise the page sizes our hardware support
 * to the IOMMU core, which will then use this information to split
 * physically contiguous memory regions it is mapping into page sizes
 * that we support.
 *
J
Joerg Roedel 已提交
64
 * 512GB Pages are not supported due to a hardware bug
65
 */
J
Joerg Roedel 已提交
66
#define AMD_IOMMU_PGSIZES	((~0xFFFUL) & ~(2ULL << 38))
67

68 69
static DEFINE_RWLOCK(amd_iommu_devtable_lock);

70 71 72 73
/* List of all available dev_data structures */
static LIST_HEAD(dev_data_list);
static DEFINE_SPINLOCK(dev_data_list_lock);

74 75
LIST_HEAD(ioapic_map);
LIST_HEAD(hpet_map);
76
LIST_HEAD(acpihid_map);
77

78 79 80 81
/*
 * Domain for untranslated devices - only allocated
 * if iommu=pt passed on kernel cmd line.
 */
82
static const struct iommu_ops amd_iommu_ops;
83

84
static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
85
int amd_iommu_max_glx_val = -1;
86

87 88
static struct dma_map_ops amd_iommu_dma_ops;

89 90 91 92 93 94 95 96 97
/*
 * This struct contains device specific data for the IOMMU
 */
struct iommu_dev_data {
	struct list_head list;		  /* For domain->dev_list */
	struct list_head dev_data_list;	  /* For global dev_data_list */
	struct protection_domain *domain; /* Domain the device is bound to */
	u16 devid;			  /* PCI Device ID */
	bool iommu_v2;			  /* Device can make use of IOMMUv2 */
98
	bool passthrough;		  /* Device is identity mapped */
99 100 101 102 103 104 105 106 107
	struct {
		bool enabled;
		int qdep;
	} ats;				  /* ATS state */
	bool pri_tlp;			  /* PASID TLB required for
					     PPR completions */
	u32 errata;			  /* Bitmap for errata to apply */
};

108 109 110
/*
 * general struct to manage commands send to an IOMMU
 */
111
struct iommu_cmd {
112 113 114
	u32 data[4];
};

115 116
struct kmem_cache *amd_iommu_irq_cache;

117
static void update_domain(struct protection_domain *domain);
118
static int protection_domain_init(struct protection_domain *domain);
119
static void detach_device(struct device *dev);
120

121 122 123 124 125 126
/*
 * For dynamic growth the aperture size is split into ranges of 128MB of
 * DMA address space each. This struct represents one such range.
 */
struct aperture_range {

127 128
	spinlock_t bitmap_lock;

129 130
	/* address allocation bitmap */
	unsigned long *bitmap;
131
	unsigned long offset;
132
	unsigned long next_bit;
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152

	/*
	 * Array of PTE pages for the aperture. In this array we save all the
	 * leaf pages of the domain page table used for the aperture. This way
	 * we don't need to walk the page table to find a specific PTE. We can
	 * just calculate its address in constant time.
	 */
	u64 *pte_pages[64];
};

/*
 * Data container for a dma_ops specific protection domain
 */
struct dma_ops_domain {
	/* generic protection domain information */
	struct protection_domain domain;

	/* size of the aperture for the mappings */
	unsigned long aperture_size;

153
	/* aperture index we start searching for free addresses */
154
	u32 __percpu *next_index;
155 156 157 158 159

	/* address space relevant data */
	struct aperture_range *aperture[APERTURE_MAX_RANGES];
};

160 161 162 163 164 165
/****************************************************************************
 *
 * Helper functions
 *
 ****************************************************************************/

166 167 168 169 170
static struct protection_domain *to_pdomain(struct iommu_domain *dom)
{
	return container_of(dom, struct protection_domain, domain);
}

171
static struct iommu_dev_data *alloc_dev_data(u16 devid)
172 173 174 175 176 177 178 179
{
	struct iommu_dev_data *dev_data;
	unsigned long flags;

	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
	if (!dev_data)
		return NULL;

180
	dev_data->devid = devid;
181 182 183 184 185 186 187 188

	spin_lock_irqsave(&dev_data_list_lock, flags);
	list_add_tail(&dev_data->dev_data_list, &dev_data_list);
	spin_unlock_irqrestore(&dev_data_list_lock, flags);

	return dev_data;
}

189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
static struct iommu_dev_data *search_dev_data(u16 devid)
{
	struct iommu_dev_data *dev_data;
	unsigned long flags;

	spin_lock_irqsave(&dev_data_list_lock, flags);
	list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {
		if (dev_data->devid == devid)
			goto out_unlock;
	}

	dev_data = NULL;

out_unlock:
	spin_unlock_irqrestore(&dev_data_list_lock, flags);

	return dev_data;
}

static struct iommu_dev_data *find_dev_data(u16 devid)
{
	struct iommu_dev_data *dev_data;

	dev_data = search_dev_data(devid);

	if (dev_data == NULL)
		dev_data = alloc_dev_data(devid);

	return dev_data;
}

220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
static inline int match_hid_uid(struct device *dev,
				struct acpihid_map_entry *entry)
{
	const char *hid, *uid;

	hid = acpi_device_hid(ACPI_COMPANION(dev));
	uid = acpi_device_uid(ACPI_COMPANION(dev));

	if (!hid || !(*hid))
		return -ENODEV;

	if (!uid || !(*uid))
		return strcmp(hid, entry->hid);

	if (!(*entry->uid))
		return strcmp(hid, entry->hid);

	return (strcmp(hid, entry->hid) || strcmp(uid, entry->uid));
}

static inline u16 get_pci_device_id(struct device *dev)
241 242 243
{
	struct pci_dev *pdev = to_pci_dev(dev);

244
	return PCI_DEVID(pdev->bus->number, pdev->devfn);
245 246
}

247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
static inline int get_acpihid_device_id(struct device *dev,
					struct acpihid_map_entry **entry)
{
	struct acpihid_map_entry *p;

	list_for_each_entry(p, &acpihid_map, list) {
		if (!match_hid_uid(dev, p)) {
			if (entry)
				*entry = p;
			return p->devid;
		}
	}
	return -EINVAL;
}

static inline int get_device_id(struct device *dev)
{
	int devid;

	if (dev_is_pci(dev))
		devid = get_pci_device_id(dev);
	else
		devid = get_acpihid_device_id(dev, NULL);

	return devid;
}

274 275 276 277 278
static struct iommu_dev_data *get_dev_data(struct device *dev)
{
	return dev->archdata.iommu;
}

279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
/*
* Find or create an IOMMU group for a acpihid device.
*/
static struct iommu_group *acpihid_device_group(struct device *dev)
{
	struct acpihid_map_entry *p, *entry = NULL;
	u16 devid;

	devid = get_acpihid_device_id(dev, &entry);
	if (devid < 0)
		return ERR_PTR(devid);

	list_for_each_entry(p, &acpihid_map, list) {
		if ((devid == p->devid) && p->group)
			entry->group = p->group;
	}

	if (!entry->group)
		entry->group = generic_device_group(dev);

	return entry->group;
}

302 303 304 305
static bool pci_iommuv2_capable(struct pci_dev *pdev)
{
	static const int caps[] = {
		PCI_EXT_CAP_ID_ATS,
306 307
		PCI_EXT_CAP_ID_PRI,
		PCI_EXT_CAP_ID_PASID,
308 309 310 311 312 313 314 315 316 317 318 319
	};
	int i, pos;

	for (i = 0; i < 3; ++i) {
		pos = pci_find_ext_capability(pdev, caps[i]);
		if (pos == 0)
			return false;
	}

	return true;
}

320 321 322 323 324 325 326 327 328
static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
{
	struct iommu_dev_data *dev_data;

	dev_data = get_dev_data(&pdev->dev);

	return dev_data->errata & (1 << erratum) ? true : false;
}

329
/*
330 331
 * This function actually applies the mapping to the page table of the
 * dma_ops domain.
332
 */
333 334
static void alloc_unity_mapping(struct dma_ops_domain *dma_dom,
				struct unity_map_entry *e)
335
{
336
	u64 addr;
337

338 339 340 341 342
	for (addr = e->address_start; addr < e->address_end;
	     addr += PAGE_SIZE) {
		if (addr < dma_dom->aperture_size)
			__set_bit(addr >> PAGE_SHIFT,
				  dma_dom->aperture[0]->bitmap);
343
	}
344
}
345

346 347 348 349 350 351 352
/*
 * Inits the unity mappings required for a specific device
 */
static void init_unity_mappings_for_device(struct device *dev,
					   struct dma_ops_domain *dma_dom)
{
	struct unity_map_entry *e;
353
	int devid;
354

355
	devid = get_device_id(dev);
356 357
	if (IS_ERR_VALUE(devid))
		return;
358

359 360 361 362 363
	list_for_each_entry(e, &amd_iommu_unity_map, list) {
		if (!(devid >= e->devid_start && devid <= e->devid_end))
			continue;
		alloc_unity_mapping(dma_dom, e);
	}
364 365
}

366 367 368 369 370 371
/*
 * This function checks if the driver got a valid device from the caller to
 * avoid dereferencing invalid pointers.
 */
static bool check_device(struct device *dev)
{
372
	int devid;
373 374 375 376 377

	if (!dev || !dev->dma_mask)
		return false;

	devid = get_device_id(dev);
378 379
	if (IS_ERR_VALUE(devid))
		return false;
380 381 382 383 384 385 386 387 388 389 390

	/* Out of our scope? */
	if (devid > amd_iommu_last_bdf)
		return false;

	if (amd_iommu_rlookup_table[devid] == NULL)
		return false;

	return true;
}

391
static void init_iommu_group(struct device *dev)
392
{
393 394
	struct dma_ops_domain *dma_domain;
	struct iommu_domain *domain;
395 396
	struct iommu_group *group;

397
	group = iommu_group_get_for_dev(dev);
398 399 400 401 402 403 404 405 406 407 408 409
	if (IS_ERR(group))
		return;

	domain = iommu_group_default_domain(group);
	if (!domain)
		goto out;

	dma_domain = to_pdomain(domain)->priv;

	init_unity_mappings_for_device(dev, dma_domain);
out:
	iommu_group_put(group);
410 411 412 413 414
}

static int iommu_init_device(struct device *dev)
{
	struct iommu_dev_data *dev_data;
415
	int devid;
416 417 418 419

	if (dev->archdata.iommu)
		return 0;

420 421 422 423 424
	devid = get_device_id(dev);
	if (IS_ERR_VALUE(devid))
		return devid;

	dev_data = find_dev_data(devid);
425 426 427
	if (!dev_data)
		return -ENOMEM;

428
	if (dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) {
429 430
		struct amd_iommu *iommu;

431
		iommu = amd_iommu_rlookup_table[dev_data->devid];
432 433 434
		dev_data->iommu_v2 = iommu->is_iommu_v2;
	}

435 436
	dev->archdata.iommu = dev_data;

A
Alex Williamson 已提交
437 438 439
	iommu_device_link(amd_iommu_rlookup_table[dev_data->devid]->iommu_dev,
			  dev);

440 441 442
	return 0;
}

443 444
static void iommu_ignore_device(struct device *dev)
{
445 446
	u16 alias;
	int devid;
447 448

	devid = get_device_id(dev);
449 450 451
	if (IS_ERR_VALUE(devid))
		return;

452 453 454 455 456 457 458 459 460
	alias = amd_iommu_alias_table[devid];

	memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
	memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));

	amd_iommu_rlookup_table[devid] = NULL;
	amd_iommu_rlookup_table[alias] = NULL;
}

461 462
static void iommu_uninit_device(struct device *dev)
{
463 464 465 466 467 468
	int devid;
	struct iommu_dev_data *dev_data;

	devid = get_device_id(dev);
	if (IS_ERR_VALUE(devid))
		return;
469

470
	dev_data = search_dev_data(devid);
471 472 473
	if (!dev_data)
		return;

474 475 476
	if (dev_data->domain)
		detach_device(dev);

A
Alex Williamson 已提交
477 478 479
	iommu_device_unlink(amd_iommu_rlookup_table[dev_data->devid]->iommu_dev,
			    dev);

480 481
	iommu_group_remove_device(dev);

482 483 484
	/* Remove dma-ops */
	dev->archdata.dma_ops = NULL;

485
	/*
486 487
	 * We keep dev_data around for unplugged devices and reuse it when the
	 * device is re-plugged - not doing so would introduce a ton of races.
488
	 */
489
}
J
Joerg Roedel 已提交
490

491 492 493 494 495 496
#ifdef CONFIG_AMD_IOMMU_STATS

/*
 * Initialization code for statistics collection
 */

497
DECLARE_STATS_COUNTER(compl_wait);
498
DECLARE_STATS_COUNTER(cnt_map_single);
499
DECLARE_STATS_COUNTER(cnt_unmap_single);
500
DECLARE_STATS_COUNTER(cnt_map_sg);
501
DECLARE_STATS_COUNTER(cnt_unmap_sg);
502
DECLARE_STATS_COUNTER(cnt_alloc_coherent);
503
DECLARE_STATS_COUNTER(cnt_free_coherent);
504
DECLARE_STATS_COUNTER(cross_page);
505
DECLARE_STATS_COUNTER(domain_flush_single);
506
DECLARE_STATS_COUNTER(domain_flush_all);
507
DECLARE_STATS_COUNTER(alloced_io_mem);
508
DECLARE_STATS_COUNTER(total_map_requests);
509 510 511 512 513
DECLARE_STATS_COUNTER(complete_ppr);
DECLARE_STATS_COUNTER(invalidate_iotlb);
DECLARE_STATS_COUNTER(invalidate_iotlb_all);
DECLARE_STATS_COUNTER(pri_requests);

514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
static struct dentry *stats_dir;
static struct dentry *de_fflush;

static void amd_iommu_stats_add(struct __iommu_counter *cnt)
{
	if (stats_dir == NULL)
		return;

	cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
				       &cnt->value);
}

static void amd_iommu_stats_init(void)
{
	stats_dir = debugfs_create_dir("amd-iommu", NULL);
	if (stats_dir == NULL)
		return;

	de_fflush  = debugfs_create_bool("fullflush", 0444, stats_dir,
533
					 &amd_iommu_unmap_flush);
534 535

	amd_iommu_stats_add(&compl_wait);
536
	amd_iommu_stats_add(&cnt_map_single);
537
	amd_iommu_stats_add(&cnt_unmap_single);
538
	amd_iommu_stats_add(&cnt_map_sg);
539
	amd_iommu_stats_add(&cnt_unmap_sg);
540
	amd_iommu_stats_add(&cnt_alloc_coherent);
541
	amd_iommu_stats_add(&cnt_free_coherent);
542
	amd_iommu_stats_add(&cross_page);
543
	amd_iommu_stats_add(&domain_flush_single);
544
	amd_iommu_stats_add(&domain_flush_all);
545
	amd_iommu_stats_add(&alloced_io_mem);
546
	amd_iommu_stats_add(&total_map_requests);
547 548 549 550
	amd_iommu_stats_add(&complete_ppr);
	amd_iommu_stats_add(&invalidate_iotlb);
	amd_iommu_stats_add(&invalidate_iotlb_all);
	amd_iommu_stats_add(&pri_requests);
551 552 553 554
}

#endif

555 556 557 558 559 560
/****************************************************************************
 *
 * Interrupt handling functions
 *
 ****************************************************************************/

561 562 563 564
static void dump_dte_entry(u16 devid)
{
	int i;

565 566
	for (i = 0; i < 4; ++i)
		pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
567 568 569
			amd_iommu_dev_table[devid].data[i]);
}

570 571 572 573 574 575 576 577 578
static void dump_command(unsigned long phys_addr)
{
	struct iommu_cmd *cmd = phys_to_virt(phys_addr);
	int i;

	for (i = 0; i < 4; ++i)
		pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
}

579
static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
580
{
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601
	int type, devid, domid, flags;
	volatile u32 *event = __evt;
	int count = 0;
	u64 address;

retry:
	type    = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
	domid   = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
	flags   = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
	address = (u64)(((u64)event[3]) << 32) | event[2];

	if (type == 0) {
		/* Did we hit the erratum? */
		if (++count == LOOP_TIMEOUT) {
			pr_err("AMD-Vi: No event written to event log\n");
			return;
		}
		udelay(1);
		goto retry;
	}
602

603
	printk(KERN_ERR "AMD-Vi: Event logged [");
604 605 606 607 608

	switch (type) {
	case EVENT_TYPE_ILL_DEV:
		printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
609
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
610
		       address, flags);
611
		dump_dte_entry(devid);
612 613 614 615
		break;
	case EVENT_TYPE_IO_FAULT:
		printk("IO_PAGE_FAULT device=%02x:%02x.%x "
		       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
616
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
617 618 619 620 621
		       domid, address, flags);
		break;
	case EVENT_TYPE_DEV_TAB_ERR:
		printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
622
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
623 624 625 626 627
		       address, flags);
		break;
	case EVENT_TYPE_PAGE_TAB_ERR:
		printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
		       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
628
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
629 630 631 632
		       domid, address, flags);
		break;
	case EVENT_TYPE_ILL_CMD:
		printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
633
		dump_command(address);
634 635 636 637 638 639 640 641
		break;
	case EVENT_TYPE_CMD_HARD_ERR:
		printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
		       "flags=0x%04x]\n", address, flags);
		break;
	case EVENT_TYPE_IOTLB_INV_TO:
		printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
		       "address=0x%016llx]\n",
642
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
643 644 645 646 647
		       address);
		break;
	case EVENT_TYPE_INV_DEV_REQ:
		printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
648
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
649 650 651 652 653
		       address, flags);
		break;
	default:
		printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
	}
654 655

	memset(__evt, 0, 4 * sizeof(u32));
656 657 658 659 660 661 662 663 664 665
}

static void iommu_poll_events(struct amd_iommu *iommu)
{
	u32 head, tail;

	head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
	tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

	while (head != tail) {
666
		iommu_print_event(iommu, iommu->evt_buf + head);
667
		head = (head + EVENT_ENTRY_SIZE) % EVT_BUFFER_SIZE;
668 669 670 671 672
	}

	writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
}

673
static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
674 675 676
{
	struct amd_iommu_fault fault;

677 678
	INC_STATS_COUNTER(pri_requests);

679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
	if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
		pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
		return;
	}

	fault.address   = raw[1];
	fault.pasid     = PPR_PASID(raw[0]);
	fault.device_id = PPR_DEVID(raw[0]);
	fault.tag       = PPR_TAG(raw[0]);
	fault.flags     = PPR_FLAGS(raw[0]);

	atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
}

static void iommu_poll_ppr_log(struct amd_iommu *iommu)
{
	u32 head, tail;

	if (iommu->ppr_log == NULL)
		return;

	head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
	tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

	while (head != tail) {
704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
		volatile u64 *raw;
		u64 entry[2];
		int i;

		raw = (u64 *)(iommu->ppr_log + head);

		/*
		 * Hardware bug: Interrupt may arrive before the entry is
		 * written to memory. If this happens we need to wait for the
		 * entry to arrive.
		 */
		for (i = 0; i < LOOP_TIMEOUT; ++i) {
			if (PPR_REQ_TYPE(raw[0]) != 0)
				break;
			udelay(1);
		}
720

721 722 723
		/* Avoid memcpy function-call overhead */
		entry[0] = raw[0];
		entry[1] = raw[1];
724

725 726 727 728 729 730 731
		/*
		 * To detect the hardware bug we need to clear the entry
		 * back to zero.
		 */
		raw[0] = raw[1] = 0UL;

		/* Update head pointer of hardware ring-buffer */
732 733
		head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
		writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
734 735 736 737 738 739

		/* Handle PPR entry */
		iommu_handle_ppr_entry(iommu, entry);

		/* Refresh ring-buffer information */
		head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
740 741 742 743
		tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
	}
}

744
irqreturn_t amd_iommu_int_thread(int irq, void *data)
745
{
746 747
	struct amd_iommu *iommu = (struct amd_iommu *) data;
	u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
748

749 750 751 752
	while (status & (MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK)) {
		/* Enable EVT and PPR interrupts again */
		writel((MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK),
			iommu->mmio_base + MMIO_STATUS_OFFSET);
753

754 755 756 757
		if (status & MMIO_STATUS_EVT_INT_MASK) {
			pr_devel("AMD-Vi: Processing IOMMU Event Log\n");
			iommu_poll_events(iommu);
		}
758

759 760 761 762
		if (status & MMIO_STATUS_PPR_INT_MASK) {
			pr_devel("AMD-Vi: Processing IOMMU PPR Log\n");
			iommu_poll_ppr_log(iommu);
		}
763

764 765 766 767 768 769 770 771 772 773 774 775 776 777 778
		/*
		 * Hardware bug: ERBT1312
		 * When re-enabling interrupt (by writing 1
		 * to clear the bit), the hardware might also try to set
		 * the interrupt bit in the event status register.
		 * In this scenario, the bit will be set, and disable
		 * subsequent interrupts.
		 *
		 * Workaround: The IOMMU driver should read back the
		 * status register and check if the interrupt bits are cleared.
		 * If not, driver will need to go through the interrupt handler
		 * again and re-clear the bits
		 */
		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
	}
779
	return IRQ_HANDLED;
780 781
}

782 783 784 785 786
irqreturn_t amd_iommu_int_handler(int irq, void *data)
{
	return IRQ_WAKE_THREAD;
}

787 788 789 790 791 792
/****************************************************************************
 *
 * IOMMU command queuing functions
 *
 ****************************************************************************/

793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812
static int wait_on_sem(volatile u64 *sem)
{
	int i = 0;

	while (*sem == 0 && i < LOOP_TIMEOUT) {
		udelay(1);
		i += 1;
	}

	if (i == LOOP_TIMEOUT) {
		pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
		return -EIO;
	}

	return 0;
}

static void copy_cmd_to_buffer(struct amd_iommu *iommu,
			       struct iommu_cmd *cmd,
			       u32 tail)
813 814 815
{
	u8 *target;

816
	target = iommu->cmd_buf + tail;
817
	tail   = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
818 819 820 821 822

	/* Copy command to buffer */
	memcpy(target, cmd, sizeof(*cmd));

	/* Tell the IOMMU about it */
823
	writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
824
}
825

826
static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
827
{
828 829
	WARN_ON(address & 0x7ULL);

830
	memset(cmd, 0, sizeof(*cmd));
831 832 833
	cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
	cmd->data[1] = upper_32_bits(__pa(address));
	cmd->data[2] = 1;
834 835 836
	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
}

837 838 839 840 841 842 843
static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
{
	memset(cmd, 0, sizeof(*cmd));
	cmd->data[0] = devid;
	CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
}

844 845 846 847
static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
				  size_t size, u16 domid, int pde)
{
	u64 pages;
848
	bool s;
849 850

	pages = iommu_num_pages(address, size, PAGE_SIZE);
851
	s     = false;
852 853 854 855 856 857 858

	if (pages > 1) {
		/*
		 * If we have to flush more than one page, flush all
		 * TLB entries for this domain
		 */
		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
859
		s = true;
860 861 862 863 864 865 866 867 868 869 870
	}

	address &= PAGE_MASK;

	memset(cmd, 0, sizeof(*cmd));
	cmd->data[1] |= domid;
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[3]  = upper_32_bits(address);
	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
	if (s) /* size bit - we flush more than one 4kb page */
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
F
Frank Arnold 已提交
871
	if (pde) /* PDE bit - we want to flush everything, not only the PTEs */
872 873 874
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
}

875 876 877 878
static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
				  u64 address, size_t size)
{
	u64 pages;
879
	bool s;
880 881

	pages = iommu_num_pages(address, size, PAGE_SIZE);
882
	s     = false;
883 884 885 886 887 888 889

	if (pages > 1) {
		/*
		 * If we have to flush more than one page, flush all
		 * TLB entries for this domain
		 */
		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
890
		s = true;
891 892 893 894 895 896 897 898 899 900 901 902 903 904 905
	}

	address &= PAGE_MASK;

	memset(cmd, 0, sizeof(*cmd));
	cmd->data[0]  = devid;
	cmd->data[0] |= (qdep & 0xff) << 24;
	cmd->data[1]  = devid;
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[3]  = upper_32_bits(address);
	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
	if (s)
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
}

906 907 908 909 910 911 912
static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
				  u64 address, bool size)
{
	memset(cmd, 0, sizeof(*cmd));

	address &= ~(0xfffULL);

913
	cmd->data[0]  = pasid;
914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931
	cmd->data[1]  = domid;
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[3]  = upper_32_bits(address);
	cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
	cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
	if (size)
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
}

static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
				  int qdep, u64 address, bool size)
{
	memset(cmd, 0, sizeof(*cmd));

	address &= ~(0xfffULL);

	cmd->data[0]  = devid;
932
	cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;
933 934
	cmd->data[0] |= (qdep  & 0xff) << 24;
	cmd->data[1]  = devid;
935
	cmd->data[1] |= (pasid & 0xff) << 16;
936 937 938 939 940 941 942 943
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
	cmd->data[3]  = upper_32_bits(address);
	if (size)
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
}

944 945 946 947 948 949 950
static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
			       int status, int tag, bool gn)
{
	memset(cmd, 0, sizeof(*cmd));

	cmd->data[0]  = devid;
	if (gn) {
951
		cmd->data[1]  = pasid;
952 953 954 955 956 957 958 959
		cmd->data[2]  = CMD_INV_IOMMU_PAGES_GN_MASK;
	}
	cmd->data[3]  = tag & 0x1ff;
	cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;

	CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
}

960 961 962 963
static void build_inv_all(struct iommu_cmd *cmd)
{
	memset(cmd, 0, sizeof(*cmd));
	CMD_SET_TYPE(cmd, CMD_INV_ALL);
964 965
}

966 967 968 969 970 971 972
static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
{
	memset(cmd, 0, sizeof(*cmd));
	cmd->data[0] = devid;
	CMD_SET_TYPE(cmd, CMD_INV_IRT);
}

973 974
/*
 * Writes the command to the IOMMUs command buffer and informs the
975
 * hardware about the new command.
976
 */
977 978 979
static int iommu_queue_command_sync(struct amd_iommu *iommu,
				    struct iommu_cmd *cmd,
				    bool sync)
980
{
981
	u32 left, tail, head, next_tail;
982 983
	unsigned long flags;

984
again:
985 986
	spin_lock_irqsave(&iommu->lock, flags);

987 988
	head      = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
	tail      = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
989 990
	next_tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
	left      = (head - next_tail) % CMD_BUFFER_SIZE;
991

992 993 994 995
	if (left <= 2) {
		struct iommu_cmd sync_cmd;
		volatile u64 sem = 0;
		int ret;
996

997 998
		build_completion_wait(&sync_cmd, (u64)&sem);
		copy_cmd_to_buffer(iommu, &sync_cmd, tail);
999

1000 1001 1002 1003 1004 1005
		spin_unlock_irqrestore(&iommu->lock, flags);

		if ((ret = wait_on_sem(&sem)) != 0)
			return ret;

		goto again;
1006 1007
	}

1008 1009 1010
	copy_cmd_to_buffer(iommu, cmd, tail);

	/* We need to sync now to make sure all commands are processed */
1011
	iommu->need_sync = sync;
1012

1013
	spin_unlock_irqrestore(&iommu->lock, flags);
1014

1015
	return 0;
1016 1017
}

1018 1019 1020 1021 1022
static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
{
	return iommu_queue_command_sync(iommu, cmd, true);
}

1023 1024 1025 1026
/*
 * This function queues a completion wait command into the command
 * buffer of an IOMMU
 */
1027
static int iommu_completion_wait(struct amd_iommu *iommu)
1028 1029
{
	struct iommu_cmd cmd;
1030
	volatile u64 sem = 0;
1031
	int ret;
1032

1033
	if (!iommu->need_sync)
1034
		return 0;
1035

1036
	build_completion_wait(&cmd, (u64)&sem);
1037

1038
	ret = iommu_queue_command_sync(iommu, &cmd, false);
1039
	if (ret)
1040
		return ret;
1041

1042
	return wait_on_sem(&sem);
1043 1044
}

1045
static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
1046
{
1047
	struct iommu_cmd cmd;
1048

1049
	build_inv_dte(&cmd, devid);
1050

1051 1052
	return iommu_queue_command(iommu, &cmd);
}
1053

1054 1055 1056
static void iommu_flush_dte_all(struct amd_iommu *iommu)
{
	u32 devid;
1057

1058 1059
	for (devid = 0; devid <= 0xffff; ++devid)
		iommu_flush_dte(iommu, devid);
1060

1061 1062
	iommu_completion_wait(iommu);
}
1063

1064 1065 1066 1067 1068 1069 1070
/*
 * This function uses heavy locking and may disable irqs for some time. But
 * this is no issue because it is only called during resume.
 */
static void iommu_flush_tlb_all(struct amd_iommu *iommu)
{
	u32 dom_id;
1071

1072 1073 1074 1075 1076 1077
	for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
		struct iommu_cmd cmd;
		build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
				      dom_id, 1);
		iommu_queue_command(iommu, &cmd);
	}
1078

1079
	iommu_completion_wait(iommu);
1080 1081
}

1082
static void iommu_flush_all(struct amd_iommu *iommu)
1083
{
1084
	struct iommu_cmd cmd;
1085

1086
	build_inv_all(&cmd);
1087

1088 1089 1090 1091
	iommu_queue_command(iommu, &cmd);
	iommu_completion_wait(iommu);
}

1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
{
	struct iommu_cmd cmd;

	build_inv_irt(&cmd, devid);

	iommu_queue_command(iommu, &cmd);
}

static void iommu_flush_irt_all(struct amd_iommu *iommu)
{
	u32 devid;

	for (devid = 0; devid <= MAX_DEV_TABLE_ENTRIES; devid++)
		iommu_flush_irt(iommu, devid);

	iommu_completion_wait(iommu);
}

1111 1112
void iommu_flush_all_caches(struct amd_iommu *iommu)
{
1113 1114 1115 1116
	if (iommu_feature(iommu, FEATURE_IA)) {
		iommu_flush_all(iommu);
	} else {
		iommu_flush_dte_all(iommu);
1117
		iommu_flush_irt_all(iommu);
1118
		iommu_flush_tlb_all(iommu);
1119 1120 1121
	}
}

1122
/*
1123
 * Command send function for flushing on-device TLB
1124
 */
1125 1126
static int device_flush_iotlb(struct iommu_dev_data *dev_data,
			      u64 address, size_t size)
1127 1128
{
	struct amd_iommu *iommu;
1129
	struct iommu_cmd cmd;
1130
	int qdep;
1131

1132 1133
	qdep     = dev_data->ats.qdep;
	iommu    = amd_iommu_rlookup_table[dev_data->devid];
1134

1135
	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
1136 1137

	return iommu_queue_command(iommu, &cmd);
1138 1139
}

1140 1141 1142
/*
 * Command send function for invalidating a device table entry
 */
1143
static int device_flush_dte(struct iommu_dev_data *dev_data)
1144
{
1145
	struct amd_iommu *iommu;
1146
	u16 alias;
1147
	int ret;
1148

1149
	iommu = amd_iommu_rlookup_table[dev_data->devid];
1150
	alias = amd_iommu_alias_table[dev_data->devid];
1151

1152
	ret = iommu_flush_dte(iommu, dev_data->devid);
1153 1154
	if (!ret && alias != dev_data->devid)
		ret = iommu_flush_dte(iommu, alias);
1155 1156 1157
	if (ret)
		return ret;

1158
	if (dev_data->ats.enabled)
1159
		ret = device_flush_iotlb(dev_data, 0, ~0UL);
1160 1161

	return ret;
1162 1163
}

1164 1165 1166 1167 1168
/*
 * TLB invalidation function which is called from the mapping functions.
 * It invalidates a single PTE if the range to flush is within a single
 * page. Otherwise it flushes the whole TLB of the IOMMU.
 */
1169 1170
static void __domain_flush_pages(struct protection_domain *domain,
				 u64 address, size_t size, int pde)
1171
{
1172
	struct iommu_dev_data *dev_data;
1173 1174
	struct iommu_cmd cmd;
	int ret = 0, i;
1175

1176
	build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
1177

1178 1179 1180 1181 1182 1183 1184 1185
	for (i = 0; i < amd_iommus_present; ++i) {
		if (!domain->dev_iommu[i])
			continue;

		/*
		 * Devices of this domain are behind this IOMMU
		 * We need a TLB flush
		 */
1186
		ret |= iommu_queue_command(amd_iommus[i], &cmd);
1187 1188
	}

1189 1190
	list_for_each_entry(dev_data, &domain->dev_list, list) {

1191
		if (!dev_data->ats.enabled)
1192 1193
			continue;

1194
		ret |= device_flush_iotlb(dev_data, address, size);
1195 1196
	}

1197
	WARN_ON(ret);
1198 1199
}

1200 1201
static void domain_flush_pages(struct protection_domain *domain,
			       u64 address, size_t size)
1202
{
1203
	__domain_flush_pages(domain, address, size, 0);
1204
}
1205

1206
/* Flush the whole IO/TLB for a given protection domain */
1207
static void domain_flush_tlb(struct protection_domain *domain)
1208
{
1209
	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
1210 1211
}

1212
/* Flush the whole IO/TLB for a given protection domain - including PDE */
1213
static void domain_flush_tlb_pde(struct protection_domain *domain)
1214
{
1215
	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
1216 1217
}

1218
static void domain_flush_complete(struct protection_domain *domain)
1219
{
1220
	int i;
1221

1222 1223 1224
	for (i = 0; i < amd_iommus_present; ++i) {
		if (!domain->dev_iommu[i])
			continue;
1225

1226 1227 1228 1229 1230
		/*
		 * Devices of this domain are behind this IOMMU
		 * We need to wait for completion of all commands.
		 */
		iommu_completion_wait(amd_iommus[i]);
1231
	}
1232 1233
}

1234

1235
/*
1236
 * This function flushes the DTEs for all devices in domain
1237
 */
1238
static void domain_flush_devices(struct protection_domain *domain)
1239
{
1240
	struct iommu_dev_data *dev_data;
1241

1242
	list_for_each_entry(dev_data, &domain->dev_list, list)
1243
		device_flush_dte(dev_data);
1244 1245
}

1246 1247 1248 1249 1250 1251 1252
/****************************************************************************
 *
 * The functions below are used the create the page table mappings for
 * unity mapped regions.
 *
 ****************************************************************************/

1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281
/*
 * This function is used to add another level to an IO page table. Adding
 * another level increases the size of the address space by 9 bits to a size up
 * to 64 bits.
 */
static bool increase_address_space(struct protection_domain *domain,
				   gfp_t gfp)
{
	u64 *pte;

	if (domain->mode == PAGE_MODE_6_LEVEL)
		/* address space already 64 bit large */
		return false;

	pte = (void *)get_zeroed_page(gfp);
	if (!pte)
		return false;

	*pte             = PM_LEVEL_PDE(domain->mode,
					virt_to_phys(domain->pt_root));
	domain->pt_root  = pte;
	domain->mode    += 1;
	domain->updated  = true;

	return true;
}

static u64 *alloc_pte(struct protection_domain *domain,
		      unsigned long address,
1282
		      unsigned long page_size,
1283 1284 1285
		      u64 **pte_page,
		      gfp_t gfp)
{
1286
	int level, end_lvl;
1287
	u64 *pte, *page;
1288 1289

	BUG_ON(!is_power_of_2(page_size));
1290 1291 1292 1293

	while (address > PM_LEVEL_SIZE(domain->mode))
		increase_address_space(domain, gfp);

1294 1295 1296 1297
	level   = domain->mode - 1;
	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
	address = PAGE_SIZE_ALIGN(address, page_size);
	end_lvl = PAGE_SIZE_LEVEL(page_size);
1298 1299

	while (level > end_lvl) {
1300 1301 1302 1303 1304
		u64 __pte, __npte;

		__pte = *pte;

		if (!IOMMU_PTE_PRESENT(__pte)) {
1305 1306 1307
			page = (u64 *)get_zeroed_page(gfp);
			if (!page)
				return NULL;
1308 1309 1310 1311 1312 1313 1314

			__npte = PM_LEVEL_PDE(level, virt_to_phys(page));

			if (cmpxchg64(pte, __pte, __npte)) {
				free_page((unsigned long)page);
				continue;
			}
1315 1316
		}

1317 1318 1319 1320
		/* No level skipping support yet */
		if (PM_PTE_LEVEL(*pte) != level)
			return NULL;

1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
		level -= 1;

		pte = IOMMU_PTE_PAGE(*pte);

		if (pte_page && level == end_lvl)
			*pte_page = pte;

		pte = &pte[PM_LEVEL_INDEX(level, address)];
	}

	return pte;
}

/*
 * This function checks if there is a PTE for a given dma address. If
 * there is one, it returns the pointer to it.
 */
1338 1339 1340
static u64 *fetch_pte(struct protection_domain *domain,
		      unsigned long address,
		      unsigned long *page_size)
1341 1342 1343 1344
{
	int level;
	u64 *pte;

1345 1346 1347
	if (address > PM_LEVEL_SIZE(domain->mode))
		return NULL;

1348 1349 1350
	level	   =  domain->mode - 1;
	pte	   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
	*page_size =  PTE_LEVEL_PAGE_SIZE(level);
1351

1352 1353 1354
	while (level > 0) {

		/* Not Present */
1355 1356 1357
		if (!IOMMU_PTE_PRESENT(*pte))
			return NULL;

1358
		/* Large PTE */
1359 1360 1361
		if (PM_PTE_LEVEL(*pte) == 7 ||
		    PM_PTE_LEVEL(*pte) == 0)
			break;
1362 1363 1364 1365 1366

		/* No level skipping support yet */
		if (PM_PTE_LEVEL(*pte) != level)
			return NULL;

1367 1368
		level -= 1;

1369
		/* Walk to the next level */
1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
		pte	   = IOMMU_PTE_PAGE(*pte);
		pte	   = &pte[PM_LEVEL_INDEX(level, address)];
		*page_size = PTE_LEVEL_PAGE_SIZE(level);
	}

	if (PM_PTE_LEVEL(*pte) == 0x07) {
		unsigned long pte_mask;

		/*
		 * If we have a series of large PTEs, make
		 * sure to return a pointer to the first one.
		 */
		*page_size = pte_mask = PTE_PAGE_SIZE(*pte);
		pte_mask   = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
		pte        = (u64 *)(((unsigned long)pte) & pte_mask);
1385 1386 1387 1388 1389
	}

	return pte;
}

1390 1391 1392 1393 1394 1395 1396
/*
 * Generic mapping functions. It maps a physical address into a DMA
 * address space. It allocates the page table pages if necessary.
 * In the future it can be extended to a generic mapping function
 * supporting all features of AMD IOMMU page tables like level skipping
 * and full 64 bit address spaces.
 */
1397 1398 1399
static int iommu_map_page(struct protection_domain *dom,
			  unsigned long bus_addr,
			  unsigned long phys_addr,
1400
			  int prot,
1401
			  unsigned long page_size)
1402
{
1403
	u64 __pte, *pte;
1404
	int i, count;
1405

1406 1407 1408
	BUG_ON(!IS_ALIGNED(bus_addr, page_size));
	BUG_ON(!IS_ALIGNED(phys_addr, page_size));

1409
	if (!(prot & IOMMU_PROT_MASK))
1410 1411
		return -EINVAL;

1412 1413
	count = PAGE_SIZE_PTE_COUNT(page_size);
	pte   = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
1414

1415 1416 1417
	if (!pte)
		return -ENOMEM;

1418 1419 1420
	for (i = 0; i < count; ++i)
		if (IOMMU_PTE_PRESENT(pte[i]))
			return -EBUSY;
1421

1422
	if (count > 1) {
1423 1424 1425 1426
		__pte = PAGE_SIZE_PTE(phys_addr, page_size);
		__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
	} else
		__pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
1427 1428 1429 1430 1431 1432

	if (prot & IOMMU_PROT_IR)
		__pte |= IOMMU_PTE_IR;
	if (prot & IOMMU_PROT_IW)
		__pte |= IOMMU_PTE_IW;

1433 1434
	for (i = 0; i < count; ++i)
		pte[i] = __pte;
1435

1436 1437
	update_domain(dom);

1438 1439 1440
	return 0;
}

1441 1442 1443
static unsigned long iommu_unmap_page(struct protection_domain *dom,
				      unsigned long bus_addr,
				      unsigned long page_size)
1444
{
1445 1446
	unsigned long long unmapped;
	unsigned long unmap_size;
1447 1448 1449 1450 1451
	u64 *pte;

	BUG_ON(!is_power_of_2(page_size));

	unmapped = 0;
1452

1453 1454
	while (unmapped < page_size) {

1455 1456 1457 1458 1459 1460
		pte = fetch_pte(dom, bus_addr, &unmap_size);

		if (pte) {
			int i, count;

			count = PAGE_SIZE_PTE_COUNT(unmap_size);
1461 1462 1463 1464 1465 1466 1467 1468
			for (i = 0; i < count; i++)
				pte[i] = 0ULL;
		}

		bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;
		unmapped += unmap_size;
	}

1469
	BUG_ON(unmapped && !is_power_of_2(unmapped));
1470

1471
	return unmapped;
1472 1473
}

1474 1475 1476 1477 1478 1479 1480 1481 1482
/****************************************************************************
 *
 * The next functions belong to the address allocator for the dma_ops
 * interface functions. They work like the allocators in the other IOMMU
 * drivers. Its basically a bitmap which marks the allocated pages in
 * the aperture. Maybe it could be enhanced in the future to a more
 * efficient allocator.
 *
 ****************************************************************************/
1483

1484
/*
1485
 * The address allocator core functions.
1486 1487 1488
 *
 * called with domain->lock held
 */
1489

1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509
/*
 * Used to reserve address ranges in the aperture (e.g. for exclusion
 * ranges.
 */
static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
				      unsigned long start_page,
				      unsigned int pages)
{
	unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;

	if (start_page + pages > last_page)
		pages = last_page - start_page;

	for (i = start_page; i < start_page + pages; ++i) {
		int index = i / APERTURE_RANGE_PAGES;
		int page  = i % APERTURE_RANGE_PAGES;
		__set_bit(page, dom->aperture[index]->bitmap);
	}
}

1510 1511 1512 1513 1514
/*
 * This function is used to add a new aperture range to an existing
 * aperture in case of dma_ops domain allocation or address allocation
 * failure.
 */
1515
static int alloc_new_range(struct dma_ops_domain *dma_dom,
1516 1517 1518
			   bool populate, gfp_t gfp)
{
	int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1519
	unsigned long i, old_size, pte_pgsize;
1520 1521 1522
	struct aperture_range *range;
	struct amd_iommu *iommu;
	unsigned long flags;
1523

1524 1525 1526 1527
#ifdef CONFIG_IOMMU_STRESS
	populate = false;
#endif

1528 1529 1530
	if (index >= APERTURE_MAX_RANGES)
		return -ENOMEM;

1531 1532
	range = kzalloc(sizeof(struct aperture_range), gfp);
	if (!range)
1533 1534
		return -ENOMEM;

1535 1536
	range->bitmap = (void *)get_zeroed_page(gfp);
	if (!range->bitmap)
1537 1538
		goto out_free;

1539
	range->offset = dma_dom->aperture_size;
1540

1541
	spin_lock_init(&range->bitmap_lock);
1542

1543 1544 1545 1546 1547 1548
	if (populate) {
		unsigned long address = dma_dom->aperture_size;
		int i, num_ptes = APERTURE_RANGE_PAGES / 512;
		u64 *pte, *pte_page;

		for (i = 0; i < num_ptes; ++i) {
1549
			pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
1550 1551 1552 1553
					&pte_page, gfp);
			if (!pte)
				goto out_free;

1554
			range->pte_pages[i] = pte_page;
1555 1556 1557 1558 1559

			address += APERTURE_RANGE_SIZE / 64;
		}
	}

1560 1561
	spin_lock_irqsave(&dma_dom->domain.lock, flags);

1562
	/* First take the bitmap_lock and then publish the range */
1563
	spin_lock(&range->bitmap_lock);
1564 1565 1566 1567

	old_size                 = dma_dom->aperture_size;
	dma_dom->aperture[index] = range;
	dma_dom->aperture_size  += APERTURE_RANGE_SIZE;
1568

1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580
	/* Reserve address range used for MSI messages */
	if (old_size < MSI_ADDR_BASE_LO &&
	    dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
		unsigned long spage;
		int pages;

		pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
		spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;

		dma_ops_reserve_addresses(dma_dom, spage, pages);
	}

1581
	/* Initialize the exclusion range if necessary */
1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592
	for_each_iommu(iommu) {
		if (iommu->exclusion_start &&
		    iommu->exclusion_start >= dma_dom->aperture[index]->offset
		    && iommu->exclusion_start < dma_dom->aperture_size) {
			unsigned long startpage;
			int pages = iommu_num_pages(iommu->exclusion_start,
						    iommu->exclusion_length,
						    PAGE_SIZE);
			startpage = iommu->exclusion_start >> PAGE_SHIFT;
			dma_ops_reserve_addresses(dma_dom, startpage, pages);
		}
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
	}

	/*
	 * Check for areas already mapped as present in the new aperture
	 * range and mark those pages as reserved in the allocator. Such
	 * mappings may already exist as a result of requested unity
	 * mappings for devices.
	 */
	for (i = dma_dom->aperture[index]->offset;
	     i < dma_dom->aperture_size;
1603
	     i += pte_pgsize) {
1604
		u64 *pte = fetch_pte(&dma_dom->domain, i, &pte_pgsize);
1605 1606 1607
		if (!pte || !IOMMU_PTE_PRESENT(*pte))
			continue;

1608 1609
		dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT,
					  pte_pgsize >> 12);
1610 1611
	}

1612 1613
	update_domain(&dma_dom->domain);

1614 1615 1616
	spin_unlock(&range->bitmap_lock);

	spin_unlock_irqrestore(&dma_dom->domain.lock, flags);
1617

1618 1619 1620
	return 0;

out_free:
1621 1622
	update_domain(&dma_dom->domain);

1623
	free_page((unsigned long)range->bitmap);
1624

1625
	kfree(range);
1626 1627 1628 1629

	return -ENOMEM;
}

1630 1631
static dma_addr_t dma_ops_aperture_alloc(struct dma_ops_domain *dom,
					 struct aperture_range *range,
1632 1633 1634
					 unsigned long pages,
					 unsigned long dma_mask,
					 unsigned long boundary_size,
1635 1636
					 unsigned long align_mask,
					 bool trylock)
1637 1638 1639
{
	unsigned long offset, limit, flags;
	dma_addr_t address;
1640
	bool flush = false;
1641 1642 1643 1644 1645

	offset = range->offset >> PAGE_SHIFT;
	limit  = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
					dma_mask >> PAGE_SHIFT);

1646 1647 1648 1649 1650 1651 1652
	if (trylock) {
		if (!spin_trylock_irqsave(&range->bitmap_lock, flags))
			return -1;
	} else {
		spin_lock_irqsave(&range->bitmap_lock, flags);
	}

1653 1654
	address = iommu_area_alloc(range->bitmap, limit, range->next_bit,
				   pages, offset, boundary_size, align_mask);
1655
	if (address == -1) {
1656 1657 1658 1659
		/* Nothing found, retry one time */
		address = iommu_area_alloc(range->bitmap, limit,
					   0, pages, offset, boundary_size,
					   align_mask);
1660 1661
		flush = true;
	}
1662 1663 1664 1665

	if (address != -1)
		range->next_bit = address + pages;

1666 1667
	spin_unlock_irqrestore(&range->bitmap_lock, flags);

1668 1669 1670 1671 1672
	if (flush) {
		domain_flush_tlb(&dom->domain);
		domain_flush_complete(&dom->domain);
	}

1673 1674 1675
	return address;
}

1676 1677 1678 1679
static unsigned long dma_ops_area_alloc(struct device *dev,
					struct dma_ops_domain *dom,
					unsigned int pages,
					unsigned long align_mask,
1680
					u64 dma_mask)
1681
{
1682
	unsigned long boundary_size, mask;
1683
	unsigned long address = -1;
1684
	bool first = true;
1685 1686 1687
	u32 start, i;

	preempt_disable();
1688

1689 1690
	mask = dma_get_seg_boundary(dev);

1691
again:
1692 1693 1694 1695 1696 1697 1698 1699
	start = this_cpu_read(*dom->next_index);

	/* Sanity check - is it really necessary? */
	if (unlikely(start > APERTURE_MAX_RANGES)) {
		start = 0;
		this_cpu_write(*dom->next_index, 0);
	}

1700 1701
	boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
				   1UL << (BITS_PER_LONG - PAGE_SHIFT);
1702

1703 1704
	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
		struct aperture_range *range;
1705 1706 1707
		int index;

		index = (start + i) % APERTURE_MAX_RANGES;
1708

1709
		range = dom->aperture[index];
1710 1711 1712

		if (!range || range->offset >= dma_mask)
			continue;
1713

1714
		address = dma_ops_aperture_alloc(dom, range, pages,
1715
						 dma_mask, boundary_size,
1716
						 align_mask, first);
1717
		if (address != -1) {
1718
			address = range->offset + (address << PAGE_SHIFT);
1719
			this_cpu_write(*dom->next_index, index);
1720 1721 1722 1723
			break;
		}
	}

1724 1725 1726 1727 1728
	if (address == -1 && first) {
		first = false;
		goto again;
	}

1729 1730
	preempt_enable();

1731 1732 1733
	return address;
}

1734 1735
static unsigned long dma_ops_alloc_addresses(struct device *dev,
					     struct dma_ops_domain *dom,
1736
					     unsigned int pages,
1737 1738
					     unsigned long align_mask,
					     u64 dma_mask)
1739
{
1740
	unsigned long address = -1;
1741

1742 1743 1744 1745
	while (address == -1) {
		address = dma_ops_area_alloc(dev, dom, pages,
					     align_mask, dma_mask);

1746
		if (address == -1 && alloc_new_range(dom, false, GFP_ATOMIC))
1747 1748
			break;
	}
1749

1750
	if (unlikely(address == -1))
1751
		address = DMA_ERROR_CODE;
1752 1753 1754 1755 1756 1757

	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);

	return address;
}

1758 1759 1760 1761 1762
/*
 * The address free function.
 *
 * called with domain->lock held
 */
1763 1764 1765 1766
static void dma_ops_free_addresses(struct dma_ops_domain *dom,
				   unsigned long address,
				   unsigned int pages)
{
1767 1768
	unsigned i = address >> APERTURE_RANGE_SHIFT;
	struct aperture_range *range = dom->aperture[i];
1769
	unsigned long flags;
1770

1771 1772
	BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);

1773 1774 1775 1776
#ifdef CONFIG_IOMMU_STRESS
	if (i < 4)
		return;
#endif
1777

1778
	if (amd_iommu_unmap_flush) {
1779 1780 1781
		domain_flush_tlb(&dom->domain);
		domain_flush_complete(&dom->domain);
	}
1782 1783

	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1784

1785
	spin_lock_irqsave(&range->bitmap_lock, flags);
1786 1787
	if (address + pages > range->next_bit)
		range->next_bit = address + pages;
A
Akinobu Mita 已提交
1788
	bitmap_clear(range->bitmap, address, pages);
1789
	spin_unlock_irqrestore(&range->bitmap_lock, flags);
1790

1791 1792
}

1793 1794 1795 1796 1797 1798 1799 1800 1801 1802
/****************************************************************************
 *
 * The next functions belong to the domain allocation. A domain is
 * allocated for every IOMMU as the default domain. If device isolation
 * is enabled, every device get its own domain. The most important thing
 * about domains is the page table mapping the DMA address space they
 * contain.
 *
 ****************************************************************************/

1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827
/*
 * This function adds a protection domain to the global protection domain list
 */
static void add_domain_to_list(struct protection_domain *domain)
{
	unsigned long flags;

	spin_lock_irqsave(&amd_iommu_pd_lock, flags);
	list_add(&domain->list, &amd_iommu_pd_list);
	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
}

/*
 * This function removes a protection domain to the global
 * protection domain list
 */
static void del_domain_from_list(struct protection_domain *domain)
{
	unsigned long flags;

	spin_lock_irqsave(&amd_iommu_pd_lock, flags);
	list_del(&domain->list);
	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
}

1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844
static u16 domain_id_alloc(void)
{
	unsigned long flags;
	int id;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
	BUG_ON(id == 0);
	if (id > 0 && id < MAX_DOMAIN_ID)
		__set_bit(id, amd_iommu_pd_alloc_bitmap);
	else
		id = 0;
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

	return id;
}

1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
static void domain_id_free(int id)
{
	unsigned long flags;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	if (id > 0 && id < MAX_DOMAIN_ID)
		__clear_bit(id, amd_iommu_pd_alloc_bitmap);
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}

1855 1856 1857 1858 1859 1860 1861 1862 1863 1864
#define DEFINE_FREE_PT_FN(LVL, FN)				\
static void free_pt_##LVL (unsigned long __pt)			\
{								\
	unsigned long p;					\
	u64 *pt;						\
	int i;							\
								\
	pt = (u64 *)__pt;					\
								\
	for (i = 0; i < 512; ++i) {				\
1865
		/* PTE present? */				\
1866 1867 1868
		if (!IOMMU_PTE_PRESENT(pt[i]))			\
			continue;				\
								\
1869 1870 1871 1872 1873
		/* Large PTE? */				\
		if (PM_PTE_LEVEL(pt[i]) == 0 ||			\
		    PM_PTE_LEVEL(pt[i]) == 7)			\
			continue;				\
								\
1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
		p = (unsigned long)IOMMU_PTE_PAGE(pt[i]);	\
		FN(p);						\
	}							\
	free_page((unsigned long)pt);				\
}

DEFINE_FREE_PT_FN(l2, free_page)
DEFINE_FREE_PT_FN(l3, free_pt_l2)
DEFINE_FREE_PT_FN(l4, free_pt_l3)
DEFINE_FREE_PT_FN(l5, free_pt_l4)
DEFINE_FREE_PT_FN(l6, free_pt_l5)

1886
static void free_pagetable(struct protection_domain *domain)
1887
{
1888
	unsigned long root = (unsigned long)domain->pt_root;
1889

1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912
	switch (domain->mode) {
	case PAGE_MODE_NONE:
		break;
	case PAGE_MODE_1_LEVEL:
		free_page(root);
		break;
	case PAGE_MODE_2_LEVEL:
		free_pt_l2(root);
		break;
	case PAGE_MODE_3_LEVEL:
		free_pt_l3(root);
		break;
	case PAGE_MODE_4_LEVEL:
		free_pt_l4(root);
		break;
	case PAGE_MODE_5_LEVEL:
		free_pt_l5(root);
		break;
	case PAGE_MODE_6_LEVEL:
		free_pt_l6(root);
		break;
	default:
		BUG();
1913 1914 1915
	}
}

1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945
static void free_gcr3_tbl_level1(u64 *tbl)
{
	u64 *ptr;
	int i;

	for (i = 0; i < 512; ++i) {
		if (!(tbl[i] & GCR3_VALID))
			continue;

		ptr = __va(tbl[i] & PAGE_MASK);

		free_page((unsigned long)ptr);
	}
}

static void free_gcr3_tbl_level2(u64 *tbl)
{
	u64 *ptr;
	int i;

	for (i = 0; i < 512; ++i) {
		if (!(tbl[i] & GCR3_VALID))
			continue;

		ptr = __va(tbl[i] & PAGE_MASK);

		free_gcr3_tbl_level1(ptr);
	}
}

1946 1947
static void free_gcr3_table(struct protection_domain *domain)
{
1948 1949 1950 1951
	if (domain->glx == 2)
		free_gcr3_tbl_level2(domain->gcr3_tbl);
	else if (domain->glx == 1)
		free_gcr3_tbl_level1(domain->gcr3_tbl);
1952 1953
	else
		BUG_ON(domain->glx != 0);
1954

1955 1956 1957
	free_page((unsigned long)domain->gcr3_tbl);
}

1958 1959 1960 1961
/*
 * Free a domain, only used if something went wrong in the
 * allocation path and we need to free an already allocated page table
 */
1962 1963
static void dma_ops_domain_free(struct dma_ops_domain *dom)
{
1964 1965
	int i;

1966 1967 1968
	if (!dom)
		return;

1969 1970
	free_percpu(dom->next_index);

1971 1972
	del_domain_from_list(&dom->domain);

1973
	free_pagetable(&dom->domain);
1974

1975 1976 1977 1978 1979 1980
	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
		if (!dom->aperture[i])
			continue;
		free_page((unsigned long)dom->aperture[i]->bitmap);
		kfree(dom->aperture[i]);
	}
1981 1982 1983 1984

	kfree(dom);
}

1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001
static int dma_ops_domain_alloc_apertures(struct dma_ops_domain *dma_dom,
					  int max_apertures)
{
	int ret, i, apertures;

	apertures = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
	ret       = 0;

	for (i = apertures; i < max_apertures; ++i) {
		ret = alloc_new_range(dma_dom, false, GFP_KERNEL);
		if (ret)
			break;
	}

	return ret;
}

2002 2003
/*
 * Allocates a new protection domain usable for the dma_ops functions.
2004
 * It also initializes the page table and the address allocator data
2005 2006
 * structures required for the dma_ops interface
 */
2007
static struct dma_ops_domain *dma_ops_domain_alloc(void)
2008 2009
{
	struct dma_ops_domain *dma_dom;
2010
	int cpu;
2011 2012 2013 2014 2015

	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
	if (!dma_dom)
		return NULL;

2016
	if (protection_domain_init(&dma_dom->domain))
2017
		goto free_dma_dom;
2018

2019 2020 2021 2022
	dma_dom->next_index = alloc_percpu(u32);
	if (!dma_dom->next_index)
		goto free_dma_dom;

2023
	dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
2024
	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2025
	dma_dom->domain.flags = PD_DMA_OPS_MASK;
2026 2027 2028 2029
	dma_dom->domain.priv = dma_dom;
	if (!dma_dom->domain.pt_root)
		goto free_dma_dom;

2030 2031
	add_domain_to_list(&dma_dom->domain);

2032
	if (alloc_new_range(dma_dom, true, GFP_KERNEL))
2033 2034
		goto free_dma_dom;

2035
	/*
2036 2037
	 * mark the first page as allocated so we never return 0 as
	 * a valid dma-address. So we can use 0 as error value
2038
	 */
2039
	dma_dom->aperture[0]->bitmap[0] = 1;
2040

2041 2042
	for_each_possible_cpu(cpu)
		*per_cpu_ptr(dma_dom->next_index, cpu) = 0;
2043 2044 2045 2046 2047 2048 2049 2050 2051

	return dma_dom;

free_dma_dom:
	dma_ops_domain_free(dma_dom);

	return NULL;
}

2052 2053 2054 2055 2056 2057 2058 2059 2060
/*
 * little helper function to check whether a given protection domain is a
 * dma_ops domain
 */
static bool dma_ops_domain(struct protection_domain *domain)
{
	return domain->flags & PD_DMA_OPS_MASK;
}

2061
static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
2062
{
2063
	u64 pte_root = 0;
2064
	u64 flags = 0;
2065

2066 2067 2068
	if (domain->mode != PAGE_MODE_NONE)
		pte_root = virt_to_phys(domain->pt_root);

2069 2070 2071
	pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
		    << DEV_ENTRY_MODE_SHIFT;
	pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
2072

2073 2074
	flags = amd_iommu_dev_table[devid].data[1];

2075 2076 2077
	if (ats)
		flags |= DTE_FLAG_IOTLB;

2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103
	if (domain->flags & PD_IOMMUV2_MASK) {
		u64 gcr3 = __pa(domain->gcr3_tbl);
		u64 glx  = domain->glx;
		u64 tmp;

		pte_root |= DTE_FLAG_GV;
		pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;

		/* First mask out possible old values for GCR3 table */
		tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
		flags    &= ~tmp;

		tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
		flags    &= ~tmp;

		/* Encode GCR3 table into DTE */
		tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
		pte_root |= tmp;

		tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
		flags    |= tmp;

		tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
		flags    |= tmp;
	}

2104 2105 2106 2107 2108
	flags &= ~(0xffffUL);
	flags |= domain->id;

	amd_iommu_dev_table[devid].data[1]  = flags;
	amd_iommu_dev_table[devid].data[0]  = pte_root;
2109 2110 2111 2112 2113
}

static void clear_dte_entry(u16 devid)
{
	/* remove entry from the device table seen by the hardware */
2114 2115
	amd_iommu_dev_table[devid].data[0]  = IOMMU_PTE_P | IOMMU_PTE_TV;
	amd_iommu_dev_table[devid].data[1] &= DTE_FLAG_MASK;
2116 2117

	amd_iommu_apply_erratum_63(devid);
2118 2119
}

2120 2121
static void do_attach(struct iommu_dev_data *dev_data,
		      struct protection_domain *domain)
2122 2123
{
	struct amd_iommu *iommu;
2124
	u16 alias;
2125
	bool ats;
2126

2127
	iommu = amd_iommu_rlookup_table[dev_data->devid];
2128
	alias = amd_iommu_alias_table[dev_data->devid];
2129
	ats   = dev_data->ats.enabled;
2130 2131 2132 2133 2134 2135 2136 2137 2138

	/* Update data structures */
	dev_data->domain = domain;
	list_add(&dev_data->list, &domain->dev_list);

	/* Do reference counting */
	domain->dev_iommu[iommu->index] += 1;
	domain->dev_cnt                 += 1;

2139 2140 2141
	/* Update device table */
	set_dte_entry(dev_data->devid, domain, ats);
	if (alias != dev_data->devid)
2142
		set_dte_entry(alias, domain, ats);
2143

2144
	device_flush_dte(dev_data);
2145 2146
}

2147
static void do_detach(struct iommu_dev_data *dev_data)
2148 2149
{
	struct amd_iommu *iommu;
2150
	u16 alias;
2151

2152 2153 2154 2155 2156 2157 2158 2159 2160
	/*
	 * First check if the device is still attached. It might already
	 * be detached from its domain because the generic
	 * iommu_detach_group code detached it and we try again here in
	 * our alias handling.
	 */
	if (!dev_data->domain)
		return;

2161
	iommu = amd_iommu_rlookup_table[dev_data->devid];
2162
	alias = amd_iommu_alias_table[dev_data->devid];
2163 2164

	/* decrease reference counters */
2165 2166 2167 2168 2169 2170
	dev_data->domain->dev_iommu[iommu->index] -= 1;
	dev_data->domain->dev_cnt                 -= 1;

	/* Update data structures */
	dev_data->domain = NULL;
	list_del(&dev_data->list);
2171
	clear_dte_entry(dev_data->devid);
2172 2173
	if (alias != dev_data->devid)
		clear_dte_entry(alias);
2174

2175
	/* Flush the DTE entry */
2176
	device_flush_dte(dev_data);
2177 2178 2179 2180 2181 2182
}

/*
 * If a device is not yet associated with a domain, this function does
 * assigns it visible for the hardware
 */
2183
static int __attach_device(struct iommu_dev_data *dev_data,
2184
			   struct protection_domain *domain)
2185
{
2186
	int ret;
2187

2188 2189 2190 2191 2192 2193
	/*
	 * Must be called with IRQs disabled. Warn here to detect early
	 * when its not.
	 */
	WARN_ON(!irqs_disabled());

2194 2195 2196
	/* lock domain */
	spin_lock(&domain->lock);

2197
	ret = -EBUSY;
2198
	if (dev_data->domain != NULL)
2199
		goto out_unlock;
2200

2201
	/* Attach alias group root */
2202
	do_attach(dev_data, domain);
2203

2204 2205 2206 2207
	ret = 0;

out_unlock:

2208 2209
	/* ready */
	spin_unlock(&domain->lock);
2210

2211
	return ret;
2212
}
2213

2214 2215 2216 2217 2218 2219 2220 2221

static void pdev_iommuv2_disable(struct pci_dev *pdev)
{
	pci_disable_ats(pdev);
	pci_disable_pri(pdev);
	pci_disable_pasid(pdev);
}

2222 2223 2224 2225 2226 2227
/* FIXME: Change generic reset-function to do the same */
static int pri_reset_while_enabled(struct pci_dev *pdev)
{
	u16 control;
	int pos;

2228
	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2229 2230 2231
	if (!pos)
		return -EINVAL;

2232 2233 2234
	pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
	control |= PCI_PRI_CTRL_RESET;
	pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
2235 2236 2237 2238

	return 0;
}

2239 2240
static int pdev_iommuv2_enable(struct pci_dev *pdev)
{
2241 2242 2243 2244 2245 2246 2247 2248
	bool reset_enable;
	int reqs, ret;

	/* FIXME: Hardcode number of outstanding requests for now */
	reqs = 32;
	if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
		reqs = 1;
	reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);
2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259

	/* Only allow access to user-accessible pages */
	ret = pci_enable_pasid(pdev, 0);
	if (ret)
		goto out_err;

	/* First reset the PRI state of the device */
	ret = pci_reset_pri(pdev);
	if (ret)
		goto out_err;

2260 2261
	/* Enable PRI */
	ret = pci_enable_pri(pdev, reqs);
2262 2263 2264
	if (ret)
		goto out_err;

2265 2266 2267 2268 2269 2270
	if (reset_enable) {
		ret = pri_reset_while_enabled(pdev);
		if (ret)
			goto out_err;
	}

2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283
	ret = pci_enable_ats(pdev, PAGE_SHIFT);
	if (ret)
		goto out_err;

	return 0;

out_err:
	pci_disable_pri(pdev);
	pci_disable_pasid(pdev);

	return ret;
}

2284
/* FIXME: Move this to PCI code */
2285
#define PCI_PRI_TLP_OFF		(1 << 15)
2286

J
Joerg Roedel 已提交
2287
static bool pci_pri_tlp_required(struct pci_dev *pdev)
2288
{
2289
	u16 status;
2290 2291
	int pos;

2292
	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2293 2294 2295
	if (!pos)
		return false;

2296
	pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
2297

2298
	return (status & PCI_PRI_TLP_OFF) ? true : false;
2299 2300
}

2301
/*
F
Frank Arnold 已提交
2302
 * If a device is not yet associated with a domain, this function
2303 2304
 * assigns it visible for the hardware
 */
2305 2306
static int attach_device(struct device *dev,
			 struct protection_domain *domain)
2307
{
2308
	struct pci_dev *pdev;
2309
	struct iommu_dev_data *dev_data;
2310
	unsigned long flags;
2311
	int ret;
2312

2313 2314
	dev_data = get_dev_data(dev);

2315 2316 2317 2318
	if (!dev_is_pci(dev))
		goto skip_ats_check;

	pdev = to_pci_dev(dev);
2319
	if (domain->flags & PD_IOMMUV2_MASK) {
2320
		if (!dev_data->passthrough)
2321 2322
			return -EINVAL;

2323 2324 2325
		if (dev_data->iommu_v2) {
			if (pdev_iommuv2_enable(pdev) != 0)
				return -EINVAL;
2326

2327 2328 2329 2330
			dev_data->ats.enabled = true;
			dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
			dev_data->pri_tlp     = pci_pri_tlp_required(pdev);
		}
2331 2332
	} else if (amd_iommu_iotlb_sup &&
		   pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
2333 2334 2335
		dev_data->ats.enabled = true;
		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
	}
2336

2337
skip_ats_check:
2338
	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2339
	ret = __attach_device(dev_data, domain);
2340 2341
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

2342 2343 2344 2345 2346
	/*
	 * We might boot into a crash-kernel here. The crashed kernel
	 * left the caches in the IOMMU dirty. So we have to flush
	 * here to evict all dirty stuff.
	 */
2347
	domain_flush_tlb_pde(domain);
2348 2349

	return ret;
2350 2351
}

2352 2353 2354
/*
 * Removes a device from a protection domain (unlocked)
 */
2355
static void __detach_device(struct iommu_dev_data *dev_data)
2356
{
2357
	struct protection_domain *domain;
2358

2359 2360 2361 2362 2363
	/*
	 * Must be called with IRQs disabled. Warn here to detect early
	 * when its not.
	 */
	WARN_ON(!irqs_disabled());
2364

2365 2366
	if (WARN_ON(!dev_data->domain))
		return;
2367

2368
	domain = dev_data->domain;
2369

2370
	spin_lock(&domain->lock);
2371

2372
	do_detach(dev_data);
2373

2374
	spin_unlock(&domain->lock);
2375 2376 2377 2378 2379
}

/*
 * Removes a device from a protection domain (with devtable_lock held)
 */
2380
static void detach_device(struct device *dev)
2381
{
2382
	struct protection_domain *domain;
2383
	struct iommu_dev_data *dev_data;
2384 2385
	unsigned long flags;

2386
	dev_data = get_dev_data(dev);
2387
	domain   = dev_data->domain;
2388

2389 2390
	/* lock device table */
	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2391
	__detach_device(dev_data);
2392
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2393

2394 2395 2396
	if (!dev_is_pci(dev))
		return;

2397
	if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2)
2398 2399
		pdev_iommuv2_disable(to_pci_dev(dev));
	else if (dev_data->ats.enabled)
2400
		pci_disable_ats(to_pci_dev(dev));
2401 2402

	dev_data->ats.enabled = false;
2403
}
2404

2405
static int amd_iommu_add_device(struct device *dev)
2406
{
2407
	struct iommu_dev_data *dev_data;
2408
	struct iommu_domain *domain;
2409
	struct amd_iommu *iommu;
2410
	int ret, devid;
2411

2412
	if (!check_device(dev) || get_dev_data(dev))
2413
		return 0;
2414

2415
	devid = get_device_id(dev);
2416 2417 2418
	if (IS_ERR_VALUE(devid))
		return devid;

2419
	iommu = amd_iommu_rlookup_table[devid];
2420

2421
	ret = iommu_init_device(dev);
2422 2423 2424 2425
	if (ret) {
		if (ret != -ENOTSUPP)
			pr_err("Failed to initialize device %s - trying to proceed anyway\n",
				dev_name(dev));
2426

2427
		iommu_ignore_device(dev);
2428
		dev->archdata.dma_ops = &nommu_dma_ops;
2429 2430 2431
		goto out;
	}
	init_iommu_group(dev);
2432

2433
	dev_data = get_dev_data(dev);
2434

2435
	BUG_ON(!dev_data);
2436

2437
	if (iommu_pass_through || dev_data->iommu_v2)
2438
		iommu_request_dm_for_dev(dev);
2439

2440 2441
	/* Domains are initialized for this device - have a look what we ended up with */
	domain = iommu_get_domain_for_dev(dev);
2442
	if (domain->type == IOMMU_DOMAIN_IDENTITY)
2443
		dev_data->passthrough = true;
2444
	else
2445
		dev->archdata.dma_ops = &amd_iommu_dma_ops;
2446

2447
out:
2448 2449 2450 2451 2452
	iommu_completion_wait(iommu);

	return 0;
}

2453
static void amd_iommu_remove_device(struct device *dev)
2454
{
2455
	struct amd_iommu *iommu;
2456
	int devid;
2457 2458 2459 2460 2461

	if (!check_device(dev))
		return;

	devid = get_device_id(dev);
2462 2463 2464
	if (IS_ERR_VALUE(devid))
		return;

2465 2466 2467 2468
	iommu = amd_iommu_rlookup_table[devid];

	iommu_uninit_device(dev);
	iommu_completion_wait(iommu);
2469 2470
}

2471 2472 2473 2474 2475 2476 2477 2478
static struct iommu_group *amd_iommu_device_group(struct device *dev)
{
	if (dev_is_pci(dev))
		return pci_device_group(dev);

	return acpihid_device_group(dev);
}

2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491
/*****************************************************************************
 *
 * The next functions belong to the dma_ops mapping/unmapping code.
 *
 *****************************************************************************/

/*
 * In the dma_ops path we only have the struct device. This function
 * finds the corresponding IOMMU, the protection domain and the
 * requestor id for a given device.
 * If the device is not yet associated with a domain this is also done
 * in this function.
 */
2492
static struct protection_domain *get_domain(struct device *dev)
2493
{
2494
	struct protection_domain *domain;
2495
	struct iommu_domain *io_domain;
2496

2497
	if (!check_device(dev))
2498
		return ERR_PTR(-EINVAL);
2499

2500
	io_domain = iommu_get_domain_for_dev(dev);
2501 2502
	if (!io_domain)
		return NULL;
2503

2504 2505
	domain = to_pdomain(io_domain);
	if (!dma_ops_domain(domain))
2506
		return ERR_PTR(-EBUSY);
2507

2508
	return domain;
2509 2510
}

2511 2512
static void update_device_table(struct protection_domain *domain)
{
2513
	struct iommu_dev_data *dev_data;
2514

2515 2516
	list_for_each_entry(dev_data, &domain->dev_list, list)
		set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
2517 2518 2519 2520 2521 2522 2523 2524
}

static void update_domain(struct protection_domain *domain)
{
	if (!domain->updated)
		return;

	update_device_table(domain);
2525 2526 2527

	domain_flush_devices(domain);
	domain_flush_tlb_pde(domain);
2528 2529 2530 2531

	domain->updated = false;
}

2532 2533 2534 2535 2536 2537
/*
 * This function fetches the PTE for a given address in the aperture
 */
static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
			    unsigned long address)
{
2538
	struct aperture_range *aperture;
2539 2540
	u64 *pte, *pte_page;

2541 2542 2543 2544 2545
	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
	if (!aperture)
		return NULL;

	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2546
	if (!pte) {
2547
		pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
2548
				GFP_ATOMIC);
2549 2550
		aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
	} else
2551
		pte += PM_LEVEL_INDEX(0, address);
2552

2553
	update_domain(&dom->domain);
2554 2555 2556 2557

	return pte;
}

2558 2559 2560 2561
/*
 * This is the generic map function. It maps one 4kb page at paddr to
 * the given address in the DMA address space for the domain.
 */
2562
static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
2563 2564 2565 2566 2567 2568 2569 2570 2571 2572
				     unsigned long address,
				     phys_addr_t paddr,
				     int direction)
{
	u64 *pte, __pte;

	WARN_ON(address > dom->aperture_size);

	paddr &= PAGE_MASK;

2573
	pte  = dma_ops_get_pte(dom, address);
2574
	if (!pte)
2575
		return DMA_ERROR_CODE;
2576 2577 2578 2579 2580 2581 2582 2583 2584 2585

	__pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;

	if (direction == DMA_TO_DEVICE)
		__pte |= IOMMU_PTE_IR;
	else if (direction == DMA_FROM_DEVICE)
		__pte |= IOMMU_PTE_IW;
	else if (direction == DMA_BIDIRECTIONAL)
		__pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;

2586
	WARN_ON_ONCE(*pte);
2587 2588 2589 2590 2591 2592

	*pte = __pte;

	return (dma_addr_t)address;
}

2593 2594 2595
/*
 * The generic unmapping function for on page in the DMA address space.
 */
2596
static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
2597 2598
				 unsigned long address)
{
2599
	struct aperture_range *aperture;
2600 2601 2602 2603 2604
	u64 *pte;

	if (address >= dom->aperture_size)
		return;

2605 2606 2607 2608 2609 2610 2611
	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
	if (!aperture)
		return;

	pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
	if (!pte)
		return;
2612

2613
	pte += PM_LEVEL_INDEX(0, address);
2614

2615
	WARN_ON_ONCE(!*pte);
2616 2617 2618 2619

	*pte = 0ULL;
}

2620 2621
/*
 * This function contains common code for mapping of a physically
J
Joerg Roedel 已提交
2622 2623
 * contiguous memory region into DMA address space. It is used by all
 * mapping functions provided with this IOMMU driver.
2624 2625
 * Must be called with the domain lock held.
 */
2626 2627 2628 2629
static dma_addr_t __map_single(struct device *dev,
			       struct dma_ops_domain *dma_dom,
			       phys_addr_t paddr,
			       size_t size,
2630
			       int dir,
2631 2632
			       bool align,
			       u64 dma_mask)
2633 2634
{
	dma_addr_t offset = paddr & ~PAGE_MASK;
2635
	dma_addr_t address, start, ret;
2636
	unsigned int pages;
2637
	unsigned long align_mask = 0;
2638 2639
	int i;

2640
	pages = iommu_num_pages(paddr, size, PAGE_SIZE);
2641 2642
	paddr &= PAGE_MASK;

2643 2644
	INC_STATS_COUNTER(total_map_requests);

2645 2646 2647
	if (pages > 1)
		INC_STATS_COUNTER(cross_page);

2648 2649 2650
	if (align)
		align_mask = (1UL << get_order(size)) - 1;

2651 2652
	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
					  dma_mask);
2653

2654 2655
	if (address == DMA_ERROR_CODE)
		goto out;
2656 2657 2658

	start = address;
	for (i = 0; i < pages; ++i) {
2659
		ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
2660
		if (ret == DMA_ERROR_CODE)
2661 2662
			goto out_unmap;

2663 2664 2665 2666 2667
		paddr += PAGE_SIZE;
		start += PAGE_SIZE;
	}
	address += offset;

2668 2669
	ADD_STATS_COUNTER(alloced_io_mem, size);

2670
	if (unlikely(amd_iommu_np_cache)) {
2671
		domain_flush_pages(&dma_dom->domain, address, size);
2672 2673
		domain_flush_complete(&dma_dom->domain);
	}
2674

2675 2676
out:
	return address;
2677 2678 2679 2680 2681

out_unmap:

	for (--i; i >= 0; --i) {
		start -= PAGE_SIZE;
2682
		dma_ops_domain_unmap(dma_dom, start);
2683 2684 2685 2686
	}

	dma_ops_free_addresses(dma_dom, address, pages);

2687
	return DMA_ERROR_CODE;
2688 2689
}

2690 2691 2692 2693
/*
 * Does the reverse of the __map_single function. Must be called with
 * the domain lock held too
 */
2694
static void __unmap_single(struct dma_ops_domain *dma_dom,
2695 2696 2697 2698
			   dma_addr_t dma_addr,
			   size_t size,
			   int dir)
{
2699
	dma_addr_t flush_addr;
2700 2701 2702
	dma_addr_t i, start;
	unsigned int pages;

2703
	if ((dma_addr == DMA_ERROR_CODE) ||
2704
	    (dma_addr + size > dma_dom->aperture_size))
2705 2706
		return;

2707
	flush_addr = dma_addr;
2708
	pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
2709 2710 2711 2712
	dma_addr &= PAGE_MASK;
	start = dma_addr;

	for (i = 0; i < pages; ++i) {
2713
		dma_ops_domain_unmap(dma_dom, start);
2714 2715 2716
		start += PAGE_SIZE;
	}

2717 2718 2719
	SUB_STATS_COUNTER(alloced_io_mem, size);

	dma_ops_free_addresses(dma_dom, dma_addr, pages);
2720 2721
}

2722 2723 2724
/*
 * The exported map_single function for dma_ops.
 */
2725 2726 2727 2728
static dma_addr_t map_page(struct device *dev, struct page *page,
			   unsigned long offset, size_t size,
			   enum dma_data_direction dir,
			   struct dma_attrs *attrs)
2729
{
2730
	phys_addr_t paddr = page_to_phys(page) + offset;
2731
	struct protection_domain *domain;
2732
	u64 dma_mask;
2733

2734 2735
	INC_STATS_COUNTER(cnt_map_single);

2736 2737
	domain = get_domain(dev);
	if (PTR_ERR(domain) == -EINVAL)
2738
		return (dma_addr_t)paddr;
2739 2740
	else if (IS_ERR(domain))
		return DMA_ERROR_CODE;
2741

2742 2743
	dma_mask = *dev->dma_mask;

2744
	return __map_single(dev, domain->priv, paddr, size, dir, false,
2745
			    dma_mask);
2746 2747
}

2748 2749 2750
/*
 * The exported unmap_single function for dma_ops.
 */
2751 2752
static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
		       enum dma_data_direction dir, struct dma_attrs *attrs)
2753 2754 2755
{
	struct protection_domain *domain;

2756 2757
	INC_STATS_COUNTER(cnt_unmap_single);

2758 2759
	domain = get_domain(dev);
	if (IS_ERR(domain))
2760 2761
		return;

2762
	__unmap_single(domain->priv, dma_addr, size, dir);
2763 2764
}

2765 2766 2767 2768
/*
 * The exported map_sg function for dma_ops (handles scatter-gather
 * lists).
 */
2769
static int map_sg(struct device *dev, struct scatterlist *sglist,
2770 2771
		  int nelems, enum dma_data_direction dir,
		  struct dma_attrs *attrs)
2772 2773 2774 2775 2776 2777
{
	struct protection_domain *domain;
	int i;
	struct scatterlist *s;
	phys_addr_t paddr;
	int mapped_elems = 0;
2778
	u64 dma_mask;
2779

2780 2781
	INC_STATS_COUNTER(cnt_map_sg);

2782
	domain = get_domain(dev);
2783
	if (IS_ERR(domain))
2784
		return 0;
2785

2786
	dma_mask = *dev->dma_mask;
2787 2788 2789 2790

	for_each_sg(sglist, s, nelems, i) {
		paddr = sg_phys(s);

2791
		s->dma_address = __map_single(dev, domain->priv,
2792 2793
					      paddr, s->length, dir, false,
					      dma_mask);
2794 2795 2796 2797 2798 2799 2800 2801 2802

		if (s->dma_address) {
			s->dma_length = s->length;
			mapped_elems++;
		} else
			goto unmap;
	}

	return mapped_elems;
2803

2804 2805 2806
unmap:
	for_each_sg(sglist, s, mapped_elems, i) {
		if (s->dma_address)
2807
			__unmap_single(domain->priv, s->dma_address,
2808 2809 2810 2811
				       s->dma_length, dir);
		s->dma_address = s->dma_length = 0;
	}

2812
	return 0;
2813 2814
}

2815 2816 2817 2818
/*
 * The exported map_sg function for dma_ops (handles scatter-gather
 * lists).
 */
2819
static void unmap_sg(struct device *dev, struct scatterlist *sglist,
2820 2821
		     int nelems, enum dma_data_direction dir,
		     struct dma_attrs *attrs)
2822 2823 2824 2825 2826
{
	struct protection_domain *domain;
	struct scatterlist *s;
	int i;

2827 2828
	INC_STATS_COUNTER(cnt_unmap_sg);

2829 2830
	domain = get_domain(dev);
	if (IS_ERR(domain))
2831 2832
		return;

2833
	for_each_sg(sglist, s, nelems, i) {
2834
		__unmap_single(domain->priv, s->dma_address,
2835 2836 2837 2838 2839
			       s->dma_length, dir);
		s->dma_address = s->dma_length = 0;
	}
}

2840 2841 2842
/*
 * The exported alloc_coherent function for dma_ops.
 */
2843
static void *alloc_coherent(struct device *dev, size_t size,
2844 2845
			    dma_addr_t *dma_addr, gfp_t flag,
			    struct dma_attrs *attrs)
2846
{
2847
	u64 dma_mask = dev->coherent_dma_mask;
2848 2849
	struct protection_domain *domain;
	struct page *page;
2850

2851 2852
	INC_STATS_COUNTER(cnt_alloc_coherent);

2853 2854
	domain = get_domain(dev);
	if (PTR_ERR(domain) == -EINVAL) {
2855 2856 2857
		page = alloc_pages(flag, get_order(size));
		*dma_addr = page_to_phys(page);
		return page_address(page);
2858 2859
	} else if (IS_ERR(domain))
		return NULL;
2860

2861
	size	  = PAGE_ALIGN(size);
2862 2863
	dma_mask  = dev->coherent_dma_mask;
	flag     &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
2864
	flag     |= __GFP_ZERO;
2865

2866 2867
	page = alloc_pages(flag | __GFP_NOWARN,  get_order(size));
	if (!page) {
2868
		if (!gfpflags_allow_blocking(flag))
2869
			return NULL;
2870

2871 2872 2873 2874 2875
		page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
						 get_order(size));
		if (!page)
			return NULL;
	}
2876

2877 2878 2879
	if (!dma_mask)
		dma_mask = *dev->dma_mask;

2880
	*dma_addr = __map_single(dev, domain->priv, page_to_phys(page),
2881
				 size, DMA_BIDIRECTIONAL, true, dma_mask);
2882

2883
	if (*dma_addr == DMA_ERROR_CODE)
2884
		goto out_free;
2885

2886
	return page_address(page);
2887 2888 2889

out_free:

2890 2891
	if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
		__free_pages(page, get_order(size));
2892 2893

	return NULL;
2894 2895
}

2896 2897 2898
/*
 * The exported free_coherent function for dma_ops.
 */
2899
static void free_coherent(struct device *dev, size_t size,
2900 2901
			  void *virt_addr, dma_addr_t dma_addr,
			  struct dma_attrs *attrs)
2902 2903
{
	struct protection_domain *domain;
2904
	struct page *page;
2905

2906 2907
	INC_STATS_COUNTER(cnt_free_coherent);

2908 2909 2910
	page = virt_to_page(virt_addr);
	size = PAGE_ALIGN(size);

2911 2912
	domain = get_domain(dev);
	if (IS_ERR(domain))
2913 2914
		goto free_mem;

2915
	__unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
2916 2917

free_mem:
2918 2919
	if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
		__free_pages(page, get_order(size));
2920 2921
}

2922 2923 2924 2925 2926 2927
/*
 * This function is called by the DMA layer to find out if we can handle a
 * particular device. It is part of the dma_ops.
 */
static int amd_iommu_dma_supported(struct device *dev, u64 mask)
{
2928
	return check_device(dev);
2929 2930
}

2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958
static int set_dma_mask(struct device *dev, u64 mask)
{
	struct protection_domain *domain;
	int max_apertures = 1;

	domain = get_domain(dev);
	if (IS_ERR(domain))
		return PTR_ERR(domain);

	if (mask == DMA_BIT_MASK(64))
		max_apertures = 8;
	else if (mask > DMA_BIT_MASK(32))
		max_apertures = 4;

	/*
	 * To prevent lock contention it doesn't make sense to allocate more
	 * apertures than online cpus
	 */
	if (max_apertures > num_online_cpus())
		max_apertures = num_online_cpus();

	if (dma_ops_domain_alloc_apertures(domain->priv, max_apertures))
		dev_err(dev, "Can't allocate %d iommu apertures\n",
			max_apertures);

	return 0;
}

2959
static struct dma_map_ops amd_iommu_dma_ops = {
2960 2961 2962 2963 2964 2965 2966 2967
	.alloc		= alloc_coherent,
	.free		= free_coherent,
	.map_page	= map_page,
	.unmap_page	= unmap_page,
	.map_sg		= map_sg,
	.unmap_sg	= unmap_sg,
	.dma_supported	= amd_iommu_dma_supported,
	.set_dma_mask	= set_dma_mask,
2968 2969
};

2970
int __init amd_iommu_init_api(void)
2971
{
2972
	return bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
2973 2974
}

2975 2976
int __init amd_iommu_init_dma_ops(void)
{
2977
	swiotlb        = iommu_pass_through ? 1 : 0;
2978 2979
	iommu_detected = 1;

2980 2981 2982 2983 2984 2985 2986 2987 2988
	/*
	 * In case we don't initialize SWIOTLB (actually the common case
	 * when AMD IOMMU is enabled), make sure there are global
	 * dma_ops set as a fall-back for devices not handled by this
	 * driver (for example non-PCI devices).
	 */
	if (!swiotlb)
		dma_ops = &nommu_dma_ops;

2989 2990
	amd_iommu_stats_init();

2991 2992 2993 2994 2995
	if (amd_iommu_unmap_flush)
		pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");
	else
		pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");

2996 2997
	return 0;
}
2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010

/*****************************************************************************
 *
 * The following functions belong to the exported interface of AMD IOMMU
 *
 * This interface allows access to lower level functions of the IOMMU
 * like protection domain handling and assignement of devices to domains
 * which is not possible with the dma_ops interface.
 *
 *****************************************************************************/

static void cleanup_domain(struct protection_domain *domain)
{
3011
	struct iommu_dev_data *entry;
3012 3013 3014 3015
	unsigned long flags;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);

3016 3017 3018 3019
	while (!list_empty(&domain->dev_list)) {
		entry = list_first_entry(&domain->dev_list,
					 struct iommu_dev_data, list);
		__detach_device(entry);
3020
	}
3021 3022 3023 3024

	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}

3025 3026 3027 3028 3029
static void protection_domain_free(struct protection_domain *domain)
{
	if (!domain)
		return;

3030 3031
	del_domain_from_list(domain);

3032 3033 3034 3035 3036 3037
	if (domain->id)
		domain_id_free(domain->id);

	kfree(domain);
}

3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049
static int protection_domain_init(struct protection_domain *domain)
{
	spin_lock_init(&domain->lock);
	mutex_init(&domain->api_lock);
	domain->id = domain_id_alloc();
	if (!domain->id)
		return -ENOMEM;
	INIT_LIST_HEAD(&domain->dev_list);

	return 0;
}

3050
static struct protection_domain *protection_domain_alloc(void)
3051 3052 3053 3054 3055
{
	struct protection_domain *domain;

	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
	if (!domain)
3056
		return NULL;
3057

3058
	if (protection_domain_init(domain))
3059 3060
		goto out_err;

3061 3062
	add_domain_to_list(domain);

3063 3064 3065 3066 3067 3068 3069 3070
	return domain;

out_err:
	kfree(domain);

	return NULL;
}

3071
static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)
3072
{
3073
	struct protection_domain *pdomain;
3074
	struct dma_ops_domain *dma_domain;
3075

3076 3077 3078 3079 3080
	switch (type) {
	case IOMMU_DOMAIN_UNMANAGED:
		pdomain = protection_domain_alloc();
		if (!pdomain)
			return NULL;
3081

3082 3083 3084 3085 3086 3087
		pdomain->mode    = PAGE_MODE_3_LEVEL;
		pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pdomain->pt_root) {
			protection_domain_free(pdomain);
			return NULL;
		}
3088

3089 3090 3091
		pdomain->domain.geometry.aperture_start = 0;
		pdomain->domain.geometry.aperture_end   = ~0ULL;
		pdomain->domain.geometry.force_aperture = true;
3092

3093 3094 3095 3096 3097 3098 3099 3100 3101
		break;
	case IOMMU_DOMAIN_DMA:
		dma_domain = dma_ops_domain_alloc();
		if (!dma_domain) {
			pr_err("AMD-Vi: Failed to allocate\n");
			return NULL;
		}
		pdomain = &dma_domain->domain;
		break;
3102 3103 3104 3105
	case IOMMU_DOMAIN_IDENTITY:
		pdomain = protection_domain_alloc();
		if (!pdomain)
			return NULL;
3106

3107 3108
		pdomain->mode = PAGE_MODE_NONE;
		break;
3109 3110 3111
	default:
		return NULL;
	}
3112

3113
	return &pdomain->domain;
3114 3115
}

3116
static void amd_iommu_domain_free(struct iommu_domain *dom)
3117
{
3118
	struct protection_domain *domain;
3119

3120
	if (!dom)
3121 3122
		return;

3123 3124
	domain = to_pdomain(dom);

3125 3126 3127 3128 3129
	if (domain->dev_cnt > 0)
		cleanup_domain(domain);

	BUG_ON(domain->dev_cnt != 0);

3130 3131
	if (domain->mode != PAGE_MODE_NONE)
		free_pagetable(domain);
3132

3133 3134 3135
	if (domain->flags & PD_IOMMUV2_MASK)
		free_gcr3_table(domain);

3136
	protection_domain_free(domain);
3137 3138
}

3139 3140 3141
static void amd_iommu_detach_device(struct iommu_domain *dom,
				    struct device *dev)
{
3142
	struct iommu_dev_data *dev_data = dev->archdata.iommu;
3143
	struct amd_iommu *iommu;
3144
	int devid;
3145

3146
	if (!check_device(dev))
3147 3148
		return;

3149
	devid = get_device_id(dev);
3150 3151
	if (IS_ERR_VALUE(devid))
		return;
3152

3153
	if (dev_data->domain != NULL)
3154
		detach_device(dev);
3155 3156 3157 3158 3159 3160 3161 3162

	iommu = amd_iommu_rlookup_table[devid];
	if (!iommu)
		return;

	iommu_completion_wait(iommu);
}

3163 3164 3165
static int amd_iommu_attach_device(struct iommu_domain *dom,
				   struct device *dev)
{
3166
	struct protection_domain *domain = to_pdomain(dom);
3167
	struct iommu_dev_data *dev_data;
3168
	struct amd_iommu *iommu;
3169
	int ret;
3170

3171
	if (!check_device(dev))
3172 3173
		return -EINVAL;

3174 3175
	dev_data = dev->archdata.iommu;

3176
	iommu = amd_iommu_rlookup_table[dev_data->devid];
3177 3178 3179
	if (!iommu)
		return -EINVAL;

3180
	if (dev_data->domain)
3181
		detach_device(dev);
3182

3183
	ret = attach_device(dev, domain);
3184 3185 3186

	iommu_completion_wait(iommu);

3187
	return ret;
3188 3189
}

3190
static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
3191
			 phys_addr_t paddr, size_t page_size, int iommu_prot)
3192
{
3193
	struct protection_domain *domain = to_pdomain(dom);
3194 3195 3196
	int prot = 0;
	int ret;

3197 3198 3199
	if (domain->mode == PAGE_MODE_NONE)
		return -EINVAL;

3200 3201 3202 3203 3204
	if (iommu_prot & IOMMU_READ)
		prot |= IOMMU_PROT_IR;
	if (iommu_prot & IOMMU_WRITE)
		prot |= IOMMU_PROT_IW;

3205
	mutex_lock(&domain->api_lock);
3206
	ret = iommu_map_page(domain, iova, paddr, prot, page_size);
3207 3208
	mutex_unlock(&domain->api_lock);

3209
	return ret;
3210 3211
}

3212 3213
static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
			   size_t page_size)
3214
{
3215
	struct protection_domain *domain = to_pdomain(dom);
3216
	size_t unmap_size;
3217

3218 3219 3220
	if (domain->mode == PAGE_MODE_NONE)
		return -EINVAL;

3221
	mutex_lock(&domain->api_lock);
3222
	unmap_size = iommu_unmap_page(domain, iova, page_size);
3223
	mutex_unlock(&domain->api_lock);
3224

3225
	domain_flush_tlb_pde(domain);
3226

3227
	return unmap_size;
3228 3229
}

3230
static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
3231
					  dma_addr_t iova)
3232
{
3233
	struct protection_domain *domain = to_pdomain(dom);
3234
	unsigned long offset_mask, pte_pgsize;
3235
	u64 *pte, __pte;
3236

3237 3238 3239
	if (domain->mode == PAGE_MODE_NONE)
		return iova;

3240
	pte = fetch_pte(domain, iova, &pte_pgsize);
3241

3242
	if (!pte || !IOMMU_PTE_PRESENT(*pte))
3243 3244
		return 0;

3245 3246
	offset_mask = pte_pgsize - 1;
	__pte	    = *pte & PM_ADDR_MASK;
3247

3248
	return (__pte & ~offset_mask) | (iova & offset_mask);
3249 3250
}

3251
static bool amd_iommu_capable(enum iommu_cap cap)
S
Sheng Yang 已提交
3252
{
3253 3254
	switch (cap) {
	case IOMMU_CAP_CACHE_COHERENCY:
3255
		return true;
3256
	case IOMMU_CAP_INTR_REMAP:
3257
		return (irq_remapping_enabled == 1);
3258 3259
	case IOMMU_CAP_NOEXEC:
		return false;
3260 3261
	}

3262
	return false;
S
Sheng Yang 已提交
3263 3264
}

3265 3266 3267 3268
static void amd_iommu_get_dm_regions(struct device *dev,
				     struct list_head *head)
{
	struct unity_map_entry *entry;
3269
	int devid;
3270 3271

	devid = get_device_id(dev);
3272 3273
	if (IS_ERR_VALUE(devid))
		return;
3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307

	list_for_each_entry(entry, &amd_iommu_unity_map, list) {
		struct iommu_dm_region *region;

		if (devid < entry->devid_start || devid > entry->devid_end)
			continue;

		region = kzalloc(sizeof(*region), GFP_KERNEL);
		if (!region) {
			pr_err("Out of memory allocating dm-regions for %s\n",
				dev_name(dev));
			return;
		}

		region->start = entry->address_start;
		region->length = entry->address_end - entry->address_start;
		if (entry->prot & IOMMU_PROT_IR)
			region->prot |= IOMMU_READ;
		if (entry->prot & IOMMU_PROT_IW)
			region->prot |= IOMMU_WRITE;

		list_add_tail(&region->list, head);
	}
}

static void amd_iommu_put_dm_regions(struct device *dev,
				     struct list_head *head)
{
	struct iommu_dm_region *entry, *next;

	list_for_each_entry_safe(entry, next, head, list)
		kfree(entry);
}

3308
static const struct iommu_ops amd_iommu_ops = {
3309
	.capable = amd_iommu_capable,
3310 3311
	.domain_alloc = amd_iommu_domain_alloc,
	.domain_free  = amd_iommu_domain_free,
3312 3313
	.attach_dev = amd_iommu_attach_device,
	.detach_dev = amd_iommu_detach_device,
3314 3315
	.map = amd_iommu_map,
	.unmap = amd_iommu_unmap,
O
Olav Haugan 已提交
3316
	.map_sg = default_iommu_map_sg,
3317
	.iova_to_phys = amd_iommu_iova_to_phys,
3318 3319
	.add_device = amd_iommu_add_device,
	.remove_device = amd_iommu_remove_device,
3320
	.device_group = amd_iommu_device_group,
3321 3322
	.get_dm_regions = amd_iommu_get_dm_regions,
	.put_dm_regions = amd_iommu_put_dm_regions,
3323
	.pgsize_bitmap	= AMD_IOMMU_PGSIZES,
3324 3325
};

3326 3327 3328 3329 3330 3331 3332 3333 3334 3335
/*****************************************************************************
 *
 * The next functions do a basic initialization of IOMMU for pass through
 * mode
 *
 * In passthrough mode the IOMMU is initialized and enabled but not used for
 * DMA-API translation.
 *
 *****************************************************************************/

3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347
/* IOMMUv2 specific functions */
int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
{
	return atomic_notifier_chain_register(&ppr_notifier, nb);
}
EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);

int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
{
	return atomic_notifier_chain_unregister(&ppr_notifier, nb);
}
EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
3348 3349 3350

void amd_iommu_domain_direct_map(struct iommu_domain *dom)
{
3351
	struct protection_domain *domain = to_pdomain(dom);
3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368
	unsigned long flags;

	spin_lock_irqsave(&domain->lock, flags);

	/* Update data structure */
	domain->mode    = PAGE_MODE_NONE;
	domain->updated = true;

	/* Make changes visible to IOMMUs */
	update_domain(domain);

	/* Page-table is not visible to IOMMU anymore, so free it */
	free_pagetable(domain);

	spin_unlock_irqrestore(&domain->lock, flags);
}
EXPORT_SYMBOL(amd_iommu_domain_direct_map);
3369 3370 3371

int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
{
3372
	struct protection_domain *domain = to_pdomain(dom);
3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415
	unsigned long flags;
	int levels, ret;

	if (pasids <= 0 || pasids > (PASID_MASK + 1))
		return -EINVAL;

	/* Number of GCR3 table levels required */
	for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
		levels += 1;

	if (levels > amd_iommu_max_glx_val)
		return -EINVAL;

	spin_lock_irqsave(&domain->lock, flags);

	/*
	 * Save us all sanity checks whether devices already in the
	 * domain support IOMMUv2. Just force that the domain has no
	 * devices attached when it is switched into IOMMUv2 mode.
	 */
	ret = -EBUSY;
	if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
		goto out;

	ret = -ENOMEM;
	domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
	if (domain->gcr3_tbl == NULL)
		goto out;

	domain->glx      = levels;
	domain->flags   |= PD_IOMMUV2_MASK;
	domain->updated  = true;

	update_domain(domain);

	ret = 0;

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449

static int __flush_pasid(struct protection_domain *domain, int pasid,
			 u64 address, bool size)
{
	struct iommu_dev_data *dev_data;
	struct iommu_cmd cmd;
	int i, ret;

	if (!(domain->flags & PD_IOMMUV2_MASK))
		return -EINVAL;

	build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);

	/*
	 * IOMMU TLB needs to be flushed before Device TLB to
	 * prevent device TLB refill from IOMMU TLB
	 */
	for (i = 0; i < amd_iommus_present; ++i) {
		if (domain->dev_iommu[i] == 0)
			continue;

		ret = iommu_queue_command(amd_iommus[i], &cmd);
		if (ret != 0)
			goto out;
	}

	/* Wait until IOMMU TLB flushes are complete */
	domain_flush_complete(domain);

	/* Now flush device TLBs */
	list_for_each_entry(dev_data, &domain->dev_list, list) {
		struct amd_iommu *iommu;
		int qdep;

3450 3451 3452 3453 3454 3455
		/*
		   There might be non-IOMMUv2 capable devices in an IOMMUv2
		 * domain.
		 */
		if (!dev_data->ats.enabled)
			continue;
3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480

		qdep  = dev_data->ats.qdep;
		iommu = amd_iommu_rlookup_table[dev_data->devid];

		build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
				      qdep, address, size);

		ret = iommu_queue_command(iommu, &cmd);
		if (ret != 0)
			goto out;
	}

	/* Wait until all device TLBs are flushed */
	domain_flush_complete(domain);

	ret = 0;

out:

	return ret;
}

static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
				  u64 address)
{
3481 3482
	INC_STATS_COUNTER(invalidate_iotlb);

3483 3484 3485 3486 3487 3488
	return __flush_pasid(domain, pasid, address, false);
}

int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
			 u64 address)
{
3489
	struct protection_domain *domain = to_pdomain(dom);
3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __amd_iommu_flush_page(domain, pasid, address);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_flush_page);

static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
{
3503 3504
	INC_STATS_COUNTER(invalidate_iotlb_all);

3505 3506 3507 3508 3509 3510
	return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
			     true);
}

int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
{
3511
	struct protection_domain *domain = to_pdomain(dom);
3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __amd_iommu_flush_tlb(domain, pasid);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_flush_tlb);

3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590
static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
{
	int index;
	u64 *pte;

	while (true) {

		index = (pasid >> (9 * level)) & 0x1ff;
		pte   = &root[index];

		if (level == 0)
			break;

		if (!(*pte & GCR3_VALID)) {
			if (!alloc)
				return NULL;

			root = (void *)get_zeroed_page(GFP_ATOMIC);
			if (root == NULL)
				return NULL;

			*pte = __pa(root) | GCR3_VALID;
		}

		root = __va(*pte & PAGE_MASK);

		level -= 1;
	}

	return pte;
}

static int __set_gcr3(struct protection_domain *domain, int pasid,
		      unsigned long cr3)
{
	u64 *pte;

	if (domain->mode != PAGE_MODE_NONE)
		return -EINVAL;

	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
	if (pte == NULL)
		return -ENOMEM;

	*pte = (cr3 & PAGE_MASK) | GCR3_VALID;

	return __amd_iommu_flush_tlb(domain, pasid);
}

static int __clear_gcr3(struct protection_domain *domain, int pasid)
{
	u64 *pte;

	if (domain->mode != PAGE_MODE_NONE)
		return -EINVAL;

	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
	if (pte == NULL)
		return 0;

	*pte = 0;

	return __amd_iommu_flush_tlb(domain, pasid);
}

int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
			      unsigned long cr3)
{
3591
	struct protection_domain *domain = to_pdomain(dom);
3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __set_gcr3(domain, pasid, cr3);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);

int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
{
3605
	struct protection_domain *domain = to_pdomain(dom);
3606 3607 3608 3609 3610 3611 3612 3613 3614 3615
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __clear_gcr3(domain, pasid);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
3616 3617 3618 3619 3620 3621 3622 3623

int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
			   int status, int tag)
{
	struct iommu_dev_data *dev_data;
	struct amd_iommu *iommu;
	struct iommu_cmd cmd;

3624 3625
	INC_STATS_COUNTER(complete_ppr);

3626 3627 3628 3629 3630 3631 3632 3633 3634
	dev_data = get_dev_data(&pdev->dev);
	iommu    = amd_iommu_rlookup_table[dev_data->devid];

	build_complete_ppr(&cmd, dev_data->devid, pasid, status,
			   tag, dev_data->pri_tlp);

	return iommu_queue_command(iommu, &cmd);
}
EXPORT_SYMBOL(amd_iommu_complete_ppr);
3635 3636 3637

struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
{
3638
	struct protection_domain *pdomain;
3639

3640 3641
	pdomain = get_domain(&pdev->dev);
	if (IS_ERR(pdomain))
3642 3643 3644
		return NULL;

	/* Only return IOMMUv2 domains */
3645
	if (!(pdomain->flags & PD_IOMMUV2_MASK))
3646 3647
		return NULL;

3648
	return &pdomain->domain;
3649 3650
}
EXPORT_SYMBOL(amd_iommu_get_v2_domain);
3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662

void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
{
	struct iommu_dev_data *dev_data;

	if (!amd_iommu_v2_supported())
		return;

	dev_data = get_dev_data(&pdev->dev);
	dev_data->errata |= (1 << erratum);
}
EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705

int amd_iommu_device_info(struct pci_dev *pdev,
                          struct amd_iommu_device_info *info)
{
	int max_pasids;
	int pos;

	if (pdev == NULL || info == NULL)
		return -EINVAL;

	if (!amd_iommu_v2_supported())
		return -EINVAL;

	memset(info, 0, sizeof(*info));

	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
	if (pos)
		info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;

	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
	if (pos)
		info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;

	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
	if (pos) {
		int features;

		max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
		max_pasids = min(max_pasids, (1 << 20));

		info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
		info->max_pasids = min(pci_max_pasids(pdev), max_pasids);

		features = pci_pasid_features(pdev);
		if (features & PCI_PASID_CAP_EXEC)
			info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
		if (features & PCI_PASID_CAP_PRIV)
			info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
	}

	return 0;
}
EXPORT_SYMBOL(amd_iommu_device_info);
3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729

#ifdef CONFIG_IRQ_REMAP

/*****************************************************************************
 *
 * Interrupt Remapping Implementation
 *
 *****************************************************************************/

union irte {
	u32 val;
	struct {
		u32 valid	: 1,
		    no_fault	: 1,
		    int_type	: 3,
		    rq_eoi	: 1,
		    dm		: 1,
		    rsvd_1	: 1,
		    destination	: 8,
		    vector	: 8,
		    rsvd_2	: 8;
	} fields;
};

3730 3731 3732 3733 3734
struct irq_2_irte {
	u16 devid; /* Device ID for IRTE table */
	u16 index; /* Index into IRTE table*/
};

3735 3736 3737 3738 3739 3740 3741 3742 3743 3744
struct amd_ir_data {
	struct irq_2_irte			irq_2_irte;
	union irte				irte_entry;
	union {
		struct msi_msg			msi_entry;
	};
};

static struct irq_chip amd_ir_chip;

3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796
#define DTE_IRQ_PHYS_ADDR_MASK	(((1ULL << 45)-1) << 6)
#define DTE_IRQ_REMAP_INTCTL    (2ULL << 60)
#define DTE_IRQ_TABLE_LEN       (8ULL << 1)
#define DTE_IRQ_REMAP_ENABLE    1ULL

static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
{
	u64 dte;

	dte	= amd_iommu_dev_table[devid].data[2];
	dte	&= ~DTE_IRQ_PHYS_ADDR_MASK;
	dte	|= virt_to_phys(table->table);
	dte	|= DTE_IRQ_REMAP_INTCTL;
	dte	|= DTE_IRQ_TABLE_LEN;
	dte	|= DTE_IRQ_REMAP_ENABLE;

	amd_iommu_dev_table[devid].data[2] = dte;
}

#define IRTE_ALLOCATED (~1U)

static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
{
	struct irq_remap_table *table = NULL;
	struct amd_iommu *iommu;
	unsigned long flags;
	u16 alias;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);

	iommu = amd_iommu_rlookup_table[devid];
	if (!iommu)
		goto out_unlock;

	table = irq_lookup_table[devid];
	if (table)
		goto out;

	alias = amd_iommu_alias_table[devid];
	table = irq_lookup_table[alias];
	if (table) {
		irq_lookup_table[devid] = table;
		set_dte_irq_entry(devid, table);
		iommu_flush_dte(iommu, devid);
		goto out;
	}

	/* Nothing there yet, allocate new irq remapping table */
	table = kzalloc(sizeof(*table), GFP_ATOMIC);
	if (!table)
		goto out;

3797 3798 3799
	/* Initialize table spin-lock */
	spin_lock_init(&table->lock);

3800 3801 3802 3803 3804 3805 3806
	if (ioapic)
		/* Keep the first 32 indexes free for IOAPIC interrupts */
		table->min_index = 32;

	table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_ATOMIC);
	if (!table->table) {
		kfree(table);
3807
		table = NULL;
3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824
		goto out;
	}

	memset(table->table, 0, MAX_IRQS_PER_TABLE * sizeof(u32));

	if (ioapic) {
		int i;

		for (i = 0; i < 32; ++i)
			table->table[i] = IRTE_ALLOCATED;
	}

	irq_lookup_table[devid] = table;
	set_dte_irq_entry(devid, table);
	iommu_flush_dte(iommu, devid);
	if (devid != alias) {
		irq_lookup_table[alias] = table;
3825
		set_dte_irq_entry(alias, table);
3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837
		iommu_flush_dte(iommu, alias);
	}

out:
	iommu_completion_wait(iommu);

out_unlock:
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

	return table;
}

3838
static int alloc_irq_index(u16 devid, int count)
3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921
{
	struct irq_remap_table *table;
	unsigned long flags;
	int index, c;

	table = get_irq_table(devid, false);
	if (!table)
		return -ENODEV;

	spin_lock_irqsave(&table->lock, flags);

	/* Scan table for free entries */
	for (c = 0, index = table->min_index;
	     index < MAX_IRQS_PER_TABLE;
	     ++index) {
		if (table->table[index] == 0)
			c += 1;
		else
			c = 0;

		if (c == count)	{
			for (; c != 0; --c)
				table->table[index - c + 1] = IRTE_ALLOCATED;

			index -= count - 1;
			goto out;
		}
	}

	index = -ENOSPC;

out:
	spin_unlock_irqrestore(&table->lock, flags);

	return index;
}

static int modify_irte(u16 devid, int index, union irte irte)
{
	struct irq_remap_table *table;
	struct amd_iommu *iommu;
	unsigned long flags;

	iommu = amd_iommu_rlookup_table[devid];
	if (iommu == NULL)
		return -EINVAL;

	table = get_irq_table(devid, false);
	if (!table)
		return -ENOMEM;

	spin_lock_irqsave(&table->lock, flags);
	table->table[index] = irte.val;
	spin_unlock_irqrestore(&table->lock, flags);

	iommu_flush_irt(iommu, devid);
	iommu_completion_wait(iommu);

	return 0;
}

static void free_irte(u16 devid, int index)
{
	struct irq_remap_table *table;
	struct amd_iommu *iommu;
	unsigned long flags;

	iommu = amd_iommu_rlookup_table[devid];
	if (iommu == NULL)
		return;

	table = get_irq_table(devid, false);
	if (!table)
		return;

	spin_lock_irqsave(&table->lock, flags);
	table->table[index] = 0;
	spin_unlock_irqrestore(&table->lock, flags);

	iommu_flush_irt(iommu, devid);
	iommu_completion_wait(iommu);
}

3922
static int get_devid(struct irq_alloc_info *info)
3923
{
3924
	int devid = -1;
3925

3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940
	switch (info->type) {
	case X86_IRQ_ALLOC_TYPE_IOAPIC:
		devid     = get_ioapic_devid(info->ioapic_id);
		break;
	case X86_IRQ_ALLOC_TYPE_HPET:
		devid     = get_hpet_devid(info->hpet_id);
		break;
	case X86_IRQ_ALLOC_TYPE_MSI:
	case X86_IRQ_ALLOC_TYPE_MSIX:
		devid = get_device_id(&info->msi_dev->dev);
		break;
	default:
		BUG_ON(1);
		break;
	}
3941

3942 3943
	return devid;
}
3944

3945 3946 3947 3948
static struct irq_domain *get_ir_irq_domain(struct irq_alloc_info *info)
{
	struct amd_iommu *iommu;
	int devid;
3949

3950 3951
	if (!info)
		return NULL;
3952

3953 3954 3955 3956 3957 3958
	devid = get_devid(info);
	if (devid >= 0) {
		iommu = amd_iommu_rlookup_table[devid];
		if (iommu)
			return iommu->ir_domain;
	}
3959

3960
	return NULL;
3961 3962
}

3963
static struct irq_domain *get_irq_domain(struct irq_alloc_info *info)
3964
{
3965 3966
	struct amd_iommu *iommu;
	int devid;
3967

3968 3969
	if (!info)
		return NULL;
3970

3971 3972 3973 3974
	switch (info->type) {
	case X86_IRQ_ALLOC_TYPE_MSI:
	case X86_IRQ_ALLOC_TYPE_MSIX:
		devid = get_device_id(&info->msi_dev->dev);
3975 3976 3977
		if (IS_ERR_VALUE(devid))
			return NULL;

3978 3979 3980
		iommu = amd_iommu_rlookup_table[devid];
		if (iommu)
			return iommu->msi_domain;
3981 3982 3983 3984
		break;
	default:
		break;
	}
3985

3986 3987
	return NULL;
}
3988

3989 3990 3991 3992 3993 3994
struct irq_remap_ops amd_iommu_irq_ops = {
	.prepare		= amd_iommu_prepare,
	.enable			= amd_iommu_enable,
	.disable		= amd_iommu_disable,
	.reenable		= amd_iommu_reenable,
	.enable_faulting	= amd_iommu_enable_faulting,
3995 3996 3997
	.get_ir_irq_domain	= get_ir_irq_domain,
	.get_irq_domain		= get_irq_domain,
};
3998

3999 4000 4001 4002 4003 4004 4005 4006 4007
static void irq_remapping_prepare_irte(struct amd_ir_data *data,
				       struct irq_cfg *irq_cfg,
				       struct irq_alloc_info *info,
				       int devid, int index, int sub_handle)
{
	struct irq_2_irte *irte_info = &data->irq_2_irte;
	struct msi_msg *msg = &data->msi_entry;
	union irte *irte = &data->irte_entry;
	struct IO_APIC_route_entry *entry;
4008

4009 4010
	data->irq_2_irte.devid = devid;
	data->irq_2_irte.index = index + sub_handle;
4011

4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033
	/* Setup IRTE for IOMMU */
	irte->val = 0;
	irte->fields.vector      = irq_cfg->vector;
	irte->fields.int_type    = apic->irq_delivery_mode;
	irte->fields.destination = irq_cfg->dest_apicid;
	irte->fields.dm          = apic->irq_dest_mode;
	irte->fields.valid       = 1;

	switch (info->type) {
	case X86_IRQ_ALLOC_TYPE_IOAPIC:
		/* Setup IOAPIC entry */
		entry = info->ioapic_entry;
		info->ioapic_entry = NULL;
		memset(entry, 0, sizeof(*entry));
		entry->vector        = index;
		entry->mask          = 0;
		entry->trigger       = info->ioapic_trigger;
		entry->polarity      = info->ioapic_polarity;
		/* Mask level triggered irqs. */
		if (info->ioapic_trigger)
			entry->mask = 1;
		break;
4034

4035 4036 4037 4038 4039 4040 4041
	case X86_IRQ_ALLOC_TYPE_HPET:
	case X86_IRQ_ALLOC_TYPE_MSI:
	case X86_IRQ_ALLOC_TYPE_MSIX:
		msg->address_hi = MSI_ADDR_BASE_HI;
		msg->address_lo = MSI_ADDR_BASE_LO;
		msg->data = irte_info->index;
		break;
4042

4043 4044 4045 4046
	default:
		BUG_ON(1);
		break;
	}
4047 4048
}

4049 4050
static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
			       unsigned int nr_irqs, void *arg)
4051
{
4052 4053 4054
	struct irq_alloc_info *info = arg;
	struct irq_data *irq_data;
	struct amd_ir_data *data;
4055
	struct irq_cfg *cfg;
4056 4057
	int i, ret, devid;
	int index = -1;
4058

4059 4060 4061 4062
	if (!info)
		return -EINVAL;
	if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_MSI &&
	    info->type != X86_IRQ_ALLOC_TYPE_MSIX)
4063 4064
		return -EINVAL;

4065 4066 4067 4068 4069 4070
	/*
	 * With IRQ remapping enabled, don't need contiguous CPU vectors
	 * to support multiple MSI interrupts.
	 */
	if (info->type == X86_IRQ_ALLOC_TYPE_MSI)
		info->flags &= ~X86_IRQ_ALLOC_CONTIGUOUS_VECTORS;
4071

4072 4073 4074
	devid = get_devid(info);
	if (devid < 0)
		return -EINVAL;
4075

4076 4077 4078
	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
	if (ret < 0)
		return ret;
4079

4080 4081 4082 4083 4084 4085
	if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) {
		if (get_irq_table(devid, true))
			index = info->ioapic_pin;
		else
			ret = -ENOMEM;
	} else {
4086
		index = alloc_irq_index(devid, nr_irqs);
4087 4088 4089 4090 4091
	}
	if (index < 0) {
		pr_warn("Failed to allocate IRTE\n");
		goto out_free_parent;
	}
4092

4093 4094 4095 4096 4097 4098 4099
	for (i = 0; i < nr_irqs; i++) {
		irq_data = irq_domain_get_irq_data(domain, virq + i);
		cfg = irqd_cfg(irq_data);
		if (!irq_data || !cfg) {
			ret = -EINVAL;
			goto out_free_data;
		}
4100

4101 4102 4103 4104 4105
		ret = -ENOMEM;
		data = kzalloc(sizeof(*data), GFP_KERNEL);
		if (!data)
			goto out_free_data;

4106 4107 4108 4109 4110 4111
		irq_data->hwirq = (devid << 16) + i;
		irq_data->chip_data = data;
		irq_data->chip = &amd_ir_chip;
		irq_remapping_prepare_irte(data, cfg, info, devid, index, i);
		irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT);
	}
4112

4113
	return 0;
4114

4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125
out_free_data:
	for (i--; i >= 0; i--) {
		irq_data = irq_domain_get_irq_data(domain, virq + i);
		if (irq_data)
			kfree(irq_data->chip_data);
	}
	for (i = 0; i < nr_irqs; i++)
		free_irte(devid, index + i);
out_free_parent:
	irq_domain_free_irqs_common(domain, virq, nr_irqs);
	return ret;
4126 4127
}

4128 4129
static void irq_remapping_free(struct irq_domain *domain, unsigned int virq,
			       unsigned int nr_irqs)
4130
{
4131 4132 4133 4134
	struct irq_2_irte *irte_info;
	struct irq_data *irq_data;
	struct amd_ir_data *data;
	int i;
4135

4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146
	for (i = 0; i < nr_irqs; i++) {
		irq_data = irq_domain_get_irq_data(domain, virq  + i);
		if (irq_data && irq_data->chip_data) {
			data = irq_data->chip_data;
			irte_info = &data->irq_2_irte;
			free_irte(irte_info->devid, irte_info->index);
			kfree(data);
		}
	}
	irq_domain_free_irqs_common(domain, virq, nr_irqs);
}
4147

4148 4149 4150 4151 4152
static void irq_remapping_activate(struct irq_domain *domain,
				   struct irq_data *irq_data)
{
	struct amd_ir_data *data = irq_data->chip_data;
	struct irq_2_irte *irte_info = &data->irq_2_irte;
4153

4154
	modify_irte(irte_info->devid, irte_info->index, data->irte_entry);
4155 4156
}

4157 4158
static void irq_remapping_deactivate(struct irq_domain *domain,
				     struct irq_data *irq_data)
4159
{
4160 4161 4162
	struct amd_ir_data *data = irq_data->chip_data;
	struct irq_2_irte *irte_info = &data->irq_2_irte;
	union irte entry;
4163

4164 4165 4166
	entry.val = 0;
	modify_irte(irte_info->devid, irte_info->index, data->irte_entry);
}
4167

4168 4169 4170 4171 4172
static struct irq_domain_ops amd_ir_domain_ops = {
	.alloc = irq_remapping_alloc,
	.free = irq_remapping_free,
	.activate = irq_remapping_activate,
	.deactivate = irq_remapping_deactivate,
4173
};
4174

4175 4176 4177 4178 4179 4180 4181 4182
static int amd_ir_set_affinity(struct irq_data *data,
			       const struct cpumask *mask, bool force)
{
	struct amd_ir_data *ir_data = data->chip_data;
	struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
	struct irq_cfg *cfg = irqd_cfg(data);
	struct irq_data *parent = data->parent_data;
	int ret;
4183

4184 4185 4186
	ret = parent->chip->irq_set_affinity(parent, mask, force);
	if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
		return ret;
4187

4188 4189 4190 4191 4192 4193 4194
	/*
	 * Atomically updates the IRTE with the new destination, vector
	 * and flushes the interrupt entry cache.
	 */
	ir_data->irte_entry.fields.vector = cfg->vector;
	ir_data->irte_entry.fields.destination = cfg->dest_apicid;
	modify_irte(irte_info->devid, irte_info->index, ir_data->irte_entry);
4195

4196 4197 4198 4199 4200
	/*
	 * After this point, all the interrupts will start arriving
	 * at the new destination. So, time to cleanup the previous
	 * vector allocation.
	 */
4201
	send_cleanup_vector(cfg);
4202 4203

	return IRQ_SET_MASK_OK_DONE;
4204 4205
}

4206
static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg)
4207
{
4208
	struct amd_ir_data *ir_data = irq_data->chip_data;
4209

4210 4211
	*msg = ir_data->msi_entry;
}
4212

4213 4214 4215 4216 4217
static struct irq_chip amd_ir_chip = {
	.irq_ack = ir_ack_apic_edge,
	.irq_set_affinity = amd_ir_set_affinity,
	.irq_compose_msi_msg = ir_compose_msi_msg,
};
4218

4219 4220 4221 4222 4223
int amd_iommu_create_irq_domain(struct amd_iommu *iommu)
{
	iommu->ir_domain = irq_domain_add_tree(NULL, &amd_ir_domain_ops, iommu);
	if (!iommu->ir_domain)
		return -ENOMEM;
4224

4225 4226
	iommu->ir_domain->parent = arch_get_ir_parent_domain();
	iommu->msi_domain = arch_create_msi_irq_domain(iommu->ir_domain);
4227 4228 4229

	return 0;
}
4230
#endif