amd_iommu.c 92.8 KB
Newer Older
1
/*
2
 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
J
Joerg Roedel 已提交
3
 * Author: Joerg Roedel <jroedel@suse.de>
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
 *         Leo Duran <leo.duran@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

20
#include <linux/ratelimit.h>
21
#include <linux/pci.h>
22
#include <linux/pci-ats.h>
A
Akinobu Mita 已提交
23
#include <linux/bitmap.h>
24
#include <linux/slab.h>
25
#include <linux/debugfs.h>
26
#include <linux/scatterlist.h>
27
#include <linux/dma-mapping.h>
28
#include <linux/iommu-helper.h>
29
#include <linux/iommu.h>
30
#include <linux/delay.h>
31
#include <linux/amd-iommu.h>
32 33
#include <linux/notifier.h>
#include <linux/export.h>
34 35
#include <linux/irq.h>
#include <linux/msi.h>
36
#include <linux/dma-contiguous.h>
37
#include <linux/irqdomain.h>
38 39 40 41
#include <asm/irq_remapping.h>
#include <asm/io_apic.h>
#include <asm/apic.h>
#include <asm/hw_irq.h>
42
#include <asm/msidef.h>
43
#include <asm/proto.h>
44
#include <asm/iommu.h>
45
#include <asm/gart.h>
46
#include <asm/dma.h>
47 48 49

#include "amd_iommu_proto.h"
#include "amd_iommu_types.h"
50
#include "irq_remapping.h"
51 52 53

#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))

54
#define LOOP_TIMEOUT	100000
55

56 57 58 59 60 61
/*
 * This bitmap is used to advertise the page sizes our hardware support
 * to the IOMMU core, which will then use this information to split
 * physically contiguous memory regions it is mapping into page sizes
 * that we support.
 *
J
Joerg Roedel 已提交
62
 * 512GB Pages are not supported due to a hardware bug
63
 */
J
Joerg Roedel 已提交
64
#define AMD_IOMMU_PGSIZES	((~0xFFFUL) & ~(2ULL << 38))
65

66 67
static DEFINE_RWLOCK(amd_iommu_devtable_lock);

68 69 70 71
/* List of all available dev_data structures */
static LIST_HEAD(dev_data_list);
static DEFINE_SPINLOCK(dev_data_list_lock);

72 73 74
LIST_HEAD(ioapic_map);
LIST_HEAD(hpet_map);

75 76 77 78
/*
 * Domain for untranslated devices - only allocated
 * if iommu=pt passed on kernel cmd line.
 */
79
static const struct iommu_ops amd_iommu_ops;
80

81
static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
82
int amd_iommu_max_glx_val = -1;
83

84 85
static struct dma_map_ops amd_iommu_dma_ops;

86 87 88 89 90 91 92 93 94
/*
 * This struct contains device specific data for the IOMMU
 */
struct iommu_dev_data {
	struct list_head list;		  /* For domain->dev_list */
	struct list_head dev_data_list;	  /* For global dev_data_list */
	struct protection_domain *domain; /* Domain the device is bound to */
	u16 devid;			  /* PCI Device ID */
	bool iommu_v2;			  /* Device can make use of IOMMUv2 */
95
	bool passthrough;		  /* Device is identity mapped */
96 97 98 99 100 101 102 103 104
	struct {
		bool enabled;
		int qdep;
	} ats;				  /* ATS state */
	bool pri_tlp;			  /* PASID TLB required for
					     PPR completions */
	u32 errata;			  /* Bitmap for errata to apply */
};

105 106 107
/*
 * general struct to manage commands send to an IOMMU
 */
108
struct iommu_cmd {
109 110 111
	u32 data[4];
};

112 113
struct kmem_cache *amd_iommu_irq_cache;

114
static void update_domain(struct protection_domain *domain);
115
static int protection_domain_init(struct protection_domain *domain);
116

117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
/*
 * For dynamic growth the aperture size is split into ranges of 128MB of
 * DMA address space each. This struct represents one such range.
 */
struct aperture_range {

	/* address allocation bitmap */
	unsigned long *bitmap;

	/*
	 * Array of PTE pages for the aperture. In this array we save all the
	 * leaf pages of the domain page table used for the aperture. This way
	 * we don't need to walk the page table to find a specific PTE. We can
	 * just calculate its address in constant time.
	 */
	u64 *pte_pages[64];

	unsigned long offset;
};

/*
 * Data container for a dma_ops specific protection domain
 */
struct dma_ops_domain {
	/* generic protection domain information */
	struct protection_domain domain;

	/* size of the aperture for the mappings */
	unsigned long aperture_size;

	/* address we start to search for free addresses */
	unsigned long next_address;

	/* address space relevant data */
	struct aperture_range *aperture[APERTURE_MAX_RANGES];

	/* This will be set to true when TLB needs to be flushed */
	bool need_flush;
};

157 158 159 160 161 162
/****************************************************************************
 *
 * Helper functions
 *
 ****************************************************************************/

163 164 165 166 167
static struct protection_domain *to_pdomain(struct iommu_domain *dom)
{
	return container_of(dom, struct protection_domain, domain);
}

168
static struct iommu_dev_data *alloc_dev_data(u16 devid)
169 170 171 172 173 174 175 176
{
	struct iommu_dev_data *dev_data;
	unsigned long flags;

	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
	if (!dev_data)
		return NULL;

177
	dev_data->devid = devid;
178 179 180 181 182 183 184 185

	spin_lock_irqsave(&dev_data_list_lock, flags);
	list_add_tail(&dev_data->dev_data_list, &dev_data_list);
	spin_unlock_irqrestore(&dev_data_list_lock, flags);

	return dev_data;
}

186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
static struct iommu_dev_data *search_dev_data(u16 devid)
{
	struct iommu_dev_data *dev_data;
	unsigned long flags;

	spin_lock_irqsave(&dev_data_list_lock, flags);
	list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {
		if (dev_data->devid == devid)
			goto out_unlock;
	}

	dev_data = NULL;

out_unlock:
	spin_unlock_irqrestore(&dev_data_list_lock, flags);

	return dev_data;
}

static struct iommu_dev_data *find_dev_data(u16 devid)
{
	struct iommu_dev_data *dev_data;

	dev_data = search_dev_data(devid);

	if (dev_data == NULL)
		dev_data = alloc_dev_data(devid);

	return dev_data;
}

217 218 219 220
static inline u16 get_device_id(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);

221
	return PCI_DEVID(pdev->bus->number, pdev->devfn);
222 223
}

224 225 226 227 228
static struct iommu_dev_data *get_dev_data(struct device *dev)
{
	return dev->archdata.iommu;
}

229 230 231 232
static bool pci_iommuv2_capable(struct pci_dev *pdev)
{
	static const int caps[] = {
		PCI_EXT_CAP_ID_ATS,
233 234
		PCI_EXT_CAP_ID_PRI,
		PCI_EXT_CAP_ID_PASID,
235 236 237 238 239 240 241 242 243 244 245 246
	};
	int i, pos;

	for (i = 0; i < 3; ++i) {
		pos = pci_find_ext_capability(pdev, caps[i]);
		if (pos == 0)
			return false;
	}

	return true;
}

247 248 249 250 251 252 253 254 255
static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
{
	struct iommu_dev_data *dev_data;

	dev_data = get_dev_data(&pdev->dev);

	return dev_data->errata & (1 << erratum) ? true : false;
}

256
/*
257 258
 * This function actually applies the mapping to the page table of the
 * dma_ops domain.
259
 */
260 261
static void alloc_unity_mapping(struct dma_ops_domain *dma_dom,
				struct unity_map_entry *e)
262
{
263
	u64 addr;
264

265 266 267 268 269
	for (addr = e->address_start; addr < e->address_end;
	     addr += PAGE_SIZE) {
		if (addr < dma_dom->aperture_size)
			__set_bit(addr >> PAGE_SHIFT,
				  dma_dom->aperture[0]->bitmap);
270
	}
271
}
272

273 274 275 276 277 278 279 280
/*
 * Inits the unity mappings required for a specific device
 */
static void init_unity_mappings_for_device(struct device *dev,
					   struct dma_ops_domain *dma_dom)
{
	struct unity_map_entry *e;
	u16 devid;
281

282
	devid = get_device_id(dev);
283

284 285 286 287 288
	list_for_each_entry(e, &amd_iommu_unity_map, list) {
		if (!(devid >= e->devid_start && devid <= e->devid_end))
			continue;
		alloc_unity_mapping(dma_dom, e);
	}
289 290
}

291 292 293 294 295 296 297 298 299 300 301
/*
 * This function checks if the driver got a valid device from the caller to
 * avoid dereferencing invalid pointers.
 */
static bool check_device(struct device *dev)
{
	u16 devid;

	if (!dev || !dev->dma_mask)
		return false;

302 303
	/* No PCI device */
	if (!dev_is_pci(dev))
304 305 306 307 308 309 310 311 312 313 314 315 316 317
		return false;

	devid = get_device_id(dev);

	/* Out of our scope? */
	if (devid > amd_iommu_last_bdf)
		return false;

	if (amd_iommu_rlookup_table[devid] == NULL)
		return false;

	return true;
}

318
static void init_iommu_group(struct device *dev)
319
{
320 321
	struct dma_ops_domain *dma_domain;
	struct iommu_domain *domain;
322 323
	struct iommu_group *group;

324
	group = iommu_group_get_for_dev(dev);
325 326 327 328 329 330 331 332 333 334 335 336
	if (IS_ERR(group))
		return;

	domain = iommu_group_default_domain(group);
	if (!domain)
		goto out;

	dma_domain = to_pdomain(domain)->priv;

	init_unity_mappings_for_device(dev, dma_domain);
out:
	iommu_group_put(group);
337 338 339 340 341 342 343 344 345 346 347 348 349 350
}

static int iommu_init_device(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct iommu_dev_data *dev_data;

	if (dev->archdata.iommu)
		return 0;

	dev_data = find_dev_data(get_device_id(dev));
	if (!dev_data)
		return -ENOMEM;

351 352 353 354 355 356 357
	if (pci_iommuv2_capable(pdev)) {
		struct amd_iommu *iommu;

		iommu              = amd_iommu_rlookup_table[dev_data->devid];
		dev_data->iommu_v2 = iommu->is_iommu_v2;
	}

358 359
	dev->archdata.iommu = dev_data;

A
Alex Williamson 已提交
360 361 362
	iommu_device_link(amd_iommu_rlookup_table[dev_data->devid]->iommu_dev,
			  dev);

363 364 365
	return 0;
}

366 367 368 369 370 371 372 373 374 375 376 377 378 379
static void iommu_ignore_device(struct device *dev)
{
	u16 devid, alias;

	devid = get_device_id(dev);
	alias = amd_iommu_alias_table[devid];

	memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
	memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));

	amd_iommu_rlookup_table[devid] = NULL;
	amd_iommu_rlookup_table[alias] = NULL;
}

380 381
static void iommu_uninit_device(struct device *dev)
{
382 383 384 385 386
	struct iommu_dev_data *dev_data = search_dev_data(get_device_id(dev));

	if (!dev_data)
		return;

A
Alex Williamson 已提交
387 388 389
	iommu_device_unlink(amd_iommu_rlookup_table[dev_data->devid]->iommu_dev,
			    dev);

390 391
	iommu_group_remove_device(dev);

392 393 394
	/* Remove dma-ops */
	dev->archdata.dma_ops = NULL;

395
	/*
396 397
	 * We keep dev_data around for unplugged devices and reuse it when the
	 * device is re-plugged - not doing so would introduce a ton of races.
398
	 */
399
}
J
Joerg Roedel 已提交
400

401 402 403 404 405 406
#ifdef CONFIG_AMD_IOMMU_STATS

/*
 * Initialization code for statistics collection
 */

407
DECLARE_STATS_COUNTER(compl_wait);
408
DECLARE_STATS_COUNTER(cnt_map_single);
409
DECLARE_STATS_COUNTER(cnt_unmap_single);
410
DECLARE_STATS_COUNTER(cnt_map_sg);
411
DECLARE_STATS_COUNTER(cnt_unmap_sg);
412
DECLARE_STATS_COUNTER(cnt_alloc_coherent);
413
DECLARE_STATS_COUNTER(cnt_free_coherent);
414
DECLARE_STATS_COUNTER(cross_page);
415
DECLARE_STATS_COUNTER(domain_flush_single);
416
DECLARE_STATS_COUNTER(domain_flush_all);
417
DECLARE_STATS_COUNTER(alloced_io_mem);
418
DECLARE_STATS_COUNTER(total_map_requests);
419 420 421 422 423
DECLARE_STATS_COUNTER(complete_ppr);
DECLARE_STATS_COUNTER(invalidate_iotlb);
DECLARE_STATS_COUNTER(invalidate_iotlb_all);
DECLARE_STATS_COUNTER(pri_requests);

424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
static struct dentry *stats_dir;
static struct dentry *de_fflush;

static void amd_iommu_stats_add(struct __iommu_counter *cnt)
{
	if (stats_dir == NULL)
		return;

	cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
				       &cnt->value);
}

static void amd_iommu_stats_init(void)
{
	stats_dir = debugfs_create_dir("amd-iommu", NULL);
	if (stats_dir == NULL)
		return;

	de_fflush  = debugfs_create_bool("fullflush", 0444, stats_dir,
443
					 &amd_iommu_unmap_flush);
444 445

	amd_iommu_stats_add(&compl_wait);
446
	amd_iommu_stats_add(&cnt_map_single);
447
	amd_iommu_stats_add(&cnt_unmap_single);
448
	amd_iommu_stats_add(&cnt_map_sg);
449
	amd_iommu_stats_add(&cnt_unmap_sg);
450
	amd_iommu_stats_add(&cnt_alloc_coherent);
451
	amd_iommu_stats_add(&cnt_free_coherent);
452
	amd_iommu_stats_add(&cross_page);
453
	amd_iommu_stats_add(&domain_flush_single);
454
	amd_iommu_stats_add(&domain_flush_all);
455
	amd_iommu_stats_add(&alloced_io_mem);
456
	amd_iommu_stats_add(&total_map_requests);
457 458 459 460
	amd_iommu_stats_add(&complete_ppr);
	amd_iommu_stats_add(&invalidate_iotlb);
	amd_iommu_stats_add(&invalidate_iotlb_all);
	amd_iommu_stats_add(&pri_requests);
461 462 463 464
}

#endif

465 466 467 468 469 470
/****************************************************************************
 *
 * Interrupt handling functions
 *
 ****************************************************************************/

471 472 473 474
static void dump_dte_entry(u16 devid)
{
	int i;

475 476
	for (i = 0; i < 4; ++i)
		pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
477 478 479
			amd_iommu_dev_table[devid].data[i]);
}

480 481 482 483 484 485 486 487 488
static void dump_command(unsigned long phys_addr)
{
	struct iommu_cmd *cmd = phys_to_virt(phys_addr);
	int i;

	for (i = 0; i < 4; ++i)
		pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
}

489
static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
490
{
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
	int type, devid, domid, flags;
	volatile u32 *event = __evt;
	int count = 0;
	u64 address;

retry:
	type    = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
	domid   = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
	flags   = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
	address = (u64)(((u64)event[3]) << 32) | event[2];

	if (type == 0) {
		/* Did we hit the erratum? */
		if (++count == LOOP_TIMEOUT) {
			pr_err("AMD-Vi: No event written to event log\n");
			return;
		}
		udelay(1);
		goto retry;
	}
512

513
	printk(KERN_ERR "AMD-Vi: Event logged [");
514 515 516 517 518

	switch (type) {
	case EVENT_TYPE_ILL_DEV:
		printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
519
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
520
		       address, flags);
521
		dump_dte_entry(devid);
522 523 524 525
		break;
	case EVENT_TYPE_IO_FAULT:
		printk("IO_PAGE_FAULT device=%02x:%02x.%x "
		       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
526
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
527 528 529 530 531
		       domid, address, flags);
		break;
	case EVENT_TYPE_DEV_TAB_ERR:
		printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
532
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
533 534 535 536 537
		       address, flags);
		break;
	case EVENT_TYPE_PAGE_TAB_ERR:
		printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
		       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
538
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
539 540 541 542
		       domid, address, flags);
		break;
	case EVENT_TYPE_ILL_CMD:
		printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
543
		dump_command(address);
544 545 546 547 548 549 550 551
		break;
	case EVENT_TYPE_CMD_HARD_ERR:
		printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
		       "flags=0x%04x]\n", address, flags);
		break;
	case EVENT_TYPE_IOTLB_INV_TO:
		printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
		       "address=0x%016llx]\n",
552
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
553 554 555 556 557
		       address);
		break;
	case EVENT_TYPE_INV_DEV_REQ:
		printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
558
		       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
559 560 561 562 563
		       address, flags);
		break;
	default:
		printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
	}
564 565

	memset(__evt, 0, 4 * sizeof(u32));
566 567 568 569 570 571 572 573 574 575
}

static void iommu_poll_events(struct amd_iommu *iommu)
{
	u32 head, tail;

	head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
	tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

	while (head != tail) {
576
		iommu_print_event(iommu, iommu->evt_buf + head);
577
		head = (head + EVENT_ENTRY_SIZE) % EVT_BUFFER_SIZE;
578 579 580 581 582
	}

	writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
}

583
static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
584 585 586
{
	struct amd_iommu_fault fault;

587 588
	INC_STATS_COUNTER(pri_requests);

589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613
	if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
		pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
		return;
	}

	fault.address   = raw[1];
	fault.pasid     = PPR_PASID(raw[0]);
	fault.device_id = PPR_DEVID(raw[0]);
	fault.tag       = PPR_TAG(raw[0]);
	fault.flags     = PPR_FLAGS(raw[0]);

	atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
}

static void iommu_poll_ppr_log(struct amd_iommu *iommu)
{
	u32 head, tail;

	if (iommu->ppr_log == NULL)
		return;

	head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
	tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

	while (head != tail) {
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629
		volatile u64 *raw;
		u64 entry[2];
		int i;

		raw = (u64 *)(iommu->ppr_log + head);

		/*
		 * Hardware bug: Interrupt may arrive before the entry is
		 * written to memory. If this happens we need to wait for the
		 * entry to arrive.
		 */
		for (i = 0; i < LOOP_TIMEOUT; ++i) {
			if (PPR_REQ_TYPE(raw[0]) != 0)
				break;
			udelay(1);
		}
630

631 632 633
		/* Avoid memcpy function-call overhead */
		entry[0] = raw[0];
		entry[1] = raw[1];
634

635 636 637 638 639 640 641
		/*
		 * To detect the hardware bug we need to clear the entry
		 * back to zero.
		 */
		raw[0] = raw[1] = 0UL;

		/* Update head pointer of hardware ring-buffer */
642 643
		head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
		writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
644 645 646 647 648 649

		/* Handle PPR entry */
		iommu_handle_ppr_entry(iommu, entry);

		/* Refresh ring-buffer information */
		head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
650 651 652 653
		tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
	}
}

654
irqreturn_t amd_iommu_int_thread(int irq, void *data)
655
{
656 657
	struct amd_iommu *iommu = (struct amd_iommu *) data;
	u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
658

659 660 661 662
	while (status & (MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK)) {
		/* Enable EVT and PPR interrupts again */
		writel((MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK),
			iommu->mmio_base + MMIO_STATUS_OFFSET);
663

664 665 666 667
		if (status & MMIO_STATUS_EVT_INT_MASK) {
			pr_devel("AMD-Vi: Processing IOMMU Event Log\n");
			iommu_poll_events(iommu);
		}
668

669 670 671 672
		if (status & MMIO_STATUS_PPR_INT_MASK) {
			pr_devel("AMD-Vi: Processing IOMMU PPR Log\n");
			iommu_poll_ppr_log(iommu);
		}
673

674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
		/*
		 * Hardware bug: ERBT1312
		 * When re-enabling interrupt (by writing 1
		 * to clear the bit), the hardware might also try to set
		 * the interrupt bit in the event status register.
		 * In this scenario, the bit will be set, and disable
		 * subsequent interrupts.
		 *
		 * Workaround: The IOMMU driver should read back the
		 * status register and check if the interrupt bits are cleared.
		 * If not, driver will need to go through the interrupt handler
		 * again and re-clear the bits
		 */
		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
	}
689
	return IRQ_HANDLED;
690 691
}

692 693 694 695 696
irqreturn_t amd_iommu_int_handler(int irq, void *data)
{
	return IRQ_WAKE_THREAD;
}

697 698 699 700 701 702
/****************************************************************************
 *
 * IOMMU command queuing functions
 *
 ****************************************************************************/

703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722
static int wait_on_sem(volatile u64 *sem)
{
	int i = 0;

	while (*sem == 0 && i < LOOP_TIMEOUT) {
		udelay(1);
		i += 1;
	}

	if (i == LOOP_TIMEOUT) {
		pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
		return -EIO;
	}

	return 0;
}

static void copy_cmd_to_buffer(struct amd_iommu *iommu,
			       struct iommu_cmd *cmd,
			       u32 tail)
723 724 725
{
	u8 *target;

726
	target = iommu->cmd_buf + tail;
727
	tail   = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
728 729 730 731 732

	/* Copy command to buffer */
	memcpy(target, cmd, sizeof(*cmd));

	/* Tell the IOMMU about it */
733
	writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
734
}
735

736
static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
737
{
738 739
	WARN_ON(address & 0x7ULL);

740
	memset(cmd, 0, sizeof(*cmd));
741 742 743
	cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
	cmd->data[1] = upper_32_bits(__pa(address));
	cmd->data[2] = 1;
744 745 746
	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
}

747 748 749 750 751 752 753
static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
{
	memset(cmd, 0, sizeof(*cmd));
	cmd->data[0] = devid;
	CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
}

754 755 756 757
static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
				  size_t size, u16 domid, int pde)
{
	u64 pages;
758
	bool s;
759 760

	pages = iommu_num_pages(address, size, PAGE_SIZE);
761
	s     = false;
762 763 764 765 766 767 768

	if (pages > 1) {
		/*
		 * If we have to flush more than one page, flush all
		 * TLB entries for this domain
		 */
		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
769
		s = true;
770 771 772 773 774 775 776 777 778 779 780
	}

	address &= PAGE_MASK;

	memset(cmd, 0, sizeof(*cmd));
	cmd->data[1] |= domid;
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[3]  = upper_32_bits(address);
	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
	if (s) /* size bit - we flush more than one 4kb page */
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
F
Frank Arnold 已提交
781
	if (pde) /* PDE bit - we want to flush everything, not only the PTEs */
782 783 784
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
}

785 786 787 788
static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
				  u64 address, size_t size)
{
	u64 pages;
789
	bool s;
790 791

	pages = iommu_num_pages(address, size, PAGE_SIZE);
792
	s     = false;
793 794 795 796 797 798 799

	if (pages > 1) {
		/*
		 * If we have to flush more than one page, flush all
		 * TLB entries for this domain
		 */
		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
800
		s = true;
801 802 803 804 805 806 807 808 809 810 811 812 813 814 815
	}

	address &= PAGE_MASK;

	memset(cmd, 0, sizeof(*cmd));
	cmd->data[0]  = devid;
	cmd->data[0] |= (qdep & 0xff) << 24;
	cmd->data[1]  = devid;
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[3]  = upper_32_bits(address);
	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
	if (s)
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
}

816 817 818 819 820 821 822
static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
				  u64 address, bool size)
{
	memset(cmd, 0, sizeof(*cmd));

	address &= ~(0xfffULL);

823
	cmd->data[0]  = pasid;
824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841
	cmd->data[1]  = domid;
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[3]  = upper_32_bits(address);
	cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
	cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
	if (size)
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
}

static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
				  int qdep, u64 address, bool size)
{
	memset(cmd, 0, sizeof(*cmd));

	address &= ~(0xfffULL);

	cmd->data[0]  = devid;
842
	cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;
843 844
	cmd->data[0] |= (qdep  & 0xff) << 24;
	cmd->data[1]  = devid;
845
	cmd->data[1] |= (pasid & 0xff) << 16;
846 847 848 849 850 851 852 853
	cmd->data[2]  = lower_32_bits(address);
	cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
	cmd->data[3]  = upper_32_bits(address);
	if (size)
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
}

854 855 856 857 858 859 860
static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
			       int status, int tag, bool gn)
{
	memset(cmd, 0, sizeof(*cmd));

	cmd->data[0]  = devid;
	if (gn) {
861
		cmd->data[1]  = pasid;
862 863 864 865 866 867 868 869
		cmd->data[2]  = CMD_INV_IOMMU_PAGES_GN_MASK;
	}
	cmd->data[3]  = tag & 0x1ff;
	cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;

	CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
}

870 871 872 873
static void build_inv_all(struct iommu_cmd *cmd)
{
	memset(cmd, 0, sizeof(*cmd));
	CMD_SET_TYPE(cmd, CMD_INV_ALL);
874 875
}

876 877 878 879 880 881 882
static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
{
	memset(cmd, 0, sizeof(*cmd));
	cmd->data[0] = devid;
	CMD_SET_TYPE(cmd, CMD_INV_IRT);
}

883 884
/*
 * Writes the command to the IOMMUs command buffer and informs the
885
 * hardware about the new command.
886
 */
887 888 889
static int iommu_queue_command_sync(struct amd_iommu *iommu,
				    struct iommu_cmd *cmd,
				    bool sync)
890
{
891
	u32 left, tail, head, next_tail;
892 893
	unsigned long flags;

894
again:
895 896
	spin_lock_irqsave(&iommu->lock, flags);

897 898
	head      = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
	tail      = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
899 900
	next_tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
	left      = (head - next_tail) % CMD_BUFFER_SIZE;
901

902 903 904 905
	if (left <= 2) {
		struct iommu_cmd sync_cmd;
		volatile u64 sem = 0;
		int ret;
906

907 908
		build_completion_wait(&sync_cmd, (u64)&sem);
		copy_cmd_to_buffer(iommu, &sync_cmd, tail);
909

910 911 912 913 914 915
		spin_unlock_irqrestore(&iommu->lock, flags);

		if ((ret = wait_on_sem(&sem)) != 0)
			return ret;

		goto again;
916 917
	}

918 919 920
	copy_cmd_to_buffer(iommu, cmd, tail);

	/* We need to sync now to make sure all commands are processed */
921
	iommu->need_sync = sync;
922

923
	spin_unlock_irqrestore(&iommu->lock, flags);
924

925
	return 0;
926 927
}

928 929 930 931 932
static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
{
	return iommu_queue_command_sync(iommu, cmd, true);
}

933 934 935 936
/*
 * This function queues a completion wait command into the command
 * buffer of an IOMMU
 */
937
static int iommu_completion_wait(struct amd_iommu *iommu)
938 939
{
	struct iommu_cmd cmd;
940
	volatile u64 sem = 0;
941
	int ret;
942

943
	if (!iommu->need_sync)
944
		return 0;
945

946
	build_completion_wait(&cmd, (u64)&sem);
947

948
	ret = iommu_queue_command_sync(iommu, &cmd, false);
949
	if (ret)
950
		return ret;
951

952
	return wait_on_sem(&sem);
953 954
}

955
static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
956
{
957
	struct iommu_cmd cmd;
958

959
	build_inv_dte(&cmd, devid);
960

961 962
	return iommu_queue_command(iommu, &cmd);
}
963

964 965 966
static void iommu_flush_dte_all(struct amd_iommu *iommu)
{
	u32 devid;
967

968 969
	for (devid = 0; devid <= 0xffff; ++devid)
		iommu_flush_dte(iommu, devid);
970

971 972
	iommu_completion_wait(iommu);
}
973

974 975 976 977 978 979 980
/*
 * This function uses heavy locking and may disable irqs for some time. But
 * this is no issue because it is only called during resume.
 */
static void iommu_flush_tlb_all(struct amd_iommu *iommu)
{
	u32 dom_id;
981

982 983 984 985 986 987
	for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
		struct iommu_cmd cmd;
		build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
				      dom_id, 1);
		iommu_queue_command(iommu, &cmd);
	}
988

989
	iommu_completion_wait(iommu);
990 991
}

992
static void iommu_flush_all(struct amd_iommu *iommu)
993
{
994
	struct iommu_cmd cmd;
995

996
	build_inv_all(&cmd);
997

998 999 1000 1001
	iommu_queue_command(iommu, &cmd);
	iommu_completion_wait(iommu);
}

1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
{
	struct iommu_cmd cmd;

	build_inv_irt(&cmd, devid);

	iommu_queue_command(iommu, &cmd);
}

static void iommu_flush_irt_all(struct amd_iommu *iommu)
{
	u32 devid;

	for (devid = 0; devid <= MAX_DEV_TABLE_ENTRIES; devid++)
		iommu_flush_irt(iommu, devid);

	iommu_completion_wait(iommu);
}

1021 1022
void iommu_flush_all_caches(struct amd_iommu *iommu)
{
1023 1024 1025 1026
	if (iommu_feature(iommu, FEATURE_IA)) {
		iommu_flush_all(iommu);
	} else {
		iommu_flush_dte_all(iommu);
1027
		iommu_flush_irt_all(iommu);
1028
		iommu_flush_tlb_all(iommu);
1029 1030 1031
	}
}

1032
/*
1033
 * Command send function for flushing on-device TLB
1034
 */
1035 1036
static int device_flush_iotlb(struct iommu_dev_data *dev_data,
			      u64 address, size_t size)
1037 1038
{
	struct amd_iommu *iommu;
1039
	struct iommu_cmd cmd;
1040
	int qdep;
1041

1042 1043
	qdep     = dev_data->ats.qdep;
	iommu    = amd_iommu_rlookup_table[dev_data->devid];
1044

1045
	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
1046 1047

	return iommu_queue_command(iommu, &cmd);
1048 1049
}

1050 1051 1052
/*
 * Command send function for invalidating a device table entry
 */
1053
static int device_flush_dte(struct iommu_dev_data *dev_data)
1054
{
1055
	struct amd_iommu *iommu;
1056
	u16 alias;
1057
	int ret;
1058

1059
	iommu = amd_iommu_rlookup_table[dev_data->devid];
1060
	alias = amd_iommu_alias_table[dev_data->devid];
1061

1062
	ret = iommu_flush_dte(iommu, dev_data->devid);
1063 1064
	if (!ret && alias != dev_data->devid)
		ret = iommu_flush_dte(iommu, alias);
1065 1066 1067
	if (ret)
		return ret;

1068
	if (dev_data->ats.enabled)
1069
		ret = device_flush_iotlb(dev_data, 0, ~0UL);
1070 1071

	return ret;
1072 1073
}

1074 1075 1076 1077 1078
/*
 * TLB invalidation function which is called from the mapping functions.
 * It invalidates a single PTE if the range to flush is within a single
 * page. Otherwise it flushes the whole TLB of the IOMMU.
 */
1079 1080
static void __domain_flush_pages(struct protection_domain *domain,
				 u64 address, size_t size, int pde)
1081
{
1082
	struct iommu_dev_data *dev_data;
1083 1084
	struct iommu_cmd cmd;
	int ret = 0, i;
1085

1086
	build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
1087

1088 1089 1090 1091 1092 1093 1094 1095
	for (i = 0; i < amd_iommus_present; ++i) {
		if (!domain->dev_iommu[i])
			continue;

		/*
		 * Devices of this domain are behind this IOMMU
		 * We need a TLB flush
		 */
1096
		ret |= iommu_queue_command(amd_iommus[i], &cmd);
1097 1098
	}

1099 1100
	list_for_each_entry(dev_data, &domain->dev_list, list) {

1101
		if (!dev_data->ats.enabled)
1102 1103
			continue;

1104
		ret |= device_flush_iotlb(dev_data, address, size);
1105 1106
	}

1107
	WARN_ON(ret);
1108 1109
}

1110 1111
static void domain_flush_pages(struct protection_domain *domain,
			       u64 address, size_t size)
1112
{
1113
	__domain_flush_pages(domain, address, size, 0);
1114
}
1115

1116
/* Flush the whole IO/TLB for a given protection domain */
1117
static void domain_flush_tlb(struct protection_domain *domain)
1118
{
1119
	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
1120 1121
}

1122
/* Flush the whole IO/TLB for a given protection domain - including PDE */
1123
static void domain_flush_tlb_pde(struct protection_domain *domain)
1124
{
1125
	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
1126 1127
}

1128
static void domain_flush_complete(struct protection_domain *domain)
1129
{
1130
	int i;
1131

1132 1133 1134
	for (i = 0; i < amd_iommus_present; ++i) {
		if (!domain->dev_iommu[i])
			continue;
1135

1136 1137 1138 1139 1140
		/*
		 * Devices of this domain are behind this IOMMU
		 * We need to wait for completion of all commands.
		 */
		iommu_completion_wait(amd_iommus[i]);
1141
	}
1142 1143
}

1144

1145
/*
1146
 * This function flushes the DTEs for all devices in domain
1147
 */
1148
static void domain_flush_devices(struct protection_domain *domain)
1149
{
1150
	struct iommu_dev_data *dev_data;
1151

1152
	list_for_each_entry(dev_data, &domain->dev_list, list)
1153
		device_flush_dte(dev_data);
1154 1155
}

1156 1157 1158 1159 1160 1161 1162
/****************************************************************************
 *
 * The functions below are used the create the page table mappings for
 * unity mapped regions.
 *
 ****************************************************************************/

1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
/*
 * This function is used to add another level to an IO page table. Adding
 * another level increases the size of the address space by 9 bits to a size up
 * to 64 bits.
 */
static bool increase_address_space(struct protection_domain *domain,
				   gfp_t gfp)
{
	u64 *pte;

	if (domain->mode == PAGE_MODE_6_LEVEL)
		/* address space already 64 bit large */
		return false;

	pte = (void *)get_zeroed_page(gfp);
	if (!pte)
		return false;

	*pte             = PM_LEVEL_PDE(domain->mode,
					virt_to_phys(domain->pt_root));
	domain->pt_root  = pte;
	domain->mode    += 1;
	domain->updated  = true;

	return true;
}

static u64 *alloc_pte(struct protection_domain *domain,
		      unsigned long address,
1192
		      unsigned long page_size,
1193 1194 1195
		      u64 **pte_page,
		      gfp_t gfp)
{
1196
	int level, end_lvl;
1197
	u64 *pte, *page;
1198 1199

	BUG_ON(!is_power_of_2(page_size));
1200 1201 1202 1203

	while (address > PM_LEVEL_SIZE(domain->mode))
		increase_address_space(domain, gfp);

1204 1205 1206 1207
	level   = domain->mode - 1;
	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
	address = PAGE_SIZE_ALIGN(address, page_size);
	end_lvl = PAGE_SIZE_LEVEL(page_size);
1208 1209 1210 1211 1212 1213 1214 1215 1216

	while (level > end_lvl) {
		if (!IOMMU_PTE_PRESENT(*pte)) {
			page = (u64 *)get_zeroed_page(gfp);
			if (!page)
				return NULL;
			*pte = PM_LEVEL_PDE(level, virt_to_phys(page));
		}

1217 1218 1219 1220
		/* No level skipping support yet */
		if (PM_PTE_LEVEL(*pte) != level)
			return NULL;

1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
		level -= 1;

		pte = IOMMU_PTE_PAGE(*pte);

		if (pte_page && level == end_lvl)
			*pte_page = pte;

		pte = &pte[PM_LEVEL_INDEX(level, address)];
	}

	return pte;
}

/*
 * This function checks if there is a PTE for a given dma address. If
 * there is one, it returns the pointer to it.
 */
1238 1239 1240
static u64 *fetch_pte(struct protection_domain *domain,
		      unsigned long address,
		      unsigned long *page_size)
1241 1242 1243 1244
{
	int level;
	u64 *pte;

1245 1246 1247
	if (address > PM_LEVEL_SIZE(domain->mode))
		return NULL;

1248 1249 1250
	level	   =  domain->mode - 1;
	pte	   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
	*page_size =  PTE_LEVEL_PAGE_SIZE(level);
1251

1252 1253 1254
	while (level > 0) {

		/* Not Present */
1255 1256 1257
		if (!IOMMU_PTE_PRESENT(*pte))
			return NULL;

1258
		/* Large PTE */
1259 1260 1261
		if (PM_PTE_LEVEL(*pte) == 7 ||
		    PM_PTE_LEVEL(*pte) == 0)
			break;
1262 1263 1264 1265 1266

		/* No level skipping support yet */
		if (PM_PTE_LEVEL(*pte) != level)
			return NULL;

1267 1268
		level -= 1;

1269
		/* Walk to the next level */
1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
		pte	   = IOMMU_PTE_PAGE(*pte);
		pte	   = &pte[PM_LEVEL_INDEX(level, address)];
		*page_size = PTE_LEVEL_PAGE_SIZE(level);
	}

	if (PM_PTE_LEVEL(*pte) == 0x07) {
		unsigned long pte_mask;

		/*
		 * If we have a series of large PTEs, make
		 * sure to return a pointer to the first one.
		 */
		*page_size = pte_mask = PTE_PAGE_SIZE(*pte);
		pte_mask   = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
		pte        = (u64 *)(((unsigned long)pte) & pte_mask);
1285 1286 1287 1288 1289
	}

	return pte;
}

1290 1291 1292 1293 1294 1295 1296
/*
 * Generic mapping functions. It maps a physical address into a DMA
 * address space. It allocates the page table pages if necessary.
 * In the future it can be extended to a generic mapping function
 * supporting all features of AMD IOMMU page tables like level skipping
 * and full 64 bit address spaces.
 */
1297 1298 1299
static int iommu_map_page(struct protection_domain *dom,
			  unsigned long bus_addr,
			  unsigned long phys_addr,
1300
			  int prot,
1301
			  unsigned long page_size)
1302
{
1303
	u64 __pte, *pte;
1304
	int i, count;
1305

1306 1307 1308
	BUG_ON(!IS_ALIGNED(bus_addr, page_size));
	BUG_ON(!IS_ALIGNED(phys_addr, page_size));

1309
	if (!(prot & IOMMU_PROT_MASK))
1310 1311
		return -EINVAL;

1312 1313
	count = PAGE_SIZE_PTE_COUNT(page_size);
	pte   = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
1314

1315 1316 1317
	if (!pte)
		return -ENOMEM;

1318 1319 1320
	for (i = 0; i < count; ++i)
		if (IOMMU_PTE_PRESENT(pte[i]))
			return -EBUSY;
1321

1322
	if (count > 1) {
1323 1324 1325 1326
		__pte = PAGE_SIZE_PTE(phys_addr, page_size);
		__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
	} else
		__pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
1327 1328 1329 1330 1331 1332

	if (prot & IOMMU_PROT_IR)
		__pte |= IOMMU_PTE_IR;
	if (prot & IOMMU_PROT_IW)
		__pte |= IOMMU_PTE_IW;

1333 1334
	for (i = 0; i < count; ++i)
		pte[i] = __pte;
1335

1336 1337
	update_domain(dom);

1338 1339 1340
	return 0;
}

1341 1342 1343
static unsigned long iommu_unmap_page(struct protection_domain *dom,
				      unsigned long bus_addr,
				      unsigned long page_size)
1344
{
1345 1346
	unsigned long long unmapped;
	unsigned long unmap_size;
1347 1348 1349 1350 1351
	u64 *pte;

	BUG_ON(!is_power_of_2(page_size));

	unmapped = 0;
1352

1353 1354
	while (unmapped < page_size) {

1355 1356 1357 1358 1359 1360
		pte = fetch_pte(dom, bus_addr, &unmap_size);

		if (pte) {
			int i, count;

			count = PAGE_SIZE_PTE_COUNT(unmap_size);
1361 1362 1363 1364 1365 1366 1367 1368
			for (i = 0; i < count; i++)
				pte[i] = 0ULL;
		}

		bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;
		unmapped += unmap_size;
	}

1369
	BUG_ON(unmapped && !is_power_of_2(unmapped));
1370

1371
	return unmapped;
1372 1373
}

1374 1375 1376 1377 1378 1379 1380 1381 1382
/****************************************************************************
 *
 * The next functions belong to the address allocator for the dma_ops
 * interface functions. They work like the allocators in the other IOMMU
 * drivers. Its basically a bitmap which marks the allocated pages in
 * the aperture. Maybe it could be enhanced in the future to a more
 * efficient allocator.
 *
 ****************************************************************************/
1383

1384
/*
1385
 * The address allocator core functions.
1386 1387 1388
 *
 * called with domain->lock held
 */
1389

1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
/*
 * Used to reserve address ranges in the aperture (e.g. for exclusion
 * ranges.
 */
static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
				      unsigned long start_page,
				      unsigned int pages)
{
	unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;

	if (start_page + pages > last_page)
		pages = last_page - start_page;

	for (i = start_page; i < start_page + pages; ++i) {
		int index = i / APERTURE_RANGE_PAGES;
		int page  = i % APERTURE_RANGE_PAGES;
		__set_bit(page, dom->aperture[index]->bitmap);
	}
}

1410 1411 1412 1413 1414
/*
 * This function is used to add a new aperture range to an existing
 * aperture in case of dma_ops domain allocation or address allocation
 * failure.
 */
1415
static int alloc_new_range(struct dma_ops_domain *dma_dom,
1416 1417 1418
			   bool populate, gfp_t gfp)
{
	int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1419
	struct amd_iommu *iommu;
1420
	unsigned long i, old_size, pte_pgsize;
1421

1422 1423 1424 1425
#ifdef CONFIG_IOMMU_STRESS
	populate = false;
#endif

1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444
	if (index >= APERTURE_MAX_RANGES)
		return -ENOMEM;

	dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
	if (!dma_dom->aperture[index])
		return -ENOMEM;

	dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
	if (!dma_dom->aperture[index]->bitmap)
		goto out_free;

	dma_dom->aperture[index]->offset = dma_dom->aperture_size;

	if (populate) {
		unsigned long address = dma_dom->aperture_size;
		int i, num_ptes = APERTURE_RANGE_PAGES / 512;
		u64 *pte, *pte_page;

		for (i = 0; i < num_ptes; ++i) {
1445
			pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455
					&pte_page, gfp);
			if (!pte)
				goto out_free;

			dma_dom->aperture[index]->pte_pages[i] = pte_page;

			address += APERTURE_RANGE_SIZE / 64;
		}
	}

1456
	old_size                = dma_dom->aperture_size;
1457 1458
	dma_dom->aperture_size += APERTURE_RANGE_SIZE;

1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
	/* Reserve address range used for MSI messages */
	if (old_size < MSI_ADDR_BASE_LO &&
	    dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
		unsigned long spage;
		int pages;

		pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
		spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;

		dma_ops_reserve_addresses(dma_dom, spage, pages);
	}

1471
	/* Initialize the exclusion range if necessary */
1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482
	for_each_iommu(iommu) {
		if (iommu->exclusion_start &&
		    iommu->exclusion_start >= dma_dom->aperture[index]->offset
		    && iommu->exclusion_start < dma_dom->aperture_size) {
			unsigned long startpage;
			int pages = iommu_num_pages(iommu->exclusion_start,
						    iommu->exclusion_length,
						    PAGE_SIZE);
			startpage = iommu->exclusion_start >> PAGE_SHIFT;
			dma_ops_reserve_addresses(dma_dom, startpage, pages);
		}
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492
	}

	/*
	 * Check for areas already mapped as present in the new aperture
	 * range and mark those pages as reserved in the allocator. Such
	 * mappings may already exist as a result of requested unity
	 * mappings for devices.
	 */
	for (i = dma_dom->aperture[index]->offset;
	     i < dma_dom->aperture_size;
1493
	     i += pte_pgsize) {
1494
		u64 *pte = fetch_pte(&dma_dom->domain, i, &pte_pgsize);
1495 1496 1497
		if (!pte || !IOMMU_PTE_PRESENT(*pte))
			continue;

1498 1499
		dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT,
					  pte_pgsize >> 12);
1500 1501
	}

1502 1503
	update_domain(&dma_dom->domain);

1504 1505 1506
	return 0;

out_free:
1507 1508
	update_domain(&dma_dom->domain);

1509 1510 1511 1512 1513 1514 1515 1516
	free_page((unsigned long)dma_dom->aperture[index]->bitmap);

	kfree(dma_dom->aperture[index]);
	dma_dom->aperture[index] = NULL;

	return -ENOMEM;
}

1517 1518 1519 1520 1521 1522 1523
static unsigned long dma_ops_area_alloc(struct device *dev,
					struct dma_ops_domain *dom,
					unsigned int pages,
					unsigned long align_mask,
					u64 dma_mask,
					unsigned long start)
{
1524
	unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1525 1526
	int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
	int i = start >> APERTURE_RANGE_SHIFT;
1527
	unsigned long boundary_size, mask;
1528 1529 1530
	unsigned long address = -1;
	unsigned long limit;

1531 1532
	next_bit >>= PAGE_SHIFT;

1533 1534 1535 1536
	mask = dma_get_seg_boundary(dev);

	boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
				   1UL << (BITS_PER_LONG - PAGE_SHIFT);
1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552

	for (;i < max_index; ++i) {
		unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;

		if (dom->aperture[i]->offset >= dma_mask)
			break;

		limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
					       dma_mask >> PAGE_SHIFT);

		address = iommu_area_alloc(dom->aperture[i]->bitmap,
					   limit, next_bit, pages, 0,
					    boundary_size, align_mask);
		if (address != -1) {
			address = dom->aperture[i]->offset +
				  (address << PAGE_SHIFT);
1553
			dom->next_address = address + (pages << PAGE_SHIFT);
1554 1555 1556 1557 1558 1559 1560 1561 1562
			break;
		}

		next_bit = 0;
	}

	return address;
}

1563 1564
static unsigned long dma_ops_alloc_addresses(struct device *dev,
					     struct dma_ops_domain *dom,
1565
					     unsigned int pages,
1566 1567
					     unsigned long align_mask,
					     u64 dma_mask)
1568 1569 1570
{
	unsigned long address;

1571 1572 1573 1574
#ifdef CONFIG_IOMMU_STRESS
	dom->next_address = 0;
	dom->need_flush = true;
#endif
1575

1576
	address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1577
				     dma_mask, dom->next_address);
1578

1579
	if (address == -1) {
1580
		dom->next_address = 0;
1581 1582
		address = dma_ops_area_alloc(dev, dom, pages, align_mask,
					     dma_mask, 0);
1583 1584
		dom->need_flush = true;
	}
1585

1586
	if (unlikely(address == -1))
1587
		address = DMA_ERROR_CODE;
1588 1589 1590 1591 1592 1593

	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);

	return address;
}

1594 1595 1596 1597 1598
/*
 * The address free function.
 *
 * called with domain->lock held
 */
1599 1600 1601 1602
static void dma_ops_free_addresses(struct dma_ops_domain *dom,
				   unsigned long address,
				   unsigned int pages)
{
1603 1604
	unsigned i = address >> APERTURE_RANGE_SHIFT;
	struct aperture_range *range = dom->aperture[i];
1605

1606 1607
	BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);

1608 1609 1610 1611
#ifdef CONFIG_IOMMU_STRESS
	if (i < 4)
		return;
#endif
1612

1613
	if (address >= dom->next_address)
1614
		dom->need_flush = true;
1615 1616

	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1617

A
Akinobu Mita 已提交
1618
	bitmap_clear(range->bitmap, address, pages);
1619

1620 1621
}

1622 1623 1624 1625 1626 1627 1628 1629 1630 1631
/****************************************************************************
 *
 * The next functions belong to the domain allocation. A domain is
 * allocated for every IOMMU as the default domain. If device isolation
 * is enabled, every device get its own domain. The most important thing
 * about domains is the page table mapping the DMA address space they
 * contain.
 *
 ****************************************************************************/

1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656
/*
 * This function adds a protection domain to the global protection domain list
 */
static void add_domain_to_list(struct protection_domain *domain)
{
	unsigned long flags;

	spin_lock_irqsave(&amd_iommu_pd_lock, flags);
	list_add(&domain->list, &amd_iommu_pd_list);
	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
}

/*
 * This function removes a protection domain to the global
 * protection domain list
 */
static void del_domain_from_list(struct protection_domain *domain)
{
	unsigned long flags;

	spin_lock_irqsave(&amd_iommu_pd_lock, flags);
	list_del(&domain->list);
	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
}

1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673
static u16 domain_id_alloc(void)
{
	unsigned long flags;
	int id;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
	BUG_ON(id == 0);
	if (id > 0 && id < MAX_DOMAIN_ID)
		__set_bit(id, amd_iommu_pd_alloc_bitmap);
	else
		id = 0;
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

	return id;
}

1674 1675 1676 1677 1678 1679 1680 1681 1682 1683
static void domain_id_free(int id)
{
	unsigned long flags;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	if (id > 0 && id < MAX_DOMAIN_ID)
		__clear_bit(id, amd_iommu_pd_alloc_bitmap);
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}

1684 1685 1686 1687 1688 1689 1690 1691 1692 1693
#define DEFINE_FREE_PT_FN(LVL, FN)				\
static void free_pt_##LVL (unsigned long __pt)			\
{								\
	unsigned long p;					\
	u64 *pt;						\
	int i;							\
								\
	pt = (u64 *)__pt;					\
								\
	for (i = 0; i < 512; ++i) {				\
1694
		/* PTE present? */				\
1695 1696 1697
		if (!IOMMU_PTE_PRESENT(pt[i]))			\
			continue;				\
								\
1698 1699 1700 1701 1702
		/* Large PTE? */				\
		if (PM_PTE_LEVEL(pt[i]) == 0 ||			\
		    PM_PTE_LEVEL(pt[i]) == 7)			\
			continue;				\
								\
1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
		p = (unsigned long)IOMMU_PTE_PAGE(pt[i]);	\
		FN(p);						\
	}							\
	free_page((unsigned long)pt);				\
}

DEFINE_FREE_PT_FN(l2, free_page)
DEFINE_FREE_PT_FN(l3, free_pt_l2)
DEFINE_FREE_PT_FN(l4, free_pt_l3)
DEFINE_FREE_PT_FN(l5, free_pt_l4)
DEFINE_FREE_PT_FN(l6, free_pt_l5)

1715
static void free_pagetable(struct protection_domain *domain)
1716
{
1717
	unsigned long root = (unsigned long)domain->pt_root;
1718

1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741
	switch (domain->mode) {
	case PAGE_MODE_NONE:
		break;
	case PAGE_MODE_1_LEVEL:
		free_page(root);
		break;
	case PAGE_MODE_2_LEVEL:
		free_pt_l2(root);
		break;
	case PAGE_MODE_3_LEVEL:
		free_pt_l3(root);
		break;
	case PAGE_MODE_4_LEVEL:
		free_pt_l4(root);
		break;
	case PAGE_MODE_5_LEVEL:
		free_pt_l5(root);
		break;
	case PAGE_MODE_6_LEVEL:
		free_pt_l6(root);
		break;
	default:
		BUG();
1742 1743 1744
	}
}

1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
static void free_gcr3_tbl_level1(u64 *tbl)
{
	u64 *ptr;
	int i;

	for (i = 0; i < 512; ++i) {
		if (!(tbl[i] & GCR3_VALID))
			continue;

		ptr = __va(tbl[i] & PAGE_MASK);

		free_page((unsigned long)ptr);
	}
}

static void free_gcr3_tbl_level2(u64 *tbl)
{
	u64 *ptr;
	int i;

	for (i = 0; i < 512; ++i) {
		if (!(tbl[i] & GCR3_VALID))
			continue;

		ptr = __va(tbl[i] & PAGE_MASK);

		free_gcr3_tbl_level1(ptr);
	}
}

1775 1776
static void free_gcr3_table(struct protection_domain *domain)
{
1777 1778 1779 1780
	if (domain->glx == 2)
		free_gcr3_tbl_level2(domain->gcr3_tbl);
	else if (domain->glx == 1)
		free_gcr3_tbl_level1(domain->gcr3_tbl);
1781 1782
	else
		BUG_ON(domain->glx != 0);
1783

1784 1785 1786
	free_page((unsigned long)domain->gcr3_tbl);
}

1787 1788 1789 1790
/*
 * Free a domain, only used if something went wrong in the
 * allocation path and we need to free an already allocated page table
 */
1791 1792
static void dma_ops_domain_free(struct dma_ops_domain *dom)
{
1793 1794
	int i;

1795 1796 1797
	if (!dom)
		return;

1798 1799
	del_domain_from_list(&dom->domain);

1800
	free_pagetable(&dom->domain);
1801

1802 1803 1804 1805 1806 1807
	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
		if (!dom->aperture[i])
			continue;
		free_page((unsigned long)dom->aperture[i]->bitmap);
		kfree(dom->aperture[i]);
	}
1808 1809 1810 1811

	kfree(dom);
}

1812 1813
/*
 * Allocates a new protection domain usable for the dma_ops functions.
1814
 * It also initializes the page table and the address allocator data
1815 1816
 * structures required for the dma_ops interface
 */
1817
static struct dma_ops_domain *dma_ops_domain_alloc(void)
1818 1819 1820 1821 1822 1823 1824
{
	struct dma_ops_domain *dma_dom;

	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
	if (!dma_dom)
		return NULL;

1825
	if (protection_domain_init(&dma_dom->domain))
1826
		goto free_dma_dom;
1827

1828
	dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
1829
	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
1830
	dma_dom->domain.flags = PD_DMA_OPS_MASK;
1831 1832 1833 1834
	dma_dom->domain.priv = dma_dom;
	if (!dma_dom->domain.pt_root)
		goto free_dma_dom;

1835 1836
	dma_dom->need_flush = false;

1837 1838
	add_domain_to_list(&dma_dom->domain);

1839
	if (alloc_new_range(dma_dom, true, GFP_KERNEL))
1840 1841
		goto free_dma_dom;

1842
	/*
1843 1844
	 * mark the first page as allocated so we never return 0 as
	 * a valid dma-address. So we can use 0 as error value
1845
	 */
1846
	dma_dom->aperture[0]->bitmap[0] = 1;
1847
	dma_dom->next_address = 0;
1848 1849 1850 1851 1852 1853 1854 1855 1856 1857


	return dma_dom;

free_dma_dom:
	dma_ops_domain_free(dma_dom);

	return NULL;
}

1858 1859 1860 1861 1862 1863 1864 1865 1866
/*
 * little helper function to check whether a given protection domain is a
 * dma_ops domain
 */
static bool dma_ops_domain(struct protection_domain *domain)
{
	return domain->flags & PD_DMA_OPS_MASK;
}

1867
static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
1868
{
1869
	u64 pte_root = 0;
1870
	u64 flags = 0;
1871

1872 1873 1874
	if (domain->mode != PAGE_MODE_NONE)
		pte_root = virt_to_phys(domain->pt_root);

1875 1876 1877
	pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
		    << DEV_ENTRY_MODE_SHIFT;
	pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
1878

1879 1880
	flags = amd_iommu_dev_table[devid].data[1];

1881 1882 1883
	if (ats)
		flags |= DTE_FLAG_IOTLB;

1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909
	if (domain->flags & PD_IOMMUV2_MASK) {
		u64 gcr3 = __pa(domain->gcr3_tbl);
		u64 glx  = domain->glx;
		u64 tmp;

		pte_root |= DTE_FLAG_GV;
		pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;

		/* First mask out possible old values for GCR3 table */
		tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
		flags    &= ~tmp;

		tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
		flags    &= ~tmp;

		/* Encode GCR3 table into DTE */
		tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
		pte_root |= tmp;

		tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
		flags    |= tmp;

		tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
		flags    |= tmp;
	}

1910 1911 1912 1913 1914
	flags &= ~(0xffffUL);
	flags |= domain->id;

	amd_iommu_dev_table[devid].data[1]  = flags;
	amd_iommu_dev_table[devid].data[0]  = pte_root;
1915 1916 1917 1918 1919
}

static void clear_dte_entry(u16 devid)
{
	/* remove entry from the device table seen by the hardware */
1920 1921
	amd_iommu_dev_table[devid].data[0]  = IOMMU_PTE_P | IOMMU_PTE_TV;
	amd_iommu_dev_table[devid].data[1] &= DTE_FLAG_MASK;
1922 1923

	amd_iommu_apply_erratum_63(devid);
1924 1925
}

1926 1927
static void do_attach(struct iommu_dev_data *dev_data,
		      struct protection_domain *domain)
1928 1929
{
	struct amd_iommu *iommu;
1930
	u16 alias;
1931
	bool ats;
1932

1933
	iommu = amd_iommu_rlookup_table[dev_data->devid];
1934
	alias = amd_iommu_alias_table[dev_data->devid];
1935
	ats   = dev_data->ats.enabled;
1936 1937 1938 1939 1940 1941 1942 1943 1944

	/* Update data structures */
	dev_data->domain = domain;
	list_add(&dev_data->list, &domain->dev_list);

	/* Do reference counting */
	domain->dev_iommu[iommu->index] += 1;
	domain->dev_cnt                 += 1;

1945 1946 1947 1948 1949
	/* Update device table */
	set_dte_entry(dev_data->devid, domain, ats);
	if (alias != dev_data->devid)
		set_dte_entry(dev_data->devid, domain, ats);

1950
	device_flush_dte(dev_data);
1951 1952
}

1953
static void do_detach(struct iommu_dev_data *dev_data)
1954 1955
{
	struct amd_iommu *iommu;
1956
	u16 alias;
1957

1958 1959 1960 1961 1962 1963 1964 1965 1966
	/*
	 * First check if the device is still attached. It might already
	 * be detached from its domain because the generic
	 * iommu_detach_group code detached it and we try again here in
	 * our alias handling.
	 */
	if (!dev_data->domain)
		return;

1967
	iommu = amd_iommu_rlookup_table[dev_data->devid];
1968
	alias = amd_iommu_alias_table[dev_data->devid];
1969 1970

	/* decrease reference counters */
1971 1972 1973 1974 1975 1976
	dev_data->domain->dev_iommu[iommu->index] -= 1;
	dev_data->domain->dev_cnt                 -= 1;

	/* Update data structures */
	dev_data->domain = NULL;
	list_del(&dev_data->list);
1977
	clear_dte_entry(dev_data->devid);
1978 1979
	if (alias != dev_data->devid)
		clear_dte_entry(alias);
1980

1981
	/* Flush the DTE entry */
1982
	device_flush_dte(dev_data);
1983 1984 1985 1986 1987 1988
}

/*
 * If a device is not yet associated with a domain, this function does
 * assigns it visible for the hardware
 */
1989
static int __attach_device(struct iommu_dev_data *dev_data,
1990
			   struct protection_domain *domain)
1991
{
1992
	int ret;
1993

1994 1995 1996 1997 1998 1999
	/*
	 * Must be called with IRQs disabled. Warn here to detect early
	 * when its not.
	 */
	WARN_ON(!irqs_disabled());

2000 2001 2002
	/* lock domain */
	spin_lock(&domain->lock);

2003
	ret = -EBUSY;
2004
	if (dev_data->domain != NULL)
2005
		goto out_unlock;
2006

2007
	/* Attach alias group root */
2008
	do_attach(dev_data, domain);
2009

2010 2011 2012 2013
	ret = 0;

out_unlock:

2014 2015
	/* ready */
	spin_unlock(&domain->lock);
2016

2017
	return ret;
2018
}
2019

2020 2021 2022 2023 2024 2025 2026 2027

static void pdev_iommuv2_disable(struct pci_dev *pdev)
{
	pci_disable_ats(pdev);
	pci_disable_pri(pdev);
	pci_disable_pasid(pdev);
}

2028 2029 2030 2031 2032 2033
/* FIXME: Change generic reset-function to do the same */
static int pri_reset_while_enabled(struct pci_dev *pdev)
{
	u16 control;
	int pos;

2034
	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2035 2036 2037
	if (!pos)
		return -EINVAL;

2038 2039 2040
	pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
	control |= PCI_PRI_CTRL_RESET;
	pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
2041 2042 2043 2044

	return 0;
}

2045 2046
static int pdev_iommuv2_enable(struct pci_dev *pdev)
{
2047 2048 2049 2050 2051 2052 2053 2054
	bool reset_enable;
	int reqs, ret;

	/* FIXME: Hardcode number of outstanding requests for now */
	reqs = 32;
	if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
		reqs = 1;
	reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);
2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065

	/* Only allow access to user-accessible pages */
	ret = pci_enable_pasid(pdev, 0);
	if (ret)
		goto out_err;

	/* First reset the PRI state of the device */
	ret = pci_reset_pri(pdev);
	if (ret)
		goto out_err;

2066 2067
	/* Enable PRI */
	ret = pci_enable_pri(pdev, reqs);
2068 2069 2070
	if (ret)
		goto out_err;

2071 2072 2073 2074 2075 2076
	if (reset_enable) {
		ret = pri_reset_while_enabled(pdev);
		if (ret)
			goto out_err;
	}

2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089
	ret = pci_enable_ats(pdev, PAGE_SHIFT);
	if (ret)
		goto out_err;

	return 0;

out_err:
	pci_disable_pri(pdev);
	pci_disable_pasid(pdev);

	return ret;
}

2090
/* FIXME: Move this to PCI code */
2091
#define PCI_PRI_TLP_OFF		(1 << 15)
2092

J
Joerg Roedel 已提交
2093
static bool pci_pri_tlp_required(struct pci_dev *pdev)
2094
{
2095
	u16 status;
2096 2097
	int pos;

2098
	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2099 2100 2101
	if (!pos)
		return false;

2102
	pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
2103

2104
	return (status & PCI_PRI_TLP_OFF) ? true : false;
2105 2106
}

2107
/*
F
Frank Arnold 已提交
2108
 * If a device is not yet associated with a domain, this function
2109 2110
 * assigns it visible for the hardware
 */
2111 2112
static int attach_device(struct device *dev,
			 struct protection_domain *domain)
2113
{
2114
	struct pci_dev *pdev = to_pci_dev(dev);
2115
	struct iommu_dev_data *dev_data;
2116
	unsigned long flags;
2117
	int ret;
2118

2119 2120
	dev_data = get_dev_data(dev);

2121
	if (domain->flags & PD_IOMMUV2_MASK) {
2122
		if (!dev_data->passthrough)
2123 2124
			return -EINVAL;

2125 2126 2127
		if (dev_data->iommu_v2) {
			if (pdev_iommuv2_enable(pdev) != 0)
				return -EINVAL;
2128

2129 2130 2131 2132
			dev_data->ats.enabled = true;
			dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
			dev_data->pri_tlp     = pci_pri_tlp_required(pdev);
		}
2133 2134
	} else if (amd_iommu_iotlb_sup &&
		   pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
2135 2136 2137
		dev_data->ats.enabled = true;
		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
	}
2138

2139
	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2140
	ret = __attach_device(dev_data, domain);
2141 2142
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

2143 2144 2145 2146 2147
	/*
	 * We might boot into a crash-kernel here. The crashed kernel
	 * left the caches in the IOMMU dirty. So we have to flush
	 * here to evict all dirty stuff.
	 */
2148
	domain_flush_tlb_pde(domain);
2149 2150

	return ret;
2151 2152
}

2153 2154 2155
/*
 * Removes a device from a protection domain (unlocked)
 */
2156
static void __detach_device(struct iommu_dev_data *dev_data)
2157
{
2158
	struct protection_domain *domain;
2159

2160 2161 2162 2163 2164
	/*
	 * Must be called with IRQs disabled. Warn here to detect early
	 * when its not.
	 */
	WARN_ON(!irqs_disabled());
2165

2166 2167
	if (WARN_ON(!dev_data->domain))
		return;
2168

2169
	domain = dev_data->domain;
2170

2171
	spin_lock(&domain->lock);
2172

2173
	do_detach(dev_data);
2174

2175
	spin_unlock(&domain->lock);
2176 2177 2178 2179 2180
}

/*
 * Removes a device from a protection domain (with devtable_lock held)
 */
2181
static void detach_device(struct device *dev)
2182
{
2183
	struct protection_domain *domain;
2184
	struct iommu_dev_data *dev_data;
2185 2186
	unsigned long flags;

2187
	dev_data = get_dev_data(dev);
2188
	domain   = dev_data->domain;
2189

2190 2191
	/* lock device table */
	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2192
	__detach_device(dev_data);
2193
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2194

2195
	if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2)
2196 2197
		pdev_iommuv2_disable(to_pci_dev(dev));
	else if (dev_data->ats.enabled)
2198
		pci_disable_ats(to_pci_dev(dev));
2199 2200

	dev_data->ats.enabled = false;
2201
}
2202

2203
static int amd_iommu_add_device(struct device *dev)
2204
{
2205
	struct iommu_dev_data *dev_data;
2206
	struct iommu_domain *domain;
2207
	struct amd_iommu *iommu;
2208
	u16 devid;
2209
	int ret;
2210

2211
	if (!check_device(dev) || get_dev_data(dev))
2212
		return 0;
2213

2214 2215
	devid = get_device_id(dev);
	iommu = amd_iommu_rlookup_table[devid];
2216

2217
	ret = iommu_init_device(dev);
2218 2219 2220 2221
	if (ret) {
		if (ret != -ENOTSUPP)
			pr_err("Failed to initialize device %s - trying to proceed anyway\n",
				dev_name(dev));
2222

2223
		iommu_ignore_device(dev);
2224
		dev->archdata.dma_ops = &nommu_dma_ops;
2225 2226 2227
		goto out;
	}
	init_iommu_group(dev);
2228

2229
	dev_data = get_dev_data(dev);
2230

2231
	BUG_ON(!dev_data);
2232

2233
	if (iommu_pass_through || dev_data->iommu_v2)
2234
		iommu_request_dm_for_dev(dev);
2235

2236 2237
	/* Domains are initialized for this device - have a look what we ended up with */
	domain = iommu_get_domain_for_dev(dev);
2238
	if (domain->type == IOMMU_DOMAIN_IDENTITY)
2239
		dev_data->passthrough = true;
2240
	else
2241
		dev->archdata.dma_ops = &amd_iommu_dma_ops;
2242

2243
out:
2244 2245 2246 2247 2248
	iommu_completion_wait(iommu);

	return 0;
}

2249
static void amd_iommu_remove_device(struct device *dev)
2250
{
2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261
	struct amd_iommu *iommu;
	u16 devid;

	if (!check_device(dev))
		return;

	devid = get_device_id(dev);
	iommu = amd_iommu_rlookup_table[devid];

	iommu_uninit_device(dev);
	iommu_completion_wait(iommu);
2262 2263
}

2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276
/*****************************************************************************
 *
 * The next functions belong to the dma_ops mapping/unmapping code.
 *
 *****************************************************************************/

/*
 * In the dma_ops path we only have the struct device. This function
 * finds the corresponding IOMMU, the protection domain and the
 * requestor id for a given device.
 * If the device is not yet associated with a domain this is also done
 * in this function.
 */
2277
static struct protection_domain *get_domain(struct device *dev)
2278
{
2279
	struct protection_domain *domain;
2280
	struct iommu_domain *io_domain;
2281

2282
	if (!check_device(dev))
2283
		return ERR_PTR(-EINVAL);
2284

2285
	io_domain = iommu_get_domain_for_dev(dev);
2286 2287
	if (!io_domain)
		return NULL;
2288

2289 2290
	domain = to_pdomain(io_domain);
	if (!dma_ops_domain(domain))
2291
		return ERR_PTR(-EBUSY);
2292

2293
	return domain;
2294 2295
}

2296 2297
static void update_device_table(struct protection_domain *domain)
{
2298
	struct iommu_dev_data *dev_data;
2299

2300 2301
	list_for_each_entry(dev_data, &domain->dev_list, list)
		set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
2302 2303 2304 2305 2306 2307 2308 2309
}

static void update_domain(struct protection_domain *domain)
{
	if (!domain->updated)
		return;

	update_device_table(domain);
2310 2311 2312

	domain_flush_devices(domain);
	domain_flush_tlb_pde(domain);
2313 2314 2315 2316

	domain->updated = false;
}

2317 2318 2319 2320 2321 2322
/*
 * This function fetches the PTE for a given address in the aperture
 */
static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
			    unsigned long address)
{
2323
	struct aperture_range *aperture;
2324 2325
	u64 *pte, *pte_page;

2326 2327 2328 2329 2330
	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
	if (!aperture)
		return NULL;

	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2331
	if (!pte) {
2332
		pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
2333
				GFP_ATOMIC);
2334 2335
		aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
	} else
2336
		pte += PM_LEVEL_INDEX(0, address);
2337

2338
	update_domain(&dom->domain);
2339 2340 2341 2342

	return pte;
}

2343 2344 2345 2346
/*
 * This is the generic map function. It maps one 4kb page at paddr to
 * the given address in the DMA address space for the domain.
 */
2347
static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
2348 2349 2350 2351 2352 2353 2354 2355 2356 2357
				     unsigned long address,
				     phys_addr_t paddr,
				     int direction)
{
	u64 *pte, __pte;

	WARN_ON(address > dom->aperture_size);

	paddr &= PAGE_MASK;

2358
	pte  = dma_ops_get_pte(dom, address);
2359
	if (!pte)
2360
		return DMA_ERROR_CODE;
2361 2362 2363 2364 2365 2366 2367 2368 2369 2370

	__pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;

	if (direction == DMA_TO_DEVICE)
		__pte |= IOMMU_PTE_IR;
	else if (direction == DMA_FROM_DEVICE)
		__pte |= IOMMU_PTE_IW;
	else if (direction == DMA_BIDIRECTIONAL)
		__pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;

2371
	WARN_ON_ONCE(*pte);
2372 2373 2374 2375 2376 2377

	*pte = __pte;

	return (dma_addr_t)address;
}

2378 2379 2380
/*
 * The generic unmapping function for on page in the DMA address space.
 */
2381
static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
2382 2383
				 unsigned long address)
{
2384
	struct aperture_range *aperture;
2385 2386 2387 2388 2389
	u64 *pte;

	if (address >= dom->aperture_size)
		return;

2390 2391 2392 2393 2394 2395 2396
	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
	if (!aperture)
		return;

	pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
	if (!pte)
		return;
2397

2398
	pte += PM_LEVEL_INDEX(0, address);
2399

2400
	WARN_ON_ONCE(!*pte);
2401 2402 2403 2404

	*pte = 0ULL;
}

2405 2406
/*
 * This function contains common code for mapping of a physically
J
Joerg Roedel 已提交
2407 2408
 * contiguous memory region into DMA address space. It is used by all
 * mapping functions provided with this IOMMU driver.
2409 2410
 * Must be called with the domain lock held.
 */
2411 2412 2413 2414
static dma_addr_t __map_single(struct device *dev,
			       struct dma_ops_domain *dma_dom,
			       phys_addr_t paddr,
			       size_t size,
2415
			       int dir,
2416 2417
			       bool align,
			       u64 dma_mask)
2418 2419
{
	dma_addr_t offset = paddr & ~PAGE_MASK;
2420
	dma_addr_t address, start, ret;
2421
	unsigned int pages;
2422
	unsigned long align_mask = 0;
2423 2424
	int i;

2425
	pages = iommu_num_pages(paddr, size, PAGE_SIZE);
2426 2427
	paddr &= PAGE_MASK;

2428 2429
	INC_STATS_COUNTER(total_map_requests);

2430 2431 2432
	if (pages > 1)
		INC_STATS_COUNTER(cross_page);

2433 2434 2435
	if (align)
		align_mask = (1UL << get_order(size)) - 1;

2436
retry:
2437 2438
	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
					  dma_mask);
2439
	if (unlikely(address == DMA_ERROR_CODE)) {
2440 2441 2442 2443 2444 2445 2446
		/*
		 * setting next_address here will let the address
		 * allocator only scan the new allocated range in the
		 * first run. This is a small optimization.
		 */
		dma_dom->next_address = dma_dom->aperture_size;

2447
		if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
2448 2449 2450
			goto out;

		/*
2451
		 * aperture was successfully enlarged by 128 MB, try
2452 2453 2454 2455
		 * allocation again
		 */
		goto retry;
	}
2456 2457 2458

	start = address;
	for (i = 0; i < pages; ++i) {
2459
		ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
2460
		if (ret == DMA_ERROR_CODE)
2461 2462
			goto out_unmap;

2463 2464 2465 2466 2467
		paddr += PAGE_SIZE;
		start += PAGE_SIZE;
	}
	address += offset;

2468 2469
	ADD_STATS_COUNTER(alloced_io_mem, size);

2470
	if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
2471
		domain_flush_tlb(&dma_dom->domain);
2472
		dma_dom->need_flush = false;
2473
	} else if (unlikely(amd_iommu_np_cache))
2474
		domain_flush_pages(&dma_dom->domain, address, size);
2475

2476 2477
out:
	return address;
2478 2479 2480 2481 2482

out_unmap:

	for (--i; i >= 0; --i) {
		start -= PAGE_SIZE;
2483
		dma_ops_domain_unmap(dma_dom, start);
2484 2485 2486 2487
	}

	dma_ops_free_addresses(dma_dom, address, pages);

2488
	return DMA_ERROR_CODE;
2489 2490
}

2491 2492 2493 2494
/*
 * Does the reverse of the __map_single function. Must be called with
 * the domain lock held too
 */
2495
static void __unmap_single(struct dma_ops_domain *dma_dom,
2496 2497 2498 2499
			   dma_addr_t dma_addr,
			   size_t size,
			   int dir)
{
2500
	dma_addr_t flush_addr;
2501 2502 2503
	dma_addr_t i, start;
	unsigned int pages;

2504
	if ((dma_addr == DMA_ERROR_CODE) ||
2505
	    (dma_addr + size > dma_dom->aperture_size))
2506 2507
		return;

2508
	flush_addr = dma_addr;
2509
	pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
2510 2511 2512 2513
	dma_addr &= PAGE_MASK;
	start = dma_addr;

	for (i = 0; i < pages; ++i) {
2514
		dma_ops_domain_unmap(dma_dom, start);
2515 2516 2517
		start += PAGE_SIZE;
	}

2518 2519
	SUB_STATS_COUNTER(alloced_io_mem, size);

2520
	dma_ops_free_addresses(dma_dom, dma_addr, pages);
2521

2522
	if (amd_iommu_unmap_flush || dma_dom->need_flush) {
2523
		domain_flush_pages(&dma_dom->domain, flush_addr, size);
2524 2525
		dma_dom->need_flush = false;
	}
2526 2527
}

2528 2529 2530
/*
 * The exported map_single function for dma_ops.
 */
2531 2532 2533 2534
static dma_addr_t map_page(struct device *dev, struct page *page,
			   unsigned long offset, size_t size,
			   enum dma_data_direction dir,
			   struct dma_attrs *attrs)
2535 2536 2537 2538
{
	unsigned long flags;
	struct protection_domain *domain;
	dma_addr_t addr;
2539
	u64 dma_mask;
2540
	phys_addr_t paddr = page_to_phys(page) + offset;
2541

2542 2543
	INC_STATS_COUNTER(cnt_map_single);

2544 2545
	domain = get_domain(dev);
	if (PTR_ERR(domain) == -EINVAL)
2546
		return (dma_addr_t)paddr;
2547 2548
	else if (IS_ERR(domain))
		return DMA_ERROR_CODE;
2549

2550 2551
	dma_mask = *dev->dma_mask;

2552
	spin_lock_irqsave(&domain->lock, flags);
2553

2554
	addr = __map_single(dev, domain->priv, paddr, size, dir, false,
2555
			    dma_mask);
2556
	if (addr == DMA_ERROR_CODE)
2557 2558
		goto out;

2559
	domain_flush_complete(domain);
2560 2561 2562 2563 2564 2565 2566

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return addr;
}

2567 2568 2569
/*
 * The exported unmap_single function for dma_ops.
 */
2570 2571
static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
		       enum dma_data_direction dir, struct dma_attrs *attrs)
2572 2573 2574 2575
{
	unsigned long flags;
	struct protection_domain *domain;

2576 2577
	INC_STATS_COUNTER(cnt_unmap_single);

2578 2579
	domain = get_domain(dev);
	if (IS_ERR(domain))
2580 2581
		return;

2582 2583
	spin_lock_irqsave(&domain->lock, flags);

2584
	__unmap_single(domain->priv, dma_addr, size, dir);
2585

2586
	domain_flush_complete(domain);
2587 2588 2589 2590

	spin_unlock_irqrestore(&domain->lock, flags);
}

2591 2592 2593 2594
/*
 * The exported map_sg function for dma_ops (handles scatter-gather
 * lists).
 */
2595
static int map_sg(struct device *dev, struct scatterlist *sglist,
2596 2597
		  int nelems, enum dma_data_direction dir,
		  struct dma_attrs *attrs)
2598 2599 2600 2601 2602 2603 2604
{
	unsigned long flags;
	struct protection_domain *domain;
	int i;
	struct scatterlist *s;
	phys_addr_t paddr;
	int mapped_elems = 0;
2605
	u64 dma_mask;
2606

2607 2608
	INC_STATS_COUNTER(cnt_map_sg);

2609
	domain = get_domain(dev);
2610
	if (IS_ERR(domain))
2611
		return 0;
2612

2613
	dma_mask = *dev->dma_mask;
2614 2615 2616 2617 2618 2619

	spin_lock_irqsave(&domain->lock, flags);

	for_each_sg(sglist, s, nelems, i) {
		paddr = sg_phys(s);

2620
		s->dma_address = __map_single(dev, domain->priv,
2621 2622
					      paddr, s->length, dir, false,
					      dma_mask);
2623 2624 2625 2626 2627 2628 2629 2630

		if (s->dma_address) {
			s->dma_length = s->length;
			mapped_elems++;
		} else
			goto unmap;
	}

2631
	domain_flush_complete(domain);
2632 2633 2634 2635 2636 2637 2638 2639

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return mapped_elems;
unmap:
	for_each_sg(sglist, s, mapped_elems, i) {
		if (s->dma_address)
2640
			__unmap_single(domain->priv, s->dma_address,
2641 2642 2643 2644 2645 2646 2647 2648 2649
				       s->dma_length, dir);
		s->dma_address = s->dma_length = 0;
	}

	mapped_elems = 0;

	goto out;
}

2650 2651 2652 2653
/*
 * The exported map_sg function for dma_ops (handles scatter-gather
 * lists).
 */
2654
static void unmap_sg(struct device *dev, struct scatterlist *sglist,
2655 2656
		     int nelems, enum dma_data_direction dir,
		     struct dma_attrs *attrs)
2657 2658 2659 2660 2661 2662
{
	unsigned long flags;
	struct protection_domain *domain;
	struct scatterlist *s;
	int i;

2663 2664
	INC_STATS_COUNTER(cnt_unmap_sg);

2665 2666
	domain = get_domain(dev);
	if (IS_ERR(domain))
2667 2668
		return;

2669 2670 2671
	spin_lock_irqsave(&domain->lock, flags);

	for_each_sg(sglist, s, nelems, i) {
2672
		__unmap_single(domain->priv, s->dma_address,
2673 2674 2675 2676
			       s->dma_length, dir);
		s->dma_address = s->dma_length = 0;
	}

2677
	domain_flush_complete(domain);
2678 2679 2680 2681

	spin_unlock_irqrestore(&domain->lock, flags);
}

2682 2683 2684
/*
 * The exported alloc_coherent function for dma_ops.
 */
2685
static void *alloc_coherent(struct device *dev, size_t size,
2686 2687
			    dma_addr_t *dma_addr, gfp_t flag,
			    struct dma_attrs *attrs)
2688
{
2689
	u64 dma_mask = dev->coherent_dma_mask;
2690 2691 2692
	struct protection_domain *domain;
	unsigned long flags;
	struct page *page;
2693

2694 2695
	INC_STATS_COUNTER(cnt_alloc_coherent);

2696 2697
	domain = get_domain(dev);
	if (PTR_ERR(domain) == -EINVAL) {
2698 2699 2700
		page = alloc_pages(flag, get_order(size));
		*dma_addr = page_to_phys(page);
		return page_address(page);
2701 2702
	} else if (IS_ERR(domain))
		return NULL;
2703

2704
	size	  = PAGE_ALIGN(size);
2705 2706
	dma_mask  = dev->coherent_dma_mask;
	flag     &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
2707
	flag     |= __GFP_ZERO;
2708

2709 2710
	page = alloc_pages(flag | __GFP_NOWARN,  get_order(size));
	if (!page) {
2711
		if (!gfpflags_allow_blocking(flag))
2712
			return NULL;
2713

2714 2715 2716 2717 2718
		page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
						 get_order(size));
		if (!page)
			return NULL;
	}
2719

2720 2721 2722
	if (!dma_mask)
		dma_mask = *dev->dma_mask;

2723 2724
	spin_lock_irqsave(&domain->lock, flags);

2725
	*dma_addr = __map_single(dev, domain->priv, page_to_phys(page),
2726
				 size, DMA_BIDIRECTIONAL, true, dma_mask);
2727

2728
	if (*dma_addr == DMA_ERROR_CODE) {
J
Jiri Slaby 已提交
2729
		spin_unlock_irqrestore(&domain->lock, flags);
2730
		goto out_free;
J
Jiri Slaby 已提交
2731
	}
2732

2733
	domain_flush_complete(domain);
2734 2735 2736

	spin_unlock_irqrestore(&domain->lock, flags);

2737
	return page_address(page);
2738 2739 2740

out_free:

2741 2742
	if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
		__free_pages(page, get_order(size));
2743 2744

	return NULL;
2745 2746
}

2747 2748 2749
/*
 * The exported free_coherent function for dma_ops.
 */
2750
static void free_coherent(struct device *dev, size_t size,
2751 2752
			  void *virt_addr, dma_addr_t dma_addr,
			  struct dma_attrs *attrs)
2753 2754
{
	struct protection_domain *domain;
2755 2756
	unsigned long flags;
	struct page *page;
2757

2758 2759
	INC_STATS_COUNTER(cnt_free_coherent);

2760 2761 2762
	page = virt_to_page(virt_addr);
	size = PAGE_ALIGN(size);

2763 2764
	domain = get_domain(dev);
	if (IS_ERR(domain))
2765 2766
		goto free_mem;

2767 2768
	spin_lock_irqsave(&domain->lock, flags);

2769
	__unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
2770

2771
	domain_flush_complete(domain);
2772 2773 2774 2775

	spin_unlock_irqrestore(&domain->lock, flags);

free_mem:
2776 2777
	if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
		__free_pages(page, get_order(size));
2778 2779
}

2780 2781 2782 2783 2784 2785
/*
 * This function is called by the DMA layer to find out if we can handle a
 * particular device. It is part of the dma_ops.
 */
static int amd_iommu_dma_supported(struct device *dev, u64 mask)
{
2786
	return check_device(dev);
2787 2788
}

2789
static struct dma_map_ops amd_iommu_dma_ops = {
2790 2791
	.alloc = alloc_coherent,
	.free = free_coherent,
2792 2793
	.map_page = map_page,
	.unmap_page = unmap_page,
2794 2795
	.map_sg = map_sg,
	.unmap_sg = unmap_sg,
2796
	.dma_supported = amd_iommu_dma_supported,
2797 2798
};

2799
int __init amd_iommu_init_api(void)
2800
{
2801
	return bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
2802 2803
}

2804 2805
int __init amd_iommu_init_dma_ops(void)
{
2806
	swiotlb        = iommu_pass_through ? 1 : 0;
2807 2808
	iommu_detected = 1;

2809 2810 2811 2812 2813 2814 2815 2816 2817
	/*
	 * In case we don't initialize SWIOTLB (actually the common case
	 * when AMD IOMMU is enabled), make sure there are global
	 * dma_ops set as a fall-back for devices not handled by this
	 * driver (for example non-PCI devices).
	 */
	if (!swiotlb)
		dma_ops = &nommu_dma_ops;

2818 2819
	amd_iommu_stats_init();

2820 2821 2822 2823 2824
	if (amd_iommu_unmap_flush)
		pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");
	else
		pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");

2825 2826
	return 0;
}
2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839

/*****************************************************************************
 *
 * The following functions belong to the exported interface of AMD IOMMU
 *
 * This interface allows access to lower level functions of the IOMMU
 * like protection domain handling and assignement of devices to domains
 * which is not possible with the dma_ops interface.
 *
 *****************************************************************************/

static void cleanup_domain(struct protection_domain *domain)
{
2840
	struct iommu_dev_data *entry;
2841 2842 2843 2844
	unsigned long flags;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);

2845 2846 2847 2848
	while (!list_empty(&domain->dev_list)) {
		entry = list_first_entry(&domain->dev_list,
					 struct iommu_dev_data, list);
		__detach_device(entry);
2849
	}
2850 2851 2852 2853

	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}

2854 2855 2856 2857 2858
static void protection_domain_free(struct protection_domain *domain)
{
	if (!domain)
		return;

2859 2860
	del_domain_from_list(domain);

2861 2862 2863 2864 2865 2866
	if (domain->id)
		domain_id_free(domain->id);

	kfree(domain);
}

2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878
static int protection_domain_init(struct protection_domain *domain)
{
	spin_lock_init(&domain->lock);
	mutex_init(&domain->api_lock);
	domain->id = domain_id_alloc();
	if (!domain->id)
		return -ENOMEM;
	INIT_LIST_HEAD(&domain->dev_list);

	return 0;
}

2879
static struct protection_domain *protection_domain_alloc(void)
2880 2881 2882 2883 2884
{
	struct protection_domain *domain;

	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
	if (!domain)
2885
		return NULL;
2886

2887
	if (protection_domain_init(domain))
2888 2889
		goto out_err;

2890 2891
	add_domain_to_list(domain);

2892 2893 2894 2895 2896 2897 2898 2899
	return domain;

out_err:
	kfree(domain);

	return NULL;
}

2900
static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)
2901
{
2902
	struct protection_domain *pdomain;
2903
	struct dma_ops_domain *dma_domain;
2904

2905 2906 2907 2908 2909
	switch (type) {
	case IOMMU_DOMAIN_UNMANAGED:
		pdomain = protection_domain_alloc();
		if (!pdomain)
			return NULL;
2910

2911 2912 2913 2914 2915 2916
		pdomain->mode    = PAGE_MODE_3_LEVEL;
		pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pdomain->pt_root) {
			protection_domain_free(pdomain);
			return NULL;
		}
2917

2918 2919 2920
		pdomain->domain.geometry.aperture_start = 0;
		pdomain->domain.geometry.aperture_end   = ~0ULL;
		pdomain->domain.geometry.force_aperture = true;
2921

2922 2923 2924 2925 2926 2927 2928 2929 2930
		break;
	case IOMMU_DOMAIN_DMA:
		dma_domain = dma_ops_domain_alloc();
		if (!dma_domain) {
			pr_err("AMD-Vi: Failed to allocate\n");
			return NULL;
		}
		pdomain = &dma_domain->domain;
		break;
2931 2932 2933 2934
	case IOMMU_DOMAIN_IDENTITY:
		pdomain = protection_domain_alloc();
		if (!pdomain)
			return NULL;
2935

2936 2937
		pdomain->mode = PAGE_MODE_NONE;
		break;
2938 2939 2940
	default:
		return NULL;
	}
2941

2942
	return &pdomain->domain;
2943 2944
}

2945
static void amd_iommu_domain_free(struct iommu_domain *dom)
2946
{
2947
	struct protection_domain *domain;
2948

2949
	if (!dom)
2950 2951
		return;

2952 2953
	domain = to_pdomain(dom);

2954 2955 2956 2957 2958
	if (domain->dev_cnt > 0)
		cleanup_domain(domain);

	BUG_ON(domain->dev_cnt != 0);

2959 2960
	if (domain->mode != PAGE_MODE_NONE)
		free_pagetable(domain);
2961

2962 2963 2964
	if (domain->flags & PD_IOMMUV2_MASK)
		free_gcr3_table(domain);

2965
	protection_domain_free(domain);
2966 2967
}

2968 2969 2970
static void amd_iommu_detach_device(struct iommu_domain *dom,
				    struct device *dev)
{
2971
	struct iommu_dev_data *dev_data = dev->archdata.iommu;
2972 2973 2974
	struct amd_iommu *iommu;
	u16 devid;

2975
	if (!check_device(dev))
2976 2977
		return;

2978
	devid = get_device_id(dev);
2979

2980
	if (dev_data->domain != NULL)
2981
		detach_device(dev);
2982 2983 2984 2985 2986 2987 2988 2989

	iommu = amd_iommu_rlookup_table[devid];
	if (!iommu)
		return;

	iommu_completion_wait(iommu);
}

2990 2991 2992
static int amd_iommu_attach_device(struct iommu_domain *dom,
				   struct device *dev)
{
2993
	struct protection_domain *domain = to_pdomain(dom);
2994
	struct iommu_dev_data *dev_data;
2995
	struct amd_iommu *iommu;
2996
	int ret;
2997

2998
	if (!check_device(dev))
2999 3000
		return -EINVAL;

3001 3002
	dev_data = dev->archdata.iommu;

3003
	iommu = amd_iommu_rlookup_table[dev_data->devid];
3004 3005 3006
	if (!iommu)
		return -EINVAL;

3007
	if (dev_data->domain)
3008
		detach_device(dev);
3009

3010
	ret = attach_device(dev, domain);
3011 3012 3013

	iommu_completion_wait(iommu);

3014
	return ret;
3015 3016
}

3017
static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
3018
			 phys_addr_t paddr, size_t page_size, int iommu_prot)
3019
{
3020
	struct protection_domain *domain = to_pdomain(dom);
3021 3022 3023
	int prot = 0;
	int ret;

3024 3025 3026
	if (domain->mode == PAGE_MODE_NONE)
		return -EINVAL;

3027 3028 3029 3030 3031
	if (iommu_prot & IOMMU_READ)
		prot |= IOMMU_PROT_IR;
	if (iommu_prot & IOMMU_WRITE)
		prot |= IOMMU_PROT_IW;

3032
	mutex_lock(&domain->api_lock);
3033
	ret = iommu_map_page(domain, iova, paddr, prot, page_size);
3034 3035
	mutex_unlock(&domain->api_lock);

3036
	return ret;
3037 3038
}

3039 3040
static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
			   size_t page_size)
3041
{
3042
	struct protection_domain *domain = to_pdomain(dom);
3043
	size_t unmap_size;
3044

3045 3046 3047
	if (domain->mode == PAGE_MODE_NONE)
		return -EINVAL;

3048
	mutex_lock(&domain->api_lock);
3049
	unmap_size = iommu_unmap_page(domain, iova, page_size);
3050
	mutex_unlock(&domain->api_lock);
3051

3052
	domain_flush_tlb_pde(domain);
3053

3054
	return unmap_size;
3055 3056
}

3057
static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
3058
					  dma_addr_t iova)
3059
{
3060
	struct protection_domain *domain = to_pdomain(dom);
3061
	unsigned long offset_mask, pte_pgsize;
3062
	u64 *pte, __pte;
3063

3064 3065 3066
	if (domain->mode == PAGE_MODE_NONE)
		return iova;

3067
	pte = fetch_pte(domain, iova, &pte_pgsize);
3068

3069
	if (!pte || !IOMMU_PTE_PRESENT(*pte))
3070 3071
		return 0;

3072 3073
	offset_mask = pte_pgsize - 1;
	__pte	    = *pte & PM_ADDR_MASK;
3074

3075
	return (__pte & ~offset_mask) | (iova & offset_mask);
3076 3077
}

3078
static bool amd_iommu_capable(enum iommu_cap cap)
S
Sheng Yang 已提交
3079
{
3080 3081
	switch (cap) {
	case IOMMU_CAP_CACHE_COHERENCY:
3082
		return true;
3083
	case IOMMU_CAP_INTR_REMAP:
3084
		return (irq_remapping_enabled == 1);
3085 3086
	case IOMMU_CAP_NOEXEC:
		return false;
3087 3088
	}

3089
	return false;
S
Sheng Yang 已提交
3090 3091
}

3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132
static void amd_iommu_get_dm_regions(struct device *dev,
				     struct list_head *head)
{
	struct unity_map_entry *entry;
	u16 devid;

	devid = get_device_id(dev);

	list_for_each_entry(entry, &amd_iommu_unity_map, list) {
		struct iommu_dm_region *region;

		if (devid < entry->devid_start || devid > entry->devid_end)
			continue;

		region = kzalloc(sizeof(*region), GFP_KERNEL);
		if (!region) {
			pr_err("Out of memory allocating dm-regions for %s\n",
				dev_name(dev));
			return;
		}

		region->start = entry->address_start;
		region->length = entry->address_end - entry->address_start;
		if (entry->prot & IOMMU_PROT_IR)
			region->prot |= IOMMU_READ;
		if (entry->prot & IOMMU_PROT_IW)
			region->prot |= IOMMU_WRITE;

		list_add_tail(&region->list, head);
	}
}

static void amd_iommu_put_dm_regions(struct device *dev,
				     struct list_head *head)
{
	struct iommu_dm_region *entry, *next;

	list_for_each_entry_safe(entry, next, head, list)
		kfree(entry);
}

3133
static const struct iommu_ops amd_iommu_ops = {
3134
	.capable = amd_iommu_capable,
3135 3136
	.domain_alloc = amd_iommu_domain_alloc,
	.domain_free  = amd_iommu_domain_free,
3137 3138
	.attach_dev = amd_iommu_attach_device,
	.detach_dev = amd_iommu_detach_device,
3139 3140
	.map = amd_iommu_map,
	.unmap = amd_iommu_unmap,
O
Olav Haugan 已提交
3141
	.map_sg = default_iommu_map_sg,
3142
	.iova_to_phys = amd_iommu_iova_to_phys,
3143 3144
	.add_device = amd_iommu_add_device,
	.remove_device = amd_iommu_remove_device,
3145
	.device_group = pci_device_group,
3146 3147
	.get_dm_regions = amd_iommu_get_dm_regions,
	.put_dm_regions = amd_iommu_put_dm_regions,
3148
	.pgsize_bitmap	= AMD_IOMMU_PGSIZES,
3149 3150
};

3151 3152 3153 3154 3155 3156 3157 3158 3159 3160
/*****************************************************************************
 *
 * The next functions do a basic initialization of IOMMU for pass through
 * mode
 *
 * In passthrough mode the IOMMU is initialized and enabled but not used for
 * DMA-API translation.
 *
 *****************************************************************************/

3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172
/* IOMMUv2 specific functions */
int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
{
	return atomic_notifier_chain_register(&ppr_notifier, nb);
}
EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);

int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
{
	return atomic_notifier_chain_unregister(&ppr_notifier, nb);
}
EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
3173 3174 3175

void amd_iommu_domain_direct_map(struct iommu_domain *dom)
{
3176
	struct protection_domain *domain = to_pdomain(dom);
3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193
	unsigned long flags;

	spin_lock_irqsave(&domain->lock, flags);

	/* Update data structure */
	domain->mode    = PAGE_MODE_NONE;
	domain->updated = true;

	/* Make changes visible to IOMMUs */
	update_domain(domain);

	/* Page-table is not visible to IOMMU anymore, so free it */
	free_pagetable(domain);

	spin_unlock_irqrestore(&domain->lock, flags);
}
EXPORT_SYMBOL(amd_iommu_domain_direct_map);
3194 3195 3196

int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
{
3197
	struct protection_domain *domain = to_pdomain(dom);
3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240
	unsigned long flags;
	int levels, ret;

	if (pasids <= 0 || pasids > (PASID_MASK + 1))
		return -EINVAL;

	/* Number of GCR3 table levels required */
	for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
		levels += 1;

	if (levels > amd_iommu_max_glx_val)
		return -EINVAL;

	spin_lock_irqsave(&domain->lock, flags);

	/*
	 * Save us all sanity checks whether devices already in the
	 * domain support IOMMUv2. Just force that the domain has no
	 * devices attached when it is switched into IOMMUv2 mode.
	 */
	ret = -EBUSY;
	if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
		goto out;

	ret = -ENOMEM;
	domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
	if (domain->gcr3_tbl == NULL)
		goto out;

	domain->glx      = levels;
	domain->flags   |= PD_IOMMUV2_MASK;
	domain->updated  = true;

	update_domain(domain);

	ret = 0;

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274

static int __flush_pasid(struct protection_domain *domain, int pasid,
			 u64 address, bool size)
{
	struct iommu_dev_data *dev_data;
	struct iommu_cmd cmd;
	int i, ret;

	if (!(domain->flags & PD_IOMMUV2_MASK))
		return -EINVAL;

	build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);

	/*
	 * IOMMU TLB needs to be flushed before Device TLB to
	 * prevent device TLB refill from IOMMU TLB
	 */
	for (i = 0; i < amd_iommus_present; ++i) {
		if (domain->dev_iommu[i] == 0)
			continue;

		ret = iommu_queue_command(amd_iommus[i], &cmd);
		if (ret != 0)
			goto out;
	}

	/* Wait until IOMMU TLB flushes are complete */
	domain_flush_complete(domain);

	/* Now flush device TLBs */
	list_for_each_entry(dev_data, &domain->dev_list, list) {
		struct amd_iommu *iommu;
		int qdep;

3275 3276 3277 3278 3279 3280
		/*
		   There might be non-IOMMUv2 capable devices in an IOMMUv2
		 * domain.
		 */
		if (!dev_data->ats.enabled)
			continue;
3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305

		qdep  = dev_data->ats.qdep;
		iommu = amd_iommu_rlookup_table[dev_data->devid];

		build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
				      qdep, address, size);

		ret = iommu_queue_command(iommu, &cmd);
		if (ret != 0)
			goto out;
	}

	/* Wait until all device TLBs are flushed */
	domain_flush_complete(domain);

	ret = 0;

out:

	return ret;
}

static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
				  u64 address)
{
3306 3307
	INC_STATS_COUNTER(invalidate_iotlb);

3308 3309 3310 3311 3312 3313
	return __flush_pasid(domain, pasid, address, false);
}

int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
			 u64 address)
{
3314
	struct protection_domain *domain = to_pdomain(dom);
3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __amd_iommu_flush_page(domain, pasid, address);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_flush_page);

static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
{
3328 3329
	INC_STATS_COUNTER(invalidate_iotlb_all);

3330 3331 3332 3333 3334 3335
	return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
			     true);
}

int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
{
3336
	struct protection_domain *domain = to_pdomain(dom);
3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __amd_iommu_flush_tlb(domain, pasid);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_flush_tlb);

3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415
static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
{
	int index;
	u64 *pte;

	while (true) {

		index = (pasid >> (9 * level)) & 0x1ff;
		pte   = &root[index];

		if (level == 0)
			break;

		if (!(*pte & GCR3_VALID)) {
			if (!alloc)
				return NULL;

			root = (void *)get_zeroed_page(GFP_ATOMIC);
			if (root == NULL)
				return NULL;

			*pte = __pa(root) | GCR3_VALID;
		}

		root = __va(*pte & PAGE_MASK);

		level -= 1;
	}

	return pte;
}

static int __set_gcr3(struct protection_domain *domain, int pasid,
		      unsigned long cr3)
{
	u64 *pte;

	if (domain->mode != PAGE_MODE_NONE)
		return -EINVAL;

	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
	if (pte == NULL)
		return -ENOMEM;

	*pte = (cr3 & PAGE_MASK) | GCR3_VALID;

	return __amd_iommu_flush_tlb(domain, pasid);
}

static int __clear_gcr3(struct protection_domain *domain, int pasid)
{
	u64 *pte;

	if (domain->mode != PAGE_MODE_NONE)
		return -EINVAL;

	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
	if (pte == NULL)
		return 0;

	*pte = 0;

	return __amd_iommu_flush_tlb(domain, pasid);
}

int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
			      unsigned long cr3)
{
3416
	struct protection_domain *domain = to_pdomain(dom);
3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __set_gcr3(domain, pasid, cr3);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);

int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
{
3430
	struct protection_domain *domain = to_pdomain(dom);
3431 3432 3433 3434 3435 3436 3437 3438 3439 3440
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&domain->lock, flags);
	ret = __clear_gcr3(domain, pasid);
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
3441 3442 3443 3444 3445 3446 3447 3448

int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
			   int status, int tag)
{
	struct iommu_dev_data *dev_data;
	struct amd_iommu *iommu;
	struct iommu_cmd cmd;

3449 3450
	INC_STATS_COUNTER(complete_ppr);

3451 3452 3453 3454 3455 3456 3457 3458 3459
	dev_data = get_dev_data(&pdev->dev);
	iommu    = amd_iommu_rlookup_table[dev_data->devid];

	build_complete_ppr(&cmd, dev_data->devid, pasid, status,
			   tag, dev_data->pri_tlp);

	return iommu_queue_command(iommu, &cmd);
}
EXPORT_SYMBOL(amd_iommu_complete_ppr);
3460 3461 3462

struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
{
3463
	struct protection_domain *pdomain;
3464

3465 3466
	pdomain = get_domain(&pdev->dev);
	if (IS_ERR(pdomain))
3467 3468 3469
		return NULL;

	/* Only return IOMMUv2 domains */
3470
	if (!(pdomain->flags & PD_IOMMUV2_MASK))
3471 3472
		return NULL;

3473
	return &pdomain->domain;
3474 3475
}
EXPORT_SYMBOL(amd_iommu_get_v2_domain);
3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487

void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
{
	struct iommu_dev_data *dev_data;

	if (!amd_iommu_v2_supported())
		return;

	dev_data = get_dev_data(&pdev->dev);
	dev_data->errata |= (1 << erratum);
}
EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530

int amd_iommu_device_info(struct pci_dev *pdev,
                          struct amd_iommu_device_info *info)
{
	int max_pasids;
	int pos;

	if (pdev == NULL || info == NULL)
		return -EINVAL;

	if (!amd_iommu_v2_supported())
		return -EINVAL;

	memset(info, 0, sizeof(*info));

	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
	if (pos)
		info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;

	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
	if (pos)
		info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;

	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
	if (pos) {
		int features;

		max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
		max_pasids = min(max_pasids, (1 << 20));

		info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
		info->max_pasids = min(pci_max_pasids(pdev), max_pasids);

		features = pci_pasid_features(pdev);
		if (features & PCI_PASID_CAP_EXEC)
			info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
		if (features & PCI_PASID_CAP_PRIV)
			info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
	}

	return 0;
}
EXPORT_SYMBOL(amd_iommu_device_info);
3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554

#ifdef CONFIG_IRQ_REMAP

/*****************************************************************************
 *
 * Interrupt Remapping Implementation
 *
 *****************************************************************************/

union irte {
	u32 val;
	struct {
		u32 valid	: 1,
		    no_fault	: 1,
		    int_type	: 3,
		    rq_eoi	: 1,
		    dm		: 1,
		    rsvd_1	: 1,
		    destination	: 8,
		    vector	: 8,
		    rsvd_2	: 8;
	} fields;
};

3555 3556 3557 3558 3559
struct irq_2_irte {
	u16 devid; /* Device ID for IRTE table */
	u16 index; /* Index into IRTE table*/
};

3560 3561 3562 3563 3564 3565 3566 3567 3568 3569
struct amd_ir_data {
	struct irq_2_irte			irq_2_irte;
	union irte				irte_entry;
	union {
		struct msi_msg			msi_entry;
	};
};

static struct irq_chip amd_ir_chip;

3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621
#define DTE_IRQ_PHYS_ADDR_MASK	(((1ULL << 45)-1) << 6)
#define DTE_IRQ_REMAP_INTCTL    (2ULL << 60)
#define DTE_IRQ_TABLE_LEN       (8ULL << 1)
#define DTE_IRQ_REMAP_ENABLE    1ULL

static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
{
	u64 dte;

	dte	= amd_iommu_dev_table[devid].data[2];
	dte	&= ~DTE_IRQ_PHYS_ADDR_MASK;
	dte	|= virt_to_phys(table->table);
	dte	|= DTE_IRQ_REMAP_INTCTL;
	dte	|= DTE_IRQ_TABLE_LEN;
	dte	|= DTE_IRQ_REMAP_ENABLE;

	amd_iommu_dev_table[devid].data[2] = dte;
}

#define IRTE_ALLOCATED (~1U)

static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
{
	struct irq_remap_table *table = NULL;
	struct amd_iommu *iommu;
	unsigned long flags;
	u16 alias;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);

	iommu = amd_iommu_rlookup_table[devid];
	if (!iommu)
		goto out_unlock;

	table = irq_lookup_table[devid];
	if (table)
		goto out;

	alias = amd_iommu_alias_table[devid];
	table = irq_lookup_table[alias];
	if (table) {
		irq_lookup_table[devid] = table;
		set_dte_irq_entry(devid, table);
		iommu_flush_dte(iommu, devid);
		goto out;
	}

	/* Nothing there yet, allocate new irq remapping table */
	table = kzalloc(sizeof(*table), GFP_ATOMIC);
	if (!table)
		goto out;

3622 3623 3624
	/* Initialize table spin-lock */
	spin_lock_init(&table->lock);

3625 3626 3627 3628 3629 3630 3631
	if (ioapic)
		/* Keep the first 32 indexes free for IOAPIC interrupts */
		table->min_index = 32;

	table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_ATOMIC);
	if (!table->table) {
		kfree(table);
3632
		table = NULL;
3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649
		goto out;
	}

	memset(table->table, 0, MAX_IRQS_PER_TABLE * sizeof(u32));

	if (ioapic) {
		int i;

		for (i = 0; i < 32; ++i)
			table->table[i] = IRTE_ALLOCATED;
	}

	irq_lookup_table[devid] = table;
	set_dte_irq_entry(devid, table);
	iommu_flush_dte(iommu, devid);
	if (devid != alias) {
		irq_lookup_table[alias] = table;
3650
		set_dte_irq_entry(alias, table);
3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662
		iommu_flush_dte(iommu, alias);
	}

out:
	iommu_completion_wait(iommu);

out_unlock:
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

	return table;
}

3663
static int alloc_irq_index(u16 devid, int count)
3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746
{
	struct irq_remap_table *table;
	unsigned long flags;
	int index, c;

	table = get_irq_table(devid, false);
	if (!table)
		return -ENODEV;

	spin_lock_irqsave(&table->lock, flags);

	/* Scan table for free entries */
	for (c = 0, index = table->min_index;
	     index < MAX_IRQS_PER_TABLE;
	     ++index) {
		if (table->table[index] == 0)
			c += 1;
		else
			c = 0;

		if (c == count)	{
			for (; c != 0; --c)
				table->table[index - c + 1] = IRTE_ALLOCATED;

			index -= count - 1;
			goto out;
		}
	}

	index = -ENOSPC;

out:
	spin_unlock_irqrestore(&table->lock, flags);

	return index;
}

static int modify_irte(u16 devid, int index, union irte irte)
{
	struct irq_remap_table *table;
	struct amd_iommu *iommu;
	unsigned long flags;

	iommu = amd_iommu_rlookup_table[devid];
	if (iommu == NULL)
		return -EINVAL;

	table = get_irq_table(devid, false);
	if (!table)
		return -ENOMEM;

	spin_lock_irqsave(&table->lock, flags);
	table->table[index] = irte.val;
	spin_unlock_irqrestore(&table->lock, flags);

	iommu_flush_irt(iommu, devid);
	iommu_completion_wait(iommu);

	return 0;
}

static void free_irte(u16 devid, int index)
{
	struct irq_remap_table *table;
	struct amd_iommu *iommu;
	unsigned long flags;

	iommu = amd_iommu_rlookup_table[devid];
	if (iommu == NULL)
		return;

	table = get_irq_table(devid, false);
	if (!table)
		return;

	spin_lock_irqsave(&table->lock, flags);
	table->table[index] = 0;
	spin_unlock_irqrestore(&table->lock, flags);

	iommu_flush_irt(iommu, devid);
	iommu_completion_wait(iommu);
}

3747
static int get_devid(struct irq_alloc_info *info)
3748
{
3749
	int devid = -1;
3750

3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765
	switch (info->type) {
	case X86_IRQ_ALLOC_TYPE_IOAPIC:
		devid     = get_ioapic_devid(info->ioapic_id);
		break;
	case X86_IRQ_ALLOC_TYPE_HPET:
		devid     = get_hpet_devid(info->hpet_id);
		break;
	case X86_IRQ_ALLOC_TYPE_MSI:
	case X86_IRQ_ALLOC_TYPE_MSIX:
		devid = get_device_id(&info->msi_dev->dev);
		break;
	default:
		BUG_ON(1);
		break;
	}
3766

3767 3768
	return devid;
}
3769

3770 3771 3772 3773
static struct irq_domain *get_ir_irq_domain(struct irq_alloc_info *info)
{
	struct amd_iommu *iommu;
	int devid;
3774

3775 3776
	if (!info)
		return NULL;
3777

3778 3779 3780 3781 3782 3783
	devid = get_devid(info);
	if (devid >= 0) {
		iommu = amd_iommu_rlookup_table[devid];
		if (iommu)
			return iommu->ir_domain;
	}
3784

3785
	return NULL;
3786 3787
}

3788
static struct irq_domain *get_irq_domain(struct irq_alloc_info *info)
3789
{
3790 3791
	struct amd_iommu *iommu;
	int devid;
3792

3793 3794
	if (!info)
		return NULL;
3795

3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808
	switch (info->type) {
	case X86_IRQ_ALLOC_TYPE_MSI:
	case X86_IRQ_ALLOC_TYPE_MSIX:
		devid = get_device_id(&info->msi_dev->dev);
		if (devid >= 0) {
			iommu = amd_iommu_rlookup_table[devid];
			if (iommu)
				return iommu->msi_domain;
		}
		break;
	default:
		break;
	}
3809

3810 3811
	return NULL;
}
3812

3813 3814 3815 3816 3817 3818
struct irq_remap_ops amd_iommu_irq_ops = {
	.prepare		= amd_iommu_prepare,
	.enable			= amd_iommu_enable,
	.disable		= amd_iommu_disable,
	.reenable		= amd_iommu_reenable,
	.enable_faulting	= amd_iommu_enable_faulting,
3819 3820 3821
	.get_ir_irq_domain	= get_ir_irq_domain,
	.get_irq_domain		= get_irq_domain,
};
3822

3823 3824 3825 3826 3827 3828 3829 3830 3831
static void irq_remapping_prepare_irte(struct amd_ir_data *data,
				       struct irq_cfg *irq_cfg,
				       struct irq_alloc_info *info,
				       int devid, int index, int sub_handle)
{
	struct irq_2_irte *irte_info = &data->irq_2_irte;
	struct msi_msg *msg = &data->msi_entry;
	union irte *irte = &data->irte_entry;
	struct IO_APIC_route_entry *entry;
3832

3833 3834
	data->irq_2_irte.devid = devid;
	data->irq_2_irte.index = index + sub_handle;
3835

3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857
	/* Setup IRTE for IOMMU */
	irte->val = 0;
	irte->fields.vector      = irq_cfg->vector;
	irte->fields.int_type    = apic->irq_delivery_mode;
	irte->fields.destination = irq_cfg->dest_apicid;
	irte->fields.dm          = apic->irq_dest_mode;
	irte->fields.valid       = 1;

	switch (info->type) {
	case X86_IRQ_ALLOC_TYPE_IOAPIC:
		/* Setup IOAPIC entry */
		entry = info->ioapic_entry;
		info->ioapic_entry = NULL;
		memset(entry, 0, sizeof(*entry));
		entry->vector        = index;
		entry->mask          = 0;
		entry->trigger       = info->ioapic_trigger;
		entry->polarity      = info->ioapic_polarity;
		/* Mask level triggered irqs. */
		if (info->ioapic_trigger)
			entry->mask = 1;
		break;
3858

3859 3860 3861 3862 3863 3864 3865
	case X86_IRQ_ALLOC_TYPE_HPET:
	case X86_IRQ_ALLOC_TYPE_MSI:
	case X86_IRQ_ALLOC_TYPE_MSIX:
		msg->address_hi = MSI_ADDR_BASE_HI;
		msg->address_lo = MSI_ADDR_BASE_LO;
		msg->data = irte_info->index;
		break;
3866

3867 3868 3869 3870
	default:
		BUG_ON(1);
		break;
	}
3871 3872
}

3873 3874
static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
			       unsigned int nr_irqs, void *arg)
3875
{
3876 3877 3878
	struct irq_alloc_info *info = arg;
	struct irq_data *irq_data;
	struct amd_ir_data *data;
3879
	struct irq_cfg *cfg;
3880 3881
	int i, ret, devid;
	int index = -1;
3882

3883 3884 3885 3886
	if (!info)
		return -EINVAL;
	if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_MSI &&
	    info->type != X86_IRQ_ALLOC_TYPE_MSIX)
3887 3888
		return -EINVAL;

3889 3890 3891 3892 3893 3894
	/*
	 * With IRQ remapping enabled, don't need contiguous CPU vectors
	 * to support multiple MSI interrupts.
	 */
	if (info->type == X86_IRQ_ALLOC_TYPE_MSI)
		info->flags &= ~X86_IRQ_ALLOC_CONTIGUOUS_VECTORS;
3895

3896 3897 3898
	devid = get_devid(info);
	if (devid < 0)
		return -EINVAL;
3899

3900 3901 3902
	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
	if (ret < 0)
		return ret;
3903

3904 3905 3906 3907 3908 3909
	if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) {
		if (get_irq_table(devid, true))
			index = info->ioapic_pin;
		else
			ret = -ENOMEM;
	} else {
3910
		index = alloc_irq_index(devid, nr_irqs);
3911 3912 3913 3914 3915
	}
	if (index < 0) {
		pr_warn("Failed to allocate IRTE\n");
		goto out_free_parent;
	}
3916

3917 3918 3919 3920 3921 3922 3923
	for (i = 0; i < nr_irqs; i++) {
		irq_data = irq_domain_get_irq_data(domain, virq + i);
		cfg = irqd_cfg(irq_data);
		if (!irq_data || !cfg) {
			ret = -EINVAL;
			goto out_free_data;
		}
3924

3925 3926 3927 3928 3929
		ret = -ENOMEM;
		data = kzalloc(sizeof(*data), GFP_KERNEL);
		if (!data)
			goto out_free_data;

3930 3931 3932 3933 3934 3935
		irq_data->hwirq = (devid << 16) + i;
		irq_data->chip_data = data;
		irq_data->chip = &amd_ir_chip;
		irq_remapping_prepare_irte(data, cfg, info, devid, index, i);
		irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT);
	}
3936

3937
	return 0;
3938

3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949
out_free_data:
	for (i--; i >= 0; i--) {
		irq_data = irq_domain_get_irq_data(domain, virq + i);
		if (irq_data)
			kfree(irq_data->chip_data);
	}
	for (i = 0; i < nr_irqs; i++)
		free_irte(devid, index + i);
out_free_parent:
	irq_domain_free_irqs_common(domain, virq, nr_irqs);
	return ret;
3950 3951
}

3952 3953
static void irq_remapping_free(struct irq_domain *domain, unsigned int virq,
			       unsigned int nr_irqs)
3954
{
3955 3956 3957 3958
	struct irq_2_irte *irte_info;
	struct irq_data *irq_data;
	struct amd_ir_data *data;
	int i;
3959

3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970
	for (i = 0; i < nr_irqs; i++) {
		irq_data = irq_domain_get_irq_data(domain, virq  + i);
		if (irq_data && irq_data->chip_data) {
			data = irq_data->chip_data;
			irte_info = &data->irq_2_irte;
			free_irte(irte_info->devid, irte_info->index);
			kfree(data);
		}
	}
	irq_domain_free_irqs_common(domain, virq, nr_irqs);
}
3971

3972 3973 3974 3975 3976
static void irq_remapping_activate(struct irq_domain *domain,
				   struct irq_data *irq_data)
{
	struct amd_ir_data *data = irq_data->chip_data;
	struct irq_2_irte *irte_info = &data->irq_2_irte;
3977

3978
	modify_irte(irte_info->devid, irte_info->index, data->irte_entry);
3979 3980
}

3981 3982
static void irq_remapping_deactivate(struct irq_domain *domain,
				     struct irq_data *irq_data)
3983
{
3984 3985 3986
	struct amd_ir_data *data = irq_data->chip_data;
	struct irq_2_irte *irte_info = &data->irq_2_irte;
	union irte entry;
3987

3988 3989 3990
	entry.val = 0;
	modify_irte(irte_info->devid, irte_info->index, data->irte_entry);
}
3991

3992 3993 3994 3995 3996
static struct irq_domain_ops amd_ir_domain_ops = {
	.alloc = irq_remapping_alloc,
	.free = irq_remapping_free,
	.activate = irq_remapping_activate,
	.deactivate = irq_remapping_deactivate,
3997
};
3998

3999 4000 4001 4002 4003 4004 4005 4006
static int amd_ir_set_affinity(struct irq_data *data,
			       const struct cpumask *mask, bool force)
{
	struct amd_ir_data *ir_data = data->chip_data;
	struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
	struct irq_cfg *cfg = irqd_cfg(data);
	struct irq_data *parent = data->parent_data;
	int ret;
4007

4008 4009 4010
	ret = parent->chip->irq_set_affinity(parent, mask, force);
	if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
		return ret;
4011

4012 4013 4014 4015 4016 4017 4018
	/*
	 * Atomically updates the IRTE with the new destination, vector
	 * and flushes the interrupt entry cache.
	 */
	ir_data->irte_entry.fields.vector = cfg->vector;
	ir_data->irte_entry.fields.destination = cfg->dest_apicid;
	modify_irte(irte_info->devid, irte_info->index, ir_data->irte_entry);
4019

4020 4021 4022 4023 4024
	/*
	 * After this point, all the interrupts will start arriving
	 * at the new destination. So, time to cleanup the previous
	 * vector allocation.
	 */
4025
	send_cleanup_vector(cfg);
4026 4027

	return IRQ_SET_MASK_OK_DONE;
4028 4029
}

4030
static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg)
4031
{
4032
	struct amd_ir_data *ir_data = irq_data->chip_data;
4033

4034 4035
	*msg = ir_data->msi_entry;
}
4036

4037 4038 4039 4040 4041
static struct irq_chip amd_ir_chip = {
	.irq_ack = ir_ack_apic_edge,
	.irq_set_affinity = amd_ir_set_affinity,
	.irq_compose_msi_msg = ir_compose_msi_msg,
};
4042

4043 4044 4045 4046 4047
int amd_iommu_create_irq_domain(struct amd_iommu *iommu)
{
	iommu->ir_domain = irq_domain_add_tree(NULL, &amd_ir_domain_ops, iommu);
	if (!iommu->ir_domain)
		return -ENOMEM;
4048

4049 4050
	iommu->ir_domain->parent = arch_get_ir_parent_domain();
	iommu->msi_domain = arch_create_msi_irq_domain(iommu->ir_domain);
4051 4052 4053

	return 0;
}
4054
#endif