amd_iommu.c 51.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
 * Author: Joerg Roedel <joerg.roedel@amd.com>
 *         Leo Duran <leo.duran@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

#include <linux/pci.h>
#include <linux/gfp.h>
#include <linux/bitops.h>
23
#include <linux/debugfs.h>
24
#include <linux/scatterlist.h>
25
#include <linux/dma-mapping.h>
26
#include <linux/iommu-helper.h>
27
#include <linux/iommu.h>
28
#include <asm/proto.h>
29
#include <asm/iommu.h>
30
#include <asm/gart.h>
31
#include <asm/amd_iommu_types.h>
32
#include <asm/amd_iommu.h>
33 34 35

#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))

36 37
#define EXIT_LOOP_COUNT 10000000

38 39
static DEFINE_RWLOCK(amd_iommu_devtable_lock);

40 41 42 43
/* A list of preallocated protection domains */
static LIST_HEAD(iommu_pd_list);
static DEFINE_SPINLOCK(iommu_pd_list_lock);

44 45 46 47
#ifdef CONFIG_IOMMU_API
static struct iommu_ops amd_iommu_ops;
#endif

48 49 50
/*
 * general struct to manage commands send to an IOMMU
 */
51
struct iommu_cmd {
52 53 54
	u32 data[4];
};

55 56
static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
			     struct unity_map_entry *e);
57
static struct dma_ops_domain *find_protection_domain(u16 devid);
58 59 60
static u64* alloc_pte(struct protection_domain *dom,
		      unsigned long address, u64
		      **pte_page, gfp_t gfp);
61 62 63
static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
				      unsigned long start_page,
				      unsigned int pages);
64

65 66 67 68
#ifndef BUS_NOTIFY_UNBOUND_DRIVER
#define BUS_NOTIFY_UNBOUND_DRIVER 0x0005
#endif

69 70 71 72 73 74
#ifdef CONFIG_AMD_IOMMU_STATS

/*
 * Initialization code for statistics collection
 */

75
DECLARE_STATS_COUNTER(compl_wait);
76
DECLARE_STATS_COUNTER(cnt_map_single);
77
DECLARE_STATS_COUNTER(cnt_unmap_single);
78
DECLARE_STATS_COUNTER(cnt_map_sg);
79
DECLARE_STATS_COUNTER(cnt_unmap_sg);
80
DECLARE_STATS_COUNTER(cnt_alloc_coherent);
81
DECLARE_STATS_COUNTER(cnt_free_coherent);
82
DECLARE_STATS_COUNTER(cross_page);
83
DECLARE_STATS_COUNTER(domain_flush_single);
84
DECLARE_STATS_COUNTER(domain_flush_all);
85
DECLARE_STATS_COUNTER(alloced_io_mem);
86
DECLARE_STATS_COUNTER(total_map_requests);
87

88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
static struct dentry *stats_dir;
static struct dentry *de_isolate;
static struct dentry *de_fflush;

static void amd_iommu_stats_add(struct __iommu_counter *cnt)
{
	if (stats_dir == NULL)
		return;

	cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
				       &cnt->value);
}

static void amd_iommu_stats_init(void)
{
	stats_dir = debugfs_create_dir("amd-iommu", NULL);
	if (stats_dir == NULL)
		return;

	de_isolate = debugfs_create_bool("isolation", 0444, stats_dir,
					 (u32 *)&amd_iommu_isolate);

	de_fflush  = debugfs_create_bool("fullflush", 0444, stats_dir,
					 (u32 *)&amd_iommu_unmap_flush);
112 113

	amd_iommu_stats_add(&compl_wait);
114
	amd_iommu_stats_add(&cnt_map_single);
115
	amd_iommu_stats_add(&cnt_unmap_single);
116
	amd_iommu_stats_add(&cnt_map_sg);
117
	amd_iommu_stats_add(&cnt_unmap_sg);
118
	amd_iommu_stats_add(&cnt_alloc_coherent);
119
	amd_iommu_stats_add(&cnt_free_coherent);
120
	amd_iommu_stats_add(&cross_page);
121
	amd_iommu_stats_add(&domain_flush_single);
122
	amd_iommu_stats_add(&domain_flush_all);
123
	amd_iommu_stats_add(&alloced_io_mem);
124
	amd_iommu_stats_add(&total_map_requests);
125 126 127 128
}

#endif

129
/* returns !0 if the IOMMU is caching non-present entries in its TLB */
130 131
static int iommu_has_npcache(struct amd_iommu *iommu)
{
132
	return iommu->cap & (1UL << IOMMU_CAP_NPCACHE);
133 134
}

135 136 137 138 139 140
/****************************************************************************
 *
 * Interrupt handling functions
 *
 ****************************************************************************/

141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
static void iommu_print_event(void *__evt)
{
	u32 *event = __evt;
	int type  = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
	int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
	int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
	int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
	u64 address = (u64)(((u64)event[3]) << 32) | event[2];

	printk(KERN_ERR "AMD IOMMU: Event logged [");

	switch (type) {
	case EVENT_TYPE_ILL_DEV:
		printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       address, flags);
		break;
	case EVENT_TYPE_IO_FAULT:
		printk("IO_PAGE_FAULT device=%02x:%02x.%x "
		       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       domid, address, flags);
		break;
	case EVENT_TYPE_DEV_TAB_ERR:
		printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       address, flags);
		break;
	case EVENT_TYPE_PAGE_TAB_ERR:
		printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
		       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       domid, address, flags);
		break;
	case EVENT_TYPE_ILL_CMD:
		printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
		break;
	case EVENT_TYPE_CMD_HARD_ERR:
		printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
		       "flags=0x%04x]\n", address, flags);
		break;
	case EVENT_TYPE_IOTLB_INV_TO:
		printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
		       "address=0x%016llx]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       address);
		break;
	case EVENT_TYPE_INV_DEV_REQ:
		printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
		       "address=0x%016llx flags=0x%04x]\n",
		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
		       address, flags);
		break;
	default:
		printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
	}
}

static void iommu_poll_events(struct amd_iommu *iommu)
{
	u32 head, tail;
	unsigned long flags;

	spin_lock_irqsave(&iommu->lock, flags);

	head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
	tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

	while (head != tail) {
		iommu_print_event(iommu->evt_buf + head);
		head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
	}

	writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

	spin_unlock_irqrestore(&iommu->lock, flags);
}

221 222
irqreturn_t amd_iommu_int_handler(int irq, void *data)
{
223 224
	struct amd_iommu *iommu;

225
	for_each_iommu(iommu)
226 227 228
		iommu_poll_events(iommu);

	return IRQ_HANDLED;
229 230
}

231 232 233 234 235 236 237 238 239 240
/****************************************************************************
 *
 * IOMMU command queuing functions
 *
 ****************************************************************************/

/*
 * Writes the command to the IOMMUs command buffer and informs the
 * hardware about the new command. Must be called with iommu->lock held.
 */
241
static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
242 243 244 245 246
{
	u32 tail, head;
	u8 *target;

	tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
247
	target = iommu->cmd_buf + tail;
248 249 250 251 252 253 254 255 256 257
	memcpy_toio(target, cmd, sizeof(*cmd));
	tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
	head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
	if (tail == head)
		return -ENOMEM;
	writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

	return 0;
}

258 259 260 261
/*
 * General queuing function for commands. Takes iommu->lock and calls
 * __iommu_queue_command().
 */
262
static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
263 264 265 266 267 268
{
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&iommu->lock, flags);
	ret = __iommu_queue_command(iommu, cmd);
269
	if (!ret)
270
		iommu->need_sync = true;
271 272 273 274 275
	spin_unlock_irqrestore(&iommu->lock, flags);

	return ret;
}

276 277 278 279 280 281 282 283 284 285
/*
 * This function waits until an IOMMU has completed a completion
 * wait command
 */
static void __iommu_wait_for_completion(struct amd_iommu *iommu)
{
	int ready = 0;
	unsigned status = 0;
	unsigned long i = 0;

286 287
	INC_STATS_COUNTER(compl_wait);

288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
	while (!ready && (i < EXIT_LOOP_COUNT)) {
		++i;
		/* wait for the bit to become one */
		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
		ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
	}

	/* set bit back to zero */
	status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
	writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);

	if (unlikely(i == EXIT_LOOP_COUNT))
		panic("AMD IOMMU: Completion wait loop failed\n");
}

/*
 * This function queues a completion wait command into the command
 * buffer of an IOMMU
 */
static int __iommu_completion_wait(struct amd_iommu *iommu)
{
	struct iommu_cmd cmd;

	 memset(&cmd, 0, sizeof(cmd));
	 cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
	 CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);

	 return __iommu_queue_command(iommu, &cmd);
}

318 319 320 321 322 323 324
/*
 * This function is called whenever we need to ensure that the IOMMU has
 * completed execution of all commands we sent. It sends a
 * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
 * us about that by writing a value to a physical address we pass with
 * the command.
 */
325 326
static int iommu_completion_wait(struct amd_iommu *iommu)
{
327 328
	int ret = 0;
	unsigned long flags;
329

330 331
	spin_lock_irqsave(&iommu->lock, flags);

332 333 334
	if (!iommu->need_sync)
		goto out;

335
	ret = __iommu_completion_wait(iommu);
336

337
	iommu->need_sync = false;
338 339

	if (ret)
340
		goto out;
341

342
	__iommu_wait_for_completion(iommu);
343

344 345
out:
	spin_unlock_irqrestore(&iommu->lock, flags);
346 347 348 349

	return 0;
}

350 351 352
/*
 * Command send function for invalidating a device table entry
 */
353 354
static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
{
355
	struct iommu_cmd cmd;
356
	int ret;
357 358 359 360 361 362 363

	BUG_ON(iommu == NULL);

	memset(&cmd, 0, sizeof(cmd));
	CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
	cmd.data[0] = devid;

364 365 366
	ret = iommu_queue_command(iommu, &cmd);

	return ret;
367 368
}

369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
					  u16 domid, int pde, int s)
{
	memset(cmd, 0, sizeof(*cmd));
	address &= PAGE_MASK;
	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
	cmd->data[1] |= domid;
	cmd->data[2] = lower_32_bits(address);
	cmd->data[3] = upper_32_bits(address);
	if (s) /* size bit - we flush more than one 4kb page */
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
	if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
		cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
}

384 385 386
/*
 * Generic command send function for invalidaing TLB entries
 */
387 388 389
static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
		u64 address, u16 domid, int pde, int s)
{
390
	struct iommu_cmd cmd;
391
	int ret;
392

393
	__iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
394

395 396 397
	ret = iommu_queue_command(iommu, &cmd);

	return ret;
398 399
}

400 401 402 403 404
/*
 * TLB invalidation function which is called from the mapping functions.
 * It invalidates a single PTE if the range to flush is within a single
 * page. Otherwise it flushes the whole TLB of the IOMMU.
 */
405 406 407
static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
		u64 address, size_t size)
{
408
	int s = 0;
409
	unsigned pages = iommu_num_pages(address, size, PAGE_SIZE);
410 411 412

	address &= PAGE_MASK;

413 414 415 416 417 418 419
	if (pages > 1) {
		/*
		 * If we have to flush more than one page, flush all
		 * TLB entries for this domain
		 */
		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
		s = 1;
420 421
	}

422 423
	iommu_queue_inv_iommu_pages(iommu, address, domid, 0, s);

424 425
	return 0;
}
426

427 428 429 430 431
/* Flush the whole IO/TLB for a given protection domain */
static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid)
{
	u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;

432 433
	INC_STATS_COUNTER(domain_flush_single);

434 435 436
	iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 1);
}

437 438 439 440 441 442 443 444 445 446
/* Flush the whole IO/TLB for a given protection domain - including PDE */
static void iommu_flush_tlb_pde(struct amd_iommu *iommu, u16 domid)
{
       u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;

       INC_STATS_COUNTER(domain_flush_single);

       iommu_queue_inv_iommu_pages(iommu, address, domid, 1, 1);
}

447 448 449 450 451 452 453 454 455 456
/*
 * This function is used to flush the IO/TLB for a given protection domain
 * on every IOMMU in the system
 */
static void iommu_flush_domain(u16 domid)
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct iommu_cmd cmd;

457 458
	INC_STATS_COUNTER(domain_flush_all);

459 460 461
	__iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
				      domid, 1, 1);

462
	for_each_iommu(iommu) {
463 464 465 466 467 468 469 470
		spin_lock_irqsave(&iommu->lock, flags);
		__iommu_queue_command(iommu, &cmd);
		__iommu_completion_wait(iommu);
		__iommu_wait_for_completion(iommu);
		spin_unlock_irqrestore(&iommu->lock, flags);
	}
}

471 472 473 474 475 476 477 478 479 480 481
void amd_iommu_flush_all_domains(void)
{
	int i;

	for (i = 1; i < MAX_DOMAIN_ID; ++i) {
		if (!test_bit(i, amd_iommu_pd_alloc_bitmap))
			continue;
		iommu_flush_domain(i);
	}
}

482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
void amd_iommu_flush_all_devices(void)
{
	struct amd_iommu *iommu;
	int i;

	for (i = 0; i <= amd_iommu_last_bdf; ++i) {
		if (amd_iommu_pd_table[i] == NULL)
			continue;

		iommu = amd_iommu_rlookup_table[i];
		if (!iommu)
			continue;

		iommu_queue_inv_dev_entry(iommu, i);
		iommu_completion_wait(iommu);
	}
}

500 501 502 503 504 505 506 507 508 509 510 511 512 513
/****************************************************************************
 *
 * The functions below are used the create the page table mappings for
 * unity mapped regions.
 *
 ****************************************************************************/

/*
 * Generic mapping functions. It maps a physical address into a DMA
 * address space. It allocates the page table pages if necessary.
 * In the future it can be extended to a generic mapping function
 * supporting all features of AMD IOMMU page tables like level skipping
 * and full 64 bit address spaces.
 */
514 515 516 517
static int iommu_map_page(struct protection_domain *dom,
			  unsigned long bus_addr,
			  unsigned long phys_addr,
			  int prot)
518
{
519
	u64 __pte, *pte;
520 521

	bus_addr  = PAGE_ALIGN(bus_addr);
522
	phys_addr = PAGE_ALIGN(phys_addr);
523 524 525 526 527

	/* only support 512GB address spaces for now */
	if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK))
		return -EINVAL;

528
	pte = alloc_pte(dom, bus_addr, NULL, GFP_KERNEL);
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543

	if (IOMMU_PTE_PRESENT(*pte))
		return -EBUSY;

	__pte = phys_addr | IOMMU_PTE_P;
	if (prot & IOMMU_PROT_IR)
		__pte |= IOMMU_PTE_IR;
	if (prot & IOMMU_PROT_IW)
		__pte |= IOMMU_PTE_IW;

	*pte = __pte;

	return 0;
}

544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
static void iommu_unmap_page(struct protection_domain *dom,
			     unsigned long bus_addr)
{
	u64 *pte;

	pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];

	if (!IOMMU_PTE_PRESENT(*pte))
		return;

	pte = IOMMU_PTE_PAGE(*pte);
	pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];

	if (!IOMMU_PTE_PRESENT(*pte))
		return;

	pte = IOMMU_PTE_PAGE(*pte);
	pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];

	*pte = 0;
}

566 567 568 569
/*
 * This function checks if a specific unity mapping entry is needed for
 * this specific IOMMU.
 */
570 571 572 573 574 575 576 577 578 579 580 581 582 583
static int iommu_for_unity_map(struct amd_iommu *iommu,
			       struct unity_map_entry *entry)
{
	u16 bdf, i;

	for (i = entry->devid_start; i <= entry->devid_end; ++i) {
		bdf = amd_iommu_alias_table[i];
		if (amd_iommu_rlookup_table[bdf] == iommu)
			return 1;
	}

	return 0;
}

584 585 586 587 588 589
/*
 * Init the unity mappings for a specific IOMMU in the system
 *
 * Basically iterates over all unity mapping entries and applies them to
 * the default domain DMA of that IOMMU if necessary.
 */
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
static int iommu_init_unity_mappings(struct amd_iommu *iommu)
{
	struct unity_map_entry *entry;
	int ret;

	list_for_each_entry(entry, &amd_iommu_unity_map, list) {
		if (!iommu_for_unity_map(iommu, entry))
			continue;
		ret = dma_ops_unity_map(iommu->default_dom, entry);
		if (ret)
			return ret;
	}

	return 0;
}

606 607 608 609
/*
 * This function actually applies the mapping to the page table of the
 * dma_ops domain.
 */
610 611 612 613 614 615 616 617
static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
			     struct unity_map_entry *e)
{
	u64 addr;
	int ret;

	for (addr = e->address_start; addr < e->address_end;
	     addr += PAGE_SIZE) {
618
		ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot);
619 620 621 622 623 624 625
		if (ret)
			return ret;
		/*
		 * if unity mapping is in aperture range mark the page
		 * as allocated in the aperture
		 */
		if (addr < dma_dom->aperture_size)
626
			__set_bit(addr >> PAGE_SHIFT,
627
				  dma_dom->aperture[0]->bitmap);
628 629 630 631 632
	}

	return 0;
}

633 634 635
/*
 * Inits the unity mappings required for a specific device
 */
636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652
static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
					  u16 devid)
{
	struct unity_map_entry *e;
	int ret;

	list_for_each_entry(e, &amd_iommu_unity_map, list) {
		if (!(devid >= e->devid_start && devid <= e->devid_end))
			continue;
		ret = dma_ops_unity_map(dma_dom, e);
		if (ret)
			return ret;
	}

	return 0;
}

653 654 655 656 657 658 659 660 661
/****************************************************************************
 *
 * The next functions belong to the address allocator for the dma_ops
 * interface functions. They work like the allocators in the other IOMMU
 * drivers. Its basically a bitmap which marks the allocated pages in
 * the aperture. Maybe it could be enhanced in the future to a more
 * efficient allocator.
 *
 ****************************************************************************/
662

663
/*
664
 * The address allocator core functions.
665 666 667
 *
 * called with domain->lock held
 */
668

669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
/*
 * This function checks if there is a PTE for a given dma address. If
 * there is one, it returns the pointer to it.
 */
static u64* fetch_pte(struct protection_domain *domain,
		      unsigned long address)
{
	u64 *pte;

	pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(address)];

	if (!IOMMU_PTE_PRESENT(*pte))
		return NULL;

	pte = IOMMU_PTE_PAGE(*pte);
	pte = &pte[IOMMU_PTE_L1_INDEX(address)];

	if (!IOMMU_PTE_PRESENT(*pte))
		return NULL;

	pte = IOMMU_PTE_PAGE(*pte);
	pte = &pte[IOMMU_PTE_L0_INDEX(address)];

	return pte;
}

695 696 697 698 699
/*
 * This function is used to add a new aperture range to an existing
 * aperture in case of dma_ops domain allocation or address allocation
 * failure.
 */
700 701
static int alloc_new_range(struct amd_iommu *iommu,
			   struct dma_ops_domain *dma_dom,
702 703 704
			   bool populate, gfp_t gfp)
{
	int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
705
	int i;
706

707 708 709 710
#ifdef CONFIG_IOMMU_STRESS
	populate = false;
#endif

711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
	if (index >= APERTURE_MAX_RANGES)
		return -ENOMEM;

	dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
	if (!dma_dom->aperture[index])
		return -ENOMEM;

	dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
	if (!dma_dom->aperture[index]->bitmap)
		goto out_free;

	dma_dom->aperture[index]->offset = dma_dom->aperture_size;

	if (populate) {
		unsigned long address = dma_dom->aperture_size;
		int i, num_ptes = APERTURE_RANGE_PAGES / 512;
		u64 *pte, *pte_page;

		for (i = 0; i < num_ptes; ++i) {
			pte = alloc_pte(&dma_dom->domain, address,
					&pte_page, gfp);
			if (!pte)
				goto out_free;

			dma_dom->aperture[index]->pte_pages[i] = pte_page;

			address += APERTURE_RANGE_SIZE / 64;
		}
	}

	dma_dom->aperture_size += APERTURE_RANGE_SIZE;

743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
	/* Intialize the exclusion range if necessary */
	if (iommu->exclusion_start &&
	    iommu->exclusion_start >= dma_dom->aperture[index]->offset &&
	    iommu->exclusion_start < dma_dom->aperture_size) {
		unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;
		int pages = iommu_num_pages(iommu->exclusion_start,
					    iommu->exclusion_length,
					    PAGE_SIZE);
		dma_ops_reserve_addresses(dma_dom, startpage, pages);
	}

	/*
	 * Check for areas already mapped as present in the new aperture
	 * range and mark those pages as reserved in the allocator. Such
	 * mappings may already exist as a result of requested unity
	 * mappings for devices.
	 */
	for (i = dma_dom->aperture[index]->offset;
	     i < dma_dom->aperture_size;
	     i += PAGE_SIZE) {
		u64 *pte = fetch_pte(&dma_dom->domain, i);
		if (!pte || !IOMMU_PTE_PRESENT(*pte))
			continue;

		dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1);
	}

770 771 772 773 774 775 776 777 778 779 780
	return 0;

out_free:
	free_page((unsigned long)dma_dom->aperture[index]->bitmap);

	kfree(dma_dom->aperture[index]);
	dma_dom->aperture[index] = NULL;

	return -ENOMEM;
}

781 782 783 784 785 786 787
static unsigned long dma_ops_area_alloc(struct device *dev,
					struct dma_ops_domain *dom,
					unsigned int pages,
					unsigned long align_mask,
					u64 dma_mask,
					unsigned long start)
{
788
	unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
789 790 791 792 793 794
	int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
	int i = start >> APERTURE_RANGE_SHIFT;
	unsigned long boundary_size;
	unsigned long address = -1;
	unsigned long limit;

795 796
	next_bit >>= PAGE_SHIFT;

797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
			PAGE_SIZE) >> PAGE_SHIFT;

	for (;i < max_index; ++i) {
		unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;

		if (dom->aperture[i]->offset >= dma_mask)
			break;

		limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
					       dma_mask >> PAGE_SHIFT);

		address = iommu_area_alloc(dom->aperture[i]->bitmap,
					   limit, next_bit, pages, 0,
					    boundary_size, align_mask);
		if (address != -1) {
			address = dom->aperture[i]->offset +
				  (address << PAGE_SHIFT);
815
			dom->next_address = address + (pages << PAGE_SHIFT);
816 817 818 819 820 821 822 823 824
			break;
		}

		next_bit = 0;
	}

	return address;
}

825 826
static unsigned long dma_ops_alloc_addresses(struct device *dev,
					     struct dma_ops_domain *dom,
827
					     unsigned int pages,
828 829
					     unsigned long align_mask,
					     u64 dma_mask)
830 831 832
{
	unsigned long address;

833 834 835 836
#ifdef CONFIG_IOMMU_STRESS
	dom->next_address = 0;
	dom->need_flush = true;
#endif
837

838
	address = dma_ops_area_alloc(dev, dom, pages, align_mask,
839
				     dma_mask, dom->next_address);
840

841
	if (address == -1) {
842
		dom->next_address = 0;
843 844
		address = dma_ops_area_alloc(dev, dom, pages, align_mask,
					     dma_mask, 0);
845 846
		dom->need_flush = true;
	}
847

848
	if (unlikely(address == -1))
849 850 851 852 853 854 855
		address = bad_dma_address;

	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);

	return address;
}

856 857 858 859 860
/*
 * The address free function.
 *
 * called with domain->lock held
 */
861 862 863 864
static void dma_ops_free_addresses(struct dma_ops_domain *dom,
				   unsigned long address,
				   unsigned int pages)
{
865 866
	unsigned i = address >> APERTURE_RANGE_SHIFT;
	struct aperture_range *range = dom->aperture[i];
867

868 869
	BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);

870 871 872 873
#ifdef CONFIG_IOMMU_STRESS
	if (i < 4)
		return;
#endif
874

875
	if (address >= dom->next_address)
876
		dom->need_flush = true;
877 878

	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
879

880 881
	iommu_area_free(range->bitmap, address, pages);

882 883
}

884 885 886 887 888 889 890 891 892 893
/****************************************************************************
 *
 * The next functions belong to the domain allocation. A domain is
 * allocated for every IOMMU as the default domain. If device isolation
 * is enabled, every device get its own domain. The most important thing
 * about domains is the page table mapping the DMA address space they
 * contain.
 *
 ****************************************************************************/

894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910
static u16 domain_id_alloc(void)
{
	unsigned long flags;
	int id;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
	BUG_ON(id == 0);
	if (id > 0 && id < MAX_DOMAIN_ID)
		__set_bit(id, amd_iommu_pd_alloc_bitmap);
	else
		id = 0;
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

	return id;
}

911 912 913 914 915 916 917 918 919 920
static void domain_id_free(int id)
{
	unsigned long flags;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	if (id > 0 && id < MAX_DOMAIN_ID)
		__clear_bit(id, amd_iommu_pd_alloc_bitmap);
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}

921 922 923 924
/*
 * Used to reserve address ranges in the aperture (e.g. for exclusion
 * ranges.
 */
925 926 927 928
static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
				      unsigned long start_page,
				      unsigned int pages)
{
929
	unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
930 931 932 933

	if (start_page + pages > last_page)
		pages = last_page - start_page;

934 935 936 937 938
	for (i = start_page; i < start_page + pages; ++i) {
		int index = i / APERTURE_RANGE_PAGES;
		int page  = i % APERTURE_RANGE_PAGES;
		__set_bit(page, dom->aperture[index]->bitmap);
	}
939 940
}

941
static void free_pagetable(struct protection_domain *domain)
942 943 944 945
{
	int i, j;
	u64 *p1, *p2, *p3;

946
	p1 = domain->pt_root;
947 948 949 950 951 952 953 954 955

	if (!p1)
		return;

	for (i = 0; i < 512; ++i) {
		if (!IOMMU_PTE_PRESENT(p1[i]))
			continue;

		p2 = IOMMU_PTE_PAGE(p1[i]);
956
		for (j = 0; j < 512; ++j) {
957 958 959 960 961 962 963 964 965 966
			if (!IOMMU_PTE_PRESENT(p2[j]))
				continue;
			p3 = IOMMU_PTE_PAGE(p2[j]);
			free_page((unsigned long)p3);
		}

		free_page((unsigned long)p2);
	}

	free_page((unsigned long)p1);
967 968

	domain->pt_root = NULL;
969 970
}

971 972 973 974
/*
 * Free a domain, only used if something went wrong in the
 * allocation path and we need to free an already allocated page table
 */
975 976
static void dma_ops_domain_free(struct dma_ops_domain *dom)
{
977 978
	int i;

979 980 981
	if (!dom)
		return;

982
	free_pagetable(&dom->domain);
983

984 985 986 987 988 989
	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
		if (!dom->aperture[i])
			continue;
		free_page((unsigned long)dom->aperture[i]->bitmap);
		kfree(dom->aperture[i]);
	}
990 991 992 993

	kfree(dom);
}

994 995 996 997 998
/*
 * Allocates a new protection domain usable for the dma_ops functions.
 * It also intializes the page table and the address allocator data
 * structures required for the dma_ops interface
 */
999
static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu)
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
{
	struct dma_ops_domain *dma_dom;

	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
	if (!dma_dom)
		return NULL;

	spin_lock_init(&dma_dom->domain.lock);

	dma_dom->domain.id = domain_id_alloc();
	if (dma_dom->domain.id == 0)
		goto free_dma_dom;
	dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
1014
	dma_dom->domain.flags = PD_DMA_OPS_MASK;
1015 1016 1017 1018
	dma_dom->domain.priv = dma_dom;
	if (!dma_dom->domain.pt_root)
		goto free_dma_dom;

1019
	dma_dom->need_flush = false;
1020
	dma_dom->target_dev = 0xffff;
1021

1022
	if (alloc_new_range(iommu, dma_dom, true, GFP_KERNEL))
1023 1024
		goto free_dma_dom;

1025
	/*
1026 1027
	 * mark the first page as allocated so we never return 0 as
	 * a valid dma-address. So we can use 0 as error value
1028
	 */
1029
	dma_dom->aperture[0]->bitmap[0] = 1;
1030
	dma_dom->next_address = 0;
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040


	return dma_dom;

free_dma_dom:
	dma_ops_domain_free(dma_dom);

	return NULL;
}

1041 1042 1043 1044 1045 1046 1047 1048 1049
/*
 * little helper function to check whether a given protection domain is a
 * dma_ops domain
 */
static bool dma_ops_domain(struct protection_domain *domain)
{
	return domain->flags & PD_DMA_OPS_MASK;
}

1050 1051 1052 1053
/*
 * Find out the protection domain structure for a given PCI device. This
 * will give us the pointer to the page table root for example.
 */
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
static struct protection_domain *domain_for_device(u16 devid)
{
	struct protection_domain *dom;
	unsigned long flags;

	read_lock_irqsave(&amd_iommu_devtable_lock, flags);
	dom = amd_iommu_pd_table[devid];
	read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

	return dom;
}

1066 1067 1068 1069
/*
 * If a device is not yet associated with a domain, this function does
 * assigns it visible for the hardware
 */
1070 1071 1072
static void attach_device(struct amd_iommu *iommu,
			  struct protection_domain *domain,
			  u16 devid)
1073 1074 1075 1076
{
	unsigned long flags;
	u64 pte_root = virt_to_phys(domain->pt_root);

1077 1078
	domain->dev_cnt += 1;

1079 1080 1081
	pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
		    << DEV_ENTRY_MODE_SHIFT;
	pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
1082 1083

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1084 1085
	amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
	amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
1086 1087 1088 1089 1090
	amd_iommu_dev_table[devid].data[2] = domain->id;

	amd_iommu_pd_table[devid] = domain;
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

1091 1092 1093 1094 1095
       /*
        * We might boot into a crash-kernel here. The crashed kernel
        * left the caches in the IOMMU dirty. So we have to flush
        * here to evict all dirty stuff.
        */
1096
	iommu_queue_inv_dev_entry(iommu, devid);
1097
	iommu_flush_tlb_pde(iommu, domain->id);
1098 1099
}

1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
/*
 * Removes a device from a protection domain (unlocked)
 */
static void __detach_device(struct protection_domain *domain, u16 devid)
{

	/* lock domain */
	spin_lock(&domain->lock);

	/* remove domain from the lookup table */
	amd_iommu_pd_table[devid] = NULL;

	/* remove entry from the device table seen by the hardware */
	amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
	amd_iommu_dev_table[devid].data[1] = 0;
	amd_iommu_dev_table[devid].data[2] = 0;

	/* decrease reference counter */
	domain->dev_cnt -= 1;

	/* ready */
	spin_unlock(&domain->lock);
}

/*
 * Removes a device from a protection domain (with devtable_lock held)
 */
static void detach_device(struct protection_domain *domain, u16 devid)
{
	unsigned long flags;

	/* lock device table */
	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	__detach_device(domain, devid);
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}
1136 1137 1138 1139 1140 1141 1142 1143 1144 1145

static int device_change_notifier(struct notifier_block *nb,
				  unsigned long action, void *data)
{
	struct device *dev = data;
	struct pci_dev *pdev = to_pci_dev(dev);
	u16 devid = calc_devid(pdev->bus->number, pdev->devfn);
	struct protection_domain *domain;
	struct dma_ops_domain *dma_domain;
	struct amd_iommu *iommu;
1146
	unsigned long flags;
1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163

	if (devid > amd_iommu_last_bdf)
		goto out;

	devid = amd_iommu_alias_table[devid];

	iommu = amd_iommu_rlookup_table[devid];
	if (iommu == NULL)
		goto out;

	domain = domain_for_device(devid);

	if (domain && !dma_ops_domain(domain))
		WARN_ONCE(1, "AMD IOMMU WARNING: device %s already bound "
			  "to a non-dma-ops domain\n", dev_name(dev));

	switch (action) {
1164
	case BUS_NOTIFY_UNBOUND_DRIVER:
1165 1166 1167
		if (!domain)
			goto out;
		detach_device(domain, devid);
1168 1169 1170 1171 1172 1173
		break;
	case BUS_NOTIFY_ADD_DEVICE:
		/* allocate a protection domain if a device is added */
		dma_domain = find_protection_domain(devid);
		if (dma_domain)
			goto out;
1174
		dma_domain = dma_ops_domain_alloc(iommu);
1175 1176 1177 1178 1179 1180 1181 1182
		if (!dma_domain)
			goto out;
		dma_domain->target_dev = devid;

		spin_lock_irqsave(&iommu_pd_list_lock, flags);
		list_add_tail(&dma_domain->list, &iommu_pd_list);
		spin_unlock_irqrestore(&iommu_pd_list_lock, flags);

1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
		break;
	default:
		goto out;
	}

	iommu_queue_inv_dev_entry(iommu, devid);
	iommu_completion_wait(iommu);

out:
	return 0;
}

struct notifier_block device_nb = {
	.notifier_call = device_change_notifier,
};
1198

1199 1200 1201 1202 1203 1204
/*****************************************************************************
 *
 * The next functions belong to the dma_ops mapping/unmapping code.
 *
 *****************************************************************************/

1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216
/*
 * This function checks if the driver got a valid device from the caller to
 * avoid dereferencing invalid pointers.
 */
static bool check_device(struct device *dev)
{
	if (!dev || !dev->dma_mask)
		return false;

	return true;
}

1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
/*
 * In this function the list of preallocated protection domains is traversed to
 * find the domain for a specific device
 */
static struct dma_ops_domain *find_protection_domain(u16 devid)
{
	struct dma_ops_domain *entry, *ret = NULL;
	unsigned long flags;

	if (list_empty(&iommu_pd_list))
		return NULL;

	spin_lock_irqsave(&iommu_pd_list_lock, flags);

	list_for_each_entry(entry, &iommu_pd_list, list) {
		if (entry->target_dev == devid) {
			ret = entry;
			break;
		}
	}

	spin_unlock_irqrestore(&iommu_pd_list_lock, flags);

	return ret;
}

1243 1244 1245 1246 1247 1248 1249
/*
 * In the dma_ops path we only have the struct device. This function
 * finds the corresponding IOMMU, the protection domain and the
 * requestor id for a given device.
 * If the device is not yet associated with a domain this is also done
 * in this function.
 */
1250 1251 1252 1253 1254 1255 1256 1257 1258
static int get_device_resources(struct device *dev,
				struct amd_iommu **iommu,
				struct protection_domain **domain,
				u16 *bdf)
{
	struct dma_ops_domain *dma_dom;
	struct pci_dev *pcidev;
	u16 _bdf;

1259 1260 1261 1262 1263 1264
	*iommu = NULL;
	*domain = NULL;
	*bdf = 0xffff;

	if (dev->bus != &pci_bus_type)
		return 0;
1265 1266

	pcidev = to_pci_dev(dev);
1267
	_bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
1268

1269
	/* device not translated by any IOMMU in the system? */
1270
	if (_bdf > amd_iommu_last_bdf)
1271 1272 1273 1274 1275 1276 1277 1278 1279
		return 0;

	*bdf = amd_iommu_alias_table[_bdf];

	*iommu = amd_iommu_rlookup_table[*bdf];
	if (*iommu == NULL)
		return 0;
	*domain = domain_for_device(*bdf);
	if (*domain == NULL) {
1280 1281 1282
		dma_dom = find_protection_domain(*bdf);
		if (!dma_dom)
			dma_dom = (*iommu)->default_dom;
1283
		*domain = &dma_dom->domain;
1284
		attach_device(*iommu, *domain, *bdf);
1285 1286
		DUMP_printk("Using protection domain %d for device %s\n",
			    (*domain)->id, dev_name(dev));
1287 1288
	}

1289
	if (domain_for_device(_bdf) == NULL)
1290
		attach_device(*iommu, *domain, _bdf);
1291

1292 1293 1294
	return 1;
}

1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
/*
 * If the pte_page is not yet allocated this function is called
 */
static u64* alloc_pte(struct protection_domain *dom,
		      unsigned long address, u64 **pte_page, gfp_t gfp)
{
	u64 *pte, *page;

	pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(address)];

	if (!IOMMU_PTE_PRESENT(*pte)) {
		page = (u64 *)get_zeroed_page(gfp);
		if (!page)
			return NULL;
		*pte = IOMMU_L2_PDE(virt_to_phys(page));
	}

	pte = IOMMU_PTE_PAGE(*pte);
	pte = &pte[IOMMU_PTE_L1_INDEX(address)];

	if (!IOMMU_PTE_PRESENT(*pte)) {
		page = (u64 *)get_zeroed_page(gfp);
		if (!page)
			return NULL;
		*pte = IOMMU_L1_PDE(virt_to_phys(page));
	}

	pte = IOMMU_PTE_PAGE(*pte);

	if (pte_page)
		*pte_page = pte;

	pte = &pte[IOMMU_PTE_L0_INDEX(address)];

	return pte;
}

/*
 * This function fetches the PTE for a given address in the aperture
 */
static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
			    unsigned long address)
{
1338
	struct aperture_range *aperture;
1339 1340
	u64 *pte, *pte_page;

1341 1342 1343 1344 1345
	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
	if (!aperture)
		return NULL;

	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
1346 1347
	if (!pte) {
		pte = alloc_pte(&dom->domain, address, &pte_page, GFP_ATOMIC);
1348 1349 1350
		aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
	} else
		pte += IOMMU_PTE_L0_INDEX(address);
1351 1352 1353 1354

	return pte;
}

1355 1356 1357 1358
/*
 * This is the generic map function. It maps one 4kb page at paddr to
 * the given address in the DMA address space for the domain.
 */
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
static dma_addr_t dma_ops_domain_map(struct amd_iommu *iommu,
				     struct dma_ops_domain *dom,
				     unsigned long address,
				     phys_addr_t paddr,
				     int direction)
{
	u64 *pte, __pte;

	WARN_ON(address > dom->aperture_size);

	paddr &= PAGE_MASK;

1371
	pte  = dma_ops_get_pte(dom, address);
1372 1373
	if (!pte)
		return bad_dma_address;
1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390

	__pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;

	if (direction == DMA_TO_DEVICE)
		__pte |= IOMMU_PTE_IR;
	else if (direction == DMA_FROM_DEVICE)
		__pte |= IOMMU_PTE_IW;
	else if (direction == DMA_BIDIRECTIONAL)
		__pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;

	WARN_ON(*pte);

	*pte = __pte;

	return (dma_addr_t)address;
}

1391 1392 1393
/*
 * The generic unmapping function for on page in the DMA address space.
 */
1394 1395 1396 1397
static void dma_ops_domain_unmap(struct amd_iommu *iommu,
				 struct dma_ops_domain *dom,
				 unsigned long address)
{
1398
	struct aperture_range *aperture;
1399 1400 1401 1402 1403
	u64 *pte;

	if (address >= dom->aperture_size)
		return;

1404 1405 1406 1407 1408 1409 1410
	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
	if (!aperture)
		return;

	pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
	if (!pte)
		return;
1411 1412 1413 1414 1415 1416 1417 1418

	pte += IOMMU_PTE_L0_INDEX(address);

	WARN_ON(!*pte);

	*pte = 0ULL;
}

1419 1420
/*
 * This function contains common code for mapping of a physically
J
Joerg Roedel 已提交
1421 1422
 * contiguous memory region into DMA address space. It is used by all
 * mapping functions provided with this IOMMU driver.
1423 1424
 * Must be called with the domain lock held.
 */
1425 1426 1427 1428 1429
static dma_addr_t __map_single(struct device *dev,
			       struct amd_iommu *iommu,
			       struct dma_ops_domain *dma_dom,
			       phys_addr_t paddr,
			       size_t size,
1430
			       int dir,
1431 1432
			       bool align,
			       u64 dma_mask)
1433 1434
{
	dma_addr_t offset = paddr & ~PAGE_MASK;
1435
	dma_addr_t address, start, ret;
1436
	unsigned int pages;
1437
	unsigned long align_mask = 0;
1438 1439
	int i;

1440
	pages = iommu_num_pages(paddr, size, PAGE_SIZE);
1441 1442
	paddr &= PAGE_MASK;

1443 1444
	INC_STATS_COUNTER(total_map_requests);

1445 1446 1447
	if (pages > 1)
		INC_STATS_COUNTER(cross_page);

1448 1449 1450
	if (align)
		align_mask = (1UL << get_order(size)) - 1;

1451
retry:
1452 1453
	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
					  dma_mask);
1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
	if (unlikely(address == bad_dma_address)) {
		/*
		 * setting next_address here will let the address
		 * allocator only scan the new allocated range in the
		 * first run. This is a small optimization.
		 */
		dma_dom->next_address = dma_dom->aperture_size;

		if (alloc_new_range(iommu, dma_dom, false, GFP_ATOMIC))
			goto out;

		/*
		 * aperture was sucessfully enlarged by 128 MB, try
		 * allocation again
		 */
		goto retry;
	}
1471 1472 1473

	start = address;
	for (i = 0; i < pages; ++i) {
1474 1475 1476 1477
		ret = dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);
		if (ret == bad_dma_address)
			goto out_unmap;

1478 1479 1480 1481 1482
		paddr += PAGE_SIZE;
		start += PAGE_SIZE;
	}
	address += offset;

1483 1484
	ADD_STATS_COUNTER(alloced_io_mem, size);

1485
	if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
1486 1487 1488
		iommu_flush_tlb(iommu, dma_dom->domain.id);
		dma_dom->need_flush = false;
	} else if (unlikely(iommu_has_npcache(iommu)))
1489 1490
		iommu_flush_pages(iommu, dma_dom->domain.id, address, size);

1491 1492
out:
	return address;
1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503

out_unmap:

	for (--i; i >= 0; --i) {
		start -= PAGE_SIZE;
		dma_ops_domain_unmap(iommu, dma_dom, start);
	}

	dma_ops_free_addresses(dma_dom, address, pages);

	return bad_dma_address;
1504 1505
}

1506 1507 1508 1509
/*
 * Does the reverse of the __map_single function. Must be called with
 * the domain lock held too
 */
1510 1511 1512 1513 1514 1515 1516 1517 1518
static void __unmap_single(struct amd_iommu *iommu,
			   struct dma_ops_domain *dma_dom,
			   dma_addr_t dma_addr,
			   size_t size,
			   int dir)
{
	dma_addr_t i, start;
	unsigned int pages;

1519 1520
	if ((dma_addr == bad_dma_address) ||
	    (dma_addr + size > dma_dom->aperture_size))
1521 1522
		return;

1523
	pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
1524 1525 1526 1527 1528 1529 1530 1531
	dma_addr &= PAGE_MASK;
	start = dma_addr;

	for (i = 0; i < pages; ++i) {
		dma_ops_domain_unmap(iommu, dma_dom, start);
		start += PAGE_SIZE;
	}

1532 1533
	SUB_STATS_COUNTER(alloced_io_mem, size);

1534
	dma_ops_free_addresses(dma_dom, dma_addr, pages);
1535

1536
	if (amd_iommu_unmap_flush || dma_dom->need_flush) {
1537
		iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size);
1538 1539
		dma_dom->need_flush = false;
	}
1540 1541
}

1542 1543 1544
/*
 * The exported map_single function for dma_ops.
 */
1545 1546 1547 1548
static dma_addr_t map_page(struct device *dev, struct page *page,
			   unsigned long offset, size_t size,
			   enum dma_data_direction dir,
			   struct dma_attrs *attrs)
1549 1550 1551 1552 1553 1554
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	u16 devid;
	dma_addr_t addr;
1555
	u64 dma_mask;
1556
	phys_addr_t paddr = page_to_phys(page) + offset;
1557

1558 1559
	INC_STATS_COUNTER(cnt_map_single);

1560 1561 1562
	if (!check_device(dev))
		return bad_dma_address;

1563
	dma_mask = *dev->dma_mask;
1564 1565 1566 1567

	get_device_resources(dev, &iommu, &domain, &devid);

	if (iommu == NULL || domain == NULL)
1568
		/* device not handled by any AMD IOMMU */
1569 1570
		return (dma_addr_t)paddr;

1571 1572 1573
	if (!dma_ops_domain(domain))
		return bad_dma_address;

1574
	spin_lock_irqsave(&domain->lock, flags);
1575 1576
	addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false,
			    dma_mask);
1577 1578 1579
	if (addr == bad_dma_address)
		goto out;

1580
	iommu_completion_wait(iommu);
1581 1582 1583 1584 1585 1586 1587

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return addr;
}

1588 1589 1590
/*
 * The exported unmap_single function for dma_ops.
 */
1591 1592
static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
		       enum dma_data_direction dir, struct dma_attrs *attrs)
1593 1594 1595 1596 1597 1598
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	u16 devid;

1599 1600
	INC_STATS_COUNTER(cnt_unmap_single);

1601 1602
	if (!check_device(dev) ||
	    !get_device_resources(dev, &iommu, &domain, &devid))
1603
		/* device not handled by any AMD IOMMU */
1604 1605
		return;

1606 1607 1608
	if (!dma_ops_domain(domain))
		return;

1609 1610 1611 1612
	spin_lock_irqsave(&domain->lock, flags);

	__unmap_single(iommu, domain->priv, dma_addr, size, dir);

1613
	iommu_completion_wait(iommu);
1614 1615 1616 1617

	spin_unlock_irqrestore(&domain->lock, flags);
}

1618 1619 1620 1621
/*
 * This is a special map_sg function which is used if we should map a
 * device which is not handled by an AMD IOMMU in the system.
 */
1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635
static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
			   int nelems, int dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sglist, s, nelems, i) {
		s->dma_address = (dma_addr_t)sg_phys(s);
		s->dma_length  = s->length;
	}

	return nelems;
}

1636 1637 1638 1639
/*
 * The exported map_sg function for dma_ops (handles scatter-gather
 * lists).
 */
1640
static int map_sg(struct device *dev, struct scatterlist *sglist,
1641 1642
		  int nelems, enum dma_data_direction dir,
		  struct dma_attrs *attrs)
1643 1644 1645 1646 1647 1648 1649 1650 1651
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	u16 devid;
	int i;
	struct scatterlist *s;
	phys_addr_t paddr;
	int mapped_elems = 0;
1652
	u64 dma_mask;
1653

1654 1655
	INC_STATS_COUNTER(cnt_map_sg);

1656 1657 1658
	if (!check_device(dev))
		return 0;

1659
	dma_mask = *dev->dma_mask;
1660 1661 1662 1663 1664 1665

	get_device_resources(dev, &iommu, &domain, &devid);

	if (!iommu || !domain)
		return map_sg_no_iommu(dev, sglist, nelems, dir);

1666 1667 1668
	if (!dma_ops_domain(domain))
		return 0;

1669 1670 1671 1672 1673 1674
	spin_lock_irqsave(&domain->lock, flags);

	for_each_sg(sglist, s, nelems, i) {
		paddr = sg_phys(s);

		s->dma_address = __map_single(dev, iommu, domain->priv,
1675 1676
					      paddr, s->length, dir, false,
					      dma_mask);
1677 1678 1679 1680 1681 1682 1683 1684

		if (s->dma_address) {
			s->dma_length = s->length;
			mapped_elems++;
		} else
			goto unmap;
	}

1685
	iommu_completion_wait(iommu);
1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return mapped_elems;
unmap:
	for_each_sg(sglist, s, mapped_elems, i) {
		if (s->dma_address)
			__unmap_single(iommu, domain->priv, s->dma_address,
				       s->dma_length, dir);
		s->dma_address = s->dma_length = 0;
	}

	mapped_elems = 0;

	goto out;
}

1704 1705 1706 1707
/*
 * The exported map_sg function for dma_ops (handles scatter-gather
 * lists).
 */
1708
static void unmap_sg(struct device *dev, struct scatterlist *sglist,
1709 1710
		     int nelems, enum dma_data_direction dir,
		     struct dma_attrs *attrs)
1711 1712 1713 1714 1715 1716 1717 1718
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	struct scatterlist *s;
	u16 devid;
	int i;

1719 1720
	INC_STATS_COUNTER(cnt_unmap_sg);

1721 1722
	if (!check_device(dev) ||
	    !get_device_resources(dev, &iommu, &domain, &devid))
1723 1724
		return;

1725 1726 1727
	if (!dma_ops_domain(domain))
		return;

1728 1729 1730 1731 1732 1733 1734 1735
	spin_lock_irqsave(&domain->lock, flags);

	for_each_sg(sglist, s, nelems, i) {
		__unmap_single(iommu, domain->priv, s->dma_address,
			       s->dma_length, dir);
		s->dma_address = s->dma_length = 0;
	}

1736
	iommu_completion_wait(iommu);
1737 1738 1739 1740

	spin_unlock_irqrestore(&domain->lock, flags);
}

1741 1742 1743
/*
 * The exported alloc_coherent function for dma_ops.
 */
1744 1745 1746 1747 1748 1749 1750 1751 1752
static void *alloc_coherent(struct device *dev, size_t size,
			    dma_addr_t *dma_addr, gfp_t flag)
{
	unsigned long flags;
	void *virt_addr;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	u16 devid;
	phys_addr_t paddr;
1753
	u64 dma_mask = dev->coherent_dma_mask;
1754

1755 1756
	INC_STATS_COUNTER(cnt_alloc_coherent);

1757 1758
	if (!check_device(dev))
		return NULL;
1759

1760 1761
	if (!get_device_resources(dev, &iommu, &domain, &devid))
		flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
1762

1763
	flag |= __GFP_ZERO;
1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
	virt_addr = (void *)__get_free_pages(flag, get_order(size));
	if (!virt_addr)
		return 0;

	paddr = virt_to_phys(virt_addr);

	if (!iommu || !domain) {
		*dma_addr = (dma_addr_t)paddr;
		return virt_addr;
	}

1775 1776 1777
	if (!dma_ops_domain(domain))
		goto out_free;

1778 1779 1780
	if (!dma_mask)
		dma_mask = *dev->dma_mask;

1781 1782 1783
	spin_lock_irqsave(&domain->lock, flags);

	*dma_addr = __map_single(dev, iommu, domain->priv, paddr,
1784
				 size, DMA_BIDIRECTIONAL, true, dma_mask);
1785

J
Jiri Slaby 已提交
1786 1787
	if (*dma_addr == bad_dma_address) {
		spin_unlock_irqrestore(&domain->lock, flags);
1788
		goto out_free;
J
Jiri Slaby 已提交
1789
	}
1790

1791
	iommu_completion_wait(iommu);
1792 1793 1794 1795

	spin_unlock_irqrestore(&domain->lock, flags);

	return virt_addr;
1796 1797 1798 1799 1800 1801

out_free:

	free_pages((unsigned long)virt_addr, get_order(size));

	return NULL;
1802 1803
}

1804 1805 1806
/*
 * The exported free_coherent function for dma_ops.
 */
1807 1808 1809 1810 1811 1812 1813 1814
static void free_coherent(struct device *dev, size_t size,
			  void *virt_addr, dma_addr_t dma_addr)
{
	unsigned long flags;
	struct amd_iommu *iommu;
	struct protection_domain *domain;
	u16 devid;

1815 1816
	INC_STATS_COUNTER(cnt_free_coherent);

1817 1818 1819
	if (!check_device(dev))
		return;

1820 1821 1822 1823 1824
	get_device_resources(dev, &iommu, &domain, &devid);

	if (!iommu || !domain)
		goto free_mem;

1825 1826 1827
	if (!dma_ops_domain(domain))
		goto free_mem;

1828 1829 1830 1831
	spin_lock_irqsave(&domain->lock, flags);

	__unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);

1832
	iommu_completion_wait(iommu);
1833 1834 1835 1836 1837 1838 1839

	spin_unlock_irqrestore(&domain->lock, flags);

free_mem:
	free_pages((unsigned long)virt_addr, get_order(size));
}

1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863
/*
 * This function is called by the DMA layer to find out if we can handle a
 * particular device. It is part of the dma_ops.
 */
static int amd_iommu_dma_supported(struct device *dev, u64 mask)
{
	u16 bdf;
	struct pci_dev *pcidev;

	/* No device or no PCI device */
	if (!dev || dev->bus != &pci_bus_type)
		return 0;

	pcidev = to_pci_dev(dev);

	bdf = calc_devid(pcidev->bus->number, pcidev->devfn);

	/* Out of our scope? */
	if (bdf > amd_iommu_last_bdf)
		return 0;

	return 1;
}

1864
/*
1865 1866
 * The function for pre-allocating protection domains.
 *
1867 1868 1869 1870
 * If the driver core informs the DMA layer if a driver grabs a device
 * we don't need to preallocate the protection domains anymore.
 * For now we have to.
 */
1871
static void prealloc_protection_domains(void)
1872 1873 1874 1875 1876 1877 1878
{
	struct pci_dev *dev = NULL;
	struct dma_ops_domain *dma_dom;
	struct amd_iommu *iommu;
	u16 devid;

	while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
1879
		devid = calc_devid(dev->bus->number, dev->devfn);
1880
		if (devid > amd_iommu_last_bdf)
1881 1882 1883 1884 1885 1886 1887
			continue;
		devid = amd_iommu_alias_table[devid];
		if (domain_for_device(devid))
			continue;
		iommu = amd_iommu_rlookup_table[devid];
		if (!iommu)
			continue;
1888
		dma_dom = dma_ops_domain_alloc(iommu);
1889 1890 1891
		if (!dma_dom)
			continue;
		init_unity_mappings_for_device(dma_dom, devid);
1892 1893 1894
		dma_dom->target_dev = devid;

		list_add_tail(&dma_dom->list, &iommu_pd_list);
1895 1896 1897
	}
}

1898
static struct dma_map_ops amd_iommu_dma_ops = {
1899 1900
	.alloc_coherent = alloc_coherent,
	.free_coherent = free_coherent,
1901 1902
	.map_page = map_page,
	.unmap_page = unmap_page,
1903 1904
	.map_sg = map_sg,
	.unmap_sg = unmap_sg,
1905
	.dma_supported = amd_iommu_dma_supported,
1906 1907
};

1908 1909 1910
/*
 * The function which clues the AMD IOMMU driver into dma_ops.
 */
1911 1912 1913 1914 1915
int __init amd_iommu_init_dma_ops(void)
{
	struct amd_iommu *iommu;
	int ret;

1916 1917 1918 1919 1920
	/*
	 * first allocate a default protection domain for every IOMMU we
	 * found in the system. Devices not assigned to any other
	 * protection domain will be assigned to the default one.
	 */
1921
	for_each_iommu(iommu) {
1922
		iommu->default_dom = dma_ops_domain_alloc(iommu);
1923 1924
		if (iommu->default_dom == NULL)
			return -ENOMEM;
1925
		iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
1926 1927 1928 1929 1930
		ret = iommu_init_unity_mappings(iommu);
		if (ret)
			goto free_domains;
	}

1931 1932 1933 1934
	/*
	 * If device isolation is enabled, pre-allocate the protection
	 * domains for each device.
	 */
1935 1936 1937 1938 1939 1940
	if (amd_iommu_isolate)
		prealloc_protection_domains();

	iommu_detected = 1;
	force_iommu = 1;
	bad_dma_address = 0;
I
Ingo Molnar 已提交
1941
#ifdef CONFIG_GART_IOMMU
1942 1943
	gart_iommu_aperture_disabled = 1;
	gart_iommu_aperture = 0;
I
Ingo Molnar 已提交
1944
#endif
1945

1946
	/* Make the driver finally visible to the drivers */
1947 1948
	dma_ops = &amd_iommu_dma_ops;

1949 1950
	register_iommu(&amd_iommu_ops);

1951 1952
	bus_register_notifier(&pci_bus_type, &device_nb);

1953 1954
	amd_iommu_stats_init();

1955 1956 1957 1958
	return 0;

free_domains:

1959
	for_each_iommu(iommu) {
1960 1961 1962 1963 1964 1965
		if (iommu->default_dom)
			dma_ops_domain_free(iommu->default_dom);
	}

	return ret;
}
1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990

/*****************************************************************************
 *
 * The following functions belong to the exported interface of AMD IOMMU
 *
 * This interface allows access to lower level functions of the IOMMU
 * like protection domain handling and assignement of devices to domains
 * which is not possible with the dma_ops interface.
 *
 *****************************************************************************/

static void cleanup_domain(struct protection_domain *domain)
{
	unsigned long flags;
	u16 devid;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);

	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
		if (amd_iommu_pd_table[devid] == domain)
			__detach_device(domain, devid);

	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}

1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017
static int amd_iommu_domain_init(struct iommu_domain *dom)
{
	struct protection_domain *domain;

	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
	if (!domain)
		return -ENOMEM;

	spin_lock_init(&domain->lock);
	domain->mode = PAGE_MODE_3_LEVEL;
	domain->id = domain_id_alloc();
	if (!domain->id)
		goto out_free;
	domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
	if (!domain->pt_root)
		goto out_free;

	dom->priv = domain;

	return 0;

out_free:
	kfree(domain);

	return -ENOMEM;
}

2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038
static void amd_iommu_domain_destroy(struct iommu_domain *dom)
{
	struct protection_domain *domain = dom->priv;

	if (!domain)
		return;

	if (domain->dev_cnt > 0)
		cleanup_domain(domain);

	BUG_ON(domain->dev_cnt != 0);

	free_pagetable(domain);

	domain_id_free(domain->id);

	kfree(domain);

	dom->priv = NULL;
}

2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064
static void amd_iommu_detach_device(struct iommu_domain *dom,
				    struct device *dev)
{
	struct protection_domain *domain = dom->priv;
	struct amd_iommu *iommu;
	struct pci_dev *pdev;
	u16 devid;

	if (dev->bus != &pci_bus_type)
		return;

	pdev = to_pci_dev(dev);

	devid = calc_devid(pdev->bus->number, pdev->devfn);

	if (devid > 0)
		detach_device(domain, devid);

	iommu = amd_iommu_rlookup_table[devid];
	if (!iommu)
		return;

	iommu_queue_inv_dev_entry(iommu, devid);
	iommu_completion_wait(iommu);
}

2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090
static int amd_iommu_attach_device(struct iommu_domain *dom,
				   struct device *dev)
{
	struct protection_domain *domain = dom->priv;
	struct protection_domain *old_domain;
	struct amd_iommu *iommu;
	struct pci_dev *pdev;
	u16 devid;

	if (dev->bus != &pci_bus_type)
		return -EINVAL;

	pdev = to_pci_dev(dev);

	devid = calc_devid(pdev->bus->number, pdev->devfn);

	if (devid >= amd_iommu_last_bdf ||
			devid != amd_iommu_alias_table[devid])
		return -EINVAL;

	iommu = amd_iommu_rlookup_table[devid];
	if (!iommu)
		return -EINVAL;

	old_domain = domain_for_device(devid);
	if (old_domain)
2091
		detach_device(old_domain, devid);
2092 2093 2094 2095 2096 2097 2098 2099

	attach_device(iommu, domain, devid);

	iommu_completion_wait(iommu);

	return 0;
}

2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
static int amd_iommu_map_range(struct iommu_domain *dom,
			       unsigned long iova, phys_addr_t paddr,
			       size_t size, int iommu_prot)
{
	struct protection_domain *domain = dom->priv;
	unsigned long i,  npages = iommu_num_pages(paddr, size, PAGE_SIZE);
	int prot = 0;
	int ret;

	if (iommu_prot & IOMMU_READ)
		prot |= IOMMU_PROT_IR;
	if (iommu_prot & IOMMU_WRITE)
		prot |= IOMMU_PROT_IW;

	iova  &= PAGE_MASK;
	paddr &= PAGE_MASK;

	for (i = 0; i < npages; ++i) {
		ret = iommu_map_page(domain, iova, paddr, prot);
		if (ret)
			return ret;

		iova  += PAGE_SIZE;
		paddr += PAGE_SIZE;
	}

	return 0;
}

2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145
static void amd_iommu_unmap_range(struct iommu_domain *dom,
				  unsigned long iova, size_t size)
{

	struct protection_domain *domain = dom->priv;
	unsigned long i,  npages = iommu_num_pages(iova, size, PAGE_SIZE);

	iova  &= PAGE_MASK;

	for (i = 0; i < npages; ++i) {
		iommu_unmap_page(domain, iova);
		iova  += PAGE_SIZE;
	}

	iommu_flush_domain(domain->id);
}

2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176
static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
					  unsigned long iova)
{
	struct protection_domain *domain = dom->priv;
	unsigned long offset = iova & ~PAGE_MASK;
	phys_addr_t paddr;
	u64 *pte;

	pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(iova)];

	if (!IOMMU_PTE_PRESENT(*pte))
		return 0;

	pte = IOMMU_PTE_PAGE(*pte);
	pte = &pte[IOMMU_PTE_L1_INDEX(iova)];

	if (!IOMMU_PTE_PRESENT(*pte))
		return 0;

	pte = IOMMU_PTE_PAGE(*pte);
	pte = &pte[IOMMU_PTE_L0_INDEX(iova)];

	if (!IOMMU_PTE_PRESENT(*pte))
		return 0;

	paddr  = *pte & IOMMU_PAGE_MASK;
	paddr |= offset;

	return paddr;
}

S
Sheng Yang 已提交
2177 2178 2179 2180 2181 2182
static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
				    unsigned long cap)
{
	return 0;
}

2183 2184 2185 2186 2187 2188 2189 2190
static struct iommu_ops amd_iommu_ops = {
	.domain_init = amd_iommu_domain_init,
	.domain_destroy = amd_iommu_domain_destroy,
	.attach_dev = amd_iommu_attach_device,
	.detach_dev = amd_iommu_detach_device,
	.map = amd_iommu_map_range,
	.unmap = amd_iommu_unmap_range,
	.iova_to_phys = amd_iommu_iova_to_phys,
S
Sheng Yang 已提交
2191
	.domain_has_cap = amd_iommu_domain_has_cap,
2192 2193
};