intel-iommu.c 99.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * Copyright (c) 2006, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
 * Place - Suite 330, Boston, MA 02111-1307 USA.
 *
17 18 19 20
 * Copyright (C) 2006-2008 Intel Corporation
 * Author: Ashok Raj <ashok.raj@intel.com>
 * Author: Shaohua Li <shaohua.li@intel.com>
 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
F
Fenghua Yu 已提交
21
 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 23 24 25
 */

#include <linux/init.h>
#include <linux/bitmap.h>
M
mark gross 已提交
26
#include <linux/debugfs.h>
27 28 29 30 31 32 33 34
#include <linux/slab.h>
#include <linux/irq.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/pci.h>
#include <linux/dmar.h>
#include <linux/dma-mapping.h>
#include <linux/mempool.h>
M
mark gross 已提交
35
#include <linux/timer.h>
K
Kay, Allen M 已提交
36
#include <linux/iova.h>
37
#include <linux/iommu.h>
K
Kay, Allen M 已提交
38
#include <linux/intel-iommu.h>
39
#include <linux/syscore_ops.h>
40
#include <linux/tboot.h>
41
#include <linux/dmi.h>
42
#include <linux/pci-ats.h>
43
#include <asm/cacheflush.h>
44
#include <asm/iommu.h>
45

F
Fenghua Yu 已提交
46 47 48
#define ROOT_SIZE		VTD_PAGE_SIZE
#define CONTEXT_SIZE		VTD_PAGE_SIZE

49 50
#define IS_BRIDGE_HOST_DEVICE(pdev) \
			    ((pdev->class >> 8) == PCI_CLASS_BRIDGE_HOST)
51 52
#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
53
#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
54 55 56 57 58 59 60

#define IOAPIC_RANGE_START	(0xfee00000)
#define IOAPIC_RANGE_END	(0xfeefffff)
#define IOVA_START_ADDR		(0x1000)

#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48

F
Fenghua Yu 已提交
61 62
#define MAX_AGAW_WIDTH 64

63 64 65 66 67 68 69 70
#define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)

/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
   to match. That way, we can use 'unsigned long' for PFNs with impunity. */
#define DOMAIN_MAX_PFN(gaw)	((unsigned long) min_t(uint64_t, \
				__DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
#define DOMAIN_MAX_ADDR(gaw)	(((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
71

72
#define IOVA_PFN(addr)		((addr) >> PAGE_SHIFT)
73
#define DMA_32BIT_PFN		IOVA_PFN(DMA_BIT_MASK(32))
74
#define DMA_64BIT_PFN		IOVA_PFN(DMA_BIT_MASK(64))
M
mark gross 已提交
75

76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
/* page table handling */
#define LEVEL_STRIDE		(9)
#define LEVEL_MASK		(((u64)1 << LEVEL_STRIDE) - 1)

static inline int agaw_to_level(int agaw)
{
	return agaw + 2;
}

static inline int agaw_to_width(int agaw)
{
	return 30 + agaw * LEVEL_STRIDE;
}

static inline int width_to_agaw(int width)
{
	return (width - 30) / LEVEL_STRIDE;
}

static inline unsigned int level_to_offset_bits(int level)
{
	return (level - 1) * LEVEL_STRIDE;
}

static inline int pfn_level_offset(unsigned long pfn, int level)
{
	return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
}

static inline unsigned long level_mask(int level)
{
	return -1UL << level_to_offset_bits(level);
}

static inline unsigned long level_size(int level)
{
	return 1UL << level_to_offset_bits(level);
}

static inline unsigned long align_to_level(unsigned long pfn, int level)
{
	return (pfn + level_size(level) - 1) & level_mask(level);
}
119

120 121 122 123 124
static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
{
	return  1 << ((lvl - 1) * LEVEL_STRIDE);
}

125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
   are never going to work. */
static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
{
	return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
}

static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
{
	return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
}
static inline unsigned long page_to_dma_pfn(struct page *pg)
{
	return mm_to_dma_pfn(page_to_pfn(pg));
}
static inline unsigned long virt_to_dma_pfn(void *p)
{
	return page_to_dma_pfn(virt_to_page(p));
}

W
Weidong Han 已提交
145 146 147
/* global iommu list, set NULL for ignored DMAR units */
static struct intel_iommu **g_iommus;

148
static void __init check_tylersburg_isoch(void);
149 150
static int rwbf_quirk;

151 152 153 154 155 156
/*
 * set to 1 to panic kernel if can't successfully enable VT-d
 * (used when kernel is launched w/ TXT)
 */
static int force_on = 0;

157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
/*
 * 0: Present
 * 1-11: Reserved
 * 12-63: Context Ptr (12 - (haw-1))
 * 64-127: Reserved
 */
struct root_entry {
	u64	val;
	u64	rsvd1;
};
#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
static inline bool root_present(struct root_entry *root)
{
	return (root->val & 1);
}
static inline void set_root_present(struct root_entry *root)
{
	root->val |= 1;
}
static inline void set_root_value(struct root_entry *root, unsigned long value)
{
	root->val |= value & VTD_PAGE_MASK;
}

static inline struct context_entry *
get_context_addr_from_root(struct root_entry *root)
{
	return (struct context_entry *)
		(root_present(root)?phys_to_virt(
		root->val & VTD_PAGE_MASK) :
		NULL);
}

190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
/*
 * low 64 bits:
 * 0: present
 * 1: fault processing disable
 * 2-3: translation type
 * 12-63: address space root
 * high 64 bits:
 * 0-2: address width
 * 3-6: aval
 * 8-23: domain id
 */
struct context_entry {
	u64 lo;
	u64 hi;
};
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249

static inline bool context_present(struct context_entry *context)
{
	return (context->lo & 1);
}
static inline void context_set_present(struct context_entry *context)
{
	context->lo |= 1;
}

static inline void context_set_fault_enable(struct context_entry *context)
{
	context->lo &= (((u64)-1) << 2) | 1;
}

static inline void context_set_translation_type(struct context_entry *context,
						unsigned long value)
{
	context->lo &= (((u64)-1) << 4) | 3;
	context->lo |= (value & 3) << 2;
}

static inline void context_set_address_root(struct context_entry *context,
					    unsigned long value)
{
	context->lo |= value & VTD_PAGE_MASK;
}

static inline void context_set_address_width(struct context_entry *context,
					     unsigned long value)
{
	context->hi |= value & 7;
}

static inline void context_set_domain_id(struct context_entry *context,
					 unsigned long value)
{
	context->hi |= (value & ((1 << 16) - 1)) << 8;
}

static inline void context_clear_entry(struct context_entry *context)
{
	context->lo = 0;
	context->hi = 0;
}
250

251 252 253 254 255
/*
 * 0: readable
 * 1: writable
 * 2-6: reserved
 * 7: super page
256 257
 * 8-10: available
 * 11: snoop behavior
258 259 260 261 262 263
 * 12-63: Host physcial address
 */
struct dma_pte {
	u64 val;
};

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
static inline void dma_clear_pte(struct dma_pte *pte)
{
	pte->val = 0;
}

static inline void dma_set_pte_readable(struct dma_pte *pte)
{
	pte->val |= DMA_PTE_READ;
}

static inline void dma_set_pte_writable(struct dma_pte *pte)
{
	pte->val |= DMA_PTE_WRITE;
}

279 280 281 282 283
static inline void dma_set_pte_snp(struct dma_pte *pte)
{
	pte->val |= DMA_PTE_SNP;
}

284 285 286 287 288 289 290
static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
{
	pte->val = (pte->val & ~3) | (prot & 3);
}

static inline u64 dma_pte_addr(struct dma_pte *pte)
{
291 292 293 294
#ifdef CONFIG_64BIT
	return pte->val & VTD_PAGE_MASK;
#else
	/* Must have a full atomic 64-bit read */
295
	return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
296
#endif
297 298
}

299
static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
300
{
301
	pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
302 303 304 305 306 307
}

static inline bool dma_pte_present(struct dma_pte *pte)
{
	return (pte->val & 3) != 0;
}
308

309 310 311 312 313
static inline int first_pte_in_page(struct dma_pte *pte)
{
	return !((unsigned long)pte & ~VTD_PAGE_MASK);
}

314 315 316 317 318 319
/*
 * This domain is a statically identity mapping domain.
 *	1. This domain creats a static 1:1 mapping to all usable memory.
 * 	2. It maps to each iommu if successful.
 *	3. Each iommu mapps to this domain if successful.
 */
320 321
static struct dmar_domain *si_domain;
static int hw_pass_through = 1;
322

W
Weidong Han 已提交
323
/* devices under the same p2p bridge are owned in one domain */
324
#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
W
Weidong Han 已提交
325

326 327 328 329 330
/* domain represents a virtual machine, more than one devices
 * across iommus may be owned in one domain, e.g. kvm guest.
 */
#define DOMAIN_FLAG_VIRTUAL_MACHINE	(1 << 1)

331 332 333
/* si_domain contains mulitple devices */
#define DOMAIN_FLAG_STATIC_IDENTITY	(1 << 2)

334 335
struct dmar_domain {
	int	id;			/* domain id */
336
	int	nid;			/* node id */
337
	unsigned long iommu_bmp;	/* bitmap of iommus this domain uses*/
338 339 340 341 342 343 344 345 346 347

	struct list_head devices; 	/* all devices' list */
	struct iova_domain iovad;	/* iova's that belong to this domain */

	struct dma_pte	*pgd;		/* virtual address */
	int		gaw;		/* max guest address width */

	/* adjusted guest address width, 0 is level 2 30-bit */
	int		agaw;

W
Weidong Han 已提交
348
	int		flags;		/* flags to find out type of domain */
W
Weidong Han 已提交
349 350

	int		iommu_coherency;/* indicate coherency of iommu access */
351
	int		iommu_snooping; /* indicate snooping control feature*/
352
	int		iommu_count;	/* reference count of iommu */
353 354 355
	int		iommu_superpage;/* Level of superpages supported:
					   0 == 4KiB (no superpages), 1 == 2MiB,
					   2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
356
	spinlock_t	iommu_lock;	/* protect iommu set in domain */
357
	u64		max_addr;	/* maximum mapped address */
358 359
};

360 361 362 363
/* PCI domain-device relationship */
struct device_domain_info {
	struct list_head link;	/* link to domain siblings */
	struct list_head global; /* link to global list */
364 365
	int segment;		/* PCI domain */
	u8 bus;			/* PCI bus number */
366
	u8 devfn;		/* PCI devfn number */
367
	struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
Y
Yu Zhao 已提交
368
	struct intel_iommu *iommu; /* IOMMU used by this device */
369 370 371
	struct dmar_domain *domain; /* pointer to domain */
};

M
mark gross 已提交
372 373 374 375
static void flush_unmaps_timeout(unsigned long data);

DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);

376 377 378 379 380 381 382 383 384
#define HIGH_WATER_MARK 250
struct deferred_flush_tables {
	int next;
	struct iova *iova[HIGH_WATER_MARK];
	struct dmar_domain *domain[HIGH_WATER_MARK];
};

static struct deferred_flush_tables *deferred_flush;

M
mark gross 已提交
385 386 387 388 389 390 391 392 393
/* bitmap for indexing intel_iommus */
static int g_num_of_iommus;

static DEFINE_SPINLOCK(async_umap_flush_lock);
static LIST_HEAD(unmaps_to_do);

static int timer_on;
static long list_size;

394 395
static void domain_remove_dev_info(struct dmar_domain *domain);

396 397 398 399 400 401
#ifdef CONFIG_DMAR_DEFAULT_ON
int dmar_disabled = 0;
#else
int dmar_disabled = 1;
#endif /*CONFIG_DMAR_DEFAULT_ON*/

402
static int dmar_map_gfx = 1;
403
static int dmar_forcedac;
M
mark gross 已提交
404
static int intel_iommu_strict;
405
static int intel_iommu_superpage = 1;
406

407 408 409
int intel_iommu_gfx_mapped;
EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);

410 411 412 413
#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
static DEFINE_SPINLOCK(device_domain_lock);
static LIST_HEAD(device_domain_list);

414 415
static struct iommu_ops intel_iommu_ops;

416 417 418 419 420
static int __init intel_iommu_setup(char *str)
{
	if (!str)
		return -EINVAL;
	while (*str) {
421 422 423 424
		if (!strncmp(str, "on", 2)) {
			dmar_disabled = 0;
			printk(KERN_INFO "Intel-IOMMU: enabled\n");
		} else if (!strncmp(str, "off", 3)) {
425
			dmar_disabled = 1;
426
			printk(KERN_INFO "Intel-IOMMU: disabled\n");
427 428 429 430
		} else if (!strncmp(str, "igfx_off", 8)) {
			dmar_map_gfx = 0;
			printk(KERN_INFO
				"Intel-IOMMU: disable GFX device mapping\n");
431
		} else if (!strncmp(str, "forcedac", 8)) {
M
mark gross 已提交
432
			printk(KERN_INFO
433 434
				"Intel-IOMMU: Forcing DAC for PCI devices\n");
			dmar_forcedac = 1;
M
mark gross 已提交
435 436 437 438
		} else if (!strncmp(str, "strict", 6)) {
			printk(KERN_INFO
				"Intel-IOMMU: disable batched IOTLB flush\n");
			intel_iommu_strict = 1;
439 440 441 442
		} else if (!strncmp(str, "sp_off", 6)) {
			printk(KERN_INFO
				"Intel-IOMMU: disable supported super page\n");
			intel_iommu_superpage = 0;
443 444 445 446 447 448 449 450 451 452 453 454 455 456
		}

		str += strcspn(str, ",");
		while (*str == ',')
			str++;
	}
	return 0;
}
__setup("intel_iommu=", intel_iommu_setup);

static struct kmem_cache *iommu_domain_cache;
static struct kmem_cache *iommu_devinfo_cache;
static struct kmem_cache *iommu_iova_cache;

457
static inline void *alloc_pgtable_page(int node)
458
{
459 460
	struct page *page;
	void *vaddr = NULL;
461

462 463 464
	page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
	if (page)
		vaddr = page_address(page);
465
	return vaddr;
466 467 468 469 470 471 472 473 474
}

static inline void free_pgtable_page(void *vaddr)
{
	free_page((unsigned long)vaddr);
}

static inline void *alloc_domain_mem(void)
{
475
	return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
476 477
}

K
Kay, Allen M 已提交
478
static void free_domain_mem(void *vaddr)
479 480 481 482 483 484
{
	kmem_cache_free(iommu_domain_cache, vaddr);
}

static inline void * alloc_devinfo_mem(void)
{
485
	return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
486 487 488 489 490 491 492 493 494
}

static inline void free_devinfo_mem(void *vaddr)
{
	kmem_cache_free(iommu_devinfo_cache, vaddr);
}

struct iova *alloc_iova_mem(void)
{
495
	return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
496 497 498 499 500 501 502
}

void free_iova_mem(struct iova *iova)
{
	kmem_cache_free(iommu_iova_cache, iova);
}

W
Weidong Han 已提交
503

F
Fenghua Yu 已提交
504
static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
W
Weidong Han 已提交
505 506 507 508 509
{
	unsigned long sagaw;
	int agaw = -1;

	sagaw = cap_sagaw(iommu->cap);
F
Fenghua Yu 已提交
510
	for (agaw = width_to_agaw(max_gaw);
W
Weidong Han 已提交
511 512 513 514 515 516 517 518
	     agaw >= 0; agaw--) {
		if (test_bit(agaw, &sagaw))
			break;
	}

	return agaw;
}

F
Fenghua Yu 已提交
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
/*
 * Calculate max SAGAW for each iommu.
 */
int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
{
	return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
}

/*
 * calculate agaw for each iommu.
 * "SAGAW" may be different across iommus, use a default agaw, and
 * get a supported less agaw for iommus that don't support the default agaw.
 */
int iommu_calculate_agaw(struct intel_iommu *iommu)
{
	return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
}

537
/* This functionin only returns single iommu in a domain */
538 539 540 541
static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
{
	int iommu_id;

542
	/* si_domain and vm domain should not get here. */
543
	BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
544
	BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
545

546 547 548 549 550 551 552
	iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
	if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
		return NULL;

	return g_iommus[iommu_id];
}

W
Weidong Han 已提交
553 554 555 556 557 558
static void domain_update_iommu_coherency(struct dmar_domain *domain)
{
	int i;

	domain->iommu_coherency = 1;

559
	for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
W
Weidong Han 已提交
560 561 562 563 564 565 566
		if (!ecap_coherent(g_iommus[i]->ecap)) {
			domain->iommu_coherency = 0;
			break;
		}
	}
}

567 568 569 570 571 572
static void domain_update_iommu_snooping(struct dmar_domain *domain)
{
	int i;

	domain->iommu_snooping = 1;

573
	for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
574 575 576 577 578 579 580
		if (!ecap_sc_support(g_iommus[i]->ecap)) {
			domain->iommu_snooping = 0;
			break;
		}
	}
}

581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
static void domain_update_iommu_superpage(struct dmar_domain *domain)
{
	int i, mask = 0xf;

	if (!intel_iommu_superpage) {
		domain->iommu_superpage = 0;
		return;
	}

	domain->iommu_superpage = 4; /* 1TiB */

	for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
		mask |= cap_super_page_val(g_iommus[i]->cap);
		if (!mask) {
			break;
		}
	}
	domain->iommu_superpage = fls(mask);
}

601 602 603 604 605
/* Some capabilities may be different across iommus */
static void domain_update_iommu_cap(struct dmar_domain *domain)
{
	domain_update_iommu_coherency(domain);
	domain_update_iommu_snooping(domain);
606
	domain_update_iommu_superpage(domain);
607 608
}

609
static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
610 611 612 613 614 615 616
{
	struct dmar_drhd_unit *drhd = NULL;
	int i;

	for_each_drhd_unit(drhd) {
		if (drhd->ignored)
			continue;
617 618
		if (segment != drhd->segment)
			continue;
619

620
		for (i = 0; i < drhd->devices_cnt; i++) {
D
Dirk Hohndel 已提交
621 622
			if (drhd->devices[i] &&
			    drhd->devices[i]->bus->number == bus &&
623 624
			    drhd->devices[i]->devfn == devfn)
				return drhd->iommu;
625 626
			if (drhd->devices[i] &&
			    drhd->devices[i]->subordinate &&
627 628 629 630
			    drhd->devices[i]->subordinate->number <= bus &&
			    drhd->devices[i]->subordinate->subordinate >= bus)
				return drhd->iommu;
		}
631 632 633 634 635 636 637 638

		if (drhd->include_all)
			return drhd->iommu;
	}

	return NULL;
}

W
Weidong Han 已提交
639 640 641 642 643 644 645
static void domain_flush_cache(struct dmar_domain *domain,
			       void *addr, int size)
{
	if (!domain->iommu_coherency)
		clflush_cache_range(addr, size);
}

646 647 648 649 650 651 652 653 654 655 656 657 658
/* Gets context entry for a given bus and devfn */
static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
		u8 bus, u8 devfn)
{
	struct root_entry *root;
	struct context_entry *context;
	unsigned long phy_addr;
	unsigned long flags;

	spin_lock_irqsave(&iommu->lock, flags);
	root = &iommu->root_entry[bus];
	context = get_context_addr_from_root(root);
	if (!context) {
659 660
		context = (struct context_entry *)
				alloc_pgtable_page(iommu->node);
661 662 663 664
		if (!context) {
			spin_unlock_irqrestore(&iommu->lock, flags);
			return NULL;
		}
F
Fenghua Yu 已提交
665
		__iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
		phy_addr = virt_to_phys((void *)context);
		set_root_value(root, phy_addr);
		set_root_present(root);
		__iommu_flush_cache(iommu, root, sizeof(*root));
	}
	spin_unlock_irqrestore(&iommu->lock, flags);
	return &context[devfn];
}

static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
{
	struct root_entry *root;
	struct context_entry *context;
	int ret;
	unsigned long flags;

	spin_lock_irqsave(&iommu->lock, flags);
	root = &iommu->root_entry[bus];
	context = get_context_addr_from_root(root);
	if (!context) {
		ret = 0;
		goto out;
	}
689
	ret = context_present(&context[devfn]);
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
out:
	spin_unlock_irqrestore(&iommu->lock, flags);
	return ret;
}

static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
{
	struct root_entry *root;
	struct context_entry *context;
	unsigned long flags;

	spin_lock_irqsave(&iommu->lock, flags);
	root = &iommu->root_entry[bus];
	context = get_context_addr_from_root(root);
	if (context) {
705
		context_clear_entry(&context[devfn]);
706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734
		__iommu_flush_cache(iommu, &context[devfn], \
			sizeof(*context));
	}
	spin_unlock_irqrestore(&iommu->lock, flags);
}

static void free_context_table(struct intel_iommu *iommu)
{
	struct root_entry *root;
	int i;
	unsigned long flags;
	struct context_entry *context;

	spin_lock_irqsave(&iommu->lock, flags);
	if (!iommu->root_entry) {
		goto out;
	}
	for (i = 0; i < ROOT_ENTRY_NR; i++) {
		root = &iommu->root_entry[i];
		context = get_context_addr_from_root(root);
		if (context)
			free_pgtable_page(context);
	}
	free_pgtable_page(iommu->root_entry);
	iommu->root_entry = NULL;
out:
	spin_unlock_irqrestore(&iommu->lock, flags);
}

735
static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
736
				      unsigned long pfn, int large_level)
737
{
738
	int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
739 740
	struct dma_pte *parent, *pte = NULL;
	int level = agaw_to_level(domain->agaw);
741
	int offset, target_level;
742 743

	BUG_ON(!domain->pgd);
744
	BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
745 746
	parent = domain->pgd;

747 748 749 750 751 752
	/* Search pte */
	if (!large_level)
		target_level = 1;
	else
		target_level = large_level;

753 754 755
	while (level > 0) {
		void *tmp_page;

756
		offset = pfn_level_offset(pfn, level);
757
		pte = &parent[offset];
758 759 760
		if (!large_level && (pte->val & DMA_PTE_LARGE_PAGE))
			break;
		if (level == target_level)
761 762
			break;

763
		if (!dma_pte_present(pte)) {
764 765
			uint64_t pteval;

766
			tmp_page = alloc_pgtable_page(domain->nid);
767

768
			if (!tmp_page)
769
				return NULL;
770

771
			domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
772
			pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
773 774 775 776 777 778 779
			if (cmpxchg64(&pte->val, 0ULL, pteval)) {
				/* Someone else set it while we were thinking; use theirs. */
				free_pgtable_page(tmp_page);
			} else {
				dma_pte_addr(pte);
				domain_flush_cache(domain, pte, sizeof(*pte));
			}
780
		}
781
		parent = phys_to_virt(dma_pte_addr(pte));
782 783 784 785 786 787
		level--;
	}

	return pte;
}

788

789
/* return address's pte at specific level */
790 791
static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
					 unsigned long pfn,
792
					 int level, int *large_page)
793 794 795 796 797 798 799
{
	struct dma_pte *parent, *pte = NULL;
	int total = agaw_to_level(domain->agaw);
	int offset;

	parent = domain->pgd;
	while (level <= total) {
800
		offset = pfn_level_offset(pfn, total);
801 802 803 804
		pte = &parent[offset];
		if (level == total)
			return pte;

805 806
		if (!dma_pte_present(pte)) {
			*large_page = total;
807
			break;
808 809 810 811 812 813 814
		}

		if (pte->val & DMA_PTE_LARGE_PAGE) {
			*large_page = total;
			return pte;
		}

815
		parent = phys_to_virt(dma_pte_addr(pte));
816 817 818 819 820 821
		total--;
	}
	return NULL;
}

/* clear last level pte, a tlb flush should be followed */
822 823 824
static void dma_pte_clear_range(struct dmar_domain *domain,
				unsigned long start_pfn,
				unsigned long last_pfn)
825
{
826
	int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
827
	unsigned int large_page = 1;
828
	struct dma_pte *first_pte, *pte;
829

830
	BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
831
	BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
832
	BUG_ON(start_pfn > last_pfn);
833

834
	/* we don't need lock here; nobody else touches the iova range */
835
	do {
836 837
		large_page = 1;
		first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
838
		if (!pte) {
839
			start_pfn = align_to_level(start_pfn + 1, large_page + 1);
840 841
			continue;
		}
842
		do {
843
			dma_clear_pte(pte);
844
			start_pfn += lvl_to_nr_pages(large_page);
845
			pte++;
846 847
		} while (start_pfn <= last_pfn && !first_pte_in_page(pte));

848 849
		domain_flush_cache(domain, first_pte,
				   (void *)pte - (void *)first_pte);
850 851

	} while (start_pfn && start_pfn <= last_pfn);
852 853 854 855
}

/* free page table pages. last level pte should already be cleared */
static void dma_pte_free_pagetable(struct dmar_domain *domain,
856 857
				   unsigned long start_pfn,
				   unsigned long last_pfn)
858
{
859
	int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
860
	struct dma_pte *first_pte, *pte;
861 862
	int total = agaw_to_level(domain->agaw);
	int level;
863
	unsigned long tmp;
864
	int large_page = 2;
865

866 867
	BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
	BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
868
	BUG_ON(start_pfn > last_pfn);
869

870
	/* We don't need lock here; nobody else touches the iova range */
871 872
	level = 2;
	while (level <= total) {
873 874
		tmp = align_to_level(start_pfn, level);

875
		/* If we can't even clear one PTE at this level, we're done */
876
		if (tmp + level_size(level) - 1 > last_pfn)
877 878
			return;

879
		do {
880 881 882 883
			large_page = level;
			first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
			if (large_page > level)
				level = large_page + 1;
884 885 886 887
			if (!pte) {
				tmp = align_to_level(tmp + 1, level + 1);
				continue;
			}
888
			do {
889 890 891 892
				if (dma_pte_present(pte)) {
					free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
					dma_clear_pte(pte);
				}
893 894
				pte++;
				tmp += level_size(level);
895 896 897
			} while (!first_pte_in_page(pte) &&
				 tmp + level_size(level) - 1 <= last_pfn);

898 899 900
			domain_flush_cache(domain, first_pte,
					   (void *)pte - (void *)first_pte);
			
901
		} while (tmp && tmp + level_size(level) - 1 <= last_pfn);
902 903 904
		level++;
	}
	/* free pgd */
905
	if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
906 907 908 909 910 911 912 913 914 915 916
		free_pgtable_page(domain->pgd);
		domain->pgd = NULL;
	}
}

/* iommu handling */
static int iommu_alloc_root_entry(struct intel_iommu *iommu)
{
	struct root_entry *root;
	unsigned long flags;

917
	root = (struct root_entry *)alloc_pgtable_page(iommu->node);
918 919 920
	if (!root)
		return -ENOMEM;

F
Fenghua Yu 已提交
921
	__iommu_flush_cache(iommu, root, ROOT_SIZE);
922 923 924 925 926 927 928 929 930 931 932

	spin_lock_irqsave(&iommu->lock, flags);
	iommu->root_entry = root;
	spin_unlock_irqrestore(&iommu->lock, flags);

	return 0;
}

static void iommu_set_root_entry(struct intel_iommu *iommu)
{
	void *addr;
933
	u32 sts;
934 935 936 937 938 939 940
	unsigned long flag;

	addr = iommu->root_entry;

	spin_lock_irqsave(&iommu->register_lock, flag);
	dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));

941
	writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
942 943 944

	/* Make sure hardware complete it */
	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
945
		      readl, (sts & DMA_GSTS_RTPS), sts);
946 947 948 949 950 951 952 953 954

	spin_unlock_irqrestore(&iommu->register_lock, flag);
}

static void iommu_flush_write_buffer(struct intel_iommu *iommu)
{
	u32 val;
	unsigned long flag;

955
	if (!rwbf_quirk && !cap_rwbf(iommu->cap))
956 957 958
		return;

	spin_lock_irqsave(&iommu->register_lock, flag);
959
	writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
960 961 962

	/* Make sure hardware complete it */
	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
963
		      readl, (!(val & DMA_GSTS_WBFS)), val);
964 965 966 967 968

	spin_unlock_irqrestore(&iommu->register_lock, flag);
}

/* return value determine if we need a write buffer flush */
969 970 971
static void __iommu_flush_context(struct intel_iommu *iommu,
				  u16 did, u16 source_id, u8 function_mask,
				  u64 type)
972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
{
	u64 val = 0;
	unsigned long flag;

	switch (type) {
	case DMA_CCMD_GLOBAL_INVL:
		val = DMA_CCMD_GLOBAL_INVL;
		break;
	case DMA_CCMD_DOMAIN_INVL:
		val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
		break;
	case DMA_CCMD_DEVICE_INVL:
		val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
			| DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
		break;
	default:
		BUG();
	}
	val |= DMA_CCMD_ICC;

	spin_lock_irqsave(&iommu->register_lock, flag);
	dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);

	/* Make sure hardware complete it */
	IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
		dmar_readq, (!(val & DMA_CCMD_ICC)), val);

	spin_unlock_irqrestore(&iommu->register_lock, flag);
}

/* return value determine if we need a write buffer flush */
1003 1004
static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
				u64 addr, unsigned int size_order, u64 type)
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
{
	int tlb_offset = ecap_iotlb_offset(iommu->ecap);
	u64 val = 0, val_iva = 0;
	unsigned long flag;

	switch (type) {
	case DMA_TLB_GLOBAL_FLUSH:
		/* global flush doesn't need set IVA_REG */
		val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
		break;
	case DMA_TLB_DSI_FLUSH:
		val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
		break;
	case DMA_TLB_PSI_FLUSH:
		val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
		/* Note: always flush non-leaf currently */
		val_iva = size_order | addr;
		break;
	default:
		BUG();
	}
	/* Note: set drain read/write */
#if 0
	/*
	 * This is probably to be super secure.. Looks like we can
	 * ignore it without any impact.
	 */
	if (cap_read_drain(iommu->cap))
		val |= DMA_TLB_READ_DRAIN;
#endif
	if (cap_write_drain(iommu->cap))
		val |= DMA_TLB_WRITE_DRAIN;

	spin_lock_irqsave(&iommu->register_lock, flag);
	/* Note: Only uses first TLB reg currently */
	if (val_iva)
		dmar_writeq(iommu->reg + tlb_offset, val_iva);
	dmar_writeq(iommu->reg + tlb_offset + 8, val);

	/* Make sure hardware complete it */
	IOMMU_WAIT_OP(iommu, tlb_offset + 8,
		dmar_readq, (!(val & DMA_TLB_IVT)), val);

	spin_unlock_irqrestore(&iommu->register_lock, flag);

	/* check IOTLB invalidation granularity */
	if (DMA_TLB_IAIG(val) == 0)
		printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
	if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
		pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
F
Fenghua Yu 已提交
1055 1056
			(unsigned long long)DMA_TLB_IIRG(type),
			(unsigned long long)DMA_TLB_IAIG(val));
1057 1058
}

Y
Yu Zhao 已提交
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
static struct device_domain_info *iommu_support_dev_iotlb(
	struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
{
	int found = 0;
	unsigned long flags;
	struct device_domain_info *info;
	struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);

	if (!ecap_dev_iotlb_support(iommu->ecap))
		return NULL;

	if (!iommu->qi)
		return NULL;

	spin_lock_irqsave(&device_domain_lock, flags);
	list_for_each_entry(info, &domain->devices, link)
		if (info->bus == bus && info->devfn == devfn) {
			found = 1;
			break;
		}
	spin_unlock_irqrestore(&device_domain_lock, flags);

	if (!found || !info->dev)
		return NULL;

	if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
		return NULL;

	if (!dmar_find_matched_atsr_unit(info->dev))
		return NULL;

	info->iommu = iommu;

	return info;
}

static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1096
{
Y
Yu Zhao 已提交
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
	if (!info)
		return;

	pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
}

static void iommu_disable_dev_iotlb(struct device_domain_info *info)
{
	if (!info->dev || !pci_ats_enabled(info->dev))
		return;

	pci_disable_ats(info->dev);
}

static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
				  u64 addr, unsigned mask)
{
	u16 sid, qdep;
	unsigned long flags;
	struct device_domain_info *info;

	spin_lock_irqsave(&device_domain_lock, flags);
	list_for_each_entry(info, &domain->devices, link) {
		if (!info->dev || !pci_ats_enabled(info->dev))
			continue;

		sid = info->bus << 8 | info->devfn;
		qdep = pci_ats_queue_depth(info->dev);
		qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
	}
	spin_unlock_irqrestore(&device_domain_lock, flags);
}

1130
static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1131
				  unsigned long pfn, unsigned int pages, int map)
1132
{
1133
	unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1134
	uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1135 1136 1137 1138

	BUG_ON(pages == 0);

	/*
1139 1140
	 * Fallback to domain selective flush if no PSI support or the size is
	 * too big.
1141 1142 1143
	 * PSI requires page size to be 2 ^ x, and the base address is naturally
	 * aligned to the size
	 */
1144 1145
	if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
		iommu->flush.flush_iotlb(iommu, did, 0, 0,
1146
						DMA_TLB_DSI_FLUSH);
1147 1148 1149
	else
		iommu->flush.flush_iotlb(iommu, did, addr, mask,
						DMA_TLB_PSI_FLUSH);
1150 1151

	/*
1152 1153
	 * In caching mode, changes of pages from non-present to present require
	 * flush. However, device IOTLB doesn't need to be flushed in this case.
1154
	 */
1155
	if (!cap_caching_mode(iommu->cap) || !map)
Y
Yu Zhao 已提交
1156
		iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1157 1158
}

M
mark gross 已提交
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
{
	u32 pmen;
	unsigned long flags;

	spin_lock_irqsave(&iommu->register_lock, flags);
	pmen = readl(iommu->reg + DMAR_PMEN_REG);
	pmen &= ~DMA_PMEN_EPM;
	writel(pmen, iommu->reg + DMAR_PMEN_REG);

	/* wait for the protected region status bit to clear */
	IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
		readl, !(pmen & DMA_PMEN_PRS), pmen);

	spin_unlock_irqrestore(&iommu->register_lock, flags);
}

1176 1177 1178 1179 1180 1181
static int iommu_enable_translation(struct intel_iommu *iommu)
{
	u32 sts;
	unsigned long flags;

	spin_lock_irqsave(&iommu->register_lock, flags);
1182 1183
	iommu->gcmd |= DMA_GCMD_TE;
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1184 1185 1186

	/* Make sure hardware complete it */
	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1187
		      readl, (sts & DMA_GSTS_TES), sts);
1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203

	spin_unlock_irqrestore(&iommu->register_lock, flags);
	return 0;
}

static int iommu_disable_translation(struct intel_iommu *iommu)
{
	u32 sts;
	unsigned long flag;

	spin_lock_irqsave(&iommu->register_lock, flag);
	iommu->gcmd &= ~DMA_GCMD_TE;
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

	/* Make sure hardware complete it */
	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1204
		      readl, (!(sts & DMA_GSTS_TES)), sts);
1205 1206 1207 1208 1209

	spin_unlock_irqrestore(&iommu->register_lock, flag);
	return 0;
}

1210

1211 1212 1213 1214 1215 1216
static int iommu_init_domains(struct intel_iommu *iommu)
{
	unsigned long ndomains;
	unsigned long nlongs;

	ndomains = cap_ndoms(iommu->cap);
Y
Yinghai Lu 已提交
1217 1218
	pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
			ndomains);
1219 1220
	nlongs = BITS_TO_LONGS(ndomains);

1221 1222
	spin_lock_init(&iommu->lock);

1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
	/* TBD: there might be 64K domains,
	 * consider other allocation for future chip
	 */
	iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
	if (!iommu->domain_ids) {
		printk(KERN_ERR "Allocating domain id array failed\n");
		return -ENOMEM;
	}
	iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
			GFP_KERNEL);
	if (!iommu->domains) {
		printk(KERN_ERR "Allocating domain array failed\n");
		return -ENOMEM;
	}

	/*
	 * if Caching mode is set, then invalid translations are tagged
	 * with domainid 0. Hence we need to pre-allocate it.
	 */
	if (cap_caching_mode(iommu->cap))
		set_bit(0, iommu->domain_ids);
	return 0;
}


static void domain_exit(struct dmar_domain *domain);
1249
static void vm_domain_exit(struct dmar_domain *domain);
1250 1251

void free_dmar_iommu(struct intel_iommu *iommu)
1252 1253 1254
{
	struct dmar_domain *domain;
	int i;
1255
	unsigned long flags;
1256

1257
	if ((iommu->domains) && (iommu->domain_ids)) {
1258
		for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
			domain = iommu->domains[i];
			clear_bit(i, iommu->domain_ids);

			spin_lock_irqsave(&domain->iommu_lock, flags);
			if (--domain->iommu_count == 0) {
				if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
					vm_domain_exit(domain);
				else
					domain_exit(domain);
			}
			spin_unlock_irqrestore(&domain->iommu_lock, flags);
1270
		}
1271 1272 1273 1274 1275 1276
	}

	if (iommu->gcmd & DMA_GCMD_TE)
		iommu_disable_translation(iommu);

	if (iommu->irq) {
1277
		irq_set_handler_data(iommu->irq, NULL);
1278 1279 1280 1281 1282 1283 1284 1285
		/* This will mask the irq */
		free_irq(iommu->irq, iommu);
		destroy_irq(iommu->irq);
	}

	kfree(iommu->domains);
	kfree(iommu->domain_ids);

W
Weidong Han 已提交
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
	g_iommus[iommu->seq_id] = NULL;

	/* if all iommus are freed, free g_iommus */
	for (i = 0; i < g_num_of_iommus; i++) {
		if (g_iommus[i])
			break;
	}

	if (i == g_num_of_iommus)
		kfree(g_iommus);

1297 1298 1299 1300
	/* free context mapping */
	free_context_table(iommu);
}

1301
static struct dmar_domain *alloc_domain(void)
1302 1303 1304 1305 1306 1307 1308
{
	struct dmar_domain *domain;

	domain = alloc_domain_mem();
	if (!domain)
		return NULL;

1309
	domain->nid = -1;
1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
	memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
	domain->flags = 0;

	return domain;
}

static int iommu_attach_domain(struct dmar_domain *domain,
			       struct intel_iommu *iommu)
{
	int num;
	unsigned long ndomains;
	unsigned long flags;

1323 1324 1325
	ndomains = cap_ndoms(iommu->cap);

	spin_lock_irqsave(&iommu->lock, flags);
1326

1327 1328 1329 1330
	num = find_first_zero_bit(iommu->domain_ids, ndomains);
	if (num >= ndomains) {
		spin_unlock_irqrestore(&iommu->lock, flags);
		printk(KERN_ERR "IOMMU: no free domain ids\n");
1331
		return -ENOMEM;
1332 1333 1334
	}

	domain->id = num;
1335
	set_bit(num, iommu->domain_ids);
1336
	set_bit(iommu->seq_id, &domain->iommu_bmp);
1337 1338 1339
	iommu->domains[num] = domain;
	spin_unlock_irqrestore(&iommu->lock, flags);

1340
	return 0;
1341 1342
}

1343 1344
static void iommu_detach_domain(struct dmar_domain *domain,
				struct intel_iommu *iommu)
1345 1346
{
	unsigned long flags;
1347 1348
	int num, ndomains;
	int found = 0;
1349

1350
	spin_lock_irqsave(&iommu->lock, flags);
1351
	ndomains = cap_ndoms(iommu->cap);
1352
	for_each_set_bit(num, iommu->domain_ids, ndomains) {
1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
		if (iommu->domains[num] == domain) {
			found = 1;
			break;
		}
	}

	if (found) {
		clear_bit(num, iommu->domain_ids);
		clear_bit(iommu->seq_id, &domain->iommu_bmp);
		iommu->domains[num] = NULL;
	}
1364
	spin_unlock_irqrestore(&iommu->lock, flags);
1365 1366 1367
}

static struct iova_domain reserved_iova_list;
M
Mark Gross 已提交
1368
static struct lock_class_key reserved_rbtree_key;
1369

1370
static int dmar_init_reserved_ranges(void)
1371 1372 1373 1374 1375
{
	struct pci_dev *pdev = NULL;
	struct iova *iova;
	int i;

D
David Miller 已提交
1376
	init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1377

M
Mark Gross 已提交
1378 1379 1380
	lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
		&reserved_rbtree_key);

1381 1382 1383
	/* IOAPIC ranges shouldn't be accessed by DMA */
	iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
		IOVA_PFN(IOAPIC_RANGE_END));
1384
	if (!iova) {
1385
		printk(KERN_ERR "Reserve IOAPIC range failed\n");
1386 1387
		return -ENODEV;
	}
1388 1389 1390 1391 1392 1393 1394 1395 1396

	/* Reserve all PCI MMIO to avoid peer-to-peer access */
	for_each_pci_dev(pdev) {
		struct resource *r;

		for (i = 0; i < PCI_NUM_RESOURCES; i++) {
			r = &pdev->resource[i];
			if (!r->flags || !(r->flags & IORESOURCE_MEM))
				continue;
1397 1398 1399
			iova = reserve_iova(&reserved_iova_list,
					    IOVA_PFN(r->start),
					    IOVA_PFN(r->end));
1400
			if (!iova) {
1401
				printk(KERN_ERR "Reserve iova failed\n");
1402 1403
				return -ENODEV;
			}
1404 1405
		}
	}
1406
	return 0;
1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
}

static void domain_reserve_special_ranges(struct dmar_domain *domain)
{
	copy_reserved_iova(&reserved_iova_list, &domain->iovad);
}

static inline int guestwidth_to_adjustwidth(int gaw)
{
	int agaw;
	int r = (gaw - 12) % 9;

	if (r == 0)
		agaw = gaw;
	else
		agaw = gaw + 9 - r;
	if (agaw > 64)
		agaw = 64;
	return agaw;
}

static int domain_init(struct dmar_domain *domain, int guest_width)
{
	struct intel_iommu *iommu;
	int adjust_width, agaw;
	unsigned long sagaw;

D
David Miller 已提交
1434
	init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1435
	spin_lock_init(&domain->iommu_lock);
1436 1437 1438 1439

	domain_reserve_special_ranges(domain);

	/* calculate AGAW */
1440
	iommu = domain_get_iommu(domain);
1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
	if (guest_width > cap_mgaw(iommu->cap))
		guest_width = cap_mgaw(iommu->cap);
	domain->gaw = guest_width;
	adjust_width = guestwidth_to_adjustwidth(guest_width);
	agaw = width_to_agaw(adjust_width);
	sagaw = cap_sagaw(iommu->cap);
	if (!test_bit(agaw, &sagaw)) {
		/* hardware doesn't support it, choose a bigger one */
		pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
		agaw = find_next_bit(&sagaw, 5, agaw);
		if (agaw >= 5)
			return -ENODEV;
	}
	domain->agaw = agaw;
	INIT_LIST_HEAD(&domain->devices);

W
Weidong Han 已提交
1457 1458 1459 1460 1461
	if (ecap_coherent(iommu->ecap))
		domain->iommu_coherency = 1;
	else
		domain->iommu_coherency = 0;

1462 1463 1464 1465 1466
	if (ecap_sc_support(iommu->ecap))
		domain->iommu_snooping = 1;
	else
		domain->iommu_snooping = 0;

1467
	domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1468
	domain->iommu_count = 1;
1469
	domain->nid = iommu->node;
1470

1471
	/* always allocate the top pgd */
1472
	domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1473 1474
	if (!domain->pgd)
		return -ENOMEM;
F
Fenghua Yu 已提交
1475
	__iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1476 1477 1478 1479 1480
	return 0;
}

static void domain_exit(struct dmar_domain *domain)
{
1481 1482
	struct dmar_drhd_unit *drhd;
	struct intel_iommu *iommu;
1483 1484 1485 1486 1487

	/* Domain 0 is reserved, so dont process it */
	if (!domain)
		return;

1488 1489 1490 1491
	/* Flush any lazy unmaps that may reference this domain */
	if (!intel_iommu_strict)
		flush_unmaps_timeout(0);

1492 1493 1494 1495 1496
	domain_remove_dev_info(domain);
	/* destroy iovas */
	put_iova_domain(&domain->iovad);

	/* clear ptes */
1497
	dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1498 1499

	/* free page tables */
1500
	dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1501

1502 1503 1504 1505
	for_each_active_iommu(iommu, drhd)
		if (test_bit(iommu->seq_id, &domain->iommu_bmp))
			iommu_detach_domain(domain, iommu);

1506 1507 1508
	free_domain_mem(domain);
}

F
Fenghua Yu 已提交
1509 1510
static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
				 u8 bus, u8 devfn, int translation)
1511 1512 1513
{
	struct context_entry *context;
	unsigned long flags;
W
Weidong Han 已提交
1514
	struct intel_iommu *iommu;
1515 1516 1517 1518 1519
	struct dma_pte *pgd;
	unsigned long num;
	unsigned long ndomains;
	int id;
	int agaw;
Y
Yu Zhao 已提交
1520
	struct device_domain_info *info = NULL;
1521 1522 1523

	pr_debug("Set context mapping for %02x:%02x.%d\n",
		bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
F
Fenghua Yu 已提交
1524

1525
	BUG_ON(!domain->pgd);
F
Fenghua Yu 已提交
1526 1527
	BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
	       translation != CONTEXT_TT_MULTI_LEVEL);
W
Weidong Han 已提交
1528

1529
	iommu = device_to_iommu(segment, bus, devfn);
W
Weidong Han 已提交
1530 1531 1532
	if (!iommu)
		return -ENODEV;

1533 1534 1535 1536
	context = device_to_context_entry(iommu, bus, devfn);
	if (!context)
		return -ENOMEM;
	spin_lock_irqsave(&iommu->lock, flags);
1537
	if (context_present(context)) {
1538 1539 1540 1541
		spin_unlock_irqrestore(&iommu->lock, flags);
		return 0;
	}

1542 1543 1544
	id = domain->id;
	pgd = domain->pgd;

1545 1546
	if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
	    domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1547 1548 1549 1550
		int found = 0;

		/* find an available domain id for this device in iommu */
		ndomains = cap_ndoms(iommu->cap);
1551
		for_each_set_bit(num, iommu->domain_ids, ndomains) {
1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
			if (iommu->domains[num] == domain) {
				id = num;
				found = 1;
				break;
			}
		}

		if (found == 0) {
			num = find_first_zero_bit(iommu->domain_ids, ndomains);
			if (num >= ndomains) {
				spin_unlock_irqrestore(&iommu->lock, flags);
				printk(KERN_ERR "IOMMU: no free domain ids\n");
				return -EFAULT;
			}

			set_bit(num, iommu->domain_ids);
			iommu->domains[num] = domain;
			id = num;
		}

		/* Skip top levels of page tables for
		 * iommu which has less agaw than default.
1574
		 * Unnecessary for PT mode.
1575
		 */
1576 1577 1578 1579 1580 1581 1582
		if (translation != CONTEXT_TT_PASS_THROUGH) {
			for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
				pgd = phys_to_virt(dma_pte_addr(pgd));
				if (!dma_pte_present(pgd)) {
					spin_unlock_irqrestore(&iommu->lock, flags);
					return -ENOMEM;
				}
1583 1584 1585 1586 1587
			}
		}
	}

	context_set_domain_id(context, id);
F
Fenghua Yu 已提交
1588

Y
Yu Zhao 已提交
1589 1590 1591 1592 1593
	if (translation != CONTEXT_TT_PASS_THROUGH) {
		info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
		translation = info ? CONTEXT_TT_DEV_IOTLB :
				     CONTEXT_TT_MULTI_LEVEL;
	}
F
Fenghua Yu 已提交
1594 1595 1596 1597
	/*
	 * In pass through mode, AW must be programmed to indicate the largest
	 * AGAW value supported by hardware. And ASR is ignored by hardware.
	 */
Y
Yu Zhao 已提交
1598
	if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
F
Fenghua Yu 已提交
1599
		context_set_address_width(context, iommu->msagaw);
Y
Yu Zhao 已提交
1600 1601 1602 1603
	else {
		context_set_address_root(context, virt_to_phys(pgd));
		context_set_address_width(context, iommu->agaw);
	}
F
Fenghua Yu 已提交
1604 1605

	context_set_translation_type(context, translation);
1606 1607
	context_set_fault_enable(context);
	context_set_present(context);
W
Weidong Han 已提交
1608
	domain_flush_cache(domain, context, sizeof(*context));
1609

1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620
	/*
	 * It's a non-present to present mapping. If hardware doesn't cache
	 * non-present entry we only need to flush the write-buffer. If the
	 * _does_ cache non-present entries, then it does so in the special
	 * domain #0, which we have to flush:
	 */
	if (cap_caching_mode(iommu->cap)) {
		iommu->flush.flush_context(iommu, 0,
					   (((u16)bus) << 8) | devfn,
					   DMA_CCMD_MASK_NOBIT,
					   DMA_CCMD_DEVICE_INVL);
1621
		iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1622
	} else {
1623
		iommu_flush_write_buffer(iommu);
1624
	}
Y
Yu Zhao 已提交
1625
	iommu_enable_dev_iotlb(info);
1626
	spin_unlock_irqrestore(&iommu->lock, flags);
1627 1628 1629 1630

	spin_lock_irqsave(&domain->iommu_lock, flags);
	if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
		domain->iommu_count++;
1631 1632
		if (domain->iommu_count == 1)
			domain->nid = iommu->node;
1633
		domain_update_iommu_cap(domain);
1634 1635
	}
	spin_unlock_irqrestore(&domain->iommu_lock, flags);
1636 1637 1638 1639
	return 0;
}

static int
F
Fenghua Yu 已提交
1640 1641
domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
			int translation)
1642 1643 1644 1645
{
	int ret;
	struct pci_dev *tmp, *parent;

1646
	ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
F
Fenghua Yu 已提交
1647 1648
					 pdev->bus->number, pdev->devfn,
					 translation);
1649 1650 1651 1652 1653 1654 1655 1656 1657 1658
	if (ret)
		return ret;

	/* dependent device mapping */
	tmp = pci_find_upstream_pcie_bridge(pdev);
	if (!tmp)
		return 0;
	/* Secondary interface's bus number and devfn 0 */
	parent = pdev->bus->self;
	while (parent != tmp) {
1659 1660 1661
		ret = domain_context_mapping_one(domain,
						 pci_domain_nr(parent->bus),
						 parent->bus->number,
F
Fenghua Yu 已提交
1662
						 parent->devfn, translation);
1663 1664 1665 1666
		if (ret)
			return ret;
		parent = parent->bus->self;
	}
1667
	if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1668
		return domain_context_mapping_one(domain,
1669
					pci_domain_nr(tmp->subordinate),
F
Fenghua Yu 已提交
1670 1671
					tmp->subordinate->number, 0,
					translation);
1672 1673
	else /* this is a legacy PCI bridge */
		return domain_context_mapping_one(domain,
1674 1675
						  pci_domain_nr(tmp->bus),
						  tmp->bus->number,
F
Fenghua Yu 已提交
1676 1677
						  tmp->devfn,
						  translation);
1678 1679
}

W
Weidong Han 已提交
1680
static int domain_context_mapped(struct pci_dev *pdev)
1681 1682 1683
{
	int ret;
	struct pci_dev *tmp, *parent;
W
Weidong Han 已提交
1684 1685
	struct intel_iommu *iommu;

1686 1687
	iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
				pdev->devfn);
W
Weidong Han 已提交
1688 1689
	if (!iommu)
		return -ENODEV;
1690

1691
	ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1692 1693 1694 1695 1696 1697 1698 1699 1700
	if (!ret)
		return ret;
	/* dependent device mapping */
	tmp = pci_find_upstream_pcie_bridge(pdev);
	if (!tmp)
		return ret;
	/* Secondary interface's bus number and devfn 0 */
	parent = pdev->bus->self;
	while (parent != tmp) {
1701
		ret = device_context_mapped(iommu, parent->bus->number,
1702
					    parent->devfn);
1703 1704 1705 1706
		if (!ret)
			return ret;
		parent = parent->bus->self;
	}
1707
	if (pci_is_pcie(tmp))
1708 1709
		return device_context_mapped(iommu, tmp->subordinate->number,
					     0);
1710
	else
1711 1712
		return device_context_mapped(iommu, tmp->bus->number,
					     tmp->devfn);
1713 1714
}

1715 1716 1717 1718 1719 1720 1721 1722
/* Returns a number of VTD pages, but aligned to MM page size */
static inline unsigned long aligned_nrpages(unsigned long host_addr,
					    size_t size)
{
	host_addr &= ~PAGE_MASK;
	return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
}

1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750
/* Return largest possible superpage level for a given mapping */
static inline int hardware_largepage_caps(struct dmar_domain *domain,
					  unsigned long iov_pfn,
					  unsigned long phy_pfn,
					  unsigned long pages)
{
	int support, level = 1;
	unsigned long pfnmerge;

	support = domain->iommu_superpage;

	/* To use a large page, the virtual *and* physical addresses
	   must be aligned to 2MiB/1GiB/etc. Lower bits set in either
	   of them will mean we have to use smaller pages. So just
	   merge them and check both at once. */
	pfnmerge = iov_pfn | phy_pfn;

	while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
		pages >>= VTD_STRIDE_SHIFT;
		if (!pages)
			break;
		pfnmerge >>= VTD_STRIDE_SHIFT;
		level++;
		support--;
	}
	return level;
}

1751 1752 1753
static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
			    struct scatterlist *sg, unsigned long phys_pfn,
			    unsigned long nr_pages, int prot)
1754 1755
{
	struct dma_pte *first_pte = NULL, *pte = NULL;
1756
	phys_addr_t uninitialized_var(pteval);
1757
	int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1758
	unsigned long sg_res;
1759 1760
	unsigned int largepage_lvl = 0;
	unsigned long lvl_pages = 0;
1761 1762 1763 1764 1765 1766 1767 1768

	BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);

	if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
		return -EINVAL;

	prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;

1769 1770 1771 1772 1773 1774 1775
	if (sg)
		sg_res = 0;
	else {
		sg_res = nr_pages + 1;
		pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
	}

1776
	while (nr_pages > 0) {
1777 1778
		uint64_t tmp;

1779
		if (!sg_res) {
1780
			sg_res = aligned_nrpages(sg->offset, sg->length);
1781 1782 1783
			sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
			sg->dma_length = sg->length;
			pteval = page_to_phys(sg_page(sg)) | prot;
1784
			phys_pfn = pteval >> VTD_PAGE_SHIFT;
1785
		}
1786

1787
		if (!pte) {
1788 1789 1790
			largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);

			first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
1791 1792
			if (!pte)
				return -ENOMEM;
1793 1794 1795 1796 1797 1798
			/* It is large page*/
			if (largepage_lvl > 1)
				pteval |= DMA_PTE_LARGE_PAGE;
			else
				pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;

1799 1800 1801 1802
		}
		/* We don't need lock here, nobody else
		 * touches the iova range
		 */
1803
		tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1804
		if (tmp) {
1805
			static int dumps = 5;
1806 1807
			printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
			       iov_pfn, tmp, (unsigned long long)pteval);
1808 1809 1810 1811 1812 1813
			if (dumps) {
				dumps--;
				debug_dma_dump_mappings(NULL);
			}
			WARN_ON(1);
		}
1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836

		lvl_pages = lvl_to_nr_pages(largepage_lvl);

		BUG_ON(nr_pages < lvl_pages);
		BUG_ON(sg_res < lvl_pages);

		nr_pages -= lvl_pages;
		iov_pfn += lvl_pages;
		phys_pfn += lvl_pages;
		pteval += lvl_pages * VTD_PAGE_SIZE;
		sg_res -= lvl_pages;

		/* If the next PTE would be the first in a new page, then we
		   need to flush the cache on the entries we've just written.
		   And then we'll need to recalculate 'pte', so clear it and
		   let it get set again in the if (!pte) block above.

		   If we're done (!nr_pages) we need to flush the cache too.

		   Also if we've been setting superpages, we may need to
		   recalculate 'pte' and switch back to smaller pages for the
		   end of the mapping, if the trailing size is not enough to
		   use another superpage (i.e. sg_res < lvl_pages). */
1837
		pte++;
1838 1839
		if (!nr_pages || first_pte_in_page(pte) ||
		    (largepage_lvl > 1 && sg_res < lvl_pages)) {
1840 1841 1842 1843
			domain_flush_cache(domain, first_pte,
					   (void *)pte - (void *)first_pte);
			pte = NULL;
		}
1844 1845

		if (!sg_res && nr_pages)
1846 1847 1848 1849 1850
			sg = sg_next(sg);
	}
	return 0;
}

1851 1852 1853
static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
				    struct scatterlist *sg, unsigned long nr_pages,
				    int prot)
1854
{
1855 1856
	return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
}
1857

1858 1859 1860 1861 1862
static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
				     unsigned long phys_pfn, unsigned long nr_pages,
				     int prot)
{
	return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1863 1864
}

1865
static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1866
{
1867 1868
	if (!iommu)
		return;
1869 1870 1871

	clear_context_table(iommu, bus, devfn);
	iommu->flush.flush_context(iommu, 0, 0, 0,
1872
					   DMA_CCMD_GLOBAL_INVL);
1873
	iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1874 1875 1876 1877 1878 1879
}

static void domain_remove_dev_info(struct dmar_domain *domain)
{
	struct device_domain_info *info;
	unsigned long flags;
1880
	struct intel_iommu *iommu;
1881 1882 1883 1884 1885 1886 1887 1888

	spin_lock_irqsave(&device_domain_lock, flags);
	while (!list_empty(&domain->devices)) {
		info = list_entry(domain->devices.next,
			struct device_domain_info, link);
		list_del(&info->link);
		list_del(&info->global);
		if (info->dev)
1889
			info->dev->dev.archdata.iommu = NULL;
1890 1891
		spin_unlock_irqrestore(&device_domain_lock, flags);

Y
Yu Zhao 已提交
1892
		iommu_disable_dev_iotlb(info);
1893
		iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1894
		iommu_detach_dev(iommu, info->bus, info->devfn);
1895 1896 1897 1898 1899 1900 1901 1902 1903
		free_devinfo_mem(info);

		spin_lock_irqsave(&device_domain_lock, flags);
	}
	spin_unlock_irqrestore(&device_domain_lock, flags);
}

/*
 * find_domain
1904
 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1905
 */
K
Kay, Allen M 已提交
1906
static struct dmar_domain *
1907 1908 1909 1910 1911
find_domain(struct pci_dev *pdev)
{
	struct device_domain_info *info;

	/* No lock here, assumes no domain exit in normal case */
1912
	info = pdev->dev.archdata.iommu;
1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927
	if (info)
		return info->domain;
	return NULL;
}

/* domain is initialized */
static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
{
	struct dmar_domain *domain, *found = NULL;
	struct intel_iommu *iommu;
	struct dmar_drhd_unit *drhd;
	struct device_domain_info *info, *tmp;
	struct pci_dev *dev_tmp;
	unsigned long flags;
	int bus = 0, devfn = 0;
1928
	int segment;
1929
	int ret;
1930 1931 1932 1933 1934

	domain = find_domain(pdev);
	if (domain)
		return domain;

1935 1936
	segment = pci_domain_nr(pdev->bus);

1937 1938
	dev_tmp = pci_find_upstream_pcie_bridge(pdev);
	if (dev_tmp) {
1939
		if (pci_is_pcie(dev_tmp)) {
1940 1941 1942 1943 1944 1945 1946 1947
			bus = dev_tmp->subordinate->number;
			devfn = 0;
		} else {
			bus = dev_tmp->bus->number;
			devfn = dev_tmp->devfn;
		}
		spin_lock_irqsave(&device_domain_lock, flags);
		list_for_each_entry(info, &device_domain_list, global) {
1948 1949
			if (info->segment == segment &&
			    info->bus == bus && info->devfn == devfn) {
1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
				found = info->domain;
				break;
			}
		}
		spin_unlock_irqrestore(&device_domain_lock, flags);
		/* pcie-pci bridge already has a domain, uses it */
		if (found) {
			domain = found;
			goto found_domain;
		}
	}

1962 1963 1964 1965
	domain = alloc_domain();
	if (!domain)
		goto error;

1966 1967 1968 1969 1970 1971 1972 1973 1974
	/* Allocate new domain for the device */
	drhd = dmar_find_matched_drhd_unit(pdev);
	if (!drhd) {
		printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
			pci_name(pdev));
		return NULL;
	}
	iommu = drhd->iommu;

1975 1976
	ret = iommu_attach_domain(domain, iommu);
	if (ret) {
1977
		free_domain_mem(domain);
1978
		goto error;
1979
	}
1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992

	if (domain_init(domain, gaw)) {
		domain_exit(domain);
		goto error;
	}

	/* register pcie-to-pci device */
	if (dev_tmp) {
		info = alloc_devinfo_mem();
		if (!info) {
			domain_exit(domain);
			goto error;
		}
1993
		info->segment = segment;
1994 1995 1996 1997 1998
		info->bus = bus;
		info->devfn = devfn;
		info->dev = NULL;
		info->domain = domain;
		/* This domain is shared by devices under p2p bridge */
W
Weidong Han 已提交
1999
		domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2000 2001 2002 2003 2004

		/* pcie-to-pci bridge already has a domain, uses it */
		found = NULL;
		spin_lock_irqsave(&device_domain_lock, flags);
		list_for_each_entry(tmp, &device_domain_list, global) {
2005 2006
			if (tmp->segment == segment &&
			    tmp->bus == bus && tmp->devfn == devfn) {
2007 2008 2009 2010 2011
				found = tmp->domain;
				break;
			}
		}
		if (found) {
2012
			spin_unlock_irqrestore(&device_domain_lock, flags);
2013 2014 2015 2016 2017 2018
			free_devinfo_mem(info);
			domain_exit(domain);
			domain = found;
		} else {
			list_add(&info->link, &domain->devices);
			list_add(&info->global, &device_domain_list);
2019
			spin_unlock_irqrestore(&device_domain_lock, flags);
2020 2021 2022 2023 2024 2025 2026
		}
	}

found_domain:
	info = alloc_devinfo_mem();
	if (!info)
		goto error;
2027
	info->segment = segment;
2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045
	info->bus = pdev->bus->number;
	info->devfn = pdev->devfn;
	info->dev = pdev;
	info->domain = domain;
	spin_lock_irqsave(&device_domain_lock, flags);
	/* somebody is fast */
	found = find_domain(pdev);
	if (found != NULL) {
		spin_unlock_irqrestore(&device_domain_lock, flags);
		if (found != domain) {
			domain_exit(domain);
			domain = found;
		}
		free_devinfo_mem(info);
		return domain;
	}
	list_add(&info->link, &domain->devices);
	list_add(&info->global, &device_domain_list);
2046
	pdev->dev.archdata.iommu = info;
2047 2048 2049 2050 2051 2052 2053
	spin_unlock_irqrestore(&device_domain_lock, flags);
	return domain;
error:
	/* recheck it here, maybe others set it */
	return find_domain(pdev);
}

2054
static int iommu_identity_mapping;
2055 2056 2057
#define IDENTMAP_ALL		1
#define IDENTMAP_GFX		2
#define IDENTMAP_AZALIA		4
2058

2059 2060 2061
static int iommu_domain_identity_map(struct dmar_domain *domain,
				     unsigned long long start,
				     unsigned long long end)
2062
{
2063 2064 2065 2066 2067
	unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
	unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;

	if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
			  dma_to_mm_pfn(last_vpfn))) {
2068
		printk(KERN_ERR "IOMMU: reserve iova failed\n");
2069
		return -ENOMEM;
2070 2071
	}

2072 2073
	pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
		 start, end, domain->id);
2074 2075 2076 2077
	/*
	 * RMRR range might have overlap with physical memory range,
	 * clear it first
	 */
2078
	dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2079

2080 2081
	return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
				  last_vpfn - first_vpfn + 1,
2082
				  DMA_PTE_READ|DMA_PTE_WRITE);
2083 2084 2085 2086 2087 2088 2089 2090 2091
}

static int iommu_prepare_identity_map(struct pci_dev *pdev,
				      unsigned long long start,
				      unsigned long long end)
{
	struct dmar_domain *domain;
	int ret;

2092
	domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2093 2094 2095
	if (!domain)
		return -ENOMEM;

2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108
	/* For _hardware_ passthrough, don't bother. But for software
	   passthrough, we do it anyway -- it may indicate a memory
	   range which is reserved in E820, so which didn't get set
	   up to start with in si_domain */
	if (domain == si_domain && hw_pass_through) {
		printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
		       pci_name(pdev), start, end);
		return 0;
	}

	printk(KERN_INFO
	       "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
	       pci_name(pdev), start, end);
2109
	
2110 2111 2112 2113 2114 2115 2116 2117 2118 2119
	if (end < start) {
		WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
			"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
			dmi_get_system_info(DMI_BIOS_VENDOR),
			dmi_get_system_info(DMI_BIOS_VERSION),
		     dmi_get_system_info(DMI_PRODUCT_VERSION));
		ret = -EIO;
		goto error;
	}

2120 2121 2122 2123 2124 2125 2126 2127 2128 2129
	if (end >> agaw_to_width(domain->agaw)) {
		WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
		     "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
		     agaw_to_width(domain->agaw),
		     dmi_get_system_info(DMI_BIOS_VENDOR),
		     dmi_get_system_info(DMI_BIOS_VERSION),
		     dmi_get_system_info(DMI_PRODUCT_VERSION));
		ret = -EIO;
		goto error;
	}
2130

2131
	ret = iommu_domain_identity_map(domain, start, end);
2132 2133 2134 2135
	if (ret)
		goto error;

	/* context entry init */
F
Fenghua Yu 已提交
2136
	ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2137 2138 2139 2140 2141 2142
	if (ret)
		goto error;

	return 0;

 error:
2143 2144 2145 2146 2147 2148 2149
	domain_exit(domain);
	return ret;
}

static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
	struct pci_dev *pdev)
{
2150
	if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2151 2152
		return 0;
	return iommu_prepare_identity_map(pdev, rmrr->base_address,
2153
		rmrr->end_address);
2154 2155
}

2156 2157 2158 2159 2160 2161 2162 2163 2164 2165
#ifdef CONFIG_DMAR_FLOPPY_WA
static inline void iommu_prepare_isa(void)
{
	struct pci_dev *pdev;
	int ret;

	pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
	if (!pdev)
		return;

2166
	printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2167
	ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
2168 2169

	if (ret)
2170 2171
		printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
		       "floppy might not work\n");
2172 2173 2174 2175 2176 2177 2178 2179 2180

}
#else
static inline void iommu_prepare_isa(void)
{
	return;
}
#endif /* !CONFIG_DMAR_FLPY_WA */

2181
static int md_domain_init(struct dmar_domain *domain, int guest_width);
2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194

static int __init si_domain_work_fn(unsigned long start_pfn,
				    unsigned long end_pfn, void *datax)
{
	int *ret = datax;

	*ret = iommu_domain_identity_map(si_domain,
					 (uint64_t)start_pfn << PAGE_SHIFT,
					 (uint64_t)end_pfn << PAGE_SHIFT);
	return *ret;

}

2195
static int __init si_domain_init(int hw)
2196 2197 2198
{
	struct dmar_drhd_unit *drhd;
	struct intel_iommu *iommu;
2199
	int nid, ret = 0;
2200 2201 2202 2203 2204

	si_domain = alloc_domain();
	if (!si_domain)
		return -EFAULT;

2205
	pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221

	for_each_active_iommu(iommu, drhd) {
		ret = iommu_attach_domain(si_domain, iommu);
		if (ret) {
			domain_exit(si_domain);
			return -EFAULT;
		}
	}

	if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
		domain_exit(si_domain);
		return -EFAULT;
	}

	si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;

2222 2223 2224
	if (hw)
		return 0;

2225 2226 2227 2228 2229 2230
	for_each_online_node(nid) {
		work_with_active_regions(nid, si_domain_work_fn, &ret);
		if (ret)
			return ret;
	}

2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242
	return 0;
}

static void domain_remove_one_dev_info(struct dmar_domain *domain,
					  struct pci_dev *pdev);
static int identity_mapping(struct pci_dev *pdev)
{
	struct device_domain_info *info;

	if (likely(!iommu_identity_mapping))
		return 0;

2243 2244 2245
	info = pdev->dev.archdata.iommu;
	if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
		return (info->domain == si_domain);
2246 2247 2248 2249 2250

	return 0;
}

static int domain_add_dev_info(struct dmar_domain *domain,
2251 2252
			       struct pci_dev *pdev,
			       int translation)
2253 2254 2255
{
	struct device_domain_info *info;
	unsigned long flags;
2256
	int ret;
2257 2258 2259 2260 2261

	info = alloc_devinfo_mem();
	if (!info)
		return -ENOMEM;

2262 2263 2264 2265 2266 2267
	ret = domain_context_mapping(domain, pdev, translation);
	if (ret) {
		free_devinfo_mem(info);
		return ret;
	}

2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282
	info->segment = pci_domain_nr(pdev->bus);
	info->bus = pdev->bus->number;
	info->devfn = pdev->devfn;
	info->dev = pdev;
	info->domain = domain;

	spin_lock_irqsave(&device_domain_lock, flags);
	list_add(&info->link, &domain->devices);
	list_add(&info->global, &device_domain_list);
	pdev->dev.archdata.iommu = info;
	spin_unlock_irqrestore(&device_domain_lock, flags);

	return 0;
}

2283 2284
static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
{
2285 2286 2287 2288 2289 2290 2291 2292
	if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
		return 1;

	if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
		return 1;

	if (!(iommu_identity_mapping & IDENTMAP_ALL))
		return 0;
2293

2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310
	/*
	 * We want to start off with all devices in the 1:1 domain, and
	 * take them out later if we find they can't access all of memory.
	 *
	 * However, we can't do this for PCI devices behind bridges,
	 * because all PCI devices behind the same bridge will end up
	 * with the same source-id on their transactions.
	 *
	 * Practically speaking, we can't change things around for these
	 * devices at run-time, because we can't be sure there'll be no
	 * DMA transactions in flight for any of their siblings.
	 * 
	 * So PCI devices (unless they're on the root bus) as well as
	 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
	 * the 1:1 domain, just in _case_ one of their siblings turns out
	 * not to be able to map all of memory.
	 */
2311
	if (!pci_is_pcie(pdev)) {
2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323
		if (!pci_is_root_bus(pdev->bus))
			return 0;
		if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
			return 0;
	} else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
		return 0;

	/* 
	 * At boot time, we don't yet know if devices will be 64-bit capable.
	 * Assume that they will -- if they turn out not to be, then we can 
	 * take them out of the 1:1 domain later.
	 */
2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336
	if (!startup) {
		/*
		 * If the device's dma_mask is less than the system's memory
		 * size then this is not a candidate for identity mapping.
		 */
		u64 dma_mask = pdev->dma_mask;

		if (pdev->dev.coherent_dma_mask &&
		    pdev->dev.coherent_dma_mask < dma_mask)
			dma_mask = pdev->dev.coherent_dma_mask;

		return dma_mask >= dma_get_required_mask(&pdev->dev);
	}
2337 2338 2339 2340

	return 1;
}

2341
static int __init iommu_prepare_static_identity_mapping(int hw)
2342 2343 2344 2345
{
	struct pci_dev *pdev = NULL;
	int ret;

2346
	ret = si_domain_init(hw);
2347 2348 2349 2350
	if (ret)
		return -EFAULT;

	for_each_pci_dev(pdev) {
2351 2352 2353
		/* Skip Host/PCI Bridge devices */
		if (IS_BRIDGE_HOST_DEVICE(pdev))
			continue;
2354
		if (iommu_should_identity_map(pdev, 1)) {
2355 2356
			printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
			       hw ? "hardware" : "software", pci_name(pdev));
2357

2358
			ret = domain_add_dev_info(si_domain, pdev,
2359
						     hw ? CONTEXT_TT_PASS_THROUGH :
2360 2361 2362 2363
						     CONTEXT_TT_MULTI_LEVEL);
			if (ret)
				return ret;
		}
2364 2365 2366 2367 2368
	}

	return 0;
}

2369
static int __init init_dmars(void)
2370 2371 2372 2373 2374
{
	struct dmar_drhd_unit *drhd;
	struct dmar_rmrr_unit *rmrr;
	struct pci_dev *pdev;
	struct intel_iommu *iommu;
2375
	int i, ret;
2376

2377 2378 2379 2380 2381 2382 2383
	/*
	 * for each drhd
	 *    allocate root
	 *    initialize and program root entry to not present
	 * endfor
	 */
	for_each_drhd_unit(drhd) {
M
mark gross 已提交
2384 2385 2386 2387 2388 2389 2390 2391
		g_num_of_iommus++;
		/*
		 * lock not needed as this is only incremented in the single
		 * threaded kernel __init code path all other access are read
		 * only
		 */
	}

W
Weidong Han 已提交
2392 2393 2394 2395 2396 2397 2398 2399
	g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
			GFP_KERNEL);
	if (!g_iommus) {
		printk(KERN_ERR "Allocating global iommu array failed\n");
		ret = -ENOMEM;
		goto error;
	}

2400 2401 2402
	deferred_flush = kzalloc(g_num_of_iommus *
		sizeof(struct deferred_flush_tables), GFP_KERNEL);
	if (!deferred_flush) {
M
mark gross 已提交
2403 2404 2405 2406 2407 2408 2409
		ret = -ENOMEM;
		goto error;
	}

	for_each_drhd_unit(drhd) {
		if (drhd->ignored)
			continue;
2410 2411

		iommu = drhd->iommu;
W
Weidong Han 已提交
2412
		g_iommus[iommu->seq_id] = iommu;
2413

2414 2415 2416 2417
		ret = iommu_init_domains(iommu);
		if (ret)
			goto error;

2418 2419 2420
		/*
		 * TBD:
		 * we could share the same root & context tables
L
Lucas De Marchi 已提交
2421
		 * among all IOMMU's. Need to Split it later.
2422 2423 2424 2425 2426 2427
		 */
		ret = iommu_alloc_root_entry(iommu);
		if (ret) {
			printk(KERN_ERR "IOMMU: allocate root entry failed\n");
			goto error;
		}
F
Fenghua Yu 已提交
2428
		if (!ecap_pass_through(iommu->ecap))
2429
			hw_pass_through = 0;
2430 2431
	}

2432 2433 2434
	/*
	 * Start from the sane iommu hardware state.
	 */
2435 2436 2437 2438 2439
	for_each_drhd_unit(drhd) {
		if (drhd->ignored)
			continue;

		iommu = drhd->iommu;
2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465

		/*
		 * If the queued invalidation is already initialized by us
		 * (for example, while enabling interrupt-remapping) then
		 * we got the things already rolling from a sane state.
		 */
		if (iommu->qi)
			continue;

		/*
		 * Clear any previous faults.
		 */
		dmar_fault(-1, iommu);
		/*
		 * Disable queued invalidation if supported and already enabled
		 * before OS handover.
		 */
		dmar_disable_qi(iommu);
	}

	for_each_drhd_unit(drhd) {
		if (drhd->ignored)
			continue;

		iommu = drhd->iommu;

2466 2467 2468 2469 2470 2471 2472
		if (dmar_enable_qi(iommu)) {
			/*
			 * Queued Invalidate not enabled, use Register Based
			 * Invalidate
			 */
			iommu->flush.flush_context = __iommu_flush_context;
			iommu->flush.flush_iotlb = __iommu_flush_iotlb;
Y
Yinghai Lu 已提交
2473
			printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2474
			       "invalidation\n",
Y
Yinghai Lu 已提交
2475
				iommu->seq_id,
2476
			       (unsigned long long)drhd->reg_base_addr);
2477 2478 2479
		} else {
			iommu->flush.flush_context = qi_flush_context;
			iommu->flush.flush_iotlb = qi_flush_iotlb;
Y
Yinghai Lu 已提交
2480
			printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2481
			       "invalidation\n",
Y
Yinghai Lu 已提交
2482
				iommu->seq_id,
2483
			       (unsigned long long)drhd->reg_base_addr);
2484 2485 2486
		}
	}

2487
	if (iommu_pass_through)
2488 2489
		iommu_identity_mapping |= IDENTMAP_ALL;

2490
#ifdef CONFIG_DMAR_BROKEN_GFX_WA
2491
	iommu_identity_mapping |= IDENTMAP_GFX;
2492
#endif
2493 2494 2495

	check_tylersburg_isoch();

2496
	/*
2497 2498 2499
	 * If pass through is not set or not enabled, setup context entries for
	 * identity mappings for rmrr, gfx, and isa and may fall back to static
	 * identity mapping if iommu_identity_mapping is set.
2500
	 */
2501 2502
	if (iommu_identity_mapping) {
		ret = iommu_prepare_static_identity_mapping(hw_pass_through);
F
Fenghua Yu 已提交
2503
		if (ret) {
2504 2505
			printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
			goto error;
2506 2507 2508
		}
	}
	/*
2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520
	 * For each rmrr
	 *   for each dev attached to rmrr
	 *   do
	 *     locate drhd for dev, alloc domain for dev
	 *     allocate free domain
	 *     allocate page table entries for rmrr
	 *     if context not allocated for bus
	 *           allocate and init context
	 *           set present in root table for this bus
	 *     init context with domain, translation etc
	 *    endfor
	 * endfor
2521
	 */
2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535
	printk(KERN_INFO "IOMMU: Setting RMRR:\n");
	for_each_rmrr_units(rmrr) {
		for (i = 0; i < rmrr->devices_cnt; i++) {
			pdev = rmrr->devices[i];
			/*
			 * some BIOS lists non-exist devices in DMAR
			 * table.
			 */
			if (!pdev)
				continue;
			ret = iommu_prepare_rmrr_dev(rmrr, pdev);
			if (ret)
				printk(KERN_ERR
				       "IOMMU: mapping reserved region failed\n");
2536
		}
F
Fenghua Yu 已提交
2537
	}
2538

2539 2540
	iommu_prepare_isa();

2541 2542 2543 2544 2545 2546 2547 2548
	/*
	 * for each drhd
	 *   enable fault log
	 *   global invalidate context cache
	 *   global invalidate iotlb
	 *   enable translation
	 */
	for_each_drhd_unit(drhd) {
2549 2550 2551 2552 2553 2554 2555
		if (drhd->ignored) {
			/*
			 * we always have to disable PMRs or DMA may fail on
			 * this device
			 */
			if (force_on)
				iommu_disable_protect_mem_regions(drhd->iommu);
2556
			continue;
2557
		}
2558 2559 2560 2561
		iommu = drhd->iommu;

		iommu_flush_write_buffer(iommu);

2562 2563 2564 2565
		ret = dmar_set_interrupt(iommu);
		if (ret)
			goto error;

2566 2567
		iommu_set_root_entry(iommu);

2568
		iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2569
		iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
M
mark gross 已提交
2570

2571 2572 2573
		ret = iommu_enable_translation(iommu);
		if (ret)
			goto error;
2574 2575

		iommu_disable_protect_mem_regions(iommu);
2576 2577 2578 2579 2580 2581 2582 2583 2584 2585
	}

	return 0;
error:
	for_each_drhd_unit(drhd) {
		if (drhd->ignored)
			continue;
		iommu = drhd->iommu;
		free_iommu(iommu);
	}
W
Weidong Han 已提交
2586
	kfree(g_iommus);
2587 2588 2589
	return ret;
}

2590
/* This takes a number of _MM_ pages, not VTD pages */
2591 2592 2593
static struct iova *intel_alloc_iova(struct device *dev,
				     struct dmar_domain *domain,
				     unsigned long nrpages, uint64_t dma_mask)
2594 2595 2596 2597
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct iova *iova = NULL;

2598 2599 2600 2601
	/* Restrict dma_mask to the width that the iommu can handle */
	dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);

	if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2602 2603
		/*
		 * First try to allocate an io virtual address in
2604
		 * DMA_BIT_MASK(32) and if that fails then try allocating
J
Joe Perches 已提交
2605
		 * from higher range
2606
		 */
2607 2608 2609 2610 2611 2612 2613 2614 2615
		iova = alloc_iova(&domain->iovad, nrpages,
				  IOVA_PFN(DMA_BIT_MASK(32)), 1);
		if (iova)
			return iova;
	}
	iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
	if (unlikely(!iova)) {
		printk(KERN_ERR "Allocating %ld-page iova for %s failed",
		       nrpages, pci_name(pdev));
2616 2617 2618 2619 2620 2621
		return NULL;
	}

	return iova;
}

2622
static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2623 2624 2625 2626 2627 2628 2629 2630 2631
{
	struct dmar_domain *domain;
	int ret;

	domain = get_domain_for_dev(pdev,
			DEFAULT_DOMAIN_ADDRESS_WIDTH);
	if (!domain) {
		printk(KERN_ERR
			"Allocating domain for %s failed", pci_name(pdev));
A
Al Viro 已提交
2632
		return NULL;
2633 2634 2635
	}

	/* make sure context mapping is ok */
W
Weidong Han 已提交
2636
	if (unlikely(!domain_context_mapped(pdev))) {
F
Fenghua Yu 已提交
2637 2638
		ret = domain_context_mapping(domain, pdev,
					     CONTEXT_TT_MULTI_LEVEL);
2639 2640 2641 2642
		if (ret) {
			printk(KERN_ERR
				"Domain context map for %s failed",
				pci_name(pdev));
A
Al Viro 已提交
2643
			return NULL;
2644
		}
2645 2646
	}

2647 2648 2649
	return domain;
}

2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661
static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
{
	struct device_domain_info *info;

	/* No lock here, assumes no domain exit in normal case */
	info = dev->dev.archdata.iommu;
	if (likely(info))
		return info->domain;

	return __get_valid_domain_for_dev(dev);
}

2662 2663 2664 2665 2666 2667
static int iommu_dummy(struct pci_dev *pdev)
{
	return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
}

/* Check if the pdev needs to go through non-identity map and unmap process.*/
2668
static int iommu_no_mapping(struct device *dev)
2669
{
2670
	struct pci_dev *pdev;
2671 2672
	int found;

2673 2674 2675 2676
	if (unlikely(dev->bus != &pci_bus_type))
		return 1;

	pdev = to_pci_dev(dev);
2677 2678 2679
	if (iommu_dummy(pdev))
		return 1;

2680
	if (!iommu_identity_mapping)
2681
		return 0;
2682 2683 2684

	found = identity_mapping(pdev);
	if (found) {
2685
		if (iommu_should_identity_map(pdev, 0))
2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701
			return 1;
		else {
			/*
			 * 32 bit DMA is removed from si_domain and fall back
			 * to non-identity mapping.
			 */
			domain_remove_one_dev_info(si_domain, pdev);
			printk(KERN_INFO "32bit %s uses non-identity mapping\n",
			       pci_name(pdev));
			return 0;
		}
	} else {
		/*
		 * In case of a detached 64 bit DMA device from vm, the device
		 * is put into si_domain for identity mapping.
		 */
2702
		if (iommu_should_identity_map(pdev, 0)) {
2703
			int ret;
2704 2705 2706 2707
			ret = domain_add_dev_info(si_domain, pdev,
						  hw_pass_through ?
						  CONTEXT_TT_PASS_THROUGH :
						  CONTEXT_TT_MULTI_LEVEL);
2708 2709 2710 2711 2712 2713 2714 2715
			if (!ret) {
				printk(KERN_INFO "64bit %s uses identity mapping\n",
				       pci_name(pdev));
				return 1;
			}
		}
	}

2716
	return 0;
2717 2718
}

2719 2720
static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
				     size_t size, int dir, u64 dma_mask)
2721 2722 2723
{
	struct pci_dev *pdev = to_pci_dev(hwdev);
	struct dmar_domain *domain;
F
Fenghua Yu 已提交
2724
	phys_addr_t start_paddr;
2725 2726
	struct iova *iova;
	int prot = 0;
I
Ingo Molnar 已提交
2727
	int ret;
2728
	struct intel_iommu *iommu;
2729
	unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2730 2731

	BUG_ON(dir == DMA_NONE);
2732

2733
	if (iommu_no_mapping(hwdev))
I
Ingo Molnar 已提交
2734
		return paddr;
2735 2736 2737 2738 2739

	domain = get_valid_domain_for_dev(pdev);
	if (!domain)
		return 0;

2740
	iommu = domain_get_iommu(domain);
2741
	size = aligned_nrpages(paddr, size);
2742

2743
	iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
2744 2745 2746
	if (!iova)
		goto error;

2747 2748 2749 2750 2751
	/*
	 * Check if DMAR supports zero-length reads on write only
	 * mappings..
	 */
	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2752
			!cap_zlr(iommu->cap))
2753 2754 2755 2756
		prot |= DMA_PTE_READ;
	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
		prot |= DMA_PTE_WRITE;
	/*
I
Ingo Molnar 已提交
2757
	 * paddr - (paddr + size) might be partial page, we should map the whole
2758
	 * page.  Note: if two part of one page are separately mapped, we
I
Ingo Molnar 已提交
2759
	 * might have two guest_addr mapping to the same host paddr, but this
2760 2761
	 * is not a big problem
	 */
2762
	ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2763
				 mm_to_dma_pfn(paddr_pfn), size, prot);
2764 2765 2766
	if (ret)
		goto error;

2767 2768
	/* it's a non-present to present mapping. Only flush if caching mode */
	if (cap_caching_mode(iommu->cap))
2769
		iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2770
	else
2771
		iommu_flush_write_buffer(iommu);
2772

2773 2774 2775
	start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
	start_paddr += paddr & ~PAGE_MASK;
	return start_paddr;
2776 2777

error:
2778 2779
	if (iova)
		__free_iova(&domain->iovad, iova);
2780
	printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
F
Fenghua Yu 已提交
2781
		pci_name(pdev), size, (unsigned long long)paddr, dir);
2782 2783 2784
	return 0;
}

2785 2786 2787 2788
static dma_addr_t intel_map_page(struct device *dev, struct page *page,
				 unsigned long offset, size_t size,
				 enum dma_data_direction dir,
				 struct dma_attrs *attrs)
2789
{
2790 2791
	return __intel_map_single(dev, page_to_phys(page) + offset, size,
				  dir, to_pci_dev(dev)->dma_mask);
2792 2793
}

M
mark gross 已提交
2794 2795
static void flush_unmaps(void)
{
2796
	int i, j;
M
mark gross 已提交
2797 2798 2799 2800 2801

	timer_on = 0;

	/* just flush them all */
	for (i = 0; i < g_num_of_iommus; i++) {
2802 2803 2804
		struct intel_iommu *iommu = g_iommus[i];
		if (!iommu)
			continue;
2805

2806 2807 2808
		if (!deferred_flush[i].next)
			continue;

2809 2810 2811
		/* In caching mode, global flushes turn emulation expensive */
		if (!cap_caching_mode(iommu->cap))
			iommu->flush.flush_iotlb(iommu, 0, 0, 0,
Y
Yu Zhao 已提交
2812
					 DMA_TLB_GLOBAL_FLUSH);
2813
		for (j = 0; j < deferred_flush[i].next; j++) {
Y
Yu Zhao 已提交
2814 2815
			unsigned long mask;
			struct iova *iova = deferred_flush[i].iova[j];
2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826
			struct dmar_domain *domain = deferred_flush[i].domain[j];

			/* On real hardware multiple invalidations are expensive */
			if (cap_caching_mode(iommu->cap))
				iommu_flush_iotlb_psi(iommu, domain->id,
				iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
			else {
				mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
				iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
						(uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
			}
Y
Yu Zhao 已提交
2827
			__free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2828
		}
2829
		deferred_flush[i].next = 0;
M
mark gross 已提交
2830 2831 2832 2833 2834 2835 2836
	}

	list_size = 0;
}

static void flush_unmaps_timeout(unsigned long data)
{
2837 2838 2839
	unsigned long flags;

	spin_lock_irqsave(&async_umap_flush_lock, flags);
M
mark gross 已提交
2840
	flush_unmaps();
2841
	spin_unlock_irqrestore(&async_umap_flush_lock, flags);
M
mark gross 已提交
2842 2843 2844 2845 2846
}

static void add_unmap(struct dmar_domain *dom, struct iova *iova)
{
	unsigned long flags;
2847
	int next, iommu_id;
2848
	struct intel_iommu *iommu;
M
mark gross 已提交
2849 2850

	spin_lock_irqsave(&async_umap_flush_lock, flags);
2851 2852 2853
	if (list_size == HIGH_WATER_MARK)
		flush_unmaps();

2854 2855
	iommu = domain_get_iommu(dom);
	iommu_id = iommu->seq_id;
2856

2857 2858 2859 2860
	next = deferred_flush[iommu_id].next;
	deferred_flush[iommu_id].domain[next] = dom;
	deferred_flush[iommu_id].iova[next] = iova;
	deferred_flush[iommu_id].next++;
M
mark gross 已提交
2861 2862 2863 2864 2865 2866 2867 2868 2869

	if (!timer_on) {
		mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
		timer_on = 1;
	}
	list_size++;
	spin_unlock_irqrestore(&async_umap_flush_lock, flags);
}

2870 2871 2872
static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
			     size_t size, enum dma_data_direction dir,
			     struct dma_attrs *attrs)
2873 2874
{
	struct pci_dev *pdev = to_pci_dev(dev);
2875
	struct dmar_domain *domain;
2876
	unsigned long start_pfn, last_pfn;
2877
	struct iova *iova;
2878
	struct intel_iommu *iommu;
2879

2880
	if (iommu_no_mapping(dev))
2881
		return;
2882

2883 2884 2885
	domain = find_domain(pdev);
	BUG_ON(!domain);

2886 2887
	iommu = domain_get_iommu(domain);

2888
	iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2889 2890
	if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
		      (unsigned long long)dev_addr))
2891 2892
		return;

2893 2894
	start_pfn = mm_to_dma_pfn(iova->pfn_lo);
	last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2895

2896 2897
	pr_debug("Device %s unmapping: pfn %lx-%lx\n",
		 pci_name(pdev), start_pfn, last_pfn);
2898

2899
	/*  clear the whole page */
2900 2901
	dma_pte_clear_range(domain, start_pfn, last_pfn);

2902
	/* free page tables */
2903 2904
	dma_pte_free_pagetable(domain, start_pfn, last_pfn);

M
mark gross 已提交
2905
	if (intel_iommu_strict) {
2906
		iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2907
				      last_pfn - start_pfn + 1, 0);
M
mark gross 已提交
2908 2909 2910 2911 2912 2913 2914 2915 2916
		/* free iova */
		__free_iova(&domain->iovad, iova);
	} else {
		add_unmap(domain, iova);
		/*
		 * queue up the release of the unmap to save the 1/6th of the
		 * cpu used up by the iotlb flush operation...
		 */
	}
2917 2918
}

2919 2920
static void *intel_alloc_coherent(struct device *hwdev, size_t size,
				  dma_addr_t *dma_handle, gfp_t flags)
2921 2922 2923 2924
{
	void *vaddr;
	int order;

F
Fenghua Yu 已提交
2925
	size = PAGE_ALIGN(size);
2926
	order = get_order(size);
2927 2928 2929 2930 2931 2932 2933 2934 2935

	if (!iommu_no_mapping(hwdev))
		flags &= ~(GFP_DMA | GFP_DMA32);
	else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
		if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
			flags |= GFP_DMA;
		else
			flags |= GFP_DMA32;
	}
2936 2937 2938 2939 2940 2941

	vaddr = (void *)__get_free_pages(flags, order);
	if (!vaddr)
		return NULL;
	memset(vaddr, 0, size);

2942 2943 2944
	*dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
					 DMA_BIDIRECTIONAL,
					 hwdev->coherent_dma_mask);
2945 2946 2947 2948 2949 2950
	if (*dma_handle)
		return vaddr;
	free_pages((unsigned long)vaddr, order);
	return NULL;
}

2951 2952
static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
				dma_addr_t dma_handle)
2953 2954 2955
{
	int order;

F
Fenghua Yu 已提交
2956
	size = PAGE_ALIGN(size);
2957 2958
	order = get_order(size);

2959
	intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2960 2961 2962
	free_pages((unsigned long)vaddr, order);
}

2963 2964 2965
static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
			   int nelems, enum dma_data_direction dir,
			   struct dma_attrs *attrs)
2966 2967 2968
{
	struct pci_dev *pdev = to_pci_dev(hwdev);
	struct dmar_domain *domain;
2969
	unsigned long start_pfn, last_pfn;
2970
	struct iova *iova;
2971
	struct intel_iommu *iommu;
2972

2973
	if (iommu_no_mapping(hwdev))
2974 2975 2976
		return;

	domain = find_domain(pdev);
2977 2978 2979
	BUG_ON(!domain);

	iommu = domain_get_iommu(domain);
2980

F
FUJITA Tomonori 已提交
2981
	iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2982 2983
	if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
		      (unsigned long long)sglist[0].dma_address))
2984 2985
		return;

2986 2987
	start_pfn = mm_to_dma_pfn(iova->pfn_lo);
	last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2988 2989

	/*  clear the whole page */
2990 2991
	dma_pte_clear_range(domain, start_pfn, last_pfn);

2992
	/* free page tables */
2993
	dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2994

2995 2996
	if (intel_iommu_strict) {
		iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2997
				      last_pfn - start_pfn + 1, 0);
2998 2999 3000 3001 3002 3003 3004 3005 3006
		/* free iova */
		__free_iova(&domain->iovad, iova);
	} else {
		add_unmap(domain, iova);
		/*
		 * queue up the release of the unmap to save the 1/6th of the
		 * cpu used up by the iotlb flush operation...
		 */
	}
3007 3008 3009
}

static int intel_nontranslate_map_sg(struct device *hddev,
F
FUJITA Tomonori 已提交
3010
	struct scatterlist *sglist, int nelems, int dir)
3011 3012
{
	int i;
F
FUJITA Tomonori 已提交
3013
	struct scatterlist *sg;
3014

F
FUJITA Tomonori 已提交
3015
	for_each_sg(sglist, sg, nelems, i) {
F
FUJITA Tomonori 已提交
3016
		BUG_ON(!sg_page(sg));
3017
		sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
F
FUJITA Tomonori 已提交
3018
		sg->dma_length = sg->length;
3019 3020 3021 3022
	}
	return nelems;
}

3023 3024
static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
			enum dma_data_direction dir, struct dma_attrs *attrs)
3025 3026 3027 3028
{
	int i;
	struct pci_dev *pdev = to_pci_dev(hwdev);
	struct dmar_domain *domain;
3029 3030 3031 3032
	size_t size = 0;
	int prot = 0;
	struct iova *iova = NULL;
	int ret;
F
FUJITA Tomonori 已提交
3033
	struct scatterlist *sg;
3034
	unsigned long start_vpfn;
3035
	struct intel_iommu *iommu;
3036 3037

	BUG_ON(dir == DMA_NONE);
3038
	if (iommu_no_mapping(hwdev))
F
FUJITA Tomonori 已提交
3039
		return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
3040

3041 3042 3043 3044
	domain = get_valid_domain_for_dev(pdev);
	if (!domain)
		return 0;

3045 3046
	iommu = domain_get_iommu(domain);

3047
	for_each_sg(sglist, sg, nelems, i)
3048
		size += aligned_nrpages(sg->offset, sg->length);
3049

3050 3051
	iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
				pdev->dma_mask);
3052
	if (!iova) {
F
FUJITA Tomonori 已提交
3053
		sglist->dma_length = 0;
3054 3055 3056 3057 3058 3059 3060 3061
		return 0;
	}

	/*
	 * Check if DMAR supports zero-length reads on write only
	 * mappings..
	 */
	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3062
			!cap_zlr(iommu->cap))
3063 3064 3065 3066
		prot |= DMA_PTE_READ;
	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
		prot |= DMA_PTE_WRITE;

3067
	start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3068

3069
	ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3070 3071 3072 3073 3074 3075 3076 3077 3078 3079
	if (unlikely(ret)) {
		/*  clear the page */
		dma_pte_clear_range(domain, start_vpfn,
				    start_vpfn + size - 1);
		/* free page tables */
		dma_pte_free_pagetable(domain, start_vpfn,
				       start_vpfn + size - 1);
		/* free iova */
		__free_iova(&domain->iovad, iova);
		return 0;
3080 3081
	}

3082 3083
	/* it's a non-present to present mapping. Only flush if caching mode */
	if (cap_caching_mode(iommu->cap))
3084
		iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
3085
	else
3086
		iommu_flush_write_buffer(iommu);
3087

3088 3089 3090
	return nelems;
}

3091 3092 3093 3094 3095
static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
	return !dma_addr;
}

3096
struct dma_map_ops intel_dma_ops = {
3097 3098 3099 3100
	.alloc_coherent = intel_alloc_coherent,
	.free_coherent = intel_free_coherent,
	.map_sg = intel_map_sg,
	.unmap_sg = intel_unmap_sg,
3101 3102
	.map_page = intel_map_page,
	.unmap_page = intel_unmap_page,
3103
	.mapping_error = intel_mapping_error,
3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187
};

static inline int iommu_domain_cache_init(void)
{
	int ret = 0;

	iommu_domain_cache = kmem_cache_create("iommu_domain",
					 sizeof(struct dmar_domain),
					 0,
					 SLAB_HWCACHE_ALIGN,

					 NULL);
	if (!iommu_domain_cache) {
		printk(KERN_ERR "Couldn't create iommu_domain cache\n");
		ret = -ENOMEM;
	}

	return ret;
}

static inline int iommu_devinfo_cache_init(void)
{
	int ret = 0;

	iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
					 sizeof(struct device_domain_info),
					 0,
					 SLAB_HWCACHE_ALIGN,
					 NULL);
	if (!iommu_devinfo_cache) {
		printk(KERN_ERR "Couldn't create devinfo cache\n");
		ret = -ENOMEM;
	}

	return ret;
}

static inline int iommu_iova_cache_init(void)
{
	int ret = 0;

	iommu_iova_cache = kmem_cache_create("iommu_iova",
					 sizeof(struct iova),
					 0,
					 SLAB_HWCACHE_ALIGN,
					 NULL);
	if (!iommu_iova_cache) {
		printk(KERN_ERR "Couldn't create iova cache\n");
		ret = -ENOMEM;
	}

	return ret;
}

static int __init iommu_init_mempool(void)
{
	int ret;
	ret = iommu_iova_cache_init();
	if (ret)
		return ret;

	ret = iommu_domain_cache_init();
	if (ret)
		goto domain_error;

	ret = iommu_devinfo_cache_init();
	if (!ret)
		return ret;

	kmem_cache_destroy(iommu_domain_cache);
domain_error:
	kmem_cache_destroy(iommu_iova_cache);

	return -ENOMEM;
}

static void __init iommu_exit_mempool(void)
{
	kmem_cache_destroy(iommu_devinfo_cache);
	kmem_cache_destroy(iommu_domain_cache);
	kmem_cache_destroy(iommu_iova_cache);

}

3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215
static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
{
	struct dmar_drhd_unit *drhd;
	u32 vtbar;
	int rc;

	/* We know that this device on this chipset has its own IOMMU.
	 * If we find it under a different IOMMU, then the BIOS is lying
	 * to us. Hope that the IOMMU for this device is actually
	 * disabled, and it needs no translation...
	 */
	rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
	if (rc) {
		/* "can't" happen */
		dev_info(&pdev->dev, "failed to run vt-d quirk\n");
		return;
	}
	vtbar &= 0xffff0000;

	/* we know that the this iommu should be at offset 0xa000 from vtbar */
	drhd = dmar_find_matched_drhd_unit(pdev);
	if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
			    TAINT_FIRMWARE_WORKAROUND,
			    "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
		pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
}
DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);

3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238
static void __init init_no_remapping_devices(void)
{
	struct dmar_drhd_unit *drhd;

	for_each_drhd_unit(drhd) {
		if (!drhd->include_all) {
			int i;
			for (i = 0; i < drhd->devices_cnt; i++)
				if (drhd->devices[i] != NULL)
					break;
			/* ignore DMAR unit if no pci devices exist */
			if (i == drhd->devices_cnt)
				drhd->ignored = 1;
		}
	}

	for_each_drhd_unit(drhd) {
		int i;
		if (drhd->ignored || drhd->include_all)
			continue;

		for (i = 0; i < drhd->devices_cnt; i++)
			if (drhd->devices[i] &&
3239
			    !IS_GFX_DEVICE(drhd->devices[i]))
3240 3241 3242 3243 3244
				break;

		if (i < drhd->devices_cnt)
			continue;

3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255
		/* This IOMMU has *only* gfx devices. Either bypass it or
		   set the gfx_mapped flag, as appropriate */
		if (dmar_map_gfx) {
			intel_iommu_gfx_mapped = 1;
		} else {
			drhd->ignored = 1;
			for (i = 0; i < drhd->devices_cnt; i++) {
				if (!drhd->devices[i])
					continue;
				drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
			}
3256 3257 3258 3259
		}
	}
}

3260 3261 3262 3263 3264 3265 3266 3267 3268 3269
#ifdef CONFIG_SUSPEND
static int init_iommu_hw(void)
{
	struct dmar_drhd_unit *drhd;
	struct intel_iommu *iommu = NULL;

	for_each_active_iommu(iommu, drhd)
		if (iommu->qi)
			dmar_reenable_qi(iommu);

3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280
	for_each_iommu(iommu, drhd) {
		if (drhd->ignored) {
			/*
			 * we always have to disable PMRs or DMA may fail on
			 * this device
			 */
			if (force_on)
				iommu_disable_protect_mem_regions(iommu);
			continue;
		}
	
3281 3282 3283 3284 3285
		iommu_flush_write_buffer(iommu);

		iommu_set_root_entry(iommu);

		iommu->flush.flush_context(iommu, 0, 0, 0,
3286
					   DMA_CCMD_GLOBAL_INVL);
3287
		iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3288
					 DMA_TLB_GLOBAL_FLUSH);
3289 3290
		if (iommu_enable_translation(iommu))
			return 1;
3291
		iommu_disable_protect_mem_regions(iommu);
3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303
	}

	return 0;
}

static void iommu_flush_all(void)
{
	struct dmar_drhd_unit *drhd;
	struct intel_iommu *iommu;

	for_each_active_iommu(iommu, drhd) {
		iommu->flush.flush_context(iommu, 0, 0, 0,
3304
					   DMA_CCMD_GLOBAL_INVL);
3305
		iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3306
					 DMA_TLB_GLOBAL_FLUSH);
3307 3308 3309
	}
}

3310
static int iommu_suspend(void)
3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349
{
	struct dmar_drhd_unit *drhd;
	struct intel_iommu *iommu = NULL;
	unsigned long flag;

	for_each_active_iommu(iommu, drhd) {
		iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
						 GFP_ATOMIC);
		if (!iommu->iommu_state)
			goto nomem;
	}

	iommu_flush_all();

	for_each_active_iommu(iommu, drhd) {
		iommu_disable_translation(iommu);

		spin_lock_irqsave(&iommu->register_lock, flag);

		iommu->iommu_state[SR_DMAR_FECTL_REG] =
			readl(iommu->reg + DMAR_FECTL_REG);
		iommu->iommu_state[SR_DMAR_FEDATA_REG] =
			readl(iommu->reg + DMAR_FEDATA_REG);
		iommu->iommu_state[SR_DMAR_FEADDR_REG] =
			readl(iommu->reg + DMAR_FEADDR_REG);
		iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
			readl(iommu->reg + DMAR_FEUADDR_REG);

		spin_unlock_irqrestore(&iommu->register_lock, flag);
	}
	return 0;

nomem:
	for_each_active_iommu(iommu, drhd)
		kfree(iommu->iommu_state);

	return -ENOMEM;
}

3350
static void iommu_resume(void)
3351 3352 3353 3354 3355 3356
{
	struct dmar_drhd_unit *drhd;
	struct intel_iommu *iommu = NULL;
	unsigned long flag;

	if (init_iommu_hw()) {
3357 3358 3359 3360
		if (force_on)
			panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
		else
			WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3361
		return;
3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383
	}

	for_each_active_iommu(iommu, drhd) {

		spin_lock_irqsave(&iommu->register_lock, flag);

		writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
			iommu->reg + DMAR_FECTL_REG);
		writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
			iommu->reg + DMAR_FEDATA_REG);
		writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
			iommu->reg + DMAR_FEADDR_REG);
		writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
			iommu->reg + DMAR_FEUADDR_REG);

		spin_unlock_irqrestore(&iommu->register_lock, flag);
	}

	for_each_active_iommu(iommu, drhd)
		kfree(iommu->iommu_state);
}

3384
static struct syscore_ops iommu_syscore_ops = {
3385 3386 3387 3388
	.resume		= iommu_resume,
	.suspend	= iommu_suspend,
};

3389
static void __init init_iommu_pm_ops(void)
3390
{
3391
	register_syscore_ops(&iommu_syscore_ops);
3392 3393 3394
}

#else
3395
static inline void init_iommu_pm_ops(void) {}
3396 3397
#endif	/* CONFIG_PM */

F
Fenghua Yu 已提交
3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410
/*
 * Here we only respond to action of unbound device from driver.
 *
 * Added device is not attached to its DMAR domain here yet. That will happen
 * when mapping the device to iova.
 */
static int device_notifier(struct notifier_block *nb,
				  unsigned long action, void *data)
{
	struct device *dev = data;
	struct pci_dev *pdev = to_pci_dev(dev);
	struct dmar_domain *domain;

3411 3412 3413
	if (iommu_no_mapping(dev))
		return 0;

F
Fenghua Yu 已提交
3414 3415 3416 3417
	domain = find_domain(pdev);
	if (!domain)
		return 0;

3418
	if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
F
Fenghua Yu 已提交
3419 3420
		domain_remove_one_dev_info(domain, pdev);

3421 3422 3423 3424 3425 3426
		if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
		    !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
		    list_empty(&domain->devices))
			domain_exit(domain);
	}

F
Fenghua Yu 已提交
3427 3428 3429 3430 3431 3432 3433
	return 0;
}

static struct notifier_block device_nb = {
	.notifier_call = device_notifier,
};

3434 3435 3436 3437
int __init intel_iommu_init(void)
{
	int ret = 0;

3438 3439 3440 3441 3442 3443
	/* VT-d is required for a TXT/tboot launch, so enforce that */
	force_on = tboot_force_iommu();

	if (dmar_table_init()) {
		if (force_on)
			panic("tboot: Failed to initialize DMAR table\n");
3444
		return 	-ENODEV;
3445
	}
3446

3447 3448 3449
	if (dmar_dev_scope_init()) {
		if (force_on)
			panic("tboot: Failed to initialize DMAR device scope\n");
3450
		return 	-ENODEV;
3451
	}
3452

3453 3454 3455 3456
	/*
	 * Check the need for DMA-remapping initialization now.
	 * Above initialization will also be used by Interrupt-remapping.
	 */
3457
	if (no_iommu || dmar_disabled)
3458 3459
		return -ENODEV;

3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470
	if (iommu_init_mempool()) {
		if (force_on)
			panic("tboot: Failed to initialize iommu memory\n");
		return 	-ENODEV;
	}

	if (dmar_init_reserved_ranges()) {
		if (force_on)
			panic("tboot: Failed to reserve iommu ranges\n");
		return 	-ENODEV;
	}
3471 3472 3473

	init_no_remapping_devices();

3474
	ret = init_dmars();
3475
	if (ret) {
3476 3477
		if (force_on)
			panic("tboot: Failed to initialize DMARs\n");
3478 3479 3480 3481 3482 3483 3484 3485
		printk(KERN_ERR "IOMMU: dmar init failed\n");
		put_iova_domain(&reserved_iova_list);
		iommu_exit_mempool();
		return ret;
	}
	printk(KERN_INFO
	"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");

M
mark gross 已提交
3486
	init_timer(&unmap_timer);
3487 3488 3489
#ifdef CONFIG_SWIOTLB
	swiotlb = 0;
#endif
3490
	dma_ops = &intel_dma_ops;
F
Fenghua Yu 已提交
3491

3492
	init_iommu_pm_ops();
3493 3494 3495

	register_iommu(&intel_iommu_ops);

F
Fenghua Yu 已提交
3496 3497
	bus_register_notifier(&pci_bus_type, &device_nb);

3498 3499
	return 0;
}
3500

3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515
static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
					   struct pci_dev *pdev)
{
	struct pci_dev *tmp, *parent;

	if (!iommu || !pdev)
		return;

	/* dependent device detach */
	tmp = pci_find_upstream_pcie_bridge(pdev);
	/* Secondary interface's bus number and devfn 0 */
	if (tmp) {
		parent = pdev->bus->self;
		while (parent != tmp) {
			iommu_detach_dev(iommu, parent->bus->number,
3516
					 parent->devfn);
3517 3518
			parent = parent->bus->self;
		}
3519
		if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3520 3521 3522
			iommu_detach_dev(iommu,
				tmp->subordinate->number, 0);
		else /* this is a legacy PCI bridge */
3523 3524
			iommu_detach_dev(iommu, tmp->bus->number,
					 tmp->devfn);
3525 3526 3527
	}
}

3528
static void domain_remove_one_dev_info(struct dmar_domain *domain,
3529 3530 3531 3532 3533 3534 3535 3536
					  struct pci_dev *pdev)
{
	struct device_domain_info *info;
	struct intel_iommu *iommu;
	unsigned long flags;
	int found = 0;
	struct list_head *entry, *tmp;

3537 3538
	iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
				pdev->devfn);
3539 3540 3541 3542 3543 3544
	if (!iommu)
		return;

	spin_lock_irqsave(&device_domain_lock, flags);
	list_for_each_safe(entry, tmp, &domain->devices) {
		info = list_entry(entry, struct device_domain_info, link);
3545 3546
		if (info->segment == pci_domain_nr(pdev->bus) &&
		    info->bus == pdev->bus->number &&
3547 3548 3549 3550 3551 3552 3553
		    info->devfn == pdev->devfn) {
			list_del(&info->link);
			list_del(&info->global);
			if (info->dev)
				info->dev->dev.archdata.iommu = NULL;
			spin_unlock_irqrestore(&device_domain_lock, flags);

Y
Yu Zhao 已提交
3554
			iommu_disable_dev_iotlb(info);
3555
			iommu_detach_dev(iommu, info->bus, info->devfn);
3556
			iommu_detach_dependent_devices(iommu, pdev);
3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570
			free_devinfo_mem(info);

			spin_lock_irqsave(&device_domain_lock, flags);

			if (found)
				break;
			else
				continue;
		}

		/* if there is no other devices under the same iommu
		 * owned by this domain, clear this iommu in iommu_bmp
		 * update iommu count and coherency
		 */
3571 3572
		if (iommu == device_to_iommu(info->segment, info->bus,
					    info->devfn))
3573 3574 3575
			found = 1;
	}

3576 3577
	spin_unlock_irqrestore(&device_domain_lock, flags);

3578 3579 3580 3581 3582
	if (found == 0) {
		unsigned long tmp_flags;
		spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
		clear_bit(iommu->seq_id, &domain->iommu_bmp);
		domain->iommu_count--;
3583
		domain_update_iommu_cap(domain);
3584
		spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3585

3586 3587 3588 3589 3590 3591 3592
		if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
		    !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
			spin_lock_irqsave(&iommu->lock, tmp_flags);
			clear_bit(domain->id, iommu->domain_ids);
			iommu->domains[domain->id] = NULL;
			spin_unlock_irqrestore(&iommu->lock, tmp_flags);
		}
3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612
	}
}

static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
{
	struct device_domain_info *info;
	struct intel_iommu *iommu;
	unsigned long flags1, flags2;

	spin_lock_irqsave(&device_domain_lock, flags1);
	while (!list_empty(&domain->devices)) {
		info = list_entry(domain->devices.next,
			struct device_domain_info, link);
		list_del(&info->link);
		list_del(&info->global);
		if (info->dev)
			info->dev->dev.archdata.iommu = NULL;

		spin_unlock_irqrestore(&device_domain_lock, flags1);

Y
Yu Zhao 已提交
3613
		iommu_disable_dev_iotlb(info);
3614
		iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3615
		iommu_detach_dev(iommu, info->bus, info->devfn);
3616
		iommu_detach_dependent_devices(iommu, info->dev);
3617 3618

		/* clear this iommu in iommu_bmp, update iommu count
3619
		 * and capabilities
3620 3621 3622 3623 3624
		 */
		spin_lock_irqsave(&domain->iommu_lock, flags2);
		if (test_and_clear_bit(iommu->seq_id,
				       &domain->iommu_bmp)) {
			domain->iommu_count--;
3625
			domain_update_iommu_cap(domain);
3626 3627 3628 3629 3630 3631 3632 3633 3634
		}
		spin_unlock_irqrestore(&domain->iommu_lock, flags2);

		free_devinfo_mem(info);
		spin_lock_irqsave(&device_domain_lock, flags1);
	}
	spin_unlock_irqrestore(&device_domain_lock, flags1);
}

3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646
/* domain id for virtual machine, it won't be set in context */
static unsigned long vm_domid;

static struct dmar_domain *iommu_alloc_vm_domain(void)
{
	struct dmar_domain *domain;

	domain = alloc_domain_mem();
	if (!domain)
		return NULL;

	domain->id = vm_domid++;
3647
	domain->nid = -1;
3648 3649 3650 3651 3652 3653
	memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
	domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;

	return domain;
}

3654
static int md_domain_init(struct dmar_domain *domain, int guest_width)
3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671
{
	int adjust_width;

	init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
	spin_lock_init(&domain->iommu_lock);

	domain_reserve_special_ranges(domain);

	/* calculate AGAW */
	domain->gaw = guest_width;
	adjust_width = guestwidth_to_adjustwidth(guest_width);
	domain->agaw = width_to_agaw(adjust_width);

	INIT_LIST_HEAD(&domain->devices);

	domain->iommu_count = 0;
	domain->iommu_coherency = 0;
3672
	domain->iommu_snooping = 0;
3673
	domain->iommu_superpage = 0;
3674
	domain->max_addr = 0;
3675
	domain->nid = -1;
3676 3677

	/* always allocate the top pgd */
3678
	domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698
	if (!domain->pgd)
		return -ENOMEM;
	domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
	return 0;
}

static void iommu_free_vm_domain(struct dmar_domain *domain)
{
	unsigned long flags;
	struct dmar_drhd_unit *drhd;
	struct intel_iommu *iommu;
	unsigned long i;
	unsigned long ndomains;

	for_each_drhd_unit(drhd) {
		if (drhd->ignored)
			continue;
		iommu = drhd->iommu;

		ndomains = cap_ndoms(iommu->cap);
3699
		for_each_set_bit(i, iommu->domain_ids, ndomains) {
3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721
			if (iommu->domains[i] == domain) {
				spin_lock_irqsave(&iommu->lock, flags);
				clear_bit(i, iommu->domain_ids);
				iommu->domains[i] = NULL;
				spin_unlock_irqrestore(&iommu->lock, flags);
				break;
			}
		}
	}
}

static void vm_domain_exit(struct dmar_domain *domain)
{
	/* Domain 0 is reserved, so dont process it */
	if (!domain)
		return;

	vm_domain_remove_all_dev_info(domain);
	/* destroy iovas */
	put_iova_domain(&domain->iovad);

	/* clear ptes */
3722
	dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3723 3724

	/* free page tables */
3725
	dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3726 3727 3728 3729 3730

	iommu_free_vm_domain(domain);
	free_domain_mem(domain);
}

3731
static int intel_iommu_domain_init(struct iommu_domain *domain)
K
Kay, Allen M 已提交
3732
{
3733
	struct dmar_domain *dmar_domain;
K
Kay, Allen M 已提交
3734

3735 3736
	dmar_domain = iommu_alloc_vm_domain();
	if (!dmar_domain) {
K
Kay, Allen M 已提交
3737
		printk(KERN_ERR
3738 3739
			"intel_iommu_domain_init: dmar_domain == NULL\n");
		return -ENOMEM;
K
Kay, Allen M 已提交
3740
	}
3741
	if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
K
Kay, Allen M 已提交
3742
		printk(KERN_ERR
3743 3744 3745
			"intel_iommu_domain_init() failed\n");
		vm_domain_exit(dmar_domain);
		return -ENOMEM;
K
Kay, Allen M 已提交
3746
	}
3747
	domain->priv = dmar_domain;
3748

3749
	return 0;
K
Kay, Allen M 已提交
3750 3751
}

3752
static void intel_iommu_domain_destroy(struct iommu_domain *domain)
K
Kay, Allen M 已提交
3753
{
3754 3755 3756 3757
	struct dmar_domain *dmar_domain = domain->priv;

	domain->priv = NULL;
	vm_domain_exit(dmar_domain);
K
Kay, Allen M 已提交
3758 3759
}

3760 3761
static int intel_iommu_attach_device(struct iommu_domain *domain,
				     struct device *dev)
K
Kay, Allen M 已提交
3762
{
3763 3764
	struct dmar_domain *dmar_domain = domain->priv;
	struct pci_dev *pdev = to_pci_dev(dev);
3765 3766
	struct intel_iommu *iommu;
	int addr_width;
3767 3768 3769 3770 3771 3772 3773

	/* normally pdev is not mapped */
	if (unlikely(domain_context_mapped(pdev))) {
		struct dmar_domain *old_domain;

		old_domain = find_domain(pdev);
		if (old_domain) {
3774 3775 3776
			if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
			    dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
				domain_remove_one_dev_info(old_domain, pdev);
3777 3778 3779 3780 3781
			else
				domain_remove_dev_info(old_domain);
		}
	}

3782 3783
	iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
				pdev->devfn);
3784 3785 3786 3787 3788
	if (!iommu)
		return -ENODEV;

	/* check if this iommu agaw is sufficient for max mapped address */
	addr_width = agaw_to_width(iommu->agaw);
3789 3790 3791 3792 3793
	if (addr_width > cap_mgaw(iommu->cap))
		addr_width = cap_mgaw(iommu->cap);

	if (dmar_domain->max_addr > (1LL << addr_width)) {
		printk(KERN_ERR "%s: iommu width (%d) is not "
3794
		       "sufficient for the mapped address (%llx)\n",
3795
		       __func__, addr_width, dmar_domain->max_addr);
3796 3797
		return -EFAULT;
	}
3798 3799 3800 3801 3802 3803 3804 3805 3806 3807
	dmar_domain->gaw = addr_width;

	/*
	 * Knock out extra levels of page tables if necessary
	 */
	while (iommu->agaw < dmar_domain->agaw) {
		struct dma_pte *pte;

		pte = dmar_domain->pgd;
		if (dma_pte_present(pte)) {
3808 3809
			dmar_domain->pgd = (struct dma_pte *)
				phys_to_virt(dma_pte_addr(pte));
3810
			free_pgtable_page(pte);
3811 3812 3813
		}
		dmar_domain->agaw--;
	}
3814

3815
	return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
K
Kay, Allen M 已提交
3816 3817
}

3818 3819
static void intel_iommu_detach_device(struct iommu_domain *domain,
				      struct device *dev)
K
Kay, Allen M 已提交
3820
{
3821 3822 3823
	struct dmar_domain *dmar_domain = domain->priv;
	struct pci_dev *pdev = to_pci_dev(dev);

3824
	domain_remove_one_dev_info(dmar_domain, pdev);
3825
}
3826

3827 3828 3829
static int intel_iommu_map(struct iommu_domain *domain,
			   unsigned long iova, phys_addr_t hpa,
			   int gfp_order, int iommu_prot)
3830
{
3831
	struct dmar_domain *dmar_domain = domain->priv;
3832
	u64 max_addr;
3833
	int prot = 0;
3834
	size_t size;
3835
	int ret;
3836

3837 3838 3839 3840
	if (iommu_prot & IOMMU_READ)
		prot |= DMA_PTE_READ;
	if (iommu_prot & IOMMU_WRITE)
		prot |= DMA_PTE_WRITE;
3841 3842
	if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
		prot |= DMA_PTE_SNP;
3843

3844
	size     = PAGE_SIZE << gfp_order;
3845
	max_addr = iova + size;
3846
	if (dmar_domain->max_addr < max_addr) {
3847 3848 3849
		u64 end;

		/* check if minimum agaw is sufficient for mapped address */
3850
		end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
3851
		if (end < max_addr) {
3852
			printk(KERN_ERR "%s: iommu width (%d) is not "
3853
			       "sufficient for the mapped address (%llx)\n",
3854
			       __func__, dmar_domain->gaw, max_addr);
3855 3856
			return -EFAULT;
		}
3857
		dmar_domain->max_addr = max_addr;
3858
	}
3859 3860
	/* Round up size to next multiple of PAGE_SIZE, if it and
	   the low bits of hpa would take us onto the next page */
3861
	size = aligned_nrpages(hpa, size);
3862 3863
	ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
				 hpa >> VTD_PAGE_SHIFT, size, prot);
3864
	return ret;
K
Kay, Allen M 已提交
3865 3866
}

3867 3868
static int intel_iommu_unmap(struct iommu_domain *domain,
			     unsigned long iova, int gfp_order)
K
Kay, Allen M 已提交
3869
{
3870
	struct dmar_domain *dmar_domain = domain->priv;
3871
	size_t size = PAGE_SIZE << gfp_order;
3872

3873 3874
	dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
			    (iova + size - 1) >> VTD_PAGE_SHIFT);
3875

3876 3877
	if (dmar_domain->max_addr == iova + size)
		dmar_domain->max_addr = iova;
3878 3879

	return gfp_order;
K
Kay, Allen M 已提交
3880 3881
}

3882 3883
static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
					    unsigned long iova)
K
Kay, Allen M 已提交
3884
{
3885
	struct dmar_domain *dmar_domain = domain->priv;
K
Kay, Allen M 已提交
3886
	struct dma_pte *pte;
3887
	u64 phys = 0;
K
Kay, Allen M 已提交
3888

3889
	pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
K
Kay, Allen M 已提交
3890
	if (pte)
3891
		phys = dma_pte_addr(pte);
K
Kay, Allen M 已提交
3892

3893
	return phys;
K
Kay, Allen M 已提交
3894
}
3895

S
Sheng Yang 已提交
3896 3897 3898 3899 3900 3901 3902
static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
				      unsigned long cap)
{
	struct dmar_domain *dmar_domain = domain->priv;

	if (cap == IOMMU_CAP_CACHE_COHERENCY)
		return dmar_domain->iommu_snooping;
3903 3904
	if (cap == IOMMU_CAP_INTR_REMAP)
		return intr_remapping_enabled;
S
Sheng Yang 已提交
3905 3906 3907 3908

	return 0;
}

3909 3910 3911 3912 3913
static struct iommu_ops intel_iommu_ops = {
	.domain_init	= intel_iommu_domain_init,
	.domain_destroy = intel_iommu_domain_destroy,
	.attach_dev	= intel_iommu_attach_device,
	.detach_dev	= intel_iommu_detach_device,
3914 3915
	.map		= intel_iommu_map,
	.unmap		= intel_iommu_unmap,
3916
	.iova_to_phys	= intel_iommu_iova_to_phys,
S
Sheng Yang 已提交
3917
	.domain_has_cap = intel_iommu_domain_has_cap,
3918
};
3919 3920 3921 3922 3923 3924 3925 3926 3927

static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
{
	/*
	 * Mobile 4 Series Chipset neglects to set RWBF capability,
	 * but needs it:
	 */
	printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
	rwbf_quirk = 1;
3928 3929 3930 3931 3932 3933

	/* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
	if (dev->revision == 0x07) {
		printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
		dmar_map_gfx = 0;
	}
3934 3935 3936
}

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
3937

3938 3939 3940 3941 3942 3943 3944 3945 3946 3947
#define GGC 0x52
#define GGC_MEMORY_SIZE_MASK	(0xf << 8)
#define GGC_MEMORY_SIZE_NONE	(0x0 << 8)
#define GGC_MEMORY_SIZE_1M	(0x1 << 8)
#define GGC_MEMORY_SIZE_2M	(0x3 << 8)
#define GGC_MEMORY_VT_ENABLED	(0x8 << 8)
#define GGC_MEMORY_SIZE_2M_VT	(0x9 << 8)
#define GGC_MEMORY_SIZE_3M_VT	(0xa << 8)
#define GGC_MEMORY_SIZE_4M_VT	(0xb << 8)

3948 3949 3950 3951
static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
{
	unsigned short ggc;

3952
	if (pci_read_config_word(dev, GGC, &ggc))
3953 3954
		return;

3955
	if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
3956 3957
		printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
		dmar_map_gfx = 0;
3958 3959 3960 3961 3962
	} else if (dmar_map_gfx) {
		/* we have to ensure the gfx device is idle before we flush */
		printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
		intel_iommu_strict = 1;
       }
3963 3964 3965 3966 3967 3968
}
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);

3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025
/* On Tylersburg chipsets, some BIOSes have been known to enable the
   ISOCH DMAR unit for the Azalia sound device, but not give it any
   TLB entries, which causes it to deadlock. Check for that.  We do
   this in a function called from init_dmars(), instead of in a PCI
   quirk, because we don't want to print the obnoxious "BIOS broken"
   message if VT-d is actually disabled.
*/
static void __init check_tylersburg_isoch(void)
{
	struct pci_dev *pdev;
	uint32_t vtisochctrl;

	/* If there's no Azalia in the system anyway, forget it. */
	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
	if (!pdev)
		return;
	pci_dev_put(pdev);

	/* System Management Registers. Might be hidden, in which case
	   we can't do the sanity check. But that's OK, because the
	   known-broken BIOSes _don't_ actually hide it, so far. */
	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
	if (!pdev)
		return;

	if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
		pci_dev_put(pdev);
		return;
	}

	pci_dev_put(pdev);

	/* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
	if (vtisochctrl & 1)
		return;

	/* Drop all bits other than the number of TLB entries */
	vtisochctrl &= 0x1c;

	/* If we have the recommended number of TLB entries (16), fine. */
	if (vtisochctrl == 0x10)
		return;

	/* Zero TLB entries? You get to ride the short bus to school. */
	if (!vtisochctrl) {
		WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
		     "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
		     dmi_get_system_info(DMI_BIOS_VENDOR),
		     dmi_get_system_info(DMI_BIOS_VERSION),
		     dmi_get_system_info(DMI_PRODUCT_VERSION));
		iommu_identity_mapping |= IDENTMAP_AZALIA;
		return;
	}
	
	printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
	       vtisochctrl);
}