pci-dma.c 9.5 KB
Newer Older
G
Glauber Costa 已提交
1
#include <linux/dma-mapping.h>
2
#include <linux/dmar.h>
3
#include <linux/bootmem.h>
G
Glauber Costa 已提交
4
#include <linux/pci.h>
5

6 7
#include <asm/proto.h>
#include <asm/dma.h>
8
#include <asm/iommu.h>
9
#include <asm/calgary.h>
10
#include <asm/amd_iommu.h>
G
Glauber Costa 已提交
11

12
static int forbid_dac __read_mostly;
G
Glauber Costa 已提交
13

14
struct dma_mapping_ops *dma_ops;
15 16
EXPORT_SYMBOL(dma_ops);

17
static int iommu_sac_force __read_mostly;
G
Glauber Costa 已提交
18

G
Glauber Costa 已提交
19 20 21 22 23 24 25 26
#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly = 0;
#endif

27 28 29 30 31 32 33 34 35 36 37
int iommu_merge __read_mostly = 0;

int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

/* This tells the BIO block layer to assume merging. Default to off
   because we cannot guarantee merging later. */
int iommu_bio_merge __read_mostly = 0;
EXPORT_SYMBOL(iommu_bio_merge);

G
Glauber Costa 已提交
38 39
dma_addr_t bad_dma_address __read_mostly = 0;
EXPORT_SYMBOL(bad_dma_address);
40

G
Glauber Costa 已提交
41 42 43 44 45 46 47 48 49
/* Dummy device used for NULL arguments (normally ISA). Better would
   be probably a smaller DMA mask, but this is bug-to-bug compatible
   to older i386. */
struct device fallback_dev = {
	.bus_id = "fallback device",
	.coherent_dma_mask = DMA_32BIT_MASK,
	.dma_mask = &fallback_dev.coherent_dma_mask,
};

G
Glauber Costa 已提交
50 51 52 53 54 55 56 57 58 59 60
int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;

	*dev->dma_mask = mask;

	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
#ifdef CONFIG_X86_64
static __initdata void *dma32_bootmem_ptr;
static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);

static int __init parse_dma32_size_opt(char *p)
{
	if (!p)
		return -EINVAL;
	dma32_bootmem_size = memparse(p, &p);
	return 0;
}
early_param("dma32_size", parse_dma32_size_opt);

void __init dma32_reserve_bootmem(void)
{
	unsigned long size, align;
Y
Yinghai Lu 已提交
77
	if (max_pfn <= MAX_DMA32_PFN)
78 79
		return;

80 81 82 83
	/*
	 * check aperture_64.c allocate_aperture() for reason about
	 * using 512M as goal
	 */
84 85 86
	align = 64ULL<<20;
	size = round_up(dma32_bootmem_size, align);
	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
87
				 512ULL<<20);
88 89 90 91 92 93 94 95
	if (dma32_bootmem_ptr)
		dma32_bootmem_size = size;
	else
		dma32_bootmem_size = 0;
}
static void __init dma32_free_bootmem(void)
{

Y
Yinghai Lu 已提交
96
	if (max_pfn <= MAX_DMA32_PFN)
97 98 99 100 101
		return;

	if (!dma32_bootmem_ptr)
		return;

102
	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121

	dma32_bootmem_ptr = NULL;
	dma32_bootmem_size = 0;
}

void __init pci_iommu_alloc(void)
{
	/* free the range so iommu could get some range less than 4G */
	dma32_free_bootmem();
	/*
	 * The order of these functions is important for
	 * fall-back/fail-over reasons
	 */
	gart_iommu_hole_init();

	detect_calgary();

	detect_intel_iommu();

122 123
	amd_iommu_detect();

124 125
	pci_swiotlb_init();
}
126 127 128 129 130 131 132 133

unsigned long iommu_num_pages(unsigned long addr, unsigned long len)
{
	unsigned long size = roundup((addr & ~PAGE_MASK) + len, PAGE_SIZE);

	return size >> PAGE_SHIFT;
}
EXPORT_SYMBOL(iommu_num_pages);
134 135
#endif

136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
/*
 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 * documentation.
 */
static __init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p, "off", 3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p, "force", 5))
			force_iommu = 1;
		if (!strncmp(p, "noforce", 7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge", 8)) {
			iommu_bio_merge = 4096;
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic", 5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic", 7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge", 5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge", 7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac", 8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
			forbid_dac = -1;
		if (!strncmp(p, "usedac", 6)) {
			forbid_dac = -1;
			return 1;
		}
#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft", 4))
			swiotlb = 1;
#endif

		gart_parse_options(p);

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

G
Glauber Costa 已提交
203 204
int dma_supported(struct device *dev, u64 mask)
{
205 206
	struct dma_mapping_ops *ops = get_dma_ops(dev);

G
Glauber Costa 已提交
207 208
#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {
209
		dev_info(dev, "PCI: Disallowing DAC for device\n");
G
Glauber Costa 已提交
210 211 212 213
		return 0;
	}
#endif

214 215
	if (ops->dma_supported)
		return ops->dma_supported(dev, mask);
G
Glauber Costa 已提交
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235

	/* Copied from i386. Doesn't make much sense, because it will
	   only work for pci_alloc_coherent.
	   The caller just has to use GFP_DMA in this case. */
	if (mask < DMA_24BIT_MASK)
		return 0;

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
236
		dev_info(dev, "Force SAC with mask %Lx\n", mask);
G
Glauber Costa 已提交
237 238 239 240 241 242 243
		return 0;
	}

	return 1;
}
EXPORT_SYMBOL(dma_supported);

G
Glauber Costa 已提交
244
/* Allocate DMA memory on node near device */
A
Adrian Bunk 已提交
245
static noinline struct page *
G
Glauber Costa 已提交
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
{
	int node;

	node = dev_to_node(dev);

	return alloc_pages_node(node, gfp, order);
}

/*
 * Allocate memory for a coherent mapping.
 */
void *
dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
		   gfp_t gfp)
{
262
	struct dma_mapping_ops *ops = get_dma_ops(dev);
G
Glauber Costa 已提交
263 264 265 266
	void *memory = NULL;
	struct page *page;
	unsigned long dma_mask = 0;
	dma_addr_t bus;
267
	int noretry = 0;
G
Glauber Costa 已提交
268 269 270 271

	/* ignore region specifiers */
	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);

272
	if (dma_alloc_from_coherent(dev, size, dma_handle, &memory))
G
Glauber Costa 已提交
273 274
		return memory;

T
Takashi Iwai 已提交
275
	if (!dev) {
G
Glauber Costa 已提交
276
		dev = &fallback_dev;
T
Takashi Iwai 已提交
277 278
		gfp |= GFP_DMA;
	}
G
Glauber Costa 已提交
279 280
	dma_mask = dev->coherent_dma_mask;
	if (dma_mask == 0)
T
Takashi Iwai 已提交
281
		dma_mask = (gfp & GFP_DMA) ? DMA_24BIT_MASK : DMA_32BIT_MASK;
G
Glauber Costa 已提交
282 283 284 285 286

	/* Device not DMA able */
	if (dev->dma_mask == NULL)
		return NULL;

287 288 289 290
	/* Don't invoke OOM killer or retry in lower 16MB DMA zone */
	if (gfp & __GFP_DMA)
		noretry = 1;

G
Glauber Costa 已提交
291 292 293 294 295
#ifdef CONFIG_X86_64
	/* Why <=? Even when the mask is smaller than 4GB it is often
	   larger than 16MB and in this case we have a chance of
	   finding fitting memory in the next higher zone first. If
	   not retry with true GFP_DMA. -AK */
296
	if (dma_mask <= DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
G
Glauber Costa 已提交
297
		gfp |= GFP_DMA32;
298 299 300
		if (dma_mask < DMA_32BIT_MASK)
			noretry = 1;
	}
G
Glauber Costa 已提交
301 302 303
#endif

 again:
304
	page = dma_alloc_pages(dev,
305
		noretry ? gfp | __GFP_NORETRY : gfp, get_order(size));
G
Glauber Costa 已提交
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
	if (page == NULL)
		return NULL;

	{
		int high, mmu;
		bus = page_to_phys(page);
		memory = page_address(page);
		high = (bus + size) >= dma_mask;
		mmu = high;
		if (force_iommu && !(gfp & GFP_DMA))
			mmu = 1;
		else if (high) {
			free_pages((unsigned long)memory,
				   get_order(size));

			/* Don't use the 16MB ZONE_DMA unless absolutely
			   needed. It's better to use remapping first. */
			if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
				gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
				goto again;
			}

			/* Let low level make its own zone decisions */
			gfp &= ~(GFP_DMA32|GFP_DMA);

331 332
			if (ops->alloc_coherent)
				return ops->alloc_coherent(dev, size,
G
Glauber Costa 已提交
333 334 335 336 337 338 339 340 341 342 343
							   dma_handle, gfp);
			return NULL;
		}

		memset(memory, 0, size);
		if (!mmu) {
			*dma_handle = bus;
			return memory;
		}
	}

344
	if (ops->alloc_coherent) {
G
Glauber Costa 已提交
345 346
		free_pages((unsigned long)memory, get_order(size));
		gfp &= ~(GFP_DMA|GFP_DMA32);
347
		return ops->alloc_coherent(dev, size, dma_handle, gfp);
G
Glauber Costa 已提交
348 349
	}

350 351
	if (ops->map_simple) {
		*dma_handle = ops->map_simple(dev, virt_to_phys(memory),
G
Glauber Costa 已提交
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
					      size,
					      PCI_DMA_BIDIRECTIONAL);
		if (*dma_handle != bad_dma_address)
			return memory;
	}

	if (panic_on_overflow)
		panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",
		      (unsigned long)size);
	free_pages((unsigned long)memory, get_order(size));
	return NULL;
}
EXPORT_SYMBOL(dma_alloc_coherent);

/*
 * Unmap coherent memory.
 * The caller must ensure that the device has finished accessing the mapping.
 */
void dma_free_coherent(struct device *dev, size_t size,
			 void *vaddr, dma_addr_t bus)
{
373 374
	struct dma_mapping_ops *ops = get_dma_ops(dev);

G
Glauber Costa 已提交
375 376
	int order = get_order(size);
	WARN_ON(irqs_disabled());	/* for portability */
377
	if (dma_release_from_coherent(dev, order, vaddr))
G
Glauber Costa 已提交
378
		return;
379 380
	if (ops->unmap_single)
		ops->unmap_single(dev, bus, size, 0);
G
Glauber Costa 已提交
381 382 383
	free_pages((unsigned long)vaddr, order);
}
EXPORT_SYMBOL(dma_free_coherent);
G
Glauber Costa 已提交
384

385 386 387 388 389 390
static int __init pci_iommu_init(void)
{
	calgary_iommu_init();

	intel_iommu_init();

391 392
	amd_iommu_init();

393
	gart_iommu_init();
G
Glauber Costa 已提交
394

395 396 397 398 399 400 401 402 403 404
	no_iommu_init();
	return 0;
}

void pci_iommu_shutdown(void)
{
	gart_iommu_shutdown();
}
/* Must execute after PCI subsystem */
fs_initcall(pci_iommu_init);
G
Glauber Costa 已提交
405 406 407 408 409 410 411 412 413 414 415 416 417 418

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

static __devinit void via_no_dac(struct pci_dev *dev)
{
	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
		printk(KERN_INFO "PCI: VIA PCI bridge detected."
				 "Disabling DAC.\n");
		forbid_dac = 1;
	}
}
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
#endif