dma-mapping.c 54.5 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2
 *  linux/arch/arm/mm/dma-mapping.c
L
Linus Torvalds 已提交
3 4 5 6 7 8 9 10 11
 *
 *  Copyright (C) 2000-2004 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 *  DMA uncached mapping support.
 */
12
#include <linux/bootmem.h>
L
Linus Torvalds 已提交
13 14
#include <linux/module.h>
#include <linux/mm.h>
15
#include <linux/gfp.h>
L
Linus Torvalds 已提交
16 17 18 19 20
#include <linux/errno.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
21
#include <linux/dma-contiguous.h>
22
#include <linux/highmem.h>
23
#include <linux/memblock.h>
24
#include <linux/slab.h>
25
#include <linux/iommu.h>
26
#include <linux/io.h>
27
#include <linux/vmalloc.h>
28
#include <linux/sizes.h>
29
#include <linux/cma.h>
L
Linus Torvalds 已提交
30

31
#include <asm/memory.h>
32
#include <asm/highmem.h>
L
Linus Torvalds 已提交
33 34
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
35
#include <asm/mach/arch.h>
36
#include <asm/dma-iommu.h>
37 38 39
#include <asm/mach/map.h>
#include <asm/system_info.h>
#include <asm/dma-contiguous.h>
40

41 42
#include "mm.h"

43 44 45 46 47 48 49 50 51 52 53 54
/*
 * The DMA API is built upon the notion of "buffer ownership".  A buffer
 * is either exclusively owned by the CPU (and therefore may be accessed
 * by it) or exclusively owned by the DMA device.  These helper functions
 * represent the transitions between these two ownership states.
 *
 * Note, however, that on later ARMs, this notion does not work due to
 * speculative prefetches.  We model our approach on the assumption that
 * the CPU does do speculative prefetches, which means we clean caches
 * before transfers and delay cache invalidation until transfer completion.
 *
 */
55
static void __dma_page_cpu_to_dev(struct page *, unsigned long,
56
		size_t, enum dma_data_direction);
57
static void __dma_page_dev_to_cpu(struct page *, unsigned long,
58 59
		size_t, enum dma_data_direction);

60 61 62 63 64 65 66 67 68 69 70 71 72 73
/**
 * arm_dma_map_page - map a portion of a page for streaming DMA
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @page: page that buffer resides in
 * @offset: offset into page for start of buffer
 * @size: size of buffer to map
 * @dir: DMA transfer direction
 *
 * Ensure that any data held in the cache is appropriately discarded
 * or written back.
 *
 * The device owns this memory once this call has completed.  The CPU
 * can regain ownership by calling dma_unmap_page().
 */
74
static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
75 76 77
	     unsigned long offset, size_t size, enum dma_data_direction dir,
	     struct dma_attrs *attrs)
{
R
Rob Herring 已提交
78
	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
79 80
		__dma_page_cpu_to_dev(page, offset, size, dir);
	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
81 82
}

R
Rob Herring 已提交
83 84 85 86 87 88 89
static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
	     unsigned long offset, size_t size, enum dma_data_direction dir,
	     struct dma_attrs *attrs)
{
	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
}

90 91 92 93 94 95 96 97 98 99 100 101 102 103
/**
 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @handle: DMA address of buffer
 * @size: size of buffer (same as passed to dma_map_page)
 * @dir: DMA transfer direction (same as passed to dma_map_page)
 *
 * Unmap a page streaming mode DMA translation.  The handle and size
 * must match what was provided in the previous dma_map_page() call.
 * All other usages are undefined.
 *
 * After this call, reads by the CPU to the buffer are guaranteed to see
 * whatever the device wrote there.
 */
104
static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
105 106 107
		size_t size, enum dma_data_direction dir,
		struct dma_attrs *attrs)
{
R
Rob Herring 已提交
108
	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
109 110
		__dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
				      handle & ~PAGE_MASK, size, dir);
111 112
}

113
static void arm_dma_sync_single_for_cpu(struct device *dev,
114 115 116 117
		dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
	unsigned int offset = handle & (PAGE_SIZE - 1);
	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
R
Rob Herring 已提交
118
	__dma_page_dev_to_cpu(page, offset, size, dir);
119 120
}

121
static void arm_dma_sync_single_for_device(struct device *dev,
122 123 124 125
		dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
	unsigned int offset = handle & (PAGE_SIZE - 1);
	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
R
Rob Herring 已提交
126
	__dma_page_cpu_to_dev(page, offset, size, dir);
127 128 129
}

struct dma_map_ops arm_dma_ops = {
130 131 132
	.alloc			= arm_dma_alloc,
	.free			= arm_dma_free,
	.mmap			= arm_dma_mmap,
133
	.get_sgtable		= arm_dma_get_sgtable,
134 135 136 137 138 139 140 141 142 143 144 145
	.map_page		= arm_dma_map_page,
	.unmap_page		= arm_dma_unmap_page,
	.map_sg			= arm_dma_map_sg,
	.unmap_sg		= arm_dma_unmap_sg,
	.sync_single_for_cpu	= arm_dma_sync_single_for_cpu,
	.sync_single_for_device	= arm_dma_sync_single_for_device,
	.sync_sg_for_cpu	= arm_dma_sync_sg_for_cpu,
	.sync_sg_for_device	= arm_dma_sync_sg_for_device,
	.set_dma_mask		= arm_dma_set_mask,
};
EXPORT_SYMBOL(arm_dma_ops);

R
Rob Herring 已提交
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
	dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs);
static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
				  dma_addr_t handle, struct dma_attrs *attrs);

struct dma_map_ops arm_coherent_dma_ops = {
	.alloc			= arm_coherent_dma_alloc,
	.free			= arm_coherent_dma_free,
	.mmap			= arm_dma_mmap,
	.get_sgtable		= arm_dma_get_sgtable,
	.map_page		= arm_coherent_dma_map_page,
	.map_sg			= arm_dma_map_sg,
	.set_dma_mask		= arm_dma_set_mask,
};
EXPORT_SYMBOL(arm_coherent_dma_ops);

162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
static int __dma_supported(struct device *dev, u64 mask, bool warn)
{
	unsigned long max_dma_pfn;

	/*
	 * If the mask allows for more memory than we can address,
	 * and we actually have that much memory, then we must
	 * indicate that DMA to this device is not supported.
	 */
	if (sizeof(mask) != sizeof(dma_addr_t) &&
	    mask > (dma_addr_t)~0 &&
	    dma_to_pfn(dev, ~0) < max_pfn) {
		if (warn) {
			dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n",
				 mask);
			dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n");
		}
		return 0;
	}

	max_dma_pfn = min(max_pfn, arm_dma_pfn_limit);

	/*
	 * Translate the device's DMA mask to a PFN limit.  This
	 * PFN number includes the page which we can DMA to.
	 */
	if (dma_to_pfn(dev, mask) < max_dma_pfn) {
		if (warn)
			dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n",
				 mask,
				 dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1,
				 max_dma_pfn + 1);
		return 0;
	}

	return 1;
}

200 201
static u64 get_coherent_dma_mask(struct device *dev)
{
202
	u64 mask = (u64)DMA_BIT_MASK(32);
203 204 205 206 207 208 209 210 211 212 213 214 215

	if (dev) {
		mask = dev->coherent_dma_mask;

		/*
		 * Sanity check the DMA mask - it must be non-zero, and
		 * must be able to be satisfied by a DMA allocation.
		 */
		if (mask == 0) {
			dev_warn(dev, "coherent DMA mask is unset\n");
			return 0;
		}

216
		if (!__dma_supported(dev, mask, true))
217 218
			return 0;
	}
L
Linus Torvalds 已提交
219

220 221 222
	return mask;
}

223 224 225 226 227 228
static void __dma_clear_buffer(struct page *page, size_t size)
{
	/*
	 * Ensure that the allocated pages are zeroed, and that any data
	 * lurking in the kernel direct-mapped region is invalidated.
	 */
229 230 231 232 233 234 235 236 237 238 239 240 241 242
	if (PageHighMem(page)) {
		phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
		phys_addr_t end = base + size;
		while (size > 0) {
			void *ptr = kmap_atomic(page);
			memset(ptr, 0, PAGE_SIZE);
			dmac_flush_range(ptr, ptr + PAGE_SIZE);
			kunmap_atomic(ptr);
			page++;
			size -= PAGE_SIZE;
		}
		outer_flush_range(base, end);
	} else {
		void *ptr = page_address(page);
243 244 245 246
		memset(ptr, 0, size);
		dmac_flush_range(ptr, ptr + size);
		outer_flush_range(__pa(ptr), __pa(ptr) + size);
	}
247 248
}

249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268
/*
 * Allocate a DMA buffer for 'dev' of size 'size' using the
 * specified gfp mask.  Note that 'size' must be page aligned.
 */
static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
{
	unsigned long order = get_order(size);
	struct page *page, *p, *e;

	page = alloc_pages(gfp, order);
	if (!page)
		return NULL;

	/*
	 * Now split the huge page and free the excess pages
	 */
	split_page(page, order);
	for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
		__free_page(p);

269
	__dma_clear_buffer(page, size);
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286

	return page;
}

/*
 * Free a DMA buffer.  'size' must be page aligned.
 */
static void __dma_free_buffer(struct page *page, size_t size)
{
	struct page *e = page + (size >> PAGE_SHIFT);

	while (page < e) {
		__free_page(page);
		page++;
	}
}

287
#ifdef CONFIG_MMU
288

289
static void *__alloc_from_contiguous(struct device *dev, size_t size,
290 291
				     pgprot_t prot, struct page **ret_page,
				     const void *caller);
292

293 294 295
static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
				 pgprot_t prot, struct page **ret_page,
				 const void *caller);
296

297 298 299
static void *
__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
	const void *caller)
300
{
301 302
	struct vm_struct *area;
	unsigned long addr;
303

304 305 306 307 308 309 310 311 312 313
	/*
	 * DMA allocation can be mapped to user space, so lets
	 * set VM_USERMAP flags too.
	 */
	area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
				  caller);
	if (!area)
		return NULL;
	addr = (unsigned long)area->addr;
	area->phys_addr = __pfn_to_phys(page_to_pfn(page));
314

315 316 317 318 319
	if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) {
		vunmap((void *)addr);
		return NULL;
	}
	return (void *)addr;
320
}
L
Linus Torvalds 已提交
321

322
static void __dma_free_remap(void *cpu_addr, size_t size)
323
{
324 325 326 327 328
	unsigned int flags = VM_ARM_DMA_CONSISTENT | VM_USERMAP;
	struct vm_struct *area = find_vm_area(cpu_addr);
	if (!area || (area->flags & flags) != flags) {
		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
		return;
329
	}
330 331
	unmap_kernel_range((unsigned long)cpu_addr, size);
	vunmap(cpu_addr);
332 333
}

334 335
#define DEFAULT_DMA_COHERENT_POOL_SIZE	SZ_256K

336 337 338 339 340 341
struct dma_pool {
	size_t size;
	spinlock_t lock;
	unsigned long *bitmap;
	unsigned long nr_pages;
	void *vaddr;
342
	struct page **pages;
343 344
};

345
static struct dma_pool atomic_pool = {
346
	.size = DEFAULT_DMA_COHERENT_POOL_SIZE,
347
};
348 349 350

static int __init early_coherent_pool(char *p)
{
351
	atomic_pool.size = memparse(p, &p);
352 353 354 355
	return 0;
}
early_param("coherent_pool", early_coherent_pool);

356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
void __init init_dma_coherent_pool_size(unsigned long size)
{
	/*
	 * Catch any attempt to set the pool size too late.
	 */
	BUG_ON(atomic_pool.vaddr);

	/*
	 * Set architecture specific coherent pool size only if
	 * it has not been changed by kernel command line parameter.
	 */
	if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
		atomic_pool.size = size;
}

371 372 373
/*
 * Initialise the coherent pool for atomic allocations.
 */
374
static int __init atomic_pool_init(void)
375
{
376
	struct dma_pool *pool = &atomic_pool;
377
	pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
378
	gfp_t gfp = GFP_KERNEL | GFP_DMA;
379 380
	unsigned long nr_pages = pool->size >> PAGE_SHIFT;
	unsigned long *bitmap;
381
	struct page *page;
382
	struct page **pages;
383
	void *ptr;
384
	int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
385

386 387 388
	bitmap = kzalloc(bitmap_size, GFP_KERNEL);
	if (!bitmap)
		goto no_bitmap;
389

390 391 392 393
	pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
	if (!pages)
		goto no_pages;

394
	if (dev_get_cma_area(NULL))
395 396
		ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page,
					      atomic_pool_init);
397
	else
398 399
		ptr = __alloc_remap_buffer(NULL, pool->size, gfp, prot, &page,
					   atomic_pool_init);
400
	if (ptr) {
401 402 403 404 405
		int i;

		for (i = 0; i < nr_pages; i++)
			pages[i] = page + i;

406 407
		spin_lock_init(&pool->lock);
		pool->vaddr = ptr;
408
		pool->pages = pages;
409 410 411 412
		pool->bitmap = bitmap;
		pool->nr_pages = nr_pages;
		pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
		       (unsigned)pool->size / 1024);
413 414
		return 0;
	}
415 416

	kfree(pages);
417
no_pages:
418 419 420 421
	kfree(bitmap);
no_bitmap:
	pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
	       (unsigned)pool->size / 1024);
422 423 424 425 426
	return -ENOMEM;
}
/*
 * CMA is activated by core_initcall, so we must be called after it.
 */
427
postcore_initcall(atomic_pool_init);
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456

struct dma_contig_early_reserve {
	phys_addr_t base;
	unsigned long size;
};

static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;

static int dma_mmu_remap_num __initdata;

void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
{
	dma_mmu_remap[dma_mmu_remap_num].base = base;
	dma_mmu_remap[dma_mmu_remap_num].size = size;
	dma_mmu_remap_num++;
}

void __init dma_contiguous_remap(void)
{
	int i;
	for (i = 0; i < dma_mmu_remap_num; i++) {
		phys_addr_t start = dma_mmu_remap[i].base;
		phys_addr_t end = start + dma_mmu_remap[i].size;
		struct map_desc map;
		unsigned long addr;

		if (end > arm_lowmem_limit)
			end = arm_lowmem_limit;
		if (start >= end)
457
			continue;
458 459 460 461 462 463 464

		map.pfn = __phys_to_pfn(start);
		map.virtual = __phys_to_virt(start);
		map.length = end - start;
		map.type = MT_MEMORY_DMA_READY;

		/*
465 466 467 468 469 470 471
		 * Clear previous low-memory mapping to ensure that the
		 * TLB does not see any conflicting entries, then flush
		 * the TLB of the old entries before creating new mappings.
		 *
		 * This ensures that any speculatively loaded TLB entries
		 * (even though they may be rare) can not cause any problems,
		 * and ensures that this code is architecturally compliant.
472 473
		 */
		for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
474
		     addr += PMD_SIZE)
475 476
			pmd_clear(pmd_off_k(addr));

477 478 479
		flush_tlb_kernel_range(__phys_to_virt(start),
				       __phys_to_virt(end));

480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
		iotable_init(&map, 1);
	}
}

static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,
			    void *data)
{
	struct page *page = virt_to_page(addr);
	pgprot_t prot = *(pgprot_t *)data;

	set_pte_ext(pte, mk_pte(page, prot), 0);
	return 0;
}

static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
{
	unsigned long start = (unsigned long) page_address(page);
	unsigned end = start + size;

	apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
	flush_tlb_kernel_range(start, end);
}

static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
				 pgprot_t prot, struct page **ret_page,
				 const void *caller)
{
	struct page *page;
	void *ptr;
	page = __dma_alloc_buffer(dev, size, gfp);
	if (!page)
		return NULL;

	ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
	if (!ptr) {
		__dma_free_buffer(page, size);
		return NULL;
	}

	*ret_page = page;
	return ptr;
}

523
static void *__alloc_from_pool(size_t size, struct page **ret_page)
524
{
525 526 527 528 529
	struct dma_pool *pool = &atomic_pool;
	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
	unsigned int pageno;
	unsigned long flags;
	void *ptr = NULL;
530
	unsigned long align_mask;
531

532 533
	if (!pool->vaddr) {
		WARN(1, "coherent pool not initialised!\n");
534 535 536 537 538 539 540 541
		return NULL;
	}

	/*
	 * Align the region allocation - allocations from pool are rather
	 * small, so align them to their order in pages, minimum is a page
	 * size. This helps reduce fragmentation of the DMA space.
	 */
542
	align_mask = (1 << get_order(size)) - 1;
543 544 545

	spin_lock_irqsave(&pool->lock, flags);
	pageno = bitmap_find_next_zero_area(pool->bitmap, pool->nr_pages,
546
					    0, count, align_mask);
547 548 549
	if (pageno < pool->nr_pages) {
		bitmap_set(pool->bitmap, pageno, count);
		ptr = pool->vaddr + PAGE_SIZE * pageno;
550
		*ret_page = pool->pages[pageno];
551 552 553 554
	} else {
		pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
			    "Please increase it with coherent_pool= kernel parameter!\n",
			    (unsigned)pool->size / 1024);
555
	}
556 557 558
	spin_unlock_irqrestore(&pool->lock, flags);

	return ptr;
559 560
}

561 562 563 564 565 566 567
static bool __in_atomic_pool(void *start, size_t size)
{
	struct dma_pool *pool = &atomic_pool;
	void *end = start + size;
	void *pool_start = pool->vaddr;
	void *pool_end = pool->vaddr + pool->size;

568
	if (start < pool_start || start >= pool_end)
569 570 571 572 573 574 575 576 577 578 579
		return false;

	if (end <= pool_end)
		return true;

	WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
	     start, end - 1, pool_start, pool_end - 1);

	return false;
}

580
static int __free_from_pool(void *start, size_t size)
581
{
582 583 584
	struct dma_pool *pool = &atomic_pool;
	unsigned long pageno, count;
	unsigned long flags;
585

586
	if (!__in_atomic_pool(start, size))
587 588
		return 0;

589 590 591 592 593 594 595
	pageno = (start - pool->vaddr) >> PAGE_SHIFT;
	count = size >> PAGE_SHIFT;

	spin_lock_irqsave(&pool->lock, flags);
	bitmap_clear(pool->bitmap, pageno, count);
	spin_unlock_irqrestore(&pool->lock, flags);

596 597 598 599
	return 1;
}

static void *__alloc_from_contiguous(struct device *dev, size_t size,
600 601
				     pgprot_t prot, struct page **ret_page,
				     const void *caller)
602 603 604 605
{
	unsigned long order = get_order(size);
	size_t count = size >> PAGE_SHIFT;
	struct page *page;
606
	void *ptr;
607 608 609 610 611 612 613

	page = dma_alloc_from_contiguous(dev, count, order);
	if (!page)
		return NULL;

	__dma_clear_buffer(page, size);

614 615 616 617 618 619 620 621 622 623
	if (PageHighMem(page)) {
		ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);
		if (!ptr) {
			dma_release_from_contiguous(dev, page, count);
			return NULL;
		}
	} else {
		__dma_remap(page, size, prot);
		ptr = page_address(page);
	}
624
	*ret_page = page;
625
	return ptr;
626 627 628
}

static void __free_from_contiguous(struct device *dev, struct page *page,
629
				   void *cpu_addr, size_t size)
630
{
631 632 633
	if (PageHighMem(page))
		__dma_free_remap(cpu_addr, size);
	else
634
		__dma_remap(page, size, PAGE_KERNEL);
635 636 637
	dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
}

638 639 640 641 642 643 644 645
static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
{
	prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
			    pgprot_writecombine(prot) :
			    pgprot_dmacoherent(prot);
	return prot;
}

646 647
#define nommu() 0

648
#else	/* !CONFIG_MMU */
649

650 651
#define nommu() 1

652
#define __get_dma_pgprot(attrs, prot)	__pgprot(0)
653
#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c)	NULL
654
#define __alloc_from_pool(size, ret_page)			NULL
655
#define __alloc_from_contiguous(dev, size, prot, ret, c)	NULL
656
#define __free_from_pool(cpu_addr, size)			0
657
#define __free_from_contiguous(dev, page, cpu_addr, size)	do { } while (0)
658
#define __dma_free_remap(cpu_addr, size)			do { } while (0)
659 660 661

#endif	/* CONFIG_MMU */

662 663
static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
				   struct page **ret_page)
664
{
665 666 667 668 669 670 671 672 673 674 675 676
	struct page *page;
	page = __dma_alloc_buffer(dev, size, gfp);
	if (!page)
		return NULL;

	*ret_page = page;
	return page_address(page);
}



static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
R
Rob Herring 已提交
677
			 gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller)
678 679
{
	u64 mask = get_coherent_dma_mask(dev);
680
	struct page *page = NULL;
681
	void *addr;
682

683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
#ifdef CONFIG_DMA_API_DEBUG
	u64 limit = (mask + 1) & ~mask;
	if (limit && size >= limit) {
		dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
			size, mask);
		return NULL;
	}
#endif

	if (!mask)
		return NULL;

	if (mask < 0xffffffffULL)
		gfp |= GFP_DMA;

698 699 700 701 702 703 704 705 706
	/*
	 * Following is a work-around (a.k.a. hack) to prevent pages
	 * with __GFP_COMP being passed to split_page() which cannot
	 * handle them.  The real problem is that this flag probably
	 * should be 0 on ARM as it is not supported on this
	 * platform; see CONFIG_HUGETLBFS.
	 */
	gfp &= ~(__GFP_COMP);

707
	*handle = DMA_ERROR_CODE;
708
	size = PAGE_ALIGN(size);
709

R
Rob Herring 已提交
710
	if (is_coherent || nommu())
711
		addr = __alloc_simple_buffer(dev, size, gfp, &page);
712
	else if (!(gfp & __GFP_WAIT))
713
		addr = __alloc_from_pool(size, &page);
714
	else if (!dev_get_cma_area(dev))
715
		addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);
716
	else
717
		addr = __alloc_from_contiguous(dev, size, prot, &page, caller);
718

719
	if (addr)
720
		*handle = pfn_to_dma(dev, page_to_pfn(page));
721

722 723
	return addr;
}
L
Linus Torvalds 已提交
724 725 726 727 728

/*
 * Allocate DMA-coherent memory space and return both the kernel remapped
 * virtual and bus address for that space.
 */
729 730
void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
		    gfp_t gfp, struct dma_attrs *attrs)
L
Linus Torvalds 已提交
731
{
732
	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
733 734 735 736 737
	void *memory;

	if (dma_alloc_from_coherent(dev, size, handle, &memory))
		return memory;

R
Rob Herring 已提交
738 739 740 741 742 743 744
	return __dma_alloc(dev, size, handle, gfp, prot, false,
			   __builtin_return_address(0));
}

static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
	dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
{
745
	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
R
Rob Herring 已提交
746 747 748 749 750 751
	void *memory;

	if (dma_alloc_from_coherent(dev, size, handle, &memory))
		return memory;

	return __dma_alloc(dev, size, handle, gfp, prot, true,
752
			   __builtin_return_address(0));
L
Linus Torvalds 已提交
753 754 755
}

/*
756
 * Create userspace mapping for the DMA-coherent memory.
L
Linus Torvalds 已提交
757
 */
758 759 760
int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
		 struct dma_attrs *attrs)
L
Linus Torvalds 已提交
761
{
762 763
	int ret = -ENXIO;
#ifdef CONFIG_MMU
764 765
	unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
766
	unsigned long pfn = dma_to_pfn(dev, dma_addr);
767 768
	unsigned long off = vma->vm_pgoff;

769 770
	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);

771 772 773
	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
		return ret;

774 775 776 777 778 779
	if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
		ret = remap_pfn_range(vma, vma->vm_start,
				      pfn + off,
				      vma->vm_end - vma->vm_start,
				      vma->vm_page_prot);
	}
780
#endif	/* CONFIG_MMU */
L
Linus Torvalds 已提交
781 782 783 784 785

	return ret;
}

/*
786
 * Free a buffer as defined by the above mapping.
L
Linus Torvalds 已提交
787
 */
R
Rob Herring 已提交
788 789 790
static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
			   dma_addr_t handle, struct dma_attrs *attrs,
			   bool is_coherent)
L
Linus Torvalds 已提交
791
{
792
	struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
793

794 795 796
	if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
		return;

797 798
	size = PAGE_ALIGN(size);

R
Rob Herring 已提交
799
	if (is_coherent || nommu()) {
800
		__dma_free_buffer(page, size);
801 802
	} else if (__free_from_pool(cpu_addr, size)) {
		return;
803
	} else if (!dev_get_cma_area(dev)) {
804
		__dma_free_remap(cpu_addr, size);
805 806 807 808 809 810
		__dma_free_buffer(page, size);
	} else {
		/*
		 * Non-atomic allocations cannot be freed with IRQs disabled
		 */
		WARN_ON(irqs_disabled());
811
		__free_from_contiguous(dev, page, cpu_addr, size);
812
	}
L
Linus Torvalds 已提交
813
}
814

R
Rob Herring 已提交
815 816 817 818 819 820 821 822 823 824 825 826
void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
		  dma_addr_t handle, struct dma_attrs *attrs)
{
	__arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
}

static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
				  dma_addr_t handle, struct dma_attrs *attrs)
{
	__arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
}

827 828 829 830 831 832 833 834 835 836 837 838 839 840 841
int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
		 void *cpu_addr, dma_addr_t handle, size_t size,
		 struct dma_attrs *attrs)
{
	struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
	int ret;

	ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
	if (unlikely(ret))
		return ret;

	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
	return 0;
}

842
static void dma_cache_maint_page(struct page *page, unsigned long offset,
843 844
	size_t size, enum dma_data_direction dir,
	void (*op)(const void *, size_t, int))
845
{
846 847 848 849 850 851
	unsigned long pfn;
	size_t left = size;

	pfn = page_to_pfn(page) + offset / PAGE_SIZE;
	offset %= PAGE_SIZE;

852 853 854 855 856 857 858 859
	/*
	 * A single sg entry may refer to multiple physically contiguous
	 * pages.  But we still need to process highmem pages individually.
	 * If highmem is not configured then the bulk of this loop gets
	 * optimized out.
	 */
	do {
		size_t len = left;
860 861
		void *vaddr;

862 863
		page = pfn_to_page(pfn);

864
		if (PageHighMem(page)) {
865
			if (len + offset > PAGE_SIZE)
866
				len = PAGE_SIZE - offset;
867 868

			if (cache_is_vipt_nonaliasing()) {
869
				vaddr = kmap_atomic(page);
870
				op(vaddr + offset, len, dir);
871
				kunmap_atomic(vaddr);
872 873 874 875 876 877
			} else {
				vaddr = kmap_high_get(page);
				if (vaddr) {
					op(vaddr + offset, len, dir);
					kunmap_high(page);
				}
878
			}
879 880
		} else {
			vaddr = page_address(page) + offset;
881
			op(vaddr, len, dir);
882 883
		}
		offset = 0;
884
		pfn++;
885 886 887
		left -= len;
	} while (left);
}
888

889 890 891 892 893 894 895
/*
 * Make an area consistent for devices.
 * Note: Drivers should NOT use this function directly, as it will break
 * platforms with CONFIG_DMABOUNCE.
 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
 */
static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
896 897
	size_t size, enum dma_data_direction dir)
{
898
	phys_addr_t paddr;
899

900
	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
901 902

	paddr = page_to_phys(page) + off;
903 904 905 906 907 908
	if (dir == DMA_FROM_DEVICE) {
		outer_inv_range(paddr, paddr + size);
	} else {
		outer_clean_range(paddr, paddr + size);
	}
	/* FIXME: non-speculating: flush on bidirectional mappings? */
909 910
}

911
static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
912 913
	size_t size, enum dma_data_direction dir)
{
914
	phys_addr_t paddr = page_to_phys(page) + off;
915 916

	/* FIXME: non-speculating: not required */
917 918
	/* in any case, don't bother invalidating if DMA to device */
	if (dir != DMA_TO_DEVICE) {
919 920
		outer_inv_range(paddr, paddr + size);

921 922
		dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
	}
923 924

	/*
925
	 * Mark the D-cache clean for these pages to avoid extra flushing.
926
	 */
927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942
	if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
		unsigned long pfn;
		size_t left = size;

		pfn = page_to_pfn(page) + off / PAGE_SIZE;
		off %= PAGE_SIZE;
		if (off) {
			pfn++;
			left -= PAGE_SIZE - off;
		}
		while (left >= PAGE_SIZE) {
			page = pfn_to_page(pfn++);
			set_bit(PG_dcache_clean, &page->flags);
			left -= PAGE_SIZE;
		}
	}
943
}
944

945
/**
946
 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
947 948 949 950 951 952 953 954 955 956 957 958 959 960
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map
 * @dir: DMA transfer direction
 *
 * Map a set of buffers described by scatterlist in streaming mode for DMA.
 * This is the scatter-gather version of the dma_map_single interface.
 * Here the scatter gather list elements are each tagged with the
 * appropriate dma address and length.  They are obtained via
 * sg_dma_{address,length}.
 *
 * Device ownership issues as mentioned for dma_map_single are the same
 * here.
 */
961 962
int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
		enum dma_data_direction dir, struct dma_attrs *attrs)
963
{
964
	struct dma_map_ops *ops = get_dma_ops(dev);
965
	struct scatterlist *s;
966
	int i, j;
967 968

	for_each_sg(sg, s, nents, i) {
969 970 971
#ifdef CONFIG_NEED_SG_DMA_LENGTH
		s->dma_length = s->length;
#endif
972 973
		s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
						s->length, dir, attrs);
974 975
		if (dma_mapping_error(dev, s->dma_address))
			goto bad_mapping;
976 977
	}
	return nents;
978 979 980

 bad_mapping:
	for_each_sg(sg, s, i, j)
981
		ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
982
	return 0;
983 984 985
}

/**
986
 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
987 988
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
989
 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
990 991 992 993 994
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 *
 * Unmap a set of streaming mode DMA translations.  Again, CPU access
 * rules concerning calls here are the same as for dma_unmap_single().
 */
995 996
void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
		enum dma_data_direction dir, struct dma_attrs *attrs)
997
{
998
	struct dma_map_ops *ops = get_dma_ops(dev);
999 1000 1001
	struct scatterlist *s;

	int i;
1002

1003
	for_each_sg(sg, s, nents, i)
1004
		ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
1005 1006 1007
}

/**
1008
 * arm_dma_sync_sg_for_cpu
1009 1010 1011 1012 1013
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
1014
void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1015 1016
			int nents, enum dma_data_direction dir)
{
1017
	struct dma_map_ops *ops = get_dma_ops(dev);
1018 1019 1020
	struct scatterlist *s;
	int i;

1021 1022 1023
	for_each_sg(sg, s, nents, i)
		ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
					 dir);
1024 1025 1026
}

/**
1027
 * arm_dma_sync_sg_for_device
1028 1029 1030 1031 1032
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
1033
void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1034 1035
			int nents, enum dma_data_direction dir)
{
1036
	struct dma_map_ops *ops = get_dma_ops(dev);
1037 1038 1039
	struct scatterlist *s;
	int i;

1040 1041 1042
	for_each_sg(sg, s, nents, i)
		ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
					    dir);
1043
}
1044

1045 1046 1047 1048 1049 1050 1051 1052
/*
 * Return whether the given device DMA address mask can be supported
 * properly.  For example, if your device can only drive the low 24-bits
 * during bus mastering, then you would pass 0x00ffffff as the mask
 * to this function.
 */
int dma_supported(struct device *dev, u64 mask)
{
1053
	return __dma_supported(dev, mask, false);
1054 1055 1056
}
EXPORT_SYMBOL(dma_supported);

1057
int arm_dma_set_mask(struct device *dev, u64 dma_mask)
1058 1059 1060 1061 1062 1063 1064 1065 1066
{
	if (!dev->dma_mask || !dma_supported(dev, dma_mask))
		return -EIO;

	*dev->dma_mask = dma_mask;

	return 0;
}

1067 1068 1069 1070 1071 1072 1073 1074
#define PREALLOC_DMA_DEBUG_ENTRIES	4096

static int __init dma_debug_do_init(void)
{
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
	return 0;
}
fs_initcall(dma_debug_do_init);
1075 1076 1077 1078 1079

#ifdef CONFIG_ARM_DMA_USE_IOMMU

/* IOMMU */

1080 1081
static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);

1082 1083 1084 1085 1086 1087
static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
				      size_t size)
{
	unsigned int order = get_order(size);
	unsigned int align = 0;
	unsigned int count, start;
1088
	size_t mapping_size = mapping->bits << PAGE_SHIFT;
1089
	unsigned long flags;
1090 1091
	dma_addr_t iova;
	int i;
1092

1093 1094 1095
	if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)
		order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;

1096 1097
	count = PAGE_ALIGN(size) >> PAGE_SHIFT;
	align = (1 << order) - 1;
1098 1099

	spin_lock_irqsave(&mapping->lock, flags);
1100 1101 1102 1103 1104 1105 1106 1107 1108
	for (i = 0; i < mapping->nr_bitmaps; i++) {
		start = bitmap_find_next_zero_area(mapping->bitmaps[i],
				mapping->bits, 0, count, align);

		if (start > mapping->bits)
			continue;

		bitmap_set(mapping->bitmaps[i], start, count);
		break;
1109 1110
	}

1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
	/*
	 * No unused range found. Try to extend the existing mapping
	 * and perform a second attempt to reserve an IO virtual
	 * address range of size bytes.
	 */
	if (i == mapping->nr_bitmaps) {
		if (extend_iommu_mapping(mapping)) {
			spin_unlock_irqrestore(&mapping->lock, flags);
			return DMA_ERROR_CODE;
		}

		start = bitmap_find_next_zero_area(mapping->bitmaps[i],
				mapping->bits, 0, count, align);

		if (start > mapping->bits) {
			spin_unlock_irqrestore(&mapping->lock, flags);
			return DMA_ERROR_CODE;
		}

		bitmap_set(mapping->bitmaps[i], start, count);
	}
1132 1133
	spin_unlock_irqrestore(&mapping->lock, flags);

1134
	iova = mapping->base + (mapping_size * i);
1135
	iova += start << PAGE_SHIFT;
1136 1137

	return iova;
1138 1139 1140 1141 1142
}

static inline void __free_iova(struct dma_iommu_mapping *mapping,
			       dma_addr_t addr, size_t size)
{
1143
	unsigned int start, count;
1144
	size_t mapping_size = mapping->bits << PAGE_SHIFT;
1145
	unsigned long flags;
1146 1147 1148 1149 1150 1151
	dma_addr_t bitmap_base;
	u32 bitmap_index;

	if (!size)
		return;

1152
	bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;
1153 1154
	BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);

1155
	bitmap_base = mapping->base + mapping_size * bitmap_index;
1156

1157
	start = (addr - bitmap_base) >>	PAGE_SHIFT;
1158

1159
	if (addr + size > bitmap_base + mapping_size) {
1160 1161 1162 1163 1164 1165 1166 1167
		/*
		 * The address range to be freed reaches into the iova
		 * range of the next bitmap. This should not happen as
		 * we don't allow this in __alloc_iova (at the
		 * moment).
		 */
		BUG();
	} else
1168
		count = size >> PAGE_SHIFT;
1169 1170

	spin_lock_irqsave(&mapping->lock, flags);
1171
	bitmap_clear(mapping->bitmaps[bitmap_index], start, count);
1172 1173 1174
	spin_unlock_irqrestore(&mapping->lock, flags);
}

1175 1176
static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
					  gfp_t gfp, struct dma_attrs *attrs)
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189
{
	struct page **pages;
	int count = size >> PAGE_SHIFT;
	int array_size = count * sizeof(struct page *);
	int i = 0;

	if (array_size <= PAGE_SIZE)
		pages = kzalloc(array_size, gfp);
	else
		pages = vzalloc(array_size);
	if (!pages)
		return NULL;

1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
	if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs))
	{
		unsigned long order = get_order(size);
		struct page *page;

		page = dma_alloc_from_contiguous(dev, count, order);
		if (!page)
			goto error;

		__dma_clear_buffer(page, size);

		for (i = 0; i < count; i++)
			pages[i] = page + i;

		return pages;
	}

1207 1208 1209 1210 1211
	/*
	 * IOMMU can map any pages, so himem can also be used here
	 */
	gfp |= __GFP_NOWARN | __GFP_HIGHMEM;

1212
	while (count) {
1213
		int j, order = __fls(count);
1214

1215
		pages[i] = alloc_pages(gfp, order);
1216
		while (!pages[i] && order)
1217
			pages[i] = alloc_pages(gfp, --order);
1218 1219 1220
		if (!pages[i])
			goto error;

1221
		if (order) {
1222
			split_page(pages[i], order);
1223 1224 1225 1226
			j = 1 << order;
			while (--j)
				pages[i + j] = pages[i] + j;
		}
1227 1228 1229 1230 1231 1232 1233 1234

		__dma_clear_buffer(pages[i], PAGE_SIZE << order);
		i += 1 << order;
		count -= 1 << order;
	}

	return pages;
error:
1235
	while (i--)
1236 1237
		if (pages[i])
			__free_pages(pages[i], 0);
1238
	if (array_size <= PAGE_SIZE)
1239 1240 1241 1242 1243 1244
		kfree(pages);
	else
		vfree(pages);
	return NULL;
}

1245 1246
static int __iommu_free_buffer(struct device *dev, struct page **pages,
			       size_t size, struct dma_attrs *attrs)
1247 1248 1249 1250
{
	int count = size >> PAGE_SHIFT;
	int array_size = count * sizeof(struct page *);
	int i;
1251 1252 1253 1254 1255 1256 1257 1258 1259

	if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) {
		dma_release_from_contiguous(dev, pages[0], count);
	} else {
		for (i = 0; i < count; i++)
			if (pages[i])
				__free_pages(pages[i], 0);
	}

1260
	if (array_size <= PAGE_SIZE)
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
		kfree(pages);
	else
		vfree(pages);
	return 0;
}

/*
 * Create a CPU mapping for a specified pages
 */
static void *
1271 1272
__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
		    const void *caller)
1273
{
1274 1275 1276
	unsigned int i, nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
	struct vm_struct *area;
	unsigned long p;
1277

1278 1279 1280
	area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
				  caller);
	if (!area)
1281 1282
		return NULL;

1283 1284 1285
	area->pages = pages;
	area->nr_pages = nr_pages;
	p = (unsigned long)area->addr;
1286

1287 1288 1289 1290 1291
	for (i = 0; i < nr_pages; i++) {
		phys_addr_t phys = __pfn_to_phys(page_to_pfn(pages[i]));
		if (ioremap_page_range(p, p + PAGE_SIZE, phys, prot))
			goto err;
		p += PAGE_SIZE;
1292
	}
1293 1294 1295 1296
	return area->addr;
err:
	unmap_kernel_range((unsigned long)area->addr, size);
	vunmap(area->addr);
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
	return NULL;
}

/*
 * Create a mapping in device IO address space for specified pages
 */
static dma_addr_t
__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
	dma_addr_t dma_addr, iova;
	int i, ret = DMA_ERROR_CODE;

	dma_addr = __alloc_iova(mapping, size);
	if (dma_addr == DMA_ERROR_CODE)
		return dma_addr;

	iova = dma_addr;
	for (i = 0; i < count; ) {
		unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
		phys_addr_t phys = page_to_phys(pages[i]);
		unsigned int len, j;

		for (j = i + 1; j < count; j++, next_pfn++)
			if (page_to_pfn(pages[j]) != next_pfn)
				break;

		len = (j - i) << PAGE_SHIFT;
1326 1327
		ret = iommu_map(mapping->domain, iova, phys, len,
				IOMMU_READ|IOMMU_WRITE);
1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355
		if (ret < 0)
			goto fail;
		iova += len;
		i = j;
	}
	return dma_addr;
fail:
	iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
	__free_iova(mapping, dma_addr, size);
	return DMA_ERROR_CODE;
}

static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;

	/*
	 * add optional in-page offset from iova to size and align
	 * result to page size
	 */
	size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
	iova &= PAGE_MASK;

	iommu_unmap(mapping->domain, iova, size);
	__free_iova(mapping, iova, size);
	return 0;
}

1356 1357 1358 1359 1360 1361 1362 1363 1364
static struct page **__atomic_get_pages(void *addr)
{
	struct dma_pool *pool = &atomic_pool;
	struct page **pages = pool->pages;
	int offs = (addr - pool->vaddr) >> PAGE_SHIFT;

	return pages + offs;
}

1365
static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
1366 1367 1368
{
	struct vm_struct *area;

1369 1370 1371
	if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
		return __atomic_get_pages(cpu_addr);

1372 1373 1374
	if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
		return cpu_addr;

1375 1376 1377 1378 1379 1380
	area = find_vm_area(cpu_addr);
	if (area && (area->flags & VM_ARM_DMA_CONSISTENT))
		return area->pages;
	return NULL;
}

1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401
static void *__iommu_alloc_atomic(struct device *dev, size_t size,
				  dma_addr_t *handle)
{
	struct page *page;
	void *addr;

	addr = __alloc_from_pool(size, &page);
	if (!addr)
		return NULL;

	*handle = __iommu_create_mapping(dev, &page, size);
	if (*handle == DMA_ERROR_CODE)
		goto err_mapping;

	return addr;

err_mapping:
	__free_from_pool(addr, size);
	return NULL;
}

1402
static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
1403 1404 1405
				dma_addr_t handle, size_t size)
{
	__iommu_remove_mapping(dev, handle, size);
1406
	__free_from_pool(cpu_addr, size);
1407 1408
}

1409 1410 1411
static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
	    dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
{
1412
	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
1413 1414 1415 1416 1417 1418
	struct page **pages;
	void *addr = NULL;

	*handle = DMA_ERROR_CODE;
	size = PAGE_ALIGN(size);

1419
	if (!(gfp & __GFP_WAIT))
1420 1421
		return __iommu_alloc_atomic(dev, size, handle);

1422 1423 1424 1425 1426 1427 1428 1429 1430
	/*
	 * Following is a work-around (a.k.a. hack) to prevent pages
	 * with __GFP_COMP being passed to split_page() which cannot
	 * handle them.  The real problem is that this flag probably
	 * should be 0 on ARM as it is not supported on this
	 * platform; see CONFIG_HUGETLBFS.
	 */
	gfp &= ~(__GFP_COMP);

1431
	pages = __iommu_alloc_buffer(dev, size, gfp, attrs);
1432 1433 1434 1435 1436 1437 1438
	if (!pages)
		return NULL;

	*handle = __iommu_create_mapping(dev, pages, size);
	if (*handle == DMA_ERROR_CODE)
		goto err_buffer;

1439 1440 1441
	if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
		return pages;

1442 1443
	addr = __iommu_alloc_remap(pages, size, gfp, prot,
				   __builtin_return_address(0));
1444 1445 1446 1447 1448 1449 1450 1451
	if (!addr)
		goto err_mapping;

	return addr;

err_mapping:
	__iommu_remove_mapping(dev, *handle, size);
err_buffer:
1452
	__iommu_free_buffer(dev, pages, size, attrs);
1453 1454 1455 1456 1457 1458 1459
	return NULL;
}

static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
		    void *cpu_addr, dma_addr_t dma_addr, size_t size,
		    struct dma_attrs *attrs)
{
1460 1461
	unsigned long uaddr = vma->vm_start;
	unsigned long usize = vma->vm_end - vma->vm_start;
1462
	struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1463 1464 1465

	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);

1466 1467
	if (!pages)
		return -ENXIO;
1468

1469 1470 1471 1472 1473 1474 1475 1476 1477
	do {
		int ret = vm_insert_page(vma, uaddr, *pages++);
		if (ret) {
			pr_err("Remapping memory failed: %d\n", ret);
			return ret;
		}
		uaddr += PAGE_SIZE;
		usize -= PAGE_SIZE;
	} while (usize > 0);
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488

	return 0;
}

/*
 * free a page as defined by the above mapping.
 * Must not be called with IRQs disabled.
 */
void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
			  dma_addr_t handle, struct dma_attrs *attrs)
{
1489
	struct page **pages;
1490 1491
	size = PAGE_ALIGN(size);

1492 1493
	if (__in_atomic_pool(cpu_addr, size)) {
		__iommu_free_atomic(dev, cpu_addr, handle, size);
1494
		return;
1495
	}
1496

1497 1498 1499
	pages = __iommu_get_pages(cpu_addr, attrs);
	if (!pages) {
		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
1500 1501 1502
		return;
	}

1503 1504 1505 1506
	if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
		unmap_kernel_range((unsigned long)cpu_addr, size);
		vunmap(cpu_addr);
	}
1507 1508

	__iommu_remove_mapping(dev, handle, size);
1509
	__iommu_free_buffer(dev, pages, size, attrs);
1510 1511
}

1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
				 void *cpu_addr, dma_addr_t dma_addr,
				 size_t size, struct dma_attrs *attrs)
{
	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
	struct page **pages = __iommu_get_pages(cpu_addr, attrs);

	if (!pages)
		return -ENXIO;

	return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
					 GFP_KERNEL);
1524 1525
}

1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546
static int __dma_direction_to_prot(enum dma_data_direction dir)
{
	int prot;

	switch (dir) {
	case DMA_BIDIRECTIONAL:
		prot = IOMMU_READ | IOMMU_WRITE;
		break;
	case DMA_TO_DEVICE:
		prot = IOMMU_READ;
		break;
	case DMA_FROM_DEVICE:
		prot = IOMMU_WRITE;
		break;
	default:
		prot = 0;
	}

	return prot;
}

1547 1548 1549 1550 1551
/*
 * Map a part of the scatter-gather list into contiguous io address space
 */
static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
			  size_t size, dma_addr_t *handle,
R
Rob Herring 已提交
1552 1553
			  enum dma_data_direction dir, struct dma_attrs *attrs,
			  bool is_coherent)
1554 1555 1556 1557 1558 1559
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova, iova_base;
	int ret = 0;
	unsigned int count;
	struct scatterlist *s;
1560
	int prot;
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572

	size = PAGE_ALIGN(size);
	*handle = DMA_ERROR_CODE;

	iova_base = iova = __alloc_iova(mapping, size);
	if (iova == DMA_ERROR_CODE)
		return -ENOMEM;

	for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
		phys_addr_t phys = page_to_phys(sg_page(s));
		unsigned int len = PAGE_ALIGN(s->offset + s->length);

R
Rob Herring 已提交
1573 1574
		if (!is_coherent &&
			!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1575 1576
			__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);

1577 1578 1579
		prot = __dma_direction_to_prot(dir);

		ret = iommu_map(mapping->domain, iova, phys, len, prot);
1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593
		if (ret < 0)
			goto fail;
		count += len >> PAGE_SHIFT;
		iova += len;
	}
	*handle = iova_base;

	return 0;
fail:
	iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
	__free_iova(mapping, iova_base, size);
	return ret;
}

R
Rob Herring 已提交
1594 1595 1596
static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
		     enum dma_data_direction dir, struct dma_attrs *attrs,
		     bool is_coherent)
1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
{
	struct scatterlist *s = sg, *dma = sg, *start = sg;
	int i, count = 0;
	unsigned int offset = s->offset;
	unsigned int size = s->offset + s->length;
	unsigned int max = dma_get_max_seg_size(dev);

	for (i = 1; i < nents; i++) {
		s = sg_next(s);

		s->dma_address = DMA_ERROR_CODE;
		s->dma_length = 0;

		if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
			if (__map_sg_chunk(dev, start, size, &dma->dma_address,
R
Rob Herring 已提交
1612
			    dir, attrs, is_coherent) < 0)
1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
				goto bad_mapping;

			dma->dma_address += offset;
			dma->dma_length = size - offset;

			size = offset = s->offset;
			start = s;
			dma = sg_next(dma);
			count += 1;
		}
		size += s->length;
	}
R
Rob Herring 已提交
1625 1626
	if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
		is_coherent) < 0)
1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640
		goto bad_mapping;

	dma->dma_address += offset;
	dma->dma_length = size - offset;

	return count+1;

bad_mapping:
	for_each_sg(sg, s, count, i)
		__iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
	return 0;
}

/**
R
Rob Herring 已提交
1641
 * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1642 1643
 * @dev: valid struct device pointer
 * @sg: list of buffers
R
Rob Herring 已提交
1644 1645
 * @nents: number of buffers to map
 * @dir: DMA transfer direction
1646
 *
R
Rob Herring 已提交
1647 1648 1649 1650
 * Map a set of i/o coherent buffers described by scatterlist in streaming
 * mode for DMA. The scatter gather list elements are merged together (if
 * possible) and tagged with the appropriate dma address and length. They are
 * obtained via sg_dma_{address,length}.
1651
 */
R
Rob Herring 已提交
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678
int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
{
	return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
}

/**
 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
 * @dev: valid struct device pointer
 * @sg: list of buffers
 * @nents: number of buffers to map
 * @dir: DMA transfer direction
 *
 * Map a set of buffers described by scatterlist in streaming mode for DMA.
 * The scatter gather list elements are merged together (if possible) and
 * tagged with the appropriate dma address and length. They are obtained via
 * sg_dma_{address,length}.
 */
int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
{
	return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
}

static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
		int nents, enum dma_data_direction dir, struct dma_attrs *attrs,
		bool is_coherent)
1679 1680 1681 1682 1683 1684 1685 1686
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i) {
		if (sg_dma_len(s))
			__iommu_remove_mapping(dev, sg_dma_address(s),
					       sg_dma_len(s));
R
Rob Herring 已提交
1687
		if (!is_coherent &&
1688
		    !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1689 1690 1691 1692 1693
			__dma_page_dev_to_cpu(sg_page(s), s->offset,
					      s->length, dir);
	}
}

R
Rob Herring 已提交
1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725
/**
 * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
 * @dev: valid struct device pointer
 * @sg: list of buffers
 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 *
 * Unmap a set of streaming mode DMA translations.  Again, CPU access
 * rules concerning calls here are the same as for dma_unmap_single().
 */
void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
{
	__iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
}

/**
 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
 * @dev: valid struct device pointer
 * @sg: list of buffers
 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 *
 * Unmap a set of streaming mode DMA translations.  Again, CPU access
 * rules concerning calls here are the same as for dma_unmap_single().
 */
void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
			enum dma_data_direction dir, struct dma_attrs *attrs)
{
	__iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
}

1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739
/**
 * arm_iommu_sync_sg_for_cpu
 * @dev: valid struct device pointer
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
			int nents, enum dma_data_direction dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i)
R
Rob Herring 已提交
1740
		__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757

}

/**
 * arm_iommu_sync_sg_for_device
 * @dev: valid struct device pointer
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
			int nents, enum dma_data_direction dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i)
R
Rob Herring 已提交
1758
		__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1759 1760 1761 1762
}


/**
R
Rob Herring 已提交
1763
 * arm_coherent_iommu_map_page
1764 1765 1766 1767 1768 1769
 * @dev: valid struct device pointer
 * @page: page that buffer resides in
 * @offset: offset into page for start of buffer
 * @size: size of buffer to map
 * @dir: DMA transfer direction
 *
R
Rob Herring 已提交
1770
 * Coherent IOMMU aware version of arm_dma_map_page()
1771
 */
R
Rob Herring 已提交
1772
static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
1773 1774 1775 1776 1777
	     unsigned long offset, size_t size, enum dma_data_direction dir,
	     struct dma_attrs *attrs)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t dma_addr;
1778
	int ret, prot, len = PAGE_ALIGN(size + offset);
1779 1780 1781 1782 1783

	dma_addr = __alloc_iova(mapping, len);
	if (dma_addr == DMA_ERROR_CODE)
		return dma_addr;

1784
	prot = __dma_direction_to_prot(dir);
1785 1786

	ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
1787 1788 1789 1790 1791 1792 1793 1794 1795
	if (ret < 0)
		goto fail;

	return dma_addr + offset;
fail:
	__free_iova(mapping, dma_addr, len);
	return DMA_ERROR_CODE;
}

R
Rob Herring 已提交
1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840
/**
 * arm_iommu_map_page
 * @dev: valid struct device pointer
 * @page: page that buffer resides in
 * @offset: offset into page for start of buffer
 * @size: size of buffer to map
 * @dir: DMA transfer direction
 *
 * IOMMU aware version of arm_dma_map_page()
 */
static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
	     unsigned long offset, size_t size, enum dma_data_direction dir,
	     struct dma_attrs *attrs)
{
	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
		__dma_page_cpu_to_dev(page, offset, size, dir);

	return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
}

/**
 * arm_coherent_iommu_unmap_page
 * @dev: valid struct device pointer
 * @handle: DMA address of buffer
 * @size: size of buffer (same as passed to dma_map_page)
 * @dir: DMA transfer direction (same as passed to dma_map_page)
 *
 * Coherent IOMMU aware version of arm_dma_unmap_page()
 */
static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
		size_t size, enum dma_data_direction dir,
		struct dma_attrs *attrs)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova = handle & PAGE_MASK;
	int offset = handle & ~PAGE_MASK;
	int len = PAGE_ALIGN(size + offset);

	if (!iova)
		return;

	iommu_unmap(mapping->domain, iova, len);
	__free_iova(mapping, iova, len);
}

1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862
/**
 * arm_iommu_unmap_page
 * @dev: valid struct device pointer
 * @handle: DMA address of buffer
 * @size: size of buffer (same as passed to dma_map_page)
 * @dir: DMA transfer direction (same as passed to dma_map_page)
 *
 * IOMMU aware version of arm_dma_unmap_page()
 */
static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
		size_t size, enum dma_data_direction dir,
		struct dma_attrs *attrs)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova = handle & PAGE_MASK;
	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
	int offset = handle & ~PAGE_MASK;
	int len = PAGE_ALIGN(size + offset);

	if (!iova)
		return;

R
Rob Herring 已提交
1863
	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880
		__dma_page_dev_to_cpu(page, offset, size, dir);

	iommu_unmap(mapping->domain, iova, len);
	__free_iova(mapping, iova, len);
}

static void arm_iommu_sync_single_for_cpu(struct device *dev,
		dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova = handle & PAGE_MASK;
	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
	unsigned int offset = handle & ~PAGE_MASK;

	if (!iova)
		return;

R
Rob Herring 已提交
1881
	__dma_page_dev_to_cpu(page, offset, size, dir);
1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901
}

static void arm_iommu_sync_single_for_device(struct device *dev,
		dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova = handle & PAGE_MASK;
	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
	unsigned int offset = handle & ~PAGE_MASK;

	if (!iova)
		return;

	__dma_page_cpu_to_dev(page, offset, size, dir);
}

struct dma_map_ops iommu_ops = {
	.alloc		= arm_iommu_alloc_attrs,
	.free		= arm_iommu_free_attrs,
	.mmap		= arm_iommu_mmap_attrs,
1902
	.get_sgtable	= arm_iommu_get_sgtable,
1903 1904 1905 1906 1907 1908 1909 1910 1911 1912

	.map_page		= arm_iommu_map_page,
	.unmap_page		= arm_iommu_unmap_page,
	.sync_single_for_cpu	= arm_iommu_sync_single_for_cpu,
	.sync_single_for_device	= arm_iommu_sync_single_for_device,

	.map_sg			= arm_iommu_map_sg,
	.unmap_sg		= arm_iommu_unmap_sg,
	.sync_sg_for_cpu	= arm_iommu_sync_sg_for_cpu,
	.sync_sg_for_device	= arm_iommu_sync_sg_for_device,
1913 1914

	.set_dma_mask		= arm_dma_set_mask,
1915 1916
};

R
Rob Herring 已提交
1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927
struct dma_map_ops iommu_coherent_ops = {
	.alloc		= arm_iommu_alloc_attrs,
	.free		= arm_iommu_free_attrs,
	.mmap		= arm_iommu_mmap_attrs,
	.get_sgtable	= arm_iommu_get_sgtable,

	.map_page	= arm_coherent_iommu_map_page,
	.unmap_page	= arm_coherent_iommu_unmap_page,

	.map_sg		= arm_coherent_iommu_map_sg,
	.unmap_sg	= arm_coherent_iommu_unmap_sg,
1928 1929

	.set_dma_mask	= arm_dma_set_mask,
R
Rob Herring 已提交
1930 1931
};

1932 1933 1934 1935
/**
 * arm_iommu_create_mapping
 * @bus: pointer to the bus holding the client device (for IOMMU calls)
 * @base: start address of the valid IO address space
1936
 * @size: maximum size of the valid IO address space
1937 1938 1939 1940 1941 1942 1943 1944 1945
 *
 * Creates a mapping structure which holds information about used/unused
 * IO address ranges, which is required to perform memory allocation and
 * mapping with IOMMU aware functions.
 *
 * The client device need to be attached to the mapping with
 * arm_iommu_attach_device function.
 */
struct dma_iommu_mapping *
1946
arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size)
1947
{
1948 1949
	unsigned int bits = size >> PAGE_SHIFT;
	unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);
1950
	struct dma_iommu_mapping *mapping;
1951
	int extensions = 1;
1952 1953
	int err = -ENOMEM;

1954
	if (!bitmap_size)
1955 1956
		return ERR_PTR(-EINVAL);

1957 1958 1959 1960 1961
	if (bitmap_size > PAGE_SIZE) {
		extensions = bitmap_size / PAGE_SIZE;
		bitmap_size = PAGE_SIZE;
	}

1962 1963 1964 1965
	mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
	if (!mapping)
		goto err;

1966 1967
	mapping->bitmap_size = bitmap_size;
	mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *),
1968 1969
				GFP_KERNEL);
	if (!mapping->bitmaps)
1970 1971
		goto err2;

1972
	mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);
1973 1974 1975 1976 1977
	if (!mapping->bitmaps[0])
		goto err3;

	mapping->nr_bitmaps = 1;
	mapping->extensions = extensions;
1978
	mapping->base = base;
1979
	mapping->bits = BITS_PER_BYTE * bitmap_size;
1980

1981 1982 1983 1984
	spin_lock_init(&mapping->lock);

	mapping->domain = iommu_domain_alloc(bus);
	if (!mapping->domain)
1985
		goto err4;
1986 1987 1988

	kref_init(&mapping->kref);
	return mapping;
1989 1990
err4:
	kfree(mapping->bitmaps[0]);
1991
err3:
1992
	kfree(mapping->bitmaps);
1993 1994 1995 1996 1997
err2:
	kfree(mapping);
err:
	return ERR_PTR(err);
}
1998
EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);
1999 2000 2001

static void release_iommu_mapping(struct kref *kref)
{
2002
	int i;
2003 2004 2005 2006
	struct dma_iommu_mapping *mapping =
		container_of(kref, struct dma_iommu_mapping, kref);

	iommu_domain_free(mapping->domain);
2007 2008 2009
	for (i = 0; i < mapping->nr_bitmaps; i++)
		kfree(mapping->bitmaps[i]);
	kfree(mapping->bitmaps);
2010 2011 2012
	kfree(mapping);
}

2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030
static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)
{
	int next_bitmap;

	if (mapping->nr_bitmaps > mapping->extensions)
		return -EINVAL;

	next_bitmap = mapping->nr_bitmaps;
	mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,
						GFP_ATOMIC);
	if (!mapping->bitmaps[next_bitmap])
		return -ENOMEM;

	mapping->nr_bitmaps++;

	return 0;
}

2031 2032 2033 2034 2035
void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
{
	if (mapping)
		kref_put(&mapping->kref, release_iommu_mapping);
}
2036
EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061

/**
 * arm_iommu_attach_device
 * @dev: valid struct device pointer
 * @mapping: io address space mapping structure (returned from
 *	arm_iommu_create_mapping)
 *
 * Attaches specified io address space mapping to the provided device,
 * this replaces the dma operations (dma_map_ops pointer) with the
 * IOMMU aware version. More than one client might be attached to
 * the same io address space mapping.
 */
int arm_iommu_attach_device(struct device *dev,
			    struct dma_iommu_mapping *mapping)
{
	int err;

	err = iommu_attach_device(mapping->domain, dev);
	if (err)
		return err;

	kref_get(&mapping->kref);
	dev->archdata.mapping = mapping;
	set_dma_ops(dev, &iommu_ops);

2062
	pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
2063 2064
	return 0;
}
2065
EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
2066

2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085
/**
 * arm_iommu_detach_device
 * @dev: valid struct device pointer
 *
 * Detaches the provided device from a previously attached map.
 * This voids the dma operations (dma_map_ops pointer)
 */
void arm_iommu_detach_device(struct device *dev)
{
	struct dma_iommu_mapping *mapping;

	mapping = to_dma_iommu_mapping(dev);
	if (!mapping) {
		dev_warn(dev, "Not attached\n");
		return;
	}

	iommu_detach_device(mapping->domain, dev);
	kref_put(&mapping->kref, release_iommu_mapping);
2086
	dev->archdata.mapping = NULL;
2087 2088 2089 2090
	set_dma_ops(dev, NULL);

	pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
}
2091
EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
2092

2093
#endif