dma-mapping.c 43.2 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2
 *  linux/arch/arm/mm/dma-mapping.c
L
Linus Torvalds 已提交
3 4 5 6 7 8 9 10 11 12 13
 *
 *  Copyright (C) 2000-2004 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 *  DMA uncached mapping support.
 */
#include <linux/module.h>
#include <linux/mm.h>
14
#include <linux/gfp.h>
L
Linus Torvalds 已提交
15 16 17 18 19
#include <linux/errno.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
20
#include <linux/dma-contiguous.h>
21
#include <linux/highmem.h>
22
#include <linux/memblock.h>
23
#include <linux/slab.h>
24 25
#include <linux/iommu.h>
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
26

27
#include <asm/memory.h>
28
#include <asm/highmem.h>
L
Linus Torvalds 已提交
29 30
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
31
#include <asm/sizes.h>
32
#include <asm/mach/arch.h>
33
#include <asm/dma-iommu.h>
34 35 36
#include <asm/mach/map.h>
#include <asm/system_info.h>
#include <asm/dma-contiguous.h>
37

38 39
#include "mm.h"

40 41 42 43 44 45 46 47 48 49 50 51
/*
 * The DMA API is built upon the notion of "buffer ownership".  A buffer
 * is either exclusively owned by the CPU (and therefore may be accessed
 * by it) or exclusively owned by the DMA device.  These helper functions
 * represent the transitions between these two ownership states.
 *
 * Note, however, that on later ARMs, this notion does not work due to
 * speculative prefetches.  We model our approach on the assumption that
 * the CPU does do speculative prefetches, which means we clean caches
 * before transfers and delay cache invalidation until transfer completion.
 *
 */
52
static void __dma_page_cpu_to_dev(struct page *, unsigned long,
53
		size_t, enum dma_data_direction);
54
static void __dma_page_dev_to_cpu(struct page *, unsigned long,
55 56
		size_t, enum dma_data_direction);

57 58 59 60 61 62 63 64 65 66 67 68 69 70
/**
 * arm_dma_map_page - map a portion of a page for streaming DMA
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @page: page that buffer resides in
 * @offset: offset into page for start of buffer
 * @size: size of buffer to map
 * @dir: DMA transfer direction
 *
 * Ensure that any data held in the cache is appropriately discarded
 * or written back.
 *
 * The device owns this memory once this call has completed.  The CPU
 * can regain ownership by calling dma_unmap_page().
 */
71
static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
72 73 74
	     unsigned long offset, size_t size, enum dma_data_direction dir,
	     struct dma_attrs *attrs)
{
75 76 77
	if (!arch_is_coherent())
		__dma_page_cpu_to_dev(page, offset, size, dir);
	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
}

/**
 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @handle: DMA address of buffer
 * @size: size of buffer (same as passed to dma_map_page)
 * @dir: DMA transfer direction (same as passed to dma_map_page)
 *
 * Unmap a page streaming mode DMA translation.  The handle and size
 * must match what was provided in the previous dma_map_page() call.
 * All other usages are undefined.
 *
 * After this call, reads by the CPU to the buffer are guaranteed to see
 * whatever the device wrote there.
 */
94
static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
95 96 97
		size_t size, enum dma_data_direction dir,
		struct dma_attrs *attrs)
{
98 99 100
	if (!arch_is_coherent())
		__dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
				      handle & ~PAGE_MASK, size, dir);
101 102
}

103
static void arm_dma_sync_single_for_cpu(struct device *dev,
104 105 106 107
		dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
	unsigned int offset = handle & (PAGE_SIZE - 1);
	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
108 109
	if (!arch_is_coherent())
		__dma_page_dev_to_cpu(page, offset, size, dir);
110 111
}

112
static void arm_dma_sync_single_for_device(struct device *dev,
113 114 115 116
		dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
	unsigned int offset = handle & (PAGE_SIZE - 1);
	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
117 118
	if (!arch_is_coherent())
		__dma_page_cpu_to_dev(page, offset, size, dir);
119 120 121 122 123
}

static int arm_dma_set_mask(struct device *dev, u64 dma_mask);

struct dma_map_ops arm_dma_ops = {
124 125 126
	.alloc			= arm_dma_alloc,
	.free			= arm_dma_free,
	.mmap			= arm_dma_mmap,
127 128 129 130 131 132 133 134 135 136 137 138
	.map_page		= arm_dma_map_page,
	.unmap_page		= arm_dma_unmap_page,
	.map_sg			= arm_dma_map_sg,
	.unmap_sg		= arm_dma_unmap_sg,
	.sync_single_for_cpu	= arm_dma_sync_single_for_cpu,
	.sync_single_for_device	= arm_dma_sync_single_for_device,
	.sync_sg_for_cpu	= arm_dma_sync_sg_for_cpu,
	.sync_sg_for_device	= arm_dma_sync_sg_for_device,
	.set_dma_mask		= arm_dma_set_mask,
};
EXPORT_SYMBOL(arm_dma_ops);

139 140
static u64 get_coherent_dma_mask(struct device *dev)
{
141
	u64 mask = (u64)arm_dma_limit;
142 143 144 145 146 147 148 149 150 151 152 153 154

	if (dev) {
		mask = dev->coherent_dma_mask;

		/*
		 * Sanity check the DMA mask - it must be non-zero, and
		 * must be able to be satisfied by a DMA allocation.
		 */
		if (mask == 0) {
			dev_warn(dev, "coherent DMA mask is unset\n");
			return 0;
		}

155
		if ((~mask) & (u64)arm_dma_limit) {
156 157
			dev_warn(dev, "coherent DMA mask %#llx is smaller "
				 "than system GFP_DMA mask %#llx\n",
158
				 mask, (u64)arm_dma_limit);
159 160 161
			return 0;
		}
	}
L
Linus Torvalds 已提交
162

163 164 165
	return mask;
}

166 167 168 169 170 171 172 173
static void __dma_clear_buffer(struct page *page, size_t size)
{
	void *ptr;
	/*
	 * Ensure that the allocated pages are zeroed, and that any data
	 * lurking in the kernel direct-mapped region is invalidated.
	 */
	ptr = page_address(page);
174 175 176 177 178
	if (ptr) {
		memset(ptr, 0, size);
		dmac_flush_range(ptr, ptr + size);
		outer_flush_range(__pa(ptr), __pa(ptr) + size);
	}
179 180
}

181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
/*
 * Allocate a DMA buffer for 'dev' of size 'size' using the
 * specified gfp mask.  Note that 'size' must be page aligned.
 */
static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
{
	unsigned long order = get_order(size);
	struct page *page, *p, *e;

	page = alloc_pages(gfp, order);
	if (!page)
		return NULL;

	/*
	 * Now split the huge page and free the excess pages
	 */
	split_page(page, order);
	for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
		__free_page(p);

201
	__dma_clear_buffer(page, size);
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218

	return page;
}

/*
 * Free a DMA buffer.  'size' must be page aligned.
 */
static void __dma_free_buffer(struct page *page, size_t size)
{
	struct page *e = page + (size >> PAGE_SHIFT);

	while (page < e) {
		__free_page(page);
		page++;
	}
}

219
#ifdef CONFIG_MMU
220

221
#define CONSISTENT_OFFSET(x)	(((unsigned long)(x) - consistent_base) >> PAGE_SHIFT)
L
Linus Torvalds 已提交
222
#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - consistent_base) >> PMD_SHIFT)
223

L
Linus Torvalds 已提交
224
/*
225
 * These are the page tables (2MB each) covering uncached, DMA consistent allocations
L
Linus Torvalds 已提交
226
 */
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
static pte_t **consistent_pte;

#define DEFAULT_CONSISTENT_DMA_SIZE SZ_2M

unsigned long consistent_base = CONSISTENT_END - DEFAULT_CONSISTENT_DMA_SIZE;

void __init init_consistent_dma_size(unsigned long size)
{
	unsigned long base = CONSISTENT_END - ALIGN(size, SZ_2M);

	BUG_ON(consistent_pte); /* Check we're called before DMA region init */
	BUG_ON(base < VMALLOC_END);

	/* Grow region to accommodate specified size  */
	if (base < consistent_base)
		consistent_base = base;
}
L
Linus Torvalds 已提交
244

245
#include "vmregion.h"
L
Linus Torvalds 已提交
246

247 248
static struct arm_vmregion_head consistent_head = {
	.vm_lock	= __SPIN_LOCK_UNLOCKED(&consistent_head.vm_lock),
L
Linus Torvalds 已提交
249 250 251 252 253 254 255 256
	.vm_list	= LIST_HEAD_INIT(consistent_head.vm_list),
	.vm_end		= CONSISTENT_END,
};

#ifdef CONFIG_HUGETLB_PAGE
#error ARM Coherent DMA allocator does not (yet) support huge TLB
#endif

257 258 259 260 261 262 263
/*
 * Initialise the consistent memory allocation.
 */
static int __init consistent_init(void)
{
	int ret = 0;
	pgd_t *pgd;
R
Russell King 已提交
264
	pud_t *pud;
265 266 267
	pmd_t *pmd;
	pte_t *pte;
	int i = 0;
268
	unsigned long base = consistent_base;
269
	unsigned long num_ptes = (CONSISTENT_END - base) >> PMD_SHIFT;
270

271
#ifndef CONFIG_ARM_DMA_USE_IOMMU
272 273
	if (cpu_architecture() >= CPU_ARCH_ARMv6)
		return 0;
274
#endif
275

276 277 278 279 280 281 282 283
	consistent_pte = kmalloc(num_ptes * sizeof(pte_t), GFP_KERNEL);
	if (!consistent_pte) {
		pr_err("%s: no memory\n", __func__);
		return -ENOMEM;
	}

	pr_debug("DMA memory: 0x%08lx - 0x%08lx:\n", base, CONSISTENT_END);
	consistent_head.vm_start = base;
284 285 286

	do {
		pgd = pgd_offset(&init_mm, base);
R
Russell King 已提交
287 288 289

		pud = pud_alloc(&init_mm, pgd, base);
		if (!pud) {
290
			pr_err("%s: no pud tables\n", __func__);
R
Russell King 已提交
291 292 293 294 295
			ret = -ENOMEM;
			break;
		}

		pmd = pmd_alloc(&init_mm, pud, base);
296
		if (!pmd) {
297
			pr_err("%s: no pmd tables\n", __func__);
298 299 300 301 302 303 304
			ret = -ENOMEM;
			break;
		}
		WARN_ON(!pmd_none(*pmd));

		pte = pte_alloc_kernel(pmd, base);
		if (!pte) {
305
			pr_err("%s: no pte tables\n", __func__);
306 307 308 309 310
			ret = -ENOMEM;
			break;
		}

		consistent_pte[i++] = pte;
311
		base += PMD_SIZE;
312 313 314 315 316 317
	} while (base < CONSISTENT_END);

	return ret;
}
core_initcall(consistent_init);

318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
static void *__alloc_from_contiguous(struct device *dev, size_t size,
				     pgprot_t prot, struct page **ret_page);

static struct arm_vmregion_head coherent_head = {
	.vm_lock	= __SPIN_LOCK_UNLOCKED(&coherent_head.vm_lock),
	.vm_list	= LIST_HEAD_INIT(coherent_head.vm_list),
};

size_t coherent_pool_size = DEFAULT_CONSISTENT_DMA_SIZE / 8;

static int __init early_coherent_pool(char *p)
{
	coherent_pool_size = memparse(p, &p);
	return 0;
}
early_param("coherent_pool", early_coherent_pool);

/*
 * Initialise the coherent pool for atomic allocations.
 */
static int __init coherent_init(void)
{
	pgprot_t prot = pgprot_dmacoherent(pgprot_kernel);
	size_t size = coherent_pool_size;
	struct page *page;
	void *ptr;

	if (cpu_architecture() < CPU_ARCH_ARMv6)
		return 0;

	ptr = __alloc_from_contiguous(NULL, size, prot, &page);
	if (ptr) {
		coherent_head.vm_start = (unsigned long) ptr;
		coherent_head.vm_end = (unsigned long) ptr + size;
		printk(KERN_INFO "DMA: preallocated %u KiB pool for atomic coherent allocations\n",
		       (unsigned)size / 1024);
		return 0;
	}
	printk(KERN_ERR "DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
	       (unsigned)size / 1024);
	return -ENOMEM;
}
/*
 * CMA is activated by core_initcall, so we must be called after it.
 */
postcore_initcall(coherent_init);

struct dma_contig_early_reserve {
	phys_addr_t base;
	unsigned long size;
};

static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;

static int dma_mmu_remap_num __initdata;

void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
{
	dma_mmu_remap[dma_mmu_remap_num].base = base;
	dma_mmu_remap[dma_mmu_remap_num].size = size;
	dma_mmu_remap_num++;
}

void __init dma_contiguous_remap(void)
{
	int i;
	for (i = 0; i < dma_mmu_remap_num; i++) {
		phys_addr_t start = dma_mmu_remap[i].base;
		phys_addr_t end = start + dma_mmu_remap[i].size;
		struct map_desc map;
		unsigned long addr;

		if (end > arm_lowmem_limit)
			end = arm_lowmem_limit;
		if (start >= end)
			return;

		map.pfn = __phys_to_pfn(start);
		map.virtual = __phys_to_virt(start);
		map.length = end - start;
		map.type = MT_MEMORY_DMA_READY;

		/*
		 * Clear previous low-memory mapping
		 */
		for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
404
		     addr += PMD_SIZE)
405 406 407 408 409 410
			pmd_clear(pmd_off_k(addr));

		iotable_init(&map, 1);
	}
}

L
Linus Torvalds 已提交
411
static void *
412 413
__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
	const void *caller)
L
Linus Torvalds 已提交
414
{
415
	struct arm_vmregion *c;
416 417
	size_t align;
	int bit;
L
Linus Torvalds 已提交
418

419
	if (!consistent_pte) {
420
		pr_err("%s: not initialised\n", __func__);
421 422 423 424
		dump_stack();
		return NULL;
	}

425 426 427 428 429 430
	/*
	 * Align the virtual region allocation - maximum alignment is
	 * a section size, minimum is a page size.  This helps reduce
	 * fragmentation of the DMA space, and also prevents allocations
	 * smaller than a section from crossing a section boundary.
	 */
431
	bit = fls(size - 1);
432 433 434 435
	if (bit > SECTION_SHIFT)
		bit = SECTION_SHIFT;
	align = 1 << bit;

L
Linus Torvalds 已提交
436 437 438
	/*
	 * Allocate a virtual address in the consistent mapping region.
	 */
439
	c = arm_vmregion_alloc(&consistent_head, align, size,
440
			    gfp & ~(__GFP_DMA | __GFP_HIGHMEM), caller);
L
Linus Torvalds 已提交
441
	if (c) {
442 443 444
		pte_t *pte;
		int idx = CONSISTENT_PTE_INDEX(c->vm_start);
		u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
L
Linus Torvalds 已提交
445

446
		pte = consistent_pte[idx] + off;
447
		c->priv = page;
L
Linus Torvalds 已提交
448 449 450 451

		do {
			BUG_ON(!pte_none(*pte));

R
Russell King 已提交
452
			set_pte_ext(pte, mk_pte(page, prot), 0);
L
Linus Torvalds 已提交
453 454
			page++;
			pte++;
455 456 457 458 459
			off++;
			if (off >= PTRS_PER_PTE) {
				off = 0;
				pte = consistent_pte[++idx];
			}
L
Linus Torvalds 已提交
460 461
		} while (size -= PAGE_SIZE);

462 463
		dsb();

L
Linus Torvalds 已提交
464 465 466 467
		return (void *)c->vm_start;
	}
	return NULL;
}
468 469 470 471 472 473 474 475 476 477 478

static void __dma_free_remap(void *cpu_addr, size_t size)
{
	struct arm_vmregion *c;
	unsigned long addr;
	pte_t *ptep;
	int idx;
	u32 off;

	c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr);
	if (!c) {
479
		pr_err("%s: trying to free invalid coherent area: %p\n",
480 481 482 483 484 485
		       __func__, cpu_addr);
		dump_stack();
		return;
	}

	if ((c->vm_end - c->vm_start) != size) {
486
		pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
		       __func__, c->vm_end - c->vm_start, size);
		dump_stack();
		size = c->vm_end - c->vm_start;
	}

	idx = CONSISTENT_PTE_INDEX(c->vm_start);
	off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
	ptep = consistent_pte[idx] + off;
	addr = c->vm_start;
	do {
		pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);

		ptep++;
		addr += PAGE_SIZE;
		off++;
		if (off >= PTRS_PER_PTE) {
			off = 0;
			ptep = consistent_pte[++idx];
		}

507
		if (pte_none(pte) || !pte_present(pte))
508 509
			pr_crit("%s: bad page in kernel page table\n",
				__func__);
510 511 512 513 514 515 516
	} while (size -= PAGE_SIZE);

	flush_tlb_kernel_range(c->vm_start, c->vm_end);

	arm_vmregion_free(&consistent_head, c);
}

517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,
			    void *data)
{
	struct page *page = virt_to_page(addr);
	pgprot_t prot = *(pgprot_t *)data;

	set_pte_ext(pte, mk_pte(page, prot), 0);
	return 0;
}

static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
{
	unsigned long start = (unsigned long) page_address(page);
	unsigned end = start + size;

	apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
	dsb();
	flush_tlb_kernel_range(start, end);
}

static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
				 pgprot_t prot, struct page **ret_page,
				 const void *caller)
{
	struct page *page;
	void *ptr;
	page = __dma_alloc_buffer(dev, size, gfp);
	if (!page)
		return NULL;

	ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
	if (!ptr) {
		__dma_free_buffer(page, size);
		return NULL;
	}

	*ret_page = page;
	return ptr;
}

static void *__alloc_from_pool(struct device *dev, size_t size,
			       struct page **ret_page, const void *caller)
{
	struct arm_vmregion *c;
	size_t align;

	if (!coherent_head.vm_start) {
		printk(KERN_ERR "%s: coherent pool not initialised!\n",
		       __func__);
		dump_stack();
		return NULL;
	}

	/*
	 * Align the region allocation - allocations from pool are rather
	 * small, so align them to their order in pages, minimum is a page
	 * size. This helps reduce fragmentation of the DMA space.
	 */
	align = PAGE_SIZE << get_order(size);
	c = arm_vmregion_alloc(&coherent_head, align, size, 0, caller);
	if (c) {
		void *ptr = (void *)c->vm_start;
		struct page *page = virt_to_page(ptr);
		*ret_page = page;
		return ptr;
	}
	return NULL;
}

static int __free_from_pool(void *cpu_addr, size_t size)
{
	unsigned long start = (unsigned long)cpu_addr;
	unsigned long end = start + size;
	struct arm_vmregion *c;

	if (start < coherent_head.vm_start || end > coherent_head.vm_end)
		return 0;

	c = arm_vmregion_find_remove(&coherent_head, (unsigned long)start);

	if ((c->vm_end - c->vm_start) != size) {
		printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
		       __func__, c->vm_end - c->vm_start, size);
		dump_stack();
		size = c->vm_end - c->vm_start;
	}

	arm_vmregion_free(&coherent_head, c);
	return 1;
}

static void *__alloc_from_contiguous(struct device *dev, size_t size,
				     pgprot_t prot, struct page **ret_page)
{
	unsigned long order = get_order(size);
	size_t count = size >> PAGE_SHIFT;
	struct page *page;

	page = dma_alloc_from_contiguous(dev, count, order);
	if (!page)
		return NULL;

	__dma_clear_buffer(page, size);
	__dma_remap(page, size, prot);

	*ret_page = page;
	return page_address(page);
}

static void __free_from_contiguous(struct device *dev, struct page *page,
				   size_t size)
{
	__dma_remap(page, size, pgprot_kernel);
	dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
}

633 634 635 636 637 638 639 640
static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
{
	prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
			    pgprot_writecombine(prot) :
			    pgprot_dmacoherent(prot);
	return prot;
}

641 642
#define nommu() 0

643
#else	/* !CONFIG_MMU */
644

645 646
#define nommu() 1

647
#define __get_dma_pgprot(attrs, prot)	__pgprot(0)
648 649 650 651 652 653
#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c)	NULL
#define __alloc_from_pool(dev, size, ret_page, c)		NULL
#define __alloc_from_contiguous(dev, size, prot, ret)		NULL
#define __free_from_pool(cpu_addr, size)			0
#define __free_from_contiguous(dev, page, size)			do { } while (0)
#define __dma_free_remap(cpu_addr, size)			do { } while (0)
654 655 656

#endif	/* CONFIG_MMU */

657 658
static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
				   struct page **ret_page)
659
{
660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
	struct page *page;
	page = __dma_alloc_buffer(dev, size, gfp);
	if (!page)
		return NULL;

	*ret_page = page;
	return page_address(page);
}



static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
			 gfp_t gfp, pgprot_t prot, const void *caller)
{
	u64 mask = get_coherent_dma_mask(dev);
675
	struct page *page;
676
	void *addr;
677

678 679 680 681 682 683 684 685 686 687 688 689 690 691 692
#ifdef CONFIG_DMA_API_DEBUG
	u64 limit = (mask + 1) & ~mask;
	if (limit && size >= limit) {
		dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
			size, mask);
		return NULL;
	}
#endif

	if (!mask)
		return NULL;

	if (mask < 0xffffffffULL)
		gfp |= GFP_DMA;

693 694 695 696 697 698 699 700 701
	/*
	 * Following is a work-around (a.k.a. hack) to prevent pages
	 * with __GFP_COMP being passed to split_page() which cannot
	 * handle them.  The real problem is that this flag probably
	 * should be 0 on ARM as it is not supported on this
	 * platform; see CONFIG_HUGETLBFS.
	 */
	gfp &= ~(__GFP_COMP);

702
	*handle = DMA_ERROR_CODE;
703
	size = PAGE_ALIGN(size);
704

705 706 707 708 709 710
	if (arch_is_coherent() || nommu())
		addr = __alloc_simple_buffer(dev, size, gfp, &page);
	else if (cpu_architecture() < CPU_ARCH_ARMv6)
		addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);
	else if (gfp & GFP_ATOMIC)
		addr = __alloc_from_pool(dev, size, &page, caller);
711
	else
712
		addr = __alloc_from_contiguous(dev, size, prot, &page);
713

714
	if (addr)
715
		*handle = pfn_to_dma(dev, page_to_pfn(page));
716

717 718
	return addr;
}
L
Linus Torvalds 已提交
719 720 721 722 723

/*
 * Allocate DMA-coherent memory space and return both the kernel remapped
 * virtual and bus address for that space.
 */
724 725
void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
		    gfp_t gfp, struct dma_attrs *attrs)
L
Linus Torvalds 已提交
726
{
727
	pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
728 729 730 731 732
	void *memory;

	if (dma_alloc_from_coherent(dev, size, handle, &memory))
		return memory;

733
	return __dma_alloc(dev, size, handle, gfp, prot,
734
			   __builtin_return_address(0));
L
Linus Torvalds 已提交
735 736 737
}

/*
738
 * Create userspace mapping for the DMA-coherent memory.
L
Linus Torvalds 已提交
739
 */
740 741 742
int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
		 void *cpu_addr, dma_addr_t dma_addr, size_t size,
		 struct dma_attrs *attrs)
L
Linus Torvalds 已提交
743
{
744 745
	int ret = -ENXIO;
#ifdef CONFIG_MMU
746
	unsigned long pfn = dma_to_pfn(dev, dma_addr);
747 748
	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);

749 750 751
	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
		return ret;

752 753 754 755
	ret = remap_pfn_range(vma, vma->vm_start,
			      pfn + vma->vm_pgoff,
			      vma->vm_end - vma->vm_start,
			      vma->vm_page_prot);
756
#endif	/* CONFIG_MMU */
L
Linus Torvalds 已提交
757 758 759 760 761

	return ret;
}

/*
762
 * Free a buffer as defined by the above mapping.
L
Linus Torvalds 已提交
763
 */
764 765
void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
		  dma_addr_t handle, struct dma_attrs *attrs)
L
Linus Torvalds 已提交
766
{
767
	struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
768

769 770 771
	if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
		return;

772 773
	size = PAGE_ALIGN(size);

774 775 776
	if (arch_is_coherent() || nommu()) {
		__dma_free_buffer(page, size);
	} else if (cpu_architecture() < CPU_ARCH_ARMv6) {
777
		__dma_free_remap(cpu_addr, size);
778 779 780 781 782 783 784 785 786 787
		__dma_free_buffer(page, size);
	} else {
		if (__free_from_pool(cpu_addr, size))
			return;
		/*
		 * Non-atomic allocations cannot be freed with IRQs disabled
		 */
		WARN_ON(irqs_disabled());
		__free_from_contiguous(dev, page, size);
	}
L
Linus Torvalds 已提交
788
}
789

790
static void dma_cache_maint_page(struct page *page, unsigned long offset,
791 792
	size_t size, enum dma_data_direction dir,
	void (*op)(const void *, size_t, int))
793 794 795 796 797 798 799 800 801 802
{
	/*
	 * A single sg entry may refer to multiple physically contiguous
	 * pages.  But we still need to process highmem pages individually.
	 * If highmem is not configured then the bulk of this loop gets
	 * optimized out.
	 */
	size_t left = size;
	do {
		size_t len = left;
803 804 805 806 807 808 809 810 811 812 813 814 815
		void *vaddr;

		if (PageHighMem(page)) {
			if (len + offset > PAGE_SIZE) {
				if (offset >= PAGE_SIZE) {
					page += offset / PAGE_SIZE;
					offset %= PAGE_SIZE;
				}
				len = PAGE_SIZE - offset;
			}
			vaddr = kmap_high_get(page);
			if (vaddr) {
				vaddr += offset;
816
				op(vaddr, len, dir);
817
				kunmap_high(page);
818
			} else if (cache_is_vipt()) {
819 820
				/* unmapped pages might still be cached */
				vaddr = kmap_atomic(page);
821
				op(vaddr + offset, len, dir);
822
				kunmap_atomic(vaddr);
823
			}
824 825
		} else {
			vaddr = page_address(page) + offset;
826
			op(vaddr, len, dir);
827 828 829 830 831 832
		}
		offset = 0;
		page++;
		left -= len;
	} while (left);
}
833

834 835 836 837 838 839 840
/*
 * Make an area consistent for devices.
 * Note: Drivers should NOT use this function directly, as it will break
 * platforms with CONFIG_DMABOUNCE.
 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
 */
static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
841 842
	size_t size, enum dma_data_direction dir)
{
843 844
	unsigned long paddr;

845
	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
846 847

	paddr = page_to_phys(page) + off;
848 849 850 851 852 853
	if (dir == DMA_FROM_DEVICE) {
		outer_inv_range(paddr, paddr + size);
	} else {
		outer_clean_range(paddr, paddr + size);
	}
	/* FIXME: non-speculating: flush on bidirectional mappings? */
854 855
}

856
static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
857 858
	size_t size, enum dma_data_direction dir)
{
859 860 861 862 863 864 865
	unsigned long paddr = page_to_phys(page) + off;

	/* FIXME: non-speculating: not required */
	/* don't bother invalidating if DMA to device */
	if (dir != DMA_TO_DEVICE)
		outer_inv_range(paddr, paddr + size);

866
	dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
867 868 869 870 871 872

	/*
	 * Mark the D-cache clean for this page to avoid extra flushing.
	 */
	if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
		set_bit(PG_dcache_clean, &page->flags);
873
}
874

875
/**
876
 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
877 878 879 880 881 882 883 884 885 886 887 888 889 890
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map
 * @dir: DMA transfer direction
 *
 * Map a set of buffers described by scatterlist in streaming mode for DMA.
 * This is the scatter-gather version of the dma_map_single interface.
 * Here the scatter gather list elements are each tagged with the
 * appropriate dma address and length.  They are obtained via
 * sg_dma_{address,length}.
 *
 * Device ownership issues as mentioned for dma_map_single are the same
 * here.
 */
891 892
int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
		enum dma_data_direction dir, struct dma_attrs *attrs)
893
{
894
	struct dma_map_ops *ops = get_dma_ops(dev);
895
	struct scatterlist *s;
896
	int i, j;
897 898

	for_each_sg(sg, s, nents, i) {
899 900 901
#ifdef CONFIG_NEED_SG_DMA_LENGTH
		s->dma_length = s->length;
#endif
902 903
		s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
						s->length, dir, attrs);
904 905
		if (dma_mapping_error(dev, s->dma_address))
			goto bad_mapping;
906 907
	}
	return nents;
908 909 910

 bad_mapping:
	for_each_sg(sg, s, i, j)
911
		ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
912
	return 0;
913 914 915
}

/**
916
 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
917 918
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
919
 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
920 921 922 923 924
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 *
 * Unmap a set of streaming mode DMA translations.  Again, CPU access
 * rules concerning calls here are the same as for dma_unmap_single().
 */
925 926
void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
		enum dma_data_direction dir, struct dma_attrs *attrs)
927
{
928
	struct dma_map_ops *ops = get_dma_ops(dev);
929 930 931
	struct scatterlist *s;

	int i;
932

933
	for_each_sg(sg, s, nents, i)
934
		ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
935 936 937
}

/**
938
 * arm_dma_sync_sg_for_cpu
939 940 941 942 943
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
944
void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
945 946
			int nents, enum dma_data_direction dir)
{
947
	struct dma_map_ops *ops = get_dma_ops(dev);
948 949 950
	struct scatterlist *s;
	int i;

951 952 953
	for_each_sg(sg, s, nents, i)
		ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
					 dir);
954 955 956
}

/**
957
 * arm_dma_sync_sg_for_device
958 959 960 961 962
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
963
void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
964 965
			int nents, enum dma_data_direction dir)
{
966
	struct dma_map_ops *ops = get_dma_ops(dev);
967 968 969
	struct scatterlist *s;
	int i;

970 971 972
	for_each_sg(sg, s, nents, i)
		ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
					    dir);
973
}
974

975 976 977 978 979 980 981 982 983 984 985 986 987 988
/*
 * Return whether the given device DMA address mask can be supported
 * properly.  For example, if your device can only drive the low 24-bits
 * during bus mastering, then you would pass 0x00ffffff as the mask
 * to this function.
 */
int dma_supported(struct device *dev, u64 mask)
{
	if (mask < (u64)arm_dma_limit)
		return 0;
	return 1;
}
EXPORT_SYMBOL(dma_supported);

989
static int arm_dma_set_mask(struct device *dev, u64 dma_mask)
990 991 992 993 994 995 996 997 998
{
	if (!dev->dma_mask || !dma_supported(dev, dma_mask))
		return -EIO;

	*dev->dma_mask = dma_mask;

	return 0;
}

999 1000 1001 1002
#define PREALLOC_DMA_DEBUG_ENTRIES	4096

static int __init dma_debug_do_init(void)
{
1003 1004 1005
#ifdef CONFIG_MMU
	arm_vmregion_create_proc("dma-mappings", &consistent_head);
#endif
1006 1007 1008 1009
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
	return 0;
}
fs_initcall(dma_debug_do_init);
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685

#ifdef CONFIG_ARM_DMA_USE_IOMMU

/* IOMMU */

static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
				      size_t size)
{
	unsigned int order = get_order(size);
	unsigned int align = 0;
	unsigned int count, start;
	unsigned long flags;

	count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) +
		 (1 << mapping->order) - 1) >> mapping->order;

	if (order > mapping->order)
		align = (1 << (order - mapping->order)) - 1;

	spin_lock_irqsave(&mapping->lock, flags);
	start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0,
					   count, align);
	if (start > mapping->bits) {
		spin_unlock_irqrestore(&mapping->lock, flags);
		return DMA_ERROR_CODE;
	}

	bitmap_set(mapping->bitmap, start, count);
	spin_unlock_irqrestore(&mapping->lock, flags);

	return mapping->base + (start << (mapping->order + PAGE_SHIFT));
}

static inline void __free_iova(struct dma_iommu_mapping *mapping,
			       dma_addr_t addr, size_t size)
{
	unsigned int start = (addr - mapping->base) >>
			     (mapping->order + PAGE_SHIFT);
	unsigned int count = ((size >> PAGE_SHIFT) +
			      (1 << mapping->order) - 1) >> mapping->order;
	unsigned long flags;

	spin_lock_irqsave(&mapping->lock, flags);
	bitmap_clear(mapping->bitmap, start, count);
	spin_unlock_irqrestore(&mapping->lock, flags);
}

static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
{
	struct page **pages;
	int count = size >> PAGE_SHIFT;
	int array_size = count * sizeof(struct page *);
	int i = 0;

	if (array_size <= PAGE_SIZE)
		pages = kzalloc(array_size, gfp);
	else
		pages = vzalloc(array_size);
	if (!pages)
		return NULL;

	while (count) {
		int j, order = __ffs(count);

		pages[i] = alloc_pages(gfp | __GFP_NOWARN, order);
		while (!pages[i] && order)
			pages[i] = alloc_pages(gfp | __GFP_NOWARN, --order);
		if (!pages[i])
			goto error;

		if (order)
			split_page(pages[i], order);
		j = 1 << order;
		while (--j)
			pages[i + j] = pages[i] + j;

		__dma_clear_buffer(pages[i], PAGE_SIZE << order);
		i += 1 << order;
		count -= 1 << order;
	}

	return pages;
error:
	while (--i)
		if (pages[i])
			__free_pages(pages[i], 0);
	if (array_size < PAGE_SIZE)
		kfree(pages);
	else
		vfree(pages);
	return NULL;
}

static int __iommu_free_buffer(struct device *dev, struct page **pages, size_t size)
{
	int count = size >> PAGE_SHIFT;
	int array_size = count * sizeof(struct page *);
	int i;
	for (i = 0; i < count; i++)
		if (pages[i])
			__free_pages(pages[i], 0);
	if (array_size < PAGE_SIZE)
		kfree(pages);
	else
		vfree(pages);
	return 0;
}

/*
 * Create a CPU mapping for a specified pages
 */
static void *
__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot)
{
	struct arm_vmregion *c;
	size_t align;
	size_t count = size >> PAGE_SHIFT;
	int bit;

	if (!consistent_pte[0]) {
		pr_err("%s: not initialised\n", __func__);
		dump_stack();
		return NULL;
	}

	/*
	 * Align the virtual region allocation - maximum alignment is
	 * a section size, minimum is a page size.  This helps reduce
	 * fragmentation of the DMA space, and also prevents allocations
	 * smaller than a section from crossing a section boundary.
	 */
	bit = fls(size - 1);
	if (bit > SECTION_SHIFT)
		bit = SECTION_SHIFT;
	align = 1 << bit;

	/*
	 * Allocate a virtual address in the consistent mapping region.
	 */
	c = arm_vmregion_alloc(&consistent_head, align, size,
			    gfp & ~(__GFP_DMA | __GFP_HIGHMEM), NULL);
	if (c) {
		pte_t *pte;
		int idx = CONSISTENT_PTE_INDEX(c->vm_start);
		int i = 0;
		u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);

		pte = consistent_pte[idx] + off;
		c->priv = pages;

		do {
			BUG_ON(!pte_none(*pte));

			set_pte_ext(pte, mk_pte(pages[i], prot), 0);
			pte++;
			off++;
			i++;
			if (off >= PTRS_PER_PTE) {
				off = 0;
				pte = consistent_pte[++idx];
			}
		} while (i < count);

		dsb();

		return (void *)c->vm_start;
	}
	return NULL;
}

/*
 * Create a mapping in device IO address space for specified pages
 */
static dma_addr_t
__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
	dma_addr_t dma_addr, iova;
	int i, ret = DMA_ERROR_CODE;

	dma_addr = __alloc_iova(mapping, size);
	if (dma_addr == DMA_ERROR_CODE)
		return dma_addr;

	iova = dma_addr;
	for (i = 0; i < count; ) {
		unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
		phys_addr_t phys = page_to_phys(pages[i]);
		unsigned int len, j;

		for (j = i + 1; j < count; j++, next_pfn++)
			if (page_to_pfn(pages[j]) != next_pfn)
				break;

		len = (j - i) << PAGE_SHIFT;
		ret = iommu_map(mapping->domain, iova, phys, len, 0);
		if (ret < 0)
			goto fail;
		iova += len;
		i = j;
	}
	return dma_addr;
fail:
	iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
	__free_iova(mapping, dma_addr, size);
	return DMA_ERROR_CODE;
}

static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;

	/*
	 * add optional in-page offset from iova to size and align
	 * result to page size
	 */
	size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
	iova &= PAGE_MASK;

	iommu_unmap(mapping->domain, iova, size);
	__free_iova(mapping, iova, size);
	return 0;
}

static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
	    dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
{
	pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
	struct page **pages;
	void *addr = NULL;

	*handle = DMA_ERROR_CODE;
	size = PAGE_ALIGN(size);

	pages = __iommu_alloc_buffer(dev, size, gfp);
	if (!pages)
		return NULL;

	*handle = __iommu_create_mapping(dev, pages, size);
	if (*handle == DMA_ERROR_CODE)
		goto err_buffer;

	addr = __iommu_alloc_remap(pages, size, gfp, prot);
	if (!addr)
		goto err_mapping;

	return addr;

err_mapping:
	__iommu_remove_mapping(dev, *handle, size);
err_buffer:
	__iommu_free_buffer(dev, pages, size);
	return NULL;
}

static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
		    void *cpu_addr, dma_addr_t dma_addr, size_t size,
		    struct dma_attrs *attrs)
{
	struct arm_vmregion *c;

	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
	c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);

	if (c) {
		struct page **pages = c->priv;

		unsigned long uaddr = vma->vm_start;
		unsigned long usize = vma->vm_end - vma->vm_start;
		int i = 0;

		do {
			int ret;

			ret = vm_insert_page(vma, uaddr, pages[i++]);
			if (ret) {
				pr_err("Remapping memory, error: %d\n", ret);
				return ret;
			}

			uaddr += PAGE_SIZE;
			usize -= PAGE_SIZE;
		} while (usize > 0);
	}
	return 0;
}

/*
 * free a page as defined by the above mapping.
 * Must not be called with IRQs disabled.
 */
void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
			  dma_addr_t handle, struct dma_attrs *attrs)
{
	struct arm_vmregion *c;
	size = PAGE_ALIGN(size);

	c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
	if (c) {
		struct page **pages = c->priv;
		__dma_free_remap(cpu_addr, size);
		__iommu_remove_mapping(dev, handle, size);
		__iommu_free_buffer(dev, pages, size);
	}
}

/*
 * Map a part of the scatter-gather list into contiguous io address space
 */
static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
			  size_t size, dma_addr_t *handle,
			  enum dma_data_direction dir)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova, iova_base;
	int ret = 0;
	unsigned int count;
	struct scatterlist *s;

	size = PAGE_ALIGN(size);
	*handle = DMA_ERROR_CODE;

	iova_base = iova = __alloc_iova(mapping, size);
	if (iova == DMA_ERROR_CODE)
		return -ENOMEM;

	for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
		phys_addr_t phys = page_to_phys(sg_page(s));
		unsigned int len = PAGE_ALIGN(s->offset + s->length);

		if (!arch_is_coherent())
			__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);

		ret = iommu_map(mapping->domain, iova, phys, len, 0);
		if (ret < 0)
			goto fail;
		count += len >> PAGE_SHIFT;
		iova += len;
	}
	*handle = iova_base;

	return 0;
fail:
	iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
	__free_iova(mapping, iova_base, size);
	return ret;
}

/**
 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
 * @dev: valid struct device pointer
 * @sg: list of buffers
 * @nents: number of buffers to map
 * @dir: DMA transfer direction
 *
 * Map a set of buffers described by scatterlist in streaming mode for DMA.
 * The scatter gather list elements are merged together (if possible) and
 * tagged with the appropriate dma address and length. They are obtained via
 * sg_dma_{address,length}.
 */
int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
		     enum dma_data_direction dir, struct dma_attrs *attrs)
{
	struct scatterlist *s = sg, *dma = sg, *start = sg;
	int i, count = 0;
	unsigned int offset = s->offset;
	unsigned int size = s->offset + s->length;
	unsigned int max = dma_get_max_seg_size(dev);

	for (i = 1; i < nents; i++) {
		s = sg_next(s);

		s->dma_address = DMA_ERROR_CODE;
		s->dma_length = 0;

		if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
			if (__map_sg_chunk(dev, start, size, &dma->dma_address,
			    dir) < 0)
				goto bad_mapping;

			dma->dma_address += offset;
			dma->dma_length = size - offset;

			size = offset = s->offset;
			start = s;
			dma = sg_next(dma);
			count += 1;
		}
		size += s->length;
	}
	if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir) < 0)
		goto bad_mapping;

	dma->dma_address += offset;
	dma->dma_length = size - offset;

	return count+1;

bad_mapping:
	for_each_sg(sg, s, count, i)
		__iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
	return 0;
}

/**
 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
 * @dev: valid struct device pointer
 * @sg: list of buffers
 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 *
 * Unmap a set of streaming mode DMA translations.  Again, CPU access
 * rules concerning calls here are the same as for dma_unmap_single().
 */
void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
			enum dma_data_direction dir, struct dma_attrs *attrs)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i) {
		if (sg_dma_len(s))
			__iommu_remove_mapping(dev, sg_dma_address(s),
					       sg_dma_len(s));
		if (!arch_is_coherent())
			__dma_page_dev_to_cpu(sg_page(s), s->offset,
					      s->length, dir);
	}
}

/**
 * arm_iommu_sync_sg_for_cpu
 * @dev: valid struct device pointer
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
			int nents, enum dma_data_direction dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i)
		if (!arch_is_coherent())
			__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);

}

/**
 * arm_iommu_sync_sg_for_device
 * @dev: valid struct device pointer
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
			int nents, enum dma_data_direction dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i)
		if (!arch_is_coherent())
			__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
}


/**
 * arm_iommu_map_page
 * @dev: valid struct device pointer
 * @page: page that buffer resides in
 * @offset: offset into page for start of buffer
 * @size: size of buffer to map
 * @dir: DMA transfer direction
 *
 * IOMMU aware version of arm_dma_map_page()
 */
static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
	     unsigned long offset, size_t size, enum dma_data_direction dir,
	     struct dma_attrs *attrs)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t dma_addr;
	int ret, len = PAGE_ALIGN(size + offset);

	if (!arch_is_coherent())
		__dma_page_cpu_to_dev(page, offset, size, dir);

	dma_addr = __alloc_iova(mapping, len);
	if (dma_addr == DMA_ERROR_CODE)
		return dma_addr;

	ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0);
	if (ret < 0)
		goto fail;

	return dma_addr + offset;
fail:
	__free_iova(mapping, dma_addr, len);
	return DMA_ERROR_CODE;
}

/**
 * arm_iommu_unmap_page
 * @dev: valid struct device pointer
 * @handle: DMA address of buffer
 * @size: size of buffer (same as passed to dma_map_page)
 * @dir: DMA transfer direction (same as passed to dma_map_page)
 *
 * IOMMU aware version of arm_dma_unmap_page()
 */
static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
		size_t size, enum dma_data_direction dir,
		struct dma_attrs *attrs)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova = handle & PAGE_MASK;
	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
	int offset = handle & ~PAGE_MASK;
	int len = PAGE_ALIGN(size + offset);

	if (!iova)
		return;

	if (!arch_is_coherent())
		__dma_page_dev_to_cpu(page, offset, size, dir);

	iommu_unmap(mapping->domain, iova, len);
	__free_iova(mapping, iova, len);
}

static void arm_iommu_sync_single_for_cpu(struct device *dev,
		dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova = handle & PAGE_MASK;
	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
	unsigned int offset = handle & ~PAGE_MASK;

	if (!iova)
		return;

	if (!arch_is_coherent())
		__dma_page_dev_to_cpu(page, offset, size, dir);
}

static void arm_iommu_sync_single_for_device(struct device *dev,
		dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova = handle & PAGE_MASK;
	struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
	unsigned int offset = handle & ~PAGE_MASK;

	if (!iova)
		return;

	__dma_page_cpu_to_dev(page, offset, size, dir);
}

struct dma_map_ops iommu_ops = {
	.alloc		= arm_iommu_alloc_attrs,
	.free		= arm_iommu_free_attrs,
	.mmap		= arm_iommu_mmap_attrs,

	.map_page		= arm_iommu_map_page,
	.unmap_page		= arm_iommu_unmap_page,
	.sync_single_for_cpu	= arm_iommu_sync_single_for_cpu,
	.sync_single_for_device	= arm_iommu_sync_single_for_device,

	.map_sg			= arm_iommu_map_sg,
	.unmap_sg		= arm_iommu_unmap_sg,
	.sync_sg_for_cpu	= arm_iommu_sync_sg_for_cpu,
	.sync_sg_for_device	= arm_iommu_sync_sg_for_device,
};

/**
 * arm_iommu_create_mapping
 * @bus: pointer to the bus holding the client device (for IOMMU calls)
 * @base: start address of the valid IO address space
 * @size: size of the valid IO address space
 * @order: accuracy of the IO addresses allocations
 *
 * Creates a mapping structure which holds information about used/unused
 * IO address ranges, which is required to perform memory allocation and
 * mapping with IOMMU aware functions.
 *
 * The client device need to be attached to the mapping with
 * arm_iommu_attach_device function.
 */
struct dma_iommu_mapping *
arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size,
			 int order)
{
	unsigned int count = size >> (PAGE_SHIFT + order);
	unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long);
	struct dma_iommu_mapping *mapping;
	int err = -ENOMEM;

	if (!count)
		return ERR_PTR(-EINVAL);

	mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
	if (!mapping)
		goto err;

	mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
	if (!mapping->bitmap)
		goto err2;

	mapping->base = base;
	mapping->bits = BITS_PER_BYTE * bitmap_size;
	mapping->order = order;
	spin_lock_init(&mapping->lock);

	mapping->domain = iommu_domain_alloc(bus);
	if (!mapping->domain)
		goto err3;

	kref_init(&mapping->kref);
	return mapping;
err3:
	kfree(mapping->bitmap);
err2:
	kfree(mapping);
err:
	return ERR_PTR(err);
}

static void release_iommu_mapping(struct kref *kref)
{
	struct dma_iommu_mapping *mapping =
		container_of(kref, struct dma_iommu_mapping, kref);

	iommu_domain_free(mapping->domain);
	kfree(mapping->bitmap);
	kfree(mapping);
}

void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
{
	if (mapping)
		kref_put(&mapping->kref, release_iommu_mapping);
}

/**
 * arm_iommu_attach_device
 * @dev: valid struct device pointer
 * @mapping: io address space mapping structure (returned from
 *	arm_iommu_create_mapping)
 *
 * Attaches specified io address space mapping to the provided device,
 * this replaces the dma operations (dma_map_ops pointer) with the
 * IOMMU aware version. More than one client might be attached to
 * the same io address space mapping.
 */
int arm_iommu_attach_device(struct device *dev,
			    struct dma_iommu_mapping *mapping)
{
	int err;

	err = iommu_attach_device(mapping->domain, dev);
	if (err)
		return err;

	kref_get(&mapping->kref);
	dev->archdata.mapping = mapping;
	set_dma_ops(dev, &iommu_ops);

	pr_info("Attached IOMMU controller to %s device.\n", dev_name(dev));
	return 0;
}

#endif