dma-mapping.c 18.1 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2
 *  linux/arch/arm/mm/dma-mapping.c
L
Linus Torvalds 已提交
3 4 5 6 7 8 9 10 11 12 13
 *
 *  Copyright (C) 2000-2004 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 *  DMA uncached mapping support.
 */
#include <linux/module.h>
#include <linux/mm.h>
14
#include <linux/gfp.h>
L
Linus Torvalds 已提交
15 16 17 18 19
#include <linux/errno.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
20
#include <linux/highmem.h>
21
#include <linux/slab.h>
L
Linus Torvalds 已提交
22

23
#include <asm/memory.h>
24
#include <asm/highmem.h>
L
Linus Torvalds 已提交
25 26
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
27
#include <asm/sizes.h>
28
#include <asm/mach/arch.h>
29

30 31
#include "mm.h"

32 33
static u64 get_coherent_dma_mask(struct device *dev)
{
34
	u64 mask = (u64)arm_dma_limit;
35 36 37 38 39 40 41 42 43 44 45 46 47

	if (dev) {
		mask = dev->coherent_dma_mask;

		/*
		 * Sanity check the DMA mask - it must be non-zero, and
		 * must be able to be satisfied by a DMA allocation.
		 */
		if (mask == 0) {
			dev_warn(dev, "coherent DMA mask is unset\n");
			return 0;
		}

48
		if ((~mask) & (u64)arm_dma_limit) {
49 50
			dev_warn(dev, "coherent DMA mask %#llx is smaller "
				 "than system GFP_DMA mask %#llx\n",
51
				 mask, (u64)arm_dma_limit);
52 53 54
			return 0;
		}
	}
L
Linus Torvalds 已提交
55

56 57 58
	return mask;
}

59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
/*
 * Allocate a DMA buffer for 'dev' of size 'size' using the
 * specified gfp mask.  Note that 'size' must be page aligned.
 */
static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
{
	unsigned long order = get_order(size);
	struct page *page, *p, *e;
	void *ptr;
	u64 mask = get_coherent_dma_mask(dev);

#ifdef CONFIG_DMA_API_DEBUG
	u64 limit = (mask + 1) & ~mask;
	if (limit && size >= limit) {
		dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
			size, mask);
		return NULL;
	}
#endif

	if (!mask)
		return NULL;

	if (mask < 0xffffffffULL)
		gfp |= GFP_DMA;

	page = alloc_pages(gfp, order);
	if (!page)
		return NULL;

	/*
	 * Now split the huge page and free the excess pages
	 */
	split_page(page, order);
	for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
		__free_page(p);

	/*
	 * Ensure that the allocated pages are zeroed, and that any data
	 * lurking in the kernel direct-mapped region is invalidated.
	 */
	ptr = page_address(page);
	memset(ptr, 0, size);
	dmac_flush_range(ptr, ptr + size);
	outer_flush_range(__pa(ptr), __pa(ptr) + size);

	return page;
}

/*
 * Free a DMA buffer.  'size' must be page aligned.
 */
static void __dma_free_buffer(struct page *page, size_t size)
{
	struct page *e = page + (size >> PAGE_SHIFT);

	while (page < e) {
		__free_page(page);
		page++;
	}
}

121
#ifdef CONFIG_MMU
122

123
#define CONSISTENT_OFFSET(x)	(((unsigned long)(x) - consistent_base) >> PAGE_SHIFT)
L
Linus Torvalds 已提交
124
#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - consistent_base) >> PMD_SHIFT)
125

L
Linus Torvalds 已提交
126
/*
127
 * These are the page tables (2MB each) covering uncached, DMA consistent allocations
L
Linus Torvalds 已提交
128
 */
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
static pte_t **consistent_pte;

#define DEFAULT_CONSISTENT_DMA_SIZE SZ_2M

unsigned long consistent_base = CONSISTENT_END - DEFAULT_CONSISTENT_DMA_SIZE;

void __init init_consistent_dma_size(unsigned long size)
{
	unsigned long base = CONSISTENT_END - ALIGN(size, SZ_2M);

	BUG_ON(consistent_pte); /* Check we're called before DMA region init */
	BUG_ON(base < VMALLOC_END);

	/* Grow region to accommodate specified size  */
	if (base < consistent_base)
		consistent_base = base;
}
L
Linus Torvalds 已提交
146

147
#include "vmregion.h"
L
Linus Torvalds 已提交
148

149 150
static struct arm_vmregion_head consistent_head = {
	.vm_lock	= __SPIN_LOCK_UNLOCKED(&consistent_head.vm_lock),
L
Linus Torvalds 已提交
151 152 153 154 155 156 157 158
	.vm_list	= LIST_HEAD_INIT(consistent_head.vm_list),
	.vm_end		= CONSISTENT_END,
};

#ifdef CONFIG_HUGETLB_PAGE
#error ARM Coherent DMA allocator does not (yet) support huge TLB
#endif

159 160 161 162 163 164 165
/*
 * Initialise the consistent memory allocation.
 */
static int __init consistent_init(void)
{
	int ret = 0;
	pgd_t *pgd;
R
Russell King 已提交
166
	pud_t *pud;
167 168 169
	pmd_t *pmd;
	pte_t *pte;
	int i = 0;
170
	unsigned long base = consistent_base;
171
	unsigned long num_ptes = (CONSISTENT_END - base) >> PMD_SHIFT;
172 173 174 175 176 177 178 179 180

	consistent_pte = kmalloc(num_ptes * sizeof(pte_t), GFP_KERNEL);
	if (!consistent_pte) {
		pr_err("%s: no memory\n", __func__);
		return -ENOMEM;
	}

	pr_debug("DMA memory: 0x%08lx - 0x%08lx:\n", base, CONSISTENT_END);
	consistent_head.vm_start = base;
181 182 183

	do {
		pgd = pgd_offset(&init_mm, base);
R
Russell King 已提交
184 185 186 187 188 189 190 191 192

		pud = pud_alloc(&init_mm, pgd, base);
		if (!pud) {
			printk(KERN_ERR "%s: no pud tables\n", __func__);
			ret = -ENOMEM;
			break;
		}

		pmd = pmd_alloc(&init_mm, pud, base);
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
		if (!pmd) {
			printk(KERN_ERR "%s: no pmd tables\n", __func__);
			ret = -ENOMEM;
			break;
		}
		WARN_ON(!pmd_none(*pmd));

		pte = pte_alloc_kernel(pmd, base);
		if (!pte) {
			printk(KERN_ERR "%s: no pte tables\n", __func__);
			ret = -ENOMEM;
			break;
		}

		consistent_pte[i++] = pte;
208
		base += PMD_SIZE;
209 210 211 212 213 214 215
	} while (base < CONSISTENT_END);

	return ret;
}

core_initcall(consistent_init);

L
Linus Torvalds 已提交
216
static void *
217 218
__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
	const void *caller)
L
Linus Torvalds 已提交
219
{
220
	struct arm_vmregion *c;
221 222
	size_t align;
	int bit;
L
Linus Torvalds 已提交
223

224
	if (!consistent_pte) {
225 226 227 228 229
		printk(KERN_ERR "%s: not initialised\n", __func__);
		dump_stack();
		return NULL;
	}

230 231 232 233 234 235
	/*
	 * Align the virtual region allocation - maximum alignment is
	 * a section size, minimum is a page size.  This helps reduce
	 * fragmentation of the DMA space, and also prevents allocations
	 * smaller than a section from crossing a section boundary.
	 */
236
	bit = fls(size - 1);
237 238 239 240
	if (bit > SECTION_SHIFT)
		bit = SECTION_SHIFT;
	align = 1 << bit;

L
Linus Torvalds 已提交
241 242 243
	/*
	 * Allocate a virtual address in the consistent mapping region.
	 */
244
	c = arm_vmregion_alloc(&consistent_head, align, size,
245
			    gfp & ~(__GFP_DMA | __GFP_HIGHMEM), caller);
L
Linus Torvalds 已提交
246
	if (c) {
247 248 249
		pte_t *pte;
		int idx = CONSISTENT_PTE_INDEX(c->vm_start);
		u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
L
Linus Torvalds 已提交
250

251
		pte = consistent_pte[idx] + off;
L
Linus Torvalds 已提交
252 253 254 255 256
		c->vm_pages = page;

		do {
			BUG_ON(!pte_none(*pte));

R
Russell King 已提交
257
			set_pte_ext(pte, mk_pte(page, prot), 0);
L
Linus Torvalds 已提交
258 259
			page++;
			pte++;
260 261 262 263 264
			off++;
			if (off >= PTRS_PER_PTE) {
				off = 0;
				pte = consistent_pte[++idx];
			}
L
Linus Torvalds 已提交
265 266
		} while (size -= PAGE_SIZE);

267 268
		dsb();

L
Linus Torvalds 已提交
269 270 271 272
		return (void *)c->vm_start;
	}
	return NULL;
}
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311

static void __dma_free_remap(void *cpu_addr, size_t size)
{
	struct arm_vmregion *c;
	unsigned long addr;
	pte_t *ptep;
	int idx;
	u32 off;

	c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr);
	if (!c) {
		printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n",
		       __func__, cpu_addr);
		dump_stack();
		return;
	}

	if ((c->vm_end - c->vm_start) != size) {
		printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
		       __func__, c->vm_end - c->vm_start, size);
		dump_stack();
		size = c->vm_end - c->vm_start;
	}

	idx = CONSISTENT_PTE_INDEX(c->vm_start);
	off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
	ptep = consistent_pte[idx] + off;
	addr = c->vm_start;
	do {
		pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);

		ptep++;
		addr += PAGE_SIZE;
		off++;
		if (off >= PTRS_PER_PTE) {
			off = 0;
			ptep = consistent_pte[++idx];
		}

312 313 314
		if (pte_none(pte) || !pte_present(pte))
			printk(KERN_CRIT "%s: bad page in kernel page table\n",
			       __func__);
315 316 317 318 319 320 321
	} while (size -= PAGE_SIZE);

	flush_tlb_kernel_range(c->vm_start, c->vm_end);

	arm_vmregion_free(&consistent_head, c);
}

322
#else	/* !CONFIG_MMU */
323

324
#define __dma_alloc_remap(page, size, gfp, prot, c)	page_address(page)
325 326 327 328
#define __dma_free_remap(addr, size)			do { } while (0)

#endif	/* CONFIG_MMU */

329 330
static void *
__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
331
	    pgprot_t prot, const void *caller)
332
{
333
	struct page *page;
334
	void *addr;
335

336 337 338 339 340 341 342 343 344
	/*
	 * Following is a work-around (a.k.a. hack) to prevent pages
	 * with __GFP_COMP being passed to split_page() which cannot
	 * handle them.  The real problem is that this flag probably
	 * should be 0 on ARM as it is not supported on this
	 * platform; see CONFIG_HUGETLBFS.
	 */
	gfp &= ~(__GFP_COMP);

345 346
	*handle = ~0;
	size = PAGE_ALIGN(size);
347

348 349 350
	page = __dma_alloc_buffer(dev, size, gfp);
	if (!page)
		return NULL;
351

352
	if (!arch_is_coherent())
353
		addr = __dma_alloc_remap(page, size, gfp, prot, caller);
354 355
	else
		addr = page_address(page);
356

357
	if (addr)
358
		*handle = pfn_to_dma(dev, page_to_pfn(page));
359 360
	else
		__dma_free_buffer(page, size);
361

362 363
	return addr;
}
L
Linus Torvalds 已提交
364 365 366 367 368 369

/*
 * Allocate DMA-coherent memory space and return both the kernel remapped
 * virtual and bus address for that space.
 */
void *
A
Al Viro 已提交
370
dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
L
Linus Torvalds 已提交
371
{
372 373 374 375 376
	void *memory;

	if (dma_alloc_from_coherent(dev, size, handle, &memory))
		return memory;

L
Linus Torvalds 已提交
377
	return __dma_alloc(dev, size, handle, gfp,
378 379
			   pgprot_dmacoherent(pgprot_kernel),
			   __builtin_return_address(0));
L
Linus Torvalds 已提交
380 381 382 383 384 385 386 387
}
EXPORT_SYMBOL(dma_alloc_coherent);

/*
 * Allocate a writecombining region, in much the same way as
 * dma_alloc_coherent above.
 */
void *
A
Al Viro 已提交
388
dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
L
Linus Torvalds 已提交
389 390
{
	return __dma_alloc(dev, size, handle, gfp,
391 392
			   pgprot_writecombine(pgprot_kernel),
			   __builtin_return_address(0));
L
Linus Torvalds 已提交
393 394 395 396 397 398
}
EXPORT_SYMBOL(dma_alloc_writecombine);

static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
		    void *cpu_addr, dma_addr_t dma_addr, size_t size)
{
399 400
	int ret = -ENXIO;
#ifdef CONFIG_MMU
401 402
	unsigned long user_size, kern_size;
	struct arm_vmregion *c;
L
Linus Torvalds 已提交
403 404 405

	user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;

406
	c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
L
Linus Torvalds 已提交
407 408 409 410 411 412 413 414 415 416 417 418 419
	if (c) {
		unsigned long off = vma->vm_pgoff;

		kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;

		if (off < kern_size &&
		    user_size <= (kern_size - off)) {
			ret = remap_pfn_range(vma, vma->vm_start,
					      page_to_pfn(c->vm_pages) + off,
					      user_size << PAGE_SHIFT,
					      vma->vm_page_prot);
		}
	}
420
#endif	/* CONFIG_MMU */
L
Linus Torvalds 已提交
421 422 423 424 425 426 427

	return ret;
}

int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
		      void *cpu_addr, dma_addr_t dma_addr, size_t size)
{
428
	vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
L
Linus Torvalds 已提交
429 430 431 432 433 434 435 436 437 438 439 440 441 442
	return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
}
EXPORT_SYMBOL(dma_mmap_coherent);

int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
			  void *cpu_addr, dma_addr_t dma_addr, size_t size)
{
	vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
	return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
}
EXPORT_SYMBOL(dma_mmap_writecombine);

/*
 * free a page as defined by the above mapping.
443
 * Must not be called with IRQs disabled.
L
Linus Torvalds 已提交
444 445 446
 */
void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
{
447 448
	WARN_ON(irqs_disabled());

449 450 451
	if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
		return;

452 453
	size = PAGE_ALIGN(size);

454 455
	if (!arch_is_coherent())
		__dma_free_remap(cpu_addr, size);
456

457
	__dma_free_buffer(pfn_to_page(dma_to_pfn(dev, handle)), size);
L
Linus Torvalds 已提交
458 459 460 461 462
}
EXPORT_SYMBOL(dma_free_coherent);

/*
 * Make an area consistent for devices.
463 464 465
 * Note: Drivers should NOT use this function directly, as it will break
 * platforms with CONFIG_DMABOUNCE.
 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
L
Linus Torvalds 已提交
466
 */
467 468 469
void ___dma_single_cpu_to_dev(const void *kaddr, size_t size,
	enum dma_data_direction dir)
{
470 471
	unsigned long paddr;

472 473 474
	BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));

	dmac_map_area(kaddr, size, dir);
475 476 477 478 479 480 481 482

	paddr = __pa(kaddr);
	if (dir == DMA_FROM_DEVICE) {
		outer_inv_range(paddr, paddr + size);
	} else {
		outer_clean_range(paddr, paddr + size);
	}
	/* FIXME: non-speculating: flush on bidirectional mappings? */
483 484 485 486 487 488
}
EXPORT_SYMBOL(___dma_single_cpu_to_dev);

void ___dma_single_dev_to_cpu(const void *kaddr, size_t size,
	enum dma_data_direction dir)
{
489 490
	BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));

491 492 493 494 495 496 497
	/* FIXME: non-speculating: not required */
	/* don't bother invalidating if DMA to device */
	if (dir != DMA_TO_DEVICE) {
		unsigned long paddr = __pa(kaddr);
		outer_inv_range(paddr, paddr + size);
	}

498
	dmac_unmap_area(kaddr, size, dir);
499 500
}
EXPORT_SYMBOL(___dma_single_dev_to_cpu);
501

502
static void dma_cache_maint_page(struct page *page, unsigned long offset,
503 504
	size_t size, enum dma_data_direction dir,
	void (*op)(const void *, size_t, int))
505 506 507 508 509 510 511 512 513 514
{
	/*
	 * A single sg entry may refer to multiple physically contiguous
	 * pages.  But we still need to process highmem pages individually.
	 * If highmem is not configured then the bulk of this loop gets
	 * optimized out.
	 */
	size_t left = size;
	do {
		size_t len = left;
515 516 517 518 519 520 521 522 523 524 525 526 527
		void *vaddr;

		if (PageHighMem(page)) {
			if (len + offset > PAGE_SIZE) {
				if (offset >= PAGE_SIZE) {
					page += offset / PAGE_SIZE;
					offset %= PAGE_SIZE;
				}
				len = PAGE_SIZE - offset;
			}
			vaddr = kmap_high_get(page);
			if (vaddr) {
				vaddr += offset;
528
				op(vaddr, len, dir);
529
				kunmap_high(page);
530
			} else if (cache_is_vipt()) {
531 532
				/* unmapped pages might still be cached */
				vaddr = kmap_atomic(page);
533
				op(vaddr + offset, len, dir);
534
				kunmap_atomic(vaddr);
535
			}
536 537
		} else {
			vaddr = page_address(page) + offset;
538
			op(vaddr, len, dir);
539 540 541 542 543 544
		}
		offset = 0;
		page++;
		left -= len;
	} while (left);
}
545 546 547 548

void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
	size_t size, enum dma_data_direction dir)
{
549 550
	unsigned long paddr;

551
	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
552 553

	paddr = page_to_phys(page) + off;
554 555 556 557 558 559
	if (dir == DMA_FROM_DEVICE) {
		outer_inv_range(paddr, paddr + size);
	} else {
		outer_clean_range(paddr, paddr + size);
	}
	/* FIXME: non-speculating: flush on bidirectional mappings? */
560 561 562 563 564 565
}
EXPORT_SYMBOL(___dma_page_cpu_to_dev);

void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
	size_t size, enum dma_data_direction dir)
{
566 567 568 569 570 571 572
	unsigned long paddr = page_to_phys(page) + off;

	/* FIXME: non-speculating: not required */
	/* don't bother invalidating if DMA to device */
	if (dir != DMA_TO_DEVICE)
		outer_inv_range(paddr, paddr + size);

573
	dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
574 575 576 577 578 579

	/*
	 * Mark the D-cache clean for this page to avoid extra flushing.
	 */
	if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
		set_bit(PG_dcache_clean, &page->flags);
580 581
}
EXPORT_SYMBOL(___dma_page_dev_to_cpu);
582

583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
/**
 * dma_map_sg - map a set of SG buffers for streaming mode DMA
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map
 * @dir: DMA transfer direction
 *
 * Map a set of buffers described by scatterlist in streaming mode for DMA.
 * This is the scatter-gather version of the dma_map_single interface.
 * Here the scatter gather list elements are each tagged with the
 * appropriate dma address and length.  They are obtained via
 * sg_dma_{address,length}.
 *
 * Device ownership issues as mentioned for dma_map_single are the same
 * here.
 */
int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
		enum dma_data_direction dir)
{
	struct scatterlist *s;
603
	int i, j;
604

605 606
	BUG_ON(!valid_dma_direction(dir));

607
	for_each_sg(sg, s, nents, i) {
608
		s->dma_address = __dma_map_page(dev, sg_page(s), s->offset,
609 610 611
						s->length, dir);
		if (dma_mapping_error(dev, s->dma_address))
			goto bad_mapping;
612
	}
613
	debug_dma_map_sg(dev, sg, nents, nents, dir);
614
	return nents;
615 616 617

 bad_mapping:
	for_each_sg(sg, s, i, j)
618
		__dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
619
	return 0;
620 621 622 623 624 625 626
}
EXPORT_SYMBOL(dma_map_sg);

/**
 * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
627
 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
628 629 630 631 632 633 634 635
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 *
 * Unmap a set of streaming mode DMA translations.  Again, CPU access
 * rules concerning calls here are the same as for dma_unmap_single().
 */
void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
		enum dma_data_direction dir)
{
636 637 638
	struct scatterlist *s;
	int i;

639 640
	debug_dma_unmap_sg(dev, sg, nents, dir);

641
	for_each_sg(sg, s, nents, i)
642
		__dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659
}
EXPORT_SYMBOL(dma_unmap_sg);

/**
 * dma_sync_sg_for_cpu
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
			int nents, enum dma_data_direction dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i) {
660 661 662 663 664 665
		if (!dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0,
					    sg_dma_len(s), dir))
			continue;

		__dma_page_dev_to_cpu(sg_page(s), s->offset,
				      s->length, dir);
666
	}
667 668

	debug_dma_sync_sg_for_cpu(dev, sg, nents, dir);
669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685
}
EXPORT_SYMBOL(dma_sync_sg_for_cpu);

/**
 * dma_sync_sg_for_device
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
			int nents, enum dma_data_direction dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i) {
686 687 688 689
		if (!dmabounce_sync_for_device(dev, sg_dma_address(s), 0,
					sg_dma_len(s), dir))
			continue;

690 691
		__dma_page_cpu_to_dev(sg_page(s), s->offset,
				      s->length, dir);
692
	}
693 694

	debug_dma_sync_sg_for_device(dev, sg, nents, dir);
695 696
}
EXPORT_SYMBOL(dma_sync_sg_for_device);
697

698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
/*
 * Return whether the given device DMA address mask can be supported
 * properly.  For example, if your device can only drive the low 24-bits
 * during bus mastering, then you would pass 0x00ffffff as the mask
 * to this function.
 */
int dma_supported(struct device *dev, u64 mask)
{
	if (mask < (u64)arm_dma_limit)
		return 0;
	return 1;
}
EXPORT_SYMBOL(dma_supported);

int dma_set_mask(struct device *dev, u64 dma_mask)
{
	if (!dev->dma_mask || !dma_supported(dev, dma_mask))
		return -EIO;

#ifndef CONFIG_DMABOUNCE
	*dev->dma_mask = dma_mask;
#endif

	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

725 726 727 728
#define PREALLOC_DMA_DEBUG_ENTRIES	4096

static int __init dma_debug_do_init(void)
{
729 730 731
#ifdef CONFIG_MMU
	arm_vmregion_create_proc("dma-mappings", &consistent_head);
#endif
732 733 734 735
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
	return 0;
}
fs_initcall(dma_debug_do_init);