dma-mapping.c 15.6 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2
 *  linux/arch/arm/mm/dma-mapping.c
L
Linus Torvalds 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
 *
 *  Copyright (C) 2000-2004 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 *  DMA uncached mapping support.
 */
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>

21
#include <asm/memory.h>
22
#include <asm/highmem.h>
L
Linus Torvalds 已提交
23 24
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
25 26 27 28 29 30
#include <asm/sizes.h>

/* Sanity check size */
#if (CONSISTENT_DMA_SIZE % SZ_2M)
#error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"
#endif
L
Linus Torvalds 已提交
31 32

#define CONSISTENT_END	(0xffe00000)
33 34
#define CONSISTENT_BASE	(CONSISTENT_END - CONSISTENT_DMA_SIZE)

L
Linus Torvalds 已提交
35
#define CONSISTENT_OFFSET(x)	(((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
36 37 38
#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT)
#define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT)

39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
static u64 get_coherent_dma_mask(struct device *dev)
{
	u64 mask = ISA_DMA_THRESHOLD;

	if (dev) {
		mask = dev->coherent_dma_mask;

		/*
		 * Sanity check the DMA mask - it must be non-zero, and
		 * must be able to be satisfied by a DMA allocation.
		 */
		if (mask == 0) {
			dev_warn(dev, "coherent DMA mask is unset\n");
			return 0;
		}

		if ((~mask) & ISA_DMA_THRESHOLD) {
			dev_warn(dev, "coherent DMA mask %#llx is smaller "
				 "than system GFP_DMA mask %#llx\n",
				 mask, (unsigned long long)ISA_DMA_THRESHOLD);
			return 0;
		}
	}
L
Linus Torvalds 已提交
62

63 64 65
	return mask;
}

66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
/*
 * Allocate a DMA buffer for 'dev' of size 'size' using the
 * specified gfp mask.  Note that 'size' must be page aligned.
 */
static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
{
	unsigned long order = get_order(size);
	struct page *page, *p, *e;
	void *ptr;
	u64 mask = get_coherent_dma_mask(dev);

#ifdef CONFIG_DMA_API_DEBUG
	u64 limit = (mask + 1) & ~mask;
	if (limit && size >= limit) {
		dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
			size, mask);
		return NULL;
	}
#endif

	if (!mask)
		return NULL;

	if (mask < 0xffffffffULL)
		gfp |= GFP_DMA;

	page = alloc_pages(gfp, order);
	if (!page)
		return NULL;

	/*
	 * Now split the huge page and free the excess pages
	 */
	split_page(page, order);
	for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
		__free_page(p);

	/*
	 * Ensure that the allocated pages are zeroed, and that any data
	 * lurking in the kernel direct-mapped region is invalidated.
	 */
	ptr = page_address(page);
	memset(ptr, 0, size);
	dmac_flush_range(ptr, ptr + size);
	outer_flush_range(__pa(ptr), __pa(ptr) + size);

	return page;
}

/*
 * Free a DMA buffer.  'size' must be page aligned.
 */
static void __dma_free_buffer(struct page *page, size_t size)
{
	struct page *e = page + (size >> PAGE_SHIFT);

	while (page < e) {
		__free_page(page);
		page++;
	}
}

128
#ifdef CONFIG_MMU
L
Linus Torvalds 已提交
129
/*
130
 * These are the page tables (2MB each) covering uncached, DMA consistent allocations
L
Linus Torvalds 已提交
131
 */
132
static pte_t *consistent_pte[NUM_CONSISTENT_PTES];
L
Linus Torvalds 已提交
133

134
#include "vmregion.h"
L
Linus Torvalds 已提交
135

136 137
static struct arm_vmregion_head consistent_head = {
	.vm_lock	= __SPIN_LOCK_UNLOCKED(&consistent_head.vm_lock),
L
Linus Torvalds 已提交
138 139 140 141 142 143 144 145 146 147
	.vm_list	= LIST_HEAD_INIT(consistent_head.vm_list),
	.vm_start	= CONSISTENT_BASE,
	.vm_end		= CONSISTENT_END,
};

#ifdef CONFIG_HUGETLB_PAGE
#error ARM Coherent DMA allocator does not (yet) support huge TLB
#endif

static void *
A
Al Viro 已提交
148
__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
L
Linus Torvalds 已提交
149 150 151
	    pgprot_t prot)
{
	struct page *page;
152
	struct arm_vmregion *c;
L
Linus Torvalds 已提交
153

154
	if (!consistent_pte[0]) {
L
Linus Torvalds 已提交
155 156 157 158 159 160 161
		printk(KERN_ERR "%s: not initialised\n", __func__);
		dump_stack();
		return NULL;
	}

	size = PAGE_ALIGN(size);

162
	page = __dma_alloc_buffer(dev, size, gfp);
L
Linus Torvalds 已提交
163 164 165 166 167 168
	if (!page)
		goto no_page;

	/*
	 * Allocate a virtual address in the consistent mapping region.
	 */
169
	c = arm_vmregion_alloc(&consistent_head, size,
L
Linus Torvalds 已提交
170 171
			    gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
	if (c) {
172 173 174
		pte_t *pte;
		int idx = CONSISTENT_PTE_INDEX(c->vm_start);
		u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
L
Linus Torvalds 已提交
175

176
		pte = consistent_pte[idx] + off;
L
Linus Torvalds 已提交
177 178 179 180 181 182 183 184 185 186 187 188 189 190
		c->vm_pages = page;

		/*
		 * Set the "dma handle"
		 */
		*handle = page_to_dma(dev, page);

		do {
			BUG_ON(!pte_none(*pte));

			/*
			 * x86 does not mark the pages reserved...
			 */
			SetPageReserved(page);
R
Russell King 已提交
191
			set_pte_ext(pte, mk_pte(page, prot), 0);
L
Linus Torvalds 已提交
192 193
			page++;
			pte++;
194 195 196 197 198
			off++;
			if (off >= PTRS_PER_PTE) {
				off = 0;
				pte = consistent_pte[++idx];
			}
L
Linus Torvalds 已提交
199 200 201 202 203 204
		} while (size -= PAGE_SIZE);

		return (void *)c->vm_start;
	}

	if (page)
205
		__dma_free_buffer(page, size);
L
Linus Torvalds 已提交
206 207 208 209
 no_page:
	*handle = ~0;
	return NULL;
}
210 211 212 213 214
#else	/* !CONFIG_MMU */
static void *
__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
	    pgprot_t prot)
{
215
	struct page *page;
216

217 218
	*handle = ~0;
	size = PAGE_ALIGN(size);
219

220 221 222
	page = __dma_alloc_buffer(dev, size, gfp);
	if (!page)
		return NULL;
223

224 225
	*handle = page_to_dma(dev, page);
	return page_address(page);
226 227
}
#endif	/* CONFIG_MMU */
L
Linus Torvalds 已提交
228 229 230 231 232 233

/*
 * Allocate DMA-coherent memory space and return both the kernel remapped
 * virtual and bus address for that space.
 */
void *
A
Al Viro 已提交
234
dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
L
Linus Torvalds 已提交
235
{
236 237 238 239 240
	void *memory;

	if (dma_alloc_from_coherent(dev, size, handle, &memory))
		return memory;

241
	if (arch_is_coherent()) {
242
		struct page *page;
243

244 245 246
		page = __dma_alloc_buffer(dev, PAGE_ALIGN(size), gfp);
		if (!page) {
			*handle = ~0;
247
			return NULL;
248
		}
249

250 251
		*handle = page_to_dma(dev, page);
		return page_address(page);
252 253
	}

L
Linus Torvalds 已提交
254 255 256 257 258 259 260 261 262 263
	return __dma_alloc(dev, size, handle, gfp,
			   pgprot_noncached(pgprot_kernel));
}
EXPORT_SYMBOL(dma_alloc_coherent);

/*
 * Allocate a writecombining region, in much the same way as
 * dma_alloc_coherent above.
 */
void *
A
Al Viro 已提交
264
dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
L
Linus Torvalds 已提交
265 266 267 268 269 270 271 272 273
{
	return __dma_alloc(dev, size, handle, gfp,
			   pgprot_writecombine(pgprot_kernel));
}
EXPORT_SYMBOL(dma_alloc_writecombine);

static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
		    void *cpu_addr, dma_addr_t dma_addr, size_t size)
{
274 275
	int ret = -ENXIO;
#ifdef CONFIG_MMU
276 277
	unsigned long user_size, kern_size;
	struct arm_vmregion *c;
L
Linus Torvalds 已提交
278 279 280

	user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;

281
	c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
L
Linus Torvalds 已提交
282 283 284 285 286 287 288 289 290 291 292 293 294
	if (c) {
		unsigned long off = vma->vm_pgoff;

		kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;

		if (off < kern_size &&
		    user_size <= (kern_size - off)) {
			ret = remap_pfn_range(vma, vma->vm_start,
					      page_to_pfn(c->vm_pages) + off,
					      user_size << PAGE_SHIFT,
					      vma->vm_page_prot);
		}
	}
295
#endif	/* CONFIG_MMU */
L
Linus Torvalds 已提交
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317

	return ret;
}

int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
		      void *cpu_addr, dma_addr_t dma_addr, size_t size)
{
	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
}
EXPORT_SYMBOL(dma_mmap_coherent);

int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
			  void *cpu_addr, dma_addr_t dma_addr, size_t size)
{
	vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
	return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
}
EXPORT_SYMBOL(dma_mmap_writecombine);

/*
 * free a page as defined by the above mapping.
318
 * Must not be called with IRQs disabled.
L
Linus Torvalds 已提交
319
 */
320
#ifdef CONFIG_MMU
L
Linus Torvalds 已提交
321 322
void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
{
323 324
	struct arm_vmregion *c;
	unsigned long addr;
L
Linus Torvalds 已提交
325
	pte_t *ptep;
326 327
	int idx;
	u32 off;
L
Linus Torvalds 已提交
328

329 330
	WARN_ON(irqs_disabled());

331 332 333
	if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
		return;

334 335
	size = PAGE_ALIGN(size);

336
	if (arch_is_coherent()) {
337
		__dma_free_buffer(dma_to_page(dev, handle), size);
338 339 340
		return;
	}

341
	c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr);
L
Linus Torvalds 已提交
342 343 344 345 346 347 348 349 350 351
	if (!c)
		goto no_area;

	if ((c->vm_end - c->vm_start) != size) {
		printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
		       __func__, c->vm_end - c->vm_start, size);
		dump_stack();
		size = c->vm_end - c->vm_start;
	}

352 353 354
	idx = CONSISTENT_PTE_INDEX(c->vm_start);
	off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
	ptep = consistent_pte[idx] + off;
L
Linus Torvalds 已提交
355 356 357 358 359 360 361
	addr = c->vm_start;
	do {
		pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
		unsigned long pfn;

		ptep++;
		addr += PAGE_SIZE;
362 363 364 365 366
		off++;
		if (off >= PTRS_PER_PTE) {
			off = 0;
			ptep = consistent_pte[++idx];
		}
L
Linus Torvalds 已提交
367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386

		if (!pte_none(pte) && pte_present(pte)) {
			pfn = pte_pfn(pte);

			if (pfn_valid(pfn)) {
				struct page *page = pfn_to_page(pfn);

				/*
				 * x86 does not mark the pages reserved...
				 */
				ClearPageReserved(page);
				continue;
			}
		}
		printk(KERN_CRIT "%s: bad page in kernel page table\n",
		       __func__);
	} while (size -= PAGE_SIZE);

	flush_tlb_kernel_range(c->vm_start, c->vm_end);

387
	arm_vmregion_free(&consistent_head, c);
388 389

	__dma_free_buffer(dma_to_page(dev, handle), size);
L
Linus Torvalds 已提交
390 391 392 393 394 395 396
	return;

 no_area:
	printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n",
	       __func__, cpu_addr);
	dump_stack();
}
397 398 399 400 401
#else	/* !CONFIG_MMU */
void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
{
	if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
		return;
402
	__dma_free_buffer(dma_to_page(dev, handle), PAGE_ALIGN(size));
403 404
}
#endif	/* CONFIG_MMU */
L
Linus Torvalds 已提交
405 406 407 408 409 410 411
EXPORT_SYMBOL(dma_free_coherent);

/*
 * Initialise the consistent memory allocation.
 */
static int __init consistent_init(void)
{
412 413
	int ret = 0;
#ifdef CONFIG_MMU
L
Linus Torvalds 已提交
414 415 416
	pgd_t *pgd;
	pmd_t *pmd;
	pte_t *pte;
417
	int i = 0;
418
	u32 base = CONSISTENT_BASE;
L
Linus Torvalds 已提交
419 420

	do {
421 422
		pgd = pgd_offset(&init_mm, base);
		pmd = pmd_alloc(&init_mm, pgd, base);
L
Linus Torvalds 已提交
423 424 425 426 427 428 429
		if (!pmd) {
			printk(KERN_ERR "%s: no pmd tables\n", __func__);
			ret = -ENOMEM;
			break;
		}
		WARN_ON(!pmd_none(*pmd));

430
		pte = pte_alloc_kernel(pmd, base);
L
Linus Torvalds 已提交
431 432 433 434 435 436
		if (!pte) {
			printk(KERN_ERR "%s: no pte tables\n", __func__);
			ret = -ENOMEM;
			break;
		}

437 438 439
		consistent_pte[i++] = pte;
		base += (1 << PGDIR_SHIFT);
	} while (base < CONSISTENT_END);
440
#endif	/* !CONFIG_MMU */
L
Linus Torvalds 已提交
441 442 443 444 445 446 447 448

	return ret;
}

core_initcall(consistent_init);

/*
 * Make an area consistent for devices.
449 450 451
 * Note: Drivers should NOT use this function directly, as it will break
 * platforms with CONFIG_DMABOUNCE.
 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
L
Linus Torvalds 已提交
452
 */
453
void dma_cache_maint(const void *start, size_t size, int direction)
L
Linus Torvalds 已提交
454
{
455 456
	void (*inner_op)(const void *, const void *);
	void (*outer_op)(unsigned long, unsigned long);
L
Linus Torvalds 已提交
457

458
	BUG_ON(!virt_addr_valid(start) || !virt_addr_valid(start + size - 1));
459

L
Linus Torvalds 已提交
460 461
	switch (direction) {
	case DMA_FROM_DEVICE:		/* invalidate only */
462 463
		inner_op = dmac_inv_range;
		outer_op = outer_inv_range;
L
Linus Torvalds 已提交
464 465
		break;
	case DMA_TO_DEVICE:		/* writeback only */
466 467
		inner_op = dmac_clean_range;
		outer_op = outer_clean_range;
L
Linus Torvalds 已提交
468 469
		break;
	case DMA_BIDIRECTIONAL:		/* writeback and invalidate */
470 471
		inner_op = dmac_flush_range;
		outer_op = outer_flush_range;
L
Linus Torvalds 已提交
472 473 474 475
		break;
	default:
		BUG();
	}
476 477 478

	inner_op(start, start + size);
	outer_op(__pa(start), __pa(start) + size);
L
Linus Torvalds 已提交
479
}
480
EXPORT_SYMBOL(dma_cache_maint);
481

482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549
static void dma_cache_maint_contiguous(struct page *page, unsigned long offset,
				       size_t size, int direction)
{
	void *vaddr;
	unsigned long paddr;
	void (*inner_op)(const void *, const void *);
	void (*outer_op)(unsigned long, unsigned long);

	switch (direction) {
	case DMA_FROM_DEVICE:		/* invalidate only */
		inner_op = dmac_inv_range;
		outer_op = outer_inv_range;
		break;
	case DMA_TO_DEVICE:		/* writeback only */
		inner_op = dmac_clean_range;
		outer_op = outer_clean_range;
		break;
	case DMA_BIDIRECTIONAL:		/* writeback and invalidate */
		inner_op = dmac_flush_range;
		outer_op = outer_flush_range;
		break;
	default:
		BUG();
	}

	if (!PageHighMem(page)) {
		vaddr = page_address(page) + offset;
		inner_op(vaddr, vaddr + size);
	} else {
		vaddr = kmap_high_get(page);
		if (vaddr) {
			vaddr += offset;
			inner_op(vaddr, vaddr + size);
			kunmap_high(page);
		}
	}

	paddr = page_to_phys(page) + offset;
	outer_op(paddr, paddr + size);
}

void dma_cache_maint_page(struct page *page, unsigned long offset,
			  size_t size, int dir)
{
	/*
	 * A single sg entry may refer to multiple physically contiguous
	 * pages.  But we still need to process highmem pages individually.
	 * If highmem is not configured then the bulk of this loop gets
	 * optimized out.
	 */
	size_t left = size;
	do {
		size_t len = left;
		if (PageHighMem(page) && len + offset > PAGE_SIZE) {
			if (offset >= PAGE_SIZE) {
				page += offset / PAGE_SIZE;
				offset %= PAGE_SIZE;
			}
			len = PAGE_SIZE - offset;
		}
		dma_cache_maint_contiguous(page, offset, len, dir);
		offset = 0;
		page++;
		left -= len;
	} while (left);
}
EXPORT_SYMBOL(dma_cache_maint_page);

550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569
/**
 * dma_map_sg - map a set of SG buffers for streaming mode DMA
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map
 * @dir: DMA transfer direction
 *
 * Map a set of buffers described by scatterlist in streaming mode for DMA.
 * This is the scatter-gather version of the dma_map_single interface.
 * Here the scatter gather list elements are each tagged with the
 * appropriate dma address and length.  They are obtained via
 * sg_dma_{address,length}.
 *
 * Device ownership issues as mentioned for dma_map_single are the same
 * here.
 */
int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
		enum dma_data_direction dir)
{
	struct scatterlist *s;
570
	int i, j;
571 572

	for_each_sg(sg, s, nents, i) {
573 574 575 576
		s->dma_address = dma_map_page(dev, sg_page(s), s->offset,
						s->length, dir);
		if (dma_mapping_error(dev, s->dma_address))
			goto bad_mapping;
577 578
	}
	return nents;
579 580 581 582 583

 bad_mapping:
	for_each_sg(sg, s, i, j)
		dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
	return 0;
584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
}
EXPORT_SYMBOL(dma_map_sg);

/**
 * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to unmap (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 *
 * Unmap a set of streaming mode DMA translations.  Again, CPU access
 * rules concerning calls here are the same as for dma_unmap_single().
 */
void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
		enum dma_data_direction dir)
{
600 601 602 603 604
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i)
		dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
}
EXPORT_SYMBOL(dma_unmap_sg);

/**
 * dma_sync_sg_for_cpu
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
			int nents, enum dma_data_direction dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i) {
622 623
		dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0,
					sg_dma_len(s), dir);
624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
	}
}
EXPORT_SYMBOL(dma_sync_sg_for_cpu);

/**
 * dma_sync_sg_for_device
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @sg: list of buffers
 * @nents: number of buffers to map (returned from dma_map_sg)
 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
 */
void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
			int nents, enum dma_data_direction dir)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i) {
642 643 644 645
		if (!dmabounce_sync_for_device(dev, sg_dma_address(s), 0,
					sg_dma_len(s), dir))
			continue;

646
		if (!arch_is_coherent())
647 648
			dma_cache_maint_page(sg_page(s), s->offset,
					     s->length, dir);
649 650 651
	}
}
EXPORT_SYMBOL(dma_sync_sg_for_device);