dma-noncoherent.c 10.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
/*
 *  PowerPC version derived from arch/arm/mm/consistent.c
 *    Copyright (C) 2001 Dan Malek (dmalek@jlc.net)
 *
 *  Copyright (C) 2000 Russell King
 *
 * Consistent memory allocators.  Used for DMA devices that want to
 * share uncached memory with the processor core.  The function return
 * is the virtual address and 'dma_handle' is the physical address.
 * Mostly stolen from the ARM port, with some changes for PowerPC.
 *						-- Dan
 *
 * Reorganized to get rid of the arch-specific consistent_* functions
 * and provide non-coherent implementations for the DMA API. -Matt
 *
 * Added in_interrupt() safe dma_alloc_coherent()/dma_free_coherent()
 * implementation. This is pulled straight from ARM and barely
 * modified. -Matt
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/sched.h>
26
#include <linux/slab.h>
L
Linus Torvalds 已提交
27 28 29 30 31 32
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/highmem.h>
#include <linux/dma-mapping.h>
33
#include <linux/export.h>
L
Linus Torvalds 已提交
34 35 36

#include <asm/tlbflush.h>

37 38
#include "mmu_decl.h"

39 40 41 42 43 44
/*
 * This address range defaults to a value that is safe for all
 * platforms which currently set CONFIG_NOT_COHERENT_CACHE. It
 * can be further configured for specific applications under
 * the "Advanced Setup" menu. -Matt
 */
45 46
#define CONSISTENT_BASE		(IOREMAP_TOP)
#define CONSISTENT_END 		(CONSISTENT_BASE + CONFIG_CONSISTENT_SIZE)
47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
#define CONSISTENT_OFFSET(x)	(((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)

/*
 * This is the page table (2MB) covering uncached, DMA consistent allocations
 */
static DEFINE_SPINLOCK(consistent_lock);

/*
 * VM region handling support.
 *
 * This should become something generic, handling VM region allocations for
 * vmalloc and similar (ioremap, module space, etc).
 *
 * I envisage vmalloc()'s supporting vm_struct becoming:
 *
 *  struct vm_struct {
 *    struct vm_region	region;
 *    unsigned long	flags;
 *    struct page	**pages;
 *    unsigned int	nr_pages;
 *    unsigned long	phys_addr;
 *  };
 *
 * get_vm_area() would then call vm_region_alloc with an appropriate
 * struct vm_region head (eg):
 *
 *  struct vm_region vmalloc_head = {
 *	.vm_list	= LIST_HEAD_INIT(vmalloc_head.vm_list),
 *	.vm_start	= VMALLOC_START,
 *	.vm_end		= VMALLOC_END,
 *  };
 *
 * However, vmalloc_head.vm_start is variable (typically, it is dependent on
 * the amount of RAM found at boot time.)  I would imagine that get_vm_area()
 * would have to initialise this each time prior to calling vm_region_alloc().
 */
struct ppc_vm_region {
	struct list_head	vm_list;
	unsigned long		vm_start;
	unsigned long		vm_end;
};

static struct ppc_vm_region consistent_head = {
	.vm_list	= LIST_HEAD_INIT(consistent_head.vm_list),
	.vm_start	= CONSISTENT_BASE,
	.vm_end		= CONSISTENT_END,
};

static struct ppc_vm_region *
ppc_vm_region_alloc(struct ppc_vm_region *head, size_t size, gfp_t gfp)
{
	unsigned long addr = head->vm_start, end = head->vm_end - size;
	unsigned long flags;
	struct ppc_vm_region *c, *new;

	new = kmalloc(sizeof(struct ppc_vm_region), gfp);
	if (!new)
		goto out;

	spin_lock_irqsave(&consistent_lock, flags);

	list_for_each_entry(c, &head->vm_list, vm_list) {
		if ((addr + size) < addr)
			goto nospc;
		if ((addr + size) <= c->vm_start)
			goto found;
		addr = c->vm_end;
		if (addr > end)
			goto nospc;
	}

 found:
	/*
	 * Insert this entry _before_ the one we found.
	 */
	list_add_tail(&new->vm_list, &c->vm_list);
	new->vm_start = addr;
	new->vm_end = addr + size;

	spin_unlock_irqrestore(&consistent_lock, flags);
	return new;

 nospc:
	spin_unlock_irqrestore(&consistent_lock, flags);
	kfree(new);
 out:
	return NULL;
}

static struct ppc_vm_region *ppc_vm_region_find(struct ppc_vm_region *head, unsigned long addr)
{
	struct ppc_vm_region *c;

	list_for_each_entry(c, &head->vm_list, vm_list) {
		if (c->vm_start == addr)
			goto out;
	}
	c = NULL;
 out:
	return c;
}

L
Linus Torvalds 已提交
149 150 151 152 153
/*
 * Allocate DMA-coherent memory space and return both the kernel remapped
 * virtual and bus address for that space.
 */
void *
154
__dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
L
Linus Torvalds 已提交
155 156
{
	struct page *page;
157
	struct ppc_vm_region *c;
L
Linus Torvalds 已提交
158
	unsigned long order;
159
	u64 mask = ISA_DMA_THRESHOLD, limit;
L
Linus Torvalds 已提交
160

161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
	if (dev) {
		mask = dev->coherent_dma_mask;

		/*
		 * Sanity check the DMA mask - it must be non-zero, and
		 * must be able to be satisfied by a DMA allocation.
		 */
		if (mask == 0) {
			dev_warn(dev, "coherent DMA mask is unset\n");
			goto no_page;
		}

		if ((~mask) & ISA_DMA_THRESHOLD) {
			dev_warn(dev, "coherent DMA mask %#llx is smaller "
				 "than system GFP_DMA mask %#llx\n",
				 mask, (unsigned long long)ISA_DMA_THRESHOLD);
			goto no_page;
		}
179 180
	}

181

L
Linus Torvalds 已提交
182 183
	size = PAGE_ALIGN(size);
	limit = (mask + 1) & ~mask;
184 185
	if ((limit && size >= limit) ||
	    size >= (CONSISTENT_END - CONSISTENT_BASE)) {
186 187
		printk(KERN_WARNING "coherent allocation too big (requested %#x mask %#Lx)\n",
		       size, mask);
L
Linus Torvalds 已提交
188 189 190 191 192
		return NULL;
	}

	order = get_order(size);

193
	/* Might be useful if we ever have a real legacy DMA zone... */
L
Linus Torvalds 已提交
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
	if (mask != 0xffffffff)
		gfp |= GFP_DMA;

	page = alloc_pages(gfp, order);
	if (!page)
		goto no_page;

	/*
	 * Invalidate any data that might be lurking in the
	 * kernel direct-mapped region for device DMA.
	 */
	{
		unsigned long kaddr = (unsigned long)page_address(page);
		memset(page_address(page), 0, size);
		flush_dcache_range(kaddr, kaddr + size);
	}

	/*
212
	 * Allocate a virtual address in the consistent mapping region.
L
Linus Torvalds 已提交
213
	 */
214 215 216 217 218
	c = ppc_vm_region_alloc(&consistent_head, size,
			    gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
	if (c) {
		unsigned long vaddr = c->vm_start;
		struct page *end = page + (1 << order);
L
Linus Torvalds 已提交
219

220 221 222 223 224 225
		split_page(page, order);

		/*
		 * Set the "dma handle"
		 */
		*handle = page_to_phys(page);
L
Linus Torvalds 已提交
226

227 228
		do {
			SetPageReserved(page);
229 230
			map_page(vaddr, page_to_phys(page),
				 pgprot_noncached(PAGE_KERNEL));
231 232 233
			page++;
			vaddr += PAGE_SIZE;
		} while (size -= PAGE_SIZE);
L
Linus Torvalds 已提交
234

235 236 237 238 239 240 241 242 243
		/*
		 * Free the otherwise unused pages.
		 */
		while (page < end) {
			__free_page(page);
			page++;
		}

		return (void *)c->vm_start;
L
Linus Torvalds 已提交
244 245 246 247
	}

	if (page)
		__free_pages(page, order);
248
 no_page:
L
Linus Torvalds 已提交
249 250 251 252 253 254 255 256 257
	return NULL;
}
EXPORT_SYMBOL(__dma_alloc_coherent);

/*
 * free a page as defined by the above mapping.
 */
void __dma_free_coherent(size_t size, void *vaddr)
{
258 259
	struct ppc_vm_region *c;
	unsigned long flags, addr;
260
	
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
	size = PAGE_ALIGN(size);

	spin_lock_irqsave(&consistent_lock, flags);

	c = ppc_vm_region_find(&consistent_head, (unsigned long)vaddr);
	if (!c)
		goto no_area;

	if ((c->vm_end - c->vm_start) != size) {
		printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
		       __func__, c->vm_end - c->vm_start, size);
		dump_stack();
		size = c->vm_end - c->vm_start;
	}

	addr = c->vm_start;
	do {
278
		pte_t *ptep;
279 280
		unsigned long pfn;

281 282 283 284 285 286 287
		ptep = pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(addr),
							       addr),
						    addr),
					 addr);
		if (!pte_none(*ptep) && pte_present(*ptep)) {
			pfn = pte_pfn(*ptep);
			pte_clear(&init_mm, addr, ptep);
288 289
			if (pfn_valid(pfn)) {
				struct page *page = pfn_to_page(pfn);
290
				__free_reserved_page(page);
291 292
			}
		}
293
		addr += PAGE_SIZE;
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
	} while (size -= PAGE_SIZE);

	flush_tlb_kernel_range(c->vm_start, c->vm_end);

	list_del(&c->vm_list);

	spin_unlock_irqrestore(&consistent_lock, flags);

	kfree(c);
	return;

 no_area:
	spin_unlock_irqrestore(&consistent_lock, flags);
	printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n",
	       __func__, vaddr);
	dump_stack();
L
Linus Torvalds 已提交
310 311 312 313 314 315 316 317 318 319 320 321 322 323
}
EXPORT_SYMBOL(__dma_free_coherent);

/*
 * make an area consistent.
 */
void __dma_sync(void *vaddr, size_t size, int direction)
{
	unsigned long start = (unsigned long)vaddr;
	unsigned long end   = start + size;

	switch (direction) {
	case DMA_NONE:
		BUG();
324 325 326 327 328 329 330 331 332
	case DMA_FROM_DEVICE:
		/*
		 * invalidate only when cache-line aligned otherwise there is
		 * the potential for discarding uncommitted data from the cache
		 */
		if ((start & (L1_CACHE_BYTES - 1)) || (size & (L1_CACHE_BYTES - 1)))
			flush_dcache_range(start, end);
		else
			invalidate_dcache_range(start, end);
L
Linus Torvalds 已提交
333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
		break;
	case DMA_TO_DEVICE:		/* writeback only */
		clean_dcache_range(start, end);
		break;
	case DMA_BIDIRECTIONAL:	/* writeback and invalidate */
		flush_dcache_range(start, end);
		break;
	}
}
EXPORT_SYMBOL(__dma_sync);

#ifdef CONFIG_HIGHMEM
/*
 * __dma_sync_page() implementation for systems using highmem.
 * In this case, each page of a buffer must be kmapped/kunmapped
 * in order to have a virtual address for __dma_sync(). This must
349
 * not sleep so kmap_atomic()/kunmap_atomic() are used.
L
Linus Torvalds 已提交
350 351 352 353 354 355 356
 *
 * Note: yes, it is possible and correct to have a buffer extend
 * beyond the first page.
 */
static inline void __dma_sync_page_highmem(struct page *page,
		unsigned long offset, size_t size, int direction)
{
P
Paolo Galtieri 已提交
357
	size_t seg_size = min((size_t)(PAGE_SIZE - offset), size);
L
Linus Torvalds 已提交
358 359
	size_t cur_size = seg_size;
	unsigned long flags, start, seg_offset = offset;
P
Paolo Galtieri 已提交
360
	int nr_segs = 1 + ((size - seg_size) + PAGE_SIZE - 1)/PAGE_SIZE;
L
Linus Torvalds 已提交
361 362 363 364 365
	int seg_nr = 0;

	local_irq_save(flags);

	do {
366
		start = (unsigned long)kmap_atomic(page + seg_nr) + seg_offset;
L
Linus Torvalds 已提交
367 368 369

		/* Sync this buffer segment */
		__dma_sync((void *)start, seg_size, direction);
370
		kunmap_atomic((void *)start);
L
Linus Torvalds 已提交
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
		seg_nr++;

		/* Calculate next buffer segment size */
		seg_size = min((size_t)PAGE_SIZE, size - cur_size);

		/* Add the segment size to our running total */
		cur_size += seg_size;
		seg_offset = 0;
	} while (seg_nr < nr_segs);

	local_irq_restore(flags);
}
#endif /* CONFIG_HIGHMEM */

/*
 * __dma_sync_page makes memory consistent. identical to __dma_sync, but
 * takes a struct page instead of a virtual address
 */
void __dma_sync_page(struct page *page, unsigned long offset,
	size_t size, int direction)
{
#ifdef CONFIG_HIGHMEM
	__dma_sync_page_highmem(page, offset, size, direction);
#else
	unsigned long start = (unsigned long)page_address(page) + offset;
	__dma_sync((void *)start, size, direction);
#endif
}
EXPORT_SYMBOL(__dma_sync_page);
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419

/*
 * Return the PFN for a given cpu virtual address returned by
 * __dma_alloc_coherent. This is used by dma_mmap_coherent()
 */
unsigned long __dma_get_coherent_pfn(unsigned long cpu_addr)
{
	/* This should always be populated, so we don't test every
	 * level. If that fails, we'll have a nice crash which
	 * will be as good as a BUG_ON()
	 */
	pgd_t *pgd = pgd_offset_k(cpu_addr);
	pud_t *pud = pud_offset(pgd, cpu_addr);
	pmd_t *pmd = pmd_offset(pud, cpu_addr);
	pte_t *ptep = pte_offset_kernel(pmd, cpu_addr);

	if (pte_none(*ptep) || !pte_present(*ptep))
		return 0;
	return pte_pfn(*ptep);
}