提交 36d0fd21 编写于 作者: L Laura Abbott 提交者: Linus Torvalds

arm: use genalloc for the atomic pool

ARM currently uses a bitmap for tracking atomic allocations.  genalloc
already handles this type of memory pool allocation so switch to using
that instead.
Signed-off-by: NLaura Abbott <lauraa@codeaurora.org>
Reviewed-by: NCatalin Marinas <catalin.marinas@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: David Riley <davidriley@chromium.org>
Cc: Olof Johansson <olof@lixom.net>
Cc: Ritesh Harjain <ritesh.harjani@gmail.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Thierry Reding <thierry.reding@gmail.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: NAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: NLinus Torvalds <torvalds@linux-foundation.org>
上级 513510dd
...@@ -14,6 +14,7 @@ config ARM ...@@ -14,6 +14,7 @@ config ARM
select CLONE_BACKWARDS select CLONE_BACKWARDS
select CPU_PM if (SUSPEND || CPU_IDLE) select CPU_PM if (SUSPEND || CPU_IDLE)
select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS
select GENERIC_ALLOCATOR
select GENERIC_ATOMIC64 if (CPU_V7M || CPU_V6 || !CPU_32v6K || !AEABI) select GENERIC_ATOMIC64 if (CPU_V7M || CPU_V6 || !CPU_32v6K || !AEABI)
select GENERIC_CLOCKEVENTS_BROADCAST if SMP select GENERIC_CLOCKEVENTS_BROADCAST if SMP
select GENERIC_IDLE_POLL_SETUP select GENERIC_IDLE_POLL_SETUP
......
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
#include <linux/bootmem.h> #include <linux/bootmem.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/genalloc.h>
#include <linux/gfp.h> #include <linux/gfp.h>
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/list.h> #include <linux/list.h>
...@@ -314,23 +315,13 @@ static void __dma_free_remap(void *cpu_addr, size_t size) ...@@ -314,23 +315,13 @@ static void __dma_free_remap(void *cpu_addr, size_t size)
} }
#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
static struct gen_pool *atomic_pool;
struct dma_pool { static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE;
size_t size;
spinlock_t lock;
unsigned long *bitmap;
unsigned long nr_pages;
void *vaddr;
struct page **pages;
};
static struct dma_pool atomic_pool = {
.size = DEFAULT_DMA_COHERENT_POOL_SIZE,
};
static int __init early_coherent_pool(char *p) static int __init early_coherent_pool(char *p)
{ {
atomic_pool.size = memparse(p, &p); atomic_pool_size = memparse(p, &p);
return 0; return 0;
} }
early_param("coherent_pool", early_coherent_pool); early_param("coherent_pool", early_coherent_pool);
...@@ -340,14 +331,14 @@ void __init init_dma_coherent_pool_size(unsigned long size) ...@@ -340,14 +331,14 @@ void __init init_dma_coherent_pool_size(unsigned long size)
/* /*
* Catch any attempt to set the pool size too late. * Catch any attempt to set the pool size too late.
*/ */
BUG_ON(atomic_pool.vaddr); BUG_ON(atomic_pool);
/* /*
* Set architecture specific coherent pool size only if * Set architecture specific coherent pool size only if
* it has not been changed by kernel command line parameter. * it has not been changed by kernel command line parameter.
*/ */
if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE) if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE)
atomic_pool.size = size; atomic_pool_size = size;
} }
/* /*
...@@ -355,52 +346,44 @@ void __init init_dma_coherent_pool_size(unsigned long size) ...@@ -355,52 +346,44 @@ void __init init_dma_coherent_pool_size(unsigned long size)
*/ */
static int __init atomic_pool_init(void) static int __init atomic_pool_init(void)
{ {
struct dma_pool *pool = &atomic_pool;
pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL); pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
gfp_t gfp = GFP_KERNEL | GFP_DMA; gfp_t gfp = GFP_KERNEL | GFP_DMA;
unsigned long nr_pages = pool->size >> PAGE_SHIFT;
unsigned long *bitmap;
struct page *page; struct page *page;
struct page **pages;
void *ptr; void *ptr;
int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
bitmap = kzalloc(bitmap_size, GFP_KERNEL);
if (!bitmap)
goto no_bitmap;
pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL); atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
if (!pages) if (!atomic_pool)
goto no_pages; goto out;
if (dev_get_cma_area(NULL)) if (dev_get_cma_area(NULL))
ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page, ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
atomic_pool_init); &page, atomic_pool_init);
else else
ptr = __alloc_remap_buffer(NULL, pool->size, gfp, prot, &page, ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
atomic_pool_init); &page, atomic_pool_init);
if (ptr) { if (ptr) {
int i; int ret;
for (i = 0; i < nr_pages; i++) ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
pages[i] = page + i; page_to_phys(page),
atomic_pool_size, -1);
spin_lock_init(&pool->lock); if (ret)
pool->vaddr = ptr; goto destroy_genpool;
pool->pages = pages;
pool->bitmap = bitmap; gen_pool_set_algo(atomic_pool,
pool->nr_pages = nr_pages; gen_pool_first_fit_order_align,
pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n", (void *)PAGE_SHIFT);
(unsigned)pool->size / 1024); pr_info("DMA: preallocated %zd KiB pool for atomic coherent allocations\n",
atomic_pool_size / 1024);
return 0; return 0;
} }
kfree(pages); destroy_genpool:
no_pages: gen_pool_destroy(atomic_pool);
kfree(bitmap); atomic_pool = NULL;
no_bitmap: out:
pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n", pr_err("DMA: failed to allocate %zx KiB pool for atomic coherent allocation\n",
(unsigned)pool->size / 1024); atomic_pool_size / 1024);
return -ENOMEM; return -ENOMEM;
} }
/* /*
...@@ -504,76 +487,36 @@ static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, ...@@ -504,76 +487,36 @@ static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
static void *__alloc_from_pool(size_t size, struct page **ret_page) static void *__alloc_from_pool(size_t size, struct page **ret_page)
{ {
struct dma_pool *pool = &atomic_pool; unsigned long val;
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
unsigned int pageno;
unsigned long flags;
void *ptr = NULL; void *ptr = NULL;
unsigned long align_mask;
if (!pool->vaddr) { if (!atomic_pool) {
WARN(1, "coherent pool not initialised!\n"); WARN(1, "coherent pool not initialised!\n");
return NULL; return NULL;
} }
/* val = gen_pool_alloc(atomic_pool, size);
* Align the region allocation - allocations from pool are rather if (val) {
* small, so align them to their order in pages, minimum is a page phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
* size. This helps reduce fragmentation of the DMA space.
*/ *ret_page = phys_to_page(phys);
align_mask = (1 << get_order(size)) - 1; ptr = (void *)val;
spin_lock_irqsave(&pool->lock, flags);
pageno = bitmap_find_next_zero_area(pool->bitmap, pool->nr_pages,
0, count, align_mask);
if (pageno < pool->nr_pages) {
bitmap_set(pool->bitmap, pageno, count);
ptr = pool->vaddr + PAGE_SIZE * pageno;
*ret_page = pool->pages[pageno];
} else {
pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
"Please increase it with coherent_pool= kernel parameter!\n",
(unsigned)pool->size / 1024);
} }
spin_unlock_irqrestore(&pool->lock, flags);
return ptr; return ptr;
} }
static bool __in_atomic_pool(void *start, size_t size) static bool __in_atomic_pool(void *start, size_t size)
{ {
struct dma_pool *pool = &atomic_pool; return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
void *end = start + size;
void *pool_start = pool->vaddr;
void *pool_end = pool->vaddr + pool->size;
if (start < pool_start || start >= pool_end)
return false;
if (end <= pool_end)
return true;
WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
start, end - 1, pool_start, pool_end - 1);
return false;
} }
static int __free_from_pool(void *start, size_t size) static int __free_from_pool(void *start, size_t size)
{ {
struct dma_pool *pool = &atomic_pool;
unsigned long pageno, count;
unsigned long flags;
if (!__in_atomic_pool(start, size)) if (!__in_atomic_pool(start, size))
return 0; return 0;
pageno = (start - pool->vaddr) >> PAGE_SHIFT; gen_pool_free(atomic_pool, (unsigned long)start, size);
count = size >> PAGE_SHIFT;
spin_lock_irqsave(&pool->lock, flags);
bitmap_clear(pool->bitmap, pageno, count);
spin_unlock_irqrestore(&pool->lock, flags);
return 1; return 1;
} }
...@@ -1316,11 +1259,13 @@ static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t si ...@@ -1316,11 +1259,13 @@ static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t si
static struct page **__atomic_get_pages(void *addr) static struct page **__atomic_get_pages(void *addr)
{ {
struct dma_pool *pool = &atomic_pool; struct page *page;
struct page **pages = pool->pages; phys_addr_t phys;
int offs = (addr - pool->vaddr) >> PAGE_SHIFT;
phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);
page = phys_to_page(phys);
return pages + offs; return (struct page **)page;
} }
static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs) static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册