uncached.c

/*
 * Copyright (C) 2001-2005 Silicon Graphics, Inc.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of version 2 of the GNU General Public License
 * as published by the Free Software Foundation.
 *
 * A simple uncached page allocator using the generic allocator. This
 * allocator first utilizes the spare (spill) pages found in the EFI
 * memmap and will then start converting cached pages to uncached ones
 * at a granule at a time. Node awareness is implemented by having a
 * pool of pages per node.
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/efi.h>
#include <linux/genalloc.h>
#include <asm/page.h>
#include <asm/pal.h>
#include <asm/system.h>
#include <asm/pgtable.h>
#include <asm/atomic.h>
#include <asm/tlbflush.h>
#include <asm/sn/arch.h>

#define DEBUG	0

#if DEBUG
#define dprintk			printk
#else
#define dprintk(x...)		do { } while (0)
#endif

void __init efi_memmap_walk_uc (efi_freemem_callback_t callback);

#define MAX_UNCACHED_GRANULES	5
static int allocated_granules;

struct gen_pool *uncached_pool[MAX_NUMNODES];


static void uncached_ipi_visibility(void *data)
{
	int status;

	status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL);
	if ((status != PAL_VISIBILITY_OK) &&
	    (status != PAL_VISIBILITY_OK_REMOTE_NEEDED))
		printk(KERN_DEBUG "pal_prefetch_visibility() returns %i on "
		       "CPU %i\n", status, get_cpu());
}


static void uncached_ipi_mc_drain(void *data)
{
	int status;
	status = ia64_pal_mc_drain();
	if (status)
		printk(KERN_WARNING "ia64_pal_mc_drain() failed with %i on "
		       "CPU %i\n", status, get_cpu());
}


static unsigned long
uncached_get_new_chunk(struct gen_pool *poolp)
{
	struct page *page;
	void *tmp;
	int status, i;
	unsigned long addr, node;

	if (allocated_granules >= MAX_UNCACHED_GRANULES)
		return 0;

	node = poolp->private;
	page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO,
				IA64_GRANULE_SHIFT-PAGE_SHIFT);

	dprintk(KERN_INFO "get_new_chunk page %p, addr %lx\n",
		page, (unsigned long)(page-vmem_map) << PAGE_SHIFT);

	/*
	 * Do magic if no mem on local node! XXX
	 */
	if (!page)
		return 0;
	tmp = page_address(page);

	/*
	 * There's a small race here where it's possible for someone to
	 * access the page through /dev/mem halfway through the conversion
	 * to uncached - not sure it's really worth bothering about
	 */
	for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++)
		SetPageUncached(&page[i]);

	flush_tlb_kernel_range(tmp, tmp + IA64_GRANULE_SIZE);

	status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL);

	dprintk(KERN_INFO "pal_prefetch_visibility() returns %i on cpu %i\n",
		status, get_cpu());

	if (!status) {
		status = smp_call_function(uncached_ipi_visibility, NULL, 0, 1);
		if (status)
			printk(KERN_WARNING "smp_call_function failed for "
			       "uncached_ipi_visibility! (%i)\n", status);
	}

	if (ia64_platform_is("sn2"))
		sn_flush_all_caches((unsigned long)tmp, IA64_GRANULE_SIZE);
	else
		flush_icache_range((unsigned long)tmp,
				   (unsigned long)tmp+IA64_GRANULE_SIZE);

	ia64_pal_mc_drain();
	status = smp_call_function(uncached_ipi_mc_drain, NULL, 0, 1);
	if (status)
		printk(KERN_WARNING "smp_call_function failed for "
		       "uncached_ipi_mc_drain! (%i)\n", status);

	addr = (unsigned long)tmp - PAGE_OFFSET + __IA64_UNCACHED_OFFSET;

	allocated_granules++;
	return addr;
}


/*
 * uncached_alloc_page
 *
 * Allocate 1 uncached page. Allocates on the requested node. If no
 * uncached pages are available on the requested node, roundrobin starting
 * with higher nodes.
 */
unsigned long
uncached_alloc_page(int nid)
{
	unsigned long maddr;

	maddr = gen_pool_alloc(uncached_pool[nid], PAGE_SIZE);

	dprintk(KERN_DEBUG "uncached_alloc_page returns %lx on node %i\n",
		maddr, nid);

	/*
	 * If no memory is availble on our local node, try the
	 * remaining nodes in the system.
	 */
	if (!maddr) {
		int i;

		for (i = MAX_NUMNODES - 1; i >= 0; i--) {
			if (i == nid || !node_online(i))
				continue;
			maddr = gen_pool_alloc(uncached_pool[i], PAGE_SIZE);
			dprintk(KERN_DEBUG "uncached_alloc_page alternate search "
				"returns %lx on node %i\n", maddr, i);
			if (maddr) {
				break;
			}
		}
	}

	return maddr;
}
EXPORT_SYMBOL(uncached_alloc_page);


/*
 * uncached_free_page
 *
 * Free a single uncached page.
 */
void
uncached_free_page(unsigned long maddr)
{
	int node;

	node = paddr_to_nid(maddr - __IA64_UNCACHED_OFFSET);

	dprintk(KERN_DEBUG "uncached_free_page(%lx) on node %i\n", maddr, node);

	if ((maddr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET)
		panic("uncached_free_page invalid address %lx\n", maddr);

	gen_pool_free(uncached_pool[node], maddr, PAGE_SIZE);
}
EXPORT_SYMBOL(uncached_free_page);


/*
 * uncached_build_memmap,
 *
 * Called at boot time to build a map of pages that can be used for
 * memory special operations.
 */
static int __init
uncached_build_memmap(unsigned long start, unsigned long end, void *arg)
{
	long length;
	unsigned long vstart, vend;
	int node;

	length = end - start;
	vstart = start + __IA64_UNCACHED_OFFSET;
	vend = end + __IA64_UNCACHED_OFFSET;

	dprintk(KERN_ERR "uncached_build_memmap(%lx %lx)\n", start, end);

	memset((char *)vstart, 0, length);

	node = paddr_to_nid(start);

	for (; vstart < vend ; vstart += PAGE_SIZE) {
		dprintk(KERN_INFO "sticking %lx into the pool!\n", vstart);
		gen_pool_free(uncached_pool[node], vstart, PAGE_SIZE);
	}

	return 0;
}


static int __init uncached_init(void) {
	int i;

	for (i = 0; i < MAX_NUMNODES; i++) {
		if (!node_online(i))
			continue;
		uncached_pool[i] = gen_pool_create(0, IA64_GRANULE_SHIFT,
						   &uncached_get_new_chunk, i);
	}

	efi_memmap_walk_uc(uncached_build_memmap);

	return 0;
}

__initcall(uncached_init);