discontig.c 12.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 * Written by: Patricia Gaughen <gone@us.ibm.com>, IBM Corporation
 * August 2002: added remote node KVA remap - Martin J. Bligh 
 *
 * Copyright (C) 2002, IBM Corp.
 *
 * All rights reserved.          
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 * NON INFRINGEMENT.  See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include <linux/mm.h>
#include <linux/bootmem.h>
#include <linux/mmzone.h>
#include <linux/highmem.h>
#include <linux/initrd.h>
#include <linux/nodemask.h>
31
#include <linux/module.h>
32
#include <linux/kexec.h>
D
Dave Hansen 已提交
33
#include <linux/pfn.h>
34

L
Linus Torvalds 已提交
35 36 37 38 39
#include <asm/e820.h>
#include <asm/setup.h>
#include <asm/mmzone.h>
#include <bios_ebda.h>

40
struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
41
EXPORT_SYMBOL(node_data);
L
Linus Torvalds 已提交
42 43 44
bootmem_data_t node0_bdata;

/*
A
Adrian Bunk 已提交
45
 * numa interface - we expect the numa architecture specific code to have
L
Linus Torvalds 已提交
46 47 48
 *                  populated the following initialisation.
 *
 * 1) node_online_map  - the map of all nodes configured (online) in the system
49
 * 2) node_start_pfn   - the starting page frame number for a node
L
Linus Torvalds 已提交
50 51
 * 3) node_end_pfn     - the ending page fram number for a node
 */
52 53
unsigned long node_start_pfn[MAX_NUMNODES] __read_mostly;
unsigned long node_end_pfn[MAX_NUMNODES] __read_mostly;
54

L
Linus Torvalds 已提交
55

56
#ifdef CONFIG_DISCONTIGMEM
L
Linus Torvalds 已提交
57
/*
58
 * 4) physnode_map     - the mapping between a pfn and owning node
L
Linus Torvalds 已提交
59 60 61 62 63 64 65 66 67 68
 * physnode_map keeps track of the physical memory layout of a generic
 * numa node on a 256Mb break (each element of the array will
 * represent 256Mb of memory and will be marked by the node id.  so,
 * if the first gig is on node 0, and the second gig is on node 1
 * physnode_map will contain:
 *
 *     physnode_map[0-3] = 0;
 *     physnode_map[4-7] = 1;
 *     physnode_map[8- ] = -1;
 */
69
s8 physnode_map[MAX_ELEMENTS] __read_mostly = { [0 ... (MAX_ELEMENTS - 1)] = -1};
70
EXPORT_SYMBOL(physnode_map);
L
Linus Torvalds 已提交
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96

void memory_present(int nid, unsigned long start, unsigned long end)
{
	unsigned long pfn;

	printk(KERN_INFO "Node: %d, start_pfn: %ld, end_pfn: %ld\n",
			nid, start, end);
	printk(KERN_DEBUG "  Setting physnode_map array to node %d for pfns:\n", nid);
	printk(KERN_DEBUG "  ");
	for (pfn = start; pfn < end; pfn += PAGES_PER_ELEMENT) {
		physnode_map[pfn / PAGES_PER_ELEMENT] = nid;
		printk("%ld ", pfn);
	}
	printk("\n");
}

unsigned long node_memmap_size_bytes(int nid, unsigned long start_pfn,
					      unsigned long end_pfn)
{
	unsigned long nr_pages = end_pfn - start_pfn;

	if (!nr_pages)
		return 0;

	return (nr_pages + 1) * sizeof(struct page);
}
97
#endif
L
Linus Torvalds 已提交
98 99 100

extern unsigned long find_max_low_pfn(void);
extern void find_max_pfn(void);
101
extern void add_one_highpage_init(struct page *, int, int);
L
Linus Torvalds 已提交
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117

extern struct e820map e820;
extern unsigned long init_pg_tables_end;
extern unsigned long highend_pfn, highstart_pfn;
extern unsigned long max_low_pfn;
extern unsigned long totalram_pages;
extern unsigned long totalhigh_pages;

#define LARGE_PAGE_BYTES (PTRS_PER_PTE * PAGE_SIZE)

unsigned long node_remap_start_pfn[MAX_NUMNODES];
unsigned long node_remap_size[MAX_NUMNODES];
unsigned long node_remap_offset[MAX_NUMNODES];
void *node_remap_start_vaddr[MAX_NUMNODES];
void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags);

118 119
void *node_remap_end_vaddr[MAX_NUMNODES];
void *node_remap_alloc_vaddr[MAX_NUMNODES];
120 121
static unsigned long kva_start_pfn;
static unsigned long kva_pages;
L
Linus Torvalds 已提交
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
/*
 * FLAT - support for basic PC memory model with discontig enabled, essentially
 *        a single node with all available processors in it with a flat
 *        memory map.
 */
int __init get_memcfg_numa_flat(void)
{
	printk("NUMA - single node, flat memory mode\n");

	/* Run the memory configuration and find the top of memory. */
	find_max_pfn();
	node_start_pfn[0] = 0;
	node_end_pfn[0] = max_pfn;
	memory_present(0, 0, max_pfn);

        /* Indicate there is one node available. */
	nodes_clear(node_online_map);
	node_set_online(0);
	return 1;
}

/*
 * Find the highest page frame number we have available for the node
 */
static void __init find_max_pfn_node(int nid)
{
	if (node_end_pfn[nid] > max_pfn)
		node_end_pfn[nid] = max_pfn;
	/*
	 * if a user has given mem=XXXX, then we need to make sure 
	 * that the node _starts_ before that, too, not just ends
	 */
	if (node_start_pfn[nid] > max_pfn)
		node_start_pfn[nid] = max_pfn;
	if (node_start_pfn[nid] > node_end_pfn[nid])
		BUG();
}

160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
/* Find the owning node for a pfn. */
int early_pfn_to_nid(unsigned long pfn)
{
	int nid;

	for_each_node(nid) {
		if (node_end_pfn[nid] == 0)
			break;
		if (node_start_pfn[nid] <= pfn && node_end_pfn[nid] >= pfn)
			return nid;
	}

	return 0;
}

L
Linus Torvalds 已提交
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
/* 
 * Allocate memory for the pg_data_t for this node via a crude pre-bootmem
 * method.  For node zero take this from the bottom of memory, for
 * subsequent nodes place them at node_remap_start_vaddr which contains
 * node local data in physically node local memory.  See setup_memory()
 * for details.
 */
static void __init allocate_pgdat(int nid)
{
	if (nid && node_has_online_mem(nid))
		NODE_DATA(nid) = (pg_data_t *)node_remap_start_vaddr[nid];
	else {
		NODE_DATA(nid) = (pg_data_t *)(__va(min_low_pfn << PAGE_SHIFT));
		min_low_pfn += PFN_UP(sizeof(pg_data_t));
	}
}

192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
void *alloc_remap(int nid, unsigned long size)
{
	void *allocation = node_remap_alloc_vaddr[nid];

	size = ALIGN(size, L1_CACHE_BYTES);

	if (!allocation || (allocation + size) >= node_remap_end_vaddr[nid])
		return 0;

	node_remap_alloc_vaddr[nid] += size;
	memset(allocation, 0, size);

	return allocation;
}

L
Linus Torvalds 已提交
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
void __init remap_numa_kva(void)
{
	void *vaddr;
	unsigned long pfn;
	int node;

	for_each_online_node(node) {
		for (pfn=0; pfn < node_remap_size[node]; pfn += PTRS_PER_PTE) {
			vaddr = node_remap_start_vaddr[node]+(pfn<<PAGE_SHIFT);
			set_pmd_pfn((ulong) vaddr, 
				node_remap_start_pfn[node] + pfn, 
				PAGE_KERNEL_LARGE);
		}
	}
}

static unsigned long calculate_numa_remap_pages(void)
{
	int nid;
	unsigned long size, reserve_pages = 0;
227
	unsigned long pfn;
L
Linus Torvalds 已提交
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245

	for_each_online_node(nid) {
		/*
		 * The acpi/srat node info can show hot-add memroy zones
		 * where memory could be added but not currently present.
		 */
		if (node_start_pfn[nid] > max_pfn)
			continue;
		if (node_end_pfn[nid] > max_pfn)
			node_end_pfn[nid] = max_pfn;

		/* ensure the remap includes space for the pgdat. */
		size = node_remap_size[nid] + sizeof(pg_data_t);

		/* convert size to large (pmd size) pages, rounding up */
		size = (size + LARGE_PAGE_BYTES - 1) / LARGE_PAGE_BYTES;
		/* now the roundup is correct, convert to PAGE_SIZE pages */
		size = size * PTRS_PER_PTE;
246 247 248 249 250 251 252 253 254 255 256 257 258

		/*
		 * Validate the region we are allocating only contains valid
		 * pages.
		 */
		for (pfn = node_end_pfn[nid] - size;
		     pfn < node_end_pfn[nid]; pfn++)
			if (!page_is_ram(pfn))
				break;

		if (pfn != node_end_pfn[nid])
			size = 0;

L
Linus Torvalds 已提交
259 260 261 262
		printk("Reserving %ld pages of KVA for lmem_map of node %d\n",
				size, nid);
		node_remap_size[nid] = size;
		node_remap_offset[nid] = reserve_pages;
263
		reserve_pages += size;
L
Linus Torvalds 已提交
264 265
		printk("Shrinking node %d from %ld pages to %ld pages\n",
			nid, node_end_pfn[nid], node_end_pfn[nid] - size);
266 267 268 269 270 271 272 273 274 275 276

		if (node_end_pfn[nid] & (PTRS_PER_PTE-1)) {
			/*
			 * Align node_end_pfn[] and node_remap_start_pfn[] to
			 * pmd boundary. remap_numa_kva will barf otherwise.
			 */
			printk("Shrinking node %d further by %ld pages for proper alignment\n",
				nid, node_end_pfn[nid] & (PTRS_PER_PTE-1));
			size +=  node_end_pfn[nid] & (PTRS_PER_PTE-1);
		}

L
Linus Torvalds 已提交
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
		node_end_pfn[nid] -= size;
		node_remap_start_pfn[nid] = node_end_pfn[nid];
	}
	printk("Reserving total of %ld pages for numa KVA remap\n",
			reserve_pages);
	return reserve_pages;
}

extern void setup_bootmem_allocator(void);
unsigned long __init setup_memory(void)
{
	int nid;
	unsigned long system_start_pfn, system_max_low_pfn;

	/*
	 * When mapping a NUMA machine we allocate the node_mem_map arrays
	 * from node local memory.  They are then mapped directly into KVA
	 * between zone normal and vmalloc space.  Calculate the size of
	 * this space and use it to adjust the boundry between ZONE_NORMAL
	 * and ZONE_HIGHMEM.
	 */
	find_max_pfn();
	get_memcfg_numa();

301
	kva_pages = calculate_numa_remap_pages();
L
Linus Torvalds 已提交
302 303 304 305

	/* partially used pages are not usable - thus round upwards */
	system_start_pfn = min_low_pfn = PFN_UP(init_pg_tables_end);

306 307 308 309 310 311 312 313 314 315 316 317
	kva_start_pfn = find_max_low_pfn() - kva_pages;

#ifdef CONFIG_BLK_DEV_INITRD
	/* Numa kva area is below the initrd */
	if (LOADER_TYPE && INITRD_START)
		kva_start_pfn = PFN_DOWN(INITRD_START)  - kva_pages;
#endif
	kva_start_pfn -= kva_start_pfn & (PTRS_PER_PTE-1);

	system_max_low_pfn = max_low_pfn = find_max_low_pfn();
	printk("kva_start_pfn ~ %ld find_max_low_pfn() ~ %ld\n",
		kva_start_pfn, max_low_pfn);
L
Linus Torvalds 已提交
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
	printk("max_pfn = %ld\n", max_pfn);
#ifdef CONFIG_HIGHMEM
	highstart_pfn = highend_pfn = max_pfn;
	if (max_pfn > system_max_low_pfn)
		highstart_pfn = system_max_low_pfn;
	printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
	       pages_to_mb(highend_pfn - highstart_pfn));
#endif
	printk(KERN_NOTICE "%ldMB LOWMEM available.\n",
			pages_to_mb(system_max_low_pfn));
	printk("min_low_pfn = %ld, max_low_pfn = %ld, highstart_pfn = %ld\n", 
			min_low_pfn, max_low_pfn, highstart_pfn);

	printk("Low memory ends at vaddr %08lx\n",
			(ulong) pfn_to_kaddr(max_low_pfn));
	for_each_online_node(nid) {
		node_remap_start_vaddr[nid] = pfn_to_kaddr(
335
				kva_start_pfn + node_remap_offset[nid]);
336 337 338 339 340 341
		/* Init the node remap allocator */
		node_remap_end_vaddr[nid] = node_remap_start_vaddr[nid] +
			(node_remap_size[nid] * PAGE_SIZE);
		node_remap_alloc_vaddr[nid] = node_remap_start_vaddr[nid] +
			ALIGN(sizeof(pg_data_t), PAGE_SIZE);

L
Linus Torvalds 已提交
342 343 344
		allocate_pgdat(nid);
		printk ("node %d will remap to vaddr %08lx - %08lx\n", nid,
			(ulong) node_remap_start_vaddr[nid],
345 346
			(ulong) pfn_to_kaddr(highstart_pfn
			   + node_remap_offset[nid] + node_remap_size[nid]));
L
Linus Torvalds 已提交
347 348 349 350 351 352 353 354 355 356 357 358
	}
	printk("High memory starts at vaddr %08lx\n",
			(ulong) pfn_to_kaddr(highstart_pfn));
	for_each_online_node(nid)
		find_max_pfn_node(nid);

	memset(NODE_DATA(0), 0, sizeof(struct pglist_data));
	NODE_DATA(0)->bdata = &node0_bdata;
	setup_bootmem_allocator();
	return max_low_pfn;
}

359 360 361 362 363
void __init numa_kva_reserve(void)
{
	reserve_bootmem(PFN_PHYS(kva_start_pfn),PFN_PHYS(kva_pages));
}

L
Linus Torvalds 已提交
364 365 366 367 368 369
void __init zone_sizes_init(void)
{
	int nid;


	for_each_online_node(nid) {
370
		unsigned long zones_size[MAX_NR_ZONES] = {0, };
L
Linus Torvalds 已提交
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401
		unsigned long *zholes_size;
		unsigned int max_dma;

		unsigned long low = max_low_pfn;
		unsigned long start = node_start_pfn[nid];
		unsigned long high = node_end_pfn[nid];

		max_dma = virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT;

		if (node_has_online_mem(nid)){
			if (start > low) {
#ifdef CONFIG_HIGHMEM
				BUG_ON(start > high);
				zones_size[ZONE_HIGHMEM] = high - start;
#endif
			} else {
				if (low < max_dma)
					zones_size[ZONE_DMA] = low;
				else {
					BUG_ON(max_dma > low);
					BUG_ON(low > high);
					zones_size[ZONE_DMA] = max_dma;
					zones_size[ZONE_NORMAL] = low - max_dma;
#ifdef CONFIG_HIGHMEM
					zones_size[ZONE_HIGHMEM] = high - low;
#endif
				}
			}
		}

		zholes_size = get_zholes_size(nid);
402 403 404

		free_area_init_node(nid, NODE_DATA(nid), zones_size, start,
				zholes_size);
L
Linus Torvalds 已提交
405 406 407 408 409 410 411 412
	}
	return;
}

void __init set_highmem_pages_init(int bad_ppro) 
{
#ifdef CONFIG_HIGHMEM
	struct zone *zone;
413
	struct page *page;
L
Linus Torvalds 已提交
414 415

	for_each_zone(zone) {
416 417
		unsigned long node_pfn, zone_start_pfn, zone_end_pfn;

L
Linus Torvalds 已提交
418 419 420 421
		if (!is_highmem(zone))
			continue;

		zone_start_pfn = zone->zone_start_pfn;
422 423 424
		zone_end_pfn = zone_start_pfn + zone->spanned_pages;

		printk("Initializing %s for node %d (%08lx:%08lx)\n",
425
				zone->name, zone_to_nid(zone),
426
				zone_start_pfn, zone_end_pfn);
L
Linus Torvalds 已提交
427

428 429 430 431
		for (node_pfn = zone_start_pfn; node_pfn < zone_end_pfn; node_pfn++) {
			if (!pfn_valid(node_pfn))
				continue;
			page = pfn_to_page(node_pfn);
432
			add_one_highpage_init(page, node_pfn, bad_ppro);
L
Linus Torvalds 已提交
433 434 435 436 437
		}
	}
	totalram_pages += totalhigh_pages;
#endif
}