numa.c 27.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * pSeries NUMA support
 *
 * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */
#include <linux/threads.h>
#include <linux/bootmem.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/module.h>
#include <linux/nodemask.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
20
#include <linux/lmb.h>
21
#include <linux/of.h>
22
#include <asm/sparsemem.h>
23
#include <asm/prom.h>
24
#include <asm/system.h>
P
Paul Mackerras 已提交
25
#include <asm/smp.h>
L
Linus Torvalds 已提交
26 27 28

static int numa_enabled = 1;

29 30
static char *cmdline __initdata;

L
Linus Torvalds 已提交
31 32 33
static int numa_debug;
#define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }

34
int numa_cpu_lookup_table[NR_CPUS];
L
Linus Torvalds 已提交
35 36
cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];
struct pglist_data *node_data[MAX_NUMNODES];
37 38 39 40 41

EXPORT_SYMBOL(numa_cpu_lookup_table);
EXPORT_SYMBOL(numa_cpumask_lookup_table);
EXPORT_SYMBOL(node_data);

L
Linus Torvalds 已提交
42
static int min_common_depth;
43
static int n_mem_addr_cells, n_mem_size_cells;
L
Linus Torvalds 已提交
44

45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
static int __cpuinit fake_numa_create_new_node(unsigned long end_pfn,
						unsigned int *nid)
{
	unsigned long long mem;
	char *p = cmdline;
	static unsigned int fake_nid;
	static unsigned long long curr_boundary;

	/*
	 * Modify node id, iff we started creating NUMA nodes
	 * We want to continue from where we left of the last time
	 */
	if (fake_nid)
		*nid = fake_nid;
	/*
	 * In case there are no more arguments to parse, the
	 * node_id should be the same as the last fake node id
	 * (we've handled this above).
	 */
	if (!p)
		return 0;

	mem = memparse(p, &p);
	if (!mem)
		return 0;

	if (mem < curr_boundary)
		return 0;

	curr_boundary = mem;

	if ((end_pfn << PAGE_SHIFT) > mem) {
		/*
		 * Skip commas and spaces
		 */
		while (*p == ',' || *p == ' ' || *p == '\t')
			p++;

		cmdline = p;
		fake_nid++;
		*nid = fake_nid;
		dbg("created new fake_node with id %d\n", fake_nid);
		return 1;
	}
	return 0;
}

92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
/*
 * get_active_region_work_fn - A helper function for get_node_active_region
 *	Returns datax set to the start_pfn and end_pfn if they contain
 *	the initial value of datax->start_pfn between them
 * @start_pfn: start page(inclusive) of region to check
 * @end_pfn: end page(exclusive) of region to check
 * @datax: comes in with ->start_pfn set to value to search for and
 *	goes out with active range if it contains it
 * Returns 1 if search value is in range else 0
 */
static int __init get_active_region_work_fn(unsigned long start_pfn,
					unsigned long end_pfn, void *datax)
{
	struct node_active_region *data;
	data = (struct node_active_region *)datax;

	if (start_pfn <= data->start_pfn && end_pfn > data->start_pfn) {
		data->start_pfn = start_pfn;
		data->end_pfn = end_pfn;
		return 1;
	}
	return 0;

}

/*
 * get_node_active_region - Return active region containing start_pfn
119
 * Active range returned is empty if none found.
120 121 122 123 124 125 126 127 128 129
 * @start_pfn: The page to return the region for.
 * @node_ar: Returned set to the active region containing start_pfn
 */
static void __init get_node_active_region(unsigned long start_pfn,
		       struct node_active_region *node_ar)
{
	int nid = early_pfn_to_nid(start_pfn);

	node_ar->nid = nid;
	node_ar->start_pfn = start_pfn;
130
	node_ar->end_pfn = start_pfn;
131 132 133
	work_with_active_regions(nid, get_active_region_work_fn, node_ar);
}

134
static void __cpuinit map_cpu_to_node(int cpu, int node)
L
Linus Torvalds 已提交
135 136
{
	numa_cpu_lookup_table[cpu] = node;
137

138 139
	dbg("adding cpu %d to node %d\n", cpu, node);

140
	if (!(cpu_isset(cpu, numa_cpumask_lookup_table[node])))
L
Linus Torvalds 已提交
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
		cpu_set(cpu, numa_cpumask_lookup_table[node]);
}

#ifdef CONFIG_HOTPLUG_CPU
static void unmap_cpu_from_node(unsigned long cpu)
{
	int node = numa_cpu_lookup_table[cpu];

	dbg("removing cpu %lu from node %d\n", cpu, node);

	if (cpu_isset(cpu, numa_cpumask_lookup_table[node])) {
		cpu_clear(cpu, numa_cpumask_lookup_table[node]);
	} else {
		printk(KERN_ERR "WARNING: cpu %lu not found in node %d\n",
		       cpu, node);
	}
}
#endif /* CONFIG_HOTPLUG_CPU */

160
static struct device_node * __cpuinit find_cpu_node(unsigned int cpu)
L
Linus Torvalds 已提交
161 162 163
{
	unsigned int hw_cpuid = get_hard_smp_processor_id(cpu);
	struct device_node *cpu_node = NULL;
164
	const unsigned int *interrupt_server, *reg;
L
Linus Torvalds 已提交
165 166 167 168
	int len;

	while ((cpu_node = of_find_node_by_type(cpu_node, "cpu")) != NULL) {
		/* Try interrupt server first */
169
		interrupt_server = of_get_property(cpu_node,
L
Linus Torvalds 已提交
170 171 172 173 174 175 176 177 178 179
					"ibm,ppc-interrupt-server#s", &len);

		len = len / sizeof(u32);

		if (interrupt_server && (len > 0)) {
			while (len--) {
				if (interrupt_server[len] == hw_cpuid)
					return cpu_node;
			}
		} else {
180
			reg = of_get_property(cpu_node, "reg", &len);
L
Linus Torvalds 已提交
181 182 183 184 185 186 187 188 189
			if (reg && (len > 0) && (reg[0] == hw_cpuid))
				return cpu_node;
		}
	}

	return NULL;
}

/* must hold reference to node during call */
190
static const int *of_get_associativity(struct device_node *dev)
L
Linus Torvalds 已提交
191
{
192
	return of_get_property(dev, "ibm,associativity", NULL);
L
Linus Torvalds 已提交
193 194
}

195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
/*
 * Returns the property linux,drconf-usable-memory if
 * it exists (the property exists only in kexec/kdump kernels,
 * added by kexec-tools)
 */
static const u32 *of_get_usable_memory(struct device_node *memory)
{
	const u32 *prop;
	u32 len;
	prop = of_get_property(memory, "linux,drconf-usable-memory", &len);
	if (!prop || len < sizeof(unsigned int))
		return 0;
	return prop;
}

210 211 212
/* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa
 * info is found.
 */
213
static int of_node_to_nid_single(struct device_node *device)
L
Linus Torvalds 已提交
214
{
215
	int nid = -1;
216
	const unsigned int *tmp;
L
Linus Torvalds 已提交
217 218

	if (min_common_depth == -1)
219
		goto out;
L
Linus Torvalds 已提交
220 221

	tmp = of_get_associativity(device);
222 223 224 225
	if (!tmp)
		goto out;

	if (tmp[0] >= min_common_depth)
226
		nid = tmp[min_common_depth];
227 228

	/* POWER4 LPAR uses 0xffff as invalid node */
229 230 231
	if (nid == 0xffff || nid >= MAX_NUMNODES)
		nid = -1;
out:
232
	return nid;
L
Linus Torvalds 已提交
233 234
}

235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
/* Walk the device tree upwards, looking for an associativity id */
int of_node_to_nid(struct device_node *device)
{
	struct device_node *tmp;
	int nid = -1;

	of_node_get(device);
	while (device) {
		nid = of_node_to_nid_single(device);
		if (nid != -1)
			break;

	        tmp = device;
		device = of_get_parent(tmp);
		of_node_put(tmp);
	}
	of_node_put(device);

	return nid;
}
EXPORT_SYMBOL_GPL(of_node_to_nid);

L
Linus Torvalds 已提交
257 258 259 260 261 262 263
/*
 * In theory, the "ibm,associativity" property may contain multiple
 * associativity lists because a resource may be multiply connected
 * into the machine.  This resource then has different associativity
 * characteristics relative to its multiple connections.  We ignore
 * this for now.  We also assume that all cpu and memory sets have
 * their distances represented at a common level.  This won't be
264
 * true for hierarchical NUMA.
L
Linus Torvalds 已提交
265 266 267 268 269 270 271 272 273
 *
 * In any case the ibm,associativity-reference-points should give
 * the correct depth for a normal NUMA system.
 *
 * - Dave Hansen <haveblue@us.ibm.com>
 */
static int __init find_min_common_depth(void)
{
	int depth;
274
	const unsigned int *ref_points;
L
Linus Torvalds 已提交
275 276 277 278 279 280 281 282 283 284 285 286 287 288
	struct device_node *rtas_root;
	unsigned int len;

	rtas_root = of_find_node_by_path("/rtas");

	if (!rtas_root)
		return -1;

	/*
	 * this property is 2 32-bit integers, each representing a level of
	 * depth in the associativity nodes.  The first is for an SMP
	 * configuration (should be all 0's) and the second is for a normal
	 * NUMA configuration.
	 */
289
	ref_points = of_get_property(rtas_root,
L
Linus Torvalds 已提交
290 291 292 293 294
			"ibm,associativity-reference-points", &len);

	if ((len >= 1) && ref_points) {
		depth = ref_points[1];
	} else {
295
		dbg("NUMA: ibm,associativity-reference-points not found.\n");
L
Linus Torvalds 已提交
296 297 298 299 300 301 302
		depth = -1;
	}
	of_node_put(rtas_root);

	return depth;
}

303
static void __init get_n_mem_cells(int *n_addr_cells, int *n_size_cells)
L
Linus Torvalds 已提交
304 305 306 307
{
	struct device_node *memory = NULL;

	memory = of_find_node_by_type(memory, "memory");
308
	if (!memory)
309
		panic("numa.c: No memory nodes found!");
310

311
	*n_addr_cells = of_n_addr_cells(memory);
312
	*n_size_cells = of_n_size_cells(memory);
313
	of_node_put(memory);
L
Linus Torvalds 已提交
314 315
}

316
static unsigned long __devinit read_n_cells(int n, const unsigned int **buf)
L
Linus Torvalds 已提交
317 318 319 320 321 322 323 324 325 326
{
	unsigned long result = 0;

	while (n--) {
		result = (result << 32) | **buf;
		(*buf)++;
	}
	return result;
}

327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
struct of_drconf_cell {
	u64	base_addr;
	u32	drc_index;
	u32	reserved;
	u32	aa_index;
	u32	flags;
};

#define DRCONF_MEM_ASSIGNED	0x00000008
#define DRCONF_MEM_AI_INVALID	0x00000040
#define DRCONF_MEM_RESERVED	0x00000080

/*
 * Read the next lmb list entry from the ibm,dynamic-memory property
 * and return the information in the provided of_drconf_cell structure.
 */
static void read_drconf_cell(struct of_drconf_cell *drmem, const u32 **cellp)
{
	const u32 *cp;

	drmem->base_addr = read_n_cells(n_mem_addr_cells, cellp);

	cp = *cellp;
	drmem->drc_index = cp[0];
	drmem->reserved = cp[1];
	drmem->aa_index = cp[2];
	drmem->flags = cp[3];

	*cellp = cp + 4;
}

/*
 * Retreive and validate the ibm,dynamic-memory property of the device tree.
 *
 * The layout of the ibm,dynamic-memory property is a number N of lmb
 * list entries followed by N lmb list entries.  Each lmb list entry
 * contains information as layed out in the of_drconf_cell struct above.
 */
static int of_get_drconf_memory(struct device_node *memory, const u32 **dm)
{
	const u32 *prop;
	u32 len, entries;

	prop = of_get_property(memory, "ibm,dynamic-memory", &len);
	if (!prop || len < sizeof(unsigned int))
		return 0;

	entries = *prop++;

	/* Now that we know the number of entries, revalidate the size
	 * of the property read in to ensure we have everything
	 */
	if (len < (entries * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int))
		return 0;

	*dm = prop;
	return entries;
}

/*
 * Retreive and validate the ibm,lmb-size property for drconf memory
 * from the device tree.
 */
static u64 of_get_lmb_size(struct device_node *memory)
{
	const u32 *prop;
	u32 len;

	prop = of_get_property(memory, "ibm,lmb-size", &len);
	if (!prop || len < sizeof(unsigned int))
		return 0;

	return read_n_cells(n_mem_size_cells, &prop);
}

struct assoc_arrays {
	u32	n_arrays;
	u32	array_sz;
	const u32 *arrays;
};

/*
 * Retreive and validate the list of associativity arrays for drconf
 * memory from the ibm,associativity-lookup-arrays property of the
 * device tree..
 *
 * The layout of the ibm,associativity-lookup-arrays property is a number N
 * indicating the number of associativity arrays, followed by a number M
 * indicating the size of each associativity array, followed by a list
 * of N associativity arrays.
 */
static int of_get_assoc_arrays(struct device_node *memory,
			       struct assoc_arrays *aa)
{
	const u32 *prop;
	u32 len;

	prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len);
	if (!prop || len < 2 * sizeof(unsigned int))
		return -1;

	aa->n_arrays = *prop++;
	aa->array_sz = *prop++;

	/* Now that we know the number of arrrays and size of each array,
	 * revalidate the size of the property read in.
	 */
	if (len < (aa->n_arrays * aa->array_sz + 2) * sizeof(unsigned int))
		return -1;

	aa->arrays = prop;
	return 0;
}

/*
 * This is like of_node_to_nid_single() for memory represented in the
 * ibm,dynamic-reconfiguration-memory node.
 */
static int of_drconf_to_nid_single(struct of_drconf_cell *drmem,
				   struct assoc_arrays *aa)
{
	int default_nid = 0;
	int nid = default_nid;
	int index;

	if (min_common_depth > 0 && min_common_depth <= aa->array_sz &&
	    !(drmem->flags & DRCONF_MEM_AI_INVALID) &&
	    drmem->aa_index < aa->n_arrays) {
		index = drmem->aa_index * aa->array_sz + min_common_depth - 1;
		nid = aa->arrays[index];

		if (nid == 0xffff || nid >= MAX_NUMNODES)
			nid = default_nid;
	}

	return nid;
}

L
Linus Torvalds 已提交
465 466 467 468
/*
 * Figure out to which domain a cpu belongs and stick it there.
 * Return the id of the domain used.
 */
469
static int __cpuinit numa_setup_cpu(unsigned long lcpu)
L
Linus Torvalds 已提交
470
{
471
	int nid = 0;
L
Linus Torvalds 已提交
472 473 474 475 476 477 478
	struct device_node *cpu = find_cpu_node(lcpu);

	if (!cpu) {
		WARN_ON(1);
		goto out;
	}

479
	nid = of_node_to_nid_single(cpu);
L
Linus Torvalds 已提交
480

481 482
	if (nid < 0 || !node_online(nid))
		nid = any_online_node(NODE_MASK_ALL);
L
Linus Torvalds 已提交
483
out:
484
	map_cpu_to_node(lcpu, nid);
L
Linus Torvalds 已提交
485 486 487

	of_node_put(cpu);

488
	return nid;
L
Linus Torvalds 已提交
489 490
}

491
static int __cpuinit cpu_numa_callback(struct notifier_block *nfb,
L
Linus Torvalds 已提交
492 493 494 495 496 497 498 499
			     unsigned long action,
			     void *hcpu)
{
	unsigned long lcpu = (unsigned long)hcpu;
	int ret = NOTIFY_DONE;

	switch (action) {
	case CPU_UP_PREPARE:
500
	case CPU_UP_PREPARE_FROZEN:
501
		numa_setup_cpu(lcpu);
L
Linus Torvalds 已提交
502 503 504 505
		ret = NOTIFY_OK;
		break;
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_DEAD:
506
	case CPU_DEAD_FROZEN:
L
Linus Torvalds 已提交
507
	case CPU_UP_CANCELED:
508
	case CPU_UP_CANCELED_FROZEN:
L
Linus Torvalds 已提交
509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
		unmap_cpu_from_node(lcpu);
		break;
		ret = NOTIFY_OK;
#endif
	}
	return ret;
}

/*
 * Check and possibly modify a memory region to enforce the memory limit.
 *
 * Returns the size the region should have to enforce the memory limit.
 * This will either be the original value of size, a truncated value,
 * or zero. If the returned value of size is 0 the region should be
 * discarded as it lies wholy above the memory limit.
 */
525 526
static unsigned long __init numa_enforce_memory_limit(unsigned long start,
						      unsigned long size)
L
Linus Torvalds 已提交
527 528 529 530
{
	/*
	 * We use lmb_end_of_DRAM() in here instead of memory_limit because
	 * we've already adjusted it for the limit and it takes care of
531 532
	 * having memory holes below the limit.  Also, in the case of
	 * iommu_is_off, memory_limit is not set but is implicitly enforced.
L
Linus Torvalds 已提交
533 534 535 536 537 538 539 540 541 542 543
	 */

	if (start + size <= lmb_end_of_DRAM())
		return size;

	if (start >= lmb_end_of_DRAM())
		return 0;

	return lmb_end_of_DRAM() - start;
}

544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
/*
 * Reads the counter for a given entry in
 * linux,drconf-usable-memory property
 */
static inline int __init read_usm_ranges(const u32 **usm)
{
	/*
	 * For each lmb in ibm,dynamic-memory a corresponding
	 * entry in linux,drconf-usable-memory property contains
	 * a counter followed by that many (base, size) duple.
	 * read the counter from linux,drconf-usable-memory
	 */
	return read_n_cells(n_mem_size_cells, usm);
}

559 560 561 562 563 564
/*
 * Extract NUMA information from the ibm,dynamic-reconfiguration-memory
 * node.  This assumes n_mem_{addr,size}_cells have been set.
 */
static void __init parse_drconf_memory(struct device_node *memory)
{
565 566 567
	const u32 *dm, *usm;
	unsigned int n, rc, ranges, is_kexec_kdump = 0;
	unsigned long lmb_size, base, size, sz;
568 569 570 571 572
	int nid;
	struct assoc_arrays aa;

	n = of_get_drconf_memory(memory, &dm);
	if (!n)
573 574
		return;

575 576 577 578 579 580
	lmb_size = of_get_lmb_size(memory);
	if (!lmb_size)
		return;

	rc = of_get_assoc_arrays(memory, &aa);
	if (rc)
581 582
		return;

583 584 585 586 587
	/* check if this is a kexec/kdump kernel */
	usm = of_get_usable_memory(memory);
	if (usm != NULL)
		is_kexec_kdump = 1;

588
	for (; n != 0; --n) {
589 590 591 592 593 594 595 596
		struct of_drconf_cell drmem;

		read_drconf_cell(&drmem, &dm);

		/* skip this block if the reserved bit is set in flags (0x80)
		   or if the block is not assigned to this partition (0x8) */
		if ((drmem.flags & DRCONF_MEM_RESERVED)
		    || !(drmem.flags & DRCONF_MEM_ASSIGNED))
597
			continue;
598

599 600 601
		base = drmem.base_addr;
		size = lmb_size;
		ranges = 1;
602

603 604 605 606 607 608 609 610 611 612 613 614 615
		if (is_kexec_kdump) {
			ranges = read_usm_ranges(&usm);
			if (!ranges) /* there are no (base, size) duple */
				continue;
		}
		do {
			if (is_kexec_kdump) {
				base = read_n_cells(n_mem_addr_cells, &usm);
				size = read_n_cells(n_mem_size_cells, &usm);
			}
			nid = of_drconf_to_nid_single(&drmem, &aa);
			fake_numa_create_new_node(
				((base + size) >> PAGE_SHIFT),
616
					   &nid);
617 618 619 620 621 622 623
			node_set_online(nid);
			sz = numa_enforce_memory_limit(base, size);
			if (sz)
				add_active_range(nid, base >> PAGE_SHIFT,
						 (base >> PAGE_SHIFT)
						 + (sz >> PAGE_SHIFT));
		} while (--ranges);
624 625 626
	}
}

L
Linus Torvalds 已提交
627 628 629 630
static int __init parse_numa_properties(void)
{
	struct device_node *cpu = NULL;
	struct device_node *memory = NULL;
631
	int default_nid = 0;
L
Linus Torvalds 已提交
632 633 634 635 636 637 638 639 640 641 642 643
	unsigned long i;

	if (numa_enabled == 0) {
		printk(KERN_WARNING "NUMA disabled by user\n");
		return -1;
	}

	min_common_depth = find_min_common_depth();

	if (min_common_depth < 0)
		return min_common_depth;

644 645
	dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth);

L
Linus Torvalds 已提交
646
	/*
647 648 649
	 * Even though we connect cpus to numa domains later in SMP
	 * init, we need to know the node ids now. This is because
	 * each node to be onlined must have NODE_DATA etc backing it.
L
Linus Torvalds 已提交
650
	 */
651
	for_each_present_cpu(i) {
652
		int nid;
L
Linus Torvalds 已提交
653 654

		cpu = find_cpu_node(i);
655
		BUG_ON(!cpu);
656
		nid = of_node_to_nid_single(cpu);
657
		of_node_put(cpu);
L
Linus Torvalds 已提交
658

659 660 661 662 663 664 665 666
		/*
		 * Don't fall back to default_nid yet -- we will plug
		 * cpus into nodes once the memory scan has discovered
		 * the topology.
		 */
		if (nid < 0)
			continue;
		node_set_online(nid);
L
Linus Torvalds 已提交
667 668
	}

669
	get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells);
L
Linus Torvalds 已提交
670 671 672 673
	memory = NULL;
	while ((memory = of_find_node_by_type(memory, "memory")) != NULL) {
		unsigned long start;
		unsigned long size;
674
		int nid;
L
Linus Torvalds 已提交
675
		int ranges;
676
		const unsigned int *memcell_buf;
L
Linus Torvalds 已提交
677 678
		unsigned int len;

679
		memcell_buf = of_get_property(memory,
680 681
			"linux,usable-memory", &len);
		if (!memcell_buf || len <= 0)
682
			memcell_buf = of_get_property(memory, "reg", &len);
L
Linus Torvalds 已提交
683 684 685
		if (!memcell_buf || len <= 0)
			continue;

686 687
		/* ranges in cell */
		ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
L
Linus Torvalds 已提交
688 689
new_range:
		/* these are order-sensitive, and modify the buffer pointer */
690 691
		start = read_n_cells(n_mem_addr_cells, &memcell_buf);
		size = read_n_cells(n_mem_size_cells, &memcell_buf);
L
Linus Torvalds 已提交
692

693 694 695 696 697
		/*
		 * Assumption: either all memory nodes or none will
		 * have associativity properties.  If none, then
		 * everything goes to default_nid.
		 */
698
		nid = of_node_to_nid_single(memory);
699 700
		if (nid < 0)
			nid = default_nid;
701 702

		fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid);
703
		node_set_online(nid);
L
Linus Torvalds 已提交
704

705
		if (!(size = numa_enforce_memory_limit(start, size))) {
L
Linus Torvalds 已提交
706 707 708 709 710 711
			if (--ranges)
				goto new_range;
			else
				continue;
		}

712 713
		add_active_range(nid, start >> PAGE_SHIFT,
				(start >> PAGE_SHIFT) + (size >> PAGE_SHIFT));
L
Linus Torvalds 已提交
714 715 716 717 718

		if (--ranges)
			goto new_range;
	}

719 720 721 722 723 724 725 726
	/*
	 * Now do the same thing for each LMB listed in the ibm,dynamic-memory
	 * property in the ibm,dynamic-reconfiguration-memory node.
	 */
	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
	if (memory)
		parse_drconf_memory(memory);

L
Linus Torvalds 已提交
727 728 729 730 731 732 733
	return 0;
}

static void __init setup_nonnuma(void)
{
	unsigned long top_of_ram = lmb_end_of_DRAM();
	unsigned long total_ram = lmb_phys_mem_size();
734
	unsigned long start_pfn, end_pfn;
735
	unsigned int i, nid = 0;
L
Linus Torvalds 已提交
736

737
	printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
L
Linus Torvalds 已提交
738
	       top_of_ram, total_ram);
739
	printk(KERN_DEBUG "Memory hole size: %ldMB\n",
L
Linus Torvalds 已提交
740 741
	       (top_of_ram - total_ram) >> 20);

742 743 744
	for (i = 0; i < lmb.memory.cnt; ++i) {
		start_pfn = lmb.memory.region[i].base >> PAGE_SHIFT;
		end_pfn = start_pfn + lmb_size_pages(&lmb.memory, i);
745 746 747 748

		fake_numa_create_new_node(end_pfn, &nid);
		add_active_range(nid, start_pfn, end_pfn);
		node_set_online(nid);
749
	}
L
Linus Torvalds 已提交
750 751
}

752 753 754 755 756 757 758 759 760
void __init dump_numa_cpu_topology(void)
{
	unsigned int node;
	unsigned int cpu, count;

	if (min_common_depth == -1 || !numa_enabled)
		return;

	for_each_online_node(node) {
761
		printk(KERN_DEBUG "Node %d CPUs:", node);
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786

		count = 0;
		/*
		 * If we used a CPU iterator here we would miss printing
		 * the holes in the cpumap.
		 */
		for (cpu = 0; cpu < NR_CPUS; cpu++) {
			if (cpu_isset(cpu, numa_cpumask_lookup_table[node])) {
				if (count == 0)
					printk(" %u", cpu);
				++count;
			} else {
				if (count > 1)
					printk("-%u", cpu - 1);
				count = 0;
			}
		}

		if (count > 1)
			printk("-%u", NR_CPUS - 1);
		printk("\n");
	}
}

static void __init dump_numa_memory_topology(void)
L
Linus Torvalds 已提交
787 788 789 790 791 792 793 794 795 796
{
	unsigned int node;
	unsigned int count;

	if (min_common_depth == -1 || !numa_enabled)
		return;

	for_each_online_node(node) {
		unsigned long i;

797
		printk(KERN_DEBUG "Node %d Memory:", node);
L
Linus Torvalds 已提交
798 799 800

		count = 0;

801 802 803
		for (i = 0; i < lmb_end_of_DRAM();
		     i += (1 << SECTION_SIZE_BITS)) {
			if (early_pfn_to_nid(i >> PAGE_SHIFT) == node) {
L
Linus Torvalds 已提交
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
				if (count == 0)
					printk(" 0x%lx", i);
				++count;
			} else {
				if (count > 0)
					printk("-0x%lx", i);
				count = 0;
			}
		}

		if (count > 0)
			printk("-0x%lx", i);
		printk("\n");
	}
}

/*
 * Allocate some memory, satisfying the lmb or bootmem allocator where
 * required. nid is the preferred node and end is the physical address of
 * the highest address in the node.
 *
 * Returns the physical address of the memory.
 */
827 828 829
static void __init *careful_allocation(int nid, unsigned long size,
				       unsigned long align,
				       unsigned long end_pfn)
L
Linus Torvalds 已提交
830
{
831
	int new_nid;
832
	unsigned long ret = __lmb_alloc_base(size, align, end_pfn << PAGE_SHIFT);
L
Linus Torvalds 已提交
833 834 835

	/* retry over all memory */
	if (!ret)
836
		ret = __lmb_alloc_base(size, align, lmb_end_of_DRAM());
L
Linus Torvalds 已提交
837 838 839 840 841 842 843 844 845

	if (!ret)
		panic("numa.c: cannot allocate %lu bytes on node %d",
		      size, nid);

	/*
	 * If the memory came from a previously allocated node, we must
	 * retry with the bootmem allocator.
	 */
846 847 848
	new_nid = early_pfn_to_nid(ret >> PAGE_SHIFT);
	if (new_nid < nid) {
		ret = (unsigned long)__alloc_bootmem_node(NODE_DATA(new_nid),
L
Linus Torvalds 已提交
849 850 851 852
				size, align, 0);

		if (!ret)
			panic("numa.c: cannot allocate %lu bytes on node %d",
853
			      size, new_nid);
L
Linus Torvalds 已提交
854

855
		ret = __pa(ret);
L
Linus Torvalds 已提交
856 857 858 859

		dbg("alloc_bootmem %lx %lx\n", ret, size);
	}

860
	return (void *)ret;
L
Linus Torvalds 已提交
861 862
}

863 864 865 866 867
static struct notifier_block __cpuinitdata ppc64_numa_nb = {
	.notifier_call = cpu_numa_callback,
	.priority = 1 /* Must run before sched domains notifier. */
};

L
Linus Torvalds 已提交
868 869 870
void __init do_init_bootmem(void)
{
	int nid;
871
	unsigned int i;
L
Linus Torvalds 已提交
872 873 874 875 876 877 878 879

	min_low_pfn = 0;
	max_low_pfn = lmb_end_of_DRAM() >> PAGE_SHIFT;
	max_pfn = max_low_pfn;

	if (parse_numa_properties())
		setup_nonnuma();
	else
880
		dump_numa_memory_topology();
L
Linus Torvalds 已提交
881 882

	register_cpu_notifier(&ppc64_numa_nb);
883 884
	cpu_numa_callback(&ppc64_numa_nb, CPU_UP_PREPARE,
			  (void *)(unsigned long)boot_cpuid);
L
Linus Torvalds 已提交
885 886

	for_each_online_node(nid) {
887
		unsigned long start_pfn, end_pfn;
L
Linus Torvalds 已提交
888 889 890
		unsigned long bootmem_paddr;
		unsigned long bootmap_pages;

891
		get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
L
Linus Torvalds 已提交
892 893

		/* Allocate the node structure node local if possible */
894
		NODE_DATA(nid) = careful_allocation(nid,
L
Linus Torvalds 已提交
895
					sizeof(struct pglist_data),
896 897
					SMP_CACHE_BYTES, end_pfn);
		NODE_DATA(nid) = __va(NODE_DATA(nid));
L
Linus Torvalds 已提交
898 899 900 901 902
		memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));

  		dbg("node %d\n", nid);
		dbg("NODE_DATA() = %p\n", NODE_DATA(nid));

903
		NODE_DATA(nid)->bdata = &bootmem_node_data[nid];
904 905
		NODE_DATA(nid)->node_start_pfn = start_pfn;
		NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
L
Linus Torvalds 已提交
906 907 908 909

		if (NODE_DATA(nid)->node_spanned_pages == 0)
  			continue;

910 911
  		dbg("start_paddr = %lx\n", start_pfn << PAGE_SHIFT);
  		dbg("end_paddr = %lx\n", end_pfn << PAGE_SHIFT);
L
Linus Torvalds 已提交
912

913 914 915 916 917
		bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn);
		bootmem_paddr = (unsigned long)careful_allocation(nid,
					bootmap_pages << PAGE_SHIFT,
					PAGE_SIZE, end_pfn);
		memset(__va(bootmem_paddr), 0, bootmap_pages << PAGE_SHIFT);
L
Linus Torvalds 已提交
918 919 920 921

		dbg("bootmap_paddr = %lx\n", bootmem_paddr);

		init_bootmem_node(NODE_DATA(nid), bootmem_paddr >> PAGE_SHIFT,
922
				  start_pfn, end_pfn);
L
Linus Torvalds 已提交
923

924
		free_bootmem_with_active_regions(nid, end_pfn);
925
	}
L
Linus Torvalds 已提交
926

927 928 929 930 931 932 933 934 935
	/* Mark reserved regions */
	for (i = 0; i < lmb.reserved.cnt; i++) {
		unsigned long physbase = lmb.reserved.region[i].base;
		unsigned long size = lmb.reserved.region[i].size;
		unsigned long start_pfn = physbase >> PAGE_SHIFT;
		unsigned long end_pfn = ((physbase + size) >> PAGE_SHIFT);
		struct node_active_region node_ar;

		get_node_active_region(start_pfn, &node_ar);
936 937 938
		while (start_pfn < end_pfn &&
			node_ar.start_pfn < node_ar.end_pfn) {
			unsigned long reserve_size = size;
939 940 941 942 943
			/*
			 * if reserved region extends past active region
			 * then trim size to active region
			 */
			if (end_pfn > node_ar.end_pfn)
944
				reserve_size = (node_ar.end_pfn << PAGE_SHIFT)
945
					- (start_pfn << PAGE_SHIFT);
946 947
			dbg("reserve_bootmem %lx %lx nid=%d\n", physbase,
				reserve_size, node_ar.nid);
948
			reserve_bootmem_node(NODE_DATA(node_ar.nid), physbase,
949
						reserve_size, BOOTMEM_DEFAULT);
950 951 952 953 954 955 956 957 958 959 960 961 962 963
			/*
			 * if reserved region is contained in the active region
			 * then done.
			 */
			if (end_pfn <= node_ar.end_pfn)
				break;

			/*
			 * reserved region extends past the active region
			 *   get next active region that contains this
			 *   reserved region
			 */
			start_pfn = node_ar.end_pfn;
			physbase = start_pfn << PAGE_SHIFT;
964
			size = size - reserve_size;
965
			get_node_active_region(start_pfn, &node_ar);
L
Linus Torvalds 已提交
966
		}
B
Bob Picco 已提交
967

L
Linus Torvalds 已提交
968
	}
969 970 971

	for_each_online_node(nid)
		sparse_memory_present_with_active_regions(nid);
L
Linus Torvalds 已提交
972 973 974 975
}

void __init paging_init(void)
{
976 977 978
	unsigned long max_zone_pfns[MAX_NR_ZONES];
	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
	max_zone_pfns[ZONE_DMA] = lmb_end_of_DRAM() >> PAGE_SHIFT;
979
	free_area_init_nodes(max_zone_pfns);
L
Linus Torvalds 已提交
980 981 982 983 984 985 986 987 988 989 990 991 992
}

static int __init early_numa(char *p)
{
	if (!p)
		return 0;

	if (strstr(p, "off"))
		numa_enabled = 0;

	if (strstr(p, "debug"))
		numa_debug = 1;

993 994 995 996
	p = strstr(p, "fake=");
	if (p)
		cmdline = p + strlen("fake=");

L
Linus Torvalds 已提交
997 998 999
	return 0;
}
early_param("numa", early_numa);
1000 1001

#ifdef CONFIG_MEMORY_HOTPLUG
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
/*
 * Validate the node associated with the memory section we are
 * trying to add.
 */
int valid_hot_add_scn(int *nid, unsigned long start, u32 lmb_size,
		      unsigned long scn_addr)
{
	nodemask_t nodes;

	if (*nid < 0 || !node_online(*nid))
		*nid = any_online_node(NODE_MASK_ALL);

	if ((scn_addr >= start) && (scn_addr < (start + lmb_size))) {
		nodes_setall(nodes);
		while (NODE_DATA(*nid)->node_spanned_pages == 0) {
			node_clear(*nid, nodes);
			*nid = any_online_node(nodes);
		}

		return 1;
	}

	return 0;
}

/*
 * Find the node associated with a hot added memory section represented
 * by the ibm,dynamic-reconfiguration-memory node.
 */
static int hot_add_drconf_scn_to_nid(struct device_node *memory,
				     unsigned long scn_addr)
{
	const u32 *dm;
	unsigned int n, rc;
	unsigned long lmb_size;
	int default_nid = any_online_node(NODE_MASK_ALL);
	int nid;
	struct assoc_arrays aa;

	n = of_get_drconf_memory(memory, &dm);
	if (!n)
		return default_nid;;

	lmb_size = of_get_lmb_size(memory);
	if (!lmb_size)
		return default_nid;

	rc = of_get_assoc_arrays(memory, &aa);
	if (rc)
		return default_nid;

	for (; n != 0; --n) {
		struct of_drconf_cell drmem;

		read_drconf_cell(&drmem, &dm);

		/* skip this block if it is reserved or not assigned to
		 * this partition */
		if ((drmem.flags & DRCONF_MEM_RESERVED)
		    || !(drmem.flags & DRCONF_MEM_ASSIGNED))
			continue;

		nid = of_drconf_to_nid_single(&drmem, &aa);

		if (valid_hot_add_scn(&nid, drmem.base_addr, lmb_size,
				      scn_addr))
			return nid;
	}

	BUG();	/* section address should be found above */
	return 0;
}

1075 1076 1077 1078 1079 1080 1081 1082
/*
 * Find the node associated with a hot added memory section.  Section
 * corresponds to a SPARSEMEM section, not an LMB.  It is assumed that
 * sections are fully contained within a single LMB.
 */
int hot_add_scn_to_nid(unsigned long scn_addr)
{
	struct device_node *memory = NULL;
1083
	int nid;
1084 1085

	if (!numa_enabled || (min_common_depth < 0))
1086 1087 1088 1089 1090 1091 1092 1093
		return any_online_node(NODE_MASK_ALL);

	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
	if (memory) {
		nid = hot_add_drconf_scn_to_nid(memory, scn_addr);
		of_node_put(memory);
		return nid;
	}
1094 1095 1096

	while ((memory = of_find_node_by_type(memory, "memory")) != NULL) {
		unsigned long start, size;
1097
		int ranges;
1098
		const unsigned int *memcell_buf;
1099 1100
		unsigned int len;

1101
		memcell_buf = of_get_property(memory, "reg", &len);
1102 1103 1104
		if (!memcell_buf || len <= 0)
			continue;

1105 1106
		/* ranges in cell */
		ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
1107 1108 1109
ha_new_range:
		start = read_n_cells(n_mem_addr_cells, &memcell_buf);
		size = read_n_cells(n_mem_size_cells, &memcell_buf);
1110
		nid = of_node_to_nid_single(memory);
1111

1112
		if (valid_hot_add_scn(&nid, start, size, scn_addr)) {
1113
			of_node_put(memory);
1114
			return nid;
1115 1116 1117 1118 1119 1120
		}

		if (--ranges)		/* process all ranges in cell */
			goto ha_new_range;
	}
	BUG();	/* section address should be found above */
1121
	return 0;
1122 1123
}
#endif /* CONFIG_MEMORY_HOTPLUG */