numa.c 19.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * pSeries NUMA support
 *
 * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */
#include <linux/threads.h>
#include <linux/bootmem.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/module.h>
#include <linux/nodemask.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
20
#include <asm/sparsemem.h>
L
Linus Torvalds 已提交
21
#include <asm/lmb.h>
22
#include <asm/system.h>
23
#include <asm/smp.h>
L
Linus Torvalds 已提交
24 25 26

static int numa_enabled = 1;

27 28
static char *cmdline __initdata;

L
Linus Torvalds 已提交
29 30 31
static int numa_debug;
#define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }

32
int numa_cpu_lookup_table[NR_CPUS];
L
Linus Torvalds 已提交
33 34
cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];
struct pglist_data *node_data[MAX_NUMNODES];
35 36 37 38 39 40

EXPORT_SYMBOL(numa_cpu_lookup_table);
EXPORT_SYMBOL(numa_cpumask_lookup_table);
EXPORT_SYMBOL(node_data);

static bootmem_data_t __initdata plat_node_bdata[MAX_NUMNODES];
L
Linus Torvalds 已提交
41
static int min_common_depth;
42
static int n_mem_addr_cells, n_mem_size_cells;
L
Linus Torvalds 已提交
43

44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
static int __cpuinit fake_numa_create_new_node(unsigned long end_pfn,
						unsigned int *nid)
{
	unsigned long long mem;
	char *p = cmdline;
	static unsigned int fake_nid = 0;
	static unsigned long long curr_boundary = 0;

	*nid = fake_nid;
	if (!p)
		return 0;

	mem = memparse(p, &p);
	if (!mem)
		return 0;

	if (mem < curr_boundary)
		return 0;

	curr_boundary = mem;

	if ((end_pfn << PAGE_SHIFT) > mem) {
		/*
		 * Skip commas and spaces
		 */
		while (*p == ',' || *p == ' ' || *p == '\t')
			p++;

		cmdline = p;
		fake_nid++;
		*nid = fake_nid;
		dbg("created new fake_node with id %d\n", fake_nid);
		return 1;
	}
	return 0;
}

81
static void __cpuinit map_cpu_to_node(int cpu, int node)
L
Linus Torvalds 已提交
82 83
{
	numa_cpu_lookup_table[cpu] = node;
84

85 86
	dbg("adding cpu %d to node %d\n", cpu, node);

87
	if (!(cpu_isset(cpu, numa_cpumask_lookup_table[node])))
L
Linus Torvalds 已提交
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
		cpu_set(cpu, numa_cpumask_lookup_table[node]);
}

#ifdef CONFIG_HOTPLUG_CPU
static void unmap_cpu_from_node(unsigned long cpu)
{
	int node = numa_cpu_lookup_table[cpu];

	dbg("removing cpu %lu from node %d\n", cpu, node);

	if (cpu_isset(cpu, numa_cpumask_lookup_table[node])) {
		cpu_clear(cpu, numa_cpumask_lookup_table[node]);
	} else {
		printk(KERN_ERR "WARNING: cpu %lu not found in node %d\n",
		       cpu, node);
	}
}
#endif /* CONFIG_HOTPLUG_CPU */

107
static struct device_node * __cpuinit find_cpu_node(unsigned int cpu)
L
Linus Torvalds 已提交
108 109 110
{
	unsigned int hw_cpuid = get_hard_smp_processor_id(cpu);
	struct device_node *cpu_node = NULL;
111
	const unsigned int *interrupt_server, *reg;
L
Linus Torvalds 已提交
112 113 114 115
	int len;

	while ((cpu_node = of_find_node_by_type(cpu_node, "cpu")) != NULL) {
		/* Try interrupt server first */
116
		interrupt_server = of_get_property(cpu_node,
L
Linus Torvalds 已提交
117 118 119 120 121 122 123 124 125 126
					"ibm,ppc-interrupt-server#s", &len);

		len = len / sizeof(u32);

		if (interrupt_server && (len > 0)) {
			while (len--) {
				if (interrupt_server[len] == hw_cpuid)
					return cpu_node;
			}
		} else {
127
			reg = of_get_property(cpu_node, "reg", &len);
L
Linus Torvalds 已提交
128 129 130 131 132 133 134 135 136
			if (reg && (len > 0) && (reg[0] == hw_cpuid))
				return cpu_node;
		}
	}

	return NULL;
}

/* must hold reference to node during call */
137
static const int *of_get_associativity(struct device_node *dev)
L
Linus Torvalds 已提交
138
{
139
	return of_get_property(dev, "ibm,associativity", NULL);
L
Linus Torvalds 已提交
140 141
}

142 143 144
/* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa
 * info is found.
 */
145
static int of_node_to_nid_single(struct device_node *device)
L
Linus Torvalds 已提交
146
{
147
	int nid = -1;
148
	const unsigned int *tmp;
L
Linus Torvalds 已提交
149 150

	if (min_common_depth == -1)
151
		goto out;
L
Linus Torvalds 已提交
152 153

	tmp = of_get_associativity(device);
154 155 156 157
	if (!tmp)
		goto out;

	if (tmp[0] >= min_common_depth)
158
		nid = tmp[min_common_depth];
159 160

	/* POWER4 LPAR uses 0xffff as invalid node */
161 162 163
	if (nid == 0xffff || nid >= MAX_NUMNODES)
		nid = -1;
out:
164
	return nid;
L
Linus Torvalds 已提交
165 166
}

167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
/* Walk the device tree upwards, looking for an associativity id */
int of_node_to_nid(struct device_node *device)
{
	struct device_node *tmp;
	int nid = -1;

	of_node_get(device);
	while (device) {
		nid = of_node_to_nid_single(device);
		if (nid != -1)
			break;

	        tmp = device;
		device = of_get_parent(tmp);
		of_node_put(tmp);
	}
	of_node_put(device);

	return nid;
}
EXPORT_SYMBOL_GPL(of_node_to_nid);

L
Linus Torvalds 已提交
189 190 191 192 193 194 195
/*
 * In theory, the "ibm,associativity" property may contain multiple
 * associativity lists because a resource may be multiply connected
 * into the machine.  This resource then has different associativity
 * characteristics relative to its multiple connections.  We ignore
 * this for now.  We also assume that all cpu and memory sets have
 * their distances represented at a common level.  This won't be
196
 * true for hierarchical NUMA.
L
Linus Torvalds 已提交
197 198 199 200 201 202 203 204 205
 *
 * In any case the ibm,associativity-reference-points should give
 * the correct depth for a normal NUMA system.
 *
 * - Dave Hansen <haveblue@us.ibm.com>
 */
static int __init find_min_common_depth(void)
{
	int depth;
206
	const unsigned int *ref_points;
L
Linus Torvalds 已提交
207 208 209 210 211 212 213 214 215 216 217 218 219 220
	struct device_node *rtas_root;
	unsigned int len;

	rtas_root = of_find_node_by_path("/rtas");

	if (!rtas_root)
		return -1;

	/*
	 * this property is 2 32-bit integers, each representing a level of
	 * depth in the associativity nodes.  The first is for an SMP
	 * configuration (should be all 0's) and the second is for a normal
	 * NUMA configuration.
	 */
221
	ref_points = of_get_property(rtas_root,
L
Linus Torvalds 已提交
222 223 224 225 226
			"ibm,associativity-reference-points", &len);

	if ((len >= 1) && ref_points) {
		depth = ref_points[1];
	} else {
227
		dbg("NUMA: ibm,associativity-reference-points not found.\n");
L
Linus Torvalds 已提交
228 229 230 231 232 233 234
		depth = -1;
	}
	of_node_put(rtas_root);

	return depth;
}

235
static void __init get_n_mem_cells(int *n_addr_cells, int *n_size_cells)
L
Linus Torvalds 已提交
236 237 238 239
{
	struct device_node *memory = NULL;

	memory = of_find_node_by_type(memory, "memory");
240
	if (!memory)
241
		panic("numa.c: No memory nodes found!");
242

243
	*n_addr_cells = of_n_addr_cells(memory);
244
	*n_size_cells = of_n_size_cells(memory);
245
	of_node_put(memory);
L
Linus Torvalds 已提交
246 247
}

248
static unsigned long __devinit read_n_cells(int n, const unsigned int **buf)
L
Linus Torvalds 已提交
249 250 251 252 253 254 255 256 257 258 259 260 261 262
{
	unsigned long result = 0;

	while (n--) {
		result = (result << 32) | **buf;
		(*buf)++;
	}
	return result;
}

/*
 * Figure out to which domain a cpu belongs and stick it there.
 * Return the id of the domain used.
 */
263
static int __cpuinit numa_setup_cpu(unsigned long lcpu)
L
Linus Torvalds 已提交
264
{
265
	int nid = 0;
L
Linus Torvalds 已提交
266 267 268 269 270 271 272
	struct device_node *cpu = find_cpu_node(lcpu);

	if (!cpu) {
		WARN_ON(1);
		goto out;
	}

273
	nid = of_node_to_nid_single(cpu);
L
Linus Torvalds 已提交
274

275 276
	if (nid < 0 || !node_online(nid))
		nid = any_online_node(NODE_MASK_ALL);
L
Linus Torvalds 已提交
277
out:
278
	map_cpu_to_node(lcpu, nid);
L
Linus Torvalds 已提交
279 280 281

	of_node_put(cpu);

282
	return nid;
L
Linus Torvalds 已提交
283 284
}

285
static int __cpuinit cpu_numa_callback(struct notifier_block *nfb,
L
Linus Torvalds 已提交
286 287 288 289 290 291 292 293
			     unsigned long action,
			     void *hcpu)
{
	unsigned long lcpu = (unsigned long)hcpu;
	int ret = NOTIFY_DONE;

	switch (action) {
	case CPU_UP_PREPARE:
294
	case CPU_UP_PREPARE_FROZEN:
295
		numa_setup_cpu(lcpu);
L
Linus Torvalds 已提交
296 297 298 299
		ret = NOTIFY_OK;
		break;
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_DEAD:
300
	case CPU_DEAD_FROZEN:
L
Linus Torvalds 已提交
301
	case CPU_UP_CANCELED:
302
	case CPU_UP_CANCELED_FROZEN:
L
Linus Torvalds 已提交
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
		unmap_cpu_from_node(lcpu);
		break;
		ret = NOTIFY_OK;
#endif
	}
	return ret;
}

/*
 * Check and possibly modify a memory region to enforce the memory limit.
 *
 * Returns the size the region should have to enforce the memory limit.
 * This will either be the original value of size, a truncated value,
 * or zero. If the returned value of size is 0 the region should be
 * discarded as it lies wholy above the memory limit.
 */
319 320
static unsigned long __init numa_enforce_memory_limit(unsigned long start,
						      unsigned long size)
L
Linus Torvalds 已提交
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
{
	/*
	 * We use lmb_end_of_DRAM() in here instead of memory_limit because
	 * we've already adjusted it for the limit and it takes care of
	 * having memory holes below the limit.
	 */

	if (! memory_limit)
		return size;

	if (start + size <= lmb_end_of_DRAM())
		return size;

	if (start >= lmb_end_of_DRAM())
		return 0;

	return lmb_end_of_DRAM() - start;
}

340 341 342 343 344 345 346 347 348
/*
 * Extract NUMA information from the ibm,dynamic-reconfiguration-memory
 * node.  This assumes n_mem_{addr,size}_cells have been set.
 */
static void __init parse_drconf_memory(struct device_node *memory)
{
	const unsigned int *lm, *dm, *aa;
	unsigned int ls, ld, la;
	unsigned int n, aam, aalen;
349
	unsigned long lmb_size, size, start;
350
	int nid, default_nid = 0;
351
	unsigned int ai, flags;
352

353 354 355
	lm = of_get_property(memory, "ibm,lmb-size", &ls);
	dm = of_get_property(memory, "ibm,dynamic-memory", &ld);
	aa = of_get_property(memory, "ibm,associativity-lookup-arrays", &la);
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390
	if (!lm || !dm || !aa ||
	    ls < sizeof(unsigned int) || ld < sizeof(unsigned int) ||
	    la < 2 * sizeof(unsigned int))
		return;

	lmb_size = read_n_cells(n_mem_size_cells, &lm);
	n = *dm++;		/* number of LMBs */
	aam = *aa++;		/* number of associativity lists */
	aalen = *aa++;		/* length of each associativity list */
	if (ld < (n * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int) ||
	    la < (aam * aalen + 2) * sizeof(unsigned int))
		return;

	for (; n != 0; --n) {
		start = read_n_cells(n_mem_addr_cells, &dm);
		ai = dm[2];
		flags = dm[3];
		dm += 4;
		/* 0x80 == reserved, 0x8 = assigned to us */
		if ((flags & 0x80) || !(flags & 0x8))
			continue;
		nid = default_nid;
		/* flags & 0x40 means associativity index is invalid */
		if (min_common_depth > 0 && min_common_depth <= aalen &&
		    (flags & 0x40) == 0 && ai < aam) {
			/* this is like of_node_to_nid_single */
			nid = aa[ai * aalen + min_common_depth - 1];
			if (nid == 0xffff || nid >= MAX_NUMNODES)
				nid = default_nid;
		}

		size = numa_enforce_memory_limit(start, lmb_size);
		if (!size)
			continue;

391 392 393
		fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid);
		node_set_online(nid);

394 395 396 397 398
		add_active_range(nid, start >> PAGE_SHIFT,
				 (start >> PAGE_SHIFT) + (size >> PAGE_SHIFT));
	}
}

L
Linus Torvalds 已提交
399 400 401 402
static int __init parse_numa_properties(void)
{
	struct device_node *cpu = NULL;
	struct device_node *memory = NULL;
403
	int default_nid = 0;
L
Linus Torvalds 已提交
404 405 406 407 408 409 410 411 412 413 414 415
	unsigned long i;

	if (numa_enabled == 0) {
		printk(KERN_WARNING "NUMA disabled by user\n");
		return -1;
	}

	min_common_depth = find_min_common_depth();

	if (min_common_depth < 0)
		return min_common_depth;

416 417
	dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth);

L
Linus Torvalds 已提交
418
	/*
419 420 421
	 * Even though we connect cpus to numa domains later in SMP
	 * init, we need to know the node ids now. This is because
	 * each node to be onlined must have NODE_DATA etc backing it.
L
Linus Torvalds 已提交
422
	 */
423
	for_each_present_cpu(i) {
424
		int nid;
L
Linus Torvalds 已提交
425 426

		cpu = find_cpu_node(i);
427
		BUG_ON(!cpu);
428
		nid = of_node_to_nid_single(cpu);
429
		of_node_put(cpu);
L
Linus Torvalds 已提交
430

431 432 433 434 435 436 437 438
		/*
		 * Don't fall back to default_nid yet -- we will plug
		 * cpus into nodes once the memory scan has discovered
		 * the topology.
		 */
		if (nid < 0)
			continue;
		node_set_online(nid);
L
Linus Torvalds 已提交
439 440
	}

441
	get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells);
L
Linus Torvalds 已提交
442 443 444 445
	memory = NULL;
	while ((memory = of_find_node_by_type(memory, "memory")) != NULL) {
		unsigned long start;
		unsigned long size;
446
		int nid;
L
Linus Torvalds 已提交
447
		int ranges;
448
		const unsigned int *memcell_buf;
L
Linus Torvalds 已提交
449 450
		unsigned int len;

451
		memcell_buf = of_get_property(memory,
452 453
			"linux,usable-memory", &len);
		if (!memcell_buf || len <= 0)
454
			memcell_buf = of_get_property(memory, "reg", &len);
L
Linus Torvalds 已提交
455 456 457
		if (!memcell_buf || len <= 0)
			continue;

458 459
		/* ranges in cell */
		ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
L
Linus Torvalds 已提交
460 461
new_range:
		/* these are order-sensitive, and modify the buffer pointer */
462 463
		start = read_n_cells(n_mem_addr_cells, &memcell_buf);
		size = read_n_cells(n_mem_size_cells, &memcell_buf);
L
Linus Torvalds 已提交
464

465 466 467 468 469
		/*
		 * Assumption: either all memory nodes or none will
		 * have associativity properties.  If none, then
		 * everything goes to default_nid.
		 */
470
		nid = of_node_to_nid_single(memory);
471 472
		if (nid < 0)
			nid = default_nid;
L
Linus Torvalds 已提交
473

474
		if (!(size = numa_enforce_memory_limit(start, size))) {
L
Linus Torvalds 已提交
475 476 477 478 479 480
			if (--ranges)
				goto new_range;
			else
				continue;
		}

481 482 483
		fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid);
		node_set_online(nid);

484 485
		add_active_range(nid, start >> PAGE_SHIFT,
				(start >> PAGE_SHIFT) + (size >> PAGE_SHIFT));
L
Linus Torvalds 已提交
486 487 488 489 490

		if (--ranges)
			goto new_range;
	}

491 492 493 494 495 496 497 498
	/*
	 * Now do the same thing for each LMB listed in the ibm,dynamic-memory
	 * property in the ibm,dynamic-reconfiguration-memory node.
	 */
	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
	if (memory)
		parse_drconf_memory(memory);

L
Linus Torvalds 已提交
499 500 501 502 503 504 505
	return 0;
}

static void __init setup_nonnuma(void)
{
	unsigned long top_of_ram = lmb_end_of_DRAM();
	unsigned long total_ram = lmb_phys_mem_size();
506
	unsigned long start_pfn, end_pfn;
507
	unsigned int i, nid = 0;
L
Linus Torvalds 已提交
508

509
	printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
L
Linus Torvalds 已提交
510
	       top_of_ram, total_ram);
511
	printk(KERN_DEBUG "Memory hole size: %ldMB\n",
L
Linus Torvalds 已提交
512 513
	       (top_of_ram - total_ram) >> 20);

514 515 516
	for (i = 0; i < lmb.memory.cnt; ++i) {
		start_pfn = lmb.memory.region[i].base >> PAGE_SHIFT;
		end_pfn = start_pfn + lmb_size_pages(&lmb.memory, i);
517 518 519 520

		fake_numa_create_new_node(end_pfn, &nid);
		add_active_range(nid, start_pfn, end_pfn);
		node_set_online(nid);
521
	}
L
Linus Torvalds 已提交
522 523
}

524 525 526 527 528 529 530 531 532
void __init dump_numa_cpu_topology(void)
{
	unsigned int node;
	unsigned int cpu, count;

	if (min_common_depth == -1 || !numa_enabled)
		return;

	for_each_online_node(node) {
533
		printk(KERN_DEBUG "Node %d CPUs:", node);
534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558

		count = 0;
		/*
		 * If we used a CPU iterator here we would miss printing
		 * the holes in the cpumap.
		 */
		for (cpu = 0; cpu < NR_CPUS; cpu++) {
			if (cpu_isset(cpu, numa_cpumask_lookup_table[node])) {
				if (count == 0)
					printk(" %u", cpu);
				++count;
			} else {
				if (count > 1)
					printk("-%u", cpu - 1);
				count = 0;
			}
		}

		if (count > 1)
			printk("-%u", NR_CPUS - 1);
		printk("\n");
	}
}

static void __init dump_numa_memory_topology(void)
L
Linus Torvalds 已提交
559 560 561 562 563 564 565 566 567 568
{
	unsigned int node;
	unsigned int count;

	if (min_common_depth == -1 || !numa_enabled)
		return;

	for_each_online_node(node) {
		unsigned long i;

569
		printk(KERN_DEBUG "Node %d Memory:", node);
L
Linus Torvalds 已提交
570 571 572

		count = 0;

573 574 575
		for (i = 0; i < lmb_end_of_DRAM();
		     i += (1 << SECTION_SIZE_BITS)) {
			if (early_pfn_to_nid(i >> PAGE_SHIFT) == node) {
L
Linus Torvalds 已提交
576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598
				if (count == 0)
					printk(" 0x%lx", i);
				++count;
			} else {
				if (count > 0)
					printk("-0x%lx", i);
				count = 0;
			}
		}

		if (count > 0)
			printk("-0x%lx", i);
		printk("\n");
	}
}

/*
 * Allocate some memory, satisfying the lmb or bootmem allocator where
 * required. nid is the preferred node and end is the physical address of
 * the highest address in the node.
 *
 * Returns the physical address of the memory.
 */
599 600 601
static void __init *careful_allocation(int nid, unsigned long size,
				       unsigned long align,
				       unsigned long end_pfn)
L
Linus Torvalds 已提交
602
{
603
	int new_nid;
604
	unsigned long ret = __lmb_alloc_base(size, align, end_pfn << PAGE_SHIFT);
L
Linus Torvalds 已提交
605 606 607

	/* retry over all memory */
	if (!ret)
608
		ret = __lmb_alloc_base(size, align, lmb_end_of_DRAM());
L
Linus Torvalds 已提交
609 610 611 612 613 614 615 616 617

	if (!ret)
		panic("numa.c: cannot allocate %lu bytes on node %d",
		      size, nid);

	/*
	 * If the memory came from a previously allocated node, we must
	 * retry with the bootmem allocator.
	 */
618 619 620
	new_nid = early_pfn_to_nid(ret >> PAGE_SHIFT);
	if (new_nid < nid) {
		ret = (unsigned long)__alloc_bootmem_node(NODE_DATA(new_nid),
L
Linus Torvalds 已提交
621 622 623 624
				size, align, 0);

		if (!ret)
			panic("numa.c: cannot allocate %lu bytes on node %d",
625
			      size, new_nid);
L
Linus Torvalds 已提交
626

627
		ret = __pa(ret);
L
Linus Torvalds 已提交
628 629 630 631

		dbg("alloc_bootmem %lx %lx\n", ret, size);
	}

632
	return (void *)ret;
L
Linus Torvalds 已提交
633 634
}

635 636 637 638 639
static struct notifier_block __cpuinitdata ppc64_numa_nb = {
	.notifier_call = cpu_numa_callback,
	.priority = 1 /* Must run before sched domains notifier. */
};

L
Linus Torvalds 已提交
640 641 642
void __init do_init_bootmem(void)
{
	int nid;
643
	unsigned int i;
L
Linus Torvalds 已提交
644 645 646 647 648 649 650 651

	min_low_pfn = 0;
	max_low_pfn = lmb_end_of_DRAM() >> PAGE_SHIFT;
	max_pfn = max_low_pfn;

	if (parse_numa_properties())
		setup_nonnuma();
	else
652
		dump_numa_memory_topology();
L
Linus Torvalds 已提交
653 654

	register_cpu_notifier(&ppc64_numa_nb);
655 656
	cpu_numa_callback(&ppc64_numa_nb, CPU_UP_PREPARE,
			  (void *)(unsigned long)boot_cpuid);
L
Linus Torvalds 已提交
657 658

	for_each_online_node(nid) {
659
		unsigned long start_pfn, end_pfn;
L
Linus Torvalds 已提交
660 661 662
		unsigned long bootmem_paddr;
		unsigned long bootmap_pages;

663
		get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
L
Linus Torvalds 已提交
664 665

		/* Allocate the node structure node local if possible */
666
		NODE_DATA(nid) = careful_allocation(nid,
L
Linus Torvalds 已提交
667
					sizeof(struct pglist_data),
668 669
					SMP_CACHE_BYTES, end_pfn);
		NODE_DATA(nid) = __va(NODE_DATA(nid));
L
Linus Torvalds 已提交
670 671 672 673 674 675
		memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));

  		dbg("node %d\n", nid);
		dbg("NODE_DATA() = %p\n", NODE_DATA(nid));

		NODE_DATA(nid)->bdata = &plat_node_bdata[nid];
676 677
		NODE_DATA(nid)->node_start_pfn = start_pfn;
		NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
L
Linus Torvalds 已提交
678 679 680 681

		if (NODE_DATA(nid)->node_spanned_pages == 0)
  			continue;

682 683
  		dbg("start_paddr = %lx\n", start_pfn << PAGE_SHIFT);
  		dbg("end_paddr = %lx\n", end_pfn << PAGE_SHIFT);
L
Linus Torvalds 已提交
684

685 686 687 688 689
		bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn);
		bootmem_paddr = (unsigned long)careful_allocation(nid,
					bootmap_pages << PAGE_SHIFT,
					PAGE_SIZE, end_pfn);
		memset(__va(bootmem_paddr), 0, bootmap_pages << PAGE_SHIFT);
L
Linus Torvalds 已提交
690 691 692 693

		dbg("bootmap_paddr = %lx\n", bootmem_paddr);

		init_bootmem_node(NODE_DATA(nid), bootmem_paddr >> PAGE_SHIFT,
694
				  start_pfn, end_pfn);
L
Linus Torvalds 已提交
695

696
		free_bootmem_with_active_regions(nid, end_pfn);
L
Linus Torvalds 已提交
697

698
		/* Mark reserved regions on this node */
L
Linus Torvalds 已提交
699
		for (i = 0; i < lmb.reserved.cnt; i++) {
700
			unsigned long physbase = lmb.reserved.region[i].base;
L
Linus Torvalds 已提交
701
			unsigned long size = lmb.reserved.region[i].size;
702 703
			unsigned long start_paddr = start_pfn << PAGE_SHIFT;
			unsigned long end_paddr = end_pfn << PAGE_SHIFT;
L
Linus Torvalds 已提交
704

705 706
			if (early_pfn_to_nid(physbase >> PAGE_SHIFT) != nid &&
			    early_pfn_to_nid((physbase+size-1) >> PAGE_SHIFT) != nid)
L
Linus Torvalds 已提交
707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
				continue;

			if (physbase < end_paddr &&
			    (physbase+size) > start_paddr) {
				/* overlaps */
				if (physbase < start_paddr) {
					size -= start_paddr - physbase;
					physbase = start_paddr;
				}

				if (size > end_paddr - physbase)
					size = end_paddr - physbase;

				dbg("reserve_bootmem %lx %lx\n", physbase,
				    size);
				reserve_bootmem_node(NODE_DATA(nid), physbase,
						     size);
			}
		}
B
Bob Picco 已提交
726

727
		sparse_memory_present_with_active_regions(nid);
L
Linus Torvalds 已提交
728 729 730 731 732
	}
}

void __init paging_init(void)
{
733 734 735
	unsigned long max_zone_pfns[MAX_NR_ZONES];
	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
	max_zone_pfns[ZONE_DMA] = lmb_end_of_DRAM() >> PAGE_SHIFT;
736
	free_area_init_nodes(max_zone_pfns);
L
Linus Torvalds 已提交
737 738 739 740 741 742 743 744 745 746 747 748 749
}

static int __init early_numa(char *p)
{
	if (!p)
		return 0;

	if (strstr(p, "off"))
		numa_enabled = 0;

	if (strstr(p, "debug"))
		numa_debug = 1;

750 751 752 753
	p = strstr(p, "fake=");
	if (p)
		cmdline = p + strlen("fake=");

L
Linus Torvalds 已提交
754 755 756
	return 0;
}
early_param("numa", early_numa);
757 758 759 760 761 762 763 764 765 766

#ifdef CONFIG_MEMORY_HOTPLUG
/*
 * Find the node associated with a hot added memory section.  Section
 * corresponds to a SPARSEMEM section, not an LMB.  It is assumed that
 * sections are fully contained within a single LMB.
 */
int hot_add_scn_to_nid(unsigned long scn_addr)
{
	struct device_node *memory = NULL;
767
	nodemask_t nodes;
768
	int default_nid = any_online_node(NODE_MASK_ALL);
769
	int nid;
770 771

	if (!numa_enabled || (min_common_depth < 0))
772
		return default_nid;
773 774 775

	while ((memory = of_find_node_by_type(memory, "memory")) != NULL) {
		unsigned long start, size;
776
		int ranges;
777
		const unsigned int *memcell_buf;
778 779
		unsigned int len;

780
		memcell_buf = of_get_property(memory, "reg", &len);
781 782 783
		if (!memcell_buf || len <= 0)
			continue;

784 785
		/* ranges in cell */
		ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells);
786 787 788
ha_new_range:
		start = read_n_cells(n_mem_addr_cells, &memcell_buf);
		size = read_n_cells(n_mem_size_cells, &memcell_buf);
789
		nid = of_node_to_nid_single(memory);
790 791

		/* Domains not present at boot default to 0 */
792 793
		if (nid < 0 || !node_online(nid))
			nid = default_nid;
794 795 796

		if ((scn_addr >= start) && (scn_addr < (start + size))) {
			of_node_put(memory);
797
			goto got_nid;
798 799 800 801 802 803
		}

		if (--ranges)		/* process all ranges in cell */
			goto ha_new_range;
	}
	BUG();	/* section address should be found above */
804
	return 0;
805 806

	/* Temporary code to ensure that returned node is not empty */
807
got_nid:
808
	nodes_setall(nodes);
809 810 811
	while (NODE_DATA(nid)->node_spanned_pages == 0) {
		node_clear(nid, nodes);
		nid = any_online_node(nodes);
812
	}
813
	return nid;
814 815
}
#endif /* CONFIG_MEMORY_HOTPLUG */
新手
引导
客服 返回
顶部