bootmem.c 15.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11
/*
 *  linux/mm/bootmem.c
 *
 *  Copyright (C) 1999 Ingo Molnar
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *
 *  simple boot-time physical memory area allocator and
 *  free memory collector. It's used to deal with reserved
 *  system memory and memory holes as well.
 */
#include <linux/init.h>
12
#include <linux/pfn.h>
L
Linus Torvalds 已提交
13 14
#include <linux/bootmem.h>
#include <linux/module.h>
15 16

#include <asm/bug.h>
L
Linus Torvalds 已提交
17
#include <asm/io.h>
18
#include <asm/processor.h>
19

L
Linus Torvalds 已提交
20 21 22 23 24 25 26 27 28 29
#include "internal.h"

/*
 * Access to this subsystem has to be serialized externally. (this is
 * true for the boot process anyway)
 */
unsigned long max_low_pfn;
unsigned long min_low_pfn;
unsigned long max_pfn;

30
static LIST_HEAD(bdata_list);
31 32 33 34 35 36 37 38
#ifdef CONFIG_CRASH_DUMP
/*
 * If we have booted due to a crash, max_pfn will be a very low value. We need
 * to know the amount of memory that the previous kernel used.
 */
unsigned long saved_max_pfn;
#endif

39 40
bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;

L
Linus Torvalds 已提交
41
/* return the number of _pages_ that will be allocated for the boot bitmap */
42
unsigned long __init bootmem_bootmap_pages(unsigned long pages)
L
Linus Torvalds 已提交
43 44 45 46 47 48 49 50 51
{
	unsigned long mapsize;

	mapsize = (pages+7)/8;
	mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK;
	mapsize >>= PAGE_SHIFT;

	return mapsize;
}
52

53 54 55
/*
 * link bdata in order
 */
56
static void __init link_bootmem(bootmem_data_t *bdata)
57 58
{
	bootmem_data_t *ent;
59

60 61 62 63 64 65 66 67 68 69 70 71 72 73
	if (list_empty(&bdata_list)) {
		list_add(&bdata->list, &bdata_list);
		return;
	}
	/* insert in order */
	list_for_each_entry(ent, &bdata_list, list) {
		if (bdata->node_boot_start < ent->node_boot_start) {
			list_add_tail(&bdata->list, &ent->list);
			return;
		}
	}
	list_add_tail(&bdata->list, &bdata_list);
}

74 75 76 77 78 79 80 81 82 83 84 85
/*
 * Given an initialised bdata, it returns the size of the boot bitmap
 */
static unsigned long __init get_mapsize(bootmem_data_t *bdata)
{
	unsigned long mapsize;
	unsigned long start = PFN_DOWN(bdata->node_boot_start);
	unsigned long end = bdata->node_low_pfn;

	mapsize = ((end - start) + 7) / 8;
	return ALIGN(mapsize, sizeof(long));
}
L
Linus Torvalds 已提交
86 87 88 89

/*
 * Called once to set up the allocator itself.
 */
90
static unsigned long __init init_bootmem_core(pg_data_t *pgdat,
L
Linus Torvalds 已提交
91 92 93
	unsigned long mapstart, unsigned long start, unsigned long end)
{
	bootmem_data_t *bdata = pgdat->bdata;
94
	unsigned long mapsize;
L
Linus Torvalds 已提交
95

96
	mminit_validate_memmodel_limits(&start, &end);
97 98
	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
	bdata->node_boot_start = PFN_PHYS(start);
L
Linus Torvalds 已提交
99
	bdata->node_low_pfn = end;
100
	link_bootmem(bdata);
L
Linus Torvalds 已提交
101 102 103 104 105

	/*
	 * Initially all pages are reserved - setup_arch() has to
	 * register free RAM areas explicitly.
	 */
106
	mapsize = get_mapsize(bdata);
L
Linus Torvalds 已提交
107 108 109 110 111 112 113 114 115 116
	memset(bdata->node_bootmem_map, 0xff, mapsize);

	return mapsize;
}

/*
 * Marks a particular physical memory range as unallocatable. Usable RAM
 * might be used for boot-time allocations - or it might get added
 * to the free page pool later on.
 */
117
static int __init can_reserve_bootmem_core(bootmem_data_t *bdata,
118
			unsigned long addr, unsigned long size, int flags)
L
Linus Torvalds 已提交
119
{
120
	unsigned long sidx, eidx;
L
Linus Torvalds 已提交
121
	unsigned long i;
122 123 124 125 126 127 128

	BUG_ON(!size);

	/* out of range, don't hold other */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return 0;
129

L
Linus Torvalds 已提交
130
	/*
131
	 * Round up to index to the range.
L
Linus Torvalds 已提交
132
	 */
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

	eidx = PFN_UP(addr + size - bdata->node_boot_start);
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);

	for (i = sidx; i < eidx; i++) {
		if (test_bit(i, bdata->node_bootmem_map)) {
			if (flags & BOOTMEM_EXCLUSIVE)
				return -EBUSY;
		}
	}

	return 0;

}

static void __init reserve_bootmem_core(bootmem_data_t *bdata,
			unsigned long addr, unsigned long size, int flags)
{
	unsigned long sidx, eidx;
	unsigned long i;

L
Linus Torvalds 已提交
159
	BUG_ON(!size);
160

161 162 163 164 165 166 167 168 169 170 171 172 173
	/* out of range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;

	/*
	 * Round up to index to the range.
	 */
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

174
	eidx = PFN_UP(addr + size - bdata->node_boot_start);
175 176
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
L
Linus Torvalds 已提交
177

178
	for (i = sidx; i < eidx; i++) {
L
Linus Torvalds 已提交
179 180 181 182 183
		if (test_and_set_bit(i, bdata->node_bootmem_map)) {
#ifdef CONFIG_DEBUG_BOOTMEM
			printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE);
#endif
		}
184
	}
L
Linus Torvalds 已提交
185 186
}

187 188
static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
				     unsigned long size)
L
Linus Torvalds 已提交
189
{
190
	unsigned long sidx, eidx;
L
Linus Torvalds 已提交
191
	unsigned long i;
192

193 194 195 196 197 198
	BUG_ON(!size);

	/* out range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;
L
Linus Torvalds 已提交
199 200 201 202 203
	/*
	 * round down end of usable mem, partially free pages are
	 * considered reserved.
	 */

204
	if (addr >= bdata->node_boot_start && addr < bdata->last_success)
L
Linus Torvalds 已提交
205 206 207
		bdata->last_success = addr;

	/*
208
	 * Round up to index to the range.
L
Linus Torvalds 已提交
209
	 */
210 211 212 213 214
	if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start))
		sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start);
	else
		sidx = 0;

215
	eidx = PFN_DOWN(addr + size - bdata->node_boot_start);
216 217
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
L
Linus Torvalds 已提交
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237

	for (i = sidx; i < eidx; i++) {
		if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map)))
			BUG();
	}
}

/*
 * We 'merge' subsequent allocations to save space. We might 'lose'
 * some fraction of a page if allocations cannot be satisfied due to
 * size constraints on boxes where there is physical RAM space
 * fragmentation - in these cases (mostly large memory boxes) this
 * is not a problem.
 *
 * On low memory boxes we get it right in 100% of the cases.
 *
 * alignment has to be a power of 2 value.
 *
 * NOTE:  This function is _not_ reentrant.
 */
238
void * __init
L
Linus Torvalds 已提交
239
__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
240
	      unsigned long align, unsigned long goal, unsigned long limit)
L
Linus Torvalds 已提交
241
{
Y
Yinghai Lu 已提交
242
	unsigned long areasize, preferred;
243
	unsigned long i, start = 0, incr, eidx, end_pfn;
L
Linus Torvalds 已提交
244
	void *ret;
Y
Yinghai Lu 已提交
245 246
	unsigned long node_boot_start;
	void *node_bootmem_map;
L
Linus Torvalds 已提交
247

248
	if (!size) {
L
Linus Torvalds 已提交
249 250 251 252 253
		printk("__alloc_bootmem_core(): zero-sized request\n");
		BUG();
	}
	BUG_ON(align & (align-1));

254 255 256 257
	/* on nodes without memory - bootmem_map is NULL */
	if (!bdata->node_bootmem_map)
		return NULL;

Y
Yinghai Lu 已提交
258 259 260 261 262 263 264 265 266 267 268 269 270
	/* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */
	node_boot_start = bdata->node_boot_start;
	node_bootmem_map = bdata->node_bootmem_map;
	if (align) {
		node_boot_start = ALIGN(bdata->node_boot_start, align);
		if (node_boot_start > bdata->node_boot_start)
			node_bootmem_map = (unsigned long *)bdata->node_bootmem_map +
			    PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG;
	}

	if (limit && node_boot_start >= limit)
		return NULL;

271 272
	end_pfn = bdata->node_low_pfn;
	limit = PFN_DOWN(limit);
273 274 275
	if (limit && end_pfn > limit)
		end_pfn = limit;

Y
Yinghai Lu 已提交
276
	eidx = end_pfn - PFN_DOWN(node_boot_start);
L
Linus Torvalds 已提交
277 278 279 280 281

	/*
	 * We try to allocate bootmem pages above 'goal'
	 * first, then we try to allocate lower pages.
	 */
282 283
	preferred = 0;
	if (goal && PFN_DOWN(goal) < end_pfn) {
Y
Yinghai Lu 已提交
284 285
		if (goal > node_boot_start)
			preferred = goal - node_boot_start;
L
Linus Torvalds 已提交
286

Y
Yinghai Lu 已提交
287 288
		if (bdata->last_success > node_boot_start &&
			bdata->last_success - node_boot_start >= preferred)
289
			if (!limit || (limit && limit > bdata->last_success))
Y
Yinghai Lu 已提交
290
				preferred = bdata->last_success - node_boot_start;
291
	}
L
Linus Torvalds 已提交
292

Y
Yinghai Lu 已提交
293
	preferred = PFN_DOWN(ALIGN(preferred, align));
294
	areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
L
Linus Torvalds 已提交
295 296 297
	incr = align >> PAGE_SHIFT ? : 1;

restart_scan:
298
	for (i = preferred; i < eidx;) {
L
Linus Torvalds 已提交
299
		unsigned long j;
300

Y
Yinghai Lu 已提交
301
		i = find_next_zero_bit(node_bootmem_map, eidx, i);
L
Linus Torvalds 已提交
302
		i = ALIGN(i, incr);
303 304
		if (i >= eidx)
			break;
Y
Yinghai Lu 已提交
305
		if (test_bit(i, node_bootmem_map)) {
306
			i += incr;
L
Linus Torvalds 已提交
307
			continue;
308
		}
L
Linus Torvalds 已提交
309 310 311
		for (j = i + 1; j < i + areasize; ++j) {
			if (j >= eidx)
				goto fail_block;
Y
Yinghai Lu 已提交
312
			if (test_bit(j, node_bootmem_map))
L
Linus Torvalds 已提交
313 314 315 316 317 318
				goto fail_block;
		}
		start = i;
		goto found;
	fail_block:
		i = ALIGN(j, incr);
319 320
		if (i == j)
			i += incr;
L
Linus Torvalds 已提交
321 322
	}

Y
Yinghai Lu 已提交
323 324
	if (preferred > 0) {
		preferred = 0;
L
Linus Torvalds 已提交
325 326 327 328 329
		goto restart_scan;
	}
	return NULL;

found:
Y
Yinghai Lu 已提交
330
	bdata->last_success = PFN_PHYS(start) + node_boot_start;
L
Linus Torvalds 已提交
331 332 333 334 335 336 337 338 339
	BUG_ON(start >= eidx);

	/*
	 * Is the next page of the previous allocation-end the start
	 * of this allocation's buffer? If yes then we can 'merge'
	 * the previous partial page with this allocation.
	 */
	if (align < PAGE_SIZE &&
	    bdata->last_offset && bdata->last_pos+1 == start) {
Y
Yinghai Lu 已提交
340
		unsigned long offset, remaining_size;
341
		offset = ALIGN(bdata->last_offset, align);
L
Linus Torvalds 已提交
342
		BUG_ON(offset > PAGE_SIZE);
343
		remaining_size = PAGE_SIZE - offset;
L
Linus Torvalds 已提交
344 345 346
		if (size < remaining_size) {
			areasize = 0;
			/* last_pos unchanged */
347 348
			bdata->last_offset = offset + size;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
Y
Yinghai Lu 已提交
349
					   offset + node_boot_start);
L
Linus Torvalds 已提交
350 351
		} else {
			remaining_size = size - remaining_size;
352 353
			areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
Y
Yinghai Lu 已提交
354
					   offset + node_boot_start);
355
			bdata->last_pos = start + areasize - 1;
L
Linus Torvalds 已提交
356 357 358 359 360 361
			bdata->last_offset = remaining_size;
		}
		bdata->last_offset &= ~PAGE_MASK;
	} else {
		bdata->last_pos = start + areasize - 1;
		bdata->last_offset = size & ~PAGE_MASK;
Y
Yinghai Lu 已提交
362
		ret = phys_to_virt(start * PAGE_SIZE + node_boot_start);
L
Linus Torvalds 已提交
363 364 365 366 367
	}

	/*
	 * Reserve the area now:
	 */
368
	for (i = start; i < start + areasize; i++)
Y
Yinghai Lu 已提交
369
		if (unlikely(test_and_set_bit(i, node_bootmem_map)))
L
Linus Torvalds 已提交
370 371 372 373 374 375 376 377
			BUG();
	memset(ret, 0, size);
	return ret;
}

static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
{
	struct page *page;
A
Andy Whitcroft 已提交
378
	unsigned long pfn;
L
Linus Torvalds 已提交
379
	bootmem_data_t *bdata = pgdat->bdata;
380
	unsigned long i, count;
L
Linus Torvalds 已提交
381 382 383 384 385 386 387 388
	unsigned long idx;
	unsigned long *map; 
	int gofast = 0;

	BUG_ON(!bdata->node_bootmem_map);

	count = 0;
	/* first extant page of the node */
389 390
	pfn = PFN_DOWN(bdata->node_boot_start);
	idx = bdata->node_low_pfn - pfn;
L
Linus Torvalds 已提交
391
	map = bdata->node_bootmem_map;
392 393 394 395 396
	/*
	 * Check if we are aligned to BITS_PER_LONG pages.  If so, we might
	 * be able to free page orders of that size at once.
	 */
	if (!(pfn & (BITS_PER_LONG-1)))
L
Linus Torvalds 已提交
397
		gofast = 1;
398

L
Linus Torvalds 已提交
399 400
	for (i = 0; i < idx; ) {
		unsigned long v = ~map[i / BITS_PER_LONG];
A
Andy Whitcroft 已提交
401

L
Linus Torvalds 已提交
402
		if (gofast && v == ~0UL) {
403
			int order;
L
Linus Torvalds 已提交
404

A
Andy Whitcroft 已提交
405
			page = pfn_to_page(pfn);
L
Linus Torvalds 已提交
406 407
			count += BITS_PER_LONG;
			order = ffs(BITS_PER_LONG) - 1;
408
			__free_pages_bootmem(page, order);
L
Linus Torvalds 已提交
409 410 411 412
			i += BITS_PER_LONG;
			page += BITS_PER_LONG;
		} else if (v) {
			unsigned long m;
A
Andy Whitcroft 已提交
413 414

			page = pfn_to_page(pfn);
L
Linus Torvalds 已提交
415 416 417
			for (m = 1; m && i < idx; m<<=1, page++, i++) {
				if (v & m) {
					count++;
418
					__free_pages_bootmem(page, 0);
L
Linus Torvalds 已提交
419 420 421
				}
			}
		} else {
422
			i += BITS_PER_LONG;
L
Linus Torvalds 已提交
423
		}
A
Andy Whitcroft 已提交
424
		pfn += BITS_PER_LONG;
L
Linus Torvalds 已提交
425 426 427 428 429 430 431
	}

	/*
	 * Now free the allocator bitmap itself, it's not
	 * needed anymore:
	 */
	page = virt_to_page(bdata->node_bootmem_map);
432
	idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT;
433
	for (i = 0; i < idx; i++, page++)
434
		__free_pages_bootmem(page, 0);
435
	count += i;
L
Linus Torvalds 已提交
436 437
	bdata->node_bootmem_map = NULL;

438
	return count;
L
Linus Torvalds 已提交
439 440
}

441
unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
442
				unsigned long startpfn, unsigned long endpfn)
L
Linus Torvalds 已提交
443
{
444
	return init_bootmem_core(pgdat, freepfn, startpfn, endpfn);
L
Linus Torvalds 已提交
445 446
}

447
int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
448
				 unsigned long size, int flags)
L
Linus Torvalds 已提交
449
{
450 451 452 453
	int ret;

	ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
	if (ret < 0)
454
		return -ENOMEM;
455
	reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
456 457

	return 0;
L
Linus Torvalds 已提交
458 459
}

460 461
void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
			      unsigned long size)
L
Linus Torvalds 已提交
462 463 464 465
{
	free_bootmem_core(pgdat->bdata, physaddr, size);
}

466
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
L
Linus Torvalds 已提交
467
{
468
	register_page_bootmem_info_node(pgdat);
469
	return free_all_bootmem_core(pgdat);
L
Linus Torvalds 已提交
470 471
}

472
unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
L
Linus Torvalds 已提交
473 474 475
{
	max_low_pfn = pages;
	min_low_pfn = start;
476
	return init_bootmem_core(NODE_DATA(0), start, 0, pages);
L
Linus Torvalds 已提交
477 478 479
}

#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
480 481
int __init reserve_bootmem(unsigned long addr, unsigned long size,
			    int flags)
L
Linus Torvalds 已提交
482
{
483 484 485 486 487 488 489 490 491 492 493 494
	bootmem_data_t *bdata;
	int ret;

	list_for_each_entry(bdata, &bdata_list, list) {
		ret = can_reserve_bootmem_core(bdata, addr, size, flags);
		if (ret < 0)
			return ret;
	}
	list_for_each_entry(bdata, &bdata_list, list)
		reserve_bootmem_core(bdata, addr, size, flags);

	return 0;
L
Linus Torvalds 已提交
495 496 497
}
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */

498
void __init free_bootmem(unsigned long addr, unsigned long size)
L
Linus Torvalds 已提交
499
{
500 501 502
	bootmem_data_t *bdata;
	list_for_each_entry(bdata, &bdata_list, list)
		free_bootmem_core(bdata, addr, size);
L
Linus Torvalds 已提交
503 504
}

505
unsigned long __init free_all_bootmem(void)
L
Linus Torvalds 已提交
506
{
507
	return free_all_bootmem_core(NODE_DATA(0));
L
Linus Torvalds 已提交
508 509
}

510 511
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
				      unsigned long goal)
L
Linus Torvalds 已提交
512
{
513
	bootmem_data_t *bdata;
L
Linus Torvalds 已提交
514 515
	void *ptr;

516 517 518 519 520
	list_for_each_entry(bdata, &bdata_list, list) {
		ptr = __alloc_bootmem_core(bdata, size, align, goal, 0);
		if (ptr)
			return ptr;
	}
521 522
	return NULL;
}
L
Linus Torvalds 已提交
523

524 525
void * __init __alloc_bootmem(unsigned long size, unsigned long align,
			      unsigned long goal)
526 527
{
	void *mem = __alloc_bootmem_nopanic(size,align,goal);
528

529 530
	if (mem)
		return mem;
L
Linus Torvalds 已提交
531 532 533 534 535 536 537 538
	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of memory");
	return NULL;
}

539

540 541
void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
				   unsigned long align, unsigned long goal)
L
Linus Torvalds 已提交
542 543 544
{
	void *ptr;

545
	ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
L
Linus Torvalds 已提交
546
	if (ptr)
547
		return ptr;
L
Linus Torvalds 已提交
548

549
	return __alloc_bootmem(size, align, goal);
L
Linus Torvalds 已提交
550 551
}

552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
#ifdef CONFIG_SPARSEMEM
void * __init alloc_bootmem_section(unsigned long size,
				    unsigned long section_nr)
{
	void *ptr;
	unsigned long limit, goal, start_nr, end_nr, pfn;
	struct pglist_data *pgdat;

	pfn = section_nr_to_pfn(section_nr);
	goal = PFN_PHYS(pfn);
	limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1;
	pgdat = NODE_DATA(early_pfn_to_nid(pfn));
	ptr = __alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
				   limit);

	if (!ptr)
		return NULL;

	start_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr)));
	end_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr) + size));
	if (start_nr != section_nr || end_nr != section_nr) {
		printk(KERN_WARNING "alloc_bootmem failed on section %ld.\n",
		       section_nr);
		free_bootmem_core(pgdat->bdata, __pa(ptr), size);
		ptr = NULL;
	}

	return ptr;
}
#endif

583 584 585
#ifndef ARCH_LOW_ADDRESS_LIMIT
#define ARCH_LOW_ADDRESS_LIMIT	0xffffffffUL
#endif
586

587 588
void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
				  unsigned long goal)
589
{
590
	bootmem_data_t *bdata;
591 592
	void *ptr;

593
	list_for_each_entry(bdata, &bdata_list, list) {
594 595
		ptr = __alloc_bootmem_core(bdata, size, align, goal,
						ARCH_LOW_ADDRESS_LIMIT);
596 597 598
		if (ptr)
			return ptr;
	}
599 600 601 602 603 604 605 606 607 608 609 610

	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of low memory");
	return NULL;
}

void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
				       unsigned long align, unsigned long goal)
{
611 612
	return __alloc_bootmem_core(pgdat->bdata, size, align, goal,
				    ARCH_LOW_ADDRESS_LIMIT);
613
}