memory_hotplug.c 51.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 *  linux/mm/memory_hotplug.c
 *
 *  Copyright (C)
 */

#include <linux/stddef.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
#include <linux/compiler.h>
13
#include <linux/export.h>
14
#include <linux/pagevec.h>
15
#include <linux/writeback.h>
16 17 18 19 20 21 22
#include <linux/slab.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/memory.h>
#include <linux/memory_hotplug.h>
#include <linux/highmem.h>
#include <linux/vmalloc.h>
23
#include <linux/ioport.h>
K
KAMEZAWA Hiroyuki 已提交
24 25 26
#include <linux/delay.h>
#include <linux/migrate.h>
#include <linux/page-isolation.h>
27
#include <linux/pfn.h>
28
#include <linux/suspend.h>
29
#include <linux/mm_inline.h>
30
#include <linux/firmware-map.h>
31
#include <linux/stop_machine.h>
32
#include <linux/hugetlb.h>
33
#include <linux/memblock.h>
34
#include <linux/bootmem.h>
35 36 37

#include <asm/tlbflush.h>

38 39
#include "internal.h"

40 41 42 43 44 45 46 47 48 49
/*
 * online_page_callback contains pointer to current page onlining function.
 * Initially it is generic_online_page(). If it is required it could be
 * changed by calling set_online_page_callback() for callback registration
 * and restore_online_page_callback() for generic callback restore.
 */

static void generic_online_page(struct page *page);

static online_page_callback_t online_page_callback = generic_online_page;
50
static DEFINE_MUTEX(online_page_callback_lock);
51

52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
/* The same as the cpu_hotplug lock, but for memory hotplug. */
static struct {
	struct task_struct *active_writer;
	struct mutex lock; /* Synchronizes accesses to refcount, */
	/*
	 * Also blocks the new readers during
	 * an ongoing mem hotplug operation.
	 */
	int refcount;

#ifdef CONFIG_DEBUG_LOCK_ALLOC
	struct lockdep_map dep_map;
#endif
} mem_hotplug = {
	.active_writer = NULL,
	.lock = __MUTEX_INITIALIZER(mem_hotplug.lock),
	.refcount = 0,
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	.dep_map = {.name = "mem_hotplug.lock" },
#endif
};

/* Lockdep annotations for get/put_online_mems() and mem_hotplug_begin/end() */
#define memhp_lock_acquire_read() lock_map_acquire_read(&mem_hotplug.dep_map)
#define memhp_lock_acquire()      lock_map_acquire(&mem_hotplug.dep_map)
#define memhp_lock_release()      lock_map_release(&mem_hotplug.dep_map)

void get_online_mems(void)
{
	might_sleep();
	if (mem_hotplug.active_writer == current)
		return;
	memhp_lock_acquire_read();
	mutex_lock(&mem_hotplug.lock);
	mem_hotplug.refcount++;
	mutex_unlock(&mem_hotplug.lock);

}
90

91
void put_online_mems(void)
92
{
93 94 95 96 97 98 99 100 101 102 103 104
	if (mem_hotplug.active_writer == current)
		return;
	mutex_lock(&mem_hotplug.lock);

	if (WARN_ON(!mem_hotplug.refcount))
		mem_hotplug.refcount++; /* try to fix things up */

	if (!--mem_hotplug.refcount && unlikely(mem_hotplug.active_writer))
		wake_up_process(mem_hotplug.active_writer);
	mutex_unlock(&mem_hotplug.lock);
	memhp_lock_release();

105 106
}

107
void mem_hotplug_begin(void)
108
{
109 110 111 112 113 114 115 116 117 118 119
	mem_hotplug.active_writer = current;

	memhp_lock_acquire();
	for (;;) {
		mutex_lock(&mem_hotplug.lock);
		if (likely(!mem_hotplug.refcount))
			break;
		__set_current_state(TASK_UNINTERRUPTIBLE);
		mutex_unlock(&mem_hotplug.lock);
		schedule();
	}
120 121
}

122
void mem_hotplug_done(void)
123 124 125 126 127
{
	mem_hotplug.active_writer = NULL;
	mutex_unlock(&mem_hotplug.lock);
	memhp_lock_release();
}
128

129 130 131 132 133 134 135 136 137 138
/* add this memory to iomem resource */
static struct resource *register_memory_resource(u64 start, u64 size)
{
	struct resource *res;
	res = kzalloc(sizeof(struct resource), GFP_KERNEL);
	BUG_ON(!res);

	res->name = "System RAM";
	res->start = start;
	res->end = start + size - 1;
139
	res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
140
	if (request_resource(&iomem_resource, res) < 0) {
141
		pr_debug("System RAM resource %pR cannot be added\n", res);
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
		kfree(res);
		res = NULL;
	}
	return res;
}

static void release_memory_resource(struct resource *res)
{
	if (!res)
		return;
	release_resource(res);
	kfree(res);
	return;
}

157
#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
158 159
void get_page_bootmem(unsigned long info,  struct page *page,
		      unsigned long type)
160
{
A
Andrea Arcangeli 已提交
161
	page->lru.next = (struct list_head *) type;
162 163 164 165 166
	SetPagePrivate(page);
	set_page_private(page, info);
	atomic_inc(&page->_count);
}

167
void put_page_bootmem(struct page *page)
168
{
A
Andrea Arcangeli 已提交
169
	unsigned long type;
170

A
Andrea Arcangeli 已提交
171 172 173
	type = (unsigned long) page->lru.next;
	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
	       type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
174 175 176 177

	if (atomic_dec_return(&page->_count) == 1) {
		ClearPagePrivate(page);
		set_page_private(page, 0);
A
Andrea Arcangeli 已提交
178
		INIT_LIST_HEAD(&page->lru);
179
		free_reserved_page(page);
180 181 182
	}
}

183 184
#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
#ifndef CONFIG_SPARSEMEM_VMEMMAP
185
static void register_page_bootmem_info_section(unsigned long start_pfn)
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
{
	unsigned long *usemap, mapsize, section_nr, i;
	struct mem_section *ms;
	struct page *page, *memmap;

	section_nr = pfn_to_section_nr(start_pfn);
	ms = __nr_to_section(section_nr);

	/* Get section's memmap address */
	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

	/*
	 * Get page for the memmap's phys address
	 * XXX: need more consideration for sparse_vmemmap...
	 */
	page = virt_to_page(memmap);
	mapsize = sizeof(struct page) * PAGES_PER_SECTION;
	mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;

	/* remember memmap's page */
	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, SECTION_INFO);

	usemap = __nr_to_section(section_nr)->pageblock_flags;
	page = virt_to_page(usemap);

	mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;

	for (i = 0; i < mapsize; i++, page++)
215
		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
216 217

}
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
#else /* CONFIG_SPARSEMEM_VMEMMAP */
static void register_page_bootmem_info_section(unsigned long start_pfn)
{
	unsigned long *usemap, mapsize, section_nr, i;
	struct mem_section *ms;
	struct page *page, *memmap;

	if (!pfn_valid(start_pfn))
		return;

	section_nr = pfn_to_section_nr(start_pfn);
	ms = __nr_to_section(section_nr);

	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

	register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);

	usemap = __nr_to_section(section_nr)->pageblock_flags;
	page = virt_to_page(usemap);

	mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;

	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
}
#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259

void register_page_bootmem_info_node(struct pglist_data *pgdat)
{
	unsigned long i, pfn, end_pfn, nr_pages;
	int node = pgdat->node_id;
	struct page *page;
	struct zone *zone;

	nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
	page = virt_to_page(pgdat);

	for (i = 0; i < nr_pages; i++, page++)
		get_page_bootmem(node, page, NODE_INFO);

	zone = &pgdat->node_zones[0];
	for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) {
260
		if (zone_is_initialized(zone)) {
261 262 263 264 265 266 267 268 269 270 271
			nr_pages = zone->wait_table_hash_nr_entries
				* sizeof(wait_queue_head_t);
			nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT;
			page = virt_to_page(zone->wait_table);

			for (i = 0; i < nr_pages; i++, page++)
				get_page_bootmem(node, page, NODE_INFO);
		}
	}

	pfn = pgdat->node_start_pfn;
272
	end_pfn = pgdat_end_pfn(pgdat);
273

274
	/* register section info */
275 276 277 278 279
	for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		/*
		 * Some platforms can assign the same pfn to multiple nodes - on
		 * node0 as well as nodeN.  To avoid registering a pfn against
		 * multiple nodes we check that this pfn does not already
280
		 * reside in some other nodes.
281 282 283 284
		 */
		if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node))
			register_page_bootmem_info_section(pfn);
	}
285
}
286
#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */
287

288 289
static void __meminit grow_zone_span(struct zone *zone, unsigned long start_pfn,
				     unsigned long end_pfn)
290 291 292 293 294
{
	unsigned long old_zone_end_pfn;

	zone_span_writelock(zone);

295
	old_zone_end_pfn = zone_end_pfn(zone);
296
	if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
297 298 299 300 301 302 303 304
		zone->zone_start_pfn = start_pfn;

	zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
				zone->zone_start_pfn;

	zone_span_writeunlock(zone);
}

305 306 307 308 309
static void resize_zone(struct zone *zone, unsigned long start_pfn,
		unsigned long end_pfn)
{
	zone_span_writelock(zone);

310 311 312 313 314 315 316 317 318 319 320
	if (end_pfn - start_pfn) {
		zone->zone_start_pfn = start_pfn;
		zone->spanned_pages = end_pfn - start_pfn;
	} else {
		/*
		 * make it consist as free_area_init_core(),
		 * if spanned_pages = 0, then keep start_pfn = 0
		 */
		zone->zone_start_pfn = 0;
		zone->spanned_pages = 0;
	}
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335

	zone_span_writeunlock(zone);
}

static void fix_zone_id(struct zone *zone, unsigned long start_pfn,
		unsigned long end_pfn)
{
	enum zone_type zid = zone_idx(zone);
	int nid = zone->zone_pgdat->node_id;
	unsigned long pfn;

	for (pfn = start_pfn; pfn < end_pfn; pfn++)
		set_page_links(pfn_to_page(pfn), zid, nid, pfn);
}

336
/* Can fail with -ENOMEM from allocating a wait table with vmalloc() or
337
 * alloc_bootmem_node_nopanic()/memblock_virt_alloc_node_nopanic() */
338 339 340 341 342 343 344 345 346
static int __ref ensure_zone_is_initialized(struct zone *zone,
			unsigned long start_pfn, unsigned long num_pages)
{
	if (!zone_is_initialized(zone))
		return init_currently_empty_zone(zone, start_pfn, num_pages,
						 MEMMAP_HOTPLUG);
	return 0;
}

347
static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2,
348 349
		unsigned long start_pfn, unsigned long end_pfn)
{
350
	int ret;
351
	unsigned long flags;
352 353
	unsigned long z1_start_pfn;

354 355 356
	ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn);
	if (ret)
		return ret;
357 358 359 360

	pgdat_resize_lock(z1->zone_pgdat, &flags);

	/* can't move pfns which are higher than @z2 */
361
	if (end_pfn > zone_end_pfn(z2))
362
		goto out_fail;
363
	/* the move out part must be at the left most of @z2 */
364 365 366 367 368 369
	if (start_pfn > z2->zone_start_pfn)
		goto out_fail;
	/* must included/overlap */
	if (end_pfn <= z2->zone_start_pfn)
		goto out_fail;

370
	/* use start_pfn for z1's start_pfn if z1 is empty */
371
	if (!zone_is_empty(z1))
372 373 374 375 376
		z1_start_pfn = z1->zone_start_pfn;
	else
		z1_start_pfn = start_pfn;

	resize_zone(z1, z1_start_pfn, end_pfn);
377
	resize_zone(z2, end_pfn, zone_end_pfn(z2));
378 379 380 381 382 383 384 385 386 387 388

	pgdat_resize_unlock(z1->zone_pgdat, &flags);

	fix_zone_id(z1, start_pfn, end_pfn);

	return 0;
out_fail:
	pgdat_resize_unlock(z1->zone_pgdat, &flags);
	return -1;
}

389
static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2,
390 391
		unsigned long start_pfn, unsigned long end_pfn)
{
392
	int ret;
393
	unsigned long flags;
394 395
	unsigned long z2_end_pfn;

396 397 398
	ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn);
	if (ret)
		return ret;
399 400 401 402 403 404 405

	pgdat_resize_lock(z1->zone_pgdat, &flags);

	/* can't move pfns which are lower than @z1 */
	if (z1->zone_start_pfn > start_pfn)
		goto out_fail;
	/* the move out part mast at the right most of @z1 */
406
	if (zone_end_pfn(z1) >  end_pfn)
407 408
		goto out_fail;
	/* must included/overlap */
409
	if (start_pfn >= zone_end_pfn(z1))
410 411
		goto out_fail;

412
	/* use end_pfn for z2's end_pfn if z2 is empty */
413
	if (!zone_is_empty(z2))
414
		z2_end_pfn = zone_end_pfn(z2);
415 416 417
	else
		z2_end_pfn = end_pfn;

418
	resize_zone(z1, z1->zone_start_pfn, start_pfn);
419
	resize_zone(z2, start_pfn, z2_end_pfn);
420 421 422 423 424 425 426 427 428 429 430

	pgdat_resize_unlock(z1->zone_pgdat, &flags);

	fix_zone_id(z2, start_pfn, end_pfn);

	return 0;
out_fail:
	pgdat_resize_unlock(z1->zone_pgdat, &flags);
	return -1;
}

431 432
static void __meminit grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn,
				      unsigned long end_pfn)
433
{
434
	unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat);
435

436
	if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
437 438 439 440 441 442
		pgdat->node_start_pfn = start_pfn;

	pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) -
					pgdat->node_start_pfn;
}

A
Al Viro 已提交
443
static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)
444 445 446 447 448
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	int nr_pages = PAGES_PER_SECTION;
	int nid = pgdat->node_id;
	int zone_type;
449
	unsigned long flags;
450
	int ret;
451 452

	zone_type = zone - pgdat->node_zones;
453 454 455
	ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages);
	if (ret)
		return ret;
456 457 458 459 460 461

	pgdat_resize_lock(zone->zone_pgdat, &flags);
	grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages);
	grow_pgdat_span(zone->zone_pgdat, phys_start_pfn,
			phys_start_pfn + nr_pages);
	pgdat_resize_unlock(zone->zone_pgdat, &flags);
D
Dave Hansen 已提交
462 463
	memmap_init_zone(nr_pages, nid, zone_type,
			 phys_start_pfn, MEMMAP_HOTPLUG);
464
	return 0;
465 466
}

467 468
static int __meminit __add_section(int nid, struct zone *zone,
					unsigned long phys_start_pfn)
469 470 471
{
	int ret;

472 473 474
	if (pfn_valid(phys_start_pfn))
		return -EEXIST;

475
	ret = sparse_add_one_section(zone, phys_start_pfn);
476 477 478 479

	if (ret < 0)
		return ret;

480 481 482 483 484
	ret = __add_zone(zone, phys_start_pfn);

	if (ret < 0)
		return ret;

485
	return register_new_memory(nid, __pfn_to_section(phys_start_pfn));
486 487
}

488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
/*
 * Reasonably generic function for adding memory.  It is
 * expected that archs that support memory hotplug will
 * call this function after deciding the zone to which to
 * add the new pages.
 */
int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
			unsigned long nr_pages)
{
	unsigned long i;
	int err = 0;
	int start_sec, end_sec;
	/* during initialize mem_map, align hot-added range to section */
	start_sec = pfn_to_section_nr(phys_start_pfn);
	end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1);

	for (i = start_sec; i <= end_sec; i++) {
505
		err = __add_section(nid, zone, section_nr_to_pfn(i));
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521

		/*
		 * EEXIST is finally dealt with by ioresource collision
		 * check. see add_memory() => register_memory_resource()
		 * Warning will be printed if there is collision.
		 */
		if (err && (err != -EEXIST))
			break;
		err = 0;
	}

	return err;
}
EXPORT_SYMBOL_GPL(__add_pages);

#ifdef CONFIG_MEMORY_HOTREMOVE
522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
/* find the smallest valid pfn in the range [start_pfn, end_pfn) */
static int find_smallest_section_pfn(int nid, struct zone *zone,
				     unsigned long start_pfn,
				     unsigned long end_pfn)
{
	struct mem_section *ms;

	for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) {
		ms = __pfn_to_section(start_pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (unlikely(pfn_to_nid(start_pfn) != nid))
			continue;

		if (zone && zone != page_zone(pfn_to_page(start_pfn)))
			continue;

		return start_pfn;
	}

	return 0;
}

/* find the biggest valid pfn in the range [start_pfn, end_pfn). */
static int find_biggest_section_pfn(int nid, struct zone *zone,
				    unsigned long start_pfn,
				    unsigned long end_pfn)
{
	struct mem_section *ms;
	unsigned long pfn;

	/* pfn is the end pfn of a memory section. */
	pfn = end_pfn - 1;
	for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) {
		ms = __pfn_to_section(pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (unlikely(pfn_to_nid(pfn) != nid))
			continue;

		if (zone && zone != page_zone(pfn_to_page(pfn)))
			continue;

		return pfn;
	}

	return 0;
}

static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
			     unsigned long end_pfn)
{
578 579 580
	unsigned long zone_start_pfn = zone->zone_start_pfn;
	unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */
	unsigned long zone_end_pfn = z;
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
	unsigned long pfn;
	struct mem_section *ms;
	int nid = zone_to_nid(zone);

	zone_span_writelock(zone);
	if (zone_start_pfn == start_pfn) {
		/*
		 * If the section is smallest section in the zone, it need
		 * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
		 * In this case, we find second smallest valid mem_section
		 * for shrinking zone.
		 */
		pfn = find_smallest_section_pfn(nid, zone, end_pfn,
						zone_end_pfn);
		if (pfn) {
			zone->zone_start_pfn = pfn;
			zone->spanned_pages = zone_end_pfn - pfn;
		}
	} else if (zone_end_pfn == end_pfn) {
		/*
		 * If the section is biggest section in the zone, it need
		 * shrink zone->spanned_pages.
		 * In this case, we find second biggest valid mem_section for
		 * shrinking zone.
		 */
		pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
					       start_pfn);
		if (pfn)
			zone->spanned_pages = pfn - zone_start_pfn + 1;
	}

	/*
	 * The section is not biggest or smallest mem_section in the zone, it
	 * only creates a hole in the zone. So in this case, we need not
	 * change the zone. But perhaps, the zone has only hole data. Thus
	 * it check the zone has only hole or not.
	 */
	pfn = zone_start_pfn;
	for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) {
		ms = __pfn_to_section(pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (page_zone(pfn_to_page(pfn)) != zone)
			continue;

		 /* If the section is current section, it continues the loop */
		if (start_pfn == pfn)
			continue;

		/* If we find valid section, we have nothing to do */
		zone_span_writeunlock(zone);
		return;
	}

	/* The zone has no valid section */
	zone->zone_start_pfn = 0;
	zone->spanned_pages = 0;
	zone_span_writeunlock(zone);
}

static void shrink_pgdat_span(struct pglist_data *pgdat,
			      unsigned long start_pfn, unsigned long end_pfn)
{
646 647 648
	unsigned long pgdat_start_pfn = pgdat->node_start_pfn;
	unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */
	unsigned long pgdat_end_pfn = p;
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723
	unsigned long pfn;
	struct mem_section *ms;
	int nid = pgdat->node_id;

	if (pgdat_start_pfn == start_pfn) {
		/*
		 * If the section is smallest section in the pgdat, it need
		 * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages.
		 * In this case, we find second smallest valid mem_section
		 * for shrinking zone.
		 */
		pfn = find_smallest_section_pfn(nid, NULL, end_pfn,
						pgdat_end_pfn);
		if (pfn) {
			pgdat->node_start_pfn = pfn;
			pgdat->node_spanned_pages = pgdat_end_pfn - pfn;
		}
	} else if (pgdat_end_pfn == end_pfn) {
		/*
		 * If the section is biggest section in the pgdat, it need
		 * shrink pgdat->node_spanned_pages.
		 * In this case, we find second biggest valid mem_section for
		 * shrinking zone.
		 */
		pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn,
					       start_pfn);
		if (pfn)
			pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1;
	}

	/*
	 * If the section is not biggest or smallest mem_section in the pgdat,
	 * it only creates a hole in the pgdat. So in this case, we need not
	 * change the pgdat.
	 * But perhaps, the pgdat has only hole data. Thus it check the pgdat
	 * has only hole or not.
	 */
	pfn = pgdat_start_pfn;
	for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) {
		ms = __pfn_to_section(pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (pfn_to_nid(pfn) != nid)
			continue;

		 /* If the section is current section, it continues the loop */
		if (start_pfn == pfn)
			continue;

		/* If we find valid section, we have nothing to do */
		return;
	}

	/* The pgdat has no valid section */
	pgdat->node_start_pfn = 0;
	pgdat->node_spanned_pages = 0;
}

static void __remove_zone(struct zone *zone, unsigned long start_pfn)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	int nr_pages = PAGES_PER_SECTION;
	int zone_type;
	unsigned long flags;

	zone_type = zone - pgdat->node_zones;

	pgdat_resize_lock(zone->zone_pgdat, &flags);
	shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
	shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages);
	pgdat_resize_unlock(zone->zone_pgdat, &flags);
}

724 725
static int __remove_section(struct zone *zone, struct mem_section *ms)
{
726 727
	unsigned long start_pfn;
	int scn_nr;
728 729 730 731 732 733 734 735 736
	int ret = -EINVAL;

	if (!valid_section(ms))
		return ret;

	ret = unregister_memory_section(ms);
	if (ret)
		return ret;

737 738 739 740
	scn_nr = __section_nr(ms);
	start_pfn = section_nr_to_pfn(scn_nr);
	__remove_zone(zone, start_pfn);

741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758
	sparse_remove_one_section(zone, ms);
	return 0;
}

/**
 * __remove_pages() - remove sections of pages from a zone
 * @zone: zone from which pages need to be removed
 * @phys_start_pfn: starting pageframe (must be aligned to start of a section)
 * @nr_pages: number of pages to remove (must be multiple of section size)
 *
 * Generic helper function to remove section mappings and sysfs entries
 * for the section of the memory we are removing. Caller needs to make
 * sure that pages are marked reserved and zones are adjust properly by
 * calling offline_pages().
 */
int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
		 unsigned long nr_pages)
{
759
	unsigned long i;
760
	int sections_to_remove;
761 762
	resource_size_t start, size;
	int ret = 0;
763 764 765 766 767 768 769

	/*
	 * We can only remove entire sections
	 */
	BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK);
	BUG_ON(nr_pages % PAGES_PER_SECTION);

770 771 772
	start = phys_start_pfn << PAGE_SHIFT;
	size = nr_pages * PAGE_SIZE;
	ret = release_mem_region_adjustable(&iomem_resource, start, size);
773 774 775 776 777 778
	if (ret) {
		resource_size_t endres = start + size - 1;

		pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
				&start, &endres, ret);
	}
779

780 781 782 783 784 785 786 787 788 789
	sections_to_remove = nr_pages / PAGES_PER_SECTION;
	for (i = 0; i < sections_to_remove; i++) {
		unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;
		ret = __remove_section(zone, __pfn_to_section(pfn));
		if (ret)
			break;
	}
	return ret;
}
EXPORT_SYMBOL_GPL(__remove_pages);
790
#endif /* CONFIG_MEMORY_HOTREMOVE */
791

792 793 794 795
int set_online_page_callback(online_page_callback_t callback)
{
	int rc = -EINVAL;

796 797
	get_online_mems();
	mutex_lock(&online_page_callback_lock);
798 799 800 801 802 803

	if (online_page_callback == generic_online_page) {
		online_page_callback = callback;
		rc = 0;
	}

804 805
	mutex_unlock(&online_page_callback_lock);
	put_online_mems();
806 807 808 809 810 811 812 813 814

	return rc;
}
EXPORT_SYMBOL_GPL(set_online_page_callback);

int restore_online_page_callback(online_page_callback_t callback)
{
	int rc = -EINVAL;

815 816
	get_online_mems();
	mutex_lock(&online_page_callback_lock);
817 818 819 820 821 822

	if (online_page_callback == callback) {
		online_page_callback = generic_online_page;
		rc = 0;
	}

823 824
	mutex_unlock(&online_page_callback_lock);
	put_online_mems();
825 826 827 828 829 830

	return rc;
}
EXPORT_SYMBOL_GPL(restore_online_page_callback);

void __online_page_set_limits(struct page *page)
831
{
832 833 834 835 836
}
EXPORT_SYMBOL_GPL(__online_page_set_limits);

void __online_page_increment_counters(struct page *page)
{
837
	adjust_managed_page_count(page, 1);
838 839
}
EXPORT_SYMBOL_GPL(__online_page_increment_counters);
840

841 842
void __online_page_free(struct page *page)
{
843
	__free_reserved_page(page);
844
}
845 846 847 848 849 850 851 852
EXPORT_SYMBOL_GPL(__online_page_free);

static void generic_online_page(struct page *page)
{
	__online_page_set_limits(page);
	__online_page_increment_counters(page);
	__online_page_free(page);
}
853

854 855
static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
			void *arg)
856 857
{
	unsigned long i;
858 859 860 861 862
	unsigned long onlined_pages = *(unsigned long *)arg;
	struct page *page;
	if (PageReserved(pfn_to_page(start_pfn)))
		for (i = 0; i < nr_pages; i++) {
			page = pfn_to_page(start_pfn + i);
863
			(*online_page_callback)(page);
864 865 866 867 868 869
			onlined_pages++;
		}
	*(unsigned long *)arg = onlined_pages;
	return 0;
}

870
#ifdef CONFIG_MOVABLE_NODE
T
Tang Chen 已提交
871 872 873 874
/*
 * When CONFIG_MOVABLE_NODE, we permit onlining of a node which doesn't have
 * normal memory.
 */
875 876 877 878
static bool can_online_high_movable(struct zone *zone)
{
	return true;
}
T
Tang Chen 已提交
879
#else /* CONFIG_MOVABLE_NODE */
880 881 882 883 884
/* ensure every online node has NORMAL memory */
static bool can_online_high_movable(struct zone *zone)
{
	return node_state(zone_to_nid(zone), N_NORMAL_MEMORY);
}
T
Tang Chen 已提交
885
#endif /* CONFIG_MOVABLE_NODE */
886

887 888 889 890 891 892 893 894
/* check which state of node_states will be changed when online memory */
static void node_states_check_changes_online(unsigned long nr_pages,
	struct zone *zone, struct memory_notify *arg)
{
	int nid = zone_to_nid(zone);
	enum zone_type zone_last = ZONE_NORMAL;

	/*
895 896 897
	 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
	 * contains nodes which have zones of 0...ZONE_NORMAL,
	 * set zone_last to ZONE_NORMAL.
898
	 *
899 900 901
	 * If we don't have HIGHMEM nor movable node,
	 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
	 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
902
	 */
903
	if (N_MEMORY == N_NORMAL_MEMORY)
904 905 906 907 908 909 910 911 912 913 914 915 916
		zone_last = ZONE_MOVABLE;

	/*
	 * if the memory to be online is in a zone of 0...zone_last, and
	 * the zones of 0...zone_last don't have memory before online, we will
	 * need to set the node to node_states[N_NORMAL_MEMORY] after
	 * the memory is online.
	 */
	if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY))
		arg->status_change_nid_normal = nid;
	else
		arg->status_change_nid_normal = -1;

917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938
#ifdef CONFIG_HIGHMEM
	/*
	 * If we have movable node, node_states[N_HIGH_MEMORY]
	 * contains nodes which have zones of 0...ZONE_HIGHMEM,
	 * set zone_last to ZONE_HIGHMEM.
	 *
	 * If we don't have movable node, node_states[N_NORMAL_MEMORY]
	 * contains nodes which have zones of 0...ZONE_MOVABLE,
	 * set zone_last to ZONE_MOVABLE.
	 */
	zone_last = ZONE_HIGHMEM;
	if (N_MEMORY == N_HIGH_MEMORY)
		zone_last = ZONE_MOVABLE;

	if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY))
		arg->status_change_nid_high = nid;
	else
		arg->status_change_nid_high = -1;
#else
	arg->status_change_nid_high = arg->status_change_nid_normal;
#endif

939 940
	/*
	 * if the node don't have memory befor online, we will need to
941
	 * set the node to node_states[N_MEMORY] after the memory
942 943
	 * is online.
	 */
944
	if (!node_state(nid, N_MEMORY))
945 946 947 948 949 950 951 952 953 954
		arg->status_change_nid = nid;
	else
		arg->status_change_nid = -1;
}

static void node_states_set_node(int node, struct memory_notify *arg)
{
	if (arg->status_change_nid_normal >= 0)
		node_set_state(node, N_NORMAL_MEMORY);

955 956 957 958
	if (arg->status_change_nid_high >= 0)
		node_set_state(node, N_HIGH_MEMORY);

	node_set_state(node, N_MEMORY);
959 960
}

961

962
/* Must be protected by mem_hotplug_begin() */
963
int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type)
964
{
965
	unsigned long flags;
966 967
	unsigned long onlined_pages = 0;
	struct zone *zone;
968
	int need_zonelists_rebuild = 0;
969 970 971 972
	int nid;
	int ret;
	struct memory_notify arg;

973 974 975 976 977 978 979
	/*
	 * This doesn't need a lock to do pfn_to_page().
	 * The section can't be removed here because of the
	 * memory_block->state_mutex.
	 */
	zone = page_zone(pfn_to_page(pfn));

980 981
	if ((zone_idx(zone) > ZONE_NORMAL ||
	    online_type == MMOP_ONLINE_MOVABLE) &&
982
	    !can_online_high_movable(zone))
983
		return -EINVAL;
984

985 986
	if (online_type == MMOP_ONLINE_KERNEL &&
	    zone_idx(zone) == ZONE_MOVABLE) {
987
		if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages))
988
			return -EINVAL;
989
	}
990 991
	if (online_type == MMOP_ONLINE_MOVABLE &&
	    zone_idx(zone) == ZONE_MOVABLE - 1) {
992
		if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages))
993
			return -EINVAL;
994 995 996 997 998
	}

	/* Previous code may changed the zone of the pfn range */
	zone = page_zone(pfn_to_page(pfn));

999 1000
	arg.start_pfn = pfn;
	arg.nr_pages = nr_pages;
1001
	node_states_check_changes_online(nr_pages, zone, &arg);
1002

1003
	nid = pfn_to_nid(pfn);
1004

1005 1006 1007 1008
	ret = memory_notify(MEM_GOING_ONLINE, &arg);
	ret = notifier_to_errno(ret);
	if (ret) {
		memory_notify(MEM_CANCEL_ONLINE, &arg);
1009
		return ret;
1010
	}
1011 1012 1013 1014 1015
	/*
	 * If this zone is not populated, then it is not in zonelist.
	 * This means the page allocator ignores this zone.
	 * So, zonelist must be updated after online.
	 */
1016
	mutex_lock(&zonelists_mutex);
1017
	if (!populated_zone(zone)) {
1018
		need_zonelists_rebuild = 1;
1019 1020
		build_all_zonelists(NULL, zone);
	}
1021

K
KAMEZAWA Hiroyuki 已提交
1022
	ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
1023
		online_pages_range);
1024
	if (ret) {
1025 1026
		if (need_zonelists_rebuild)
			zone_pcp_reset(zone);
1027
		mutex_unlock(&zonelists_mutex);
1028 1029 1030 1031
		printk(KERN_DEBUG "online_pages [mem %#010llx-%#010llx] failed\n",
		       (unsigned long long) pfn << PAGE_SHIFT,
		       (((unsigned long long) pfn + nr_pages)
			    << PAGE_SHIFT) - 1);
1032
		memory_notify(MEM_CANCEL_ONLINE, &arg);
1033
		return ret;
1034 1035
	}

1036
	zone->present_pages += onlined_pages;
1037 1038

	pgdat_resize_lock(zone->zone_pgdat, &flags);
1039
	zone->zone_pgdat->node_present_pages += onlined_pages;
1040 1041
	pgdat_resize_unlock(zone->zone_pgdat, &flags);

1042
	if (onlined_pages) {
1043
		node_states_set_node(zone_to_nid(zone), &arg);
1044
		if (need_zonelists_rebuild)
1045
			build_all_zonelists(NULL, NULL);
1046 1047 1048
		else
			zone_pcp_update(zone);
	}
1049

1050
	mutex_unlock(&zonelists_mutex);
1051 1052 1053

	init_per_zone_wmark_min();

1054
	if (onlined_pages)
1055
		kswapd_run(zone_to_nid(zone));
1056

1057
	vm_total_pages = nr_free_pagecache_pages();
1058

1059
	writeback_set_ratelimit();
1060 1061 1062

	if (onlined_pages)
		memory_notify(MEM_ONLINE, &arg);
1063
	return 0;
1064
}
1065
#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
1066

1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
static void reset_node_present_pages(pg_data_t *pgdat)
{
	struct zone *z;

	for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
		z->present_pages = 0;

	pgdat->node_present_pages = 0;
}

1077 1078
/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
1079 1080 1081 1082
{
	struct pglist_data *pgdat;
	unsigned long zones_size[MAX_NR_ZONES] = {0};
	unsigned long zholes_size[MAX_NR_ZONES] = {0};
1083
	unsigned long start_pfn = PFN_DOWN(start);
1084

1085 1086 1087 1088 1089
	pgdat = NODE_DATA(nid);
	if (!pgdat) {
		pgdat = arch_alloc_nodedata(nid);
		if (!pgdat)
			return NULL;
1090

1091
		arch_refresh_nodedata(nid, pgdat);
1092 1093 1094 1095
	} else {
		/* Reset the nr_zones and classzone_idx to 0 before reuse */
		pgdat->nr_zones = 0;
		pgdat->classzone_idx = 0;
1096
	}
1097 1098 1099 1100

	/* we can use NODE_DATA(nid) from here */

	/* init node's zones as empty zones, we don't have any present pages.*/
1101
	free_area_init_node(nid, zones_size, start_pfn, zholes_size);
1102

1103 1104 1105 1106
	/*
	 * The node we allocated has no zone fallback lists. For avoiding
	 * to access not-initialized zonelist, build here.
	 */
1107
	mutex_lock(&zonelists_mutex);
1108
	build_all_zonelists(pgdat, NULL);
1109
	mutex_unlock(&zonelists_mutex);
1110

1111 1112 1113 1114 1115 1116 1117 1118
	/*
	 * zone->managed_pages is set to an approximate value in
	 * free_area_init_core(), which will cause
	 * /sys/device/system/node/nodeX/meminfo has wrong data.
	 * So reset it to 0 before any memory is onlined.
	 */
	reset_node_managed_pages(pgdat);

1119 1120 1121 1122 1123 1124 1125
	/*
	 * When memory is hot-added, all the memory is in offline state. So
	 * clear all zones' present_pages because they will be updated in
	 * online_pages() and offline_pages().
	 */
	reset_node_present_pages(pgdat);

1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
	return pgdat;
}

static void rollback_node_hotadd(int nid, pg_data_t *pgdat)
{
	arch_refresh_nodedata(nid, NULL);
	arch_free_nodedata(pgdat);
	return;
}

1136

1137 1138 1139
/**
 * try_online_node - online a node if offlined
 *
1140 1141
 * called by cpu_up() to online a node without onlined memory.
 */
1142
int try_online_node(int nid)
1143 1144 1145 1146
{
	pg_data_t	*pgdat;
	int	ret;

1147 1148 1149
	if (node_online(nid))
		return 0;

1150
	mem_hotplug_begin();
1151
	pgdat = hotadd_new_pgdat(nid, 0);
1152
	if (!pgdat) {
1153
		pr_err("Cannot online node %d due to NULL pgdat\n", nid);
1154 1155 1156 1157 1158 1159 1160
		ret = -ENOMEM;
		goto out;
	}
	node_set_online(nid);
	ret = register_one_node(nid);
	BUG_ON(ret);

1161 1162 1163 1164 1165 1166
	if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
		mutex_lock(&zonelists_mutex);
		build_all_zonelists(NULL, NULL);
		mutex_unlock(&zonelists_mutex);
	}

1167
out:
1168
	mem_hotplug_done();
1169 1170 1171
	return ret;
}

1172 1173
static int check_hotplug_memory_range(u64 start, u64 size)
{
1174
	u64 start_pfn = PFN_DOWN(start);
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
	u64 nr_pages = size >> PAGE_SHIFT;

	/* Memory range must be aligned with section */
	if ((start_pfn & ~PAGE_SECTION_MASK) ||
	    (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) {
		pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n",
				(unsigned long long)start,
				(unsigned long long)size);
		return -EINVAL;
	}

	return 0;
}

1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216
/*
 * If movable zone has already been setup, newly added memory should be check.
 * If its address is higher than movable zone, it should be added as movable.
 * Without this check, movable zone may overlap with other zone.
 */
static int should_add_memory_movable(int nid, u64 start, u64 size)
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	pg_data_t *pgdat = NODE_DATA(nid);
	struct zone *movable_zone = pgdat->node_zones + ZONE_MOVABLE;

	if (zone_is_empty(movable_zone))
		return 0;

	if (movable_zone->zone_start_pfn <= start_pfn)
		return 1;

	return 0;
}

int zone_for_memory(int nid, u64 start, u64 size, int zone_default)
{
	if (should_add_memory_movable(nid, start, size))
		return ZONE_MOVABLE;

	return zone_default;
}

A
Al Viro 已提交
1217 1218
/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
int __ref add_memory(int nid, u64 start, u64 size)
1219
{
1220
	pg_data_t *pgdat = NULL;
1221 1222
	bool new_pgdat;
	bool new_node;
1223
	struct resource *res;
1224 1225
	int ret;

1226 1227 1228 1229
	ret = check_hotplug_memory_range(start, size);
	if (ret)
		return ret;

1230
	res = register_memory_resource(start, size);
1231
	ret = -EEXIST;
1232
	if (!res)
1233
		return ret;
1234

1235 1236 1237 1238
	{	/* Stupid hack to suppress address-never-null warning */
		void *p = NODE_DATA(nid);
		new_pgdat = !p;
	}
1239

1240
	mem_hotplug_begin();
1241

1242 1243
	new_node = !node_online(nid);
	if (new_node) {
1244
		pgdat = hotadd_new_pgdat(nid, start);
1245
		ret = -ENOMEM;
1246
		if (!pgdat)
1247
			goto error;
1248 1249
	}

1250 1251 1252
	/* call arch's memory hotadd */
	ret = arch_add_memory(nid, start, size);

1253 1254 1255
	if (ret < 0)
		goto error;

1256
	/* we online node here. we can't roll back from here. */
1257 1258
	node_set_online(nid);

1259
	if (new_node) {
1260 1261 1262 1263 1264 1265 1266 1267 1268
		ret = register_one_node(nid);
		/*
		 * If sysfs file of new node can't create, cpu on the node
		 * can't be hot-added. There is no rollback way now.
		 * So, check by BUG_ON() to catch it reluctantly..
		 */
		BUG_ON(ret);
	}

1269 1270 1271
	/* create new memmap entry */
	firmware_map_add_hotplug(start, start + size, "System RAM");

1272 1273
	goto out;

1274 1275 1276 1277
error:
	/* rollback pgdat allocation and others */
	if (new_pgdat)
		rollback_node_hotadd(nid, pgdat);
1278
	release_memory_resource(res);
1279

1280
out:
1281
	mem_hotplug_done();
1282 1283 1284
	return ret;
}
EXPORT_SYMBOL_GPL(add_memory);
K
KAMEZAWA Hiroyuki 已提交
1285 1286

#ifdef CONFIG_MEMORY_HOTREMOVE
1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305
/*
 * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy
 * set and the size of the free page is given by page_order(). Using this,
 * the function determines if the pageblock contains only free pages.
 * Due to buddy contraints, a free page at least the size of a pageblock will
 * be located at the start of the pageblock
 */
static inline int pageblock_free(struct page *page)
{
	return PageBuddy(page) && page_order(page) >= pageblock_order;
}

/* Return the start of the next active pageblock after a given page */
static struct page *next_active_pageblock(struct page *page)
{
	/* Ensure the starting page is pageblock-aligned */
	BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1));

	/* If the entire pageblock is free, move to the end of free page */
1306 1307 1308 1309 1310 1311 1312
	if (pageblock_free(page)) {
		int order;
		/* be careful. we don't have locks, page_order can be changed.*/
		order = page_order(page);
		if ((order < MAX_ORDER) && (order >= pageblock_order))
			return page + (1 << order);
	}
1313

1314
	return page + pageblock_nr_pages;
1315 1316 1317 1318 1319 1320 1321 1322 1323 1324
}

/* Checks if this range of memory is likely to be hot-removable. */
int is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
{
	struct page *page = pfn_to_page(start_pfn);
	struct page *end_page = page + nr_pages;

	/* Check the starting page of each pageblock within the range */
	for (; page < end_page; page = next_active_pageblock(page)) {
1325
		if (!is_pageblock_removable_nolock(page))
1326
			return 0;
1327
		cond_resched();
1328 1329 1330 1331 1332 1333
	}

	/* All pageblocks in the memory block are likely to be hot-removable */
	return 1;
}

K
KAMEZAWA Hiroyuki 已提交
1334 1335 1336
/*
 * Confirm all pages in a range [start, end) is belongs to the same zone.
 */
1337
int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn)
K
KAMEZAWA Hiroyuki 已提交
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
{
	unsigned long pfn;
	struct zone *zone = NULL;
	struct page *page;
	int i;
	for (pfn = start_pfn;
	     pfn < end_pfn;
	     pfn += MAX_ORDER_NR_PAGES) {
		i = 0;
		/* This is just a CONFIG_HOLES_IN_ZONE check.*/
		while ((i < MAX_ORDER_NR_PAGES) && !pfn_valid_within(pfn + i))
			i++;
		if (i == MAX_ORDER_NR_PAGES)
			continue;
		page = pfn_to_page(pfn + i);
		if (zone && page_zone(page) != zone)
			return 0;
		zone = page_zone(page);
	}
	return 1;
}

/*
1361 1362 1363 1364
 * Scan pfn range [start,end) to find movable/migratable pages (LRU pages
 * and hugepages). We scan pfn because it's much easier than scanning over
 * linked list. This function returns the pfn of the first found movable
 * page if it's found, otherwise 0.
K
KAMEZAWA Hiroyuki 已提交
1365
 */
1366
static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
K
KAMEZAWA Hiroyuki 已提交
1367 1368 1369 1370 1371 1372 1373 1374
{
	unsigned long pfn;
	struct page *page;
	for (pfn = start; pfn < end; pfn++) {
		if (pfn_valid(pfn)) {
			page = pfn_to_page(pfn);
			if (PageLRU(page))
				return pfn;
1375
			if (PageHuge(page)) {
1376
				if (page_huge_active(page))
1377 1378 1379 1380 1381
					return pfn;
				else
					pfn = round_up(pfn + 1,
						1 << compound_order(page)) - 1;
			}
K
KAMEZAWA Hiroyuki 已提交
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401
		}
	}
	return 0;
}

#define NR_OFFLINE_AT_ONCE_PAGES	(256)
static int
do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
{
	unsigned long pfn;
	struct page *page;
	int move_pages = NR_OFFLINE_AT_ONCE_PAGES;
	int not_managed = 0;
	int ret = 0;
	LIST_HEAD(source);

	for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) {
		if (!pfn_valid(pfn))
			continue;
		page = pfn_to_page(pfn);
1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414

		if (PageHuge(page)) {
			struct page *head = compound_head(page);
			pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1;
			if (compound_order(head) > PFN_SECTION_SHIFT) {
				ret = -EBUSY;
				break;
			}
			if (isolate_huge_page(page, &source))
				move_pages -= 1 << compound_order(head);
			continue;
		}

1415
		if (!get_page_unless_zero(page))
K
KAMEZAWA Hiroyuki 已提交
1416 1417 1418 1419 1420
			continue;
		/*
		 * We can skip free pages. And we can only deal with pages on
		 * LRU.
		 */
1421
		ret = isolate_lru_page(page);
K
KAMEZAWA Hiroyuki 已提交
1422
		if (!ret) { /* Success */
1423
			put_page(page);
1424
			list_add_tail(&page->lru, &source);
K
KAMEZAWA Hiroyuki 已提交
1425
			move_pages--;
1426 1427 1428
			inc_zone_page_state(page, NR_ISOLATED_ANON +
					    page_is_file_cache(page));

K
KAMEZAWA Hiroyuki 已提交
1429 1430
		} else {
#ifdef CONFIG_DEBUG_VM
1431 1432
			printk(KERN_ALERT "removing pfn %lx from LRU failed\n",
			       pfn);
1433
			dump_page(page, "failed to remove from LRU");
K
KAMEZAWA Hiroyuki 已提交
1434
#endif
1435
			put_page(page);
L
Lucas De Marchi 已提交
1436
			/* Because we don't have big zone->lock. we should
1437 1438 1439
			   check this again here. */
			if (page_count(page)) {
				not_managed++;
1440
				ret = -EBUSY;
1441 1442
				break;
			}
K
KAMEZAWA Hiroyuki 已提交
1443 1444
		}
	}
1445 1446
	if (!list_empty(&source)) {
		if (not_managed) {
1447
			putback_movable_pages(&source);
1448 1449
			goto out;
		}
1450 1451 1452 1453 1454

		/*
		 * alloc_migrate_target should be improooooved!!
		 * migrate_pages returns # of failed pages.
		 */
1455
		ret = migrate_pages(&source, alloc_migrate_target, NULL, 0,
1456
					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
1457
		if (ret)
1458
			putback_movable_pages(&source);
K
KAMEZAWA Hiroyuki 已提交
1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477
	}
out:
	return ret;
}

/*
 * remove from free_area[] and mark all as Reserved.
 */
static int
offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
			void *data)
{
	__offline_isolated_pages(start, start + nr_pages);
	return 0;
}

static void
offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
{
K
KAMEZAWA Hiroyuki 已提交
1478
	walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL,
K
KAMEZAWA Hiroyuki 已提交
1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490
				offline_isolated_pages_cb);
}

/*
 * Check all pages in range, recoreded as memory resource, are isolated.
 */
static int
check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
			void *data)
{
	int ret;
	long offlined = *(long *)data;
1491
	ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
K
KAMEZAWA Hiroyuki 已提交
1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
	offlined = nr_pages;
	if (!ret)
		*(long *)data += offlined;
	return ret;
}

static long
check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
{
	long offlined = 0;
	int ret;

K
KAMEZAWA Hiroyuki 已提交
1504
	ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined,
K
KAMEZAWA Hiroyuki 已提交
1505 1506 1507 1508 1509 1510
			check_pages_isolated_cb);
	if (ret < 0)
		offlined = (long)ret;
	return offlined;
}

1511
#ifdef CONFIG_MOVABLE_NODE
T
Tang Chen 已提交
1512 1513 1514 1515
/*
 * When CONFIG_MOVABLE_NODE, we permit offlining of a node which doesn't have
 * normal memory.
 */
1516 1517 1518 1519
static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
{
	return true;
}
T
Tang Chen 已提交
1520
#else /* CONFIG_MOVABLE_NODE */
1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
/* ensure the node has NORMAL memory if it is still online */
static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	unsigned long present_pages = 0;
	enum zone_type zt;

	for (zt = 0; zt <= ZONE_NORMAL; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;

	if (present_pages > nr_pages)
		return true;

	present_pages = 0;
	for (; zt <= ZONE_MOVABLE; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;

	/*
	 * we can't offline the last normal memory until all
	 * higher memory is offlined.
	 */
	return present_pages == 0;
}
T
Tang Chen 已提交
1544
#endif /* CONFIG_MOVABLE_NODE */
1545

1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568
static int __init cmdline_parse_movable_node(char *p)
{
#ifdef CONFIG_MOVABLE_NODE
	/*
	 * Memory used by the kernel cannot be hot-removed because Linux
	 * cannot migrate the kernel pages. When memory hotplug is
	 * enabled, we should prevent memblock from allocating memory
	 * for the kernel.
	 *
	 * ACPI SRAT records all hotpluggable memory ranges. But before
	 * SRAT is parsed, we don't know about it.
	 *
	 * The kernel image is loaded into memory at very early time. We
	 * cannot prevent this anyway. So on NUMA system, we set any
	 * node the kernel resides in as un-hotpluggable.
	 *
	 * Since on modern servers, one node could have double-digit
	 * gigabytes memory, we can assume the memory around the kernel
	 * image is also un-hotpluggable. So before SRAT is parsed, just
	 * allocate memory near the kernel image to try the best to keep
	 * the kernel away from hotpluggable memory.
	 */
	memblock_set_bottom_up(true);
1569
	movable_node_enabled = true;
1570 1571 1572 1573 1574 1575 1576
#else
	pr_warn("movable_node option not supported\n");
#endif
	return 0;
}
early_param("movable_node", cmdline_parse_movable_node);

1577 1578 1579 1580 1581 1582 1583 1584 1585
/* check which state of node_states will be changed when offline memory */
static void node_states_check_changes_offline(unsigned long nr_pages,
		struct zone *zone, struct memory_notify *arg)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	unsigned long present_pages = 0;
	enum zone_type zt, zone_last = ZONE_NORMAL;

	/*
1586 1587 1588
	 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
	 * contains nodes which have zones of 0...ZONE_NORMAL,
	 * set zone_last to ZONE_NORMAL.
1589
	 *
1590 1591 1592
	 * If we don't have HIGHMEM nor movable node,
	 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
	 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
1593
	 */
1594
	if (N_MEMORY == N_NORMAL_MEMORY)
1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610
		zone_last = ZONE_MOVABLE;

	/*
	 * check whether node_states[N_NORMAL_MEMORY] will be changed.
	 * If the memory to be offline is in a zone of 0...zone_last,
	 * and it is the last present memory, 0...zone_last will
	 * become empty after offline , thus we can determind we will
	 * need to clear the node from node_states[N_NORMAL_MEMORY].
	 */
	for (zt = 0; zt <= zone_last; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;
	if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
		arg->status_change_nid_normal = zone_to_nid(zone);
	else
		arg->status_change_nid_normal = -1;

1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
#ifdef CONFIG_HIGHMEM
	/*
	 * If we have movable node, node_states[N_HIGH_MEMORY]
	 * contains nodes which have zones of 0...ZONE_HIGHMEM,
	 * set zone_last to ZONE_HIGHMEM.
	 *
	 * If we don't have movable node, node_states[N_NORMAL_MEMORY]
	 * contains nodes which have zones of 0...ZONE_MOVABLE,
	 * set zone_last to ZONE_MOVABLE.
	 */
	zone_last = ZONE_HIGHMEM;
	if (N_MEMORY == N_HIGH_MEMORY)
		zone_last = ZONE_MOVABLE;

	for (; zt <= zone_last; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;
	if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
		arg->status_change_nid_high = zone_to_nid(zone);
	else
		arg->status_change_nid_high = -1;
#else
	arg->status_change_nid_high = arg->status_change_nid_normal;
#endif

1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658
	/*
	 * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE
	 */
	zone_last = ZONE_MOVABLE;

	/*
	 * check whether node_states[N_HIGH_MEMORY] will be changed
	 * If we try to offline the last present @nr_pages from the node,
	 * we can determind we will need to clear the node from
	 * node_states[N_HIGH_MEMORY].
	 */
	for (; zt <= zone_last; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;
	if (nr_pages >= present_pages)
		arg->status_change_nid = zone_to_nid(zone);
	else
		arg->status_change_nid = -1;
}

static void node_states_clear_node(int node, struct memory_notify *arg)
{
	if (arg->status_change_nid_normal >= 0)
		node_clear_state(node, N_NORMAL_MEMORY);

1659 1660
	if ((N_MEMORY != N_NORMAL_MEMORY) &&
	    (arg->status_change_nid_high >= 0))
1661
		node_clear_state(node, N_HIGH_MEMORY);
1662 1663 1664 1665

	if ((N_MEMORY != N_HIGH_MEMORY) &&
	    (arg->status_change_nid >= 0))
		node_clear_state(node, N_MEMORY);
1666 1667
}

1668
static int __ref __offline_pages(unsigned long start_pfn,
K
KAMEZAWA Hiroyuki 已提交
1669 1670 1671 1672
		  unsigned long end_pfn, unsigned long timeout)
{
	unsigned long pfn, nr_pages, expire;
	long offlined_pages;
1673
	int ret, drain, retry_max, node;
1674
	unsigned long flags;
K
KAMEZAWA Hiroyuki 已提交
1675
	struct zone *zone;
1676
	struct memory_notify arg;
K
KAMEZAWA Hiroyuki 已提交
1677 1678 1679 1680 1681 1682 1683 1684 1685 1686

	/* at least, alignment against pageblock is necessary */
	if (!IS_ALIGNED(start_pfn, pageblock_nr_pages))
		return -EINVAL;
	if (!IS_ALIGNED(end_pfn, pageblock_nr_pages))
		return -EINVAL;
	/* This makes hotplug much easier...and readable.
	   we assume this for now. .*/
	if (!test_pages_in_a_zone(start_pfn, end_pfn))
		return -EINVAL;
1687 1688 1689 1690 1691

	zone = page_zone(pfn_to_page(start_pfn));
	node = zone_to_nid(zone);
	nr_pages = end_pfn - start_pfn;

1692
	if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages))
1693
		return -EINVAL;
1694

K
KAMEZAWA Hiroyuki 已提交
1695
	/* set above range as isolated */
1696 1697
	ret = start_isolate_page_range(start_pfn, end_pfn,
				       MIGRATE_MOVABLE, true);
K
KAMEZAWA Hiroyuki 已提交
1698
	if (ret)
1699
		return ret;
1700 1701 1702

	arg.start_pfn = start_pfn;
	arg.nr_pages = nr_pages;
1703
	node_states_check_changes_offline(nr_pages, zone, &arg);
1704 1705 1706 1707 1708 1709

	ret = memory_notify(MEM_GOING_OFFLINE, &arg);
	ret = notifier_to_errno(ret);
	if (ret)
		goto failed_removal;

K
KAMEZAWA Hiroyuki 已提交
1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725
	pfn = start_pfn;
	expire = jiffies + timeout;
	drain = 0;
	retry_max = 5;
repeat:
	/* start memory hot removal */
	ret = -EAGAIN;
	if (time_after(jiffies, expire))
		goto failed_removal;
	ret = -EINTR;
	if (signal_pending(current))
		goto failed_removal;
	ret = 0;
	if (drain) {
		lru_add_drain_all();
		cond_resched();
1726
		drain_all_pages(zone);
K
KAMEZAWA Hiroyuki 已提交
1727 1728
	}

1729 1730
	pfn = scan_movable_pages(start_pfn, end_pfn);
	if (pfn) { /* We have movable pages */
K
KAMEZAWA Hiroyuki 已提交
1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743
		ret = do_migrate_range(pfn, end_pfn);
		if (!ret) {
			drain = 1;
			goto repeat;
		} else {
			if (ret < 0)
				if (--retry_max == 0)
					goto failed_removal;
			yield();
			drain = 1;
			goto repeat;
		}
	}
1744
	/* drain all zone's lru pagevec, this is asynchronous... */
K
KAMEZAWA Hiroyuki 已提交
1745 1746
	lru_add_drain_all();
	yield();
1747
	/* drain pcp pages, this is synchronous. */
1748
	drain_all_pages(zone);
1749 1750 1751 1752 1753
	/*
	 * dissolve free hugepages in the memory block before doing offlining
	 * actually in order to make hugetlbfs's object counting consistent.
	 */
	dissolve_free_huge_pages(start_pfn, end_pfn);
K
KAMEZAWA Hiroyuki 已提交
1754 1755 1756 1757 1758 1759 1760
	/* check again */
	offlined_pages = check_pages_isolated(start_pfn, end_pfn);
	if (offlined_pages < 0) {
		ret = -EBUSY;
		goto failed_removal;
	}
	printk(KERN_INFO "Offlined Pages %ld\n", offlined_pages);
1761
	/* Ok, all of our target is isolated.
K
KAMEZAWA Hiroyuki 已提交
1762 1763
	   We cannot do rollback at this point. */
	offline_isolated_pages(start_pfn, end_pfn);
1764
	/* reset pagetype flags and makes migrate type to be MOVABLE */
1765
	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
K
KAMEZAWA Hiroyuki 已提交
1766
	/* removal success */
1767
	adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages);
K
KAMEZAWA Hiroyuki 已提交
1768
	zone->present_pages -= offlined_pages;
1769 1770

	pgdat_resize_lock(zone->zone_pgdat, &flags);
K
KAMEZAWA Hiroyuki 已提交
1771
	zone->zone_pgdat->node_present_pages -= offlined_pages;
1772
	pgdat_resize_unlock(zone->zone_pgdat, &flags);
1773

1774 1775
	init_per_zone_wmark_min();

1776
	if (!populated_zone(zone)) {
1777
		zone_pcp_reset(zone);
1778 1779 1780 1781 1782
		mutex_lock(&zonelists_mutex);
		build_all_zonelists(NULL, NULL);
		mutex_unlock(&zonelists_mutex);
	} else
		zone_pcp_update(zone);
1783

1784 1785
	node_states_clear_node(node, &arg);
	if (arg.status_change_nid >= 0)
1786
		kswapd_stop(node);
1787

K
KAMEZAWA Hiroyuki 已提交
1788 1789
	vm_total_pages = nr_free_pagecache_pages();
	writeback_set_ratelimit();
1790 1791

	memory_notify(MEM_OFFLINE, &arg);
K
KAMEZAWA Hiroyuki 已提交
1792 1793 1794
	return 0;

failed_removal:
1795 1796 1797
	printk(KERN_INFO "memory offlining [mem %#010llx-%#010llx] failed\n",
	       (unsigned long long) start_pfn << PAGE_SHIFT,
	       ((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
1798
	memory_notify(MEM_CANCEL_OFFLINE, &arg);
K
KAMEZAWA Hiroyuki 已提交
1799
	/* pushback to free area */
1800
	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
K
KAMEZAWA Hiroyuki 已提交
1801 1802
	return ret;
}
1803

1804
/* Must be protected by mem_hotplug_begin() */
1805 1806 1807 1808
int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
{
	return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ);
}
1809
#endif /* CONFIG_MEMORY_HOTREMOVE */
1810

1811 1812 1813
/**
 * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn)
 * @start_pfn: start pfn of the memory range
1814
 * @end_pfn: end pfn of the memory range
1815 1816 1817 1818 1819 1820 1821 1822
 * @arg: argument passed to func
 * @func: callback for each memory section walked
 *
 * This function walks through all present mem sections in range
 * [start_pfn, end_pfn) and call func on each mem section.
 *
 * Returns the return value of func.
 */
1823
int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn,
1824
		void *arg, int (*func)(struct memory_block *, void *))
1825
{
1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846
	struct memory_block *mem = NULL;
	struct mem_section *section;
	unsigned long pfn, section_nr;
	int ret;

	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		section_nr = pfn_to_section_nr(pfn);
		if (!present_section_nr(section_nr))
			continue;

		section = __nr_to_section(section_nr);
		/* same memblock? */
		if (mem)
			if ((section_nr >= mem->start_section_nr) &&
			    (section_nr <= mem->end_section_nr))
				continue;

		mem = find_memory_block_hinted(section, mem);
		if (!mem)
			continue;

1847
		ret = func(mem, arg);
1848
		if (ret) {
1849 1850
			kobject_put(&mem->dev.kobj);
			return ret;
1851 1852 1853 1854 1855 1856
		}
	}

	if (mem)
		kobject_put(&mem->dev.kobj);

1857 1858 1859
	return 0;
}

1860
#ifdef CONFIG_MEMORY_HOTREMOVE
1861
static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
1862 1863 1864
{
	int ret = !is_memblock_offlined(mem);

1865 1866 1867 1868 1869
	if (unlikely(ret)) {
		phys_addr_t beginpa, endpa;

		beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
		endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;
1870
		pr_warn("removing memory fails, because memory "
1871 1872 1873
			"[%pa-%pa] is onlined\n",
			&beginpa, &endpa);
	}
1874 1875 1876 1877

	return ret;
}

1878
static int check_cpu_on_node(pg_data_t *pgdat)
1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893
{
	int cpu;

	for_each_present_cpu(cpu) {
		if (cpu_to_node(cpu) == pgdat->node_id)
			/*
			 * the cpu on this node isn't removed, and we can't
			 * offline this node.
			 */
			return -EBUSY;
	}

	return 0;
}

1894
static void unmap_cpu_on_node(pg_data_t *pgdat)
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904
{
#ifdef CONFIG_ACPI_NUMA
	int cpu;

	for_each_possible_cpu(cpu)
		if (cpu_to_node(cpu) == pgdat->node_id)
			numa_clear_node(cpu);
#endif
}

1905
static int check_and_unmap_cpu_on_node(pg_data_t *pgdat)
1906
{
1907
	int ret;
1908

1909
	ret = check_cpu_on_node(pgdat);
1910 1911 1912 1913 1914 1915 1916 1917
	if (ret)
		return ret;

	/*
	 * the node will be offlined when we come here, so we can clear
	 * the cpu_to_node() now.
	 */

1918
	unmap_cpu_on_node(pgdat);
1919 1920 1921
	return 0;
}

1922 1923 1924 1925 1926 1927 1928 1929
/**
 * try_offline_node
 *
 * Offline a node if all memory sections and cpus of the node are removed.
 *
 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
 * and online/offline operations before this call.
 */
1930
void try_offline_node(int nid)
1931
{
1932 1933 1934
	pg_data_t *pgdat = NODE_DATA(nid);
	unsigned long start_pfn = pgdat->node_start_pfn;
	unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
1935
	unsigned long pfn;
1936
	int i;
1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953

	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		unsigned long section_nr = pfn_to_section_nr(pfn);

		if (!present_section_nr(section_nr))
			continue;

		if (pfn_to_nid(pfn) != nid)
			continue;

		/*
		 * some memory sections of this node are not removed, and we
		 * can't offline node now.
		 */
		return;
	}

1954
	if (check_and_unmap_cpu_on_node(pgdat))
1955 1956 1957 1958 1959 1960 1961 1962
		return;

	/*
	 * all memory/cpu of this node are removed, we can offline this
	 * node now.
	 */
	node_set_offline(nid);
	unregister_one_node(nid);
1963 1964 1965 1966 1967

	/* free waittable in each zone */
	for (i = 0; i < MAX_NR_ZONES; i++) {
		struct zone *zone = pgdat->node_zones + i;

1968 1969 1970 1971
		/*
		 * wait_table may be allocated from boot memory,
		 * here only free if it's allocated by vmalloc.
		 */
1972
		if (is_vmalloc_addr(zone->wait_table)) {
1973
			vfree(zone->wait_table);
1974 1975
			zone->wait_table = NULL;
		}
1976
	}
1977
}
1978
EXPORT_SYMBOL(try_offline_node);
1979

1980 1981 1982 1983 1984 1985 1986
/**
 * remove_memory
 *
 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
 * and online/offline operations before this call, as required by
 * try_offline_node().
 */
1987
void __ref remove_memory(int nid, u64 start, u64 size)
1988
{
1989
	int ret;
1990

1991 1992
	BUG_ON(check_hotplug_memory_range(start, size));

1993
	mem_hotplug_begin();
1994 1995

	/*
1996 1997 1998
	 * All memory blocks must be offlined before removing memory.  Check
	 * whether all memory blocks in question are offline and trigger a BUG()
	 * if this is not the case.
1999
	 */
2000
	ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL,
2001
				check_memblock_offlined_cb);
2002
	if (ret)
2003
		BUG();
2004

2005 2006 2007
	/* remove memmap entry */
	firmware_map_remove(start, start + size, "System RAM");

2008 2009
	arch_remove_memory(start, size);

2010 2011
	try_offline_node(nid);

2012
	mem_hotplug_done();
2013 2014
}
EXPORT_SYMBOL_GPL(remove_memory);
2015
#endif /* CONFIG_MEMORY_HOTREMOVE */