memory_hotplug.c 54.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 *  linux/mm/memory_hotplug.c
 *
 *  Copyright (C)
 */

#include <linux/stddef.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
#include <linux/compiler.h>
13
#include <linux/export.h>
14
#include <linux/pagevec.h>
15
#include <linux/writeback.h>
16 17 18 19
#include <linux/slab.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/memory.h>
20
#include <linux/memremap.h>
21 22 23
#include <linux/memory_hotplug.h>
#include <linux/highmem.h>
#include <linux/vmalloc.h>
24
#include <linux/ioport.h>
K
KAMEZAWA Hiroyuki 已提交
25 26 27
#include <linux/delay.h>
#include <linux/migrate.h>
#include <linux/page-isolation.h>
28
#include <linux/pfn.h>
29
#include <linux/suspend.h>
30
#include <linux/mm_inline.h>
31
#include <linux/firmware-map.h>
32
#include <linux/stop_machine.h>
33
#include <linux/hugetlb.h>
34
#include <linux/memblock.h>
35
#include <linux/bootmem.h>
36
#include <linux/compaction.h>
37 38 39

#include <asm/tlbflush.h>

40 41
#include "internal.h"

42 43 44 45 46 47 48 49 50 51
/*
 * online_page_callback contains pointer to current page onlining function.
 * Initially it is generic_online_page(). If it is required it could be
 * changed by calling set_online_page_callback() for callback registration
 * and restore_online_page_callback() for generic callback restore.
 */

static void generic_online_page(struct page *page);

static online_page_callback_t online_page_callback = generic_online_page;
52
static DEFINE_MUTEX(online_page_callback_lock);
53

54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
/* The same as the cpu_hotplug lock, but for memory hotplug. */
static struct {
	struct task_struct *active_writer;
	struct mutex lock; /* Synchronizes accesses to refcount, */
	/*
	 * Also blocks the new readers during
	 * an ongoing mem hotplug operation.
	 */
	int refcount;

#ifdef CONFIG_DEBUG_LOCK_ALLOC
	struct lockdep_map dep_map;
#endif
} mem_hotplug = {
	.active_writer = NULL,
	.lock = __MUTEX_INITIALIZER(mem_hotplug.lock),
	.refcount = 0,
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	.dep_map = {.name = "mem_hotplug.lock" },
#endif
};

/* Lockdep annotations for get/put_online_mems() and mem_hotplug_begin/end() */
#define memhp_lock_acquire_read() lock_map_acquire_read(&mem_hotplug.dep_map)
#define memhp_lock_acquire()      lock_map_acquire(&mem_hotplug.dep_map)
#define memhp_lock_release()      lock_map_release(&mem_hotplug.dep_map)

81
#ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE
82
bool memhp_auto_online;
83 84 85
#else
bool memhp_auto_online = true;
#endif
86 87
EXPORT_SYMBOL_GPL(memhp_auto_online);

88 89 90 91 92 93 94 95 96 97 98
static int __init setup_memhp_default_state(char *str)
{
	if (!strcmp(str, "online"))
		memhp_auto_online = true;
	else if (!strcmp(str, "offline"))
		memhp_auto_online = false;

	return 1;
}
__setup("memhp_default_state=", setup_memhp_default_state);

99 100 101 102 103 104 105 106 107 108 109
void get_online_mems(void)
{
	might_sleep();
	if (mem_hotplug.active_writer == current)
		return;
	memhp_lock_acquire_read();
	mutex_lock(&mem_hotplug.lock);
	mem_hotplug.refcount++;
	mutex_unlock(&mem_hotplug.lock);

}
110

111
void put_online_mems(void)
112
{
113 114 115 116 117 118 119 120 121 122 123 124
	if (mem_hotplug.active_writer == current)
		return;
	mutex_lock(&mem_hotplug.lock);

	if (WARN_ON(!mem_hotplug.refcount))
		mem_hotplug.refcount++; /* try to fix things up */

	if (!--mem_hotplug.refcount && unlikely(mem_hotplug.active_writer))
		wake_up_process(mem_hotplug.active_writer);
	mutex_unlock(&mem_hotplug.lock);
	memhp_lock_release();

125 126
}

127
void mem_hotplug_begin(void)
128
{
129 130 131 132 133 134 135 136 137 138 139
	mem_hotplug.active_writer = current;

	memhp_lock_acquire();
	for (;;) {
		mutex_lock(&mem_hotplug.lock);
		if (likely(!mem_hotplug.refcount))
			break;
		__set_current_state(TASK_UNINTERRUPTIBLE);
		mutex_unlock(&mem_hotplug.lock);
		schedule();
	}
140 141
}

142
void mem_hotplug_done(void)
143 144 145 146 147
{
	mem_hotplug.active_writer = NULL;
	mutex_unlock(&mem_hotplug.lock);
	memhp_lock_release();
}
148

149 150 151 152 153
/* add this memory to iomem resource */
static struct resource *register_memory_resource(u64 start, u64 size)
{
	struct resource *res;
	res = kzalloc(sizeof(struct resource), GFP_KERNEL);
154 155
	if (!res)
		return ERR_PTR(-ENOMEM);
156 157 158 159

	res->name = "System RAM";
	res->start = start;
	res->end = start + size - 1;
160
	res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
161
	if (request_resource(&iomem_resource, res) < 0) {
162
		pr_debug("System RAM resource %pR cannot be added\n", res);
163
		kfree(res);
164
		return ERR_PTR(-EEXIST);
165 166 167 168 169 170 171 172 173 174 175 176 177
	}
	return res;
}

static void release_memory_resource(struct resource *res)
{
	if (!res)
		return;
	release_resource(res);
	kfree(res);
	return;
}

178
#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
179 180
void get_page_bootmem(unsigned long info,  struct page *page,
		      unsigned long type)
181
{
A
Andrea Arcangeli 已提交
182
	page->lru.next = (struct list_head *) type;
183 184
	SetPagePrivate(page);
	set_page_private(page, info);
185
	page_ref_inc(page);
186 187
}

188
void put_page_bootmem(struct page *page)
189
{
A
Andrea Arcangeli 已提交
190
	unsigned long type;
191

A
Andrea Arcangeli 已提交
192 193 194
	type = (unsigned long) page->lru.next;
	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
	       type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
195

196
	if (page_ref_dec_return(page) == 1) {
197 198
		ClearPagePrivate(page);
		set_page_private(page, 0);
A
Andrea Arcangeli 已提交
199
		INIT_LIST_HEAD(&page->lru);
200
		free_reserved_page(page);
201 202 203
	}
}

204 205
#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
#ifndef CONFIG_SPARSEMEM_VMEMMAP
206
static void register_page_bootmem_info_section(unsigned long start_pfn)
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
{
	unsigned long *usemap, mapsize, section_nr, i;
	struct mem_section *ms;
	struct page *page, *memmap;

	section_nr = pfn_to_section_nr(start_pfn);
	ms = __nr_to_section(section_nr);

	/* Get section's memmap address */
	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

	/*
	 * Get page for the memmap's phys address
	 * XXX: need more consideration for sparse_vmemmap...
	 */
	page = virt_to_page(memmap);
	mapsize = sizeof(struct page) * PAGES_PER_SECTION;
	mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;

	/* remember memmap's page */
	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, SECTION_INFO);

	usemap = __nr_to_section(section_nr)->pageblock_flags;
	page = virt_to_page(usemap);

	mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;

	for (i = 0; i < mapsize; i++, page++)
236
		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
237 238

}
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
#else /* CONFIG_SPARSEMEM_VMEMMAP */
static void register_page_bootmem_info_section(unsigned long start_pfn)
{
	unsigned long *usemap, mapsize, section_nr, i;
	struct mem_section *ms;
	struct page *page, *memmap;

	if (!pfn_valid(start_pfn))
		return;

	section_nr = pfn_to_section_nr(start_pfn);
	ms = __nr_to_section(section_nr);

	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

	register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);

	usemap = __nr_to_section(section_nr)->pageblock_flags;
	page = virt_to_page(usemap);

	mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;

	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
}
#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280

void register_page_bootmem_info_node(struct pglist_data *pgdat)
{
	unsigned long i, pfn, end_pfn, nr_pages;
	int node = pgdat->node_id;
	struct page *page;
	struct zone *zone;

	nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
	page = virt_to_page(pgdat);

	for (i = 0; i < nr_pages; i++, page++)
		get_page_bootmem(node, page, NODE_INFO);

	zone = &pgdat->node_zones[0];
	for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) {
281
		if (zone_is_initialized(zone)) {
282 283 284 285 286 287 288 289 290 291 292
			nr_pages = zone->wait_table_hash_nr_entries
				* sizeof(wait_queue_head_t);
			nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT;
			page = virt_to_page(zone->wait_table);

			for (i = 0; i < nr_pages; i++, page++)
				get_page_bootmem(node, page, NODE_INFO);
		}
	}

	pfn = pgdat->node_start_pfn;
293
	end_pfn = pgdat_end_pfn(pgdat);
294

295
	/* register section info */
296 297 298 299 300
	for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		/*
		 * Some platforms can assign the same pfn to multiple nodes - on
		 * node0 as well as nodeN.  To avoid registering a pfn against
		 * multiple nodes we check that this pfn does not already
301
		 * reside in some other nodes.
302
		 */
303
		if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
304 305
			register_page_bootmem_info_section(pfn);
	}
306
}
307
#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */
308

309 310
static void __meminit grow_zone_span(struct zone *zone, unsigned long start_pfn,
				     unsigned long end_pfn)
311 312 313 314 315
{
	unsigned long old_zone_end_pfn;

	zone_span_writelock(zone);

316
	old_zone_end_pfn = zone_end_pfn(zone);
317
	if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
318 319 320 321 322 323 324 325
		zone->zone_start_pfn = start_pfn;

	zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
				zone->zone_start_pfn;

	zone_span_writeunlock(zone);
}

326 327 328 329 330
static void resize_zone(struct zone *zone, unsigned long start_pfn,
		unsigned long end_pfn)
{
	zone_span_writelock(zone);

331 332 333 334 335 336 337 338 339 340 341
	if (end_pfn - start_pfn) {
		zone->zone_start_pfn = start_pfn;
		zone->spanned_pages = end_pfn - start_pfn;
	} else {
		/*
		 * make it consist as free_area_init_core(),
		 * if spanned_pages = 0, then keep start_pfn = 0
		 */
		zone->zone_start_pfn = 0;
		zone->spanned_pages = 0;
	}
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356

	zone_span_writeunlock(zone);
}

static void fix_zone_id(struct zone *zone, unsigned long start_pfn,
		unsigned long end_pfn)
{
	enum zone_type zid = zone_idx(zone);
	int nid = zone->zone_pgdat->node_id;
	unsigned long pfn;

	for (pfn = start_pfn; pfn < end_pfn; pfn++)
		set_page_links(pfn_to_page(pfn), zid, nid, pfn);
}

357
/* Can fail with -ENOMEM from allocating a wait table with vmalloc() or
358
 * alloc_bootmem_node_nopanic()/memblock_virt_alloc_node_nopanic() */
359 360 361 362
static int __ref ensure_zone_is_initialized(struct zone *zone,
			unsigned long start_pfn, unsigned long num_pages)
{
	if (!zone_is_initialized(zone))
363 364
		return init_currently_empty_zone(zone, start_pfn, num_pages);

365 366 367
	return 0;
}

368
static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2,
369 370
		unsigned long start_pfn, unsigned long end_pfn)
{
371
	int ret;
372
	unsigned long flags;
373 374
	unsigned long z1_start_pfn;

375 376 377
	ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn);
	if (ret)
		return ret;
378 379 380 381

	pgdat_resize_lock(z1->zone_pgdat, &flags);

	/* can't move pfns which are higher than @z2 */
382
	if (end_pfn > zone_end_pfn(z2))
383
		goto out_fail;
384
	/* the move out part must be at the left most of @z2 */
385 386 387 388 389 390
	if (start_pfn > z2->zone_start_pfn)
		goto out_fail;
	/* must included/overlap */
	if (end_pfn <= z2->zone_start_pfn)
		goto out_fail;

391
	/* use start_pfn for z1's start_pfn if z1 is empty */
392
	if (!zone_is_empty(z1))
393 394 395 396 397
		z1_start_pfn = z1->zone_start_pfn;
	else
		z1_start_pfn = start_pfn;

	resize_zone(z1, z1_start_pfn, end_pfn);
398
	resize_zone(z2, end_pfn, zone_end_pfn(z2));
399 400 401 402 403 404 405 406 407 408 409

	pgdat_resize_unlock(z1->zone_pgdat, &flags);

	fix_zone_id(z1, start_pfn, end_pfn);

	return 0;
out_fail:
	pgdat_resize_unlock(z1->zone_pgdat, &flags);
	return -1;
}

410
static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2,
411 412
		unsigned long start_pfn, unsigned long end_pfn)
{
413
	int ret;
414
	unsigned long flags;
415 416
	unsigned long z2_end_pfn;

417 418 419
	ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn);
	if (ret)
		return ret;
420 421 422 423 424 425 426

	pgdat_resize_lock(z1->zone_pgdat, &flags);

	/* can't move pfns which are lower than @z1 */
	if (z1->zone_start_pfn > start_pfn)
		goto out_fail;
	/* the move out part mast at the right most of @z1 */
427
	if (zone_end_pfn(z1) >  end_pfn)
428 429
		goto out_fail;
	/* must included/overlap */
430
	if (start_pfn >= zone_end_pfn(z1))
431 432
		goto out_fail;

433
	/* use end_pfn for z2's end_pfn if z2 is empty */
434
	if (!zone_is_empty(z2))
435
		z2_end_pfn = zone_end_pfn(z2);
436 437 438
	else
		z2_end_pfn = end_pfn;

439
	resize_zone(z1, z1->zone_start_pfn, start_pfn);
440
	resize_zone(z2, start_pfn, z2_end_pfn);
441 442 443 444 445 446 447 448 449 450 451

	pgdat_resize_unlock(z1->zone_pgdat, &flags);

	fix_zone_id(z2, start_pfn, end_pfn);

	return 0;
out_fail:
	pgdat_resize_unlock(z1->zone_pgdat, &flags);
	return -1;
}

452 453
static void __meminit grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn,
				      unsigned long end_pfn)
454
{
455
	unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat);
456

457
	if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
458 459 460 461 462 463
		pgdat->node_start_pfn = start_pfn;

	pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) -
					pgdat->node_start_pfn;
}

A
Al Viro 已提交
464
static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)
465 466 467 468 469
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	int nr_pages = PAGES_PER_SECTION;
	int nid = pgdat->node_id;
	int zone_type;
470
	unsigned long flags, pfn;
471
	int ret;
472 473

	zone_type = zone - pgdat->node_zones;
474 475 476
	ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages);
	if (ret)
		return ret;
477 478 479 480 481 482

	pgdat_resize_lock(zone->zone_pgdat, &flags);
	grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages);
	grow_pgdat_span(zone->zone_pgdat, phys_start_pfn,
			phys_start_pfn + nr_pages);
	pgdat_resize_unlock(zone->zone_pgdat, &flags);
D
Dave Hansen 已提交
483 484
	memmap_init_zone(nr_pages, nid, zone_type,
			 phys_start_pfn, MEMMAP_HOTPLUG);
485 486 487 488 489 490 491 492

	/* online_page_range is called later and expects pages reserved */
	for (pfn = phys_start_pfn; pfn < phys_start_pfn + nr_pages; pfn++) {
		if (!pfn_valid(pfn))
			continue;

		SetPageReserved(pfn_to_page(pfn));
	}
493
	return 0;
494 495
}

496 497
static int __meminit __add_section(int nid, struct zone *zone,
					unsigned long phys_start_pfn)
498 499 500
{
	int ret;

501 502 503
	if (pfn_valid(phys_start_pfn))
		return -EEXIST;

504
	ret = sparse_add_one_section(zone, phys_start_pfn);
505 506 507 508

	if (ret < 0)
		return ret;

509 510 511 512 513
	ret = __add_zone(zone, phys_start_pfn);

	if (ret < 0)
		return ret;

514
	return register_new_memory(nid, __pfn_to_section(phys_start_pfn));
515 516
}

517 518 519 520 521 522 523 524 525 526 527 528
/*
 * Reasonably generic function for adding memory.  It is
 * expected that archs that support memory hotplug will
 * call this function after deciding the zone to which to
 * add the new pages.
 */
int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
			unsigned long nr_pages)
{
	unsigned long i;
	int err = 0;
	int start_sec, end_sec;
529 530
	struct vmem_altmap *altmap;

531 532
	clear_zone_contiguous(zone);

533 534 535 536
	/* during initialize mem_map, align hot-added range to section */
	start_sec = pfn_to_section_nr(phys_start_pfn);
	end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1);

537 538 539 540 541 542 543 544
	altmap = to_vmem_altmap((unsigned long) pfn_to_page(phys_start_pfn));
	if (altmap) {
		/*
		 * Validate altmap is within bounds of the total request
		 */
		if (altmap->base_pfn != phys_start_pfn
				|| vmem_altmap_offset(altmap) > nr_pages) {
			pr_warn_once("memory add fail, invalid altmap\n");
545 546
			err = -EINVAL;
			goto out;
547 548 549 550
		}
		altmap->alloc = 0;
	}

551
	for (i = start_sec; i <= end_sec; i++) {
552
		err = __add_section(nid, zone, section_nr_to_pfn(i));
553 554 555 556 557 558 559 560 561 562

		/*
		 * EEXIST is finally dealt with by ioresource collision
		 * check. see add_memory() => register_memory_resource()
		 * Warning will be printed if there is collision.
		 */
		if (err && (err != -EEXIST))
			break;
		err = 0;
	}
563
	vmemmap_populate_print_last();
564 565
out:
	set_zone_contiguous(zone);
566 567 568 569 570
	return err;
}
EXPORT_SYMBOL_GPL(__add_pages);

#ifdef CONFIG_MEMORY_HOTREMOVE
571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
/* find the smallest valid pfn in the range [start_pfn, end_pfn) */
static int find_smallest_section_pfn(int nid, struct zone *zone,
				     unsigned long start_pfn,
				     unsigned long end_pfn)
{
	struct mem_section *ms;

	for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) {
		ms = __pfn_to_section(start_pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (unlikely(pfn_to_nid(start_pfn) != nid))
			continue;

		if (zone && zone != page_zone(pfn_to_page(start_pfn)))
			continue;

		return start_pfn;
	}

	return 0;
}

/* find the biggest valid pfn in the range [start_pfn, end_pfn). */
static int find_biggest_section_pfn(int nid, struct zone *zone,
				    unsigned long start_pfn,
				    unsigned long end_pfn)
{
	struct mem_section *ms;
	unsigned long pfn;

	/* pfn is the end pfn of a memory section. */
	pfn = end_pfn - 1;
	for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) {
		ms = __pfn_to_section(pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (unlikely(pfn_to_nid(pfn) != nid))
			continue;

		if (zone && zone != page_zone(pfn_to_page(pfn)))
			continue;

		return pfn;
	}

	return 0;
}

static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
			     unsigned long end_pfn)
{
627 628 629
	unsigned long zone_start_pfn = zone->zone_start_pfn;
	unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */
	unsigned long zone_end_pfn = z;
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
	unsigned long pfn;
	struct mem_section *ms;
	int nid = zone_to_nid(zone);

	zone_span_writelock(zone);
	if (zone_start_pfn == start_pfn) {
		/*
		 * If the section is smallest section in the zone, it need
		 * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
		 * In this case, we find second smallest valid mem_section
		 * for shrinking zone.
		 */
		pfn = find_smallest_section_pfn(nid, zone, end_pfn,
						zone_end_pfn);
		if (pfn) {
			zone->zone_start_pfn = pfn;
			zone->spanned_pages = zone_end_pfn - pfn;
		}
	} else if (zone_end_pfn == end_pfn) {
		/*
		 * If the section is biggest section in the zone, it need
		 * shrink zone->spanned_pages.
		 * In this case, we find second biggest valid mem_section for
		 * shrinking zone.
		 */
		pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
					       start_pfn);
		if (pfn)
			zone->spanned_pages = pfn - zone_start_pfn + 1;
	}

	/*
	 * The section is not biggest or smallest mem_section in the zone, it
	 * only creates a hole in the zone. So in this case, we need not
	 * change the zone. But perhaps, the zone has only hole data. Thus
	 * it check the zone has only hole or not.
	 */
	pfn = zone_start_pfn;
	for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) {
		ms = __pfn_to_section(pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (page_zone(pfn_to_page(pfn)) != zone)
			continue;

		 /* If the section is current section, it continues the loop */
		if (start_pfn == pfn)
			continue;

		/* If we find valid section, we have nothing to do */
		zone_span_writeunlock(zone);
		return;
	}

	/* The zone has no valid section */
	zone->zone_start_pfn = 0;
	zone->spanned_pages = 0;
	zone_span_writeunlock(zone);
}

static void shrink_pgdat_span(struct pglist_data *pgdat,
			      unsigned long start_pfn, unsigned long end_pfn)
{
695 696 697
	unsigned long pgdat_start_pfn = pgdat->node_start_pfn;
	unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */
	unsigned long pgdat_end_pfn = p;
698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772
	unsigned long pfn;
	struct mem_section *ms;
	int nid = pgdat->node_id;

	if (pgdat_start_pfn == start_pfn) {
		/*
		 * If the section is smallest section in the pgdat, it need
		 * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages.
		 * In this case, we find second smallest valid mem_section
		 * for shrinking zone.
		 */
		pfn = find_smallest_section_pfn(nid, NULL, end_pfn,
						pgdat_end_pfn);
		if (pfn) {
			pgdat->node_start_pfn = pfn;
			pgdat->node_spanned_pages = pgdat_end_pfn - pfn;
		}
	} else if (pgdat_end_pfn == end_pfn) {
		/*
		 * If the section is biggest section in the pgdat, it need
		 * shrink pgdat->node_spanned_pages.
		 * In this case, we find second biggest valid mem_section for
		 * shrinking zone.
		 */
		pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn,
					       start_pfn);
		if (pfn)
			pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1;
	}

	/*
	 * If the section is not biggest or smallest mem_section in the pgdat,
	 * it only creates a hole in the pgdat. So in this case, we need not
	 * change the pgdat.
	 * But perhaps, the pgdat has only hole data. Thus it check the pgdat
	 * has only hole or not.
	 */
	pfn = pgdat_start_pfn;
	for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) {
		ms = __pfn_to_section(pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (pfn_to_nid(pfn) != nid)
			continue;

		 /* If the section is current section, it continues the loop */
		if (start_pfn == pfn)
			continue;

		/* If we find valid section, we have nothing to do */
		return;
	}

	/* The pgdat has no valid section */
	pgdat->node_start_pfn = 0;
	pgdat->node_spanned_pages = 0;
}

static void __remove_zone(struct zone *zone, unsigned long start_pfn)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	int nr_pages = PAGES_PER_SECTION;
	int zone_type;
	unsigned long flags;

	zone_type = zone - pgdat->node_zones;

	pgdat_resize_lock(zone->zone_pgdat, &flags);
	shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
	shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages);
	pgdat_resize_unlock(zone->zone_pgdat, &flags);
}

773 774
static int __remove_section(struct zone *zone, struct mem_section *ms,
		unsigned long map_offset)
775
{
776 777
	unsigned long start_pfn;
	int scn_nr;
778 779 780 781 782 783 784 785 786
	int ret = -EINVAL;

	if (!valid_section(ms))
		return ret;

	ret = unregister_memory_section(ms);
	if (ret)
		return ret;

787 788 789 790
	scn_nr = __section_nr(ms);
	start_pfn = section_nr_to_pfn(scn_nr);
	__remove_zone(zone, start_pfn);

791
	sparse_remove_one_section(zone, ms, map_offset);
792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
	return 0;
}

/**
 * __remove_pages() - remove sections of pages from a zone
 * @zone: zone from which pages need to be removed
 * @phys_start_pfn: starting pageframe (must be aligned to start of a section)
 * @nr_pages: number of pages to remove (must be multiple of section size)
 *
 * Generic helper function to remove section mappings and sysfs entries
 * for the section of the memory we are removing. Caller needs to make
 * sure that pages are marked reserved and zones are adjust properly by
 * calling offline_pages().
 */
int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
		 unsigned long nr_pages)
{
809
	unsigned long i;
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835
	unsigned long map_offset = 0;
	int sections_to_remove, ret = 0;

	/* In the ZONE_DEVICE case device driver owns the memory region */
	if (is_dev_zone(zone)) {
		struct page *page = pfn_to_page(phys_start_pfn);
		struct vmem_altmap *altmap;

		altmap = to_vmem_altmap((unsigned long) page);
		if (altmap)
			map_offset = vmem_altmap_offset(altmap);
	} else {
		resource_size_t start, size;

		start = phys_start_pfn << PAGE_SHIFT;
		size = nr_pages * PAGE_SIZE;

		ret = release_mem_region_adjustable(&iomem_resource, start,
					size);
		if (ret) {
			resource_size_t endres = start + size - 1;

			pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
					&start, &endres, ret);
		}
	}
836

837 838
	clear_zone_contiguous(zone);

839 840 841 842 843 844 845 846 847
	/*
	 * We can only remove entire sections
	 */
	BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK);
	BUG_ON(nr_pages % PAGES_PER_SECTION);

	sections_to_remove = nr_pages / PAGES_PER_SECTION;
	for (i = 0; i < sections_to_remove; i++) {
		unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;
848 849 850

		ret = __remove_section(zone, __pfn_to_section(pfn), map_offset);
		map_offset = 0;
851 852 853
		if (ret)
			break;
	}
854 855 856

	set_zone_contiguous(zone);

857 858 859
	return ret;
}
EXPORT_SYMBOL_GPL(__remove_pages);
860
#endif /* CONFIG_MEMORY_HOTREMOVE */
861

862 863 864 865
int set_online_page_callback(online_page_callback_t callback)
{
	int rc = -EINVAL;

866 867
	get_online_mems();
	mutex_lock(&online_page_callback_lock);
868 869 870 871 872 873

	if (online_page_callback == generic_online_page) {
		online_page_callback = callback;
		rc = 0;
	}

874 875
	mutex_unlock(&online_page_callback_lock);
	put_online_mems();
876 877 878 879 880 881 882 883 884

	return rc;
}
EXPORT_SYMBOL_GPL(set_online_page_callback);

int restore_online_page_callback(online_page_callback_t callback)
{
	int rc = -EINVAL;

885 886
	get_online_mems();
	mutex_lock(&online_page_callback_lock);
887 888 889 890 891 892

	if (online_page_callback == callback) {
		online_page_callback = generic_online_page;
		rc = 0;
	}

893 894
	mutex_unlock(&online_page_callback_lock);
	put_online_mems();
895 896 897 898 899 900

	return rc;
}
EXPORT_SYMBOL_GPL(restore_online_page_callback);

void __online_page_set_limits(struct page *page)
901
{
902 903 904 905 906
}
EXPORT_SYMBOL_GPL(__online_page_set_limits);

void __online_page_increment_counters(struct page *page)
{
907
	adjust_managed_page_count(page, 1);
908 909
}
EXPORT_SYMBOL_GPL(__online_page_increment_counters);
910

911 912
void __online_page_free(struct page *page)
{
913
	__free_reserved_page(page);
914
}
915 916 917 918 919 920 921 922
EXPORT_SYMBOL_GPL(__online_page_free);

static void generic_online_page(struct page *page)
{
	__online_page_set_limits(page);
	__online_page_increment_counters(page);
	__online_page_free(page);
}
923

924 925
static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
			void *arg)
926 927
{
	unsigned long i;
928 929 930 931 932
	unsigned long onlined_pages = *(unsigned long *)arg;
	struct page *page;
	if (PageReserved(pfn_to_page(start_pfn)))
		for (i = 0; i < nr_pages; i++) {
			page = pfn_to_page(start_pfn + i);
933
			(*online_page_callback)(page);
934 935 936 937 938 939
			onlined_pages++;
		}
	*(unsigned long *)arg = onlined_pages;
	return 0;
}

940
#ifdef CONFIG_MOVABLE_NODE
T
Tang Chen 已提交
941 942 943 944
/*
 * When CONFIG_MOVABLE_NODE, we permit onlining of a node which doesn't have
 * normal memory.
 */
945 946 947 948
static bool can_online_high_movable(struct zone *zone)
{
	return true;
}
T
Tang Chen 已提交
949
#else /* CONFIG_MOVABLE_NODE */
950 951 952 953 954
/* ensure every online node has NORMAL memory */
static bool can_online_high_movable(struct zone *zone)
{
	return node_state(zone_to_nid(zone), N_NORMAL_MEMORY);
}
T
Tang Chen 已提交
955
#endif /* CONFIG_MOVABLE_NODE */
956

957 958 959 960 961 962 963 964
/* check which state of node_states will be changed when online memory */
static void node_states_check_changes_online(unsigned long nr_pages,
	struct zone *zone, struct memory_notify *arg)
{
	int nid = zone_to_nid(zone);
	enum zone_type zone_last = ZONE_NORMAL;

	/*
965 966 967
	 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
	 * contains nodes which have zones of 0...ZONE_NORMAL,
	 * set zone_last to ZONE_NORMAL.
968
	 *
969 970 971
	 * If we don't have HIGHMEM nor movable node,
	 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
	 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
972
	 */
973
	if (N_MEMORY == N_NORMAL_MEMORY)
974 975 976 977 978 979 980 981 982 983 984 985 986
		zone_last = ZONE_MOVABLE;

	/*
	 * if the memory to be online is in a zone of 0...zone_last, and
	 * the zones of 0...zone_last don't have memory before online, we will
	 * need to set the node to node_states[N_NORMAL_MEMORY] after
	 * the memory is online.
	 */
	if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY))
		arg->status_change_nid_normal = nid;
	else
		arg->status_change_nid_normal = -1;

987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
#ifdef CONFIG_HIGHMEM
	/*
	 * If we have movable node, node_states[N_HIGH_MEMORY]
	 * contains nodes which have zones of 0...ZONE_HIGHMEM,
	 * set zone_last to ZONE_HIGHMEM.
	 *
	 * If we don't have movable node, node_states[N_NORMAL_MEMORY]
	 * contains nodes which have zones of 0...ZONE_MOVABLE,
	 * set zone_last to ZONE_MOVABLE.
	 */
	zone_last = ZONE_HIGHMEM;
	if (N_MEMORY == N_HIGH_MEMORY)
		zone_last = ZONE_MOVABLE;

	if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY))
		arg->status_change_nid_high = nid;
	else
		arg->status_change_nid_high = -1;
#else
	arg->status_change_nid_high = arg->status_change_nid_normal;
#endif

1009 1010
	/*
	 * if the node don't have memory befor online, we will need to
1011
	 * set the node to node_states[N_MEMORY] after the memory
1012 1013
	 * is online.
	 */
1014
	if (!node_state(nid, N_MEMORY))
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
		arg->status_change_nid = nid;
	else
		arg->status_change_nid = -1;
}

static void node_states_set_node(int node, struct memory_notify *arg)
{
	if (arg->status_change_nid_normal >= 0)
		node_set_state(node, N_NORMAL_MEMORY);

1025 1026 1027 1028
	if (arg->status_change_nid_high >= 0)
		node_set_state(node, N_HIGH_MEMORY);

	node_set_state(node, N_MEMORY);
1029 1030
}

1031

1032
/* Must be protected by mem_hotplug_begin() */
1033
int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type)
1034
{
1035
	unsigned long flags;
1036 1037
	unsigned long onlined_pages = 0;
	struct zone *zone;
1038
	int need_zonelists_rebuild = 0;
1039 1040 1041 1042
	int nid;
	int ret;
	struct memory_notify arg;

1043 1044 1045 1046 1047 1048 1049
	/*
	 * This doesn't need a lock to do pfn_to_page().
	 * The section can't be removed here because of the
	 * memory_block->state_mutex.
	 */
	zone = page_zone(pfn_to_page(pfn));

1050 1051
	if ((zone_idx(zone) > ZONE_NORMAL ||
	    online_type == MMOP_ONLINE_MOVABLE) &&
1052
	    !can_online_high_movable(zone))
1053
		return -EINVAL;
1054

1055 1056
	if (online_type == MMOP_ONLINE_KERNEL &&
	    zone_idx(zone) == ZONE_MOVABLE) {
1057
		if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages))
1058
			return -EINVAL;
1059
	}
1060 1061
	if (online_type == MMOP_ONLINE_MOVABLE &&
	    zone_idx(zone) == ZONE_MOVABLE - 1) {
1062
		if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages))
1063
			return -EINVAL;
1064 1065 1066 1067 1068
	}

	/* Previous code may changed the zone of the pfn range */
	zone = page_zone(pfn_to_page(pfn));

1069 1070
	arg.start_pfn = pfn;
	arg.nr_pages = nr_pages;
1071
	node_states_check_changes_online(nr_pages, zone, &arg);
1072

1073
	nid = zone_to_nid(zone);
1074

1075 1076
	ret = memory_notify(MEM_GOING_ONLINE, &arg);
	ret = notifier_to_errno(ret);
1077 1078 1079
	if (ret)
		goto failed_addition;

1080 1081 1082 1083 1084
	/*
	 * If this zone is not populated, then it is not in zonelist.
	 * This means the page allocator ignores this zone.
	 * So, zonelist must be updated after online.
	 */
1085
	mutex_lock(&zonelists_mutex);
1086
	if (!populated_zone(zone)) {
1087
		need_zonelists_rebuild = 1;
1088 1089
		build_all_zonelists(NULL, zone);
	}
1090

K
KAMEZAWA Hiroyuki 已提交
1091
	ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
1092
		online_pages_range);
1093
	if (ret) {
1094 1095
		if (need_zonelists_rebuild)
			zone_pcp_reset(zone);
1096
		mutex_unlock(&zonelists_mutex);
1097
		goto failed_addition;
1098 1099
	}

1100
	zone->present_pages += onlined_pages;
1101 1102

	pgdat_resize_lock(zone->zone_pgdat, &flags);
1103
	zone->zone_pgdat->node_present_pages += onlined_pages;
1104 1105
	pgdat_resize_unlock(zone->zone_pgdat, &flags);

1106
	if (onlined_pages) {
1107
		node_states_set_node(nid, &arg);
1108
		if (need_zonelists_rebuild)
1109
			build_all_zonelists(NULL, NULL);
1110 1111 1112
		else
			zone_pcp_update(zone);
	}
1113

1114
	mutex_unlock(&zonelists_mutex);
1115 1116 1117

	init_per_zone_wmark_min();

1118
	if (onlined_pages) {
1119
		kswapd_run(nid);
1120 1121
		kcompactd_run(nid);
	}
1122

1123
	vm_total_pages = nr_free_pagecache_pages();
1124

1125
	writeback_set_ratelimit();
1126 1127 1128

	if (onlined_pages)
		memory_notify(MEM_ONLINE, &arg);
1129
	return 0;
1130 1131 1132 1133 1134 1135 1136

failed_addition:
	pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
		 (unsigned long long) pfn << PAGE_SHIFT,
		 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
	memory_notify(MEM_CANCEL_ONLINE, &arg);
	return ret;
1137
}
1138
#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
1139

1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
static void reset_node_present_pages(pg_data_t *pgdat)
{
	struct zone *z;

	for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
		z->present_pages = 0;

	pgdat->node_present_pages = 0;
}

1150 1151
/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
1152 1153 1154 1155
{
	struct pglist_data *pgdat;
	unsigned long zones_size[MAX_NR_ZONES] = {0};
	unsigned long zholes_size[MAX_NR_ZONES] = {0};
1156
	unsigned long start_pfn = PFN_DOWN(start);
1157

1158 1159 1160 1161 1162
	pgdat = NODE_DATA(nid);
	if (!pgdat) {
		pgdat = arch_alloc_nodedata(nid);
		if (!pgdat)
			return NULL;
1163

1164
		arch_refresh_nodedata(nid, pgdat);
1165 1166 1167 1168
	} else {
		/* Reset the nr_zones and classzone_idx to 0 before reuse */
		pgdat->nr_zones = 0;
		pgdat->classzone_idx = 0;
1169
	}
1170 1171 1172 1173

	/* we can use NODE_DATA(nid) from here */

	/* init node's zones as empty zones, we don't have any present pages.*/
1174
	free_area_init_node(nid, zones_size, start_pfn, zholes_size);
1175

1176 1177 1178 1179
	/*
	 * The node we allocated has no zone fallback lists. For avoiding
	 * to access not-initialized zonelist, build here.
	 */
1180
	mutex_lock(&zonelists_mutex);
1181
	build_all_zonelists(pgdat, NULL);
1182
	mutex_unlock(&zonelists_mutex);
1183

1184 1185 1186 1187 1188 1189 1190 1191
	/*
	 * zone->managed_pages is set to an approximate value in
	 * free_area_init_core(), which will cause
	 * /sys/device/system/node/nodeX/meminfo has wrong data.
	 * So reset it to 0 before any memory is onlined.
	 */
	reset_node_managed_pages(pgdat);

1192 1193 1194 1195 1196 1197 1198
	/*
	 * When memory is hot-added, all the memory is in offline state. So
	 * clear all zones' present_pages because they will be updated in
	 * online_pages() and offline_pages().
	 */
	reset_node_present_pages(pgdat);

1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
	return pgdat;
}

static void rollback_node_hotadd(int nid, pg_data_t *pgdat)
{
	arch_refresh_nodedata(nid, NULL);
	arch_free_nodedata(pgdat);
	return;
}

1209

1210 1211 1212
/**
 * try_online_node - online a node if offlined
 *
1213 1214
 * called by cpu_up() to online a node without onlined memory.
 */
1215
int try_online_node(int nid)
1216 1217 1218 1219
{
	pg_data_t	*pgdat;
	int	ret;

1220 1221 1222
	if (node_online(nid))
		return 0;

1223
	mem_hotplug_begin();
1224
	pgdat = hotadd_new_pgdat(nid, 0);
1225
	if (!pgdat) {
1226
		pr_err("Cannot online node %d due to NULL pgdat\n", nid);
1227 1228 1229 1230 1231 1232 1233
		ret = -ENOMEM;
		goto out;
	}
	node_set_online(nid);
	ret = register_one_node(nid);
	BUG_ON(ret);

1234 1235 1236 1237 1238 1239
	if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
		mutex_lock(&zonelists_mutex);
		build_all_zonelists(NULL, NULL);
		mutex_unlock(&zonelists_mutex);
	}

1240
out:
1241
	mem_hotplug_done();
1242 1243 1244
	return ret;
}

1245 1246
static int check_hotplug_memory_range(u64 start, u64 size)
{
1247
	u64 start_pfn = PFN_DOWN(start);
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261
	u64 nr_pages = size >> PAGE_SHIFT;

	/* Memory range must be aligned with section */
	if ((start_pfn & ~PAGE_SECTION_MASK) ||
	    (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) {
		pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n",
				(unsigned long long)start,
				(unsigned long long)size);
		return -EINVAL;
	}

	return 0;
}

1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281
/*
 * If movable zone has already been setup, newly added memory should be check.
 * If its address is higher than movable zone, it should be added as movable.
 * Without this check, movable zone may overlap with other zone.
 */
static int should_add_memory_movable(int nid, u64 start, u64 size)
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	pg_data_t *pgdat = NODE_DATA(nid);
	struct zone *movable_zone = pgdat->node_zones + ZONE_MOVABLE;

	if (zone_is_empty(movable_zone))
		return 0;

	if (movable_zone->zone_start_pfn <= start_pfn)
		return 1;

	return 0;
}

1282 1283
int zone_for_memory(int nid, u64 start, u64 size, int zone_default,
		bool for_device)
1284
{
1285 1286 1287 1288
#ifdef CONFIG_ZONE_DEVICE
	if (for_device)
		return ZONE_DEVICE;
#endif
1289 1290 1291 1292 1293 1294
	if (should_add_memory_movable(nid, start, size))
		return ZONE_MOVABLE;

	return zone_default;
}

1295 1296 1297 1298 1299
static int online_memory_block(struct memory_block *mem, void *arg)
{
	return memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE);
}

A
Al Viro 已提交
1300
/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
1301
int __ref add_memory_resource(int nid, struct resource *res, bool online)
1302
{
1303
	u64 start, size;
1304
	pg_data_t *pgdat = NULL;
1305 1306
	bool new_pgdat;
	bool new_node;
1307 1308
	int ret;

1309 1310 1311
	start = res->start;
	size = resource_size(res);

1312 1313 1314 1315
	ret = check_hotplug_memory_range(start, size);
	if (ret)
		return ret;

1316 1317 1318 1319
	{	/* Stupid hack to suppress address-never-null warning */
		void *p = NODE_DATA(nid);
		new_pgdat = !p;
	}
1320

1321
	mem_hotplug_begin();
1322

1323 1324 1325 1326 1327 1328 1329 1330
	/*
	 * Add new range to memblock so that when hotadd_new_pgdat() is called
	 * to allocate new pgdat, get_pfn_range_for_nid() will be able to find
	 * this new range and calculate total pages correctly.  The range will
	 * be removed at hot-remove time.
	 */
	memblock_add_node(start, size, nid);

1331 1332
	new_node = !node_online(nid);
	if (new_node) {
1333
		pgdat = hotadd_new_pgdat(nid, start);
1334
		ret = -ENOMEM;
1335
		if (!pgdat)
1336
			goto error;
1337 1338
	}

1339
	/* call arch's memory hotadd */
1340
	ret = arch_add_memory(nid, start, size, false);
1341

1342 1343 1344
	if (ret < 0)
		goto error;

1345
	/* we online node here. we can't roll back from here. */
1346 1347
	node_set_online(nid);

1348
	if (new_node) {
1349 1350 1351 1352 1353 1354 1355 1356 1357
		ret = register_one_node(nid);
		/*
		 * If sysfs file of new node can't create, cpu on the node
		 * can't be hot-added. There is no rollback way now.
		 * So, check by BUG_ON() to catch it reluctantly..
		 */
		BUG_ON(ret);
	}

1358 1359 1360
	/* create new memmap entry */
	firmware_map_add_hotplug(start, start + size, "System RAM");

1361 1362 1363 1364 1365
	/* online pages if requested */
	if (online)
		walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1),
				  NULL, online_memory_block);

1366 1367
	goto out;

1368 1369 1370 1371
error:
	/* rollback pgdat allocation and others */
	if (new_pgdat)
		rollback_node_hotadd(nid, pgdat);
1372
	memblock_remove(start, size);
1373

1374
out:
1375
	mem_hotplug_done();
1376 1377
	return ret;
}
1378 1379 1380 1381 1382 1383 1384 1385
EXPORT_SYMBOL_GPL(add_memory_resource);

int __ref add_memory(int nid, u64 start, u64 size)
{
	struct resource *res;
	int ret;

	res = register_memory_resource(start, size);
1386 1387
	if (IS_ERR(res))
		return PTR_ERR(res);
1388

1389
	ret = add_memory_resource(nid, res, memhp_auto_online);
1390 1391 1392 1393
	if (ret < 0)
		release_memory_resource(res);
	return ret;
}
1394
EXPORT_SYMBOL_GPL(add_memory);
K
KAMEZAWA Hiroyuki 已提交
1395 1396

#ifdef CONFIG_MEMORY_HOTREMOVE
1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415
/*
 * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy
 * set and the size of the free page is given by page_order(). Using this,
 * the function determines if the pageblock contains only free pages.
 * Due to buddy contraints, a free page at least the size of a pageblock will
 * be located at the start of the pageblock
 */
static inline int pageblock_free(struct page *page)
{
	return PageBuddy(page) && page_order(page) >= pageblock_order;
}

/* Return the start of the next active pageblock after a given page */
static struct page *next_active_pageblock(struct page *page)
{
	/* Ensure the starting page is pageblock-aligned */
	BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1));

	/* If the entire pageblock is free, move to the end of free page */
1416 1417 1418 1419 1420 1421 1422
	if (pageblock_free(page)) {
		int order;
		/* be careful. we don't have locks, page_order can be changed.*/
		order = page_order(page);
		if ((order < MAX_ORDER) && (order >= pageblock_order))
			return page + (1 << order);
	}
1423

1424
	return page + pageblock_nr_pages;
1425 1426 1427
}

/* Checks if this range of memory is likely to be hot-removable. */
1428
bool is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
1429 1430 1431 1432 1433 1434
{
	struct page *page = pfn_to_page(start_pfn);
	struct page *end_page = page + nr_pages;

	/* Check the starting page of each pageblock within the range */
	for (; page < end_page; page = next_active_pageblock(page)) {
1435
		if (!is_pageblock_removable_nolock(page))
1436
			return false;
1437
		cond_resched();
1438 1439 1440
	}

	/* All pageblocks in the memory block are likely to be hot-removable */
1441
	return true;
1442 1443
}

K
KAMEZAWA Hiroyuki 已提交
1444 1445 1446
/*
 * Confirm all pages in a range [start, end) is belongs to the same zone.
 */
1447
int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn)
K
KAMEZAWA Hiroyuki 已提交
1448
{
1449
	unsigned long pfn, sec_end_pfn;
K
KAMEZAWA Hiroyuki 已提交
1450 1451 1452
	struct zone *zone = NULL;
	struct page *page;
	int i;
1453
	for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn);
K
KAMEZAWA Hiroyuki 已提交
1454
	     pfn < end_pfn;
1455 1456 1457
	     pfn = sec_end_pfn + 1, sec_end_pfn += PAGES_PER_SECTION) {
		/* Make sure the memory section is present first */
		if (!present_section_nr(pfn_to_section_nr(pfn)))
K
KAMEZAWA Hiroyuki 已提交
1458
			continue;
1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472
		for (; pfn < sec_end_pfn && pfn < end_pfn;
		     pfn += MAX_ORDER_NR_PAGES) {
			i = 0;
			/* This is just a CONFIG_HOLES_IN_ZONE check.*/
			while ((i < MAX_ORDER_NR_PAGES) &&
				!pfn_valid_within(pfn + i))
				i++;
			if (i == MAX_ORDER_NR_PAGES)
				continue;
			page = pfn_to_page(pfn + i);
			if (zone && page_zone(page) != zone)
				return 0;
			zone = page_zone(page);
		}
K
KAMEZAWA Hiroyuki 已提交
1473 1474 1475 1476 1477
	}
	return 1;
}

/*
1478 1479 1480 1481
 * Scan pfn range [start,end) to find movable/migratable pages (LRU pages
 * and hugepages). We scan pfn because it's much easier than scanning over
 * linked list. This function returns the pfn of the first found movable
 * page if it's found, otherwise 0.
K
KAMEZAWA Hiroyuki 已提交
1482
 */
1483
static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
K
KAMEZAWA Hiroyuki 已提交
1484 1485 1486 1487 1488 1489 1490 1491
{
	unsigned long pfn;
	struct page *page;
	for (pfn = start; pfn < end; pfn++) {
		if (pfn_valid(pfn)) {
			page = pfn_to_page(pfn);
			if (PageLRU(page))
				return pfn;
1492
			if (PageHuge(page)) {
1493
				if (page_huge_active(page))
1494 1495 1496 1497 1498
					return pfn;
				else
					pfn = round_up(pfn + 1,
						1 << compound_order(page)) - 1;
			}
K
KAMEZAWA Hiroyuki 已提交
1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518
		}
	}
	return 0;
}

#define NR_OFFLINE_AT_ONCE_PAGES	(256)
static int
do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
{
	unsigned long pfn;
	struct page *page;
	int move_pages = NR_OFFLINE_AT_ONCE_PAGES;
	int not_managed = 0;
	int ret = 0;
	LIST_HEAD(source);

	for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) {
		if (!pfn_valid(pfn))
			continue;
		page = pfn_to_page(pfn);
1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531

		if (PageHuge(page)) {
			struct page *head = compound_head(page);
			pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1;
			if (compound_order(head) > PFN_SECTION_SHIFT) {
				ret = -EBUSY;
				break;
			}
			if (isolate_huge_page(page, &source))
				move_pages -= 1 << compound_order(head);
			continue;
		}

1532
		if (!get_page_unless_zero(page))
K
KAMEZAWA Hiroyuki 已提交
1533 1534 1535 1536 1537
			continue;
		/*
		 * We can skip free pages. And we can only deal with pages on
		 * LRU.
		 */
1538
		ret = isolate_lru_page(page);
K
KAMEZAWA Hiroyuki 已提交
1539
		if (!ret) { /* Success */
1540
			put_page(page);
1541
			list_add_tail(&page->lru, &source);
K
KAMEZAWA Hiroyuki 已提交
1542
			move_pages--;
1543 1544 1545
			inc_zone_page_state(page, NR_ISOLATED_ANON +
					    page_is_file_cache(page));

K
KAMEZAWA Hiroyuki 已提交
1546 1547
		} else {
#ifdef CONFIG_DEBUG_VM
1548
			pr_alert("removing pfn %lx from LRU failed\n", pfn);
1549
			dump_page(page, "failed to remove from LRU");
K
KAMEZAWA Hiroyuki 已提交
1550
#endif
1551
			put_page(page);
L
Lucas De Marchi 已提交
1552
			/* Because we don't have big zone->lock. we should
1553 1554 1555
			   check this again here. */
			if (page_count(page)) {
				not_managed++;
1556
				ret = -EBUSY;
1557 1558
				break;
			}
K
KAMEZAWA Hiroyuki 已提交
1559 1560
		}
	}
1561 1562
	if (!list_empty(&source)) {
		if (not_managed) {
1563
			putback_movable_pages(&source);
1564 1565
			goto out;
		}
1566 1567 1568 1569 1570

		/*
		 * alloc_migrate_target should be improooooved!!
		 * migrate_pages returns # of failed pages.
		 */
1571
		ret = migrate_pages(&source, alloc_migrate_target, NULL, 0,
1572
					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
1573
		if (ret)
1574
			putback_movable_pages(&source);
K
KAMEZAWA Hiroyuki 已提交
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593
	}
out:
	return ret;
}

/*
 * remove from free_area[] and mark all as Reserved.
 */
static int
offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
			void *data)
{
	__offline_isolated_pages(start, start + nr_pages);
	return 0;
}

static void
offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
{
K
KAMEZAWA Hiroyuki 已提交
1594
	walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL,
K
KAMEZAWA Hiroyuki 已提交
1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606
				offline_isolated_pages_cb);
}

/*
 * Check all pages in range, recoreded as memory resource, are isolated.
 */
static int
check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
			void *data)
{
	int ret;
	long offlined = *(long *)data;
1607
	ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
K
KAMEZAWA Hiroyuki 已提交
1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619
	offlined = nr_pages;
	if (!ret)
		*(long *)data += offlined;
	return ret;
}

static long
check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
{
	long offlined = 0;
	int ret;

K
KAMEZAWA Hiroyuki 已提交
1620
	ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined,
K
KAMEZAWA Hiroyuki 已提交
1621 1622 1623 1624 1625 1626
			check_pages_isolated_cb);
	if (ret < 0)
		offlined = (long)ret;
	return offlined;
}

1627
#ifdef CONFIG_MOVABLE_NODE
T
Tang Chen 已提交
1628 1629 1630 1631
/*
 * When CONFIG_MOVABLE_NODE, we permit offlining of a node which doesn't have
 * normal memory.
 */
1632 1633 1634 1635
static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
{
	return true;
}
T
Tang Chen 已提交
1636
#else /* CONFIG_MOVABLE_NODE */
1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659
/* ensure the node has NORMAL memory if it is still online */
static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	unsigned long present_pages = 0;
	enum zone_type zt;

	for (zt = 0; zt <= ZONE_NORMAL; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;

	if (present_pages > nr_pages)
		return true;

	present_pages = 0;
	for (; zt <= ZONE_MOVABLE; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;

	/*
	 * we can't offline the last normal memory until all
	 * higher memory is offlined.
	 */
	return present_pages == 0;
}
T
Tang Chen 已提交
1660
#endif /* CONFIG_MOVABLE_NODE */
1661

1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684
static int __init cmdline_parse_movable_node(char *p)
{
#ifdef CONFIG_MOVABLE_NODE
	/*
	 * Memory used by the kernel cannot be hot-removed because Linux
	 * cannot migrate the kernel pages. When memory hotplug is
	 * enabled, we should prevent memblock from allocating memory
	 * for the kernel.
	 *
	 * ACPI SRAT records all hotpluggable memory ranges. But before
	 * SRAT is parsed, we don't know about it.
	 *
	 * The kernel image is loaded into memory at very early time. We
	 * cannot prevent this anyway. So on NUMA system, we set any
	 * node the kernel resides in as un-hotpluggable.
	 *
	 * Since on modern servers, one node could have double-digit
	 * gigabytes memory, we can assume the memory around the kernel
	 * image is also un-hotpluggable. So before SRAT is parsed, just
	 * allocate memory near the kernel image to try the best to keep
	 * the kernel away from hotpluggable memory.
	 */
	memblock_set_bottom_up(true);
1685
	movable_node_enabled = true;
1686 1687 1688 1689 1690 1691 1692
#else
	pr_warn("movable_node option not supported\n");
#endif
	return 0;
}
early_param("movable_node", cmdline_parse_movable_node);

1693 1694 1695 1696 1697 1698 1699 1700 1701
/* check which state of node_states will be changed when offline memory */
static void node_states_check_changes_offline(unsigned long nr_pages,
		struct zone *zone, struct memory_notify *arg)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	unsigned long present_pages = 0;
	enum zone_type zt, zone_last = ZONE_NORMAL;

	/*
1702 1703 1704
	 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
	 * contains nodes which have zones of 0...ZONE_NORMAL,
	 * set zone_last to ZONE_NORMAL.
1705
	 *
1706 1707 1708
	 * If we don't have HIGHMEM nor movable node,
	 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
	 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
1709
	 */
1710
	if (N_MEMORY == N_NORMAL_MEMORY)
1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726
		zone_last = ZONE_MOVABLE;

	/*
	 * check whether node_states[N_NORMAL_MEMORY] will be changed.
	 * If the memory to be offline is in a zone of 0...zone_last,
	 * and it is the last present memory, 0...zone_last will
	 * become empty after offline , thus we can determind we will
	 * need to clear the node from node_states[N_NORMAL_MEMORY].
	 */
	for (zt = 0; zt <= zone_last; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;
	if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
		arg->status_change_nid_normal = zone_to_nid(zone);
	else
		arg->status_change_nid_normal = -1;

1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750
#ifdef CONFIG_HIGHMEM
	/*
	 * If we have movable node, node_states[N_HIGH_MEMORY]
	 * contains nodes which have zones of 0...ZONE_HIGHMEM,
	 * set zone_last to ZONE_HIGHMEM.
	 *
	 * If we don't have movable node, node_states[N_NORMAL_MEMORY]
	 * contains nodes which have zones of 0...ZONE_MOVABLE,
	 * set zone_last to ZONE_MOVABLE.
	 */
	zone_last = ZONE_HIGHMEM;
	if (N_MEMORY == N_HIGH_MEMORY)
		zone_last = ZONE_MOVABLE;

	for (; zt <= zone_last; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;
	if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
		arg->status_change_nid_high = zone_to_nid(zone);
	else
		arg->status_change_nid_high = -1;
#else
	arg->status_change_nid_high = arg->status_change_nid_normal;
#endif

1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
	/*
	 * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE
	 */
	zone_last = ZONE_MOVABLE;

	/*
	 * check whether node_states[N_HIGH_MEMORY] will be changed
	 * If we try to offline the last present @nr_pages from the node,
	 * we can determind we will need to clear the node from
	 * node_states[N_HIGH_MEMORY].
	 */
	for (; zt <= zone_last; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;
	if (nr_pages >= present_pages)
		arg->status_change_nid = zone_to_nid(zone);
	else
		arg->status_change_nid = -1;
}

static void node_states_clear_node(int node, struct memory_notify *arg)
{
	if (arg->status_change_nid_normal >= 0)
		node_clear_state(node, N_NORMAL_MEMORY);

1775 1776
	if ((N_MEMORY != N_NORMAL_MEMORY) &&
	    (arg->status_change_nid_high >= 0))
1777
		node_clear_state(node, N_HIGH_MEMORY);
1778 1779 1780 1781

	if ((N_MEMORY != N_HIGH_MEMORY) &&
	    (arg->status_change_nid >= 0))
		node_clear_state(node, N_MEMORY);
1782 1783
}

1784
static int __ref __offline_pages(unsigned long start_pfn,
K
KAMEZAWA Hiroyuki 已提交
1785 1786 1787 1788
		  unsigned long end_pfn, unsigned long timeout)
{
	unsigned long pfn, nr_pages, expire;
	long offlined_pages;
1789
	int ret, drain, retry_max, node;
1790
	unsigned long flags;
K
KAMEZAWA Hiroyuki 已提交
1791
	struct zone *zone;
1792
	struct memory_notify arg;
K
KAMEZAWA Hiroyuki 已提交
1793 1794 1795 1796 1797 1798 1799 1800 1801 1802

	/* at least, alignment against pageblock is necessary */
	if (!IS_ALIGNED(start_pfn, pageblock_nr_pages))
		return -EINVAL;
	if (!IS_ALIGNED(end_pfn, pageblock_nr_pages))
		return -EINVAL;
	/* This makes hotplug much easier...and readable.
	   we assume this for now. .*/
	if (!test_pages_in_a_zone(start_pfn, end_pfn))
		return -EINVAL;
1803 1804 1805 1806 1807

	zone = page_zone(pfn_to_page(start_pfn));
	node = zone_to_nid(zone);
	nr_pages = end_pfn - start_pfn;

1808
	if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages))
1809
		return -EINVAL;
1810

K
KAMEZAWA Hiroyuki 已提交
1811
	/* set above range as isolated */
1812 1813
	ret = start_isolate_page_range(start_pfn, end_pfn,
				       MIGRATE_MOVABLE, true);
K
KAMEZAWA Hiroyuki 已提交
1814
	if (ret)
1815
		return ret;
1816 1817 1818

	arg.start_pfn = start_pfn;
	arg.nr_pages = nr_pages;
1819
	node_states_check_changes_offline(nr_pages, zone, &arg);
1820 1821 1822 1823 1824 1825

	ret = memory_notify(MEM_GOING_OFFLINE, &arg);
	ret = notifier_to_errno(ret);
	if (ret)
		goto failed_removal;

K
KAMEZAWA Hiroyuki 已提交
1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841
	pfn = start_pfn;
	expire = jiffies + timeout;
	drain = 0;
	retry_max = 5;
repeat:
	/* start memory hot removal */
	ret = -EAGAIN;
	if (time_after(jiffies, expire))
		goto failed_removal;
	ret = -EINTR;
	if (signal_pending(current))
		goto failed_removal;
	ret = 0;
	if (drain) {
		lru_add_drain_all();
		cond_resched();
1842
		drain_all_pages(zone);
K
KAMEZAWA Hiroyuki 已提交
1843 1844
	}

1845 1846
	pfn = scan_movable_pages(start_pfn, end_pfn);
	if (pfn) { /* We have movable pages */
K
KAMEZAWA Hiroyuki 已提交
1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
		ret = do_migrate_range(pfn, end_pfn);
		if (!ret) {
			drain = 1;
			goto repeat;
		} else {
			if (ret < 0)
				if (--retry_max == 0)
					goto failed_removal;
			yield();
			drain = 1;
			goto repeat;
		}
	}
1860
	/* drain all zone's lru pagevec, this is asynchronous... */
K
KAMEZAWA Hiroyuki 已提交
1861 1862
	lru_add_drain_all();
	yield();
1863
	/* drain pcp pages, this is synchronous. */
1864
	drain_all_pages(zone);
1865 1866 1867 1868 1869
	/*
	 * dissolve free hugepages in the memory block before doing offlining
	 * actually in order to make hugetlbfs's object counting consistent.
	 */
	dissolve_free_huge_pages(start_pfn, end_pfn);
K
KAMEZAWA Hiroyuki 已提交
1870 1871 1872 1873 1874 1875
	/* check again */
	offlined_pages = check_pages_isolated(start_pfn, end_pfn);
	if (offlined_pages < 0) {
		ret = -EBUSY;
		goto failed_removal;
	}
1876
	pr_info("Offlined Pages %ld\n", offlined_pages);
1877
	/* Ok, all of our target is isolated.
K
KAMEZAWA Hiroyuki 已提交
1878 1879
	   We cannot do rollback at this point. */
	offline_isolated_pages(start_pfn, end_pfn);
1880
	/* reset pagetype flags and makes migrate type to be MOVABLE */
1881
	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
K
KAMEZAWA Hiroyuki 已提交
1882
	/* removal success */
1883
	adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages);
K
KAMEZAWA Hiroyuki 已提交
1884
	zone->present_pages -= offlined_pages;
1885 1886

	pgdat_resize_lock(zone->zone_pgdat, &flags);
K
KAMEZAWA Hiroyuki 已提交
1887
	zone->zone_pgdat->node_present_pages -= offlined_pages;
1888
	pgdat_resize_unlock(zone->zone_pgdat, &flags);
1889

1890 1891
	init_per_zone_wmark_min();

1892
	if (!populated_zone(zone)) {
1893
		zone_pcp_reset(zone);
1894 1895 1896 1897 1898
		mutex_lock(&zonelists_mutex);
		build_all_zonelists(NULL, NULL);
		mutex_unlock(&zonelists_mutex);
	} else
		zone_pcp_update(zone);
1899

1900
	node_states_clear_node(node, &arg);
1901
	if (arg.status_change_nid >= 0) {
1902
		kswapd_stop(node);
1903 1904
		kcompactd_stop(node);
	}
1905

K
KAMEZAWA Hiroyuki 已提交
1906 1907
	vm_total_pages = nr_free_pagecache_pages();
	writeback_set_ratelimit();
1908 1909

	memory_notify(MEM_OFFLINE, &arg);
K
KAMEZAWA Hiroyuki 已提交
1910 1911 1912
	return 0;

failed_removal:
1913 1914 1915
	pr_debug("memory offlining [mem %#010llx-%#010llx] failed\n",
		 (unsigned long long) start_pfn << PAGE_SHIFT,
		 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
1916
	memory_notify(MEM_CANCEL_OFFLINE, &arg);
K
KAMEZAWA Hiroyuki 已提交
1917
	/* pushback to free area */
1918
	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
K
KAMEZAWA Hiroyuki 已提交
1919 1920
	return ret;
}
1921

1922
/* Must be protected by mem_hotplug_begin() */
1923 1924 1925 1926
int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
{
	return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ);
}
1927
#endif /* CONFIG_MEMORY_HOTREMOVE */
1928

1929 1930 1931
/**
 * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn)
 * @start_pfn: start pfn of the memory range
1932
 * @end_pfn: end pfn of the memory range
1933 1934 1935 1936 1937 1938 1939 1940
 * @arg: argument passed to func
 * @func: callback for each memory section walked
 *
 * This function walks through all present mem sections in range
 * [start_pfn, end_pfn) and call func on each mem section.
 *
 * Returns the return value of func.
 */
1941
int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn,
1942
		void *arg, int (*func)(struct memory_block *, void *))
1943
{
1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964
	struct memory_block *mem = NULL;
	struct mem_section *section;
	unsigned long pfn, section_nr;
	int ret;

	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		section_nr = pfn_to_section_nr(pfn);
		if (!present_section_nr(section_nr))
			continue;

		section = __nr_to_section(section_nr);
		/* same memblock? */
		if (mem)
			if ((section_nr >= mem->start_section_nr) &&
			    (section_nr <= mem->end_section_nr))
				continue;

		mem = find_memory_block_hinted(section, mem);
		if (!mem)
			continue;

1965
		ret = func(mem, arg);
1966
		if (ret) {
1967 1968
			kobject_put(&mem->dev.kobj);
			return ret;
1969 1970 1971 1972 1973 1974
		}
	}

	if (mem)
		kobject_put(&mem->dev.kobj);

1975 1976 1977
	return 0;
}

1978
#ifdef CONFIG_MEMORY_HOTREMOVE
1979
static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
1980 1981 1982
{
	int ret = !is_memblock_offlined(mem);

1983 1984 1985 1986 1987
	if (unlikely(ret)) {
		phys_addr_t beginpa, endpa;

		beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
		endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;
J
Joe Perches 已提交
1988
		pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n",
1989 1990
			&beginpa, &endpa);
	}
1991 1992 1993 1994

	return ret;
}

1995
static int check_cpu_on_node(pg_data_t *pgdat)
1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010
{
	int cpu;

	for_each_present_cpu(cpu) {
		if (cpu_to_node(cpu) == pgdat->node_id)
			/*
			 * the cpu on this node isn't removed, and we can't
			 * offline this node.
			 */
			return -EBUSY;
	}

	return 0;
}

2011
static void unmap_cpu_on_node(pg_data_t *pgdat)
2012 2013 2014 2015 2016 2017 2018 2019 2020 2021
{
#ifdef CONFIG_ACPI_NUMA
	int cpu;

	for_each_possible_cpu(cpu)
		if (cpu_to_node(cpu) == pgdat->node_id)
			numa_clear_node(cpu);
#endif
}

2022
static int check_and_unmap_cpu_on_node(pg_data_t *pgdat)
2023
{
2024
	int ret;
2025

2026
	ret = check_cpu_on_node(pgdat);
2027 2028 2029 2030 2031 2032 2033 2034
	if (ret)
		return ret;

	/*
	 * the node will be offlined when we come here, so we can clear
	 * the cpu_to_node() now.
	 */

2035
	unmap_cpu_on_node(pgdat);
2036 2037 2038
	return 0;
}

2039 2040 2041 2042 2043 2044 2045 2046
/**
 * try_offline_node
 *
 * Offline a node if all memory sections and cpus of the node are removed.
 *
 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
 * and online/offline operations before this call.
 */
2047
void try_offline_node(int nid)
2048
{
2049 2050 2051
	pg_data_t *pgdat = NODE_DATA(nid);
	unsigned long start_pfn = pgdat->node_start_pfn;
	unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
2052
	unsigned long pfn;
2053
	int i;
2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070

	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		unsigned long section_nr = pfn_to_section_nr(pfn);

		if (!present_section_nr(section_nr))
			continue;

		if (pfn_to_nid(pfn) != nid)
			continue;

		/*
		 * some memory sections of this node are not removed, and we
		 * can't offline node now.
		 */
		return;
	}

2071
	if (check_and_unmap_cpu_on_node(pgdat))
2072 2073 2074 2075 2076 2077 2078 2079
		return;

	/*
	 * all memory/cpu of this node are removed, we can offline this
	 * node now.
	 */
	node_set_offline(nid);
	unregister_one_node(nid);
2080 2081 2082 2083 2084

	/* free waittable in each zone */
	for (i = 0; i < MAX_NR_ZONES; i++) {
		struct zone *zone = pgdat->node_zones + i;

2085 2086 2087 2088
		/*
		 * wait_table may be allocated from boot memory,
		 * here only free if it's allocated by vmalloc.
		 */
2089
		if (is_vmalloc_addr(zone->wait_table)) {
2090
			vfree(zone->wait_table);
2091 2092
			zone->wait_table = NULL;
		}
2093
	}
2094
}
2095
EXPORT_SYMBOL(try_offline_node);
2096

2097 2098 2099 2100 2101 2102 2103
/**
 * remove_memory
 *
 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
 * and online/offline operations before this call, as required by
 * try_offline_node().
 */
2104
void __ref remove_memory(int nid, u64 start, u64 size)
2105
{
2106
	int ret;
2107

2108 2109
	BUG_ON(check_hotplug_memory_range(start, size));

2110
	mem_hotplug_begin();
2111 2112

	/*
2113 2114 2115
	 * All memory blocks must be offlined before removing memory.  Check
	 * whether all memory blocks in question are offline and trigger a BUG()
	 * if this is not the case.
2116
	 */
2117
	ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL,
2118
				check_memblock_offlined_cb);
2119
	if (ret)
2120
		BUG();
2121

2122 2123
	/* remove memmap entry */
	firmware_map_remove(start, start + size, "System RAM");
2124 2125
	memblock_free(start, size);
	memblock_remove(start, size);
2126

2127 2128
	arch_remove_memory(start, size);

2129 2130
	try_offline_node(nid);

2131
	mem_hotplug_done();
2132 2133
}
EXPORT_SYMBOL_GPL(remove_memory);
2134
#endif /* CONFIG_MEMORY_HOTREMOVE */