memory_hotplug.c 48.1 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7 8 9
/*
 *  linux/mm/memory_hotplug.c
 *
 *  Copyright (C)
 */

#include <linux/stddef.h>
#include <linux/mm.h>
10
#include <linux/sched/signal.h>
11 12 13 14
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
#include <linux/compiler.h>
15
#include <linux/export.h>
16
#include <linux/pagevec.h>
17
#include <linux/writeback.h>
18 19 20 21
#include <linux/slab.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/memory.h>
22
#include <linux/memremap.h>
23 24 25
#include <linux/memory_hotplug.h>
#include <linux/highmem.h>
#include <linux/vmalloc.h>
26
#include <linux/ioport.h>
K
KAMEZAWA Hiroyuki 已提交
27 28 29
#include <linux/delay.h>
#include <linux/migrate.h>
#include <linux/page-isolation.h>
30
#include <linux/pfn.h>
31
#include <linux/suspend.h>
32
#include <linux/mm_inline.h>
33
#include <linux/firmware-map.h>
34
#include <linux/stop_machine.h>
35
#include <linux/hugetlb.h>
36
#include <linux/memblock.h>
37
#include <linux/compaction.h>
38
#include <linux/rmap.h>
39 40 41

#include <asm/tlbflush.h>

42
#include "internal.h"
43
#include "shuffle.h"
44

45 46 47 48 49 50 51 52
/*
 * online_page_callback contains pointer to current page onlining function.
 * Initially it is generic_online_page(). If it is required it could be
 * changed by calling set_online_page_callback() for callback registration
 * and restore_online_page_callback() for generic callback restore.
 */

static online_page_callback_t online_page_callback = generic_online_page;
53
static DEFINE_MUTEX(online_page_callback_lock);
54

55
DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock);
56

57 58 59 60
void get_online_mems(void)
{
	percpu_down_read(&mem_hotplug_lock);
}
61

62 63 64 65
void put_online_mems(void)
{
	percpu_up_read(&mem_hotplug_lock);
}
66

67 68
bool movable_node_enabled = false;

69
#ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE
70
bool memhp_auto_online;
71 72 73
#else
bool memhp_auto_online = true;
#endif
74 75
EXPORT_SYMBOL_GPL(memhp_auto_online);

76 77 78 79 80 81 82 83 84 85 86
static int __init setup_memhp_default_state(char *str)
{
	if (!strcmp(str, "online"))
		memhp_auto_online = true;
	else if (!strcmp(str, "offline"))
		memhp_auto_online = false;

	return 1;
}
__setup("memhp_default_state=", setup_memhp_default_state);

87
void mem_hotplug_begin(void)
88
{
89 90
	cpus_read_lock();
	percpu_down_write(&mem_hotplug_lock);
91 92
}

93
void mem_hotplug_done(void)
94
{
95 96
	percpu_up_write(&mem_hotplug_lock);
	cpus_read_unlock();
97
}
98

99 100
u64 max_mem_size = U64_MAX;

101 102 103
/* add this memory to iomem resource */
static struct resource *register_memory_resource(u64 start, u64 size)
{
104 105 106
	struct resource *res;
	unsigned long flags =  IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
	char *resource_name = "System RAM";
107 108 109 110

	if (start + size > max_mem_size)
		return ERR_PTR(-E2BIG);

111 112 113 114 115 116 117 118 119 120 121
	/*
	 * Request ownership of the new memory range.  This might be
	 * a child of an existing resource that was present but
	 * not marked as busy.
	 */
	res = __request_region(&iomem_resource, start, size,
			       resource_name, flags);

	if (!res) {
		pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n",
				start, start + size);
122
		return ERR_PTR(-EEXIST);
123 124 125 126 127 128 129 130 131 132 133 134
	}
	return res;
}

static void release_memory_resource(struct resource *res)
{
	if (!res)
		return;
	release_resource(res);
	kfree(res);
}

135
#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
136 137
void get_page_bootmem(unsigned long info,  struct page *page,
		      unsigned long type)
138
{
139
	page->freelist = (void *)type;
140 141
	SetPagePrivate(page);
	set_page_private(page, info);
142
	page_ref_inc(page);
143 144
}

145
void put_page_bootmem(struct page *page)
146
{
A
Andrea Arcangeli 已提交
147
	unsigned long type;
148

149
	type = (unsigned long) page->freelist;
A
Andrea Arcangeli 已提交
150 151
	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
	       type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
152

153
	if (page_ref_dec_return(page) == 1) {
154
		page->freelist = NULL;
155 156
		ClearPagePrivate(page);
		set_page_private(page, 0);
A
Andrea Arcangeli 已提交
157
		INIT_LIST_HEAD(&page->lru);
158
		free_reserved_page(page);
159 160 161
	}
}

162 163
#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
#ifndef CONFIG_SPARSEMEM_VMEMMAP
164
static void register_page_bootmem_info_section(unsigned long start_pfn)
165
{
166
	unsigned long mapsize, section_nr, i;
167 168
	struct mem_section *ms;
	struct page *page, *memmap;
169
	struct mem_section_usage *usage;
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188

	section_nr = pfn_to_section_nr(start_pfn);
	ms = __nr_to_section(section_nr);

	/* Get section's memmap address */
	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

	/*
	 * Get page for the memmap's phys address
	 * XXX: need more consideration for sparse_vmemmap...
	 */
	page = virt_to_page(memmap);
	mapsize = sizeof(struct page) * PAGES_PER_SECTION;
	mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;

	/* remember memmap's page */
	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, SECTION_INFO);

189 190
	usage = ms->usage;
	page = virt_to_page(usage);
191

192
	mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
193 194

	for (i = 0; i < mapsize; i++, page++)
195
		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
196 197

}
198 199 200
#else /* CONFIG_SPARSEMEM_VMEMMAP */
static void register_page_bootmem_info_section(unsigned long start_pfn)
{
201
	unsigned long mapsize, section_nr, i;
202 203
	struct mem_section *ms;
	struct page *page, *memmap;
204
	struct mem_section_usage *usage;
205 206 207 208 209 210 211 212

	section_nr = pfn_to_section_nr(start_pfn);
	ms = __nr_to_section(section_nr);

	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

	register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);

213 214
	usage = ms->usage;
	page = virt_to_page(usage);
215

216
	mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
217 218 219 220 221

	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
}
#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
222

223
void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
224 225 226 227 228 229 230 231 232 233 234 235
{
	unsigned long i, pfn, end_pfn, nr_pages;
	int node = pgdat->node_id;
	struct page *page;

	nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
	page = virt_to_page(pgdat);

	for (i = 0; i < nr_pages; i++, page++)
		get_page_bootmem(node, page, NODE_INFO);

	pfn = pgdat->node_start_pfn;
236
	end_pfn = pgdat_end_pfn(pgdat);
237

238
	/* register section info */
239 240 241 242 243
	for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		/*
		 * Some platforms can assign the same pfn to multiple nodes - on
		 * node0 as well as nodeN.  To avoid registering a pfn against
		 * multiple nodes we check that this pfn does not already
244
		 * reside in some other nodes.
245
		 */
246
		if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
247 248
			register_page_bootmem_info_section(pfn);
	}
249
}
250
#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */
251

252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
static int check_pfn_span(unsigned long pfn, unsigned long nr_pages,
		const char *reason)
{
	/*
	 * Disallow all operations smaller than a sub-section and only
	 * allow operations smaller than a section for
	 * SPARSEMEM_VMEMMAP. Note that check_hotplug_memory_range()
	 * enforces a larger memory_block_size_bytes() granularity for
	 * memory that will be marked online, so this check should only
	 * fire for direct arch_{add,remove}_memory() users outside of
	 * add_memory_resource().
	 */
	unsigned long min_align;

	if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP))
		min_align = PAGES_PER_SUBSECTION;
	else
		min_align = PAGES_PER_SECTION;
	if (!IS_ALIGNED(pfn, min_align)
			|| !IS_ALIGNED(nr_pages, min_align)) {
		WARN(1, "Misaligned __%s_pages start: %#lx end: #%lx\n",
				reason, pfn, pfn + nr_pages - 1);
		return -EINVAL;
	}
	return 0;
}

279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
static int check_hotplug_memory_addressable(unsigned long pfn,
					    unsigned long nr_pages)
{
	const u64 max_addr = PFN_PHYS(pfn + nr_pages) - 1;

	if (max_addr >> MAX_PHYSMEM_BITS) {
		const u64 max_allowed = (1ull << (MAX_PHYSMEM_BITS + 1)) - 1;
		WARN(1,
		     "Hotplugged memory exceeds maximum addressable address, range=%#llx-%#llx, maximum=%#llx\n",
		     (u64)PFN_PHYS(pfn), max_addr, max_allowed);
		return -E2BIG;
	}

	return 0;
}

295 296 297 298 299 300
/*
 * Reasonably generic function for adding memory.  It is
 * expected that archs that support memory hotplug will
 * call this function after deciding the zone to which to
 * add the new pages.
 */
301 302
int __ref __add_pages(int nid, unsigned long pfn, unsigned long nr_pages,
		struct mhp_restrictions *restrictions)
303
{
304 305
	int err;
	unsigned long nr, start_sec, end_sec;
306
	struct vmem_altmap *altmap = restrictions->altmap;
307

308 309 310 311
	err = check_hotplug_memory_addressable(pfn, nr_pages);
	if (err)
		return err;

312 313 314 315
	if (altmap) {
		/*
		 * Validate altmap is within bounds of the total request
		 */
316
		if (altmap->base_pfn != pfn
317 318
				|| vmem_altmap_offset(altmap) > nr_pages) {
			pr_warn_once("memory add fail, invalid altmap\n");
319
			return -EINVAL;
320 321 322 323
		}
		altmap->alloc = 0;
	}

324 325 326 327 328 329
	err = check_pfn_span(pfn, nr_pages, "add");
	if (err)
		return err;

	start_sec = pfn_to_section_nr(pfn);
	end_sec = pfn_to_section_nr(pfn + nr_pages - 1);
330
	for (nr = start_sec; nr <= end_sec; nr++) {
331 332 333 334
		unsigned long pfns;

		pfns = min(nr_pages, PAGES_PER_SECTION
				- (pfn & ~PAGE_SECTION_MASK));
335 336 337
		err = sparse_add_section(nid, pfn, pfns, altmap);
		if (err)
			break;
338 339
		pfn += pfns;
		nr_pages -= pfns;
340
		cond_resched();
341
	}
342
	vmemmap_populate_print_last();
343 344 345
	return err;
}

346
/* find the smallest valid pfn in the range [start_pfn, end_pfn) */
347
static unsigned long find_smallest_section_pfn(int nid, struct zone *zone,
348 349 350
				     unsigned long start_pfn,
				     unsigned long end_pfn)
{
351
	for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) {
352
		if (unlikely(!pfn_to_online_page(start_pfn)))
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
			continue;

		if (unlikely(pfn_to_nid(start_pfn) != nid))
			continue;

		if (zone && zone != page_zone(pfn_to_page(start_pfn)))
			continue;

		return start_pfn;
	}

	return 0;
}

/* find the biggest valid pfn in the range [start_pfn, end_pfn). */
368
static unsigned long find_biggest_section_pfn(int nid, struct zone *zone,
369 370 371 372 373 374 375
				    unsigned long start_pfn,
				    unsigned long end_pfn)
{
	unsigned long pfn;

	/* pfn is the end pfn of a memory section. */
	pfn = end_pfn - 1;
376
	for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) {
377
		if (unlikely(!pfn_to_online_page(pfn)))
378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394
			continue;

		if (unlikely(pfn_to_nid(pfn) != nid))
			continue;

		if (zone && zone != page_zone(pfn_to_page(pfn)))
			continue;

		return pfn;
	}

	return 0;
}

static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
			     unsigned long end_pfn)
{
395 396 397
	unsigned long zone_start_pfn = zone->zone_start_pfn;
	unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */
	unsigned long zone_end_pfn = z;
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434
	unsigned long pfn;
	int nid = zone_to_nid(zone);

	zone_span_writelock(zone);
	if (zone_start_pfn == start_pfn) {
		/*
		 * If the section is smallest section in the zone, it need
		 * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
		 * In this case, we find second smallest valid mem_section
		 * for shrinking zone.
		 */
		pfn = find_smallest_section_pfn(nid, zone, end_pfn,
						zone_end_pfn);
		if (pfn) {
			zone->zone_start_pfn = pfn;
			zone->spanned_pages = zone_end_pfn - pfn;
		}
	} else if (zone_end_pfn == end_pfn) {
		/*
		 * If the section is biggest section in the zone, it need
		 * shrink zone->spanned_pages.
		 * In this case, we find second biggest valid mem_section for
		 * shrinking zone.
		 */
		pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
					       start_pfn);
		if (pfn)
			zone->spanned_pages = pfn - zone_start_pfn + 1;
	}

	/*
	 * The section is not biggest or smallest mem_section in the zone, it
	 * only creates a hole in the zone. So in this case, we need not
	 * change the zone. But perhaps, the zone has only hole data. Thus
	 * it check the zone has only hole or not.
	 */
	pfn = zone_start_pfn;
435
	for (; pfn < zone_end_pfn; pfn += PAGES_PER_SUBSECTION) {
436
		if (unlikely(!pfn_to_online_page(pfn)))
437 438 439 440 441
			continue;

		if (page_zone(pfn_to_page(pfn)) != zone)
			continue;

442 443
		/* Skip range to be removed */
		if (pfn >= start_pfn && pfn < end_pfn)
444 445 446 447 448 449 450 451 452 453 454 455 456
			continue;

		/* If we find valid section, we have nothing to do */
		zone_span_writeunlock(zone);
		return;
	}

	/* The zone has no valid section */
	zone->zone_start_pfn = 0;
	zone->spanned_pages = 0;
	zone_span_writeunlock(zone);
}

457
static void update_pgdat_span(struct pglist_data *pgdat)
458
{
459 460 461 462 463 464 465 466 467
	unsigned long node_start_pfn = 0, node_end_pfn = 0;
	struct zone *zone;

	for (zone = pgdat->node_zones;
	     zone < pgdat->node_zones + MAX_NR_ZONES; zone++) {
		unsigned long zone_end_pfn = zone->zone_start_pfn +
					     zone->spanned_pages;

		/* No need to lock the zones, they can't change. */
468 469 470 471 472 473 474 475
		if (!zone->spanned_pages)
			continue;
		if (!node_end_pfn) {
			node_start_pfn = zone->zone_start_pfn;
			node_end_pfn = zone_end_pfn;
			continue;
		}

476 477 478 479
		if (zone_end_pfn > node_end_pfn)
			node_end_pfn = zone_end_pfn;
		if (zone->zone_start_pfn < node_start_pfn)
			node_start_pfn = zone->zone_start_pfn;
480 481
	}

482 483
	pgdat->node_start_pfn = node_start_pfn;
	pgdat->node_spanned_pages = node_end_pfn - node_start_pfn;
484 485
}

486 487 488
void __ref remove_pfn_range_from_zone(struct zone *zone,
				      unsigned long start_pfn,
				      unsigned long nr_pages)
489 490 491 492
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	unsigned long flags;

493 494 495
	/* Poison struct pages because they are now uninitialized again. */
	page_init_poison(pfn_to_page(start_pfn), sizeof(struct page) * nr_pages);

496 497 498 499 500 501 502 503 504 505
#ifdef CONFIG_ZONE_DEVICE
	/*
	 * Zone shrinking code cannot properly deal with ZONE_DEVICE. So
	 * we will not try to shrink the zones - which is okay as
	 * set_zone_contiguous() cannot deal with ZONE_DEVICE either way.
	 */
	if (zone_idx(zone) == ZONE_DEVICE)
		return;
#endif

506 507
	clear_zone_contiguous(zone);

508 509
	pgdat_resize_lock(zone->zone_pgdat, &flags);
	shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
510
	update_pgdat_span(pgdat);
511
	pgdat_resize_unlock(zone->zone_pgdat, &flags);
512 513

	set_zone_contiguous(zone);
514 515
}

516 517 518
static void __remove_section(unsigned long pfn, unsigned long nr_pages,
			     unsigned long map_offset,
			     struct vmem_altmap *altmap)
519
{
520
	struct mem_section *ms = __nr_to_section(pfn_to_section_nr(pfn));
521

522 523
	if (WARN_ON_ONCE(!valid_section(ms)))
		return;
524

525
	sparse_remove_section(ms, pfn, nr_pages, map_offset, altmap);
526 527 528
}

/**
529
 * __remove_pages() - remove sections of pages
530
 * @pfn: starting pageframe (must be aligned to start of a section)
531
 * @nr_pages: number of pages to remove (must be multiple of section size)
532
 * @altmap: alternative device page map or %NULL if default memmap is used
533 534 535 536 537 538
 *
 * Generic helper function to remove section mappings and sysfs entries
 * for the section of the memory we are removing. Caller needs to make
 * sure that pages are marked reserved and zones are adjust properly by
 * calling offline_pages().
 */
539 540
void __remove_pages(unsigned long pfn, unsigned long nr_pages,
		    struct vmem_altmap *altmap)
541
{
542
	unsigned long map_offset = 0;
543
	unsigned long nr, start_sec, end_sec;
544

545
	map_offset = vmem_altmap_offset(altmap);
546

547 548
	if (check_pfn_span(pfn, nr_pages, "remove"))
		return;
549

550 551
	start_sec = pfn_to_section_nr(pfn);
	end_sec = pfn_to_section_nr(pfn + nr_pages - 1);
552
	for (nr = start_sec; nr <= end_sec; nr++) {
553
		unsigned long pfns;
554

555
		cond_resched();
556 557
		pfns = min(nr_pages, PAGES_PER_SECTION
				- (pfn & ~PAGE_SECTION_MASK));
558
		__remove_section(pfn, pfns, map_offset, altmap);
559 560
		pfn += pfns;
		nr_pages -= pfns;
561
		map_offset = 0;
562 563 564
	}
}

565 566 567 568
int set_online_page_callback(online_page_callback_t callback)
{
	int rc = -EINVAL;

569 570
	get_online_mems();
	mutex_lock(&online_page_callback_lock);
571 572 573 574 575 576

	if (online_page_callback == generic_online_page) {
		online_page_callback = callback;
		rc = 0;
	}

577 578
	mutex_unlock(&online_page_callback_lock);
	put_online_mems();
579 580 581 582 583 584 585 586 587

	return rc;
}
EXPORT_SYMBOL_GPL(set_online_page_callback);

int restore_online_page_callback(online_page_callback_t callback)
{
	int rc = -EINVAL;

588 589
	get_online_mems();
	mutex_lock(&online_page_callback_lock);
590 591 592 593 594 595

	if (online_page_callback == callback) {
		online_page_callback = generic_online_page;
		rc = 0;
	}

596 597
	mutex_unlock(&online_page_callback_lock);
	put_online_mems();
598 599 600 601 602

	return rc;
}
EXPORT_SYMBOL_GPL(restore_online_page_callback);

603
void generic_online_page(struct page *page, unsigned int order)
604
{
605
	kernel_map_pages(page, 1 << order, 1);
606 607 608 609 610 611 612
	__free_pages_core(page, order);
	totalram_pages_add(1UL << order);
#ifdef CONFIG_HIGHMEM
	if (PageHighMem(page))
		totalhigh_pages_add(1UL << order);
#endif
}
613
EXPORT_SYMBOL_GPL(generic_online_page);
614

615 616
static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
			void *arg)
617
{
618 619 620 621 622 623 624 625 626 627 628
	const unsigned long end_pfn = start_pfn + nr_pages;
	unsigned long pfn;
	int order;

	/*
	 * Online the pages. The callback might decide to keep some pages
	 * PG_reserved (to add them to the buddy later), but we still account
	 * them as being online/belonging to this zone ("present").
	 */
	for (pfn = start_pfn; pfn < end_pfn; pfn += 1ul << order) {
		order = min(MAX_ORDER - 1, get_order(PFN_PHYS(end_pfn - pfn)));
629 630 631
		/* __free_pages_core() wants pfns to be aligned to the order */
		if (WARN_ON_ONCE(!IS_ALIGNED(pfn, 1ul << order)))
			order = 0;
632 633
		(*online_page_callback)(pfn_to_page(pfn), order);
	}
634

635 636
	/* mark all involved sections as online */
	online_mem_sections(start_pfn, end_pfn);
637

638
	*(unsigned long *)arg += nr_pages;
639 640 641
	return 0;
}

642 643 644 645 646 647
/* check which state of node_states will be changed when online memory */
static void node_states_check_changes_online(unsigned long nr_pages,
	struct zone *zone, struct memory_notify *arg)
{
	int nid = zone_to_nid(zone);

648 649 650
	arg->status_change_nid = NUMA_NO_NODE;
	arg->status_change_nid_normal = NUMA_NO_NODE;
	arg->status_change_nid_high = NUMA_NO_NODE;
651

652 653 654
	if (!node_state(nid, N_MEMORY))
		arg->status_change_nid = nid;
	if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY))
655
		arg->status_change_nid_normal = nid;
656
#ifdef CONFIG_HIGHMEM
657
	if (zone_idx(zone) <= ZONE_HIGHMEM && !node_state(nid, N_HIGH_MEMORY))
658 659
		arg->status_change_nid_high = nid;
#endif
660 661 662 663 664 665 666
}

static void node_states_set_node(int node, struct memory_notify *arg)
{
	if (arg->status_change_nid_normal >= 0)
		node_set_state(node, N_NORMAL_MEMORY);

667 668 669
	if (arg->status_change_nid_high >= 0)
		node_set_state(node, N_HIGH_MEMORY);

670 671
	if (arg->status_change_nid >= 0)
		node_set_state(node, N_MEMORY);
672 673
}

674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn,
		unsigned long nr_pages)
{
	unsigned long old_end_pfn = zone_end_pfn(zone);

	if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
		zone->zone_start_pfn = start_pfn;

	zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn;
}

static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn,
                                     unsigned long nr_pages)
{
	unsigned long old_end_pfn = pgdat_end_pfn(pgdat);

	if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
		pgdat->node_start_pfn = start_pfn;

	pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn;

695 696 697 698 699 700
}
/*
 * Associate the pfn range with the given zone, initializing the memmaps
 * and resizing the pgdat/zone data to span the added pages. After this
 * call, all affected pages are PG_reserved.
 */
701 702
void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
		unsigned long nr_pages, struct vmem_altmap *altmap)
703 704 705 706
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	int nid = pgdat->node_id;
	unsigned long flags;
707

708 709 710 711 712
	clear_zone_contiguous(zone);

	/* TODO Huh pgdat is irqsave while zone is not. It used to be like that before */
	pgdat_resize_lock(pgdat, &flags);
	zone_span_writelock(zone);
713 714
	if (zone_is_empty(zone))
		init_currently_empty_zone(zone, start_pfn, nr_pages);
715 716 717 718 719 720 721 722 723 724 725
	resize_zone_range(zone, start_pfn, nr_pages);
	zone_span_writeunlock(zone);
	resize_pgdat_range(pgdat, start_pfn, nr_pages);
	pgdat_resize_unlock(pgdat, &flags);

	/*
	 * TODO now we have a visible range of pages which are not associated
	 * with their zone properly. Not nice but set_pfnblock_flags_mask
	 * expects the zone spans the pfn range. All the pages in the range
	 * are reserved so nobody should be touching them so we should be safe
	 */
726 727
	memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn,
			MEMMAP_HOTPLUG, altmap);
728 729 730 731

	set_zone_contiguous(zone);
}

732 733 734 735 736
/*
 * Returns a default kernel memory zone for the given pfn range.
 * If no kernel zone covers this pfn range it will automatically go
 * to the ZONE_NORMAL.
 */
737
static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn,
738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
		unsigned long nr_pages)
{
	struct pglist_data *pgdat = NODE_DATA(nid);
	int zid;

	for (zid = 0; zid <= ZONE_NORMAL; zid++) {
		struct zone *zone = &pgdat->node_zones[zid];

		if (zone_intersects(zone, start_pfn, nr_pages))
			return zone;
	}

	return &pgdat->node_zones[ZONE_NORMAL];
}

753 754
static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn,
		unsigned long nr_pages)
755
{
756 757 758 759 760
	struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn,
			nr_pages);
	struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
	bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages);
	bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages);
761 762

	/*
763 764
	 * We inherit the existing zone in a simple case where zones do not
	 * overlap in the given range
765
	 */
766 767
	if (in_kernel ^ in_movable)
		return (in_kernel) ? kernel_zone : movable_zone;
768

769 770 771 772 773 774
	/*
	 * If the range doesn't belong to any zone or two zones overlap in the
	 * given range then we use movable zone only if movable_node is
	 * enabled because we always online to a kernel zone by default.
	 */
	return movable_node_enabled ? movable_zone : kernel_zone;
775 776
}

777 778
struct zone * zone_for_pfn_range(int online_type, int nid, unsigned start_pfn,
		unsigned long nr_pages)
779
{
780 781
	if (online_type == MMOP_ONLINE_KERNEL)
		return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages);
782

783 784
	if (online_type == MMOP_ONLINE_MOVABLE)
		return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
785

786
	return default_zone_for_pfn(nid, start_pfn, nr_pages);
787 788
}

789 790
int __ref online_pages(unsigned long pfn, unsigned long nr_pages,
		       int online_type, int nid)
791
{
792
	unsigned long flags;
793 794
	unsigned long onlined_pages = 0;
	struct zone *zone;
795
	int need_zonelists_rebuild = 0;
796 797
	int ret;
	struct memory_notify arg;
798

799 800
	mem_hotplug_begin();

801
	/* associate pfn range with the zone */
802 803
	zone = zone_for_pfn_range(online_type, nid, pfn, nr_pages);
	move_pfn_range_to_zone(zone, pfn, nr_pages, NULL);
804

805 806
	arg.start_pfn = pfn;
	arg.nr_pages = nr_pages;
807
	node_states_check_changes_online(nr_pages, zone, &arg);
808 809 810

	ret = memory_notify(MEM_GOING_ONLINE, &arg);
	ret = notifier_to_errno(ret);
811 812 813
	if (ret)
		goto failed_addition;

814 815 816 817 818
	/*
	 * If this zone is not populated, then it is not in zonelist.
	 * This means the page allocator ignores this zone.
	 * So, zonelist must be updated after online.
	 */
819
	if (!populated_zone(zone)) {
820
		need_zonelists_rebuild = 1;
821
		setup_zone_pageset(zone);
822
	}
823

K
KAMEZAWA Hiroyuki 已提交
824
	ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
825
		online_pages_range);
826
	if (ret) {
827
		/* not a single memory resource was applicable */
828 829
		if (need_zonelists_rebuild)
			zone_pcp_reset(zone);
830
		goto failed_addition;
831 832
	}

833
	zone->present_pages += onlined_pages;
834 835

	pgdat_resize_lock(zone->zone_pgdat, &flags);
836
	zone->zone_pgdat->node_present_pages += onlined_pages;
837 838
	pgdat_resize_unlock(zone->zone_pgdat, &flags);

839 840
	shuffle_zone(zone);

841 842 843 844 845
	node_states_set_node(nid, &arg);
	if (need_zonelists_rebuild)
		build_all_zonelists(NULL);
	else
		zone_pcp_update(zone);
846

847 848
	init_per_zone_wmark_min();

849 850
	kswapd_run(nid);
	kcompactd_run(nid);
851

852
	vm_total_pages = nr_free_pagecache_pages();
853

854
	writeback_set_ratelimit();
855

856
	memory_notify(MEM_ONLINE, &arg);
857
	mem_hotplug_done();
858
	return 0;
859 860 861 862 863 864

failed_addition:
	pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
		 (unsigned long long) pfn << PAGE_SHIFT,
		 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
	memory_notify(MEM_CANCEL_ONLINE, &arg);
865
	remove_pfn_range_from_zone(zone, pfn, nr_pages);
866
	mem_hotplug_done();
867
	return ret;
868
}
869
#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
870

871 872 873 874 875 876 877 878 879 880
static void reset_node_present_pages(pg_data_t *pgdat)
{
	struct zone *z;

	for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
		z->present_pages = 0;

	pgdat->node_present_pages = 0;
}

881 882
/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
883 884
{
	struct pglist_data *pgdat;
885
	unsigned long start_pfn = PFN_DOWN(start);
886

887 888 889 890 891
	pgdat = NODE_DATA(nid);
	if (!pgdat) {
		pgdat = arch_alloc_nodedata(nid);
		if (!pgdat)
			return NULL;
892

893 894
		pgdat->per_cpu_nodestats =
			alloc_percpu(struct per_cpu_nodestat);
895
		arch_refresh_nodedata(nid, pgdat);
896
	} else {
897
		int cpu;
898 899 900 901 902
		/*
		 * Reset the nr_zones, order and classzone_idx before reuse.
		 * Note that kswapd will init kswapd_classzone_idx properly
		 * when it starts in the near future.
		 */
903
		pgdat->nr_zones = 0;
904 905
		pgdat->kswapd_order = 0;
		pgdat->kswapd_classzone_idx = 0;
906 907 908 909 910 911
		for_each_online_cpu(cpu) {
			struct per_cpu_nodestat *p;

			p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
			memset(p, 0, sizeof(*p));
		}
912
	}
913 914 915

	/* we can use NODE_DATA(nid) from here */

916 917 918
	pgdat->node_id = nid;
	pgdat->node_start_pfn = start_pfn;

919
	/* init node's zones as empty zones, we don't have any present pages.*/
920
	free_area_init_core_hotplug(nid);
921

922 923 924 925
	/*
	 * The node we allocated has no zone fallback lists. For avoiding
	 * to access not-initialized zonelist, build here.
	 */
926
	build_all_zonelists(pgdat);
927

928 929 930 931 932
	/*
	 * When memory is hot-added, all the memory is in offline state. So
	 * clear all zones' present_pages because they will be updated in
	 * online_pages() and offline_pages().
	 */
933
	reset_node_managed_pages(pgdat);
934 935
	reset_node_present_pages(pgdat);

936 937 938
	return pgdat;
}

939
static void rollback_node_hotadd(int nid)
940
{
941 942
	pg_data_t *pgdat = NODE_DATA(nid);

943
	arch_refresh_nodedata(nid, NULL);
944
	free_percpu(pgdat->per_cpu_nodestats);
945 946 947
	arch_free_nodedata(pgdat);
}

948

949 950
/**
 * try_online_node - online a node if offlined
951
 * @nid: the node ID
952 953
 * @start: start addr of the node
 * @set_node_online: Whether we want to online the node
954
 * called by cpu_up() to online a node without onlined memory.
955 956 957 958 959
 *
 * Returns:
 * 1 -> a new node has been allocated
 * 0 -> the node is already online
 * -ENOMEM -> the node could not be allocated
960
 */
961
static int __try_online_node(int nid, u64 start, bool set_node_online)
962
{
963 964
	pg_data_t *pgdat;
	int ret = 1;
965

966 967 968
	if (node_online(nid))
		return 0;

969
	pgdat = hotadd_new_pgdat(nid, start);
970
	if (!pgdat) {
971
		pr_err("Cannot online node %d due to NULL pgdat\n", nid);
972 973 974
		ret = -ENOMEM;
		goto out;
	}
975 976 977 978 979 980

	if (set_node_online) {
		node_set_online(nid);
		ret = register_one_node(nid);
		BUG_ON(ret);
	}
981
out:
982 983 984 985 986 987 988 989 990 991 992 993
	return ret;
}

/*
 * Users of this function always want to online/register the node
 */
int try_online_node(int nid)
{
	int ret;

	mem_hotplug_begin();
	ret =  __try_online_node(nid, 0, true);
994
	mem_hotplug_done();
995 996 997
	return ret;
}

998 999
static int check_hotplug_memory_range(u64 start, u64 size)
{
1000
	/* memory range must be block size aligned */
1001 1002
	if (!size || !IS_ALIGNED(start, memory_block_size_bytes()) ||
	    !IS_ALIGNED(size, memory_block_size_bytes())) {
1003
		pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx",
1004
		       memory_block_size_bytes(), start, size);
1005 1006 1007 1008 1009 1010
		return -EINVAL;
	}

	return 0;
}

1011 1012
static int online_memory_block(struct memory_block *mem, void *arg)
{
1013
	return device_online(&mem->dev);
1014 1015
}

1016 1017 1018 1019 1020 1021
/*
 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
 * and online/offline operations (triggered e.g. by sysfs).
 *
 * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG
 */
1022
int __ref add_memory_resource(int nid, struct resource *res)
1023
{
1024
	struct mhp_restrictions restrictions = {};
1025
	u64 start, size;
1026
	bool new_node = false;
1027 1028
	int ret;

1029 1030 1031
	start = res->start;
	size = resource_size(res);

1032 1033 1034 1035
	ret = check_hotplug_memory_range(start, size);
	if (ret)
		return ret;

1036
	mem_hotplug_begin();
1037

1038 1039 1040 1041 1042 1043 1044 1045
	/*
	 * Add new range to memblock so that when hotadd_new_pgdat() is called
	 * to allocate new pgdat, get_pfn_range_for_nid() will be able to find
	 * this new range and calculate total pages correctly.  The range will
	 * be removed at hot-remove time.
	 */
	memblock_add_node(start, size, nid);

1046 1047 1048 1049
	ret = __try_online_node(nid, start, false);
	if (ret < 0)
		goto error;
	new_node = ret;
1050

1051
	/* call arch's memory hotadd */
1052
	ret = arch_add_memory(nid, start, size, &restrictions);
1053 1054 1055
	if (ret < 0)
		goto error;

1056 1057 1058 1059 1060 1061 1062
	/* create memory block devices after memory was added */
	ret = create_memory_block_devices(start, size);
	if (ret) {
		arch_remove_memory(nid, start, size, NULL);
		goto error;
	}

1063
	if (new_node) {
1064
		/* If sysfs file of new node can't be created, cpu on the node
1065 1066
		 * can't be hot-added. There is no rollback way now.
		 * So, check by BUG_ON() to catch it reluctantly..
1067
		 * We online node here. We can't roll back from here.
1068
		 */
1069 1070
		node_set_online(nid);
		ret = __register_one_node(nid);
1071 1072 1073
		BUG_ON(ret);
	}

1074
	/* link memory sections under this node.*/
1075
	ret = link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1));
1076 1077
	BUG_ON(ret);

1078 1079 1080
	/* create new memmap entry */
	firmware_map_add_hotplug(start, start + size, "System RAM");

1081 1082 1083
	/* device_online() will take the lock when calling online_pages() */
	mem_hotplug_done();

1084
	/* online pages if requested */
1085
	if (memhp_auto_online)
1086
		walk_memory_blocks(start, size, NULL, online_memory_block);
1087

1088
	return ret;
1089 1090
error:
	/* rollback pgdat allocation and others */
1091 1092
	if (new_node)
		rollback_node_hotadd(nid);
1093
	memblock_remove(start, size);
1094
	mem_hotplug_done();
1095 1096
	return ret;
}
1097

1098 1099
/* requires device_hotplug_lock, see add_memory_resource() */
int __ref __add_memory(int nid, u64 start, u64 size)
1100 1101 1102 1103 1104
{
	struct resource *res;
	int ret;

	res = register_memory_resource(start, size);
1105 1106
	if (IS_ERR(res))
		return PTR_ERR(res);
1107

1108
	ret = add_memory_resource(nid, res);
1109 1110 1111 1112
	if (ret < 0)
		release_memory_resource(res);
	return ret;
}
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123

int add_memory(int nid, u64 start, u64 size)
{
	int rc;

	lock_device_hotplug();
	rc = __add_memory(nid, start, size);
	unlock_device_hotplug();

	return rc;
}
1124
EXPORT_SYMBOL_GPL(add_memory);
K
KAMEZAWA Hiroyuki 已提交
1125 1126

#ifdef CONFIG_MEMORY_HOTREMOVE
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
/*
 * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy
 * set and the size of the free page is given by page_order(). Using this,
 * the function determines if the pageblock contains only free pages.
 * Due to buddy contraints, a free page at least the size of a pageblock will
 * be located at the start of the pageblock
 */
static inline int pageblock_free(struct page *page)
{
	return PageBuddy(page) && page_order(page) >= pageblock_order;
}

1139 1140
/* Return the pfn of the start of the next active pageblock after a given pfn */
static unsigned long next_active_pageblock(unsigned long pfn)
1141
{
1142 1143
	struct page *page = pfn_to_page(pfn);

1144
	/* Ensure the starting page is pageblock-aligned */
1145
	BUG_ON(pfn & (pageblock_nr_pages - 1));
1146 1147

	/* If the entire pageblock is free, move to the end of free page */
1148 1149 1150 1151 1152
	if (pageblock_free(page)) {
		int order;
		/* be careful. we don't have locks, page_order can be changed.*/
		order = page_order(page);
		if ((order < MAX_ORDER) && (order >= pageblock_order))
1153
			return pfn + (1 << order);
1154
	}
1155

1156
	return pfn + pageblock_nr_pages;
1157 1158
}

1159
static bool is_pageblock_removable_nolock(unsigned long pfn)
1160
{
1161
	struct page *page = pfn_to_page(pfn);
1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178
	struct zone *zone;

	/*
	 * We have to be careful here because we are iterating over memory
	 * sections which are not zone aware so we might end up outside of
	 * the zone but still within the section.
	 * We have to take care about the node as well. If the node is offline
	 * its NODE_DATA will be NULL - see page_zone.
	 */
	if (!node_online(page_to_nid(page)))
		return false;

	zone = page_zone(page);
	pfn = page_to_pfn(page);
	if (!zone_spans_pfn(zone, pfn))
		return false;

1179
	return !has_unmovable_pages(zone, page, MIGRATE_MOVABLE,
1180
				    MEMORY_OFFLINE);
1181 1182
}

1183
/* Checks if this range of memory is likely to be hot-removable. */
1184
bool is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
1185
{
1186 1187 1188 1189
	unsigned long end_pfn, pfn;

	end_pfn = min(start_pfn + nr_pages,
			zone_end_pfn(page_zone(pfn_to_page(start_pfn))));
1190 1191

	/* Check the starting page of each pageblock within the range */
1192 1193
	for (pfn = start_pfn; pfn < end_pfn; pfn = next_active_pageblock(pfn)) {
		if (!is_pageblock_removable_nolock(pfn))
1194
			return false;
1195
		cond_resched();
1196 1197 1198
	}

	/* All pageblocks in the memory block are likely to be hot-removable */
1199
	return true;
1200 1201
}

K
KAMEZAWA Hiroyuki 已提交
1202
/*
1203
 * Confirm all pages in a range [start, end) belong to the same zone.
1204
 * When true, return its valid [start, end).
K
KAMEZAWA Hiroyuki 已提交
1205
 */
1206 1207
int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn,
			 unsigned long *valid_start, unsigned long *valid_end)
K
KAMEZAWA Hiroyuki 已提交
1208
{
1209
	unsigned long pfn, sec_end_pfn;
1210
	unsigned long start, end;
K
KAMEZAWA Hiroyuki 已提交
1211 1212 1213
	struct zone *zone = NULL;
	struct page *page;
	int i;
1214
	for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn + 1);
K
KAMEZAWA Hiroyuki 已提交
1215
	     pfn < end_pfn;
1216
	     pfn = sec_end_pfn, sec_end_pfn += PAGES_PER_SECTION) {
1217 1218
		/* Make sure the memory section is present first */
		if (!present_section_nr(pfn_to_section_nr(pfn)))
K
KAMEZAWA Hiroyuki 已提交
1219
			continue;
1220 1221 1222 1223 1224 1225 1226
		for (; pfn < sec_end_pfn && pfn < end_pfn;
		     pfn += MAX_ORDER_NR_PAGES) {
			i = 0;
			/* This is just a CONFIG_HOLES_IN_ZONE check.*/
			while ((i < MAX_ORDER_NR_PAGES) &&
				!pfn_valid_within(pfn + i))
				i++;
1227
			if (i == MAX_ORDER_NR_PAGES || pfn + i >= end_pfn)
1228
				continue;
1229 1230 1231
			/* Check if we got outside of the zone */
			if (zone && !zone_spans_pfn(zone, pfn + i))
				return 0;
1232 1233 1234
			page = pfn_to_page(pfn + i);
			if (zone && page_zone(page) != zone)
				return 0;
1235 1236
			if (!zone)
				start = pfn + i;
1237
			zone = page_zone(page);
1238
			end = pfn + MAX_ORDER_NR_PAGES;
1239
		}
K
KAMEZAWA Hiroyuki 已提交
1240
	}
1241

1242 1243
	if (zone) {
		*valid_start = start;
1244
		*valid_end = min(end, end_pfn);
1245
		return 1;
1246
	} else {
1247
		return 0;
1248
	}
K
KAMEZAWA Hiroyuki 已提交
1249 1250 1251
}

/*
1252 1253 1254 1255
 * Scan pfn range [start,end) to find movable/migratable pages (LRU pages,
 * non-lru movable pages and hugepages). We scan pfn because it's much
 * easier than scanning over linked list. This function returns the pfn
 * of the first found movable page if it's found, otherwise 0.
K
KAMEZAWA Hiroyuki 已提交
1256
 */
1257
static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
K
KAMEZAWA Hiroyuki 已提交
1258 1259
{
	unsigned long pfn;
1260

K
KAMEZAWA Hiroyuki 已提交
1261
	for (pfn = start; pfn < end; pfn++) {
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
		struct page *page, *head;
		unsigned long skip;

		if (!pfn_valid(pfn))
			continue;
		page = pfn_to_page(pfn);
		if (PageLRU(page))
			return pfn;
		if (__PageMovable(page))
			return pfn;

		if (!PageHuge(page))
			continue;
		head = compound_head(page);
1276
		if (page_huge_active(head))
1277
			return pfn;
1278
		skip = compound_nr(head) - (page - head);
1279
		pfn += skip - 1;
K
KAMEZAWA Hiroyuki 已提交
1280 1281 1282 1283
	}
	return 0;
}

1284
static struct page *new_node_page(struct page *page, unsigned long private)
1285 1286
{
	int nid = page_to_nid(page);
1287
	nodemask_t nmask = node_states[N_MEMORY];
1288 1289 1290 1291 1292 1293 1294 1295 1296

	/*
	 * try to allocate from a different node but reuse this node if there
	 * are no other online nodes to be used (e.g. we are offlining a part
	 * of the only existing node)
	 */
	node_clear(nid, nmask);
	if (nodes_empty(nmask))
		node_set(nid, nmask);
1297

1298
	return new_page_nodemask(page, nid, &nmask);
1299 1300
}

K
KAMEZAWA Hiroyuki 已提交
1301 1302 1303 1304 1305 1306 1307 1308
static int
do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
{
	unsigned long pfn;
	struct page *page;
	int ret = 0;
	LIST_HEAD(source);

1309
	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
K
KAMEZAWA Hiroyuki 已提交
1310 1311 1312
		if (!pfn_valid(pfn))
			continue;
		page = pfn_to_page(pfn);
1313 1314 1315

		if (PageHuge(page)) {
			struct page *head = compound_head(page);
1316
			pfn = page_to_pfn(head) + compound_nr(head) - 1;
1317
			isolate_huge_page(head, &source);
1318
			continue;
M
Michal Hocko 已提交
1319
		} else if (PageTransHuge(page))
1320 1321
			pfn = page_to_pfn(compound_head(page))
				+ hpage_nr_pages(page) - 1;
1322

1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
		/*
		 * HWPoison pages have elevated reference counts so the migration would
		 * fail on them. It also doesn't make any sense to migrate them in the
		 * first place. Still try to unmap such a page in case it is still mapped
		 * (e.g. current hwpoison implementation doesn't unmap KSM pages but keep
		 * the unmap as the catch all safety net).
		 */
		if (PageHWPoison(page)) {
			if (WARN_ON(PageLRU(page)))
				isolate_lru_page(page);
			if (page_mapped(page))
				try_to_unmap(page, TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS);
			continue;
		}

1338
		if (!get_page_unless_zero(page))
K
KAMEZAWA Hiroyuki 已提交
1339 1340
			continue;
		/*
1341 1342
		 * We can skip free pages. And we can deal with pages on
		 * LRU and non-lru movable pages.
K
KAMEZAWA Hiroyuki 已提交
1343
		 */
1344 1345 1346 1347
		if (PageLRU(page))
			ret = isolate_lru_page(page);
		else
			ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE);
K
KAMEZAWA Hiroyuki 已提交
1348
		if (!ret) { /* Success */
1349
			list_add_tail(&page->lru, &source);
1350 1351 1352
			if (!__PageMovable(page))
				inc_node_page_state(page, NR_ISOLATED_ANON +
						    page_is_file_cache(page));
1353

K
KAMEZAWA Hiroyuki 已提交
1354
		} else {
1355
			pr_warn("failed to isolate pfn %lx\n", pfn);
1356
			dump_page(page, "isolation failed");
K
KAMEZAWA Hiroyuki 已提交
1357
		}
1358
		put_page(page);
K
KAMEZAWA Hiroyuki 已提交
1359
	}
1360
	if (!list_empty(&source)) {
1361 1362
		/* Allocate a new page from the nearest neighbor node */
		ret = migrate_pages(&source, new_node_page, NULL, 0,
1363
					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
1364 1365 1366 1367 1368 1369
		if (ret) {
			list_for_each_entry(page, &source, lru) {
				pr_warn("migrating pfn %lx failed ret:%d ",
				       page_to_pfn(page), ret);
				dump_page(page, "migration failure");
			}
1370
			putback_movable_pages(&source);
1371
		}
K
KAMEZAWA Hiroyuki 已提交
1372
	}
1373

K
KAMEZAWA Hiroyuki 已提交
1374 1375 1376
	return ret;
}

1377
/* Mark all sections offline and remove all free pages from the buddy. */
K
KAMEZAWA Hiroyuki 已提交
1378 1379 1380 1381
static int
offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
			void *data)
{
1382
	unsigned long *offlined_pages = (unsigned long *)data;
K
KAMEZAWA Hiroyuki 已提交
1383

1384 1385
	*offlined_pages += __offline_isolated_pages(start, start + nr_pages);
	return 0;
K
KAMEZAWA Hiroyuki 已提交
1386 1387 1388 1389 1390 1391 1392 1393 1394
}

/*
 * Check all pages in range, recoreded as memory resource, are isolated.
 */
static int
check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
			void *data)
{
1395 1396
	return test_pages_isolated(start_pfn, start_pfn + nr_pages,
				   MEMORY_OFFLINE);
K
KAMEZAWA Hiroyuki 已提交
1397 1398
}

1399 1400
static int __init cmdline_parse_movable_node(char *p)
{
1401
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1402
	movable_node_enabled = true;
1403 1404 1405
#else
	pr_warn("movable_node parameter depends on CONFIG_HAVE_MEMBLOCK_NODE_MAP to work properly\n");
#endif
1406 1407 1408 1409
	return 0;
}
early_param("movable_node", cmdline_parse_movable_node);

1410 1411 1412 1413 1414 1415
/* check which state of node_states will be changed when offline memory */
static void node_states_check_changes_offline(unsigned long nr_pages,
		struct zone *zone, struct memory_notify *arg)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	unsigned long present_pages = 0;
1416
	enum zone_type zt;
1417

1418 1419 1420
	arg->status_change_nid = NUMA_NO_NODE;
	arg->status_change_nid_normal = NUMA_NO_NODE;
	arg->status_change_nid_high = NUMA_NO_NODE;
1421 1422

	/*
1423 1424 1425 1426 1427 1428
	 * Check whether node_states[N_NORMAL_MEMORY] will be changed.
	 * If the memory to be offline is within the range
	 * [0..ZONE_NORMAL], and it is the last present memory there,
	 * the zones in that range will become empty after the offlining,
	 * thus we can determine that we need to clear the node from
	 * node_states[N_NORMAL_MEMORY].
1429
	 */
1430
	for (zt = 0; zt <= ZONE_NORMAL; zt++)
1431
		present_pages += pgdat->node_zones[zt].present_pages;
1432
	if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages)
1433 1434
		arg->status_change_nid_normal = zone_to_nid(zone);

1435 1436
#ifdef CONFIG_HIGHMEM
	/*
1437 1438 1439 1440 1441 1442
	 * node_states[N_HIGH_MEMORY] contains nodes which
	 * have normal memory or high memory.
	 * Here we add the present_pages belonging to ZONE_HIGHMEM.
	 * If the zone is within the range of [0..ZONE_HIGHMEM), and
	 * we determine that the zones in that range become empty,
	 * we need to clear the node for N_HIGH_MEMORY.
1443
	 */
1444 1445
	present_pages += pgdat->node_zones[ZONE_HIGHMEM].present_pages;
	if (zone_idx(zone) <= ZONE_HIGHMEM && nr_pages >= present_pages)
1446 1447 1448
		arg->status_change_nid_high = zone_to_nid(zone);
#endif

1449
	/*
1450 1451 1452 1453 1454 1455 1456 1457
	 * We have accounted the pages from [0..ZONE_NORMAL), and
	 * in case of CONFIG_HIGHMEM the pages from ZONE_HIGHMEM
	 * as well.
	 * Here we count the possible pages from ZONE_MOVABLE.
	 * If after having accounted all the pages, we see that the nr_pages
	 * to be offlined is over or equal to the accounted pages,
	 * we know that the node will become empty, and so, we can clear
	 * it for N_MEMORY as well.
1458
	 */
1459
	present_pages += pgdat->node_zones[ZONE_MOVABLE].present_pages;
1460 1461 1462 1463 1464 1465 1466 1467 1468 1469

	if (nr_pages >= present_pages)
		arg->status_change_nid = zone_to_nid(zone);
}

static void node_states_clear_node(int node, struct memory_notify *arg)
{
	if (arg->status_change_nid_normal >= 0)
		node_clear_state(node, N_NORMAL_MEMORY);

1470
	if (arg->status_change_nid_high >= 0)
1471
		node_clear_state(node, N_HIGH_MEMORY);
1472

1473
	if (arg->status_change_nid >= 0)
1474
		node_clear_state(node, N_MEMORY);
1475 1476
}

1477 1478 1479 1480 1481 1482 1483 1484 1485
static int count_system_ram_pages_cb(unsigned long start_pfn,
				     unsigned long nr_pages, void *data)
{
	unsigned long *nr_system_ram_pages = data;

	*nr_system_ram_pages += nr_pages;
	return 0;
}

1486
static int __ref __offline_pages(unsigned long start_pfn,
1487
		  unsigned long end_pfn)
K
KAMEZAWA Hiroyuki 已提交
1488
{
1489
	unsigned long pfn, nr_pages = 0;
1490
	unsigned long offlined_pages = 0;
1491
	int ret, node, nr_isolate_pageblock;
1492
	unsigned long flags;
1493
	unsigned long valid_start, valid_end;
K
KAMEZAWA Hiroyuki 已提交
1494
	struct zone *zone;
1495
	struct memory_notify arg;
1496
	char *reason;
K
KAMEZAWA Hiroyuki 已提交
1497

1498 1499
	mem_hotplug_begin();

1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
	/*
	 * Don't allow to offline memory blocks that contain holes.
	 * Consequently, memory blocks with holes can never get onlined
	 * via the hotplug path - online_pages() - as hotplugged memory has
	 * no holes. This way, we e.g., don't have to worry about marking
	 * memory holes PG_reserved, don't need pfn_valid() checks, and can
	 * avoid using walk_system_ram_range() later.
	 */
	walk_system_ram_range(start_pfn, end_pfn - start_pfn, &nr_pages,
			      count_system_ram_pages_cb);
	if (nr_pages != end_pfn - start_pfn) {
		ret = -EINVAL;
		reason = "memory holes";
		goto failed_removal;
	}

K
KAMEZAWA Hiroyuki 已提交
1516 1517
	/* This makes hotplug much easier...and readable.
	   we assume this for now. .*/
1518 1519
	if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start,
				  &valid_end)) {
1520 1521 1522
		ret = -EINVAL;
		reason = "multizone range";
		goto failed_removal;
1523
	}
1524

1525
	zone = page_zone(pfn_to_page(valid_start));
1526 1527
	node = zone_to_nid(zone);

K
KAMEZAWA Hiroyuki 已提交
1528
	/* set above range as isolated */
1529
	ret = start_isolate_page_range(start_pfn, end_pfn,
1530
				       MIGRATE_MOVABLE,
1531
				       MEMORY_OFFLINE | REPORT_FAILURE);
1532
	if (ret < 0) {
1533 1534
		reason = "failure to isolate range";
		goto failed_removal;
1535
	}
1536
	nr_isolate_pageblock = ret;
1537 1538 1539

	arg.start_pfn = start_pfn;
	arg.nr_pages = nr_pages;
1540
	node_states_check_changes_offline(nr_pages, zone, &arg);
1541 1542 1543

	ret = memory_notify(MEM_GOING_OFFLINE, &arg);
	ret = notifier_to_errno(ret);
1544 1545 1546 1547
	if (ret) {
		reason = "notifier failure";
		goto failed_removal_isolated;
	}
1548

1549 1550 1551 1552 1553 1554 1555
	do {
		for (pfn = start_pfn; pfn;) {
			if (signal_pending(current)) {
				ret = -EINTR;
				reason = "signal backoff";
				goto failed_removal_isolated;
			}
1556

1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568
			cond_resched();
			lru_add_drain_all();

			pfn = scan_movable_pages(pfn, end_pfn);
			if (pfn) {
				/*
				 * TODO: fatal migration failures should bail
				 * out
				 */
				do_migrate_range(pfn, end_pfn);
			}
		}
K
KAMEZAWA Hiroyuki 已提交
1569

1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580
		/*
		 * Dissolve free hugepages in the memory block before doing
		 * offlining actually in order to make hugetlbfs's object
		 * counting consistent.
		 */
		ret = dissolve_free_huge_pages(start_pfn, end_pfn);
		if (ret) {
			reason = "failure to dissolve huge pages";
			goto failed_removal_isolated;
		}
		/* check again */
1581 1582 1583
		ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn,
					    NULL, check_pages_isolated_cb);
	} while (ret);
1584

1585
	/* Ok, all of our target is isolated.
K
KAMEZAWA Hiroyuki 已提交
1586
	   We cannot do rollback at this point. */
1587 1588 1589
	walk_system_ram_range(start_pfn, end_pfn - start_pfn,
			      &offlined_pages, offline_isolated_pages_cb);
	pr_info("Offlined Pages %ld\n", offlined_pages);
1590 1591 1592 1593 1594 1595 1596 1597 1598
	/*
	 * Onlining will reset pagetype flags and makes migrate type
	 * MOVABLE, so just need to decrease the number of isolated
	 * pageblocks zone counter here.
	 */
	spin_lock_irqsave(&zone->lock, flags);
	zone->nr_isolate_pageblock -= nr_isolate_pageblock;
	spin_unlock_irqrestore(&zone->lock, flags);

K
KAMEZAWA Hiroyuki 已提交
1599
	/* removal success */
1600
	adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages);
K
KAMEZAWA Hiroyuki 已提交
1601
	zone->present_pages -= offlined_pages;
1602 1603

	pgdat_resize_lock(zone->zone_pgdat, &flags);
K
KAMEZAWA Hiroyuki 已提交
1604
	zone->zone_pgdat->node_present_pages -= offlined_pages;
1605
	pgdat_resize_unlock(zone->zone_pgdat, &flags);
1606

1607 1608
	init_per_zone_wmark_min();

1609
	if (!populated_zone(zone)) {
1610
		zone_pcp_reset(zone);
1611
		build_all_zonelists(NULL);
1612 1613
	} else
		zone_pcp_update(zone);
1614

1615
	node_states_clear_node(node, &arg);
1616
	if (arg.status_change_nid >= 0) {
1617
		kswapd_stop(node);
1618 1619
		kcompactd_stop(node);
	}
1620

K
KAMEZAWA Hiroyuki 已提交
1621 1622
	vm_total_pages = nr_free_pagecache_pages();
	writeback_set_ratelimit();
1623 1624

	memory_notify(MEM_OFFLINE, &arg);
1625
	remove_pfn_range_from_zone(zone, start_pfn, nr_pages);
1626
	mem_hotplug_done();
K
KAMEZAWA Hiroyuki 已提交
1627 1628
	return 0;

1629 1630
failed_removal_isolated:
	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1631
	memory_notify(MEM_CANCEL_OFFLINE, &arg);
K
KAMEZAWA Hiroyuki 已提交
1632
failed_removal:
1633
	pr_debug("memory offlining [mem %#010llx-%#010llx] failed due to %s\n",
1634
		 (unsigned long long) start_pfn << PAGE_SHIFT,
1635 1636
		 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1,
		 reason);
K
KAMEZAWA Hiroyuki 已提交
1637
	/* pushback to free area */
1638
	mem_hotplug_done();
K
KAMEZAWA Hiroyuki 已提交
1639 1640
	return ret;
}
1641

1642 1643
int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
{
1644
	return __offline_pages(start_pfn, start_pfn + nr_pages);
1645 1646
}

1647
static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
1648 1649 1650
{
	int ret = !is_memblock_offlined(mem);

1651 1652 1653 1654
	if (unlikely(ret)) {
		phys_addr_t beginpa, endpa;

		beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
1655
		endpa = beginpa + memory_block_size_bytes() - 1;
J
Joe Perches 已提交
1656
		pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n",
1657
			&beginpa, &endpa);
1658

1659 1660 1661
		return -EBUSY;
	}
	return 0;
1662 1663
}

1664
static int check_cpu_on_node(pg_data_t *pgdat)
1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679
{
	int cpu;

	for_each_present_cpu(cpu) {
		if (cpu_to_node(cpu) == pgdat->node_id)
			/*
			 * the cpu on this node isn't removed, and we can't
			 * offline this node.
			 */
			return -EBUSY;
	}

	return 0;
}

1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691
static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg)
{
	int nid = *(int *)arg;

	/*
	 * If a memory block belongs to multiple nodes, the stored nid is not
	 * reliable. However, such blocks are always online (e.g., cannot get
	 * offlined) and, therefore, are still spanned by the node.
	 */
	return mem->nid == nid ? -EEXIST : 0;
}

1692 1693
/**
 * try_offline_node
1694
 * @nid: the node ID
1695 1696 1697 1698 1699 1700
 *
 * Offline a node if all memory sections and cpus of the node are removed.
 *
 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
 * and online/offline operations before this call.
 */
1701
void try_offline_node(int nid)
1702
{
1703
	pg_data_t *pgdat = NODE_DATA(nid);
1704
	int rc;
1705

1706 1707 1708 1709 1710 1711 1712
	/*
	 * If the node still spans pages (especially ZONE_DEVICE), don't
	 * offline it. A node spans memory after move_pfn_range_to_zone(),
	 * e.g., after the memory block was onlined.
	 */
	if (pgdat->node_spanned_pages)
		return;
1713

1714 1715 1716 1717 1718 1719 1720
	/*
	 * Especially offline memory blocks might not be spanned by the
	 * node. They will get spanned by the node once they get onlined.
	 * However, they link to the node in sysfs and can get onlined later.
	 */
	rc = for_each_memory_block(&nid, check_no_memblock_for_node_cb);
	if (rc)
1721 1722
		return;

1723
	if (check_cpu_on_node(pgdat))
1724 1725 1726 1727 1728 1729 1730 1731 1732
		return;

	/*
	 * all memory/cpu of this node are removed, we can offline this
	 * node now.
	 */
	node_set_offline(nid);
	unregister_one_node(nid);
}
1733
EXPORT_SYMBOL(try_offline_node);
1734

1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
static void __release_memory_resource(resource_size_t start,
				      resource_size_t size)
{
	int ret;

	/*
	 * When removing memory in the same granularity as it was added,
	 * this function never fails. It might only fail if resources
	 * have to be adjusted or split. We'll ignore the error, as
	 * removing of memory cannot fail.
	 */
	ret = release_mem_region_adjustable(&iomem_resource, start, size);
	if (ret) {
		resource_size_t endres = start + size - 1;

		pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
			&start, &endres, ret);
	}
}

1755
static int __ref try_remove_memory(int nid, u64 start, u64 size)
1756
{
1757
	int rc = 0;
1758

1759 1760
	BUG_ON(check_hotplug_memory_range(start, size));

1761
	/*
1762
	 * All memory blocks must be offlined before removing memory.  Check
1763
	 * whether all memory blocks in question are offline and return error
1764
	 * if this is not the case.
1765
	 */
1766
	rc = walk_memory_blocks(start, size, NULL, check_memblock_offlined_cb);
1767 1768
	if (rc)
		goto done;
1769

1770 1771
	/* remove memmap entry */
	firmware_map_remove(start, start + size, "System RAM");
1772

1773 1774 1775 1776
	/*
	 * Memory block device removal under the device_hotplug_lock is
	 * a barrier against racing online attempts.
	 */
1777
	remove_memory_block_devices(start, size);
1778

1779 1780
	mem_hotplug_begin();

1781
	arch_remove_memory(nid, start, size, NULL);
1782 1783
	memblock_free(start, size);
	memblock_remove(start, size);
1784
	__release_memory_resource(start, size);
1785

1786 1787
	try_offline_node(nid);

1788
done:
1789
	mem_hotplug_done();
1790
	return rc;
1791
}
1792

1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806
/**
 * remove_memory
 * @nid: the node ID
 * @start: physical address of the region to remove
 * @size: size of the region to remove
 *
 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
 * and online/offline operations before this call, as required by
 * try_offline_node().
 */
void __remove_memory(int nid, u64 start, u64 size)
{

	/*
S
Souptick Joarder 已提交
1807
	 * trigger BUG() if some memory is not offlined prior to calling this
1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818
	 * function
	 */
	if (try_remove_memory(nid, start, size))
		BUG();
}

/*
 * Remove memory if every memory block is offline, otherwise return -EBUSY is
 * some memory is not offline
 */
int remove_memory(int nid, u64 start, u64 size)
1819
{
1820 1821
	int rc;

1822
	lock_device_hotplug();
1823
	rc  = try_remove_memory(nid, start, size);
1824
	unlock_device_hotplug();
1825 1826

	return rc;
1827
}
1828
EXPORT_SYMBOL_GPL(remove_memory);
1829
#endif /* CONFIG_MEMORY_HOTREMOVE */