page_alloc.c 227.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 *  linux/mm/page_alloc.c
 *
 *  Manages the free list, the system allocates free pages here.
 *  Note that kmalloc() lives in slab.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *  Swap reorganised 29.12.95, Stephen Tweedie
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
 *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
 *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
 */

#include <linux/stddef.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
22
#include <linux/jiffies.h>
L
Linus Torvalds 已提交
23
#include <linux/bootmem.h>
24
#include <linux/memblock.h>
L
Linus Torvalds 已提交
25
#include <linux/compiler.h>
26
#include <linux/kernel.h>
27
#include <linux/kasan.h>
L
Linus Torvalds 已提交
28 29 30 31 32
#include <linux/module.h>
#include <linux/suspend.h>
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
33
#include <linux/ratelimit.h>
34
#include <linux/oom.h>
L
Linus Torvalds 已提交
35 36 37 38
#include <linux/topology.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
39
#include <linux/memory_hotplug.h>
L
Linus Torvalds 已提交
40 41
#include <linux/nodemask.h>
#include <linux/vmalloc.h>
42
#include <linux/vmstat.h>
43
#include <linux/mempolicy.h>
44
#include <linux/memremap.h>
45
#include <linux/stop_machine.h>
46 47
#include <linux/sort.h>
#include <linux/pfn.h>
48
#include <linux/backing-dev.h>
49
#include <linux/fault-inject.h>
K
KAMEZAWA Hiroyuki 已提交
50
#include <linux/page-isolation.h>
51
#include <linux/page_ext.h>
52
#include <linux/debugobjects.h>
53
#include <linux/kmemleak.h>
54
#include <linux/compaction.h>
55
#include <trace/events/kmem.h>
56
#include <trace/events/oom.h>
57
#include <linux/prefetch.h>
58
#include <linux/mm_inline.h>
59
#include <linux/migrate.h>
60
#include <linux/hugetlb.h>
61
#include <linux/sched/rt.h>
62
#include <linux/sched/mm.h>
63
#include <linux/page_owner.h>
64
#include <linux/kthread.h>
65
#include <linux/memcontrol.h>
66
#include <linux/ftrace.h>
67
#include <linux/lockdep.h>
68
#include <linux/nmi.h>
69
#include <linux/psi.h>
L
Linus Torvalds 已提交
70

71
#include <asm/sections.h>
L
Linus Torvalds 已提交
72
#include <asm/tlbflush.h>
73
#include <asm/div64.h>
L
Linus Torvalds 已提交
74
#include "internal.h"
A
Alexander Duyck 已提交
75
#include "page_reporting.h"
L
Linus Torvalds 已提交
76

77 78
/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
static DEFINE_MUTEX(pcp_batch_high_lock);
79
#define MIN_PERCPU_PAGELIST_FRACTION	(8)
80

81 82 83 84 85
#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
DEFINE_PER_CPU(int, numa_node);
EXPORT_PER_CPU_SYMBOL(numa_node);
#endif

86 87
DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);

88 89 90 91 92 93 94 95 96
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
/*
 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
 * defined in <linux/topology.h>.
 */
DEFINE_PER_CPU(int, _numa_mem_);		/* Kernel "local memory" node */
EXPORT_PER_CPU_SYMBOL(_numa_mem_);
97
int _node_numa_mem_[MAX_NUMNODES];
98 99
#endif

100 101 102 103
/* work_structs for global per-cpu drains */
DEFINE_MUTEX(pcpu_drain_mutex);
DEFINE_PER_CPU(struct work_struct, pcpu_drain);

104
#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
105
volatile unsigned long latent_entropy __latent_entropy;
106 107 108
EXPORT_SYMBOL(latent_entropy);
#endif

L
Linus Torvalds 已提交
109
/*
110
 * Array of node states.
L
Linus Torvalds 已提交
111
 */
112 113 114 115 116 117 118
nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
	[N_POSSIBLE] = NODE_MASK_ALL,
	[N_ONLINE] = { { [0] = 1UL } },
#ifndef CONFIG_NUMA
	[N_NORMAL_MEMORY] = { { [0] = 1UL } },
#ifdef CONFIG_HIGHMEM
	[N_HIGH_MEMORY] = { { [0] = 1UL } },
119 120
#endif
	[N_MEMORY] = { { [0] = 1UL } },
121 122 123 124 125
	[N_CPU] = { { [0] = 1UL } },
#endif	/* NUMA */
};
EXPORT_SYMBOL(node_states);

126 127 128
/* Protect totalram_pages and zone->managed_pages */
static DEFINE_SPINLOCK(managed_page_count_lock);

129
unsigned long totalram_pages __read_mostly;
130
unsigned long totalreserve_pages __read_mostly;
131
unsigned long totalcma_pages __read_mostly;
132

133
int percpu_pagelist_fraction;
134
gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
L
Linus Torvalds 已提交
135

136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
/*
 * A cached value of the page's pageblock's migratetype, used when the page is
 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
 * Also the migratetype set in the page does not necessarily match the pcplist
 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
 * other index - this ensures that it will be put on the correct CMA freelist.
 */
static inline int get_pcppage_migratetype(struct page *page)
{
	return page->index;
}

static inline void set_pcppage_migratetype(struct page *page, int migratetype)
{
	page->index = migratetype;
}

154 155 156 157 158
#ifdef CONFIG_PM_SLEEP
/*
 * The following functions are used by the suspend/hibernate code to temporarily
 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
 * while devices are suspended.  To avoid races with the suspend/hibernate code,
159 160 161 162
 * they should always be called with system_transition_mutex held
 * (gfp_allowed_mask also should only be modified with system_transition_mutex
 * held, unless the suspend/hibernate code is guaranteed not to run in parallel
 * with that modification).
163
 */
164 165 166 167

static gfp_t saved_gfp_mask;

void pm_restore_gfp_mask(void)
168
{
169
	WARN_ON(!mutex_is_locked(&system_transition_mutex));
170 171 172 173
	if (saved_gfp_mask) {
		gfp_allowed_mask = saved_gfp_mask;
		saved_gfp_mask = 0;
	}
174 175
}

176
void pm_restrict_gfp_mask(void)
177
{
178
	WARN_ON(!mutex_is_locked(&system_transition_mutex));
179 180
	WARN_ON(saved_gfp_mask);
	saved_gfp_mask = gfp_allowed_mask;
181
	gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
182
}
183 184 185

bool pm_suspended_storage(void)
{
186
	if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
187 188 189
		return false;
	return true;
}
190 191
#endif /* CONFIG_PM_SLEEP */

192
#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
193
unsigned int pageblock_order __read_mostly;
194 195
#endif

196
static void __free_pages_ok(struct page *page, unsigned int order);
197

L
Linus Torvalds 已提交
198 199 200 201 202 203
/*
 * results with 256, 32 in the lowmem_reserve sysctl:
 *	1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
 *	1G machine -> (16M dma, 784M normal, 224M high)
 *	NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
 *	HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
Y
Yaowei Bai 已提交
204
 *	HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
A
Andi Kleen 已提交
205 206 207
 *
 * TBD: should special case ZONE_DMA32 machines here - in those we normally
 * don't need any ZONE_NORMAL reservation
L
Linus Torvalds 已提交
208
 */
209
int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
210
#ifdef CONFIG_ZONE_DMA
211
	[ZONE_DMA] = 256,
212
#endif
213
#ifdef CONFIG_ZONE_DMA32
214
	[ZONE_DMA32] = 256,
215
#endif
216
	[ZONE_NORMAL] = 32,
217
#ifdef CONFIG_HIGHMEM
218
	[ZONE_HIGHMEM] = 0,
219
#endif
220
	[ZONE_MOVABLE] = 0,
221
};
L
Linus Torvalds 已提交
222 223 224

EXPORT_SYMBOL(totalram_pages);

225
static char * const zone_names[MAX_NR_ZONES] = {
226
#ifdef CONFIG_ZONE_DMA
227
	 "DMA",
228
#endif
229
#ifdef CONFIG_ZONE_DMA32
230
	 "DMA32",
231
#endif
232
	 "Normal",
233
#ifdef CONFIG_HIGHMEM
M
Mel Gorman 已提交
234
	 "HighMem",
235
#endif
M
Mel Gorman 已提交
236
	 "Movable",
237 238 239
#ifdef CONFIG_ZONE_DEVICE
	 "Device",
#endif
240 241
};

242 243 244 245 246 247 248 249 250 251 252 253 254
char * const migratetype_names[MIGRATE_TYPES] = {
	"Unmovable",
	"Movable",
	"Reclaimable",
	"HighAtomic",
#ifdef CONFIG_CMA
	"CMA",
#endif
#ifdef CONFIG_MEMORY_ISOLATION
	"Isolate",
#endif
};

255 256 257 258 259 260
compound_page_dtor * const compound_page_dtors[] = {
	NULL,
	free_compound_page,
#ifdef CONFIG_HUGETLB_PAGE
	free_huge_page,
#endif
261 262 263
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	free_transhuge_page,
#endif
264 265
};

L
Linus Torvalds 已提交
266
int min_free_kbytes = 1024;
267
int user_min_free_kbytes = -1;
268
int watermark_scale_factor = 10;
L
Linus Torvalds 已提交
269

270 271 272
static unsigned long nr_kernel_pages __meminitdata;
static unsigned long nr_all_pages __meminitdata;
static unsigned long dma_reserve __meminitdata;
L
Linus Torvalds 已提交
273

T
Tejun Heo 已提交
274
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
275 276 277
static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __meminitdata;
static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __meminitdata;
static unsigned long required_kernelcore __initdata;
278
static unsigned long required_kernelcore_percent __initdata;
279
static unsigned long required_movablecore __initdata;
280
static unsigned long required_movablecore_percent __initdata;
281 282
static unsigned long zone_movable_pfn[MAX_NUMNODES] __meminitdata;
static bool mirrored_kernelcore __meminitdata;
T
Tejun Heo 已提交
283 284 285 286 287

/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
int movable_zone;
EXPORT_SYMBOL(movable_zone);
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
288

M
Miklos Szeredi 已提交
289 290
#if MAX_NUMNODES > 1
int nr_node_ids __read_mostly = MAX_NUMNODES;
291
int nr_online_nodes __read_mostly = 1;
M
Miklos Szeredi 已提交
292
EXPORT_SYMBOL(nr_node_ids);
293
EXPORT_SYMBOL(nr_online_nodes);
M
Miklos Szeredi 已提交
294 295
#endif

296 297
int page_group_by_mobility_disabled __read_mostly;

298
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
/*
 * During boot we initialize deferred pages on-demand, as needed, but once
 * page_alloc_init_late() has finished, the deferred pages are all initialized,
 * and we can permanently disable that path.
 */
static DEFINE_STATIC_KEY_TRUE(deferred_pages);

/*
 * Calling kasan_free_pages() only after deferred memory initialization
 * has completed. Poisoning pages during deferred memory init will greatly
 * lengthen the process and cause problem in large memory systems as the
 * deferred pages initialization is done with interrupt disabled.
 *
 * Assuming that there will be no reference to those newly initialized
 * pages before they are ever allocated, this should have no effect on
 * KASAN memory tracking as the poison will be properly inserted at page
 * allocation time. The only corner case is when pages are allocated by
 * on-demand allocation and then freed again before the deferred pages
 * initialization is done, but this is not likely to happen.
 */
static inline void kasan_free_nondeferred_pages(struct page *page, int order)
{
	if (!static_branch_unlikely(&deferred_pages))
		kasan_free_pages(page, order);
}

325
/* Returns true if the struct page for the pfn is uninitialised */
326
static inline bool __meminit early_page_uninitialised(unsigned long pfn)
327
{
328 329 330
	int nid = early_pfn_to_nid(pfn);

	if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
331 332 333 334 335 336 337 338 339 340 341 342 343
		return true;

	return false;
}

/*
 * Returns false when the remaining initialisation should be deferred until
 * later in the boot cycle when it can be parallelised.
 */
static inline bool update_defer_init(pg_data_t *pgdat,
				unsigned long pfn, unsigned long zone_end,
				unsigned long *nr_initialised)
{
344
	/* Always populate low zones for address-constrained allocations */
345 346 347
	if (zone_end < pgdat_end_pfn(pgdat))
		return true;
	(*nr_initialised)++;
348
	if ((*nr_initialised > pgdat->static_init_pgcnt) &&
349 350 351 352 353 354 355 356
	    (pfn & (PAGES_PER_SECTION - 1)) == 0) {
		pgdat->first_deferred_pfn = pfn;
		return false;
	}

	return true;
}
#else
357 358
#define kasan_free_nondeferred_pages(p, o)	kasan_free_pages(p, o)

359 360 361 362 363 364 365 366 367 368 369 370 371
static inline bool early_page_uninitialised(unsigned long pfn)
{
	return false;
}

static inline bool update_defer_init(pg_data_t *pgdat,
				unsigned long pfn, unsigned long zone_end,
				unsigned long *nr_initialised)
{
	return true;
}
#endif

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
/* Return a pointer to the bitmap storing bits affecting a block of pages */
static inline unsigned long *get_pageblock_bitmap(struct page *page,
							unsigned long pfn)
{
#ifdef CONFIG_SPARSEMEM
	return __pfn_to_section(pfn)->pageblock_flags;
#else
	return page_zone(page)->pageblock_flags;
#endif /* CONFIG_SPARSEMEM */
}

static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
{
#ifdef CONFIG_SPARSEMEM
	pfn &= (PAGES_PER_SECTION-1);
	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
#else
	pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
#endif /* CONFIG_SPARSEMEM */
}

/**
 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
 * @page: The page within the block of interest
 * @pfn: The target page frame number
 * @end_bitidx: The last bit of interest to retrieve
 * @mask: mask of bits that the caller is interested in
 *
 * Return: pageblock_bits flags
 */
static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
					unsigned long pfn,
					unsigned long end_bitidx,
					unsigned long mask)
{
	unsigned long *bitmap;
	unsigned long bitidx, word_bitidx;
	unsigned long word;

	bitmap = get_pageblock_bitmap(page, pfn);
	bitidx = pfn_to_bitidx(page, pfn);
	word_bitidx = bitidx / BITS_PER_LONG;
	bitidx &= (BITS_PER_LONG-1);

	word = bitmap[word_bitidx];
	bitidx += end_bitidx;
	return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
}

unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
					unsigned long end_bitidx,
					unsigned long mask)
{
	return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
}

static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
{
	return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
}

/**
 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
 * @page: The page within the block of interest
 * @flags: The flags to set
 * @pfn: The target page frame number
 * @end_bitidx: The last bit of interest
 * @mask: mask of bits that the caller is interested in
 */
void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
					unsigned long pfn,
					unsigned long end_bitidx,
					unsigned long mask)
{
	unsigned long *bitmap;
	unsigned long bitidx, word_bitidx;
	unsigned long old_word, word;

	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);

	bitmap = get_pageblock_bitmap(page, pfn);
	bitidx = pfn_to_bitidx(page, pfn);
	word_bitidx = bitidx / BITS_PER_LONG;
	bitidx &= (BITS_PER_LONG-1);

	VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);

	bitidx += end_bitidx;
	mask <<= (BITS_PER_LONG - bitidx - 1);
	flags <<= (BITS_PER_LONG - bitidx - 1);

	word = READ_ONCE(bitmap[word_bitidx]);
	for (;;) {
		old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
		if (word == old_word)
			break;
		word = old_word;
	}
}
472

473
void set_pageblock_migratetype(struct page *page, int migratetype)
474
{
475 476
	if (unlikely(page_group_by_mobility_disabled &&
		     migratetype < MIGRATE_PCPTYPES))
477 478
		migratetype = MIGRATE_UNMOVABLE;

479 480 481 482
	set_pageblock_flags_group(page, (unsigned long)migratetype,
					PB_migrate, PB_migrate_end);
}

N
Nick Piggin 已提交
483
#ifdef CONFIG_DEBUG_VM
484
static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
L
Linus Torvalds 已提交
485
{
486 487 488
	int ret = 0;
	unsigned seq;
	unsigned long pfn = page_to_pfn(page);
489
	unsigned long sp, start_pfn;
490

491 492
	do {
		seq = zone_span_seqbegin(zone);
493 494
		start_pfn = zone->zone_start_pfn;
		sp = zone->spanned_pages;
495
		if (!zone_spans_pfn(zone, pfn))
496 497 498
			ret = 1;
	} while (zone_span_seqretry(zone, seq));

499
	if (ret)
500 501 502
		pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
			pfn, zone_to_nid(zone), zone->name,
			start_pfn, start_pfn + sp);
503

504
	return ret;
505 506 507 508
}

static int page_is_consistent(struct zone *zone, struct page *page)
{
509
	if (!pfn_valid_within(page_to_pfn(page)))
510
		return 0;
L
Linus Torvalds 已提交
511
	if (zone != page_zone(page))
512 513 514 515 516 517 518
		return 0;

	return 1;
}
/*
 * Temporary debugging check for pages not lying within a given zone.
 */
519
static int __maybe_unused bad_range(struct zone *zone, struct page *page)
520 521
{
	if (page_outside_zone_boundaries(zone, page))
L
Linus Torvalds 已提交
522
		return 1;
523 524 525
	if (!page_is_consistent(zone, page))
		return 1;

L
Linus Torvalds 已提交
526 527
	return 0;
}
N
Nick Piggin 已提交
528
#else
529
static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
N
Nick Piggin 已提交
530 531 532 533 534
{
	return 0;
}
#endif

535 536
static void bad_page(struct page *page, const char *reason,
		unsigned long bad_flags)
L
Linus Torvalds 已提交
537
{
538 539 540 541 542 543 544 545 546 547 548 549 550 551
	static unsigned long resume;
	static unsigned long nr_shown;
	static unsigned long nr_unshown;

	/*
	 * Allow a burst of 60 reports, then keep quiet for that minute;
	 * or allow a steady drip of one report per second.
	 */
	if (nr_shown == 60) {
		if (time_before(jiffies, resume)) {
			nr_unshown++;
			goto out;
		}
		if (nr_unshown) {
552
			pr_alert(
553
			      "BUG: Bad page state: %lu messages suppressed\n",
554 555 556 557 558 559 560 561
				nr_unshown);
			nr_unshown = 0;
		}
		nr_shown = 0;
	}
	if (nr_shown++ == 0)
		resume = jiffies + 60 * HZ;

562
	pr_alert("BUG: Bad page state in process %s  pfn:%05lx\n",
563
		current->comm, page_to_pfn(page));
564 565 566 567 568
	__dump_page(page, reason);
	bad_flags &= page->flags;
	if (bad_flags)
		pr_alert("bad because of flags: %#lx(%pGp)\n",
						bad_flags, &bad_flags);
569
	dump_page_owner(page);
570

571
	print_modules();
L
Linus Torvalds 已提交
572
	dump_stack();
573
out:
574
	/* Leave bad fields for debug, except PageBuddy could make trouble */
575
	page_mapcount_reset(page); /* remove PageBuddy */
576
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
L
Linus Torvalds 已提交
577 578 579 580 581
}

/*
 * Higher-order pages are called "compound pages".  They are structured thusly:
 *
582
 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
L
Linus Torvalds 已提交
583
 *
584 585
 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
L
Linus Torvalds 已提交
586
 *
587 588
 * The first tail page's ->compound_dtor holds the offset in array of compound
 * page destructors. See compound_page_dtors.
L
Linus Torvalds 已提交
589
 *
590
 * The first tail page's ->compound_order holds the order of allocation.
591
 * This usage means that zero-order pages may not be compound.
L
Linus Torvalds 已提交
592
 */
593

594
void free_compound_page(struct page *page)
595
{
596
	mem_cgroup_uncharge(page);
597
	__free_pages_ok(page, compound_order(page));
598 599
}

600
void prep_compound_page(struct page *page, unsigned int order)
601 602 603 604
{
	int i;
	int nr_pages = 1 << order;

605
	set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
606 607 608 609
	set_compound_order(page, order);
	__SetPageHead(page);
	for (i = 1; i < nr_pages; i++) {
		struct page *p = page + i;
610
		set_page_count(p, 0);
611
		p->mapping = TAIL_MAPPING;
612
		set_compound_head(p, page);
613
	}
614
	atomic_set(compound_mapcount_ptr(page), -1);
615 616
}

617 618
#ifdef CONFIG_DEBUG_PAGEALLOC
unsigned int _debug_guardpage_minorder;
619 620
bool _debug_pagealloc_enabled __read_mostly
			= IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
621
EXPORT_SYMBOL(_debug_pagealloc_enabled);
622 623
bool _debug_guardpage_enabled __read_mostly;

624 625 626 627
static int __init early_debug_pagealloc(char *buf)
{
	if (!buf)
		return -EINVAL;
628
	return kstrtobool(buf, &_debug_pagealloc_enabled);
629 630 631
}
early_param("debug_pagealloc", early_debug_pagealloc);

632 633
static bool need_debug_guardpage(void)
{
634 635 636 637
	/* If we don't use debug_pagealloc, we don't need guard page */
	if (!debug_pagealloc_enabled())
		return false;

638 639 640
	if (!debug_guardpage_minorder())
		return false;

641 642 643 644 645
	return true;
}

static void init_debug_guardpage(void)
{
646 647 648
	if (!debug_pagealloc_enabled())
		return;

649 650 651
	if (!debug_guardpage_minorder())
		return;

652 653 654 655 656 657 658
	_debug_guardpage_enabled = true;
}

struct page_ext_operations debug_guardpage_ops = {
	.need = need_debug_guardpage,
	.init = init_debug_guardpage,
};
659 660 661 662 663 664

static int __init debug_guardpage_minorder_setup(char *buf)
{
	unsigned long res;

	if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
665
		pr_err("Bad debug_guardpage_minorder value\n");
666 667 668
		return 0;
	}
	_debug_guardpage_minorder = res;
669
	pr_info("Setting debug_guardpage_minorder to %lu\n", res);
670 671
	return 0;
}
672
early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
673

674
static inline bool set_page_guard(struct zone *zone, struct page *page,
675
				unsigned int order, int migratetype)
676
{
677 678 679
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
680 681 682 683
		return false;

	if (order >= debug_guardpage_minorder())
		return false;
684 685

	page_ext = lookup_page_ext(page);
686
	if (unlikely(!page_ext))
687
		return false;
688

689 690
	__set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);

691 692 693 694
	INIT_LIST_HEAD(&page->lru);
	set_page_private(page, order);
	/* Guard pages are not available for any usage */
	__mod_zone_freepage_state(zone, -(1 << order), migratetype);
695 696

	return true;
697 698
}

699 700
static inline void clear_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype)
701
{
702 703 704 705 706 707
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
		return;

	page_ext = lookup_page_ext(page);
708 709 710
	if (unlikely(!page_ext))
		return;

711 712
	__clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);

713 714 715
	set_page_private(page, 0);
	if (!is_migrate_isolate(migratetype))
		__mod_zone_freepage_state(zone, (1 << order), migratetype);
716 717
}
#else
718
struct page_ext_operations debug_guardpage_ops;
719 720
static inline bool set_page_guard(struct zone *zone, struct page *page,
			unsigned int order, int migratetype) { return false; }
721 722
static inline void clear_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype) {}
723 724
#endif

725
static inline void set_page_order(struct page *page, unsigned int order)
726
{
H
Hugh Dickins 已提交
727
	set_page_private(page, order);
728
	__SetPageBuddy(page);
L
Linus Torvalds 已提交
729 730 731 732
}

/*
 * This function checks whether a page is free && is the buddy
733
 * we can coalesce a page and its buddy if
734
 * (a) the buddy is not in a hole (check before calling!) &&
735
 * (b) the buddy is in the buddy system &&
736 737
 * (c) a page and its buddy have the same order &&
 * (d) a page and its buddy are in the same zone.
738
 *
739 740
 * For recording whether a page is in the buddy system, we set PageBuddy.
 * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
L
Linus Torvalds 已提交
741
 *
742
 * For recording page's order, we use page_private(page).
L
Linus Torvalds 已提交
743
 */
744
static inline int page_is_buddy(struct page *page, struct page *buddy,
745
							unsigned int order)
L
Linus Torvalds 已提交
746
{
747
	if (page_is_guard(buddy) && page_order(buddy) == order) {
748 749 750
		if (page_zone_id(page) != page_zone_id(buddy))
			return 0;

751 752
		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);

753 754 755
		return 1;
	}

756
	if (PageBuddy(buddy) && page_order(buddy) == order) {
757 758 759 760 761 762 763 764
		/*
		 * zone check is done late to avoid uselessly
		 * calculating zone/node ids for pages that could
		 * never merge.
		 */
		if (page_zone_id(page) != page_zone_id(buddy))
			return 0;

765 766
		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);

767
		return 1;
768
	}
769
	return 0;
L
Linus Torvalds 已提交
770 771
}

772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
/* Used for pages not on another list */
static inline void add_to_free_list(struct page *page, struct zone *zone,
				    unsigned int order, int migratetype)
{
	struct free_area *area = &zone->free_area[order];

	list_add(&page->lru, &area->free_list[migratetype]);
	area->nr_free++;
}

/* Used for pages not on another list */
static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
					 unsigned int order, int migratetype)
{
	struct free_area *area = &zone->free_area[order];

	list_add_tail(&page->lru, &area->free_list[migratetype]);
	area->nr_free++;
}

/* Used for pages which are on another list */
static inline void move_to_free_list(struct page *page, struct zone *zone,
				     unsigned int order, int migratetype)
{
	struct free_area *area = &zone->free_area[order];

	list_move(&page->lru, &area->free_list[migratetype]);
}

static inline void del_page_from_free_list(struct page *page, struct zone *zone,
					   unsigned int order)
{
A
Alexander Duyck 已提交
804 805 806 807
	/* clear reported state and update reported page count */
	if (page_reported(page))
		__ClearPageReported(page);

808 809 810 811 812 813
	list_del(&page->lru);
	__ClearPageBuddy(page);
	set_page_private(page, 0);
	zone->free_area[order].nr_free--;
}

L
Linus Torvalds 已提交
814 815 816 817 818 819 820 821 822 823 824 825 826
/*
 * Freeing function for a buddy system allocator.
 *
 * The concept of a buddy system is to maintain direct-mapped table
 * (containing bit values) for memory blocks of various "orders".
 * The bottom level table contains the map for the smallest allocatable
 * units of memory (here, pages), and each level above it describes
 * pairs of units from the levels below, hence, "buddies".
 * At a high level, all that happens here is marking the table entry
 * at the bottom level available, and propagating the changes upward
 * as necessary, plus some accounting needed to play nicely with other
 * parts of the VM system.
 * At each level, we keep a list of pages, which are heads of continuous
827 828
 * free pages of length of (1 << order) and marked with PageBuddy.
 * Page's order is recorded in page_private(page) field.
L
Linus Torvalds 已提交
829
 * So when we are allocating or freeing one, we can derive the state of the
830 831
 * other.  That is, if we allocate a small block, and both were
 * free, the remainder of the region must be split into blocks.
L
Linus Torvalds 已提交
832
 * If a block is freed, and its buddy is also free, then this
833
 * triggers coalescing into a block of larger size.
L
Linus Torvalds 已提交
834
 *
835
 * -- nyc
L
Linus Torvalds 已提交
836 837
 */

N
Nick Piggin 已提交
838
static inline void __free_one_page(struct page *page,
839
		unsigned long pfn,
840
		struct zone *zone, unsigned int order,
A
Alexander Duyck 已提交
841
		int migratetype, bool report)
L
Linus Torvalds 已提交
842
{
843 844
	unsigned long combined_pfn;
	unsigned long uninitialized_var(buddy_pfn);
845
	struct page *buddy;
846 847 848
	unsigned int max_order;

	max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
L
Linus Torvalds 已提交
849

850
	VM_BUG_ON(!zone_is_initialized(zone));
851
	VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
L
Linus Torvalds 已提交
852

853
	VM_BUG_ON(migratetype == -1);
854
	if (likely(!is_migrate_isolate(migratetype)))
855
		__mod_zone_freepage_state(zone, 1 << order, migratetype);
856

857
	VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
858
	VM_BUG_ON_PAGE(bad_range(zone, page), page);
L
Linus Torvalds 已提交
859

860
continue_merging:
861
	while (order < max_order - 1) {
862 863
		buddy_pfn = __find_buddy_pfn(pfn, order);
		buddy = page + (buddy_pfn - pfn);
864 865 866

		if (!pfn_valid_within(buddy_pfn))
			goto done_merging;
867
		if (!page_is_buddy(page, buddy, order))
868
			goto done_merging;
869 870 871 872
		/*
		 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
		 * merge with it and move up one order.
		 */
873
		if (page_is_guard(buddy))
874
			clear_page_guard(zone, buddy, order, migratetype);
875
		else
876
			del_page_from_free_list(buddy, zone, order);
877 878 879
		combined_pfn = buddy_pfn & pfn;
		page = page + (combined_pfn - pfn);
		pfn = combined_pfn;
L
Linus Torvalds 已提交
880 881
		order++;
	}
882 883 884 885 886 887 888 889 890 891 892 893
	if (max_order < MAX_ORDER) {
		/* If we are here, it means order is >= pageblock_order.
		 * We want to prevent merge between freepages on isolate
		 * pageblock and normal pageblock. Without this, pageblock
		 * isolation could cause incorrect freepage or CMA accounting.
		 *
		 * We don't want to hit this code for the more frequent
		 * low-order merging.
		 */
		if (unlikely(has_isolate_pageblock(zone))) {
			int buddy_mt;

894 895
			buddy_pfn = __find_buddy_pfn(pfn, order);
			buddy = page + (buddy_pfn - pfn);
896 897 898 899 900 901 902 903 904 905 906 907
			buddy_mt = get_pageblock_migratetype(buddy);

			if (migratetype != buddy_mt
					&& (is_migrate_isolate(migratetype) ||
						is_migrate_isolate(buddy_mt)))
				goto done_merging;
		}
		max_order++;
		goto continue_merging;
	}

done_merging:
L
Linus Torvalds 已提交
908
	set_page_order(page, order);
909 910 911 912 913 914 915 916 917

	/*
	 * If this is not the largest possible page, check if the buddy
	 * of the next-highest order is free. If it is, it's possible
	 * that pages are being freed that will coalesce soon. In case,
	 * that is happening, add the free page to the tail of the list
	 * so it's less likely to be used soon and more likely to be merged
	 * as a higher order page
	 */
918
	if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn)) {
919
		struct page *higher_page, *higher_buddy;
920 921 922 923
		combined_pfn = buddy_pfn & pfn;
		higher_page = page + (combined_pfn - pfn);
		buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
		higher_buddy = higher_page + (buddy_pfn - combined_pfn);
924 925
		if (pfn_valid_within(buddy_pfn) &&
		    page_is_buddy(higher_page, higher_buddy, order + 1)) {
926
			add_to_free_list_tail(page, zone, order,
927 928
					      migratetype);
			return;
929 930 931
		}
	}

932
	add_to_free_list(page, zone, order, migratetype);
A
Alexander Duyck 已提交
933 934 935 936

	/* Notify page reporting subsystem of freed page */
	if (report)
		page_reporting_notify_free(order);
L
Linus Torvalds 已提交
937 938
}

939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960
/*
 * A bad page could be due to a number of fields. Instead of multiple branches,
 * try and check multiple fields with one check. The caller must do a detailed
 * check if necessary.
 */
static inline bool page_expected_state(struct page *page,
					unsigned long check_flags)
{
	if (unlikely(atomic_read(&page->_mapcount) != -1))
		return false;

	if (unlikely((unsigned long)page->mapping |
			page_ref_count(page) |
#ifdef CONFIG_MEMCG
			(unsigned long)page->mem_cgroup |
#endif
			(page->flags & check_flags)))
		return false;

	return true;
}

961
static void free_pages_check_bad(struct page *page)
L
Linus Torvalds 已提交
962
{
963 964 965 966 967
	const char *bad_reason;
	unsigned long bad_flags;

	bad_reason = NULL;
	bad_flags = 0;
968

969
	if (unlikely(atomic_read(&page->_mapcount) != -1))
970 971 972
		bad_reason = "nonzero mapcount";
	if (unlikely(page->mapping != NULL))
		bad_reason = "non-NULL mapping";
973
	if (unlikely(page_ref_count(page) != 0))
974
		bad_reason = "nonzero _refcount";
975 976 977 978
	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
		bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
		bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
	}
979 980 981 982
#ifdef CONFIG_MEMCG
	if (unlikely(page->mem_cgroup))
		bad_reason = "page still charged to cgroup";
#endif
983
	bad_page(page, bad_reason, bad_flags);
984 985 986 987
}

static inline int free_pages_check(struct page *page)
{
988
	if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
989 990 991 992
		return 0;

	/* Something has gone sideways, find it */
	free_pages_check_bad(page);
993
	return 1;
L
Linus Torvalds 已提交
994 995
}

996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
static int free_tail_pages_check(struct page *head_page, struct page *page)
{
	int ret = 1;

	/*
	 * We rely page->lru.next never has bit 0 set, unless the page
	 * is PageTail(). Let's make sure that's true even for poisoned ->lru.
	 */
	BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);

	if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
		ret = 0;
		goto out;
	}
	switch (page - head_page) {
	case 1:
1012
		/* the first tail page: ->mapping may be compound_mapcount() */
1013 1014 1015 1016 1017 1018 1019 1020
		if (unlikely(compound_mapcount(page))) {
			bad_page(page, "nonzero compound_mapcount", 0);
			goto out;
		}
		break;
	case 2:
		/*
		 * the second tail page: ->mapping is
M
Matthew Wilcox 已提交
1021
		 * deferred_list.next -- ignore value.
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
		 */
		break;
	default:
		if (page->mapping != TAIL_MAPPING) {
			bad_page(page, "corrupted mapping in tail page", 0);
			goto out;
		}
		break;
	}
	if (unlikely(!PageTail(page))) {
		bad_page(page, "PageTail not set", 0);
		goto out;
	}
	if (unlikely(compound_head(page) != head_page)) {
		bad_page(page, "compound_head not consistent", 0);
		goto out;
	}
	ret = 0;
out:
	page->mapping = NULL;
	clear_compound_head(page);
	return ret;
}

1046 1047
static __always_inline bool free_pages_prepare(struct page *page,
					unsigned int order, bool check_free)
1048
{
1049
	int bad = 0;
1050 1051 1052

	VM_BUG_ON_PAGE(PageTail(page), page);

1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
	trace_mm_page_free(page, order);

	/*
	 * Check tail pages before head page information is cleared to
	 * avoid checking PageCompound for order-0 pages.
	 */
	if (unlikely(order)) {
		bool compound = PageCompound(page);
		int i;

		VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1064

1065 1066
		if (compound)
			ClearPageDoubleMap(page);
1067 1068 1069 1070 1071 1072 1073
		for (i = 1; i < (1 << order); i++) {
			if (compound)
				bad += free_tail_pages_check(page, page + i);
			if (unlikely(free_pages_check(page + i))) {
				bad++;
				continue;
			}
G
Gavin Shan 已提交
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084

			/*
			 * The page age information is stored in page flags
			 * or node's page array. We need to explicitly clear
			 * it in both cases. Otherwise, the stale age will
			 * be provided when it's allocated again. Also, we
			 * maintain age information for each page in the
			 * compound page, So we have to clear them one by one.
			 */
			kidled_set_page_age(page_pgdat(page + i),
					    page_to_pfn(page + i), 0);
1085 1086 1087
			(page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
		}
	}
1088
	if (PageMappingFlags(page))
1089
		page->mapping = NULL;
1090
	if (memcg_kmem_enabled() && PageKmemcg(page))
1091
		memcg_kmem_uncharge(page, order);
1092 1093 1094 1095
	if (check_free)
		bad += free_pages_check(page);
	if (bad)
		return false;
1096

1097
	page_cpupid_reset_last(page);
G
Gavin Shan 已提交
1098
	kidled_set_page_age(page_pgdat(page), page_to_pfn(page), 0);
1099 1100
	page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
	reset_page_owner(page, order);
1101 1102 1103

	if (!PageHighMem(page)) {
		debug_check_no_locks_freed(page_address(page),
1104
					   PAGE_SIZE << order);
1105
		debug_check_no_obj_freed(page_address(page),
1106
					   PAGE_SIZE << order);
1107
	}
1108 1109 1110
	arch_free_page(page, order);
	kernel_poison_pages(page, 1 << order, 0);
	kernel_map_pages(page, 1 << order, 0);
1111
	kasan_free_nondeferred_pages(page, order);
1112 1113 1114 1115

	return true;
}

1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
#ifdef CONFIG_DEBUG_VM
static inline bool free_pcp_prepare(struct page *page)
{
	return free_pages_prepare(page, 0, true);
}

static inline bool bulkfree_pcp_prepare(struct page *page)
{
	return false;
}
#else
static bool free_pcp_prepare(struct page *page)
{
	return free_pages_prepare(page, 0, false);
}

1132 1133 1134 1135 1136 1137
static bool bulkfree_pcp_prepare(struct page *page)
{
	return free_pages_check(page);
}
#endif /* CONFIG_DEBUG_VM */

1138 1139 1140 1141 1142 1143 1144 1145 1146
static inline void prefetch_buddy(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);
	unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
	struct page *buddy = page + (buddy_pfn - pfn);

	prefetch(buddy);
}

L
Linus Torvalds 已提交
1147
/*
1148
 * Frees a number of pages from the PCP lists
L
Linus Torvalds 已提交
1149
 * Assumes all pages on list are in same zone, and of same order.
1150
 * count is the number of pages to free.
L
Linus Torvalds 已提交
1151 1152 1153 1154 1155 1156 1157
 *
 * If the zone was previously in an "all pages pinned" state then look to
 * see if this freeing clears that state.
 *
 * And clear the zone's pages_scanned counter, to hold off the "all pages are
 * pinned" detection logic.
 */
1158 1159
static void free_pcppages_bulk(struct zone *zone, int count,
					struct per_cpu_pages *pcp)
L
Linus Torvalds 已提交
1160
{
1161
	int migratetype = 0;
1162
	int batch_free = 0;
1163
	int prefetch_nr = 0;
1164
	bool isolated_pageblocks;
1165 1166
	struct page *page, *tmp;
	LIST_HEAD(head);
1167

1168
	while (count) {
1169 1170 1171
		struct list_head *list;

		/*
1172 1173 1174 1175 1176
		 * Remove pages from lists in a round-robin fashion. A
		 * batch_free count is maintained that is incremented when an
		 * empty list is encountered.  This is so more pages are freed
		 * off fuller lists instead of spinning excessively around empty
		 * lists
1177 1178
		 */
		do {
1179
			batch_free++;
1180 1181 1182 1183
			if (++migratetype == MIGRATE_PCPTYPES)
				migratetype = 0;
			list = &pcp->lists[migratetype];
		} while (list_empty(list));
N
Nick Piggin 已提交
1184

1185 1186
		/* This is the only non-empty list. Free them all. */
		if (batch_free == MIGRATE_PCPTYPES)
1187
			batch_free = count;
1188

1189
		do {
1190
			page = list_last_entry(list, struct page, lru);
1191
			/* must delete to avoid corrupting pcp list */
1192
			list_del(&page->lru);
1193
			pcp->count--;
1194

1195 1196 1197
			if (bulkfree_pcp_prepare(page))
				continue;

1198
			list_add_tail(&page->lru, &head);
1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210

			/*
			 * We are going to put the page back to the global
			 * pool, prefetch its buddy to speed up later access
			 * under zone->lock. It is believed the overhead of
			 * an additional test and calculating buddy_pfn here
			 * can be offset by reduced memory latency later. To
			 * avoid excessive prefetching due to large count, only
			 * prefetch buddy for the first pcp->batch nr of pages.
			 */
			if (prefetch_nr++ < pcp->batch)
				prefetch_buddy(page);
1211
		} while (--count && --batch_free && !list_empty(list));
L
Linus Torvalds 已提交
1212
	}
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228

	spin_lock(&zone->lock);
	isolated_pageblocks = has_isolate_pageblock(zone);

	/*
	 * Use safe version since after __free_one_page(),
	 * page->lru.next will not point to original list.
	 */
	list_for_each_entry_safe(page, tmp, &head, lru) {
		int mt = get_pcppage_migratetype(page);
		/* MIGRATE_ISOLATE page should not go to pcplists */
		VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
		/* Pageblock could have been isolated meanwhile */
		if (unlikely(isolated_pageblocks))
			mt = get_pageblock_migratetype(page);

A
Alexander Duyck 已提交
1229
		__free_one_page(page, page_to_pfn(page), zone, 0, mt, true);
1230 1231
		trace_mm_page_pcpu_drain(page, 0, mt);
	}
1232
	spin_unlock(&zone->lock);
L
Linus Torvalds 已提交
1233 1234
}

1235 1236
static void free_one_page(struct zone *zone,
				struct page *page, unsigned long pfn,
1237
				unsigned int order,
1238
				int migratetype)
L
Linus Torvalds 已提交
1239
{
1240
	spin_lock(&zone->lock);
1241 1242 1243 1244
	if (unlikely(has_isolate_pageblock(zone) ||
		is_migrate_isolate(migratetype))) {
		migratetype = get_pfnblock_migratetype(page, pfn);
	}
A
Alexander Duyck 已提交
1245
	__free_one_page(page, pfn, zone, order, migratetype, true);
1246
	spin_unlock(&zone->lock);
N
Nick Piggin 已提交
1247 1248
}

1249
static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1250
				unsigned long zone, int nid)
1251
{
1252
	mm_zero_struct_page(page);
1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
	set_page_links(page, zone, nid, pfn);
	init_page_count(page);
	page_mapcount_reset(page);
	page_cpupid_reset_last(page);

	INIT_LIST_HEAD(&page->lru);
#ifdef WANT_PAGE_VIRTUAL
	/* The shift won't overflow because ZONE_NORMAL is below 4G. */
	if (!is_highmem_idx(zone))
		set_page_address(page, __va(pfn << PAGE_SHIFT));
#endif
}

1266
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1267
static void __meminit init_reserved_page(unsigned long pfn)
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
{
	pg_data_t *pgdat;
	int nid, zid;

	if (!early_page_uninitialised(pfn))
		return;

	nid = early_pfn_to_nid(pfn);
	pgdat = NODE_DATA(nid);

	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
		struct zone *zone = &pgdat->node_zones[zid];

		if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
			break;
	}
1284
	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
1285 1286 1287 1288 1289 1290 1291
}
#else
static inline void init_reserved_page(unsigned long pfn)
{
}
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */

1292 1293 1294 1295 1296 1297
/*
 * Initialised pages do not have PageReserved set. This function is
 * called for each range allocated by the bootmem allocator and
 * marks the pages PageReserved. The remaining valid pages are later
 * sent to the buddy page allocator.
 */
1298
void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
1299 1300 1301 1302
{
	unsigned long start_pfn = PFN_DOWN(start);
	unsigned long end_pfn = PFN_UP(end);

1303 1304 1305 1306 1307
	for (; start_pfn < end_pfn; start_pfn++) {
		if (pfn_valid(start_pfn)) {
			struct page *page = pfn_to_page(start_pfn);

			init_reserved_page(start_pfn);
1308 1309 1310 1311

			/* Avoid false-positive PageTail() */
			INIT_LIST_HEAD(&page->lru);

1312 1313 1314
			SetPageReserved(page);
		}
	}
1315 1316
}

1317 1318
static void __free_pages_ok(struct page *page, unsigned int order)
{
1319
	unsigned long flags;
M
Minchan Kim 已提交
1320
	int migratetype;
1321
	unsigned long pfn = page_to_pfn(page);
1322

1323
	if (!free_pages_prepare(page, order, true))
1324 1325
		return;

1326
	migratetype = get_pfnblock_migratetype(page, pfn);
1327 1328
	local_irq_save(flags);
	__count_vm_events(PGFREE, 1 << order);
1329
	free_one_page(page_zone(page), page, pfn, order, migratetype);
1330
	local_irq_restore(flags);
L
Linus Torvalds 已提交
1331 1332
}

1333
static void __init __free_pages_boot_core(struct page *page, unsigned int order)
1334
{
1335
	unsigned int nr_pages = 1 << order;
1336
	struct page *p = page;
1337
	unsigned int loop;
1338

1339 1340 1341
	prefetchw(p);
	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
		prefetchw(p + 1);
1342 1343
		__ClearPageReserved(p);
		set_page_count(p, 0);
1344
	}
1345 1346
	__ClearPageReserved(p);
	set_page_count(p, 0);
1347

1348
	page_zone(page)->managed_pages += nr_pages;
1349 1350
	set_page_refcounted(page);
	__free_pages(page, order);
1351 1352
}

1353 1354
#if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \
	defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1355

1356 1357 1358 1359
static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;

int __meminit early_pfn_to_nid(unsigned long pfn)
{
1360
	static DEFINE_SPINLOCK(early_pfn_lock);
1361 1362
	int nid;

1363
	spin_lock(&early_pfn_lock);
1364
	nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1365
	if (nid < 0)
1366
		nid = first_online_node;
1367 1368 1369
	spin_unlock(&early_pfn_lock);

	return nid;
1370 1371 1372 1373
}
#endif

#ifdef CONFIG_NODES_SPAN_OTHER_NODES
1374 1375
/* Only safe to use early in boot when initialisation is single-threaded */
static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1376 1377 1378
{
	int nid;

1379
	nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
	if (nid >= 0 && nid != node)
		return false;
	return true;
}

#else
static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
{
	return true;
}
#endif


1393
void __init __free_pages_bootmem(struct page *page, unsigned long pfn,
1394 1395 1396 1397
							unsigned int order)
{
	if (early_page_uninitialised(pfn))
		return;
1398
	return __free_pages_boot_core(page, order);
1399 1400
}

1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429
/*
 * Check that the whole (or subset of) a pageblock given by the interval of
 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
 * with the migration of free compaction scanner. The scanners then need to
 * use only pfn_valid_within() check for arches that allow holes within
 * pageblocks.
 *
 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
 *
 * It's possible on some configurations to have a setup like node0 node1 node0
 * i.e. it's possible that all pages within a zones range of pages do not
 * belong to a single zone. We assume that a border between node0 and node1
 * can occur within a single pageblock, but not a node0 node1 node0
 * interleaving within a single pageblock. It is therefore sufficient to check
 * the first and last page of a pageblock and avoid checking each individual
 * page in a pageblock.
 */
struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
				     unsigned long end_pfn, struct zone *zone)
{
	struct page *start_page;
	struct page *end_page;

	/* end_pfn is one past the range we are checking */
	end_pfn--;

	if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
		return NULL;

1430 1431 1432
	start_page = pfn_to_online_page(start_pfn);
	if (!start_page)
		return NULL;
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471

	if (page_zone(start_page) != zone)
		return NULL;

	end_page = pfn_to_page(end_pfn);

	/* This gives a shorter code than deriving page_zone(end_page) */
	if (page_zone_id(start_page) != page_zone_id(end_page))
		return NULL;

	return start_page;
}

void set_zone_contiguous(struct zone *zone)
{
	unsigned long block_start_pfn = zone->zone_start_pfn;
	unsigned long block_end_pfn;

	block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
	for (; block_start_pfn < zone_end_pfn(zone);
			block_start_pfn = block_end_pfn,
			 block_end_pfn += pageblock_nr_pages) {

		block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));

		if (!__pageblock_pfn_to_page(block_start_pfn,
					     block_end_pfn, zone))
			return;
	}

	/* We confirm that there is no hole */
	zone->contiguous = true;
}

void clear_zone_contiguous(struct zone *zone)
{
	zone->contiguous = false;
}

1472
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1473 1474
static void __init deferred_free_range(unsigned long pfn,
				       unsigned long nr_pages)
1475
{
1476 1477
	struct page *page;
	unsigned long i;
1478

1479
	if (!nr_pages)
1480 1481
		return;

1482 1483
	page = pfn_to_page(pfn);

1484
	/* Free a large naturally-aligned chunk if possible */
1485 1486
	if (nr_pages == pageblock_nr_pages &&
	    (pfn & (pageblock_nr_pages - 1)) == 0) {
1487
		set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1488
		__free_pages_boot_core(page, pageblock_order);
1489 1490 1491
		return;
	}

1492 1493 1494
	for (i = 0; i < nr_pages; i++, page++, pfn++) {
		if ((pfn & (pageblock_nr_pages - 1)) == 0)
			set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1495
		__free_pages_boot_core(page, 0);
1496
	}
1497 1498
}

1499 1500 1501 1502 1503 1504 1505 1506 1507
/* Completion tracking for deferred_init_memmap() threads */
static atomic_t pgdat_init_n_undone __initdata;
static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);

static inline void __init pgdat_init_report_one_done(void)
{
	if (atomic_dec_and_test(&pgdat_init_n_undone))
		complete(&pgdat_init_all_done_comp);
}
1508

1509
/*
1510 1511 1512 1513 1514 1515 1516 1517
 * Returns true if page needs to be initialized or freed to buddy allocator.
 *
 * First we check if pfn is valid on architectures where it is possible to have
 * holes within pageblock_nr_pages. On systems where it is not possible, this
 * function is optimized out.
 *
 * Then, we check if a current large page is valid by only checking the validity
 * of the head pfn.
1518
 */
1519
static inline bool __init deferred_pfn_valid(unsigned long pfn)
1520
{
1521 1522 1523 1524 1525 1526
	if (!pfn_valid_within(pfn))
		return false;
	if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
		return false;
	return true;
}
1527

1528 1529 1530 1531
/*
 * Free pages to buddy allocator. Try to free aligned pages in
 * pageblock_nr_pages sizes.
 */
1532
static void __init deferred_free_pages(unsigned long pfn,
1533 1534 1535 1536
				       unsigned long end_pfn)
{
	unsigned long nr_pgmask = pageblock_nr_pages - 1;
	unsigned long nr_free = 0;
1537

1538
	for (; pfn < end_pfn; pfn++) {
1539
		if (!deferred_pfn_valid(pfn)) {
1540 1541 1542 1543 1544
			deferred_free_range(pfn - nr_free, nr_free);
			nr_free = 0;
		} else if (!(pfn & nr_pgmask)) {
			deferred_free_range(pfn - nr_free, nr_free);
			nr_free = 1;
1545
			touch_nmi_watchdog();
1546 1547 1548 1549 1550 1551
		} else {
			nr_free++;
		}
	}
	/* Free the last block of pages to allocator */
	deferred_free_range(pfn - nr_free, nr_free);
1552 1553
}

1554 1555 1556 1557 1558
/*
 * Initialize struct pages.  We minimize pfn page lookups and scheduler checks
 * by performing it only once every pageblock_nr_pages.
 * Return number of pages initialized.
 */
1559
static unsigned long  __init deferred_init_pages(struct zone *zone,
1560 1561
						 unsigned long pfn,
						 unsigned long end_pfn)
1562 1563
{
	unsigned long nr_pgmask = pageblock_nr_pages - 1;
1564
	int nid = zone_to_nid(zone);
1565
	unsigned long nr_pages = 0;
1566
	int zid = zone_idx(zone);
1567 1568
	struct page *page = NULL;

1569
	for (; pfn < end_pfn; pfn++) {
1570
		if (!deferred_pfn_valid(pfn)) {
1571
			page = NULL;
1572
			continue;
1573
		} else if (!page || !(pfn & nr_pgmask)) {
1574
			page = pfn_to_page(pfn);
1575
			touch_nmi_watchdog();
1576 1577
		} else {
			page++;
1578
		}
1579
		__init_single_page(page, pfn, zid, nid);
1580
		nr_pages++;
1581
	}
1582
	return (nr_pages);
1583 1584
}

1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
/*
 * This function is meant to pre-load the iterator for the zone init.
 * Specifically it walks through the ranges until we are caught up to the
 * first_init_pfn value and exits there. If we never encounter the value we
 * return false indicating there are no valid ranges left.
 */
static bool __init
deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
				    unsigned long *spfn, unsigned long *epfn,
				    unsigned long first_init_pfn)
{
	u64 j;

	/*
	 * Start out by walking through the ranges in this zone that have
	 * already been initialized. We don't need to do anything with them
	 * so we just need to flush them out of the system.
	 */
	for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
		if (*epfn <= first_init_pfn)
			continue;
		if (*spfn < first_init_pfn)
			*spfn = first_init_pfn;
		*i = j;
		return true;
	}

	return false;
}

/*
 * Initialize and free pages. We do it in two loops: first we initialize
 * struct page, then free to buddy allocator, because while we are
 * freeing pages we can access pages that are ahead (computing buddy
 * page in __free_one_page()).
 *
 * In order to try and keep some memory in the cache we have the loop
 * broken along max page order boundaries. This way we will not cause
 * any issues with the buddy page computation.
 */
static unsigned long __init
deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
		       unsigned long *end_pfn)
{
	unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
	unsigned long spfn = *start_pfn, epfn = *end_pfn;
	unsigned long nr_pages = 0;
	u64 j = *i;

	/* First we loop through and initialize the page values */
	for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
		unsigned long t;

		if (mo_pfn <= *start_pfn)
			break;

		t = min(mo_pfn, *end_pfn);
		nr_pages += deferred_init_pages(zone, *start_pfn, t);

		if (mo_pfn < *end_pfn) {
			*start_pfn = mo_pfn;
			break;
		}
	}

	/* Reset values and now loop through freeing pages as needed */
	swap(j, *i);

	for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
		unsigned long t;

		if (mo_pfn <= spfn)
			break;

		t = min(mo_pfn, epfn);
		deferred_free_pages(spfn, t);

		if (mo_pfn <= epfn)
			break;
	}

	return nr_pages;
}

1669 1670 1671 1672 1673
/*
 * Release the pending interrupts for every TICK_PAGE_COUNT pages.
 */
#define TICK_PAGE_COUNT	(32 * 1024)

1674
/* Initialise remaining memory on a node */
1675
static int __init deferred_init_memmap(void *data)
1676
{
1677
	pg_data_t *pgdat = data;
1678
	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
1679
	unsigned long spfn = 0, epfn = 0, nr_pages = 0, prev_nr_pages = 0;
1680
	unsigned long first_init_pfn, flags;
1681 1682
	unsigned long start = jiffies;
	struct zone *zone;
1683
	int zid;
1684
	u64 i;
1685

1686 1687 1688 1689
	/* Bind memory initialisation thread to a local node if possible */
	if (!cpumask_empty(cpumask))
		set_cpus_allowed_ptr(current, cpumask);

1690
again:
1691 1692
	pgdat_resize_lock(pgdat, &flags);
	first_init_pfn = pgdat->first_deferred_pfn;
1693
	if (first_init_pfn == ULONG_MAX) {
1694
		pgdat_resize_unlock(pgdat, &flags);
1695
		pgdat_init_report_one_done();
1696 1697 1698
		return 0;
	}

1699 1700 1701 1702 1703 1704 1705 1706 1707 1708
	/* Sanity check boundaries */
	BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
	BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));

	/* Only the highest zone is deferred so find it */
	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
		zone = pgdat->node_zones + zid;
		if (first_init_pfn < zone_end_pfn(zone))
			break;
	}
1709 1710 1711 1712 1713

	/* If the zone is empty somebody else may have cleared out the zone */
	if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
						 first_init_pfn))
		goto zone_empty;
1714

1715
	/*
1716 1717 1718
	 * Initialize and free pages in MAX_ORDER sized increments so
	 * that we can avoid introducing any issues with the buddy
	 * allocator.
1719
	 */
1720
	while (spfn < epfn) {
1721
		nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
		/*
		 * Release the interrupts for every TICK_PAGE_COUNT pages
		 * (128MB) to give tick timer the chance to update the
		 * system jiffies.
		 */
		if ((nr_pages - prev_nr_pages) > TICK_PAGE_COUNT) {
			prev_nr_pages = nr_pages;
			pgdat->first_deferred_pfn = spfn;
			pgdat_resize_unlock(pgdat, &flags);
			goto again;
		}
	}

1735
zone_empty:
1736
	pgdat->first_deferred_pfn = ULONG_MAX;
1737
	pgdat_resize_unlock(pgdat, &flags);
1738 1739 1740 1741

	/* Sanity check that the next zone really is unpopulated */
	WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));

1742 1743
	pr_info("node %d initialised, %lu pages in %ums\n",
		pgdat->node_id,	nr_pages, jiffies_to_msecs(jiffies - start));
1744 1745

	pgdat_init_report_one_done();
1746 1747
	return 0;
}
1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767

/*
 * If this zone has deferred pages, try to grow it by initializing enough
 * deferred pages to satisfy the allocation specified by order, rounded up to
 * the nearest PAGES_PER_SECTION boundary.  So we're adding memory in increments
 * of SECTION_SIZE bytes by initializing struct pages in increments of
 * PAGES_PER_SECTION * sizeof(struct page) bytes.
 *
 * Return true when zone was grown, otherwise return false. We return true even
 * when we grow less than requested, to let the caller decide if there are
 * enough pages to satisfy the allocation.
 *
 * Note: We use noinline because this function is needed only during boot, and
 * it is called from a __ref function _deferred_grow_zone. This way we are
 * making sure that it is not inlined into permanent text section.
 */
static noinline bool __init
deferred_grow_zone(struct zone *zone, unsigned int order)
{
	unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
1768
	pg_data_t *pgdat = zone->zone_pgdat;
1769
	unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
1770 1771
	unsigned long spfn, epfn, flags;
	unsigned long nr_pages = 0;
1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
	u64 i;

	/* Only the last zone may have deferred pages */
	if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
		return false;

	pgdat_resize_lock(pgdat, &flags);

	/*
	 * If deferred pages have been initialized while we were waiting for
	 * the lock, return true, as the zone was grown.  The caller will retry
	 * this zone.  We won't return to this function since the caller also
	 * has this static branch.
	 */
	if (!static_branch_unlikely(&deferred_pages)) {
		pgdat_resize_unlock(pgdat, &flags);
		return true;
	}

	/*
	 * If someone grew this zone while we were waiting for spinlock, return
	 * true, as there might be enough pages already.
	 */
	if (first_deferred_pfn != pgdat->first_deferred_pfn) {
		pgdat_resize_unlock(pgdat, &flags);
		return true;
	}

1800 1801 1802 1803
	/* If the zone is empty somebody else may have cleared out the zone */
	if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
						 first_deferred_pfn)) {
		pgdat->first_deferred_pfn = ULONG_MAX;
1804
		pgdat_resize_unlock(pgdat, &flags);
1805 1806
		/* Retry only once. */
		return first_deferred_pfn != ULONG_MAX;
1807 1808
	}

1809 1810 1811 1812 1813 1814 1815 1816 1817 1818
	/*
	 * Initialize and free pages in MAX_ORDER sized increments so
	 * that we can avoid introducing any issues with the buddy
	 * allocator.
	 */
	while (spfn < epfn) {
		/* update our first deferred PFN for this section */
		first_deferred_pfn = spfn;

		nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
1819

1820 1821 1822
		/* We should only stop along section boundaries */
		if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
			continue;
1823

1824
		/* If our quota has been met we can stop here */
1825 1826 1827 1828
		if (nr_pages >= nr_pages_needed)
			break;
	}

1829
	pgdat->first_deferred_pfn = spfn;
1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846
	pgdat_resize_unlock(pgdat, &flags);

	return nr_pages > 0;
}

/*
 * deferred_grow_zone() is __init, but it is called from
 * get_page_from_freelist() during early boot until deferred_pages permanently
 * disables this call. This is why we have refdata wrapper to avoid warning,
 * and to ensure that the function body gets unloaded.
 */
static bool __ref
_deferred_grow_zone(struct zone *zone, unsigned int order)
{
	return deferred_grow_zone(zone, order);
}

1847
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1848 1849 1850

void __init page_alloc_init_late(void)
{
1851 1852 1853
	struct zone *zone;

#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1854 1855
	int nid;

1856 1857
	/* There will be num_node_state(N_MEMORY) threads */
	atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
1858 1859 1860 1861 1862
	for_each_node_state(nid, N_MEMORY) {
		kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
	}

	/* Block until all are initialised */
1863
	wait_for_completion(&pgdat_init_all_done_comp);
1864

1865 1866 1867 1868 1869 1870 1871 1872
	/*
	 * The number of managed pages has changed due to the initialisation
	 * so the pcpu batch and high limits needs to be updated or the limits
	 * will be artificially small.
	 */
	for_each_populated_zone(zone)
		zone_pcp_update(zone);

1873 1874 1875 1876 1877 1878
	/*
	 * We initialized the rest of the deferred pages.  Permanently disable
	 * on-demand struct page initialization.
	 */
	static_branch_disable(&deferred_pages);

1879 1880
	/* Reinit limits that are based on free pages after the kernel is up */
	files_maxfiles_init();
1881
#endif
P
Pavel Tatashin 已提交
1882 1883 1884 1885
#ifdef CONFIG_ARCH_DISCARD_MEMBLOCK
	/* Discard memblock private memory */
	memblock_discard();
#endif
1886 1887 1888

	for_each_populated_zone(zone)
		set_zone_contiguous(zone);
1889 1890
}

1891
#ifdef CONFIG_CMA
1892
/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
1893 1894 1895 1896 1897 1898 1899 1900
void __init init_cma_reserved_pageblock(struct page *page)
{
	unsigned i = pageblock_nr_pages;
	struct page *p = page;

	do {
		__ClearPageReserved(p);
		set_page_count(p, 0);
1901
	} while (++p, --i);
1902 1903

	set_pageblock_migratetype(page, MIGRATE_CMA);
1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917

	if (pageblock_order >= MAX_ORDER) {
		i = pageblock_nr_pages;
		p = page;
		do {
			set_page_refcounted(p);
			__free_pages(p, MAX_ORDER - 1);
			p += MAX_ORDER_NR_PAGES;
		} while (i -= MAX_ORDER_NR_PAGES);
	} else {
		set_page_refcounted(page);
		__free_pages(page, pageblock_order);
	}

1918
	adjust_managed_page_count(page, pageblock_nr_pages);
1919 1920
}
#endif
L
Linus Torvalds 已提交
1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933

/*
 * The order of subdivision here is critical for the IO subsystem.
 * Please do not alter this order without good reasons and regression
 * testing. Specifically, as large blocks of memory are subdivided,
 * the order in which smaller blocks are delivered depends on the order
 * they're subdivided in this function. This is the primary factor
 * influencing the order in which pages are delivered to the IO
 * subsystem according to empirical testing, and this is also justified
 * by considering the behavior of a buddy system containing a single
 * large block of memory acted on by a series of small allocations.
 * This behavior is a critical factor in sglist merging's success.
 *
1934
 * -- nyc
L
Linus Torvalds 已提交
1935
 */
N
Nick Piggin 已提交
1936
static inline void expand(struct zone *zone, struct page *page,
1937
	int low, int high, int migratetype)
L
Linus Torvalds 已提交
1938 1939 1940 1941 1942 1943
{
	unsigned long size = 1 << high;

	while (high > low) {
		high--;
		size >>= 1;
1944
		VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
1945

1946 1947 1948 1949 1950 1951 1952
		/*
		 * Mark as guard pages (or page), that will allow to
		 * merge back to allocator when buddy will be freed.
		 * Corresponding page table entries will not be touched,
		 * pages will stay not present in virtual address space
		 */
		if (set_page_guard(zone, &page[size], high, migratetype))
1953
			continue;
1954

1955
		add_to_free_list(&page[size], zone, high, migratetype);
L
Linus Torvalds 已提交
1956 1957 1958 1959
		set_page_order(&page[size], high);
	}
}

1960
static void check_new_page_bad(struct page *page)
L
Linus Torvalds 已提交
1961
{
1962 1963
	const char *bad_reason = NULL;
	unsigned long bad_flags = 0;
1964

1965
	if (unlikely(atomic_read(&page->_mapcount) != -1))
1966 1967 1968
		bad_reason = "nonzero mapcount";
	if (unlikely(page->mapping != NULL))
		bad_reason = "non-NULL mapping";
1969
	if (unlikely(page_ref_count(page) != 0))
1970
		bad_reason = "nonzero _count";
1971 1972 1973
	if (unlikely(page->flags & __PG_HWPOISON)) {
		bad_reason = "HWPoisoned (hardware-corrupted)";
		bad_flags = __PG_HWPOISON;
1974 1975 1976
		/* Don't complain about hwpoisoned pages */
		page_mapcount_reset(page); /* remove PageBuddy */
		return;
1977
	}
1978 1979 1980 1981
	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
		bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
		bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
	}
1982 1983 1984 1985
#ifdef CONFIG_MEMCG
	if (unlikely(page->mem_cgroup))
		bad_reason = "page still charged to cgroup";
#endif
1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
	bad_page(page, bad_reason, bad_flags);
}

/*
 * This page is about to be returned from the page allocator
 */
static inline int check_new_page(struct page *page)
{
	if (likely(page_expected_state(page,
				PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
		return 0;

	check_new_page_bad(page);
	return 1;
2000 2001
}

2002
static inline bool free_pages_prezeroed(void)
2003 2004
{
	return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
2005
		page_poisoning_enabled();
2006 2007
}

2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041
#ifdef CONFIG_DEBUG_VM
static bool check_pcp_refill(struct page *page)
{
	return false;
}

static bool check_new_pcp(struct page *page)
{
	return check_new_page(page);
}
#else
static bool check_pcp_refill(struct page *page)
{
	return check_new_page(page);
}
static bool check_new_pcp(struct page *page)
{
	return false;
}
#endif /* CONFIG_DEBUG_VM */

static bool check_new_pages(struct page *page, unsigned int order)
{
	int i;
	for (i = 0; i < (1 << order); i++) {
		struct page *p = page + i;

		if (unlikely(check_new_page(p)))
			return true;
	}

	return false;
}

2042 2043 2044 2045 2046 2047 2048 2049 2050
inline void post_alloc_hook(struct page *page, unsigned int order,
				gfp_t gfp_flags)
{
	set_page_private(page, 0);
	set_page_refcounted(page);

	arch_alloc_page(page, order);
	kernel_map_pages(page, 1 << order, 1);
	kasan_alloc_pages(page, order);
Q
Qian Cai 已提交
2051
	kernel_poison_pages(page, 1 << order, 1);
2052 2053 2054
	set_page_owner(page, order, gfp_flags);
}

2055
static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
2056
							unsigned int alloc_flags)
2057 2058
{
	int i;
2059

2060
	post_alloc_hook(page, order, gfp_flags);
N
Nick Piggin 已提交
2061

2062
	if (!free_pages_prezeroed() && (gfp_flags & __GFP_ZERO))
2063 2064
		for (i = 0; i < (1 << order); i++)
			clear_highpage(page + i);
N
Nick Piggin 已提交
2065 2066 2067 2068

	if (order && (gfp_flags & __GFP_COMP))
		prep_compound_page(page, order);

2069
	/*
2070
	 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
2071 2072 2073 2074
	 * allocate the page. The expectation is that the caller is taking
	 * steps that will free more memory. The caller should avoid the page
	 * being used for !PFMEMALLOC purposes.
	 */
2075 2076 2077 2078
	if (alloc_flags & ALLOC_NO_WATERMARKS)
		set_page_pfmemalloc(page);
	else
		clear_page_pfmemalloc(page);
L
Linus Torvalds 已提交
2079 2080
}

2081 2082 2083 2084
/*
 * Go through the free lists for the given migratetype and remove
 * the smallest available page from the freelists
 */
2085
static __always_inline
2086
struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
2087 2088 2089
						int migratetype)
{
	unsigned int current_order;
2090
	struct free_area *area;
2091 2092 2093 2094 2095
	struct page *page;

	/* Find a page of the appropriate size in the preferred list */
	for (current_order = order; current_order < MAX_ORDER; ++current_order) {
		area = &(zone->free_area[current_order]);
2096
		page = get_page_from_free_area(area, migratetype);
2097 2098
		if (!page)
			continue;
2099 2100
		del_page_from_free_list(page, zone, current_order);
		expand(zone, page, order, current_order, migratetype);
2101
		set_pcppage_migratetype(page, migratetype);
2102 2103 2104 2105 2106 2107 2108
		return page;
	}

	return NULL;
}


2109 2110 2111 2112
/*
 * This array describes the order lists are fallen back to when
 * the free lists for the desirable migrate type are depleted
 */
2113
static int fallbacks[MIGRATE_TYPES][4] = {
2114 2115 2116
	[MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_TYPES },
	[MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_TYPES },
	[MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
2117
#ifdef CONFIG_CMA
2118
	[MIGRATE_CMA]         = { MIGRATE_TYPES }, /* Never used */
2119
#endif
2120
#ifdef CONFIG_MEMORY_ISOLATION
2121
	[MIGRATE_ISOLATE]     = { MIGRATE_TYPES }, /* Never used */
2122
#endif
2123 2124
};

2125
#ifdef CONFIG_CMA
2126
static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2127 2128 2129 2130 2131 2132 2133 2134 2135
					unsigned int order)
{
	return __rmqueue_smallest(zone, order, MIGRATE_CMA);
}
#else
static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
					unsigned int order) { return NULL; }
#endif

2136 2137
/*
 * Move the free pages in a range to the free lists of the requested type.
2138
 * Note that start_page and end_pages are not aligned on a pageblock
2139 2140
 * boundary. If alignment is required, use move_freepages_block()
 */
2141
static int move_freepages(struct zone *zone,
A
Adrian Bunk 已提交
2142
			  struct page *start_page, struct page *end_page,
2143
			  int migratetype, int *num_movable)
2144 2145
{
	struct page *page;
2146
	unsigned int order;
2147
	int pages_moved = 0;
2148 2149 2150 2151 2152 2153 2154

#ifndef CONFIG_HOLES_IN_ZONE
	/*
	 * page_zone is not safe to call in this context when
	 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
	 * anyway as we check zone boundaries in move_freepages_block().
	 * Remove at a later date when no bug reports exist related to
M
Mel Gorman 已提交
2155
	 * grouping pages by mobility
2156
	 */
2157 2158 2159
	VM_BUG_ON(pfn_valid(page_to_pfn(start_page)) &&
	          pfn_valid(page_to_pfn(end_page)) &&
	          page_zone(start_page) != page_zone(end_page));
2160 2161
#endif

2162 2163 2164
	if (num_movable)
		*num_movable = 0;

2165 2166 2167 2168 2169 2170
	for (page = start_page; page <= end_page;) {
		if (!pfn_valid_within(page_to_pfn(page))) {
			page++;
			continue;
		}

2171 2172 2173
		/* Make sure we are not inadvertently changing nodes */
		VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);

2174
		if (!PageBuddy(page)) {
2175 2176 2177 2178 2179 2180 2181 2182 2183
			/*
			 * We assume that pages that could be isolated for
			 * migration are movable. But we don't actually try
			 * isolating, as that would be expensive.
			 */
			if (num_movable &&
					(PageLRU(page) || __PageMovable(page)))
				(*num_movable)++;

2184 2185 2186 2187 2188
			page++;
			continue;
		}

		order = page_order(page);
2189
		move_to_free_list(page, zone, order, migratetype);
2190
		page += 1 << order;
2191
		pages_moved += 1 << order;
2192 2193
	}

2194
	return pages_moved;
2195 2196
}

2197
int move_freepages_block(struct zone *zone, struct page *page,
2198
				int migratetype, int *num_movable)
2199 2200 2201 2202 2203
{
	unsigned long start_pfn, end_pfn;
	struct page *start_page, *end_page;

	start_pfn = page_to_pfn(page);
2204
	start_pfn = start_pfn & ~(pageblock_nr_pages-1);
2205
	start_page = pfn_to_page(start_pfn);
2206 2207
	end_page = start_page + pageblock_nr_pages - 1;
	end_pfn = start_pfn + pageblock_nr_pages - 1;
2208 2209

	/* Do not cross zone boundaries */
2210
	if (!zone_spans_pfn(zone, start_pfn))
2211
		start_page = page;
2212
	if (!zone_spans_pfn(zone, end_pfn))
2213 2214
		return 0;

2215 2216
	return move_freepages(zone, start_page, end_page, migratetype,
								num_movable);
2217 2218
}

2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229
static void change_pageblock_range(struct page *pageblock_page,
					int start_order, int migratetype)
{
	int nr_pageblocks = 1 << (start_order - pageblock_order);

	while (nr_pageblocks--) {
		set_pageblock_migratetype(pageblock_page, migratetype);
		pageblock_page += pageblock_nr_pages;
	}
}

2230
/*
2231 2232 2233 2234 2235 2236 2237 2238 2239 2240
 * When we are falling back to another migratetype during allocation, try to
 * steal extra free pages from the same pageblocks to satisfy further
 * allocations, instead of polluting multiple pageblocks.
 *
 * If we are stealing a relatively large buddy page, it is likely there will
 * be more free pages in the pageblock, so try to steal them all. For
 * reclaimable and unmovable allocations, we steal regardless of page size,
 * as fragmentation caused by those allocations polluting movable pageblocks
 * is worse than movable allocations stealing from unmovable and reclaimable
 * pageblocks.
2241
 */
2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265
static bool can_steal_fallback(unsigned int order, int start_mt)
{
	/*
	 * Leaving this order check is intended, although there is
	 * relaxed order check in next check. The reason is that
	 * we can actually steal whole pageblock if this condition met,
	 * but, below check doesn't guarantee it and that is just heuristic
	 * so could be changed anytime.
	 */
	if (order >= pageblock_order)
		return true;

	if (order >= pageblock_order / 2 ||
		start_mt == MIGRATE_RECLAIMABLE ||
		start_mt == MIGRATE_UNMOVABLE ||
		page_group_by_mobility_disabled)
		return true;

	return false;
}

/*
 * This function implements actual steal behaviour. If order is large enough,
 * we can steal whole pageblock. If not, we first move freepages in this
2266 2267 2268 2269
 * pageblock to our migratetype and determine how many already-allocated pages
 * are there in the pageblock with a compatible migratetype. If at least half
 * of pages are free or compatible, we can change migratetype of the pageblock
 * itself, so pages freed in the future will be put on the correct free list.
2270 2271
 */
static void steal_suitable_fallback(struct zone *zone, struct page *page,
2272
					int start_type, bool whole_block)
2273
{
2274
	unsigned int current_order = page_order(page);
2275 2276 2277 2278
	int free_pages, movable_pages, alike_pages;
	int old_block_type;

	old_block_type = get_pageblock_migratetype(page);
2279

2280 2281 2282 2283
	/*
	 * This can happen due to races and we want to prevent broken
	 * highatomic accounting.
	 */
2284
	if (is_migrate_highatomic(old_block_type))
2285 2286
		goto single_page;

2287 2288 2289
	/* Take ownership for orders >= pageblock_order */
	if (current_order >= pageblock_order) {
		change_pageblock_range(page, current_order, start_type);
2290
		goto single_page;
2291 2292
	}

2293 2294 2295 2296
	/* We are not allowed to try stealing from the whole block */
	if (!whole_block)
		goto single_page;

2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320
	free_pages = move_freepages_block(zone, page, start_type,
						&movable_pages);
	/*
	 * Determine how many pages are compatible with our allocation.
	 * For movable allocation, it's the number of movable pages which
	 * we just obtained. For other types it's a bit more tricky.
	 */
	if (start_type == MIGRATE_MOVABLE) {
		alike_pages = movable_pages;
	} else {
		/*
		 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
		 * to MOVABLE pageblock, consider all non-movable pages as
		 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
		 * vice versa, be conservative since we can't distinguish the
		 * exact migratetype of non-movable pages.
		 */
		if (old_block_type == MIGRATE_MOVABLE)
			alike_pages = pageblock_nr_pages
						- (free_pages + movable_pages);
		else
			alike_pages = 0;
	}

2321
	/* moving whole block can fail due to zone boundary conditions */
2322
	if (!free_pages)
2323
		goto single_page;
2324

2325 2326 2327 2328 2329
	/*
	 * If a sufficient number of pages in the block are either free or of
	 * comparable migratability as our allocation, claim the whole block.
	 */
	if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
2330 2331
			page_group_by_mobility_disabled)
		set_pageblock_migratetype(page, start_type);
2332 2333 2334 2335

	return;

single_page:
2336
	move_to_free_list(page, zone, current_order, start_type);
2337 2338
}

2339 2340 2341 2342 2343 2344 2345 2346
/*
 * Check whether there is a suitable fallback freepage with requested order.
 * If only_stealable is true, this function returns fallback_mt only if
 * we can steal other freepages all together. This would help to reduce
 * fragmentation due to mixed migratetype pages in one pageblock.
 */
int find_suitable_fallback(struct free_area *area, unsigned int order,
			int migratetype, bool only_stealable, bool *can_steal)
2347 2348 2349 2350 2351 2352 2353 2354 2355 2356
{
	int i;
	int fallback_mt;

	if (area->nr_free == 0)
		return -1;

	*can_steal = false;
	for (i = 0;; i++) {
		fallback_mt = fallbacks[migratetype][i];
2357
		if (fallback_mt == MIGRATE_TYPES)
2358 2359
			break;

2360
		if (free_area_empty(area, fallback_mt))
2361
			continue;
2362

2363 2364 2365
		if (can_steal_fallback(order, migratetype))
			*can_steal = true;

2366 2367 2368 2369 2370
		if (!only_stealable)
			return fallback_mt;

		if (*can_steal)
			return fallback_mt;
2371
	}
2372 2373

	return -1;
2374 2375
}

2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
/*
 * Reserve a pageblock for exclusive use of high-order atomic allocations if
 * there are no empty page blocks that contain a page with a suitable order
 */
static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
				unsigned int alloc_order)
{
	int mt;
	unsigned long max_managed, flags;

	/*
	 * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
	 * Check is race-prone but harmless.
	 */
	max_managed = (zone->managed_pages / 100) + pageblock_nr_pages;
	if (zone->nr_reserved_highatomic >= max_managed)
		return;

	spin_lock_irqsave(&zone->lock, flags);

	/* Recheck the nr_reserved_highatomic limit under the lock */
	if (zone->nr_reserved_highatomic >= max_managed)
		goto out_unlock;

	/* Yoink! */
	mt = get_pageblock_migratetype(page);
2402 2403
	if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
	    && !is_migrate_cma(mt)) {
2404 2405
		zone->nr_reserved_highatomic += pageblock_nr_pages;
		set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2406
		move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417
	}

out_unlock:
	spin_unlock_irqrestore(&zone->lock, flags);
}

/*
 * Used when an allocation is about to fail under memory pressure. This
 * potentially hurts the reliability of high-order allocations when under
 * intense memory pressure but failed atomic allocations should be easier
 * to recover from than an OOM.
2418 2419 2420
 *
 * If @force is true, try to unreserve a pageblock even though highatomic
 * pageblock is exhausted.
2421
 */
2422 2423
static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
						bool force)
2424 2425 2426 2427 2428 2429 2430
{
	struct zonelist *zonelist = ac->zonelist;
	unsigned long flags;
	struct zoneref *z;
	struct zone *zone;
	struct page *page;
	int order;
2431
	bool ret;
2432 2433 2434

	for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
								ac->nodemask) {
2435 2436 2437 2438 2439 2440
		/*
		 * Preserve at least one pageblock unless memory pressure
		 * is really high.
		 */
		if (!force && zone->nr_reserved_highatomic <=
					pageblock_nr_pages)
2441 2442 2443 2444 2445 2446
			continue;

		spin_lock_irqsave(&zone->lock, flags);
		for (order = 0; order < MAX_ORDER; order++) {
			struct free_area *area = &(zone->free_area[order]);

2447
			page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
2448
			if (!page)
2449 2450 2451
				continue;

			/*
2452 2453 2454 2455 2456
			 * In page freeing path, migratetype change is racy so
			 * we can counter several free pages in a pageblock
			 * in this loop althoug we changed the pageblock type
			 * from highatomic to ac->migratetype. So we should
			 * adjust the count once.
2457
			 */
2458
			if (is_migrate_highatomic_page(page)) {
2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469
				/*
				 * It should never happen but changes to
				 * locking could inadvertently allow a per-cpu
				 * drain to add pages to MIGRATE_HIGHATOMIC
				 * while unreserving so be safe and watch for
				 * underflows.
				 */
				zone->nr_reserved_highatomic -= min(
						pageblock_nr_pages,
						zone->nr_reserved_highatomic);
			}
2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480

			/*
			 * Convert to ac->migratetype and avoid the normal
			 * pageblock stealing heuristics. Minimally, the caller
			 * is doing the work and needs the pages. More
			 * importantly, if the block was always converted to
			 * MIGRATE_UNMOVABLE or another type then the number
			 * of pageblocks that cannot be completely freed
			 * may increase.
			 */
			set_pageblock_migratetype(page, ac->migratetype);
2481 2482
			ret = move_freepages_block(zone, page, ac->migratetype,
									NULL);
2483 2484 2485 2486
			if (ret) {
				spin_unlock_irqrestore(&zone->lock, flags);
				return ret;
			}
2487 2488 2489
		}
		spin_unlock_irqrestore(&zone->lock, flags);
	}
2490 2491

	return false;
2492 2493
}

2494 2495 2496 2497 2498
/*
 * Try finding a free buddy page on the fallback list and put it on the free
 * list of requested migratetype, possibly along with other pages from the same
 * block, depending on fragmentation avoidance heuristics. Returns true if
 * fallback was found so that __rmqueue_smallest() can grab it.
2499 2500 2501 2502
 *
 * The use of signed ints for order and current_order is a deliberate
 * deviation from the rest of this file, to make the for loop
 * condition simpler.
2503
 */
2504
static __always_inline bool
2505
__rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
2506
{
2507
	struct free_area *area;
2508
	int current_order;
2509
	struct page *page;
2510 2511
	int fallback_mt;
	bool can_steal;
2512

2513 2514 2515 2516 2517
	/*
	 * Find the largest available free page in the other list. This roughly
	 * approximates finding the pageblock with the most free pages, which
	 * would be too costly to do exactly.
	 */
2518
	for (current_order = MAX_ORDER - 1; current_order >= order;
2519
				--current_order) {
2520 2521
		area = &(zone->free_area[current_order]);
		fallback_mt = find_suitable_fallback(area, current_order,
2522
				start_migratetype, false, &can_steal);
2523 2524
		if (fallback_mt == -1)
			continue;
2525

2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536
		/*
		 * We cannot steal all free pages from the pageblock and the
		 * requested migratetype is movable. In that case it's better to
		 * steal and split the smallest available page instead of the
		 * largest available page, because even if the next movable
		 * allocation falls back into a different pageblock than this
		 * one, it won't cause permanent fragmentation.
		 */
		if (!can_steal && start_migratetype == MIGRATE_MOVABLE
					&& current_order > order)
			goto find_smallest;
2537

2538 2539
		goto do_steal;
	}
2540

2541
	return false;
2542

2543 2544 2545 2546 2547 2548 2549 2550
find_smallest:
	for (current_order = order; current_order < MAX_ORDER;
							current_order++) {
		area = &(zone->free_area[current_order]);
		fallback_mt = find_suitable_fallback(area, current_order,
				start_migratetype, false, &can_steal);
		if (fallback_mt != -1)
			break;
2551 2552
	}

2553 2554 2555 2556 2557 2558 2559
	/*
	 * This should not happen - we already found a suitable fallback
	 * when looking for the largest page.
	 */
	VM_BUG_ON(current_order == MAX_ORDER);

do_steal:
2560
	page = get_page_from_free_area(area, fallback_mt);
2561 2562 2563 2564 2565 2566 2567 2568

	steal_suitable_fallback(zone, page, start_migratetype, can_steal);

	trace_mm_page_alloc_extfrag(page, order, current_order,
		start_migratetype, fallback_mt);

	return true;

2569 2570
}

2571
/*
L
Linus Torvalds 已提交
2572 2573 2574
 * Do the hard work of removing an element from the buddy allocator.
 * Call me with the zone->lock already held.
 */
2575 2576
static __always_inline struct page *
__rmqueue(struct zone *zone, unsigned int order, int migratetype)
L
Linus Torvalds 已提交
2577 2578 2579
{
	struct page *page;

2580
retry:
2581
	page = __rmqueue_smallest(zone, order, migratetype);
2582
	if (unlikely(!page)) {
2583 2584 2585
		if (migratetype == MIGRATE_MOVABLE)
			page = __rmqueue_cma_fallback(zone, order);

2586 2587
		if (!page && __rmqueue_fallback(zone, order, migratetype))
			goto retry;
2588 2589
	}

2590
	trace_mm_page_alloc_zone_locked(page, order, migratetype);
2591
	return page;
L
Linus Torvalds 已提交
2592 2593
}

2594
/*
L
Linus Torvalds 已提交
2595 2596 2597 2598
 * Obtain a specified number of elements from the buddy allocator, all under
 * a single hold of the lock, for efficiency.  Add them to the supplied list.
 * Returns the number of new pages which were placed at *list.
 */
2599
static int rmqueue_bulk(struct zone *zone, unsigned int order,
2600
			unsigned long count, struct list_head *list,
M
Mel Gorman 已提交
2601
			int migratetype)
L
Linus Torvalds 已提交
2602
{
2603
	int i, alloced = 0;
2604

2605
	spin_lock(&zone->lock);
L
Linus Torvalds 已提交
2606
	for (i = 0; i < count; ++i) {
2607
		struct page *page = __rmqueue(zone, order, migratetype);
N
Nick Piggin 已提交
2608
		if (unlikely(page == NULL))
L
Linus Torvalds 已提交
2609
			break;
2610

2611 2612 2613
		if (unlikely(check_pcp_refill(page)))
			continue;

2614
		/*
2615 2616 2617 2618 2619 2620 2621 2622
		 * Split buddy pages returned by expand() are received here in
		 * physical page order. The page is added to the tail of
		 * caller's list. From the callers perspective, the linked list
		 * is ordered by page number under some conditions. This is
		 * useful for IO devices that can forward direction from the
		 * head, thus also in the physical page order. This is useful
		 * for IO devices that can merge IO requests if the physical
		 * pages are ordered properly.
2623
		 */
2624
		list_add_tail(&page->lru, list);
2625
		alloced++;
2626
		if (is_migrate_cma(get_pcppage_migratetype(page)))
2627 2628
			__mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
					      -(1 << order));
L
Linus Torvalds 已提交
2629
	}
2630 2631 2632 2633 2634 2635 2636

	/*
	 * i pages were removed from the buddy list even if some leak due
	 * to check_pcp_refill failing so adjust NR_FREE_PAGES based
	 * on i. Do not confuse with 'alloced' which is the number of
	 * pages added to the pcp list.
	 */
2637
	__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
2638
	spin_unlock(&zone->lock);
2639
	return alloced;
L
Linus Torvalds 已提交
2640 2641
}

2642
#ifdef CONFIG_NUMA
2643
/*
2644 2645 2646 2647
 * Called from the vmstat counter updater to drain pagesets of this
 * currently executing processor on remote nodes after they have
 * expired.
 *
2648 2649
 * Note that this function must be called with the thread pinned to
 * a single processor.
2650
 */
2651
void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
2652 2653
{
	unsigned long flags;
2654
	int to_drain, batch;
2655

2656
	local_irq_save(flags);
2657
	batch = READ_ONCE(pcp->batch);
2658
	to_drain = min(pcp->count, batch);
2659
	if (to_drain > 0)
2660
		free_pcppages_bulk(zone, to_drain, pcp);
2661
	local_irq_restore(flags);
2662 2663 2664
}
#endif

2665
/*
2666
 * Drain pcplists of the indicated processor and zone.
2667 2668 2669 2670 2671
 *
 * The processor must either be the current processor and the
 * thread pinned to the current processor or a processor that
 * is not online.
 */
2672
static void drain_pages_zone(unsigned int cpu, struct zone *zone)
L
Linus Torvalds 已提交
2673
{
N
Nick Piggin 已提交
2674
	unsigned long flags;
2675 2676
	struct per_cpu_pageset *pset;
	struct per_cpu_pages *pcp;
L
Linus Torvalds 已提交
2677

2678 2679
	local_irq_save(flags);
	pset = per_cpu_ptr(zone->pageset, cpu);
L
Linus Torvalds 已提交
2680

2681
	pcp = &pset->pcp;
2682
	if (pcp->count)
2683 2684 2685
		free_pcppages_bulk(zone, pcp->count, pcp);
	local_irq_restore(flags);
}
2686

2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699
/*
 * Drain pcplists of all zones on the indicated processor.
 *
 * The processor must either be the current processor and the
 * thread pinned to the current processor or a processor that
 * is not online.
 */
static void drain_pages(unsigned int cpu)
{
	struct zone *zone;

	for_each_populated_zone(zone) {
		drain_pages_zone(cpu, zone);
L
Linus Torvalds 已提交
2700 2701 2702
	}
}

2703 2704
/*
 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
2705 2706 2707
 *
 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
 * the single zone's pages.
2708
 */
2709
void drain_local_pages(struct zone *zone)
2710
{
2711 2712 2713 2714 2715 2716
	int cpu = smp_processor_id();

	if (zone)
		drain_pages_zone(cpu, zone);
	else
		drain_pages(cpu);
2717 2718
}

2719 2720
static void drain_local_pages_wq(struct work_struct *work)
{
2721 2722 2723 2724 2725 2726 2727 2728
	/*
	 * drain_all_pages doesn't use proper cpu hotplug protection so
	 * we can race with cpu offline when the WQ can move this from
	 * a cpu pinned worker to an unbound one. We can operate on a different
	 * cpu which is allright but we also have to make sure to not move to
	 * a different one.
	 */
	preempt_disable();
2729
	drain_local_pages(NULL);
2730
	preempt_enable();
2731 2732
}

2733
/*
2734 2735
 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
 *
2736 2737
 * When zone parameter is non-NULL, spill just the single zone's pages.
 *
2738
 * Note that this can be extremely slow as the draining happens in a workqueue.
2739
 */
2740
void drain_all_pages(struct zone *zone)
2741
{
2742 2743 2744 2745 2746 2747 2748 2749
	int cpu;

	/*
	 * Allocate in the BSS so we wont require allocation in
	 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
	 */
	static cpumask_t cpus_with_pcps;

2750 2751 2752 2753 2754 2755 2756
	/*
	 * Make sure nobody triggers this path before mm_percpu_wq is fully
	 * initialized.
	 */
	if (WARN_ON_ONCE(!mm_percpu_wq))
		return;

2757 2758 2759 2760 2761 2762 2763 2764 2765 2766
	/*
	 * Do not drain if one is already in progress unless it's specific to
	 * a zone. Such callers are primarily CMA and memory hotplug and need
	 * the drain to be complete when the call returns.
	 */
	if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
		if (!zone)
			return;
		mutex_lock(&pcpu_drain_mutex);
	}
2767

2768 2769 2770 2771 2772 2773 2774
	/*
	 * We don't care about racing with CPU hotplug event
	 * as offline notification will cause the notified
	 * cpu to drain that CPU pcps and on_each_cpu_mask
	 * disables preemption as part of its processing
	 */
	for_each_online_cpu(cpu) {
2775 2776
		struct per_cpu_pageset *pcp;
		struct zone *z;
2777
		bool has_pcps = false;
2778 2779

		if (zone) {
2780
			pcp = per_cpu_ptr(zone->pageset, cpu);
2781
			if (pcp->pcp.count)
2782
				has_pcps = true;
2783 2784 2785 2786 2787 2788 2789
		} else {
			for_each_populated_zone(z) {
				pcp = per_cpu_ptr(z->pageset, cpu);
				if (pcp->pcp.count) {
					has_pcps = true;
					break;
				}
2790 2791
			}
		}
2792

2793 2794 2795 2796 2797
		if (has_pcps)
			cpumask_set_cpu(cpu, &cpus_with_pcps);
		else
			cpumask_clear_cpu(cpu, &cpus_with_pcps);
	}
2798

2799 2800 2801
	for_each_cpu(cpu, &cpus_with_pcps) {
		struct work_struct *work = per_cpu_ptr(&pcpu_drain, cpu);
		INIT_WORK(work, drain_local_pages_wq);
2802
		queue_work_on(cpu, mm_percpu_wq, work);
2803
	}
2804 2805 2806 2807
	for_each_cpu(cpu, &cpus_with_pcps)
		flush_work(per_cpu_ptr(&pcpu_drain, cpu));

	mutex_unlock(&pcpu_drain_mutex);
2808 2809
}

2810
#ifdef CONFIG_HIBERNATION
L
Linus Torvalds 已提交
2811

2812 2813 2814 2815 2816
/*
 * Touch the watchdog for every WD_PAGE_COUNT pages.
 */
#define WD_PAGE_COUNT	(128*1024)

L
Linus Torvalds 已提交
2817 2818
void mark_free_pages(struct zone *zone)
{
2819
	unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
2820
	unsigned long flags;
2821
	unsigned int order, t;
2822
	struct page *page;
L
Linus Torvalds 已提交
2823

2824
	if (zone_is_empty(zone))
L
Linus Torvalds 已提交
2825 2826 2827
		return;

	spin_lock_irqsave(&zone->lock, flags);
2828

2829
	max_zone_pfn = zone_end_pfn(zone);
2830 2831
	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
		if (pfn_valid(pfn)) {
2832
			page = pfn_to_page(pfn);
2833

2834 2835 2836 2837 2838
			if (!--page_count) {
				touch_nmi_watchdog();
				page_count = WD_PAGE_COUNT;
			}

2839 2840 2841
			if (page_zone(page) != zone)
				continue;

2842 2843
			if (!swsusp_page_is_forbidden(page))
				swsusp_unset_page_free(page);
2844
		}
L
Linus Torvalds 已提交
2845

2846
	for_each_migratetype_order(order, t) {
2847 2848
		list_for_each_entry(page,
				&zone->free_area[order].free_list[t], lru) {
2849
			unsigned long i;
L
Linus Torvalds 已提交
2850

2851
			pfn = page_to_pfn(page);
2852 2853 2854 2855 2856
			for (i = 0; i < (1UL << order); i++) {
				if (!--page_count) {
					touch_nmi_watchdog();
					page_count = WD_PAGE_COUNT;
				}
2857
				swsusp_set_page_free(pfn_to_page(pfn + i));
2858
			}
2859
		}
2860
	}
L
Linus Torvalds 已提交
2861 2862
	spin_unlock_irqrestore(&zone->lock, flags);
}
2863
#endif /* CONFIG_PM */
L
Linus Torvalds 已提交
2864

2865
static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
L
Linus Torvalds 已提交
2866
{
2867
	int migratetype;
L
Linus Torvalds 已提交
2868

2869
	if (!free_pcp_prepare(page))
2870
		return false;
2871

2872
	migratetype = get_pfnblock_migratetype(page, pfn);
2873
	set_pcppage_migratetype(page, migratetype);
2874 2875 2876
	return true;
}

2877
static void free_unref_page_commit(struct page *page, unsigned long pfn)
2878 2879 2880 2881 2882 2883
{
	struct zone *zone = page_zone(page);
	struct per_cpu_pages *pcp;
	int migratetype;

	migratetype = get_pcppage_migratetype(page);
2884
	__count_vm_event(PGFREE);
2885

2886 2887 2888
	/*
	 * We only track unmovable, reclaimable and movable on pcp lists.
	 * Free ISOLATE pages back to the allocator because they are being
2889
	 * offlined but treat HIGHATOMIC as movable pages so we can get those
2890 2891 2892 2893
	 * areas back if necessary. Otherwise, we may have to free
	 * excessively into the page allocator
	 */
	if (migratetype >= MIGRATE_PCPTYPES) {
2894
		if (unlikely(is_migrate_isolate(migratetype))) {
2895
			free_one_page(zone, page, pfn, 0, migratetype);
2896
			return;
2897 2898 2899 2900
		}
		migratetype = MIGRATE_MOVABLE;
	}

2901
	pcp = &this_cpu_ptr(zone->pageset)->pcp;
2902
	list_add(&page->lru, &pcp->lists[migratetype]);
L
Linus Torvalds 已提交
2903
	pcp->count++;
N
Nick Piggin 已提交
2904
	if (pcp->count >= pcp->high) {
2905
		unsigned long batch = READ_ONCE(pcp->batch);
2906
		free_pcppages_bulk(zone, batch, pcp);
N
Nick Piggin 已提交
2907
	}
2908
}
2909

2910 2911 2912
/*
 * Free a 0-order page
 */
2913
void free_unref_page(struct page *page)
2914 2915 2916 2917
{
	unsigned long flags;
	unsigned long pfn = page_to_pfn(page);

2918
	if (!free_unref_page_prepare(page, pfn))
2919 2920 2921
		return;

	local_irq_save(flags);
2922
	free_unref_page_commit(page, pfn);
2923
	local_irq_restore(flags);
L
Linus Torvalds 已提交
2924 2925
}

2926 2927 2928
/*
 * Free a list of 0-order pages
 */
2929
void free_unref_page_list(struct list_head *list)
2930 2931
{
	struct page *page, *next;
2932
	unsigned long flags, pfn;
2933
	int batch_count = 0;
2934 2935 2936 2937

	/* Prepare pages for freeing */
	list_for_each_entry_safe(page, next, list, lru) {
		pfn = page_to_pfn(page);
2938
		if (!free_unref_page_prepare(page, pfn))
2939 2940 2941
			list_del(&page->lru);
		set_page_private(page, pfn);
	}
2942

2943
	local_irq_save(flags);
2944
	list_for_each_entry_safe(page, next, list, lru) {
2945 2946 2947
		unsigned long pfn = page_private(page);

		set_page_private(page, 0);
2948 2949
		trace_mm_page_free_batched(page);
		free_unref_page_commit(page, pfn);
2950 2951 2952 2953 2954 2955 2956 2957 2958 2959

		/*
		 * Guard against excessive IRQ disabled times when we get
		 * a large list of pages to free.
		 */
		if (++batch_count == SWAP_CLUSTER_MAX) {
			local_irq_restore(flags);
			batch_count = 0;
			local_irq_save(flags);
		}
2960
	}
2961
	local_irq_restore(flags);
2962 2963
}

N
Nick Piggin 已提交
2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975
/*
 * split_page takes a non-compound higher-order page, and splits it into
 * n (1<<order) sub-pages: page[0..n]
 * Each sub-page must be freed individually.
 *
 * Note: this is probably too low level an operation for use in drivers.
 * Please consult with lkml before using this in your driver.
 */
void split_page(struct page *page, unsigned int order)
{
	int i;

2976 2977
	VM_BUG_ON_PAGE(PageCompound(page), page);
	VM_BUG_ON_PAGE(!page_count(page), page);
2978

2979
	for (i = 1; i < (1 << order); i++)
2980
		set_page_refcounted(page + i);
2981
	split_page_owner(page, order);
N
Nick Piggin 已提交
2982
}
K
K. Y. Srinivasan 已提交
2983
EXPORT_SYMBOL_GPL(split_page);
N
Nick Piggin 已提交
2984

2985
int __isolate_free_page(struct page *page, unsigned int order)
2986 2987 2988
{
	unsigned long watermark;
	struct zone *zone;
2989
	int mt;
2990 2991 2992 2993

	BUG_ON(!PageBuddy(page));

	zone = page_zone(page);
2994
	mt = get_pageblock_migratetype(page);
2995

2996
	if (!is_migrate_isolate(mt)) {
2997 2998 2999 3000 3001 3002 3003
		/*
		 * Obey watermarks as if the page was being allocated. We can
		 * emulate a high-order watermark check with a raised order-0
		 * watermark, because we already know our high-order page
		 * exists.
		 */
		watermark = min_wmark_pages(zone) + (1UL << order);
3004
		if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
3005 3006
			return 0;

3007
		__mod_zone_freepage_state(zone, -(1UL << order), mt);
3008
	}
3009 3010

	/* Remove page from free list */
3011

3012
	del_page_from_free_list(page, zone, order);
3013

3014 3015 3016 3017
	/*
	 * Set the pageblock if the isolated page is at least half of a
	 * pageblock
	 */
3018 3019
	if (order >= pageblock_order - 1) {
		struct page *endpage = page + (1 << order) - 1;
3020 3021
		for (; page < endpage; page += pageblock_nr_pages) {
			int mt = get_pageblock_migratetype(page);
M
Minchan Kim 已提交
3022
			if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
3023
			    && !is_migrate_highatomic(mt))
3024 3025 3026
				set_pageblock_migratetype(page,
							  MIGRATE_MOVABLE);
		}
3027 3028
	}

3029

3030
	return 1UL << order;
3031 3032
}

3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048
/**
 * __putback_isolated_page - Return a now-isolated page back where we got it
 * @page: Page that was isolated
 * @order: Order of the isolated page
 *
 * This function is meant to return a page pulled from the free lists via
 * __isolate_free_page back to the free lists they were pulled from.
 */
void __putback_isolated_page(struct page *page, unsigned int order, int mt)
{
	struct zone *zone = page_zone(page);

	/* zone lock should be held when this function is called */
	lockdep_assert_held(&zone->lock);

	/* Return isolated page to tail of freelist. */
A
Alexander Duyck 已提交
3049
	__free_one_page(page, page_to_pfn(page), zone, order, mt, false);
3050 3051
}

3052 3053 3054 3055 3056
/*
 * Update NUMA hit/miss statistics
 *
 * Must be called with interrupts disabled.
 */
M
Michal Hocko 已提交
3057
static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
3058 3059
{
#ifdef CONFIG_NUMA
3060
	enum numa_stat_item local_stat = NUMA_LOCAL;
3061

3062 3063 3064 3065
	/* skip numa counters update if numa stats is disabled */
	if (!static_branch_likely(&vm_numa_stat_key))
		return;

3066
	if (zone_to_nid(z) != numa_node_id())
3067 3068
		local_stat = NUMA_OTHER;

3069
	if (zone_to_nid(z) == zone_to_nid(preferred_zone))
3070
		__inc_numa_state(z, NUMA_HIT);
3071
	else {
3072 3073
		__inc_numa_state(z, NUMA_MISS);
		__inc_numa_state(preferred_zone, NUMA_FOREIGN);
3074
	}
3075
	__inc_numa_state(z, local_stat);
3076 3077 3078
#endif
}

3079 3080
/* Remove page from the per-cpu list, caller must protect the list */
static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
M
Mel Gorman 已提交
3081
			struct per_cpu_pages *pcp,
3082 3083 3084 3085 3086 3087 3088 3089
			struct list_head *list)
{
	struct page *page;

	do {
		if (list_empty(list)) {
			pcp->count += rmqueue_bulk(zone, 0,
					pcp->batch, list,
M
Mel Gorman 已提交
3090
					migratetype);
3091 3092 3093 3094
			if (unlikely(list_empty(list)))
				return NULL;
		}

M
Mel Gorman 已提交
3095
		page = list_first_entry(list, struct page, lru);
3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110
		list_del(&page->lru);
		pcp->count--;
	} while (check_new_pcp(page));

	return page;
}

/* Lock and remove page from the per-cpu list */
static struct page *rmqueue_pcplist(struct zone *preferred_zone,
			struct zone *zone, unsigned int order,
			gfp_t gfp_flags, int migratetype)
{
	struct per_cpu_pages *pcp;
	struct list_head *list;
	struct page *page;
3111
	unsigned long flags;
3112

3113
	local_irq_save(flags);
3114 3115
	pcp = &this_cpu_ptr(zone->pageset)->pcp;
	list = &pcp->lists[migratetype];
M
Mel Gorman 已提交
3116
	page = __rmqueue_pcplist(zone,  migratetype, pcp, list);
3117 3118 3119 3120
	if (page) {
		__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
		zone_statistics(preferred_zone, zone);
	}
3121
	local_irq_restore(flags);
3122 3123 3124
	return page;
}

L
Linus Torvalds 已提交
3125
/*
3126
 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
L
Linus Torvalds 已提交
3127
 */
3128
static inline
3129
struct page *rmqueue(struct zone *preferred_zone,
3130
			struct zone *zone, unsigned int order,
3131 3132
			gfp_t gfp_flags, unsigned int alloc_flags,
			int migratetype)
L
Linus Torvalds 已提交
3133 3134
{
	unsigned long flags;
3135
	struct page *page;
L
Linus Torvalds 已提交
3136

3137
	if (likely(order == 0)) {
3138 3139 3140 3141
		page = rmqueue_pcplist(preferred_zone, zone, order,
				gfp_flags, migratetype);
		goto out;
	}
3142

3143 3144 3145 3146 3147 3148
	/*
	 * We most definitely don't want callers attempting to
	 * allocate greater than order-1 page units with __GFP_NOFAIL.
	 */
	WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
	spin_lock_irqsave(&zone->lock, flags);
3149

3150 3151 3152 3153 3154 3155 3156
	do {
		page = NULL;
		if (alloc_flags & ALLOC_HARDER) {
			page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
			if (page)
				trace_mm_page_alloc_zone_locked(page, order, migratetype);
		}
N
Nick Piggin 已提交
3157
		if (!page)
3158 3159 3160 3161 3162 3163 3164
			page = __rmqueue(zone, order, migratetype);
	} while (page && check_new_pages(page, order));
	spin_unlock(&zone->lock);
	if (!page)
		goto failed;
	__mod_zone_freepage_state(zone, -(1 << order),
				  get_pcppage_migratetype(page));
L
Linus Torvalds 已提交
3165

3166
	__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
M
Michal Hocko 已提交
3167
	zone_statistics(preferred_zone, zone);
N
Nick Piggin 已提交
3168
	local_irq_restore(flags);
L
Linus Torvalds 已提交
3169

3170 3171
out:
	VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
L
Linus Torvalds 已提交
3172
	return page;
N
Nick Piggin 已提交
3173 3174 3175 3176

failed:
	local_irq_restore(flags);
	return NULL;
L
Linus Torvalds 已提交
3177 3178
}

3179 3180
#ifdef CONFIG_FAIL_PAGE_ALLOC

3181
static struct {
3182 3183
	struct fault_attr attr;

3184
	bool ignore_gfp_highmem;
3185
	bool ignore_gfp_reclaim;
3186
	u32 min_order;
3187 3188
} fail_page_alloc = {
	.attr = FAULT_ATTR_INITIALIZER,
3189
	.ignore_gfp_reclaim = true,
3190
	.ignore_gfp_highmem = true,
3191
	.min_order = 1,
3192 3193 3194 3195 3196 3197 3198 3199
};

static int __init setup_fail_page_alloc(char *str)
{
	return setup_fault_attr(&fail_page_alloc.attr, str);
}
__setup("fail_page_alloc=", setup_fail_page_alloc);

3200
static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3201
{
3202
	if (order < fail_page_alloc.min_order)
3203
		return false;
3204
	if (gfp_mask & __GFP_NOFAIL)
3205
		return false;
3206
	if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
3207
		return false;
3208 3209
	if (fail_page_alloc.ignore_gfp_reclaim &&
			(gfp_mask & __GFP_DIRECT_RECLAIM))
3210
		return false;
3211 3212 3213 3214 3215 3216 3217 3218

	return should_fail(&fail_page_alloc.attr, 1 << order);
}

#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS

static int __init fail_page_alloc_debugfs(void)
{
3219
	umode_t mode = S_IFREG | 0600;
3220 3221
	struct dentry *dir;

3222 3223 3224 3225
	dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
					&fail_page_alloc.attr);
	if (IS_ERR(dir))
		return PTR_ERR(dir);
3226

3227
	if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
3228
				&fail_page_alloc.ignore_gfp_reclaim))
3229 3230 3231 3232 3233 3234 3235 3236 3237 3238
		goto fail;
	if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
				&fail_page_alloc.ignore_gfp_highmem))
		goto fail;
	if (!debugfs_create_u32("min-order", mode, dir,
				&fail_page_alloc.min_order))
		goto fail;

	return 0;
fail:
3239
	debugfs_remove_recursive(dir);
3240

3241
	return -ENOMEM;
3242 3243 3244 3245 3246 3247 3248 3249
}

late_initcall(fail_page_alloc_debugfs);

#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */

#else /* CONFIG_FAIL_PAGE_ALLOC */

3250
static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3251
{
3252
	return false;
3253 3254 3255 3256
}

#endif /* CONFIG_FAIL_PAGE_ALLOC */

L
Linus Torvalds 已提交
3257
/*
3258 3259 3260 3261
 * Return true if free base pages are above 'mark'. For high-order checks it
 * will return true of the order-0 watermark is reached and there is at least
 * one free page of a suitable size. Checking now avoids taking the zone lock
 * to check in the allocation paths if no pages are free.
L
Linus Torvalds 已提交
3262
 */
3263 3264 3265
bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
			 int classzone_idx, unsigned int alloc_flags,
			 long free_pages)
L
Linus Torvalds 已提交
3266
{
3267
	long min = mark;
L
Linus Torvalds 已提交
3268
	int o;
3269
	const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
L
Linus Torvalds 已提交
3270

3271 3272 3273 3274 3275 3276 3277 3278
	/* apply negative memory.wmark_min_adj */
	if ((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN) {
		int min_adj = memcg_get_wmark_min_adj(current);

		if (min_adj < 0)
			min -= mark * (-min_adj) / 100;
	}

3279
	/* free_pages may go negative - that's OK */
3280
	free_pages -= (1 << order) - 1;
3281

R
Rohit Seth 已提交
3282
	if (alloc_flags & ALLOC_HIGH)
L
Linus Torvalds 已提交
3283
		min -= min / 2;
3284 3285 3286 3287 3288 3289

	/*
	 * If the caller does not have rights to ALLOC_HARDER then subtract
	 * the high-atomic reserves. This will over-estimate the size of the
	 * atomic reserve but it avoids a search.
	 */
3290
	if (likely(!alloc_harder)) {
3291
		free_pages -= z->nr_reserved_highatomic;
3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304
	} else {
		/*
		 * OOM victims can try even harder than normal ALLOC_HARDER
		 * users on the grounds that it's definitely going to be in
		 * the exit path shortly and free memory. Any allocation it
		 * makes during the free path will be small and short-lived.
		 */
		if (alloc_flags & ALLOC_OOM)
			min -= min / 2;
		else
			min -= min / 4;
	}

3305 3306 3307 3308 3309 3310
	/*
	 * Only happens due to memory.wmark_min_adj.
	 * Guarantee safe min after memory.wmark_min_adj?
	 */
	if (min < mark / 4)
		min = mark / 4;
3311

3312 3313 3314 3315 3316 3317
#ifdef CONFIG_CMA
	/* If allocation can't use CMA areas don't use free CMA pages */
	if (!(alloc_flags & ALLOC_CMA))
		free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
#endif

3318 3319 3320 3321 3322 3323
	/*
	 * Check watermarks for an order-0 allocation request. If these
	 * are not met, then a high-order request also cannot go ahead
	 * even if a suitable page happened to be free.
	 */
	if (free_pages <= min + z->lowmem_reserve[classzone_idx])
3324
		return false;
L
Linus Torvalds 已提交
3325

3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338
	/* If this is an order-0 request then the watermark is fine */
	if (!order)
		return true;

	/* For a high-order request, check at least one suitable page is free */
	for (o = order; o < MAX_ORDER; o++) {
		struct free_area *area = &z->free_area[o];
		int mt;

		if (!area->nr_free)
			continue;

		for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
3339
			if (!free_area_empty(area, mt))
3340 3341 3342 3343
				return true;
		}

#ifdef CONFIG_CMA
3344
		if ((alloc_flags & ALLOC_CMA) &&
3345
		    !free_area_empty(area, MIGRATE_CMA)) {
3346
			return true;
3347
		}
3348
#endif
3349 3350 3351
		if (alloc_harder &&
			!list_empty(&area->free_list[MIGRATE_HIGHATOMIC]))
			return true;
L
Linus Torvalds 已提交
3352
	}
3353
	return false;
3354 3355
}

3356
bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3357
		      int classzone_idx, unsigned int alloc_flags)
3358 3359 3360 3361 3362
{
	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
					zone_page_state(z, NR_FREE_PAGES));
}

3363 3364 3365 3366
static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
		unsigned long mark, int classzone_idx, unsigned int alloc_flags)
{
	long free_pages = zone_page_state(z, NR_FREE_PAGES);
3367 3368 3369 3370 3371 3372 3373
	long cma_pages = 0;

#ifdef CONFIG_CMA
	/* If allocation can't use CMA areas don't use free CMA pages */
	if (!(alloc_flags & ALLOC_CMA))
		cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES);
#endif
3374 3375 3376 3377 3378 3379 3380 3381

	/*
	 * Fast check for order-0 only. If this fails then the reserves
	 * need to be calculated. There is a corner case where the check
	 * passes but only the high-order atomic reserve are free. If
	 * the caller is !atomic then it'll uselessly search the free
	 * list. That corner case is then slower but it is harmless.
	 */
3382
	if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx])
3383 3384 3385 3386 3387 3388
		return true;

	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
					free_pages);
}

3389
bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
3390
			unsigned long mark, int classzone_idx)
3391 3392 3393 3394 3395 3396
{
	long free_pages = zone_page_state(z, NR_FREE_PAGES);

	if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
		free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);

3397
	return __zone_watermark_ok(z, order, mark, classzone_idx, 0,
3398
								free_pages);
L
Linus Torvalds 已提交
3399 3400
}

3401
#ifdef CONFIG_NUMA
3402 3403
static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
{
3404
	return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
3405
				RECLAIM_DISTANCE;
3406
}
3407
#else	/* CONFIG_NUMA */
3408 3409 3410 3411
static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
{
	return true;
}
3412 3413
#endif	/* CONFIG_NUMA */

R
Rohit Seth 已提交
3414
/*
3415
 * get_page_from_freelist goes through the zonelist trying to allocate
R
Rohit Seth 已提交
3416 3417 3418
 * a page.
 */
static struct page *
3419 3420
get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
						const struct alloc_context *ac)
M
Martin Hicks 已提交
3421
{
3422
	struct zoneref *z = ac->preferred_zoneref;
3423
	struct zone *zone;
3424 3425
	struct pglist_data *last_pgdat_dirty_limit = NULL;

R
Rohit Seth 已提交
3426
	/*
3427
	 * Scan zonelist, looking for a zone with enough free.
3428
	 * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
R
Rohit Seth 已提交
3429
	 */
3430
	for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3431
								ac->nodemask) {
3432
		struct page *page;
3433 3434
		unsigned long mark;

3435 3436
		if (cpusets_enabled() &&
			(alloc_flags & ALLOC_CPUSET) &&
3437
			!__cpuset_zone_allowed(zone, gfp_mask))
3438
				continue;
3439 3440
		/*
		 * When allocating a page cache page for writing, we
3441 3442
		 * want to get it from a node that is within its dirty
		 * limit, such that no single node holds more than its
3443
		 * proportional share of globally allowed dirty pages.
3444
		 * The dirty limits take into account the node's
3445 3446 3447 3448 3449
		 * lowmem reserves and high watermark so that kswapd
		 * should be able to balance it without having to
		 * write pages from its LRU list.
		 *
		 * XXX: For now, allow allocations to potentially
3450
		 * exceed the per-node dirty limit in the slowpath
3451
		 * (spread_dirty_pages unset) before going into reclaim,
3452
		 * which is important when on a NUMA setup the allowed
3453
		 * nodes are together not big enough to reach the
3454
		 * global limit.  The proper fix for these situations
3455
		 * will require awareness of nodes in the
3456 3457
		 * dirty-throttling and the flusher threads.
		 */
3458 3459 3460 3461 3462 3463 3464 3465 3466
		if (ac->spread_dirty_pages) {
			if (last_pgdat_dirty_limit == zone->zone_pgdat)
				continue;

			if (!node_dirty_ok(zone->zone_pgdat)) {
				last_pgdat_dirty_limit = zone->zone_pgdat;
				continue;
			}
		}
R
Rohit Seth 已提交
3467

3468
		mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
3469
		if (!zone_watermark_fast(zone, order, mark,
3470
				       ac_classzone_idx(ac), alloc_flags)) {
3471 3472
			int ret;

3473 3474 3475 3476 3477 3478 3479 3480 3481 3482
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
			/*
			 * Watermark failed for this zone, but see if we can
			 * grow this zone if it contains deferred pages.
			 */
			if (static_branch_unlikely(&deferred_pages)) {
				if (_deferred_grow_zone(zone, order))
					goto try_this_zone;
			}
#endif
3483 3484 3485 3486 3487
			/* Checked here to keep the fast path fast */
			BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
			if (alloc_flags & ALLOC_NO_WATERMARKS)
				goto try_this_zone;

3488
			if (node_reclaim_mode == 0 ||
3489
			    !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
3490 3491
				continue;

3492
			ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
3493
			switch (ret) {
3494
			case NODE_RECLAIM_NOSCAN:
3495
				/* did not scan */
3496
				continue;
3497
			case NODE_RECLAIM_FULL:
3498
				/* scanned but unreclaimable */
3499
				continue;
3500 3501
			default:
				/* did we reclaim enough */
3502
				if (zone_watermark_ok(zone, order, mark,
3503
						ac_classzone_idx(ac), alloc_flags))
3504 3505 3506
					goto try_this_zone;

				continue;
3507
			}
R
Rohit Seth 已提交
3508 3509
		}

3510
try_this_zone:
3511
		page = rmqueue(ac->preferred_zoneref->zone, zone, order,
3512
				gfp_mask, alloc_flags, ac->migratetype);
3513
		if (page) {
3514
			prep_new_page(page, order, gfp_mask, alloc_flags);
3515 3516 3517 3518 3519 3520 3521 3522

			/*
			 * If this is a high-order atomic allocation then check
			 * if the pageblock should be reserved for the future
			 */
			if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
				reserve_highatomic_pageblock(page, zone, order);

3523
			return page;
3524 3525 3526 3527 3528 3529 3530 3531
		} else {
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
			/* Try again if zone has deferred pages */
			if (static_branch_unlikely(&deferred_pages)) {
				if (_deferred_grow_zone(zone, order))
					goto try_this_zone;
			}
#endif
3532
		}
3533
	}
3534

3535
	return NULL;
M
Martin Hicks 已提交
3536 3537
}

3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551
/*
 * Large machines with many possible nodes should not always dump per-node
 * meminfo in irq context.
 */
static inline bool should_suppress_show_mem(void)
{
	bool ret = false;

#if NODES_SHIFT > 8
	ret = in_interrupt();
#endif
	return ret;
}

3552
static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
3553 3554
{
	unsigned int filter = SHOW_MEM_FILTER_NODES;
3555
	static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
3556

3557
	if (should_suppress_show_mem() || !__ratelimit(&show_mem_rs))
3558 3559 3560 3561 3562 3563 3564 3565
		return;

	/*
	 * This documents exceptions given to allocations in certain
	 * contexts that are allowed to allocate outside current's set
	 * of allowed nodes.
	 */
	if (!(gfp_mask & __GFP_NOMEMALLOC))
3566
		if (tsk_is_oom_victim(current) ||
3567 3568
		    (current->flags & (PF_MEMALLOC | PF_EXITING)))
			filter &= ~SHOW_MEM_FILTER_NODES;
3569
	if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
3570 3571
		filter &= ~SHOW_MEM_FILTER_NODES;

3572
	show_mem(filter, nodemask);
3573 3574
}

3575
void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
3576 3577 3578 3579 3580 3581
{
	struct va_format vaf;
	va_list args;
	static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
				      DEFAULT_RATELIMIT_BURST);

3582
	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
3583 3584
		return;

3585 3586 3587
	va_start(args, fmt);
	vaf.fmt = fmt;
	vaf.va = &args;
M
Michal Hocko 已提交
3588 3589 3590
	pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl\n",
			current->comm, &vaf, gfp_mask, &gfp_mask,
			nodemask_pr_args(nodemask));
3591
	va_end(args);
J
Joe Perches 已提交
3592

3593
	cpuset_print_current_mems_allowed();
J
Joe Perches 已提交
3594

3595
	dump_stack();
3596
	warn_alloc_show_mem(gfp_mask, nodemask);
3597 3598
}

3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618
static inline struct page *
__alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
			      unsigned int alloc_flags,
			      const struct alloc_context *ac)
{
	struct page *page;

	page = get_page_from_freelist(gfp_mask, order,
			alloc_flags|ALLOC_CPUSET, ac);
	/*
	 * fallback to ignore cpuset restriction if our nodes
	 * are depleted
	 */
	if (!page)
		page = get_page_from_freelist(gfp_mask, order,
				alloc_flags, ac);

	return page;
}

3619 3620
static inline struct page *
__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
3621
	const struct alloc_context *ac, unsigned long *did_some_progress)
3622
{
3623 3624 3625
	struct oom_control oc = {
		.zonelist = ac->zonelist,
		.nodemask = ac->nodemask,
3626
		.memcg = NULL,
3627 3628 3629
		.gfp_mask = gfp_mask,
		.order = order,
	};
3630 3631
	struct page *page;

3632 3633 3634
	*did_some_progress = 0;

	/*
3635 3636
	 * Acquire the oom lock.  If that fails, somebody else is
	 * making progress for us.
3637
	 */
3638
	if (!mutex_trylock(&oom_lock)) {
3639
		*did_some_progress = 1;
3640
		schedule_timeout_uninterruptible(1);
L
Linus Torvalds 已提交
3641 3642
		return NULL;
	}
3643

3644 3645 3646
	/*
	 * Go through the zonelist yet one more time, keep very high watermark
	 * here, this is only to catch a parallel oom killing, we must fail if
3647 3648 3649
	 * we're still under heavy pressure. But make sure that this reclaim
	 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
	 * allocation which will never fail due to oom_lock already held.
3650
	 */
3651 3652 3653
	page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
				      ~__GFP_DIRECT_RECLAIM, order,
				      ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
R
Rohit Seth 已提交
3654
	if (page)
3655 3656
		goto out;

3657 3658 3659 3660 3661 3662
	/* Coredumps can quickly deplete all memory reserves */
	if (current->flags & PF_DUMPCORE)
		goto out;
	/* The OOM killer will not help higher order allocs */
	if (order > PAGE_ALLOC_COSTLY_ORDER)
		goto out;
3663 3664 3665 3666 3667 3668 3669 3670
	/*
	 * We have already exhausted all our reclaim opportunities without any
	 * success so it is time to admit defeat. We will skip the OOM killer
	 * because it is very likely that the caller has a more reasonable
	 * fallback than shooting a random task.
	 */
	if (gfp_mask & __GFP_RETRY_MAYFAIL)
		goto out;
3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688
	/* The OOM killer does not needlessly kill tasks for lowmem */
	if (ac->high_zoneidx < ZONE_NORMAL)
		goto out;
	if (pm_suspended_storage())
		goto out;
	/*
	 * XXX: GFP_NOFS allocations should rather fail than rely on
	 * other request to make a forward progress.
	 * We are in an unfortunate situation where out_of_memory cannot
	 * do much for this context but let's try it to at least get
	 * access to memory reserved if the current task is killed (see
	 * out_of_memory). Once filesystems are ready to handle allocation
	 * failures more gracefully we should just bail out here.
	 */

	/* The OOM killer may not free memory on a specific node */
	if (gfp_mask & __GFP_THISNODE)
		goto out;
3689

3690
	/* Exhausted what can be done so it's blame time */
3691
	if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
3692
		*did_some_progress = 1;
3693

3694 3695 3696 3697 3698 3699
		/*
		 * Help non-failing allocations by giving them access to memory
		 * reserves
		 */
		if (gfp_mask & __GFP_NOFAIL)
			page = __alloc_pages_cpuset_fallback(gfp_mask, order,
3700 3701
					ALLOC_NO_WATERMARKS, ac);
	}
3702
out:
3703
	mutex_unlock(&oom_lock);
3704 3705 3706
	return page;
}

3707 3708 3709 3710 3711 3712
/*
 * Maximum number of compaction retries wit a progress before OOM
 * killer is consider as the only way to move forward.
 */
#define MAX_COMPACT_RETRIES 16

3713 3714 3715 3716
#ifdef CONFIG_COMPACTION
/* Try memory compaction for high-order allocations before reclaim */
static struct page *
__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3717
		unsigned int alloc_flags, const struct alloc_context *ac,
3718
		enum compact_priority prio, enum compact_result *compact_result)
3719
{
3720
	struct page *page;
3721
	unsigned long pflags;
3722
	unsigned int noreclaim_flag;
3723
	u64 start;
3724 3725

	if (!order)
3726 3727
		return NULL;

3728
	psi_memstall_enter(&pflags);
3729
	memcg_lat_stat_start(&start);
3730
	noreclaim_flag = memalloc_noreclaim_save();
3731

3732
	*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
3733
									prio);
3734

3735
	memalloc_noreclaim_restore(noreclaim_flag);
3736
	memcg_lat_stat_end(MEM_LAT_DIRECT_COMPACT, start);
3737
	psi_memstall_leave(&pflags);
3738

3739
	if (*compact_result <= COMPACT_INACTIVE)
3740
		return NULL;
3741

3742 3743 3744 3745 3746
	/*
	 * At least in one zone compaction wasn't deferred or skipped, so let's
	 * count a compaction stall
	 */
	count_vm_event(COMPACTSTALL);
3747

3748
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3749

3750 3751
	if (page) {
		struct zone *zone = page_zone(page);
3752

3753 3754 3755 3756 3757
		zone->compact_blockskip_flush = false;
		compaction_defer_reset(zone, order, true);
		count_vm_event(COMPACTSUCCESS);
		return page;
	}
3758

3759 3760 3761 3762 3763
	/*
	 * It's bad if compaction run occurs and fails. The most likely reason
	 * is that pages exist, but not enough to satisfy watermarks.
	 */
	count_vm_event(COMPACTFAIL);
3764

3765
	cond_resched();
3766 3767 3768

	return NULL;
}
3769

3770 3771 3772 3773
static inline bool
should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
		     enum compact_result compact_result,
		     enum compact_priority *compact_priority,
3774
		     int *compaction_retries)
3775 3776
{
	int max_retries = MAX_COMPACT_RETRIES;
3777
	int min_priority;
3778 3779 3780
	bool ret = false;
	int retries = *compaction_retries;
	enum compact_priority priority = *compact_priority;
3781 3782 3783 3784

	if (!order)
		return false;

3785 3786 3787
	if (compaction_made_progress(compact_result))
		(*compaction_retries)++;

3788 3789 3790 3791 3792
	/*
	 * compaction considers all the zone as desperately out of memory
	 * so it doesn't really make much sense to retry except when the
	 * failure could be caused by insufficient priority
	 */
3793 3794
	if (compaction_failed(compact_result))
		goto check_priority;
3795 3796 3797 3798 3799 3800 3801

	/*
	 * make sure the compaction wasn't deferred or didn't bail out early
	 * due to locks contention before we declare that we should give up.
	 * But do not retry if the given zonelist is not suitable for
	 * compaction.
	 */
3802 3803 3804 3805
	if (compaction_withdrawn(compact_result)) {
		ret = compaction_zonelist_suitable(ac, order, alloc_flags);
		goto out;
	}
3806 3807

	/*
3808
	 * !costly requests are much more important than __GFP_RETRY_MAYFAIL
3809 3810 3811 3812 3813 3814 3815 3816
	 * costly ones because they are de facto nofail and invoke OOM
	 * killer to move on while costly can fail and users are ready
	 * to cope with that. 1/4 retries is rather arbitrary but we
	 * would need much more detailed feedback from compaction to
	 * make a better decision.
	 */
	if (order > PAGE_ALLOC_COSTLY_ORDER)
		max_retries /= 4;
3817 3818 3819 3820
	if (*compaction_retries <= max_retries) {
		ret = true;
		goto out;
	}
3821

3822 3823 3824 3825 3826
	/*
	 * Make sure there are attempts at the highest priority if we exhausted
	 * all retries or failed at the lower priorities.
	 */
check_priority:
3827 3828
	min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
			MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
3829

3830
	if (*compact_priority > min_priority) {
3831 3832
		(*compact_priority)--;
		*compaction_retries = 0;
3833
		ret = true;
3834
	}
3835 3836 3837
out:
	trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
	return ret;
3838
}
3839 3840 3841
#else
static inline struct page *
__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3842
		unsigned int alloc_flags, const struct alloc_context *ac,
3843
		enum compact_priority prio, enum compact_result *compact_result)
3844
{
3845
	*compact_result = COMPACT_SKIPPED;
3846 3847
	return NULL;
}
3848 3849

static inline bool
3850 3851
should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
		     enum compact_result compact_result,
3852
		     enum compact_priority *compact_priority,
3853
		     int *compaction_retries)
3854
{
3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872
	struct zone *zone;
	struct zoneref *z;

	if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
		return false;

	/*
	 * There are setups with compaction disabled which would prefer to loop
	 * inside the allocator rather than hit the oom killer prematurely.
	 * Let's give them a good hope and keep retrying while the order-0
	 * watermarks are OK.
	 */
	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
					ac->nodemask) {
		if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
					ac_classzone_idx(ac), alloc_flags))
			return true;
	}
3873 3874
	return false;
}
3875
#endif /* CONFIG_COMPACTION */
3876

3877
#ifdef CONFIG_LOCKDEP
3878
static struct lockdep_map __fs_reclaim_map =
3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889
	STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);

static bool __need_fs_reclaim(gfp_t gfp_mask)
{
	gfp_mask = current_gfp_context(gfp_mask);

	/* no reclaim without waiting on it */
	if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
		return false;

	/* this guy won't enter reclaim */
T
Tetsuo Handa 已提交
3890
	if (current->flags & PF_MEMALLOC)
3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902
		return false;

	/* We're only interested __GFP_FS allocations for now */
	if (!(gfp_mask & __GFP_FS))
		return false;

	if (gfp_mask & __GFP_NOLOCKDEP)
		return false;

	return true;
}

3903 3904 3905 3906 3907 3908 3909 3910 3911 3912
void __fs_reclaim_acquire(void)
{
	lock_map_acquire(&__fs_reclaim_map);
}

void __fs_reclaim_release(void)
{
	lock_map_release(&__fs_reclaim_map);
}

3913 3914 3915
void fs_reclaim_acquire(gfp_t gfp_mask)
{
	if (__need_fs_reclaim(gfp_mask))
3916
		__fs_reclaim_acquire();
3917 3918 3919 3920 3921 3922
}
EXPORT_SYMBOL_GPL(fs_reclaim_acquire);

void fs_reclaim_release(gfp_t gfp_mask)
{
	if (__need_fs_reclaim(gfp_mask))
3923
		__fs_reclaim_release();
3924 3925 3926 3927
}
EXPORT_SYMBOL_GPL(fs_reclaim_release);
#endif

3928 3929
/* Perform direct synchronous page reclaim */
static int
3930 3931
__perform_reclaim(gfp_t gfp_mask, unsigned int order,
					const struct alloc_context *ac)
3932 3933
{
	struct reclaim_state reclaim_state;
3934
	int progress;
3935
	unsigned int noreclaim_flag;
3936
	unsigned long pflags;
3937
	u64 start;
3938 3939 3940 3941 3942

	cond_resched();

	/* We now go into synchronous reclaim */
	cpuset_memory_pressure_bump();
3943
	psi_memstall_enter(&pflags);
3944
	memcg_lat_stat_start(&start);
3945
	fs_reclaim_acquire(gfp_mask);
3946
	noreclaim_flag = memalloc_noreclaim_save();
3947
	reclaim_state.reclaimed_slab = 0;
3948
	current->reclaim_state = &reclaim_state;
3949

3950 3951
	progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
								ac->nodemask);
3952

3953
	current->reclaim_state = NULL;
3954
	memalloc_noreclaim_restore(noreclaim_flag);
3955
	fs_reclaim_release(gfp_mask);
3956
	memcg_lat_stat_end(MEM_LAT_GLOBAL_DIRECT_RECLAIM, start);
3957
	psi_memstall_leave(&pflags);
3958 3959 3960

	cond_resched();

3961 3962 3963 3964 3965 3966
	return progress;
}

/* The really slow allocator path where we enter direct reclaim */
static inline struct page *
__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
3967
		unsigned int alloc_flags, const struct alloc_context *ac,
3968
		unsigned long *did_some_progress)
3969 3970 3971 3972
{
	struct page *page = NULL;
	bool drained = false;

3973
	*did_some_progress = __perform_reclaim(gfp_mask, order, ac);
3974 3975
	if (unlikely(!(*did_some_progress)))
		return NULL;
3976

3977
retry:
3978
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3979 3980 3981

	/*
	 * If an allocation failed after direct reclaim, it could be because
3982 3983
	 * pages are pinned on the per-cpu lists or in high alloc reserves.
	 * Shrink them them and try again
3984 3985
	 */
	if (!page && !drained) {
3986
		unreserve_highatomic_pageblock(ac, false);
3987
		drain_all_pages(NULL);
3988 3989 3990 3991
		drained = true;
		goto retry;
	}

3992 3993 3994
	return page;
}

3995 3996
static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
			     const struct alloc_context *ac)
3997 3998 3999
{
	struct zoneref *z;
	struct zone *zone;
4000
	pg_data_t *last_pgdat = NULL;
4001
	enum zone_type high_zoneidx = ac->high_zoneidx;
4002

4003 4004
	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx,
					ac->nodemask) {
4005
		if (last_pgdat != zone->zone_pgdat)
4006
			wakeup_kswapd(zone, gfp_mask, order, high_zoneidx);
4007 4008
		last_pgdat = zone->zone_pgdat;
	}
4009 4010
}

4011
static inline unsigned int
4012 4013
gfp_to_alloc_flags(gfp_t gfp_mask)
{
4014
	unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
L
Linus Torvalds 已提交
4015

4016
	/* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
4017
	BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
4018

4019 4020 4021 4022
	/*
	 * The caller may dip into page reserves a bit more if the caller
	 * cannot run direct reclaim, or if the caller has realtime scheduling
	 * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
4023
	 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
4024
	 */
4025
	alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
L
Linus Torvalds 已提交
4026

4027
	if (gfp_mask & __GFP_ATOMIC) {
4028
		/*
4029 4030
		 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
		 * if it can't schedule.
4031
		 */
4032
		if (!(gfp_mask & __GFP_NOMEMALLOC))
4033
			alloc_flags |= ALLOC_HARDER;
4034
		/*
4035
		 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
4036
		 * comment for __cpuset_node_allowed().
4037
		 */
4038
		alloc_flags &= ~ALLOC_CPUSET;
4039
	} else if (unlikely(rt_task(current)) && !in_interrupt())
4040 4041
		alloc_flags |= ALLOC_HARDER;

4042 4043 4044 4045
#ifdef CONFIG_CMA
	if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
		alloc_flags |= ALLOC_CMA;
#endif
4046 4047 4048
	return alloc_flags;
}

4049
static bool oom_reserves_allowed(struct task_struct *tsk)
4050
{
4051 4052 4053 4054 4055 4056 4057 4058
	if (!tsk_is_oom_victim(tsk))
		return false;

	/*
	 * !MMU doesn't have oom reaper so give access to memory reserves
	 * only to the thread with TIF_MEMDIE set
	 */
	if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4059 4060
		return false;

4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071
	return true;
}

/*
 * Distinguish requests which really need access to full memory
 * reserves from oom victims which can live with a portion of it
 */
static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
{
	if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
		return 0;
4072
	if (gfp_mask & __GFP_MEMALLOC)
4073
		return ALLOC_NO_WATERMARKS;
4074
	if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4075 4076 4077 4078 4079 4080 4081
		return ALLOC_NO_WATERMARKS;
	if (!in_interrupt()) {
		if (current->flags & PF_MEMALLOC)
			return ALLOC_NO_WATERMARKS;
		else if (oom_reserves_allowed(current))
			return ALLOC_OOM;
	}
4082

4083 4084 4085 4086 4087 4088
	return 0;
}

bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
{
	return !!__gfp_pfmemalloc_flags(gfp_mask);
4089 4090
}

M
Michal Hocko 已提交
4091 4092 4093
/*
 * Checks whether it makes sense to retry the reclaim to make a forward progress
 * for the given allocation request.
4094 4095 4096 4097
 *
 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
 * without success, or when we couldn't even meet the watermark if we
 * reclaimed all remaining pages on the LRU lists.
M
Michal Hocko 已提交
4098 4099 4100 4101 4102 4103
 *
 * Returns true if a retry is viable or false to enter the oom path.
 */
static inline bool
should_reclaim_retry(gfp_t gfp_mask, unsigned order,
		     struct alloc_context *ac, int alloc_flags,
4104
		     bool did_some_progress, int *no_progress_loops)
M
Michal Hocko 已提交
4105 4106 4107 4108
{
	struct zone *zone;
	struct zoneref *z;

4109 4110 4111 4112 4113 4114 4115 4116 4117 4118
	/*
	 * Costly allocations might have made a progress but this doesn't mean
	 * their order will become available due to high fragmentation so
	 * always increment the no progress counter for them
	 */
	if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
		*no_progress_loops = 0;
	else
		(*no_progress_loops)++;

M
Michal Hocko 已提交
4119 4120 4121 4122
	/*
	 * Make sure we converge to OOM if we cannot make any progress
	 * several times in the row.
	 */
4123 4124
	if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
		/* Before OOM, exhaust highatomic_reserve */
4125
		return unreserve_highatomic_pageblock(ac, true);
4126
	}
M
Michal Hocko 已提交
4127

4128 4129 4130 4131 4132
	/*
	 * Keep reclaiming pages while there is a chance this will lead
	 * somewhere.  If none of the target zones can satisfy our allocation
	 * request even if all reclaimable pages are considered then we are
	 * screwed and have to go OOM.
M
Michal Hocko 已提交
4133 4134 4135 4136
	 */
	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
					ac->nodemask) {
		unsigned long available;
4137
		unsigned long reclaimable;
4138 4139
		unsigned long min_wmark = min_wmark_pages(zone);
		bool wmark;
M
Michal Hocko 已提交
4140

4141 4142
		available = reclaimable = zone_reclaimable_pages(zone);
		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
M
Michal Hocko 已提交
4143 4144

		/*
4145 4146
		 * Would the allocation succeed if we reclaimed all
		 * reclaimable pages?
M
Michal Hocko 已提交
4147
		 */
4148 4149 4150 4151 4152
		wmark = __zone_watermark_ok(zone, order, min_wmark,
				ac_classzone_idx(ac), alloc_flags, available);
		trace_reclaim_retry_zone(z, order, reclaimable,
				available, min_wmark, *no_progress_loops, wmark);
		if (wmark) {
4153 4154 4155 4156 4157 4158 4159
			/*
			 * If we didn't make any progress and have a lot of
			 * dirty + writeback pages then we should wait for
			 * an IO to complete to slow down the reclaim and
			 * prevent from pre mature OOM
			 */
			if (!did_some_progress) {
4160
				unsigned long write_pending;
4161

4162 4163
				write_pending = zone_page_state_snapshot(zone,
							NR_ZONE_WRITE_PENDING);
4164

4165
				if (2 * write_pending > reclaimable) {
4166 4167 4168 4169
					congestion_wait(BLK_RW_ASYNC, HZ/10);
					return true;
				}
			}
4170

4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184
			/*
			 * Memory allocation/reclaim might be called from a WQ
			 * context and the current implementation of the WQ
			 * concurrency control doesn't recognize that
			 * a particular WQ is congested if the worker thread is
			 * looping without ever sleeping. Therefore we have to
			 * do a short sleep here rather than calling
			 * cond_resched().
			 */
			if (current->flags & PF_WQ_WORKER)
				schedule_timeout_uninterruptible(1);
			else
				cond_resched();

M
Michal Hocko 已提交
4185 4186 4187 4188 4189 4190 4191
			return true;
		}
	}

	return false;
}

4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224
static inline bool
check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
{
	/*
	 * It's possible that cpuset's mems_allowed and the nodemask from
	 * mempolicy don't intersect. This should be normally dealt with by
	 * policy_nodemask(), but it's possible to race with cpuset update in
	 * such a way the check therein was true, and then it became false
	 * before we got our cpuset_mems_cookie here.
	 * This assumes that for all allocations, ac->nodemask can come only
	 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
	 * when it does not intersect with the cpuset restrictions) or the
	 * caller can deal with a violated nodemask.
	 */
	if (cpusets_enabled() && ac->nodemask &&
			!cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
		ac->nodemask = NULL;
		return true;
	}

	/*
	 * When updating a task's mems_allowed or mempolicy nodemask, it is
	 * possible to race with parallel threads in such a way that our
	 * allocation can fail while the mask is being updated. If we are about
	 * to fail, check if the cpuset changed during allocation and if so,
	 * retry.
	 */
	if (read_mems_allowed_retry(cpuset_mems_cookie))
		return true;

	return false;
}

4225 4226
static inline struct page *
__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
4227
						struct alloc_context *ac)
4228
{
4229
	bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
4230
	const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
4231
	struct page *page = NULL;
4232
	unsigned int alloc_flags;
4233
	unsigned long did_some_progress;
4234
	enum compact_priority compact_priority;
4235
	enum compact_result compact_result;
4236 4237 4238
	int compaction_retries;
	int no_progress_loops;
	unsigned int cpuset_mems_cookie;
4239
	int reserve_flags;
L
Linus Torvalds 已提交
4240

4241 4242 4243 4244 4245 4246 4247 4248
	/*
	 * We also sanity check to catch abuse of atomic reserves being used by
	 * callers that are not in atomic context.
	 */
	if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
				(__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
		gfp_mask &= ~__GFP_ATOMIC;

4249 4250 4251 4252 4253
retry_cpuset:
	compaction_retries = 0;
	no_progress_loops = 0;
	compact_priority = DEF_COMPACT_PRIORITY;
	cpuset_mems_cookie = read_mems_allowed_begin();
4254 4255 4256 4257 4258 4259 4260 4261

	/*
	 * The fast path uses conservative alloc_flags to succeed only until
	 * kswapd needs to be woken up, and to avoid the cost of setting up
	 * alloc_flags precisely. So we do that now.
	 */
	alloc_flags = gfp_to_alloc_flags(gfp_mask);

4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272
	/*
	 * We need to recalculate the starting point for the zonelist iterator
	 * because we might have used different nodemask in the fast path, or
	 * there was a cpuset modification and we are retrying - otherwise we
	 * could end up iterating over non-eligible zones endlessly.
	 */
	ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
					ac->high_zoneidx, ac->nodemask);
	if (!ac->preferred_zoneref->zone)
		goto nopage;

4273
	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
4274
		wake_all_kswapds(order, gfp_mask, ac);
4275 4276 4277 4278 4279 4280 4281 4282 4283

	/*
	 * The adjusted alloc_flags might result in immediate success, so try
	 * that first
	 */
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
	if (page)
		goto got_pg;

4284 4285
	/*
	 * For costly allocations, try direct compaction first, as it's likely
4286 4287 4288 4289 4290 4291
	 * that we have enough base pages and don't need to reclaim. For non-
	 * movable high-order allocations, do that as well, as compaction will
	 * try prevent permanent fragmentation by migrating from blocks of the
	 * same migratetype.
	 * Don't try this for allocations that are allowed to ignore
	 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
4292
	 */
4293 4294 4295 4296
	if (can_direct_reclaim &&
			(costly_order ||
			   (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
			&& !gfp_pfmemalloc_allowed(gfp_mask)) {
4297 4298
		page = __alloc_pages_direct_compact(gfp_mask, order,
						alloc_flags, ac,
4299
						INIT_COMPACT_PRIORITY,
4300 4301 4302 4303
						&compact_result);
		if (page)
			goto got_pg;

4304 4305 4306 4307
		/*
		 * Checks for costly allocations with __GFP_NORETRY, which
		 * includes THP page fault allocations
		 */
4308
		if (costly_order && (gfp_mask & __GFP_NORETRY)) {
4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320
			/*
			 * If compaction is deferred for high-order allocations,
			 * it is because sync compaction recently failed. If
			 * this is the case and the caller requested a THP
			 * allocation, we do not want to heavily disrupt the
			 * system, so we fail the allocation instead of entering
			 * direct reclaim.
			 */
			if (compact_result == COMPACT_DEFERRED)
				goto nopage;

			/*
4321 4322
			 * Looks like reclaim/compaction is worth trying, but
			 * sync compaction could be very expensive, so keep
4323
			 * using async compaction.
4324
			 */
4325
			compact_priority = INIT_COMPACT_PRIORITY;
4326 4327
		}
	}
4328

4329
retry:
4330
	/* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
4331
	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
4332
		wake_all_kswapds(order, gfp_mask, ac);
4333

4334 4335 4336
	reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
	if (reserve_flags)
		alloc_flags = reserve_flags;
4337

4338
	/*
4339 4340 4341
	 * Reset the nodemask and zonelist iterators if memory policies can be
	 * ignored. These allocations are high priority and system rather than
	 * user oriented.
4342
	 */
4343
	if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
4344
		ac->nodemask = NULL;
4345 4346 4347 4348
		ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
					ac->high_zoneidx, ac->nodemask);
	}

4349
	/* Attempt with potentially adjusted zonelist and alloc_flags */
4350
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
R
Rohit Seth 已提交
4351 4352
	if (page)
		goto got_pg;
L
Linus Torvalds 已提交
4353

4354
	/* Caller is not willing to reclaim, we can't balance anything */
4355
	if (!can_direct_reclaim)
L
Linus Torvalds 已提交
4356 4357
		goto nopage;

4358 4359
	/* Avoid recursion of direct reclaim */
	if (current->flags & PF_MEMALLOC)
4360 4361
		goto nopage;

4362 4363 4364 4365 4366 4367 4368
	/* Try direct reclaim and then allocating */
	page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
							&did_some_progress);
	if (page)
		goto got_pg;

	/* Try direct compaction and then allocating */
4369
	page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
4370
					compact_priority, &compact_result);
4371 4372
	if (page)
		goto got_pg;
4373

4374 4375
	/* Do not loop if specifically requested */
	if (gfp_mask & __GFP_NORETRY)
4376
		goto nopage;
4377

M
Michal Hocko 已提交
4378 4379
	/*
	 * Do not retry costly high order allocations unless they are
4380
	 * __GFP_RETRY_MAYFAIL
M
Michal Hocko 已提交
4381
	 */
4382
	if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
4383
		goto nopage;
M
Michal Hocko 已提交
4384 4385

	if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
4386
				 did_some_progress > 0, &no_progress_loops))
M
Michal Hocko 已提交
4387 4388
		goto retry;

4389 4390 4391 4392 4393 4394 4395
	/*
	 * It doesn't make any sense to retry for the compaction if the order-0
	 * reclaim is not able to make any progress because the current
	 * implementation of the compaction depends on the sufficient amount
	 * of free memory (see __compaction_suitable)
	 */
	if (did_some_progress > 0 &&
4396
			should_compact_retry(ac, order, alloc_flags,
4397
				compact_result, &compact_priority,
4398
				&compaction_retries))
4399 4400
		goto retry;

4401 4402 4403

	/* Deal with possible cpuset update races before we start OOM killing */
	if (check_retry_cpuset(cpuset_mems_cookie, ac))
4404 4405
		goto retry_cpuset;

4406 4407 4408 4409 4410
	/* Reclaim has failed us, start killing things */
	page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
	if (page)
		goto got_pg;

4411
	/* Avoid allocations with no watermarks from looping endlessly */
4412 4413
	if (tsk_is_oom_victim(current) &&
	    (alloc_flags == ALLOC_OOM ||
4414
	     (gfp_mask & __GFP_NOMEMALLOC)))
4415 4416
		goto nopage;

4417
	/* Retry as long as the OOM killer is making progress */
M
Michal Hocko 已提交
4418 4419
	if (did_some_progress) {
		no_progress_loops = 0;
4420
		goto retry;
M
Michal Hocko 已提交
4421
	}
4422

L
Linus Torvalds 已提交
4423
nopage:
4424 4425
	/* Deal with possible cpuset update races before we fail */
	if (check_retry_cpuset(cpuset_mems_cookie, ac))
4426 4427
		goto retry_cpuset;

4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454
	/*
	 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
	 * we always retry
	 */
	if (gfp_mask & __GFP_NOFAIL) {
		/*
		 * All existing users of the __GFP_NOFAIL are blockable, so warn
		 * of any new users that actually require GFP_NOWAIT
		 */
		if (WARN_ON_ONCE(!can_direct_reclaim))
			goto fail;

		/*
		 * PF_MEMALLOC request from this context is rather bizarre
		 * because we cannot reclaim anything and only can loop waiting
		 * for somebody to do a work for us
		 */
		WARN_ON_ONCE(current->flags & PF_MEMALLOC);

		/*
		 * non failing costly orders are a hard requirement which we
		 * are not prepared for much so let's warn about these users
		 * so that we can identify them and convert them to something
		 * else.
		 */
		WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER);

4455 4456 4457 4458 4459 4460 4461 4462 4463 4464
		/*
		 * Help non-failing allocations by giving them access to memory
		 * reserves but do not use ALLOC_NO_WATERMARKS because this
		 * could deplete whole memory reserves which would just make
		 * the situation worse
		 */
		page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
		if (page)
			goto got_pg;

4465 4466 4467 4468
		cond_resched();
		goto retry;
	}
fail:
4469
	warn_alloc(gfp_mask, ac->nodemask,
4470
			"page allocation failure: order:%u", order);
L
Linus Torvalds 已提交
4471
got_pg:
4472 4473 4474 4475

	if (ac->migratetype == MIGRATE_MOVABLE)
		memcg_check_wmark_min_adj(current, ac);

4476
	return page;
L
Linus Torvalds 已提交
4477
}
4478

4479
static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
4480
		int preferred_nid, nodemask_t *nodemask,
4481 4482
		struct alloc_context *ac, gfp_t *alloc_mask,
		unsigned int *alloc_flags)
4483
{
4484
	ac->high_zoneidx = gfp_zone(gfp_mask);
4485
	ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
4486 4487
	ac->nodemask = nodemask;
	ac->migratetype = gfpflags_to_migratetype(gfp_mask);
4488

4489
	if (cpusets_enabled()) {
4490 4491 4492
		*alloc_mask |= __GFP_HARDWALL;
		if (!ac->nodemask)
			ac->nodemask = &cpuset_current_mems_allowed;
4493 4494
		else
			*alloc_flags |= ALLOC_CPUSET;
4495 4496
	}

4497 4498
	fs_reclaim_acquire(gfp_mask);
	fs_reclaim_release(gfp_mask);
4499

4500
	might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
4501 4502

	if (should_fail_alloc_page(gfp_mask, order))
4503
		return false;
4504

4505 4506 4507
	if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE)
		*alloc_flags |= ALLOC_CMA;

4508 4509
	return true;
}
4510

4511
/* Determine whether to spread dirty pages and what the first usable zone */
4512
static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac)
4513
{
4514
	/* Dirty zone balancing only done in the fast path */
4515
	ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
4516

4517 4518 4519 4520 4521
	/*
	 * The preferred zone is used for statistics but crucially it is
	 * also used as the starting point for the zonelist iterator. It
	 * may get reset for allocations that ignore memory policies.
	 */
4522 4523 4524 4525 4526 4527 4528 4529
	ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
					ac->high_zoneidx, ac->nodemask);
}

/*
 * This is the 'heart' of the zoned buddy allocator.
 */
struct page *
4530 4531
__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
							nodemask_t *nodemask)
4532 4533 4534
{
	struct page *page;
	unsigned int alloc_flags = ALLOC_WMARK_LOW;
4535
	gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */
4536 4537
	struct alloc_context ac = { };

4538 4539 4540 4541 4542 4543 4544 4545 4546
	/*
	 * There are several places where we assume that the order value is sane
	 * so bail out early if the request is out of bound.
	 */
	if (unlikely(order >= MAX_ORDER)) {
		WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
		return NULL;
	}

4547
	gfp_mask &= gfp_allowed_mask;
4548
	alloc_mask = gfp_mask;
4549
	if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))
4550 4551
		return NULL;

4552
	finalise_ac(gfp_mask, &ac);
4553

4554
	/* First allocation attempt */
4555
	page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
4556 4557
	if (likely(page))
		goto out;
4558

4559
	/*
4560 4561 4562 4563
	 * Apply scoped allocation constraints. This is mainly about GFP_NOFS
	 * resp. GFP_NOIO which has to be inherited for all allocation requests
	 * from a particular context which has been marked by
	 * memalloc_no{fs,io}_{save,restore}.
4564
	 */
4565
	alloc_mask = current_gfp_context(gfp_mask);
4566
	ac.spread_dirty_pages = false;
4567

4568 4569 4570 4571
	/*
	 * Restore the original nodemask if it was potentially replaced with
	 * &cpuset_current_mems_allowed to optimize the fast-path attempt.
	 */
4572
	if (unlikely(ac.nodemask != nodemask))
4573
		ac.nodemask = nodemask;
4574

4575
	page = __alloc_pages_slowpath(alloc_mask, order, &ac);
4576

4577
out:
4578 4579 4580 4581
	if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
	    unlikely(memcg_kmem_charge(page, gfp_mask, order) != 0)) {
		__free_pages(page, order);
		page = NULL;
4582 4583
	}

4584 4585
	trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);

4586
	return page;
L
Linus Torvalds 已提交
4587
}
4588
EXPORT_SYMBOL(__alloc_pages_nodemask);
L
Linus Torvalds 已提交
4589 4590

/*
4591 4592 4593
 * Common helper functions. Never use with __GFP_HIGHMEM because the returned
 * address cannot represent highmem pages. Use alloc_pages and then kmap if
 * you need to access high mem.
L
Linus Torvalds 已提交
4594
 */
H
Harvey Harrison 已提交
4595
unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
L
Linus Torvalds 已提交
4596
{
4597 4598
	struct page *page;

4599
	page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
L
Linus Torvalds 已提交
4600 4601 4602 4603 4604 4605
	if (!page)
		return 0;
	return (unsigned long) page_address(page);
}
EXPORT_SYMBOL(__get_free_pages);

H
Harvey Harrison 已提交
4606
unsigned long get_zeroed_page(gfp_t gfp_mask)
L
Linus Torvalds 已提交
4607
{
4608
	return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
L
Linus Torvalds 已提交
4609 4610 4611
}
EXPORT_SYMBOL(get_zeroed_page);

4612
static inline void free_the_page(struct page *page, unsigned int order)
L
Linus Torvalds 已提交
4613
{
4614 4615 4616 4617
	if (order == 0)		/* Via pcp? */
		free_unref_page(page);
	else
		__free_pages_ok(page, order);
L
Linus Torvalds 已提交
4618 4619
}

4620 4621 4622 4623 4624
void __free_pages(struct page *page, unsigned int order)
{
	if (put_page_testzero(page))
		free_the_page(page, order);
}
L
Linus Torvalds 已提交
4625 4626
EXPORT_SYMBOL(__free_pages);

H
Harvey Harrison 已提交
4627
void free_pages(unsigned long addr, unsigned int order)
L
Linus Torvalds 已提交
4628 4629
{
	if (addr != 0) {
N
Nick Piggin 已提交
4630
		VM_BUG_ON(!virt_addr_valid((void *)addr));
L
Linus Torvalds 已提交
4631 4632 4633 4634 4635 4636
		__free_pages(virt_to_page((void *)addr), order);
	}
}

EXPORT_SYMBOL(free_pages);

4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647
/*
 * Page Fragment:
 *  An arbitrary-length arbitrary-offset area of memory which resides
 *  within a 0 or higher order page.  Multiple fragments within that page
 *  are individually refcounted, in the page's reference counter.
 *
 * The page_frag functions below provide a simple allocation framework for
 * page fragments.  This is used by the network stack and network device
 * drivers to provide a backing region of memory for use as either an
 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
 */
4648 4649
static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
					     gfp_t gfp_mask)
4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668
{
	struct page *page = NULL;
	gfp_t gfp = gfp_mask;

#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
	gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
		    __GFP_NOMEMALLOC;
	page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
				PAGE_FRAG_CACHE_MAX_ORDER);
	nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
#endif
	if (unlikely(!page))
		page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);

	nc->va = page ? page_address(page) : NULL;

	return page;
}

4669
void __page_frag_cache_drain(struct page *page, unsigned int count)
4670 4671 4672
{
	VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);

4673 4674
	if (page_ref_sub_and_test(page, count))
		free_the_page(page, compound_order(page));
4675
}
4676
EXPORT_SYMBOL(__page_frag_cache_drain);
4677

4678 4679
void *page_frag_alloc(struct page_frag_cache *nc,
		      unsigned int fragsz, gfp_t gfp_mask)
4680 4681 4682 4683 4684 4685 4686
{
	unsigned int size = PAGE_SIZE;
	struct page *page;
	int offset;

	if (unlikely(!nc->va)) {
refill:
4687
		page = __page_frag_cache_refill(nc, gfp_mask);
4688 4689 4690 4691 4692 4693 4694 4695 4696 4697
		if (!page)
			return NULL;

#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
		/* if size can vary use size else just use PAGE_SIZE */
		size = nc->size;
#endif
		/* Even if we own the page, we do not use atomic_set().
		 * This would break get_page_unless_zero() users.
		 */
4698
		page_ref_add(page, size);
4699 4700

		/* reset page count bias and offset to start of new frag */
4701
		nc->pfmemalloc = page_is_pfmemalloc(page);
4702
		nc->pagecnt_bias = size + 1;
4703 4704 4705 4706 4707 4708 4709
		nc->offset = size;
	}

	offset = nc->offset - fragsz;
	if (unlikely(offset < 0)) {
		page = virt_to_page(nc->va);

4710
		if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
4711 4712 4713 4714 4715 4716 4717
			goto refill;

#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
		/* if size can vary use size else just use PAGE_SIZE */
		size = nc->size;
#endif
		/* OK, page count is 0, we can safely set it */
4718
		set_page_count(page, size + 1);
4719 4720

		/* reset page count bias and offset to start of new frag */
4721
		nc->pagecnt_bias = size + 1;
4722 4723 4724 4725 4726 4727 4728 4729
		offset = size - fragsz;
	}

	nc->pagecnt_bias--;
	nc->offset = offset;

	return nc->va + offset;
}
4730
EXPORT_SYMBOL(page_frag_alloc);
4731 4732 4733 4734

/*
 * Frees a page fragment allocated out of either a compound or order 0 page.
 */
4735
void page_frag_free(void *addr)
4736 4737 4738
{
	struct page *page = virt_to_head_page(addr);

4739 4740
	if (unlikely(put_page_testzero(page)))
		free_the_page(page, compound_order(page));
4741
}
4742
EXPORT_SYMBOL(page_frag_free);
4743

4744 4745
static void *make_alloc_exact(unsigned long addr, unsigned int order,
		size_t size)
A
Andi Kleen 已提交
4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759
{
	if (addr) {
		unsigned long alloc_end = addr + (PAGE_SIZE << order);
		unsigned long used = addr + PAGE_ALIGN(size);

		split_page(virt_to_page((void *)addr), order);
		while (used < alloc_end) {
			free_page(used);
			used += PAGE_SIZE;
		}
	}
	return (void *)addr;
}

4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778
/**
 * alloc_pages_exact - allocate an exact number physically-contiguous pages.
 * @size: the number of bytes to allocate
 * @gfp_mask: GFP flags for the allocation
 *
 * This function is similar to alloc_pages(), except that it allocates the
 * minimum number of pages to satisfy the request.  alloc_pages() can only
 * allocate memory in power-of-two pages.
 *
 * This function is also limited by MAX_ORDER.
 *
 * Memory allocated by this function must be released by free_pages_exact().
 */
void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
{
	unsigned int order = get_order(size);
	unsigned long addr;

	addr = __get_free_pages(gfp_mask, order);
A
Andi Kleen 已提交
4779
	return make_alloc_exact(addr, order, size);
4780 4781 4782
}
EXPORT_SYMBOL(alloc_pages_exact);

A
Andi Kleen 已提交
4783 4784 4785
/**
 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
 *			   pages on a node.
4786
 * @nid: the preferred node ID where memory should be allocated
A
Andi Kleen 已提交
4787 4788 4789 4790 4791 4792
 * @size: the number of bytes to allocate
 * @gfp_mask: GFP flags for the allocation
 *
 * Like alloc_pages_exact(), but try to allocate on node nid first before falling
 * back.
 */
4793
void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
A
Andi Kleen 已提交
4794
{
4795
	unsigned int order = get_order(size);
A
Andi Kleen 已提交
4796 4797 4798 4799 4800 4801
	struct page *p = alloc_pages_node(nid, gfp_mask, order);
	if (!p)
		return NULL;
	return make_alloc_exact((unsigned long)page_address(p), order, size);
}

4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820
/**
 * free_pages_exact - release memory allocated via alloc_pages_exact()
 * @virt: the value returned by alloc_pages_exact.
 * @size: size of allocation, same value as passed to alloc_pages_exact().
 *
 * Release the memory allocated by a previous call to alloc_pages_exact.
 */
void free_pages_exact(void *virt, size_t size)
{
	unsigned long addr = (unsigned long)virt;
	unsigned long end = addr + PAGE_ALIGN(size);

	while (addr < end) {
		free_page(addr);
		addr += PAGE_SIZE;
	}
}
EXPORT_SYMBOL(free_pages_exact);

4821 4822 4823 4824 4825 4826 4827
/**
 * nr_free_zone_pages - count number of pages beyond high watermark
 * @offset: The zone index of the highest zone
 *
 * nr_free_zone_pages() counts the number of counts pages which are beyond the
 * high watermark within all zones at or below a given zone index.  For each
 * zone, the number of pages is calculated as:
4828 4829
 *
 *     nr_free_zone_pages = managed_pages - high_pages
4830
 */
4831
static unsigned long nr_free_zone_pages(int offset)
L
Linus Torvalds 已提交
4832
{
4833
	struct zoneref *z;
4834 4835
	struct zone *zone;

4836
	/* Just pick one node, since fallback list is circular */
4837
	unsigned long sum = 0;
L
Linus Torvalds 已提交
4838

4839
	struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
L
Linus Torvalds 已提交
4840

4841
	for_each_zone_zonelist(zone, z, zonelist, offset) {
4842
		unsigned long size = zone->managed_pages;
4843
		unsigned long high = high_wmark_pages(zone);
4844 4845
		if (size > high)
			sum += size - high;
L
Linus Torvalds 已提交
4846 4847 4848 4849 4850
	}

	return sum;
}

4851 4852 4853 4854 4855
/**
 * nr_free_buffer_pages - count number of pages beyond high watermark
 *
 * nr_free_buffer_pages() counts the number of pages which are beyond the high
 * watermark within ZONE_DMA and ZONE_NORMAL.
L
Linus Torvalds 已提交
4856
 */
4857
unsigned long nr_free_buffer_pages(void)
L
Linus Torvalds 已提交
4858
{
A
Al Viro 已提交
4859
	return nr_free_zone_pages(gfp_zone(GFP_USER));
L
Linus Torvalds 已提交
4860
}
4861
EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
L
Linus Torvalds 已提交
4862

4863 4864 4865 4866 4867
/**
 * nr_free_pagecache_pages - count number of pages beyond high watermark
 *
 * nr_free_pagecache_pages() counts the number of pages which are beyond the
 * high watermark within all zones.
L
Linus Torvalds 已提交
4868
 */
4869
unsigned long nr_free_pagecache_pages(void)
L
Linus Torvalds 已提交
4870
{
M
Mel Gorman 已提交
4871
	return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
L
Linus Torvalds 已提交
4872
}
4873 4874

static inline void show_node(struct zone *zone)
L
Linus Torvalds 已提交
4875
{
4876
	if (IS_ENABLED(CONFIG_NUMA))
4877
		printk("Node %d ", zone_to_nid(zone));
L
Linus Torvalds 已提交
4878 4879
}

4880 4881 4882 4883 4884 4885 4886 4887 4888 4889
long si_mem_available(void)
{
	long available;
	unsigned long pagecache;
	unsigned long wmark_low = 0;
	unsigned long pages[NR_LRU_LISTS];
	struct zone *zone;
	int lru;

	for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
4890
		pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
4891 4892

	for_each_zone(zone)
4893
		wmark_low += low_wmark_pages(zone);
4894 4895 4896 4897 4898

	/*
	 * Estimate the amount of memory available for userspace allocations,
	 * without causing swapping.
	 */
4899
	available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913

	/*
	 * Not all the page cache can be freed, otherwise the system will
	 * start swapping. Assume at least half of the page cache, or the
	 * low watermark worth of cache, needs to stay.
	 */
	pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
	pagecache -= min(pagecache / 2, wmark_low);
	available += pagecache;

	/*
	 * Part of the reclaimable slab consists of items that are in use,
	 * and cannot be freed. Cap this estimate at the low watermark.
	 */
4914 4915 4916
	available += global_node_page_state(NR_SLAB_RECLAIMABLE) -
		     min(global_node_page_state(NR_SLAB_RECLAIMABLE) / 2,
			 wmark_low);
4917

4918 4919 4920 4921 4922 4923 4924
	/*
	 * Part of the kernel memory, which can be released under memory
	 * pressure.
	 */
	available += global_node_page_state(NR_INDIRECTLY_RECLAIMABLE_BYTES) >>
		PAGE_SHIFT;

4925 4926 4927 4928 4929 4930
	if (available < 0)
		available = 0;
	return available;
}
EXPORT_SYMBOL_GPL(si_mem_available);

L
Linus Torvalds 已提交
4931 4932 4933
void si_meminfo(struct sysinfo *val)
{
	val->totalram = totalram_pages;
4934
	val->sharedram = global_node_page_state(NR_SHMEM);
4935
	val->freeram = global_zone_page_state(NR_FREE_PAGES);
L
Linus Torvalds 已提交
4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946
	val->bufferram = nr_blockdev_pages();
	val->totalhigh = totalhigh_pages;
	val->freehigh = nr_free_highpages();
	val->mem_unit = PAGE_SIZE;
}

EXPORT_SYMBOL(si_meminfo);

#ifdef CONFIG_NUMA
void si_meminfo_node(struct sysinfo *val, int nid)
{
4947 4948
	int zone_type;		/* needs to be signed */
	unsigned long managed_pages = 0;
4949 4950
	unsigned long managed_highpages = 0;
	unsigned long free_highpages = 0;
L
Linus Torvalds 已提交
4951 4952
	pg_data_t *pgdat = NODE_DATA(nid);

4953 4954 4955
	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
		managed_pages += pgdat->node_zones[zone_type].managed_pages;
	val->totalram = managed_pages;
4956
	val->sharedram = node_page_state(pgdat, NR_SHMEM);
4957
	val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
4958
#ifdef CONFIG_HIGHMEM
4959 4960 4961 4962 4963 4964 4965 4966 4967 4968
	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
		struct zone *zone = &pgdat->node_zones[zone_type];

		if (is_highmem(zone)) {
			managed_highpages += zone->managed_pages;
			free_highpages += zone_page_state(zone, NR_FREE_PAGES);
		}
	}
	val->totalhigh = managed_highpages;
	val->freehigh = free_highpages;
4969
#else
4970 4971
	val->totalhigh = managed_highpages;
	val->freehigh = free_highpages;
4972
#endif
L
Linus Torvalds 已提交
4973 4974 4975 4976
	val->mem_unit = PAGE_SIZE;
}
#endif

4977
/*
4978 4979
 * Determine whether the node should be displayed or not, depending on whether
 * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
4980
 */
4981
static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
4982 4983
{
	if (!(flags & SHOW_MEM_FILTER_NODES))
4984
		return false;
4985

4986 4987 4988 4989 4990 4991 4992 4993 4994
	/*
	 * no node mask - aka implicit memory numa policy. Do not bother with
	 * the synchronization - read_mems_allowed_begin - because we do not
	 * have to be precise here.
	 */
	if (!nodemask)
		nodemask = &cpuset_current_mems_allowed;

	return !node_isset(nid, *nodemask);
4995 4996
}

L
Linus Torvalds 已提交
4997 4998
#define K(x) ((x) << (PAGE_SHIFT-10))

4999 5000 5001 5002 5003
static void show_migration_types(unsigned char type)
{
	static const char types[MIGRATE_TYPES] = {
		[MIGRATE_UNMOVABLE]	= 'U',
		[MIGRATE_MOVABLE]	= 'M',
5004 5005
		[MIGRATE_RECLAIMABLE]	= 'E',
		[MIGRATE_HIGHATOMIC]	= 'H',
5006 5007 5008
#ifdef CONFIG_CMA
		[MIGRATE_CMA]		= 'C',
#endif
5009
#ifdef CONFIG_MEMORY_ISOLATION
5010
		[MIGRATE_ISOLATE]	= 'I',
5011
#endif
5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022
	};
	char tmp[MIGRATE_TYPES + 1];
	char *p = tmp;
	int i;

	for (i = 0; i < MIGRATE_TYPES; i++) {
		if (type & (1 << i))
			*p++ = types[i];
	}

	*p = '\0';
5023
	printk(KERN_CONT "(%s) ", tmp);
5024 5025
}

L
Linus Torvalds 已提交
5026 5027 5028 5029
/*
 * Show free area list (used inside shift_scroll-lock stuff)
 * We also calculate the percentage fragmentation. We do this by counting the
 * memory on each free list with the exception of the first item on the list.
5030 5031 5032 5033
 *
 * Bits in @filter:
 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
 *   cpuset.
L
Linus Torvalds 已提交
5034
 */
5035
void show_free_areas(unsigned int filter, nodemask_t *nodemask)
L
Linus Torvalds 已提交
5036
{
5037
	unsigned long free_pcp = 0;
5038
	int cpu;
L
Linus Torvalds 已提交
5039
	struct zone *zone;
M
Mel Gorman 已提交
5040
	pg_data_t *pgdat;
L
Linus Torvalds 已提交
5041

5042
	for_each_populated_zone(zone) {
5043
		if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5044
			continue;
5045

5046 5047
		for_each_online_cpu(cpu)
			free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
L
Linus Torvalds 已提交
5048 5049
	}

K
KOSAKI Motohiro 已提交
5050 5051
	printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
		" active_file:%lu inactive_file:%lu isolated_file:%lu\n"
5052 5053
		" unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
		" slab_reclaimable:%lu slab_unreclaimable:%lu\n"
5054
		" mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
5055
		" free:%lu free_pcp:%lu free_cma:%lu\n",
M
Mel Gorman 已提交
5056 5057 5058 5059 5060 5061 5062
		global_node_page_state(NR_ACTIVE_ANON),
		global_node_page_state(NR_INACTIVE_ANON),
		global_node_page_state(NR_ISOLATED_ANON),
		global_node_page_state(NR_ACTIVE_FILE),
		global_node_page_state(NR_INACTIVE_FILE),
		global_node_page_state(NR_ISOLATED_FILE),
		global_node_page_state(NR_UNEVICTABLE),
5063 5064 5065
		global_node_page_state(NR_FILE_DIRTY),
		global_node_page_state(NR_WRITEBACK),
		global_node_page_state(NR_UNSTABLE_NFS),
5066 5067
		global_node_page_state(NR_SLAB_RECLAIMABLE),
		global_node_page_state(NR_SLAB_UNRECLAIMABLE),
5068
		global_node_page_state(NR_FILE_MAPPED),
5069
		global_node_page_state(NR_SHMEM),
5070 5071 5072
		global_zone_page_state(NR_PAGETABLE),
		global_zone_page_state(NR_BOUNCE),
		global_zone_page_state(NR_FREE_PAGES),
5073
		free_pcp,
5074
		global_zone_page_state(NR_FREE_CMA_PAGES));
L
Linus Torvalds 已提交
5075

M
Mel Gorman 已提交
5076
	for_each_online_pgdat(pgdat) {
5077
		if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
5078 5079
			continue;

M
Mel Gorman 已提交
5080 5081 5082 5083 5084 5085 5086 5087
		printk("Node %d"
			" active_anon:%lukB"
			" inactive_anon:%lukB"
			" active_file:%lukB"
			" inactive_file:%lukB"
			" unevictable:%lukB"
			" isolated(anon):%lukB"
			" isolated(file):%lukB"
5088
			" mapped:%lukB"
5089 5090 5091 5092 5093 5094 5095 5096 5097 5098
			" dirty:%lukB"
			" writeback:%lukB"
			" shmem:%lukB"
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
			" shmem_thp: %lukB"
			" shmem_pmdmapped: %lukB"
			" anon_thp: %lukB"
#endif
			" writeback_tmp:%lukB"
			" unstable:%lukB"
M
Mel Gorman 已提交
5099 5100 5101 5102 5103 5104 5105 5106 5107 5108
			" all_unreclaimable? %s"
			"\n",
			pgdat->node_id,
			K(node_page_state(pgdat, NR_ACTIVE_ANON)),
			K(node_page_state(pgdat, NR_INACTIVE_ANON)),
			K(node_page_state(pgdat, NR_ACTIVE_FILE)),
			K(node_page_state(pgdat, NR_INACTIVE_FILE)),
			K(node_page_state(pgdat, NR_UNEVICTABLE)),
			K(node_page_state(pgdat, NR_ISOLATED_ANON)),
			K(node_page_state(pgdat, NR_ISOLATED_FILE)),
5109
			K(node_page_state(pgdat, NR_FILE_MAPPED)),
5110 5111
			K(node_page_state(pgdat, NR_FILE_DIRTY)),
			K(node_page_state(pgdat, NR_WRITEBACK)),
5112
			K(node_page_state(pgdat, NR_SHMEM)),
5113 5114 5115 5116 5117 5118 5119 5120
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
			K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
			K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
					* HPAGE_PMD_NR),
			K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
#endif
			K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
			K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
5121 5122
			pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
				"yes" : "no");
M
Mel Gorman 已提交
5123 5124
	}

5125
	for_each_populated_zone(zone) {
L
Linus Torvalds 已提交
5126 5127
		int i;

5128
		if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5129
			continue;
5130 5131 5132 5133 5134

		free_pcp = 0;
		for_each_online_cpu(cpu)
			free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;

L
Linus Torvalds 已提交
5135
		show_node(zone);
5136 5137
		printk(KERN_CONT
			"%s"
L
Linus Torvalds 已提交
5138 5139 5140 5141
			" free:%lukB"
			" min:%lukB"
			" low:%lukB"
			" high:%lukB"
M
Minchan Kim 已提交
5142 5143 5144 5145 5146
			" active_anon:%lukB"
			" inactive_anon:%lukB"
			" active_file:%lukB"
			" inactive_file:%lukB"
			" unevictable:%lukB"
5147
			" writepending:%lukB"
L
Linus Torvalds 已提交
5148
			" present:%lukB"
5149
			" managed:%lukB"
5150
			" mlocked:%lukB"
5151
			" kernel_stack:%lukB"
5152 5153
			" pagetables:%lukB"
			" bounce:%lukB"
5154 5155
			" free_pcp:%lukB"
			" local_pcp:%ukB"
5156
			" free_cma:%lukB"
L
Linus Torvalds 已提交
5157 5158
			"\n",
			zone->name,
5159
			K(zone_page_state(zone, NR_FREE_PAGES)),
5160 5161 5162
			K(min_wmark_pages(zone)),
			K(low_wmark_pages(zone)),
			K(high_wmark_pages(zone)),
M
Minchan Kim 已提交
5163 5164 5165 5166 5167
			K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
			K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
			K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
			K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
			K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
5168
			K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
L
Linus Torvalds 已提交
5169
			K(zone->present_pages),
5170
			K(zone->managed_pages),
5171
			K(zone_page_state(zone, NR_MLOCK)),
5172
			zone_page_state(zone, NR_KERNEL_STACK_KB),
5173 5174
			K(zone_page_state(zone, NR_PAGETABLE)),
			K(zone_page_state(zone, NR_BOUNCE)),
5175 5176
			K(free_pcp),
			K(this_cpu_read(zone->pageset->pcp.count)),
5177
			K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
L
Linus Torvalds 已提交
5178 5179
		printk("lowmem_reserve[]:");
		for (i = 0; i < MAX_NR_ZONES; i++)
5180 5181
			printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
		printk(KERN_CONT "\n");
L
Linus Torvalds 已提交
5182 5183
	}

5184
	for_each_populated_zone(zone) {
5185 5186
		unsigned int order;
		unsigned long nr[MAX_ORDER], flags, total = 0;
5187
		unsigned char types[MAX_ORDER];
L
Linus Torvalds 已提交
5188

5189
		if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5190
			continue;
L
Linus Torvalds 已提交
5191
		show_node(zone);
5192
		printk(KERN_CONT "%s: ", zone->name);
L
Linus Torvalds 已提交
5193 5194 5195

		spin_lock_irqsave(&zone->lock, flags);
		for (order = 0; order < MAX_ORDER; order++) {
5196 5197 5198 5199
			struct free_area *area = &zone->free_area[order];
			int type;

			nr[order] = area->nr_free;
5200
			total += nr[order] << order;
5201 5202 5203

			types[order] = 0;
			for (type = 0; type < MIGRATE_TYPES; type++) {
5204
				if (!free_area_empty(area, type))
5205 5206
					types[order] |= 1 << type;
			}
L
Linus Torvalds 已提交
5207 5208
		}
		spin_unlock_irqrestore(&zone->lock, flags);
5209
		for (order = 0; order < MAX_ORDER; order++) {
5210 5211
			printk(KERN_CONT "%lu*%lukB ",
			       nr[order], K(1UL) << order);
5212 5213 5214
			if (nr[order])
				show_migration_types(types[order]);
		}
5215
		printk(KERN_CONT "= %lukB\n", K(total));
L
Linus Torvalds 已提交
5216 5217
	}

5218 5219
	hugetlb_show_meminfo();

5220
	printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
5221

L
Linus Torvalds 已提交
5222 5223 5224
	show_swap_cache_info();
}

5225 5226 5227 5228 5229 5230
static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
{
	zoneref->zone = zone;
	zoneref->zone_idx = zone_idx(zone);
}

L
Linus Torvalds 已提交
5231 5232
/*
 * Builds allocation fallback zone lists.
5233 5234
 *
 * Add all populated zones of a node to the zonelist.
L
Linus Torvalds 已提交
5235
 */
5236
static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
L
Linus Torvalds 已提交
5237
{
5238
	struct zone *zone;
5239
	enum zone_type zone_type = MAX_NR_ZONES;
5240
	int nr_zones = 0;
5241 5242

	do {
5243
		zone_type--;
5244
		zone = pgdat->node_zones + zone_type;
5245
		if (managed_zone(zone)) {
5246
			zoneref_set_zone(zone, &zonerefs[nr_zones++]);
5247
			check_highest_zone(zone_type);
L
Linus Torvalds 已提交
5248
		}
5249
	} while (zone_type);
5250

5251
	return nr_zones;
L
Linus Torvalds 已提交
5252 5253 5254
}

#ifdef CONFIG_NUMA
5255 5256 5257

static int __parse_numa_zonelist_order(char *s)
{
5258 5259 5260 5261 5262 5263 5264 5265
	/*
	 * We used to support different zonlists modes but they turned
	 * out to be just not useful. Let's keep the warning in place
	 * if somebody still use the cmd line parameter so that we do
	 * not fail it silently
	 */
	if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
		pr_warn("Ignoring unsupported numa_zonelist_order value:  %s\n", s);
5266 5267 5268 5269 5270 5271 5272
		return -EINVAL;
	}
	return 0;
}

static __init int setup_numa_zonelist_order(char *s)
{
5273 5274 5275
	if (!s)
		return 0;

5276
	return __parse_numa_zonelist_order(s);
5277 5278 5279
}
early_param("numa_zonelist_order", setup_numa_zonelist_order);

5280 5281
char numa_zonelist_order[] = "Node";

5282 5283 5284
/*
 * sysctl handler for numa_zonelist_order
 */
5285
int numa_zonelist_order_handler(struct ctl_table *table, int write,
5286
		void __user *buffer, size_t *length,
5287 5288
		loff_t *ppos)
{
5289
	char *str;
5290 5291
	int ret;

5292 5293 5294 5295 5296
	if (!write)
		return proc_dostring(table, write, buffer, length, ppos);
	str = memdup_user_nul(buffer, 16);
	if (IS_ERR(str))
		return PTR_ERR(str);
5297

5298 5299
	ret = __parse_numa_zonelist_order(str);
	kfree(str);
5300
	return ret;
5301 5302 5303
}


5304
#define MAX_NODE_LOAD (nr_online_nodes)
5305 5306
static int node_load[MAX_NUMNODES];

L
Linus Torvalds 已提交
5307
/**
5308
 * find_next_best_node - find the next node that should appear in a given node's fallback list
L
Linus Torvalds 已提交
5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320
 * @node: node whose fallback list we're appending
 * @used_node_mask: nodemask_t of already used nodes
 *
 * We use a number of factors to determine which is the next node that should
 * appear on a given node's fallback list.  The node should not have appeared
 * already in @node's fallback list, and it should be the next closest node
 * according to the distance array (which contains arbitrary distance values
 * from each node to each node in the system), and should also prefer nodes
 * with no CPUs, since presumably they'll have very little allocation pressure
 * on them otherwise.
 * It returns -1 if no node is found.
 */
5321
static int find_next_best_node(int node, nodemask_t *used_node_mask)
L
Linus Torvalds 已提交
5322
{
5323
	int n, val;
L
Linus Torvalds 已提交
5324
	int min_val = INT_MAX;
D
David Rientjes 已提交
5325
	int best_node = NUMA_NO_NODE;
5326
	const struct cpumask *tmp = cpumask_of_node(0);
L
Linus Torvalds 已提交
5327

5328 5329 5330 5331 5332
	/* Use the local node if we haven't already */
	if (!node_isset(node, *used_node_mask)) {
		node_set(node, *used_node_mask);
		return node;
	}
L
Linus Torvalds 已提交
5333

5334
	for_each_node_state(n, N_MEMORY) {
L
Linus Torvalds 已提交
5335 5336 5337 5338 5339 5340 5341 5342

		/* Don't want a node to appear more than once */
		if (node_isset(n, *used_node_mask))
			continue;

		/* Use the distance array to find the distance */
		val = node_distance(node, n);

5343 5344 5345
		/* Penalize nodes under us ("prefer the next node") */
		val += (n < node);

L
Linus Torvalds 已提交
5346
		/* Give preference to headless and unused nodes */
5347 5348
		tmp = cpumask_of_node(n);
		if (!cpumask_empty(tmp))
L
Linus Torvalds 已提交
5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366
			val += PENALTY_FOR_NODE_WITH_CPUS;

		/* Slight preference for less loaded node */
		val *= (MAX_NODE_LOAD*MAX_NUMNODES);
		val += node_load[n];

		if (val < min_val) {
			min_val = val;
			best_node = n;
		}
	}

	if (best_node >= 0)
		node_set(best_node, *used_node_mask);

	return best_node;
}

5367 5368 5369 5370 5371 5372

/*
 * Build zonelists ordered by node and zones within node.
 * This results in maximum locality--normal zone overflows into local
 * DMA zone, if any--but risks exhausting DMA zone.
 */
5373 5374
static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
		unsigned nr_nodes)
L
Linus Torvalds 已提交
5375
{
5376 5377 5378 5379 5380 5381 5382 5383 5384
	struct zoneref *zonerefs;
	int i;

	zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;

	for (i = 0; i < nr_nodes; i++) {
		int nr_zones;

		pg_data_t *node = NODE_DATA(node_order[i]);
5385

5386 5387 5388 5389 5390
		nr_zones = build_zonerefs_node(node, zonerefs);
		zonerefs += nr_zones;
	}
	zonerefs->zone = NULL;
	zonerefs->zone_idx = 0;
5391 5392
}

5393 5394 5395 5396 5397
/*
 * Build gfp_thisnode zonelists
 */
static void build_thisnode_zonelists(pg_data_t *pgdat)
{
5398 5399
	struct zoneref *zonerefs;
	int nr_zones;
5400

5401 5402 5403 5404 5405
	zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
	nr_zones = build_zonerefs_node(pgdat, zonerefs);
	zonerefs += nr_zones;
	zonerefs->zone = NULL;
	zonerefs->zone_idx = 0;
5406 5407
}

5408 5409 5410 5411 5412 5413 5414 5415 5416
/*
 * Build zonelists ordered by zone and nodes within zones.
 * This results in conserving DMA zone[s] until all Normal memory is
 * exhausted, but results in overflowing to remote node while memory
 * may still exist in local DMA zone.
 */

static void build_zonelists(pg_data_t *pgdat)
{
5417 5418
	static int node_order[MAX_NUMNODES];
	int node, load, nr_nodes = 0;
L
Linus Torvalds 已提交
5419
	nodemask_t used_mask;
5420
	int local_node, prev_node;
L
Linus Torvalds 已提交
5421 5422 5423

	/* NUMA-aware ordering of nodes */
	local_node = pgdat->node_id;
5424
	load = nr_online_nodes;
L
Linus Torvalds 已提交
5425 5426
	prev_node = local_node;
	nodes_clear(used_mask);
5427 5428

	memset(node_order, 0, sizeof(node_order));
L
Linus Torvalds 已提交
5429 5430 5431 5432 5433 5434
	while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
		/*
		 * We don't want to pressure a particular node.
		 * So adding penalty to the first node in same
		 * distance group to make it round-robin.
		 */
5435 5436
		if (node_distance(local_node, node) !=
		    node_distance(local_node, prev_node))
5437 5438
			node_load[node] = load;

5439
		node_order[nr_nodes++] = node;
L
Linus Torvalds 已提交
5440 5441 5442
		prev_node = node;
		load--;
	}
5443

5444
	build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
5445
	build_thisnode_zonelists(pgdat);
L
Linus Torvalds 已提交
5446 5447
}

5448 5449 5450 5451 5452 5453 5454 5455 5456
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
/*
 * Return node id of node used for "local" allocations.
 * I.e., first node id of first zone in arg node's generic zonelist.
 * Used for initializing percpu 'numa_mem', which is used primarily
 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
 */
int local_memory_node(int node)
{
5457
	struct zoneref *z;
5458

5459
	z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
5460
				   gfp_zone(GFP_KERNEL),
5461
				   NULL);
5462
	return zone_to_nid(z->zone);
5463 5464
}
#endif
5465

5466 5467
static void setup_min_unmapped_ratio(void);
static void setup_min_slab_ratio(void);
L
Linus Torvalds 已提交
5468 5469
#else	/* CONFIG_NUMA */

5470
static void build_zonelists(pg_data_t *pgdat)
L
Linus Torvalds 已提交
5471
{
5472
	int node, local_node;
5473 5474
	struct zoneref *zonerefs;
	int nr_zones;
L
Linus Torvalds 已提交
5475 5476 5477

	local_node = pgdat->node_id;

5478 5479 5480
	zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
	nr_zones = build_zonerefs_node(pgdat, zonerefs);
	zonerefs += nr_zones;
L
Linus Torvalds 已提交
5481

5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492
	/*
	 * Now we build the zonelist so that it contains the zones
	 * of all the other nodes.
	 * We don't want to pressure a particular node, so when
	 * building the zones for node N, we make sure that the
	 * zones coming right after the local ones are those from
	 * node N+1 (modulo N)
	 */
	for (node = local_node + 1; node < MAX_NUMNODES; node++) {
		if (!node_online(node))
			continue;
5493 5494
		nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
		zonerefs += nr_zones;
L
Linus Torvalds 已提交
5495
	}
5496 5497 5498
	for (node = 0; node < local_node; node++) {
		if (!node_online(node))
			continue;
5499 5500
		nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
		zonerefs += nr_zones;
5501 5502
	}

5503 5504
	zonerefs->zone = NULL;
	zonerefs->zone_idx = 0;
L
Linus Torvalds 已提交
5505 5506 5507 5508
}

#endif	/* CONFIG_NUMA */

5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525
/*
 * Boot pageset table. One per cpu which is going to be used for all
 * zones and all nodes. The parameters will be set in such a way
 * that an item put on a list will immediately be handed over to
 * the buddy list. This is safe since pageset manipulation is done
 * with interrupts disabled.
 *
 * The boot_pagesets must be kept even after bootup is complete for
 * unused processors and/or zones. They do play a role for bootstrapping
 * hotplugged processors.
 *
 * zoneinfo_show() and maybe other functions do
 * not check if the processor is online before following the pageset pointer.
 * Other parts of the kernel may not check if the zone is available.
 */
static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
5526
static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
5527

5528
static void __build_all_zonelists(void *data)
L
Linus Torvalds 已提交
5529
{
5530
	int nid;
5531
	int __maybe_unused cpu;
5532
	pg_data_t *self = data;
5533 5534 5535
	static DEFINE_SPINLOCK(lock);

	spin_lock(&lock);
5536

5537 5538 5539
#ifdef CONFIG_NUMA
	memset(node_load, 0, sizeof(node_load));
#endif
5540

5541 5542 5543 5544
	/*
	 * This node is hotadded and no memory is yet present.   So just
	 * building zonelists is fine - no need to touch other nodes.
	 */
5545 5546
	if (self && !node_online(self->node_id)) {
		build_zonelists(self);
5547 5548 5549
	} else {
		for_each_online_node(nid) {
			pg_data_t *pgdat = NODE_DATA(nid);
5550

5551 5552
			build_zonelists(pgdat);
		}
5553

5554 5555 5556 5557 5558 5559 5560 5561 5562
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
		/*
		 * We now know the "local memory node" for each node--
		 * i.e., the node of the first zone in the generic zonelist.
		 * Set up numa_mem percpu variable for on-line cpus.  During
		 * boot, only the boot cpu should be on-line;  we'll init the
		 * secondary cpus' numa_mem as they come on-line.  During
		 * node/memory hotplug, we'll fixup all on-line cpus.
		 */
5563
		for_each_online_cpu(cpu)
5564
			set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
5565
#endif
5566
	}
5567 5568

	spin_unlock(&lock);
5569 5570
}

5571 5572 5573
static noinline void __init
build_all_zonelists_init(void)
{
5574 5575
	int cpu;

5576
	__build_all_zonelists(NULL);
5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593

	/*
	 * Initialize the boot_pagesets that are going to be used
	 * for bootstrapping processors. The real pagesets for
	 * each zone will be allocated later when the per cpu
	 * allocator is available.
	 *
	 * boot_pagesets are used also for bootstrapping offline
	 * cpus if the system is already booted because the pagesets
	 * are needed to initialize allocators on a specific cpu too.
	 * F.e. the percpu allocator needs the page allocator which
	 * needs the percpu allocator in order to allocate its pagesets
	 * (a chicken-egg dilemma).
	 */
	for_each_possible_cpu(cpu)
		setup_pageset(&per_cpu(boot_pageset, cpu), 0);

5594 5595 5596 5597
	mminit_verify_zonelist();
	cpuset_init_current_mems_allowed();
}

5598 5599
/*
 * unless system_state == SYSTEM_BOOTING.
5600
 *
5601
 * __ref due to call of __init annotated helper build_all_zonelists_init
5602
 * [protected by SYSTEM_BOOTING].
5603
 */
5604
void __ref build_all_zonelists(pg_data_t *pgdat)
5605 5606
{
	if (system_state == SYSTEM_BOOTING) {
5607
		build_all_zonelists_init();
5608
	} else {
5609
		__build_all_zonelists(pgdat);
5610 5611
		/* cpuset refresh routine should be here */
	}
5612
	vm_total_pages = nr_free_pagecache_pages();
5613 5614 5615 5616 5617 5618 5619
	/*
	 * Disable grouping by mobility if the number of pages in the
	 * system is too low to allow the mechanism to work. It would be
	 * more accurate, but expensive to check per-zone. This check is
	 * made on memory-hotadd so a system can start with mobility
	 * disabled and enable it later
	 */
5620
	if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
5621 5622 5623 5624
		page_group_by_mobility_disabled = 1;
	else
		page_group_by_mobility_disabled = 0;

5625
	pr_info("Built %i zonelists, mobility grouping %s.  Total pages: %ld\n",
J
Joe Perches 已提交
5626 5627 5628
		nr_online_nodes,
		page_group_by_mobility_disabled ? "off" : "on",
		vm_total_pages);
5629
#ifdef CONFIG_NUMA
5630
	pr_info("Policy zone: %s\n", zone_names[policy_zone]);
5631
#endif
L
Linus Torvalds 已提交
5632 5633 5634 5635 5636 5637 5638
}

/*
 * Initially all pages are reserved - free ones are freed
 * up by free_all_bootmem() once the early boot process is
 * done. Non-atomic initialization, single-pass.
 */
5639
void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
5640 5641
		unsigned long start_pfn, enum memmap_context context,
		struct vmem_altmap *altmap)
L
Linus Torvalds 已提交
5642
{
A
Andy Whitcroft 已提交
5643
	unsigned long end_pfn = start_pfn + size;
5644
	pg_data_t *pgdat = NODE_DATA(nid);
A
Andy Whitcroft 已提交
5645
	unsigned long pfn;
5646
	unsigned long nr_initialised = 0;
5647
	struct page *page;
5648 5649 5650
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
	struct memblock_region *r = NULL, *tmp;
#endif
L
Linus Torvalds 已提交
5651

5652 5653 5654
	if (highest_memmap_pfn < end_pfn - 1)
		highest_memmap_pfn = end_pfn - 1;

5655 5656 5657 5658 5659 5660 5661
	/*
	 * Honor reservation requested by the driver for this ZONE_DEVICE
	 * memory
	 */
	if (altmap && start_pfn == altmap->base_pfn)
		start_pfn += altmap->reserve;

5662
	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
D
Dave Hansen 已提交
5663
		/*
5664 5665
		 * There can be holes in boot-time mem_map[]s handed to this
		 * function.  They do not exist on hotplugged memory.
D
Dave Hansen 已提交
5666
		 */
5667 5668 5669
		if (context != MEMMAP_EARLY)
			goto not_early;

5670
		if (!early_pfn_valid(pfn))
5671 5672 5673 5674 5675
			continue;
		if (!early_pfn_in_nid(pfn, nid))
			continue;
		if (!update_defer_init(pgdat, pfn, end_pfn, &nr_initialised))
			break;
5676 5677

#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694
		/*
		 * Check given memblock attribute by firmware which can affect
		 * kernel memory layout.  If zone==ZONE_MOVABLE but memory is
		 * mirrored, it's an overlapped memmap init. skip it.
		 */
		if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
			if (!r || pfn >= memblock_region_memory_end_pfn(r)) {
				for_each_memblock(memory, tmp)
					if (pfn < memblock_region_memory_end_pfn(tmp))
						break;
				r = tmp;
			}
			if (pfn >= memblock_region_memory_base_pfn(r) &&
			    memblock_is_mirror(r)) {
				/* already initialized as NORMAL */
				pfn = memblock_region_memory_end_pfn(r);
				continue;
5695
			}
D
Dave Hansen 已提交
5696
		}
5697
#endif
5698

5699
not_early:
5700 5701 5702 5703 5704
		page = pfn_to_page(pfn);
		__init_single_page(page, pfn, zone, nid);
		if (context == MEMMAP_HOTPLUG)
			SetPageReserved(page);

5705 5706 5707 5708 5709
		/*
		 * Mark the block movable so that blocks are reserved for
		 * movable at startup. This will force kernel allocations
		 * to reserve their blocks rather than leaking throughout
		 * the address space during boot when many long-lived
5710
		 * kernel allocations are made.
5711 5712 5713 5714 5715
		 *
		 * bitmap is created for zone's valid pfn range. but memmap
		 * can be created for invalid pages (for alignment)
		 * check here not to call set_pageblock_migratetype() against
		 * pfn out of zone.
5716 5717 5718
		 *
		 * Please note that MEMMAP_HOTPLUG path doesn't clear memmap
		 * because this is done early in sparse_add_one_section
5719 5720 5721
		 */
		if (!(pfn & (pageblock_nr_pages - 1))) {
			set_pageblock_migratetype(page, MIGRATE_MOVABLE);
5722
			cond_resched();
5723
		}
L
Linus Torvalds 已提交
5724 5725 5726
	}
}

5727
static void __meminit zone_init_free_lists(struct zone *zone)
L
Linus Torvalds 已提交
5728
{
5729
	unsigned int order, t;
5730 5731
	for_each_migratetype_order(order, t) {
		INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
L
Linus Torvalds 已提交
5732 5733 5734 5735 5736 5737
		zone->free_area[order].nr_free = 0;
	}
}

#ifndef __HAVE_ARCH_MEMMAP_INIT
#define memmap_init(size, nid, zone, start_pfn) \
5738
	memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY, NULL)
L
Linus Torvalds 已提交
5739 5740
#endif

5741
static int zone_batchsize(struct zone *zone)
5742
{
5743
#ifdef CONFIG_MMU
5744 5745 5746 5747
	int batch;

	/*
	 * The per-cpu-pages pools are set to around 1000th of the
5748
	 * size of the zone.
5749
	 */
5750
	batch = zone->managed_pages / 1024;
5751 5752 5753
	/* But no more than a meg. */
	if (batch * PAGE_SIZE > 1024 * 1024)
		batch = (1024 * 1024) / PAGE_SIZE;
5754 5755 5756 5757 5758
	batch /= 4;		/* We effectively *= 4 below */
	if (batch < 1)
		batch = 1;

	/*
5759 5760 5761
	 * Clamp the batch to a 2^n - 1 value. Having a power
	 * of 2 value was found to be more likely to have
	 * suboptimal cache aliasing properties in some cases.
5762
	 *
5763 5764 5765 5766
	 * For example if 2 tasks are alternately allocating
	 * batches of pages, one task can end up with a lot
	 * of pages of one half of the possible page colors
	 * and the other with pages of the other colors.
5767
	 */
5768
	batch = rounddown_pow_of_two(batch + batch/2) - 1;
5769

5770
	return batch;
5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787

#else
	/* The deferral and batching of frees should be suppressed under NOMMU
	 * conditions.
	 *
	 * The problem is that NOMMU needs to be able to allocate large chunks
	 * of contiguous memory as there's no hardware page translation to
	 * assemble apparent contiguous memory from discontiguous pages.
	 *
	 * Queueing large contiguous runs of pages for batching, however,
	 * causes the pages to actually be freed in smaller chunks.  As there
	 * can be a significant delay between the individual batches being
	 * recycled, this leads to the once large chunks of space being
	 * fragmented and becoming unavailable for high-order allocations.
	 */
	return 0;
#endif
5788 5789
}

5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816
/*
 * pcp->high and pcp->batch values are related and dependent on one another:
 * ->batch must never be higher then ->high.
 * The following function updates them in a safe manner without read side
 * locking.
 *
 * Any new users of pcp->batch and pcp->high should ensure they can cope with
 * those fields changing asynchronously (acording the the above rule).
 *
 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
 * outside of boot time (or some other assurance that no concurrent updaters
 * exist).
 */
static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
		unsigned long batch)
{
       /* start with a fail safe value for batch */
	pcp->batch = 1;
	smp_wmb();

       /* Update high, then batch, in order */
	pcp->high = high;
	smp_wmb();

	pcp->batch = batch;
}

5817
/* a companion to pageset_set_high() */
5818 5819
static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
{
5820
	pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
5821 5822
}

5823
static void pageset_init(struct per_cpu_pageset *p)
5824 5825
{
	struct per_cpu_pages *pcp;
5826
	int migratetype;
5827

5828 5829
	memset(p, 0, sizeof(*p));

5830
	pcp = &p->pcp;
5831
	pcp->count = 0;
5832 5833
	for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
		INIT_LIST_HEAD(&pcp->lists[migratetype]);
5834 5835
}

5836 5837 5838 5839 5840 5841
static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
{
	pageset_init(p);
	pageset_set_batch(p, batch);
}

5842
/*
5843
 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
5844 5845
 * to the value high for the pageset p.
 */
5846
static void pageset_set_high(struct per_cpu_pageset *p,
5847 5848
				unsigned long high)
{
5849 5850 5851
	unsigned long batch = max(1UL, high / 4);
	if ((high / 4) > (PAGE_SHIFT * 8))
		batch = PAGE_SHIFT * 8;
5852

5853
	pageset_update(&p->pcp, high, batch);
5854 5855
}

5856 5857
static void pageset_set_high_and_batch(struct zone *zone,
				       struct per_cpu_pageset *pcp)
5858 5859
{
	if (percpu_pagelist_fraction)
5860
		pageset_set_high(pcp,
5861 5862 5863 5864 5865 5866
			(zone->managed_pages /
				percpu_pagelist_fraction));
	else
		pageset_set_batch(pcp, zone_batchsize(zone));
}

5867 5868 5869 5870 5871 5872 5873 5874
static void __meminit zone_pageset_init(struct zone *zone, int cpu)
{
	struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);

	pageset_init(pcp);
	pageset_set_high_and_batch(zone, pcp);
}

5875
void __meminit setup_zone_pageset(struct zone *zone)
5876 5877 5878
{
	int cpu;
	zone->pageset = alloc_percpu(struct per_cpu_pageset);
5879 5880
	for_each_possible_cpu(cpu)
		zone_pageset_init(zone, cpu);
5881 5882
}

5883
/*
5884 5885
 * Allocate per cpu pagesets and initialize them.
 * Before this call only boot pagesets were available.
5886
 */
5887
void __init setup_per_cpu_pageset(void)
5888
{
5889
	struct pglist_data *pgdat;
5890
	struct zone *zone;
5891

5892 5893
	for_each_populated_zone(zone)
		setup_zone_pageset(zone);
5894 5895 5896 5897

	for_each_online_pgdat(pgdat)
		pgdat->per_cpu_nodestats =
			alloc_percpu(struct per_cpu_nodestat);
5898 5899
}

5900
static __meminit void zone_pcp_init(struct zone *zone)
5901
{
5902 5903 5904 5905 5906 5907
	/*
	 * per cpu subsystem is not up at this point. The following code
	 * relies on the ability of the linker to provide the
	 * offset of a (static) per cpu variable into the per cpu area.
	 */
	zone->pageset = &boot_pageset;
5908

5909
	if (populated_zone(zone))
5910 5911 5912
		printk(KERN_DEBUG "  %s zone: %lu pages, LIFO batch:%u\n",
			zone->name, zone->present_pages,
					 zone_batchsize(zone));
5913 5914
}

5915
void __meminit init_currently_empty_zone(struct zone *zone,
5916
					unsigned long zone_start_pfn,
5917
					unsigned long size)
5918 5919
{
	struct pglist_data *pgdat = zone->zone_pgdat;
5920
	int zone_idx = zone_idx(zone) + 1;
5921

5922 5923
	if (zone_idx > pgdat->nr_zones)
		pgdat->nr_zones = zone_idx;
5924 5925 5926

	zone->zone_start_pfn = zone_start_pfn;

5927 5928 5929 5930 5931 5932
	mminit_dprintk(MMINIT_TRACE, "memmap_init",
			"Initialising map node %d zone %lu pfns %lu -> %lu\n",
			pgdat->node_id,
			(unsigned long)zone_idx(zone),
			zone_start_pfn, (zone_start_pfn + size));

5933
	zone_init_free_lists(zone);
5934
	zone->initialized = 1;
5935 5936
}

T
Tejun Heo 已提交
5937
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5938
#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
5939

5940 5941 5942
/*
 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
 */
5943 5944
int __meminit __early_pfn_to_nid(unsigned long pfn,
					struct mminit_pfnnid_cache *state)
5945
{
5946
	unsigned long start_pfn, end_pfn;
5947
	int nid;
5948

5949 5950
	if (state->last_start <= pfn && pfn < state->last_end)
		return state->last_nid;
5951

5952 5953
	nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
	if (nid != -1) {
5954 5955 5956
		state->last_start = start_pfn;
		state->last_end = end_pfn;
		state->last_nid = nid;
5957 5958 5959
	}

	return nid;
5960 5961 5962 5963
}
#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */

/**
5964
 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range
5965
 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
5966
 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid
5967
 *
5968 5969 5970
 * If an architecture guarantees that all ranges registered contain no holes
 * and may be freed, this this function may be used instead of calling
 * memblock_free_early_nid() manually.
5971
 */
5972
void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
5973
{
5974 5975
	unsigned long start_pfn, end_pfn;
	int i, this_nid;
5976

5977 5978 5979
	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
		start_pfn = min(start_pfn, max_low_pfn);
		end_pfn = min(end_pfn, max_low_pfn);
5980

5981
		if (start_pfn < end_pfn)
5982 5983 5984
			memblock_free_early_nid(PFN_PHYS(start_pfn),
					(end_pfn - start_pfn) << PAGE_SHIFT,
					this_nid);
5985 5986 5987
	}
}

5988 5989
/**
 * sparse_memory_present_with_active_regions - Call memory_present for each active range
5990
 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
5991
 *
5992 5993
 * If an architecture guarantees that all ranges registered contain no holes and may
 * be freed, this function may be used instead of calling memory_present() manually.
5994 5995 5996
 */
void __init sparse_memory_present_with_active_regions(int nid)
{
5997 5998
	unsigned long start_pfn, end_pfn;
	int i, this_nid;
5999

6000 6001
	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
		memory_present(this_nid, start_pfn, end_pfn);
6002 6003 6004 6005
}

/**
 * get_pfn_range_for_nid - Return the start and end page frames for a node
6006 6007 6008
 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
6009 6010
 *
 * It returns the start and end page frame of a node based on information
6011
 * provided by memblock_set_node(). If called for a node
6012
 * with no available memory, a warning is printed and the start and end
6013
 * PFNs will be 0.
6014
 */
6015
void __meminit get_pfn_range_for_nid(unsigned int nid,
6016 6017
			unsigned long *start_pfn, unsigned long *end_pfn)
{
6018
	unsigned long this_start_pfn, this_end_pfn;
6019
	int i;
6020

6021 6022 6023
	*start_pfn = -1UL;
	*end_pfn = 0;

6024 6025 6026
	for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
		*start_pfn = min(*start_pfn, this_start_pfn);
		*end_pfn = max(*end_pfn, this_end_pfn);
6027 6028
	}

6029
	if (*start_pfn == -1UL)
6030 6031 6032
		*start_pfn = 0;
}

M
Mel Gorman 已提交
6033 6034 6035 6036 6037
/*
 * This finds a zone that can be used for ZONE_MOVABLE pages. The
 * assumption is made that zones within a node are ordered in monotonic
 * increasing memory addresses so that the "highest" populated zone is used
 */
A
Adrian Bunk 已提交
6038
static void __init find_usable_zone_for_movable(void)
M
Mel Gorman 已提交
6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055
{
	int zone_index;
	for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
		if (zone_index == ZONE_MOVABLE)
			continue;

		if (arch_zone_highest_possible_pfn[zone_index] >
				arch_zone_lowest_possible_pfn[zone_index])
			break;
	}

	VM_BUG_ON(zone_index == -1);
	movable_zone = zone_index;
}

/*
 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
L
Lucas De Marchi 已提交
6056
 * because it is sized independent of architecture. Unlike the other zones,
M
Mel Gorman 已提交
6057 6058 6059 6060 6061 6062 6063
 * the starting point for ZONE_MOVABLE is not fixed. It may be different
 * in each node depending on the size of each node and how evenly kernelcore
 * is distributed. This helper function adjusts the zone ranges
 * provided by the architecture for a given node by using the end of the
 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
 * zones within a node are in order of monotonic increases memory addresses
 */
A
Adrian Bunk 已提交
6064
static void __meminit adjust_zone_range_for_zone_movable(int nid,
M
Mel Gorman 已提交
6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078
					unsigned long zone_type,
					unsigned long node_start_pfn,
					unsigned long node_end_pfn,
					unsigned long *zone_start_pfn,
					unsigned long *zone_end_pfn)
{
	/* Only adjust if ZONE_MOVABLE is on this node */
	if (zone_movable_pfn[nid]) {
		/* Size ZONE_MOVABLE */
		if (zone_type == ZONE_MOVABLE) {
			*zone_start_pfn = zone_movable_pfn[nid];
			*zone_end_pfn = min(node_end_pfn,
				arch_zone_highest_possible_pfn[movable_zone]);

6079 6080 6081 6082 6083 6084
		/* Adjust for ZONE_MOVABLE starting within this range */
		} else if (!mirrored_kernelcore &&
			*zone_start_pfn < zone_movable_pfn[nid] &&
			*zone_end_pfn > zone_movable_pfn[nid]) {
			*zone_end_pfn = zone_movable_pfn[nid];

M
Mel Gorman 已提交
6085 6086 6087 6088 6089 6090
		/* Check if this whole range is within ZONE_MOVABLE */
		} else if (*zone_start_pfn >= zone_movable_pfn[nid])
			*zone_start_pfn = *zone_end_pfn;
	}
}

6091 6092 6093 6094
/*
 * Return the number of pages a zone spans in a node, including holes
 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
 */
P
Paul Mundt 已提交
6095
static unsigned long __meminit zone_spanned_pages_in_node(int nid,
6096
					unsigned long zone_type,
6097 6098
					unsigned long node_start_pfn,
					unsigned long node_end_pfn,
6099 6100
					unsigned long *zone_start_pfn,
					unsigned long *zone_end_pfn,
6101 6102
					unsigned long *ignored)
{
6103 6104
	unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
	unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
6105
	/* When hotadd a new node from cpu_up(), the node should be empty */
6106 6107 6108
	if (!node_start_pfn && !node_end_pfn)
		return 0;

6109
	/* Get the start and end of the zone */
6110 6111
	*zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
	*zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
M
Mel Gorman 已提交
6112 6113
	adjust_zone_range_for_zone_movable(nid, zone_type,
				node_start_pfn, node_end_pfn,
6114
				zone_start_pfn, zone_end_pfn);
6115 6116

	/* Check that this node has pages within the zone's required range */
6117
	if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
6118 6119 6120
		return 0;

	/* Move the zone boundaries inside the node if necessary */
6121 6122
	*zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
	*zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
6123 6124

	/* Return the spanned pages */
6125
	return *zone_end_pfn - *zone_start_pfn;
6126 6127 6128 6129
}

/*
 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
6130
 * then all holes in the requested range will be accounted for.
6131
 */
6132
unsigned long __meminit __absent_pages_in_range(int nid,
6133 6134 6135
				unsigned long range_start_pfn,
				unsigned long range_end_pfn)
{
6136 6137 6138
	unsigned long nr_absent = range_end_pfn - range_start_pfn;
	unsigned long start_pfn, end_pfn;
	int i;
6139

6140 6141 6142 6143
	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
		start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
		end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
		nr_absent -= end_pfn - start_pfn;
6144
	}
6145
	return nr_absent;
6146 6147 6148 6149 6150 6151 6152
}

/**
 * absent_pages_in_range - Return number of page frames in holes within a range
 * @start_pfn: The start PFN to start searching for holes
 * @end_pfn: The end PFN to stop searching for holes
 *
6153
 * It returns the number of pages frames in memory holes within a range.
6154 6155 6156 6157 6158 6159 6160 6161
 */
unsigned long __init absent_pages_in_range(unsigned long start_pfn,
							unsigned long end_pfn)
{
	return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
}

/* Return the number of page frames in holes in a zone on a node */
P
Paul Mundt 已提交
6162
static unsigned long __meminit zone_absent_pages_in_node(int nid,
6163
					unsigned long zone_type,
6164 6165
					unsigned long node_start_pfn,
					unsigned long node_end_pfn,
6166 6167
					unsigned long *ignored)
{
6168 6169
	unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
	unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
6170
	unsigned long zone_start_pfn, zone_end_pfn;
6171
	unsigned long nr_absent;
6172

6173
	/* When hotadd a new node from cpu_up(), the node should be empty */
6174 6175 6176
	if (!node_start_pfn && !node_end_pfn)
		return 0;

6177 6178
	zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
	zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
6179

M
Mel Gorman 已提交
6180 6181 6182
	adjust_zone_range_for_zone_movable(nid, zone_type,
			node_start_pfn, node_end_pfn,
			&zone_start_pfn, &zone_end_pfn);
6183 6184 6185 6186 6187 6188 6189
	nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);

	/*
	 * ZONE_MOVABLE handling.
	 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
	 * and vice versa.
	 */
6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206
	if (mirrored_kernelcore && zone_movable_pfn[nid]) {
		unsigned long start_pfn, end_pfn;
		struct memblock_region *r;

		for_each_memblock(memory, r) {
			start_pfn = clamp(memblock_region_memory_base_pfn(r),
					  zone_start_pfn, zone_end_pfn);
			end_pfn = clamp(memblock_region_memory_end_pfn(r),
					zone_start_pfn, zone_end_pfn);

			if (zone_type == ZONE_MOVABLE &&
			    memblock_is_mirror(r))
				nr_absent += end_pfn - start_pfn;

			if (zone_type == ZONE_NORMAL &&
			    !memblock_is_mirror(r))
				nr_absent += end_pfn - start_pfn;
6207 6208 6209 6210
		}
	}

	return nr_absent;
6211
}
6212

T
Tejun Heo 已提交
6213
#else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
P
Paul Mundt 已提交
6214
static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
6215
					unsigned long zone_type,
6216 6217
					unsigned long node_start_pfn,
					unsigned long node_end_pfn,
6218 6219
					unsigned long *zone_start_pfn,
					unsigned long *zone_end_pfn,
6220 6221
					unsigned long *zones_size)
{
6222 6223 6224 6225 6226 6227 6228 6229
	unsigned int zone;

	*zone_start_pfn = node_start_pfn;
	for (zone = 0; zone < zone_type; zone++)
		*zone_start_pfn += zones_size[zone];

	*zone_end_pfn = *zone_start_pfn + zones_size[zone_type];

6230 6231 6232
	return zones_size[zone_type];
}

P
Paul Mundt 已提交
6233
static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
6234
						unsigned long zone_type,
6235 6236
						unsigned long node_start_pfn,
						unsigned long node_end_pfn,
6237 6238 6239 6240 6241 6242 6243
						unsigned long *zholes_size)
{
	if (!zholes_size)
		return 0;

	return zholes_size[zone_type];
}
6244

T
Tejun Heo 已提交
6245
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
6246

6247
static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
6248 6249 6250 6251
						unsigned long node_start_pfn,
						unsigned long node_end_pfn,
						unsigned long *zones_size,
						unsigned long *zholes_size)
6252
{
6253
	unsigned long realtotalpages = 0, totalpages = 0;
6254 6255
	enum zone_type i;

6256 6257
	for (i = 0; i < MAX_NR_ZONES; i++) {
		struct zone *zone = pgdat->node_zones + i;
6258
		unsigned long zone_start_pfn, zone_end_pfn;
6259
		unsigned long size, real_size;
6260

6261 6262 6263
		size = zone_spanned_pages_in_node(pgdat->node_id, i,
						  node_start_pfn,
						  node_end_pfn,
6264 6265
						  &zone_start_pfn,
						  &zone_end_pfn,
6266 6267
						  zones_size);
		real_size = size - zone_absent_pages_in_node(pgdat->node_id, i,
6268 6269
						  node_start_pfn, node_end_pfn,
						  zholes_size);
6270 6271 6272 6273
		if (size)
			zone->zone_start_pfn = zone_start_pfn;
		else
			zone->zone_start_pfn = 0;
6274 6275 6276 6277 6278 6279 6280 6281
		zone->spanned_pages = size;
		zone->present_pages = real_size;

		totalpages += size;
		realtotalpages += real_size;
	}

	pgdat->node_spanned_pages = totalpages;
6282 6283 6284 6285 6286
	pgdat->node_present_pages = realtotalpages;
	printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
							realtotalpages);
}

6287 6288 6289
#ifndef CONFIG_SPARSEMEM
/*
 * Calculate the size of the zone->blockflags rounded to an unsigned long
6290 6291
 * Start by making sure zonesize is a multiple of pageblock_order by rounding
 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
6292 6293 6294
 * round what is now in bits to nearest long in bits, then return it in
 * bytes.
 */
6295
static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
6296 6297 6298
{
	unsigned long usemapsize;

6299
	zonesize += zone_start_pfn & (pageblock_nr_pages-1);
6300 6301
	usemapsize = roundup(zonesize, pageblock_nr_pages);
	usemapsize = usemapsize >> pageblock_order;
6302 6303 6304 6305 6306 6307
	usemapsize *= NR_PAGEBLOCK_BITS;
	usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));

	return usemapsize / 8;
}

P
Pavel Tatashin 已提交
6308
static void __ref setup_usemap(struct pglist_data *pgdat,
6309 6310 6311
				struct zone *zone,
				unsigned long zone_start_pfn,
				unsigned long zonesize)
6312
{
6313
	unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
6314
	zone->pageblock_flags = NULL;
6315
	if (usemapsize)
6316 6317 6318
		zone->pageblock_flags =
			memblock_virt_alloc_node_nopanic(usemapsize,
							 pgdat->node_id);
6319 6320
}
#else
6321 6322
static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
				unsigned long zone_start_pfn, unsigned long zonesize) {}
6323 6324
#endif /* CONFIG_SPARSEMEM */

6325
#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
6326

6327
/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
6328
void __init set_pageblock_order(void)
6329
{
6330 6331
	unsigned int order;

6332 6333 6334 6335
	/* Check that pageblock_nr_pages has not already been setup */
	if (pageblock_order)
		return;

6336 6337 6338 6339 6340
	if (HPAGE_SHIFT > PAGE_SHIFT)
		order = HUGETLB_PAGE_ORDER;
	else
		order = MAX_ORDER - 1;

6341 6342
	/*
	 * Assume the largest contiguous order of interest is a huge page.
6343 6344
	 * This value may be variable depending on boot parameters on IA64 and
	 * powerpc.
6345 6346 6347 6348 6349
	 */
	pageblock_order = order;
}
#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */

6350 6351
/*
 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
6352 6353 6354
 * is unused as pageblock_order is set at compile-time. See
 * include/linux/pageblock-flags.h for the values of pageblock_order based on
 * the kernel config
6355
 */
6356
void __init set_pageblock_order(void)
6357 6358
{
}
6359 6360 6361

#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */

6362
static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
P
Pavel Tatashin 已提交
6363
						unsigned long present_pages)
6364 6365 6366 6367 6368 6369 6370 6371
{
	unsigned long pages = spanned_pages;

	/*
	 * Provide a more accurate estimation if there are holes within
	 * the zone and SPARSEMEM is in use. If there are holes within the
	 * zone, each populated memory region may cost us one or two extra
	 * memmap pages due to alignment because memmap pages for each
6372
	 * populated regions may not be naturally aligned on page boundary.
6373 6374 6375 6376 6377 6378 6379 6380 6381
	 * So the (present_pages >> 4) heuristic is a tradeoff for that.
	 */
	if (spanned_pages > present_pages + (present_pages >> 4) &&
	    IS_ENABLED(CONFIG_SPARSEMEM))
		pages = present_pages;

	return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
}

6382 6383 6384
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static void pgdat_init_split_queue(struct pglist_data *pgdat)
{
6385 6386 6387 6388 6389
	struct deferred_split *ds_queue = &pgdat->deferred_split_queue;

	spin_lock_init(&ds_queue->split_queue_lock);
	INIT_LIST_HEAD(&ds_queue->split_queue);
	ds_queue->split_queue_len = 0;
6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403
}
#else
static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
#endif

#ifdef CONFIG_COMPACTION
static void pgdat_init_kcompactd(struct pglist_data *pgdat)
{
	init_waitqueue_head(&pgdat->kcompactd_wait);
}
#else
static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
#endif

6404
static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
L
Linus Torvalds 已提交
6405
{
6406
	pgdat_resize_init(pgdat);
6407 6408 6409 6410

	pgdat_init_split_queue(pgdat);
	pgdat_init_kcompactd(pgdat);

L
Linus Torvalds 已提交
6411
	init_waitqueue_head(&pgdat->kswapd_wait);
6412
	init_waitqueue_head(&pgdat->pfmemalloc_wait);
6413

6414
	pgdat_page_ext_init(pgdat);
6415
	spin_lock_init(&pgdat->lru_lock);
6416
	lruvec_init(node_lruvec(pgdat));
6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462
}

static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
							unsigned long remaining_pages)
{
	zone->managed_pages = remaining_pages;
	zone_set_nid(zone, nid);
	zone->name = zone_names[idx];
	zone->zone_pgdat = NODE_DATA(nid);
	spin_lock_init(&zone->lock);
	zone_seqlock_init(zone);
	zone_pcp_init(zone);
}

/*
 * Set up the zone data structures
 * - init pgdat internals
 * - init all zones belonging to this node
 *
 * NOTE: this function is only called during memory hotplug
 */
#ifdef CONFIG_MEMORY_HOTPLUG
void __ref free_area_init_core_hotplug(int nid)
{
	enum zone_type z;
	pg_data_t *pgdat = NODE_DATA(nid);

	pgdat_init_internals(pgdat);
	for (z = 0; z < MAX_NR_ZONES; z++)
		zone_init_internals(&pgdat->node_zones[z], z, nid, 0);
}
#endif

/*
 * Set up the zone data structures:
 *   - mark all pages reserved
 *   - mark all memory queues empty
 *   - clear the memory bitmaps
 *
 * NOTE: pgdat should get zeroed by caller.
 * NOTE: this function is only called during early init.
 */
static void __init free_area_init_core(struct pglist_data *pgdat)
{
	enum zone_type j;
	int nid = pgdat->node_id;
6463

6464
	pgdat_init_internals(pgdat);
6465 6466
	pgdat->per_cpu_nodestats = &boot_nodestats;

L
Linus Torvalds 已提交
6467 6468
	for (j = 0; j < MAX_NR_ZONES; j++) {
		struct zone *zone = pgdat->node_zones + j;
6469
		unsigned long size, freesize, memmap_pages;
6470
		unsigned long zone_start_pfn = zone->zone_start_pfn;
L
Linus Torvalds 已提交
6471

6472
		size = zone->spanned_pages;
6473
		freesize = zone->present_pages;
L
Linus Torvalds 已提交
6474

6475
		/*
6476
		 * Adjust freesize so that it accounts for how much memory
6477 6478 6479
		 * is used by this zone for memmap. This affects the watermark
		 * and per-cpu initialisations
		 */
6480
		memmap_pages = calc_memmap_size(size, freesize);
6481 6482 6483 6484 6485 6486 6487 6488
		if (!is_highmem_idx(j)) {
			if (freesize >= memmap_pages) {
				freesize -= memmap_pages;
				if (memmap_pages)
					printk(KERN_DEBUG
					       "  %s zone: %lu pages used for memmap\n",
					       zone_names[j], memmap_pages);
			} else
6489
				pr_warn("  %s zone: %lu pages exceeds freesize %lu\n",
6490 6491
					zone_names[j], memmap_pages, freesize);
		}
6492

6493
		/* Account for reserved pages */
6494 6495
		if (j == 0 && freesize > dma_reserve) {
			freesize -= dma_reserve;
Y
Yinghai Lu 已提交
6496
			printk(KERN_DEBUG "  %s zone: %lu pages reserved\n",
6497
					zone_names[0], dma_reserve);
6498 6499
		}

6500
		if (!is_highmem_idx(j))
6501
			nr_kernel_pages += freesize;
6502 6503 6504
		/* Charge for highmem memmap if there are enough kernel pages */
		else if (nr_kernel_pages > memmap_pages * 2)
			nr_kernel_pages -= memmap_pages;
6505
		nr_all_pages += freesize;
L
Linus Torvalds 已提交
6506

6507 6508 6509 6510 6511
		/*
		 * Set an approximate value for lowmem here, it will be adjusted
		 * when the bootmem allocator frees pages into the buddy system.
		 * And all highmem pages will be managed by the buddy system.
		 */
6512
		zone_init_internals(zone, j, nid, freesize);
6513

6514
		if (!size)
L
Linus Torvalds 已提交
6515 6516
			continue;

6517
		set_pageblock_order();
6518 6519
		setup_usemap(pgdat, zone, zone_start_pfn, size);
		init_currently_empty_zone(zone, zone_start_pfn, size);
6520
		memmap_init(size, nid, j, zone_start_pfn);
L
Linus Torvalds 已提交
6521 6522 6523
	}
}

6524
#ifdef CONFIG_FLAT_NODE_MEM_MAP
6525
static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
L
Linus Torvalds 已提交
6526
{
6527
	unsigned long __maybe_unused start = 0;
L
Laura Abbott 已提交
6528 6529
	unsigned long __maybe_unused offset = 0;

L
Linus Torvalds 已提交
6530 6531 6532 6533
	/* Skip empty nodes */
	if (!pgdat->node_spanned_pages)
		return;

6534 6535
	start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
	offset = pgdat->node_start_pfn - start;
L
Linus Torvalds 已提交
6536 6537
	/* ia64 gets its own node_mem_map, before this, without bootmem */
	if (!pgdat->node_mem_map) {
6538
		unsigned long size, end;
A
Andy Whitcroft 已提交
6539 6540
		struct page *map;

6541 6542 6543 6544 6545
		/*
		 * The zone's endpoints aren't required to be MAX_ORDER
		 * aligned but the node_mem_map endpoints must be in order
		 * for the buddy allocator to function correctly.
		 */
6546
		end = pgdat_end_pfn(pgdat);
6547 6548
		end = ALIGN(end, MAX_ORDER_NR_PAGES);
		size =  (end - start) * sizeof(struct page);
6549
		map = memblock_virt_alloc_node_nopanic(size, pgdat->node_id);
L
Laura Abbott 已提交
6550
		pgdat->node_mem_map = map + offset;
L
Linus Torvalds 已提交
6551
	}
6552 6553 6554
	pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
				__func__, pgdat->node_id, (unsigned long)pgdat,
				(unsigned long)pgdat->node_mem_map);
6555
#ifndef CONFIG_NEED_MULTIPLE_NODES
L
Linus Torvalds 已提交
6556 6557 6558
	/*
	 * With no DISCONTIG, the global mem_map is just set as node 0's
	 */
6559
	if (pgdat == NODE_DATA(0)) {
L
Linus Torvalds 已提交
6560
		mem_map = NODE_DATA(0)->node_mem_map;
L
Laura Abbott 已提交
6561
#if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM)
6562
		if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
L
Laura Abbott 已提交
6563
			mem_map -= offset;
T
Tejun Heo 已提交
6564
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
6565
	}
L
Linus Torvalds 已提交
6566 6567
#endif
}
6568 6569 6570
#else
static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { }
#endif /* CONFIG_FLAT_NODE_MEM_MAP */
L
Linus Torvalds 已提交
6571

6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
{
	/*
	 * We start only with one section of pages, more pages are added as
	 * needed until the rest of deferred pages are initialized.
	 */
	pgdat->static_init_pgcnt = min_t(unsigned long, PAGES_PER_SECTION,
						pgdat->node_spanned_pages);
	pgdat->first_deferred_pfn = ULONG_MAX;
}
#else
static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
#endif

6587
void __init free_area_init_node(int nid, unsigned long *zones_size,
P
Pavel Tatashin 已提交
6588 6589
				   unsigned long node_start_pfn,
				   unsigned long *zholes_size)
L
Linus Torvalds 已提交
6590
{
6591
	pg_data_t *pgdat = NODE_DATA(nid);
6592 6593
	unsigned long start_pfn = 0;
	unsigned long end_pfn = 0;
6594

6595
	/* pg_data_t should be reset to zero when it's allocated */
6596
	WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx);
6597

L
Linus Torvalds 已提交
6598 6599
	pgdat->node_id = nid;
	pgdat->node_start_pfn = node_start_pfn;
6600
	pgdat->per_cpu_nodestats = NULL;
6601 6602
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
	get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
6603
	pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
6604 6605
		(u64)start_pfn << PAGE_SHIFT,
		end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
6606 6607
#else
	start_pfn = node_start_pfn;
6608 6609 6610
#endif
	calculate_node_totalpages(pgdat, start_pfn, end_pfn,
				  zones_size, zholes_size);
L
Linus Torvalds 已提交
6611 6612

	alloc_node_mem_map(pgdat);
6613
	pgdat_set_deferred_range(pgdat);
L
Linus Torvalds 已提交
6614

6615
	free_area_init_core(pgdat);
L
Linus Torvalds 已提交
6616 6617
}

6618
#if defined(CONFIG_HAVE_MEMBLOCK) && !defined(CONFIG_FLAT_NODE_MEM_MAP)
6619 6620 6621 6622 6623 6624 6625
/*
 * Only struct pages that are backed by physical memory are zeroed and
 * initialized by going through __init_single_page(). But, there are some
 * struct pages which are reserved in memblock allocator and their fields
 * may be accessed (for example page_to_pfn() on some configuration accesses
 * flags). We must explicitly zero those struct pages.
 */
6626
void __init zero_resv_unavail(void)
6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638
{
	phys_addr_t start, end;
	unsigned long pfn;
	u64 i, pgcnt;

	/*
	 * Loop through ranges that are reserved, but do not have reported
	 * physical memory backing.
	 */
	pgcnt = 0;
	for_each_resv_unavail_range(i, &start, &end) {
		for (pfn = PFN_DOWN(start); pfn < PFN_UP(end); pfn++) {
6639 6640 6641
			if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) {
				pfn = ALIGN_DOWN(pfn, pageblock_nr_pages)
					+ pageblock_nr_pages - 1;
6642
				continue;
6643
			}
6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658
			mm_zero_struct_page(pfn_to_page(pfn));
			pgcnt++;
		}
	}

	/*
	 * Struct pages that do not have backing memory. This could be because
	 * firmware is using some of this memory, or for some other reasons.
	 * Once memblock is changed so such behaviour is not allowed: i.e.
	 * list of "reserved" memory must be a subset of list of "memory", then
	 * this code can be removed.
	 */
	if (pgcnt)
		pr_info("Reserved but unavailable: %lld pages", pgcnt);
}
6659
#endif /* CONFIG_HAVE_MEMBLOCK && !CONFIG_FLAT_NODE_MEM_MAP */
6660

T
Tejun Heo 已提交
6661
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
M
Miklos Szeredi 已提交
6662 6663 6664 6665 6666

#if MAX_NUMNODES > 1
/*
 * Figure out the number of possible node ids.
 */
6667
void __init setup_nr_node_ids(void)
M
Miklos Szeredi 已提交
6668
{
6669
	unsigned int highest;
M
Miklos Szeredi 已提交
6670

6671
	highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
M
Miklos Szeredi 已提交
6672 6673 6674 6675
	nr_node_ids = highest + 1;
}
#endif

6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697
/**
 * node_map_pfn_alignment - determine the maximum internode alignment
 *
 * This function should be called after node map is populated and sorted.
 * It calculates the maximum power of two alignment which can distinguish
 * all the nodes.
 *
 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)).  If the
 * nodes are shifted by 256MiB, 256MiB.  Note that if only the last node is
 * shifted, 1GiB is enough and this function will indicate so.
 *
 * This is used to test whether pfn -> nid mapping of the chosen memory
 * model has fine enough granularity to avoid incorrect mapping for the
 * populated node map.
 *
 * Returns the determined alignment in pfn's.  0 if there is no alignment
 * requirement (single node).
 */
unsigned long __init node_map_pfn_alignment(void)
{
	unsigned long accl_mask = 0, last_end = 0;
6698
	unsigned long start, end, mask;
6699
	int last_nid = -1;
6700
	int i, nid;
6701

6702
	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725
		if (!start || last_nid < 0 || last_nid == nid) {
			last_nid = nid;
			last_end = end;
			continue;
		}

		/*
		 * Start with a mask granular enough to pin-point to the
		 * start pfn and tick off bits one-by-one until it becomes
		 * too coarse to separate the current node from the last.
		 */
		mask = ~((1 << __ffs(start)) - 1);
		while (mask && last_end <= (start & (mask << 1)))
			mask <<= 1;

		/* accumulate all internode masks */
		accl_mask |= mask;
	}

	/* convert mask to number of pages */
	return ~accl_mask + 1;
}

6726
/* Find the lowest pfn for a node */
A
Adrian Bunk 已提交
6727
static unsigned long __init find_min_pfn_for_node(int nid)
6728
{
6729
	unsigned long min_pfn = ULONG_MAX;
6730 6731
	unsigned long start_pfn;
	int i;
6732

6733 6734
	for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
		min_pfn = min(min_pfn, start_pfn);
6735

6736
	if (min_pfn == ULONG_MAX) {
6737
		pr_warn("Could not find start_pfn for node %d\n", nid);
6738 6739 6740 6741
		return 0;
	}

	return min_pfn;
6742 6743 6744 6745 6746 6747
}

/**
 * find_min_pfn_with_active_regions - Find the minimum PFN registered
 *
 * It returns the minimum PFN based on information provided via
6748
 * memblock_set_node().
6749 6750 6751 6752 6753 6754
 */
unsigned long __init find_min_pfn_with_active_regions(void)
{
	return find_min_pfn_for_node(MAX_NUMNODES);
}

6755 6756 6757
/*
 * early_calculate_totalpages()
 * Sum pages in active regions for movable zone.
6758
 * Populate N_MEMORY for calculating usable_nodes.
6759
 */
A
Adrian Bunk 已提交
6760
static unsigned long __init early_calculate_totalpages(void)
6761 6762
{
	unsigned long totalpages = 0;
6763 6764 6765 6766 6767
	unsigned long start_pfn, end_pfn;
	int i, nid;

	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
		unsigned long pages = end_pfn - start_pfn;
6768

6769 6770
		totalpages += pages;
		if (pages)
6771
			node_set_state(nid, N_MEMORY);
6772
	}
6773
	return totalpages;
6774 6775
}

M
Mel Gorman 已提交
6776 6777 6778 6779 6780 6781
/*
 * Find the PFN the Movable zone begins in each node. Kernel memory
 * is spread evenly between nodes as long as the nodes have enough
 * memory. When they don't, some nodes will have more kernelcore than
 * others
 */
6782
static void __init find_zone_movable_pfns_for_nodes(void)
M
Mel Gorman 已提交
6783 6784 6785 6786
{
	int i, nid;
	unsigned long usable_startpfn;
	unsigned long kernelcore_node, kernelcore_remaining;
6787
	/* save the state before borrow the nodemask */
6788
	nodemask_t saved_node_state = node_states[N_MEMORY];
6789
	unsigned long totalpages = early_calculate_totalpages();
6790
	int usable_nodes = nodes_weight(node_states[N_MEMORY]);
E
Emil Medve 已提交
6791
	struct memblock_region *r;
6792 6793 6794 6795 6796 6797 6798 6799 6800

	/* Need to find movable_zone earlier when movable_node is specified. */
	find_usable_zone_for_movable();

	/*
	 * If movable_node is specified, ignore kernelcore and movablecore
	 * options.
	 */
	if (movable_node_is_enabled()) {
E
Emil Medve 已提交
6801 6802
		for_each_memblock(memory, r) {
			if (!memblock_is_hotpluggable(r))
6803 6804
				continue;

E
Emil Medve 已提交
6805
			nid = r->nid;
6806

E
Emil Medve 已提交
6807
			usable_startpfn = PFN_DOWN(r->base);
6808 6809 6810 6811 6812 6813 6814
			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
				min(usable_startpfn, zone_movable_pfn[nid]) :
				usable_startpfn;
		}

		goto out2;
	}
M
Mel Gorman 已提交
6815

6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845
	/*
	 * If kernelcore=mirror is specified, ignore movablecore option
	 */
	if (mirrored_kernelcore) {
		bool mem_below_4gb_not_mirrored = false;

		for_each_memblock(memory, r) {
			if (memblock_is_mirror(r))
				continue;

			nid = r->nid;

			usable_startpfn = memblock_region_memory_base_pfn(r);

			if (usable_startpfn < 0x100000) {
				mem_below_4gb_not_mirrored = true;
				continue;
			}

			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
				min(usable_startpfn, zone_movable_pfn[nid]) :
				usable_startpfn;
		}

		if (mem_below_4gb_not_mirrored)
			pr_warn("This configuration results in unmirrored kernel memory.");

		goto out2;
	}

6846
	/*
6847 6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858
	 * If kernelcore=nn% or movablecore=nn% was specified, calculate the
	 * amount of necessary memory.
	 */
	if (required_kernelcore_percent)
		required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
				       10000UL;
	if (required_movablecore_percent)
		required_movablecore = (totalpages * 100 * required_movablecore_percent) /
					10000UL;

	/*
	 * If movablecore= was specified, calculate what size of
6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873
	 * kernelcore that corresponds so that memory usable for
	 * any allocation type is evenly spread. If both kernelcore
	 * and movablecore are specified, then the value of kernelcore
	 * will be used for required_kernelcore if it's greater than
	 * what movablecore would have allowed.
	 */
	if (required_movablecore) {
		unsigned long corepages;

		/*
		 * Round-up so that ZONE_MOVABLE is at least as large as what
		 * was requested by the user
		 */
		required_movablecore =
			roundup(required_movablecore, MAX_ORDER_NR_PAGES);
6874
		required_movablecore = min(totalpages, required_movablecore);
6875 6876 6877 6878 6879
		corepages = totalpages - required_movablecore;

		required_kernelcore = max(required_kernelcore, corepages);
	}

6880 6881 6882 6883 6884
	/*
	 * If kernelcore was not specified or kernelcore size is larger
	 * than totalpages, there is no ZONE_MOVABLE.
	 */
	if (!required_kernelcore || required_kernelcore >= totalpages)
6885
		goto out;
M
Mel Gorman 已提交
6886 6887 6888 6889 6890 6891 6892

	/* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
	usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];

restart:
	/* Spread kernelcore memory as evenly as possible throughout nodes */
	kernelcore_node = required_kernelcore / usable_nodes;
6893
	for_each_node_state(nid, N_MEMORY) {
6894 6895
		unsigned long start_pfn, end_pfn;

M
Mel Gorman 已提交
6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911
		/*
		 * Recalculate kernelcore_node if the division per node
		 * now exceeds what is necessary to satisfy the requested
		 * amount of memory for the kernel
		 */
		if (required_kernelcore < kernelcore_node)
			kernelcore_node = required_kernelcore / usable_nodes;

		/*
		 * As the map is walked, we track how much memory is usable
		 * by the kernel using kernelcore_remaining. When it is
		 * 0, the rest of the node is usable by ZONE_MOVABLE
		 */
		kernelcore_remaining = kernelcore_node;

		/* Go through each range of PFNs within this node */
6912
		for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
M
Mel Gorman 已提交
6913 6914
			unsigned long size_pages;

6915
			start_pfn = max(start_pfn, zone_movable_pfn[nid]);
M
Mel Gorman 已提交
6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957
			if (start_pfn >= end_pfn)
				continue;

			/* Account for what is only usable for kernelcore */
			if (start_pfn < usable_startpfn) {
				unsigned long kernel_pages;
				kernel_pages = min(end_pfn, usable_startpfn)
								- start_pfn;

				kernelcore_remaining -= min(kernel_pages,
							kernelcore_remaining);
				required_kernelcore -= min(kernel_pages,
							required_kernelcore);

				/* Continue if range is now fully accounted */
				if (end_pfn <= usable_startpfn) {

					/*
					 * Push zone_movable_pfn to the end so
					 * that if we have to rebalance
					 * kernelcore across nodes, we will
					 * not double account here
					 */
					zone_movable_pfn[nid] = end_pfn;
					continue;
				}
				start_pfn = usable_startpfn;
			}

			/*
			 * The usable PFN range for ZONE_MOVABLE is from
			 * start_pfn->end_pfn. Calculate size_pages as the
			 * number of pages used as kernelcore
			 */
			size_pages = end_pfn - start_pfn;
			if (size_pages > kernelcore_remaining)
				size_pages = kernelcore_remaining;
			zone_movable_pfn[nid] = start_pfn + size_pages;

			/*
			 * Some kernelcore has been met, update counts and
			 * break if the kernelcore for this node has been
6958
			 * satisfied
M
Mel Gorman 已提交
6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971
			 */
			required_kernelcore -= min(required_kernelcore,
								size_pages);
			kernelcore_remaining -= size_pages;
			if (!kernelcore_remaining)
				break;
		}
	}

	/*
	 * If there is still required_kernelcore, we do another pass with one
	 * less node in the count. This will push zone_movable_pfn[nid] further
	 * along on the nodes that still have memory until kernelcore is
6972
	 * satisfied
M
Mel Gorman 已提交
6973 6974 6975 6976 6977
	 */
	usable_nodes--;
	if (usable_nodes && required_kernelcore > usable_nodes)
		goto restart;

6978
out2:
M
Mel Gorman 已提交
6979 6980 6981 6982
	/* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
	for (nid = 0; nid < MAX_NUMNODES; nid++)
		zone_movable_pfn[nid] =
			roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
6983

6984
out:
6985
	/* restore the node_state */
6986
	node_states[N_MEMORY] = saved_node_state;
M
Mel Gorman 已提交
6987 6988
}

6989 6990
/* Any regular or high memory on that node ? */
static void check_for_memory(pg_data_t *pgdat, int nid)
6991 6992 6993
{
	enum zone_type zone_type;

6994 6995 6996 6997
	if (N_MEMORY == N_NORMAL_MEMORY)
		return;

	for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
6998
		struct zone *zone = &pgdat->node_zones[zone_type];
6999
		if (populated_zone(zone)) {
7000 7001 7002 7003
			node_set_state(nid, N_HIGH_MEMORY);
			if (N_NORMAL_MEMORY != N_HIGH_MEMORY &&
			    zone_type <= ZONE_NORMAL)
				node_set_state(nid, N_NORMAL_MEMORY);
7004 7005
			break;
		}
7006 7007 7008
	}
}

7009 7010
/**
 * free_area_init_nodes - Initialise all pg_data_t and zone data
7011
 * @max_zone_pfn: an array of max PFNs for each zone
7012 7013
 *
 * This will call free_area_init_node() for each active node in the system.
7014
 * Using the page ranges provided by memblock_set_node(), the size of each
7015 7016 7017 7018 7019 7020 7021 7022 7023
 * zone in each node and their holes is calculated. If the maximum PFN
 * between two adjacent zones match, it is assumed that the zone is empty.
 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
 * starts where the previous one ended. For example, ZONE_DMA32 starts
 * at arch_max_dma_pfn.
 */
void __init free_area_init_nodes(unsigned long *max_zone_pfn)
{
7024 7025
	unsigned long start_pfn, end_pfn;
	int i, nid;
7026

7027 7028 7029 7030 7031
	/* Record where the zone boundaries are */
	memset(arch_zone_lowest_possible_pfn, 0,
				sizeof(arch_zone_lowest_possible_pfn));
	memset(arch_zone_highest_possible_pfn, 0,
				sizeof(arch_zone_highest_possible_pfn));
7032 7033 7034 7035

	start_pfn = find_min_pfn_with_active_regions();

	for (i = 0; i < MAX_NR_ZONES; i++) {
M
Mel Gorman 已提交
7036 7037
		if (i == ZONE_MOVABLE)
			continue;
7038 7039 7040 7041 7042 7043

		end_pfn = max(max_zone_pfn[i], start_pfn);
		arch_zone_lowest_possible_pfn[i] = start_pfn;
		arch_zone_highest_possible_pfn[i] = end_pfn;

		start_pfn = end_pfn;
7044
	}
M
Mel Gorman 已提交
7045 7046 7047

	/* Find the PFNs that ZONE_MOVABLE begins at in each node */
	memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
7048
	find_zone_movable_pfns_for_nodes();
7049 7050

	/* Print out the zone ranges */
7051
	pr_info("Zone ranges:\n");
M
Mel Gorman 已提交
7052 7053 7054
	for (i = 0; i < MAX_NR_ZONES; i++) {
		if (i == ZONE_MOVABLE)
			continue;
7055
		pr_info("  %-8s ", zone_names[i]);
7056 7057
		if (arch_zone_lowest_possible_pfn[i] ==
				arch_zone_highest_possible_pfn[i])
7058
			pr_cont("empty\n");
7059
		else
7060 7061 7062 7063
			pr_cont("[mem %#018Lx-%#018Lx]\n",
				(u64)arch_zone_lowest_possible_pfn[i]
					<< PAGE_SHIFT,
				((u64)arch_zone_highest_possible_pfn[i]
7064
					<< PAGE_SHIFT) - 1);
M
Mel Gorman 已提交
7065 7066 7067
	}

	/* Print out the PFNs ZONE_MOVABLE begins at in each node */
7068
	pr_info("Movable zone start for each node\n");
M
Mel Gorman 已提交
7069 7070
	for (i = 0; i < MAX_NUMNODES; i++) {
		if (zone_movable_pfn[i])
7071 7072
			pr_info("  Node %d: %#018Lx\n", i,
			       (u64)zone_movable_pfn[i] << PAGE_SHIFT);
M
Mel Gorman 已提交
7073
	}
7074

7075
	/* Print out the early node map */
7076
	pr_info("Early memory node ranges\n");
7077
	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
7078 7079 7080
		pr_info("  node %3d: [mem %#018Lx-%#018Lx]\n", nid,
			(u64)start_pfn << PAGE_SHIFT,
			((u64)end_pfn << PAGE_SHIFT) - 1);
7081 7082

	/* Initialise every node */
7083
	mminit_verify_pageflags_layout();
7084
	setup_nr_node_ids();
7085
	zero_resv_unavail();
7086 7087
	for_each_online_node(nid) {
		pg_data_t *pgdat = NODE_DATA(nid);
7088
		free_area_init_node(nid, NULL,
7089
				find_min_pfn_for_node(nid), NULL);
7090 7091 7092

		/* Any memory on that node */
		if (pgdat->node_present_pages)
7093 7094
			node_set_state(nid, N_MEMORY);
		check_for_memory(pgdat, nid);
7095 7096
	}
}
M
Mel Gorman 已提交
7097

7098 7099
static int __init cmdline_parse_core(char *p, unsigned long *core,
				     unsigned long *percent)
M
Mel Gorman 已提交
7100 7101
{
	unsigned long long coremem;
7102 7103
	char *endptr;

M
Mel Gorman 已提交
7104 7105 7106
	if (!p)
		return -EINVAL;

7107 7108 7109 7110 7111
	/* Value may be a percentage of total memory, otherwise bytes */
	coremem = simple_strtoull(p, &endptr, 0);
	if (*endptr == '%') {
		/* Paranoid check for percent values greater than 100 */
		WARN_ON(coremem > 100);
M
Mel Gorman 已提交
7112

7113 7114 7115 7116 7117
		*percent = coremem;
	} else {
		coremem = memparse(p, &p);
		/* Paranoid check that UL is enough for the coremem value */
		WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
M
Mel Gorman 已提交
7118

7119 7120 7121
		*core = coremem >> PAGE_SHIFT;
		*percent = 0UL;
	}
M
Mel Gorman 已提交
7122 7123
	return 0;
}
M
Mel Gorman 已提交
7124

7125 7126 7127 7128 7129 7130
/*
 * kernelcore=size sets the amount of memory for use for allocations that
 * cannot be reclaimed or migrated.
 */
static int __init cmdline_parse_kernelcore(char *p)
{
7131 7132 7133 7134 7135 7136
	/* parse kernelcore=mirror */
	if (parse_option_str(p, "mirror")) {
		mirrored_kernelcore = true;
		return 0;
	}

7137 7138
	return cmdline_parse_core(p, &required_kernelcore,
				  &required_kernelcore_percent);
7139 7140 7141 7142 7143 7144 7145 7146
}

/*
 * movablecore=size sets the amount of memory for use for allocations that
 * can be reclaimed or migrated.
 */
static int __init cmdline_parse_movablecore(char *p)
{
7147 7148
	return cmdline_parse_core(p, &required_movablecore,
				  &required_movablecore_percent);
7149 7150
}

M
Mel Gorman 已提交
7151
early_param("kernelcore", cmdline_parse_kernelcore);
7152
early_param("movablecore", cmdline_parse_movablecore);
M
Mel Gorman 已提交
7153

T
Tejun Heo 已提交
7154
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
7155

7156 7157 7158 7159 7160
void adjust_managed_page_count(struct page *page, long count)
{
	spin_lock(&managed_page_count_lock);
	page_zone(page)->managed_pages += count;
	totalram_pages += count;
7161 7162 7163 7164
#ifdef CONFIG_HIGHMEM
	if (PageHighMem(page))
		totalhigh_pages += count;
#endif
7165 7166
	spin_unlock(&managed_page_count_lock);
}
7167
EXPORT_SYMBOL(adjust_managed_page_count);
7168

7169
unsigned long free_reserved_area(void *start, void *end, int poison, char *s)
7170
{
7171 7172
	void *pos;
	unsigned long pages = 0;
7173

7174 7175 7176
	start = (void *)PAGE_ALIGN((unsigned long)start);
	end = (void *)((unsigned long)end & PAGE_MASK);
	for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187
		struct page *page = virt_to_page(pos);
		void *direct_map_addr;

		/*
		 * 'direct_map_addr' might be different from 'pos'
		 * because some architectures' virt_to_page()
		 * work with aliases.  Getting the direct map
		 * address ensures that we get a _writeable_
		 * alias for the memset().
		 */
		direct_map_addr = page_address(page);
7188
		if ((unsigned int)poison <= 0xFF)
7189 7190 7191
			memset(direct_map_addr, poison, PAGE_SIZE);

		free_reserved_page(page);
7192 7193 7194
	}

	if (pages && s)
7195 7196
		pr_info("Freeing %s memory: %ldK\n",
			s, pages << (PAGE_SHIFT - 10));
7197 7198 7199

	return pages;
}
7200
EXPORT_SYMBOL(free_reserved_area);
7201

7202 7203 7204 7205 7206
#ifdef	CONFIG_HIGHMEM
void free_highmem_page(struct page *page)
{
	__free_reserved_page(page);
	totalram_pages++;
7207
	page_zone(page)->managed_pages++;
7208 7209 7210 7211
	totalhigh_pages++;
}
#endif

7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233

void __init mem_init_print_info(const char *str)
{
	unsigned long physpages, codesize, datasize, rosize, bss_size;
	unsigned long init_code_size, init_data_size;

	physpages = get_num_physpages();
	codesize = _etext - _stext;
	datasize = _edata - _sdata;
	rosize = __end_rodata - __start_rodata;
	bss_size = __bss_stop - __bss_start;
	init_data_size = __init_end - __init_begin;
	init_code_size = _einittext - _sinittext;

	/*
	 * Detect special cases and adjust section sizes accordingly:
	 * 1) .init.* may be embedded into .data sections
	 * 2) .init.text.* may be out of [__init_begin, __init_end],
	 *    please refer to arch/tile/kernel/vmlinux.lds.S.
	 * 3) .rodata.* may be embedded into .text or .data sections.
	 */
#define adj_init_size(start, end, size, pos, adj) \
7234 7235 7236 7237
	do { \
		if (start <= pos && pos < end && size > adj) \
			size -= adj; \
	} while (0)
7238 7239 7240 7241 7242 7243 7244 7245 7246 7247

	adj_init_size(__init_begin, __init_end, init_data_size,
		     _sinittext, init_code_size);
	adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
	adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
	adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
	adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);

#undef	adj_init_size

J
Joe Perches 已提交
7248
	pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
7249
#ifdef	CONFIG_HIGHMEM
J
Joe Perches 已提交
7250
		", %luK highmem"
7251
#endif
J
Joe Perches 已提交
7252 7253 7254 7255 7256 7257 7258
		"%s%s)\n",
		nr_free_pages() << (PAGE_SHIFT - 10),
		physpages << (PAGE_SHIFT - 10),
		codesize >> 10, datasize >> 10, rosize >> 10,
		(init_data_size + init_code_size) >> 10, bss_size >> 10,
		(physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10),
		totalcma_pages << (PAGE_SHIFT - 10),
7259
#ifdef	CONFIG_HIGHMEM
J
Joe Perches 已提交
7260
		totalhigh_pages << (PAGE_SHIFT - 10),
7261
#endif
J
Joe Perches 已提交
7262
		str ? ", " : "", str ? str : "");
7263 7264
}

7265
/**
7266 7267
 * set_dma_reserve - set the specified number of pages reserved in the first zone
 * @new_dma_reserve: The number of pages to mark reserved
7268
 *
7269
 * The per-cpu batchsize and zone watermarks are determined by managed_pages.
7270 7271
 * In the DMA zone, a significant percentage may be consumed by kernel image
 * and other unfreeable allocations which can skew the watermarks badly. This
7272 7273 7274
 * function may optionally be used to account for unfreeable pages in the
 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
 * smaller per-cpu batchsize.
7275 7276 7277 7278 7279 7280
 */
void __init set_dma_reserve(unsigned long new_dma_reserve)
{
	dma_reserve = new_dma_reserve;
}

L
Linus Torvalds 已提交
7281 7282
void __init free_area_init(unsigned long *zones_size)
{
7283
	zero_resv_unavail();
7284
	free_area_init_node(0, zones_size,
L
Linus Torvalds 已提交
7285 7286 7287
			__pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
}

7288
static int page_alloc_cpu_dead(unsigned int cpu)
L
Linus Torvalds 已提交
7289 7290
{

7291 7292
	lru_add_drain_cpu(cpu);
	drain_pages(cpu);
7293

7294 7295 7296 7297 7298 7299 7300
	/*
	 * Spill the event counters of the dead processor
	 * into the current processors event counters.
	 * This artificially elevates the count of the current
	 * processor.
	 */
	vm_events_fold_cpu(cpu);
7301

7302 7303 7304 7305 7306 7307 7308 7309 7310
	/*
	 * Zero the differential counters of the dead processor
	 * so that the vm statistics are consistent.
	 *
	 * This is only okay since the processor is dead and cannot
	 * race with what we are doing.
	 */
	cpu_vm_stats_fold(cpu);
	return 0;
L
Linus Torvalds 已提交
7311 7312 7313 7314
}

void __init page_alloc_init(void)
{
7315 7316 7317 7318 7319 7320
	int ret;

	ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
					"mm/page_alloc:dead", NULL,
					page_alloc_cpu_dead);
	WARN_ON(ret < 0);
L
Linus Torvalds 已提交
7321 7322
}

7323
/*
7324
 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
7325 7326 7327 7328 7329 7330
 *	or min_free_kbytes changes.
 */
static void calculate_totalreserve_pages(void)
{
	struct pglist_data *pgdat;
	unsigned long reserve_pages = 0;
7331
	enum zone_type i, j;
7332 7333

	for_each_online_pgdat(pgdat) {
7334 7335 7336

		pgdat->totalreserve_pages = 0;

7337 7338
		for (i = 0; i < MAX_NR_ZONES; i++) {
			struct zone *zone = pgdat->node_zones + i;
7339
			long max = 0;
7340 7341 7342 7343 7344 7345 7346

			/* Find valid and maximum lowmem_reserve in the zone */
			for (j = i; j < MAX_NR_ZONES; j++) {
				if (zone->lowmem_reserve[j] > max)
					max = zone->lowmem_reserve[j];
			}

7347 7348
			/* we treat the high watermark as reserved pages. */
			max += high_wmark_pages(zone);
7349

7350 7351
			if (max > zone->managed_pages)
				max = zone->managed_pages;
7352

7353
			pgdat->totalreserve_pages += max;
7354

7355 7356 7357 7358 7359 7360
			reserve_pages += max;
		}
	}
	totalreserve_pages = reserve_pages;
}

L
Linus Torvalds 已提交
7361 7362
/*
 * setup_per_zone_lowmem_reserve - called whenever
7363
 *	sysctl_lowmem_reserve_ratio changes.  Ensures that each zone
L
Linus Torvalds 已提交
7364 7365 7366 7367 7368 7369
 *	has a correct pages reserved value, so an adequate number of
 *	pages are left in the zone after a successful __alloc_pages().
 */
static void setup_per_zone_lowmem_reserve(void)
{
	struct pglist_data *pgdat;
7370
	enum zone_type j, idx;
L
Linus Torvalds 已提交
7371

7372
	for_each_online_pgdat(pgdat) {
L
Linus Torvalds 已提交
7373 7374
		for (j = 0; j < MAX_NR_ZONES; j++) {
			struct zone *zone = pgdat->node_zones + j;
7375
			unsigned long managed_pages = zone->managed_pages;
L
Linus Torvalds 已提交
7376 7377 7378

			zone->lowmem_reserve[j] = 0;

7379 7380
			idx = j;
			while (idx) {
L
Linus Torvalds 已提交
7381 7382
				struct zone *lower_zone;

7383
				idx--;
L
Linus Torvalds 已提交
7384
				lower_zone = pgdat->node_zones + idx;
7385 7386 7387 7388 7389 7390 7391 7392

				if (sysctl_lowmem_reserve_ratio[idx] < 1) {
					sysctl_lowmem_reserve_ratio[idx] = 0;
					lower_zone->lowmem_reserve[j] = 0;
				} else {
					lower_zone->lowmem_reserve[j] =
						managed_pages / sysctl_lowmem_reserve_ratio[idx];
				}
7393
				managed_pages += lower_zone->managed_pages;
L
Linus Torvalds 已提交
7394 7395 7396
			}
		}
	}
7397 7398 7399

	/* update totalreserve_pages */
	calculate_totalreserve_pages();
L
Linus Torvalds 已提交
7400 7401
}

7402
static void __setup_per_zone_wmarks(void)
L
Linus Torvalds 已提交
7403 7404 7405 7406 7407 7408 7409 7410 7411
{
	unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
	unsigned long lowmem_pages = 0;
	struct zone *zone;
	unsigned long flags;

	/* Calculate total number of !ZONE_HIGHMEM pages */
	for_each_zone(zone) {
		if (!is_highmem(zone))
7412
			lowmem_pages += zone->managed_pages;
L
Linus Torvalds 已提交
7413 7414 7415
	}

	for_each_zone(zone) {
7416 7417
		u64 tmp;

7418
		spin_lock_irqsave(&zone->lock, flags);
7419
		tmp = (u64)pages_min * zone->managed_pages;
7420
		do_div(tmp, lowmem_pages);
L
Linus Torvalds 已提交
7421 7422
		if (is_highmem(zone)) {
			/*
N
Nick Piggin 已提交
7423 7424 7425 7426
			 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
			 * need highmem pages, so cap pages_min to a small
			 * value here.
			 *
7427
			 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
Y
Yaowei Bai 已提交
7428
			 * deltas control asynch page reclaim, and so should
N
Nick Piggin 已提交
7429
			 * not be capped for highmem.
L
Linus Torvalds 已提交
7430
			 */
7431
			unsigned long min_pages;
L
Linus Torvalds 已提交
7432

7433
			min_pages = zone->managed_pages / 1024;
7434
			min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
7435
			zone->_watermark[WMARK_MIN] = min_pages;
L
Linus Torvalds 已提交
7436
		} else {
N
Nick Piggin 已提交
7437 7438
			/*
			 * If it's a lowmem zone, reserve a number of pages
L
Linus Torvalds 已提交
7439 7440
			 * proportionate to the zone's size.
			 */
7441
			zone->_watermark[WMARK_MIN] = tmp;
L
Linus Torvalds 已提交
7442 7443
		}

7444 7445 7446 7447 7448 7449 7450 7451 7452
		/*
		 * Set the kswapd watermarks distance according to the
		 * scale factor in proportion to available memory, but
		 * ensure a minimum size on small systems.
		 */
		tmp = max_t(u64, tmp >> 2,
			    mult_frac(zone->managed_pages,
				      watermark_scale_factor, 10000));

7453 7454
		zone->_watermark[WMARK_LOW]  = min_wmark_pages(zone) + tmp;
		zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
7455

7456
		spin_unlock_irqrestore(&zone->lock, flags);
L
Linus Torvalds 已提交
7457
	}
7458 7459 7460

	/* update totalreserve_pages */
	calculate_totalreserve_pages();
L
Linus Torvalds 已提交
7461 7462
}

7463 7464 7465 7466 7467 7468 7469 7470 7471
/**
 * setup_per_zone_wmarks - called when min_free_kbytes changes
 * or when memory is hot-{added|removed}
 *
 * Ensures that the watermark[min,low,high] values for each zone are set
 * correctly with respect to min_free_kbytes.
 */
void setup_per_zone_wmarks(void)
{
7472 7473 7474
	static DEFINE_SPINLOCK(lock);

	spin_lock(&lock);
7475
	__setup_per_zone_wmarks();
7476
	spin_unlock(&lock);
7477 7478
}

L
Linus Torvalds 已提交
7479 7480 7481 7482 7483 7484 7485
/*
 * Initialise min_free_kbytes.
 *
 * For small machines we want it small (128k min).  For large machines
 * we want it large (64MB max).  But it is not linear, because network
 * bandwidth does not increase linearly with machine size.  We use
 *
7486
 *	min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
L
Linus Torvalds 已提交
7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498 7499 7500 7501 7502
 *	min_free_kbytes = sqrt(lowmem_kbytes * 16)
 *
 * which yields
 *
 * 16MB:	512k
 * 32MB:	724k
 * 64MB:	1024k
 * 128MB:	1448k
 * 256MB:	2048k
 * 512MB:	2896k
 * 1024MB:	4096k
 * 2048MB:	5792k
 * 4096MB:	8192k
 * 8192MB:	11584k
 * 16384MB:	16384k
 */
7503
int __meminit init_per_zone_wmark_min(void)
L
Linus Torvalds 已提交
7504 7505
{
	unsigned long lowmem_kbytes;
7506
	int new_min_free_kbytes;
L
Linus Torvalds 已提交
7507 7508

	lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520
	new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);

	if (new_min_free_kbytes > user_min_free_kbytes) {
		min_free_kbytes = new_min_free_kbytes;
		if (min_free_kbytes < 128)
			min_free_kbytes = 128;
		if (min_free_kbytes > 65536)
			min_free_kbytes = 65536;
	} else {
		pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
				new_min_free_kbytes, user_min_free_kbytes);
	}
7521
	setup_per_zone_wmarks();
7522
	refresh_zone_stat_thresholds();
L
Linus Torvalds 已提交
7523
	setup_per_zone_lowmem_reserve();
7524 7525 7526 7527 7528 7529

#ifdef CONFIG_NUMA
	setup_min_unmapped_ratio();
	setup_min_slab_ratio();
#endif

L
Linus Torvalds 已提交
7530 7531
	return 0;
}
7532
core_initcall(init_per_zone_wmark_min)
L
Linus Torvalds 已提交
7533 7534

/*
7535
 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
L
Linus Torvalds 已提交
7536 7537 7538
 *	that we can call two helper functions whenever min_free_kbytes
 *	changes.
 */
7539
int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
7540
	void __user *buffer, size_t *length, loff_t *ppos)
L
Linus Torvalds 已提交
7541
{
7542 7543 7544 7545 7546 7547
	int rc;

	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
	if (rc)
		return rc;

7548 7549
	if (write) {
		user_min_free_kbytes = min_free_kbytes;
7550
		setup_per_zone_wmarks();
7551
	}
L
Linus Torvalds 已提交
7552 7553 7554
	return 0;
}

7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569
int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
	void __user *buffer, size_t *length, loff_t *ppos)
{
	int rc;

	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
	if (rc)
		return rc;

	if (write)
		setup_per_zone_wmarks();

	return 0;
}

7570
#ifdef CONFIG_NUMA
7571
static void setup_min_unmapped_ratio(void)
7572
{
7573
	pg_data_t *pgdat;
7574 7575
	struct zone *zone;

7576
	for_each_online_pgdat(pgdat)
7577
		pgdat->min_unmapped_pages = 0;
7578

7579
	for_each_zone(zone)
7580
		zone->zone_pgdat->min_unmapped_pages += (zone->managed_pages *
7581 7582
				sysctl_min_unmapped_ratio) / 100;
}
7583

7584 7585

int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
7586
	void __user *buffer, size_t *length, loff_t *ppos)
7587 7588 7589
{
	int rc;

7590
	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7591 7592 7593
	if (rc)
		return rc;

7594 7595 7596 7597 7598 7599 7600 7601 7602 7603
	setup_min_unmapped_ratio();

	return 0;
}

static void setup_min_slab_ratio(void)
{
	pg_data_t *pgdat;
	struct zone *zone;

7604 7605 7606
	for_each_online_pgdat(pgdat)
		pgdat->min_slab_pages = 0;

7607
	for_each_zone(zone)
7608
		zone->zone_pgdat->min_slab_pages += (zone->managed_pages *
7609
				sysctl_min_slab_ratio) / 100;
7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622
}

int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
	void __user *buffer, size_t *length, loff_t *ppos)
{
	int rc;

	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
	if (rc)
		return rc;

	setup_min_slab_ratio();

7623 7624
	return 0;
}
7625 7626
#endif

L
Linus Torvalds 已提交
7627 7628 7629 7630 7631 7632
/*
 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
 *	proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
 *	whenever sysctl_lowmem_reserve_ratio changes.
 *
 * The reserve ratio obviously has absolutely no relation with the
7633
 * minimum watermarks. The lowmem reserve ratio can only make sense
L
Linus Torvalds 已提交
7634 7635
 * if in function of the boot time zone sizes.
 */
7636
int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
7637
	void __user *buffer, size_t *length, loff_t *ppos)
L
Linus Torvalds 已提交
7638
{
7639
	proc_dointvec_minmax(table, write, buffer, length, ppos);
L
Linus Torvalds 已提交
7640 7641 7642 7643
	setup_per_zone_lowmem_reserve();
	return 0;
}

7644 7645
/*
 * percpu_pagelist_fraction - changes the pcp->high for each zone on each
7646 7647
 * cpu.  It is the fraction of total pages in each zone that a hot per cpu
 * pagelist can have before it gets flushed back to buddy allocator.
7648
 */
7649
int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
7650
	void __user *buffer, size_t *length, loff_t *ppos)
7651 7652
{
	struct zone *zone;
7653
	int old_percpu_pagelist_fraction;
7654 7655
	int ret;

7656 7657 7658
	mutex_lock(&pcp_batch_high_lock);
	old_percpu_pagelist_fraction = percpu_pagelist_fraction;

7659
	ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673
	if (!write || ret < 0)
		goto out;

	/* Sanity checking to avoid pcp imbalance */
	if (percpu_pagelist_fraction &&
	    percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
		percpu_pagelist_fraction = old_percpu_pagelist_fraction;
		ret = -EINVAL;
		goto out;
	}

	/* No change? */
	if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
		goto out;
7674

7675
	for_each_populated_zone(zone) {
7676 7677
		unsigned int cpu;

7678
		for_each_possible_cpu(cpu)
7679 7680
			pageset_set_high_and_batch(zone,
					per_cpu_ptr(zone->pageset, cpu));
7681
	}
7682
out:
7683
	mutex_unlock(&pcp_batch_high_lock);
7684
	return ret;
7685 7686
}

7687
#ifdef CONFIG_NUMA
7688
int hashdist = HASHDIST_DEFAULT;
L
Linus Torvalds 已提交
7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699

static int __init set_hashdist(char *str)
{
	if (!str)
		return 0;
	hashdist = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("hashdist=", set_hashdist);
#endif

7700 7701 7702 7703 7704 7705 7706 7707 7708 7709 7710
#ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
/*
 * Returns the number of pages that arch has reserved but
 * is not known to alloc_large_system_hash().
 */
static unsigned long __init arch_reserved_kernel_pages(void)
{
	return 0;
}
#endif

P
Pavel Tatashin 已提交
7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723 7724 7725
/*
 * Adaptive scale is meant to reduce sizes of hash tables on large memory
 * machines. As memory size is increased the scale is also increased but at
 * slower pace.  Starting from ADAPT_SCALE_BASE (64G), every time memory
 * quadruples the scale is increased by one, which means the size of hash table
 * only doubles, instead of quadrupling as well.
 * Because 32-bit systems cannot have large physical memory, where this scaling
 * makes sense, it is disabled on such platforms.
 */
#if __BITS_PER_LONG > 32
#define ADAPT_SCALE_BASE	(64ul << 30)
#define ADAPT_SCALE_SHIFT	2
#define ADAPT_SCALE_NPAGES	(ADAPT_SCALE_BASE >> PAGE_SHIFT)
#endif

L
Linus Torvalds 已提交
7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738
/*
 * allocate a large system hash table from bootmem
 * - it is assumed that the hash table must contain an exact power-of-2
 *   quantity of entries
 * - limit is the number of hash buckets, not the total allocation size
 */
void *__init alloc_large_system_hash(const char *tablename,
				     unsigned long bucketsize,
				     unsigned long numentries,
				     int scale,
				     int flags,
				     unsigned int *_hash_shift,
				     unsigned int *_hash_mask,
7739 7740
				     unsigned long low_limit,
				     unsigned long high_limit)
L
Linus Torvalds 已提交
7741
{
7742
	unsigned long long max = high_limit;
L
Linus Torvalds 已提交
7743 7744
	unsigned long log2qty, size;
	void *table = NULL;
7745
	gfp_t gfp_flags;
L
Linus Torvalds 已提交
7746 7747 7748 7749

	/* allow the kernel cmdline to have a say */
	if (!numentries) {
		/* round applicable memory size up to nearest megabyte */
A
Andrew Morton 已提交
7750
		numentries = nr_kernel_pages;
7751
		numentries -= arch_reserved_kernel_pages();
7752 7753 7754 7755

		/* It isn't necessary when PAGE_SIZE >= 1MB */
		if (PAGE_SHIFT < 20)
			numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
L
Linus Torvalds 已提交
7756

P
Pavel Tatashin 已提交
7757 7758 7759 7760 7761 7762 7763 7764 7765 7766
#if __BITS_PER_LONG > 32
		if (!high_limit) {
			unsigned long adapt;

			for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
			     adapt <<= ADAPT_SCALE_SHIFT)
				scale++;
		}
#endif

L
Linus Torvalds 已提交
7767 7768 7769 7770 7771
		/* limit to 1 bucket per 2^scale bytes of low memory */
		if (scale > PAGE_SHIFT)
			numentries >>= (scale - PAGE_SHIFT);
		else
			numentries <<= (PAGE_SHIFT - scale);
7772 7773

		/* Make sure we've got at least a 0-order allocation.. */
7774 7775 7776 7777 7778 7779 7780 7781
		if (unlikely(flags & HASH_SMALL)) {
			/* Makes no sense without HASH_EARLY */
			WARN_ON(!(flags & HASH_EARLY));
			if (!(numentries >> *_hash_shift)) {
				numentries = 1UL << *_hash_shift;
				BUG_ON(!numentries);
			}
		} else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
7782
			numentries = PAGE_SIZE / bucketsize;
L
Linus Torvalds 已提交
7783
	}
7784
	numentries = roundup_pow_of_two(numentries);
L
Linus Torvalds 已提交
7785 7786 7787 7788 7789 7790

	/* limit allocation size to 1/16 total memory by default */
	if (max == 0) {
		max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
		do_div(max, bucketsize);
	}
7791
	max = min(max, 0x80000000ULL);
L
Linus Torvalds 已提交
7792

7793 7794
	if (numentries < low_limit)
		numentries = low_limit;
L
Linus Torvalds 已提交
7795 7796 7797
	if (numentries > max)
		numentries = max;

7798
	log2qty = ilog2(numentries);
L
Linus Torvalds 已提交
7799

7800
	gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
L
Linus Torvalds 已提交
7801 7802
	do {
		size = bucketsize << log2qty;
7803 7804 7805 7806 7807 7808
		if (flags & HASH_EARLY) {
			if (flags & HASH_ZERO)
				table = memblock_virt_alloc_nopanic(size, 0);
			else
				table = memblock_virt_alloc_raw(size, 0);
		} else if (hashdist) {
7809
			table = __vmalloc(size, gfp_flags, PAGE_KERNEL);
7810
		} else {
7811 7812
			/*
			 * If bucketsize is not a power-of-two, we may free
7813 7814
			 * some pages at the end of hash table which
			 * alloc_pages_exact() automatically does
7815
			 */
7816
			if (get_order(size) < MAX_ORDER) {
7817 7818
				table = alloc_pages_exact(size, gfp_flags);
				kmemleak_alloc(table, size, 1, gfp_flags);
7819
			}
L
Linus Torvalds 已提交
7820 7821 7822 7823 7824 7825
		}
	} while (!table && size > PAGE_SIZE && --log2qty);

	if (!table)
		panic("Failed to allocate %s hash table\n", tablename);

7826 7827
	pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n",
		tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size);
L
Linus Torvalds 已提交
7828 7829 7830 7831 7832 7833 7834 7835

	if (_hash_shift)
		*_hash_shift = log2qty;
	if (_hash_mask)
		*_hash_mask = (1 << log2qty) - 1;

	return table;
}
7836

K
KAMEZAWA Hiroyuki 已提交
7837
/*
7838 7839 7840
 * This function checks whether pageblock includes unmovable pages or not.
 * If @count is not zero, it is okay to include less @count unmovable pages
 *
7841
 * PageLRU check without isolation or lru_lock could race so that
7842 7843 7844
 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable
 * check without lock_page also may miss some movable non-lru pages at
 * race condition. So you can't expect this function should be exact.
K
KAMEZAWA Hiroyuki 已提交
7845
 */
7846
bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
7847
			 int migratetype,
7848
			 bool skip_hwpoisoned_pages)
7849 7850
{
	unsigned long pfn, iter, found;
7851

7852
	/*
7853 7854 7855 7856 7857
	 * TODO we could make this much more efficient by not checking every
	 * page in the range if we know all of them are in MOVABLE_ZONE and
	 * that the movable zone guarantees that pages are migratable but
	 * the later is not the case right now unfortunatelly. E.g. movablecore
	 * can still lead to having bootmem allocations in zone_movable.
7858 7859
	 */

7860 7861 7862 7863 7864 7865 7866 7867 7868
	/*
	 * CMA allocations (alloc_contig_range) really need to mark isolate
	 * CMA pageblocks even when they are not movable in fact so consider
	 * them movable here.
	 */
	if (is_migrate_cma(migratetype) &&
			is_migrate_cma(get_pageblock_migratetype(page)))
		return false;

7869 7870 7871 7872
	pfn = page_to_pfn(page);
	for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
		unsigned long check = pfn + iter;

7873
		if (!pfn_valid_within(check))
7874
			continue;
7875

7876
		page = pfn_to_page(check);
7877

7878
		if (PageReserved(page))
7879
			goto unmovable;
7880

7881 7882 7883 7884 7885 7886 7887 7888
		/*
		 * If the zone is movable and we have ruled out all reserved
		 * pages then it should be reasonably safe to assume the rest
		 * is movable.
		 */
		if (zone_idx(zone) == ZONE_MOVABLE)
			continue;

7889 7890 7891 7892 7893 7894
		/*
		 * Hugepages are not in LRU lists, but they're movable.
		 * We need not scan over tail pages bacause we don't
		 * handle each tail page individually in migration.
		 */
		if (PageHuge(page)) {
7895 7896
			struct page *head = compound_head(page);
			unsigned int skip_pages;
7897

7898
			if (!hugepage_migration_supported(page_hstate(head)))
7899 7900
				goto unmovable;

7901 7902
			skip_pages = (1 << compound_order(head)) - (page - head);
			iter += skip_pages - 1;
7903 7904 7905
			continue;
		}

7906 7907 7908 7909
		/*
		 * We can't use page_count without pin a page
		 * because another CPU can free compound page.
		 * This check already skips compound tails of THP
7910
		 * because their page->_refcount is zero at all time.
7911
		 */
7912
		if (!page_ref_count(page)) {
7913 7914 7915 7916
			if (PageBuddy(page))
				iter += (1 << page_order(page)) - 1;
			continue;
		}
7917

7918 7919 7920 7921 7922 7923 7924
		/*
		 * The HWPoisoned page may be not in buddy system, and
		 * page_count() is not 0.
		 */
		if (skip_hwpoisoned_pages && PageHWPoison(page))
			continue;

7925 7926 7927
		if (__PageMovable(page))
			continue;

7928 7929 7930
		if (!PageLRU(page))
			found++;
		/*
7931 7932 7933
		 * If there are RECLAIMABLE pages, we need to check
		 * it.  But now, memory offline itself doesn't call
		 * shrink_node_slabs() and it still to be fixed.
7934 7935 7936 7937 7938 7939 7940 7941 7942 7943
		 */
		/*
		 * If the page is not RAM, page_count()should be 0.
		 * we don't need more check. This is an _used_ not-movable page.
		 *
		 * The problematic thing here is PG_reserved pages. PG_reserved
		 * is set to both of a memory hole page and a _used_ kernel
		 * page at boot.
		 */
		if (found > count)
7944
			goto unmovable;
7945
	}
7946
	return false;
7947 7948 7949
unmovable:
	WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE);
	return true;
7950 7951
}

7952
#if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)
7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966

static unsigned long pfn_max_align_down(unsigned long pfn)
{
	return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
			     pageblock_nr_pages) - 1);
}

static unsigned long pfn_max_align_up(unsigned long pfn)
{
	return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
				pageblock_nr_pages));
}

/* [start, end) must belong to a single zone. */
7967 7968
static int __alloc_contig_migrate_range(struct compact_control *cc,
					unsigned long start, unsigned long end)
7969 7970
{
	/* This function is based on compact_zone() from compaction.c. */
7971
	unsigned long nr_reclaimed;
7972 7973 7974 7975
	unsigned long pfn = start;
	unsigned int tries = 0;
	int ret = 0;

7976
	migrate_prep();
7977

7978
	while (pfn < end || !list_empty(&cc->migratepages)) {
7979 7980 7981 7982 7983
		if (fatal_signal_pending(current)) {
			ret = -EINTR;
			break;
		}

7984 7985
		if (list_empty(&cc->migratepages)) {
			cc->nr_migratepages = 0;
7986
			pfn = isolate_migratepages_range(cc, pfn, end);
7987 7988 7989 7990 7991 7992 7993 7994 7995 7996
			if (!pfn) {
				ret = -EINTR;
				break;
			}
			tries = 0;
		} else if (++tries == 5) {
			ret = ret < 0 ? ret : -EBUSY;
			break;
		}

7997 7998 7999
		nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
							&cc->migratepages);
		cc->nr_migratepages -= nr_reclaimed;
8000

8001
		ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
8002
				    NULL, 0, cc->mode, MR_CONTIG_RANGE);
8003
	}
8004 8005 8006 8007 8008
	if (ret < 0) {
		putback_movable_pages(&cc->migratepages);
		return ret;
	}
	return 0;
8009 8010 8011 8012 8013 8014
}

/**
 * alloc_contig_range() -- tries to allocate given range of pages
 * @start:	start PFN to allocate
 * @end:	one-past-the-last PFN to allocate
8015 8016 8017 8018
 * @migratetype:	migratetype of the underlaying pageblocks (either
 *			#MIGRATE_MOVABLE or #MIGRATE_CMA).  All pageblocks
 *			in range must have the same migratetype and it must
 *			be either of the two.
8019
 * @gfp_mask:	GFP mask to use during compaction
8020 8021
 *
 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
8022
 * aligned.  The PFN range must belong to a single zone.
8023
 *
8024 8025 8026
 * The first thing this routine does is attempt to MIGRATE_ISOLATE all
 * pageblocks in the range.  Once isolated, the pageblocks should not
 * be modified by others.
8027 8028 8029 8030 8031
 *
 * Returns zero on success or negative error code.  On success all
 * pages which PFN is in [start, end) are allocated for the caller and
 * need to be freed with free_contig_range().
 */
8032
int alloc_contig_range(unsigned long start, unsigned long end,
8033
		       unsigned migratetype, gfp_t gfp_mask)
8034 8035
{
	unsigned long outer_start, outer_end;
8036 8037
	unsigned int order;
	int ret = 0;
8038

8039 8040 8041 8042
	struct compact_control cc = {
		.nr_migratepages = 0,
		.order = -1,
		.zone = page_zone(pfn_to_page(start)),
8043
		.mode = MIGRATE_SYNC,
8044
		.ignore_skip_hint = true,
8045
		.no_set_skip_hint = true,
8046
		.gfp_mask = current_gfp_context(gfp_mask),
8047 8048 8049
	};
	INIT_LIST_HEAD(&cc.migratepages);

8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074
	/*
	 * What we do here is we mark all pageblocks in range as
	 * MIGRATE_ISOLATE.  Because pageblock and max order pages may
	 * have different sizes, and due to the way page allocator
	 * work, we align the range to biggest of the two pages so
	 * that page allocator won't try to merge buddies from
	 * different pageblocks and change MIGRATE_ISOLATE to some
	 * other migration type.
	 *
	 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
	 * migrate the pages from an unaligned range (ie. pages that
	 * we are interested in).  This will put all the pages in
	 * range back to page allocator as MIGRATE_ISOLATE.
	 *
	 * When this is done, we take the pages in range from page
	 * allocator removing them from the buddy system.  This way
	 * page allocator will never consider using them.
	 *
	 * This lets us mark the pageblocks back as
	 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
	 * aligned range but not in the unaligned, original range are
	 * put back to page allocator so that buddy can use them.
	 */

	ret = start_isolate_page_range(pfn_max_align_down(start),
8075 8076
				       pfn_max_align_up(end), migratetype,
				       false);
8077
	if (ret)
8078
		return ret;
8079

8080 8081
	/*
	 * In case of -EBUSY, we'd like to know which page causes problem.
8082 8083 8084 8085 8086 8087 8088
	 * So, just fall through. test_pages_isolated() has a tracepoint
	 * which will report the busy page.
	 *
	 * It is possible that busy pages could become available before
	 * the call to test_pages_isolated, and the range will actually be
	 * allocated.  So, if we fall through be sure to clear ret so that
	 * -EBUSY is not accidentally used or returned to caller.
8089
	 */
8090
	ret = __alloc_contig_migrate_range(&cc, start, end);
8091
	if (ret && ret != -EBUSY)
8092
		goto done;
8093
	ret =0;
8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112

	/*
	 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
	 * aligned blocks that are marked as MIGRATE_ISOLATE.  What's
	 * more, all pages in [start, end) are free in page allocator.
	 * What we are going to do is to allocate all pages from
	 * [start, end) (that is remove them from page allocator).
	 *
	 * The only problem is that pages at the beginning and at the
	 * end of interesting range may be not aligned with pages that
	 * page allocator holds, ie. they can be part of higher order
	 * pages.  Because of this, we reserve the bigger range and
	 * once this is done free the pages we are not interested in.
	 *
	 * We don't have to hold zone->lock here because the pages are
	 * isolated thus they won't get removed from buddy.
	 */

	lru_add_drain_all();
8113
	drain_all_pages(cc.zone);
8114 8115 8116 8117 8118

	order = 0;
	outer_start = start;
	while (!PageBuddy(pfn_to_page(outer_start))) {
		if (++order >= MAX_ORDER) {
8119 8120
			outer_start = start;
			break;
8121 8122 8123 8124
		}
		outer_start &= ~0UL << order;
	}

8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137
	if (outer_start != start) {
		order = page_order(pfn_to_page(outer_start));

		/*
		 * outer_start page could be small order buddy page and
		 * it doesn't include start page. Adjust outer_start
		 * in this case to report failed page properly
		 * on tracepoint in test_pages_isolated()
		 */
		if (outer_start + (1UL << order) <= start)
			outer_start = start;
	}

8138
	/* Make sure the range is really isolated. */
8139
	if (test_pages_isolated(outer_start, end, false)) {
8140
		pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
8141
			__func__, outer_start, end);
8142 8143 8144 8145
		ret = -EBUSY;
		goto done;
	}

8146
	/* Grab isolated pages from freelists. */
8147
	outer_end = isolate_freepages_range(&cc, outer_start, end);
8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160
	if (!outer_end) {
		ret = -EBUSY;
		goto done;
	}

	/* Free head and tail (if any) */
	if (start != outer_start)
		free_contig_range(outer_start, start - outer_start);
	if (end != outer_end)
		free_contig_range(end, outer_end - end);

done:
	undo_isolate_page_range(pfn_max_align_down(start),
8161
				pfn_max_align_up(end), migratetype);
8162 8163 8164 8165 8166
	return ret;
}

void free_contig_range(unsigned long pfn, unsigned nr_pages)
{
8167 8168 8169 8170 8171 8172 8173 8174 8175
	unsigned int count = 0;

	for (; nr_pages--; pfn++) {
		struct page *page = pfn_to_page(pfn);

		count += page_count(page) != 1;
		__free_page(page);
	}
	WARN(count != 0, "%d pages are still in use!\n", count);
8176 8177 8178
}
#endif

8179 8180 8181 8182
/*
 * The zone indicated has a new number of managed_pages; batch sizes and percpu
 * page high values need to be recalulated.
 */
8183 8184
void __meminit zone_pcp_update(struct zone *zone)
{
8185
	unsigned cpu;
8186
	mutex_lock(&pcp_batch_high_lock);
8187
	for_each_possible_cpu(cpu)
8188 8189
		pageset_set_high_and_batch(zone,
				per_cpu_ptr(zone->pageset, cpu));
8190
	mutex_unlock(&pcp_batch_high_lock);
8191 8192
}

8193 8194 8195
void zone_pcp_reset(struct zone *zone)
{
	unsigned long flags;
8196 8197
	int cpu;
	struct per_cpu_pageset *pset;
8198 8199 8200 8201

	/* avoid races with drain_pages()  */
	local_irq_save(flags);
	if (zone->pageset != &boot_pageset) {
8202 8203 8204 8205
		for_each_online_cpu(cpu) {
			pset = per_cpu_ptr(zone->pageset, cpu);
			drain_zonestat(zone, pset);
		}
8206 8207 8208 8209 8210 8211
		free_percpu(zone->pageset);
		zone->pageset = &boot_pageset;
	}
	local_irq_restore(flags);
}

8212
#ifdef CONFIG_MEMORY_HOTREMOVE
K
KAMEZAWA Hiroyuki 已提交
8213
/*
8214 8215
 * All pages in the range must be in a single zone and isolated
 * before calling this.
K
KAMEZAWA Hiroyuki 已提交
8216 8217 8218 8219 8220 8221
 */
void
__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
{
	struct page *page;
	struct zone *zone;
8222
	unsigned int order, i;
K
KAMEZAWA Hiroyuki 已提交
8223 8224 8225 8226 8227 8228 8229 8230
	unsigned long pfn;
	unsigned long flags;
	/* find the first valid pfn */
	for (pfn = start_pfn; pfn < end_pfn; pfn++)
		if (pfn_valid(pfn))
			break;
	if (pfn == end_pfn)
		return;
8231
	offline_mem_sections(pfn, end_pfn);
K
KAMEZAWA Hiroyuki 已提交
8232 8233 8234 8235 8236 8237 8238 8239 8240
	zone = page_zone(pfn_to_page(pfn));
	spin_lock_irqsave(&zone->lock, flags);
	pfn = start_pfn;
	while (pfn < end_pfn) {
		if (!pfn_valid(pfn)) {
			pfn++;
			continue;
		}
		page = pfn_to_page(pfn);
8241 8242 8243 8244 8245 8246 8247 8248 8249 8250
		/*
		 * The HWPoisoned page may be not in buddy system, and
		 * page_count() is not 0.
		 */
		if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
			pfn++;
			SetPageReserved(page);
			continue;
		}

K
KAMEZAWA Hiroyuki 已提交
8251 8252 8253 8254
		BUG_ON(page_count(page));
		BUG_ON(!PageBuddy(page));
		order = page_order(page);
#ifdef CONFIG_DEBUG_VM
8255 8256
		pr_info("remove from free list %lx %d %lx\n",
			pfn, 1 << order, end_pfn);
K
KAMEZAWA Hiroyuki 已提交
8257
#endif
8258
		del_page_from_free_list(page, zone, order);
K
KAMEZAWA Hiroyuki 已提交
8259 8260 8261 8262 8263 8264 8265
		for (i = 0; i < (1 << order); i++)
			SetPageReserved((page+i));
		pfn += (1 << order);
	}
	spin_unlock_irqrestore(&zone->lock, flags);
}
#endif
8266 8267 8268 8269 8270 8271

bool is_free_buddy_page(struct page *page)
{
	struct zone *zone = page_zone(page);
	unsigned long pfn = page_to_pfn(page);
	unsigned long flags;
8272
	unsigned int order;
8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284

	spin_lock_irqsave(&zone->lock, flags);
	for (order = 0; order < MAX_ORDER; order++) {
		struct page *page_head = page - (pfn & ((1 << order) - 1));

		if (PageBuddy(page_head) && page_order(page_head) >= order)
			break;
	}
	spin_unlock_irqrestore(&zone->lock, flags);

	return order < MAX_ORDER;
}
8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308 8309 8310 8311 8312 8313 8314

#ifdef CONFIG_MEMORY_FAILURE
/*
 * Set PG_hwpoison flag if a given page is confirmed to be a free page.  This
 * test is performed under the zone lock to prevent a race against page
 * allocation.
 */
bool set_hwpoison_free_buddy_page(struct page *page)
{
	struct zone *zone = page_zone(page);
	unsigned long pfn = page_to_pfn(page);
	unsigned long flags;
	unsigned int order;
	bool hwpoisoned = false;

	spin_lock_irqsave(&zone->lock, flags);
	for (order = 0; order < MAX_ORDER; order++) {
		struct page *page_head = page - (pfn & ((1 << order) - 1));

		if (PageBuddy(page_head) && page_order(page_head) >= order) {
			if (!TestSetPageHWPoison(page))
				hwpoisoned = true;
			break;
		}
	}
	spin_unlock_irqrestore(&zone->lock, flags);

	return hwpoisoned;
}
#endif