page_alloc.c 227.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 *  linux/mm/page_alloc.c
 *
 *  Manages the free list, the system allocates free pages here.
 *  Note that kmalloc() lives in slab.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *  Swap reorganised 29.12.95, Stephen Tweedie
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
 *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
 *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
 */

#include <linux/stddef.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
22
#include <linux/jiffies.h>
L
Linus Torvalds 已提交
23
#include <linux/bootmem.h>
24
#include <linux/memblock.h>
L
Linus Torvalds 已提交
25
#include <linux/compiler.h>
26
#include <linux/kernel.h>
27
#include <linux/kasan.h>
L
Linus Torvalds 已提交
28 29 30 31 32
#include <linux/module.h>
#include <linux/suspend.h>
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
33
#include <linux/ratelimit.h>
34
#include <linux/oom.h>
L
Linus Torvalds 已提交
35 36 37 38
#include <linux/topology.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
39
#include <linux/memory_hotplug.h>
L
Linus Torvalds 已提交
40 41
#include <linux/nodemask.h>
#include <linux/vmalloc.h>
42
#include <linux/vmstat.h>
43
#include <linux/mempolicy.h>
44
#include <linux/memremap.h>
45
#include <linux/stop_machine.h>
46 47
#include <linux/sort.h>
#include <linux/pfn.h>
48
#include <linux/backing-dev.h>
49
#include <linux/fault-inject.h>
K
KAMEZAWA Hiroyuki 已提交
50
#include <linux/page-isolation.h>
51
#include <linux/page_ext.h>
52
#include <linux/debugobjects.h>
53
#include <linux/kmemleak.h>
54
#include <linux/compaction.h>
55
#include <trace/events/kmem.h>
56
#include <trace/events/oom.h>
57
#include <linux/prefetch.h>
58
#include <linux/mm_inline.h>
59
#include <linux/migrate.h>
60
#include <linux/hugetlb.h>
61
#include <linux/sched/rt.h>
62
#include <linux/sched/mm.h>
63
#include <linux/page_owner.h>
64
#include <linux/kthread.h>
65
#include <linux/memcontrol.h>
66
#include <linux/ftrace.h>
67
#include <linux/lockdep.h>
68
#include <linux/nmi.h>
69
#include <linux/psi.h>
L
Linus Torvalds 已提交
70

71
#include <asm/sections.h>
L
Linus Torvalds 已提交
72
#include <asm/tlbflush.h>
73
#include <asm/div64.h>
L
Linus Torvalds 已提交
74
#include "internal.h"
A
Alexander Duyck 已提交
75
#include "page_reporting.h"
L
Linus Torvalds 已提交
76

77 78
/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
static DEFINE_MUTEX(pcp_batch_high_lock);
79
#define MIN_PERCPU_PAGELIST_FRACTION	(8)
80

81 82 83 84 85
#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
DEFINE_PER_CPU(int, numa_node);
EXPORT_PER_CPU_SYMBOL(numa_node);
#endif

86 87
DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);

88 89 90 91 92 93 94 95 96
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
/*
 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
 * defined in <linux/topology.h>.
 */
DEFINE_PER_CPU(int, _numa_mem_);		/* Kernel "local memory" node */
EXPORT_PER_CPU_SYMBOL(_numa_mem_);
97
int _node_numa_mem_[MAX_NUMNODES];
98 99
#endif

100 101 102 103
/* work_structs for global per-cpu drains */
DEFINE_MUTEX(pcpu_drain_mutex);
DEFINE_PER_CPU(struct work_struct, pcpu_drain);

104
#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
105
volatile unsigned long latent_entropy __latent_entropy;
106 107 108
EXPORT_SYMBOL(latent_entropy);
#endif

L
Linus Torvalds 已提交
109
/*
110
 * Array of node states.
L
Linus Torvalds 已提交
111
 */
112 113 114 115 116 117 118
nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
	[N_POSSIBLE] = NODE_MASK_ALL,
	[N_ONLINE] = { { [0] = 1UL } },
#ifndef CONFIG_NUMA
	[N_NORMAL_MEMORY] = { { [0] = 1UL } },
#ifdef CONFIG_HIGHMEM
	[N_HIGH_MEMORY] = { { [0] = 1UL } },
119 120
#endif
	[N_MEMORY] = { { [0] = 1UL } },
121 122 123 124 125
	[N_CPU] = { { [0] = 1UL } },
#endif	/* NUMA */
};
EXPORT_SYMBOL(node_states);

126 127 128
/* Protect totalram_pages and zone->managed_pages */
static DEFINE_SPINLOCK(managed_page_count_lock);

129
unsigned long totalram_pages __read_mostly;
130
unsigned long totalreserve_pages __read_mostly;
131
unsigned long totalcma_pages __read_mostly;
132

133
int percpu_pagelist_fraction;
134
gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
L
Linus Torvalds 已提交
135

136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
/*
 * A cached value of the page's pageblock's migratetype, used when the page is
 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
 * Also the migratetype set in the page does not necessarily match the pcplist
 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
 * other index - this ensures that it will be put on the correct CMA freelist.
 */
static inline int get_pcppage_migratetype(struct page *page)
{
	return page->index;
}

static inline void set_pcppage_migratetype(struct page *page, int migratetype)
{
	page->index = migratetype;
}

154 155 156 157 158
#ifdef CONFIG_PM_SLEEP
/*
 * The following functions are used by the suspend/hibernate code to temporarily
 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
 * while devices are suspended.  To avoid races with the suspend/hibernate code,
159 160 161 162
 * they should always be called with system_transition_mutex held
 * (gfp_allowed_mask also should only be modified with system_transition_mutex
 * held, unless the suspend/hibernate code is guaranteed not to run in parallel
 * with that modification).
163
 */
164 165 166 167

static gfp_t saved_gfp_mask;

void pm_restore_gfp_mask(void)
168
{
169
	WARN_ON(!mutex_is_locked(&system_transition_mutex));
170 171 172 173
	if (saved_gfp_mask) {
		gfp_allowed_mask = saved_gfp_mask;
		saved_gfp_mask = 0;
	}
174 175
}

176
void pm_restrict_gfp_mask(void)
177
{
178
	WARN_ON(!mutex_is_locked(&system_transition_mutex));
179 180
	WARN_ON(saved_gfp_mask);
	saved_gfp_mask = gfp_allowed_mask;
181
	gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
182
}
183 184 185

bool pm_suspended_storage(void)
{
186
	if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
187 188 189
		return false;
	return true;
}
190 191
#endif /* CONFIG_PM_SLEEP */

192
#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
193
unsigned int pageblock_order __read_mostly;
194 195
#endif

196
static void __free_pages_ok(struct page *page, unsigned int order);
197

L
Linus Torvalds 已提交
198 199 200 201 202 203
/*
 * results with 256, 32 in the lowmem_reserve sysctl:
 *	1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
 *	1G machine -> (16M dma, 784M normal, 224M high)
 *	NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
 *	HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
Y
Yaowei Bai 已提交
204
 *	HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
A
Andi Kleen 已提交
205 206 207
 *
 * TBD: should special case ZONE_DMA32 machines here - in those we normally
 * don't need any ZONE_NORMAL reservation
L
Linus Torvalds 已提交
208
 */
209
int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
210
#ifdef CONFIG_ZONE_DMA
211
	[ZONE_DMA] = 256,
212
#endif
213
#ifdef CONFIG_ZONE_DMA32
214
	[ZONE_DMA32] = 256,
215
#endif
216
	[ZONE_NORMAL] = 32,
217
#ifdef CONFIG_HIGHMEM
218
	[ZONE_HIGHMEM] = 0,
219
#endif
220
	[ZONE_MOVABLE] = 0,
221
};
L
Linus Torvalds 已提交
222 223 224

EXPORT_SYMBOL(totalram_pages);

225
static char * const zone_names[MAX_NR_ZONES] = {
226
#ifdef CONFIG_ZONE_DMA
227
	 "DMA",
228
#endif
229
#ifdef CONFIG_ZONE_DMA32
230
	 "DMA32",
231
#endif
232
	 "Normal",
233
#ifdef CONFIG_HIGHMEM
M
Mel Gorman 已提交
234
	 "HighMem",
235
#endif
M
Mel Gorman 已提交
236
	 "Movable",
237 238 239
#ifdef CONFIG_ZONE_DEVICE
	 "Device",
#endif
240 241
};

242 243 244 245 246 247 248 249 250 251 252 253 254
char * const migratetype_names[MIGRATE_TYPES] = {
	"Unmovable",
	"Movable",
	"Reclaimable",
	"HighAtomic",
#ifdef CONFIG_CMA
	"CMA",
#endif
#ifdef CONFIG_MEMORY_ISOLATION
	"Isolate",
#endif
};

255 256 257 258 259 260
compound_page_dtor * const compound_page_dtors[] = {
	NULL,
	free_compound_page,
#ifdef CONFIG_HUGETLB_PAGE
	free_huge_page,
#endif
261 262 263
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	free_transhuge_page,
#endif
264 265
};

L
Linus Torvalds 已提交
266
int min_free_kbytes = 1024;
267
int user_min_free_kbytes = -1;
268
int watermark_scale_factor = 10;
L
Linus Torvalds 已提交
269

270 271 272
static unsigned long nr_kernel_pages __meminitdata;
static unsigned long nr_all_pages __meminitdata;
static unsigned long dma_reserve __meminitdata;
L
Linus Torvalds 已提交
273

T
Tejun Heo 已提交
274
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
275 276 277
static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __meminitdata;
static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __meminitdata;
static unsigned long required_kernelcore __initdata;
278
static unsigned long required_kernelcore_percent __initdata;
279
static unsigned long required_movablecore __initdata;
280
static unsigned long required_movablecore_percent __initdata;
281 282
static unsigned long zone_movable_pfn[MAX_NUMNODES] __meminitdata;
static bool mirrored_kernelcore __meminitdata;
T
Tejun Heo 已提交
283 284 285 286 287

/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
int movable_zone;
EXPORT_SYMBOL(movable_zone);
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
288

M
Miklos Szeredi 已提交
289 290
#if MAX_NUMNODES > 1
int nr_node_ids __read_mostly = MAX_NUMNODES;
291
int nr_online_nodes __read_mostly = 1;
M
Miklos Szeredi 已提交
292
EXPORT_SYMBOL(nr_node_ids);
293
EXPORT_SYMBOL(nr_online_nodes);
M
Miklos Szeredi 已提交
294 295
#endif

296 297
int page_group_by_mobility_disabled __read_mostly;

298
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
/*
 * During boot we initialize deferred pages on-demand, as needed, but once
 * page_alloc_init_late() has finished, the deferred pages are all initialized,
 * and we can permanently disable that path.
 */
static DEFINE_STATIC_KEY_TRUE(deferred_pages);

/*
 * Calling kasan_free_pages() only after deferred memory initialization
 * has completed. Poisoning pages during deferred memory init will greatly
 * lengthen the process and cause problem in large memory systems as the
 * deferred pages initialization is done with interrupt disabled.
 *
 * Assuming that there will be no reference to those newly initialized
 * pages before they are ever allocated, this should have no effect on
 * KASAN memory tracking as the poison will be properly inserted at page
 * allocation time. The only corner case is when pages are allocated by
 * on-demand allocation and then freed again before the deferred pages
 * initialization is done, but this is not likely to happen.
 */
static inline void kasan_free_nondeferred_pages(struct page *page, int order)
{
	if (!static_branch_unlikely(&deferred_pages))
		kasan_free_pages(page, order);
}

325
/* Returns true if the struct page for the pfn is uninitialised */
326
static inline bool __meminit early_page_uninitialised(unsigned long pfn)
327
{
328 329 330
	int nid = early_pfn_to_nid(pfn);

	if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
331 332 333 334 335 336 337 338 339 340 341 342 343
		return true;

	return false;
}

/*
 * Returns false when the remaining initialisation should be deferred until
 * later in the boot cycle when it can be parallelised.
 */
static inline bool update_defer_init(pg_data_t *pgdat,
				unsigned long pfn, unsigned long zone_end,
				unsigned long *nr_initialised)
{
344
	/* Always populate low zones for address-constrained allocations */
345 346 347
	if (zone_end < pgdat_end_pfn(pgdat))
		return true;
	(*nr_initialised)++;
348
	if ((*nr_initialised > pgdat->static_init_pgcnt) &&
349 350 351 352 353 354 355 356
	    (pfn & (PAGES_PER_SECTION - 1)) == 0) {
		pgdat->first_deferred_pfn = pfn;
		return false;
	}

	return true;
}
#else
357 358
#define kasan_free_nondeferred_pages(p, o)	kasan_free_pages(p, o)

359 360 361 362 363 364 365 366 367 368 369 370 371
static inline bool early_page_uninitialised(unsigned long pfn)
{
	return false;
}

static inline bool update_defer_init(pg_data_t *pgdat,
				unsigned long pfn, unsigned long zone_end,
				unsigned long *nr_initialised)
{
	return true;
}
#endif

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
/* Return a pointer to the bitmap storing bits affecting a block of pages */
static inline unsigned long *get_pageblock_bitmap(struct page *page,
							unsigned long pfn)
{
#ifdef CONFIG_SPARSEMEM
	return __pfn_to_section(pfn)->pageblock_flags;
#else
	return page_zone(page)->pageblock_flags;
#endif /* CONFIG_SPARSEMEM */
}

static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
{
#ifdef CONFIG_SPARSEMEM
	pfn &= (PAGES_PER_SECTION-1);
	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
#else
	pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
#endif /* CONFIG_SPARSEMEM */
}

/**
 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
 * @page: The page within the block of interest
 * @pfn: The target page frame number
 * @end_bitidx: The last bit of interest to retrieve
 * @mask: mask of bits that the caller is interested in
 *
 * Return: pageblock_bits flags
 */
static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
					unsigned long pfn,
					unsigned long end_bitidx,
					unsigned long mask)
{
	unsigned long *bitmap;
	unsigned long bitidx, word_bitidx;
	unsigned long word;

	bitmap = get_pageblock_bitmap(page, pfn);
	bitidx = pfn_to_bitidx(page, pfn);
	word_bitidx = bitidx / BITS_PER_LONG;
	bitidx &= (BITS_PER_LONG-1);

	word = bitmap[word_bitidx];
	bitidx += end_bitidx;
	return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
}

unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
					unsigned long end_bitidx,
					unsigned long mask)
{
	return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
}

static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
{
	return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
}

/**
 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
 * @page: The page within the block of interest
 * @flags: The flags to set
 * @pfn: The target page frame number
 * @end_bitidx: The last bit of interest
 * @mask: mask of bits that the caller is interested in
 */
void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
					unsigned long pfn,
					unsigned long end_bitidx,
					unsigned long mask)
{
	unsigned long *bitmap;
	unsigned long bitidx, word_bitidx;
	unsigned long old_word, word;

	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);

	bitmap = get_pageblock_bitmap(page, pfn);
	bitidx = pfn_to_bitidx(page, pfn);
	word_bitidx = bitidx / BITS_PER_LONG;
	bitidx &= (BITS_PER_LONG-1);

	VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);

	bitidx += end_bitidx;
	mask <<= (BITS_PER_LONG - bitidx - 1);
	flags <<= (BITS_PER_LONG - bitidx - 1);

	word = READ_ONCE(bitmap[word_bitidx]);
	for (;;) {
		old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
		if (word == old_word)
			break;
		word = old_word;
	}
}
472

473
void set_pageblock_migratetype(struct page *page, int migratetype)
474
{
475 476
	if (unlikely(page_group_by_mobility_disabled &&
		     migratetype < MIGRATE_PCPTYPES))
477 478
		migratetype = MIGRATE_UNMOVABLE;

479 480 481 482
	set_pageblock_flags_group(page, (unsigned long)migratetype,
					PB_migrate, PB_migrate_end);
}

N
Nick Piggin 已提交
483
#ifdef CONFIG_DEBUG_VM
484
static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
L
Linus Torvalds 已提交
485
{
486 487 488
	int ret = 0;
	unsigned seq;
	unsigned long pfn = page_to_pfn(page);
489
	unsigned long sp, start_pfn;
490

491 492
	do {
		seq = zone_span_seqbegin(zone);
493 494
		start_pfn = zone->zone_start_pfn;
		sp = zone->spanned_pages;
495
		if (!zone_spans_pfn(zone, pfn))
496 497 498
			ret = 1;
	} while (zone_span_seqretry(zone, seq));

499
	if (ret)
500 501 502
		pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
			pfn, zone_to_nid(zone), zone->name,
			start_pfn, start_pfn + sp);
503

504
	return ret;
505 506 507 508
}

static int page_is_consistent(struct zone *zone, struct page *page)
{
509
	if (!pfn_valid_within(page_to_pfn(page)))
510
		return 0;
L
Linus Torvalds 已提交
511
	if (zone != page_zone(page))
512 513 514 515 516 517 518
		return 0;

	return 1;
}
/*
 * Temporary debugging check for pages not lying within a given zone.
 */
519
static int __maybe_unused bad_range(struct zone *zone, struct page *page)
520 521
{
	if (page_outside_zone_boundaries(zone, page))
L
Linus Torvalds 已提交
522
		return 1;
523 524 525
	if (!page_is_consistent(zone, page))
		return 1;

L
Linus Torvalds 已提交
526 527
	return 0;
}
N
Nick Piggin 已提交
528
#else
529
static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
N
Nick Piggin 已提交
530 531 532 533 534
{
	return 0;
}
#endif

535 536
static void bad_page(struct page *page, const char *reason,
		unsigned long bad_flags)
L
Linus Torvalds 已提交
537
{
538 539 540 541 542 543 544 545 546 547 548 549 550 551
	static unsigned long resume;
	static unsigned long nr_shown;
	static unsigned long nr_unshown;

	/*
	 * Allow a burst of 60 reports, then keep quiet for that minute;
	 * or allow a steady drip of one report per second.
	 */
	if (nr_shown == 60) {
		if (time_before(jiffies, resume)) {
			nr_unshown++;
			goto out;
		}
		if (nr_unshown) {
552
			pr_alert(
553
			      "BUG: Bad page state: %lu messages suppressed\n",
554 555 556 557 558 559 560 561
				nr_unshown);
			nr_unshown = 0;
		}
		nr_shown = 0;
	}
	if (nr_shown++ == 0)
		resume = jiffies + 60 * HZ;

562
	pr_alert("BUG: Bad page state in process %s  pfn:%05lx\n",
563
		current->comm, page_to_pfn(page));
564 565 566 567 568
	__dump_page(page, reason);
	bad_flags &= page->flags;
	if (bad_flags)
		pr_alert("bad because of flags: %#lx(%pGp)\n",
						bad_flags, &bad_flags);
569
	dump_page_owner(page);
570

571
	print_modules();
L
Linus Torvalds 已提交
572
	dump_stack();
573
out:
574
	/* Leave bad fields for debug, except PageBuddy could make trouble */
575
	page_mapcount_reset(page); /* remove PageBuddy */
576
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
L
Linus Torvalds 已提交
577 578 579 580 581
}

/*
 * Higher-order pages are called "compound pages".  They are structured thusly:
 *
582
 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
L
Linus Torvalds 已提交
583
 *
584 585
 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
L
Linus Torvalds 已提交
586
 *
587 588
 * The first tail page's ->compound_dtor holds the offset in array of compound
 * page destructors. See compound_page_dtors.
L
Linus Torvalds 已提交
589
 *
590
 * The first tail page's ->compound_order holds the order of allocation.
591
 * This usage means that zero-order pages may not be compound.
L
Linus Torvalds 已提交
592
 */
593

594
void free_compound_page(struct page *page)
595
{
596
	mem_cgroup_uncharge(page);
597
	__free_pages_ok(page, compound_order(page));
598 599
}

600
void prep_compound_page(struct page *page, unsigned int order)
601 602 603 604
{
	int i;
	int nr_pages = 1 << order;

605
	set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
606 607 608 609
	set_compound_order(page, order);
	__SetPageHead(page);
	for (i = 1; i < nr_pages; i++) {
		struct page *p = page + i;
610
		set_page_count(p, 0);
611
		p->mapping = TAIL_MAPPING;
612
		set_compound_head(p, page);
613
	}
614
	atomic_set(compound_mapcount_ptr(page), -1);
615 616
}

617 618
#ifdef CONFIG_DEBUG_PAGEALLOC
unsigned int _debug_guardpage_minorder;
619 620
bool _debug_pagealloc_enabled __read_mostly
			= IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
621
EXPORT_SYMBOL(_debug_pagealloc_enabled);
622 623
bool _debug_guardpage_enabled __read_mostly;

624 625 626 627
static int __init early_debug_pagealloc(char *buf)
{
	if (!buf)
		return -EINVAL;
628
	return kstrtobool(buf, &_debug_pagealloc_enabled);
629 630 631
}
early_param("debug_pagealloc", early_debug_pagealloc);

632 633
static bool need_debug_guardpage(void)
{
634 635 636 637
	/* If we don't use debug_pagealloc, we don't need guard page */
	if (!debug_pagealloc_enabled())
		return false;

638 639 640
	if (!debug_guardpage_minorder())
		return false;

641 642 643 644 645
	return true;
}

static void init_debug_guardpage(void)
{
646 647 648
	if (!debug_pagealloc_enabled())
		return;

649 650 651
	if (!debug_guardpage_minorder())
		return;

652 653 654 655 656 657 658
	_debug_guardpage_enabled = true;
}

struct page_ext_operations debug_guardpage_ops = {
	.need = need_debug_guardpage,
	.init = init_debug_guardpage,
};
659 660 661 662 663 664

static int __init debug_guardpage_minorder_setup(char *buf)
{
	unsigned long res;

	if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
665
		pr_err("Bad debug_guardpage_minorder value\n");
666 667 668
		return 0;
	}
	_debug_guardpage_minorder = res;
669
	pr_info("Setting debug_guardpage_minorder to %lu\n", res);
670 671
	return 0;
}
672
early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
673

674
static inline bool set_page_guard(struct zone *zone, struct page *page,
675
				unsigned int order, int migratetype)
676
{
677 678 679
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
680 681 682 683
		return false;

	if (order >= debug_guardpage_minorder())
		return false;
684 685

	page_ext = lookup_page_ext(page);
686
	if (unlikely(!page_ext))
687
		return false;
688

689 690
	__set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);

691 692 693 694
	INIT_LIST_HEAD(&page->lru);
	set_page_private(page, order);
	/* Guard pages are not available for any usage */
	__mod_zone_freepage_state(zone, -(1 << order), migratetype);
695 696

	return true;
697 698
}

699 700
static inline void clear_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype)
701
{
702 703 704 705 706 707
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
		return;

	page_ext = lookup_page_ext(page);
708 709 710
	if (unlikely(!page_ext))
		return;

711 712
	__clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);

713 714 715
	set_page_private(page, 0);
	if (!is_migrate_isolate(migratetype))
		__mod_zone_freepage_state(zone, (1 << order), migratetype);
716 717
}
#else
718
struct page_ext_operations debug_guardpage_ops;
719 720
static inline bool set_page_guard(struct zone *zone, struct page *page,
			unsigned int order, int migratetype) { return false; }
721 722
static inline void clear_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype) {}
723 724
#endif

725
static inline void set_page_order(struct page *page, unsigned int order)
726
{
H
Hugh Dickins 已提交
727
	set_page_private(page, order);
728
	__SetPageBuddy(page);
L
Linus Torvalds 已提交
729 730 731 732
}

/*
 * This function checks whether a page is free && is the buddy
733
 * we can coalesce a page and its buddy if
734
 * (a) the buddy is not in a hole (check before calling!) &&
735
 * (b) the buddy is in the buddy system &&
736 737
 * (c) a page and its buddy have the same order &&
 * (d) a page and its buddy are in the same zone.
738
 *
739 740
 * For recording whether a page is in the buddy system, we set PageBuddy.
 * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
L
Linus Torvalds 已提交
741
 *
742
 * For recording page's order, we use page_private(page).
L
Linus Torvalds 已提交
743
 */
744
static inline int page_is_buddy(struct page *page, struct page *buddy,
745
							unsigned int order)
L
Linus Torvalds 已提交
746
{
747
	if (page_is_guard(buddy) && page_order(buddy) == order) {
748 749 750
		if (page_zone_id(page) != page_zone_id(buddy))
			return 0;

751 752
		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);

753 754 755
		return 1;
	}

756
	if (PageBuddy(buddy) && page_order(buddy) == order) {
757 758 759 760 761 762 763 764
		/*
		 * zone check is done late to avoid uselessly
		 * calculating zone/node ids for pages that could
		 * never merge.
		 */
		if (page_zone_id(page) != page_zone_id(buddy))
			return 0;

765 766
		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);

767
		return 1;
768
	}
769
	return 0;
L
Linus Torvalds 已提交
770 771
}

772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
/* Used for pages not on another list */
static inline void add_to_free_list(struct page *page, struct zone *zone,
				    unsigned int order, int migratetype)
{
	struct free_area *area = &zone->free_area[order];

	list_add(&page->lru, &area->free_list[migratetype]);
	area->nr_free++;
}

/* Used for pages not on another list */
static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
					 unsigned int order, int migratetype)
{
	struct free_area *area = &zone->free_area[order];

	list_add_tail(&page->lru, &area->free_list[migratetype]);
	area->nr_free++;
}

/* Used for pages which are on another list */
static inline void move_to_free_list(struct page *page, struct zone *zone,
				     unsigned int order, int migratetype)
{
	struct free_area *area = &zone->free_area[order];

	list_move(&page->lru, &area->free_list[migratetype]);
}

static inline void del_page_from_free_list(struct page *page, struct zone *zone,
					   unsigned int order)
{
A
Alexander Duyck 已提交
804 805 806 807
	/* clear reported state and update reported page count */
	if (page_reported(page))
		__ClearPageReported(page);

808 809 810 811 812 813
	list_del(&page->lru);
	__ClearPageBuddy(page);
	set_page_private(page, 0);
	zone->free_area[order].nr_free--;
}

L
Linus Torvalds 已提交
814 815 816 817 818 819 820 821 822 823 824 825 826
/*
 * Freeing function for a buddy system allocator.
 *
 * The concept of a buddy system is to maintain direct-mapped table
 * (containing bit values) for memory blocks of various "orders".
 * The bottom level table contains the map for the smallest allocatable
 * units of memory (here, pages), and each level above it describes
 * pairs of units from the levels below, hence, "buddies".
 * At a high level, all that happens here is marking the table entry
 * at the bottom level available, and propagating the changes upward
 * as necessary, plus some accounting needed to play nicely with other
 * parts of the VM system.
 * At each level, we keep a list of pages, which are heads of continuous
827 828
 * free pages of length of (1 << order) and marked with PageBuddy.
 * Page's order is recorded in page_private(page) field.
L
Linus Torvalds 已提交
829
 * So when we are allocating or freeing one, we can derive the state of the
830 831
 * other.  That is, if we allocate a small block, and both were
 * free, the remainder of the region must be split into blocks.
L
Linus Torvalds 已提交
832
 * If a block is freed, and its buddy is also free, then this
833
 * triggers coalescing into a block of larger size.
L
Linus Torvalds 已提交
834
 *
835
 * -- nyc
L
Linus Torvalds 已提交
836 837
 */

N
Nick Piggin 已提交
838
static inline void __free_one_page(struct page *page,
839
		unsigned long pfn,
840
		struct zone *zone, unsigned int order,
A
Alexander Duyck 已提交
841
		int migratetype, bool report)
L
Linus Torvalds 已提交
842
{
843 844
	unsigned long combined_pfn;
	unsigned long uninitialized_var(buddy_pfn);
845
	struct page *buddy;
846 847 848
	unsigned int max_order;

	max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
L
Linus Torvalds 已提交
849

850
	VM_BUG_ON(!zone_is_initialized(zone));
851
	VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
L
Linus Torvalds 已提交
852

853
	VM_BUG_ON(migratetype == -1);
854
	if (likely(!is_migrate_isolate(migratetype)))
855
		__mod_zone_freepage_state(zone, 1 << order, migratetype);
856

857
	VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
858
	VM_BUG_ON_PAGE(bad_range(zone, page), page);
L
Linus Torvalds 已提交
859

860
continue_merging:
861
	while (order < max_order - 1) {
862 863
		buddy_pfn = __find_buddy_pfn(pfn, order);
		buddy = page + (buddy_pfn - pfn);
864 865 866

		if (!pfn_valid_within(buddy_pfn))
			goto done_merging;
867
		if (!page_is_buddy(page, buddy, order))
868
			goto done_merging;
869 870 871 872
		/*
		 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
		 * merge with it and move up one order.
		 */
873
		if (page_is_guard(buddy))
874
			clear_page_guard(zone, buddy, order, migratetype);
875
		else
876
			del_page_from_free_list(buddy, zone, order);
877 878 879
		combined_pfn = buddy_pfn & pfn;
		page = page + (combined_pfn - pfn);
		pfn = combined_pfn;
L
Linus Torvalds 已提交
880 881
		order++;
	}
882 883 884 885 886 887 888 889 890 891 892 893
	if (max_order < MAX_ORDER) {
		/* If we are here, it means order is >= pageblock_order.
		 * We want to prevent merge between freepages on isolate
		 * pageblock and normal pageblock. Without this, pageblock
		 * isolation could cause incorrect freepage or CMA accounting.
		 *
		 * We don't want to hit this code for the more frequent
		 * low-order merging.
		 */
		if (unlikely(has_isolate_pageblock(zone))) {
			int buddy_mt;

894 895
			buddy_pfn = __find_buddy_pfn(pfn, order);
			buddy = page + (buddy_pfn - pfn);
896 897 898 899 900 901 902 903 904 905 906 907
			buddy_mt = get_pageblock_migratetype(buddy);

			if (migratetype != buddy_mt
					&& (is_migrate_isolate(migratetype) ||
						is_migrate_isolate(buddy_mt)))
				goto done_merging;
		}
		max_order++;
		goto continue_merging;
	}

done_merging:
L
Linus Torvalds 已提交
908
	set_page_order(page, order);
909 910 911 912 913 914 915 916 917

	/*
	 * If this is not the largest possible page, check if the buddy
	 * of the next-highest order is free. If it is, it's possible
	 * that pages are being freed that will coalesce soon. In case,
	 * that is happening, add the free page to the tail of the list
	 * so it's less likely to be used soon and more likely to be merged
	 * as a higher order page
	 */
918
	if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn)) {
919
		struct page *higher_page, *higher_buddy;
920 921 922 923
		combined_pfn = buddy_pfn & pfn;
		higher_page = page + (combined_pfn - pfn);
		buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
		higher_buddy = higher_page + (buddy_pfn - combined_pfn);
924 925
		if (pfn_valid_within(buddy_pfn) &&
		    page_is_buddy(higher_page, higher_buddy, order + 1)) {
926
			add_to_free_list_tail(page, zone, order,
927 928
					      migratetype);
			return;
929 930 931
		}
	}

932
	add_to_free_list(page, zone, order, migratetype);
A
Alexander Duyck 已提交
933 934 935 936

	/* Notify page reporting subsystem of freed page */
	if (report)
		page_reporting_notify_free(order);
L
Linus Torvalds 已提交
937 938
}

939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960
/*
 * A bad page could be due to a number of fields. Instead of multiple branches,
 * try and check multiple fields with one check. The caller must do a detailed
 * check if necessary.
 */
static inline bool page_expected_state(struct page *page,
					unsigned long check_flags)
{
	if (unlikely(atomic_read(&page->_mapcount) != -1))
		return false;

	if (unlikely((unsigned long)page->mapping |
			page_ref_count(page) |
#ifdef CONFIG_MEMCG
			(unsigned long)page->mem_cgroup |
#endif
			(page->flags & check_flags)))
		return false;

	return true;
}

961
static void free_pages_check_bad(struct page *page)
L
Linus Torvalds 已提交
962
{
963 964 965 966 967
	const char *bad_reason;
	unsigned long bad_flags;

	bad_reason = NULL;
	bad_flags = 0;
968

969
	if (unlikely(atomic_read(&page->_mapcount) != -1))
970 971 972
		bad_reason = "nonzero mapcount";
	if (unlikely(page->mapping != NULL))
		bad_reason = "non-NULL mapping";
973
	if (unlikely(page_ref_count(page) != 0))
974
		bad_reason = "nonzero _refcount";
975 976 977 978
	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
		bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
		bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
	}
979 980 981 982
#ifdef CONFIG_MEMCG
	if (unlikely(page->mem_cgroup))
		bad_reason = "page still charged to cgroup";
#endif
983
	bad_page(page, bad_reason, bad_flags);
984 985 986 987
}

static inline int free_pages_check(struct page *page)
{
988
	if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
989 990 991 992
		return 0;

	/* Something has gone sideways, find it */
	free_pages_check_bad(page);
993
	return 1;
L
Linus Torvalds 已提交
994 995
}

996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
static int free_tail_pages_check(struct page *head_page, struct page *page)
{
	int ret = 1;

	/*
	 * We rely page->lru.next never has bit 0 set, unless the page
	 * is PageTail(). Let's make sure that's true even for poisoned ->lru.
	 */
	BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);

	if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
		ret = 0;
		goto out;
	}
	switch (page - head_page) {
	case 1:
1012
		/* the first tail page: ->mapping may be compound_mapcount() */
1013 1014 1015 1016 1017 1018 1019 1020
		if (unlikely(compound_mapcount(page))) {
			bad_page(page, "nonzero compound_mapcount", 0);
			goto out;
		}
		break;
	case 2:
		/*
		 * the second tail page: ->mapping is
M
Matthew Wilcox 已提交
1021
		 * deferred_list.next -- ignore value.
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
		 */
		break;
	default:
		if (page->mapping != TAIL_MAPPING) {
			bad_page(page, "corrupted mapping in tail page", 0);
			goto out;
		}
		break;
	}
	if (unlikely(!PageTail(page))) {
		bad_page(page, "PageTail not set", 0);
		goto out;
	}
	if (unlikely(compound_head(page) != head_page)) {
		bad_page(page, "compound_head not consistent", 0);
		goto out;
	}
	ret = 0;
out:
	page->mapping = NULL;
	clear_compound_head(page);
	return ret;
}

1046 1047
static __always_inline bool free_pages_prepare(struct page *page,
					unsigned int order, bool check_free)
1048
{
1049
	int bad = 0;
1050 1051 1052

	VM_BUG_ON_PAGE(PageTail(page), page);

1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
	trace_mm_page_free(page, order);

	/*
	 * Check tail pages before head page information is cleared to
	 * avoid checking PageCompound for order-0 pages.
	 */
	if (unlikely(order)) {
		bool compound = PageCompound(page);
		int i;

		VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1064

1065 1066
		if (compound)
			ClearPageDoubleMap(page);
1067 1068 1069 1070 1071 1072 1073
		for (i = 1; i < (1 << order); i++) {
			if (compound)
				bad += free_tail_pages_check(page, page + i);
			if (unlikely(free_pages_check(page + i))) {
				bad++;
				continue;
			}
G
Gavin Shan 已提交
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084

			/*
			 * The page age information is stored in page flags
			 * or node's page array. We need to explicitly clear
			 * it in both cases. Otherwise, the stale age will
			 * be provided when it's allocated again. Also, we
			 * maintain age information for each page in the
			 * compound page, So we have to clear them one by one.
			 */
			kidled_set_page_age(page_pgdat(page + i),
					    page_to_pfn(page + i), 0);
1085 1086 1087
			(page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
		}
	}
1088
	if (PageMappingFlags(page))
1089
		page->mapping = NULL;
1090
	if (memcg_kmem_enabled() && PageKmemcg(page))
1091
		memcg_kmem_uncharge(page, order);
1092 1093 1094 1095
	if (check_free)
		bad += free_pages_check(page);
	if (bad)
		return false;
1096

1097
	page_cpupid_reset_last(page);
G
Gavin Shan 已提交
1098
	kidled_set_page_age(page_pgdat(page), page_to_pfn(page), 0);
1099 1100
	page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
	reset_page_owner(page, order);
1101 1102 1103

	if (!PageHighMem(page)) {
		debug_check_no_locks_freed(page_address(page),
1104
					   PAGE_SIZE << order);
1105
		debug_check_no_obj_freed(page_address(page),
1106
					   PAGE_SIZE << order);
1107
	}
1108 1109 1110
	arch_free_page(page, order);
	kernel_poison_pages(page, 1 << order, 0);
	kernel_map_pages(page, 1 << order, 0);
1111
	kasan_free_nondeferred_pages(page, order);
1112 1113 1114 1115

	return true;
}

1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
#ifdef CONFIG_DEBUG_VM
static inline bool free_pcp_prepare(struct page *page)
{
	return free_pages_prepare(page, 0, true);
}

static inline bool bulkfree_pcp_prepare(struct page *page)
{
	return false;
}
#else
static bool free_pcp_prepare(struct page *page)
{
	return free_pages_prepare(page, 0, false);
}

1132 1133 1134 1135 1136 1137
static bool bulkfree_pcp_prepare(struct page *page)
{
	return free_pages_check(page);
}
#endif /* CONFIG_DEBUG_VM */

1138 1139 1140 1141 1142 1143 1144 1145 1146
static inline void prefetch_buddy(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);
	unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
	struct page *buddy = page + (buddy_pfn - pfn);

	prefetch(buddy);
}

L
Linus Torvalds 已提交
1147
/*
1148
 * Frees a number of pages from the PCP lists
L
Linus Torvalds 已提交
1149
 * Assumes all pages on list are in same zone, and of same order.
1150
 * count is the number of pages to free.
L
Linus Torvalds 已提交
1151 1152 1153 1154 1155 1156 1157
 *
 * If the zone was previously in an "all pages pinned" state then look to
 * see if this freeing clears that state.
 *
 * And clear the zone's pages_scanned counter, to hold off the "all pages are
 * pinned" detection logic.
 */
1158 1159
static void free_pcppages_bulk(struct zone *zone, int count,
					struct per_cpu_pages *pcp)
L
Linus Torvalds 已提交
1160
{
1161
	int migratetype = 0;
1162
	int batch_free = 0;
1163
	int prefetch_nr = 0;
1164
	bool isolated_pageblocks;
1165 1166
	struct page *page, *tmp;
	LIST_HEAD(head);
1167

1168
	while (count) {
1169 1170 1171
		struct list_head *list;

		/*
1172 1173 1174 1175 1176
		 * Remove pages from lists in a round-robin fashion. A
		 * batch_free count is maintained that is incremented when an
		 * empty list is encountered.  This is so more pages are freed
		 * off fuller lists instead of spinning excessively around empty
		 * lists
1177 1178
		 */
		do {
1179
			batch_free++;
1180 1181 1182 1183
			if (++migratetype == MIGRATE_PCPTYPES)
				migratetype = 0;
			list = &pcp->lists[migratetype];
		} while (list_empty(list));
N
Nick Piggin 已提交
1184

1185 1186
		/* This is the only non-empty list. Free them all. */
		if (batch_free == MIGRATE_PCPTYPES)
1187
			batch_free = count;
1188

1189
		do {
1190
			page = list_last_entry(list, struct page, lru);
1191
			/* must delete to avoid corrupting pcp list */
1192
			list_del(&page->lru);
1193
			pcp->count--;
1194

1195 1196 1197
			if (bulkfree_pcp_prepare(page))
				continue;

1198
			list_add_tail(&page->lru, &head);
1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210

			/*
			 * We are going to put the page back to the global
			 * pool, prefetch its buddy to speed up later access
			 * under zone->lock. It is believed the overhead of
			 * an additional test and calculating buddy_pfn here
			 * can be offset by reduced memory latency later. To
			 * avoid excessive prefetching due to large count, only
			 * prefetch buddy for the first pcp->batch nr of pages.
			 */
			if (prefetch_nr++ < pcp->batch)
				prefetch_buddy(page);
1211
		} while (--count && --batch_free && !list_empty(list));
L
Linus Torvalds 已提交
1212
	}
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228

	spin_lock(&zone->lock);
	isolated_pageblocks = has_isolate_pageblock(zone);

	/*
	 * Use safe version since after __free_one_page(),
	 * page->lru.next will not point to original list.
	 */
	list_for_each_entry_safe(page, tmp, &head, lru) {
		int mt = get_pcppage_migratetype(page);
		/* MIGRATE_ISOLATE page should not go to pcplists */
		VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
		/* Pageblock could have been isolated meanwhile */
		if (unlikely(isolated_pageblocks))
			mt = get_pageblock_migratetype(page);

A
Alexander Duyck 已提交
1229
		__free_one_page(page, page_to_pfn(page), zone, 0, mt, true);
1230 1231
		trace_mm_page_pcpu_drain(page, 0, mt);
	}
1232
	spin_unlock(&zone->lock);
L
Linus Torvalds 已提交
1233 1234
}

1235 1236
static void free_one_page(struct zone *zone,
				struct page *page, unsigned long pfn,
1237
				unsigned int order,
1238
				int migratetype)
L
Linus Torvalds 已提交
1239
{
1240
	spin_lock(&zone->lock);
1241 1242 1243 1244
	if (unlikely(has_isolate_pageblock(zone) ||
		is_migrate_isolate(migratetype))) {
		migratetype = get_pfnblock_migratetype(page, pfn);
	}
A
Alexander Duyck 已提交
1245
	__free_one_page(page, pfn, zone, order, migratetype, true);
1246
	spin_unlock(&zone->lock);
N
Nick Piggin 已提交
1247 1248
}

1249
static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1250
				unsigned long zone, int nid)
1251
{
1252
	mm_zero_struct_page(page);
1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
	set_page_links(page, zone, nid, pfn);
	init_page_count(page);
	page_mapcount_reset(page);
	page_cpupid_reset_last(page);

	INIT_LIST_HEAD(&page->lru);
#ifdef WANT_PAGE_VIRTUAL
	/* The shift won't overflow because ZONE_NORMAL is below 4G. */
	if (!is_highmem_idx(zone))
		set_page_address(page, __va(pfn << PAGE_SHIFT));
#endif
}

1266
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1267
static void __meminit init_reserved_page(unsigned long pfn)
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
{
	pg_data_t *pgdat;
	int nid, zid;

	if (!early_page_uninitialised(pfn))
		return;

	nid = early_pfn_to_nid(pfn);
	pgdat = NODE_DATA(nid);

	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
		struct zone *zone = &pgdat->node_zones[zid];

		if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
			break;
	}
1284
	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
1285 1286 1287 1288 1289 1290 1291
}
#else
static inline void init_reserved_page(unsigned long pfn)
{
}
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */

1292 1293 1294 1295 1296 1297
/*
 * Initialised pages do not have PageReserved set. This function is
 * called for each range allocated by the bootmem allocator and
 * marks the pages PageReserved. The remaining valid pages are later
 * sent to the buddy page allocator.
 */
1298
void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
1299 1300 1301 1302
{
	unsigned long start_pfn = PFN_DOWN(start);
	unsigned long end_pfn = PFN_UP(end);

1303 1304 1305 1306 1307
	for (; start_pfn < end_pfn; start_pfn++) {
		if (pfn_valid(start_pfn)) {
			struct page *page = pfn_to_page(start_pfn);

			init_reserved_page(start_pfn);
1308 1309 1310 1311

			/* Avoid false-positive PageTail() */
			INIT_LIST_HEAD(&page->lru);

1312 1313 1314
			SetPageReserved(page);
		}
	}
1315 1316
}

1317 1318
static void __free_pages_ok(struct page *page, unsigned int order)
{
1319
	unsigned long flags;
M
Minchan Kim 已提交
1320
	int migratetype;
1321
	unsigned long pfn = page_to_pfn(page);
1322

1323
	if (!free_pages_prepare(page, order, true))
1324 1325
		return;

1326
	migratetype = get_pfnblock_migratetype(page, pfn);
1327 1328
	local_irq_save(flags);
	__count_vm_events(PGFREE, 1 << order);
1329
	free_one_page(page_zone(page), page, pfn, order, migratetype);
1330
	local_irq_restore(flags);
L
Linus Torvalds 已提交
1331 1332
}

1333
static void __init __free_pages_boot_core(struct page *page, unsigned int order)
1334
{
1335
	unsigned int nr_pages = 1 << order;
1336
	struct page *p = page;
1337
	unsigned int loop;
1338

1339 1340 1341
	prefetchw(p);
	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
		prefetchw(p + 1);
1342 1343
		__ClearPageReserved(p);
		set_page_count(p, 0);
1344
	}
1345 1346
	__ClearPageReserved(p);
	set_page_count(p, 0);
1347

1348
	page_zone(page)->managed_pages += nr_pages;
1349 1350
	set_page_refcounted(page);
	__free_pages(page, order);
1351 1352
}

1353 1354
#if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \
	defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1355

1356 1357 1358 1359
static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;

int __meminit early_pfn_to_nid(unsigned long pfn)
{
1360
	static DEFINE_SPINLOCK(early_pfn_lock);
1361 1362
	int nid;

1363
	spin_lock(&early_pfn_lock);
1364
	nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1365
	if (nid < 0)
1366
		nid = first_online_node;
1367 1368 1369
	spin_unlock(&early_pfn_lock);

	return nid;
1370 1371 1372 1373
}
#endif

#ifdef CONFIG_NODES_SPAN_OTHER_NODES
1374 1375
/* Only safe to use early in boot when initialisation is single-threaded */
static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1376 1377 1378
{
	int nid;

1379
	nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
	if (nid >= 0 && nid != node)
		return false;
	return true;
}

#else
static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
{
	return true;
}
#endif


1393
void __init __free_pages_bootmem(struct page *page, unsigned long pfn,
1394 1395 1396 1397
							unsigned int order)
{
	if (early_page_uninitialised(pfn))
		return;
1398
	return __free_pages_boot_core(page, order);
1399 1400
}

1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429
/*
 * Check that the whole (or subset of) a pageblock given by the interval of
 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
 * with the migration of free compaction scanner. The scanners then need to
 * use only pfn_valid_within() check for arches that allow holes within
 * pageblocks.
 *
 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
 *
 * It's possible on some configurations to have a setup like node0 node1 node0
 * i.e. it's possible that all pages within a zones range of pages do not
 * belong to a single zone. We assume that a border between node0 and node1
 * can occur within a single pageblock, but not a node0 node1 node0
 * interleaving within a single pageblock. It is therefore sufficient to check
 * the first and last page of a pageblock and avoid checking each individual
 * page in a pageblock.
 */
struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
				     unsigned long end_pfn, struct zone *zone)
{
	struct page *start_page;
	struct page *end_page;

	/* end_pfn is one past the range we are checking */
	end_pfn--;

	if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
		return NULL;

1430 1431 1432
	start_page = pfn_to_online_page(start_pfn);
	if (!start_page)
		return NULL;
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471

	if (page_zone(start_page) != zone)
		return NULL;

	end_page = pfn_to_page(end_pfn);

	/* This gives a shorter code than deriving page_zone(end_page) */
	if (page_zone_id(start_page) != page_zone_id(end_page))
		return NULL;

	return start_page;
}

void set_zone_contiguous(struct zone *zone)
{
	unsigned long block_start_pfn = zone->zone_start_pfn;
	unsigned long block_end_pfn;

	block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
	for (; block_start_pfn < zone_end_pfn(zone);
			block_start_pfn = block_end_pfn,
			 block_end_pfn += pageblock_nr_pages) {

		block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));

		if (!__pageblock_pfn_to_page(block_start_pfn,
					     block_end_pfn, zone))
			return;
	}

	/* We confirm that there is no hole */
	zone->contiguous = true;
}

void clear_zone_contiguous(struct zone *zone)
{
	zone->contiguous = false;
}

1472
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1473 1474
static void __init deferred_free_range(unsigned long pfn,
				       unsigned long nr_pages)
1475
{
1476 1477
	struct page *page;
	unsigned long i;
1478

1479
	if (!nr_pages)
1480 1481
		return;

1482 1483
	page = pfn_to_page(pfn);

1484
	/* Free a large naturally-aligned chunk if possible */
1485 1486
	if (nr_pages == pageblock_nr_pages &&
	    (pfn & (pageblock_nr_pages - 1)) == 0) {
1487
		set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1488
		__free_pages_boot_core(page, pageblock_order);
1489 1490 1491
		return;
	}

1492 1493 1494
	for (i = 0; i < nr_pages; i++, page++, pfn++) {
		if ((pfn & (pageblock_nr_pages - 1)) == 0)
			set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1495
		__free_pages_boot_core(page, 0);
1496
	}
1497 1498
}

1499 1500 1501 1502 1503 1504 1505 1506 1507
/* Completion tracking for deferred_init_memmap() threads */
static atomic_t pgdat_init_n_undone __initdata;
static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);

static inline void __init pgdat_init_report_one_done(void)
{
	if (atomic_dec_and_test(&pgdat_init_n_undone))
		complete(&pgdat_init_all_done_comp);
}
1508

1509
/*
1510 1511 1512 1513 1514 1515 1516 1517
 * Returns true if page needs to be initialized or freed to buddy allocator.
 *
 * First we check if pfn is valid on architectures where it is possible to have
 * holes within pageblock_nr_pages. On systems where it is not possible, this
 * function is optimized out.
 *
 * Then, we check if a current large page is valid by only checking the validity
 * of the head pfn.
1518
 */
1519
static inline bool __init deferred_pfn_valid(unsigned long pfn)
1520
{
1521 1522 1523 1524 1525 1526
	if (!pfn_valid_within(pfn))
		return false;
	if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
		return false;
	return true;
}
1527

1528 1529 1530 1531
/*
 * Free pages to buddy allocator. Try to free aligned pages in
 * pageblock_nr_pages sizes.
 */
1532
static void __init deferred_free_pages(unsigned long pfn,
1533 1534 1535 1536
				       unsigned long end_pfn)
{
	unsigned long nr_pgmask = pageblock_nr_pages - 1;
	unsigned long nr_free = 0;
1537

1538
	for (; pfn < end_pfn; pfn++) {
1539
		if (!deferred_pfn_valid(pfn)) {
1540 1541 1542 1543 1544
			deferred_free_range(pfn - nr_free, nr_free);
			nr_free = 0;
		} else if (!(pfn & nr_pgmask)) {
			deferred_free_range(pfn - nr_free, nr_free);
			nr_free = 1;
1545
			touch_nmi_watchdog();
1546 1547 1548 1549 1550 1551
		} else {
			nr_free++;
		}
	}
	/* Free the last block of pages to allocator */
	deferred_free_range(pfn - nr_free, nr_free);
1552 1553
}

1554 1555 1556 1557 1558
/*
 * Initialize struct pages.  We minimize pfn page lookups and scheduler checks
 * by performing it only once every pageblock_nr_pages.
 * Return number of pages initialized.
 */
1559
static unsigned long  __init deferred_init_pages(struct zone *zone,
1560 1561
						 unsigned long pfn,
						 unsigned long end_pfn)
1562 1563
{
	unsigned long nr_pgmask = pageblock_nr_pages - 1;
1564
	int nid = zone_to_nid(zone);
1565
	unsigned long nr_pages = 0;
1566
	int zid = zone_idx(zone);
1567 1568
	struct page *page = NULL;

1569
	for (; pfn < end_pfn; pfn++) {
1570
		if (!deferred_pfn_valid(pfn)) {
1571
			page = NULL;
1572
			continue;
1573
		} else if (!page || !(pfn & nr_pgmask)) {
1574
			page = pfn_to_page(pfn);
1575
			touch_nmi_watchdog();
1576 1577
		} else {
			page++;
1578
		}
1579
		__init_single_page(page, pfn, zid, nid);
1580
		nr_pages++;
1581
	}
1582
	return (nr_pages);
1583 1584
}

1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
/*
 * This function is meant to pre-load the iterator for the zone init.
 * Specifically it walks through the ranges until we are caught up to the
 * first_init_pfn value and exits there. If we never encounter the value we
 * return false indicating there are no valid ranges left.
 */
static bool __init
deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
				    unsigned long *spfn, unsigned long *epfn,
				    unsigned long first_init_pfn)
{
	u64 j;

	/*
	 * Start out by walking through the ranges in this zone that have
	 * already been initialized. We don't need to do anything with them
	 * so we just need to flush them out of the system.
	 */
	for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
		if (*epfn <= first_init_pfn)
			continue;
		if (*spfn < first_init_pfn)
			*spfn = first_init_pfn;
		*i = j;
		return true;
	}

	return false;
}

/*
 * Initialize and free pages. We do it in two loops: first we initialize
 * struct page, then free to buddy allocator, because while we are
 * freeing pages we can access pages that are ahead (computing buddy
 * page in __free_one_page()).
 *
 * In order to try and keep some memory in the cache we have the loop
 * broken along max page order boundaries. This way we will not cause
 * any issues with the buddy page computation.
 */
static unsigned long __init
deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
		       unsigned long *end_pfn)
{
	unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
	unsigned long spfn = *start_pfn, epfn = *end_pfn;
	unsigned long nr_pages = 0;
	u64 j = *i;

	/* First we loop through and initialize the page values */
	for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
		unsigned long t;

		if (mo_pfn <= *start_pfn)
			break;

		t = min(mo_pfn, *end_pfn);
		nr_pages += deferred_init_pages(zone, *start_pfn, t);

		if (mo_pfn < *end_pfn) {
			*start_pfn = mo_pfn;
			break;
		}
	}

	/* Reset values and now loop through freeing pages as needed */
	swap(j, *i);

	for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
		unsigned long t;

		if (mo_pfn <= spfn)
			break;

		t = min(mo_pfn, epfn);
		deferred_free_pages(spfn, t);

		if (mo_pfn <= epfn)
			break;
	}

	return nr_pages;
}

1669 1670 1671 1672 1673
/*
 * Release the pending interrupts for every TICK_PAGE_COUNT pages.
 */
#define TICK_PAGE_COUNT	(32 * 1024)

1674
/* Initialise remaining memory on a node */
1675
static int __init deferred_init_memmap(void *data)
1676
{
1677
	pg_data_t *pgdat = data;
1678
	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
1679
	unsigned long spfn = 0, epfn = 0, nr_pages = 0, prev_nr_pages = 0;
1680
	unsigned long first_init_pfn, flags;
1681 1682
	unsigned long start = jiffies;
	struct zone *zone;
1683
	int zid;
1684
	u64 i;
1685

1686 1687 1688 1689
	/* Bind memory initialisation thread to a local node if possible */
	if (!cpumask_empty(cpumask))
		set_cpus_allowed_ptr(current, cpumask);

1690
again:
1691 1692
	pgdat_resize_lock(pgdat, &flags);
	first_init_pfn = pgdat->first_deferred_pfn;
1693
	if (first_init_pfn == ULONG_MAX) {
1694
		pgdat_resize_unlock(pgdat, &flags);
1695
		pgdat_init_report_one_done();
1696 1697 1698
		return 0;
	}

1699 1700 1701 1702 1703 1704 1705 1706 1707 1708
	/* Sanity check boundaries */
	BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
	BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));

	/* Only the highest zone is deferred so find it */
	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
		zone = pgdat->node_zones + zid;
		if (first_init_pfn < zone_end_pfn(zone))
			break;
	}
1709 1710 1711 1712 1713

	/* If the zone is empty somebody else may have cleared out the zone */
	if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
						 first_init_pfn))
		goto zone_empty;
1714

1715
	/*
1716 1717 1718
	 * Initialize and free pages in MAX_ORDER sized increments so
	 * that we can avoid introducing any issues with the buddy
	 * allocator.
1719
	 */
1720
	while (spfn < epfn) {
1721
		nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
		/*
		 * Release the interrupts for every TICK_PAGE_COUNT pages
		 * (128MB) to give tick timer the chance to update the
		 * system jiffies.
		 */
		if ((nr_pages - prev_nr_pages) > TICK_PAGE_COUNT) {
			prev_nr_pages = nr_pages;
			pgdat->first_deferred_pfn = spfn;
			pgdat_resize_unlock(pgdat, &flags);
			goto again;
		}
	}

1735
zone_empty:
1736
	pgdat->first_deferred_pfn = ULONG_MAX;
1737
	pgdat_resize_unlock(pgdat, &flags);
1738 1739 1740 1741

	/* Sanity check that the next zone really is unpopulated */
	WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));

1742 1743
	pr_info("node %d initialised, %lu pages in %ums\n",
		pgdat->node_id,	nr_pages, jiffies_to_msecs(jiffies - start));
1744 1745

	pgdat_init_report_one_done();
1746 1747
	return 0;
}
1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767

/*
 * If this zone has deferred pages, try to grow it by initializing enough
 * deferred pages to satisfy the allocation specified by order, rounded up to
 * the nearest PAGES_PER_SECTION boundary.  So we're adding memory in increments
 * of SECTION_SIZE bytes by initializing struct pages in increments of
 * PAGES_PER_SECTION * sizeof(struct page) bytes.
 *
 * Return true when zone was grown, otherwise return false. We return true even
 * when we grow less than requested, to let the caller decide if there are
 * enough pages to satisfy the allocation.
 *
 * Note: We use noinline because this function is needed only during boot, and
 * it is called from a __ref function _deferred_grow_zone. This way we are
 * making sure that it is not inlined into permanent text section.
 */
static noinline bool __init
deferred_grow_zone(struct zone *zone, unsigned int order)
{
	unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
1768
	pg_data_t *pgdat = zone->zone_pgdat;
1769
	unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
1770 1771
	unsigned long spfn, epfn, flags;
	unsigned long nr_pages = 0;
1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
	u64 i;

	/* Only the last zone may have deferred pages */
	if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
		return false;

	pgdat_resize_lock(pgdat, &flags);

	/*
	 * If deferred pages have been initialized while we were waiting for
	 * the lock, return true, as the zone was grown.  The caller will retry
	 * this zone.  We won't return to this function since the caller also
	 * has this static branch.
	 */
	if (!static_branch_unlikely(&deferred_pages)) {
		pgdat_resize_unlock(pgdat, &flags);
		return true;
	}

	/*
	 * If someone grew this zone while we were waiting for spinlock, return
	 * true, as there might be enough pages already.
	 */
	if (first_deferred_pfn != pgdat->first_deferred_pfn) {
		pgdat_resize_unlock(pgdat, &flags);
		return true;
	}

1800 1801 1802 1803
	/* If the zone is empty somebody else may have cleared out the zone */
	if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
						 first_deferred_pfn)) {
		pgdat->first_deferred_pfn = ULONG_MAX;
1804
		pgdat_resize_unlock(pgdat, &flags);
1805 1806
		/* Retry only once. */
		return first_deferred_pfn != ULONG_MAX;
1807 1808
	}

1809 1810 1811 1812 1813 1814 1815 1816 1817 1818
	/*
	 * Initialize and free pages in MAX_ORDER sized increments so
	 * that we can avoid introducing any issues with the buddy
	 * allocator.
	 */
	while (spfn < epfn) {
		/* update our first deferred PFN for this section */
		first_deferred_pfn = spfn;

		nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
1819

1820 1821 1822
		/* We should only stop along section boundaries */
		if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
			continue;
1823

1824
		/* If our quota has been met we can stop here */
1825 1826 1827 1828
		if (nr_pages >= nr_pages_needed)
			break;
	}

1829
	pgdat->first_deferred_pfn = spfn;
1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846
	pgdat_resize_unlock(pgdat, &flags);

	return nr_pages > 0;
}

/*
 * deferred_grow_zone() is __init, but it is called from
 * get_page_from_freelist() during early boot until deferred_pages permanently
 * disables this call. This is why we have refdata wrapper to avoid warning,
 * and to ensure that the function body gets unloaded.
 */
static bool __ref
_deferred_grow_zone(struct zone *zone, unsigned int order)
{
	return deferred_grow_zone(zone, order);
}

1847
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1848 1849 1850

void __init page_alloc_init_late(void)
{
1851 1852 1853
	struct zone *zone;

#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1854 1855
	int nid;

1856 1857
	/* There will be num_node_state(N_MEMORY) threads */
	atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
1858 1859 1860 1861 1862
	for_each_node_state(nid, N_MEMORY) {
		kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
	}

	/* Block until all are initialised */
1863
	wait_for_completion(&pgdat_init_all_done_comp);
1864

1865 1866 1867 1868 1869 1870 1871 1872
	/*
	 * The number of managed pages has changed due to the initialisation
	 * so the pcpu batch and high limits needs to be updated or the limits
	 * will be artificially small.
	 */
	for_each_populated_zone(zone)
		zone_pcp_update(zone);

1873 1874 1875 1876 1877 1878
	/*
	 * We initialized the rest of the deferred pages.  Permanently disable
	 * on-demand struct page initialization.
	 */
	static_branch_disable(&deferred_pages);

1879 1880
	/* Reinit limits that are based on free pages after the kernel is up */
	files_maxfiles_init();
1881
#endif
P
Pavel Tatashin 已提交
1882 1883 1884 1885
#ifdef CONFIG_ARCH_DISCARD_MEMBLOCK
	/* Discard memblock private memory */
	memblock_discard();
#endif
1886 1887 1888

	for_each_populated_zone(zone)
		set_zone_contiguous(zone);
1889 1890
}

1891
#ifdef CONFIG_CMA
1892
/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
1893 1894 1895 1896 1897 1898 1899 1900
void __init init_cma_reserved_pageblock(struct page *page)
{
	unsigned i = pageblock_nr_pages;
	struct page *p = page;

	do {
		__ClearPageReserved(p);
		set_page_count(p, 0);
1901
	} while (++p, --i);
1902 1903

	set_pageblock_migratetype(page, MIGRATE_CMA);
1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917

	if (pageblock_order >= MAX_ORDER) {
		i = pageblock_nr_pages;
		p = page;
		do {
			set_page_refcounted(p);
			__free_pages(p, MAX_ORDER - 1);
			p += MAX_ORDER_NR_PAGES;
		} while (i -= MAX_ORDER_NR_PAGES);
	} else {
		set_page_refcounted(page);
		__free_pages(page, pageblock_order);
	}

1918
	adjust_managed_page_count(page, pageblock_nr_pages);
1919 1920
}
#endif
L
Linus Torvalds 已提交
1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933

/*
 * The order of subdivision here is critical for the IO subsystem.
 * Please do not alter this order without good reasons and regression
 * testing. Specifically, as large blocks of memory are subdivided,
 * the order in which smaller blocks are delivered depends on the order
 * they're subdivided in this function. This is the primary factor
 * influencing the order in which pages are delivered to the IO
 * subsystem according to empirical testing, and this is also justified
 * by considering the behavior of a buddy system containing a single
 * large block of memory acted on by a series of small allocations.
 * This behavior is a critical factor in sglist merging's success.
 *
1934
 * -- nyc
L
Linus Torvalds 已提交
1935
 */
N
Nick Piggin 已提交
1936
static inline void expand(struct zone *zone, struct page *page,
1937
	int low, int high, int migratetype)
L
Linus Torvalds 已提交
1938 1939 1940 1941 1942 1943
{
	unsigned long size = 1 << high;

	while (high > low) {
		high--;
		size >>= 1;
1944
		VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
1945

1946 1947 1948 1949 1950 1951 1952
		/*
		 * Mark as guard pages (or page), that will allow to
		 * merge back to allocator when buddy will be freed.
		 * Corresponding page table entries will not be touched,
		 * pages will stay not present in virtual address space
		 */
		if (set_page_guard(zone, &page[size], high, migratetype))
1953
			continue;
1954

1955
		add_to_free_list(&page[size], zone, high, migratetype);
L
Linus Torvalds 已提交
1956 1957 1958 1959
		set_page_order(&page[size], high);
	}
}

1960
static void check_new_page_bad(struct page *page)
L
Linus Torvalds 已提交
1961
{
1962 1963
	const char *bad_reason = NULL;
	unsigned long bad_flags = 0;
1964

1965
	if (unlikely(atomic_read(&page->_mapcount) != -1))
1966 1967 1968
		bad_reason = "nonzero mapcount";
	if (unlikely(page->mapping != NULL))
		bad_reason = "non-NULL mapping";
1969
	if (unlikely(page_ref_count(page) != 0))
1970
		bad_reason = "nonzero _count";
1971 1972 1973
	if (unlikely(page->flags & __PG_HWPOISON)) {
		bad_reason = "HWPoisoned (hardware-corrupted)";
		bad_flags = __PG_HWPOISON;
1974 1975 1976
		/* Don't complain about hwpoisoned pages */
		page_mapcount_reset(page); /* remove PageBuddy */
		return;
1977
	}
1978 1979 1980 1981
	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
		bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
		bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
	}
1982 1983 1984 1985
#ifdef CONFIG_MEMCG
	if (unlikely(page->mem_cgroup))
		bad_reason = "page still charged to cgroup";
#endif
1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
	bad_page(page, bad_reason, bad_flags);
}

/*
 * This page is about to be returned from the page allocator
 */
static inline int check_new_page(struct page *page)
{
	if (likely(page_expected_state(page,
				PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
		return 0;

	check_new_page_bad(page);
	return 1;
2000 2001
}

2002
static inline bool free_pages_prezeroed(void)
2003 2004
{
	return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
2005
		page_poisoning_enabled();
2006 2007
}

2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041
#ifdef CONFIG_DEBUG_VM
static bool check_pcp_refill(struct page *page)
{
	return false;
}

static bool check_new_pcp(struct page *page)
{
	return check_new_page(page);
}
#else
static bool check_pcp_refill(struct page *page)
{
	return check_new_page(page);
}
static bool check_new_pcp(struct page *page)
{
	return false;
}
#endif /* CONFIG_DEBUG_VM */

static bool check_new_pages(struct page *page, unsigned int order)
{
	int i;
	for (i = 0; i < (1 << order); i++) {
		struct page *p = page + i;

		if (unlikely(check_new_page(p)))
			return true;
	}

	return false;
}

2042 2043 2044 2045 2046 2047 2048 2049 2050
inline void post_alloc_hook(struct page *page, unsigned int order,
				gfp_t gfp_flags)
{
	set_page_private(page, 0);
	set_page_refcounted(page);

	arch_alloc_page(page, order);
	kernel_map_pages(page, 1 << order, 1);
	kasan_alloc_pages(page, order);
Q
Qian Cai 已提交
2051
	kernel_poison_pages(page, 1 << order, 1);
2052 2053 2054
	set_page_owner(page, order, gfp_flags);
}

2055
static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
2056
							unsigned int alloc_flags)
2057 2058
{
	int i;
2059

2060
	post_alloc_hook(page, order, gfp_flags);
N
Nick Piggin 已提交
2061

2062
	if (!free_pages_prezeroed() && (gfp_flags & __GFP_ZERO))
2063 2064
		for (i = 0; i < (1 << order); i++)
			clear_highpage(page + i);
N
Nick Piggin 已提交
2065 2066 2067 2068

	if (order && (gfp_flags & __GFP_COMP))
		prep_compound_page(page, order);

2069
	/*
2070
	 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
2071 2072 2073 2074
	 * allocate the page. The expectation is that the caller is taking
	 * steps that will free more memory. The caller should avoid the page
	 * being used for !PFMEMALLOC purposes.
	 */
2075 2076 2077 2078
	if (alloc_flags & ALLOC_NO_WATERMARKS)
		set_page_pfmemalloc(page);
	else
		clear_page_pfmemalloc(page);
L
Linus Torvalds 已提交
2079 2080
}

2081 2082 2083 2084
/*
 * Go through the free lists for the given migratetype and remove
 * the smallest available page from the freelists
 */
2085
static __always_inline
2086
struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
2087 2088 2089
						int migratetype)
{
	unsigned int current_order;
2090
	struct free_area *area;
2091 2092 2093 2094 2095
	struct page *page;

	/* Find a page of the appropriate size in the preferred list */
	for (current_order = order; current_order < MAX_ORDER; ++current_order) {
		area = &(zone->free_area[current_order]);
2096
		page = get_page_from_free_area(area, migratetype);
2097 2098
		if (!page)
			continue;
2099 2100
		del_page_from_free_list(page, zone, current_order);
		expand(zone, page, order, current_order, migratetype);
2101
		set_pcppage_migratetype(page, migratetype);
2102 2103 2104 2105 2106 2107 2108
		return page;
	}

	return NULL;
}


2109 2110 2111 2112
/*
 * This array describes the order lists are fallen back to when
 * the free lists for the desirable migrate type are depleted
 */
2113
static int fallbacks[MIGRATE_TYPES][4] = {
2114 2115 2116
	[MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_TYPES },
	[MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_TYPES },
	[MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
2117
#ifdef CONFIG_CMA
2118
	[MIGRATE_CMA]         = { MIGRATE_TYPES }, /* Never used */
2119
#endif
2120
#ifdef CONFIG_MEMORY_ISOLATION
2121
	[MIGRATE_ISOLATE]     = { MIGRATE_TYPES }, /* Never used */
2122
#endif
2123 2124
};

2125
#ifdef CONFIG_CMA
2126
static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2127 2128 2129 2130 2131 2132 2133 2134 2135
					unsigned int order)
{
	return __rmqueue_smallest(zone, order, MIGRATE_CMA);
}
#else
static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
					unsigned int order) { return NULL; }
#endif

2136 2137
/*
 * Move the free pages in a range to the free lists of the requested type.
2138
 * Note that start_page and end_pages are not aligned on a pageblock
2139 2140
 * boundary. If alignment is required, use move_freepages_block()
 */
2141
static int move_freepages(struct zone *zone,
A
Adrian Bunk 已提交
2142
			  struct page *start_page, struct page *end_page,
2143
			  int migratetype, int *num_movable)
2144 2145
{
	struct page *page;
2146
	unsigned int order;
2147
	int pages_moved = 0;
2148 2149 2150 2151 2152 2153 2154

#ifndef CONFIG_HOLES_IN_ZONE
	/*
	 * page_zone is not safe to call in this context when
	 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
	 * anyway as we check zone boundaries in move_freepages_block().
	 * Remove at a later date when no bug reports exist related to
M
Mel Gorman 已提交
2155
	 * grouping pages by mobility
2156
	 */
2157 2158 2159
	VM_BUG_ON(pfn_valid(page_to_pfn(start_page)) &&
	          pfn_valid(page_to_pfn(end_page)) &&
	          page_zone(start_page) != page_zone(end_page));
2160 2161
#endif

2162 2163 2164
	if (num_movable)
		*num_movable = 0;

2165 2166 2167 2168 2169 2170
	for (page = start_page; page <= end_page;) {
		if (!pfn_valid_within(page_to_pfn(page))) {
			page++;
			continue;
		}

2171 2172 2173
		/* Make sure we are not inadvertently changing nodes */
		VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);

2174
		if (!PageBuddy(page)) {
2175 2176 2177 2178 2179 2180 2181 2182 2183
			/*
			 * We assume that pages that could be isolated for
			 * migration are movable. But we don't actually try
			 * isolating, as that would be expensive.
			 */
			if (num_movable &&
					(PageLRU(page) || __PageMovable(page)))
				(*num_movable)++;

2184 2185 2186 2187 2188
			page++;
			continue;
		}

		order = page_order(page);
2189
		move_to_free_list(page, zone, order, migratetype);
2190
		page += 1 << order;
2191
		pages_moved += 1 << order;
2192 2193
	}

2194
	return pages_moved;
2195 2196
}

2197
int move_freepages_block(struct zone *zone, struct page *page,
2198
				int migratetype, int *num_movable)
2199 2200 2201 2202 2203
{
	unsigned long start_pfn, end_pfn;
	struct page *start_page, *end_page;

	start_pfn = page_to_pfn(page);
2204
	start_pfn = start_pfn & ~(pageblock_nr_pages-1);
2205
	start_page = pfn_to_page(start_pfn);
2206 2207
	end_page = start_page + pageblock_nr_pages - 1;
	end_pfn = start_pfn + pageblock_nr_pages - 1;
2208 2209

	/* Do not cross zone boundaries */
2210
	if (!zone_spans_pfn(zone, start_pfn))
2211
		start_page = page;
2212
	if (!zone_spans_pfn(zone, end_pfn))
2213 2214
		return 0;

2215 2216
	return move_freepages(zone, start_page, end_page, migratetype,
								num_movable);
2217 2218
}

2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229
static void change_pageblock_range(struct page *pageblock_page,
					int start_order, int migratetype)
{
	int nr_pageblocks = 1 << (start_order - pageblock_order);

	while (nr_pageblocks--) {
		set_pageblock_migratetype(pageblock_page, migratetype);
		pageblock_page += pageblock_nr_pages;
	}
}

2230
/*
2231 2232 2233 2234 2235 2236 2237 2238 2239 2240
 * When we are falling back to another migratetype during allocation, try to
 * steal extra free pages from the same pageblocks to satisfy further
 * allocations, instead of polluting multiple pageblocks.
 *
 * If we are stealing a relatively large buddy page, it is likely there will
 * be more free pages in the pageblock, so try to steal them all. For
 * reclaimable and unmovable allocations, we steal regardless of page size,
 * as fragmentation caused by those allocations polluting movable pageblocks
 * is worse than movable allocations stealing from unmovable and reclaimable
 * pageblocks.
2241
 */
2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265
static bool can_steal_fallback(unsigned int order, int start_mt)
{
	/*
	 * Leaving this order check is intended, although there is
	 * relaxed order check in next check. The reason is that
	 * we can actually steal whole pageblock if this condition met,
	 * but, below check doesn't guarantee it and that is just heuristic
	 * so could be changed anytime.
	 */
	if (order >= pageblock_order)
		return true;

	if (order >= pageblock_order / 2 ||
		start_mt == MIGRATE_RECLAIMABLE ||
		start_mt == MIGRATE_UNMOVABLE ||
		page_group_by_mobility_disabled)
		return true;

	return false;
}

/*
 * This function implements actual steal behaviour. If order is large enough,
 * we can steal whole pageblock. If not, we first move freepages in this
2266 2267 2268 2269
 * pageblock to our migratetype and determine how many already-allocated pages
 * are there in the pageblock with a compatible migratetype. If at least half
 * of pages are free or compatible, we can change migratetype of the pageblock
 * itself, so pages freed in the future will be put on the correct free list.
2270 2271
 */
static void steal_suitable_fallback(struct zone *zone, struct page *page,
2272
					int start_type, bool whole_block)
2273
{
2274
	unsigned int current_order = page_order(page);
2275 2276 2277 2278
	int free_pages, movable_pages, alike_pages;
	int old_block_type;

	old_block_type = get_pageblock_migratetype(page);
2279

2280 2281 2282 2283
	/*
	 * This can happen due to races and we want to prevent broken
	 * highatomic accounting.
	 */
2284
	if (is_migrate_highatomic(old_block_type))
2285 2286
		goto single_page;

2287 2288 2289
	/* Take ownership for orders >= pageblock_order */
	if (current_order >= pageblock_order) {
		change_pageblock_range(page, current_order, start_type);
2290
		goto single_page;
2291 2292
	}

2293 2294 2295 2296
	/* We are not allowed to try stealing from the whole block */
	if (!whole_block)
		goto single_page;

2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320
	free_pages = move_freepages_block(zone, page, start_type,
						&movable_pages);
	/*
	 * Determine how many pages are compatible with our allocation.
	 * For movable allocation, it's the number of movable pages which
	 * we just obtained. For other types it's a bit more tricky.
	 */
	if (start_type == MIGRATE_MOVABLE) {
		alike_pages = movable_pages;
	} else {
		/*
		 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
		 * to MOVABLE pageblock, consider all non-movable pages as
		 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
		 * vice versa, be conservative since we can't distinguish the
		 * exact migratetype of non-movable pages.
		 */
		if (old_block_type == MIGRATE_MOVABLE)
			alike_pages = pageblock_nr_pages
						- (free_pages + movable_pages);
		else
			alike_pages = 0;
	}

2321
	/* moving whole block can fail due to zone boundary conditions */
2322
	if (!free_pages)
2323
		goto single_page;
2324

2325 2326 2327 2328 2329
	/*
	 * If a sufficient number of pages in the block are either free or of
	 * comparable migratability as our allocation, claim the whole block.
	 */
	if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
2330 2331
			page_group_by_mobility_disabled)
		set_pageblock_migratetype(page, start_type);
2332 2333 2334 2335

	return;

single_page:
2336
	move_to_free_list(page, zone, current_order, start_type);
2337 2338
}

2339 2340 2341 2342 2343 2344 2345 2346
/*
 * Check whether there is a suitable fallback freepage with requested order.
 * If only_stealable is true, this function returns fallback_mt only if
 * we can steal other freepages all together. This would help to reduce
 * fragmentation due to mixed migratetype pages in one pageblock.
 */
int find_suitable_fallback(struct free_area *area, unsigned int order,
			int migratetype, bool only_stealable, bool *can_steal)
2347 2348 2349 2350 2351 2352 2353 2354 2355 2356
{
	int i;
	int fallback_mt;

	if (area->nr_free == 0)
		return -1;

	*can_steal = false;
	for (i = 0;; i++) {
		fallback_mt = fallbacks[migratetype][i];
2357
		if (fallback_mt == MIGRATE_TYPES)
2358 2359
			break;

2360
		if (free_area_empty(area, fallback_mt))
2361
			continue;
2362

2363 2364 2365
		if (can_steal_fallback(order, migratetype))
			*can_steal = true;

2366 2367 2368 2369 2370
		if (!only_stealable)
			return fallback_mt;

		if (*can_steal)
			return fallback_mt;
2371
	}
2372 2373

	return -1;
2374 2375
}

2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
/*
 * Reserve a pageblock for exclusive use of high-order atomic allocations if
 * there are no empty page blocks that contain a page with a suitable order
 */
static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
				unsigned int alloc_order)
{
	int mt;
	unsigned long max_managed, flags;

	/*
	 * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
	 * Check is race-prone but harmless.
	 */
	max_managed = (zone->managed_pages / 100) + pageblock_nr_pages;
	if (zone->nr_reserved_highatomic >= max_managed)
		return;

	spin_lock_irqsave(&zone->lock, flags);

	/* Recheck the nr_reserved_highatomic limit under the lock */
	if (zone->nr_reserved_highatomic >= max_managed)
		goto out_unlock;

	/* Yoink! */
	mt = get_pageblock_migratetype(page);
2402 2403
	if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
	    && !is_migrate_cma(mt)) {
2404 2405
		zone->nr_reserved_highatomic += pageblock_nr_pages;
		set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2406
		move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417
	}

out_unlock:
	spin_unlock_irqrestore(&zone->lock, flags);
}

/*
 * Used when an allocation is about to fail under memory pressure. This
 * potentially hurts the reliability of high-order allocations when under
 * intense memory pressure but failed atomic allocations should be easier
 * to recover from than an OOM.
2418 2419 2420
 *
 * If @force is true, try to unreserve a pageblock even though highatomic
 * pageblock is exhausted.
2421
 */
2422 2423
static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
						bool force)
2424 2425 2426 2427 2428 2429 2430
{
	struct zonelist *zonelist = ac->zonelist;
	unsigned long flags;
	struct zoneref *z;
	struct zone *zone;
	struct page *page;
	int order;
2431
	bool ret;
2432 2433 2434

	for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
								ac->nodemask) {
2435 2436 2437 2438 2439 2440
		/*
		 * Preserve at least one pageblock unless memory pressure
		 * is really high.
		 */
		if (!force && zone->nr_reserved_highatomic <=
					pageblock_nr_pages)
2441 2442 2443 2444 2445 2446
			continue;

		spin_lock_irqsave(&zone->lock, flags);
		for (order = 0; order < MAX_ORDER; order++) {
			struct free_area *area = &(zone->free_area[order]);

2447
			page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
2448
			if (!page)
2449 2450 2451
				continue;

			/*
2452 2453 2454 2455 2456
			 * In page freeing path, migratetype change is racy so
			 * we can counter several free pages in a pageblock
			 * in this loop althoug we changed the pageblock type
			 * from highatomic to ac->migratetype. So we should
			 * adjust the count once.
2457
			 */
2458
			if (is_migrate_highatomic_page(page)) {
2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469
				/*
				 * It should never happen but changes to
				 * locking could inadvertently allow a per-cpu
				 * drain to add pages to MIGRATE_HIGHATOMIC
				 * while unreserving so be safe and watch for
				 * underflows.
				 */
				zone->nr_reserved_highatomic -= min(
						pageblock_nr_pages,
						zone->nr_reserved_highatomic);
			}
2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480

			/*
			 * Convert to ac->migratetype and avoid the normal
			 * pageblock stealing heuristics. Minimally, the caller
			 * is doing the work and needs the pages. More
			 * importantly, if the block was always converted to
			 * MIGRATE_UNMOVABLE or another type then the number
			 * of pageblocks that cannot be completely freed
			 * may increase.
			 */
			set_pageblock_migratetype(page, ac->migratetype);
2481 2482
			ret = move_freepages_block(zone, page, ac->migratetype,
									NULL);
2483 2484 2485 2486
			if (ret) {
				spin_unlock_irqrestore(&zone->lock, flags);
				return ret;
			}
2487 2488 2489
		}
		spin_unlock_irqrestore(&zone->lock, flags);
	}
2490 2491

	return false;
2492 2493
}

2494 2495 2496 2497 2498
/*
 * Try finding a free buddy page on the fallback list and put it on the free
 * list of requested migratetype, possibly along with other pages from the same
 * block, depending on fragmentation avoidance heuristics. Returns true if
 * fallback was found so that __rmqueue_smallest() can grab it.
2499 2500 2501 2502
 *
 * The use of signed ints for order and current_order is a deliberate
 * deviation from the rest of this file, to make the for loop
 * condition simpler.
2503
 */
2504
static __always_inline bool
2505
__rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
2506
{
2507
	struct free_area *area;
2508
	int current_order;
2509
	struct page *page;
2510 2511
	int fallback_mt;
	bool can_steal;
2512

2513 2514 2515 2516 2517
	/*
	 * Find the largest available free page in the other list. This roughly
	 * approximates finding the pageblock with the most free pages, which
	 * would be too costly to do exactly.
	 */
2518
	for (current_order = MAX_ORDER - 1; current_order >= order;
2519
				--current_order) {
2520 2521
		area = &(zone->free_area[current_order]);
		fallback_mt = find_suitable_fallback(area, current_order,
2522
				start_migratetype, false, &can_steal);
2523 2524
		if (fallback_mt == -1)
			continue;
2525

2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536
		/*
		 * We cannot steal all free pages from the pageblock and the
		 * requested migratetype is movable. In that case it's better to
		 * steal and split the smallest available page instead of the
		 * largest available page, because even if the next movable
		 * allocation falls back into a different pageblock than this
		 * one, it won't cause permanent fragmentation.
		 */
		if (!can_steal && start_migratetype == MIGRATE_MOVABLE
					&& current_order > order)
			goto find_smallest;
2537

2538 2539
		goto do_steal;
	}
2540

2541
	return false;
2542

2543 2544 2545 2546 2547 2548 2549 2550
find_smallest:
	for (current_order = order; current_order < MAX_ORDER;
							current_order++) {
		area = &(zone->free_area[current_order]);
		fallback_mt = find_suitable_fallback(area, current_order,
				start_migratetype, false, &can_steal);
		if (fallback_mt != -1)
			break;
2551 2552
	}

2553 2554 2555 2556 2557 2558 2559
	/*
	 * This should not happen - we already found a suitable fallback
	 * when looking for the largest page.
	 */
	VM_BUG_ON(current_order == MAX_ORDER);

do_steal:
2560
	page = get_page_from_free_area(area, fallback_mt);
2561 2562 2563 2564 2565 2566 2567 2568

	steal_suitable_fallback(zone, page, start_migratetype, can_steal);

	trace_mm_page_alloc_extfrag(page, order, current_order,
		start_migratetype, fallback_mt);

	return true;

2569 2570
}

2571
/*
L
Linus Torvalds 已提交
2572 2573 2574
 * Do the hard work of removing an element from the buddy allocator.
 * Call me with the zone->lock already held.
 */
2575 2576
static __always_inline struct page *
__rmqueue(struct zone *zone, unsigned int order, int migratetype)
L
Linus Torvalds 已提交
2577 2578 2579
{
	struct page *page;

2580
retry:
2581
	page = __rmqueue_smallest(zone, order, migratetype);
2582
	if (unlikely(!page)) {
2583 2584 2585
		if (migratetype == MIGRATE_MOVABLE)
			page = __rmqueue_cma_fallback(zone, order);

2586 2587
		if (!page && __rmqueue_fallback(zone, order, migratetype))
			goto retry;
2588 2589
	}

2590
	trace_mm_page_alloc_zone_locked(page, order, migratetype);
2591
	return page;
L
Linus Torvalds 已提交
2592 2593
}

2594
/*
L
Linus Torvalds 已提交
2595 2596 2597 2598
 * Obtain a specified number of elements from the buddy allocator, all under
 * a single hold of the lock, for efficiency.  Add them to the supplied list.
 * Returns the number of new pages which were placed at *list.
 */
2599
static int rmqueue_bulk(struct zone *zone, unsigned int order,
2600
			unsigned long count, struct list_head *list,
M
Mel Gorman 已提交
2601
			int migratetype)
L
Linus Torvalds 已提交
2602
{
2603
	int i, alloced = 0;
2604

2605
	spin_lock(&zone->lock);
L
Linus Torvalds 已提交
2606
	for (i = 0; i < count; ++i) {
2607
		struct page *page = __rmqueue(zone, order, migratetype);
N
Nick Piggin 已提交
2608
		if (unlikely(page == NULL))
L
Linus Torvalds 已提交
2609
			break;
2610

2611 2612 2613
		if (unlikely(check_pcp_refill(page)))
			continue;

2614
		/*
2615 2616 2617 2618 2619 2620 2621 2622
		 * Split buddy pages returned by expand() are received here in
		 * physical page order. The page is added to the tail of
		 * caller's list. From the callers perspective, the linked list
		 * is ordered by page number under some conditions. This is
		 * useful for IO devices that can forward direction from the
		 * head, thus also in the physical page order. This is useful
		 * for IO devices that can merge IO requests if the physical
		 * pages are ordered properly.
2623
		 */
2624
		list_add_tail(&page->lru, list);
2625
		alloced++;
2626
		if (is_migrate_cma(get_pcppage_migratetype(page)))
2627 2628
			__mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
					      -(1 << order));
L
Linus Torvalds 已提交
2629
	}
2630 2631 2632 2633 2634 2635 2636

	/*
	 * i pages were removed from the buddy list even if some leak due
	 * to check_pcp_refill failing so adjust NR_FREE_PAGES based
	 * on i. Do not confuse with 'alloced' which is the number of
	 * pages added to the pcp list.
	 */
2637
	__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
2638
	spin_unlock(&zone->lock);
2639
	return alloced;
L
Linus Torvalds 已提交
2640 2641
}

2642
#ifdef CONFIG_NUMA
2643
/*
2644 2645 2646 2647
 * Called from the vmstat counter updater to drain pagesets of this
 * currently executing processor on remote nodes after they have
 * expired.
 *
2648 2649
 * Note that this function must be called with the thread pinned to
 * a single processor.
2650
 */
2651
void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
2652 2653
{
	unsigned long flags;
2654
	int to_drain, batch;
2655

2656
	local_irq_save(flags);
2657
	batch = READ_ONCE(pcp->batch);
2658
	to_drain = min(pcp->count, batch);
2659
	if (to_drain > 0)
2660
		free_pcppages_bulk(zone, to_drain, pcp);
2661
	local_irq_restore(flags);
2662 2663 2664
}
#endif

2665
/*
2666
 * Drain pcplists of the indicated processor and zone.
2667 2668 2669 2670 2671
 *
 * The processor must either be the current processor and the
 * thread pinned to the current processor or a processor that
 * is not online.
 */
2672
static void drain_pages_zone(unsigned int cpu, struct zone *zone)
L
Linus Torvalds 已提交
2673
{
N
Nick Piggin 已提交
2674
	unsigned long flags;
2675 2676
	struct per_cpu_pageset *pset;
	struct per_cpu_pages *pcp;
L
Linus Torvalds 已提交
2677

2678 2679
	local_irq_save(flags);
	pset = per_cpu_ptr(zone->pageset, cpu);
L
Linus Torvalds 已提交
2680

2681
	pcp = &pset->pcp;
2682
	if (pcp->count)
2683 2684 2685
		free_pcppages_bulk(zone, pcp->count, pcp);
	local_irq_restore(flags);
}
2686

2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699
/*
 * Drain pcplists of all zones on the indicated processor.
 *
 * The processor must either be the current processor and the
 * thread pinned to the current processor or a processor that
 * is not online.
 */
static void drain_pages(unsigned int cpu)
{
	struct zone *zone;

	for_each_populated_zone(zone) {
		drain_pages_zone(cpu, zone);
L
Linus Torvalds 已提交
2700 2701 2702
	}
}

2703 2704
/*
 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
2705 2706 2707
 *
 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
 * the single zone's pages.
2708
 */
2709
void drain_local_pages(struct zone *zone)
2710
{
2711 2712 2713 2714 2715 2716
	int cpu = smp_processor_id();

	if (zone)
		drain_pages_zone(cpu, zone);
	else
		drain_pages(cpu);
2717 2718
}

2719 2720
static void drain_local_pages_wq(struct work_struct *work)
{
2721 2722 2723 2724 2725 2726 2727 2728
	/*
	 * drain_all_pages doesn't use proper cpu hotplug protection so
	 * we can race with cpu offline when the WQ can move this from
	 * a cpu pinned worker to an unbound one. We can operate on a different
	 * cpu which is allright but we also have to make sure to not move to
	 * a different one.
	 */
	preempt_disable();
2729
	drain_local_pages(NULL);
2730
	preempt_enable();
2731 2732
}

2733
/*
2734 2735
 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
 *
2736 2737
 * When zone parameter is non-NULL, spill just the single zone's pages.
 *
2738
 * Note that this can be extremely slow as the draining happens in a workqueue.
2739
 */
2740
void drain_all_pages(struct zone *zone)
2741
{
2742 2743 2744 2745 2746 2747 2748 2749
	int cpu;

	/*
	 * Allocate in the BSS so we wont require allocation in
	 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
	 */
	static cpumask_t cpus_with_pcps;

2750 2751 2752 2753 2754 2755 2756
	/*
	 * Make sure nobody triggers this path before mm_percpu_wq is fully
	 * initialized.
	 */
	if (WARN_ON_ONCE(!mm_percpu_wq))
		return;

2757 2758 2759 2760 2761 2762 2763 2764 2765 2766
	/*
	 * Do not drain if one is already in progress unless it's specific to
	 * a zone. Such callers are primarily CMA and memory hotplug and need
	 * the drain to be complete when the call returns.
	 */
	if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
		if (!zone)
			return;
		mutex_lock(&pcpu_drain_mutex);
	}
2767

2768 2769 2770 2771 2772 2773 2774
	/*
	 * We don't care about racing with CPU hotplug event
	 * as offline notification will cause the notified
	 * cpu to drain that CPU pcps and on_each_cpu_mask
	 * disables preemption as part of its processing
	 */
	for_each_online_cpu(cpu) {
2775 2776
		struct per_cpu_pageset *pcp;
		struct zone *z;
2777
		bool has_pcps = false;
2778 2779

		if (zone) {
2780
			pcp = per_cpu_ptr(zone->pageset, cpu);
2781
			if (pcp->pcp.count)
2782
				has_pcps = true;
2783 2784 2785 2786 2787 2788 2789
		} else {
			for_each_populated_zone(z) {
				pcp = per_cpu_ptr(z->pageset, cpu);
				if (pcp->pcp.count) {
					has_pcps = true;
					break;
				}
2790 2791
			}
		}
2792

2793 2794 2795 2796 2797
		if (has_pcps)
			cpumask_set_cpu(cpu, &cpus_with_pcps);
		else
			cpumask_clear_cpu(cpu, &cpus_with_pcps);
	}
2798

2799 2800 2801
	for_each_cpu(cpu, &cpus_with_pcps) {
		struct work_struct *work = per_cpu_ptr(&pcpu_drain, cpu);
		INIT_WORK(work, drain_local_pages_wq);
2802
		queue_work_on(cpu, mm_percpu_wq, work);
2803
	}
2804 2805 2806 2807
	for_each_cpu(cpu, &cpus_with_pcps)
		flush_work(per_cpu_ptr(&pcpu_drain, cpu));

	mutex_unlock(&pcpu_drain_mutex);
2808 2809
}

2810
#ifdef CONFIG_HIBERNATION
L
Linus Torvalds 已提交
2811

2812 2813 2814 2815 2816
/*
 * Touch the watchdog for every WD_PAGE_COUNT pages.
 */
#define WD_PAGE_COUNT	(128*1024)

L
Linus Torvalds 已提交
2817 2818
void mark_free_pages(struct zone *zone)
{
2819
	unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
2820
	unsigned long flags;
2821
	unsigned int order, t;
2822
	struct page *page;
L
Linus Torvalds 已提交
2823

2824
	if (zone_is_empty(zone))
L
Linus Torvalds 已提交
2825 2826 2827
		return;

	spin_lock_irqsave(&zone->lock, flags);
2828

2829
	max_zone_pfn = zone_end_pfn(zone);
2830 2831
	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
		if (pfn_valid(pfn)) {
2832
			page = pfn_to_page(pfn);
2833

2834 2835 2836 2837 2838
			if (!--page_count) {
				touch_nmi_watchdog();
				page_count = WD_PAGE_COUNT;
			}

2839 2840 2841
			if (page_zone(page) != zone)
				continue;

2842 2843
			if (!swsusp_page_is_forbidden(page))
				swsusp_unset_page_free(page);
2844
		}
L
Linus Torvalds 已提交
2845

2846
	for_each_migratetype_order(order, t) {
2847 2848
		list_for_each_entry(page,
				&zone->free_area[order].free_list[t], lru) {
2849
			unsigned long i;
L
Linus Torvalds 已提交
2850

2851
			pfn = page_to_pfn(page);
2852 2853 2854 2855 2856
			for (i = 0; i < (1UL << order); i++) {
				if (!--page_count) {
					touch_nmi_watchdog();
					page_count = WD_PAGE_COUNT;
				}
2857
				swsusp_set_page_free(pfn_to_page(pfn + i));
2858
			}
2859
		}
2860
	}
L
Linus Torvalds 已提交
2861 2862
	spin_unlock_irqrestore(&zone->lock, flags);
}
2863
#endif /* CONFIG_PM */
L
Linus Torvalds 已提交
2864

2865
static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
L
Linus Torvalds 已提交
2866
{
2867
	int migratetype;
L
Linus Torvalds 已提交
2868

2869
	if (!free_pcp_prepare(page))
2870
		return false;
2871

2872
	migratetype = get_pfnblock_migratetype(page, pfn);
2873
	set_pcppage_migratetype(page, migratetype);
2874 2875 2876
	return true;
}

2877
static void free_unref_page_commit(struct page *page, unsigned long pfn)
2878 2879 2880 2881 2882 2883
{
	struct zone *zone = page_zone(page);
	struct per_cpu_pages *pcp;
	int migratetype;

	migratetype = get_pcppage_migratetype(page);
2884
	__count_vm_event(PGFREE);
2885

2886 2887 2888
	/*
	 * We only track unmovable, reclaimable and movable on pcp lists.
	 * Free ISOLATE pages back to the allocator because they are being
2889
	 * offlined but treat HIGHATOMIC as movable pages so we can get those
2890 2891 2892 2893
	 * areas back if necessary. Otherwise, we may have to free
	 * excessively into the page allocator
	 */
	if (migratetype >= MIGRATE_PCPTYPES) {
2894
		if (unlikely(is_migrate_isolate(migratetype))) {
2895
			free_one_page(zone, page, pfn, 0, migratetype);
2896
			return;
2897 2898 2899 2900
		}
		migratetype = MIGRATE_MOVABLE;
	}

2901
	pcp = &this_cpu_ptr(zone->pageset)->pcp;
2902
	list_add(&page->lru, &pcp->lists[migratetype]);
L
Linus Torvalds 已提交
2903
	pcp->count++;
N
Nick Piggin 已提交
2904
	if (pcp->count >= pcp->high) {
2905
		unsigned long batch = READ_ONCE(pcp->batch);
2906
		free_pcppages_bulk(zone, batch, pcp);
N
Nick Piggin 已提交
2907
	}
2908
}
2909

2910 2911 2912
/*
 * Free a 0-order page
 */
2913
void free_unref_page(struct page *page)
2914 2915 2916 2917
{
	unsigned long flags;
	unsigned long pfn = page_to_pfn(page);

2918
	if (!free_unref_page_prepare(page, pfn))
2919 2920 2921
		return;

	local_irq_save(flags);
2922
	free_unref_page_commit(page, pfn);
2923
	local_irq_restore(flags);
L
Linus Torvalds 已提交
2924 2925
}

2926 2927 2928
/*
 * Free a list of 0-order pages
 */
2929
void free_unref_page_list(struct list_head *list)
2930 2931
{
	struct page *page, *next;
2932
	unsigned long flags, pfn;
2933
	int batch_count = 0;
2934 2935 2936 2937

	/* Prepare pages for freeing */
	list_for_each_entry_safe(page, next, list, lru) {
		pfn = page_to_pfn(page);
2938
		if (!free_unref_page_prepare(page, pfn))
2939 2940 2941
			list_del(&page->lru);
		set_page_private(page, pfn);
	}
2942

2943
	local_irq_save(flags);
2944
	list_for_each_entry_safe(page, next, list, lru) {
2945 2946 2947
		unsigned long pfn = page_private(page);

		set_page_private(page, 0);
2948 2949
		trace_mm_page_free_batched(page);
		free_unref_page_commit(page, pfn);
2950 2951 2952 2953 2954 2955 2956 2957 2958 2959

		/*
		 * Guard against excessive IRQ disabled times when we get
		 * a large list of pages to free.
		 */
		if (++batch_count == SWAP_CLUSTER_MAX) {
			local_irq_restore(flags);
			batch_count = 0;
			local_irq_save(flags);
		}
2960
	}
2961
	local_irq_restore(flags);
2962 2963
}

N
Nick Piggin 已提交
2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975
/*
 * split_page takes a non-compound higher-order page, and splits it into
 * n (1<<order) sub-pages: page[0..n]
 * Each sub-page must be freed individually.
 *
 * Note: this is probably too low level an operation for use in drivers.
 * Please consult with lkml before using this in your driver.
 */
void split_page(struct page *page, unsigned int order)
{
	int i;

2976 2977
	VM_BUG_ON_PAGE(PageCompound(page), page);
	VM_BUG_ON_PAGE(!page_count(page), page);
2978

2979
	for (i = 1; i < (1 << order); i++)
2980
		set_page_refcounted(page + i);
2981
	split_page_owner(page, order);
N
Nick Piggin 已提交
2982
}
K
K. Y. Srinivasan 已提交
2983
EXPORT_SYMBOL_GPL(split_page);
N
Nick Piggin 已提交
2984

2985
int __isolate_free_page(struct page *page, unsigned int order)
2986 2987 2988
{
	unsigned long watermark;
	struct zone *zone;
2989
	int mt;
2990 2991 2992 2993

	BUG_ON(!PageBuddy(page));

	zone = page_zone(page);
2994
	mt = get_pageblock_migratetype(page);
2995

2996
	if (!is_migrate_isolate(mt)) {
2997 2998 2999 3000 3001 3002 3003
		/*
		 * Obey watermarks as if the page was being allocated. We can
		 * emulate a high-order watermark check with a raised order-0
		 * watermark, because we already know our high-order page
		 * exists.
		 */
		watermark = min_wmark_pages(zone) + (1UL << order);
3004
		if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
3005 3006
			return 0;

3007
		__mod_zone_freepage_state(zone, -(1UL << order), mt);
3008
	}
3009 3010

	/* Remove page from free list */
3011

3012
	del_page_from_free_list(page, zone, order);
3013

3014 3015 3016 3017
	/*
	 * Set the pageblock if the isolated page is at least half of a
	 * pageblock
	 */
3018 3019
	if (order >= pageblock_order - 1) {
		struct page *endpage = page + (1 << order) - 1;
3020 3021
		for (; page < endpage; page += pageblock_nr_pages) {
			int mt = get_pageblock_migratetype(page);
M
Minchan Kim 已提交
3022
			if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
3023
			    && !is_migrate_highatomic(mt))
3024 3025 3026
				set_pageblock_migratetype(page,
							  MIGRATE_MOVABLE);
		}
3027 3028
	}

3029

3030
	return 1UL << order;
3031 3032
}

3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048
/**
 * __putback_isolated_page - Return a now-isolated page back where we got it
 * @page: Page that was isolated
 * @order: Order of the isolated page
 *
 * This function is meant to return a page pulled from the free lists via
 * __isolate_free_page back to the free lists they were pulled from.
 */
void __putback_isolated_page(struct page *page, unsigned int order, int mt)
{
	struct zone *zone = page_zone(page);

	/* zone lock should be held when this function is called */
	lockdep_assert_held(&zone->lock);

	/* Return isolated page to tail of freelist. */
A
Alexander Duyck 已提交
3049
	__free_one_page(page, page_to_pfn(page), zone, order, mt, false);
3050 3051
}

3052 3053 3054 3055 3056
/*
 * Update NUMA hit/miss statistics
 *
 * Must be called with interrupts disabled.
 */
M
Michal Hocko 已提交
3057
static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
3058 3059
{
#ifdef CONFIG_NUMA
3060
	enum numa_stat_item local_stat = NUMA_LOCAL;
3061

3062 3063 3064 3065
	/* skip numa counters update if numa stats is disabled */
	if (!static_branch_likely(&vm_numa_stat_key))
		return;

3066
	if (zone_to_nid(z) != numa_node_id())
3067 3068
		local_stat = NUMA_OTHER;

3069
	if (zone_to_nid(z) == zone_to_nid(preferred_zone))
3070
		__inc_numa_state(z, NUMA_HIT);
3071
	else {
3072 3073
		__inc_numa_state(z, NUMA_MISS);
		__inc_numa_state(preferred_zone, NUMA_FOREIGN);
3074
	}
3075
	__inc_numa_state(z, local_stat);
3076 3077 3078
#endif
}

3079 3080
/* Remove page from the per-cpu list, caller must protect the list */
static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
M
Mel Gorman 已提交
3081
			struct per_cpu_pages *pcp,
3082 3083 3084 3085 3086 3087 3088 3089
			struct list_head *list)
{
	struct page *page;

	do {
		if (list_empty(list)) {
			pcp->count += rmqueue_bulk(zone, 0,
					pcp->batch, list,
M
Mel Gorman 已提交
3090
					migratetype);
3091 3092 3093 3094
			if (unlikely(list_empty(list)))
				return NULL;
		}

M
Mel Gorman 已提交
3095
		page = list_first_entry(list, struct page, lru);
3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110
		list_del(&page->lru);
		pcp->count--;
	} while (check_new_pcp(page));

	return page;
}

/* Lock and remove page from the per-cpu list */
static struct page *rmqueue_pcplist(struct zone *preferred_zone,
			struct zone *zone, unsigned int order,
			gfp_t gfp_flags, int migratetype)
{
	struct per_cpu_pages *pcp;
	struct list_head *list;
	struct page *page;
3111
	unsigned long flags;
3112

3113
	local_irq_save(flags);
3114 3115
	pcp = &this_cpu_ptr(zone->pageset)->pcp;
	list = &pcp->lists[migratetype];
M
Mel Gorman 已提交
3116
	page = __rmqueue_pcplist(zone,  migratetype, pcp, list);
3117 3118 3119 3120
	if (page) {
		__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
		zone_statistics(preferred_zone, zone);
	}
3121
	local_irq_restore(flags);
3122 3123 3124
	return page;
}

L
Linus Torvalds 已提交
3125
/*
3126
 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
L
Linus Torvalds 已提交
3127
 */
3128
static inline
3129
struct page *rmqueue(struct zone *preferred_zone,
3130
			struct zone *zone, unsigned int order,
3131 3132
			gfp_t gfp_flags, unsigned int alloc_flags,
			int migratetype)
L
Linus Torvalds 已提交
3133 3134
{
	unsigned long flags;
3135
	struct page *page;
L
Linus Torvalds 已提交
3136

3137
	if (likely(order == 0)) {
3138 3139 3140 3141
		page = rmqueue_pcplist(preferred_zone, zone, order,
				gfp_flags, migratetype);
		goto out;
	}
3142

3143 3144 3145 3146 3147 3148
	/*
	 * We most definitely don't want callers attempting to
	 * allocate greater than order-1 page units with __GFP_NOFAIL.
	 */
	WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
	spin_lock_irqsave(&zone->lock, flags);
3149

3150 3151 3152 3153 3154 3155 3156
	do {
		page = NULL;
		if (alloc_flags & ALLOC_HARDER) {
			page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
			if (page)
				trace_mm_page_alloc_zone_locked(page, order, migratetype);
		}
N
Nick Piggin 已提交
3157
		if (!page)
3158 3159 3160 3161 3162 3163 3164
			page = __rmqueue(zone, order, migratetype);
	} while (page && check_new_pages(page, order));
	spin_unlock(&zone->lock);
	if (!page)
		goto failed;
	__mod_zone_freepage_state(zone, -(1 << order),
				  get_pcppage_migratetype(page));
L
Linus Torvalds 已提交
3165

3166
	__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
M
Michal Hocko 已提交
3167
	zone_statistics(preferred_zone, zone);
N
Nick Piggin 已提交
3168
	local_irq_restore(flags);
L
Linus Torvalds 已提交
3169

3170 3171
out:
	VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
L
Linus Torvalds 已提交
3172
	return page;
N
Nick Piggin 已提交
3173 3174 3175 3176

failed:
	local_irq_restore(flags);
	return NULL;
L
Linus Torvalds 已提交
3177 3178
}

3179 3180
#ifdef CONFIG_FAIL_PAGE_ALLOC

3181
static struct {
3182 3183
	struct fault_attr attr;

3184
	bool ignore_gfp_highmem;
3185
	bool ignore_gfp_reclaim;
3186
	u32 min_order;
3187 3188
} fail_page_alloc = {
	.attr = FAULT_ATTR_INITIALIZER,
3189
	.ignore_gfp_reclaim = true,
3190
	.ignore_gfp_highmem = true,
3191
	.min_order = 1,
3192 3193 3194 3195 3196 3197 3198 3199
};

static int __init setup_fail_page_alloc(char *str)
{
	return setup_fault_attr(&fail_page_alloc.attr, str);
}
__setup("fail_page_alloc=", setup_fail_page_alloc);

3200
static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3201
{
3202
	if (order < fail_page_alloc.min_order)
3203
		return false;
3204
	if (gfp_mask & __GFP_NOFAIL)
3205
		return false;
3206
	if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
3207
		return false;
3208 3209
	if (fail_page_alloc.ignore_gfp_reclaim &&
			(gfp_mask & __GFP_DIRECT_RECLAIM))
3210
		return false;
3211 3212 3213 3214 3215 3216 3217 3218

	return should_fail(&fail_page_alloc.attr, 1 << order);
}

#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS

static int __init fail_page_alloc_debugfs(void)
{
3219
	umode_t mode = S_IFREG | 0600;
3220 3221
	struct dentry *dir;

3222 3223 3224 3225
	dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
					&fail_page_alloc.attr);
	if (IS_ERR(dir))
		return PTR_ERR(dir);
3226

3227
	if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
3228
				&fail_page_alloc.ignore_gfp_reclaim))
3229 3230 3231 3232 3233 3234 3235 3236 3237 3238
		goto fail;
	if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
				&fail_page_alloc.ignore_gfp_highmem))
		goto fail;
	if (!debugfs_create_u32("min-order", mode, dir,
				&fail_page_alloc.min_order))
		goto fail;

	return 0;
fail:
3239
	debugfs_remove_recursive(dir);
3240

3241
	return -ENOMEM;
3242 3243 3244 3245 3246 3247 3248 3249
}

late_initcall(fail_page_alloc_debugfs);

#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */

#else /* CONFIG_FAIL_PAGE_ALLOC */

3250
static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3251
{
3252
	return false;
3253 3254 3255 3256
}

#endif /* CONFIG_FAIL_PAGE_ALLOC */

L
Linus Torvalds 已提交
3257
/*
3258 3259 3260 3261
 * Return true if free base pages are above 'mark'. For high-order checks it
 * will return true of the order-0 watermark is reached and there is at least
 * one free page of a suitable size. Checking now avoids taking the zone lock
 * to check in the allocation paths if no pages are free.
L
Linus Torvalds 已提交
3262
 */
3263 3264 3265
bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
			 int classzone_idx, unsigned int alloc_flags,
			 long free_pages)
L
Linus Torvalds 已提交
3266
{
3267
	long min = mark;
L
Linus Torvalds 已提交
3268
	int o;
3269
	const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
L
Linus Torvalds 已提交
3270

3271 3272 3273 3274 3275 3276 3277 3278
	/* apply negative memory.wmark_min_adj */
	if ((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN) {
		int min_adj = memcg_get_wmark_min_adj(current);

		if (min_adj < 0)
			min -= mark * (-min_adj) / 100;
	}

3279
	/* free_pages may go negative - that's OK */
3280
	free_pages -= (1 << order) - 1;
3281

R
Rohit Seth 已提交
3282
	if (alloc_flags & ALLOC_HIGH)
L
Linus Torvalds 已提交
3283
		min -= min / 2;
3284 3285 3286 3287 3288 3289

	/*
	 * If the caller does not have rights to ALLOC_HARDER then subtract
	 * the high-atomic reserves. This will over-estimate the size of the
	 * atomic reserve but it avoids a search.
	 */
3290
	if (likely(!alloc_harder)) {
3291
		free_pages -= z->nr_reserved_highatomic;
3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304
	} else {
		/*
		 * OOM victims can try even harder than normal ALLOC_HARDER
		 * users on the grounds that it's definitely going to be in
		 * the exit path shortly and free memory. Any allocation it
		 * makes during the free path will be small and short-lived.
		 */
		if (alloc_flags & ALLOC_OOM)
			min -= min / 2;
		else
			min -= min / 4;
	}

3305 3306 3307 3308 3309 3310
	/*
	 * Only happens due to memory.wmark_min_adj.
	 * Guarantee safe min after memory.wmark_min_adj?
	 */
	if (min < mark / 4)
		min = mark / 4;
3311

3312 3313 3314 3315 3316 3317
#ifdef CONFIG_CMA
	/* If allocation can't use CMA areas don't use free CMA pages */
	if (!(alloc_flags & ALLOC_CMA))
		free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
#endif

3318 3319 3320 3321 3322 3323
	/*
	 * Check watermarks for an order-0 allocation request. If these
	 * are not met, then a high-order request also cannot go ahead
	 * even if a suitable page happened to be free.
	 */
	if (free_pages <= min + z->lowmem_reserve[classzone_idx])
3324
		return false;
L
Linus Torvalds 已提交
3325

3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338
	/* If this is an order-0 request then the watermark is fine */
	if (!order)
		return true;

	/* For a high-order request, check at least one suitable page is free */
	for (o = order; o < MAX_ORDER; o++) {
		struct free_area *area = &z->free_area[o];
		int mt;

		if (!area->nr_free)
			continue;

		for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
3339
			if (!free_area_empty(area, mt))
3340 3341 3342 3343
				return true;
		}

#ifdef CONFIG_CMA
3344
		if ((alloc_flags & ALLOC_CMA) &&
3345
		    !free_area_empty(area, MIGRATE_CMA)) {
3346
			return true;
3347
		}
3348
#endif
3349 3350 3351
		if (alloc_harder &&
			!list_empty(&area->free_list[MIGRATE_HIGHATOMIC]))
			return true;
L
Linus Torvalds 已提交
3352
	}
3353
	return false;
3354 3355
}

3356
bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3357
		      int classzone_idx, unsigned int alloc_flags)
3358 3359 3360 3361 3362
{
	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
					zone_page_state(z, NR_FREE_PAGES));
}

3363 3364 3365 3366
static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
		unsigned long mark, int classzone_idx, unsigned int alloc_flags)
{
	long free_pages = zone_page_state(z, NR_FREE_PAGES);
3367 3368 3369 3370 3371 3372 3373
	long cma_pages = 0;

#ifdef CONFIG_CMA
	/* If allocation can't use CMA areas don't use free CMA pages */
	if (!(alloc_flags & ALLOC_CMA))
		cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES);
#endif
3374 3375 3376 3377 3378 3379 3380 3381

	/*
	 * Fast check for order-0 only. If this fails then the reserves
	 * need to be calculated. There is a corner case where the check
	 * passes but only the high-order atomic reserve are free. If
	 * the caller is !atomic then it'll uselessly search the free
	 * list. That corner case is then slower but it is harmless.
	 */
3382
	if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx])
3383 3384 3385 3386 3387 3388
		return true;

	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
					free_pages);
}

3389
bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
3390
			unsigned long mark, int classzone_idx)
3391 3392 3393 3394 3395 3396
{
	long free_pages = zone_page_state(z, NR_FREE_PAGES);

	if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
		free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);

3397
	return __zone_watermark_ok(z, order, mark, classzone_idx, 0,
3398
								free_pages);
L
Linus Torvalds 已提交
3399 3400
}

3401
#ifdef CONFIG_NUMA
3402 3403
static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
{
3404
	return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
3405
				RECLAIM_DISTANCE;
3406
}
3407
#else	/* CONFIG_NUMA */
3408 3409 3410 3411
static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
{
	return true;
}
3412 3413
#endif	/* CONFIG_NUMA */

R
Rohit Seth 已提交
3414
/*
3415
 * get_page_from_freelist goes through the zonelist trying to allocate
R
Rohit Seth 已提交
3416 3417 3418
 * a page.
 */
static struct page *
3419 3420
get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
						const struct alloc_context *ac)
M
Martin Hicks 已提交
3421
{
3422
	struct zoneref *z = ac->preferred_zoneref;
3423
	struct zone *zone;
3424 3425
	struct pglist_data *last_pgdat_dirty_limit = NULL;

R
Rohit Seth 已提交
3426
	/*
3427
	 * Scan zonelist, looking for a zone with enough free.
3428
	 * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
R
Rohit Seth 已提交
3429
	 */
3430
	for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3431
								ac->nodemask) {
3432
		struct page *page;
3433 3434
		unsigned long mark;

3435 3436
		if (cpusets_enabled() &&
			(alloc_flags & ALLOC_CPUSET) &&
3437
			!__cpuset_zone_allowed(zone, gfp_mask))
3438
				continue;
3439 3440
		/*
		 * When allocating a page cache page for writing, we
3441 3442
		 * want to get it from a node that is within its dirty
		 * limit, such that no single node holds more than its
3443
		 * proportional share of globally allowed dirty pages.
3444
		 * The dirty limits take into account the node's
3445 3446 3447 3448 3449
		 * lowmem reserves and high watermark so that kswapd
		 * should be able to balance it without having to
		 * write pages from its LRU list.
		 *
		 * XXX: For now, allow allocations to potentially
3450
		 * exceed the per-node dirty limit in the slowpath
3451
		 * (spread_dirty_pages unset) before going into reclaim,
3452
		 * which is important when on a NUMA setup the allowed
3453
		 * nodes are together not big enough to reach the
3454
		 * global limit.  The proper fix for these situations
3455
		 * will require awareness of nodes in the
3456 3457
		 * dirty-throttling and the flusher threads.
		 */
3458 3459 3460 3461 3462 3463 3464 3465 3466
		if (ac->spread_dirty_pages) {
			if (last_pgdat_dirty_limit == zone->zone_pgdat)
				continue;

			if (!node_dirty_ok(zone->zone_pgdat)) {
				last_pgdat_dirty_limit = zone->zone_pgdat;
				continue;
			}
		}
R
Rohit Seth 已提交
3467

3468
		mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
3469
		if (!zone_watermark_fast(zone, order, mark,
3470
				       ac_classzone_idx(ac), alloc_flags)) {
3471 3472
			int ret;

3473 3474 3475 3476 3477 3478 3479 3480 3481 3482
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
			/*
			 * Watermark failed for this zone, but see if we can
			 * grow this zone if it contains deferred pages.
			 */
			if (static_branch_unlikely(&deferred_pages)) {
				if (_deferred_grow_zone(zone, order))
					goto try_this_zone;
			}
#endif
3483 3484 3485 3486 3487
			/* Checked here to keep the fast path fast */
			BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
			if (alloc_flags & ALLOC_NO_WATERMARKS)
				goto try_this_zone;

3488
			if (node_reclaim_mode == 0 ||
3489
			    !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
3490 3491
				continue;

3492
			ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
3493
			switch (ret) {
3494
			case NODE_RECLAIM_NOSCAN:
3495
				/* did not scan */
3496
				continue;
3497
			case NODE_RECLAIM_FULL:
3498
				/* scanned but unreclaimable */
3499
				continue;
3500 3501
			default:
				/* did we reclaim enough */
3502
				if (zone_watermark_ok(zone, order, mark,
3503
						ac_classzone_idx(ac), alloc_flags))
3504 3505 3506
					goto try_this_zone;

				continue;
3507
			}
R
Rohit Seth 已提交
3508 3509
		}

3510
try_this_zone:
3511
		page = rmqueue(ac->preferred_zoneref->zone, zone, order,
3512
				gfp_mask, alloc_flags, ac->migratetype);
3513
		if (page) {
3514
			prep_new_page(page, order, gfp_mask, alloc_flags);
3515 3516 3517 3518 3519 3520 3521 3522

			/*
			 * If this is a high-order atomic allocation then check
			 * if the pageblock should be reserved for the future
			 */
			if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
				reserve_highatomic_pageblock(page, zone, order);

3523
			return page;
3524 3525 3526 3527 3528 3529 3530 3531
		} else {
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
			/* Try again if zone has deferred pages */
			if (static_branch_unlikely(&deferred_pages)) {
				if (_deferred_grow_zone(zone, order))
					goto try_this_zone;
			}
#endif
3532
		}
3533
	}
3534

3535
	return NULL;
M
Martin Hicks 已提交
3536 3537
}

3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551
/*
 * Large machines with many possible nodes should not always dump per-node
 * meminfo in irq context.
 */
static inline bool should_suppress_show_mem(void)
{
	bool ret = false;

#if NODES_SHIFT > 8
	ret = in_interrupt();
#endif
	return ret;
}

3552
static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
3553 3554
{
	unsigned int filter = SHOW_MEM_FILTER_NODES;
3555
	static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
3556

3557
	if (should_suppress_show_mem() || !__ratelimit(&show_mem_rs))
3558 3559 3560 3561 3562 3563 3564 3565
		return;

	/*
	 * This documents exceptions given to allocations in certain
	 * contexts that are allowed to allocate outside current's set
	 * of allowed nodes.
	 */
	if (!(gfp_mask & __GFP_NOMEMALLOC))
3566
		if (tsk_is_oom_victim(current) ||
3567 3568
		    (current->flags & (PF_MEMALLOC | PF_EXITING)))
			filter &= ~SHOW_MEM_FILTER_NODES;
3569
	if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
3570 3571
		filter &= ~SHOW_MEM_FILTER_NODES;

3572
	show_mem(filter, nodemask);
3573 3574
}

3575
void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
3576 3577 3578 3579 3580 3581
{
	struct va_format vaf;
	va_list args;
	static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
				      DEFAULT_RATELIMIT_BURST);

3582
	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
3583 3584
		return;

3585 3586 3587
	va_start(args, fmt);
	vaf.fmt = fmt;
	vaf.va = &args;
M
Michal Hocko 已提交
3588 3589 3590
	pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl\n",
			current->comm, &vaf, gfp_mask, &gfp_mask,
			nodemask_pr_args(nodemask));
3591
	va_end(args);
J
Joe Perches 已提交
3592

3593
	cpuset_print_current_mems_allowed();
J
Joe Perches 已提交
3594

3595
	dump_stack();
3596
	warn_alloc_show_mem(gfp_mask, nodemask);
3597 3598
}

3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618
static inline struct page *
__alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
			      unsigned int alloc_flags,
			      const struct alloc_context *ac)
{
	struct page *page;

	page = get_page_from_freelist(gfp_mask, order,
			alloc_flags|ALLOC_CPUSET, ac);
	/*
	 * fallback to ignore cpuset restriction if our nodes
	 * are depleted
	 */
	if (!page)
		page = get_page_from_freelist(gfp_mask, order,
				alloc_flags, ac);

	return page;
}

3619 3620
static inline struct page *
__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
3621
	const struct alloc_context *ac, unsigned long *did_some_progress)
3622
{
3623 3624 3625
	struct oom_control oc = {
		.zonelist = ac->zonelist,
		.nodemask = ac->nodemask,
3626
		.memcg = NULL,
3627 3628 3629
		.gfp_mask = gfp_mask,
		.order = order,
	};
3630 3631
	struct page *page;

3632 3633 3634
	*did_some_progress = 0;

	/*
3635 3636
	 * Acquire the oom lock.  If that fails, somebody else is
	 * making progress for us.
3637
	 */
3638
	if (!mutex_trylock(&oom_lock)) {
3639
		*did_some_progress = 1;
3640
		schedule_timeout_uninterruptible(1);
L
Linus Torvalds 已提交
3641 3642
		return NULL;
	}
3643

3644 3645 3646
	/*
	 * Go through the zonelist yet one more time, keep very high watermark
	 * here, this is only to catch a parallel oom killing, we must fail if
3647 3648 3649
	 * we're still under heavy pressure. But make sure that this reclaim
	 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
	 * allocation which will never fail due to oom_lock already held.
3650
	 */
3651 3652 3653
	page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
				      ~__GFP_DIRECT_RECLAIM, order,
				      ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
R
Rohit Seth 已提交
3654
	if (page)
3655 3656
		goto out;

3657 3658 3659 3660 3661 3662
	/* Coredumps can quickly deplete all memory reserves */
	if (current->flags & PF_DUMPCORE)
		goto out;
	/* The OOM killer will not help higher order allocs */
	if (order > PAGE_ALLOC_COSTLY_ORDER)
		goto out;
3663 3664 3665 3666 3667 3668 3669 3670
	/*
	 * We have already exhausted all our reclaim opportunities without any
	 * success so it is time to admit defeat. We will skip the OOM killer
	 * because it is very likely that the caller has a more reasonable
	 * fallback than shooting a random task.
	 */
	if (gfp_mask & __GFP_RETRY_MAYFAIL)
		goto out;
3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688
	/* The OOM killer does not needlessly kill tasks for lowmem */
	if (ac->high_zoneidx < ZONE_NORMAL)
		goto out;
	if (pm_suspended_storage())
		goto out;
	/*
	 * XXX: GFP_NOFS allocations should rather fail than rely on
	 * other request to make a forward progress.
	 * We are in an unfortunate situation where out_of_memory cannot
	 * do much for this context but let's try it to at least get
	 * access to memory reserved if the current task is killed (see
	 * out_of_memory). Once filesystems are ready to handle allocation
	 * failures more gracefully we should just bail out here.
	 */

	/* The OOM killer may not free memory on a specific node */
	if (gfp_mask & __GFP_THISNODE)
		goto out;
3689

3690
	/* Exhausted what can be done so it's blame time */
3691
	if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
3692
		*did_some_progress = 1;
3693

3694 3695 3696 3697 3698 3699
		/*
		 * Help non-failing allocations by giving them access to memory
		 * reserves
		 */
		if (gfp_mask & __GFP_NOFAIL)
			page = __alloc_pages_cpuset_fallback(gfp_mask, order,
3700 3701
					ALLOC_NO_WATERMARKS, ac);
	}
3702
out:
3703
	mutex_unlock(&oom_lock);
3704 3705 3706
	return page;
}

3707 3708 3709 3710 3711 3712
/*
 * Maximum number of compaction retries wit a progress before OOM
 * killer is consider as the only way to move forward.
 */
#define MAX_COMPACT_RETRIES 16

3713 3714 3715 3716
#ifdef CONFIG_COMPACTION
/* Try memory compaction for high-order allocations before reclaim */
static struct page *
__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3717
		unsigned int alloc_flags, const struct alloc_context *ac,
3718
		enum compact_priority prio, enum compact_result *compact_result)
3719
{
3720
	struct page *page;
3721
	unsigned long pflags;
3722
	unsigned int noreclaim_flag;
3723 3724

	if (!order)
3725 3726
		return NULL;

3727
	psi_memstall_enter(&pflags);
3728
	noreclaim_flag = memalloc_noreclaim_save();
3729

3730
	*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
3731
									prio);
3732

3733
	memalloc_noreclaim_restore(noreclaim_flag);
3734
	psi_memstall_leave(&pflags);
3735

3736
	if (*compact_result <= COMPACT_INACTIVE)
3737
		return NULL;
3738

3739 3740 3741 3742 3743
	/*
	 * At least in one zone compaction wasn't deferred or skipped, so let's
	 * count a compaction stall
	 */
	count_vm_event(COMPACTSTALL);
3744

3745
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3746

3747 3748
	if (page) {
		struct zone *zone = page_zone(page);
3749

3750 3751 3752 3753 3754
		zone->compact_blockskip_flush = false;
		compaction_defer_reset(zone, order, true);
		count_vm_event(COMPACTSUCCESS);
		return page;
	}
3755

3756 3757 3758 3759 3760
	/*
	 * It's bad if compaction run occurs and fails. The most likely reason
	 * is that pages exist, but not enough to satisfy watermarks.
	 */
	count_vm_event(COMPACTFAIL);
3761

3762
	cond_resched();
3763 3764 3765

	return NULL;
}
3766

3767 3768 3769 3770
static inline bool
should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
		     enum compact_result compact_result,
		     enum compact_priority *compact_priority,
3771
		     int *compaction_retries)
3772 3773
{
	int max_retries = MAX_COMPACT_RETRIES;
3774
	int min_priority;
3775 3776 3777
	bool ret = false;
	int retries = *compaction_retries;
	enum compact_priority priority = *compact_priority;
3778 3779 3780 3781

	if (!order)
		return false;

3782 3783 3784
	if (compaction_made_progress(compact_result))
		(*compaction_retries)++;

3785 3786 3787 3788 3789
	/*
	 * compaction considers all the zone as desperately out of memory
	 * so it doesn't really make much sense to retry except when the
	 * failure could be caused by insufficient priority
	 */
3790 3791
	if (compaction_failed(compact_result))
		goto check_priority;
3792 3793 3794 3795 3796 3797 3798

	/*
	 * make sure the compaction wasn't deferred or didn't bail out early
	 * due to locks contention before we declare that we should give up.
	 * But do not retry if the given zonelist is not suitable for
	 * compaction.
	 */
3799 3800 3801 3802
	if (compaction_withdrawn(compact_result)) {
		ret = compaction_zonelist_suitable(ac, order, alloc_flags);
		goto out;
	}
3803 3804

	/*
3805
	 * !costly requests are much more important than __GFP_RETRY_MAYFAIL
3806 3807 3808 3809 3810 3811 3812 3813
	 * costly ones because they are de facto nofail and invoke OOM
	 * killer to move on while costly can fail and users are ready
	 * to cope with that. 1/4 retries is rather arbitrary but we
	 * would need much more detailed feedback from compaction to
	 * make a better decision.
	 */
	if (order > PAGE_ALLOC_COSTLY_ORDER)
		max_retries /= 4;
3814 3815 3816 3817
	if (*compaction_retries <= max_retries) {
		ret = true;
		goto out;
	}
3818

3819 3820 3821 3822 3823
	/*
	 * Make sure there are attempts at the highest priority if we exhausted
	 * all retries or failed at the lower priorities.
	 */
check_priority:
3824 3825
	min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
			MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
3826

3827
	if (*compact_priority > min_priority) {
3828 3829
		(*compact_priority)--;
		*compaction_retries = 0;
3830
		ret = true;
3831
	}
3832 3833 3834
out:
	trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
	return ret;
3835
}
3836 3837 3838
#else
static inline struct page *
__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3839
		unsigned int alloc_flags, const struct alloc_context *ac,
3840
		enum compact_priority prio, enum compact_result *compact_result)
3841
{
3842
	*compact_result = COMPACT_SKIPPED;
3843 3844
	return NULL;
}
3845 3846

static inline bool
3847 3848
should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
		     enum compact_result compact_result,
3849
		     enum compact_priority *compact_priority,
3850
		     int *compaction_retries)
3851
{
3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869
	struct zone *zone;
	struct zoneref *z;

	if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
		return false;

	/*
	 * There are setups with compaction disabled which would prefer to loop
	 * inside the allocator rather than hit the oom killer prematurely.
	 * Let's give them a good hope and keep retrying while the order-0
	 * watermarks are OK.
	 */
	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
					ac->nodemask) {
		if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
					ac_classzone_idx(ac), alloc_flags))
			return true;
	}
3870 3871
	return false;
}
3872
#endif /* CONFIG_COMPACTION */
3873

3874
#ifdef CONFIG_LOCKDEP
3875
static struct lockdep_map __fs_reclaim_map =
3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886
	STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);

static bool __need_fs_reclaim(gfp_t gfp_mask)
{
	gfp_mask = current_gfp_context(gfp_mask);

	/* no reclaim without waiting on it */
	if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
		return false;

	/* this guy won't enter reclaim */
T
Tetsuo Handa 已提交
3887
	if (current->flags & PF_MEMALLOC)
3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899
		return false;

	/* We're only interested __GFP_FS allocations for now */
	if (!(gfp_mask & __GFP_FS))
		return false;

	if (gfp_mask & __GFP_NOLOCKDEP)
		return false;

	return true;
}

3900 3901 3902 3903 3904 3905 3906 3907 3908 3909
void __fs_reclaim_acquire(void)
{
	lock_map_acquire(&__fs_reclaim_map);
}

void __fs_reclaim_release(void)
{
	lock_map_release(&__fs_reclaim_map);
}

3910 3911 3912
void fs_reclaim_acquire(gfp_t gfp_mask)
{
	if (__need_fs_reclaim(gfp_mask))
3913
		__fs_reclaim_acquire();
3914 3915 3916 3917 3918 3919
}
EXPORT_SYMBOL_GPL(fs_reclaim_acquire);

void fs_reclaim_release(gfp_t gfp_mask)
{
	if (__need_fs_reclaim(gfp_mask))
3920
		__fs_reclaim_release();
3921 3922 3923 3924
}
EXPORT_SYMBOL_GPL(fs_reclaim_release);
#endif

3925 3926
/* Perform direct synchronous page reclaim */
static int
3927 3928
__perform_reclaim(gfp_t gfp_mask, unsigned int order,
					const struct alloc_context *ac)
3929 3930
{
	struct reclaim_state reclaim_state;
3931
	int progress;
3932
	unsigned int noreclaim_flag;
3933
	unsigned long pflags;
3934
	u64 start;
3935 3936 3937 3938 3939

	cond_resched();

	/* We now go into synchronous reclaim */
	cpuset_memory_pressure_bump();
3940
	psi_memstall_enter(&pflags);
3941
	start = ktime_get_ns();
3942
	fs_reclaim_acquire(gfp_mask);
3943
	noreclaim_flag = memalloc_noreclaim_save();
3944
	reclaim_state.reclaimed_slab = 0;
3945
	current->reclaim_state = &reclaim_state;
3946

3947 3948
	progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
								ac->nodemask);
3949

3950
	current->reclaim_state = NULL;
3951
	memalloc_noreclaim_restore(noreclaim_flag);
3952
	fs_reclaim_release(gfp_mask);
3953
	memcg_lat_stat_update(GLOBAL_DIRECT_RECLAIM, (ktime_get_ns() - start));
3954
	psi_memstall_leave(&pflags);
3955 3956 3957

	cond_resched();

3958 3959 3960 3961 3962 3963
	return progress;
}

/* The really slow allocator path where we enter direct reclaim */
static inline struct page *
__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
3964
		unsigned int alloc_flags, const struct alloc_context *ac,
3965
		unsigned long *did_some_progress)
3966 3967 3968 3969
{
	struct page *page = NULL;
	bool drained = false;

3970
	*did_some_progress = __perform_reclaim(gfp_mask, order, ac);
3971 3972
	if (unlikely(!(*did_some_progress)))
		return NULL;
3973

3974
retry:
3975
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3976 3977 3978

	/*
	 * If an allocation failed after direct reclaim, it could be because
3979 3980
	 * pages are pinned on the per-cpu lists or in high alloc reserves.
	 * Shrink them them and try again
3981 3982
	 */
	if (!page && !drained) {
3983
		unreserve_highatomic_pageblock(ac, false);
3984
		drain_all_pages(NULL);
3985 3986 3987 3988
		drained = true;
		goto retry;
	}

3989 3990 3991
	return page;
}

3992 3993
static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
			     const struct alloc_context *ac)
3994 3995 3996
{
	struct zoneref *z;
	struct zone *zone;
3997
	pg_data_t *last_pgdat = NULL;
3998
	enum zone_type high_zoneidx = ac->high_zoneidx;
3999

4000 4001
	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx,
					ac->nodemask) {
4002
		if (last_pgdat != zone->zone_pgdat)
4003
			wakeup_kswapd(zone, gfp_mask, order, high_zoneidx);
4004 4005
		last_pgdat = zone->zone_pgdat;
	}
4006 4007
}

4008
static inline unsigned int
4009 4010
gfp_to_alloc_flags(gfp_t gfp_mask)
{
4011
	unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
L
Linus Torvalds 已提交
4012

4013
	/* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
4014
	BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
4015

4016 4017 4018 4019
	/*
	 * The caller may dip into page reserves a bit more if the caller
	 * cannot run direct reclaim, or if the caller has realtime scheduling
	 * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
4020
	 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
4021
	 */
4022
	alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
L
Linus Torvalds 已提交
4023

4024
	if (gfp_mask & __GFP_ATOMIC) {
4025
		/*
4026 4027
		 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
		 * if it can't schedule.
4028
		 */
4029
		if (!(gfp_mask & __GFP_NOMEMALLOC))
4030
			alloc_flags |= ALLOC_HARDER;
4031
		/*
4032
		 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
4033
		 * comment for __cpuset_node_allowed().
4034
		 */
4035
		alloc_flags &= ~ALLOC_CPUSET;
4036
	} else if (unlikely(rt_task(current)) && !in_interrupt())
4037 4038
		alloc_flags |= ALLOC_HARDER;

4039 4040 4041 4042
#ifdef CONFIG_CMA
	if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
		alloc_flags |= ALLOC_CMA;
#endif
4043 4044 4045
	return alloc_flags;
}

4046
static bool oom_reserves_allowed(struct task_struct *tsk)
4047
{
4048 4049 4050 4051 4052 4053 4054 4055
	if (!tsk_is_oom_victim(tsk))
		return false;

	/*
	 * !MMU doesn't have oom reaper so give access to memory reserves
	 * only to the thread with TIF_MEMDIE set
	 */
	if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4056 4057
		return false;

4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068
	return true;
}

/*
 * Distinguish requests which really need access to full memory
 * reserves from oom victims which can live with a portion of it
 */
static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
{
	if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
		return 0;
4069
	if (gfp_mask & __GFP_MEMALLOC)
4070
		return ALLOC_NO_WATERMARKS;
4071
	if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4072 4073 4074 4075 4076 4077 4078
		return ALLOC_NO_WATERMARKS;
	if (!in_interrupt()) {
		if (current->flags & PF_MEMALLOC)
			return ALLOC_NO_WATERMARKS;
		else if (oom_reserves_allowed(current))
			return ALLOC_OOM;
	}
4079

4080 4081 4082 4083 4084 4085
	return 0;
}

bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
{
	return !!__gfp_pfmemalloc_flags(gfp_mask);
4086 4087
}

M
Michal Hocko 已提交
4088 4089 4090
/*
 * Checks whether it makes sense to retry the reclaim to make a forward progress
 * for the given allocation request.
4091 4092 4093 4094
 *
 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
 * without success, or when we couldn't even meet the watermark if we
 * reclaimed all remaining pages on the LRU lists.
M
Michal Hocko 已提交
4095 4096 4097 4098 4099 4100
 *
 * Returns true if a retry is viable or false to enter the oom path.
 */
static inline bool
should_reclaim_retry(gfp_t gfp_mask, unsigned order,
		     struct alloc_context *ac, int alloc_flags,
4101
		     bool did_some_progress, int *no_progress_loops)
M
Michal Hocko 已提交
4102 4103 4104 4105
{
	struct zone *zone;
	struct zoneref *z;

4106 4107 4108 4109 4110 4111 4112 4113 4114 4115
	/*
	 * Costly allocations might have made a progress but this doesn't mean
	 * their order will become available due to high fragmentation so
	 * always increment the no progress counter for them
	 */
	if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
		*no_progress_loops = 0;
	else
		(*no_progress_loops)++;

M
Michal Hocko 已提交
4116 4117 4118 4119
	/*
	 * Make sure we converge to OOM if we cannot make any progress
	 * several times in the row.
	 */
4120 4121
	if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
		/* Before OOM, exhaust highatomic_reserve */
4122
		return unreserve_highatomic_pageblock(ac, true);
4123
	}
M
Michal Hocko 已提交
4124

4125 4126 4127 4128 4129
	/*
	 * Keep reclaiming pages while there is a chance this will lead
	 * somewhere.  If none of the target zones can satisfy our allocation
	 * request even if all reclaimable pages are considered then we are
	 * screwed and have to go OOM.
M
Michal Hocko 已提交
4130 4131 4132 4133
	 */
	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
					ac->nodemask) {
		unsigned long available;
4134
		unsigned long reclaimable;
4135 4136
		unsigned long min_wmark = min_wmark_pages(zone);
		bool wmark;
M
Michal Hocko 已提交
4137

4138 4139
		available = reclaimable = zone_reclaimable_pages(zone);
		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
M
Michal Hocko 已提交
4140 4141

		/*
4142 4143
		 * Would the allocation succeed if we reclaimed all
		 * reclaimable pages?
M
Michal Hocko 已提交
4144
		 */
4145 4146 4147 4148 4149
		wmark = __zone_watermark_ok(zone, order, min_wmark,
				ac_classzone_idx(ac), alloc_flags, available);
		trace_reclaim_retry_zone(z, order, reclaimable,
				available, min_wmark, *no_progress_loops, wmark);
		if (wmark) {
4150 4151 4152 4153 4154 4155 4156
			/*
			 * If we didn't make any progress and have a lot of
			 * dirty + writeback pages then we should wait for
			 * an IO to complete to slow down the reclaim and
			 * prevent from pre mature OOM
			 */
			if (!did_some_progress) {
4157
				unsigned long write_pending;
4158

4159 4160
				write_pending = zone_page_state_snapshot(zone,
							NR_ZONE_WRITE_PENDING);
4161

4162
				if (2 * write_pending > reclaimable) {
4163 4164 4165 4166
					congestion_wait(BLK_RW_ASYNC, HZ/10);
					return true;
				}
			}
4167

4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181
			/*
			 * Memory allocation/reclaim might be called from a WQ
			 * context and the current implementation of the WQ
			 * concurrency control doesn't recognize that
			 * a particular WQ is congested if the worker thread is
			 * looping without ever sleeping. Therefore we have to
			 * do a short sleep here rather than calling
			 * cond_resched().
			 */
			if (current->flags & PF_WQ_WORKER)
				schedule_timeout_uninterruptible(1);
			else
				cond_resched();

M
Michal Hocko 已提交
4182 4183 4184 4185 4186 4187 4188
			return true;
		}
	}

	return false;
}

4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221
static inline bool
check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
{
	/*
	 * It's possible that cpuset's mems_allowed and the nodemask from
	 * mempolicy don't intersect. This should be normally dealt with by
	 * policy_nodemask(), but it's possible to race with cpuset update in
	 * such a way the check therein was true, and then it became false
	 * before we got our cpuset_mems_cookie here.
	 * This assumes that for all allocations, ac->nodemask can come only
	 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
	 * when it does not intersect with the cpuset restrictions) or the
	 * caller can deal with a violated nodemask.
	 */
	if (cpusets_enabled() && ac->nodemask &&
			!cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
		ac->nodemask = NULL;
		return true;
	}

	/*
	 * When updating a task's mems_allowed or mempolicy nodemask, it is
	 * possible to race with parallel threads in such a way that our
	 * allocation can fail while the mask is being updated. If we are about
	 * to fail, check if the cpuset changed during allocation and if so,
	 * retry.
	 */
	if (read_mems_allowed_retry(cpuset_mems_cookie))
		return true;

	return false;
}

4222 4223
static inline struct page *
__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
4224
						struct alloc_context *ac)
4225
{
4226
	bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
4227
	const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
4228
	struct page *page = NULL;
4229
	unsigned int alloc_flags;
4230
	unsigned long did_some_progress;
4231
	enum compact_priority compact_priority;
4232
	enum compact_result compact_result;
4233 4234 4235
	int compaction_retries;
	int no_progress_loops;
	unsigned int cpuset_mems_cookie;
4236
	int reserve_flags;
L
Linus Torvalds 已提交
4237

4238 4239 4240 4241 4242 4243 4244 4245
	/*
	 * We also sanity check to catch abuse of atomic reserves being used by
	 * callers that are not in atomic context.
	 */
	if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
				(__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
		gfp_mask &= ~__GFP_ATOMIC;

4246 4247 4248 4249 4250
retry_cpuset:
	compaction_retries = 0;
	no_progress_loops = 0;
	compact_priority = DEF_COMPACT_PRIORITY;
	cpuset_mems_cookie = read_mems_allowed_begin();
4251 4252 4253 4254 4255 4256 4257 4258

	/*
	 * The fast path uses conservative alloc_flags to succeed only until
	 * kswapd needs to be woken up, and to avoid the cost of setting up
	 * alloc_flags precisely. So we do that now.
	 */
	alloc_flags = gfp_to_alloc_flags(gfp_mask);

4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269
	/*
	 * We need to recalculate the starting point for the zonelist iterator
	 * because we might have used different nodemask in the fast path, or
	 * there was a cpuset modification and we are retrying - otherwise we
	 * could end up iterating over non-eligible zones endlessly.
	 */
	ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
					ac->high_zoneidx, ac->nodemask);
	if (!ac->preferred_zoneref->zone)
		goto nopage;

4270
	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
4271
		wake_all_kswapds(order, gfp_mask, ac);
4272 4273 4274 4275 4276 4277 4278 4279 4280

	/*
	 * The adjusted alloc_flags might result in immediate success, so try
	 * that first
	 */
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
	if (page)
		goto got_pg;

4281 4282
	/*
	 * For costly allocations, try direct compaction first, as it's likely
4283 4284 4285 4286 4287 4288
	 * that we have enough base pages and don't need to reclaim. For non-
	 * movable high-order allocations, do that as well, as compaction will
	 * try prevent permanent fragmentation by migrating from blocks of the
	 * same migratetype.
	 * Don't try this for allocations that are allowed to ignore
	 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
4289
	 */
4290 4291 4292 4293
	if (can_direct_reclaim &&
			(costly_order ||
			   (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
			&& !gfp_pfmemalloc_allowed(gfp_mask)) {
4294 4295
		page = __alloc_pages_direct_compact(gfp_mask, order,
						alloc_flags, ac,
4296
						INIT_COMPACT_PRIORITY,
4297 4298 4299 4300
						&compact_result);
		if (page)
			goto got_pg;

4301 4302 4303 4304
		/*
		 * Checks for costly allocations with __GFP_NORETRY, which
		 * includes THP page fault allocations
		 */
4305
		if (costly_order && (gfp_mask & __GFP_NORETRY)) {
4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317
			/*
			 * If compaction is deferred for high-order allocations,
			 * it is because sync compaction recently failed. If
			 * this is the case and the caller requested a THP
			 * allocation, we do not want to heavily disrupt the
			 * system, so we fail the allocation instead of entering
			 * direct reclaim.
			 */
			if (compact_result == COMPACT_DEFERRED)
				goto nopage;

			/*
4318 4319
			 * Looks like reclaim/compaction is worth trying, but
			 * sync compaction could be very expensive, so keep
4320
			 * using async compaction.
4321
			 */
4322
			compact_priority = INIT_COMPACT_PRIORITY;
4323 4324
		}
	}
4325

4326
retry:
4327
	/* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
4328
	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
4329
		wake_all_kswapds(order, gfp_mask, ac);
4330

4331 4332 4333
	reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
	if (reserve_flags)
		alloc_flags = reserve_flags;
4334

4335
	/*
4336 4337 4338
	 * Reset the nodemask and zonelist iterators if memory policies can be
	 * ignored. These allocations are high priority and system rather than
	 * user oriented.
4339
	 */
4340
	if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
4341
		ac->nodemask = NULL;
4342 4343 4344 4345
		ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
					ac->high_zoneidx, ac->nodemask);
	}

4346
	/* Attempt with potentially adjusted zonelist and alloc_flags */
4347
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
R
Rohit Seth 已提交
4348 4349
	if (page)
		goto got_pg;
L
Linus Torvalds 已提交
4350

4351
	/* Caller is not willing to reclaim, we can't balance anything */
4352
	if (!can_direct_reclaim)
L
Linus Torvalds 已提交
4353 4354
		goto nopage;

4355 4356
	/* Avoid recursion of direct reclaim */
	if (current->flags & PF_MEMALLOC)
4357 4358
		goto nopage;

4359 4360 4361 4362 4363 4364 4365
	/* Try direct reclaim and then allocating */
	page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
							&did_some_progress);
	if (page)
		goto got_pg;

	/* Try direct compaction and then allocating */
4366
	page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
4367
					compact_priority, &compact_result);
4368 4369
	if (page)
		goto got_pg;
4370

4371 4372
	/* Do not loop if specifically requested */
	if (gfp_mask & __GFP_NORETRY)
4373
		goto nopage;
4374

M
Michal Hocko 已提交
4375 4376
	/*
	 * Do not retry costly high order allocations unless they are
4377
	 * __GFP_RETRY_MAYFAIL
M
Michal Hocko 已提交
4378
	 */
4379
	if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
4380
		goto nopage;
M
Michal Hocko 已提交
4381 4382

	if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
4383
				 did_some_progress > 0, &no_progress_loops))
M
Michal Hocko 已提交
4384 4385
		goto retry;

4386 4387 4388 4389 4390 4391 4392
	/*
	 * It doesn't make any sense to retry for the compaction if the order-0
	 * reclaim is not able to make any progress because the current
	 * implementation of the compaction depends on the sufficient amount
	 * of free memory (see __compaction_suitable)
	 */
	if (did_some_progress > 0 &&
4393
			should_compact_retry(ac, order, alloc_flags,
4394
				compact_result, &compact_priority,
4395
				&compaction_retries))
4396 4397
		goto retry;

4398 4399 4400

	/* Deal with possible cpuset update races before we start OOM killing */
	if (check_retry_cpuset(cpuset_mems_cookie, ac))
4401 4402
		goto retry_cpuset;

4403 4404 4405 4406 4407
	/* Reclaim has failed us, start killing things */
	page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
	if (page)
		goto got_pg;

4408
	/* Avoid allocations with no watermarks from looping endlessly */
4409 4410
	if (tsk_is_oom_victim(current) &&
	    (alloc_flags == ALLOC_OOM ||
4411
	     (gfp_mask & __GFP_NOMEMALLOC)))
4412 4413
		goto nopage;

4414
	/* Retry as long as the OOM killer is making progress */
M
Michal Hocko 已提交
4415 4416
	if (did_some_progress) {
		no_progress_loops = 0;
4417
		goto retry;
M
Michal Hocko 已提交
4418
	}
4419

L
Linus Torvalds 已提交
4420
nopage:
4421 4422
	/* Deal with possible cpuset update races before we fail */
	if (check_retry_cpuset(cpuset_mems_cookie, ac))
4423 4424
		goto retry_cpuset;

4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451
	/*
	 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
	 * we always retry
	 */
	if (gfp_mask & __GFP_NOFAIL) {
		/*
		 * All existing users of the __GFP_NOFAIL are blockable, so warn
		 * of any new users that actually require GFP_NOWAIT
		 */
		if (WARN_ON_ONCE(!can_direct_reclaim))
			goto fail;

		/*
		 * PF_MEMALLOC request from this context is rather bizarre
		 * because we cannot reclaim anything and only can loop waiting
		 * for somebody to do a work for us
		 */
		WARN_ON_ONCE(current->flags & PF_MEMALLOC);

		/*
		 * non failing costly orders are a hard requirement which we
		 * are not prepared for much so let's warn about these users
		 * so that we can identify them and convert them to something
		 * else.
		 */
		WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER);

4452 4453 4454 4455 4456 4457 4458 4459 4460 4461
		/*
		 * Help non-failing allocations by giving them access to memory
		 * reserves but do not use ALLOC_NO_WATERMARKS because this
		 * could deplete whole memory reserves which would just make
		 * the situation worse
		 */
		page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
		if (page)
			goto got_pg;

4462 4463 4464 4465
		cond_resched();
		goto retry;
	}
fail:
4466
	warn_alloc(gfp_mask, ac->nodemask,
4467
			"page allocation failure: order:%u", order);
L
Linus Torvalds 已提交
4468
got_pg:
4469 4470 4471 4472

	if (ac->migratetype == MIGRATE_MOVABLE)
		memcg_check_wmark_min_adj(current, ac);

4473
	return page;
L
Linus Torvalds 已提交
4474
}
4475

4476
static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
4477
		int preferred_nid, nodemask_t *nodemask,
4478 4479
		struct alloc_context *ac, gfp_t *alloc_mask,
		unsigned int *alloc_flags)
4480
{
4481
	ac->high_zoneidx = gfp_zone(gfp_mask);
4482
	ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
4483 4484
	ac->nodemask = nodemask;
	ac->migratetype = gfpflags_to_migratetype(gfp_mask);
4485

4486
	if (cpusets_enabled()) {
4487 4488 4489
		*alloc_mask |= __GFP_HARDWALL;
		if (!ac->nodemask)
			ac->nodemask = &cpuset_current_mems_allowed;
4490 4491
		else
			*alloc_flags |= ALLOC_CPUSET;
4492 4493
	}

4494 4495
	fs_reclaim_acquire(gfp_mask);
	fs_reclaim_release(gfp_mask);
4496

4497
	might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
4498 4499

	if (should_fail_alloc_page(gfp_mask, order))
4500
		return false;
4501

4502 4503 4504
	if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE)
		*alloc_flags |= ALLOC_CMA;

4505 4506
	return true;
}
4507

4508
/* Determine whether to spread dirty pages and what the first usable zone */
4509
static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac)
4510
{
4511
	/* Dirty zone balancing only done in the fast path */
4512
	ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
4513

4514 4515 4516 4517 4518
	/*
	 * The preferred zone is used for statistics but crucially it is
	 * also used as the starting point for the zonelist iterator. It
	 * may get reset for allocations that ignore memory policies.
	 */
4519 4520 4521 4522 4523 4524 4525 4526
	ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
					ac->high_zoneidx, ac->nodemask);
}

/*
 * This is the 'heart' of the zoned buddy allocator.
 */
struct page *
4527 4528
__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
							nodemask_t *nodemask)
4529 4530 4531
{
	struct page *page;
	unsigned int alloc_flags = ALLOC_WMARK_LOW;
4532
	gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */
4533 4534
	struct alloc_context ac = { };

4535 4536 4537 4538 4539 4540 4541 4542 4543
	/*
	 * There are several places where we assume that the order value is sane
	 * so bail out early if the request is out of bound.
	 */
	if (unlikely(order >= MAX_ORDER)) {
		WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
		return NULL;
	}

4544
	gfp_mask &= gfp_allowed_mask;
4545
	alloc_mask = gfp_mask;
4546
	if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))
4547 4548
		return NULL;

4549
	finalise_ac(gfp_mask, &ac);
4550

4551
	/* First allocation attempt */
4552
	page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
4553 4554
	if (likely(page))
		goto out;
4555

4556
	/*
4557 4558 4559 4560
	 * Apply scoped allocation constraints. This is mainly about GFP_NOFS
	 * resp. GFP_NOIO which has to be inherited for all allocation requests
	 * from a particular context which has been marked by
	 * memalloc_no{fs,io}_{save,restore}.
4561
	 */
4562
	alloc_mask = current_gfp_context(gfp_mask);
4563
	ac.spread_dirty_pages = false;
4564

4565 4566 4567 4568
	/*
	 * Restore the original nodemask if it was potentially replaced with
	 * &cpuset_current_mems_allowed to optimize the fast-path attempt.
	 */
4569
	if (unlikely(ac.nodemask != nodemask))
4570
		ac.nodemask = nodemask;
4571

4572
	page = __alloc_pages_slowpath(alloc_mask, order, &ac);
4573

4574
out:
4575 4576 4577 4578
	if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
	    unlikely(memcg_kmem_charge(page, gfp_mask, order) != 0)) {
		__free_pages(page, order);
		page = NULL;
4579 4580
	}

4581 4582
	trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);

4583
	return page;
L
Linus Torvalds 已提交
4584
}
4585
EXPORT_SYMBOL(__alloc_pages_nodemask);
L
Linus Torvalds 已提交
4586 4587

/*
4588 4589 4590
 * Common helper functions. Never use with __GFP_HIGHMEM because the returned
 * address cannot represent highmem pages. Use alloc_pages and then kmap if
 * you need to access high mem.
L
Linus Torvalds 已提交
4591
 */
H
Harvey Harrison 已提交
4592
unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
L
Linus Torvalds 已提交
4593
{
4594 4595
	struct page *page;

4596
	page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
L
Linus Torvalds 已提交
4597 4598 4599 4600 4601 4602
	if (!page)
		return 0;
	return (unsigned long) page_address(page);
}
EXPORT_SYMBOL(__get_free_pages);

H
Harvey Harrison 已提交
4603
unsigned long get_zeroed_page(gfp_t gfp_mask)
L
Linus Torvalds 已提交
4604
{
4605
	return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
L
Linus Torvalds 已提交
4606 4607 4608
}
EXPORT_SYMBOL(get_zeroed_page);

4609
static inline void free_the_page(struct page *page, unsigned int order)
L
Linus Torvalds 已提交
4610
{
4611 4612 4613 4614
	if (order == 0)		/* Via pcp? */
		free_unref_page(page);
	else
		__free_pages_ok(page, order);
L
Linus Torvalds 已提交
4615 4616
}

4617 4618 4619 4620 4621
void __free_pages(struct page *page, unsigned int order)
{
	if (put_page_testzero(page))
		free_the_page(page, order);
}
L
Linus Torvalds 已提交
4622 4623
EXPORT_SYMBOL(__free_pages);

H
Harvey Harrison 已提交
4624
void free_pages(unsigned long addr, unsigned int order)
L
Linus Torvalds 已提交
4625 4626
{
	if (addr != 0) {
N
Nick Piggin 已提交
4627
		VM_BUG_ON(!virt_addr_valid((void *)addr));
L
Linus Torvalds 已提交
4628 4629 4630 4631 4632 4633
		__free_pages(virt_to_page((void *)addr), order);
	}
}

EXPORT_SYMBOL(free_pages);

4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644
/*
 * Page Fragment:
 *  An arbitrary-length arbitrary-offset area of memory which resides
 *  within a 0 or higher order page.  Multiple fragments within that page
 *  are individually refcounted, in the page's reference counter.
 *
 * The page_frag functions below provide a simple allocation framework for
 * page fragments.  This is used by the network stack and network device
 * drivers to provide a backing region of memory for use as either an
 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
 */
4645 4646
static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
					     gfp_t gfp_mask)
4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665
{
	struct page *page = NULL;
	gfp_t gfp = gfp_mask;

#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
	gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
		    __GFP_NOMEMALLOC;
	page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
				PAGE_FRAG_CACHE_MAX_ORDER);
	nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
#endif
	if (unlikely(!page))
		page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);

	nc->va = page ? page_address(page) : NULL;

	return page;
}

4666
void __page_frag_cache_drain(struct page *page, unsigned int count)
4667 4668 4669
{
	VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);

4670 4671
	if (page_ref_sub_and_test(page, count))
		free_the_page(page, compound_order(page));
4672
}
4673
EXPORT_SYMBOL(__page_frag_cache_drain);
4674

4675 4676
void *page_frag_alloc(struct page_frag_cache *nc,
		      unsigned int fragsz, gfp_t gfp_mask)
4677 4678 4679 4680 4681 4682 4683
{
	unsigned int size = PAGE_SIZE;
	struct page *page;
	int offset;

	if (unlikely(!nc->va)) {
refill:
4684
		page = __page_frag_cache_refill(nc, gfp_mask);
4685 4686 4687 4688 4689 4690 4691 4692 4693 4694
		if (!page)
			return NULL;

#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
		/* if size can vary use size else just use PAGE_SIZE */
		size = nc->size;
#endif
		/* Even if we own the page, we do not use atomic_set().
		 * This would break get_page_unless_zero() users.
		 */
4695
		page_ref_add(page, size);
4696 4697

		/* reset page count bias and offset to start of new frag */
4698
		nc->pfmemalloc = page_is_pfmemalloc(page);
4699
		nc->pagecnt_bias = size + 1;
4700 4701 4702 4703 4704 4705 4706
		nc->offset = size;
	}

	offset = nc->offset - fragsz;
	if (unlikely(offset < 0)) {
		page = virt_to_page(nc->va);

4707
		if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
4708 4709 4710 4711 4712 4713 4714
			goto refill;

#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
		/* if size can vary use size else just use PAGE_SIZE */
		size = nc->size;
#endif
		/* OK, page count is 0, we can safely set it */
4715
		set_page_count(page, size + 1);
4716 4717

		/* reset page count bias and offset to start of new frag */
4718
		nc->pagecnt_bias = size + 1;
4719 4720 4721 4722 4723 4724 4725 4726
		offset = size - fragsz;
	}

	nc->pagecnt_bias--;
	nc->offset = offset;

	return nc->va + offset;
}
4727
EXPORT_SYMBOL(page_frag_alloc);
4728 4729 4730 4731

/*
 * Frees a page fragment allocated out of either a compound or order 0 page.
 */
4732
void page_frag_free(void *addr)
4733 4734 4735
{
	struct page *page = virt_to_head_page(addr);

4736 4737
	if (unlikely(put_page_testzero(page)))
		free_the_page(page, compound_order(page));
4738
}
4739
EXPORT_SYMBOL(page_frag_free);
4740

4741 4742
static void *make_alloc_exact(unsigned long addr, unsigned int order,
		size_t size)
A
Andi Kleen 已提交
4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756
{
	if (addr) {
		unsigned long alloc_end = addr + (PAGE_SIZE << order);
		unsigned long used = addr + PAGE_ALIGN(size);

		split_page(virt_to_page((void *)addr), order);
		while (used < alloc_end) {
			free_page(used);
			used += PAGE_SIZE;
		}
	}
	return (void *)addr;
}

4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775
/**
 * alloc_pages_exact - allocate an exact number physically-contiguous pages.
 * @size: the number of bytes to allocate
 * @gfp_mask: GFP flags for the allocation
 *
 * This function is similar to alloc_pages(), except that it allocates the
 * minimum number of pages to satisfy the request.  alloc_pages() can only
 * allocate memory in power-of-two pages.
 *
 * This function is also limited by MAX_ORDER.
 *
 * Memory allocated by this function must be released by free_pages_exact().
 */
void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
{
	unsigned int order = get_order(size);
	unsigned long addr;

	addr = __get_free_pages(gfp_mask, order);
A
Andi Kleen 已提交
4776
	return make_alloc_exact(addr, order, size);
4777 4778 4779
}
EXPORT_SYMBOL(alloc_pages_exact);

A
Andi Kleen 已提交
4780 4781 4782
/**
 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
 *			   pages on a node.
4783
 * @nid: the preferred node ID where memory should be allocated
A
Andi Kleen 已提交
4784 4785 4786 4787 4788 4789
 * @size: the number of bytes to allocate
 * @gfp_mask: GFP flags for the allocation
 *
 * Like alloc_pages_exact(), but try to allocate on node nid first before falling
 * back.
 */
4790
void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
A
Andi Kleen 已提交
4791
{
4792
	unsigned int order = get_order(size);
A
Andi Kleen 已提交
4793 4794 4795 4796 4797 4798
	struct page *p = alloc_pages_node(nid, gfp_mask, order);
	if (!p)
		return NULL;
	return make_alloc_exact((unsigned long)page_address(p), order, size);
}

4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817
/**
 * free_pages_exact - release memory allocated via alloc_pages_exact()
 * @virt: the value returned by alloc_pages_exact.
 * @size: size of allocation, same value as passed to alloc_pages_exact().
 *
 * Release the memory allocated by a previous call to alloc_pages_exact.
 */
void free_pages_exact(void *virt, size_t size)
{
	unsigned long addr = (unsigned long)virt;
	unsigned long end = addr + PAGE_ALIGN(size);

	while (addr < end) {
		free_page(addr);
		addr += PAGE_SIZE;
	}
}
EXPORT_SYMBOL(free_pages_exact);

4818 4819 4820 4821 4822 4823 4824
/**
 * nr_free_zone_pages - count number of pages beyond high watermark
 * @offset: The zone index of the highest zone
 *
 * nr_free_zone_pages() counts the number of counts pages which are beyond the
 * high watermark within all zones at or below a given zone index.  For each
 * zone, the number of pages is calculated as:
4825 4826
 *
 *     nr_free_zone_pages = managed_pages - high_pages
4827
 */
4828
static unsigned long nr_free_zone_pages(int offset)
L
Linus Torvalds 已提交
4829
{
4830
	struct zoneref *z;
4831 4832
	struct zone *zone;

4833
	/* Just pick one node, since fallback list is circular */
4834
	unsigned long sum = 0;
L
Linus Torvalds 已提交
4835

4836
	struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
L
Linus Torvalds 已提交
4837

4838
	for_each_zone_zonelist(zone, z, zonelist, offset) {
4839
		unsigned long size = zone->managed_pages;
4840
		unsigned long high = high_wmark_pages(zone);
4841 4842
		if (size > high)
			sum += size - high;
L
Linus Torvalds 已提交
4843 4844 4845 4846 4847
	}

	return sum;
}

4848 4849 4850 4851 4852
/**
 * nr_free_buffer_pages - count number of pages beyond high watermark
 *
 * nr_free_buffer_pages() counts the number of pages which are beyond the high
 * watermark within ZONE_DMA and ZONE_NORMAL.
L
Linus Torvalds 已提交
4853
 */
4854
unsigned long nr_free_buffer_pages(void)
L
Linus Torvalds 已提交
4855
{
A
Al Viro 已提交
4856
	return nr_free_zone_pages(gfp_zone(GFP_USER));
L
Linus Torvalds 已提交
4857
}
4858
EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
L
Linus Torvalds 已提交
4859

4860 4861 4862 4863 4864
/**
 * nr_free_pagecache_pages - count number of pages beyond high watermark
 *
 * nr_free_pagecache_pages() counts the number of pages which are beyond the
 * high watermark within all zones.
L
Linus Torvalds 已提交
4865
 */
4866
unsigned long nr_free_pagecache_pages(void)
L
Linus Torvalds 已提交
4867
{
M
Mel Gorman 已提交
4868
	return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
L
Linus Torvalds 已提交
4869
}
4870 4871

static inline void show_node(struct zone *zone)
L
Linus Torvalds 已提交
4872
{
4873
	if (IS_ENABLED(CONFIG_NUMA))
4874
		printk("Node %d ", zone_to_nid(zone));
L
Linus Torvalds 已提交
4875 4876
}

4877 4878 4879 4880 4881 4882 4883 4884 4885 4886
long si_mem_available(void)
{
	long available;
	unsigned long pagecache;
	unsigned long wmark_low = 0;
	unsigned long pages[NR_LRU_LISTS];
	struct zone *zone;
	int lru;

	for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
4887
		pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
4888 4889 4890 4891 4892 4893 4894 4895

	for_each_zone(zone)
		wmark_low += zone->watermark[WMARK_LOW];

	/*
	 * Estimate the amount of memory available for userspace allocations,
	 * without causing swapping.
	 */
4896
	available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910

	/*
	 * Not all the page cache can be freed, otherwise the system will
	 * start swapping. Assume at least half of the page cache, or the
	 * low watermark worth of cache, needs to stay.
	 */
	pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
	pagecache -= min(pagecache / 2, wmark_low);
	available += pagecache;

	/*
	 * Part of the reclaimable slab consists of items that are in use,
	 * and cannot be freed. Cap this estimate at the low watermark.
	 */
4911 4912 4913
	available += global_node_page_state(NR_SLAB_RECLAIMABLE) -
		     min(global_node_page_state(NR_SLAB_RECLAIMABLE) / 2,
			 wmark_low);
4914

4915 4916 4917 4918 4919 4920 4921
	/*
	 * Part of the kernel memory, which can be released under memory
	 * pressure.
	 */
	available += global_node_page_state(NR_INDIRECTLY_RECLAIMABLE_BYTES) >>
		PAGE_SHIFT;

4922 4923 4924 4925 4926 4927
	if (available < 0)
		available = 0;
	return available;
}
EXPORT_SYMBOL_GPL(si_mem_available);

L
Linus Torvalds 已提交
4928 4929 4930
void si_meminfo(struct sysinfo *val)
{
	val->totalram = totalram_pages;
4931
	val->sharedram = global_node_page_state(NR_SHMEM);
4932
	val->freeram = global_zone_page_state(NR_FREE_PAGES);
L
Linus Torvalds 已提交
4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943
	val->bufferram = nr_blockdev_pages();
	val->totalhigh = totalhigh_pages;
	val->freehigh = nr_free_highpages();
	val->mem_unit = PAGE_SIZE;
}

EXPORT_SYMBOL(si_meminfo);

#ifdef CONFIG_NUMA
void si_meminfo_node(struct sysinfo *val, int nid)
{
4944 4945
	int zone_type;		/* needs to be signed */
	unsigned long managed_pages = 0;
4946 4947
	unsigned long managed_highpages = 0;
	unsigned long free_highpages = 0;
L
Linus Torvalds 已提交
4948 4949
	pg_data_t *pgdat = NODE_DATA(nid);

4950 4951 4952
	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
		managed_pages += pgdat->node_zones[zone_type].managed_pages;
	val->totalram = managed_pages;
4953
	val->sharedram = node_page_state(pgdat, NR_SHMEM);
4954
	val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
4955
#ifdef CONFIG_HIGHMEM
4956 4957 4958 4959 4960 4961 4962 4963 4964 4965
	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
		struct zone *zone = &pgdat->node_zones[zone_type];

		if (is_highmem(zone)) {
			managed_highpages += zone->managed_pages;
			free_highpages += zone_page_state(zone, NR_FREE_PAGES);
		}
	}
	val->totalhigh = managed_highpages;
	val->freehigh = free_highpages;
4966
#else
4967 4968
	val->totalhigh = managed_highpages;
	val->freehigh = free_highpages;
4969
#endif
L
Linus Torvalds 已提交
4970 4971 4972 4973
	val->mem_unit = PAGE_SIZE;
}
#endif

4974
/*
4975 4976
 * Determine whether the node should be displayed or not, depending on whether
 * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
4977
 */
4978
static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
4979 4980
{
	if (!(flags & SHOW_MEM_FILTER_NODES))
4981
		return false;
4982

4983 4984 4985 4986 4987 4988 4989 4990 4991
	/*
	 * no node mask - aka implicit memory numa policy. Do not bother with
	 * the synchronization - read_mems_allowed_begin - because we do not
	 * have to be precise here.
	 */
	if (!nodemask)
		nodemask = &cpuset_current_mems_allowed;

	return !node_isset(nid, *nodemask);
4992 4993
}

L
Linus Torvalds 已提交
4994 4995
#define K(x) ((x) << (PAGE_SHIFT-10))

4996 4997 4998 4999 5000
static void show_migration_types(unsigned char type)
{
	static const char types[MIGRATE_TYPES] = {
		[MIGRATE_UNMOVABLE]	= 'U',
		[MIGRATE_MOVABLE]	= 'M',
5001 5002
		[MIGRATE_RECLAIMABLE]	= 'E',
		[MIGRATE_HIGHATOMIC]	= 'H',
5003 5004 5005
#ifdef CONFIG_CMA
		[MIGRATE_CMA]		= 'C',
#endif
5006
#ifdef CONFIG_MEMORY_ISOLATION
5007
		[MIGRATE_ISOLATE]	= 'I',
5008
#endif
5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019
	};
	char tmp[MIGRATE_TYPES + 1];
	char *p = tmp;
	int i;

	for (i = 0; i < MIGRATE_TYPES; i++) {
		if (type & (1 << i))
			*p++ = types[i];
	}

	*p = '\0';
5020
	printk(KERN_CONT "(%s) ", tmp);
5021 5022
}

L
Linus Torvalds 已提交
5023 5024 5025 5026
/*
 * Show free area list (used inside shift_scroll-lock stuff)
 * We also calculate the percentage fragmentation. We do this by counting the
 * memory on each free list with the exception of the first item on the list.
5027 5028 5029 5030
 *
 * Bits in @filter:
 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
 *   cpuset.
L
Linus Torvalds 已提交
5031
 */
5032
void show_free_areas(unsigned int filter, nodemask_t *nodemask)
L
Linus Torvalds 已提交
5033
{
5034
	unsigned long free_pcp = 0;
5035
	int cpu;
L
Linus Torvalds 已提交
5036
	struct zone *zone;
M
Mel Gorman 已提交
5037
	pg_data_t *pgdat;
L
Linus Torvalds 已提交
5038

5039
	for_each_populated_zone(zone) {
5040
		if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5041
			continue;
5042

5043 5044
		for_each_online_cpu(cpu)
			free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
L
Linus Torvalds 已提交
5045 5046
	}

K
KOSAKI Motohiro 已提交
5047 5048
	printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
		" active_file:%lu inactive_file:%lu isolated_file:%lu\n"
5049 5050
		" unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
		" slab_reclaimable:%lu slab_unreclaimable:%lu\n"
5051
		" mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
5052
		" free:%lu free_pcp:%lu free_cma:%lu\n",
M
Mel Gorman 已提交
5053 5054 5055 5056 5057 5058 5059
		global_node_page_state(NR_ACTIVE_ANON),
		global_node_page_state(NR_INACTIVE_ANON),
		global_node_page_state(NR_ISOLATED_ANON),
		global_node_page_state(NR_ACTIVE_FILE),
		global_node_page_state(NR_INACTIVE_FILE),
		global_node_page_state(NR_ISOLATED_FILE),
		global_node_page_state(NR_UNEVICTABLE),
5060 5061 5062
		global_node_page_state(NR_FILE_DIRTY),
		global_node_page_state(NR_WRITEBACK),
		global_node_page_state(NR_UNSTABLE_NFS),
5063 5064
		global_node_page_state(NR_SLAB_RECLAIMABLE),
		global_node_page_state(NR_SLAB_UNRECLAIMABLE),
5065
		global_node_page_state(NR_FILE_MAPPED),
5066
		global_node_page_state(NR_SHMEM),
5067 5068 5069
		global_zone_page_state(NR_PAGETABLE),
		global_zone_page_state(NR_BOUNCE),
		global_zone_page_state(NR_FREE_PAGES),
5070
		free_pcp,
5071
		global_zone_page_state(NR_FREE_CMA_PAGES));
L
Linus Torvalds 已提交
5072

M
Mel Gorman 已提交
5073
	for_each_online_pgdat(pgdat) {
5074
		if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
5075 5076
			continue;

M
Mel Gorman 已提交
5077 5078 5079 5080 5081 5082 5083 5084
		printk("Node %d"
			" active_anon:%lukB"
			" inactive_anon:%lukB"
			" active_file:%lukB"
			" inactive_file:%lukB"
			" unevictable:%lukB"
			" isolated(anon):%lukB"
			" isolated(file):%lukB"
5085
			" mapped:%lukB"
5086 5087 5088 5089 5090 5091 5092 5093 5094 5095
			" dirty:%lukB"
			" writeback:%lukB"
			" shmem:%lukB"
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
			" shmem_thp: %lukB"
			" shmem_pmdmapped: %lukB"
			" anon_thp: %lukB"
#endif
			" writeback_tmp:%lukB"
			" unstable:%lukB"
M
Mel Gorman 已提交
5096 5097 5098 5099 5100 5101 5102 5103 5104 5105
			" all_unreclaimable? %s"
			"\n",
			pgdat->node_id,
			K(node_page_state(pgdat, NR_ACTIVE_ANON)),
			K(node_page_state(pgdat, NR_INACTIVE_ANON)),
			K(node_page_state(pgdat, NR_ACTIVE_FILE)),
			K(node_page_state(pgdat, NR_INACTIVE_FILE)),
			K(node_page_state(pgdat, NR_UNEVICTABLE)),
			K(node_page_state(pgdat, NR_ISOLATED_ANON)),
			K(node_page_state(pgdat, NR_ISOLATED_FILE)),
5106
			K(node_page_state(pgdat, NR_FILE_MAPPED)),
5107 5108
			K(node_page_state(pgdat, NR_FILE_DIRTY)),
			K(node_page_state(pgdat, NR_WRITEBACK)),
5109
			K(node_page_state(pgdat, NR_SHMEM)),
5110 5111 5112 5113 5114 5115 5116 5117
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
			K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
			K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
					* HPAGE_PMD_NR),
			K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
#endif
			K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
			K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
5118 5119
			pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
				"yes" : "no");
M
Mel Gorman 已提交
5120 5121
	}

5122
	for_each_populated_zone(zone) {
L
Linus Torvalds 已提交
5123 5124
		int i;

5125
		if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5126
			continue;
5127 5128 5129 5130 5131

		free_pcp = 0;
		for_each_online_cpu(cpu)
			free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;

L
Linus Torvalds 已提交
5132
		show_node(zone);
5133 5134
		printk(KERN_CONT
			"%s"
L
Linus Torvalds 已提交
5135 5136 5137 5138
			" free:%lukB"
			" min:%lukB"
			" low:%lukB"
			" high:%lukB"
M
Minchan Kim 已提交
5139 5140 5141 5142 5143
			" active_anon:%lukB"
			" inactive_anon:%lukB"
			" active_file:%lukB"
			" inactive_file:%lukB"
			" unevictable:%lukB"
5144
			" writepending:%lukB"
L
Linus Torvalds 已提交
5145
			" present:%lukB"
5146
			" managed:%lukB"
5147
			" mlocked:%lukB"
5148
			" kernel_stack:%lukB"
5149 5150
			" pagetables:%lukB"
			" bounce:%lukB"
5151 5152
			" free_pcp:%lukB"
			" local_pcp:%ukB"
5153
			" free_cma:%lukB"
L
Linus Torvalds 已提交
5154 5155
			"\n",
			zone->name,
5156
			K(zone_page_state(zone, NR_FREE_PAGES)),
5157 5158 5159
			K(min_wmark_pages(zone)),
			K(low_wmark_pages(zone)),
			K(high_wmark_pages(zone)),
M
Minchan Kim 已提交
5160 5161 5162 5163 5164
			K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
			K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
			K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
			K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
			K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
5165
			K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
L
Linus Torvalds 已提交
5166
			K(zone->present_pages),
5167
			K(zone->managed_pages),
5168
			K(zone_page_state(zone, NR_MLOCK)),
5169
			zone_page_state(zone, NR_KERNEL_STACK_KB),
5170 5171
			K(zone_page_state(zone, NR_PAGETABLE)),
			K(zone_page_state(zone, NR_BOUNCE)),
5172 5173
			K(free_pcp),
			K(this_cpu_read(zone->pageset->pcp.count)),
5174
			K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
L
Linus Torvalds 已提交
5175 5176
		printk("lowmem_reserve[]:");
		for (i = 0; i < MAX_NR_ZONES; i++)
5177 5178
			printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
		printk(KERN_CONT "\n");
L
Linus Torvalds 已提交
5179 5180
	}

5181
	for_each_populated_zone(zone) {
5182 5183
		unsigned int order;
		unsigned long nr[MAX_ORDER], flags, total = 0;
5184
		unsigned char types[MAX_ORDER];
L
Linus Torvalds 已提交
5185

5186
		if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5187
			continue;
L
Linus Torvalds 已提交
5188
		show_node(zone);
5189
		printk(KERN_CONT "%s: ", zone->name);
L
Linus Torvalds 已提交
5190 5191 5192

		spin_lock_irqsave(&zone->lock, flags);
		for (order = 0; order < MAX_ORDER; order++) {
5193 5194 5195 5196
			struct free_area *area = &zone->free_area[order];
			int type;

			nr[order] = area->nr_free;
5197
			total += nr[order] << order;
5198 5199 5200

			types[order] = 0;
			for (type = 0; type < MIGRATE_TYPES; type++) {
5201
				if (!free_area_empty(area, type))
5202 5203
					types[order] |= 1 << type;
			}
L
Linus Torvalds 已提交
5204 5205
		}
		spin_unlock_irqrestore(&zone->lock, flags);
5206
		for (order = 0; order < MAX_ORDER; order++) {
5207 5208
			printk(KERN_CONT "%lu*%lukB ",
			       nr[order], K(1UL) << order);
5209 5210 5211
			if (nr[order])
				show_migration_types(types[order]);
		}
5212
		printk(KERN_CONT "= %lukB\n", K(total));
L
Linus Torvalds 已提交
5213 5214
	}

5215 5216
	hugetlb_show_meminfo();

5217
	printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
5218

L
Linus Torvalds 已提交
5219 5220 5221
	show_swap_cache_info();
}

5222 5223 5224 5225 5226 5227
static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
{
	zoneref->zone = zone;
	zoneref->zone_idx = zone_idx(zone);
}

L
Linus Torvalds 已提交
5228 5229
/*
 * Builds allocation fallback zone lists.
5230 5231
 *
 * Add all populated zones of a node to the zonelist.
L
Linus Torvalds 已提交
5232
 */
5233
static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
L
Linus Torvalds 已提交
5234
{
5235
	struct zone *zone;
5236
	enum zone_type zone_type = MAX_NR_ZONES;
5237
	int nr_zones = 0;
5238 5239

	do {
5240
		zone_type--;
5241
		zone = pgdat->node_zones + zone_type;
5242
		if (managed_zone(zone)) {
5243
			zoneref_set_zone(zone, &zonerefs[nr_zones++]);
5244
			check_highest_zone(zone_type);
L
Linus Torvalds 已提交
5245
		}
5246
	} while (zone_type);
5247

5248
	return nr_zones;
L
Linus Torvalds 已提交
5249 5250 5251
}

#ifdef CONFIG_NUMA
5252 5253 5254

static int __parse_numa_zonelist_order(char *s)
{
5255 5256 5257 5258 5259 5260 5261 5262
	/*
	 * We used to support different zonlists modes but they turned
	 * out to be just not useful. Let's keep the warning in place
	 * if somebody still use the cmd line parameter so that we do
	 * not fail it silently
	 */
	if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
		pr_warn("Ignoring unsupported numa_zonelist_order value:  %s\n", s);
5263 5264 5265 5266 5267 5268 5269
		return -EINVAL;
	}
	return 0;
}

static __init int setup_numa_zonelist_order(char *s)
{
5270 5271 5272
	if (!s)
		return 0;

5273
	return __parse_numa_zonelist_order(s);
5274 5275 5276
}
early_param("numa_zonelist_order", setup_numa_zonelist_order);

5277 5278
char numa_zonelist_order[] = "Node";

5279 5280 5281
/*
 * sysctl handler for numa_zonelist_order
 */
5282
int numa_zonelist_order_handler(struct ctl_table *table, int write,
5283
		void __user *buffer, size_t *length,
5284 5285
		loff_t *ppos)
{
5286
	char *str;
5287 5288
	int ret;

5289 5290 5291 5292 5293
	if (!write)
		return proc_dostring(table, write, buffer, length, ppos);
	str = memdup_user_nul(buffer, 16);
	if (IS_ERR(str))
		return PTR_ERR(str);
5294

5295 5296
	ret = __parse_numa_zonelist_order(str);
	kfree(str);
5297
	return ret;
5298 5299 5300
}


5301
#define MAX_NODE_LOAD (nr_online_nodes)
5302 5303
static int node_load[MAX_NUMNODES];

L
Linus Torvalds 已提交
5304
/**
5305
 * find_next_best_node - find the next node that should appear in a given node's fallback list
L
Linus Torvalds 已提交
5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317
 * @node: node whose fallback list we're appending
 * @used_node_mask: nodemask_t of already used nodes
 *
 * We use a number of factors to determine which is the next node that should
 * appear on a given node's fallback list.  The node should not have appeared
 * already in @node's fallback list, and it should be the next closest node
 * according to the distance array (which contains arbitrary distance values
 * from each node to each node in the system), and should also prefer nodes
 * with no CPUs, since presumably they'll have very little allocation pressure
 * on them otherwise.
 * It returns -1 if no node is found.
 */
5318
static int find_next_best_node(int node, nodemask_t *used_node_mask)
L
Linus Torvalds 已提交
5319
{
5320
	int n, val;
L
Linus Torvalds 已提交
5321
	int min_val = INT_MAX;
D
David Rientjes 已提交
5322
	int best_node = NUMA_NO_NODE;
5323
	const struct cpumask *tmp = cpumask_of_node(0);
L
Linus Torvalds 已提交
5324

5325 5326 5327 5328 5329
	/* Use the local node if we haven't already */
	if (!node_isset(node, *used_node_mask)) {
		node_set(node, *used_node_mask);
		return node;
	}
L
Linus Torvalds 已提交
5330

5331
	for_each_node_state(n, N_MEMORY) {
L
Linus Torvalds 已提交
5332 5333 5334 5335 5336 5337 5338 5339

		/* Don't want a node to appear more than once */
		if (node_isset(n, *used_node_mask))
			continue;

		/* Use the distance array to find the distance */
		val = node_distance(node, n);

5340 5341 5342
		/* Penalize nodes under us ("prefer the next node") */
		val += (n < node);

L
Linus Torvalds 已提交
5343
		/* Give preference to headless and unused nodes */
5344 5345
		tmp = cpumask_of_node(n);
		if (!cpumask_empty(tmp))
L
Linus Torvalds 已提交
5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363
			val += PENALTY_FOR_NODE_WITH_CPUS;

		/* Slight preference for less loaded node */
		val *= (MAX_NODE_LOAD*MAX_NUMNODES);
		val += node_load[n];

		if (val < min_val) {
			min_val = val;
			best_node = n;
		}
	}

	if (best_node >= 0)
		node_set(best_node, *used_node_mask);

	return best_node;
}

5364 5365 5366 5367 5368 5369

/*
 * Build zonelists ordered by node and zones within node.
 * This results in maximum locality--normal zone overflows into local
 * DMA zone, if any--but risks exhausting DMA zone.
 */
5370 5371
static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
		unsigned nr_nodes)
L
Linus Torvalds 已提交
5372
{
5373 5374 5375 5376 5377 5378 5379 5380 5381
	struct zoneref *zonerefs;
	int i;

	zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;

	for (i = 0; i < nr_nodes; i++) {
		int nr_zones;

		pg_data_t *node = NODE_DATA(node_order[i]);
5382

5383 5384 5385 5386 5387
		nr_zones = build_zonerefs_node(node, zonerefs);
		zonerefs += nr_zones;
	}
	zonerefs->zone = NULL;
	zonerefs->zone_idx = 0;
5388 5389
}

5390 5391 5392 5393 5394
/*
 * Build gfp_thisnode zonelists
 */
static void build_thisnode_zonelists(pg_data_t *pgdat)
{
5395 5396
	struct zoneref *zonerefs;
	int nr_zones;
5397

5398 5399 5400 5401 5402
	zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
	nr_zones = build_zonerefs_node(pgdat, zonerefs);
	zonerefs += nr_zones;
	zonerefs->zone = NULL;
	zonerefs->zone_idx = 0;
5403 5404
}

5405 5406 5407 5408 5409 5410 5411 5412 5413
/*
 * Build zonelists ordered by zone and nodes within zones.
 * This results in conserving DMA zone[s] until all Normal memory is
 * exhausted, but results in overflowing to remote node while memory
 * may still exist in local DMA zone.
 */

static void build_zonelists(pg_data_t *pgdat)
{
5414 5415
	static int node_order[MAX_NUMNODES];
	int node, load, nr_nodes = 0;
L
Linus Torvalds 已提交
5416
	nodemask_t used_mask;
5417
	int local_node, prev_node;
L
Linus Torvalds 已提交
5418 5419 5420

	/* NUMA-aware ordering of nodes */
	local_node = pgdat->node_id;
5421
	load = nr_online_nodes;
L
Linus Torvalds 已提交
5422 5423
	prev_node = local_node;
	nodes_clear(used_mask);
5424 5425

	memset(node_order, 0, sizeof(node_order));
L
Linus Torvalds 已提交
5426 5427 5428 5429 5430 5431
	while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
		/*
		 * We don't want to pressure a particular node.
		 * So adding penalty to the first node in same
		 * distance group to make it round-robin.
		 */
5432 5433
		if (node_distance(local_node, node) !=
		    node_distance(local_node, prev_node))
5434 5435
			node_load[node] = load;

5436
		node_order[nr_nodes++] = node;
L
Linus Torvalds 已提交
5437 5438 5439
		prev_node = node;
		load--;
	}
5440

5441
	build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
5442
	build_thisnode_zonelists(pgdat);
L
Linus Torvalds 已提交
5443 5444
}

5445 5446 5447 5448 5449 5450 5451 5452 5453
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
/*
 * Return node id of node used for "local" allocations.
 * I.e., first node id of first zone in arg node's generic zonelist.
 * Used for initializing percpu 'numa_mem', which is used primarily
 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
 */
int local_memory_node(int node)
{
5454
	struct zoneref *z;
5455

5456
	z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
5457
				   gfp_zone(GFP_KERNEL),
5458
				   NULL);
5459
	return zone_to_nid(z->zone);
5460 5461
}
#endif
5462

5463 5464
static void setup_min_unmapped_ratio(void);
static void setup_min_slab_ratio(void);
L
Linus Torvalds 已提交
5465 5466
#else	/* CONFIG_NUMA */

5467
static void build_zonelists(pg_data_t *pgdat)
L
Linus Torvalds 已提交
5468
{
5469
	int node, local_node;
5470 5471
	struct zoneref *zonerefs;
	int nr_zones;
L
Linus Torvalds 已提交
5472 5473 5474

	local_node = pgdat->node_id;

5475 5476 5477
	zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
	nr_zones = build_zonerefs_node(pgdat, zonerefs);
	zonerefs += nr_zones;
L
Linus Torvalds 已提交
5478

5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489
	/*
	 * Now we build the zonelist so that it contains the zones
	 * of all the other nodes.
	 * We don't want to pressure a particular node, so when
	 * building the zones for node N, we make sure that the
	 * zones coming right after the local ones are those from
	 * node N+1 (modulo N)
	 */
	for (node = local_node + 1; node < MAX_NUMNODES; node++) {
		if (!node_online(node))
			continue;
5490 5491
		nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
		zonerefs += nr_zones;
L
Linus Torvalds 已提交
5492
	}
5493 5494 5495
	for (node = 0; node < local_node; node++) {
		if (!node_online(node))
			continue;
5496 5497
		nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
		zonerefs += nr_zones;
5498 5499
	}

5500 5501
	zonerefs->zone = NULL;
	zonerefs->zone_idx = 0;
L
Linus Torvalds 已提交
5502 5503 5504 5505
}

#endif	/* CONFIG_NUMA */

5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522
/*
 * Boot pageset table. One per cpu which is going to be used for all
 * zones and all nodes. The parameters will be set in such a way
 * that an item put on a list will immediately be handed over to
 * the buddy list. This is safe since pageset manipulation is done
 * with interrupts disabled.
 *
 * The boot_pagesets must be kept even after bootup is complete for
 * unused processors and/or zones. They do play a role for bootstrapping
 * hotplugged processors.
 *
 * zoneinfo_show() and maybe other functions do
 * not check if the processor is online before following the pageset pointer.
 * Other parts of the kernel may not check if the zone is available.
 */
static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
5523
static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
5524

5525
static void __build_all_zonelists(void *data)
L
Linus Torvalds 已提交
5526
{
5527
	int nid;
5528
	int __maybe_unused cpu;
5529
	pg_data_t *self = data;
5530 5531 5532
	static DEFINE_SPINLOCK(lock);

	spin_lock(&lock);
5533

5534 5535 5536
#ifdef CONFIG_NUMA
	memset(node_load, 0, sizeof(node_load));
#endif
5537

5538 5539 5540 5541
	/*
	 * This node is hotadded and no memory is yet present.   So just
	 * building zonelists is fine - no need to touch other nodes.
	 */
5542 5543
	if (self && !node_online(self->node_id)) {
		build_zonelists(self);
5544 5545 5546
	} else {
		for_each_online_node(nid) {
			pg_data_t *pgdat = NODE_DATA(nid);
5547

5548 5549
			build_zonelists(pgdat);
		}
5550

5551 5552 5553 5554 5555 5556 5557 5558 5559
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
		/*
		 * We now know the "local memory node" for each node--
		 * i.e., the node of the first zone in the generic zonelist.
		 * Set up numa_mem percpu variable for on-line cpus.  During
		 * boot, only the boot cpu should be on-line;  we'll init the
		 * secondary cpus' numa_mem as they come on-line.  During
		 * node/memory hotplug, we'll fixup all on-line cpus.
		 */
5560
		for_each_online_cpu(cpu)
5561
			set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
5562
#endif
5563
	}
5564 5565

	spin_unlock(&lock);
5566 5567
}

5568 5569 5570
static noinline void __init
build_all_zonelists_init(void)
{
5571 5572
	int cpu;

5573
	__build_all_zonelists(NULL);
5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590

	/*
	 * Initialize the boot_pagesets that are going to be used
	 * for bootstrapping processors. The real pagesets for
	 * each zone will be allocated later when the per cpu
	 * allocator is available.
	 *
	 * boot_pagesets are used also for bootstrapping offline
	 * cpus if the system is already booted because the pagesets
	 * are needed to initialize allocators on a specific cpu too.
	 * F.e. the percpu allocator needs the page allocator which
	 * needs the percpu allocator in order to allocate its pagesets
	 * (a chicken-egg dilemma).
	 */
	for_each_possible_cpu(cpu)
		setup_pageset(&per_cpu(boot_pageset, cpu), 0);

5591 5592 5593 5594
	mminit_verify_zonelist();
	cpuset_init_current_mems_allowed();
}

5595 5596
/*
 * unless system_state == SYSTEM_BOOTING.
5597
 *
5598
 * __ref due to call of __init annotated helper build_all_zonelists_init
5599
 * [protected by SYSTEM_BOOTING].
5600
 */
5601
void __ref build_all_zonelists(pg_data_t *pgdat)
5602 5603
{
	if (system_state == SYSTEM_BOOTING) {
5604
		build_all_zonelists_init();
5605
	} else {
5606
		__build_all_zonelists(pgdat);
5607 5608
		/* cpuset refresh routine should be here */
	}
5609
	vm_total_pages = nr_free_pagecache_pages();
5610 5611 5612 5613 5614 5615 5616
	/*
	 * Disable grouping by mobility if the number of pages in the
	 * system is too low to allow the mechanism to work. It would be
	 * more accurate, but expensive to check per-zone. This check is
	 * made on memory-hotadd so a system can start with mobility
	 * disabled and enable it later
	 */
5617
	if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
5618 5619 5620 5621
		page_group_by_mobility_disabled = 1;
	else
		page_group_by_mobility_disabled = 0;

5622
	pr_info("Built %i zonelists, mobility grouping %s.  Total pages: %ld\n",
J
Joe Perches 已提交
5623 5624 5625
		nr_online_nodes,
		page_group_by_mobility_disabled ? "off" : "on",
		vm_total_pages);
5626
#ifdef CONFIG_NUMA
5627
	pr_info("Policy zone: %s\n", zone_names[policy_zone]);
5628
#endif
L
Linus Torvalds 已提交
5629 5630 5631 5632 5633 5634 5635
}

/*
 * Initially all pages are reserved - free ones are freed
 * up by free_all_bootmem() once the early boot process is
 * done. Non-atomic initialization, single-pass.
 */
5636
void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
5637 5638
		unsigned long start_pfn, enum memmap_context context,
		struct vmem_altmap *altmap)
L
Linus Torvalds 已提交
5639
{
A
Andy Whitcroft 已提交
5640
	unsigned long end_pfn = start_pfn + size;
5641
	pg_data_t *pgdat = NODE_DATA(nid);
A
Andy Whitcroft 已提交
5642
	unsigned long pfn;
5643
	unsigned long nr_initialised = 0;
5644
	struct page *page;
5645 5646 5647
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
	struct memblock_region *r = NULL, *tmp;
#endif
L
Linus Torvalds 已提交
5648

5649 5650 5651
	if (highest_memmap_pfn < end_pfn - 1)
		highest_memmap_pfn = end_pfn - 1;

5652 5653 5654 5655 5656 5657 5658
	/*
	 * Honor reservation requested by the driver for this ZONE_DEVICE
	 * memory
	 */
	if (altmap && start_pfn == altmap->base_pfn)
		start_pfn += altmap->reserve;

5659
	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
D
Dave Hansen 已提交
5660
		/*
5661 5662
		 * There can be holes in boot-time mem_map[]s handed to this
		 * function.  They do not exist on hotplugged memory.
D
Dave Hansen 已提交
5663
		 */
5664 5665 5666
		if (context != MEMMAP_EARLY)
			goto not_early;

5667
		if (!early_pfn_valid(pfn))
5668 5669 5670 5671 5672
			continue;
		if (!early_pfn_in_nid(pfn, nid))
			continue;
		if (!update_defer_init(pgdat, pfn, end_pfn, &nr_initialised))
			break;
5673 5674

#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691
		/*
		 * Check given memblock attribute by firmware which can affect
		 * kernel memory layout.  If zone==ZONE_MOVABLE but memory is
		 * mirrored, it's an overlapped memmap init. skip it.
		 */
		if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
			if (!r || pfn >= memblock_region_memory_end_pfn(r)) {
				for_each_memblock(memory, tmp)
					if (pfn < memblock_region_memory_end_pfn(tmp))
						break;
				r = tmp;
			}
			if (pfn >= memblock_region_memory_base_pfn(r) &&
			    memblock_is_mirror(r)) {
				/* already initialized as NORMAL */
				pfn = memblock_region_memory_end_pfn(r);
				continue;
5692
			}
D
Dave Hansen 已提交
5693
		}
5694
#endif
5695

5696
not_early:
5697 5698 5699 5700 5701
		page = pfn_to_page(pfn);
		__init_single_page(page, pfn, zone, nid);
		if (context == MEMMAP_HOTPLUG)
			SetPageReserved(page);

5702 5703 5704 5705 5706
		/*
		 * Mark the block movable so that blocks are reserved for
		 * movable at startup. This will force kernel allocations
		 * to reserve their blocks rather than leaking throughout
		 * the address space during boot when many long-lived
5707
		 * kernel allocations are made.
5708 5709 5710 5711 5712
		 *
		 * bitmap is created for zone's valid pfn range. but memmap
		 * can be created for invalid pages (for alignment)
		 * check here not to call set_pageblock_migratetype() against
		 * pfn out of zone.
5713 5714 5715
		 *
		 * Please note that MEMMAP_HOTPLUG path doesn't clear memmap
		 * because this is done early in sparse_add_one_section
5716 5717 5718
		 */
		if (!(pfn & (pageblock_nr_pages - 1))) {
			set_pageblock_migratetype(page, MIGRATE_MOVABLE);
5719
			cond_resched();
5720
		}
L
Linus Torvalds 已提交
5721 5722 5723
	}
}

5724
static void __meminit zone_init_free_lists(struct zone *zone)
L
Linus Torvalds 已提交
5725
{
5726
	unsigned int order, t;
5727 5728
	for_each_migratetype_order(order, t) {
		INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
L
Linus Torvalds 已提交
5729 5730 5731 5732 5733 5734
		zone->free_area[order].nr_free = 0;
	}
}

#ifndef __HAVE_ARCH_MEMMAP_INIT
#define memmap_init(size, nid, zone, start_pfn) \
5735
	memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY, NULL)
L
Linus Torvalds 已提交
5736 5737
#endif

5738
static int zone_batchsize(struct zone *zone)
5739
{
5740
#ifdef CONFIG_MMU
5741 5742 5743 5744
	int batch;

	/*
	 * The per-cpu-pages pools are set to around 1000th of the
5745
	 * size of the zone.
5746
	 */
5747
	batch = zone->managed_pages / 1024;
5748 5749 5750
	/* But no more than a meg. */
	if (batch * PAGE_SIZE > 1024 * 1024)
		batch = (1024 * 1024) / PAGE_SIZE;
5751 5752 5753 5754 5755
	batch /= 4;		/* We effectively *= 4 below */
	if (batch < 1)
		batch = 1;

	/*
5756 5757 5758
	 * Clamp the batch to a 2^n - 1 value. Having a power
	 * of 2 value was found to be more likely to have
	 * suboptimal cache aliasing properties in some cases.
5759
	 *
5760 5761 5762 5763
	 * For example if 2 tasks are alternately allocating
	 * batches of pages, one task can end up with a lot
	 * of pages of one half of the possible page colors
	 * and the other with pages of the other colors.
5764
	 */
5765
	batch = rounddown_pow_of_two(batch + batch/2) - 1;
5766

5767
	return batch;
5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784

#else
	/* The deferral and batching of frees should be suppressed under NOMMU
	 * conditions.
	 *
	 * The problem is that NOMMU needs to be able to allocate large chunks
	 * of contiguous memory as there's no hardware page translation to
	 * assemble apparent contiguous memory from discontiguous pages.
	 *
	 * Queueing large contiguous runs of pages for batching, however,
	 * causes the pages to actually be freed in smaller chunks.  As there
	 * can be a significant delay between the individual batches being
	 * recycled, this leads to the once large chunks of space being
	 * fragmented and becoming unavailable for high-order allocations.
	 */
	return 0;
#endif
5785 5786
}

5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813
/*
 * pcp->high and pcp->batch values are related and dependent on one another:
 * ->batch must never be higher then ->high.
 * The following function updates them in a safe manner without read side
 * locking.
 *
 * Any new users of pcp->batch and pcp->high should ensure they can cope with
 * those fields changing asynchronously (acording the the above rule).
 *
 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
 * outside of boot time (or some other assurance that no concurrent updaters
 * exist).
 */
static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
		unsigned long batch)
{
       /* start with a fail safe value for batch */
	pcp->batch = 1;
	smp_wmb();

       /* Update high, then batch, in order */
	pcp->high = high;
	smp_wmb();

	pcp->batch = batch;
}

5814
/* a companion to pageset_set_high() */
5815 5816
static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
{
5817
	pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
5818 5819
}

5820
static void pageset_init(struct per_cpu_pageset *p)
5821 5822
{
	struct per_cpu_pages *pcp;
5823
	int migratetype;
5824

5825 5826
	memset(p, 0, sizeof(*p));

5827
	pcp = &p->pcp;
5828
	pcp->count = 0;
5829 5830
	for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
		INIT_LIST_HEAD(&pcp->lists[migratetype]);
5831 5832
}

5833 5834 5835 5836 5837 5838
static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
{
	pageset_init(p);
	pageset_set_batch(p, batch);
}

5839
/*
5840
 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
5841 5842
 * to the value high for the pageset p.
 */
5843
static void pageset_set_high(struct per_cpu_pageset *p,
5844 5845
				unsigned long high)
{
5846 5847 5848
	unsigned long batch = max(1UL, high / 4);
	if ((high / 4) > (PAGE_SHIFT * 8))
		batch = PAGE_SHIFT * 8;
5849

5850
	pageset_update(&p->pcp, high, batch);
5851 5852
}

5853 5854
static void pageset_set_high_and_batch(struct zone *zone,
				       struct per_cpu_pageset *pcp)
5855 5856
{
	if (percpu_pagelist_fraction)
5857
		pageset_set_high(pcp,
5858 5859 5860 5861 5862 5863
			(zone->managed_pages /
				percpu_pagelist_fraction));
	else
		pageset_set_batch(pcp, zone_batchsize(zone));
}

5864 5865 5866 5867 5868 5869 5870 5871
static void __meminit zone_pageset_init(struct zone *zone, int cpu)
{
	struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);

	pageset_init(pcp);
	pageset_set_high_and_batch(zone, pcp);
}

5872
void __meminit setup_zone_pageset(struct zone *zone)
5873 5874 5875
{
	int cpu;
	zone->pageset = alloc_percpu(struct per_cpu_pageset);
5876 5877
	for_each_possible_cpu(cpu)
		zone_pageset_init(zone, cpu);
5878 5879
}

5880
/*
5881 5882
 * Allocate per cpu pagesets and initialize them.
 * Before this call only boot pagesets were available.
5883
 */
5884
void __init setup_per_cpu_pageset(void)
5885
{
5886
	struct pglist_data *pgdat;
5887
	struct zone *zone;
5888

5889 5890
	for_each_populated_zone(zone)
		setup_zone_pageset(zone);
5891 5892 5893 5894

	for_each_online_pgdat(pgdat)
		pgdat->per_cpu_nodestats =
			alloc_percpu(struct per_cpu_nodestat);
5895 5896
}

5897
static __meminit void zone_pcp_init(struct zone *zone)
5898
{
5899 5900 5901 5902 5903 5904
	/*
	 * per cpu subsystem is not up at this point. The following code
	 * relies on the ability of the linker to provide the
	 * offset of a (static) per cpu variable into the per cpu area.
	 */
	zone->pageset = &boot_pageset;
5905

5906
	if (populated_zone(zone))
5907 5908 5909
		printk(KERN_DEBUG "  %s zone: %lu pages, LIFO batch:%u\n",
			zone->name, zone->present_pages,
					 zone_batchsize(zone));
5910 5911
}

5912
void __meminit init_currently_empty_zone(struct zone *zone,
5913
					unsigned long zone_start_pfn,
5914
					unsigned long size)
5915 5916
{
	struct pglist_data *pgdat = zone->zone_pgdat;
5917
	int zone_idx = zone_idx(zone) + 1;
5918

5919 5920
	if (zone_idx > pgdat->nr_zones)
		pgdat->nr_zones = zone_idx;
5921 5922 5923

	zone->zone_start_pfn = zone_start_pfn;

5924 5925 5926 5927 5928 5929
	mminit_dprintk(MMINIT_TRACE, "memmap_init",
			"Initialising map node %d zone %lu pfns %lu -> %lu\n",
			pgdat->node_id,
			(unsigned long)zone_idx(zone),
			zone_start_pfn, (zone_start_pfn + size));

5930
	zone_init_free_lists(zone);
5931
	zone->initialized = 1;
5932 5933
}

T
Tejun Heo 已提交
5934
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5935
#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
5936

5937 5938 5939
/*
 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
 */
5940 5941
int __meminit __early_pfn_to_nid(unsigned long pfn,
					struct mminit_pfnnid_cache *state)
5942
{
5943
	unsigned long start_pfn, end_pfn;
5944
	int nid;
5945

5946 5947
	if (state->last_start <= pfn && pfn < state->last_end)
		return state->last_nid;
5948

5949 5950
	nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
	if (nid != -1) {
5951 5952 5953
		state->last_start = start_pfn;
		state->last_end = end_pfn;
		state->last_nid = nid;
5954 5955 5956
	}

	return nid;
5957 5958 5959 5960
}
#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */

/**
5961
 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range
5962
 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
5963
 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid
5964
 *
5965 5966 5967
 * If an architecture guarantees that all ranges registered contain no holes
 * and may be freed, this this function may be used instead of calling
 * memblock_free_early_nid() manually.
5968
 */
5969
void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
5970
{
5971 5972
	unsigned long start_pfn, end_pfn;
	int i, this_nid;
5973

5974 5975 5976
	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
		start_pfn = min(start_pfn, max_low_pfn);
		end_pfn = min(end_pfn, max_low_pfn);
5977

5978
		if (start_pfn < end_pfn)
5979 5980 5981
			memblock_free_early_nid(PFN_PHYS(start_pfn),
					(end_pfn - start_pfn) << PAGE_SHIFT,
					this_nid);
5982 5983 5984
	}
}

5985 5986
/**
 * sparse_memory_present_with_active_regions - Call memory_present for each active range
5987
 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
5988
 *
5989 5990
 * If an architecture guarantees that all ranges registered contain no holes and may
 * be freed, this function may be used instead of calling memory_present() manually.
5991 5992 5993
 */
void __init sparse_memory_present_with_active_regions(int nid)
{
5994 5995
	unsigned long start_pfn, end_pfn;
	int i, this_nid;
5996

5997 5998
	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
		memory_present(this_nid, start_pfn, end_pfn);
5999 6000 6001 6002
}

/**
 * get_pfn_range_for_nid - Return the start and end page frames for a node
6003 6004 6005
 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
6006 6007
 *
 * It returns the start and end page frame of a node based on information
6008
 * provided by memblock_set_node(). If called for a node
6009
 * with no available memory, a warning is printed and the start and end
6010
 * PFNs will be 0.
6011
 */
6012
void __meminit get_pfn_range_for_nid(unsigned int nid,
6013 6014
			unsigned long *start_pfn, unsigned long *end_pfn)
{
6015
	unsigned long this_start_pfn, this_end_pfn;
6016
	int i;
6017

6018 6019 6020
	*start_pfn = -1UL;
	*end_pfn = 0;

6021 6022 6023
	for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
		*start_pfn = min(*start_pfn, this_start_pfn);
		*end_pfn = max(*end_pfn, this_end_pfn);
6024 6025
	}

6026
	if (*start_pfn == -1UL)
6027 6028 6029
		*start_pfn = 0;
}

M
Mel Gorman 已提交
6030 6031 6032 6033 6034
/*
 * This finds a zone that can be used for ZONE_MOVABLE pages. The
 * assumption is made that zones within a node are ordered in monotonic
 * increasing memory addresses so that the "highest" populated zone is used
 */
A
Adrian Bunk 已提交
6035
static void __init find_usable_zone_for_movable(void)
M
Mel Gorman 已提交
6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052
{
	int zone_index;
	for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
		if (zone_index == ZONE_MOVABLE)
			continue;

		if (arch_zone_highest_possible_pfn[zone_index] >
				arch_zone_lowest_possible_pfn[zone_index])
			break;
	}

	VM_BUG_ON(zone_index == -1);
	movable_zone = zone_index;
}

/*
 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
L
Lucas De Marchi 已提交
6053
 * because it is sized independent of architecture. Unlike the other zones,
M
Mel Gorman 已提交
6054 6055 6056 6057 6058 6059 6060
 * the starting point for ZONE_MOVABLE is not fixed. It may be different
 * in each node depending on the size of each node and how evenly kernelcore
 * is distributed. This helper function adjusts the zone ranges
 * provided by the architecture for a given node by using the end of the
 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
 * zones within a node are in order of monotonic increases memory addresses
 */
A
Adrian Bunk 已提交
6061
static void __meminit adjust_zone_range_for_zone_movable(int nid,
M
Mel Gorman 已提交
6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075
					unsigned long zone_type,
					unsigned long node_start_pfn,
					unsigned long node_end_pfn,
					unsigned long *zone_start_pfn,
					unsigned long *zone_end_pfn)
{
	/* Only adjust if ZONE_MOVABLE is on this node */
	if (zone_movable_pfn[nid]) {
		/* Size ZONE_MOVABLE */
		if (zone_type == ZONE_MOVABLE) {
			*zone_start_pfn = zone_movable_pfn[nid];
			*zone_end_pfn = min(node_end_pfn,
				arch_zone_highest_possible_pfn[movable_zone]);

6076 6077 6078 6079 6080 6081
		/* Adjust for ZONE_MOVABLE starting within this range */
		} else if (!mirrored_kernelcore &&
			*zone_start_pfn < zone_movable_pfn[nid] &&
			*zone_end_pfn > zone_movable_pfn[nid]) {
			*zone_end_pfn = zone_movable_pfn[nid];

M
Mel Gorman 已提交
6082 6083 6084 6085 6086 6087
		/* Check if this whole range is within ZONE_MOVABLE */
		} else if (*zone_start_pfn >= zone_movable_pfn[nid])
			*zone_start_pfn = *zone_end_pfn;
	}
}

6088 6089 6090 6091
/*
 * Return the number of pages a zone spans in a node, including holes
 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
 */
P
Paul Mundt 已提交
6092
static unsigned long __meminit zone_spanned_pages_in_node(int nid,
6093
					unsigned long zone_type,
6094 6095
					unsigned long node_start_pfn,
					unsigned long node_end_pfn,
6096 6097
					unsigned long *zone_start_pfn,
					unsigned long *zone_end_pfn,
6098 6099
					unsigned long *ignored)
{
6100 6101
	unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
	unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
6102
	/* When hotadd a new node from cpu_up(), the node should be empty */
6103 6104 6105
	if (!node_start_pfn && !node_end_pfn)
		return 0;

6106
	/* Get the start and end of the zone */
6107 6108
	*zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
	*zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
M
Mel Gorman 已提交
6109 6110
	adjust_zone_range_for_zone_movable(nid, zone_type,
				node_start_pfn, node_end_pfn,
6111
				zone_start_pfn, zone_end_pfn);
6112 6113

	/* Check that this node has pages within the zone's required range */
6114
	if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
6115 6116 6117
		return 0;

	/* Move the zone boundaries inside the node if necessary */
6118 6119
	*zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
	*zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
6120 6121

	/* Return the spanned pages */
6122
	return *zone_end_pfn - *zone_start_pfn;
6123 6124 6125 6126
}

/*
 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
6127
 * then all holes in the requested range will be accounted for.
6128
 */
6129
unsigned long __meminit __absent_pages_in_range(int nid,
6130 6131 6132
				unsigned long range_start_pfn,
				unsigned long range_end_pfn)
{
6133 6134 6135
	unsigned long nr_absent = range_end_pfn - range_start_pfn;
	unsigned long start_pfn, end_pfn;
	int i;
6136

6137 6138 6139 6140
	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
		start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
		end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
		nr_absent -= end_pfn - start_pfn;
6141
	}
6142
	return nr_absent;
6143 6144 6145 6146 6147 6148 6149
}

/**
 * absent_pages_in_range - Return number of page frames in holes within a range
 * @start_pfn: The start PFN to start searching for holes
 * @end_pfn: The end PFN to stop searching for holes
 *
6150
 * It returns the number of pages frames in memory holes within a range.
6151 6152 6153 6154 6155 6156 6157 6158
 */
unsigned long __init absent_pages_in_range(unsigned long start_pfn,
							unsigned long end_pfn)
{
	return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
}

/* Return the number of page frames in holes in a zone on a node */
P
Paul Mundt 已提交
6159
static unsigned long __meminit zone_absent_pages_in_node(int nid,
6160
					unsigned long zone_type,
6161 6162
					unsigned long node_start_pfn,
					unsigned long node_end_pfn,
6163 6164
					unsigned long *ignored)
{
6165 6166
	unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
	unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
6167
	unsigned long zone_start_pfn, zone_end_pfn;
6168
	unsigned long nr_absent;
6169

6170
	/* When hotadd a new node from cpu_up(), the node should be empty */
6171 6172 6173
	if (!node_start_pfn && !node_end_pfn)
		return 0;

6174 6175
	zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
	zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
6176

M
Mel Gorman 已提交
6177 6178 6179
	adjust_zone_range_for_zone_movable(nid, zone_type,
			node_start_pfn, node_end_pfn,
			&zone_start_pfn, &zone_end_pfn);
6180 6181 6182 6183 6184 6185 6186
	nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);

	/*
	 * ZONE_MOVABLE handling.
	 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
	 * and vice versa.
	 */
6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203
	if (mirrored_kernelcore && zone_movable_pfn[nid]) {
		unsigned long start_pfn, end_pfn;
		struct memblock_region *r;

		for_each_memblock(memory, r) {
			start_pfn = clamp(memblock_region_memory_base_pfn(r),
					  zone_start_pfn, zone_end_pfn);
			end_pfn = clamp(memblock_region_memory_end_pfn(r),
					zone_start_pfn, zone_end_pfn);

			if (zone_type == ZONE_MOVABLE &&
			    memblock_is_mirror(r))
				nr_absent += end_pfn - start_pfn;

			if (zone_type == ZONE_NORMAL &&
			    !memblock_is_mirror(r))
				nr_absent += end_pfn - start_pfn;
6204 6205 6206 6207
		}
	}

	return nr_absent;
6208
}
6209

T
Tejun Heo 已提交
6210
#else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
P
Paul Mundt 已提交
6211
static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
6212
					unsigned long zone_type,
6213 6214
					unsigned long node_start_pfn,
					unsigned long node_end_pfn,
6215 6216
					unsigned long *zone_start_pfn,
					unsigned long *zone_end_pfn,
6217 6218
					unsigned long *zones_size)
{
6219 6220 6221 6222 6223 6224 6225 6226
	unsigned int zone;

	*zone_start_pfn = node_start_pfn;
	for (zone = 0; zone < zone_type; zone++)
		*zone_start_pfn += zones_size[zone];

	*zone_end_pfn = *zone_start_pfn + zones_size[zone_type];

6227 6228 6229
	return zones_size[zone_type];
}

P
Paul Mundt 已提交
6230
static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
6231
						unsigned long zone_type,
6232 6233
						unsigned long node_start_pfn,
						unsigned long node_end_pfn,
6234 6235 6236 6237 6238 6239 6240
						unsigned long *zholes_size)
{
	if (!zholes_size)
		return 0;

	return zholes_size[zone_type];
}
6241

T
Tejun Heo 已提交
6242
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
6243

6244
static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
6245 6246 6247 6248
						unsigned long node_start_pfn,
						unsigned long node_end_pfn,
						unsigned long *zones_size,
						unsigned long *zholes_size)
6249
{
6250
	unsigned long realtotalpages = 0, totalpages = 0;
6251 6252
	enum zone_type i;

6253 6254
	for (i = 0; i < MAX_NR_ZONES; i++) {
		struct zone *zone = pgdat->node_zones + i;
6255
		unsigned long zone_start_pfn, zone_end_pfn;
6256
		unsigned long size, real_size;
6257

6258 6259 6260
		size = zone_spanned_pages_in_node(pgdat->node_id, i,
						  node_start_pfn,
						  node_end_pfn,
6261 6262
						  &zone_start_pfn,
						  &zone_end_pfn,
6263 6264
						  zones_size);
		real_size = size - zone_absent_pages_in_node(pgdat->node_id, i,
6265 6266
						  node_start_pfn, node_end_pfn,
						  zholes_size);
6267 6268 6269 6270
		if (size)
			zone->zone_start_pfn = zone_start_pfn;
		else
			zone->zone_start_pfn = 0;
6271 6272 6273 6274 6275 6276 6277 6278
		zone->spanned_pages = size;
		zone->present_pages = real_size;

		totalpages += size;
		realtotalpages += real_size;
	}

	pgdat->node_spanned_pages = totalpages;
6279 6280 6281 6282 6283
	pgdat->node_present_pages = realtotalpages;
	printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
							realtotalpages);
}

6284 6285 6286
#ifndef CONFIG_SPARSEMEM
/*
 * Calculate the size of the zone->blockflags rounded to an unsigned long
6287 6288
 * Start by making sure zonesize is a multiple of pageblock_order by rounding
 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
6289 6290 6291
 * round what is now in bits to nearest long in bits, then return it in
 * bytes.
 */
6292
static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
6293 6294 6295
{
	unsigned long usemapsize;

6296
	zonesize += zone_start_pfn & (pageblock_nr_pages-1);
6297 6298
	usemapsize = roundup(zonesize, pageblock_nr_pages);
	usemapsize = usemapsize >> pageblock_order;
6299 6300 6301 6302 6303 6304
	usemapsize *= NR_PAGEBLOCK_BITS;
	usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));

	return usemapsize / 8;
}

P
Pavel Tatashin 已提交
6305
static void __ref setup_usemap(struct pglist_data *pgdat,
6306 6307 6308
				struct zone *zone,
				unsigned long zone_start_pfn,
				unsigned long zonesize)
6309
{
6310
	unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
6311
	zone->pageblock_flags = NULL;
6312
	if (usemapsize)
6313 6314 6315
		zone->pageblock_flags =
			memblock_virt_alloc_node_nopanic(usemapsize,
							 pgdat->node_id);
6316 6317
}
#else
6318 6319
static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
				unsigned long zone_start_pfn, unsigned long zonesize) {}
6320 6321
#endif /* CONFIG_SPARSEMEM */

6322
#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
6323

6324
/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
6325
void __init set_pageblock_order(void)
6326
{
6327 6328
	unsigned int order;

6329 6330 6331 6332
	/* Check that pageblock_nr_pages has not already been setup */
	if (pageblock_order)
		return;

6333 6334 6335 6336 6337
	if (HPAGE_SHIFT > PAGE_SHIFT)
		order = HUGETLB_PAGE_ORDER;
	else
		order = MAX_ORDER - 1;

6338 6339
	/*
	 * Assume the largest contiguous order of interest is a huge page.
6340 6341
	 * This value may be variable depending on boot parameters on IA64 and
	 * powerpc.
6342 6343 6344 6345 6346
	 */
	pageblock_order = order;
}
#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */

6347 6348
/*
 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
6349 6350 6351
 * is unused as pageblock_order is set at compile-time. See
 * include/linux/pageblock-flags.h for the values of pageblock_order based on
 * the kernel config
6352
 */
6353
void __init set_pageblock_order(void)
6354 6355
{
}
6356 6357 6358

#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */

6359
static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
P
Pavel Tatashin 已提交
6360
						unsigned long present_pages)
6361 6362 6363 6364 6365 6366 6367 6368
{
	unsigned long pages = spanned_pages;

	/*
	 * Provide a more accurate estimation if there are holes within
	 * the zone and SPARSEMEM is in use. If there are holes within the
	 * zone, each populated memory region may cost us one or two extra
	 * memmap pages due to alignment because memmap pages for each
6369
	 * populated regions may not be naturally aligned on page boundary.
6370 6371 6372 6373 6374 6375 6376 6377 6378
	 * So the (present_pages >> 4) heuristic is a tradeoff for that.
	 */
	if (spanned_pages > present_pages + (present_pages >> 4) &&
	    IS_ENABLED(CONFIG_SPARSEMEM))
		pages = present_pages;

	return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
}

6379 6380 6381
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static void pgdat_init_split_queue(struct pglist_data *pgdat)
{
6382 6383 6384 6385 6386
	struct deferred_split *ds_queue = &pgdat->deferred_split_queue;

	spin_lock_init(&ds_queue->split_queue_lock);
	INIT_LIST_HEAD(&ds_queue->split_queue);
	ds_queue->split_queue_len = 0;
6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400
}
#else
static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
#endif

#ifdef CONFIG_COMPACTION
static void pgdat_init_kcompactd(struct pglist_data *pgdat)
{
	init_waitqueue_head(&pgdat->kcompactd_wait);
}
#else
static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
#endif

6401
static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
L
Linus Torvalds 已提交
6402
{
6403
	pgdat_resize_init(pgdat);
6404 6405 6406 6407

	pgdat_init_split_queue(pgdat);
	pgdat_init_kcompactd(pgdat);

L
Linus Torvalds 已提交
6408
	init_waitqueue_head(&pgdat->kswapd_wait);
6409
	init_waitqueue_head(&pgdat->pfmemalloc_wait);
6410

6411
	pgdat_page_ext_init(pgdat);
6412
	spin_lock_init(&pgdat->lru_lock);
6413
	lruvec_init(node_lruvec(pgdat));
6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459
}

static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
							unsigned long remaining_pages)
{
	zone->managed_pages = remaining_pages;
	zone_set_nid(zone, nid);
	zone->name = zone_names[idx];
	zone->zone_pgdat = NODE_DATA(nid);
	spin_lock_init(&zone->lock);
	zone_seqlock_init(zone);
	zone_pcp_init(zone);
}

/*
 * Set up the zone data structures
 * - init pgdat internals
 * - init all zones belonging to this node
 *
 * NOTE: this function is only called during memory hotplug
 */
#ifdef CONFIG_MEMORY_HOTPLUG
void __ref free_area_init_core_hotplug(int nid)
{
	enum zone_type z;
	pg_data_t *pgdat = NODE_DATA(nid);

	pgdat_init_internals(pgdat);
	for (z = 0; z < MAX_NR_ZONES; z++)
		zone_init_internals(&pgdat->node_zones[z], z, nid, 0);
}
#endif

/*
 * Set up the zone data structures:
 *   - mark all pages reserved
 *   - mark all memory queues empty
 *   - clear the memory bitmaps
 *
 * NOTE: pgdat should get zeroed by caller.
 * NOTE: this function is only called during early init.
 */
static void __init free_area_init_core(struct pglist_data *pgdat)
{
	enum zone_type j;
	int nid = pgdat->node_id;
6460

6461
	pgdat_init_internals(pgdat);
6462 6463
	pgdat->per_cpu_nodestats = &boot_nodestats;

L
Linus Torvalds 已提交
6464 6465
	for (j = 0; j < MAX_NR_ZONES; j++) {
		struct zone *zone = pgdat->node_zones + j;
6466
		unsigned long size, freesize, memmap_pages;
6467
		unsigned long zone_start_pfn = zone->zone_start_pfn;
L
Linus Torvalds 已提交
6468

6469
		size = zone->spanned_pages;
6470
		freesize = zone->present_pages;
L
Linus Torvalds 已提交
6471

6472
		/*
6473
		 * Adjust freesize so that it accounts for how much memory
6474 6475 6476
		 * is used by this zone for memmap. This affects the watermark
		 * and per-cpu initialisations
		 */
6477
		memmap_pages = calc_memmap_size(size, freesize);
6478 6479 6480 6481 6482 6483 6484 6485
		if (!is_highmem_idx(j)) {
			if (freesize >= memmap_pages) {
				freesize -= memmap_pages;
				if (memmap_pages)
					printk(KERN_DEBUG
					       "  %s zone: %lu pages used for memmap\n",
					       zone_names[j], memmap_pages);
			} else
6486
				pr_warn("  %s zone: %lu pages exceeds freesize %lu\n",
6487 6488
					zone_names[j], memmap_pages, freesize);
		}
6489

6490
		/* Account for reserved pages */
6491 6492
		if (j == 0 && freesize > dma_reserve) {
			freesize -= dma_reserve;
Y
Yinghai Lu 已提交
6493
			printk(KERN_DEBUG "  %s zone: %lu pages reserved\n",
6494
					zone_names[0], dma_reserve);
6495 6496
		}

6497
		if (!is_highmem_idx(j))
6498
			nr_kernel_pages += freesize;
6499 6500 6501
		/* Charge for highmem memmap if there are enough kernel pages */
		else if (nr_kernel_pages > memmap_pages * 2)
			nr_kernel_pages -= memmap_pages;
6502
		nr_all_pages += freesize;
L
Linus Torvalds 已提交
6503

6504 6505 6506 6507 6508
		/*
		 * Set an approximate value for lowmem here, it will be adjusted
		 * when the bootmem allocator frees pages into the buddy system.
		 * And all highmem pages will be managed by the buddy system.
		 */
6509
		zone_init_internals(zone, j, nid, freesize);
6510

6511
		if (!size)
L
Linus Torvalds 已提交
6512 6513
			continue;

6514
		set_pageblock_order();
6515 6516
		setup_usemap(pgdat, zone, zone_start_pfn, size);
		init_currently_empty_zone(zone, zone_start_pfn, size);
6517
		memmap_init(size, nid, j, zone_start_pfn);
L
Linus Torvalds 已提交
6518 6519 6520
	}
}

6521
#ifdef CONFIG_FLAT_NODE_MEM_MAP
6522
static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
L
Linus Torvalds 已提交
6523
{
6524
	unsigned long __maybe_unused start = 0;
L
Laura Abbott 已提交
6525 6526
	unsigned long __maybe_unused offset = 0;

L
Linus Torvalds 已提交
6527 6528 6529 6530
	/* Skip empty nodes */
	if (!pgdat->node_spanned_pages)
		return;

6531 6532
	start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
	offset = pgdat->node_start_pfn - start;
L
Linus Torvalds 已提交
6533 6534
	/* ia64 gets its own node_mem_map, before this, without bootmem */
	if (!pgdat->node_mem_map) {
6535
		unsigned long size, end;
A
Andy Whitcroft 已提交
6536 6537
		struct page *map;

6538 6539 6540 6541 6542
		/*
		 * The zone's endpoints aren't required to be MAX_ORDER
		 * aligned but the node_mem_map endpoints must be in order
		 * for the buddy allocator to function correctly.
		 */
6543
		end = pgdat_end_pfn(pgdat);
6544 6545
		end = ALIGN(end, MAX_ORDER_NR_PAGES);
		size =  (end - start) * sizeof(struct page);
6546
		map = memblock_virt_alloc_node_nopanic(size, pgdat->node_id);
L
Laura Abbott 已提交
6547
		pgdat->node_mem_map = map + offset;
L
Linus Torvalds 已提交
6548
	}
6549 6550 6551
	pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
				__func__, pgdat->node_id, (unsigned long)pgdat,
				(unsigned long)pgdat->node_mem_map);
6552
#ifndef CONFIG_NEED_MULTIPLE_NODES
L
Linus Torvalds 已提交
6553 6554 6555
	/*
	 * With no DISCONTIG, the global mem_map is just set as node 0's
	 */
6556
	if (pgdat == NODE_DATA(0)) {
L
Linus Torvalds 已提交
6557
		mem_map = NODE_DATA(0)->node_mem_map;
L
Laura Abbott 已提交
6558
#if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM)
6559
		if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
L
Laura Abbott 已提交
6560
			mem_map -= offset;
T
Tejun Heo 已提交
6561
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
6562
	}
L
Linus Torvalds 已提交
6563 6564
#endif
}
6565 6566 6567
#else
static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { }
#endif /* CONFIG_FLAT_NODE_MEM_MAP */
L
Linus Torvalds 已提交
6568

6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
{
	/*
	 * We start only with one section of pages, more pages are added as
	 * needed until the rest of deferred pages are initialized.
	 */
	pgdat->static_init_pgcnt = min_t(unsigned long, PAGES_PER_SECTION,
						pgdat->node_spanned_pages);
	pgdat->first_deferred_pfn = ULONG_MAX;
}
#else
static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
#endif

6584
void __init free_area_init_node(int nid, unsigned long *zones_size,
P
Pavel Tatashin 已提交
6585 6586
				   unsigned long node_start_pfn,
				   unsigned long *zholes_size)
L
Linus Torvalds 已提交
6587
{
6588
	pg_data_t *pgdat = NODE_DATA(nid);
6589 6590
	unsigned long start_pfn = 0;
	unsigned long end_pfn = 0;
6591

6592
	/* pg_data_t should be reset to zero when it's allocated */
6593
	WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx);
6594

L
Linus Torvalds 已提交
6595 6596
	pgdat->node_id = nid;
	pgdat->node_start_pfn = node_start_pfn;
6597
	pgdat->per_cpu_nodestats = NULL;
6598 6599
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
	get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
6600
	pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
6601 6602
		(u64)start_pfn << PAGE_SHIFT,
		end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
6603 6604
#else
	start_pfn = node_start_pfn;
6605 6606 6607
#endif
	calculate_node_totalpages(pgdat, start_pfn, end_pfn,
				  zones_size, zholes_size);
L
Linus Torvalds 已提交
6608 6609

	alloc_node_mem_map(pgdat);
6610
	pgdat_set_deferred_range(pgdat);
L
Linus Torvalds 已提交
6611

6612
	free_area_init_core(pgdat);
L
Linus Torvalds 已提交
6613 6614
}

6615
#if defined(CONFIG_HAVE_MEMBLOCK) && !defined(CONFIG_FLAT_NODE_MEM_MAP)
6616 6617 6618 6619 6620 6621 6622
/*
 * Only struct pages that are backed by physical memory are zeroed and
 * initialized by going through __init_single_page(). But, there are some
 * struct pages which are reserved in memblock allocator and their fields
 * may be accessed (for example page_to_pfn() on some configuration accesses
 * flags). We must explicitly zero those struct pages.
 */
6623
void __init zero_resv_unavail(void)
6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635
{
	phys_addr_t start, end;
	unsigned long pfn;
	u64 i, pgcnt;

	/*
	 * Loop through ranges that are reserved, but do not have reported
	 * physical memory backing.
	 */
	pgcnt = 0;
	for_each_resv_unavail_range(i, &start, &end) {
		for (pfn = PFN_DOWN(start); pfn < PFN_UP(end); pfn++) {
6636 6637 6638
			if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) {
				pfn = ALIGN_DOWN(pfn, pageblock_nr_pages)
					+ pageblock_nr_pages - 1;
6639
				continue;
6640
			}
6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655
			mm_zero_struct_page(pfn_to_page(pfn));
			pgcnt++;
		}
	}

	/*
	 * Struct pages that do not have backing memory. This could be because
	 * firmware is using some of this memory, or for some other reasons.
	 * Once memblock is changed so such behaviour is not allowed: i.e.
	 * list of "reserved" memory must be a subset of list of "memory", then
	 * this code can be removed.
	 */
	if (pgcnt)
		pr_info("Reserved but unavailable: %lld pages", pgcnt);
}
6656
#endif /* CONFIG_HAVE_MEMBLOCK && !CONFIG_FLAT_NODE_MEM_MAP */
6657

T
Tejun Heo 已提交
6658
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
M
Miklos Szeredi 已提交
6659 6660 6661 6662 6663

#if MAX_NUMNODES > 1
/*
 * Figure out the number of possible node ids.
 */
6664
void __init setup_nr_node_ids(void)
M
Miklos Szeredi 已提交
6665
{
6666
	unsigned int highest;
M
Miklos Szeredi 已提交
6667

6668
	highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
M
Miklos Szeredi 已提交
6669 6670 6671 6672
	nr_node_ids = highest + 1;
}
#endif

6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694
/**
 * node_map_pfn_alignment - determine the maximum internode alignment
 *
 * This function should be called after node map is populated and sorted.
 * It calculates the maximum power of two alignment which can distinguish
 * all the nodes.
 *
 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)).  If the
 * nodes are shifted by 256MiB, 256MiB.  Note that if only the last node is
 * shifted, 1GiB is enough and this function will indicate so.
 *
 * This is used to test whether pfn -> nid mapping of the chosen memory
 * model has fine enough granularity to avoid incorrect mapping for the
 * populated node map.
 *
 * Returns the determined alignment in pfn's.  0 if there is no alignment
 * requirement (single node).
 */
unsigned long __init node_map_pfn_alignment(void)
{
	unsigned long accl_mask = 0, last_end = 0;
6695
	unsigned long start, end, mask;
6696
	int last_nid = -1;
6697
	int i, nid;
6698

6699
	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722
		if (!start || last_nid < 0 || last_nid == nid) {
			last_nid = nid;
			last_end = end;
			continue;
		}

		/*
		 * Start with a mask granular enough to pin-point to the
		 * start pfn and tick off bits one-by-one until it becomes
		 * too coarse to separate the current node from the last.
		 */
		mask = ~((1 << __ffs(start)) - 1);
		while (mask && last_end <= (start & (mask << 1)))
			mask <<= 1;

		/* accumulate all internode masks */
		accl_mask |= mask;
	}

	/* convert mask to number of pages */
	return ~accl_mask + 1;
}

6723
/* Find the lowest pfn for a node */
A
Adrian Bunk 已提交
6724
static unsigned long __init find_min_pfn_for_node(int nid)
6725
{
6726
	unsigned long min_pfn = ULONG_MAX;
6727 6728
	unsigned long start_pfn;
	int i;
6729

6730 6731
	for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
		min_pfn = min(min_pfn, start_pfn);
6732

6733
	if (min_pfn == ULONG_MAX) {
6734
		pr_warn("Could not find start_pfn for node %d\n", nid);
6735 6736 6737 6738
		return 0;
	}

	return min_pfn;
6739 6740 6741 6742 6743 6744
}

/**
 * find_min_pfn_with_active_regions - Find the minimum PFN registered
 *
 * It returns the minimum PFN based on information provided via
6745
 * memblock_set_node().
6746 6747 6748 6749 6750 6751
 */
unsigned long __init find_min_pfn_with_active_regions(void)
{
	return find_min_pfn_for_node(MAX_NUMNODES);
}

6752 6753 6754
/*
 * early_calculate_totalpages()
 * Sum pages in active regions for movable zone.
6755
 * Populate N_MEMORY for calculating usable_nodes.
6756
 */
A
Adrian Bunk 已提交
6757
static unsigned long __init early_calculate_totalpages(void)
6758 6759
{
	unsigned long totalpages = 0;
6760 6761 6762 6763 6764
	unsigned long start_pfn, end_pfn;
	int i, nid;

	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
		unsigned long pages = end_pfn - start_pfn;
6765

6766 6767
		totalpages += pages;
		if (pages)
6768
			node_set_state(nid, N_MEMORY);
6769
	}
6770
	return totalpages;
6771 6772
}

M
Mel Gorman 已提交
6773 6774 6775 6776 6777 6778
/*
 * Find the PFN the Movable zone begins in each node. Kernel memory
 * is spread evenly between nodes as long as the nodes have enough
 * memory. When they don't, some nodes will have more kernelcore than
 * others
 */
6779
static void __init find_zone_movable_pfns_for_nodes(void)
M
Mel Gorman 已提交
6780 6781 6782 6783
{
	int i, nid;
	unsigned long usable_startpfn;
	unsigned long kernelcore_node, kernelcore_remaining;
6784
	/* save the state before borrow the nodemask */
6785
	nodemask_t saved_node_state = node_states[N_MEMORY];
6786
	unsigned long totalpages = early_calculate_totalpages();
6787
	int usable_nodes = nodes_weight(node_states[N_MEMORY]);
E
Emil Medve 已提交
6788
	struct memblock_region *r;
6789 6790 6791 6792 6793 6794 6795 6796 6797

	/* Need to find movable_zone earlier when movable_node is specified. */
	find_usable_zone_for_movable();

	/*
	 * If movable_node is specified, ignore kernelcore and movablecore
	 * options.
	 */
	if (movable_node_is_enabled()) {
E
Emil Medve 已提交
6798 6799
		for_each_memblock(memory, r) {
			if (!memblock_is_hotpluggable(r))
6800 6801
				continue;

E
Emil Medve 已提交
6802
			nid = r->nid;
6803

E
Emil Medve 已提交
6804
			usable_startpfn = PFN_DOWN(r->base);
6805 6806 6807 6808 6809 6810 6811
			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
				min(usable_startpfn, zone_movable_pfn[nid]) :
				usable_startpfn;
		}

		goto out2;
	}
M
Mel Gorman 已提交
6812

6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842
	/*
	 * If kernelcore=mirror is specified, ignore movablecore option
	 */
	if (mirrored_kernelcore) {
		bool mem_below_4gb_not_mirrored = false;

		for_each_memblock(memory, r) {
			if (memblock_is_mirror(r))
				continue;

			nid = r->nid;

			usable_startpfn = memblock_region_memory_base_pfn(r);

			if (usable_startpfn < 0x100000) {
				mem_below_4gb_not_mirrored = true;
				continue;
			}

			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
				min(usable_startpfn, zone_movable_pfn[nid]) :
				usable_startpfn;
		}

		if (mem_below_4gb_not_mirrored)
			pr_warn("This configuration results in unmirrored kernel memory.");

		goto out2;
	}

6843
	/*
6844 6845 6846 6847 6848 6849 6850 6851 6852 6853 6854 6855
	 * If kernelcore=nn% or movablecore=nn% was specified, calculate the
	 * amount of necessary memory.
	 */
	if (required_kernelcore_percent)
		required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
				       10000UL;
	if (required_movablecore_percent)
		required_movablecore = (totalpages * 100 * required_movablecore_percent) /
					10000UL;

	/*
	 * If movablecore= was specified, calculate what size of
6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870
	 * kernelcore that corresponds so that memory usable for
	 * any allocation type is evenly spread. If both kernelcore
	 * and movablecore are specified, then the value of kernelcore
	 * will be used for required_kernelcore if it's greater than
	 * what movablecore would have allowed.
	 */
	if (required_movablecore) {
		unsigned long corepages;

		/*
		 * Round-up so that ZONE_MOVABLE is at least as large as what
		 * was requested by the user
		 */
		required_movablecore =
			roundup(required_movablecore, MAX_ORDER_NR_PAGES);
6871
		required_movablecore = min(totalpages, required_movablecore);
6872 6873 6874 6875 6876
		corepages = totalpages - required_movablecore;

		required_kernelcore = max(required_kernelcore, corepages);
	}

6877 6878 6879 6880 6881
	/*
	 * If kernelcore was not specified or kernelcore size is larger
	 * than totalpages, there is no ZONE_MOVABLE.
	 */
	if (!required_kernelcore || required_kernelcore >= totalpages)
6882
		goto out;
M
Mel Gorman 已提交
6883 6884 6885 6886 6887 6888 6889

	/* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
	usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];

restart:
	/* Spread kernelcore memory as evenly as possible throughout nodes */
	kernelcore_node = required_kernelcore / usable_nodes;
6890
	for_each_node_state(nid, N_MEMORY) {
6891 6892
		unsigned long start_pfn, end_pfn;

M
Mel Gorman 已提交
6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908
		/*
		 * Recalculate kernelcore_node if the division per node
		 * now exceeds what is necessary to satisfy the requested
		 * amount of memory for the kernel
		 */
		if (required_kernelcore < kernelcore_node)
			kernelcore_node = required_kernelcore / usable_nodes;

		/*
		 * As the map is walked, we track how much memory is usable
		 * by the kernel using kernelcore_remaining. When it is
		 * 0, the rest of the node is usable by ZONE_MOVABLE
		 */
		kernelcore_remaining = kernelcore_node;

		/* Go through each range of PFNs within this node */
6909
		for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
M
Mel Gorman 已提交
6910 6911
			unsigned long size_pages;

6912
			start_pfn = max(start_pfn, zone_movable_pfn[nid]);
M
Mel Gorman 已提交
6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954
			if (start_pfn >= end_pfn)
				continue;

			/* Account for what is only usable for kernelcore */
			if (start_pfn < usable_startpfn) {
				unsigned long kernel_pages;
				kernel_pages = min(end_pfn, usable_startpfn)
								- start_pfn;

				kernelcore_remaining -= min(kernel_pages,
							kernelcore_remaining);
				required_kernelcore -= min(kernel_pages,
							required_kernelcore);

				/* Continue if range is now fully accounted */
				if (end_pfn <= usable_startpfn) {

					/*
					 * Push zone_movable_pfn to the end so
					 * that if we have to rebalance
					 * kernelcore across nodes, we will
					 * not double account here
					 */
					zone_movable_pfn[nid] = end_pfn;
					continue;
				}
				start_pfn = usable_startpfn;
			}

			/*
			 * The usable PFN range for ZONE_MOVABLE is from
			 * start_pfn->end_pfn. Calculate size_pages as the
			 * number of pages used as kernelcore
			 */
			size_pages = end_pfn - start_pfn;
			if (size_pages > kernelcore_remaining)
				size_pages = kernelcore_remaining;
			zone_movable_pfn[nid] = start_pfn + size_pages;

			/*
			 * Some kernelcore has been met, update counts and
			 * break if the kernelcore for this node has been
6955
			 * satisfied
M
Mel Gorman 已提交
6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968
			 */
			required_kernelcore -= min(required_kernelcore,
								size_pages);
			kernelcore_remaining -= size_pages;
			if (!kernelcore_remaining)
				break;
		}
	}

	/*
	 * If there is still required_kernelcore, we do another pass with one
	 * less node in the count. This will push zone_movable_pfn[nid] further
	 * along on the nodes that still have memory until kernelcore is
6969
	 * satisfied
M
Mel Gorman 已提交
6970 6971 6972 6973 6974
	 */
	usable_nodes--;
	if (usable_nodes && required_kernelcore > usable_nodes)
		goto restart;

6975
out2:
M
Mel Gorman 已提交
6976 6977 6978 6979
	/* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
	for (nid = 0; nid < MAX_NUMNODES; nid++)
		zone_movable_pfn[nid] =
			roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
6980

6981
out:
6982
	/* restore the node_state */
6983
	node_states[N_MEMORY] = saved_node_state;
M
Mel Gorman 已提交
6984 6985
}

6986 6987
/* Any regular or high memory on that node ? */
static void check_for_memory(pg_data_t *pgdat, int nid)
6988 6989 6990
{
	enum zone_type zone_type;

6991 6992 6993 6994
	if (N_MEMORY == N_NORMAL_MEMORY)
		return;

	for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
6995
		struct zone *zone = &pgdat->node_zones[zone_type];
6996
		if (populated_zone(zone)) {
6997 6998 6999 7000
			node_set_state(nid, N_HIGH_MEMORY);
			if (N_NORMAL_MEMORY != N_HIGH_MEMORY &&
			    zone_type <= ZONE_NORMAL)
				node_set_state(nid, N_NORMAL_MEMORY);
7001 7002
			break;
		}
7003 7004 7005
	}
}

7006 7007
/**
 * free_area_init_nodes - Initialise all pg_data_t and zone data
7008
 * @max_zone_pfn: an array of max PFNs for each zone
7009 7010
 *
 * This will call free_area_init_node() for each active node in the system.
7011
 * Using the page ranges provided by memblock_set_node(), the size of each
7012 7013 7014 7015 7016 7017 7018 7019 7020
 * zone in each node and their holes is calculated. If the maximum PFN
 * between two adjacent zones match, it is assumed that the zone is empty.
 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
 * starts where the previous one ended. For example, ZONE_DMA32 starts
 * at arch_max_dma_pfn.
 */
void __init free_area_init_nodes(unsigned long *max_zone_pfn)
{
7021 7022
	unsigned long start_pfn, end_pfn;
	int i, nid;
7023

7024 7025 7026 7027 7028
	/* Record where the zone boundaries are */
	memset(arch_zone_lowest_possible_pfn, 0,
				sizeof(arch_zone_lowest_possible_pfn));
	memset(arch_zone_highest_possible_pfn, 0,
				sizeof(arch_zone_highest_possible_pfn));
7029 7030 7031 7032

	start_pfn = find_min_pfn_with_active_regions();

	for (i = 0; i < MAX_NR_ZONES; i++) {
M
Mel Gorman 已提交
7033 7034
		if (i == ZONE_MOVABLE)
			continue;
7035 7036 7037 7038 7039 7040

		end_pfn = max(max_zone_pfn[i], start_pfn);
		arch_zone_lowest_possible_pfn[i] = start_pfn;
		arch_zone_highest_possible_pfn[i] = end_pfn;

		start_pfn = end_pfn;
7041
	}
M
Mel Gorman 已提交
7042 7043 7044

	/* Find the PFNs that ZONE_MOVABLE begins at in each node */
	memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
7045
	find_zone_movable_pfns_for_nodes();
7046 7047

	/* Print out the zone ranges */
7048
	pr_info("Zone ranges:\n");
M
Mel Gorman 已提交
7049 7050 7051
	for (i = 0; i < MAX_NR_ZONES; i++) {
		if (i == ZONE_MOVABLE)
			continue;
7052
		pr_info("  %-8s ", zone_names[i]);
7053 7054
		if (arch_zone_lowest_possible_pfn[i] ==
				arch_zone_highest_possible_pfn[i])
7055
			pr_cont("empty\n");
7056
		else
7057 7058 7059 7060
			pr_cont("[mem %#018Lx-%#018Lx]\n",
				(u64)arch_zone_lowest_possible_pfn[i]
					<< PAGE_SHIFT,
				((u64)arch_zone_highest_possible_pfn[i]
7061
					<< PAGE_SHIFT) - 1);
M
Mel Gorman 已提交
7062 7063 7064
	}

	/* Print out the PFNs ZONE_MOVABLE begins at in each node */
7065
	pr_info("Movable zone start for each node\n");
M
Mel Gorman 已提交
7066 7067
	for (i = 0; i < MAX_NUMNODES; i++) {
		if (zone_movable_pfn[i])
7068 7069
			pr_info("  Node %d: %#018Lx\n", i,
			       (u64)zone_movable_pfn[i] << PAGE_SHIFT);
M
Mel Gorman 已提交
7070
	}
7071

7072
	/* Print out the early node map */
7073
	pr_info("Early memory node ranges\n");
7074
	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
7075 7076 7077
		pr_info("  node %3d: [mem %#018Lx-%#018Lx]\n", nid,
			(u64)start_pfn << PAGE_SHIFT,
			((u64)end_pfn << PAGE_SHIFT) - 1);
7078 7079

	/* Initialise every node */
7080
	mminit_verify_pageflags_layout();
7081
	setup_nr_node_ids();
7082
	zero_resv_unavail();
7083 7084
	for_each_online_node(nid) {
		pg_data_t *pgdat = NODE_DATA(nid);
7085
		free_area_init_node(nid, NULL,
7086
				find_min_pfn_for_node(nid), NULL);
7087 7088 7089

		/* Any memory on that node */
		if (pgdat->node_present_pages)
7090 7091
			node_set_state(nid, N_MEMORY);
		check_for_memory(pgdat, nid);
7092 7093
	}
}
M
Mel Gorman 已提交
7094

7095 7096
static int __init cmdline_parse_core(char *p, unsigned long *core,
				     unsigned long *percent)
M
Mel Gorman 已提交
7097 7098
{
	unsigned long long coremem;
7099 7100
	char *endptr;

M
Mel Gorman 已提交
7101 7102 7103
	if (!p)
		return -EINVAL;

7104 7105 7106 7107 7108
	/* Value may be a percentage of total memory, otherwise bytes */
	coremem = simple_strtoull(p, &endptr, 0);
	if (*endptr == '%') {
		/* Paranoid check for percent values greater than 100 */
		WARN_ON(coremem > 100);
M
Mel Gorman 已提交
7109

7110 7111 7112 7113 7114
		*percent = coremem;
	} else {
		coremem = memparse(p, &p);
		/* Paranoid check that UL is enough for the coremem value */
		WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
M
Mel Gorman 已提交
7115

7116 7117 7118
		*core = coremem >> PAGE_SHIFT;
		*percent = 0UL;
	}
M
Mel Gorman 已提交
7119 7120
	return 0;
}
M
Mel Gorman 已提交
7121

7122 7123 7124 7125 7126 7127
/*
 * kernelcore=size sets the amount of memory for use for allocations that
 * cannot be reclaimed or migrated.
 */
static int __init cmdline_parse_kernelcore(char *p)
{
7128 7129 7130 7131 7132 7133
	/* parse kernelcore=mirror */
	if (parse_option_str(p, "mirror")) {
		mirrored_kernelcore = true;
		return 0;
	}

7134 7135
	return cmdline_parse_core(p, &required_kernelcore,
				  &required_kernelcore_percent);
7136 7137 7138 7139 7140 7141 7142 7143
}

/*
 * movablecore=size sets the amount of memory for use for allocations that
 * can be reclaimed or migrated.
 */
static int __init cmdline_parse_movablecore(char *p)
{
7144 7145
	return cmdline_parse_core(p, &required_movablecore,
				  &required_movablecore_percent);
7146 7147
}

M
Mel Gorman 已提交
7148
early_param("kernelcore", cmdline_parse_kernelcore);
7149
early_param("movablecore", cmdline_parse_movablecore);
M
Mel Gorman 已提交
7150

T
Tejun Heo 已提交
7151
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
7152

7153 7154 7155 7156 7157
void adjust_managed_page_count(struct page *page, long count)
{
	spin_lock(&managed_page_count_lock);
	page_zone(page)->managed_pages += count;
	totalram_pages += count;
7158 7159 7160 7161
#ifdef CONFIG_HIGHMEM
	if (PageHighMem(page))
		totalhigh_pages += count;
#endif
7162 7163
	spin_unlock(&managed_page_count_lock);
}
7164
EXPORT_SYMBOL(adjust_managed_page_count);
7165

7166
unsigned long free_reserved_area(void *start, void *end, int poison, char *s)
7167
{
7168 7169
	void *pos;
	unsigned long pages = 0;
7170

7171 7172 7173
	start = (void *)PAGE_ALIGN((unsigned long)start);
	end = (void *)((unsigned long)end & PAGE_MASK);
	for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184
		struct page *page = virt_to_page(pos);
		void *direct_map_addr;

		/*
		 * 'direct_map_addr' might be different from 'pos'
		 * because some architectures' virt_to_page()
		 * work with aliases.  Getting the direct map
		 * address ensures that we get a _writeable_
		 * alias for the memset().
		 */
		direct_map_addr = page_address(page);
7185
		if ((unsigned int)poison <= 0xFF)
7186 7187 7188
			memset(direct_map_addr, poison, PAGE_SIZE);

		free_reserved_page(page);
7189 7190 7191
	}

	if (pages && s)
7192 7193
		pr_info("Freeing %s memory: %ldK\n",
			s, pages << (PAGE_SHIFT - 10));
7194 7195 7196

	return pages;
}
7197
EXPORT_SYMBOL(free_reserved_area);
7198

7199 7200 7201 7202 7203
#ifdef	CONFIG_HIGHMEM
void free_highmem_page(struct page *page)
{
	__free_reserved_page(page);
	totalram_pages++;
7204
	page_zone(page)->managed_pages++;
7205 7206 7207 7208
	totalhigh_pages++;
}
#endif

7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230

void __init mem_init_print_info(const char *str)
{
	unsigned long physpages, codesize, datasize, rosize, bss_size;
	unsigned long init_code_size, init_data_size;

	physpages = get_num_physpages();
	codesize = _etext - _stext;
	datasize = _edata - _sdata;
	rosize = __end_rodata - __start_rodata;
	bss_size = __bss_stop - __bss_start;
	init_data_size = __init_end - __init_begin;
	init_code_size = _einittext - _sinittext;

	/*
	 * Detect special cases and adjust section sizes accordingly:
	 * 1) .init.* may be embedded into .data sections
	 * 2) .init.text.* may be out of [__init_begin, __init_end],
	 *    please refer to arch/tile/kernel/vmlinux.lds.S.
	 * 3) .rodata.* may be embedded into .text or .data sections.
	 */
#define adj_init_size(start, end, size, pos, adj) \
7231 7232 7233 7234
	do { \
		if (start <= pos && pos < end && size > adj) \
			size -= adj; \
	} while (0)
7235 7236 7237 7238 7239 7240 7241 7242 7243 7244

	adj_init_size(__init_begin, __init_end, init_data_size,
		     _sinittext, init_code_size);
	adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
	adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
	adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
	adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);

#undef	adj_init_size

J
Joe Perches 已提交
7245
	pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
7246
#ifdef	CONFIG_HIGHMEM
J
Joe Perches 已提交
7247
		", %luK highmem"
7248
#endif
J
Joe Perches 已提交
7249 7250 7251 7252 7253 7254 7255
		"%s%s)\n",
		nr_free_pages() << (PAGE_SHIFT - 10),
		physpages << (PAGE_SHIFT - 10),
		codesize >> 10, datasize >> 10, rosize >> 10,
		(init_data_size + init_code_size) >> 10, bss_size >> 10,
		(physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10),
		totalcma_pages << (PAGE_SHIFT - 10),
7256
#ifdef	CONFIG_HIGHMEM
J
Joe Perches 已提交
7257
		totalhigh_pages << (PAGE_SHIFT - 10),
7258
#endif
J
Joe Perches 已提交
7259
		str ? ", " : "", str ? str : "");
7260 7261
}

7262
/**
7263 7264
 * set_dma_reserve - set the specified number of pages reserved in the first zone
 * @new_dma_reserve: The number of pages to mark reserved
7265
 *
7266
 * The per-cpu batchsize and zone watermarks are determined by managed_pages.
7267 7268
 * In the DMA zone, a significant percentage may be consumed by kernel image
 * and other unfreeable allocations which can skew the watermarks badly. This
7269 7270 7271
 * function may optionally be used to account for unfreeable pages in the
 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
 * smaller per-cpu batchsize.
7272 7273 7274 7275 7276 7277
 */
void __init set_dma_reserve(unsigned long new_dma_reserve)
{
	dma_reserve = new_dma_reserve;
}

L
Linus Torvalds 已提交
7278 7279
void __init free_area_init(unsigned long *zones_size)
{
7280
	zero_resv_unavail();
7281
	free_area_init_node(0, zones_size,
L
Linus Torvalds 已提交
7282 7283 7284
			__pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
}

7285
static int page_alloc_cpu_dead(unsigned int cpu)
L
Linus Torvalds 已提交
7286 7287
{

7288 7289
	lru_add_drain_cpu(cpu);
	drain_pages(cpu);
7290

7291 7292 7293 7294 7295 7296 7297
	/*
	 * Spill the event counters of the dead processor
	 * into the current processors event counters.
	 * This artificially elevates the count of the current
	 * processor.
	 */
	vm_events_fold_cpu(cpu);
7298

7299 7300 7301 7302 7303 7304 7305 7306 7307
	/*
	 * Zero the differential counters of the dead processor
	 * so that the vm statistics are consistent.
	 *
	 * This is only okay since the processor is dead and cannot
	 * race with what we are doing.
	 */
	cpu_vm_stats_fold(cpu);
	return 0;
L
Linus Torvalds 已提交
7308 7309 7310 7311
}

void __init page_alloc_init(void)
{
7312 7313 7314 7315 7316 7317
	int ret;

	ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
					"mm/page_alloc:dead", NULL,
					page_alloc_cpu_dead);
	WARN_ON(ret < 0);
L
Linus Torvalds 已提交
7318 7319
}

7320
/*
7321
 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
7322 7323 7324 7325 7326 7327
 *	or min_free_kbytes changes.
 */
static void calculate_totalreserve_pages(void)
{
	struct pglist_data *pgdat;
	unsigned long reserve_pages = 0;
7328
	enum zone_type i, j;
7329 7330

	for_each_online_pgdat(pgdat) {
7331 7332 7333

		pgdat->totalreserve_pages = 0;

7334 7335
		for (i = 0; i < MAX_NR_ZONES; i++) {
			struct zone *zone = pgdat->node_zones + i;
7336
			long max = 0;
7337 7338 7339 7340 7341 7342 7343

			/* Find valid and maximum lowmem_reserve in the zone */
			for (j = i; j < MAX_NR_ZONES; j++) {
				if (zone->lowmem_reserve[j] > max)
					max = zone->lowmem_reserve[j];
			}

7344 7345
			/* we treat the high watermark as reserved pages. */
			max += high_wmark_pages(zone);
7346

7347 7348
			if (max > zone->managed_pages)
				max = zone->managed_pages;
7349

7350
			pgdat->totalreserve_pages += max;
7351

7352 7353 7354 7355 7356 7357
			reserve_pages += max;
		}
	}
	totalreserve_pages = reserve_pages;
}

L
Linus Torvalds 已提交
7358 7359
/*
 * setup_per_zone_lowmem_reserve - called whenever
7360
 *	sysctl_lowmem_reserve_ratio changes.  Ensures that each zone
L
Linus Torvalds 已提交
7361 7362 7363 7364 7365 7366
 *	has a correct pages reserved value, so an adequate number of
 *	pages are left in the zone after a successful __alloc_pages().
 */
static void setup_per_zone_lowmem_reserve(void)
{
	struct pglist_data *pgdat;
7367
	enum zone_type j, idx;
L
Linus Torvalds 已提交
7368

7369
	for_each_online_pgdat(pgdat) {
L
Linus Torvalds 已提交
7370 7371
		for (j = 0; j < MAX_NR_ZONES; j++) {
			struct zone *zone = pgdat->node_zones + j;
7372
			unsigned long managed_pages = zone->managed_pages;
L
Linus Torvalds 已提交
7373 7374 7375

			zone->lowmem_reserve[j] = 0;

7376 7377
			idx = j;
			while (idx) {
L
Linus Torvalds 已提交
7378 7379
				struct zone *lower_zone;

7380
				idx--;
L
Linus Torvalds 已提交
7381
				lower_zone = pgdat->node_zones + idx;
7382 7383 7384 7385 7386 7387 7388 7389

				if (sysctl_lowmem_reserve_ratio[idx] < 1) {
					sysctl_lowmem_reserve_ratio[idx] = 0;
					lower_zone->lowmem_reserve[j] = 0;
				} else {
					lower_zone->lowmem_reserve[j] =
						managed_pages / sysctl_lowmem_reserve_ratio[idx];
				}
7390
				managed_pages += lower_zone->managed_pages;
L
Linus Torvalds 已提交
7391 7392 7393
			}
		}
	}
7394 7395 7396

	/* update totalreserve_pages */
	calculate_totalreserve_pages();
L
Linus Torvalds 已提交
7397 7398
}

7399
static void __setup_per_zone_wmarks(void)
L
Linus Torvalds 已提交
7400 7401 7402 7403 7404 7405 7406 7407 7408
{
	unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
	unsigned long lowmem_pages = 0;
	struct zone *zone;
	unsigned long flags;

	/* Calculate total number of !ZONE_HIGHMEM pages */
	for_each_zone(zone) {
		if (!is_highmem(zone))
7409
			lowmem_pages += zone->managed_pages;
L
Linus Torvalds 已提交
7410 7411 7412
	}

	for_each_zone(zone) {
7413 7414
		u64 tmp;

7415
		spin_lock_irqsave(&zone->lock, flags);
7416
		tmp = (u64)pages_min * zone->managed_pages;
7417
		do_div(tmp, lowmem_pages);
L
Linus Torvalds 已提交
7418 7419
		if (is_highmem(zone)) {
			/*
N
Nick Piggin 已提交
7420 7421 7422 7423
			 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
			 * need highmem pages, so cap pages_min to a small
			 * value here.
			 *
7424
			 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
Y
Yaowei Bai 已提交
7425
			 * deltas control asynch page reclaim, and so should
N
Nick Piggin 已提交
7426
			 * not be capped for highmem.
L
Linus Torvalds 已提交
7427
			 */
7428
			unsigned long min_pages;
L
Linus Torvalds 已提交
7429

7430
			min_pages = zone->managed_pages / 1024;
7431
			min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
7432
			zone->watermark[WMARK_MIN] = min_pages;
L
Linus Torvalds 已提交
7433
		} else {
N
Nick Piggin 已提交
7434 7435
			/*
			 * If it's a lowmem zone, reserve a number of pages
L
Linus Torvalds 已提交
7436 7437
			 * proportionate to the zone's size.
			 */
7438
			zone->watermark[WMARK_MIN] = tmp;
L
Linus Torvalds 已提交
7439 7440
		}

7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451
		/*
		 * Set the kswapd watermarks distance according to the
		 * scale factor in proportion to available memory, but
		 * ensure a minimum size on small systems.
		 */
		tmp = max_t(u64, tmp >> 2,
			    mult_frac(zone->managed_pages,
				      watermark_scale_factor, 10000));

		zone->watermark[WMARK_LOW]  = min_wmark_pages(zone) + tmp;
		zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
7452

7453
		spin_unlock_irqrestore(&zone->lock, flags);
L
Linus Torvalds 已提交
7454
	}
7455 7456 7457

	/* update totalreserve_pages */
	calculate_totalreserve_pages();
L
Linus Torvalds 已提交
7458 7459
}

7460 7461 7462 7463 7464 7465 7466 7467 7468
/**
 * setup_per_zone_wmarks - called when min_free_kbytes changes
 * or when memory is hot-{added|removed}
 *
 * Ensures that the watermark[min,low,high] values for each zone are set
 * correctly with respect to min_free_kbytes.
 */
void setup_per_zone_wmarks(void)
{
7469 7470 7471
	static DEFINE_SPINLOCK(lock);

	spin_lock(&lock);
7472
	__setup_per_zone_wmarks();
7473
	spin_unlock(&lock);
7474 7475
}

L
Linus Torvalds 已提交
7476 7477 7478 7479 7480 7481 7482
/*
 * Initialise min_free_kbytes.
 *
 * For small machines we want it small (128k min).  For large machines
 * we want it large (64MB max).  But it is not linear, because network
 * bandwidth does not increase linearly with machine size.  We use
 *
7483
 *	min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
L
Linus Torvalds 已提交
7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498 7499
 *	min_free_kbytes = sqrt(lowmem_kbytes * 16)
 *
 * which yields
 *
 * 16MB:	512k
 * 32MB:	724k
 * 64MB:	1024k
 * 128MB:	1448k
 * 256MB:	2048k
 * 512MB:	2896k
 * 1024MB:	4096k
 * 2048MB:	5792k
 * 4096MB:	8192k
 * 8192MB:	11584k
 * 16384MB:	16384k
 */
7500
int __meminit init_per_zone_wmark_min(void)
L
Linus Torvalds 已提交
7501 7502
{
	unsigned long lowmem_kbytes;
7503
	int new_min_free_kbytes;
L
Linus Torvalds 已提交
7504 7505

	lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517
	new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);

	if (new_min_free_kbytes > user_min_free_kbytes) {
		min_free_kbytes = new_min_free_kbytes;
		if (min_free_kbytes < 128)
			min_free_kbytes = 128;
		if (min_free_kbytes > 65536)
			min_free_kbytes = 65536;
	} else {
		pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
				new_min_free_kbytes, user_min_free_kbytes);
	}
7518
	setup_per_zone_wmarks();
7519
	refresh_zone_stat_thresholds();
L
Linus Torvalds 已提交
7520
	setup_per_zone_lowmem_reserve();
7521 7522 7523 7524 7525 7526

#ifdef CONFIG_NUMA
	setup_min_unmapped_ratio();
	setup_min_slab_ratio();
#endif

L
Linus Torvalds 已提交
7527 7528
	return 0;
}
7529
core_initcall(init_per_zone_wmark_min)
L
Linus Torvalds 已提交
7530 7531

/*
7532
 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
L
Linus Torvalds 已提交
7533 7534 7535
 *	that we can call two helper functions whenever min_free_kbytes
 *	changes.
 */
7536
int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
7537
	void __user *buffer, size_t *length, loff_t *ppos)
L
Linus Torvalds 已提交
7538
{
7539 7540 7541 7542 7543 7544
	int rc;

	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
	if (rc)
		return rc;

7545 7546
	if (write) {
		user_min_free_kbytes = min_free_kbytes;
7547
		setup_per_zone_wmarks();
7548
	}
L
Linus Torvalds 已提交
7549 7550 7551
	return 0;
}

7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566
int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
	void __user *buffer, size_t *length, loff_t *ppos)
{
	int rc;

	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
	if (rc)
		return rc;

	if (write)
		setup_per_zone_wmarks();

	return 0;
}

7567
#ifdef CONFIG_NUMA
7568
static void setup_min_unmapped_ratio(void)
7569
{
7570
	pg_data_t *pgdat;
7571 7572
	struct zone *zone;

7573
	for_each_online_pgdat(pgdat)
7574
		pgdat->min_unmapped_pages = 0;
7575

7576
	for_each_zone(zone)
7577
		zone->zone_pgdat->min_unmapped_pages += (zone->managed_pages *
7578 7579
				sysctl_min_unmapped_ratio) / 100;
}
7580

7581 7582

int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
7583
	void __user *buffer, size_t *length, loff_t *ppos)
7584 7585 7586
{
	int rc;

7587
	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7588 7589 7590
	if (rc)
		return rc;

7591 7592 7593 7594 7595 7596 7597 7598 7599 7600
	setup_min_unmapped_ratio();

	return 0;
}

static void setup_min_slab_ratio(void)
{
	pg_data_t *pgdat;
	struct zone *zone;

7601 7602 7603
	for_each_online_pgdat(pgdat)
		pgdat->min_slab_pages = 0;

7604
	for_each_zone(zone)
7605
		zone->zone_pgdat->min_slab_pages += (zone->managed_pages *
7606
				sysctl_min_slab_ratio) / 100;
7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619
}

int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
	void __user *buffer, size_t *length, loff_t *ppos)
{
	int rc;

	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
	if (rc)
		return rc;

	setup_min_slab_ratio();

7620 7621
	return 0;
}
7622 7623
#endif

L
Linus Torvalds 已提交
7624 7625 7626 7627 7628 7629
/*
 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
 *	proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
 *	whenever sysctl_lowmem_reserve_ratio changes.
 *
 * The reserve ratio obviously has absolutely no relation with the
7630
 * minimum watermarks. The lowmem reserve ratio can only make sense
L
Linus Torvalds 已提交
7631 7632
 * if in function of the boot time zone sizes.
 */
7633
int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
7634
	void __user *buffer, size_t *length, loff_t *ppos)
L
Linus Torvalds 已提交
7635
{
7636
	proc_dointvec_minmax(table, write, buffer, length, ppos);
L
Linus Torvalds 已提交
7637 7638 7639 7640
	setup_per_zone_lowmem_reserve();
	return 0;
}

7641 7642
/*
 * percpu_pagelist_fraction - changes the pcp->high for each zone on each
7643 7644
 * cpu.  It is the fraction of total pages in each zone that a hot per cpu
 * pagelist can have before it gets flushed back to buddy allocator.
7645
 */
7646
int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
7647
	void __user *buffer, size_t *length, loff_t *ppos)
7648 7649
{
	struct zone *zone;
7650
	int old_percpu_pagelist_fraction;
7651 7652
	int ret;

7653 7654 7655
	mutex_lock(&pcp_batch_high_lock);
	old_percpu_pagelist_fraction = percpu_pagelist_fraction;

7656
	ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670
	if (!write || ret < 0)
		goto out;

	/* Sanity checking to avoid pcp imbalance */
	if (percpu_pagelist_fraction &&
	    percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
		percpu_pagelist_fraction = old_percpu_pagelist_fraction;
		ret = -EINVAL;
		goto out;
	}

	/* No change? */
	if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
		goto out;
7671

7672
	for_each_populated_zone(zone) {
7673 7674
		unsigned int cpu;

7675
		for_each_possible_cpu(cpu)
7676 7677
			pageset_set_high_and_batch(zone,
					per_cpu_ptr(zone->pageset, cpu));
7678
	}
7679
out:
7680
	mutex_unlock(&pcp_batch_high_lock);
7681
	return ret;
7682 7683
}

7684
#ifdef CONFIG_NUMA
7685
int hashdist = HASHDIST_DEFAULT;
L
Linus Torvalds 已提交
7686 7687 7688 7689 7690 7691 7692 7693 7694 7695 7696

static int __init set_hashdist(char *str)
{
	if (!str)
		return 0;
	hashdist = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("hashdist=", set_hashdist);
#endif

7697 7698 7699 7700 7701 7702 7703 7704 7705 7706 7707
#ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
/*
 * Returns the number of pages that arch has reserved but
 * is not known to alloc_large_system_hash().
 */
static unsigned long __init arch_reserved_kernel_pages(void)
{
	return 0;
}
#endif

P
Pavel Tatashin 已提交
7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722
/*
 * Adaptive scale is meant to reduce sizes of hash tables on large memory
 * machines. As memory size is increased the scale is also increased but at
 * slower pace.  Starting from ADAPT_SCALE_BASE (64G), every time memory
 * quadruples the scale is increased by one, which means the size of hash table
 * only doubles, instead of quadrupling as well.
 * Because 32-bit systems cannot have large physical memory, where this scaling
 * makes sense, it is disabled on such platforms.
 */
#if __BITS_PER_LONG > 32
#define ADAPT_SCALE_BASE	(64ul << 30)
#define ADAPT_SCALE_SHIFT	2
#define ADAPT_SCALE_NPAGES	(ADAPT_SCALE_BASE >> PAGE_SHIFT)
#endif

L
Linus Torvalds 已提交
7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735
/*
 * allocate a large system hash table from bootmem
 * - it is assumed that the hash table must contain an exact power-of-2
 *   quantity of entries
 * - limit is the number of hash buckets, not the total allocation size
 */
void *__init alloc_large_system_hash(const char *tablename,
				     unsigned long bucketsize,
				     unsigned long numentries,
				     int scale,
				     int flags,
				     unsigned int *_hash_shift,
				     unsigned int *_hash_mask,
7736 7737
				     unsigned long low_limit,
				     unsigned long high_limit)
L
Linus Torvalds 已提交
7738
{
7739
	unsigned long long max = high_limit;
L
Linus Torvalds 已提交
7740 7741
	unsigned long log2qty, size;
	void *table = NULL;
7742
	gfp_t gfp_flags;
L
Linus Torvalds 已提交
7743 7744 7745 7746

	/* allow the kernel cmdline to have a say */
	if (!numentries) {
		/* round applicable memory size up to nearest megabyte */
A
Andrew Morton 已提交
7747
		numentries = nr_kernel_pages;
7748
		numentries -= arch_reserved_kernel_pages();
7749 7750 7751 7752

		/* It isn't necessary when PAGE_SIZE >= 1MB */
		if (PAGE_SHIFT < 20)
			numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
L
Linus Torvalds 已提交
7753

P
Pavel Tatashin 已提交
7754 7755 7756 7757 7758 7759 7760 7761 7762 7763
#if __BITS_PER_LONG > 32
		if (!high_limit) {
			unsigned long adapt;

			for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
			     adapt <<= ADAPT_SCALE_SHIFT)
				scale++;
		}
#endif

L
Linus Torvalds 已提交
7764 7765 7766 7767 7768
		/* limit to 1 bucket per 2^scale bytes of low memory */
		if (scale > PAGE_SHIFT)
			numentries >>= (scale - PAGE_SHIFT);
		else
			numentries <<= (PAGE_SHIFT - scale);
7769 7770

		/* Make sure we've got at least a 0-order allocation.. */
7771 7772 7773 7774 7775 7776 7777 7778
		if (unlikely(flags & HASH_SMALL)) {
			/* Makes no sense without HASH_EARLY */
			WARN_ON(!(flags & HASH_EARLY));
			if (!(numentries >> *_hash_shift)) {
				numentries = 1UL << *_hash_shift;
				BUG_ON(!numentries);
			}
		} else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
7779
			numentries = PAGE_SIZE / bucketsize;
L
Linus Torvalds 已提交
7780
	}
7781
	numentries = roundup_pow_of_two(numentries);
L
Linus Torvalds 已提交
7782 7783 7784 7785 7786 7787

	/* limit allocation size to 1/16 total memory by default */
	if (max == 0) {
		max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
		do_div(max, bucketsize);
	}
7788
	max = min(max, 0x80000000ULL);
L
Linus Torvalds 已提交
7789

7790 7791
	if (numentries < low_limit)
		numentries = low_limit;
L
Linus Torvalds 已提交
7792 7793 7794
	if (numentries > max)
		numentries = max;

7795
	log2qty = ilog2(numentries);
L
Linus Torvalds 已提交
7796

7797
	gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
L
Linus Torvalds 已提交
7798 7799
	do {
		size = bucketsize << log2qty;
7800 7801 7802 7803 7804 7805
		if (flags & HASH_EARLY) {
			if (flags & HASH_ZERO)
				table = memblock_virt_alloc_nopanic(size, 0);
			else
				table = memblock_virt_alloc_raw(size, 0);
		} else if (hashdist) {
7806
			table = __vmalloc(size, gfp_flags, PAGE_KERNEL);
7807
		} else {
7808 7809
			/*
			 * If bucketsize is not a power-of-two, we may free
7810 7811
			 * some pages at the end of hash table which
			 * alloc_pages_exact() automatically does
7812
			 */
7813
			if (get_order(size) < MAX_ORDER) {
7814 7815
				table = alloc_pages_exact(size, gfp_flags);
				kmemleak_alloc(table, size, 1, gfp_flags);
7816
			}
L
Linus Torvalds 已提交
7817 7818 7819 7820 7821 7822
		}
	} while (!table && size > PAGE_SIZE && --log2qty);

	if (!table)
		panic("Failed to allocate %s hash table\n", tablename);

7823 7824
	pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n",
		tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size);
L
Linus Torvalds 已提交
7825 7826 7827 7828 7829 7830 7831 7832

	if (_hash_shift)
		*_hash_shift = log2qty;
	if (_hash_mask)
		*_hash_mask = (1 << log2qty) - 1;

	return table;
}
7833

K
KAMEZAWA Hiroyuki 已提交
7834
/*
7835 7836 7837
 * This function checks whether pageblock includes unmovable pages or not.
 * If @count is not zero, it is okay to include less @count unmovable pages
 *
7838
 * PageLRU check without isolation or lru_lock could race so that
7839 7840 7841
 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable
 * check without lock_page also may miss some movable non-lru pages at
 * race condition. So you can't expect this function should be exact.
K
KAMEZAWA Hiroyuki 已提交
7842
 */
7843
bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
7844
			 int migratetype,
7845
			 bool skip_hwpoisoned_pages)
7846 7847
{
	unsigned long pfn, iter, found;
7848

7849
	/*
7850 7851 7852 7853 7854
	 * TODO we could make this much more efficient by not checking every
	 * page in the range if we know all of them are in MOVABLE_ZONE and
	 * that the movable zone guarantees that pages are migratable but
	 * the later is not the case right now unfortunatelly. E.g. movablecore
	 * can still lead to having bootmem allocations in zone_movable.
7855 7856
	 */

7857 7858 7859 7860 7861 7862 7863 7864 7865
	/*
	 * CMA allocations (alloc_contig_range) really need to mark isolate
	 * CMA pageblocks even when they are not movable in fact so consider
	 * them movable here.
	 */
	if (is_migrate_cma(migratetype) &&
			is_migrate_cma(get_pageblock_migratetype(page)))
		return false;

7866 7867 7868 7869
	pfn = page_to_pfn(page);
	for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
		unsigned long check = pfn + iter;

7870
		if (!pfn_valid_within(check))
7871
			continue;
7872

7873
		page = pfn_to_page(check);
7874

7875
		if (PageReserved(page))
7876
			goto unmovable;
7877

7878 7879 7880 7881 7882 7883 7884 7885
		/*
		 * If the zone is movable and we have ruled out all reserved
		 * pages then it should be reasonably safe to assume the rest
		 * is movable.
		 */
		if (zone_idx(zone) == ZONE_MOVABLE)
			continue;

7886 7887 7888 7889 7890 7891
		/*
		 * Hugepages are not in LRU lists, but they're movable.
		 * We need not scan over tail pages bacause we don't
		 * handle each tail page individually in migration.
		 */
		if (PageHuge(page)) {
7892 7893
			struct page *head = compound_head(page);
			unsigned int skip_pages;
7894

7895
			if (!hugepage_migration_supported(page_hstate(head)))
7896 7897
				goto unmovable;

7898 7899
			skip_pages = (1 << compound_order(head)) - (page - head);
			iter += skip_pages - 1;
7900 7901 7902
			continue;
		}

7903 7904 7905 7906
		/*
		 * We can't use page_count without pin a page
		 * because another CPU can free compound page.
		 * This check already skips compound tails of THP
7907
		 * because their page->_refcount is zero at all time.
7908
		 */
7909
		if (!page_ref_count(page)) {
7910 7911 7912 7913
			if (PageBuddy(page))
				iter += (1 << page_order(page)) - 1;
			continue;
		}
7914

7915 7916 7917 7918 7919 7920 7921
		/*
		 * The HWPoisoned page may be not in buddy system, and
		 * page_count() is not 0.
		 */
		if (skip_hwpoisoned_pages && PageHWPoison(page))
			continue;

7922 7923 7924
		if (__PageMovable(page))
			continue;

7925 7926 7927
		if (!PageLRU(page))
			found++;
		/*
7928 7929 7930
		 * If there are RECLAIMABLE pages, we need to check
		 * it.  But now, memory offline itself doesn't call
		 * shrink_node_slabs() and it still to be fixed.
7931 7932 7933 7934 7935 7936 7937 7938 7939 7940
		 */
		/*
		 * If the page is not RAM, page_count()should be 0.
		 * we don't need more check. This is an _used_ not-movable page.
		 *
		 * The problematic thing here is PG_reserved pages. PG_reserved
		 * is set to both of a memory hole page and a _used_ kernel
		 * page at boot.
		 */
		if (found > count)
7941
			goto unmovable;
7942
	}
7943
	return false;
7944 7945 7946
unmovable:
	WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE);
	return true;
7947 7948
}

7949
#if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)
7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963

static unsigned long pfn_max_align_down(unsigned long pfn)
{
	return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
			     pageblock_nr_pages) - 1);
}

static unsigned long pfn_max_align_up(unsigned long pfn)
{
	return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
				pageblock_nr_pages));
}

/* [start, end) must belong to a single zone. */
7964 7965
static int __alloc_contig_migrate_range(struct compact_control *cc,
					unsigned long start, unsigned long end)
7966 7967
{
	/* This function is based on compact_zone() from compaction.c. */
7968
	unsigned long nr_reclaimed;
7969 7970 7971 7972
	unsigned long pfn = start;
	unsigned int tries = 0;
	int ret = 0;

7973
	migrate_prep();
7974

7975
	while (pfn < end || !list_empty(&cc->migratepages)) {
7976 7977 7978 7979 7980
		if (fatal_signal_pending(current)) {
			ret = -EINTR;
			break;
		}

7981 7982
		if (list_empty(&cc->migratepages)) {
			cc->nr_migratepages = 0;
7983
			pfn = isolate_migratepages_range(cc, pfn, end);
7984 7985 7986 7987 7988 7989 7990 7991 7992 7993
			if (!pfn) {
				ret = -EINTR;
				break;
			}
			tries = 0;
		} else if (++tries == 5) {
			ret = ret < 0 ? ret : -EBUSY;
			break;
		}

7994 7995 7996
		nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
							&cc->migratepages);
		cc->nr_migratepages -= nr_reclaimed;
7997

7998
		ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
7999
				    NULL, 0, cc->mode, MR_CONTIG_RANGE);
8000
	}
8001 8002 8003 8004 8005
	if (ret < 0) {
		putback_movable_pages(&cc->migratepages);
		return ret;
	}
	return 0;
8006 8007 8008 8009 8010 8011
}

/**
 * alloc_contig_range() -- tries to allocate given range of pages
 * @start:	start PFN to allocate
 * @end:	one-past-the-last PFN to allocate
8012 8013 8014 8015
 * @migratetype:	migratetype of the underlaying pageblocks (either
 *			#MIGRATE_MOVABLE or #MIGRATE_CMA).  All pageblocks
 *			in range must have the same migratetype and it must
 *			be either of the two.
8016
 * @gfp_mask:	GFP mask to use during compaction
8017 8018
 *
 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
8019
 * aligned.  The PFN range must belong to a single zone.
8020
 *
8021 8022 8023
 * The first thing this routine does is attempt to MIGRATE_ISOLATE all
 * pageblocks in the range.  Once isolated, the pageblocks should not
 * be modified by others.
8024 8025 8026 8027 8028
 *
 * Returns zero on success or negative error code.  On success all
 * pages which PFN is in [start, end) are allocated for the caller and
 * need to be freed with free_contig_range().
 */
8029
int alloc_contig_range(unsigned long start, unsigned long end,
8030
		       unsigned migratetype, gfp_t gfp_mask)
8031 8032
{
	unsigned long outer_start, outer_end;
8033 8034
	unsigned int order;
	int ret = 0;
8035

8036 8037 8038 8039
	struct compact_control cc = {
		.nr_migratepages = 0,
		.order = -1,
		.zone = page_zone(pfn_to_page(start)),
8040
		.mode = MIGRATE_SYNC,
8041
		.ignore_skip_hint = true,
8042
		.no_set_skip_hint = true,
8043
		.gfp_mask = current_gfp_context(gfp_mask),
8044 8045 8046
	};
	INIT_LIST_HEAD(&cc.migratepages);

8047 8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071
	/*
	 * What we do here is we mark all pageblocks in range as
	 * MIGRATE_ISOLATE.  Because pageblock and max order pages may
	 * have different sizes, and due to the way page allocator
	 * work, we align the range to biggest of the two pages so
	 * that page allocator won't try to merge buddies from
	 * different pageblocks and change MIGRATE_ISOLATE to some
	 * other migration type.
	 *
	 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
	 * migrate the pages from an unaligned range (ie. pages that
	 * we are interested in).  This will put all the pages in
	 * range back to page allocator as MIGRATE_ISOLATE.
	 *
	 * When this is done, we take the pages in range from page
	 * allocator removing them from the buddy system.  This way
	 * page allocator will never consider using them.
	 *
	 * This lets us mark the pageblocks back as
	 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
	 * aligned range but not in the unaligned, original range are
	 * put back to page allocator so that buddy can use them.
	 */

	ret = start_isolate_page_range(pfn_max_align_down(start),
8072 8073
				       pfn_max_align_up(end), migratetype,
				       false);
8074
	if (ret)
8075
		return ret;
8076

8077 8078
	/*
	 * In case of -EBUSY, we'd like to know which page causes problem.
8079 8080 8081 8082 8083 8084 8085
	 * So, just fall through. test_pages_isolated() has a tracepoint
	 * which will report the busy page.
	 *
	 * It is possible that busy pages could become available before
	 * the call to test_pages_isolated, and the range will actually be
	 * allocated.  So, if we fall through be sure to clear ret so that
	 * -EBUSY is not accidentally used or returned to caller.
8086
	 */
8087
	ret = __alloc_contig_migrate_range(&cc, start, end);
8088
	if (ret && ret != -EBUSY)
8089
		goto done;
8090
	ret =0;
8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109

	/*
	 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
	 * aligned blocks that are marked as MIGRATE_ISOLATE.  What's
	 * more, all pages in [start, end) are free in page allocator.
	 * What we are going to do is to allocate all pages from
	 * [start, end) (that is remove them from page allocator).
	 *
	 * The only problem is that pages at the beginning and at the
	 * end of interesting range may be not aligned with pages that
	 * page allocator holds, ie. they can be part of higher order
	 * pages.  Because of this, we reserve the bigger range and
	 * once this is done free the pages we are not interested in.
	 *
	 * We don't have to hold zone->lock here because the pages are
	 * isolated thus they won't get removed from buddy.
	 */

	lru_add_drain_all();
8110
	drain_all_pages(cc.zone);
8111 8112 8113 8114 8115

	order = 0;
	outer_start = start;
	while (!PageBuddy(pfn_to_page(outer_start))) {
		if (++order >= MAX_ORDER) {
8116 8117
			outer_start = start;
			break;
8118 8119 8120 8121
		}
		outer_start &= ~0UL << order;
	}

8122 8123 8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134
	if (outer_start != start) {
		order = page_order(pfn_to_page(outer_start));

		/*
		 * outer_start page could be small order buddy page and
		 * it doesn't include start page. Adjust outer_start
		 * in this case to report failed page properly
		 * on tracepoint in test_pages_isolated()
		 */
		if (outer_start + (1UL << order) <= start)
			outer_start = start;
	}

8135
	/* Make sure the range is really isolated. */
8136
	if (test_pages_isolated(outer_start, end, false)) {
8137
		pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
8138
			__func__, outer_start, end);
8139 8140 8141 8142
		ret = -EBUSY;
		goto done;
	}

8143
	/* Grab isolated pages from freelists. */
8144
	outer_end = isolate_freepages_range(&cc, outer_start, end);
8145 8146 8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157
	if (!outer_end) {
		ret = -EBUSY;
		goto done;
	}

	/* Free head and tail (if any) */
	if (start != outer_start)
		free_contig_range(outer_start, start - outer_start);
	if (end != outer_end)
		free_contig_range(end, outer_end - end);

done:
	undo_isolate_page_range(pfn_max_align_down(start),
8158
				pfn_max_align_up(end), migratetype);
8159 8160 8161 8162 8163
	return ret;
}

void free_contig_range(unsigned long pfn, unsigned nr_pages)
{
8164 8165 8166 8167 8168 8169 8170 8171 8172
	unsigned int count = 0;

	for (; nr_pages--; pfn++) {
		struct page *page = pfn_to_page(pfn);

		count += page_count(page) != 1;
		__free_page(page);
	}
	WARN(count != 0, "%d pages are still in use!\n", count);
8173 8174 8175
}
#endif

8176 8177 8178 8179
/*
 * The zone indicated has a new number of managed_pages; batch sizes and percpu
 * page high values need to be recalulated.
 */
8180 8181
void __meminit zone_pcp_update(struct zone *zone)
{
8182
	unsigned cpu;
8183
	mutex_lock(&pcp_batch_high_lock);
8184
	for_each_possible_cpu(cpu)
8185 8186
		pageset_set_high_and_batch(zone,
				per_cpu_ptr(zone->pageset, cpu));
8187
	mutex_unlock(&pcp_batch_high_lock);
8188 8189
}

8190 8191 8192
void zone_pcp_reset(struct zone *zone)
{
	unsigned long flags;
8193 8194
	int cpu;
	struct per_cpu_pageset *pset;
8195 8196 8197 8198

	/* avoid races with drain_pages()  */
	local_irq_save(flags);
	if (zone->pageset != &boot_pageset) {
8199 8200 8201 8202
		for_each_online_cpu(cpu) {
			pset = per_cpu_ptr(zone->pageset, cpu);
			drain_zonestat(zone, pset);
		}
8203 8204 8205 8206 8207 8208
		free_percpu(zone->pageset);
		zone->pageset = &boot_pageset;
	}
	local_irq_restore(flags);
}

8209
#ifdef CONFIG_MEMORY_HOTREMOVE
K
KAMEZAWA Hiroyuki 已提交
8210
/*
8211 8212
 * All pages in the range must be in a single zone and isolated
 * before calling this.
K
KAMEZAWA Hiroyuki 已提交
8213 8214 8215 8216 8217 8218
 */
void
__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
{
	struct page *page;
	struct zone *zone;
8219
	unsigned int order, i;
K
KAMEZAWA Hiroyuki 已提交
8220 8221 8222 8223 8224 8225 8226 8227
	unsigned long pfn;
	unsigned long flags;
	/* find the first valid pfn */
	for (pfn = start_pfn; pfn < end_pfn; pfn++)
		if (pfn_valid(pfn))
			break;
	if (pfn == end_pfn)
		return;
8228
	offline_mem_sections(pfn, end_pfn);
K
KAMEZAWA Hiroyuki 已提交
8229 8230 8231 8232 8233 8234 8235 8236 8237
	zone = page_zone(pfn_to_page(pfn));
	spin_lock_irqsave(&zone->lock, flags);
	pfn = start_pfn;
	while (pfn < end_pfn) {
		if (!pfn_valid(pfn)) {
			pfn++;
			continue;
		}
		page = pfn_to_page(pfn);
8238 8239 8240 8241 8242 8243 8244 8245 8246 8247
		/*
		 * The HWPoisoned page may be not in buddy system, and
		 * page_count() is not 0.
		 */
		if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
			pfn++;
			SetPageReserved(page);
			continue;
		}

K
KAMEZAWA Hiroyuki 已提交
8248 8249 8250 8251
		BUG_ON(page_count(page));
		BUG_ON(!PageBuddy(page));
		order = page_order(page);
#ifdef CONFIG_DEBUG_VM
8252 8253
		pr_info("remove from free list %lx %d %lx\n",
			pfn, 1 << order, end_pfn);
K
KAMEZAWA Hiroyuki 已提交
8254
#endif
8255
		del_page_from_free_list(page, zone, order);
K
KAMEZAWA Hiroyuki 已提交
8256 8257 8258 8259 8260 8261 8262
		for (i = 0; i < (1 << order); i++)
			SetPageReserved((page+i));
		pfn += (1 << order);
	}
	spin_unlock_irqrestore(&zone->lock, flags);
}
#endif
8263 8264 8265 8266 8267 8268

bool is_free_buddy_page(struct page *page)
{
	struct zone *zone = page_zone(page);
	unsigned long pfn = page_to_pfn(page);
	unsigned long flags;
8269
	unsigned int order;
8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281

	spin_lock_irqsave(&zone->lock, flags);
	for (order = 0; order < MAX_ORDER; order++) {
		struct page *page_head = page - (pfn & ((1 << order) - 1));

		if (PageBuddy(page_head) && page_order(page_head) >= order)
			break;
	}
	spin_unlock_irqrestore(&zone->lock, flags);

	return order < MAX_ORDER;
}
8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308 8309 8310 8311

#ifdef CONFIG_MEMORY_FAILURE
/*
 * Set PG_hwpoison flag if a given page is confirmed to be a free page.  This
 * test is performed under the zone lock to prevent a race against page
 * allocation.
 */
bool set_hwpoison_free_buddy_page(struct page *page)
{
	struct zone *zone = page_zone(page);
	unsigned long pfn = page_to_pfn(page);
	unsigned long flags;
	unsigned int order;
	bool hwpoisoned = false;

	spin_lock_irqsave(&zone->lock, flags);
	for (order = 0; order < MAX_ORDER; order++) {
		struct page *page_head = page - (pfn & ((1 << order) - 1));

		if (PageBuddy(page_head) && page_order(page_head) >= order) {
			if (!TestSetPageHWPoison(page))
				hwpoisoned = true;
			break;
		}
	}
	spin_unlock_irqrestore(&zone->lock, flags);

	return hwpoisoned;
}
#endif