mmzone.h 40.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4
#ifndef _LINUX_MMZONE_H
#define _LINUX_MMZONE_H

#ifndef __ASSEMBLY__
C
Christoph Lameter 已提交
5
#ifndef __GENERATING_BOUNDS_H
L
Linus Torvalds 已提交
6 7 8 9

#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/wait.h>
10
#include <linux/bitops.h>
L
Linus Torvalds 已提交
11 12 13 14
#include <linux/cache.h>
#include <linux/threads.h>
#include <linux/numa.h>
#include <linux/init.h>
15
#include <linux/seqlock.h>
16
#include <linux/nodemask.h>
17
#include <linux/pageblock-flags.h>
18
#include <linux/page-flags-layout.h>
A
Arun Sharma 已提交
19
#include <linux/atomic.h>
R
Ralf Baechle 已提交
20
#include <asm/page.h>
L
Linus Torvalds 已提交
21 22 23 24 25 26 27

/* Free memory management - zoned buddy allocator.  */
#ifndef CONFIG_FORCE_MAX_ZONEORDER
#define MAX_ORDER 11
#else
#define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
#endif
28
#define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1))
L
Linus Torvalds 已提交
29

A
Andy Whitcroft 已提交
30 31 32
/*
 * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed
 * costly to service.  That is between allocation orders which should
33
 * coalesce naturally under reasonable reclaim pressure and those which
A
Andy Whitcroft 已提交
34 35 36 37
 * will not.
 */
#define PAGE_ALLOC_COSTLY_ORDER 3

38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
enum {
	MIGRATE_UNMOVABLE,
	MIGRATE_RECLAIMABLE,
	MIGRATE_MOVABLE,
	MIGRATE_PCPTYPES,	/* the number of types on the pcp lists */
	MIGRATE_RESERVE = MIGRATE_PCPTYPES,
#ifdef CONFIG_CMA
	/*
	 * MIGRATE_CMA migration type is designed to mimic the way
	 * ZONE_MOVABLE works.  Only movable pages can be allocated
	 * from MIGRATE_CMA pageblocks and page allocator never
	 * implicitly change migration type of MIGRATE_CMA pageblock.
	 *
	 * The way to use it is to change migratetype of a range of
	 * pageblocks to MIGRATE_CMA which can be done by
	 * __free_pageblock_cma() function.  What is important though
	 * is that a range of pageblocks must be aligned to
	 * MAX_ORDER_NR_PAGES should biggest page be bigger then
	 * a single pageblock.
	 */
	MIGRATE_CMA,
#endif
60
#ifdef CONFIG_MEMORY_ISOLATION
61
	MIGRATE_ISOLATE,	/* can't allocate from here */
62
#endif
63 64 65 66 67 68 69 70
	MIGRATE_TYPES
};

#ifdef CONFIG_CMA
#  define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA)
#else
#  define is_migrate_cma(migratetype) false
#endif
71 72 73 74 75

#define for_each_migratetype_order(order, type) \
	for (order = 0; order < MAX_ORDER; order++) \
		for (type = 0; type < MIGRATE_TYPES; type++)

76 77
extern int page_group_by_mobility_disabled;

78 79 80
#define NR_MIGRATETYPE_BITS (PB_migrate_end - PB_migrate + 1)
#define MIGRATETYPE_MASK ((1UL << NR_MIGRATETYPE_BITS) - 1)

81 82 83 84 85
#define get_pageblock_migratetype(page)					\
	get_pfnblock_flags_mask(page, page_to_pfn(page),		\
			PB_migrate_end, MIGRATETYPE_MASK)

static inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
86
{
87
	BUILD_BUG_ON(PB_migrate_end - PB_migrate != 2);
88 89
	return get_pfnblock_flags_mask(page, pfn, PB_migrate_end,
					MIGRATETYPE_MASK);
90 91
}

L
Linus Torvalds 已提交
92
struct free_area {
93
	struct list_head	free_list[MIGRATE_TYPES];
L
Linus Torvalds 已提交
94 95 96 97 98 99 100 101 102 103 104 105 106 107
	unsigned long		nr_free;
};

struct pglist_data;

/*
 * zone->lock and zone->lru_lock are two of the hottest locks in the kernel.
 * So add a wild amount of padding here to ensure that they fall into separate
 * cachelines.  There are very few zone structures in the machine, so space
 * consumption is not a concern here.
 */
#if defined(CONFIG_SMP)
struct zone_padding {
	char x[0];
108
} ____cacheline_internodealigned_in_smp;
L
Linus Torvalds 已提交
109 110 111 112 113
#define ZONE_PADDING(name)	struct zone_padding name;
#else
#define ZONE_PADDING(name)
#endif

114
enum zone_stat_item {
115
	/* First 128 byte cacheline (assuming 64 bit words) */
116
	NR_FREE_PAGES,
117
	NR_ALLOC_BATCH,
118
	NR_LRU_BASE,
119 120 121 122
	NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */
	NR_ACTIVE_ANON,		/*  "     "     "   "       "         */
	NR_INACTIVE_FILE,	/*  "     "     "   "       "         */
	NR_ACTIVE_FILE,		/*  "     "     "   "       "         */
L
Lee Schermerhorn 已提交
123
	NR_UNEVICTABLE,		/*  "     "     "   "       "         */
N
Nick Piggin 已提交
124
	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
125 126
	NR_ANON_PAGES,	/* Mapped anonymous pages */
	NR_FILE_MAPPED,	/* pagecache pages mapped into pagetables.
127
			   only modified from process context */
128
	NR_FILE_PAGES,
129
	NR_FILE_DIRTY,
130
	NR_WRITEBACK,
131 132 133
	NR_SLAB_RECLAIMABLE,
	NR_SLAB_UNRECLAIMABLE,
	NR_PAGETABLE,		/* used for pagetables */
134 135
	NR_KERNEL_STACK,
	/* Second 128 byte cacheline */
136
	NR_UNSTABLE_NFS,	/* NFS unstable pages */
137
	NR_BOUNCE,
138
	NR_VMSCAN_WRITE,
139
	NR_VMSCAN_IMMEDIATE,	/* Prioritise for reclaim when writeback ends */
140
	NR_WRITEBACK_TEMP,	/* Writeback using temporary buffers */
K
KOSAKI Motohiro 已提交
141 142
	NR_ISOLATED_ANON,	/* Temporary isolated pages from anon lru */
	NR_ISOLATED_FILE,	/* Temporary isolated pages from file lru */
143
	NR_SHMEM,		/* shmem pages (included tmpfs/GEM pages) */
144 145
	NR_DIRTIED,		/* page dirtyings since bootup */
	NR_WRITTEN,		/* page writings since bootup */
146 147 148 149 150 151 152 153
#ifdef CONFIG_NUMA
	NUMA_HIT,		/* allocated in intended node */
	NUMA_MISS,		/* allocated in non intended node */
	NUMA_FOREIGN,		/* was intended here, hit elsewhere */
	NUMA_INTERLEAVE_HIT,	/* interleaver preferred this zone */
	NUMA_LOCAL,		/* allocation from local node */
	NUMA_OTHER,		/* allocation from other node */
#endif
154 155
	WORKINGSET_REFAULT,
	WORKINGSET_ACTIVATE,
156
	WORKINGSET_NODERECLAIM,
157
	NR_ANON_TRANSPARENT_HUGEPAGES,
158
	NR_FREE_CMA_PAGES,
159 160
	NR_VM_ZONE_STAT_ITEMS };

161 162 163 164 165 166 167 168 169 170 171 172 173
/*
 * We do arithmetic on the LRU lists in various places in the code,
 * so it is important to keep the active lists LRU_ACTIVE higher in
 * the array than the corresponding inactive lists, and to keep
 * the *_FILE lists LRU_FILE higher than the corresponding _ANON lists.
 *
 * This has to be kept in sync with the statistics in zone_stat_item
 * above and the descriptions in vmstat_text in mm/vmstat.c
 */
#define LRU_BASE 0
#define LRU_ACTIVE 1
#define LRU_FILE 2

174
enum lru_list {
175 176 177 178
	LRU_INACTIVE_ANON = LRU_BASE,
	LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE,
	LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
	LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
L
Lee Schermerhorn 已提交
179 180 181
	LRU_UNEVICTABLE,
	NR_LRU_LISTS
};
182

H
Hugh Dickins 已提交
183
#define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
184

H
Hugh Dickins 已提交
185
#define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
L
Lee Schermerhorn 已提交
186

H
Hugh Dickins 已提交
187
static inline int is_file_lru(enum lru_list lru)
188
{
H
Hugh Dickins 已提交
189
	return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
190 191
}

H
Hugh Dickins 已提交
192
static inline int is_active_lru(enum lru_list lru)
193
{
H
Hugh Dickins 已提交
194
	return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);
195 196
}

H
Hugh Dickins 已提交
197
static inline int is_unevictable_lru(enum lru_list lru)
L
Lee Schermerhorn 已提交
198
{
H
Hugh Dickins 已提交
199
	return (lru == LRU_UNEVICTABLE);
L
Lee Schermerhorn 已提交
200 201
}

202 203 204
struct zone_reclaim_stat {
	/*
	 * The pageout code in vmscan.c keeps track of how many of the
205
	 * mem/swap backed and file backed pages are referenced.
206 207 208 209 210 211 212 213 214
	 * The higher the rotated/scanned ratio, the more valuable
	 * that cache is.
	 *
	 * The anon LRU stats live in [0], file LRU stats in [1]
	 */
	unsigned long		recent_rotated[2];
	unsigned long		recent_scanned[2];
};

215 216
struct lruvec {
	struct list_head lists[NR_LRU_LISTS];
217
	struct zone_reclaim_stat reclaim_stat;
A
Andrew Morton 已提交
218
#ifdef CONFIG_MEMCG
219 220
	struct zone *zone;
#endif
221 222
};

223 224 225 226 227
/* Mask used at gathering information at once (see memcontrol.c) */
#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
#define LRU_ALL	     ((1 << NR_LRU_LISTS) - 1)

228
/* Isolate clean file */
229
#define ISOLATE_CLEAN		((__force isolate_mode_t)0x1)
230
/* Isolate unmapped file */
231
#define ISOLATE_UNMAPPED	((__force isolate_mode_t)0x2)
232
/* Isolate for asynchronous migration */
233
#define ISOLATE_ASYNC_MIGRATE	((__force isolate_mode_t)0x4)
M
Minchan Kim 已提交
234 235
/* Isolate unevictable pages */
#define ISOLATE_UNEVICTABLE	((__force isolate_mode_t)0x8)
236 237 238 239

/* LRU Isolation modes. */
typedef unsigned __bitwise__ isolate_mode_t;

240 241 242 243 244 245 246 247 248 249 250
enum zone_watermarks {
	WMARK_MIN,
	WMARK_LOW,
	WMARK_HIGH,
	NR_WMARK
};

#define min_wmark_pages(z) (z->watermark[WMARK_MIN])
#define low_wmark_pages(z) (z->watermark[WMARK_LOW])
#define high_wmark_pages(z) (z->watermark[WMARK_HIGH])

L
Linus Torvalds 已提交
251 252 253 254
struct per_cpu_pages {
	int count;		/* number of pages in the list */
	int high;		/* high watermark, emptying needed */
	int batch;		/* chunk size for buddy add/remove */
255 256 257

	/* Lists of pages, one per migrate type stored on the pcp-lists */
	struct list_head lists[MIGRATE_PCPTYPES];
L
Linus Torvalds 已提交
258 259 260
};

struct per_cpu_pageset {
261
	struct per_cpu_pages pcp;
262 263 264
#ifdef CONFIG_NUMA
	s8 expire;
#endif
265
#ifdef CONFIG_SMP
266
	s8 stat_threshold;
267 268
	s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
#endif
269
};
270

C
Christoph Lameter 已提交
271 272
#endif /* !__GENERATING_BOUNDS.H */

273
enum zone_type {
274
#ifdef CONFIG_ZONE_DMA
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
	/*
	 * ZONE_DMA is used when there are devices that are not able
	 * to do DMA to all of addressable memory (ZONE_NORMAL). Then we
	 * carve out the portion of memory that is needed for these devices.
	 * The range is arch specific.
	 *
	 * Some examples
	 *
	 * Architecture		Limit
	 * ---------------------------
	 * parisc, ia64, sparc	<4G
	 * s390			<2G
	 * arm			Various
	 * alpha		Unlimited or 0-16MB.
	 *
	 * i386, x86_64 and multiple other arches
	 * 			<16M.
	 */
	ZONE_DMA,
294
#endif
295
#ifdef CONFIG_ZONE_DMA32
296 297 298 299 300 301
	/*
	 * x86_64 needs two ZONE_DMAs because it supports devices that are
	 * only able to do DMA to the lower 16M but also 32 bit devices that
	 * can only do DMA areas below 4G.
	 */
	ZONE_DMA32,
302
#endif
303 304 305 306 307 308
	/*
	 * Normal addressable memory is in ZONE_NORMAL. DMA operations can be
	 * performed on pages in ZONE_NORMAL if the DMA devices support
	 * transfers to all addressable memory.
	 */
	ZONE_NORMAL,
309
#ifdef CONFIG_HIGHMEM
310 311 312 313 314 315 316 317 318
	/*
	 * A memory area that is only addressable by the kernel through
	 * mapping portions into its own address space. This is for example
	 * used by i386 to allow the kernel to address the memory beyond
	 * 900MB. The kernel will set up special mappings (page
	 * table entries on i386) for each page that the kernel needs to
	 * access.
	 */
	ZONE_HIGHMEM,
319
#endif
M
Mel Gorman 已提交
320
	ZONE_MOVABLE,
C
Christoph Lameter 已提交
321
	__MAX_NR_ZONES
322
};
L
Linus Torvalds 已提交
323

C
Christoph Lameter 已提交
324 325
#ifndef __GENERATING_BOUNDS_H

L
Linus Torvalds 已提交
326 327
struct zone {
	/* Fields commonly accessed by the page allocator */
328 329 330 331

	/* zone watermarks, access with *_wmark_pages(zone) macros */
	unsigned long watermark[NR_WMARK];

332 333 334 335 336 337 338
	/*
	 * When free pages are below this point, additional steps are taken
	 * when reading the number of free pages to avoid per-cpu counter
	 * drift allowing watermarks to be breached
	 */
	unsigned long percpu_drift_mark;

L
Linus Torvalds 已提交
339 340 341 342 343 344 345 346 347 348
	/*
	 * We don't know if the memory that we're going to allocate will be freeable
	 * or/and it will be released eventually, so to avoid totally wasting several
	 * GB of ram we must reserve some of the lower zone memory (otherwise we risk
	 * to run OOM on the lower zones despite there's tons of freeable ram
	 * on the higher zones). This array is recalculated at runtime if the
	 * sysctl_lowmem_reserve_ratio sysctl changes.
	 */
	unsigned long		lowmem_reserve[MAX_NR_ZONES];

349 350 351 352 353 354
	/*
	 * This is a per-zone reserve of pages that should not be
	 * considered dirtyable memory.
	 */
	unsigned long		dirty_balance_reserve;

355
#ifdef CONFIG_NUMA
356
	int node;
357 358 359
	/*
	 * zone reclaim becomes active if more unmapped pages exist.
	 */
360
	unsigned long		min_unmapped_pages;
361
	unsigned long		min_slab_pages;
362
#endif
363
	struct per_cpu_pageset __percpu *pageset;
L
Linus Torvalds 已提交
364 365 366 367
	/*
	 * free areas of different sizes
	 */
	spinlock_t		lock;
368
#if defined CONFIG_COMPACTION || defined CONFIG_CMA
369 370
	/* Set to true when the PG_migrate_skip bits should be cleared */
	bool			compact_blockskip_flush;
371

372
	/* pfn where compaction free scanner should start */
373
	unsigned long		compact_cached_free_pfn;
374 375
	/* pfn where async and sync compaction migration scanner should start */
	unsigned long		compact_cached_migrate_pfn[2];
376
#endif
377 378 379 380
#ifdef CONFIG_MEMORY_HOTPLUG
	/* see spanned/present_pages for more description */
	seqlock_t		span_seqlock;
#endif
L
Linus Torvalds 已提交
381 382
	struct free_area	free_area[MAX_ORDER];

383 384
#ifndef CONFIG_SPARSEMEM
	/*
385
	 * Flags for a pageblock_nr_pages block. See pageblock-flags.h.
386 387 388 389 390
	 * In SPARSEMEM, this map is stored in struct mem_section
	 */
	unsigned long		*pageblock_flags;
#endif /* CONFIG_SPARSEMEM */

391 392 393 394 395 396 397 398
#ifdef CONFIG_COMPACTION
	/*
	 * On compaction failure, 1<<compact_defer_shift compactions
	 * are skipped before trying again. The number attempted since
	 * last failure is tracked with compact_considered.
	 */
	unsigned int		compact_considered;
	unsigned int		compact_defer_shift;
399
	int			compact_order_failed;
400
#endif
L
Linus Torvalds 已提交
401 402 403 404

	ZONE_PADDING(_pad1_)

	/* Fields commonly accessed by the page reclaim scanner */
405 406
	spinlock_t		lru_lock;
	struct lruvec		lruvec;
407

408 409 410
	/* Evictions & activations on the inactive file list */
	atomic_long_t		inactive_age;

L
Linus Torvalds 已提交
411
	unsigned long		pages_scanned;	   /* since last reclaim */
412
	unsigned long		flags;		   /* zone flags, see below */
M
Martin Hicks 已提交
413

414 415
	/* Zone statistics */
	atomic_long_t		vm_stat[NR_VM_ZONE_STAT_ITEMS];
416

417 418 419 420 421 422
	/*
	 * The target ratio of ACTIVE_ANON to INACTIVE_ANON pages on
	 * this zone's LRU.  Maintained by the pageout code.
	 */
	unsigned int inactive_ratio;

L
Linus Torvalds 已提交
423 424 425 426 427 428

	ZONE_PADDING(_pad2_)
	/* Rarely used or read-mostly fields */

	/*
	 * wait_table		-- the array holding the hash table
429
	 * wait_table_hash_nr_entries	-- the size of the hash table array
L
Linus Torvalds 已提交
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451
	 * wait_table_bits	-- wait_table_size == (1 << wait_table_bits)
	 *
	 * The purpose of all these is to keep track of the people
	 * waiting for a page to become available and make them
	 * runnable again when possible. The trouble is that this
	 * consumes a lot of space, especially when so few things
	 * wait on pages at a given time. So instead of using
	 * per-page waitqueues, we use a waitqueue hash table.
	 *
	 * The bucket discipline is to sleep on the same queue when
	 * colliding and wake all in that wait queue when removing.
	 * When something wakes, it must check to be sure its page is
	 * truly available, a la thundering herd. The cost of a
	 * collision is great, but given the expected load of the
	 * table, they should be so rare as to be outweighed by the
	 * benefits from the saved space.
	 *
	 * __wait_on_page_locked() and unlock_page() in mm/filemap.c, are the
	 * primary users of these fields, and in mm/page_alloc.c
	 * free_area_init_core() performs the initialization of them.
	 */
	wait_queue_head_t	* wait_table;
452
	unsigned long		wait_table_hash_nr_entries;
L
Linus Torvalds 已提交
453 454 455 456 457 458 459 460 461
	unsigned long		wait_table_bits;

	/*
	 * Discontig memory support fields.
	 */
	struct pglist_data	*zone_pgdat;
	/* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
	unsigned long		zone_start_pfn;

462
	/*
463 464 465
	 * spanned_pages is the total pages spanned by the zone, including
	 * holes, which is calculated as:
	 * 	spanned_pages = zone_end_pfn - zone_start_pfn;
466
	 *
467 468
	 * present_pages is physical pages existing within the zone, which
	 * is calculated as:
469
	 *	present_pages = spanned_pages - absent_pages(pages in holes);
470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
	 *
	 * managed_pages is present pages managed by the buddy system, which
	 * is calculated as (reserved_pages includes pages allocated by the
	 * bootmem allocator):
	 *	managed_pages = present_pages - reserved_pages;
	 *
	 * So present_pages may be used by memory hotplug or memory power
	 * management logic to figure out unmanaged pages by checking
	 * (present_pages - managed_pages). And managed_pages should be used
	 * by page allocator and vm scanner to calculate all kinds of watermarks
	 * and thresholds.
	 *
	 * Locking rules:
	 *
	 * zone_start_pfn and spanned_pages are protected by span_seqlock.
	 * It is a seqlock because it has to be read outside of zone->lock,
	 * and it is done in the main allocator path.  But, it is written
	 * quite infrequently.
	 *
	 * The span_seq lock is declared along with zone->lock because it is
490 491
	 * frequently read in proximity to zone->lock.  It's good to
	 * give them a chance of being in the same cacheline.
492
	 *
493
	 * Write access to present_pages at runtime should be protected by
494 495
	 * mem_hotplug_begin/end(). Any reader who can't tolerant drift of
	 * present_pages should get_online_mems() to get a stable value.
496 497 498 499 500 501
	 *
	 * Read access to managed_pages should be safe because it's unsigned
	 * long. Write access to zone->managed_pages and totalram_pages are
	 * protected by managed_page_count_lock at runtime. Idealy only
	 * adjust_managed_page_count() should be used instead of directly
	 * touching zone->managed_pages and totalram_pages.
502
	 */
503 504 505
	unsigned long		spanned_pages;
	unsigned long		present_pages;
	unsigned long		managed_pages;
L
Linus Torvalds 已提交
506

507 508 509 510 511 512
	/*
	 * Number of MIGRATE_RESEVE page block. To maintain for just
	 * optimization. Protected by zone->lock.
	 */
	int			nr_migrate_reserve_block;

L
Linus Torvalds 已提交
513 514 515
	/*
	 * rarely used fields:
	 */
516
	const char		*name;
517
} ____cacheline_internodealigned_in_smp;
L
Linus Torvalds 已提交
518

519 520
typedef enum {
	ZONE_RECLAIM_LOCKED,		/* prevents concurrent reclaim */
D
David Rientjes 已提交
521
	ZONE_OOM_LOCKED,		/* zone is in OOM killer zonelist */
522 523 524
	ZONE_CONGESTED,			/* zone has many dirty pages backed by
					 * a congested BDI
					 */
525 526 527 528
	ZONE_TAIL_LRU_DIRTY,		/* reclaim scanning has recently found
					 * many dirty file pages at the tail
					 * of the LRU.
					 */
529 530 531
	ZONE_WRITEBACK,			/* reclaim scanning has recently found
					 * many pages under writeback
					 */
532 533 534 535 536 537
} zone_flags_t;

static inline void zone_set_flag(struct zone *zone, zone_flags_t flag)
{
	set_bit(flag, &zone->flags);
}
538 539 540 541 542 543

static inline int zone_test_and_set_flag(struct zone *zone, zone_flags_t flag)
{
	return test_and_set_bit(flag, &zone->flags);
}

544 545 546 547 548
static inline void zone_clear_flag(struct zone *zone, zone_flags_t flag)
{
	clear_bit(flag, &zone->flags);
}

549 550 551 552 553
static inline int zone_is_reclaim_congested(const struct zone *zone)
{
	return test_bit(ZONE_CONGESTED, &zone->flags);
}

554 555 556 557 558
static inline int zone_is_reclaim_dirty(const struct zone *zone)
{
	return test_bit(ZONE_TAIL_LRU_DIRTY, &zone->flags);
}

559 560 561 562 563
static inline int zone_is_reclaim_writeback(const struct zone *zone)
{
	return test_bit(ZONE_WRITEBACK, &zone->flags);
}

564 565 566 567
static inline int zone_is_reclaim_locked(const struct zone *zone)
{
	return test_bit(ZONE_RECLAIM_LOCKED, &zone->flags);
}
568

D
David Rientjes 已提交
569 570 571 572
static inline int zone_is_oom_locked(const struct zone *zone)
{
	return test_bit(ZONE_OOM_LOCKED, &zone->flags);
}
573

R
Russ Anderson 已提交
574
static inline unsigned long zone_end_pfn(const struct zone *zone)
575 576 577 578 579 580 581 582 583
{
	return zone->zone_start_pfn + zone->spanned_pages;
}

static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn)
{
	return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);
}

584 585 586 587 588 589 590 591 592 593
static inline bool zone_is_initialized(struct zone *zone)
{
	return !!zone->wait_table;
}

static inline bool zone_is_empty(struct zone *zone)
{
	return zone->spanned_pages == 0;
}

L
Linus Torvalds 已提交
594 595 596 597 598 599 600
/*
 * The "priority" of VM scanning is how much of the queues we will scan in one
 * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the
 * queues ("queue_length >> 12") during an aging round.
 */
#define DEF_PRIORITY 12

601 602 603 604
/* Maximum number of zones on a zonelist */
#define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)

#ifdef CONFIG_NUMA
605 606

/*
607
 * The NUMA zonelists are doubled because we need zonelists that restrict the
608
 * allocations to a single node for __GFP_THISNODE.
609
 *
610
 * [0]	: Zonelist with fallback
611
 * [1]	: No fallback (__GFP_THISNODE)
612
 */
613
#define MAX_ZONELISTS 2
614 615


616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676
/*
 * We cache key information from each zonelist for smaller cache
 * footprint when scanning for free pages in get_page_from_freelist().
 *
 * 1) The BITMAP fullzones tracks which zones in a zonelist have come
 *    up short of free memory since the last time (last_fullzone_zap)
 *    we zero'd fullzones.
 * 2) The array z_to_n[] maps each zone in the zonelist to its node
 *    id, so that we can efficiently evaluate whether that node is
 *    set in the current tasks mems_allowed.
 *
 * Both fullzones and z_to_n[] are one-to-one with the zonelist,
 * indexed by a zones offset in the zonelist zones[] array.
 *
 * The get_page_from_freelist() routine does two scans.  During the
 * first scan, we skip zones whose corresponding bit in 'fullzones'
 * is set or whose corresponding node in current->mems_allowed (which
 * comes from cpusets) is not set.  During the second scan, we bypass
 * this zonelist_cache, to ensure we look methodically at each zone.
 *
 * Once per second, we zero out (zap) fullzones, forcing us to
 * reconsider nodes that might have regained more free memory.
 * The field last_full_zap is the time we last zapped fullzones.
 *
 * This mechanism reduces the amount of time we waste repeatedly
 * reexaming zones for free memory when they just came up low on
 * memory momentarilly ago.
 *
 * The zonelist_cache struct members logically belong in struct
 * zonelist.  However, the mempolicy zonelists constructed for
 * MPOL_BIND are intentionally variable length (and usually much
 * shorter).  A general purpose mechanism for handling structs with
 * multiple variable length members is more mechanism than we want
 * here.  We resort to some special case hackery instead.
 *
 * The MPOL_BIND zonelists don't need this zonelist_cache (in good
 * part because they are shorter), so we put the fixed length stuff
 * at the front of the zonelist struct, ending in a variable length
 * zones[], as is needed by MPOL_BIND.
 *
 * Then we put the optional zonelist cache on the end of the zonelist
 * struct.  This optional stuff is found by a 'zlcache_ptr' pointer in
 * the fixed length portion at the front of the struct.  This pointer
 * both enables us to find the zonelist cache, and in the case of
 * MPOL_BIND zonelists, (which will just set the zlcache_ptr to NULL)
 * to know that the zonelist cache is not there.
 *
 * The end result is that struct zonelists come in two flavors:
 *  1) The full, fixed length version, shown below, and
 *  2) The custom zonelists for MPOL_BIND.
 * The custom MPOL_BIND zonelists have a NULL zlcache_ptr and no zlcache.
 *
 * Even though there may be multiple CPU cores on a node modifying
 * fullzones or last_full_zap in the same zonelist_cache at the same
 * time, we don't lock it.  This is just hint data - if it is wrong now
 * and then, the allocator will still function, perhaps a bit slower.
 */


struct zonelist_cache {
	unsigned short z_to_n[MAX_ZONES_PER_ZONELIST];		/* zone->nid */
677
	DECLARE_BITMAP(fullzones, MAX_ZONES_PER_ZONELIST);	/* zone full? */
678 679 680
	unsigned long last_full_zap;		/* when last zap'd (jiffies) */
};
#else
681
#define MAX_ZONELISTS 1
682 683 684
struct zonelist_cache;
#endif

685 686 687 688 689 690 691 692 693
/*
 * This struct contains information about a zone in a zonelist. It is stored
 * here to avoid dereferences into large structures and lookups of tables
 */
struct zoneref {
	struct zone *zone;	/* Pointer to actual zone */
	int zone_idx;		/* zone_idx(zoneref->zone) */
};

L
Linus Torvalds 已提交
694 695 696 697 698 699
/*
 * One allocation request operates on a zonelist. A zonelist
 * is a list of zones, the first one is the 'goal' of the
 * allocation, the other zones are fallback zones, in decreasing
 * priority.
 *
700 701
 * If zlcache_ptr is not NULL, then it is just the address of zlcache,
 * as explained above.  If zlcache_ptr is NULL, there is no zlcache.
702 703 704 705 706 707 708 709
 * *
 * To speed the reading of the zonelist, the zonerefs contain the zone index
 * of the entry being read. Helper functions to access information given
 * a struct zoneref are
 *
 * zonelist_zone()	- Return the struct zone * for an entry in _zonerefs
 * zonelist_zone_idx()	- Return the index of the zone for an entry
 * zonelist_node_idx()	- Return the index of the node for an entry
L
Linus Torvalds 已提交
710 711
 */
struct zonelist {
712
	struct zonelist_cache *zlcache_ptr;		     // NULL or &zlcache
713
	struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];
714 715 716
#ifdef CONFIG_NUMA
	struct zonelist_cache zlcache;			     // optional ...
#endif
L
Linus Torvalds 已提交
717 718
};

T
Tejun Heo 已提交
719
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
720 721 722 723 724
struct node_active_region {
	unsigned long start_pfn;
	unsigned long end_pfn;
	int nid;
};
T
Tejun Heo 已提交
725
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
L
Linus Torvalds 已提交
726

727 728 729 730 731
#ifndef CONFIG_DISCONTIGMEM
/* The array of struct pages - for discontigmem use pgdat->lmem_map */
extern struct page *mem_map;
#endif

L
Linus Torvalds 已提交
732 733 734 735 736 737 738 739 740 741 742 743 744 745
/*
 * The pg_data_t structure is used in machines with CONFIG_DISCONTIGMEM
 * (mostly NUMA machines?) to denote a higher-level memory zone than the
 * zone denotes.
 *
 * On NUMA machines, each NUMA node would have a pg_data_t to describe
 * it's memory layout.
 *
 * Memory statistics and page replacement data structures are maintained on a
 * per-zone basis.
 */
struct bootmem_data;
typedef struct pglist_data {
	struct zone node_zones[MAX_NR_ZONES];
746
	struct zonelist node_zonelists[MAX_ZONELISTS];
L
Linus Torvalds 已提交
747
	int nr_zones;
748
#ifdef CONFIG_FLAT_NODE_MEM_MAP	/* means !SPARSEMEM */
L
Linus Torvalds 已提交
749
	struct page *node_mem_map;
A
Andrew Morton 已提交
750
#ifdef CONFIG_MEMCG
751 752
	struct page_cgroup *node_page_cgroup;
#endif
A
Andy Whitcroft 已提交
753
#endif
754
#ifndef CONFIG_NO_BOOTMEM
L
Linus Torvalds 已提交
755
	struct bootmem_data *bdata;
756
#endif
757 758 759 760 761 762
#ifdef CONFIG_MEMORY_HOTPLUG
	/*
	 * Must be held any time you expect node_start_pfn, node_present_pages
	 * or node_spanned_pages stay constant.  Holding this will also
	 * guarantee that any pfn_valid() stays that way.
	 *
763 764 765
	 * pgdat_resize_lock() and pgdat_resize_unlock() are provided to
	 * manipulate node_size_lock without checking for CONFIG_MEMORY_HOTPLUG.
	 *
766
	 * Nests above zone->lock and zone->span_seqlock
767 768 769
	 */
	spinlock_t node_size_lock;
#endif
L
Linus Torvalds 已提交
770 771 772 773 774 775
	unsigned long node_start_pfn;
	unsigned long node_present_pages; /* total number of physical pages */
	unsigned long node_spanned_pages; /* total size of physical page
					     range, including holes */
	int node_id;
	wait_queue_head_t kswapd_wait;
776
	wait_queue_head_t pfmemalloc_wait;
777 778
	struct task_struct *kswapd;	/* Protected by
					   mem_hotplug_begin/end() */
L
Linus Torvalds 已提交
779
	int kswapd_max_order;
780
	enum zone_type classzone_idx;
781
#ifdef CONFIG_NUMA_BALANCING
782
	/* Lock serializing the migrate rate limiting window */
783 784 785 786 787 788 789 790
	spinlock_t numabalancing_migrate_lock;

	/* Rate limiting time interval */
	unsigned long numabalancing_migrate_next_window;

	/* Number of pages migrated during the rate limiting time interval */
	unsigned long numabalancing_migrate_nr_pages;
#endif
L
Linus Torvalds 已提交
791 792 793 794
} pg_data_t;

#define node_present_pages(nid)	(NODE_DATA(nid)->node_present_pages)
#define node_spanned_pages(nid)	(NODE_DATA(nid)->node_spanned_pages)
A
Andy Whitcroft 已提交
795
#ifdef CONFIG_FLAT_NODE_MEM_MAP
796
#define pgdat_page_nr(pgdat, pagenr)	((pgdat)->node_mem_map + (pagenr))
A
Andy Whitcroft 已提交
797 798 799
#else
#define pgdat_page_nr(pgdat, pagenr)	pfn_to_page((pgdat)->node_start_pfn + (pagenr))
#endif
800
#define nid_page_nr(nid, pagenr) 	pgdat_page_nr(NODE_DATA(nid),(pagenr))
L
Linus Torvalds 已提交
801

802
#define node_start_pfn(nid)	(NODE_DATA(nid)->node_start_pfn)
803
#define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
804

805 806 807 808 809 810 811 812 813
static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
{
	return pgdat->node_start_pfn + pgdat->node_spanned_pages;
}

static inline bool pgdat_is_empty(pg_data_t *pgdat)
{
	return !pgdat->node_start_pfn && !pgdat->node_spanned_pages;
}
814

815 816
#include <linux/memory_hotplug.h>

817
extern struct mutex zonelists_mutex;
818
void build_all_zonelists(pg_data_t *pgdat, struct zone *zone);
819
void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx);
820 821 822 823
bool zone_watermark_ok(struct zone *z, unsigned int order,
		unsigned long mark, int classzone_idx, int alloc_flags);
bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
		unsigned long mark, int classzone_idx, int alloc_flags);
D
Dave Hansen 已提交
824 825 826 827
enum memmap_context {
	MEMMAP_EARLY,
	MEMMAP_HOTPLUG,
};
828
extern int init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
D
Dave Hansen 已提交
829 830
				     unsigned long size,
				     enum memmap_context context);
831

832
extern void lruvec_init(struct lruvec *lruvec);
833 834 835

static inline struct zone *lruvec_zone(struct lruvec *lruvec)
{
A
Andrew Morton 已提交
836
#ifdef CONFIG_MEMCG
837 838 839 840 841 842
	return lruvec->zone;
#else
	return container_of(lruvec, struct zone, lruvec);
#endif
}

L
Linus Torvalds 已提交
843 844 845 846 847 848
#ifdef CONFIG_HAVE_MEMORY_PRESENT
void memory_present(int nid, unsigned long start, unsigned long end);
#else
static inline void memory_present(int nid, unsigned long start, unsigned long end) {}
#endif

849 850 851 852 853 854
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
int local_memory_node(int node_id);
#else
static inline int local_memory_node(int node_id) { return node_id; };
#endif

L
Linus Torvalds 已提交
855 856 857 858 859 860 861 862 863
#ifdef CONFIG_NEED_NODE_MEMMAP_SIZE
unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long);
#endif

/*
 * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc.
 */
#define zone_idx(zone)		((zone) - (zone)->zone_pgdat->node_zones)

864 865 866 867 868
static inline int populated_zone(struct zone *zone)
{
	return (!!zone->present_pages);
}

M
Mel Gorman 已提交
869 870 871 872
extern int movable_zone;

static inline int zone_movable_is_highmem(void)
{
873
#if defined(CONFIG_HIGHMEM) && defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
M
Mel Gorman 已提交
874 875 876 877 878 879
	return movable_zone == ZONE_HIGHMEM;
#else
	return 0;
#endif
}

880
static inline int is_highmem_idx(enum zone_type idx)
L
Linus Torvalds 已提交
881
{
882
#ifdef CONFIG_HIGHMEM
M
Mel Gorman 已提交
883 884
	return (idx == ZONE_HIGHMEM ||
		(idx == ZONE_MOVABLE && zone_movable_is_highmem()));
885 886 887
#else
	return 0;
#endif
L
Linus Torvalds 已提交
888 889 890 891 892 893 894 895 896 897
}

/**
 * is_highmem - helper function to quickly check if a struct zone is a 
 *              highmem zone or not.  This is an attempt to keep references
 *              to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum.
 * @zone - pointer to struct zone variable
 */
static inline int is_highmem(struct zone *zone)
{
898
#ifdef CONFIG_HIGHMEM
899 900 901 902
	int zone_off = (char *)zone - (char *)zone->zone_pgdat->node_zones;
	return zone_off == ZONE_HIGHMEM * sizeof(*zone) ||
	       (zone_off == ZONE_MOVABLE * sizeof(*zone) &&
		zone_movable_is_highmem());
903 904 905
#else
	return 0;
#endif
L
Linus Torvalds 已提交
906 907 908 909
}

/* These two functions are used to setup the per zone pages min values */
struct ctl_table;
910
int min_free_kbytes_sysctl_handler(struct ctl_table *, int,
L
Linus Torvalds 已提交
911 912
					void __user *, size_t *, loff_t *);
extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1];
913
int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int,
L
Linus Torvalds 已提交
914
					void __user *, size_t *, loff_t *);
915
int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *, int,
916
					void __user *, size_t *, loff_t *);
917
int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int,
918
			void __user *, size_t *, loff_t *);
919
int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int,
920
			void __user *, size_t *, loff_t *);
L
Linus Torvalds 已提交
921

922
extern int numa_zonelist_order_handler(struct ctl_table *, int,
923
			void __user *, size_t *, loff_t *);
924 925 926
extern char numa_zonelist_order[];
#define NUMA_ZONELIST_ORDER_LEN 16	/* string buffer size */

927
#ifndef CONFIG_NEED_MULTIPLE_NODES
L
Linus Torvalds 已提交
928 929 930 931 932

extern struct pglist_data contig_page_data;
#define NODE_DATA(nid)		(&contig_page_data)
#define NODE_MEM_MAP(nid)	mem_map

933
#else /* CONFIG_NEED_MULTIPLE_NODES */
L
Linus Torvalds 已提交
934 935 936

#include <asm/mmzone.h>

937
#endif /* !CONFIG_NEED_MULTIPLE_NODES */
938

939 940 941
extern struct pglist_data *first_online_pgdat(void);
extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
extern struct zone *next_zone(struct zone *zone);
942 943

/**
944
 * for_each_online_pgdat - helper macro to iterate over all online nodes
945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962
 * @pgdat - pointer to a pg_data_t variable
 */
#define for_each_online_pgdat(pgdat)			\
	for (pgdat = first_online_pgdat();		\
	     pgdat;					\
	     pgdat = next_online_pgdat(pgdat))
/**
 * for_each_zone - helper macro to iterate over all memory zones
 * @zone - pointer to struct zone variable
 *
 * The user only needs to declare the zone variable, for_each_zone
 * fills it in.
 */
#define for_each_zone(zone)			        \
	for (zone = (first_online_pgdat())->node_zones; \
	     zone;					\
	     zone = next_zone(zone))

963 964 965 966 967 968 969 970
#define for_each_populated_zone(zone)		        \
	for (zone = (first_online_pgdat())->node_zones; \
	     zone;					\
	     zone = next_zone(zone))			\
		if (!populated_zone(zone))		\
			; /* do nothing */		\
		else

971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990
static inline struct zone *zonelist_zone(struct zoneref *zoneref)
{
	return zoneref->zone;
}

static inline int zonelist_zone_idx(struct zoneref *zoneref)
{
	return zoneref->zone_idx;
}

static inline int zonelist_node_idx(struct zoneref *zoneref)
{
#ifdef CONFIG_NUMA
	/* zone_to_nid not available in this context */
	return zoneref->zone->node;
#else
	return 0;
#endif /* CONFIG_NUMA */
}

991 992 993 994 995 996 997 998 999
/**
 * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point
 * @z - The cursor used as a starting point for the search
 * @highest_zoneidx - The zone index of the highest zone to return
 * @nodes - An optional nodemask to filter the zonelist with
 * @zone - The first suitable zone found is returned via this parameter
 *
 * This function returns the next zone at or below a given zone index that is
 * within the allowed nodemask using a cursor as the starting point for the
1000 1001 1002
 * search. The zoneref returned is a cursor that represents the current zone
 * being examined. It should be advanced by one before calling
 * next_zones_zonelist again.
1003 1004 1005 1006 1007
 */
struct zoneref *next_zones_zonelist(struct zoneref *z,
					enum zone_type highest_zoneidx,
					nodemask_t *nodes,
					struct zone **zone);
1008

1009 1010 1011 1012 1013 1014 1015 1016 1017
/**
 * first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist
 * @zonelist - The zonelist to search for a suitable zone
 * @highest_zoneidx - The zone index of the highest zone to return
 * @nodes - An optional nodemask to filter the zonelist with
 * @zone - The first suitable zone found is returned via this parameter
 *
 * This function returns the first zone at or below a given zone index that is
 * within the allowed nodemask. The zoneref returned is a cursor that can be
1018 1019
 * used to iterate the zonelist with next_zones_zonelist by advancing it by
 * one before calling.
1020
 */
1021
static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
1022 1023 1024
					enum zone_type highest_zoneidx,
					nodemask_t *nodes,
					struct zone **zone)
1025
{
1026 1027
	return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes,
								zone);
1028 1029
}

1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
/**
 * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask
 * @zone - The current zone in the iterator
 * @z - The current pointer within zonelist->zones being iterated
 * @zlist - The zonelist being iterated
 * @highidx - The zone index of the highest zone to return
 * @nodemask - Nodemask allowed by the allocator
 *
 * This iterator iterates though all zones at or below a given zone index and
 * within a given nodemask
 */
#define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
	for (z = first_zones_zonelist(zlist, highidx, nodemask, &zone);	\
		zone;							\
1044
		z = next_zones_zonelist(++z, highidx, nodemask, &zone))	\
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055

/**
 * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index
 * @zone - The current zone in the iterator
 * @z - The current pointer within zonelist->zones being iterated
 * @zlist - The zonelist being iterated
 * @highidx - The zone index of the highest zone to return
 *
 * This iterator iterates though all zones at or below a given zone index.
 */
#define for_each_zone_zonelist(zone, z, zlist, highidx) \
1056
	for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL)
1057

A
Andy Whitcroft 已提交
1058 1059 1060 1061
#ifdef CONFIG_SPARSEMEM
#include <asm/sparsemem.h>
#endif

1062
#if !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) && \
T
Tejun Heo 已提交
1063
	!defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1064 1065 1066 1067
static inline unsigned long early_pfn_to_nid(unsigned long pfn)
{
	return 0;
}
1068 1069
#endif

1070 1071 1072 1073
#ifdef CONFIG_FLATMEM
#define pfn_to_nid(pfn)		(0)
#endif

A
Andy Whitcroft 已提交
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
#ifdef CONFIG_SPARSEMEM

/*
 * SECTION_SHIFT    		#bits space required to store a section #
 *
 * PA_SECTION_SHIFT		physical address to/from section number
 * PFN_SECTION_SHIFT		pfn to/from section number
 */
#define PA_SECTION_SHIFT	(SECTION_SIZE_BITS)
#define PFN_SECTION_SHIFT	(SECTION_SIZE_BITS - PAGE_SHIFT)

#define NR_MEM_SECTIONS		(1UL << SECTIONS_SHIFT)

#define PAGES_PER_SECTION       (1UL << PFN_SECTION_SHIFT)
#define PAGE_SECTION_MASK	(~(PAGES_PER_SECTION-1))

1090
#define SECTION_BLOCKFLAGS_BITS \
1091
	((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
1092

A
Andy Whitcroft 已提交
1093 1094 1095 1096
#if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS
#error Allocator MAX_ORDER exceeds SECTION_SIZE
#endif

1097 1098 1099
#define pfn_to_section_nr(pfn) ((pfn) >> PFN_SECTION_SHIFT)
#define section_nr_to_pfn(sec) ((sec) << PFN_SECTION_SHIFT)

1100 1101 1102
#define SECTION_ALIGN_UP(pfn)	(((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)
#define SECTION_ALIGN_DOWN(pfn)	((pfn) & PAGE_SECTION_MASK)

A
Andy Whitcroft 已提交
1103
struct page;
1104
struct page_cgroup;
A
Andy Whitcroft 已提交
1105
struct mem_section {
A
Andy Whitcroft 已提交
1106 1107 1108 1109 1110
	/*
	 * This is, logically, a pointer to an array of struct
	 * pages.  However, it is stored with some other magic.
	 * (see sparse.c::sparse_init_one_section())
	 *
1111 1112 1113 1114
	 * Additionally during early boot we encode node id of
	 * the location of the section here to guide allocation.
	 * (see sparse.c::memory_present())
	 *
A
Andy Whitcroft 已提交
1115 1116 1117 1118
	 * Making it a UL at least makes someone do a cast
	 * before using it wrong.
	 */
	unsigned long section_mem_map;
1119 1120 1121

	/* See declaration of similar field in struct zone */
	unsigned long *pageblock_flags;
A
Andrew Morton 已提交
1122
#ifdef CONFIG_MEMCG
1123 1124 1125 1126 1127 1128 1129
	/*
	 * If !SPARSEMEM, pgdat doesn't have page_cgroup pointer. We use
	 * section. (see memcontrol.h/page_cgroup.h about this.)
	 */
	struct page_cgroup *page_cgroup;
	unsigned long pad;
#endif
1130 1131 1132 1133
	/*
	 * WARNING: mem_section must be a power-of-2 in size for the
	 * calculation and use of SECTION_ROOT_MASK to make sense.
	 */
A
Andy Whitcroft 已提交
1134 1135
};

1136 1137 1138 1139 1140
#ifdef CONFIG_SPARSEMEM_EXTREME
#define SECTIONS_PER_ROOT       (PAGE_SIZE / sizeof (struct mem_section))
#else
#define SECTIONS_PER_ROOT	1
#endif
B
Bob Picco 已提交
1141

1142
#define SECTION_NR_TO_ROOT(sec)	((sec) / SECTIONS_PER_ROOT)
1143
#define NR_SECTION_ROOTS	DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)
1144
#define SECTION_ROOT_MASK	(SECTIONS_PER_ROOT - 1)
B
Bob Picco 已提交
1145

1146 1147
#ifdef CONFIG_SPARSEMEM_EXTREME
extern struct mem_section *mem_section[NR_SECTION_ROOTS];
B
Bob Picco 已提交
1148
#else
1149 1150
extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
#endif
A
Andy Whitcroft 已提交
1151

A
Andy Whitcroft 已提交
1152 1153
static inline struct mem_section *__nr_to_section(unsigned long nr)
{
1154 1155 1156
	if (!mem_section[SECTION_NR_TO_ROOT(nr)])
		return NULL;
	return &mem_section[SECTION_NR_TO_ROOT(nr)][nr & SECTION_ROOT_MASK];
A
Andy Whitcroft 已提交
1157
}
1158
extern int __section_nr(struct mem_section* ms);
1159
extern unsigned long usemap_size(void);
A
Andy Whitcroft 已提交
1160 1161 1162 1163 1164 1165 1166 1167 1168 1169

/*
 * We use the lower bits of the mem_map pointer to store
 * a little bit of information.  There should be at least
 * 3 bits here due to 32-bit alignment.
 */
#define	SECTION_MARKED_PRESENT	(1UL<<0)
#define SECTION_HAS_MEM_MAP	(1UL<<1)
#define SECTION_MAP_LAST_BIT	(1UL<<2)
#define SECTION_MAP_MASK	(~(SECTION_MAP_LAST_BIT-1))
1170
#define SECTION_NID_SHIFT	2
A
Andy Whitcroft 已提交
1171 1172 1173 1174 1175 1176 1177 1178

static inline struct page *__section_mem_map_addr(struct mem_section *section)
{
	unsigned long map = section->section_mem_map;
	map &= SECTION_MAP_MASK;
	return (struct page *)map;
}

1179
static inline int present_section(struct mem_section *section)
A
Andy Whitcroft 已提交
1180
{
B
Bob Picco 已提交
1181
	return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));
A
Andy Whitcroft 已提交
1182 1183
}

1184 1185 1186 1187 1188 1189
static inline int present_section_nr(unsigned long nr)
{
	return present_section(__nr_to_section(nr));
}

static inline int valid_section(struct mem_section *section)
A
Andy Whitcroft 已提交
1190
{
B
Bob Picco 已提交
1191
	return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));
A
Andy Whitcroft 已提交
1192 1193 1194 1195 1196 1197 1198
}

static inline int valid_section_nr(unsigned long nr)
{
	return valid_section(__nr_to_section(nr));
}

A
Andy Whitcroft 已提交
1199 1200
static inline struct mem_section *__pfn_to_section(unsigned long pfn)
{
A
Andy Whitcroft 已提交
1201
	return __nr_to_section(pfn_to_section_nr(pfn));
A
Andy Whitcroft 已提交
1202 1203
}

1204
#ifndef CONFIG_HAVE_ARCH_PFN_VALID
A
Andy Whitcroft 已提交
1205 1206 1207 1208
static inline int pfn_valid(unsigned long pfn)
{
	if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
		return 0;
A
Andy Whitcroft 已提交
1209
	return valid_section(__nr_to_section(pfn_to_section_nr(pfn)));
A
Andy Whitcroft 已提交
1210
}
1211
#endif
A
Andy Whitcroft 已提交
1212

1213 1214 1215 1216 1217 1218 1219
static inline int pfn_present(unsigned long pfn)
{
	if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
		return 0;
	return present_section(__nr_to_section(pfn_to_section_nr(pfn)));
}

A
Andy Whitcroft 已提交
1220 1221 1222 1223 1224 1225
/*
 * These are _only_ used during initialisation, therefore they
 * can use __initdata ...  They could have names to indicate
 * this restriction.
 */
#ifdef CONFIG_NUMA
1226 1227 1228 1229 1230
#define pfn_to_nid(pfn)							\
({									\
	unsigned long __pfn_to_nid_pfn = (pfn);				\
	page_to_nid(pfn_to_page(__pfn_to_nid_pfn));			\
})
1231 1232
#else
#define pfn_to_nid(pfn)		(0)
A
Andy Whitcroft 已提交
1233 1234 1235 1236 1237 1238
#endif

#define early_pfn_valid(pfn)	pfn_valid(pfn)
void sparse_init(void);
#else
#define sparse_init()	do {} while (0)
1239
#define sparse_index_init(_sec, _nid)  do {} while (0)
A
Andy Whitcroft 已提交
1240 1241
#endif /* CONFIG_SPARSEMEM */

1242
#ifdef CONFIG_NODES_SPAN_OTHER_NODES
1243
bool early_pfn_in_nid(unsigned long pfn, int nid);
1244 1245 1246 1247
#else
#define early_pfn_in_nid(pfn, nid)	(1)
#endif

A
Andy Whitcroft 已提交
1248 1249 1250 1251 1252 1253 1254
#ifndef early_pfn_valid
#define early_pfn_valid(pfn)	(1)
#endif

void memory_present(int nid, unsigned long start, unsigned long end);
unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long);

1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
/*
 * If it is possible to have holes within a MAX_ORDER_NR_PAGES, then we
 * need to check pfn validility within that MAX_ORDER_NR_PAGES block.
 * pfn_valid_within() should be used in this case; we optimise this away
 * when we have no holes within a MAX_ORDER_NR_PAGES block.
 */
#ifdef CONFIG_HOLES_IN_ZONE
#define pfn_valid_within(pfn) pfn_valid(pfn)
#else
#define pfn_valid_within(pfn) (1)
#endif

1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
#ifdef CONFIG_ARCH_HAS_HOLES_MEMORYMODEL
/*
 * pfn_valid() is meant to be able to tell if a given PFN has valid memmap
 * associated with it or not. In FLATMEM, it is expected that holes always
 * have valid memmap as long as there is valid PFNs either side of the hole.
 * In SPARSEMEM, it is assumed that a valid section has a memmap for the
 * entire section.
 *
 * However, an ARM, and maybe other embedded architectures in the future
 * free memmap backing holes to save memory on the assumption the memmap is
 * never used. The page_zone linkages are then broken even though pfn_valid()
 * returns true. A walker of the full memmap must then do this additional
 * check to ensure the memmap they are looking at is sane by making sure
 * the zone and PFN linkages are still valid. This is expensive, but walkers
 * of the full memmap are extremely rare.
 */
int memmap_valid_within(unsigned long pfn,
					struct page *page, struct zone *zone);
#else
static inline int memmap_valid_within(unsigned long pfn,
					struct page *page, struct zone *zone)
{
	return 1;
}
#endif /* CONFIG_ARCH_HAS_HOLES_MEMORYMODEL */

C
Christoph Lameter 已提交
1293
#endif /* !__GENERATING_BOUNDS.H */
L
Linus Torvalds 已提交
1294 1295
#endif /* !__ASSEMBLY__ */
#endif /* _LINUX_MMZONE_H */