page-flags.h 20.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * Macros for manipulating and testing page->flags
 */

#ifndef PAGE_FLAGS_H
#define PAGE_FLAGS_H

A
Andrew Morton 已提交
8
#include <linux/types.h>
9
#include <linux/bug.h>
10
#include <linux/mmdebug.h>
11
#ifndef __GENERATING_BOUNDS_H
12
#include <linux/mm_types.h>
13
#include <generated/bounds.h>
14
#endif /* !__GENERATING_BOUNDS_H */
A
Andrew Morton 已提交
15

L
Linus Torvalds 已提交
16 17 18 19 20 21
/*
 * Various page->flags bits:
 *
 * PG_reserved is set for special pages, which can never be swapped out. Some
 * of them might not even exist (eg empty_bad_page)...
 *
N
Nick Piggin 已提交
22 23 24
 * The PG_private bitflag is set on pagecache pages if they contain filesystem
 * specific data (which is normally at page->private). It can be used by
 * private allocations for its own usage.
L
Linus Torvalds 已提交
25
 *
N
Nick Piggin 已提交
26 27 28 29 30 31 32 33 34
 * During initiation of disk I/O, PG_locked is set. This bit is set before I/O
 * and cleared when writeback _starts_ or when read _completes_. PG_writeback
 * is set before writeback starts and cleared when it finishes.
 *
 * PG_locked also pins a page in pagecache, and blocks truncation of the file
 * while it is held.
 *
 * page_waitqueue(page) is a wait queue of all tasks waiting for the page
 * to become unlocked.
L
Linus Torvalds 已提交
35 36 37 38
 *
 * PG_uptodate tells whether the page's contents is valid.  When a read
 * completes, the page becomes uptodate, unless a disk I/O error happened.
 *
N
Nick Piggin 已提交
39 40
 * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
 * file-backed pagecache (see mm/vmscan.c).
L
Linus Torvalds 已提交
41 42 43 44 45 46 47 48 49 50 51
 *
 * PG_error is set to indicate that an I/O error occurred on this page.
 *
 * PG_arch_1 is an architecture specific page state bit.  The generic code
 * guarantees that this bit is cleared for a page when it first is entered into
 * the page cache.
 *
 * PG_highmem pages are not permanently mapped into the kernel virtual address
 * space, they need to be kmapped separately for doing IO on the pages.  The
 * struct page (these bits with information) are always mapped into kernel
 * address space...
N
Nick Piggin 已提交
52
 *
53 54 55
 * PG_hwpoison indicates that a page got corrupted in hardware and contains
 * data with incorrect ECC bits that triggered a machine check. Accessing is
 * not safe since it may cause another machine check. Don't touch!
L
Linus Torvalds 已提交
56 57 58 59
 */

/*
 * Don't use the *_dontuse flags.  Use the macros.  Otherwise you'll break
60 61 62 63 64 65 66
 * locked- and dirty-page accounting.
 *
 * The page flags field is split into two parts, the main flags area
 * which extends from the low bits upwards, and the fields area which
 * extends from the high bits downwards.
 *
 *  | FIELD | ... | FLAGS |
67 68
 *  N-1           ^       0
 *               (NR_PAGEFLAGS)
69
 *
70 71 72
 * The fields area is reserved for fields mapping zone, node (for NUMA) and
 * SPARSEMEM section (for variants of SPARSEMEM that require section ids like
 * SPARSEMEM_EXTREME with !SPARSEMEM_VMEMMAP).
L
Linus Torvalds 已提交
73
 */
74 75 76 77 78 79 80 81 82 83 84 85 86
enum pageflags {
	PG_locked,		/* Page is locked. Don't touch. */
	PG_error,
	PG_referenced,
	PG_uptodate,
	PG_dirty,
	PG_lru,
	PG_active,
	PG_slab,
	PG_owner_priv_1,	/* Owner use. If pagecache, fs may use*/
	PG_arch_1,
	PG_reserved,
	PG_private,		/* If pagecache, has fs-private data */
87
	PG_private_2,		/* If pagecache, has fs aux data */
88
	PG_writeback,		/* Page is under writeback */
89
	PG_head,		/* A head page */
90 91 92
	PG_swapcache,		/* Swap page: swp_entry_t in private */
	PG_mappedtodisk,	/* Has blocks allocated on-disk */
	PG_reclaim,		/* To be reclaimed asap */
R
Rik van Riel 已提交
93
	PG_swapbacked,		/* Page is backed by RAM/swap */
L
Lee Schermerhorn 已提交
94
	PG_unevictable,		/* Page is "unevictable"  */
H
Hugh Dickins 已提交
95
#ifdef CONFIG_MMU
N
Nick Piggin 已提交
96
	PG_mlocked,		/* Page is vma mlocked */
L
Lee Schermerhorn 已提交
97
#endif
98
#ifdef CONFIG_ARCH_USES_PG_UNCACHED
99
	PG_uncached,		/* Page has been mapped as uncached */
100 101 102
#endif
#ifdef CONFIG_MEMORY_FAILURE
	PG_hwpoison,		/* hardware poisoned page. Don't touch */
A
Andrea Arcangeli 已提交
103
#endif
104 105 106
#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
	PG_young,
	PG_idle,
A
Andrew Morton 已提交
107
#endif
108 109 110 111 112
	__NR_PAGEFLAGS,

	/* Filesystems */
	PG_checked = PG_owner_priv_1,

113 114 115 116 117 118
	/* Two page bits are conscripted by FS-Cache to maintain local caching
	 * state.  These bits are set on pages belonging to the netfs's inodes
	 * when those inodes are being locally cached.
	 */
	PG_fscache = PG_private_2,	/* page backed by cache */

119
	/* XEN */
120
	/* Pinned in Xen as a read-only pagetable page. */
121
	PG_pinned = PG_owner_priv_1,
122
	/* Pinned as part of domain save (see xen_mm_pin_all()). */
123
	PG_savepinned = PG_dirty,
124 125
	/* Has a grant mapping of another (foreign) domain's page. */
	PG_foreign = PG_owner_priv_1,
126

127 128
	/* SLOB */
	PG_slob_free = PG_private,
129 130 131

	/* Compound pages. Stored in first tail page's flags */
	PG_double_map = PG_private_2,
132
};
L
Linus Torvalds 已提交
133

134 135
#ifndef __GENERATING_BOUNDS_H

136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
struct page;	/* forward declaration */

static inline struct page *compound_head(struct page *page)
{
	unsigned long head = READ_ONCE(page->compound_head);

	if (unlikely(head & 1))
		return (struct page *) (head - 1);
	return page;
}

static inline int PageTail(struct page *page)
{
	return READ_ONCE(page->compound_head) & 1;
}

static inline int PageCompound(struct page *page)
{
	return test_bit(PG_head, &page->flags) || PageTail(page);
}

157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
/*
 * Page flags policies wrt compound pages
 *
 * PF_ANY:
 *     the page flag is relevant for small, head and tail pages.
 *
 * PF_HEAD:
 *     for compound page all operations related to the page flag applied to
 *     head page.
 *
 * PF_NO_TAIL:
 *     modifications of the page flag must be done on small or head pages,
 *     checks can be done on tail pages too.
 *
 * PF_NO_COMPOUND:
 *     the page flag is not relevant for compound pages.
 */
#define PF_ANY(page, enforce)	page
#define PF_HEAD(page, enforce)	compound_head(page)
#define PF_NO_TAIL(page, enforce) ({					\
		VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page);	\
		compound_head(page);})
179
#define PF_NO_COMPOUND(page, enforce) ({				\
180 181 182
		VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page);	\
		page;})

183 184 185
/*
 * Macros to create function definitions for page flags
 */
186 187 188
#define TESTPAGEFLAG(uname, lname, policy)				\
static inline int Page##uname(struct page *page)			\
	{ return test_bit(PG_##lname, &policy(page, 0)->flags); }
189

190
#define SETPAGEFLAG(uname, lname, policy)				\
191
static inline void SetPage##uname(struct page *page)			\
192
	{ set_bit(PG_##lname, &policy(page, 1)->flags); }
193

194
#define CLEARPAGEFLAG(uname, lname, policy)				\
195
static inline void ClearPage##uname(struct page *page)			\
196
	{ clear_bit(PG_##lname, &policy(page, 1)->flags); }
197

198
#define __SETPAGEFLAG(uname, lname, policy)				\
199
static inline void __SetPage##uname(struct page *page)			\
200
	{ __set_bit(PG_##lname, &policy(page, 1)->flags); }
201

202
#define __CLEARPAGEFLAG(uname, lname, policy)				\
203
static inline void __ClearPage##uname(struct page *page)		\
204
	{ __clear_bit(PG_##lname, &policy(page, 1)->flags); }
205

206
#define TESTSETFLAG(uname, lname, policy)				\
207
static inline int TestSetPage##uname(struct page *page)			\
208
	{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
209

210
#define TESTCLEARFLAG(uname, lname, policy)				\
211
static inline int TestClearPage##uname(struct page *page)		\
212
	{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
213

214 215 216 217
#define PAGEFLAG(uname, lname, policy)					\
	TESTPAGEFLAG(uname, lname, policy)				\
	SETPAGEFLAG(uname, lname, policy)				\
	CLEARPAGEFLAG(uname, lname, policy)
218

219 220 221 222
#define __PAGEFLAG(uname, lname, policy)				\
	TESTPAGEFLAG(uname, lname, policy)				\
	__SETPAGEFLAG(uname, lname, policy)				\
	__CLEARPAGEFLAG(uname, lname, policy)
223

224 225 226
#define TESTSCFLAG(uname, lname, policy)				\
	TESTSETFLAG(uname, lname, policy)				\
	TESTCLEARFLAG(uname, lname, policy)
227

228 229 230
#define TESTPAGEFLAG_FALSE(uname)					\
static inline int Page##uname(const struct page *page) { return 0; }

231 232 233 234 235 236 237 238 239
#define SETPAGEFLAG_NOOP(uname)						\
static inline void SetPage##uname(struct page *page) {  }

#define CLEARPAGEFLAG_NOOP(uname)					\
static inline void ClearPage##uname(struct page *page) {  }

#define __CLEARPAGEFLAG_NOOP(uname)					\
static inline void __ClearPage##uname(struct page *page) {  }

240 241 242
#define TESTSETFLAG_FALSE(uname)					\
static inline int TestSetPage##uname(struct page *page) { return 0; }

243 244 245
#define TESTCLEARFLAG_FALSE(uname)					\
static inline int TestClearPage##uname(struct page *page) { return 0; }

246 247 248 249 250 251
#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname)			\
	SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)

#define TESTSCFLAG_FALSE(uname)						\
	TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)

252
__PAGEFLAG(Locked, locked, PF_NO_TAIL)
253
PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND)
254 255 256
PAGEFLAG(Referenced, referenced, PF_HEAD)
	TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
	__SETPAGEFLAG(Referenced, referenced, PF_HEAD)
257 258
PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
	__CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
259 260 261
PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
	TESTCLEARFLAG(Active, active, PF_HEAD)
262 263
__PAGEFLAG(Slab, slab, PF_NO_TAIL)
__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
264
PAGEFLAG(Checked, checked, PF_NO_COMPOUND)	   /* Used by some filesystems */
265 266 267 268 269 270 271

/* Xen */
PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
	TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);

272 273
PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
	__CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
274 275 276
PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
	__CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
	__SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
277

278 279 280 281 282
/*
 * Private page markings that may be used by the filesystem that owns the page
 * for its own purposes.
 * - PG_private and PG_private_2 cause releasepage() and co to be invoked
 */
283 284 285 286 287
PAGEFLAG(Private, private, PF_ANY) __SETPAGEFLAG(Private, private, PF_ANY)
	__CLEARPAGEFLAG(Private, private, PF_ANY)
PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
	TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
288

289 290 291 292
/*
 * Only test-and-set exist for PG_writeback.  The unconditional operators are
 * risky: they bypass page accounting.
 */
293 294 295
TESTPAGEFLAG(Writeback, writeback, PF_NO_COMPOUND)
	TESTSCFLAG(Writeback, writeback, PF_NO_COMPOUND)
PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_COMPOUND)
296

297
/* PG_readahead is only used for reads; PG_reclaim is only for writes */
298 299 300 301
PAGEFLAG(Reclaim, reclaim, PF_NO_COMPOUND)
	TESTCLEARFLAG(Reclaim, reclaim, PF_NO_COMPOUND)
PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
	TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
302 303

#ifdef CONFIG_HIGHMEM
L
Linus Torvalds 已提交
304
/*
305 306
 * Must use a macro here due to header dependency issues. page_zone() is not
 * available at this point.
L
Linus Torvalds 已提交
307
 */
V
Vineet Gupta 已提交
308
#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
309
#else
310
PAGEFLAG_FALSE(HighMem)
311 312 313
#endif

#ifdef CONFIG_SWAP
314
PAGEFLAG(SwapCache, swapcache, PF_NO_COMPOUND)
315
#else
316
PAGEFLAG_FALSE(SwapCache)
317 318
#endif

319 320 321
PAGEFLAG(Unevictable, unevictable, PF_HEAD)
	__CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
	TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
N
Nick Piggin 已提交
322

H
Hugh Dickins 已提交
323
#ifdef CONFIG_MMU
324 325 326
PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
	__CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
	TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
L
Lee Schermerhorn 已提交
327
#else
328
PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
329
	TESTSCFLAG_FALSE(Mlocked)
L
Lee Schermerhorn 已提交
330 331
#endif

332
#ifdef CONFIG_ARCH_USES_PG_UNCACHED
333
PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
334
#else
335
PAGEFLAG_FALSE(Uncached)
336
#endif
L
Linus Torvalds 已提交
337

338
#ifdef CONFIG_MEMORY_FAILURE
339 340
PAGEFLAG(HWPoison, hwpoison, PF_ANY)
TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
341 342 343 344 345 346
#define __PG_HWPOISON (1UL << PG_hwpoison)
#else
PAGEFLAG_FALSE(HWPoison)
#define __PG_HWPOISON 0
#endif

347
#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
348 349 350 351
TESTPAGEFLAG(Young, young, PF_ANY)
SETPAGEFLAG(Young, young, PF_ANY)
TESTCLEARFLAG(Young, young, PF_ANY)
PAGEFLAG(Idle, idle, PF_ANY)
352 353
#endif

354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
/*
 * On an anonymous page mapped into a user virtual memory area,
 * page->mapping points to its anon_vma, not to a struct address_space;
 * with the PAGE_MAPPING_ANON bit set to distinguish it.  See rmap.h.
 *
 * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,
 * the PAGE_MAPPING_KSM bit may be set along with the PAGE_MAPPING_ANON bit;
 * and then page->mapping points, not to an anon_vma, but to a private
 * structure which KSM associates with that merged page.  See ksm.h.
 *
 * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is currently never used.
 *
 * Please note that, confusingly, "page_mapping" refers to the inode
 * address_space which maps the page from disk; whereas "page_mapped"
 * refers to user virtual address space into which the page is mapped.
 */
#define PAGE_MAPPING_ANON	1
#define PAGE_MAPPING_KSM	2
#define PAGE_MAPPING_FLAGS	(PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)

static inline int PageAnon(struct page *page)
{
376
	page = compound_head(page);
377 378 379 380 381 382 383 384 385 386 387 388
	return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
}

#ifdef CONFIG_KSM
/*
 * A KSM page is one of those write-protected "shared pages" or "merged pages"
 * which KSM maps into multiple mms, wherever identical anonymous page content
 * is found in VM_MERGEABLE vmas.  It's a PageAnon page, pointing not to any
 * anon_vma, but to that page's node of the stable tree.
 */
static inline int PageKsm(struct page *page)
{
389
	page = compound_head(page);
390 391 392 393 394 395 396
	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
				(PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
}
#else
TESTPAGEFLAG_FALSE(Ksm)
#endif

W
Wu Fengguang 已提交
397 398
u64 stable_page_flags(struct page *page);

N
Nick Piggin 已提交
399 400
static inline int PageUptodate(struct page *page)
{
401 402 403
	int ret;
	page = compound_head(page);
	ret = test_bit(PG_uptodate, &(page)->flags);
N
Nick Piggin 已提交
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419
	/*
	 * Must ensure that the data we read out of the page is loaded
	 * _after_ we've loaded page->flags to check for PageUptodate.
	 * We can skip the barrier if the page is not uptodate, because
	 * we wouldn't be reading anything from it.
	 *
	 * See SetPageUptodate() for the other side of the story.
	 */
	if (ret)
		smp_rmb();

	return ret;
}

static inline void __SetPageUptodate(struct page *page)
{
420
	VM_BUG_ON_PAGE(PageTail(page), page);
N
Nick Piggin 已提交
421
	smp_wmb();
422
	__set_bit(PG_uptodate, &page->flags);
N
Nick Piggin 已提交
423 424
}

425 426
static inline void SetPageUptodate(struct page *page)
{
427
	VM_BUG_ON_PAGE(PageTail(page), page);
N
Nick Piggin 已提交
428 429 430 431 432 433
	/*
	 * Memory barrier must be issued before setting the PG_uptodate bit,
	 * so that all previous stores issued in order to bring the page
	 * uptodate are actually visible before PageUptodate becomes true.
	 */
	smp_wmb();
434
	set_bit(PG_uptodate, &page->flags);
N
Nick Piggin 已提交
435 436
}

437
CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
L
Linus Torvalds 已提交
438

439
int test_clear_page_writeback(struct page *page);
440 441 442 443 444 445
int __test_set_page_writeback(struct page *page, bool keep_write);

#define test_set_page_writeback(page)			\
	__test_set_page_writeback(page, false)
#define test_set_page_writeback_keepwrite(page)	\
	__test_set_page_writeback(page, true)
L
Linus Torvalds 已提交
446

447 448 449 450
static inline void set_page_writeback(struct page *page)
{
	test_set_page_writeback(page);
}
L
Linus Torvalds 已提交
451

452 453 454 455 456
static inline void set_page_writeback_keepwrite(struct page *page)
{
	test_set_page_writeback_keepwrite(page);
}

457
__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
458

459
static inline void set_compound_head(struct page *page, struct page *head)
460
{
461
	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
462 463
}

464
static inline void clear_compound_head(struct page *page)
465
{
466
	WRITE_ONCE(page->compound_head, 0);
467
}
468

A
Andrea Arcangeli 已提交
469 470 471
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline void ClearPageCompound(struct page *page)
{
472 473
	BUG_ON(!PageHead(page));
	ClearPageHead(page);
A
Andrea Arcangeli 已提交
474 475 476
}
#endif

477
#define PG_head_mask ((1L << PG_head))
478

479 480 481
#ifdef CONFIG_HUGETLB_PAGE
int PageHuge(struct page *page);
int PageHeadHuge(struct page *page);
482
bool page_huge_active(struct page *page);
483 484 485
#else
TESTPAGEFLAG_FALSE(Huge)
TESTPAGEFLAG_FALSE(HeadHuge)
486 487 488 489 490

static inline bool page_huge_active(struct page *page)
{
	return 0;
}
491 492
#endif

493

494
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
495 496 497 498 499 500 501 502 503 504
/*
 * PageHuge() only returns true for hugetlbfs pages, but not for
 * normal or transparent huge pages.
 *
 * PageTransHuge() returns true for both transparent huge and
 * hugetlbfs pages, but not normal pages. PageTransHuge() can only be
 * called only in the core VM paths where hugetlbfs pages can't exist.
 */
static inline int PageTransHuge(struct page *page)
{
505
	VM_BUG_ON_PAGE(PageTail(page), page);
506 507 508
	return PageHead(page);
}

509 510 511 512 513
/*
 * PageTransCompound returns true for both transparent huge pages
 * and hugetlbfs pages, so it should only be called when it's known
 * that hugetlbfs pages aren't involved.
 */
514 515 516 517
static inline int PageTransCompound(struct page *page)
{
	return PageCompound(page);
}
518

519 520 521 522 523 524 525 526 527 528
/*
 * PageTransTail returns true for both transparent huge pages
 * and hugetlbfs pages, so it should only be called when it's known
 * that hugetlbfs pages aren't involved.
 */
static inline int PageTransTail(struct page *page)
{
	return PageTail(page);
}

529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
/*
 * PageDoubleMap indicates that the compound page is mapped with PTEs as well
 * as PMDs.
 *
 * This is required for optimization of rmap operations for THP: we can postpone
 * per small page mapcount accounting (and its overhead from atomic operations)
 * until the first PMD split.
 *
 * For the page PageDoubleMap means ->_mapcount in all sub-pages is offset up
 * by one. This reference will go away with last compound_mapcount.
 *
 * See also __split_huge_pmd_locked() and page_remove_anon_compound_rmap().
 */
static inline int PageDoubleMap(struct page *page)
{
	return PageHead(page) && test_bit(PG_double_map, &page[1].flags);
}

static inline int TestSetPageDoubleMap(struct page *page)
{
	VM_BUG_ON_PAGE(!PageHead(page), page);
	return test_and_set_bit(PG_double_map, &page[1].flags);
}

static inline int TestClearPageDoubleMap(struct page *page)
{
	VM_BUG_ON_PAGE(!PageHead(page), page);
	return test_and_clear_bit(PG_double_map, &page[1].flags);
}

559
#else
560 561 562
TESTPAGEFLAG_FALSE(TransHuge)
TESTPAGEFLAG_FALSE(TransCompound)
TESTPAGEFLAG_FALSE(TransTail)
563 564 565
TESTPAGEFLAG_FALSE(DoubleMap)
	TESTSETFLAG_FALSE(DoubleMap)
	TESTCLEARFLAG_FALSE(DoubleMap)
566 567
#endif

568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614
/*
 * PageBuddy() indicate that the page is free and in the buddy system
 * (see mm/page_alloc.c).
 *
 * PAGE_BUDDY_MAPCOUNT_VALUE must be <= -2 but better not too close to
 * -2 so that an underflow of the page_mapcount() won't be mistaken
 * for a genuine PAGE_BUDDY_MAPCOUNT_VALUE. -128 can be created very
 * efficiently by most CPU architectures.
 */
#define PAGE_BUDDY_MAPCOUNT_VALUE (-128)

static inline int PageBuddy(struct page *page)
{
	return atomic_read(&page->_mapcount) == PAGE_BUDDY_MAPCOUNT_VALUE;
}

static inline void __SetPageBuddy(struct page *page)
{
	VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page);
	atomic_set(&page->_mapcount, PAGE_BUDDY_MAPCOUNT_VALUE);
}

static inline void __ClearPageBuddy(struct page *page)
{
	VM_BUG_ON_PAGE(!PageBuddy(page), page);
	atomic_set(&page->_mapcount, -1);
}

#define PAGE_BALLOON_MAPCOUNT_VALUE (-256)

static inline int PageBalloon(struct page *page)
{
	return atomic_read(&page->_mapcount) == PAGE_BALLOON_MAPCOUNT_VALUE;
}

static inline void __SetPageBalloon(struct page *page)
{
	VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page);
	atomic_set(&page->_mapcount, PAGE_BALLOON_MAPCOUNT_VALUE);
}

static inline void __ClearPageBalloon(struct page *page)
{
	VM_BUG_ON_PAGE(!PageBalloon(page), page);
	atomic_set(&page->_mapcount, -1);
}

615 616 617 618 619 620
/*
 * If network-based swap is enabled, sl*b must keep track of whether pages
 * were allocated from pfmemalloc reserves.
 */
static inline int PageSlabPfmemalloc(struct page *page)
{
621
	VM_BUG_ON_PAGE(!PageSlab(page), page);
622 623 624 625 626
	return PageActive(page);
}

static inline void SetPageSlabPfmemalloc(struct page *page)
{
627
	VM_BUG_ON_PAGE(!PageSlab(page), page);
628 629 630 631 632
	SetPageActive(page);
}

static inline void __ClearPageSlabPfmemalloc(struct page *page)
{
633
	VM_BUG_ON_PAGE(!PageSlab(page), page);
634 635 636 637 638
	__ClearPageActive(page);
}

static inline void ClearPageSlabPfmemalloc(struct page *page)
{
639
	VM_BUG_ON_PAGE(!PageSlab(page), page);
640 641 642
	ClearPageActive(page);
}

H
Hugh Dickins 已提交
643
#ifdef CONFIG_MMU
644 645
#define __PG_MLOCKED		(1 << PG_mlocked)
#else
N
Nick Piggin 已提交
646
#define __PG_MLOCKED		0
L
Lee Schermerhorn 已提交
647 648
#endif

649 650 651 652
/*
 * Flags checked when a page is freed.  Pages being freed should not have
 * these flags set.  It they are, there is a problem.
 */
653
#define PAGE_FLAGS_CHECK_AT_FREE \
654 655
	(1 << PG_lru	 | 1 << PG_locked    | \
	 1 << PG_private | 1 << PG_private_2 | \
A
Andrea Arcangeli 已提交
656
	 1 << PG_writeback | 1 << PG_reserved | \
657
	 1 << PG_slab	 | 1 << PG_swapcache | 1 << PG_active | \
658
	 1 << PG_unevictable | __PG_MLOCKED)
659 660 661

/*
 * Flags checked when a page is prepped for return by the page allocator.
662
 * Pages being prepped should not have these flags set.  It they are set,
663
 * there has been a kernel bug or struct page corruption.
664 665 666
 *
 * __PG_HWPOISON is exceptional because it needs to be kept beyond page's
 * alloc-free cycle to prevent from reusing the page.
667
 */
668 669
#define PAGE_FLAGS_CHECK_AT_PREP	\
	(((1 << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
670

671 672
#define PAGE_FLAGS_PRIVATE				\
	(1 << PG_private | 1 << PG_private_2)
673 674 675 676 677 678 679
/**
 * page_has_private - Determine if page has private stuff
 * @page: The page to be checked
 *
 * Determine if a page has private stuff, indicating that release routines
 * should be invoked upon it.
 */
680 681 682 683 684
static inline int page_has_private(struct page *page)
{
	return !!(page->flags & PAGE_FLAGS_PRIVATE);
}

685 686 687 688
#undef PF_ANY
#undef PF_HEAD
#undef PF_NO_TAIL
#undef PF_NO_COMPOUND
689
#endif /* !__GENERATING_BOUNDS_H */
690

L
Linus Torvalds 已提交
691
#endif	/* PAGE_FLAGS_H */