page-flags.h 16.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * Macros for manipulating and testing page->flags
 */

#ifndef PAGE_FLAGS_H
#define PAGE_FLAGS_H

A
Andrew Morton 已提交
8
#include <linux/types.h>
9
#include <linux/bug.h>
10
#include <linux/mmdebug.h>
11
#ifndef __GENERATING_BOUNDS_H
12
#include <linux/mm_types.h>
13
#include <generated/bounds.h>
14
#endif /* !__GENERATING_BOUNDS_H */
A
Andrew Morton 已提交
15

L
Linus Torvalds 已提交
16 17 18 19 20 21
/*
 * Various page->flags bits:
 *
 * PG_reserved is set for special pages, which can never be swapped out. Some
 * of them might not even exist (eg empty_bad_page)...
 *
N
Nick Piggin 已提交
22 23 24
 * The PG_private bitflag is set on pagecache pages if they contain filesystem
 * specific data (which is normally at page->private). It can be used by
 * private allocations for its own usage.
L
Linus Torvalds 已提交
25
 *
N
Nick Piggin 已提交
26 27 28 29 30 31 32 33 34
 * During initiation of disk I/O, PG_locked is set. This bit is set before I/O
 * and cleared when writeback _starts_ or when read _completes_. PG_writeback
 * is set before writeback starts and cleared when it finishes.
 *
 * PG_locked also pins a page in pagecache, and blocks truncation of the file
 * while it is held.
 *
 * page_waitqueue(page) is a wait queue of all tasks waiting for the page
 * to become unlocked.
L
Linus Torvalds 已提交
35 36 37 38
 *
 * PG_uptodate tells whether the page's contents is valid.  When a read
 * completes, the page becomes uptodate, unless a disk I/O error happened.
 *
N
Nick Piggin 已提交
39 40
 * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
 * file-backed pagecache (see mm/vmscan.c).
L
Linus Torvalds 已提交
41 42 43 44 45 46 47 48 49 50 51
 *
 * PG_error is set to indicate that an I/O error occurred on this page.
 *
 * PG_arch_1 is an architecture specific page state bit.  The generic code
 * guarantees that this bit is cleared for a page when it first is entered into
 * the page cache.
 *
 * PG_highmem pages are not permanently mapped into the kernel virtual address
 * space, they need to be kmapped separately for doing IO on the pages.  The
 * struct page (these bits with information) are always mapped into kernel
 * address space...
N
Nick Piggin 已提交
52
 *
53 54 55
 * PG_hwpoison indicates that a page got corrupted in hardware and contains
 * data with incorrect ECC bits that triggered a machine check. Accessing is
 * not safe since it may cause another machine check. Don't touch!
L
Linus Torvalds 已提交
56 57 58 59
 */

/*
 * Don't use the *_dontuse flags.  Use the macros.  Otherwise you'll break
60 61 62 63 64 65 66
 * locked- and dirty-page accounting.
 *
 * The page flags field is split into two parts, the main flags area
 * which extends from the low bits upwards, and the fields area which
 * extends from the high bits downwards.
 *
 *  | FIELD | ... | FLAGS |
67 68
 *  N-1           ^       0
 *               (NR_PAGEFLAGS)
69
 *
70 71 72
 * The fields area is reserved for fields mapping zone, node (for NUMA) and
 * SPARSEMEM section (for variants of SPARSEMEM that require section ids like
 * SPARSEMEM_EXTREME with !SPARSEMEM_VMEMMAP).
L
Linus Torvalds 已提交
73
 */
74 75 76 77 78 79 80 81 82 83 84 85 86
enum pageflags {
	PG_locked,		/* Page is locked. Don't touch. */
	PG_error,
	PG_referenced,
	PG_uptodate,
	PG_dirty,
	PG_lru,
	PG_active,
	PG_slab,
	PG_owner_priv_1,	/* Owner use. If pagecache, fs may use*/
	PG_arch_1,
	PG_reserved,
	PG_private,		/* If pagecache, has fs-private data */
87
	PG_private_2,		/* If pagecache, has fs aux data */
88
	PG_writeback,		/* Page is under writeback */
89 90 91 92
#ifdef CONFIG_PAGEFLAGS_EXTENDED
	PG_head,		/* A head page */
	PG_tail,		/* A tail page */
#else
93
	PG_compound,		/* A compound page */
94
#endif
95 96 97
	PG_swapcache,		/* Swap page: swp_entry_t in private */
	PG_mappedtodisk,	/* Has blocks allocated on-disk */
	PG_reclaim,		/* To be reclaimed asap */
R
Rik van Riel 已提交
98
	PG_swapbacked,		/* Page is backed by RAM/swap */
L
Lee Schermerhorn 已提交
99
	PG_unevictable,		/* Page is "unevictable"  */
H
Hugh Dickins 已提交
100
#ifdef CONFIG_MMU
N
Nick Piggin 已提交
101
	PG_mlocked,		/* Page is vma mlocked */
L
Lee Schermerhorn 已提交
102
#endif
103
#ifdef CONFIG_ARCH_USES_PG_UNCACHED
104
	PG_uncached,		/* Page has been mapped as uncached */
105 106 107
#endif
#ifdef CONFIG_MEMORY_FAILURE
	PG_hwpoison,		/* hardware poisoned page. Don't touch */
A
Andrea Arcangeli 已提交
108 109 110
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	PG_compound_lock,
A
Andrew Morton 已提交
111
#endif
112 113 114 115 116
	__NR_PAGEFLAGS,

	/* Filesystems */
	PG_checked = PG_owner_priv_1,

117 118 119 120 121 122
	/* Two page bits are conscripted by FS-Cache to maintain local caching
	 * state.  These bits are set on pages belonging to the netfs's inodes
	 * when those inodes are being locally cached.
	 */
	PG_fscache = PG_private_2,	/* page backed by cache */

123 124 125
	/* XEN */
	PG_pinned = PG_owner_priv_1,
	PG_savepinned = PG_dirty,
126

127 128
	/* SLOB */
	PG_slob_free = PG_private,
129
};
L
Linus Torvalds 已提交
130

131 132
#ifndef __GENERATING_BOUNDS_H

133 134 135 136
/*
 * Macros to create function definitions for page flags
 */
#define TESTPAGEFLAG(uname, lname)					\
137
static inline int Page##uname(const struct page *page)			\
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
			{ return test_bit(PG_##lname, &page->flags); }

#define SETPAGEFLAG(uname, lname)					\
static inline void SetPage##uname(struct page *page)			\
			{ set_bit(PG_##lname, &page->flags); }

#define CLEARPAGEFLAG(uname, lname)					\
static inline void ClearPage##uname(struct page *page)			\
			{ clear_bit(PG_##lname, &page->flags); }

#define __SETPAGEFLAG(uname, lname)					\
static inline void __SetPage##uname(struct page *page)			\
			{ __set_bit(PG_##lname, &page->flags); }

#define __CLEARPAGEFLAG(uname, lname)					\
static inline void __ClearPage##uname(struct page *page)		\
			{ __clear_bit(PG_##lname, &page->flags); }

#define TESTSETFLAG(uname, lname)					\
static inline int TestSetPage##uname(struct page *page)			\
		{ return test_and_set_bit(PG_##lname, &page->flags); }

#define TESTCLEARFLAG(uname, lname)					\
static inline int TestClearPage##uname(struct page *page)		\
		{ return test_and_clear_bit(PG_##lname, &page->flags); }

164 165 166
#define __TESTCLEARFLAG(uname, lname)					\
static inline int __TestClearPage##uname(struct page *page)		\
		{ return __test_and_clear_bit(PG_##lname, &page->flags); }
167 168 169 170 171 172 173

#define PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname)		\
	SETPAGEFLAG(uname, lname) CLEARPAGEFLAG(uname, lname)

#define __PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname)		\
	__SETPAGEFLAG(uname, lname)  __CLEARPAGEFLAG(uname, lname)

174
#define PAGEFLAG_FALSE(uname) 						\
175
static inline int Page##uname(const struct page *page)			\
176 177
			{ return 0; }

178 179 180
#define TESTSCFLAG(uname, lname)					\
	TESTSETFLAG(uname, lname) TESTCLEARFLAG(uname, lname)

181 182 183 184 185 186 187 188 189 190 191 192
#define SETPAGEFLAG_NOOP(uname)						\
static inline void SetPage##uname(struct page *page) {  }

#define CLEARPAGEFLAG_NOOP(uname)					\
static inline void ClearPage##uname(struct page *page) {  }

#define __CLEARPAGEFLAG_NOOP(uname)					\
static inline void __ClearPage##uname(struct page *page) {  }

#define TESTCLEARFLAG_FALSE(uname)					\
static inline int TestClearPage##uname(struct page *page) { return 0; }

193 194 195
#define __TESTCLEARFLAG_FALSE(uname)					\
static inline int __TestClearPage##uname(struct page *page) { return 0; }

196 197
struct page;	/* forward declaration */

198
TESTPAGEFLAG(Locked, locked)
199
PAGEFLAG(Error, error) TESTCLEARFLAG(Error, error)
200
PAGEFLAG(Referenced, referenced) TESTCLEARFLAG(Referenced, referenced)
201
	__SETPAGEFLAG(Referenced, referenced)
202 203 204
PAGEFLAG(Dirty, dirty) TESTSCFLAG(Dirty, dirty) __CLEARPAGEFLAG(Dirty, dirty)
PAGEFLAG(LRU, lru) __CLEARPAGEFLAG(LRU, lru)
PAGEFLAG(Active, active) __CLEARPAGEFLAG(Active, active)
L
Lee Schermerhorn 已提交
205
	TESTCLEARFLAG(Active, active)
206
__PAGEFLAG(Slab, slab)
207 208 209
PAGEFLAG(Checked, checked)		/* Used by some filesystems */
PAGEFLAG(Pinned, pinned) TESTSCFLAG(Pinned, pinned)	/* Xen */
PAGEFLAG(SavePinned, savepinned);			/* Xen */
210
PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved)
R
Rik van Riel 已提交
211
PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked)
212
	__SETPAGEFLAG(SwapBacked, swapbacked)
213

214 215
__PAGEFLAG(SlobFree, slob_free)

216 217 218 219 220 221 222 223 224 225
/*
 * Private page markings that may be used by the filesystem that owns the page
 * for its own purposes.
 * - PG_private and PG_private_2 cause releasepage() and co to be invoked
 */
PAGEFLAG(Private, private) __SETPAGEFLAG(Private, private)
	__CLEARPAGEFLAG(Private, private)
PAGEFLAG(Private2, private_2) TESTSCFLAG(Private2, private_2)
PAGEFLAG(OwnerPriv1, owner_priv_1) TESTCLEARFLAG(OwnerPriv1, owner_priv_1)

226 227 228 229 230 231 232
/*
 * Only test-and-set exist for PG_writeback.  The unconditional operators are
 * risky: they bypass page accounting.
 */
TESTPAGEFLAG(Writeback, writeback) TESTSCFLAG(Writeback, writeback)
PAGEFLAG(MappedToDisk, mappedtodisk)

233
/* PG_readahead is only used for reads; PG_reclaim is only for writes */
234
PAGEFLAG(Reclaim, reclaim) TESTCLEARFLAG(Reclaim, reclaim)
235
PAGEFLAG(Readahead, reclaim) TESTCLEARFLAG(Readahead, reclaim)
236 237

#ifdef CONFIG_HIGHMEM
L
Linus Torvalds 已提交
238
/*
239 240
 * Must use a macro here due to header dependency issues. page_zone() is not
 * available at this point.
L
Linus Torvalds 已提交
241
 */
242
#define PageHighMem(__p) is_highmem(page_zone(__p))
243
#else
244
PAGEFLAG_FALSE(HighMem)
245 246 247 248 249
#endif

#ifdef CONFIG_SWAP
PAGEFLAG(SwapCache, swapcache)
#else
250
PAGEFLAG_FALSE(SwapCache)
251
	SETPAGEFLAG_NOOP(SwapCache) CLEARPAGEFLAG_NOOP(SwapCache)
252 253
#endif

L
Lee Schermerhorn 已提交
254 255
PAGEFLAG(Unevictable, unevictable) __CLEARPAGEFLAG(Unevictable, unevictable)
	TESTCLEARFLAG(Unevictable, unevictable)
N
Nick Piggin 已提交
256

H
Hugh Dickins 已提交
257
#ifdef CONFIG_MMU
N
Nick Piggin 已提交
258
PAGEFLAG(Mlocked, mlocked) __CLEARPAGEFLAG(Mlocked, mlocked)
259
	TESTSCFLAG(Mlocked, mlocked) __TESTCLEARFLAG(Mlocked, mlocked)
L
Lee Schermerhorn 已提交
260
#else
261 262
PAGEFLAG_FALSE(Mlocked) SETPAGEFLAG_NOOP(Mlocked)
	TESTCLEARFLAG_FALSE(Mlocked) __TESTCLEARFLAG_FALSE(Mlocked)
L
Lee Schermerhorn 已提交
263 264
#endif

265
#ifdef CONFIG_ARCH_USES_PG_UNCACHED
266
PAGEFLAG(Uncached, uncached)
267
#else
268
PAGEFLAG_FALSE(Uncached)
269
#endif
L
Linus Torvalds 已提交
270

271 272
#ifdef CONFIG_MEMORY_FAILURE
PAGEFLAG(HWPoison, hwpoison)
W
Wu Fengguang 已提交
273
TESTSCFLAG(HWPoison, hwpoison)
274 275 276 277 278 279
#define __PG_HWPOISON (1UL << PG_hwpoison)
#else
PAGEFLAG_FALSE(HWPoison)
#define __PG_HWPOISON 0
#endif

W
Wu Fengguang 已提交
280 281
u64 stable_page_flags(struct page *page);

N
Nick Piggin 已提交
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
static inline int PageUptodate(struct page *page)
{
	int ret = test_bit(PG_uptodate, &(page)->flags);

	/*
	 * Must ensure that the data we read out of the page is loaded
	 * _after_ we've loaded page->flags to check for PageUptodate.
	 * We can skip the barrier if the page is not uptodate, because
	 * we wouldn't be reading anything from it.
	 *
	 * See SetPageUptodate() for the other side of the story.
	 */
	if (ret)
		smp_rmb();

	return ret;
}

static inline void __SetPageUptodate(struct page *page)
{
	smp_wmb();
	__set_bit(PG_uptodate, &(page)->flags);
}

306 307
static inline void SetPageUptodate(struct page *page)
{
N
Nick Piggin 已提交
308 309 310 311 312 313 314 315 316
	/*
	 * Memory barrier must be issued before setting the PG_uptodate bit,
	 * so that all previous stores issued in order to bring the page
	 * uptodate are actually visible before PageUptodate becomes true.
	 */
	smp_wmb();
	set_bit(PG_uptodate, &(page)->flags);
}

317
CLEARPAGEFLAG(Uptodate, uptodate)
L
Linus Torvalds 已提交
318

319
extern void cancel_dirty_page(struct page *page, unsigned int account_size);
F
Fengguang Wu 已提交
320

321
int test_clear_page_writeback(struct page *page);
322 323 324 325 326 327
int __test_set_page_writeback(struct page *page, bool keep_write);

#define test_set_page_writeback(page)			\
	__test_set_page_writeback(page, false)
#define test_set_page_writeback_keepwrite(page)	\
	__test_set_page_writeback(page, true)
L
Linus Torvalds 已提交
328

329 330 331 332
static inline void set_page_writeback(struct page *page)
{
	test_set_page_writeback(page);
}
L
Linus Torvalds 已提交
333

334 335 336 337 338
static inline void set_page_writeback_keepwrite(struct page *page)
{
	test_set_page_writeback_keepwrite(page);
}

339 340 341 342 343
#ifdef CONFIG_PAGEFLAGS_EXTENDED
/*
 * System with lots of page flags available. This allows separate
 * flags for PageHead() and PageTail() checks of compound pages so that bit
 * tests can be used in performance sensitive paths. PageCompound is
344 345 346
 * generally not used in hot code paths except arch/powerpc/mm/init_64.c
 * and arch/powerpc/kvm/book3s_64_vio_hv.c which use it to detect huge pages
 * and avoid handling those in real mode.
347
 */
A
Andrea Arcangeli 已提交
348
__PAGEFLAG(Head, head) CLEARPAGEFLAG(Head, head)
349 350 351 352 353 354 355
__PAGEFLAG(Tail, tail)

static inline int PageCompound(struct page *page)
{
	return page->flags & ((1L << PG_head) | (1L << PG_tail));

}
A
Andrea Arcangeli 已提交
356 357 358 359 360 361 362
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline void ClearPageCompound(struct page *page)
{
	BUG_ON(!PageHead(page));
	ClearPageHead(page);
}
#endif
363 364 365

#define PG_head_mask ((1L << PG_head))

366 367 368 369 370 371 372
#else
/*
 * Reduce page flag use as much as possible by overlapping
 * compound page flags with the flags used for page cache pages. Possible
 * because PageCompound is always set for compound pages and not for
 * pages on the LRU and/or pagecache.
 */
373
TESTPAGEFLAG(Compound, compound)
374
__SETPAGEFLAG(Head, compound)  __CLEARPAGEFLAG(Head, compound)
L
Linus Torvalds 已提交
375

376
/*
377
 * PG_reclaim is used in combination with PG_compound to mark the
378 379 380 381
 * head and tail of a compound page. This saves one page flag
 * but makes it impossible to use compound pages for the page cache.
 * The PG_reclaim bit would have to be used for reclaim or readahead
 * if compound pages enter the page cache.
382 383 384
 *
 * PG_compound & PG_reclaim	=> Tail page
 * PG_compound & ~PG_reclaim	=> Head page
385
 */
386
#define PG_head_mask ((1L << PG_compound))
387 388
#define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim))

389 390 391 392 393
static inline int PageHead(struct page *page)
{
	return ((page->flags & PG_head_tail_mask) == PG_head_mask);
}

394 395 396 397
static inline int PageTail(struct page *page)
{
	return ((page->flags & PG_head_tail_mask) == PG_head_tail_mask);
}
398 399 400 401 402 403 404 405 406 407 408

static inline void __SetPageTail(struct page *page)
{
	page->flags |= PG_head_tail_mask;
}

static inline void __ClearPageTail(struct page *page)
{
	page->flags &= ~PG_head_tail_mask;
}

A
Andrea Arcangeli 已提交
409 410 411 412 413 414 415 416
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline void ClearPageCompound(struct page *page)
{
	BUG_ON((page->flags & PG_head_tail_mask) != (1 << PG_compound));
	clear_bit(PG_compound, &page->flags);
}
#endif

417
#endif /* !PAGEFLAGS_EXTENDED */
418

419
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
420 421 422 423 424 425 426 427 428 429
/*
 * PageHuge() only returns true for hugetlbfs pages, but not for
 * normal or transparent huge pages.
 *
 * PageTransHuge() returns true for both transparent huge and
 * hugetlbfs pages, but not normal pages. PageTransHuge() can only be
 * called only in the core VM paths where hugetlbfs pages can't exist.
 */
static inline int PageTransHuge(struct page *page)
{
430
	VM_BUG_ON_PAGE(PageTail(page), page);
431 432 433
	return PageHead(page);
}

434 435 436 437 438
/*
 * PageTransCompound returns true for both transparent huge pages
 * and hugetlbfs pages, so it should only be called when it's known
 * that hugetlbfs pages aren't involved.
 */
439 440 441 442
static inline int PageTransCompound(struct page *page)
{
	return PageCompound(page);
}
443

444 445 446 447 448 449 450 451 452 453
/*
 * PageTransTail returns true for both transparent huge pages
 * and hugetlbfs pages, so it should only be called when it's known
 * that hugetlbfs pages aren't involved.
 */
static inline int PageTransTail(struct page *page)
{
	return PageTail(page);
}

454
#else
455 456 457 458 459 460

static inline int PageTransHuge(struct page *page)
{
	return 0;
}

461 462 463 464
static inline int PageTransCompound(struct page *page)
{
	return 0;
}
465 466 467 468 469

static inline int PageTransTail(struct page *page)
{
	return 0;
}
470 471
#endif

472 473 474 475 476 477
/*
 * If network-based swap is enabled, sl*b must keep track of whether pages
 * were allocated from pfmemalloc reserves.
 */
static inline int PageSlabPfmemalloc(struct page *page)
{
478
	VM_BUG_ON_PAGE(!PageSlab(page), page);
479 480 481 482 483
	return PageActive(page);
}

static inline void SetPageSlabPfmemalloc(struct page *page)
{
484
	VM_BUG_ON_PAGE(!PageSlab(page), page);
485 486 487 488 489
	SetPageActive(page);
}

static inline void __ClearPageSlabPfmemalloc(struct page *page)
{
490
	VM_BUG_ON_PAGE(!PageSlab(page), page);
491 492 493 494 495
	__ClearPageActive(page);
}

static inline void ClearPageSlabPfmemalloc(struct page *page)
{
496
	VM_BUG_ON_PAGE(!PageSlab(page), page);
497 498 499
	ClearPageActive(page);
}

H
Hugh Dickins 已提交
500
#ifdef CONFIG_MMU
501 502
#define __PG_MLOCKED		(1 << PG_mlocked)
#else
N
Nick Piggin 已提交
503
#define __PG_MLOCKED		0
L
Lee Schermerhorn 已提交
504 505
#endif

A
Andrea Arcangeli 已提交
506 507 508 509 510 511
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define __PG_COMPOUND_LOCK		(1 << PG_compound_lock)
#else
#define __PG_COMPOUND_LOCK		0
#endif

512 513 514 515
/*
 * Flags checked when a page is freed.  Pages being freed should not have
 * these flags set.  It they are, there is a problem.
 */
516
#define PAGE_FLAGS_CHECK_AT_FREE \
517 518
	(1 << PG_lru	 | 1 << PG_locked    | \
	 1 << PG_private | 1 << PG_private_2 | \
A
Andrea Arcangeli 已提交
519
	 1 << PG_writeback | 1 << PG_reserved | \
520
	 1 << PG_slab	 | 1 << PG_swapcache | 1 << PG_active | \
A
Andrea Arcangeli 已提交
521 522
	 1 << PG_unevictable | __PG_MLOCKED | __PG_HWPOISON | \
	 __PG_COMPOUND_LOCK)
523 524 525

/*
 * Flags checked when a page is prepped for return by the page allocator.
526 527
 * Pages being prepped should not have any flags set.  It they are set,
 * there has been a kernel bug or struct page corruption.
528
 */
529
#define PAGE_FLAGS_CHECK_AT_PREP	((1 << NR_PAGEFLAGS) - 1)
530

531 532
#define PAGE_FLAGS_PRIVATE				\
	(1 << PG_private | 1 << PG_private_2)
533 534 535 536 537 538 539
/**
 * page_has_private - Determine if page has private stuff
 * @page: The page to be checked
 *
 * Determine if a page has private stuff, indicating that release routines
 * should be invoked upon it.
 */
540 541 542 543 544 545
static inline int page_has_private(struct page *page)
{
	return !!(page->flags & PAGE_FLAGS_PRIVATE);
}

#endif /* !__GENERATING_BOUNDS_H */
546

L
Linus Torvalds 已提交
547
#endif	/* PAGE_FLAGS_H */