mm.h 70.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
#ifndef _LINUX_MM_H
#define _LINUX_MM_H

#include <linux/errno.h>

#ifdef __KERNEL__

8
#include <linux/mmdebug.h>
L
Linus Torvalds 已提交
9
#include <linux/gfp.h>
10
#include <linux/bug.h>
L
Linus Torvalds 已提交
11 12 13
#include <linux/list.h>
#include <linux/mmzone.h>
#include <linux/rbtree.h>
14
#include <linux/atomic.h>
15
#include <linux/debug_locks.h>
16
#include <linux/mm_types.h>
17
#include <linux/range.h>
18
#include <linux/pfn.h>
A
Andrea Arcangeli 已提交
19
#include <linux/bit_spinlock.h>
20
#include <linux/shrinker.h>
21
#include <linux/resource.h>
22
#include <linux/page_ext.h>
L
Linus Torvalds 已提交
23 24 25

struct mempolicy;
struct anon_vma;
26
struct anon_vma_chain;
A
Alexey Dobriyan 已提交
27
struct file_ra_state;
A
Alexey Dobriyan 已提交
28
struct user_struct;
A
Alexey Dobriyan 已提交
29
struct writeback_control;
30
struct bdi_writeback;
L
Linus Torvalds 已提交
31

32
#ifndef CONFIG_NEED_MULTIPLE_NODES	/* Don't use mapnrs, do it properly */
L
Linus Torvalds 已提交
33
extern unsigned long max_mapnr;
34 35 36 37 38 39 40

static inline void set_max_mapnr(unsigned long limit)
{
	max_mapnr = limit;
}
#else
static inline void set_max_mapnr(unsigned long limit) { }
L
Linus Torvalds 已提交
41 42
#endif

43
extern unsigned long totalram_pages;
L
Linus Torvalds 已提交
44 45 46 47 48 49 50 51 52 53 54 55 56
extern void * high_memory;
extern int page_cluster;

#ifdef CONFIG_SYSCTL
extern int sysctl_legacy_va_layout;
#else
#define sysctl_legacy_va_layout 0
#endif

#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/processor.h>

57 58 59 60
#ifndef __pa_symbol
#define __pa_symbol(x)  __pa(RELOC_HIDE((unsigned long)(x), 0))
#endif

61 62 63 64 65 66 67 68 69 70 71
/*
 * To prevent common memory management code establishing
 * a zero page mapping on a read fault.
 * This macro should be defined within <asm/pgtable.h>.
 * s390 does this to prevent multiplexing of hardware bits
 * related to the physical page in case of virtualization.
 */
#ifndef mm_forbids_zeropage
#define mm_forbids_zeropage(X)	(0)
#endif

72
extern unsigned long sysctl_user_reserve_kbytes;
73
extern unsigned long sysctl_admin_reserve_kbytes;
74

75 76 77 78 79 80 81 82 83
extern int sysctl_overcommit_memory;
extern int sysctl_overcommit_ratio;
extern unsigned long sysctl_overcommit_kbytes;

extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *,
				    size_t *, loff_t *);
extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
				    size_t *, loff_t *);

L
Linus Torvalds 已提交
84 85
#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))

86 87 88
/* to align the pointer to the (next) page boundary */
#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)

89 90 91
/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
#define PAGE_ALIGNED(addr)	IS_ALIGNED((unsigned long)addr, PAGE_SIZE)

L
Linus Torvalds 已提交
92 93 94 95 96 97 98 99 100
/*
 * Linux kernel virtual memory manager primitives.
 * The idea being to have a "virtual" mm in the same way
 * we have a virtual fs - giving a cleaner interface to the
 * mm details, and allowing different kinds of memory mappings
 * (from shared memory to executable loading to arbitrary
 * mmap() functions).
 */

101 102
extern struct kmem_cache *vm_area_cachep;

L
Linus Torvalds 已提交
103
#ifndef CONFIG_MMU
104 105
extern struct rb_root nommu_region_tree;
extern struct rw_semaphore nommu_region_sem;
L
Linus Torvalds 已提交
106 107 108 109 110

extern unsigned int kobjsize(const void *objp);
#endif

/*
H
Hugh Dickins 已提交
111
 * vm_flags in vm_area_struct, see mm_types.h.
L
Linus Torvalds 已提交
112
 */
113 114
#define VM_NONE		0x00000000

L
Linus Torvalds 已提交
115 116 117 118 119
#define VM_READ		0x00000001	/* currently active flags */
#define VM_WRITE	0x00000002
#define VM_EXEC		0x00000004
#define VM_SHARED	0x00000008

120
/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
L
Linus Torvalds 已提交
121 122 123 124 125 126
#define VM_MAYREAD	0x00000010	/* limits for mprotect() etc */
#define VM_MAYWRITE	0x00000020
#define VM_MAYEXEC	0x00000040
#define VM_MAYSHARE	0x00000080

#define VM_GROWSDOWN	0x00000100	/* general info on the segment */
127
#define VM_UFFD_MISSING	0x00000200	/* missing pages tracking */
128
#define VM_PFNMAP	0x00000400	/* Page-ranges managed without "struct page", just pure PFN */
L
Linus Torvalds 已提交
129
#define VM_DENYWRITE	0x00000800	/* ETXTBSY on write attempts.. */
130
#define VM_UFFD_WP	0x00001000	/* wrprotect pages tracking */
L
Linus Torvalds 已提交
131 132 133 134 135 136 137 138 139 140 141

#define VM_LOCKED	0x00002000
#define VM_IO           0x00004000	/* Memory mapped I/O or similar */

					/* Used by sys_madvise() */
#define VM_SEQ_READ	0x00008000	/* App will access data sequentially */
#define VM_RAND_READ	0x00010000	/* App will not benefit from clustered reads */

#define VM_DONTCOPY	0x00020000      /* Do not copy this vma on fork */
#define VM_DONTEXPAND	0x00040000	/* Cannot expand with mremap() */
#define VM_ACCOUNT	0x00100000	/* Is a VM accounted object */
142
#define VM_NORESERVE	0x00200000	/* should the VM suppress accounting */
L
Linus Torvalds 已提交
143
#define VM_HUGETLB	0x00400000	/* Huge TLB Page VM */
144
#define VM_ARCH_1	0x01000000	/* Architecture-specific flag */
145
#define VM_ARCH_2	0x02000000
146
#define VM_DONTDUMP	0x04000000	/* Do not include in the core dump */
147

148 149 150 151 152 153
#ifdef CONFIG_MEM_SOFT_DIRTY
# define VM_SOFTDIRTY	0x08000000	/* Not soft dirty clean area */
#else
# define VM_SOFTDIRTY	0
#endif

J
Jared Hulbert 已提交
154
#define VM_MIXEDMAP	0x10000000	/* Can contain "struct page" and pure PFN pages */
155 156
#define VM_HUGEPAGE	0x20000000	/* MADV_HUGEPAGE marked this vma */
#define VM_NOHUGEPAGE	0x40000000	/* MADV_NOHUGEPAGE marked this vma */
H
Hugh Dickins 已提交
157
#define VM_MERGEABLE	0x80000000	/* KSM may merge identical pages */
L
Linus Torvalds 已提交
158

159 160 161 162 163 164
#if defined(CONFIG_X86)
# define VM_PAT		VM_ARCH_1	/* PAT reserves whole VMA at once (x86) */
#elif defined(CONFIG_PPC)
# define VM_SAO		VM_ARCH_1	/* Strong Access Ordering (powerpc) */
#elif defined(CONFIG_PARISC)
# define VM_GROWSUP	VM_ARCH_1
165 166
#elif defined(CONFIG_METAG)
# define VM_GROWSUP	VM_ARCH_1
167 168 169 170 171 172
#elif defined(CONFIG_IA64)
# define VM_GROWSUP	VM_ARCH_1
#elif !defined(CONFIG_MMU)
# define VM_MAPPED_COPY	VM_ARCH_1	/* T if mapped copy of data (nommu mmap) */
#endif

173 174 175 176 177
#if defined(CONFIG_X86)
/* MPX specific bounds table or bounds directory */
# define VM_MPX		VM_ARCH_2
#endif

178 179 180 181
#ifndef VM_GROWSUP
# define VM_GROWSUP	VM_NONE
#endif

182 183 184
/* Bits set in the VMA until the stack is in its final location */
#define VM_STACK_INCOMPLETE_SETUP	(VM_RAND_READ | VM_SEQ_READ)

L
Linus Torvalds 已提交
185 186 187 188 189 190 191 192 193 194
#ifndef VM_STACK_DEFAULT_FLAGS		/* arch can override this */
#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
#endif

#ifdef CONFIG_STACK_GROWSUP
#define VM_STACK_FLAGS	(VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
#else
#define VM_STACK_FLAGS	(VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
#endif

N
Nick Piggin 已提交
195
/*
196 197
 * Special vmas that are non-mergable, non-mlock()able.
 * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
N
Nick Piggin 已提交
198
 */
199
#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
N
Nick Piggin 已提交
200

201 202 203
/* This mask defines which mm->def_flags a process can inherit its parent */
#define VM_INIT_DEF_MASK	VM_NOHUGEPAGE

L
Linus Torvalds 已提交
204 205 206 207 208 209
/*
 * mapping from the currently active vm_flags protection bits (the
 * low four bits) to a page protection mask..
 */
extern pgprot_t protection_map[16];

N
Nick Piggin 已提交
210
#define FAULT_FLAG_WRITE	0x01	/* Fault was a write access */
211 212 213 214 215 216
#define FAULT_FLAG_MKWRITE	0x02	/* Fault was mkwrite of existing pte */
#define FAULT_FLAG_ALLOW_RETRY	0x04	/* Retry fault if blocking */
#define FAULT_FLAG_RETRY_NOWAIT	0x08	/* Don't drop mmap_sem and wait when retrying */
#define FAULT_FLAG_KILLABLE	0x10	/* The fault task is in SIGKILL killable region */
#define FAULT_FLAG_TRIED	0x20	/* Second try */
#define FAULT_FLAG_USER		0x40	/* The fault originated in userspace */
N
Nick Piggin 已提交
217

218
/*
N
Nick Piggin 已提交
219
 * vm_fault is filled by the the pagefault handler and passed to the vma's
N
Nick Piggin 已提交
220 221
 * ->fault function. The vma's ->fault is responsible for returning a bitmask
 * of VM_FAULT_xxx flags that give details about how the fault was handled.
222
 *
223
 * pgoff should be used in favour of virtual_address, if possible.
224
 */
N
Nick Piggin 已提交
225 226 227 228 229
struct vm_fault {
	unsigned int flags;		/* FAULT_FLAG_xxx flags */
	pgoff_t pgoff;			/* Logical page offset based on vma */
	void __user *virtual_address;	/* Faulting virtual address */

230
	struct page *cow_page;		/* Handler may choose to COW */
N
Nick Piggin 已提交
231
	struct page *page;		/* ->fault handlers should return a
N
Nick Piggin 已提交
232
					 * page here, unless VM_FAULT_NOPAGE
N
Nick Piggin 已提交
233
					 * is set (which is also implied by
N
Nick Piggin 已提交
234
					 * VM_FAULT_ERROR).
N
Nick Piggin 已提交
235
					 */
236 237 238 239
	/* for ->map_pages() only */
	pgoff_t max_pgoff;		/* map pages for offset from pgoff till
					 * max_pgoff inclusive */
	pte_t *pte;			/* pte entry associated with ->pgoff */
240
};
L
Linus Torvalds 已提交
241 242 243 244 245 246 247 248 249

/*
 * These are the virtual MM functions - opening of an area, closing and
 * unmapping it (needed to keep files on disk up-to-date etc), pointer
 * to the functions called when a no-page or a wp-page exception occurs. 
 */
struct vm_operations_struct {
	void (*open)(struct vm_area_struct * area);
	void (*close)(struct vm_area_struct * area);
250
	int (*mremap)(struct vm_area_struct * area);
N
Nick Piggin 已提交
251
	int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
M
Matthew Wilcox 已提交
252 253
	int (*pmd_fault)(struct vm_area_struct *, unsigned long address,
						pmd_t *, unsigned int flags);
254
	void (*map_pages)(struct vm_area_struct *vma, struct vm_fault *vmf);
255 256 257

	/* notification that a previously read-only page is about to become
	 * writable, if an error is returned it will cause a SIGBUS */
258
	int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
259

260 261 262
	/* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */
	int (*pfn_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);

263 264 265 266 267
	/* called by access_process_vm when get_user_pages() fails, typically
	 * for use by special VMAs that can switch between memory and hardware
	 */
	int (*access)(struct vm_area_struct *vma, unsigned long addr,
		      void *buf, int len, int write);
268 269 270 271 272 273

	/* Called by the /proc/PID/maps code to ask the vma whether it
	 * has a special name.  Returning non-NULL will also cause this
	 * vma to be dumped unconditionally. */
	const char *(*name)(struct vm_area_struct *vma);

L
Linus Torvalds 已提交
274
#ifdef CONFIG_NUMA
275 276 277 278 279 280 281
	/*
	 * set_policy() op must add a reference to any non-NULL @new mempolicy
	 * to hold the policy upon return.  Caller should pass NULL @new to
	 * remove a policy and fall back to surrounding context--i.e. do not
	 * install a MPOL_DEFAULT policy, nor the task or system default
	 * mempolicy.
	 */
L
Linus Torvalds 已提交
282
	int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
283 284 285 286 287 288 289 290 291 292 293

	/*
	 * get_policy() op must add reference [mpol_get()] to any policy at
	 * (vma,addr) marked as MPOL_SHARED.  The shared policy infrastructure
	 * in mm/mempolicy.c will do this automatically.
	 * get_policy() must NOT add a ref if the policy at (vma,addr) is not
	 * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
	 * If no [shared/vma] mempolicy exists at the addr, get_policy() op
	 * must return NULL--i.e., do not "fallback" to task or system default
	 * policy.
	 */
L
Linus Torvalds 已提交
294 295 296
	struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
					unsigned long addr);
#endif
297 298 299 300 301 302 303
	/*
	 * Called by vm_normal_page() for special PTEs to find the
	 * page for @addr.  This is useful if the default behavior
	 * (using pte_page()) would not find the correct page.
	 */
	struct page *(*find_special_page)(struct vm_area_struct *vma,
					  unsigned long addr);
L
Linus Torvalds 已提交
304 305 306 307 308
};

struct mmu_gather;
struct inode;

A
Andrew Morton 已提交
309 310
#define page_private(page)		((page)->private)
#define set_page_private(page, v)	((page)->private = (v))
H
Hugh Dickins 已提交
311

L
Linus Torvalds 已提交
312 313 314 315 316
/*
 * FIXME: take this include out, include page-flags.h in
 * files which need it (119 of them)
 */
#include <linux/page-flags.h>
317
#include <linux/huge_mm.h>
L
Linus Torvalds 已提交
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332

/*
 * Methods to modify the page usage count.
 *
 * What counts for a page usage:
 * - cache mapping   (page->mapping)
 * - private data    (page->private)
 * - page mapped in a task's page tables, each mapping
 *   is counted separately
 *
 * Also, many kernel routines increase the page count before a critical
 * routine so they can be sure the page doesn't go away from under them.
 */

/*
N
Nick Piggin 已提交
333
 * Drop a ref, return true if the refcount fell to zero (the page has no users)
L
Linus Torvalds 已提交
334
 */
335 336
static inline int put_page_testzero(struct page *page)
{
337
	VM_BUG_ON_PAGE(atomic_read(&page->_count) == 0, page);
338
	return atomic_dec_and_test(&page->_count);
339
}
L
Linus Torvalds 已提交
340 341

/*
342 343
 * Try to grab a ref unless the page has a refcount of zero, return false if
 * that is the case.
344 345
 * This can be called when MMU is off so it must not access
 * any of the virtual mappings.
L
Linus Torvalds 已提交
346
 */
347 348
static inline int get_page_unless_zero(struct page *page)
{
349
	return atomic_inc_not_zero(&page->_count);
350
}
L
Linus Torvalds 已提交
351

352
extern int page_is_ram(unsigned long pfn);
353
extern int region_is_ram(resource_size_t phys_addr, unsigned long size);
354

355
/* Support for virtually mapped pages */
356 357
struct page *vmalloc_to_page(const void *addr);
unsigned long vmalloc_to_pfn(const void *addr);
358

359 360 361 362 363 364
/*
 * Determine if an address is within the vmalloc range
 *
 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
 * is no special casing required.
 */
365 366
static inline int is_vmalloc_addr(const void *x)
{
367
#ifdef CONFIG_MMU
368 369 370
	unsigned long addr = (unsigned long)x;

	return addr >= VMALLOC_START && addr < VMALLOC_END;
371 372
#else
	return 0;
373
#endif
374
}
375 376 377
#ifdef CONFIG_MMU
extern int is_vmalloc_or_module_addr(const void *x);
#else
378
static inline int is_vmalloc_or_module_addr(const void *x)
379 380 381 382
{
	return 0;
}
#endif
383

A
Al Viro 已提交
384 385
extern void kvfree(const void *addr);

A
Andrea Arcangeli 已提交
386 387 388
static inline void compound_lock(struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
389
	VM_BUG_ON_PAGE(PageSlab(page), page);
A
Andrea Arcangeli 已提交
390 391 392 393 394 395 396
	bit_spin_lock(PG_compound_lock, &page->flags);
#endif
}

static inline void compound_unlock(struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
397
	VM_BUG_ON_PAGE(PageSlab(page), page);
A
Andrea Arcangeli 已提交
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
	bit_spin_unlock(PG_compound_lock, &page->flags);
#endif
}

static inline unsigned long compound_lock_irqsave(struct page *page)
{
	unsigned long uninitialized_var(flags);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	local_irq_save(flags);
	compound_lock(page);
#endif
	return flags;
}

static inline void compound_unlock_irqrestore(struct page *page,
					      unsigned long flags)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	compound_unlock(page);
	local_irq_restore(flags);
#endif
}

421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
static inline struct page *compound_head_by_tail(struct page *tail)
{
	struct page *head = tail->first_page;

	/*
	 * page->first_page may be a dangling pointer to an old
	 * compound page, so recheck that it is still a tail
	 * page before returning.
	 */
	smp_rmb();
	if (likely(PageTail(tail)))
		return head;
	return tail;
}

436 437 438 439 440 441
/*
 * Since either compound page could be dismantled asynchronously in THP
 * or we access asynchronously arbitrary positioned struct page, there
 * would be tail flag race. To handle this race, we should call
 * smp_rmb() before checking tail flag. compound_head_by_tail() did it.
 */
442 443
static inline struct page *compound_head(struct page *page)
{
444 445
	if (unlikely(PageTail(page)))
		return compound_head_by_tail(page);
446 447 448
	return page;
}

449 450 451 452 453 454 455 456 457 458 459 460
/*
 * If we access compound page synchronously such as access to
 * allocated page, there is no need to handle tail flag race, so we can
 * check tail flag directly without any synchronization primitive.
 */
static inline struct page *compound_head_fast(struct page *page)
{
	if (unlikely(PageTail(page)))
		return page->first_page;
	return page;
}

461 462 463 464 465
/*
 * The atomic page->_mapcount, starts from -1: so that transitions
 * both from it and to it can be tracked, using atomic_inc_and_test
 * and atomic_add_negative(-1).
 */
466
static inline void page_mapcount_reset(struct page *page)
467 468 469 470 471 472
{
	atomic_set(&(page)->_mapcount, -1);
}

static inline int page_mapcount(struct page *page)
{
473 474
	VM_BUG_ON_PAGE(PageSlab(page), page);
	return atomic_read(&page->_mapcount) + 1;
475 476
}

H
Hugh Dickins 已提交
477
static inline int page_count(struct page *page)
L
Linus Torvalds 已提交
478
{
479
	return atomic_read(&compound_head(page)->_count);
L
Linus Torvalds 已提交
480 481
}

482 483
static inline bool __compound_tail_refcounted(struct page *page)
{
484
	return PageAnon(page) && !PageSlab(page) && !PageHeadHuge(page);
485 486 487 488 489 490 491 492 493 494 495 496
}

/*
 * This takes a head page as parameter and tells if the
 * tail page reference counting can be skipped.
 *
 * For this to be safe, PageSlab and PageHeadHuge must remain true on
 * any given page where they return true here, until all tail pins
 * have been released.
 */
static inline bool compound_tail_refcounted(struct page *page)
{
497
	VM_BUG_ON_PAGE(!PageHead(page), page);
498 499 500
	return __compound_tail_refcounted(page);
}

A
Andrea Arcangeli 已提交
501 502 503
static inline void get_huge_page_tail(struct page *page)
{
	/*
504
	 * __split_huge_page_refcount() cannot run from under us.
A
Andrea Arcangeli 已提交
505
	 */
506 507 508
	VM_BUG_ON_PAGE(!PageTail(page), page);
	VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
	VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page);
509
	if (compound_tail_refcounted(page->first_page))
510
		atomic_inc(&page->_mapcount);
A
Andrea Arcangeli 已提交
511 512
}

513 514
extern bool __get_page_tail(struct page *page);

L
Linus Torvalds 已提交
515 516
static inline void get_page(struct page *page)
{
517 518 519
	if (unlikely(PageTail(page)))
		if (likely(__get_page_tail(page)))
			return;
520 521
	/*
	 * Getting a normal page or the head of a compound page
522
	 * requires to already have an elevated page->_count.
523
	 */
524
	VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page);
L
Linus Torvalds 已提交
525 526 527
	atomic_inc(&page->_count);
}

528 529 530
static inline struct page *virt_to_head_page(const void *x)
{
	struct page *page = virt_to_page(x);
531 532 533 534 535 536 537 538

	/*
	 * We don't need to worry about synchronization of tail flag
	 * when we call virt_to_head_page() since it is only called for
	 * already allocated page and this page won't be freed until
	 * this virt_to_head_page() is finished. So use _fast variant.
	 */
	return compound_head_fast(page);
539 540
}

541 542 543 544 545 546 547 548 549
/*
 * Setup the page count before being freed into the page allocator for
 * the first time (boot or memory hotplug)
 */
static inline void init_page_count(struct page *page)
{
	atomic_set(&page->_count, 1);
}

L
Linus Torvalds 已提交
550
void put_page(struct page *page);
551
void put_pages_list(struct list_head *pages);
L
Linus Torvalds 已提交
552

N
Nick Piggin 已提交
553
void split_page(struct page *page, unsigned int order);
554
int split_free_page(struct page *page);
N
Nick Piggin 已提交
555

556 557 558 559 560 561 562 563 564
/*
 * Compound pages have a destructor function.  Provide a
 * prototype for that function and accessor functions.
 * These are _only_ valid on the head of a PG_compound page.
 */

static inline void set_compound_page_dtor(struct page *page,
						compound_page_dtor *dtor)
{
565
	page[1].compound_dtor = dtor;
566 567 568 569
}

static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
{
570
	return page[1].compound_dtor;
571 572
}

573 574
static inline int compound_order(struct page *page)
{
575
	if (!PageHead(page))
576
		return 0;
577
	return page[1].compound_order;
578 579 580 581
}

static inline void set_compound_order(struct page *page, unsigned long order)
{
582
	page[1].compound_order = order;
583 584
}

585
#ifdef CONFIG_MMU
A
Andrea Arcangeli 已提交
586 587 588 589 590 591 592 593 594 595 596 597
/*
 * Do pte_mkwrite, but only if the vma says VM_WRITE.  We do this when
 * servicing faults for write access.  In the normal case, do always want
 * pte_mkwrite.  But get_user_pages can cause write faults for mappings
 * that do not have writing enabled, when used by access_process_vm.
 */
static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
{
	if (likely(vma->vm_flags & VM_WRITE))
		pte = pte_mkwrite(pte);
	return pte;
}
598 599 600

void do_set_pte(struct vm_area_struct *vma, unsigned long address,
		struct page *page, pte_t *pte, bool write, bool anon);
601
#endif
A
Andrea Arcangeli 已提交
602

L
Linus Torvalds 已提交
603 604 605 606 607 608 609
/*
 * Multiple processes may "see" the same page. E.g. for untouched
 * mappings of /dev/null, all processes see the same page full of
 * zeroes, and text pages of executables and shared libraries have
 * only one copy in memory, at most, normally.
 *
 * For the non-reserved pages, page_count(page) denotes a reference count.
610 611
 *   page_count() == 0 means the page is free. page->lru is then used for
 *   freelist management in the buddy allocator.
N
Nick Piggin 已提交
612
 *   page_count() > 0  means the page has been allocated.
L
Linus Torvalds 已提交
613
 *
N
Nick Piggin 已提交
614 615 616 617 618
 * Pages are allocated by the slab allocator in order to provide memory
 * to kmalloc and kmem_cache_alloc. In this case, the management of the
 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
 * unless a particular usage is carefully commented. (the responsibility of
 * freeing the kmalloc memory is the caller's, of course).
L
Linus Torvalds 已提交
619
 *
N
Nick Piggin 已提交
620 621 622 623 624 625 626 627 628
 * A page may be used by anyone else who does a __get_free_page().
 * In this case, page_count still tracks the references, and should only
 * be used through the normal accessor functions. The top bits of page->flags
 * and page->virtual store page management information, but all other fields
 * are unused and could be used privately, carefully. The management of this
 * page is the responsibility of the one who allocated it, and those who have
 * subsequently been given references to it.
 *
 * The other pages (we may call them "pagecache pages") are completely
L
Linus Torvalds 已提交
629 630 631
 * managed by the Linux memory manager: I/O, buffers, swapping etc.
 * The following discussion applies only to them.
 *
N
Nick Piggin 已提交
632 633 634 635
 * A pagecache page contains an opaque `private' member, which belongs to the
 * page's address_space. Usually, this is the address of a circular list of
 * the page's disk buffers. PG_private must be set to tell the VM to call
 * into the filesystem to release these pages.
L
Linus Torvalds 已提交
636
 *
N
Nick Piggin 已提交
637 638 639
 * A page may belong to an inode's memory mapping. In this case, page->mapping
 * is the pointer to the inode, and page->index is the file offset of the page,
 * in units of PAGE_CACHE_SIZE.
L
Linus Torvalds 已提交
640
 *
N
Nick Piggin 已提交
641 642 643
 * If pagecache pages are not associated with an inode, they are said to be
 * anonymous pages. These may become associated with the swapcache, and in that
 * case PG_swapcache is set, and page->private is an offset into the swapcache.
L
Linus Torvalds 已提交
644
 *
N
Nick Piggin 已提交
645 646 647
 * In either case (swapcache or inode backed), the pagecache itself holds one
 * reference to the page. Setting PG_private should also increment the
 * refcount. The each user mapping also has a reference to the page.
L
Linus Torvalds 已提交
648
 *
N
Nick Piggin 已提交
649 650 651 652
 * The pagecache pages are stored in a per-mapping radix tree, which is
 * rooted at mapping->page_tree, and indexed by offset.
 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
 * lists, we instead now tag pages as dirty/writeback in the radix tree.
L
Linus Torvalds 已提交
653
 *
N
Nick Piggin 已提交
654
 * All pagecache pages may be subject to I/O:
L
Linus Torvalds 已提交
655 656
 * - inode pages may need to be read from disk,
 * - inode pages which have been modified and are MAP_SHARED may need
N
Nick Piggin 已提交
657 658 659 660
 *   to be written back to the inode on disk,
 * - anonymous pages (including MAP_PRIVATE file mappings) which have been
 *   modified may need to be swapped out to swap space and (later) to be read
 *   back into memory.
L
Linus Torvalds 已提交
661 662 663 664 665 666
 */

/*
 * The zone field is never updated after free_area_init_core()
 * sets it, so none of the operations on it need to be atomic.
 */
667

668
/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
669
#define SECTIONS_PGOFF		((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
A
Andy Whitcroft 已提交
670 671
#define NODES_PGOFF		(SECTIONS_PGOFF - NODES_WIDTH)
#define ZONES_PGOFF		(NODES_PGOFF - ZONES_WIDTH)
672
#define LAST_CPUPID_PGOFF	(ZONES_PGOFF - LAST_CPUPID_WIDTH)
A
Andy Whitcroft 已提交
673

674
/*
L
Lucas De Marchi 已提交
675
 * Define the bit shifts to access each section.  For non-existent
676 677 678
 * sections we define the shift as 0; that plus a 0 mask ensures
 * the compiler will optimise away reference to them.
 */
A
Andy Whitcroft 已提交
679 680 681
#define SECTIONS_PGSHIFT	(SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
#define NODES_PGSHIFT		(NODES_PGOFF * (NODES_WIDTH != 0))
#define ZONES_PGSHIFT		(ZONES_PGOFF * (ZONES_WIDTH != 0))
682
#define LAST_CPUPID_PGSHIFT	(LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
683

684 685
/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
#ifdef NODE_NOT_IN_PAGE_FLAGS
686
#define ZONEID_SHIFT		(SECTIONS_SHIFT + ZONES_SHIFT)
687 688
#define ZONEID_PGOFF		((SECTIONS_PGOFF < ZONES_PGOFF)? \
						SECTIONS_PGOFF : ZONES_PGOFF)
A
Andy Whitcroft 已提交
689
#else
690
#define ZONEID_SHIFT		(NODES_SHIFT + ZONES_SHIFT)
691 692
#define ZONEID_PGOFF		((NODES_PGOFF < ZONES_PGOFF)? \
						NODES_PGOFF : ZONES_PGOFF)
693 694
#endif

695
#define ZONEID_PGSHIFT		(ZONEID_PGOFF * (ZONEID_SHIFT != 0))
696

697 698
#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
699 700
#endif

A
Andy Whitcroft 已提交
701 702 703
#define ZONES_MASK		((1UL << ZONES_WIDTH) - 1)
#define NODES_MASK		((1UL << NODES_WIDTH) - 1)
#define SECTIONS_MASK		((1UL << SECTIONS_WIDTH) - 1)
704
#define LAST_CPUPID_MASK	((1UL << LAST_CPUPID_SHIFT) - 1)
705
#define ZONEID_MASK		((1UL << ZONEID_SHIFT) - 1)
706

I
Ian Campbell 已提交
707
static inline enum zone_type page_zonenum(const struct page *page)
L
Linus Torvalds 已提交
708
{
709
	return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
L
Linus Torvalds 已提交
710 711
}

C
Cody P Schafer 已提交
712 713 714 715
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
#define SECTION_IN_PAGE_FLAGS
#endif

716
/*
717 718 719 720 721 722
 * The identification function is mainly used by the buddy allocator for
 * determining if two pages could be buddies. We are not really identifying
 * the zone since we could be using the section number id if we do not have
 * node id available in page flags.
 * We only guarantee that it will return the same value for two combinable
 * pages in a zone.
723
 */
724 725
static inline int page_zone_id(struct page *page)
{
726
	return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
727 728
}

729
static inline int zone_to_nid(struct zone *zone)
730
{
731 732 733 734 735
#ifdef CONFIG_NUMA
	return zone->node;
#else
	return 0;
#endif
736 737
}

738
#ifdef NODE_NOT_IN_PAGE_FLAGS
I
Ian Campbell 已提交
739
extern int page_to_nid(const struct page *page);
740
#else
I
Ian Campbell 已提交
741
static inline int page_to_nid(const struct page *page)
A
Andy Whitcroft 已提交
742
{
743
	return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
A
Andy Whitcroft 已提交
744
}
745 746
#endif

747
#ifdef CONFIG_NUMA_BALANCING
748
static inline int cpu_pid_to_cpupid(int cpu, int pid)
749
{
750
	return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
751 752
}

753
static inline int cpupid_to_pid(int cpupid)
754
{
755
	return cpupid & LAST__PID_MASK;
756
}
757

758
static inline int cpupid_to_cpu(int cpupid)
759
{
760
	return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
761 762
}

763
static inline int cpupid_to_nid(int cpupid)
764
{
765
	return cpu_to_node(cpupid_to_cpu(cpupid));
766 767
}

768
static inline bool cpupid_pid_unset(int cpupid)
769
{
770
	return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
771 772
}

773
static inline bool cpupid_cpu_unset(int cpupid)
774
{
775
	return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
776 777
}

778 779 780 781 782 783
static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
{
	return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
}

#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
784 785
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
786
{
787
	return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
788
}
789 790 791 792 793 794

static inline int page_cpupid_last(struct page *page)
{
	return page->_last_cpupid;
}
static inline void page_cpupid_reset_last(struct page *page)
795
{
796
	page->_last_cpupid = -1 & LAST_CPUPID_MASK;
797 798
}
#else
799
static inline int page_cpupid_last(struct page *page)
800
{
801
	return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
802 803
}

804
extern int page_cpupid_xchg_last(struct page *page, int cpupid);
805

806
static inline void page_cpupid_reset_last(struct page *page)
807
{
808
	int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;
M
Mel Gorman 已提交
809

810 811
	page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
	page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
812
}
813 814 815
#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
#else /* !CONFIG_NUMA_BALANCING */
static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
816
{
817
	return page_to_nid(page); /* XXX */
818 819
}

820
static inline int page_cpupid_last(struct page *page)
821
{
822
	return page_to_nid(page); /* XXX */
823 824
}

825
static inline int cpupid_to_nid(int cpupid)
826 827 828 829
{
	return -1;
}

830
static inline int cpupid_to_pid(int cpupid)
831 832 833 834
{
	return -1;
}

835
static inline int cpupid_to_cpu(int cpupid)
836 837 838 839
{
	return -1;
}

840 841 842 843 844 845
static inline int cpu_pid_to_cpupid(int nid, int pid)
{
	return -1;
}

static inline bool cpupid_pid_unset(int cpupid)
846 847 848 849
{
	return 1;
}

850
static inline void page_cpupid_reset_last(struct page *page)
851 852
{
}
853 854 855 856 857

static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
{
	return false;
}
858
#endif /* CONFIG_NUMA_BALANCING */
859

I
Ian Campbell 已提交
860
static inline struct zone *page_zone(const struct page *page)
861 862 863 864
{
	return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
}

C
Cody P Schafer 已提交
865
#ifdef SECTION_IN_PAGE_FLAGS
866 867 868 869 870 871
static inline void set_page_section(struct page *page, unsigned long section)
{
	page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
	page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
}

872
static inline unsigned long page_to_section(const struct page *page)
A
Andy Whitcroft 已提交
873 874 875
{
	return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
}
876
#endif
A
Andy Whitcroft 已提交
877

878
static inline void set_page_zone(struct page *page, enum zone_type zone)
879 880 881 882
{
	page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
	page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
}
883

884 885 886 887
static inline void set_page_node(struct page *page, unsigned long node)
{
	page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
	page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
L
Linus Torvalds 已提交
888
}
889

890
static inline void set_page_links(struct page *page, enum zone_type zone,
A
Andy Whitcroft 已提交
891
	unsigned long node, unsigned long pfn)
L
Linus Torvalds 已提交
892
{
893 894
	set_page_zone(page, zone);
	set_page_node(page, node);
C
Cody P Schafer 已提交
895
#ifdef SECTION_IN_PAGE_FLAGS
A
Andy Whitcroft 已提交
896
	set_page_section(page, pfn_to_section_nr(pfn));
897
#endif
L
Linus Torvalds 已提交
898 899
}

900 901 902 903 904
/*
 * Some inline functions in vmstat.h depend on page_zone()
 */
#include <linux/vmstat.h>

I
Ian Campbell 已提交
905
static __always_inline void *lowmem_page_address(const struct page *page)
L
Linus Torvalds 已提交
906
{
907
	return __va(PFN_PHYS(page_to_pfn(page)));
L
Linus Torvalds 已提交
908 909 910 911 912 913 914
}

#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
#define HASHED_PAGE_VIRTUAL
#endif

#if defined(WANT_PAGE_VIRTUAL)
915 916 917 918 919 920 921 922
static inline void *page_address(const struct page *page)
{
	return page->virtual;
}
static inline void set_page_address(struct page *page, void *address)
{
	page->virtual = address;
}
L
Linus Torvalds 已提交
923 924 925 926
#define page_address_init()  do { } while(0)
#endif

#if defined(HASHED_PAGE_VIRTUAL)
927
void *page_address(const struct page *page);
L
Linus Torvalds 已提交
928 929 930 931 932 933 934 935 936 937
void set_page_address(struct page *page, void *virtual);
void page_address_init(void);
#endif

#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
#define page_address(page) lowmem_page_address(page)
#define set_page_address(page, address)  do { } while(0)
#define page_address_init()  do { } while(0)
#endif

938 939
extern void *page_rmapping(struct page *page);
extern struct anon_vma *page_anon_vma(struct page *page);
S
Shaohua Li 已提交
940
extern struct address_space *page_mapping(struct page *page);
L
Linus Torvalds 已提交
941

942 943 944 945 946 947 948 949 950 951 952
extern struct address_space *__page_file_mapping(struct page *);

static inline
struct address_space *page_file_mapping(struct page *page)
{
	if (unlikely(PageSwapCache(page)))
		return __page_file_mapping(page);

	return page->mapping;
}

L
Linus Torvalds 已提交
953 954 955 956 957 958 959
/*
 * Return the pagecache index of the passed page.  Regular pagecache pages
 * use ->index whereas swapcache pages use ->private
 */
static inline pgoff_t page_index(struct page *page)
{
	if (unlikely(PageSwapCache(page)))
H
Hugh Dickins 已提交
960
		return page_private(page);
L
Linus Torvalds 已提交
961 962 963
	return page->index;
}

964 965 966 967 968 969 970 971 972 973 974 975 976 977
extern pgoff_t __page_file_index(struct page *page);

/*
 * Return the file index of the page. Regular pagecache pages use ->index
 * whereas swapcache pages use swp_offset(->private)
 */
static inline pgoff_t page_file_index(struct page *page)
{
	if (unlikely(PageSwapCache(page)))
		return __page_file_index(page);

	return page->index;
}

L
Linus Torvalds 已提交
978 979 980 981 982 983 984 985
/*
 * Return true if this page is mapped into pagetables.
 */
static inline int page_mapped(struct page *page)
{
	return atomic_read(&(page)->_mapcount) >= 0;
}

986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
/*
 * Return true only if the page has been allocated with
 * ALLOC_NO_WATERMARKS and the low watermark was not
 * met implying that the system is under some pressure.
 */
static inline bool page_is_pfmemalloc(struct page *page)
{
	/*
	 * Page index cannot be this large so this must be
	 * a pfmemalloc page.
	 */
	return page->index == -1UL;
}

/*
 * Only to be called by the page allocator on a freshly allocated
 * page.
 */
static inline void set_page_pfmemalloc(struct page *page)
{
	page->index = -1UL;
}

static inline void clear_page_pfmemalloc(struct page *page)
{
	page->index = 0;
}

L
Linus Torvalds 已提交
1014 1015 1016 1017 1018
/*
 * Different kinds of faults, as returned by handle_mm_fault().
 * Used to decide whether a process gets delivered SIGBUS or
 * just gets major/minor fault counters bumped up.
 */
N
Nick Piggin 已提交
1019

N
Nick Piggin 已提交
1020
#define VM_FAULT_MINOR	0 /* For backwards compat. Remove me quickly. */
N
Nick Piggin 已提交
1021

N
Nick Piggin 已提交
1022 1023 1024 1025
#define VM_FAULT_OOM	0x0001
#define VM_FAULT_SIGBUS	0x0002
#define VM_FAULT_MAJOR	0x0004
#define VM_FAULT_WRITE	0x0008	/* Special case for get_user_pages */
1026 1027
#define VM_FAULT_HWPOISON 0x0010	/* Hit poisoned small page */
#define VM_FAULT_HWPOISON_LARGE 0x0020  /* Hit poisoned large page. Index encoded in upper bits */
1028
#define VM_FAULT_SIGSEGV 0x0040
N
Nick Piggin 已提交
1029

N
Nick Piggin 已提交
1030 1031
#define VM_FAULT_NOPAGE	0x0100	/* ->fault installed the pte, not return page */
#define VM_FAULT_LOCKED	0x0200	/* ->fault locked the returned page */
1032
#define VM_FAULT_RETRY	0x0400	/* ->fault blocked, must retry */
1033
#define VM_FAULT_FALLBACK 0x0800	/* huge page fault failed, fall back to small */
L
Linus Torvalds 已提交
1034

1035 1036
#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */

1037 1038 1039
#define VM_FAULT_ERROR	(VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \
			 VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE | \
			 VM_FAULT_FALLBACK)
1040 1041 1042 1043

/* Encode hstate index for a hwpoisoned large page */
#define VM_FAULT_SET_HINDEX(x) ((x) << 12)
#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
N
Nick Piggin 已提交
1044

1045 1046 1047 1048 1049
/*
 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
 */
extern void pagefault_out_of_memory(void);

L
Linus Torvalds 已提交
1050 1051
#define offset_in_page(p)	((unsigned long)(p) & ~PAGE_MASK)

1052
/*
1053
 * Flags passed to show_mem() and show_free_areas() to suppress output in
1054 1055
 * various contexts.
 */
1056
#define SHOW_MEM_FILTER_NODES		(0x0001u)	/* disallowed nodes */
1057

1058 1059
extern void show_free_areas(unsigned int flags);
extern bool skip_free_areas_node(unsigned int flags, int nid);
L
Linus Torvalds 已提交
1060 1061

int shmem_zero_setup(struct vm_area_struct *);
1062 1063 1064 1065 1066 1067 1068 1069
#ifdef CONFIG_SHMEM
bool shmem_mapping(struct address_space *mapping);
#else
static inline bool shmem_mapping(struct address_space *mapping)
{
	return false;
}
#endif
L
Linus Torvalds 已提交
1070

A
Alexey Dobriyan 已提交
1071
extern int can_do_mlock(void);
L
Linus Torvalds 已提交
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
extern int user_shm_lock(size_t, struct user_struct *);
extern void user_shm_unlock(size_t, struct user_struct *);

/*
 * Parameter block passed down to zap_pte_range in exceptional cases.
 */
struct zap_details {
	struct address_space *check_mapping;	/* Check page->mapping if set */
	pgoff_t	first_index;			/* Lowest page->index to unmap */
	pgoff_t last_index;			/* Highest page->index to unmap */
};

N
Nick Piggin 已提交
1084 1085 1086
struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
		pte_t pte);

1087 1088
int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
		unsigned long size);
A
Al Viro 已提交
1089
void zap_page_range(struct vm_area_struct *vma, unsigned long address,
L
Linus Torvalds 已提交
1090
		unsigned long size, struct zap_details *);
1091 1092
void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
		unsigned long start, unsigned long end);
1093 1094 1095 1096

/**
 * mm_walk - callbacks for walk_page_range
 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
1097 1098 1099
 *	       this handler is required to be able to handle
 *	       pmd_trans_huge() pmds.  They may simply choose to
 *	       split_huge_page() instead of handling it explicitly.
1100 1101
 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
 * @pte_hole: if set, called for each hole at all levels
1102
 * @hugetlb_entry: if set, called for each hugetlb entry
N
Naoya Horiguchi 已提交
1103 1104 1105 1106 1107 1108 1109 1110
 * @test_walk: caller specific callback function to determine whether
 *             we walk over the current vma or not. A positive returned
 *             value means "do page table walk over the current vma,"
 *             and a negative one means "abort current page table walk
 *             right now." 0 means "skip the current vma."
 * @mm:        mm_struct representing the target process of page table walk
 * @vma:       vma currently walked (NULL if walking outside vmas)
 * @private:   private data for callbacks' usage
1111
 *
N
Naoya Horiguchi 已提交
1112
 * (see the comment on walk_page_range() for more details)
1113 1114
 */
struct mm_walk {
1115 1116 1117 1118 1119 1120 1121 1122 1123
	int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
			 unsigned long next, struct mm_walk *walk);
	int (*pte_entry)(pte_t *pte, unsigned long addr,
			 unsigned long next, struct mm_walk *walk);
	int (*pte_hole)(unsigned long addr, unsigned long next,
			struct mm_walk *walk);
	int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
			     unsigned long addr, unsigned long next,
			     struct mm_walk *walk);
N
Naoya Horiguchi 已提交
1124 1125
	int (*test_walk)(unsigned long addr, unsigned long next,
			struct mm_walk *walk);
D
Dave Hansen 已提交
1126
	struct mm_struct *mm;
N
Naoya Horiguchi 已提交
1127
	struct vm_area_struct *vma;
D
Dave Hansen 已提交
1128
	void *private;
1129 1130
};

D
Dave Hansen 已提交
1131 1132
int walk_page_range(unsigned long addr, unsigned long end,
		struct mm_walk *walk);
N
Naoya Horiguchi 已提交
1133
int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk);
1134
void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
1135
		unsigned long end, unsigned long floor, unsigned long ceiling);
L
Linus Torvalds 已提交
1136 1137 1138 1139
int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
			struct vm_area_struct *vma);
void unmap_mapping_range(struct address_space *mapping,
		loff_t const holebegin, loff_t const holelen, int even_cows);
J
Johannes Weiner 已提交
1140 1141
int follow_pfn(struct vm_area_struct *vma, unsigned long address,
	unsigned long *pfn);
1142 1143
int follow_phys(struct vm_area_struct *vma, unsigned long address,
		unsigned int flags, unsigned long *prot, resource_size_t *phys);
1144 1145
int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
			void *buf, int len, int write);
L
Linus Torvalds 已提交
1146 1147 1148 1149 1150 1151 1152

static inline void unmap_shared_mapping_range(struct address_space *mapping,
		loff_t const holebegin, loff_t const holelen)
{
	unmap_mapping_range(mapping, holebegin, holelen, 0);
}

1153
extern void truncate_pagecache(struct inode *inode, loff_t new);
1154
extern void truncate_setsize(struct inode *inode, loff_t newsize);
1155
void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
1156
void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
1157
int truncate_inode_page(struct address_space *mapping, struct page *page);
1158
int generic_error_remove_page(struct address_space *mapping, struct page *page);
1159 1160
int invalidate_inode_page(struct page *page);

1161
#ifdef CONFIG_MMU
N
Nick Piggin 已提交
1162
extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
1163
			unsigned long address, unsigned int flags);
1164 1165
extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
			    unsigned long address, unsigned int fault_flags);
1166 1167 1168
#else
static inline int handle_mm_fault(struct mm_struct *mm,
			struct vm_area_struct *vma, unsigned long address,
1169
			unsigned int flags)
1170 1171 1172 1173 1174
{
	/* should never happen if there's no MMU */
	BUG();
	return VM_FAULT_SIGBUS;
}
1175 1176 1177 1178 1179 1180 1181 1182
static inline int fixup_user_fault(struct task_struct *tsk,
		struct mm_struct *mm, unsigned long address,
		unsigned int fault_flags)
{
	/* should never happen if there's no MMU */
	BUG();
	return -EFAULT;
}
1183
#endif
N
Nick Piggin 已提交
1184

L
Linus Torvalds 已提交
1185
extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
S
Stephen Wilson 已提交
1186 1187
extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
		void *buf, int len, int write);
L
Linus Torvalds 已提交
1188

1189 1190 1191 1192 1193 1194 1195 1196
long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
		      unsigned long start, unsigned long nr_pages,
		      unsigned int foll_flags, struct page **pages,
		      struct vm_area_struct **vmas, int *nonblocking);
long get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
		    unsigned long start, unsigned long nr_pages,
		    int write, int force, struct page **pages,
		    struct vm_area_struct **vmas);
1197 1198 1199 1200
long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm,
		    unsigned long start, unsigned long nr_pages,
		    int write, int force, struct page **pages,
		    int *locked);
1201 1202 1203 1204
long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
			       unsigned long start, unsigned long nr_pages,
			       int write, int force, struct page **pages,
			       unsigned int gup_flags);
1205 1206 1207
long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
		    unsigned long start, unsigned long nr_pages,
		    int write, int force, struct page **pages);
1208 1209
int get_user_pages_fast(unsigned long start, int nr_pages, int write,
			struct page **pages);
1210 1211 1212 1213
struct kvec;
int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
			struct page **pages);
int get_kernel_page(unsigned long start, int write, struct page **pages);
H
Hugh Dickins 已提交
1214
struct page *get_dump_page(unsigned long addr);
L
Linus Torvalds 已提交
1215

1216
extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1217 1218
extern void do_invalidatepage(struct page *page, unsigned int offset,
			      unsigned int length);
1219

L
Linus Torvalds 已提交
1220
int __set_page_dirty_nobuffers(struct page *page);
1221
int __set_page_dirty_no_writeback(struct page *page);
L
Linus Torvalds 已提交
1222 1223
int redirty_page_for_writepage(struct writeback_control *wbc,
				struct page *page);
1224 1225 1226
void account_page_dirtied(struct page *page, struct address_space *mapping,
			  struct mem_cgroup *memcg);
void account_page_cleaned(struct page *page, struct address_space *mapping,
1227
			  struct mem_cgroup *memcg, struct bdi_writeback *wb);
1228
int set_page_dirty(struct page *page);
L
Linus Torvalds 已提交
1229
int set_page_dirty_lock(struct page *page);
1230
void cancel_dirty_page(struct page *page);
L
Linus Torvalds 已提交
1231
int clear_page_dirty_for_io(struct page *page);
1232

1233
int get_cmdline(struct task_struct *task, char *buffer, int buflen);
L
Linus Torvalds 已提交
1234

1235
/* Is the vma a continuation of the stack vma above it? */
1236
static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
1237 1238 1239 1240
{
	return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
}

1241 1242 1243 1244 1245
static inline bool vma_is_anonymous(struct vm_area_struct *vma)
{
	return !vma->vm_ops;
}

1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
static inline int stack_guard_page_start(struct vm_area_struct *vma,
					     unsigned long addr)
{
	return (vma->vm_flags & VM_GROWSDOWN) &&
		(vma->vm_start == addr) &&
		!vma_growsdown(vma->vm_prev, addr);
}

/* Is the vma a continuation of the stack vma below it? */
static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
{
	return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
}

static inline int stack_guard_page_end(struct vm_area_struct *vma,
					   unsigned long addr)
{
	return (vma->vm_flags & VM_GROWSUP) &&
		(vma->vm_end == addr) &&
		!vma_growsup(vma->vm_next, addr);
}

1268 1269
extern struct task_struct *task_of_stack(struct task_struct *task,
				struct vm_area_struct *vma, bool in_group);
1270

1271 1272
extern unsigned long move_page_tables(struct vm_area_struct *vma,
		unsigned long old_addr, struct vm_area_struct *new_vma,
1273 1274
		unsigned long new_addr, unsigned long len,
		bool need_rmap_locks);
1275 1276
extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
			      unsigned long end, pgprot_t newprot,
1277
			      int dirty_accountable, int prot_numa);
1278 1279 1280
extern int mprotect_fixup(struct vm_area_struct *vma,
			  struct vm_area_struct **pprev, unsigned long start,
			  unsigned long end, unsigned long newflags);
L
Linus Torvalds 已提交
1281

1282 1283 1284 1285 1286
/*
 * doesn't attempt to fault and will return short.
 */
int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
			  struct page **pages);
K
KAMEZAWA Hiroyuki 已提交
1287 1288 1289 1290 1291
/*
 * per-process(per-mm_struct) statistics.
 */
static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
{
1292 1293 1294 1295 1296 1297 1298 1299 1300
	long val = atomic_long_read(&mm->rss_stat.count[member]);

#ifdef SPLIT_RSS_COUNTING
	/*
	 * counter is updated in asynchronous manner and may go to minus.
	 * But it's never be expected number for users.
	 */
	if (val < 0)
		val = 0;
1301
#endif
1302 1303
	return (unsigned long)val;
}
K
KAMEZAWA Hiroyuki 已提交
1304 1305 1306

static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
{
1307
	atomic_long_add(value, &mm->rss_stat.count[member]);
K
KAMEZAWA Hiroyuki 已提交
1308 1309 1310 1311
}

static inline void inc_mm_counter(struct mm_struct *mm, int member)
{
1312
	atomic_long_inc(&mm->rss_stat.count[member]);
K
KAMEZAWA Hiroyuki 已提交
1313 1314 1315 1316
}

static inline void dec_mm_counter(struct mm_struct *mm, int member)
{
1317
	atomic_long_dec(&mm->rss_stat.count[member]);
K
KAMEZAWA Hiroyuki 已提交
1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
}

static inline unsigned long get_mm_rss(struct mm_struct *mm)
{
	return get_mm_counter(mm, MM_FILEPAGES) +
		get_mm_counter(mm, MM_ANONPAGES);
}

static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
{
	return max(mm->hiwater_rss, get_mm_rss(mm));
}

static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
{
	return max(mm->hiwater_vm, mm->total_vm);
}

static inline void update_hiwater_rss(struct mm_struct *mm)
{
	unsigned long _rss = get_mm_rss(mm);

	if ((mm)->hiwater_rss < _rss)
		(mm)->hiwater_rss = _rss;
}

static inline void update_hiwater_vm(struct mm_struct *mm)
{
	if (mm->hiwater_vm < mm->total_vm)
		mm->hiwater_vm = mm->total_vm;
}

1350 1351 1352 1353 1354
static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
{
	mm->hiwater_rss = get_mm_rss(mm);
}

K
KAMEZAWA Hiroyuki 已提交
1355 1356 1357 1358 1359 1360 1361 1362 1363
static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
					 struct mm_struct *mm)
{
	unsigned long hiwater_rss = get_mm_hiwater_rss(mm);

	if (*maxrss < hiwater_rss)
		*maxrss = hiwater_rss;
}

K
KAMEZAWA Hiroyuki 已提交
1364
#if defined(SPLIT_RSS_COUNTING)
1365
void sync_mm_rss(struct mm_struct *mm);
K
KAMEZAWA Hiroyuki 已提交
1366
#else
1367
static inline void sync_mm_rss(struct mm_struct *mm)
K
KAMEZAWA Hiroyuki 已提交
1368 1369 1370
{
}
#endif
1371

A
Alexey Dobriyan 已提交
1372
int vma_wants_writenotify(struct vm_area_struct *vma);
1373

1374 1375 1376 1377 1378 1379 1380 1381 1382
extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
			       spinlock_t **ptl);
static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
				    spinlock_t **ptl)
{
	pte_t *ptep;
	__cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
	return ptep;
}
1383

N
Nick Piggin 已提交
1384 1385 1386 1387 1388 1389 1390
#ifdef __PAGETABLE_PUD_FOLDED
static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
						unsigned long address)
{
	return 0;
}
#else
1391
int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
N
Nick Piggin 已提交
1392 1393
#endif

1394
#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
N
Nick Piggin 已提交
1395 1396 1397 1398 1399
static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
						unsigned long address)
{
	return 0;
}
1400

1401 1402
static inline void mm_nr_pmds_init(struct mm_struct *mm) {}

1403 1404 1405 1406 1407 1408 1409 1410
static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
{
	return 0;
}

static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}

N
Nick Piggin 已提交
1411
#else
1412
int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
1413

1414 1415 1416 1417 1418
static inline void mm_nr_pmds_init(struct mm_struct *mm)
{
	atomic_long_set(&mm->nr_pmds, 0);
}

1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
{
	return atomic_long_read(&mm->nr_pmds);
}

static inline void mm_inc_nr_pmds(struct mm_struct *mm)
{
	atomic_long_inc(&mm->nr_pmds);
}

static inline void mm_dec_nr_pmds(struct mm_struct *mm)
{
	atomic_long_dec(&mm->nr_pmds);
}
N
Nick Piggin 已提交
1433 1434
#endif

1435 1436
int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
		pmd_t *pmd, unsigned long address);
1437 1438
int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);

L
Linus Torvalds 已提交
1439 1440 1441 1442
/*
 * The following ifdef needed to get the 4level-fixup.h header to work.
 * Remove it when 4level-fixup.h has been removed.
 */
1443
#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
L
Linus Torvalds 已提交
1444 1445
static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
{
1446 1447
	return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
		NULL: pud_offset(pgd, address);
L
Linus Torvalds 已提交
1448 1449 1450 1451
}

static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
{
1452 1453
	return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
		NULL: pmd_offset(pud, address);
L
Linus Torvalds 已提交
1454
}
1455 1456
#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */

1457
#if USE_SPLIT_PTE_PTLOCKS
1458
#if ALLOC_SPLIT_PTLOCKS
1459
void __init ptlock_cache_init(void);
1460 1461 1462 1463 1464 1465 1466
extern bool ptlock_alloc(struct page *page);
extern void ptlock_free(struct page *page);

static inline spinlock_t *ptlock_ptr(struct page *page)
{
	return page->ptl;
}
1467
#else /* ALLOC_SPLIT_PTLOCKS */
1468 1469 1470 1471
static inline void ptlock_cache_init(void)
{
}

1472 1473 1474 1475
static inline bool ptlock_alloc(struct page *page)
{
	return true;
}
1476

1477 1478 1479 1480 1481 1482
static inline void ptlock_free(struct page *page)
{
}

static inline spinlock_t *ptlock_ptr(struct page *page)
{
1483
	return &page->ptl;
1484
}
1485
#endif /* ALLOC_SPLIT_PTLOCKS */
1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501

static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
	return ptlock_ptr(pmd_page(*pmd));
}

static inline bool ptlock_init(struct page *page)
{
	/*
	 * prep_new_page() initialize page->private (and therefore page->ptl)
	 * with 0. Make sure nobody took it in use in between.
	 *
	 * It can happen if arch try to use slab for page table allocation:
	 * slab code uses page->slab_cache and page->first_page (for tail
	 * pages), which share storage with page->ptl.
	 */
1502
	VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
	if (!ptlock_alloc(page))
		return false;
	spin_lock_init(ptlock_ptr(page));
	return true;
}

/* Reset page->mapping so free_pages_check won't complain. */
static inline void pte_lock_deinit(struct page *page)
{
	page->mapping = NULL;
	ptlock_free(page);
}

1516
#else	/* !USE_SPLIT_PTE_PTLOCKS */
H
Hugh Dickins 已提交
1517 1518 1519
/*
 * We use mm->page_table_lock to guard all pagetable pages of the mm.
 */
1520 1521 1522 1523
static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
	return &mm->page_table_lock;
}
1524
static inline void ptlock_cache_init(void) {}
1525 1526
static inline bool ptlock_init(struct page *page) { return true; }
static inline void pte_lock_deinit(struct page *page) {}
1527
#endif /* USE_SPLIT_PTE_PTLOCKS */
H
Hugh Dickins 已提交
1528

1529 1530 1531 1532 1533 1534
static inline void pgtable_init(void)
{
	ptlock_cache_init();
	pgtable_cache_init();
}

1535
static inline bool pgtable_page_ctor(struct page *page)
1536 1537
{
	inc_zone_page_state(page, NR_PAGETABLE);
1538
	return ptlock_init(page);
1539 1540 1541 1542 1543 1544 1545 1546
}

static inline void pgtable_page_dtor(struct page *page)
{
	pte_lock_deinit(page);
	dec_zone_page_state(page, NR_PAGETABLE);
}

H
Hugh Dickins 已提交
1547 1548
#define pte_offset_map_lock(mm, pmd, address, ptlp)	\
({							\
H
Hugh Dickins 已提交
1549
	spinlock_t *__ptl = pte_lockptr(mm, pmd);	\
H
Hugh Dickins 已提交
1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560
	pte_t *__pte = pte_offset_map(pmd, address);	\
	*(ptlp) = __ptl;				\
	spin_lock(__ptl);				\
	__pte;						\
})

#define pte_unmap_unlock(pte, ptl)	do {		\
	spin_unlock(ptl);				\
	pte_unmap(pte);					\
} while (0)

1561 1562 1563 1564
#define pte_alloc_map(mm, vma, pmd, address)				\
	((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma,	\
							pmd, address))?	\
	 NULL: pte_offset_map(pmd, address))
1565

H
Hugh Dickins 已提交
1566
#define pte_alloc_map_lock(mm, pmd, address, ptlp)	\
1567 1568
	((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL,	\
							pmd, address))?	\
H
Hugh Dickins 已提交
1569 1570
		NULL: pte_offset_map_lock(mm, pmd, address, ptlp))

1571
#define pte_alloc_kernel(pmd, address)			\
1572
	((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
1573
		NULL: pte_offset_kernel(pmd, address))
L
Linus Torvalds 已提交
1574

1575 1576
#if USE_SPLIT_PMD_PTLOCKS

1577 1578 1579 1580 1581 1582
static struct page *pmd_to_page(pmd_t *pmd)
{
	unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
	return virt_to_page((void *)((unsigned long) pmd & mask));
}

1583 1584
static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
1585
	return ptlock_ptr(pmd_to_page(pmd));
1586 1587 1588 1589 1590 1591 1592
}

static inline bool pgtable_pmd_page_ctor(struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	page->pmd_huge_pte = NULL;
#endif
1593
	return ptlock_init(page);
1594 1595 1596 1597 1598
}

static inline void pgtable_pmd_page_dtor(struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1599
	VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
1600
#endif
1601
	ptlock_free(page);
1602 1603
}

1604
#define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte)
1605 1606 1607

#else

1608 1609 1610 1611 1612
static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
	return &mm->page_table_lock;
}

1613 1614 1615
static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
static inline void pgtable_pmd_page_dtor(struct page *page) {}

1616
#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
1617

1618 1619
#endif

1620 1621 1622 1623 1624 1625 1626
static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
{
	spinlock_t *ptl = pmd_lockptr(mm, pmd);
	spin_lock(ptl);
	return ptl;
}

L
Linus Torvalds 已提交
1627
extern void free_area_init(unsigned long * zones_size);
1628 1629
extern void free_area_init_node(int nid, unsigned long * zones_size,
		unsigned long zone_start_pfn, unsigned long *zholes_size);
1630 1631
extern void free_initmem(void);

1632 1633 1634
/*
 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
 * into the buddy system. The freed pages will be poisoned with pattern
1635
 * "poison" if it's within range [0, UCHAR_MAX].
1636 1637
 * Return pages freed into the buddy system.
 */
1638
extern unsigned long free_reserved_area(void *start, void *end,
1639
					int poison, char *s);
1640

1641 1642 1643 1644 1645 1646 1647
#ifdef	CONFIG_HIGHMEM
/*
 * Free a highmem page into the buddy system, adjusting totalhigh_pages
 * and totalram_pages.
 */
extern void free_highmem_page(struct page *page);
#endif
1648

1649
extern void adjust_managed_page_count(struct page *page, long count);
1650
extern void mem_init_print_info(const char *str);
1651

1652 1653
extern void reserve_bootmem_region(unsigned long start, unsigned long end);

1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675
/* Free the reserved page into the buddy system, so it gets managed. */
static inline void __free_reserved_page(struct page *page)
{
	ClearPageReserved(page);
	init_page_count(page);
	__free_page(page);
}

static inline void free_reserved_page(struct page *page)
{
	__free_reserved_page(page);
	adjust_managed_page_count(page, 1);
}

static inline void mark_page_reserved(struct page *page)
{
	SetPageReserved(page);
	adjust_managed_page_count(page, -1);
}

/*
 * Default method to free all the __init memory into the buddy system.
1676 1677 1678
 * The freed pages will be poisoned with pattern "poison" if it's within
 * range [0, UCHAR_MAX].
 * Return pages freed into the buddy system.
1679 1680 1681 1682 1683
 */
static inline unsigned long free_initmem_default(int poison)
{
	extern char __init_begin[], __init_end[];

1684
	return free_reserved_area(&__init_begin, &__init_end,
1685 1686 1687
				  poison, "unused kernel");
}

1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698
static inline unsigned long get_num_physpages(void)
{
	int nid;
	unsigned long phys_pages = 0;

	for_each_online_node(nid)
		phys_pages += node_present_pages(nid);

	return phys_pages;
}

T
Tejun Heo 已提交
1699
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1700
/*
T
Tejun Heo 已提交
1701
 * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
1702 1703 1704 1705 1706 1707
 * zones, allocate the backing mem_map and account for memory holes in a more
 * architecture independent manner. This is a substitute for creating the
 * zone_sizes[] and zholes_size[] arrays and passing them to
 * free_area_init_node()
 *
 * An architecture is expected to register range of page frames backed by
T
Tejun Heo 已提交
1708
 * physical memory with memblock_add[_node]() before calling
1709 1710 1711 1712 1713 1714
 * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
 * usage, an architecture is expected to do something like
 *
 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
 * 							 max_highmem_pfn};
 * for_each_valid_physical_page_range()
T
Tejun Heo 已提交
1715
 * 	memblock_add_node(base, size, nid)
1716 1717
 * free_area_init_nodes(max_zone_pfns);
 *
T
Tejun Heo 已提交
1718 1719 1720 1721
 * free_bootmem_with_active_regions() calls free_bootmem_node() for each
 * registered physical page range.  Similarly
 * sparse_memory_present_with_active_regions() calls memory_present() for
 * each range when SPARSEMEM is enabled.
1722 1723
 *
 * See mm/page_alloc.c for more information on each function exposed by
T
Tejun Heo 已提交
1724
 * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
1725 1726
 */
extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1727
unsigned long node_map_pfn_alignment(void);
1728 1729
unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
						unsigned long end_pfn);
1730 1731 1732 1733 1734 1735 1736 1737
extern unsigned long absent_pages_in_range(unsigned long start_pfn,
						unsigned long end_pfn);
extern void get_pfn_range_for_nid(unsigned int nid,
			unsigned long *start_pfn, unsigned long *end_pfn);
extern unsigned long find_min_pfn_with_active_regions(void);
extern void free_bootmem_with_active_regions(int nid,
						unsigned long max_low_pfn);
extern void sparse_memory_present_with_active_regions(int nid);
1738

T
Tejun Heo 已提交
1739
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
1740

T
Tejun Heo 已提交
1741
#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
1742
    !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1743 1744
static inline int __early_pfn_to_nid(unsigned long pfn,
					struct mminit_pfnnid_cache *state)
1745 1746 1747 1748 1749 1750 1751
{
	return 0;
}
#else
/* please see mm/page_alloc.c */
extern int __meminit early_pfn_to_nid(unsigned long pfn);
/* there is a per-arch backend function. */
1752 1753
extern int __meminit __early_pfn_to_nid(unsigned long pfn,
					struct mminit_pfnnid_cache *state);
1754 1755
#endif

1756
extern void set_dma_reserve(unsigned long new_dma_reserve);
D
Dave Hansen 已提交
1757 1758
extern void memmap_init_zone(unsigned long, int, unsigned long,
				unsigned long, enum memmap_context);
1759
extern void setup_per_zone_wmarks(void);
1760
extern int __meminit init_per_zone_wmark_min(void);
L
Linus Torvalds 已提交
1761
extern void mem_init(void);
1762
extern void __init mmap_init(void);
1763
extern void show_mem(unsigned int flags);
L
Linus Torvalds 已提交
1764 1765 1766
extern void si_meminfo(struct sysinfo * val);
extern void si_meminfo_node(struct sysinfo *val, int nid);

J
Joe Perches 已提交
1767 1768
extern __printf(3, 4)
void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...);
1769

1770 1771
extern void setup_per_cpu_pageset(void);

1772
extern void zone_pcp_update(struct zone *zone);
1773
extern void zone_pcp_reset(struct zone *zone);
1774

1775 1776 1777
/* page_alloc.c */
extern int min_free_kbytes;

1778
/* nommu.c */
1779
extern atomic_long_t mmap_pages_allocated;
1780
extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
1781

1782 1783 1784
/* interval_tree.c */
void vma_interval_tree_insert(struct vm_area_struct *node,
			      struct rb_root *root);
M
Michel Lespinasse 已提交
1785 1786 1787
void vma_interval_tree_insert_after(struct vm_area_struct *node,
				    struct vm_area_struct *prev,
				    struct rb_root *root);
1788 1789 1790 1791 1792 1793 1794 1795 1796 1797
void vma_interval_tree_remove(struct vm_area_struct *node,
			      struct rb_root *root);
struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
				unsigned long start, unsigned long last);
struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
				unsigned long start, unsigned long last);

#define vma_interval_tree_foreach(vma, root, start, last)		\
	for (vma = vma_interval_tree_iter_first(root, start, last);	\
	     vma; vma = vma_interval_tree_iter_next(vma, start, last))
L
Linus Torvalds 已提交
1798

1799 1800 1801 1802 1803 1804 1805 1806
void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
				   struct rb_root *root);
void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
				   struct rb_root *root);
struct anon_vma_chain *anon_vma_interval_tree_iter_first(
	struct rb_root *root, unsigned long start, unsigned long last);
struct anon_vma_chain *anon_vma_interval_tree_iter_next(
	struct anon_vma_chain *node, unsigned long start, unsigned long last);
1807 1808 1809
#ifdef CONFIG_DEBUG_VM_RB
void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
#endif
1810 1811 1812 1813 1814

#define anon_vma_interval_tree_foreach(avc, root, start, last)		 \
	for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
	     avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))

L
Linus Torvalds 已提交
1815
/* mmap.c */
1816
extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1817
extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
L
Linus Torvalds 已提交
1818 1819 1820 1821
	unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
extern struct vm_area_struct *vma_merge(struct mm_struct *,
	struct vm_area_struct *prev, unsigned long addr, unsigned long end,
	unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1822
	struct mempolicy *, struct vm_userfaultfd_ctx);
L
Linus Torvalds 已提交
1823 1824 1825 1826 1827 1828
extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
extern int split_vma(struct mm_struct *,
	struct vm_area_struct *, unsigned long addr, int new_below);
extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
	struct rb_node **, struct rb_node *);
1829
extern void unlink_file_vma(struct vm_area_struct *);
L
Linus Torvalds 已提交
1830
extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1831 1832
	unsigned long addr, unsigned long len, pgoff_t pgoff,
	bool *need_rmap_locks);
L
Linus Torvalds 已提交
1833
extern void exit_mmap(struct mm_struct *);
M
Matt Helsley 已提交
1834

1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848
static inline int check_data_rlimit(unsigned long rlim,
				    unsigned long new,
				    unsigned long start,
				    unsigned long end_data,
				    unsigned long start_data)
{
	if (rlim < RLIM_INFINITY) {
		if (((new - start) + (end_data - start_data)) > rlim)
			return -ENOSPC;
	}

	return 0;
}

1849 1850 1851
extern int mm_take_all_locks(struct mm_struct *mm);
extern void mm_drop_all_locks(struct mm_struct *mm);

1852 1853
extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
extern struct file *get_mm_exe_file(struct mm_struct *mm);
M
Matt Helsley 已提交
1854

A
akpm@osdl.org 已提交
1855
extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
1856 1857
extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
				   unsigned long addr, unsigned long len,
1858 1859 1860
				   unsigned long flags,
				   const struct vm_special_mapping *spec);
/* This is an obsolete alternative to _install_special_mapping. */
1861 1862 1863
extern int install_special_mapping(struct mm_struct *mm,
				   unsigned long addr, unsigned long len,
				   unsigned long flags, struct page **pages);
L
Linus Torvalds 已提交
1864 1865 1866

extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);

M
Miklos Szeredi 已提交
1867
extern unsigned long mmap_region(struct file *file, unsigned long addr,
1868
	unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
1869 1870
extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
	unsigned long len, unsigned long prot, unsigned long flags,
1871
	unsigned long pgoff, unsigned long *populate);
L
Linus Torvalds 已提交
1872 1873
extern int do_munmap(struct mm_struct *, unsigned long, size_t);

1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
#ifdef CONFIG_MMU
extern int __mm_populate(unsigned long addr, unsigned long len,
			 int ignore_errors);
static inline void mm_populate(unsigned long addr, unsigned long len)
{
	/* Ignore errors */
	(void) __mm_populate(addr, len, 1);
}
#else
static inline void mm_populate(unsigned long addr, unsigned long len) {}
#endif

1886 1887
/* These take the mm semaphore themselves */
extern unsigned long vm_brk(unsigned long, unsigned long);
A
Al Viro 已提交
1888
extern int vm_munmap(unsigned long, size_t);
1889 1890 1891
extern unsigned long vm_mmap(struct file *, unsigned long,
        unsigned long, unsigned long,
        unsigned long, unsigned long);
L
Linus Torvalds 已提交
1892

1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917
struct vm_unmapped_area_info {
#define VM_UNMAPPED_AREA_TOPDOWN 1
	unsigned long flags;
	unsigned long length;
	unsigned long low_limit;
	unsigned long high_limit;
	unsigned long align_mask;
	unsigned long align_offset;
};

extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);

/*
 * Search for an unmapped address range.
 *
 * We are looking for a range that:
 * - does not intersect with any VMA;
 * - is contained within the [low_limit, high_limit) interval;
 * - is at least the desired size.
 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
 */
static inline unsigned long
vm_unmapped_area(struct vm_unmapped_area_info *info)
{
1918
	if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
1919
		return unmapped_area_topdown(info);
1920 1921
	else
		return unmapped_area(info);
1922 1923
}

1924
/* truncate.c */
L
Linus Torvalds 已提交
1925
extern void truncate_inode_pages(struct address_space *, loff_t);
1926 1927
extern void truncate_inode_pages_range(struct address_space *,
				       loff_t lstart, loff_t lend);
1928
extern void truncate_inode_pages_final(struct address_space *);
L
Linus Torvalds 已提交
1929 1930

/* generic vm_area_ops exported for stackable file systems */
N
Nick Piggin 已提交
1931
extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
1932
extern void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf);
1933
extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
L
Linus Torvalds 已提交
1934 1935 1936

/* mm/page-writeback.c */
int write_one_page(struct page *page, int wait);
N
Nick Piggin 已提交
1937
void task_dirty_inc(struct task_struct *tsk);
L
Linus Torvalds 已提交
1938 1939 1940 1941 1942 1943

/* readahead.c */
#define VM_MAX_READAHEAD	128	/* kbytes */
#define VM_MIN_READAHEAD	16	/* kbytes (includes current page) */

int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
A
Andrew Morton 已提交
1944
			pgoff_t offset, unsigned long nr_to_read);
1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958

void page_cache_sync_readahead(struct address_space *mapping,
			       struct file_ra_state *ra,
			       struct file *filp,
			       pgoff_t offset,
			       unsigned long size);

void page_cache_async_readahead(struct address_space *mapping,
				struct file_ra_state *ra,
				struct file *filp,
				struct page *pg,
				pgoff_t offset,
				unsigned long size);

L
Linus Torvalds 已提交
1959 1960
unsigned long max_sane_readahead(unsigned long nr);

1961
/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
H
Hugh Dickins 已提交
1962
extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
1963 1964 1965 1966

/* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
extern int expand_downwards(struct vm_area_struct *vma,
		unsigned long address);
1967
#if VM_GROWSUP
H
Hugh Dickins 已提交
1968
extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
1969
#else
1970
  #define expand_upwards(vma, address) (0)
1971
#endif
L
Linus Torvalds 已提交
1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993

/* Look up the first VMA which satisfies  addr < vm_end,  NULL if none. */
extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
					     struct vm_area_struct **pprev);

/* Look up the first VMA which intersects the interval start_addr..end_addr-1,
   NULL if none.  Assume start_addr < end_addr. */
static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
{
	struct vm_area_struct * vma = find_vma(mm,start_addr);

	if (vma && end_addr <= vma->vm_start)
		vma = NULL;
	return vma;
}

static inline unsigned long vma_pages(struct vm_area_struct *vma)
{
	return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
}

1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005
/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
				unsigned long vm_start, unsigned long vm_end)
{
	struct vm_area_struct *vma = find_vma(mm, vm_start);

	if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
		vma = NULL;

	return vma;
}

2006
#ifdef CONFIG_MMU
2007
pgprot_t vm_get_page_prot(unsigned long vm_flags);
2008
void vma_set_page_prot(struct vm_area_struct *vma);
2009 2010 2011 2012 2013
#else
static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
{
	return __pgprot(0);
}
2014 2015 2016 2017
static inline void vma_set_page_prot(struct vm_area_struct *vma)
{
	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
}
2018 2019
#endif

2020
#ifdef CONFIG_NUMA_BALANCING
2021
unsigned long change_prot_numa(struct vm_area_struct *vma,
L
Lee Schermerhorn 已提交
2022 2023 2024
			unsigned long start, unsigned long end);
#endif

2025 2026 2027
struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
			unsigned long pfn, unsigned long size, pgprot_t);
2028
int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
N
Nick Piggin 已提交
2029 2030
int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
			unsigned long pfn);
N
Nick Piggin 已提交
2031 2032
int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
			unsigned long pfn);
2033 2034
int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);

2035

2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046
struct page *follow_page_mask(struct vm_area_struct *vma,
			      unsigned long address, unsigned int foll_flags,
			      unsigned int *page_mask);

static inline struct page *follow_page(struct vm_area_struct *vma,
		unsigned long address, unsigned int foll_flags)
{
	unsigned int unused_page_mask;
	return follow_page_mask(vma, address, foll_flags, &unused_page_mask);
}

2047 2048 2049
#define FOLL_WRITE	0x01	/* check pte is writable */
#define FOLL_TOUCH	0x02	/* mark page accessed */
#define FOLL_GET	0x04	/* do get_page on page */
H
Hugh Dickins 已提交
2050
#define FOLL_DUMP	0x08	/* give error on hole if it would be zero */
H
Hugh Dickins 已提交
2051
#define FOLL_FORCE	0x10	/* get_user_pages read/write w/o permission */
2052 2053
#define FOLL_NOWAIT	0x20	/* if a disk transfer is needed, start the IO
				 * and return without waiting upon it */
2054
#define FOLL_POPULATE	0x40	/* fault in page */
2055
#define FOLL_SPLIT	0x80	/* don't return transhuge pages, split them */
2056
#define FOLL_HWPOISON	0x100	/* check page is hwpoisoned */
2057
#define FOLL_NUMA	0x200	/* force NUMA hinting page fault */
2058
#define FOLL_MIGRATION	0x400	/* wait for page to replace migration entry */
2059
#define FOLL_TRIED	0x800	/* a retry, previous pass started an IO */
L
Linus Torvalds 已提交
2060

2061
typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
2062 2063 2064 2065
			void *data);
extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
			       unsigned long size, pte_fn_t fn, void *data);

L
Linus Torvalds 已提交
2066
#ifdef CONFIG_PROC_FS
2067
void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
L
Linus Torvalds 已提交
2068
#else
2069
static inline void vm_stat_account(struct mm_struct *mm,
L
Linus Torvalds 已提交
2070 2071
			unsigned long flags, struct file *file, long pages)
{
2072
	mm->total_vm += pages;
L
Linus Torvalds 已提交
2073 2074 2075
}
#endif /* CONFIG_PROC_FS */

2076
#ifdef CONFIG_DEBUG_PAGEALLOC
2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092
extern bool _debug_pagealloc_enabled;
extern void __kernel_map_pages(struct page *page, int numpages, int enable);

static inline bool debug_pagealloc_enabled(void)
{
	return _debug_pagealloc_enabled;
}

static inline void
kernel_map_pages(struct page *page, int numpages, int enable)
{
	if (!debug_pagealloc_enabled())
		return;

	__kernel_map_pages(page, numpages, enable);
}
2093 2094 2095
#ifdef CONFIG_HIBERNATION
extern bool kernel_page_present(struct page *page);
#endif /* CONFIG_HIBERNATION */
2096
#else
L
Linus Torvalds 已提交
2097
static inline void
N
Nick Piggin 已提交
2098
kernel_map_pages(struct page *page, int numpages, int enable) {}
2099 2100 2101
#ifdef CONFIG_HIBERNATION
static inline bool kernel_page_present(struct page *page) { return true; }
#endif /* CONFIG_HIBERNATION */
L
Linus Torvalds 已提交
2102 2103
#endif

2104
#ifdef __HAVE_ARCH_GATE_AREA
2105
extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
2106 2107
extern int in_gate_area_no_mm(unsigned long addr);
extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
L
Linus Torvalds 已提交
2108
#else
2109 2110 2111 2112 2113 2114 2115 2116 2117
static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
{
	return NULL;
}
static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
{
	return 0;
}
L
Linus Torvalds 已提交
2118 2119
#endif	/* __HAVE_ARCH_GATE_AREA */

2120 2121
#ifdef CONFIG_SYSCTL
extern int sysctl_drop_caches;
2122
int drop_caches_sysctl_handler(struct ctl_table *, int,
A
Andrew Morton 已提交
2123
					void __user *, size_t *, loff_t *);
2124 2125
#endif

2126 2127
void drop_slab(void);
void drop_slab_node(int nid);
A
Andrew Morton 已提交
2128

2129 2130 2131
#ifndef CONFIG_MMU
#define randomize_va_space 0
#else
2132
extern int randomize_va_space;
2133
#endif
2134

2135
const char * arch_vma_name(struct vm_area_struct *vma);
2136
void print_vma_addr(char *prefix, unsigned long rip);
2137

2138 2139 2140 2141 2142 2143
void sparse_mem_maps_populate_node(struct page **map_map,
				   unsigned long pnum_begin,
				   unsigned long pnum_end,
				   unsigned long map_count,
				   int nodeid);

2144
struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
2145 2146 2147 2148
pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
2149
void *vmemmap_alloc_block(unsigned long size, int node);
2150
void *vmemmap_alloc_block_buf(unsigned long size, int node);
2151
void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
2152 2153 2154
int vmemmap_populate_basepages(unsigned long start, unsigned long end,
			       int node);
int vmemmap_populate(unsigned long start, unsigned long end, int node);
2155
void vmemmap_populate_print_last(void);
2156
#ifdef CONFIG_MEMORY_HOTPLUG
2157
void vmemmap_free(unsigned long start, unsigned long end);
2158
#endif
2159 2160
void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
				  unsigned long size);
2161

2162 2163
enum mf_flags {
	MF_COUNT_INCREASED = 1 << 0,
2164
	MF_ACTION_REQUIRED = 1 << 1,
2165
	MF_MUST_KILL = 1 << 2,
2166
	MF_SOFT_OFFLINE = 1 << 3,
2167
};
2168
extern int memory_failure(unsigned long pfn, int trapno, int flags);
2169
extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
W
Wu Fengguang 已提交
2170
extern int unpoison_memory(unsigned long pfn);
2171
extern int get_hwpoison_page(struct page *page);
2172 2173
extern int sysctl_memory_failure_early_kill;
extern int sysctl_memory_failure_recovery;
2174
extern void shake_page(struct page *p, int access);
2175
extern atomic_long_t num_poisoned_pages;
2176
extern int soft_offline_page(struct page *page, int flags);
2177

2178 2179 2180 2181

/*
 * Error handlers for various types of pages.
 */
2182
enum mf_result {
2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211
	MF_IGNORED,	/* Error: cannot be handled */
	MF_FAILED,	/* Error: handling failed */
	MF_DELAYED,	/* Will be handled later */
	MF_RECOVERED,	/* Successfully recovered */
};

enum mf_action_page_type {
	MF_MSG_KERNEL,
	MF_MSG_KERNEL_HIGH_ORDER,
	MF_MSG_SLAB,
	MF_MSG_DIFFERENT_COMPOUND,
	MF_MSG_POISONED_HUGE,
	MF_MSG_HUGE,
	MF_MSG_FREE_HUGE,
	MF_MSG_UNMAP_FAILED,
	MF_MSG_DIRTY_SWAPCACHE,
	MF_MSG_CLEAN_SWAPCACHE,
	MF_MSG_DIRTY_MLOCKED_LRU,
	MF_MSG_CLEAN_MLOCKED_LRU,
	MF_MSG_DIRTY_UNEVICTABLE_LRU,
	MF_MSG_CLEAN_UNEVICTABLE_LRU,
	MF_MSG_DIRTY_LRU,
	MF_MSG_CLEAN_LRU,
	MF_MSG_TRUNCATED_LRU,
	MF_MSG_BUDDY,
	MF_MSG_BUDDY_2ND,
	MF_MSG_UNKNOWN,
};

A
Andrea Arcangeli 已提交
2212 2213 2214 2215 2216 2217 2218 2219 2220
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
extern void clear_huge_page(struct page *page,
			    unsigned long addr,
			    unsigned int pages_per_huge_page);
extern void copy_user_huge_page(struct page *dst, struct page *src,
				unsigned long addr, struct vm_area_struct *vma,
				unsigned int pages_per_huge_page);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */

2221 2222 2223
extern struct page_ext_operations debug_guardpage_ops;
extern struct page_ext_operations page_poisoning_ops;

2224 2225
#ifdef CONFIG_DEBUG_PAGEALLOC
extern unsigned int _debug_guardpage_minorder;
2226
extern bool _debug_guardpage_enabled;
2227 2228 2229 2230 2231 2232

static inline unsigned int debug_guardpage_minorder(void)
{
	return _debug_guardpage_minorder;
}

2233 2234 2235 2236 2237
static inline bool debug_guardpage_enabled(void)
{
	return _debug_guardpage_enabled;
}

2238 2239
static inline bool page_is_guard(struct page *page)
{
2240 2241 2242 2243 2244 2245 2246
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
		return false;

	page_ext = lookup_page_ext(page);
	return test_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
2247 2248 2249
}
#else
static inline unsigned int debug_guardpage_minorder(void) { return 0; }
2250
static inline bool debug_guardpage_enabled(void) { return false; }
2251 2252 2253
static inline bool page_is_guard(struct page *page) { return false; }
#endif /* CONFIG_DEBUG_PAGEALLOC */

2254 2255 2256 2257 2258 2259
#if MAX_NUMNODES > 1
void __init setup_nr_node_ids(void);
#else
static inline void setup_nr_node_ids(void) {}
#endif

L
Linus Torvalds 已提交
2260 2261
#endif /* __KERNEL__ */
#endif /* _LINUX_MM_H */