internal.h 7.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/* internal.h: mm/ internal definitions
 *
 * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */
11 12 13 14
#ifndef __MM_INTERNAL_H
#define __MM_INTERNAL_H

#include <linux/mm.h>
L
Linus Torvalds 已提交
15

16 17 18
void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
		unsigned long floor, unsigned long ceiling);

A
Andi Kleen 已提交
19
extern void prep_compound_page(struct page *page, unsigned long order);
20
extern void prep_compound_gigantic_page(struct page *page, unsigned long order);
A
Andi Kleen 已提交
21

22
static inline void set_page_count(struct page *page, int v)
N
Nick Piggin 已提交
23
{
24 25 26 27 28 29 30 31 32
	atomic_set(&page->_count, v);
}

/*
 * Turn a non-refcounted page (->_count == 0) into refcounted with
 * a count of one.
 */
static inline void set_page_refcounted(struct page *page)
{
Q
Qi Yong 已提交
33
	VM_BUG_ON(PageTail(page));
N
Nick Piggin 已提交
34
	VM_BUG_ON(atomic_read(&page->_count));
N
Nick Piggin 已提交
35 36 37
	set_page_count(page, 1);
}

38 39 40 41 42
static inline void __put_page(struct page *page)
{
	atomic_dec(&page->_count);
}

L
Lee Schermerhorn 已提交
43 44 45
/*
 * in mm/vmscan.c:
 */
46
extern int isolate_lru_page(struct page *page);
L
Lee Schermerhorn 已提交
47
extern void putback_lru_page(struct page *page);
48

L
Lee Schermerhorn 已提交
49 50 51
/*
 * in mm/page_alloc.c
 */
52
extern unsigned long highest_memmap_pfn;
53
extern void __free_pages_bootmem(struct page *page, unsigned int order);
54

55 56 57 58 59 60 61 62 63 64
/*
 * function for dealing with page's order in buddy system.
 * zone->lock is already acquired when we use these.
 * So, we don't need atomic page->flags operations here.
 */
static inline unsigned long page_order(struct page *page)
{
	VM_BUG_ON(!PageBuddy(page));
	return page_private(page);
}
65

66
#ifdef CONFIG_HAVE_MLOCK
67
extern long mlock_vma_pages_range(struct vm_area_struct *vma,
N
Nick Piggin 已提交
68
			unsigned long start, unsigned long end);
69 70 71 72 73 74
extern void munlock_vma_pages_range(struct vm_area_struct *vma,
			unsigned long start, unsigned long end);
static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
{
	munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
}
75
#endif
N
Nick Piggin 已提交
76

L
Lee Schermerhorn 已提交
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
#ifdef CONFIG_UNEVICTABLE_LRU
/*
 * unevictable_migrate_page() called only from migrate_page_copy() to
 * migrate unevictable flag to new page.
 * Note that the old page has been isolated from the LRU lists at this
 * point so we don't need to worry about LRU statistics.
 */
static inline void unevictable_migrate_page(struct page *new, struct page *old)
{
	if (TestClearPageUnevictable(old))
		SetPageUnevictable(new);
}
#else
static inline void unevictable_migrate_page(struct page *new, struct page *old)
{
}
#endif

95
#ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT
N
Nick Piggin 已提交
96 97 98 99 100 101 102 103 104 105 106 107
/*
 * Called only in fault path via page_evictable() for a new page
 * to determine if it's being mapped into a LOCKED vma.
 * If so, mark page as mlocked.
 */
static inline int is_mlocked_vma(struct vm_area_struct *vma, struct page *page)
{
	VM_BUG_ON(PageLRU(page));

	if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED))
		return 0;

N
Nick Piggin 已提交
108 109 110 111
	if (!TestSetPageMlocked(page)) {
		inc_zone_page_state(page, NR_MLOCK);
		count_vm_event(UNEVICTABLE_PGMLOCKED);
	}
N
Nick Piggin 已提交
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
	return 1;
}

/*
 * must be called with vma's mmap_sem held for read, and page locked.
 */
extern void mlock_vma_page(struct page *page);

/*
 * Clear the page's PageMlocked().  This can be useful in a situation where
 * we want to unconditionally remove a page from the pagecache -- e.g.,
 * on truncation or freeing.
 *
 * It is legal to call this function for any page, mlocked or not.
 * If called for a page that is still mapped by mlocked vmas, all we do
 * is revert to lazy LRU behaviour -- semantics are not broken.
 */
extern void __clear_page_mlock(struct page *page);
static inline void clear_page_mlock(struct page *page)
{
	if (unlikely(TestClearPageMlocked(page)))
		__clear_page_mlock(page);
}

/*
 * mlock_migrate_page - called only from migrate_page_copy() to
N
Nick Piggin 已提交
138
 * migrate the Mlocked page flag; update statistics.
N
Nick Piggin 已提交
139 140 141
 */
static inline void mlock_migrate_page(struct page *newpage, struct page *page)
{
N
Nick Piggin 已提交
142 143 144 145 146
	if (TestClearPageMlocked(page)) {
		unsigned long flags;

		local_irq_save(flags);
		__dec_zone_page_state(page, NR_MLOCK);
N
Nick Piggin 已提交
147
		SetPageMlocked(newpage);
N
Nick Piggin 已提交
148 149 150
		__inc_zone_page_state(newpage, NR_MLOCK);
		local_irq_restore(flags);
	}
N
Nick Piggin 已提交
151 152
}

153
#else /* CONFIG_HAVE_MLOCKED_PAGE_BIT */
N
Nick Piggin 已提交
154 155 156 157 158 159 160 161
static inline int is_mlocked_vma(struct vm_area_struct *v, struct page *p)
{
	return 0;
}
static inline void clear_page_mlock(struct page *page) { }
static inline void mlock_vma_page(struct page *page) { }
static inline void mlock_migrate_page(struct page *new, struct page *old) { }

162
#endif /* CONFIG_HAVE_MLOCKED_PAGE_BIT */
L
Lee Schermerhorn 已提交
163

164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
/*
 * Return the mem_map entry representing the 'offset' subpage within
 * the maximally aligned gigantic page 'base'.  Handle any discontiguity
 * in the mem_map at MAX_ORDER_NR_PAGES boundaries.
 */
static inline struct page *mem_map_offset(struct page *base, int offset)
{
	if (unlikely(offset >= MAX_ORDER_NR_PAGES))
		return pfn_to_page(page_to_pfn(base) + offset);
	return base + offset;
}

/*
 * Iterator over all subpages withing the maximally aligned gigantic
 * page 'base'.  Handle any discontiguity in the mem_map.
 */
static inline struct page *mem_map_next(struct page *iter,
						struct page *base, int offset)
{
	if (unlikely((offset & (MAX_ORDER_NR_PAGES - 1)) == 0)) {
		unsigned long pfn = page_to_pfn(base) + offset;
		if (!pfn_valid(pfn))
			return NULL;
		return pfn_to_page(pfn);
	}
	return iter + 1;
}

192 193 194 195 196 197 198 199 200 201 202 203
/*
 * FLATMEM and DISCONTIGMEM configurations use alloc_bootmem_node,
 * so all functions starting at paging_init should be marked __init
 * in those cases. SPARSEMEM, however, allows for memory hotplug,
 * and alloc_bootmem_node is not used.
 */
#ifdef CONFIG_SPARSEMEM
#define __paginginit __meminit
#else
#define __paginginit __init
#endif

204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
/* Memory initialisation debug and verification */
enum mminit_level {
	MMINIT_WARNING,
	MMINIT_VERIFY,
	MMINIT_TRACE
};

#ifdef CONFIG_DEBUG_MEMORY_INIT

extern int mminit_loglevel;

#define mminit_dprintk(level, prefix, fmt, arg...) \
do { \
	if (level < mminit_loglevel) { \
		printk(level <= MMINIT_WARNING ? KERN_WARNING : KERN_DEBUG); \
		printk(KERN_CONT "mminit::" prefix " " fmt, ##arg); \
	} \
} while (0)

223 224 225
extern void mminit_verify_pageflags_layout(void);
extern void mminit_verify_page_links(struct page *page,
		enum zone_type zone, unsigned long nid, unsigned long pfn);
226
extern void mminit_verify_zonelist(void);
227

228 229 230 231 232 233 234
#else

static inline void mminit_dprintk(enum mminit_level level,
				const char *prefix, const char *fmt, ...)
{
}

235 236 237 238 239 240 241 242
static inline void mminit_verify_pageflags_layout(void)
{
}

static inline void mminit_verify_page_links(struct page *page,
		enum zone_type zone, unsigned long nid, unsigned long pfn)
{
}
243 244 245 246

static inline void mminit_verify_zonelist(void)
{
}
247
#endif /* CONFIG_DEBUG_MEMORY_INIT */
248 249 250 251 252 253 254 255 256 257 258 259

/* mminit_validate_memmodel_limits is independent of CONFIG_DEBUG_MEMORY_INIT */
#if defined(CONFIG_SPARSEMEM)
extern void mminit_validate_memmodel_limits(unsigned long *start_pfn,
				unsigned long *end_pfn);
#else
static inline void mminit_validate_memmodel_limits(unsigned long *start_pfn,
				unsigned long *end_pfn)
{
}
#endif /* CONFIG_SPARSEMEM */

N
Nick Piggin 已提交
260 261 262
#define GUP_FLAGS_WRITE                  0x1
#define GUP_FLAGS_FORCE                  0x2
#define GUP_FLAGS_IGNORE_VMA_PERMISSIONS 0x4
263
#define GUP_FLAGS_IGNORE_SIGKILL         0x8
N
Nick Piggin 已提交
264 265 266 267 268

int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
		     unsigned long start, int len, int flags,
		     struct page **pages, struct vm_area_struct **vmas);

269
#endif