pgtable_64.h 7.8 KB
Newer Older
H
H. Peter Anvin 已提交
1 2
#ifndef _ASM_X86_PGTABLE_64_H
#define _ASM_X86_PGTABLE_64_H
L
Linus Torvalds 已提交
3

4
#include <linux/const.h>
5 6
#ifndef __ASSEMBLY__

L
Linus Torvalds 已提交
7 8 9 10 11
/*
 * This file contains the functions and defines necessary to modify and use
 * the x86-64 page table tree.
 */
#include <asm/processor.h>
J
Jiri Slaby 已提交
12
#include <linux/bitops.h>
L
Linus Torvalds 已提交
13 14 15 16 17 18
#include <linux/threads.h>
#include <asm/pda.h>

extern pud_t level3_kernel_pgt[512];
extern pud_t level3_ident_pgt[512];
extern pmd_t level2_kernel_pgt[512];
J
Jeremy Fitzhardinge 已提交
19 20
extern pmd_t level2_fixmap_pgt[512];
extern pmd_t level2_ident_pgt[512];
L
Linus Torvalds 已提交
21 22
extern pgd_t init_level4_pgt[];

23
#define swapper_pg_dir init_level4_pgt
L
Linus Torvalds 已提交
24 25 26

extern void paging_init(void);

27 28
#endif /* !__ASSEMBLY__ */

J
Jeremy Fitzhardinge 已提交
29
#define SHARED_KERNEL_PMD	0
30

L
Linus Torvalds 已提交
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
/*
 * PGDIR_SHIFT determines what a top-level page table entry can map
 */
#define PGDIR_SHIFT	39
#define PTRS_PER_PGD	512

/*
 * 3rd level page
 */
#define PUD_SHIFT	30
#define PTRS_PER_PUD	512

/*
 * PMD_SHIFT determines the size of the area a middle-level
 * page table can map
 */
#define PMD_SHIFT	21
#define PTRS_PER_PMD	512

/*
 * entries per page directory level
 */
#define PTRS_PER_PTE	512

55 56
#ifndef __ASSEMBLY__

57 58 59 60 61 62 63 64 65 66 67 68
#define pte_ERROR(e)					\
	printk("%s:%d: bad pte %p(%016lx).\n",		\
	       __FILE__, __LINE__, &(e), pte_val(e))
#define pmd_ERROR(e)					\
	printk("%s:%d: bad pmd %p(%016lx).\n",		\
	       __FILE__, __LINE__, &(e), pmd_val(e))
#define pud_ERROR(e)					\
	printk("%s:%d: bad pud %p(%016lx).\n",		\
	       __FILE__, __LINE__, &(e), pud_val(e))
#define pgd_ERROR(e)					\
	printk("%s:%d: bad pgd %p(%016lx).\n",		\
	       __FILE__, __LINE__, &(e), pgd_val(e))
L
Linus Torvalds 已提交
69 70 71 72

#define pgd_none(x)	(!pgd_val(x))
#define pud_none(x)	(!pud_val(x))

73 74
struct mm_struct;

75 76 77
void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte);


78 79
static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
				    pte_t *ptep)
L
Linus Torvalds 已提交
80
{
81 82
	*ptep = native_make_pte(0);
}
L
Linus Torvalds 已提交
83

84
static inline void native_set_pte(pte_t *ptep, pte_t pte)
L
Linus Torvalds 已提交
85
{
86 87
	*ptep = pte;
}
L
Linus Torvalds 已提交
88

89 90 91 92 93
static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
{
	native_set_pte(ptep, pte);
}

94
static inline pte_t native_ptep_get_and_clear(pte_t *xp)
L
Linus Torvalds 已提交
95
{
96 97 98
#ifdef CONFIG_SMP
	return native_make_pte(xchg(&xp->pte, 0));
#else
99 100
	/* native_local_ptep_get_and_clear,
	   but duplicated because of cyclic dependency */
101 102 103 104
	pte_t ret = *xp;
	native_pte_clear(NULL, 0, xp);
	return ret;
#endif
L
Linus Torvalds 已提交
105 106
}

107
static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
L
Linus Torvalds 已提交
108
{
109
	*pmdp = pmd;
L
Linus Torvalds 已提交
110 111
}

112
static inline void native_pmd_clear(pmd_t *pmd)
L
Linus Torvalds 已提交
113
{
114 115
	native_set_pmd(pmd, native_make_pmd(0));
}
L
Linus Torvalds 已提交
116

117
static inline void native_set_pud(pud_t *pudp, pud_t pud)
L
Linus Torvalds 已提交
118
{
119
	*pudp = pud;
L
Linus Torvalds 已提交
120 121
}

122 123 124 125
static inline void native_pud_clear(pud_t *pud)
{
	native_set_pud(pud, native_make_pud(0));
}
126

127 128 129 130
static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
{
	*pgdp = pgd;
}
131

132
static inline void native_pgd_clear(pgd_t *pgd)
133
{
134
	native_set_pgd(pgd, native_make_pgd(0));
135 136
}

137 138
#endif /* !__ASSEMBLY__ */

139 140 141 142 143 144
#define PMD_SIZE	(_AC(1, UL) << PMD_SHIFT)
#define PMD_MASK	(~(PMD_SIZE - 1))
#define PUD_SIZE	(_AC(1, UL) << PUD_SHIFT)
#define PUD_MASK	(~(PUD_SIZE - 1))
#define PGDIR_SIZE	(_AC(1, UL) << PGDIR_SHIFT)
#define PGDIR_MASK	(~(PGDIR_SIZE - 1))
L
Linus Torvalds 已提交
145 146


I
Ingo Molnar 已提交
147
#define MAXMEM		 _AC(__AC(1, UL) << MAX_PHYSMEM_BITS, UL)
148 149
#define VMALLOC_START    _AC(0xffffc20000000000, UL)
#define VMALLOC_END      _AC(0xffffe1ffffffffff, UL)
150
#define VMEMMAP_START	 _AC(0xffffe20000000000, UL)
151
#define MODULES_VADDR    _AC(0xffffffffa0000000, UL)
152
#define MODULES_END      _AC(0xffffffffff000000, UL)
L
Linus Torvalds 已提交
153 154
#define MODULES_LEN   (MODULES_END - MODULES_VADDR)

155 156
#ifndef __ASSEMBLY__

H
Hugh Dickins 已提交
157
static inline int pgd_bad(pgd_t pgd)
J
Jan Beulich 已提交
158
{
159
	return (pgd_val(pgd) & ~(PTE_PFN_MASK | _PAGE_USER)) != _KERNPG_TABLE;
J
Jan Beulich 已提交
160
}
L
Linus Torvalds 已提交
161

H
Hugh Dickins 已提交
162
static inline int pud_bad(pud_t pud)
L
Linus Torvalds 已提交
163
{
164
	return (pud_val(pud) & ~(PTE_PFN_MASK | _PAGE_USER)) != _KERNPG_TABLE;
J
Jan Beulich 已提交
165 166
}

H
Hugh Dickins 已提交
167
static inline int pmd_bad(pmd_t pmd)
J
Jan Beulich 已提交
168
{
169
	return (pmd_val(pmd) & ~(PTE_PFN_MASK | _PAGE_USER)) != _KERNPG_TABLE;
L
Linus Torvalds 已提交
170 171
}

172
#define pages_to_mb(x)	((x) >> (20 - PAGE_SHIFT))   /* FIXME: is this right? */
L
Linus Torvalds 已提交
173 174 175 176 177 178 179 180 181

/*
 * Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 */

/*
 * Level 4 access.
 */
182
#define pgd_page_vaddr(pgd)						\
183
	((unsigned long)__va((unsigned long)pgd_val((pgd)) & PTE_PFN_MASK))
184
#define pgd_page(pgd)		(pfn_to_page(pgd_val((pgd)) >> PAGE_SHIFT))
L
Linus Torvalds 已提交
185
#define pgd_present(pgd) (pgd_val(pgd) & _PAGE_PRESENT)
186
static inline int pgd_large(pgd_t pgd) { return 0; }
187
#define mk_kernel_pgd(address) __pgd((address) | _KERNPG_TABLE)
L
Linus Torvalds 已提交
188 189 190

/* PUD - Level3 access */
/* to find an entry in a page-table-directory. */
191 192 193 194 195 196
#define pud_page_vaddr(pud)						\
	((unsigned long)__va(pud_val((pud)) & PHYSICAL_PAGE_MASK))
#define pud_page(pud)	(pfn_to_page(pud_val((pud)) >> PAGE_SHIFT))
#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
#define pud_offset(pgd, address)					\
	((pud_t *)pgd_page_vaddr(*(pgd)) + pud_index((address)))
L
Linus Torvalds 已提交
197

198 199
static inline int pud_large(pud_t pte)
{
200 201
	return (pud_val(pte) & (_PAGE_PSE | _PAGE_PRESENT)) ==
		(_PAGE_PSE | _PAGE_PRESENT);
202 203
}

L
Linus Torvalds 已提交
204
/* PMD  - Level 2 access */
205
#define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val((pmd)) & PTE_PFN_MASK))
206 207 208 209 210 211 212 213 214 215 216 217 218
#define pmd_page(pmd)		(pfn_to_page(pmd_val((pmd)) >> PAGE_SHIFT))

#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
#define pmd_offset(dir, address) ((pmd_t *)pud_page_vaddr(*(dir)) + \
				  pmd_index(address))
#define pmd_none(x)	(!pmd_val((x)))
#define pmd_present(x)	(pmd_val((x)) & _PAGE_PRESENT)
#define pfn_pmd(nr, prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val((prot))))
#define pmd_pfn(x)  ((pmd_val((x)) & __PHYSICAL_MASK) >> PAGE_SHIFT)

#define pte_to_pgoff(pte) ((pte_val((pte)) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT)
#define pgoff_to_pte(off) ((pte_t) { .pte = ((off) << PAGE_SHIFT) |	\
					    _PAGE_FILE })
L
Linus Torvalds 已提交
219 220 221 222 223
#define PTE_FILE_MAX_BITS __PHYSICAL_MASK_SHIFT

/* PTE - Level 1 access. */

/* page, protection -> pte */
224 225 226
#define mk_pte(page, pgprot)	pfn_pte(page_to_pfn((page)), (pgprot))

#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
227
#define pte_offset_kernel(dir, address) ((pte_t *) pmd_page_vaddr(*(dir)) + \
228
					 pte_index((address)))
L
Linus Torvalds 已提交
229 230

/* x86-64 always has all page tables mapped. */
231 232
#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address))
#define pte_offset_map_nested(dir, address) pte_offset_kernel((dir), (address))
L
Linus Torvalds 已提交
233
#define pte_unmap(pte) /* NOP */
234
#define pte_unmap_nested(pte) /* NOP */
L
Linus Torvalds 已提交
235

236
#define update_mmu_cache(vma, address, pte) do { } while (0)
L
Linus Torvalds 已提交
237

I
Ingo Molnar 已提交
238 239
extern int direct_gbpages;

L
Linus Torvalds 已提交
240
/* Encode and de-code a swap entry */
J
Jan Beulich 已提交
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
#if _PAGE_BIT_FILE < _PAGE_BIT_PROTNONE
#define SWP_TYPE_BITS (_PAGE_BIT_FILE - _PAGE_BIT_PRESENT - 1)
#define SWP_OFFSET_SHIFT (_PAGE_BIT_PROTNONE + 1)
#else
#define SWP_TYPE_BITS (_PAGE_BIT_PROTNONE - _PAGE_BIT_PRESENT - 1)
#define SWP_OFFSET_SHIFT (_PAGE_BIT_FILE + 1)
#endif

#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS)

#define __swp_type(x)			(((x).val >> (_PAGE_BIT_PRESENT + 1)) \
					 & ((1U << SWP_TYPE_BITS) - 1))
#define __swp_offset(x)			((x).val >> SWP_OFFSET_SHIFT)
#define __swp_entry(type, offset)	((swp_entry_t) { \
					 ((type) << (_PAGE_BIT_PRESENT + 1)) \
					 | ((offset) << SWP_OFFSET_SHIFT) })
257
#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val((pte)) })
258
#define __swp_entry_to_pte(x)		((pte_t) { .pte = (x).val })
L
Linus Torvalds 已提交
259

260
extern int kern_addr_valid(unsigned long addr);
261
extern void cleanup_highmap(void);
L
Linus Torvalds 已提交
262

263 264
#define io_remap_pfn_range(vma, vaddr, pfn, size, prot)	\
	remap_pfn_range(vma, vaddr, pfn, size, prot)
L
Linus Torvalds 已提交
265 266

#define HAVE_ARCH_UNMAPPED_AREA
J
Jiri Kosina 已提交
267
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
L
Linus Torvalds 已提交
268 269

#define pgtable_cache_init()   do { } while (0)
270
#define check_pgt_cache()      do { } while (0)
L
Linus Torvalds 已提交
271 272 273 274 275 276

#define PAGE_AGP    PAGE_KERNEL_NOCACHE
#define HAVE_PAGE_AGP 1

/* fs/proc/kcore.c */
#define	kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK)
277 278 279 280
#define	kc_offset_to_vaddr(o)				\
	(((o) & (1UL << (__VIRTUAL_MASK_SHIFT - 1)))	\
	 ? ((o) | ~__VIRTUAL_MASK)			\
	 : (o))
L
Linus Torvalds 已提交
281 282

#define __HAVE_ARCH_PTE_SAME
283
#endif /* !__ASSEMBLY__ */
L
Linus Torvalds 已提交
284

H
H. Peter Anvin 已提交
285
#endif /* _ASM_X86_PGTABLE_64_H */