pgtable_64.h 8.9 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
H
H. Peter Anvin 已提交
2 3
#ifndef _ASM_X86_PGTABLE_64_H
#define _ASM_X86_PGTABLE_64_H
L
Linus Torvalds 已提交
4

5
#include <linux/const.h>
6 7
#include <asm/pgtable_64_types.h>

8 9
#ifndef __ASSEMBLY__

L
Linus Torvalds 已提交
10 11 12 13 14
/*
 * This file contains the functions and defines necessary to modify and use
 * the x86-64 page table tree.
 */
#include <asm/processor.h>
J
Jiri Slaby 已提交
15
#include <linux/bitops.h>
L
Linus Torvalds 已提交
16 17
#include <linux/threads.h>

18 19
extern p4d_t level4_kernel_pgt[512];
extern p4d_t level4_ident_pgt[512];
L
Linus Torvalds 已提交
20 21 22
extern pud_t level3_kernel_pgt[512];
extern pud_t level3_ident_pgt[512];
extern pmd_t level2_kernel_pgt[512];
J
Jeremy Fitzhardinge 已提交
23 24
extern pmd_t level2_fixmap_pgt[512];
extern pmd_t level2_ident_pgt[512];
25
extern pte_t level1_fixmap_pgt[512];
26
extern pgd_t init_top_pgt[];
L
Linus Torvalds 已提交
27

28
#define swapper_pg_dir init_top_pgt
L
Linus Torvalds 已提交
29 30

extern void paging_init(void);
31
static inline void sync_initial_page_table(void) { }
L
Linus Torvalds 已提交
32

33
#define pte_ERROR(e)					\
34
	pr_err("%s:%d: bad pte %p(%016lx)\n",		\
35 36
	       __FILE__, __LINE__, &(e), pte_val(e))
#define pmd_ERROR(e)					\
37
	pr_err("%s:%d: bad pmd %p(%016lx)\n",		\
38 39
	       __FILE__, __LINE__, &(e), pmd_val(e))
#define pud_ERROR(e)					\
40
	pr_err("%s:%d: bad pud %p(%016lx)\n",		\
41
	       __FILE__, __LINE__, &(e), pud_val(e))
42 43 44 45 46 47 48

#if CONFIG_PGTABLE_LEVELS >= 5
#define p4d_ERROR(e)					\
	pr_err("%s:%d: bad p4d %p(%016lx)\n",		\
	       __FILE__, __LINE__, &(e), p4d_val(e))
#endif

49
#define pgd_ERROR(e)					\
50
	pr_err("%s:%d: bad pgd %p(%016lx)\n",		\
51
	       __FILE__, __LINE__, &(e), pgd_val(e))
L
Linus Torvalds 已提交
52

53 54
struct mm_struct;

55
void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte);
56 57
void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte);

58 59
static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
				    pte_t *ptep)
L
Linus Torvalds 已提交
60
{
61 62
	*ptep = native_make_pte(0);
}
L
Linus Torvalds 已提交
63

64
static inline void native_set_pte(pte_t *ptep, pte_t pte)
L
Linus Torvalds 已提交
65
{
66 67
	*ptep = pte;
}
L
Linus Torvalds 已提交
68

69 70 71 72 73
static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
{
	native_set_pte(ptep, pte);
}

74 75 76 77 78 79 80 81 82 83
static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
{
	*pmdp = pmd;
}

static inline void native_pmd_clear(pmd_t *pmd)
{
	native_set_pmd(pmd, native_make_pmd(0));
}

84
static inline pte_t native_ptep_get_and_clear(pte_t *xp)
L
Linus Torvalds 已提交
85
{
86 87 88
#ifdef CONFIG_SMP
	return native_make_pte(xchg(&xp->pte, 0));
#else
89 90
	/* native_local_ptep_get_and_clear,
	   but duplicated because of cyclic dependency */
91 92 93 94
	pte_t ret = *xp;
	native_pte_clear(NULL, 0, xp);
	return ret;
#endif
L
Linus Torvalds 已提交
95 96
}

97
static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
L
Linus Torvalds 已提交
98
{
99 100 101 102 103 104 105 106 107
#ifdef CONFIG_SMP
	return native_make_pmd(xchg(&xp->pmd, 0));
#else
	/* native_local_pmdp_get_and_clear,
	   but duplicated because of cyclic dependency */
	pmd_t ret = *xp;
	native_pmd_clear(xp);
	return ret;
#endif
108
}
L
Linus Torvalds 已提交
109

110
static inline void native_set_pud(pud_t *pudp, pud_t pud)
L
Linus Torvalds 已提交
111
{
112
	*pudp = pud;
L
Linus Torvalds 已提交
113 114
}

115 116 117 118
static inline void native_pud_clear(pud_t *pud)
{
	native_set_pud(pud, native_make_pud(0));
}
119

120 121 122 123 124 125 126 127 128 129 130 131 132
static inline pud_t native_pudp_get_and_clear(pud_t *xp)
{
#ifdef CONFIG_SMP
	return native_make_pud(xchg(&xp->pud, 0));
#else
	/* native_local_pudp_get_and_clear,
	 * but duplicated because of cyclic dependency
	 */
	pud_t ret = *xp;

	native_pud_clear(xp);
	return ret;
#endif
133 134
}

135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
#ifdef CONFIG_PAGE_TABLE_ISOLATION
/*
 * All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages
 * (8k-aligned and 8k in size).  The kernel one is at the beginning 4k and
 * the user one is in the last 4k.  To switch between them, you
 * just need to flip the 12th bit in their addresses.
 */
#define PTI_PGTABLE_SWITCH_BIT	PAGE_SHIFT

/*
 * This generates better code than the inline assembly in
 * __set_bit().
 */
static inline void *ptr_set_bit(void *ptr, int bit)
{
	unsigned long __ptr = (unsigned long)ptr;

	__ptr |= BIT(bit);
	return (void *)__ptr;
}
static inline void *ptr_clear_bit(void *ptr, int bit)
{
	unsigned long __ptr = (unsigned long)ptr;

	__ptr &= ~BIT(bit);
	return (void *)__ptr;
}

static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
{
	return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
}

static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
{
	return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
}

static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
{
	return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
}

static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
{
	return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
}
#endif /* CONFIG_PAGE_TABLE_ISOLATION */

/*
 * Page table pages are page-aligned.  The lower half of the top
 * level is used for userspace and the top half for the kernel.
 *
 * Returns true for parts of the PGD that map userspace and
 * false for the parts that map the kernel.
 */
static inline bool pgdp_maps_userspace(void *__ptr)
{
	unsigned long ptr = (unsigned long)__ptr;

	return (ptr & ~PAGE_MASK) < (PAGE_SIZE / 2);
}

#ifdef CONFIG_PAGE_TABLE_ISOLATION
pgd_t __pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd);

/*
 * Take a PGD location (pgdp) and a pgd value that needs to be set there.
 * Populates the user and returns the resulting PGD that must be set in
 * the kernel copy of the page tables.
 */
static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd)
{
	if (!static_cpu_has(X86_FEATURE_PTI))
		return pgd;
	return __pti_set_user_pgd(pgdp, pgd);
}
#else
static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd)
{
	return pgd;
}
#endif

219
static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)
220
{
221 222
	pgd_t pgd;

223
	if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) {
224 225 226 227
		*p4dp = p4d;
		return;
	}

228
	pgd = native_make_pgd(native_p4d_val(p4d));
229
	pgd = pti_set_user_pgd((pgd_t *)p4dp, pgd);
230
	*p4dp = native_make_p4d(native_pgd_val(pgd));
231 232
}

233
static inline void native_p4d_clear(p4d_t *p4d)
234
{
235
	native_set_p4d(p4d, native_make_p4d(0));
236 237
}

238 239
static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
{
240
	*pgdp = pti_set_user_pgd(pgdp, pgd);
241
}
242

243
static inline void native_pgd_clear(pgd_t *pgd)
244
{
245
	native_set_pgd(pgd, native_make_pgd(0));
246 247
}

248
extern void sync_global_pgds(unsigned long start, unsigned long end);
249

L
Linus Torvalds 已提交
250 251 252 253 254 255 256 257
/*
 * Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 */

/*
 * Level 4 access.
 */
258
static inline int pgd_large(pgd_t pgd) { return 0; }
259
#define mk_kernel_pgd(address) __pgd((address) | _KERNPG_TABLE)
L
Linus Torvalds 已提交
260 261 262 263 264 265 266 267

/* PUD - Level3 access */

/* PMD  - Level 2 access */

/* PTE - Level 1 access. */

/* x86-64 always has all page tables mapped. */
268
#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address))
269
#define pte_unmap(pte) ((void)(pte))/* NOP */
L
Linus Torvalds 已提交
270

271 272 273
/*
 * Encode and de-code a swap entry
 *
274 275 276
 * |     ...            | 11| 10|  9|8|7|6|5| 4| 3|2| 1|0| <- bit number
 * |     ...            |SW3|SW2|SW1|G|L|D|A|CD|WT|U| W|P| <- bit names
 * | OFFSET (14->63) | TYPE (9-13)  |0|0|X|X| X| X|X|SD|0| <- swp entry
277 278 279 280 281 282
 *
 * G (8) is aliased and used as a PROT_NONE indicator for
 * !present ptes.  We need to start storing swap entries above
 * there.  We also need to avoid using A and D because of an
 * erratum where they can be incorrectly set by hardware on
 * non-present PTEs.
283 284 285 286 287 288
 *
 * SD (1) in swp entry is used to store soft dirty bit, which helps us
 * remember soft dirty over page migration
 *
 * Bit 7 in swp entry should be 0 because pmd_present checks not only P,
 * but also L and G.
289 290
 */
#define SWP_TYPE_FIRST_BIT (_PAGE_BIT_PROTNONE + 1)
291
#define SWP_TYPE_BITS 5
292
/* Place the offset above the type: */
293
#define SWP_OFFSET_FIRST_BIT (SWP_TYPE_FIRST_BIT + SWP_TYPE_BITS)
J
Jan Beulich 已提交
294 295 296

#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS)

297
#define __swp_type(x)			(((x).val >> (SWP_TYPE_FIRST_BIT)) \
J
Jan Beulich 已提交
298
					 & ((1U << SWP_TYPE_BITS) - 1))
299
#define __swp_offset(x)			((x).val >> SWP_OFFSET_FIRST_BIT)
J
Jan Beulich 已提交
300
#define __swp_entry(type, offset)	((swp_entry_t) { \
301 302
					 ((type) << (SWP_TYPE_FIRST_BIT)) \
					 | ((offset) << SWP_OFFSET_FIRST_BIT) })
303
#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val((pte)) })
304
#define __pmd_to_swp_entry(pmd)		((swp_entry_t) { pmd_val((pmd)) })
305
#define __swp_entry_to_pte(x)		((pte_t) { .pte = (x).val })
306
#define __swp_entry_to_pmd(x)		((pmd_t) { .pmd = (x).val })
L
Linus Torvalds 已提交
307

308
extern int kern_addr_valid(unsigned long addr);
309
extern void cleanup_highmap(void);
L
Linus Torvalds 已提交
310 311

#define HAVE_ARCH_UNMAPPED_AREA
J
Jiri Kosina 已提交
312
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
L
Linus Torvalds 已提交
313 314

#define pgtable_cache_init()   do { } while (0)
315
#define check_pgt_cache()      do { } while (0)
L
Linus Torvalds 已提交
316 317 318 319 320 321

#define PAGE_AGP    PAGE_KERNEL_NOCACHE
#define HAVE_PAGE_AGP 1

/* fs/proc/kcore.c */
#define	kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK)
322
#define	kc_offset_to_vaddr(o) ((o) | ~__VIRTUAL_MASK)
L
Linus Torvalds 已提交
323 324

#define __HAVE_ARCH_PTE_SAME
325

326 327 328 329 330
#define vmemmap ((struct page *)VMEMMAP_START)

extern void init_extra_mapping_uc(unsigned long phys, unsigned long size);
extern void init_extra_mapping_wb(unsigned long phys, unsigned long size);

331 332 333 334 335 336 337 338 339 340 341 342 343 344
#define gup_fast_permitted gup_fast_permitted
static inline bool gup_fast_permitted(unsigned long start, int nr_pages,
		int write)
{
	unsigned long len, end;

	len = (unsigned long)nr_pages << PAGE_SHIFT;
	end = start + len;
	if (end < start)
		return false;
	if (end >> __VIRTUAL_MASK_SHIFT)
		return false;
	return true;
}
345

346
#endif /* !__ASSEMBLY__ */
H
H. Peter Anvin 已提交
347
#endif /* _ASM_X86_PGTABLE_64_H */