pgtable-3level.h 10.3 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
H
H. Peter Anvin 已提交
2 3
#ifndef _ASM_X86_PGTABLE_3LEVEL_H
#define _ASM_X86_PGTABLE_3LEVEL_H
L
Linus Torvalds 已提交
4 5 6 7 8 9 10 11

/*
 * Intel Physical Address Extension (PAE) Mode - three-level page
 * tables on PPro+ CPUs.
 *
 * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
 */

12
#define pte_ERROR(e)							\
13
	pr_err("%s:%d: bad pte %p(%08lx%08lx)\n",			\
14 15
	       __FILE__, __LINE__, &(e), (e).pte_high, (e).pte_low)
#define pmd_ERROR(e)							\
16
	pr_err("%s:%d: bad pmd %p(%016Lx)\n",				\
17 18
	       __FILE__, __LINE__, &(e), pmd_val(e))
#define pgd_ERROR(e)							\
19
	pr_err("%s:%d: bad pgd %p(%016Lx)\n",				\
20
	       __FILE__, __LINE__, &(e), pgd_val(e))
21

L
Linus Torvalds 已提交
22 23 24 25 26 27
/* Rules for using set_pte: the pte being assigned *must* be
 * either not present or in a state where the hardware will
 * not attempt to update the pte.  In places where this is
 * not possible, use pte_get_and_clear to obtain the old pte
 * value and then use set_pte to update it.  -ben
 */
28
static inline void native_set_pte(pte_t *ptep, pte_t pte)
L
Linus Torvalds 已提交
29 30 31 32 33 34
{
	ptep->pte_high = pte.pte_high;
	smp_wmb();
	ptep->pte_low = pte.pte_low;
}

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
#define pmd_read_atomic pmd_read_atomic
/*
 * pte_offset_map_lock on 32bit PAE kernels was reading the pmd_t with
 * a "*pmdp" dereference done by gcc. Problem is, in certain places
 * where pte_offset_map_lock is called, concurrent page faults are
 * allowed, if the mmap_sem is hold for reading. An example is mincore
 * vs page faults vs MADV_DONTNEED. On the page fault side
 * pmd_populate rightfully does a set_64bit, but if we're reading the
 * pmd_t with a "*pmdp" on the mincore side, a SMP race can happen
 * because gcc will not read the 64bit of the pmd atomically. To fix
 * this all places running pmd_offset_map_lock() while holding the
 * mmap_sem in read mode, shall read the pmdp pointer using this
 * function to know if the pmd is null nor not, and in turn to know if
 * they can run pmd_offset_map_lock or pmd_trans_huge or other pmd
 * operations.
 *
51 52 53
 * Without THP if the mmap_sem is hold for reading, the pmd can only
 * transition from null to not null while pmd_read_atomic runs. So
 * we can always return atomic pmd values with this function.
54 55
 *
 * With THP if the mmap_sem is hold for reading, the pmd can become
56 57 58 59 60 61 62 63 64 65 66 67 68 69
 * trans_huge or none or point to a pte (and in turn become "stable")
 * at any time under pmd_read_atomic. We could read it really
 * atomically here with a atomic64_read for the THP enabled case (and
 * it would be a whole lot simpler), but to avoid using cmpxchg8b we
 * only return an atomic pmdval if the low part of the pmdval is later
 * found stable (i.e. pointing to a pte). And we're returning a none
 * pmdval if the low part of the pmd is none. In some cases the high
 * and low part of the pmdval returned may not be consistent if THP is
 * enabled (the low part may point to previously mapped hugepage,
 * while the high part may point to a more recently mapped hugepage),
 * but pmd_none_or_trans_huge_or_clear_bad() only needs the low part
 * of the pmd to be read atomically to decide if the pmd is unstable
 * or not, with the only exception of when the low part of the pmd is
 * zero in which case we return a none pmd.
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
 */
static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
{
	pmdval_t ret;
	u32 *tmp = (u32 *)pmdp;

	ret = (pmdval_t) (*tmp);
	if (ret) {
		/*
		 * If the low part is null, we must not read the high part
		 * or we can end up with a partial pmd.
		 */
		smp_rmb();
		ret |= ((pmdval_t)*(tmp + 1)) << 32;
	}

	return (pmd_t) { ret };
}

89 90
static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
{
91
	set_64bit((unsigned long long *)(ptep), native_pte_val(pte));
92
}
93

94 95
static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
{
96
	set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd));
97
}
98

99 100
static inline void native_set_pud(pud_t *pudp, pud_t pud)
{
101 102 103
#ifdef CONFIG_PAGE_TABLE_ISOLATION
	pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd);
#endif
104
	set_64bit((unsigned long long *)(pudp), native_pud_val(pud));
105
}
L
Linus Torvalds 已提交
106

107 108 109 110 111
/*
 * For PTEs and PDEs, we must clear the P-bit first when clearing a page table
 * entry, so clear the bottom half first and enforce ordering with a compiler
 * barrier.
 */
112 113
static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
				    pte_t *ptep)
114 115 116 117 118 119
{
	ptep->pte_low = 0;
	smp_wmb();
	ptep->pte_high = 0;
}

120
static inline void native_pmd_clear(pmd_t *pmd)
121 122 123 124 125 126
{
	u32 *tmp = (u32 *)pmd;
	*tmp = 0;
	smp_wmb();
	*(tmp + 1) = 0;
}
127

128 129 130 131
static inline void native_pud_clear(pud_t *pudp)
{
}

132 133 134 135 136
static inline void pud_clear(pud_t *pudp)
{
	set_pud(pudp, __pud(0));

	/*
137 138 139 140
	 * According to Intel App note "TLBs, Paging-Structure Caches,
	 * and Their Invalidation", April 2007, document 317080-001,
	 * section 8.1: in PAE mode we explicitly have to flush the
	 * TLB via cr3 if the top-level pgd is changed...
141
	 *
142 143 144
	 * Currently all places where pud_clear() is called either have
	 * flush_tlb_mm() followed or don't need TLB flush (x86_64 code or
	 * pud_clear_bad()), so we don't need TLB flush here.
145 146
	 */
}
147

148
#ifdef CONFIG_SMP
149
static inline pte_t native_ptep_get_and_clear(pte_t *ptep)
L
Linus Torvalds 已提交
150 151 152 153 154 155 156 157 158 159
{
	pte_t res;

	/* xchg acts as a barrier before the setting of the high bits */
	res.pte_low = xchg(&ptep->pte_low, 0);
	res.pte_high = ptep->pte_high;
	ptep->pte_high = 0;

	return res;
}
160 161 162
#else
#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
#endif
L
Linus Torvalds 已提交
163

J
Johannes Weiner 已提交
164 165 166 167 168 169 170
union split_pmd {
	struct {
		u32 pmd_low;
		u32 pmd_high;
	};
	pmd_t pmd;
};
171 172

#ifdef CONFIG_SMP
J
Johannes Weiner 已提交
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
{
	union split_pmd res, *orig = (union split_pmd *)pmdp;

	/* xchg acts as a barrier before setting of the high bits */
	res.pmd_low = xchg(&orig->pmd_low, 0);
	res.pmd_high = orig->pmd_high;
	orig->pmd_high = 0;

	return res.pmd;
}
#else
#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
#endif

188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
#ifndef pmdp_establish
#define pmdp_establish pmdp_establish
static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
		unsigned long address, pmd_t *pmdp, pmd_t pmd)
{
	pmd_t old;

	/*
	 * If pmd has present bit cleared we can get away without expensive
	 * cmpxchg64: we can update pmdp half-by-half without racing with
	 * anybody.
	 */
	if (!(pmd_val(pmd) & _PAGE_PRESENT)) {
		union split_pmd old, new, *ptr;

		ptr = (union split_pmd *)pmdp;

		new.pmd = pmd;

		/* xchg acts as a barrier before setting of the high bits */
		old.pmd_low = xchg(&ptr->pmd_low, new.pmd_low);
		old.pmd_high = ptr->pmd_high;
		ptr->pmd_high = new.pmd_high;
		return old.pmd;
	}

	do {
		old = *pmdp;
	} while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);

	return old;
}
#endif

222 223 224 225 226 227 228 229 230 231 232 233 234
#ifdef CONFIG_SMP
union split_pud {
	struct {
		u32 pud_low;
		u32 pud_high;
	};
	pud_t pud;
};

static inline pud_t native_pudp_get_and_clear(pud_t *pudp)
{
	union split_pud res, *orig = (union split_pud *)pudp;

235 236 237 238
#ifdef CONFIG_PAGE_TABLE_ISOLATION
	pti_set_user_pgtbl(&pudp->p4d.pgd, __pgd(0));
#endif

239 240 241 242 243 244 245 246 247 248 249
	/* xchg acts as a barrier before setting of the high bits */
	res.pud_low = xchg(&orig->pud_low, 0);
	res.pud_high = orig->pud_high;
	orig->pud_high = 0;

	return res.pud;
}
#else
#define native_pudp_get_and_clear(xp) native_local_pudp_get_and_clear(xp)
#endif

L
Linus Torvalds 已提交
250
/* Encode and de-code a swap entry */
251 252 253 254 255 256 257
#define SWP_TYPE_BITS		5

#define SWP_OFFSET_FIRST_BIT	(_PAGE_BIT_PROTNONE + 1)

/* We always extract/encode the offset by shifting it all the way up, and then down again */
#define SWP_OFFSET_SHIFT	(SWP_OFFSET_FIRST_BIT + SWP_TYPE_BITS)

J
Jan Beulich 已提交
258
#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > 5)
L
Linus Torvalds 已提交
259 260 261
#define __swp_type(x)			(((x).val) & 0x1f)
#define __swp_offset(x)			((x).val >> 5)
#define __swp_entry(type, offset)	((swp_entry_t){(type) | (offset) << 5})
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287

/*
 * Normally, __swp_entry() converts from arch-independent swp_entry_t to
 * arch-dependent swp_entry_t, and __swp_entry_to_pte() just stores the result
 * to pte. But here we have 32bit swp_entry_t and 64bit pte, and need to use the
 * whole 64 bits. Thus, we shift the "real" arch-dependent conversion to
 * __swp_entry_to_pte() through the following helper macro based on 64bit
 * __swp_entry().
 */
#define __swp_pteval_entry(type, offset) ((pteval_t) { \
	(~(pteval_t)(offset) << SWP_OFFSET_SHIFT >> SWP_TYPE_BITS) \
	| ((pteval_t)(type) << (64 - SWP_TYPE_BITS)) })

#define __swp_entry_to_pte(x)	((pte_t){ .pte = \
		__swp_pteval_entry(__swp_type(x), __swp_offset(x)) })
/*
 * Analogically, __pte_to_swp_entry() doesn't just extract the arch-dependent
 * swp_entry_t, but also has to convert it from 64bit to the 32bit
 * intermediate representation, using the following macros based on 64bit
 * __swp_type() and __swp_offset().
 */
#define __pteval_swp_type(x) ((unsigned long)((x).pte >> (64 - SWP_TYPE_BITS)))
#define __pteval_swp_offset(x) ((unsigned long)(~((x).pte) << SWP_TYPE_BITS >> SWP_OFFSET_SHIFT))

#define __pte_to_swp_entry(pte)	(__swp_entry(__pteval_swp_type(pte), \
					     __pteval_swp_offset(pte)))
L
Linus Torvalds 已提交
288

289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
#define gup_get_pte gup_get_pte
/*
 * WARNING: only to be used in the get_user_pages_fast() implementation.
 *
 * With get_user_pages_fast(), we walk down the pagetables without taking
 * any locks.  For this we would like to load the pointers atomically,
 * but that is not possible (without expensive cmpxchg8b) on PAE.  What
 * we do have is the guarantee that a PTE will only either go from not
 * present to present, or present to not present or both -- it will not
 * switch to a completely different present page without a TLB flush in
 * between; something that we are blocking by holding interrupts off.
 *
 * Setting ptes from not present to present goes:
 *
 *   ptep->pte_high = h;
 *   smp_wmb();
 *   ptep->pte_low = l;
 *
 * And present to not present goes:
 *
 *   ptep->pte_low = 0;
 *   smp_wmb();
 *   ptep->pte_high = 0;
 *
 * We must ensure here that the load of pte_low sees 'l' iff pte_high
 * sees 'h'. We load pte_high *after* loading pte_low, which ensures we
 * don't see an older value of pte_high.  *Then* we recheck pte_low,
 * which ensures that we haven't picked up a changed pte high. We might
 * have gotten rubbish values from pte_low and pte_high, but we are
 * guaranteed that pte_low will not have the present bit set *unless*
 * it is 'l'. Because get_user_pages_fast() only operates on present ptes
 * we're safe.
 */
static inline pte_t gup_get_pte(pte_t *ptep)
{
	pte_t pte;

	do {
		pte.pte_low = ptep->pte_low;
		smp_rmb();
		pte.pte_high = ptep->pte_high;
		smp_rmb();
	} while (unlikely(pte.pte_low != ptep->pte_low));

	return pte;
}

336 337
#include <asm/pgtable-invert.h>

H
H. Peter Anvin 已提交
338
#endif /* _ASM_X86_PGTABLE_3LEVEL_H */