pgtable.h 42.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 *  S390 version
3
 *    Copyright IBM Corp. 1999, 2000
L
Linus Torvalds 已提交
4 5 6 7 8 9 10 11 12 13 14
 *    Author(s): Hartmut Penner (hp@de.ibm.com)
 *               Ulrich Weigand (weigand@de.ibm.com)
 *               Martin Schwidefsky (schwidefsky@de.ibm.com)
 *
 *  Derived from "include/asm-i386/pgtable.h"
 */

#ifndef _ASM_S390_PGTABLE_H
#define _ASM_S390_PGTABLE_H

/*
15 16 17
 * The Linux memory management assumes a three-level page table setup.
 * For s390 64 bit we use up to four of the five levels the hardware
 * provides (region first tables are not used).
L
Linus Torvalds 已提交
18 19 20 21 22 23 24 25 26
 *
 * The "pgd_xxx()" functions are trivial for a folded two-level
 * setup: the pgd is never bad, and a pmd always exists (as it's folded
 * into the pgd entry)
 *
 * This file contains the functions and defines necessary to modify and use
 * the S390 page table tree.
 */
#ifndef __ASSEMBLY__
27
#include <linux/sched.h>
28
#include <linux/mm_types.h>
29
#include <linux/page-flags.h>
30
#include <linux/radix-tree.h>
31
#include <linux/atomic.h>
L
Linus Torvalds 已提交
32
#include <asm/bug.h>
33
#include <asm/page.h>
L
Linus Torvalds 已提交
34 35 36

extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096)));
extern void paging_init(void);
37
extern void vmem_map_init(void);
38 39
pmd_t *vmem_pmd_alloc(void);
pte_t *vmem_pte_alloc(void);
L
Linus Torvalds 已提交
40

41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
enum {
	PG_DIRECT_MAP_4K = 0,
	PG_DIRECT_MAP_1M,
	PG_DIRECT_MAP_2G,
	PG_DIRECT_MAP_MAX
};

extern atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX];

static inline void update_page_count(int level, long count)
{
	if (IS_ENABLED(CONFIG_PROC_FS))
		atomic_long_add(count, &direct_pages_count[level]);
}

struct seq_file;
void arch_report_meminfo(struct seq_file *m);

L
Linus Torvalds 已提交
59 60 61 62
/*
 * The S390 doesn't have any external MMU info: the kernel page
 * tables contain all the necessary information.
 */
63
#define update_mmu_cache(vma, address, ptep)     do { } while (0)
64
#define update_mmu_cache_pmd(vma, address, ptep) do { } while (0)
L
Linus Torvalds 已提交
65 66

/*
67
 * ZERO_PAGE is a global shared page that is always zero; used
L
Linus Torvalds 已提交
68 69
 * for zero-mapped memory areas etc..
 */
70 71 72 73 74 75 76

extern unsigned long empty_zero_page;
extern unsigned long zero_page_mask;

#define ZERO_PAGE(vaddr) \
	(virt_to_page((void *)(empty_zero_page + \
	 (((unsigned long)(vaddr)) &zero_page_mask))))
77
#define __HAVE_COLOR_ZERO_PAGE
78

79
/* TODO: s390 cannot support io_remap_pfn_range... */
L
Linus Torvalds 已提交
80 81 82 83 84 85 86
#endif /* !__ASSEMBLY__ */

/*
 * PMD_SHIFT determines the size of the area a second-level page
 * table can map
 * PGDIR_SHIFT determines what a third-level page table entry can map
 */
H
Heiko Carstens 已提交
87 88 89
#define PMD_SHIFT	20
#define PUD_SHIFT	31
#define PGDIR_SHIFT	42
L
Linus Torvalds 已提交
90 91 92

#define PMD_SIZE        (1UL << PMD_SHIFT)
#define PMD_MASK        (~(PMD_SIZE-1))
M
Martin Schwidefsky 已提交
93 94
#define PUD_SIZE	(1UL << PUD_SHIFT)
#define PUD_MASK	(~(PUD_SIZE-1))
95 96
#define PGDIR_SIZE	(1UL << PGDIR_SHIFT)
#define PGDIR_MASK	(~(PGDIR_SIZE-1))
L
Linus Torvalds 已提交
97 98 99 100 101 102 103

/*
 * entries per page directory level: the S390 is two-level, so
 * we don't really have any PMD directory physically.
 * for S390 segment-table entries are combined to one PGD
 * that leads to 1024 pte per pgd
 */
104 105
#define PTRS_PER_PTE	256
#define PTRS_PER_PMD	2048
106
#define PTRS_PER_PUD	2048
107
#define PTRS_PER_PGD	2048
L
Linus Torvalds 已提交
108

109
#define FIRST_USER_ADDRESS  0UL
110

L
Linus Torvalds 已提交
111 112 113 114
#define pte_ERROR(e) \
	printk("%s:%d: bad pte %p.\n", __FILE__, __LINE__, (void *) pte_val(e))
#define pmd_ERROR(e) \
	printk("%s:%d: bad pmd %p.\n", __FILE__, __LINE__, (void *) pmd_val(e))
M
Martin Schwidefsky 已提交
115 116
#define pud_ERROR(e) \
	printk("%s:%d: bad pud %p.\n", __FILE__, __LINE__, (void *) pud_val(e))
L
Linus Torvalds 已提交
117 118 119 120 121
#define pgd_ERROR(e) \
	printk("%s:%d: bad pgd %p.\n", __FILE__, __LINE__, (void *) pgd_val(e))

#ifndef __ASSEMBLY__
/*
122 123
 * The vmalloc and module area will always be on the topmost area of the
 * kernel mapping. We reserve 128GB (64bit) for vmalloc and modules.
124 125 126 127
 * On 64 bit kernels we have a 2GB area at the top of the vmalloc area where
 * modules will reside. That makes sure that inter module branches always
 * happen without trampolines and in addition the placement within a 2GB frame
 * is branch prediction unit friendly.
H
Heiko Carstens 已提交
128
 */
129
extern unsigned long VMALLOC_START;
130 131
extern unsigned long VMALLOC_END;
extern struct page *vmemmap;
132

133
#define VMEM_MAX_PHYS ((unsigned long) vmemmap)
134

135 136 137 138 139 140
extern unsigned long MODULES_VADDR;
extern unsigned long MODULES_END;
#define MODULES_VADDR	MODULES_VADDR
#define MODULES_END	MODULES_END
#define MODULES_LEN	(1UL << 31)

141 142 143 144 145 146 147 148 149 150
static inline int is_module_addr(void *addr)
{
	BUILD_BUG_ON(MODULES_LEN > (1UL << 31));
	if (addr < (void *)MODULES_VADDR)
		return 0;
	if (addr > (void *)MODULES_END)
		return 0;
	return 1;
}

L
Linus Torvalds 已提交
151 152
/*
 * A 64 bit pagetable entry of S390 has following format:
153
 * |			 PFRA			      |0IPC|  OS  |
L
Linus Torvalds 已提交
154 155 156 157 158
 * 0000000000111111111122222222223333333333444444444455555555556666
 * 0123456789012345678901234567890123456789012345678901234567890123
 *
 * I Page-Invalid Bit:    Page is not available for address-translation
 * P Page-Protection Bit: Store access not possible for page
159
 * C Change-bit override: HW is not required to set change bit
L
Linus Torvalds 已提交
160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
 *
 * A 64 bit segmenttable entry of S390 has following format:
 * |        P-table origin                              |      TT
 * 0000000000111111111122222222223333333333444444444455555555556666
 * 0123456789012345678901234567890123456789012345678901234567890123
 *
 * I Segment-Invalid Bit:    Segment is not available for address-translation
 * C Common-Segment Bit:     Segment is not private (PoP 3-30)
 * P Page-Protection Bit: Store access not possible for page
 * TT Type 00
 *
 * A 64 bit region table entry of S390 has following format:
 * |        S-table origin                             |   TF  TTTL
 * 0000000000111111111122222222223333333333444444444455555555556666
 * 0123456789012345678901234567890123456789012345678901234567890123
 *
 * I Segment-Invalid Bit:    Segment is not available for address-translation
 * TT Type 01
 * TF
M
Martin Schwidefsky 已提交
179
 * TL Table length
L
Linus Torvalds 已提交
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
 *
 * The 64 bit regiontable origin of S390 has following format:
 * |      region table origon                          |       DTTL
 * 0000000000111111111122222222223333333333444444444455555555556666
 * 0123456789012345678901234567890123456789012345678901234567890123
 *
 * X Space-Switch event:
 * G Segment-Invalid Bit:  
 * P Private-Space Bit:    
 * S Storage-Alteration:
 * R Real space
 * TL Table-Length:
 *
 * A storage key has the following format:
 * | ACC |F|R|C|0|
 *  0   3 4 5 6 7
 * ACC: access key
 * F  : fetch protection bit
 * R  : referenced bit
 * C  : changed bit
 */

/* Hardware bits in the page table entry */
203
#define _PAGE_PROTECT	0x200		/* HW read-only bit  */
M
Martin Schwidefsky 已提交
204
#define _PAGE_INVALID	0x400		/* HW invalid bit    */
205
#define _PAGE_LARGE	0x800		/* Bit to mark a large pte */
206 207

/* Software bits in the page table entry */
208 209 210
#define _PAGE_PRESENT	0x001		/* SW pte present bit */
#define _PAGE_YOUNG	0x004		/* SW pte young bit */
#define _PAGE_DIRTY	0x008		/* SW pte dirty bit */
211 212 213
#define _PAGE_READ	0x010		/* SW pte read bit */
#define _PAGE_WRITE	0x020		/* SW pte write bit */
#define _PAGE_SPECIAL	0x040		/* SW associated with special page */
214
#define _PAGE_UNUSED	0x080		/* SW bit for pgste usage state */
N
Nick Piggin 已提交
215
#define __HAVE_ARCH_PTE_SPECIAL
L
Linus Torvalds 已提交
216

217 218 219 220 221 222
#ifdef CONFIG_MEM_SOFT_DIRTY
#define _PAGE_SOFT_DIRTY 0x002		/* SW pte soft dirty bit */
#else
#define _PAGE_SOFT_DIRTY 0x000
#endif

223
/* Set of bits not changed in pte_modify */
224
#define _PAGE_CHG_MASK		(PAGE_MASK | _PAGE_SPECIAL | _PAGE_DIRTY | \
225
				 _PAGE_YOUNG | _PAGE_SOFT_DIRTY)
226

M
Martin Schwidefsky 已提交
227
/*
228 229 230 231
 * handle_pte_fault uses pte_present and pte_none to find out the pte type
 * WITHOUT holding the page table lock. The _PAGE_PRESENT bit is used to
 * distinguish present from not-present ptes. It is changed only with the page
 * table lock held.
M
Martin Schwidefsky 已提交
232
 *
233
 * The following table gives the different possible bit combinations for
234 235
 * the pte hardware and software bits in the last 12 bits of a pte
 * (. unassigned bit, x don't care, t swap type):
M
Martin Schwidefsky 已提交
236
 *
237 238 239
 *				842100000000
 *				000084210000
 *				000000008421
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
 *				.IR.uswrdy.p
 * empty			.10.00000000
 * swap				.11..ttttt.0
 * prot-none, clean, old	.11.xx0000.1
 * prot-none, clean, young	.11.xx0001.1
 * prot-none, dirty, old	.10.xx0010.1
 * prot-none, dirty, young	.10.xx0011.1
 * read-only, clean, old	.11.xx0100.1
 * read-only, clean, young	.01.xx0101.1
 * read-only, dirty, old	.11.xx0110.1
 * read-only, dirty, young	.01.xx0111.1
 * read-write, clean, old	.11.xx1100.1
 * read-write, clean, young	.01.xx1101.1
 * read-write, dirty, old	.10.xx1110.1
 * read-write, dirty, young	.00.xx1111.1
 * HW-bits: R read-only, I invalid
 * SW-bits: p present, y young, d dirty, r read, w write, s special,
 *	    u unused, l large
258
 *
259 260 261
 * pte_none    is true for the bit pattern .10.00000000, pte == 0x400
 * pte_swap    is true for the bit pattern .11..ooooo.0, (pte & 0x201) == 0x200
 * pte_present is true for the bit pattern .xx.xxxxxx.1, (pte & 0x001) == 0x001
M
Martin Schwidefsky 已提交
262 263
 */

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
/* Bits in the segment/region table address-space-control-element */
#define _ASCE_ORIGIN		~0xfffUL/* segment table origin		    */
#define _ASCE_PRIVATE_SPACE	0x100	/* private space control	    */
#define _ASCE_ALT_EVENT		0x80	/* storage alteration event control */
#define _ASCE_SPACE_SWITCH	0x40	/* space switch event		    */
#define _ASCE_REAL_SPACE	0x20	/* real space control		    */
#define _ASCE_TYPE_MASK		0x0c	/* asce table type mask		    */
#define _ASCE_TYPE_REGION1	0x0c	/* region first table type	    */
#define _ASCE_TYPE_REGION2	0x08	/* region second table type	    */
#define _ASCE_TYPE_REGION3	0x04	/* region third table type	    */
#define _ASCE_TYPE_SEGMENT	0x00	/* segment table type		    */
#define _ASCE_TABLE_LENGTH	0x03	/* region table length		    */

/* Bits in the region table entry */
#define _REGION_ENTRY_ORIGIN	~0xfffUL/* region/segment table origin	    */
279 280
#define _REGION_ENTRY_PROTECT	0x200	/* region protection bit	    */
#define _REGION_ENTRY_INVALID	0x20	/* invalid region table entry	    */
281 282 283 284 285 286 287
#define _REGION_ENTRY_TYPE_MASK	0x0c	/* region/segment table type mask   */
#define _REGION_ENTRY_TYPE_R1	0x0c	/* region first table type	    */
#define _REGION_ENTRY_TYPE_R2	0x08	/* region second table type	    */
#define _REGION_ENTRY_TYPE_R3	0x04	/* region third table type	    */
#define _REGION_ENTRY_LENGTH	0x03	/* region third length		    */

#define _REGION1_ENTRY		(_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
288
#define _REGION1_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INVALID)
289
#define _REGION2_ENTRY		(_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
290
#define _REGION2_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INVALID)
291
#define _REGION3_ENTRY		(_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH)
292
#define _REGION3_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID)
293

294 295 296
#define _REGION3_ENTRY_ORIGIN_LARGE ~0x7fffffffUL /* large page address	     */
#define _REGION3_ENTRY_ORIGIN  ~0x7ffUL/* region third table origin	     */

297 298 299 300 301 302 303 304 305 306 307 308
#define _REGION3_ENTRY_DIRTY	0x2000	/* SW region dirty bit */
#define _REGION3_ENTRY_YOUNG	0x1000	/* SW region young bit */
#define _REGION3_ENTRY_LARGE	0x0400	/* RTTE-format control, large page  */
#define _REGION3_ENTRY_READ	0x0002	/* SW region read bit */
#define _REGION3_ENTRY_WRITE	0x0001	/* SW region write bit */

#ifdef CONFIG_MEM_SOFT_DIRTY
#define _REGION3_ENTRY_SOFT_DIRTY 0x4000 /* SW region soft dirty bit */
#else
#define _REGION3_ENTRY_SOFT_DIRTY 0x0000 /* SW region soft dirty bit */
#endif

L
Linus Torvalds 已提交
309
/* Bits in the segment table entry */
310
#define _SEGMENT_ENTRY_BITS	0xfffffffffffffe33UL
311
#define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff0ff33UL
312
#define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address	    */
313
#define _SEGMENT_ENTRY_ORIGIN	~0x7ffUL/* segment table origin		    */
314 315
#define _SEGMENT_ENTRY_PROTECT	0x200	/* page protection bit		    */
#define _SEGMENT_ENTRY_INVALID	0x20	/* invalid segment table entry	    */
L
Linus Torvalds 已提交
316

317
#define _SEGMENT_ENTRY		(0)
318
#define _SEGMENT_ENTRY_EMPTY	(_SEGMENT_ENTRY_INVALID)
319

320 321 322 323 324
#define _SEGMENT_ENTRY_DIRTY	0x2000	/* SW segment dirty bit */
#define _SEGMENT_ENTRY_YOUNG	0x1000	/* SW segment young bit */
#define _SEGMENT_ENTRY_LARGE	0x0400	/* STE-format control, large page */
#define _SEGMENT_ENTRY_READ	0x0002	/* SW segment read bit */
#define _SEGMENT_ENTRY_WRITE	0x0001	/* SW segment write bit */
325

326 327 328 329 330 331
#ifdef CONFIG_MEM_SOFT_DIRTY
#define _SEGMENT_ENTRY_SOFT_DIRTY 0x4000 /* SW segment soft dirty bit */
#else
#define _SEGMENT_ENTRY_SOFT_DIRTY 0x0000 /* SW segment soft dirty bit */
#endif

332
/*
333 334
 * Segment table and region3 table entry encoding
 * (R = read-only, I = invalid, y = young bit):
335
 *				dy..R...I...rw
336 337 338 339
 * prot-none, clean, old	00..1...1...00
 * prot-none, clean, young	01..1...1...00
 * prot-none, dirty, old	10..1...1...00
 * prot-none, dirty, young	11..1...1...00
340 341 342 343
 * read-only, clean, old	00..1...1...10
 * read-only, clean, young	01..1...0...10
 * read-only, dirty, old	10..1...1...10
 * read-only, dirty, young	11..1...0...10
344 345 346 347
 * read-write, clean, old	00..1...1...11
 * read-write, clean, young	01..1...0...11
 * read-write, dirty, old	10..0...1...11
 * read-write, dirty, young	11..0...0...11
348 349
 * The segment table origin is used to distinguish empty (origin==0) from
 * read-write, old segment table entries (origin!=0)
350 351
 * HW-bits: R read-only, I invalid
 * SW-bits: y young, d dirty, r read, w write
352
 */
353

354
/* Page status table bits for virtualization */
355 356 357 358 359 360 361
#define PGSTE_ACC_BITS	0xf000000000000000UL
#define PGSTE_FP_BIT	0x0800000000000000UL
#define PGSTE_PCL_BIT	0x0080000000000000UL
#define PGSTE_HR_BIT	0x0040000000000000UL
#define PGSTE_HC_BIT	0x0020000000000000UL
#define PGSTE_GR_BIT	0x0004000000000000UL
#define PGSTE_GC_BIT	0x0002000000000000UL
362 363
#define PGSTE_UC_BIT	0x0000800000000000UL	/* user dirty (migration) */
#define PGSTE_IN_BIT	0x0000400000000000UL	/* IPTE notify bit */
364

365 366 367 368 369 370
/* Guest Page State used for virtualization */
#define _PGSTE_GPS_ZERO		0x0000000080000000UL
#define _PGSTE_GPS_USAGE_MASK	0x0000000003000000UL
#define _PGSTE_GPS_USAGE_STABLE 0x0000000000000000UL
#define _PGSTE_GPS_USAGE_UNUSED 0x0000000001000000UL

L
Linus Torvalds 已提交
371
/*
372 373 374
 * A user page table pointer has the space-switch-event bit, the
 * private-space-control bit and the storage-alteration-event-control
 * bit set. A kernel page table pointer doesn't need them.
L
Linus Torvalds 已提交
375
 */
376 377
#define _ASCE_USER_BITS		(_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \
				 _ASCE_ALT_EVENT)
L
Linus Torvalds 已提交
378 379

/*
380
 * Page protection definitions.
L
Linus Torvalds 已提交
381
 */
382
#define PAGE_NONE	__pgprot(_PAGE_PRESENT | _PAGE_INVALID)
383 384 385 386 387 388 389 390 391 392 393
#define PAGE_READ	__pgprot(_PAGE_PRESENT | _PAGE_READ | \
				 _PAGE_INVALID | _PAGE_PROTECT)
#define PAGE_WRITE	__pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
				 _PAGE_INVALID | _PAGE_PROTECT)

#define PAGE_SHARED	__pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
				 _PAGE_YOUNG | _PAGE_DIRTY)
#define PAGE_KERNEL	__pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
				 _PAGE_YOUNG | _PAGE_DIRTY)
#define PAGE_KERNEL_RO	__pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_YOUNG | \
				 _PAGE_PROTECT)
L
Linus Torvalds 已提交
394 395

/*
396 397 398
 * On s390 the page table entry has an invalid bit and a read-only bit.
 * Read permission implies execute permission and write permission
 * implies read permission.
L
Linus Torvalds 已提交
399 400
 */
         /*xwr*/
401
#define __P000	PAGE_NONE
402 403 404 405 406 407 408
#define __P001	PAGE_READ
#define __P010	PAGE_READ
#define __P011	PAGE_READ
#define __P100	PAGE_READ
#define __P101	PAGE_READ
#define __P110	PAGE_READ
#define __P111	PAGE_READ
409 410

#define __S000	PAGE_NONE
411 412 413 414 415 416 417
#define __S001	PAGE_READ
#define __S010	PAGE_WRITE
#define __S011	PAGE_WRITE
#define __S100	PAGE_READ
#define __S101	PAGE_READ
#define __S110	PAGE_WRITE
#define __S111	PAGE_WRITE
L
Linus Torvalds 已提交
418

419 420 421
/*
 * Segment entry (large page) protection definitions.
 */
422 423
#define SEGMENT_NONE	__pgprot(_SEGMENT_ENTRY_INVALID | \
				 _SEGMENT_ENTRY_PROTECT)
424 425 426 427
#define SEGMENT_READ	__pgprot(_SEGMENT_ENTRY_PROTECT | \
				 _SEGMENT_ENTRY_READ)
#define SEGMENT_WRITE	__pgprot(_SEGMENT_ENTRY_READ | \
				 _SEGMENT_ENTRY_WRITE)
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
#define SEGMENT_KERNEL	__pgprot(_SEGMENT_ENTRY |	\
				 _SEGMENT_ENTRY_LARGE |	\
				 _SEGMENT_ENTRY_READ |	\
				 _SEGMENT_ENTRY_WRITE | \
				 _SEGMENT_ENTRY_YOUNG | \
				 _SEGMENT_ENTRY_DIRTY)
#define SEGMENT_KERNEL_RO __pgprot(_SEGMENT_ENTRY |	\
				 _SEGMENT_ENTRY_LARGE |	\
				 _SEGMENT_ENTRY_READ |	\
				 _SEGMENT_ENTRY_YOUNG |	\
				 _SEGMENT_ENTRY_PROTECT)

/*
 * Region3 entry (large page) protection definitions.
 */

#define REGION3_KERNEL	__pgprot(_REGION_ENTRY_TYPE_R3 | \
				 _REGION3_ENTRY_LARGE |	 \
				 _REGION3_ENTRY_READ |	 \
				 _REGION3_ENTRY_WRITE |	 \
				 _REGION3_ENTRY_YOUNG |	 \
				 _REGION3_ENTRY_DIRTY)
#define REGION3_KERNEL_RO __pgprot(_REGION_ENTRY_TYPE_R3 | \
				   _REGION3_ENTRY_LARGE |  \
				   _REGION3_ENTRY_READ |   \
				   _REGION3_ENTRY_YOUNG |  \
				   _REGION_ENTRY_PROTECT)
455

456 457 458 459 460 461 462 463
static inline int mm_has_pgste(struct mm_struct *mm)
{
#ifdef CONFIG_PGSTE
	if (unlikely(mm->context.has_pgste))
		return 1;
#endif
	return 0;
}
464

465 466 467 468 469 470 471 472 473
static inline int mm_alloc_pgste(struct mm_struct *mm)
{
#ifdef CONFIG_PGSTE
	if (unlikely(mm->context.alloc_pgste))
		return 1;
#endif
	return 0;
}

474 475 476 477 478
/*
 * In the case that a guest uses storage keys
 * faults should no longer be backed by zero pages
 */
#define mm_forbids_zeropage mm_use_skey
479 480 481 482 483 484 485 486 487
static inline int mm_use_skey(struct mm_struct *mm)
{
#ifdef CONFIG_PGSTE
	if (mm->context.use_skey)
		return 1;
#endif
	return 0;
}

488 489 490 491 492 493 494 495 496 497 498 499 500
static inline void csp(unsigned int *ptr, unsigned int old, unsigned int new)
{
	register unsigned long reg2 asm("2") = old;
	register unsigned long reg3 asm("3") = new;
	unsigned long address = (unsigned long)ptr | 1;

	asm volatile(
		"	csp	%0,%3"
		: "+d" (reg2), "+m" (*ptr)
		: "d" (reg3), "d" (address)
		: "cc");
}

501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
static inline void cspg(unsigned long *ptr, unsigned long old, unsigned long new)
{
	register unsigned long reg2 asm("2") = old;
	register unsigned long reg3 asm("3") = new;
	unsigned long address = (unsigned long)ptr | 1;

	asm volatile(
		"	.insn	rre,0xb98a0000,%0,%3"
		: "+d" (reg2), "+m" (*ptr)
		: "d" (reg3), "d" (address)
		: "cc");
}

#define CRDTE_DTT_PAGE		0x00UL
#define CRDTE_DTT_SEGMENT	0x10UL
#define CRDTE_DTT_REGION3	0x14UL
#define CRDTE_DTT_REGION2	0x18UL
#define CRDTE_DTT_REGION1	0x1cUL

static inline void crdte(unsigned long old, unsigned long new,
			 unsigned long table, unsigned long dtt,
			 unsigned long address, unsigned long asce)
{
	register unsigned long reg2 asm("2") = old;
	register unsigned long reg3 asm("3") = new;
	register unsigned long reg4 asm("4") = table | dtt;
	register unsigned long reg5 asm("5") = address;

	asm volatile(".insn rrf,0xb98f0000,%0,%2,%4,0"
		     : "+d" (reg2)
		     : "d" (reg3), "d" (reg4), "d" (reg5), "a" (asce)
		     : "memory", "cc");
}

L
Linus Torvalds 已提交
535 536 537
/*
 * pgd/pmd/pte query functions
 */
538 539
static inline int pgd_present(pgd_t pgd)
{
M
Martin Schwidefsky 已提交
540 541
	if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
		return 1;
542 543 544 545 546
	return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL;
}

static inline int pgd_none(pgd_t pgd)
{
M
Martin Schwidefsky 已提交
547 548
	if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
		return 0;
549
	return (pgd_val(pgd) & _REGION_ENTRY_INVALID) != 0UL;
550 551 552 553
}

static inline int pgd_bad(pgd_t pgd)
{
M
Martin Schwidefsky 已提交
554 555 556 557 558
	/*
	 * With dynamic page table levels the pgd can be a region table
	 * entry or a segment table entry. Check for the bit that are
	 * invalid for either table entry.
	 */
559
	unsigned long mask =
560
		~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID &
561 562 563
		~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
	return (pgd_val(pgd) & mask) != 0;
}
M
Martin Schwidefsky 已提交
564 565

static inline int pud_present(pud_t pud)
L
Linus Torvalds 已提交
566
{
M
Martin Schwidefsky 已提交
567 568
	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
		return 1;
569
	return (pud_val(pud) & _REGION_ENTRY_ORIGIN) != 0UL;
L
Linus Torvalds 已提交
570 571
}

M
Martin Schwidefsky 已提交
572
static inline int pud_none(pud_t pud)
L
Linus Torvalds 已提交
573
{
M
Martin Schwidefsky 已提交
574 575
	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
		return 0;
576
	return (pud_val(pud) & _REGION_ENTRY_INVALID) != 0UL;
L
Linus Torvalds 已提交
577 578
}

579 580 581 582 583 584 585
static inline int pud_large(pud_t pud)
{
	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) != _REGION_ENTRY_TYPE_R3)
		return 0;
	return !!(pud_val(pud) & _REGION3_ENTRY_LARGE);
}

586 587 588 589 590 591 592 593 594 595
static inline unsigned long pud_pfn(pud_t pud)
{
	unsigned long origin_mask;

	origin_mask = _REGION3_ENTRY_ORIGIN;
	if (pud_large(pud))
		origin_mask = _REGION3_ENTRY_ORIGIN_LARGE;
	return (pud_val(pud) & origin_mask) >> PAGE_SHIFT;
}

M
Martin Schwidefsky 已提交
596
static inline int pud_bad(pud_t pud)
L
Linus Torvalds 已提交
597
{
M
Martin Schwidefsky 已提交
598 599 600 601 602
	/*
	 * With dynamic page table levels the pud can be a region table
	 * entry or a segment table entry. Check for the bit that are
	 * invalid for either table entry.
	 */
603
	unsigned long mask =
604
		~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID &
605 606
		~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
	return (pud_val(pud) & mask) != 0;
L
Linus Torvalds 已提交
607 608
}

609
static inline int pmd_present(pmd_t pmd)
L
Linus Torvalds 已提交
610
{
611
	return pmd_val(pmd) != _SEGMENT_ENTRY_INVALID;
L
Linus Torvalds 已提交
612 613
}

614
static inline int pmd_none(pmd_t pmd)
L
Linus Torvalds 已提交
615
{
616
	return pmd_val(pmd) == _SEGMENT_ENTRY_INVALID;
L
Linus Torvalds 已提交
617 618
}

619 620
static inline int pmd_large(pmd_t pmd)
{
621
	return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0;
622 623
}

624
static inline unsigned long pmd_pfn(pmd_t pmd)
625
{
626 627 628 629 630 631
	unsigned long origin_mask;

	origin_mask = _SEGMENT_ENTRY_ORIGIN;
	if (pmd_large(pmd))
		origin_mask = _SEGMENT_ENTRY_ORIGIN_LARGE;
	return (pmd_val(pmd) & origin_mask) >> PAGE_SHIFT;
632 633
}

634
static inline int pmd_bad(pmd_t pmd)
L
Linus Torvalds 已提交
635
{
636 637 638
	if (pmd_large(pmd))
		return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS_LARGE) != 0;
	return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0;
L
Linus Torvalds 已提交
639 640
}

641 642 643
#define __HAVE_ARCH_PMD_WRITE
static inline int pmd_write(pmd_t pmd)
{
644 645 646 647 648 649 650 651 652
	return (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) != 0;
}

static inline int pmd_dirty(pmd_t pmd)
{
	int dirty = 1;
	if (pmd_large(pmd))
		dirty = (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) != 0;
	return dirty;
653 654 655 656
}

static inline int pmd_young(pmd_t pmd)
{
657 658
	int young = 1;
	if (pmd_large(pmd))
659 660
		young = (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) != 0;
	return young;
661 662
}

663
static inline int pte_present(pte_t pte)
L
Linus Torvalds 已提交
664
{
665 666
	/* Bit pattern: (pte & 0x001) == 0x001 */
	return (pte_val(pte) & _PAGE_PRESENT) != 0;
L
Linus Torvalds 已提交
667 668
}

669
static inline int pte_none(pte_t pte)
L
Linus Torvalds 已提交
670
{
671 672
	/* Bit pattern: pte == 0x400 */
	return pte_val(pte) == _PAGE_INVALID;
L
Linus Torvalds 已提交
673 674
}

675 676
static inline int pte_swap(pte_t pte)
{
677 678 679
	/* Bit pattern: (pte & 0x201) == 0x200 */
	return (pte_val(pte) & (_PAGE_PROTECT | _PAGE_PRESENT))
		== _PAGE_PROTECT;
680 681
}

N
Nick Piggin 已提交
682 683
static inline int pte_special(pte_t pte)
{
N
Nick Piggin 已提交
684
	return (pte_val(pte) & _PAGE_SPECIAL);
N
Nick Piggin 已提交
685 686
}

M
Martin Schwidefsky 已提交
687
#define __HAVE_ARCH_PTE_SAME
688 689 690 691
static inline int pte_same(pte_t a, pte_t b)
{
	return pte_val(a) == pte_val(b);
}
L
Linus Torvalds 已提交
692

693 694 695 696 697 698 699 700 701 702 703 704 705
#ifdef CONFIG_NUMA_BALANCING
static inline int pte_protnone(pte_t pte)
{
	return pte_present(pte) && !(pte_val(pte) & _PAGE_READ);
}

static inline int pmd_protnone(pmd_t pmd)
{
	/* pmd_large(pmd) implies pmd_present(pmd) */
	return pmd_large(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ);
}
#endif

706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
static inline int pte_soft_dirty(pte_t pte)
{
	return pte_val(pte) & _PAGE_SOFT_DIRTY;
}
#define pte_swp_soft_dirty pte_soft_dirty

static inline pte_t pte_mksoft_dirty(pte_t pte)
{
	pte_val(pte) |= _PAGE_SOFT_DIRTY;
	return pte;
}
#define pte_swp_mksoft_dirty pte_mksoft_dirty

static inline pte_t pte_clear_soft_dirty(pte_t pte)
{
	pte_val(pte) &= ~_PAGE_SOFT_DIRTY;
	return pte;
}
#define pte_swp_clear_soft_dirty pte_clear_soft_dirty

static inline int pmd_soft_dirty(pmd_t pmd)
{
	return pmd_val(pmd) & _SEGMENT_ENTRY_SOFT_DIRTY;
}

static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
{
	pmd_val(pmd) |= _SEGMENT_ENTRY_SOFT_DIRTY;
	return pmd;
}

static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
{
	pmd_val(pmd) &= ~_SEGMENT_ENTRY_SOFT_DIRTY;
	return pmd;
}

L
Linus Torvalds 已提交
743 744 745 746
/*
 * query functions pte_write/pte_dirty/pte_young only work if
 * pte_present() is true. Undefined behaviour if not..
 */
747
static inline int pte_write(pte_t pte)
L
Linus Torvalds 已提交
748
{
749
	return (pte_val(pte) & _PAGE_WRITE) != 0;
L
Linus Torvalds 已提交
750 751
}

752
static inline int pte_dirty(pte_t pte)
L
Linus Torvalds 已提交
753
{
754
	return (pte_val(pte) & _PAGE_DIRTY) != 0;
L
Linus Torvalds 已提交
755 756
}

757
static inline int pte_young(pte_t pte)
L
Linus Torvalds 已提交
758
{
759
	return (pte_val(pte) & _PAGE_YOUNG) != 0;
L
Linus Torvalds 已提交
760 761
}

762 763 764 765 766 767
#define __HAVE_ARCH_PTE_UNUSED
static inline int pte_unused(pte_t pte)
{
	return pte_val(pte) & _PAGE_UNUSED;
}

L
Linus Torvalds 已提交
768 769 770 771
/*
 * pgd/pmd/pte modification functions
 */

772
static inline void pgd_clear(pgd_t *pgd)
773
{
M
Martin Schwidefsky 已提交
774 775
	if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
		pgd_val(*pgd) = _REGION2_ENTRY_EMPTY;
776 777
}

778
static inline void pud_clear(pud_t *pud)
L
Linus Torvalds 已提交
779
{
M
Martin Schwidefsky 已提交
780 781
	if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		pud_val(*pud) = _REGION3_ENTRY_EMPTY;
L
Linus Torvalds 已提交
782 783
}

784
static inline void pmd_clear(pmd_t *pmdp)
L
Linus Torvalds 已提交
785
{
786
	pmd_val(*pmdp) = _SEGMENT_ENTRY_INVALID;
L
Linus Torvalds 已提交
787 788
}

789
static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
L
Linus Torvalds 已提交
790
{
791
	pte_val(*ptep) = _PAGE_INVALID;
L
Linus Torvalds 已提交
792 793 794 795 796 797
}

/*
 * The following pte modification functions only work if
 * pte_present() is true. Undefined behaviour if not..
 */
798
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
L
Linus Torvalds 已提交
799
{
800
	pte_val(pte) &= _PAGE_CHG_MASK;
L
Linus Torvalds 已提交
801
	pte_val(pte) |= pgprot_val(newprot);
802 803 804 805 806 807 808 809 810 811
	/*
	 * newprot for PAGE_NONE, PAGE_READ and PAGE_WRITE has the
	 * invalid bit set, clear it again for readable, young pages
	 */
	if ((pte_val(pte) & _PAGE_YOUNG) && (pte_val(pte) & _PAGE_READ))
		pte_val(pte) &= ~_PAGE_INVALID;
	/*
	 * newprot for PAGE_READ and PAGE_WRITE has the page protection
	 * bit set, clear it again for writable, dirty pages
	 */
812 813
	if ((pte_val(pte) & _PAGE_DIRTY) && (pte_val(pte) & _PAGE_WRITE))
		pte_val(pte) &= ~_PAGE_PROTECT;
L
Linus Torvalds 已提交
814 815 816
	return pte;
}

817
static inline pte_t pte_wrprotect(pte_t pte)
L
Linus Torvalds 已提交
818
{
819 820
	pte_val(pte) &= ~_PAGE_WRITE;
	pte_val(pte) |= _PAGE_PROTECT;
L
Linus Torvalds 已提交
821 822 823
	return pte;
}

824
static inline pte_t pte_mkwrite(pte_t pte)
L
Linus Torvalds 已提交
825
{
826 827 828
	pte_val(pte) |= _PAGE_WRITE;
	if (pte_val(pte) & _PAGE_DIRTY)
		pte_val(pte) &= ~_PAGE_PROTECT;
L
Linus Torvalds 已提交
829 830 831
	return pte;
}

832
static inline pte_t pte_mkclean(pte_t pte)
L
Linus Torvalds 已提交
833
{
834 835
	pte_val(pte) &= ~_PAGE_DIRTY;
	pte_val(pte) |= _PAGE_PROTECT;
L
Linus Torvalds 已提交
836 837 838
	return pte;
}

839
static inline pte_t pte_mkdirty(pte_t pte)
L
Linus Torvalds 已提交
840
{
841
	pte_val(pte) |= _PAGE_DIRTY | _PAGE_SOFT_DIRTY;
842 843
	if (pte_val(pte) & _PAGE_WRITE)
		pte_val(pte) &= ~_PAGE_PROTECT;
L
Linus Torvalds 已提交
844 845 846
	return pte;
}

847
static inline pte_t pte_mkold(pte_t pte)
L
Linus Torvalds 已提交
848
{
849
	pte_val(pte) &= ~_PAGE_YOUNG;
850
	pte_val(pte) |= _PAGE_INVALID;
L
Linus Torvalds 已提交
851 852 853
	return pte;
}

854
static inline pte_t pte_mkyoung(pte_t pte)
L
Linus Torvalds 已提交
855
{
856 857 858
	pte_val(pte) |= _PAGE_YOUNG;
	if (pte_val(pte) & _PAGE_READ)
		pte_val(pte) &= ~_PAGE_INVALID;
L
Linus Torvalds 已提交
859 860 861
	return pte;
}

N
Nick Piggin 已提交
862 863
static inline pte_t pte_mkspecial(pte_t pte)
{
N
Nick Piggin 已提交
864
	pte_val(pte) |= _PAGE_SPECIAL;
N
Nick Piggin 已提交
865 866 867
	return pte;
}

868 869 870
#ifdef CONFIG_HUGETLB_PAGE
static inline pte_t pte_mkhuge(pte_t pte)
{
871
	pte_val(pte) |= _PAGE_LARGE;
872 873 874 875
	return pte;
}
#endif

876
static inline void __ptep_ipte(unsigned long address, pte_t *ptep)
L
Linus Torvalds 已提交
877
{
878 879 880 881 882 883 884 885
	unsigned long pto = (unsigned long) ptep;

	/* Invalidation + global TLB flush for the pte */
	asm volatile(
		"	ipte	%2,%3"
		: "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address));
}

886 887 888 889 890 891 892 893 894 895
static inline void __ptep_ipte_local(unsigned long address, pte_t *ptep)
{
	unsigned long pto = (unsigned long) ptep;

	/* Invalidation + local TLB flush for the pte */
	asm volatile(
		"	.insn rrf,0xb2210000,%2,%3,0,1"
		: "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address));
}

896 897 898 899 900 901 902 903 904 905 906 907
static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep)
{
	unsigned long pto = (unsigned long) ptep;

	/* Invalidate a range of ptes + global TLB flush of the ptes */
	do {
		asm volatile(
			"	.insn rrf,0xb2210000,%2,%0,%1,0"
			: "+a" (address), "+a" (nr) : "a" (pto) : "memory");
	} while (nr != 255);
}

908
/*
909 910 911 912 913 914 915 916 917 918 919
 * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
 * both clear the TLB for the unmapped pte. The reason is that
 * ptep_get_and_clear is used in common code (e.g. change_pte_range)
 * to modify an active pte. The sequence is
 *   1) ptep_get_and_clear
 *   2) set_pte_at
 *   3) flush_tlb_range
 * On s390 the tlb needs to get flushed with the modification of the pte
 * if the pte is active. The only way how this can be implemented is to
 * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
 * is a nop.
920
 */
921 922
pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, pte_t *, pte_t);
pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t);
923

924 925 926 927
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
					    unsigned long addr, pte_t *ptep)
{
928
	pte_t pte = *ptep;
929

930 931
	pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte));
	return pte_young(pte);
932 933 934 935 936 937 938 939 940
}

#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
					 unsigned long address, pte_t *ptep)
{
	return ptep_test_and_clear_young(vma, address, ptep);
}

M
Martin Schwidefsky 已提交
941
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
942
static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
943
				       unsigned long addr, pte_t *ptep)
944
{
945
	return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
946 947 948
}

#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
949 950
pte_t ptep_modify_prot_start(struct mm_struct *, unsigned long, pte_t *);
void ptep_modify_prot_commit(struct mm_struct *, unsigned long, pte_t *, pte_t);
M
Martin Schwidefsky 已提交
951 952

#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
M
Martin Schwidefsky 已提交
953
static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
954
				     unsigned long addr, pte_t *ptep)
M
Martin Schwidefsky 已提交
955
{
956
	return ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID));
L
Linus Torvalds 已提交
957 958
}

M
Martin Schwidefsky 已提交
959 960 961 962 963 964 965 966 967
/*
 * The batched pte unmap code uses ptep_get_and_clear_full to clear the
 * ptes. Here an optimization is possible. tlb_gather_mmu flushes all
 * tlbs of an mm if it can guarantee that the ptes of the mm_struct
 * cannot be accessed while the batched unmap is running. In this case
 * full==1 and a simple pte_clear is enough. See tlb.h.
 */
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
968
					    unsigned long addr,
M
Martin Schwidefsky 已提交
969
					    pte_t *ptep, int full)
L
Linus Torvalds 已提交
970
{
971 972 973 974
	if (full) {
		pte_t pte = *ptep;
		*ptep = __pte(_PAGE_INVALID);
		return pte;
975
	}
976
	return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
L
Linus Torvalds 已提交
977 978
}

M
Martin Schwidefsky 已提交
979
#define __HAVE_ARCH_PTEP_SET_WRPROTECT
980 981
static inline void ptep_set_wrprotect(struct mm_struct *mm,
				      unsigned long addr, pte_t *ptep)
982 983 984
{
	pte_t pte = *ptep;

985 986
	if (pte_write(pte))
		ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte));
987
}
M
Martin Schwidefsky 已提交
988 989

#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
990
static inline int ptep_set_access_flags(struct vm_area_struct *vma,
991
					unsigned long addr, pte_t *ptep,
992 993
					pte_t entry, int dirty)
{
994
	if (pte_same(*ptep, entry))
995
		return 0;
996 997 998
	ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);
	return 1;
}
999

1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
/*
 * Additional functions to handle KVM guest page tables
 */
void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
		     pte_t *ptep, pte_t entry);
void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
void ptep_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
		     pte_t *ptep , int reset);
void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep);

bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long address);
int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
			  unsigned char key, bool nq);
unsigned char get_guest_storage_key(struct mm_struct *mm, unsigned long addr);
1015

1016 1017 1018 1019 1020 1021 1022 1023 1024
/*
 * Certain architectures need to do special things when PTEs
 * within a page table are directly modified.  Thus, the following
 * hook is made available.
 */
static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
			      pte_t *ptep, pte_t entry)
{
	if (mm_has_pgste(mm))
1025
		ptep_set_pte_at(mm, addr, ptep, entry);
1026
	else
1027
		*ptep = entry;
1028
}
L
Linus Torvalds 已提交
1029 1030 1031 1032 1033 1034 1035 1036 1037

/*
 * Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 */
static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
{
	pte_t __pte;
	pte_val(__pte) = physpage + pgprot_val(pgprot);
1038
	return pte_mkyoung(__pte);
L
Linus Torvalds 已提交
1039 1040
}

1041 1042
static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
{
1043
	unsigned long physpage = page_to_phys(page);
1044
	pte_t __pte = mk_pte_phys(physpage, pgprot);
L
Linus Torvalds 已提交
1045

1046 1047
	if (pte_write(__pte) && PageDirty(page))
		__pte = pte_mkdirty(__pte);
1048
	return __pte;
1049 1050
}

M
Martin Schwidefsky 已提交
1051 1052 1053 1054
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1))
L
Linus Torvalds 已提交
1055

M
Martin Schwidefsky 已提交
1056 1057
#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
#define pgd_offset_k(address) pgd_offset(&init_mm, address)
L
Linus Torvalds 已提交
1058

M
Martin Schwidefsky 已提交
1059 1060
#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
#define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN)
1061
#define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN)
L
Linus Torvalds 已提交
1062

1063 1064
static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
{
M
Martin Schwidefsky 已提交
1065 1066 1067
	pud_t *pud = (pud_t *) pgd;
	if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
		pud = (pud_t *) pgd_deref(*pgd);
1068 1069
	return pud  + pud_index(address);
}
L
Linus Torvalds 已提交
1070

M
Martin Schwidefsky 已提交
1071
static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
L
Linus Torvalds 已提交
1072
{
M
Martin Schwidefsky 已提交
1073 1074 1075
	pmd_t *pmd = (pmd_t *) pud;
	if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		pmd = (pmd_t *) pud_deref(*pud);
M
Martin Schwidefsky 已提交
1076
	return pmd + pmd_index(address);
L
Linus Torvalds 已提交
1077 1078
}

M
Martin Schwidefsky 已提交
1079 1080 1081
#define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot))
#define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
#define pte_page(x) pfn_to_page(pte_pfn(x))
L
Linus Torvalds 已提交
1082

1083
#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
L
Linus Torvalds 已提交
1084

M
Martin Schwidefsky 已提交
1085 1086 1087
/* Find an entry in the lowest level page table.. */
#define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr))
#define pte_offset_kernel(pmd, address) pte_offset(pmd,address)
L
Linus Torvalds 已提交
1088 1089 1090
#define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address)
#define pte_unmap(pte) do { } while (0)

1091
static inline pmd_t pmd_wrprotect(pmd_t pmd)
1092
{
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
	pmd_val(pmd) &= ~_SEGMENT_ENTRY_WRITE;
	pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
	return pmd;
}

static inline pmd_t pmd_mkwrite(pmd_t pmd)
{
	pmd_val(pmd) |= _SEGMENT_ENTRY_WRITE;
	if (pmd_large(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY))
		return pmd;
	pmd_val(pmd) &= ~_SEGMENT_ENTRY_PROTECT;
	return pmd;
}

static inline pmd_t pmd_mkclean(pmd_t pmd)
{
	if (pmd_large(pmd)) {
		pmd_val(pmd) &= ~_SEGMENT_ENTRY_DIRTY;
1111
		pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
1112 1113 1114 1115 1116 1117 1118
	}
	return pmd;
}

static inline pmd_t pmd_mkdirty(pmd_t pmd)
{
	if (pmd_large(pmd)) {
1119 1120
		pmd_val(pmd) |= _SEGMENT_ENTRY_DIRTY |
				_SEGMENT_ENTRY_SOFT_DIRTY;
1121 1122 1123 1124 1125 1126
		if (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE)
			pmd_val(pmd) &= ~_SEGMENT_ENTRY_PROTECT;
	}
	return pmd;
}

1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
static inline pud_t pud_wrprotect(pud_t pud)
{
	pud_val(pud) &= ~_REGION3_ENTRY_WRITE;
	pud_val(pud) |= _REGION_ENTRY_PROTECT;
	return pud;
}

static inline pud_t pud_mkwrite(pud_t pud)
{
	pud_val(pud) |= _REGION3_ENTRY_WRITE;
	if (pud_large(pud) && !(pud_val(pud) & _REGION3_ENTRY_DIRTY))
		return pud;
	pud_val(pud) &= ~_REGION_ENTRY_PROTECT;
	return pud;
}

static inline pud_t pud_mkclean(pud_t pud)
{
	if (pud_large(pud)) {
		pud_val(pud) &= ~_REGION3_ENTRY_DIRTY;
		pud_val(pud) |= _REGION_ENTRY_PROTECT;
	}
	return pud;
}

static inline pud_t pud_mkdirty(pud_t pud)
{
	if (pud_large(pud)) {
		pud_val(pud) |= _REGION3_ENTRY_DIRTY |
				_REGION3_ENTRY_SOFT_DIRTY;
		if (pud_val(pud) & _REGION3_ENTRY_WRITE)
			pud_val(pud) &= ~_REGION_ENTRY_PROTECT;
	}
	return pud;
}

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
{
	/*
	 * pgprot is PAGE_NONE, PAGE_READ, or PAGE_WRITE (see __Pxxx / __Sxxx)
	 * Convert to segment table entry format.
	 */
	if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE))
		return pgprot_val(SEGMENT_NONE);
	if (pgprot_val(pgprot) == pgprot_val(PAGE_READ))
		return pgprot_val(SEGMENT_READ);
	return pgprot_val(SEGMENT_WRITE);
}

1177 1178 1179
static inline pmd_t pmd_mkyoung(pmd_t pmd)
{
	if (pmd_large(pmd)) {
1180
		pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG;
1181 1182
		if (pmd_val(pmd) & _SEGMENT_ENTRY_READ)
			pmd_val(pmd) &= ~_SEGMENT_ENTRY_INVALID;
1183 1184 1185 1186 1187 1188
	}
	return pmd;
}

static inline pmd_t pmd_mkold(pmd_t pmd)
{
1189
	if (pmd_large(pmd)) {
1190 1191 1192 1193 1194 1195
		pmd_val(pmd) &= ~_SEGMENT_ENTRY_YOUNG;
		pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID;
	}
	return pmd;
}

1196 1197
static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
{
1198 1199 1200
	if (pmd_large(pmd)) {
		pmd_val(pmd) &= _SEGMENT_ENTRY_ORIGIN_LARGE |
			_SEGMENT_ENTRY_DIRTY | _SEGMENT_ENTRY_YOUNG |
1201
			_SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_SOFT_DIRTY;
1202 1203 1204 1205 1206 1207 1208 1209
		pmd_val(pmd) |= massage_pgprot_pmd(newprot);
		if (!(pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY))
			pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
		if (!(pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG))
			pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID;
		return pmd;
	}
	pmd_val(pmd) &= _SEGMENT_ENTRY_ORIGIN;
1210 1211 1212 1213
	pmd_val(pmd) |= massage_pgprot_pmd(newprot);
	return pmd;
}

1214
static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
1215
{
1216 1217
	pmd_t __pmd;
	pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot);
1218
	return __pmd;
1219 1220
}

1221 1222
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLB_PAGE */

1223 1224
static inline void __pmdp_csp(pmd_t *pmdp)
{
1225 1226
	csp((unsigned int *)pmdp + 1, pmd_val(*pmdp),
	    pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252
}

static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp)
{
	unsigned long sto;

	sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t);
	asm volatile(
		"	.insn	rrf,0xb98e0000,%2,%3,0,0"
		: "=m" (*pmdp)
		: "m" (*pmdp), "a" (sto), "a" ((address & HPAGE_MASK))
		: "cc" );
}

static inline void __pmdp_idte_local(unsigned long address, pmd_t *pmdp)
{
	unsigned long sto;

	sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t);
	asm volatile(
		"	.insn	rrf,0xb98e0000,%2,%3,0,1"
		: "=m" (*pmdp)
		: "m" (*pmdp), "a" (sto), "a" ((address & HPAGE_MASK))
		: "cc" );
}

1253 1254
pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
pmd_t pmdp_xchg_lazy(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
1255

1256 1257 1258 1259 1260 1261 1262 1263
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#define __HAVE_ARCH_PGTABLE_DEPOSIT
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
				pgtable_t pgtable);

#define __HAVE_ARCH_PGTABLE_WITHDRAW
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
1264

1265 1266 1267 1268
#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
					unsigned long addr, pmd_t *pmdp,
					pmd_t entry, int dirty)
1269
{
1270
	VM_BUG_ON(addr & ~HPAGE_MASK);
1271

1272 1273 1274 1275 1276 1277 1278
	entry = pmd_mkyoung(entry);
	if (dirty)
		entry = pmd_mkdirty(entry);
	if (pmd_val(*pmdp) == pmd_val(entry))
		return 0;
	pmdp_xchg_direct(vma->vm_mm, addr, pmdp, entry);
	return 1;
1279 1280
}

1281 1282 1283 1284 1285
#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
					    unsigned long addr, pmd_t *pmdp)
{
	pmd_t pmd = *pmdp;
1286

1287 1288 1289
	pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd_mkold(pmd));
	return pmd_young(pmd);
}
1290

1291 1292 1293 1294 1295 1296 1297
#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
					 unsigned long addr, pmd_t *pmdp)
{
	VM_BUG_ON(addr & ~HPAGE_MASK);
	return pmdp_test_and_clear_young(vma, addr, pmdp);
}
1298 1299 1300 1301 1302 1303 1304 1305 1306 1307

static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
			      pmd_t *pmdp, pmd_t entry)
{
	*pmdp = entry;
}

static inline pmd_t pmd_mkhuge(pmd_t pmd)
{
	pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
1308 1309
	pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG;
	pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
1310 1311 1312
	return pmd;
}

1313 1314
#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1315
					    unsigned long addr, pmd_t *pmdp)
1316
{
1317
	return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID));
1318 1319
}

1320 1321
#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm,
1322
						 unsigned long addr,
1323
						 pmd_t *pmdp, int full)
1324
{
1325 1326 1327 1328 1329 1330
	if (full) {
		pmd_t pmd = *pmdp;
		*pmdp = __pmd(_SEGMENT_ENTRY_INVALID);
		return pmd;
	}
	return pmdp_xchg_lazy(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID));
1331 1332
}

1333 1334
#define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
1335
					  unsigned long addr, pmd_t *pmdp)
1336
{
1337
	return pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp);
1338 1339 1340 1341
}

#define __HAVE_ARCH_PMDP_INVALIDATE
static inline void pmdp_invalidate(struct vm_area_struct *vma,
1342
				   unsigned long addr, pmd_t *pmdp)
1343
{
1344
	pmdp_xchg_direct(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID));
1345 1346
}

1347 1348
#define __HAVE_ARCH_PMDP_SET_WRPROTECT
static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1349
				      unsigned long addr, pmd_t *pmdp)
1350 1351 1352
{
	pmd_t pmd = *pmdp;

1353 1354
	if (pmd_write(pmd))
		pmd = pmdp_xchg_lazy(mm, addr, pmdp, pmd_wrprotect(pmd));
1355 1356
}

1357 1358 1359 1360
static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
					unsigned long address,
					pmd_t *pmdp)
{
1361
	return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
1362 1363 1364
}
#define pmdp_collapse_flush pmdp_collapse_flush

1365 1366 1367 1368 1369 1370 1371 1372
#define pfn_pmd(pfn, pgprot)	mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot))
#define mk_pmd(page, pgprot)	pfn_pmd(page_to_pfn(page), (pgprot))

static inline int pmd_trans_huge(pmd_t pmd)
{
	return pmd_val(pmd) & _SEGMENT_ENTRY_LARGE;
}

1373
#define has_transparent_hugepage has_transparent_hugepage
1374 1375 1376 1377
static inline int has_transparent_hugepage(void)
{
	return MACHINE_HAS_HPAGE ? 1 : 0;
}
1378 1379
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

L
Linus Torvalds 已提交
1380 1381 1382
/*
 * 64 bit swap entry format:
 * A page-table entry has some bits we have to treat in a special way.
1383
 * Bits 52 and bit 55 have to be zero, otherwise a specification
L
Linus Torvalds 已提交
1384
 * exception will occur instead of a page translation exception. The
1385
 * specification exception has the bad habit not to store necessary
L
Linus Torvalds 已提交
1386
 * information in the lowcore.
1387 1388 1389 1390 1391 1392 1393 1394
 * Bits 54 and 63 are used to indicate the page type.
 * A swap pte is indicated by bit pattern (pte & 0x201) == 0x200
 * This leaves the bits 0-51 and bits 56-62 to store type and offset.
 * We use the 5 bits from 57-61 for the type and the 52 bits from 0-51
 * for the offset.
 * |			  offset			|01100|type |00|
 * |0000000000111111111122222222223333333333444444444455|55555|55566|66|
 * |0123456789012345678901234567890123456789012345678901|23456|78901|23|
L
Linus Torvalds 已提交
1395
 */
H
Heiko Carstens 已提交
1396

1397 1398 1399 1400
#define __SWP_OFFSET_MASK	((1UL << 52) - 1)
#define __SWP_OFFSET_SHIFT	12
#define __SWP_TYPE_MASK		((1UL << 5) - 1)
#define __SWP_TYPE_SHIFT	2
H
Heiko Carstens 已提交
1401

1402
static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
L
Linus Torvalds 已提交
1403 1404
{
	pte_t pte;
1405 1406 1407 1408

	pte_val(pte) = _PAGE_INVALID | _PAGE_PROTECT;
	pte_val(pte) |= (offset & __SWP_OFFSET_MASK) << __SWP_OFFSET_SHIFT;
	pte_val(pte) |= (type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT;
L
Linus Torvalds 已提交
1409 1410 1411
	return pte;
}

1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425
static inline unsigned long __swp_type(swp_entry_t entry)
{
	return (entry.val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK;
}

static inline unsigned long __swp_offset(swp_entry_t entry)
{
	return (entry.val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK;
}

static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset)
{
	return (swp_entry_t) { pte_val(mk_swap_pte(type, offset)) };
}
L
Linus Torvalds 已提交
1426 1427 1428 1429 1430 1431 1432 1433

#define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
#define __swp_entry_to_pte(x)	((pte_t) { (x).val })

#endif /* !__ASSEMBLY__ */

#define kern_addr_valid(addr)   (1)

1434 1435
extern int vmem_add_mapping(unsigned long start, unsigned long size);
extern int vmem_remove_mapping(unsigned long start, unsigned long size);
1436
extern int s390_enable_sie(void);
1437
extern int s390_enable_skey(void);
1438
extern void s390_reset_cmma(struct mm_struct *mm);
H
Heiko Carstens 已提交
1439

1440 1441 1442 1443
/* s390 has a private copy of get unmapped area to deal with cache synonyms */
#define HAVE_ARCH_UNMAPPED_AREA
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN

L
Linus Torvalds 已提交
1444 1445 1446
/*
 * No page table caches to initialise
 */
1447 1448
static inline void pgtable_cache_init(void) { }
static inline void check_pgt_cache(void) { }
L
Linus Torvalds 已提交
1449 1450 1451 1452

#include <asm-generic/pgtable.h>

#endif /* _S390_PAGE_H */