1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  *  S390 version
4  *    Copyright IBM Corp. 1999, 2000
5  *    Author(s): Hartmut Penner (hp@de.ibm.com)
6  *               Ulrich Weigand (weigand@de.ibm.com)
7  *               Martin Schwidefsky (schwidefsky@de.ibm.com)
8  *
9  *  Derived from "include/asm-i386/pgtable.h"
10  */
11 
12 #ifndef _ASM_S390_PGTABLE_H
13 #define _ASM_S390_PGTABLE_H
14 
15 #include <linux/sched.h>
16 #include <linux/mm_types.h>
17 #include <linux/page-flags.h>
18 #include <linux/radix-tree.h>
19 #include <linux/atomic.h>
20 #include <asm/sections.h>
21 #include <asm/bug.h>
22 #include <asm/page.h>
23 #include <asm/uv.h>
24 
25 extern pgd_t swapper_pg_dir[];
26 extern void paging_init(void);
27 extern unsigned long s390_invalid_asce;
28 
29 enum {
30 	PG_DIRECT_MAP_4K = 0,
31 	PG_DIRECT_MAP_1M,
32 	PG_DIRECT_MAP_2G,
33 	PG_DIRECT_MAP_MAX
34 };
35 
36 extern atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX];
37 
update_page_count(int level,long count)38 static inline void update_page_count(int level, long count)
39 {
40 	if (IS_ENABLED(CONFIG_PROC_FS))
41 		atomic_long_add(count, &direct_pages_count[level]);
42 }
43 
44 struct seq_file;
45 void arch_report_meminfo(struct seq_file *m);
46 
47 /*
48  * The S390 doesn't have any external MMU info: the kernel page
49  * tables contain all the necessary information.
50  */
51 #define update_mmu_cache(vma, address, ptep)     do { } while (0)
52 #define update_mmu_cache_pmd(vma, address, ptep) do { } while (0)
53 
54 /*
55  * ZERO_PAGE is a global shared page that is always zero; used
56  * for zero-mapped memory areas etc..
57  */
58 
59 extern unsigned long empty_zero_page;
60 extern unsigned long zero_page_mask;
61 
62 #define ZERO_PAGE(vaddr) \
63 	(virt_to_page((void *)(empty_zero_page + \
64 	 (((unsigned long)(vaddr)) &zero_page_mask))))
65 #define __HAVE_COLOR_ZERO_PAGE
66 
67 /* TODO: s390 cannot support io_remap_pfn_range... */
68 
69 #define pte_ERROR(e) \
70 	pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
71 #define pmd_ERROR(e) \
72 	pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
73 #define pud_ERROR(e) \
74 	pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
75 #define p4d_ERROR(e) \
76 	pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e))
77 #define pgd_ERROR(e) \
78 	pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e))
79 
80 /*
81  * The vmalloc and module area will always be on the topmost area of the
82  * kernel mapping. 512GB are reserved for vmalloc by default.
83  * At the top of the vmalloc area a 2GB area is reserved where modules
84  * will reside. That makes sure that inter module branches always
85  * happen without trampolines and in addition the placement within a
86  * 2GB frame is branch prediction unit friendly.
87  */
88 extern unsigned long __bootdata_preserved(VMALLOC_START);
89 extern unsigned long __bootdata_preserved(VMALLOC_END);
90 #define VMALLOC_DEFAULT_SIZE	((512UL << 30) - MODULES_LEN)
91 extern struct page *__bootdata_preserved(vmemmap);
92 extern unsigned long __bootdata_preserved(vmemmap_size);
93 
94 #define VMEM_MAX_PHYS ((unsigned long) vmemmap)
95 
96 extern unsigned long __bootdata_preserved(MODULES_VADDR);
97 extern unsigned long __bootdata_preserved(MODULES_END);
98 #define MODULES_VADDR	MODULES_VADDR
99 #define MODULES_END	MODULES_END
100 #define MODULES_LEN	(1UL << 31)
101 
is_module_addr(void * addr)102 static inline int is_module_addr(void *addr)
103 {
104 	BUILD_BUG_ON(MODULES_LEN > (1UL << 31));
105 	if (addr < (void *)MODULES_VADDR)
106 		return 0;
107 	if (addr > (void *)MODULES_END)
108 		return 0;
109 	return 1;
110 }
111 
112 /*
113  * A 64 bit pagetable entry of S390 has following format:
114  * |			 PFRA			      |0IPC|  OS  |
115  * 0000000000111111111122222222223333333333444444444455555555556666
116  * 0123456789012345678901234567890123456789012345678901234567890123
117  *
118  * I Page-Invalid Bit:    Page is not available for address-translation
119  * P Page-Protection Bit: Store access not possible for page
120  * C Change-bit override: HW is not required to set change bit
121  *
122  * A 64 bit segmenttable entry of S390 has following format:
123  * |        P-table origin                              |      TT
124  * 0000000000111111111122222222223333333333444444444455555555556666
125  * 0123456789012345678901234567890123456789012345678901234567890123
126  *
127  * I Segment-Invalid Bit:    Segment is not available for address-translation
128  * C Common-Segment Bit:     Segment is not private (PoP 3-30)
129  * P Page-Protection Bit: Store access not possible for page
130  * TT Type 00
131  *
132  * A 64 bit region table entry of S390 has following format:
133  * |        S-table origin                             |   TF  TTTL
134  * 0000000000111111111122222222223333333333444444444455555555556666
135  * 0123456789012345678901234567890123456789012345678901234567890123
136  *
137  * I Segment-Invalid Bit:    Segment is not available for address-translation
138  * TT Type 01
139  * TF
140  * TL Table length
141  *
142  * The 64 bit regiontable origin of S390 has following format:
143  * |      region table origon                          |       DTTL
144  * 0000000000111111111122222222223333333333444444444455555555556666
145  * 0123456789012345678901234567890123456789012345678901234567890123
146  *
147  * X Space-Switch event:
148  * G Segment-Invalid Bit:
149  * P Private-Space Bit:
150  * S Storage-Alteration:
151  * R Real space
152  * TL Table-Length:
153  *
154  * A storage key has the following format:
155  * | ACC |F|R|C|0|
156  *  0   3 4 5 6 7
157  * ACC: access key
158  * F  : fetch protection bit
159  * R  : referenced bit
160  * C  : changed bit
161  */
162 
163 /* Hardware bits in the page table entry */
164 #define _PAGE_NOEXEC	0x100		/* HW no-execute bit  */
165 #define _PAGE_PROTECT	0x200		/* HW read-only bit  */
166 #define _PAGE_INVALID	0x400		/* HW invalid bit    */
167 #define _PAGE_LARGE	0x800		/* Bit to mark a large pte */
168 
169 /* Software bits in the page table entry */
170 #define _PAGE_PRESENT	0x001		/* SW pte present bit */
171 #define _PAGE_YOUNG	0x004		/* SW pte young bit */
172 #define _PAGE_DIRTY	0x008		/* SW pte dirty bit */
173 #define _PAGE_READ	0x010		/* SW pte read bit */
174 #define _PAGE_WRITE	0x020		/* SW pte write bit */
175 #define _PAGE_SPECIAL	0x040		/* SW associated with special page */
176 #define _PAGE_UNUSED	0x080		/* SW bit for pgste usage state */
177 
178 #ifdef CONFIG_MEM_SOFT_DIRTY
179 #define _PAGE_SOFT_DIRTY 0x002		/* SW pte soft dirty bit */
180 #else
181 #define _PAGE_SOFT_DIRTY 0x000
182 #endif
183 
184 #define _PAGE_SWP_EXCLUSIVE _PAGE_LARGE	/* SW pte exclusive swap bit */
185 
186 /* Set of bits not changed in pte_modify */
187 #define _PAGE_CHG_MASK		(PAGE_MASK | _PAGE_SPECIAL | _PAGE_DIRTY | \
188 				 _PAGE_YOUNG | _PAGE_SOFT_DIRTY)
189 
190 /*
191  * handle_pte_fault uses pte_present and pte_none to find out the pte type
192  * WITHOUT holding the page table lock. The _PAGE_PRESENT bit is used to
193  * distinguish present from not-present ptes. It is changed only with the page
194  * table lock held.
195  *
196  * The following table gives the different possible bit combinations for
197  * the pte hardware and software bits in the last 12 bits of a pte
198  * (. unassigned bit, x don't care, t swap type):
199  *
200  *				842100000000
201  *				000084210000
202  *				000000008421
203  *				.IR.uswrdy.p
204  * empty			.10.00000000
205  * swap				.11..ttttt.0
206  * prot-none, clean, old	.11.xx0000.1
207  * prot-none, clean, young	.11.xx0001.1
208  * prot-none, dirty, old	.11.xx0010.1
209  * prot-none, dirty, young	.11.xx0011.1
210  * read-only, clean, old	.11.xx0100.1
211  * read-only, clean, young	.01.xx0101.1
212  * read-only, dirty, old	.11.xx0110.1
213  * read-only, dirty, young	.01.xx0111.1
214  * read-write, clean, old	.11.xx1100.1
215  * read-write, clean, young	.01.xx1101.1
216  * read-write, dirty, old	.10.xx1110.1
217  * read-write, dirty, young	.00.xx1111.1
218  * HW-bits: R read-only, I invalid
219  * SW-bits: p present, y young, d dirty, r read, w write, s special,
220  *	    u unused, l large
221  *
222  * pte_none    is true for the bit pattern .10.00000000, pte == 0x400
223  * pte_swap    is true for the bit pattern .11..ooooo.0, (pte & 0x201) == 0x200
224  * pte_present is true for the bit pattern .xx.xxxxxx.1, (pte & 0x001) == 0x001
225  */
226 
227 /* Bits in the segment/region table address-space-control-element */
228 #define _ASCE_ORIGIN		~0xfffUL/* region/segment table origin	    */
229 #define _ASCE_PRIVATE_SPACE	0x100	/* private space control	    */
230 #define _ASCE_ALT_EVENT		0x80	/* storage alteration event control */
231 #define _ASCE_SPACE_SWITCH	0x40	/* space switch event		    */
232 #define _ASCE_REAL_SPACE	0x20	/* real space control		    */
233 #define _ASCE_TYPE_MASK		0x0c	/* asce table type mask		    */
234 #define _ASCE_TYPE_REGION1	0x0c	/* region first table type	    */
235 #define _ASCE_TYPE_REGION2	0x08	/* region second table type	    */
236 #define _ASCE_TYPE_REGION3	0x04	/* region third table type	    */
237 #define _ASCE_TYPE_SEGMENT	0x00	/* segment table type		    */
238 #define _ASCE_TABLE_LENGTH	0x03	/* region table length		    */
239 
240 /* Bits in the region table entry */
241 #define _REGION_ENTRY_ORIGIN	~0xfffUL/* region/segment table origin	    */
242 #define _REGION_ENTRY_PROTECT	0x200	/* region protection bit	    */
243 #define _REGION_ENTRY_NOEXEC	0x100	/* region no-execute bit	    */
244 #define _REGION_ENTRY_OFFSET	0xc0	/* region table offset		    */
245 #define _REGION_ENTRY_INVALID	0x20	/* invalid region table entry	    */
246 #define _REGION_ENTRY_TYPE_MASK	0x0c	/* region table type mask	    */
247 #define _REGION_ENTRY_TYPE_R1	0x0c	/* region first table type	    */
248 #define _REGION_ENTRY_TYPE_R2	0x08	/* region second table type	    */
249 #define _REGION_ENTRY_TYPE_R3	0x04	/* region third table type	    */
250 #define _REGION_ENTRY_LENGTH	0x03	/* region third length		    */
251 
252 #define _REGION1_ENTRY		(_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
253 #define _REGION1_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INVALID)
254 #define _REGION2_ENTRY		(_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
255 #define _REGION2_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INVALID)
256 #define _REGION3_ENTRY		(_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH)
257 #define _REGION3_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID)
258 
259 #define _REGION3_ENTRY_ORIGIN_LARGE ~0x7fffffffUL /* large page address	     */
260 #define _REGION3_ENTRY_DIRTY	0x2000	/* SW region dirty bit */
261 #define _REGION3_ENTRY_YOUNG	0x1000	/* SW region young bit */
262 #define _REGION3_ENTRY_LARGE	0x0400	/* RTTE-format control, large page  */
263 #define _REGION3_ENTRY_READ	0x0002	/* SW region read bit */
264 #define _REGION3_ENTRY_WRITE	0x0001	/* SW region write bit */
265 
266 #ifdef CONFIG_MEM_SOFT_DIRTY
267 #define _REGION3_ENTRY_SOFT_DIRTY 0x4000 /* SW region soft dirty bit */
268 #else
269 #define _REGION3_ENTRY_SOFT_DIRTY 0x0000 /* SW region soft dirty bit */
270 #endif
271 
272 #define _REGION_ENTRY_BITS	 0xfffffffffffff22fUL
273 
274 /* Bits in the segment table entry */
275 #define _SEGMENT_ENTRY_BITS			0xfffffffffffffe33UL
276 #define _SEGMENT_ENTRY_HARDWARE_BITS		0xfffffffffffffe30UL
277 #define _SEGMENT_ENTRY_HARDWARE_BITS_LARGE	0xfffffffffff00730UL
278 #define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address	    */
279 #define _SEGMENT_ENTRY_ORIGIN	~0x7ffUL/* page table origin		    */
280 #define _SEGMENT_ENTRY_PROTECT	0x200	/* segment protection bit	    */
281 #define _SEGMENT_ENTRY_NOEXEC	0x100	/* segment no-execute bit	    */
282 #define _SEGMENT_ENTRY_INVALID	0x20	/* invalid segment table entry	    */
283 #define _SEGMENT_ENTRY_TYPE_MASK 0x0c	/* segment table type mask	    */
284 
285 #define _SEGMENT_ENTRY		(0)
286 #define _SEGMENT_ENTRY_EMPTY	(_SEGMENT_ENTRY_INVALID)
287 
288 #define _SEGMENT_ENTRY_DIRTY	0x2000	/* SW segment dirty bit */
289 #define _SEGMENT_ENTRY_YOUNG	0x1000	/* SW segment young bit */
290 #define _SEGMENT_ENTRY_LARGE	0x0400	/* STE-format control, large page */
291 #define _SEGMENT_ENTRY_WRITE	0x0002	/* SW segment write bit */
292 #define _SEGMENT_ENTRY_READ	0x0001	/* SW segment read bit */
293 
294 #ifdef CONFIG_MEM_SOFT_DIRTY
295 #define _SEGMENT_ENTRY_SOFT_DIRTY 0x4000 /* SW segment soft dirty bit */
296 #else
297 #define _SEGMENT_ENTRY_SOFT_DIRTY 0x0000 /* SW segment soft dirty bit */
298 #endif
299 
300 #define _CRST_ENTRIES	2048	/* number of region/segment table entries */
301 #define _PAGE_ENTRIES	256	/* number of page table entries	*/
302 
303 #define _CRST_TABLE_SIZE (_CRST_ENTRIES * 8)
304 #define _PAGE_TABLE_SIZE (_PAGE_ENTRIES * 8)
305 
306 #define _REGION1_SHIFT	53
307 #define _REGION2_SHIFT	42
308 #define _REGION3_SHIFT	31
309 #define _SEGMENT_SHIFT	20
310 
311 #define _REGION1_INDEX	(0x7ffUL << _REGION1_SHIFT)
312 #define _REGION2_INDEX	(0x7ffUL << _REGION2_SHIFT)
313 #define _REGION3_INDEX	(0x7ffUL << _REGION3_SHIFT)
314 #define _SEGMENT_INDEX	(0x7ffUL << _SEGMENT_SHIFT)
315 #define _PAGE_INDEX	(0xffUL  << _PAGE_SHIFT)
316 
317 #define _REGION1_SIZE	(1UL << _REGION1_SHIFT)
318 #define _REGION2_SIZE	(1UL << _REGION2_SHIFT)
319 #define _REGION3_SIZE	(1UL << _REGION3_SHIFT)
320 #define _SEGMENT_SIZE	(1UL << _SEGMENT_SHIFT)
321 
322 #define _REGION1_MASK	(~(_REGION1_SIZE - 1))
323 #define _REGION2_MASK	(~(_REGION2_SIZE - 1))
324 #define _REGION3_MASK	(~(_REGION3_SIZE - 1))
325 #define _SEGMENT_MASK	(~(_SEGMENT_SIZE - 1))
326 
327 #define PMD_SHIFT	_SEGMENT_SHIFT
328 #define PUD_SHIFT	_REGION3_SHIFT
329 #define P4D_SHIFT	_REGION2_SHIFT
330 #define PGDIR_SHIFT	_REGION1_SHIFT
331 
332 #define PMD_SIZE	_SEGMENT_SIZE
333 #define PUD_SIZE	_REGION3_SIZE
334 #define P4D_SIZE	_REGION2_SIZE
335 #define PGDIR_SIZE	_REGION1_SIZE
336 
337 #define PMD_MASK	_SEGMENT_MASK
338 #define PUD_MASK	_REGION3_MASK
339 #define P4D_MASK	_REGION2_MASK
340 #define PGDIR_MASK	_REGION1_MASK
341 
342 #define PTRS_PER_PTE	_PAGE_ENTRIES
343 #define PTRS_PER_PMD	_CRST_ENTRIES
344 #define PTRS_PER_PUD	_CRST_ENTRIES
345 #define PTRS_PER_P4D	_CRST_ENTRIES
346 #define PTRS_PER_PGD	_CRST_ENTRIES
347 
348 /*
349  * Segment table and region3 table entry encoding
350  * (R = read-only, I = invalid, y = young bit):
351  *				dy..R...I...wr
352  * prot-none, clean, old	00..1...1...00
353  * prot-none, clean, young	01..1...1...00
354  * prot-none, dirty, old	10..1...1...00
355  * prot-none, dirty, young	11..1...1...00
356  * read-only, clean, old	00..1...1...01
357  * read-only, clean, young	01..1...0...01
358  * read-only, dirty, old	10..1...1...01
359  * read-only, dirty, young	11..1...0...01
360  * read-write, clean, old	00..1...1...11
361  * read-write, clean, young	01..1...0...11
362  * read-write, dirty, old	10..0...1...11
363  * read-write, dirty, young	11..0...0...11
364  * The segment table origin is used to distinguish empty (origin==0) from
365  * read-write, old segment table entries (origin!=0)
366  * HW-bits: R read-only, I invalid
367  * SW-bits: y young, d dirty, r read, w write
368  */
369 
370 /* Page status table bits for virtualization */
371 #define PGSTE_ACC_BITS	0xf000000000000000UL
372 #define PGSTE_FP_BIT	0x0800000000000000UL
373 #define PGSTE_PCL_BIT	0x0080000000000000UL
374 #define PGSTE_HR_BIT	0x0040000000000000UL
375 #define PGSTE_HC_BIT	0x0020000000000000UL
376 #define PGSTE_GR_BIT	0x0004000000000000UL
377 #define PGSTE_GC_BIT	0x0002000000000000UL
378 #define PGSTE_UC_BIT	0x0000800000000000UL	/* user dirty (migration) */
379 #define PGSTE_IN_BIT	0x0000400000000000UL	/* IPTE notify bit */
380 #define PGSTE_VSIE_BIT	0x0000200000000000UL	/* ref'd in a shadow table */
381 
382 /* Guest Page State used for virtualization */
383 #define _PGSTE_GPS_ZERO			0x0000000080000000UL
384 #define _PGSTE_GPS_NODAT		0x0000000040000000UL
385 #define _PGSTE_GPS_USAGE_MASK		0x0000000003000000UL
386 #define _PGSTE_GPS_USAGE_STABLE		0x0000000000000000UL
387 #define _PGSTE_GPS_USAGE_UNUSED		0x0000000001000000UL
388 #define _PGSTE_GPS_USAGE_POT_VOLATILE	0x0000000002000000UL
389 #define _PGSTE_GPS_USAGE_VOLATILE	_PGSTE_GPS_USAGE_MASK
390 
391 /*
392  * A user page table pointer has the space-switch-event bit, the
393  * private-space-control bit and the storage-alteration-event-control
394  * bit set. A kernel page table pointer doesn't need them.
395  */
396 #define _ASCE_USER_BITS		(_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \
397 				 _ASCE_ALT_EVENT)
398 
399 /*
400  * Page protection definitions.
401  */
402 #define PAGE_NONE	__pgprot(_PAGE_PRESENT | _PAGE_INVALID | _PAGE_PROTECT)
403 #define PAGE_RO		__pgprot(_PAGE_PRESENT | _PAGE_READ | \
404 				 _PAGE_NOEXEC  | _PAGE_INVALID | _PAGE_PROTECT)
405 #define PAGE_RX		__pgprot(_PAGE_PRESENT | _PAGE_READ | \
406 				 _PAGE_INVALID | _PAGE_PROTECT)
407 #define PAGE_RW		__pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
408 				 _PAGE_NOEXEC  | _PAGE_INVALID | _PAGE_PROTECT)
409 #define PAGE_RWX	__pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
410 				 _PAGE_INVALID | _PAGE_PROTECT)
411 
412 #define PAGE_SHARED	__pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
413 				 _PAGE_YOUNG | _PAGE_DIRTY | _PAGE_NOEXEC)
414 #define PAGE_KERNEL	__pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
415 				 _PAGE_YOUNG | _PAGE_DIRTY | _PAGE_NOEXEC)
416 #define PAGE_KERNEL_RO	__pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_YOUNG | \
417 				 _PAGE_PROTECT | _PAGE_NOEXEC)
418 #define PAGE_KERNEL_EXEC __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
419 				  _PAGE_YOUNG |	_PAGE_DIRTY)
420 
421 /*
422  * On s390 the page table entry has an invalid bit and a read-only bit.
423  * Read permission implies execute permission and write permission
424  * implies read permission.
425  */
426          /*xwr*/
427 #define __P000	PAGE_NONE
428 #define __P001	PAGE_RO
429 #define __P010	PAGE_RO
430 #define __P011	PAGE_RO
431 #define __P100	PAGE_RX
432 #define __P101	PAGE_RX
433 #define __P110	PAGE_RX
434 #define __P111	PAGE_RX
435 
436 #define __S000	PAGE_NONE
437 #define __S001	PAGE_RO
438 #define __S010	PAGE_RW
439 #define __S011	PAGE_RW
440 #define __S100	PAGE_RX
441 #define __S101	PAGE_RX
442 #define __S110	PAGE_RWX
443 #define __S111	PAGE_RWX
444 
445 /*
446  * Segment entry (large page) protection definitions.
447  */
448 #define SEGMENT_NONE	__pgprot(_SEGMENT_ENTRY_INVALID | \
449 				 _SEGMENT_ENTRY_PROTECT)
450 #define SEGMENT_RO	__pgprot(_SEGMENT_ENTRY_PROTECT | \
451 				 _SEGMENT_ENTRY_READ | \
452 				 _SEGMENT_ENTRY_NOEXEC)
453 #define SEGMENT_RX	__pgprot(_SEGMENT_ENTRY_PROTECT | \
454 				 _SEGMENT_ENTRY_READ)
455 #define SEGMENT_RW	__pgprot(_SEGMENT_ENTRY_READ | \
456 				 _SEGMENT_ENTRY_WRITE | \
457 				 _SEGMENT_ENTRY_NOEXEC)
458 #define SEGMENT_RWX	__pgprot(_SEGMENT_ENTRY_READ | \
459 				 _SEGMENT_ENTRY_WRITE)
460 #define SEGMENT_KERNEL	__pgprot(_SEGMENT_ENTRY |	\
461 				 _SEGMENT_ENTRY_LARGE |	\
462 				 _SEGMENT_ENTRY_READ |	\
463 				 _SEGMENT_ENTRY_WRITE | \
464 				 _SEGMENT_ENTRY_YOUNG | \
465 				 _SEGMENT_ENTRY_DIRTY | \
466 				 _SEGMENT_ENTRY_NOEXEC)
467 #define SEGMENT_KERNEL_RO __pgprot(_SEGMENT_ENTRY |	\
468 				 _SEGMENT_ENTRY_LARGE |	\
469 				 _SEGMENT_ENTRY_READ |	\
470 				 _SEGMENT_ENTRY_YOUNG |	\
471 				 _SEGMENT_ENTRY_PROTECT | \
472 				 _SEGMENT_ENTRY_NOEXEC)
473 #define SEGMENT_KERNEL_EXEC __pgprot(_SEGMENT_ENTRY |	\
474 				 _SEGMENT_ENTRY_LARGE |	\
475 				 _SEGMENT_ENTRY_READ |	\
476 				 _SEGMENT_ENTRY_WRITE | \
477 				 _SEGMENT_ENTRY_YOUNG |	\
478 				 _SEGMENT_ENTRY_DIRTY)
479 
480 /*
481  * Region3 entry (large page) protection definitions.
482  */
483 
484 #define REGION3_KERNEL	__pgprot(_REGION_ENTRY_TYPE_R3 | \
485 				 _REGION3_ENTRY_LARGE |	 \
486 				 _REGION3_ENTRY_READ |	 \
487 				 _REGION3_ENTRY_WRITE |	 \
488 				 _REGION3_ENTRY_YOUNG |	 \
489 				 _REGION3_ENTRY_DIRTY | \
490 				 _REGION_ENTRY_NOEXEC)
491 #define REGION3_KERNEL_RO __pgprot(_REGION_ENTRY_TYPE_R3 | \
492 				   _REGION3_ENTRY_LARGE |  \
493 				   _REGION3_ENTRY_READ |   \
494 				   _REGION3_ENTRY_YOUNG |  \
495 				   _REGION_ENTRY_PROTECT | \
496 				   _REGION_ENTRY_NOEXEC)
497 
mm_p4d_folded(struct mm_struct * mm)498 static inline bool mm_p4d_folded(struct mm_struct *mm)
499 {
500 	return mm->context.asce_limit <= _REGION1_SIZE;
501 }
502 #define mm_p4d_folded(mm) mm_p4d_folded(mm)
503 
mm_pud_folded(struct mm_struct * mm)504 static inline bool mm_pud_folded(struct mm_struct *mm)
505 {
506 	return mm->context.asce_limit <= _REGION2_SIZE;
507 }
508 #define mm_pud_folded(mm) mm_pud_folded(mm)
509 
mm_pmd_folded(struct mm_struct * mm)510 static inline bool mm_pmd_folded(struct mm_struct *mm)
511 {
512 	return mm->context.asce_limit <= _REGION3_SIZE;
513 }
514 #define mm_pmd_folded(mm) mm_pmd_folded(mm)
515 
mm_has_pgste(struct mm_struct * mm)516 static inline int mm_has_pgste(struct mm_struct *mm)
517 {
518 #ifdef CONFIG_PGSTE
519 	if (unlikely(mm->context.has_pgste))
520 		return 1;
521 #endif
522 	return 0;
523 }
524 
mm_is_protected(struct mm_struct * mm)525 static inline int mm_is_protected(struct mm_struct *mm)
526 {
527 #ifdef CONFIG_PGSTE
528 	if (unlikely(atomic_read(&mm->context.is_protected)))
529 		return 1;
530 #endif
531 	return 0;
532 }
533 
mm_alloc_pgste(struct mm_struct * mm)534 static inline int mm_alloc_pgste(struct mm_struct *mm)
535 {
536 #ifdef CONFIG_PGSTE
537 	if (unlikely(mm->context.alloc_pgste))
538 		return 1;
539 #endif
540 	return 0;
541 }
542 
clear_pte_bit(pte_t pte,pgprot_t prot)543 static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
544 {
545 	return __pte(pte_val(pte) & ~pgprot_val(prot));
546 }
547 
set_pte_bit(pte_t pte,pgprot_t prot)548 static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot)
549 {
550 	return __pte(pte_val(pte) | pgprot_val(prot));
551 }
552 
clear_pmd_bit(pmd_t pmd,pgprot_t prot)553 static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot)
554 {
555 	return __pmd(pmd_val(pmd) & ~pgprot_val(prot));
556 }
557 
set_pmd_bit(pmd_t pmd,pgprot_t prot)558 static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot)
559 {
560 	return __pmd(pmd_val(pmd) | pgprot_val(prot));
561 }
562 
clear_pud_bit(pud_t pud,pgprot_t prot)563 static inline pud_t clear_pud_bit(pud_t pud, pgprot_t prot)
564 {
565 	return __pud(pud_val(pud) & ~pgprot_val(prot));
566 }
567 
set_pud_bit(pud_t pud,pgprot_t prot)568 static inline pud_t set_pud_bit(pud_t pud, pgprot_t prot)
569 {
570 	return __pud(pud_val(pud) | pgprot_val(prot));
571 }
572 
573 /*
574  * In the case that a guest uses storage keys
575  * faults should no longer be backed by zero pages
576  */
577 #define mm_forbids_zeropage mm_has_pgste
mm_uses_skeys(struct mm_struct * mm)578 static inline int mm_uses_skeys(struct mm_struct *mm)
579 {
580 #ifdef CONFIG_PGSTE
581 	if (mm->context.uses_skeys)
582 		return 1;
583 #endif
584 	return 0;
585 }
586 
csp(unsigned int * ptr,unsigned int old,unsigned int new)587 static inline void csp(unsigned int *ptr, unsigned int old, unsigned int new)
588 {
589 	union register_pair r1 = { .even = old, .odd = new, };
590 	unsigned long address = (unsigned long)ptr | 1;
591 
592 	asm volatile(
593 		"	csp	%[r1],%[address]"
594 		: [r1] "+&d" (r1.pair), "+m" (*ptr)
595 		: [address] "d" (address)
596 		: "cc");
597 }
598 
cspg(unsigned long * ptr,unsigned long old,unsigned long new)599 static inline void cspg(unsigned long *ptr, unsigned long old, unsigned long new)
600 {
601 	union register_pair r1 = { .even = old, .odd = new, };
602 	unsigned long address = (unsigned long)ptr | 1;
603 
604 	asm volatile(
605 		"	cspg	%[r1],%[address]"
606 		: [r1] "+&d" (r1.pair), "+m" (*ptr)
607 		: [address] "d" (address)
608 		: "cc");
609 }
610 
611 #define CRDTE_DTT_PAGE		0x00UL
612 #define CRDTE_DTT_SEGMENT	0x10UL
613 #define CRDTE_DTT_REGION3	0x14UL
614 #define CRDTE_DTT_REGION2	0x18UL
615 #define CRDTE_DTT_REGION1	0x1cUL
616 
crdte(unsigned long old,unsigned long new,unsigned long * table,unsigned long dtt,unsigned long address,unsigned long asce)617 static inline void crdte(unsigned long old, unsigned long new,
618 			 unsigned long *table, unsigned long dtt,
619 			 unsigned long address, unsigned long asce)
620 {
621 	union register_pair r1 = { .even = old, .odd = new, };
622 	union register_pair r2 = { .even = __pa(table) | dtt, .odd = address, };
623 
624 	asm volatile(".insn rrf,0xb98f0000,%[r1],%[r2],%[asce],0"
625 		     : [r1] "+&d" (r1.pair)
626 		     : [r2] "d" (r2.pair), [asce] "a" (asce)
627 		     : "memory", "cc");
628 }
629 
630 /*
631  * pgd/p4d/pud/pmd/pte query functions
632  */
pgd_folded(pgd_t pgd)633 static inline int pgd_folded(pgd_t pgd)
634 {
635 	return (pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R1;
636 }
637 
pgd_present(pgd_t pgd)638 static inline int pgd_present(pgd_t pgd)
639 {
640 	if (pgd_folded(pgd))
641 		return 1;
642 	return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL;
643 }
644 
pgd_none(pgd_t pgd)645 static inline int pgd_none(pgd_t pgd)
646 {
647 	if (pgd_folded(pgd))
648 		return 0;
649 	return (pgd_val(pgd) & _REGION_ENTRY_INVALID) != 0UL;
650 }
651 
pgd_bad(pgd_t pgd)652 static inline int pgd_bad(pgd_t pgd)
653 {
654 	if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R1)
655 		return 0;
656 	return (pgd_val(pgd) & ~_REGION_ENTRY_BITS) != 0;
657 }
658 
pgd_pfn(pgd_t pgd)659 static inline unsigned long pgd_pfn(pgd_t pgd)
660 {
661 	unsigned long origin_mask;
662 
663 	origin_mask = _REGION_ENTRY_ORIGIN;
664 	return (pgd_val(pgd) & origin_mask) >> PAGE_SHIFT;
665 }
666 
p4d_folded(p4d_t p4d)667 static inline int p4d_folded(p4d_t p4d)
668 {
669 	return (p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2;
670 }
671 
p4d_present(p4d_t p4d)672 static inline int p4d_present(p4d_t p4d)
673 {
674 	if (p4d_folded(p4d))
675 		return 1;
676 	return (p4d_val(p4d) & _REGION_ENTRY_ORIGIN) != 0UL;
677 }
678 
p4d_none(p4d_t p4d)679 static inline int p4d_none(p4d_t p4d)
680 {
681 	if (p4d_folded(p4d))
682 		return 0;
683 	return p4d_val(p4d) == _REGION2_ENTRY_EMPTY;
684 }
685 
p4d_pfn(p4d_t p4d)686 static inline unsigned long p4d_pfn(p4d_t p4d)
687 {
688 	unsigned long origin_mask;
689 
690 	origin_mask = _REGION_ENTRY_ORIGIN;
691 	return (p4d_val(p4d) & origin_mask) >> PAGE_SHIFT;
692 }
693 
pud_folded(pud_t pud)694 static inline int pud_folded(pud_t pud)
695 {
696 	return (pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3;
697 }
698 
pud_present(pud_t pud)699 static inline int pud_present(pud_t pud)
700 {
701 	if (pud_folded(pud))
702 		return 1;
703 	return (pud_val(pud) & _REGION_ENTRY_ORIGIN) != 0UL;
704 }
705 
pud_none(pud_t pud)706 static inline int pud_none(pud_t pud)
707 {
708 	if (pud_folded(pud))
709 		return 0;
710 	return pud_val(pud) == _REGION3_ENTRY_EMPTY;
711 }
712 
713 #define pud_leaf	pud_large
pud_large(pud_t pud)714 static inline int pud_large(pud_t pud)
715 {
716 	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) != _REGION_ENTRY_TYPE_R3)
717 		return 0;
718 	return !!(pud_val(pud) & _REGION3_ENTRY_LARGE);
719 }
720 
721 #define pmd_leaf	pmd_large
pmd_large(pmd_t pmd)722 static inline int pmd_large(pmd_t pmd)
723 {
724 	return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0;
725 }
726 
pmd_bad(pmd_t pmd)727 static inline int pmd_bad(pmd_t pmd)
728 {
729 	if ((pmd_val(pmd) & _SEGMENT_ENTRY_TYPE_MASK) > 0 || pmd_large(pmd))
730 		return 1;
731 	return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0;
732 }
733 
pud_bad(pud_t pud)734 static inline int pud_bad(pud_t pud)
735 {
736 	unsigned long type = pud_val(pud) & _REGION_ENTRY_TYPE_MASK;
737 
738 	if (type > _REGION_ENTRY_TYPE_R3 || pud_large(pud))
739 		return 1;
740 	if (type < _REGION_ENTRY_TYPE_R3)
741 		return 0;
742 	return (pud_val(pud) & ~_REGION_ENTRY_BITS) != 0;
743 }
744 
p4d_bad(p4d_t p4d)745 static inline int p4d_bad(p4d_t p4d)
746 {
747 	unsigned long type = p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK;
748 
749 	if (type > _REGION_ENTRY_TYPE_R2)
750 		return 1;
751 	if (type < _REGION_ENTRY_TYPE_R2)
752 		return 0;
753 	return (p4d_val(p4d) & ~_REGION_ENTRY_BITS) != 0;
754 }
755 
pmd_present(pmd_t pmd)756 static inline int pmd_present(pmd_t pmd)
757 {
758 	return pmd_val(pmd) != _SEGMENT_ENTRY_EMPTY;
759 }
760 
pmd_none(pmd_t pmd)761 static inline int pmd_none(pmd_t pmd)
762 {
763 	return pmd_val(pmd) == _SEGMENT_ENTRY_EMPTY;
764 }
765 
766 #define pmd_write pmd_write
pmd_write(pmd_t pmd)767 static inline int pmd_write(pmd_t pmd)
768 {
769 	return (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) != 0;
770 }
771 
772 #define pud_write pud_write
pud_write(pud_t pud)773 static inline int pud_write(pud_t pud)
774 {
775 	return (pud_val(pud) & _REGION3_ENTRY_WRITE) != 0;
776 }
777 
pmd_dirty(pmd_t pmd)778 static inline int pmd_dirty(pmd_t pmd)
779 {
780 	return (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) != 0;
781 }
782 
pmd_young(pmd_t pmd)783 static inline int pmd_young(pmd_t pmd)
784 {
785 	return (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) != 0;
786 }
787 
pte_present(pte_t pte)788 static inline int pte_present(pte_t pte)
789 {
790 	/* Bit pattern: (pte & 0x001) == 0x001 */
791 	return (pte_val(pte) & _PAGE_PRESENT) != 0;
792 }
793 
pte_none(pte_t pte)794 static inline int pte_none(pte_t pte)
795 {
796 	/* Bit pattern: pte == 0x400 */
797 	return pte_val(pte) == _PAGE_INVALID;
798 }
799 
pte_swap(pte_t pte)800 static inline int pte_swap(pte_t pte)
801 {
802 	/* Bit pattern: (pte & 0x201) == 0x200 */
803 	return (pte_val(pte) & (_PAGE_PROTECT | _PAGE_PRESENT))
804 		== _PAGE_PROTECT;
805 }
806 
pte_special(pte_t pte)807 static inline int pte_special(pte_t pte)
808 {
809 	return (pte_val(pte) & _PAGE_SPECIAL);
810 }
811 
812 #define __HAVE_ARCH_PTE_SAME
pte_same(pte_t a,pte_t b)813 static inline int pte_same(pte_t a, pte_t b)
814 {
815 	return pte_val(a) == pte_val(b);
816 }
817 
818 #ifdef CONFIG_NUMA_BALANCING
pte_protnone(pte_t pte)819 static inline int pte_protnone(pte_t pte)
820 {
821 	return pte_present(pte) && !(pte_val(pte) & _PAGE_READ);
822 }
823 
pmd_protnone(pmd_t pmd)824 static inline int pmd_protnone(pmd_t pmd)
825 {
826 	/* pmd_large(pmd) implies pmd_present(pmd) */
827 	return pmd_large(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ);
828 }
829 #endif
830 
831 #define __HAVE_ARCH_PTE_SWP_EXCLUSIVE
pte_swp_exclusive(pte_t pte)832 static inline int pte_swp_exclusive(pte_t pte)
833 {
834 	return pte_val(pte) & _PAGE_SWP_EXCLUSIVE;
835 }
836 
pte_swp_mkexclusive(pte_t pte)837 static inline pte_t pte_swp_mkexclusive(pte_t pte)
838 {
839 	return set_pte_bit(pte, __pgprot(_PAGE_SWP_EXCLUSIVE));
840 }
841 
pte_swp_clear_exclusive(pte_t pte)842 static inline pte_t pte_swp_clear_exclusive(pte_t pte)
843 {
844 	return clear_pte_bit(pte, __pgprot(_PAGE_SWP_EXCLUSIVE));
845 }
846 
pte_soft_dirty(pte_t pte)847 static inline int pte_soft_dirty(pte_t pte)
848 {
849 	return pte_val(pte) & _PAGE_SOFT_DIRTY;
850 }
851 #define pte_swp_soft_dirty pte_soft_dirty
852 
pte_mksoft_dirty(pte_t pte)853 static inline pte_t pte_mksoft_dirty(pte_t pte)
854 {
855 	return set_pte_bit(pte, __pgprot(_PAGE_SOFT_DIRTY));
856 }
857 #define pte_swp_mksoft_dirty pte_mksoft_dirty
858 
pte_clear_soft_dirty(pte_t pte)859 static inline pte_t pte_clear_soft_dirty(pte_t pte)
860 {
861 	return clear_pte_bit(pte, __pgprot(_PAGE_SOFT_DIRTY));
862 }
863 #define pte_swp_clear_soft_dirty pte_clear_soft_dirty
864 
pmd_soft_dirty(pmd_t pmd)865 static inline int pmd_soft_dirty(pmd_t pmd)
866 {
867 	return pmd_val(pmd) & _SEGMENT_ENTRY_SOFT_DIRTY;
868 }
869 
pmd_mksoft_dirty(pmd_t pmd)870 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
871 {
872 	return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY));
873 }
874 
pmd_clear_soft_dirty(pmd_t pmd)875 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
876 {
877 	return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY));
878 }
879 
880 /*
881  * query functions pte_write/pte_dirty/pte_young only work if
882  * pte_present() is true. Undefined behaviour if not..
883  */
pte_write(pte_t pte)884 static inline int pte_write(pte_t pte)
885 {
886 	return (pte_val(pte) & _PAGE_WRITE) != 0;
887 }
888 
pte_dirty(pte_t pte)889 static inline int pte_dirty(pte_t pte)
890 {
891 	return (pte_val(pte) & _PAGE_DIRTY) != 0;
892 }
893 
pte_young(pte_t pte)894 static inline int pte_young(pte_t pte)
895 {
896 	return (pte_val(pte) & _PAGE_YOUNG) != 0;
897 }
898 
899 #define __HAVE_ARCH_PTE_UNUSED
pte_unused(pte_t pte)900 static inline int pte_unused(pte_t pte)
901 {
902 	return pte_val(pte) & _PAGE_UNUSED;
903 }
904 
905 /*
906  * Extract the pgprot value from the given pte while at the same time making it
907  * usable for kernel address space mappings where fault driven dirty and
908  * young/old accounting is not supported, i.e _PAGE_PROTECT and _PAGE_INVALID
909  * must not be set.
910  */
pte_pgprot(pte_t pte)911 static inline pgprot_t pte_pgprot(pte_t pte)
912 {
913 	unsigned long pte_flags = pte_val(pte) & _PAGE_CHG_MASK;
914 
915 	if (pte_write(pte))
916 		pte_flags |= pgprot_val(PAGE_KERNEL);
917 	else
918 		pte_flags |= pgprot_val(PAGE_KERNEL_RO);
919 	pte_flags |= pte_val(pte) & mio_wb_bit_mask;
920 
921 	return __pgprot(pte_flags);
922 }
923 
924 /*
925  * pgd/pmd/pte modification functions
926  */
927 
set_pgd(pgd_t * pgdp,pgd_t pgd)928 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
929 {
930 	WRITE_ONCE(*pgdp, pgd);
931 }
932 
set_p4d(p4d_t * p4dp,p4d_t p4d)933 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
934 {
935 	WRITE_ONCE(*p4dp, p4d);
936 }
937 
set_pud(pud_t * pudp,pud_t pud)938 static inline void set_pud(pud_t *pudp, pud_t pud)
939 {
940 	WRITE_ONCE(*pudp, pud);
941 }
942 
set_pmd(pmd_t * pmdp,pmd_t pmd)943 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
944 {
945 	WRITE_ONCE(*pmdp, pmd);
946 }
947 
set_pte(pte_t * ptep,pte_t pte)948 static inline void set_pte(pte_t *ptep, pte_t pte)
949 {
950 	WRITE_ONCE(*ptep, pte);
951 }
952 
pgd_clear(pgd_t * pgd)953 static inline void pgd_clear(pgd_t *pgd)
954 {
955 	if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1)
956 		set_pgd(pgd, __pgd(_REGION1_ENTRY_EMPTY));
957 }
958 
p4d_clear(p4d_t * p4d)959 static inline void p4d_clear(p4d_t *p4d)
960 {
961 	if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
962 		set_p4d(p4d, __p4d(_REGION2_ENTRY_EMPTY));
963 }
964 
pud_clear(pud_t * pud)965 static inline void pud_clear(pud_t *pud)
966 {
967 	if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
968 		set_pud(pud, __pud(_REGION3_ENTRY_EMPTY));
969 }
970 
pmd_clear(pmd_t * pmdp)971 static inline void pmd_clear(pmd_t *pmdp)
972 {
973 	set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
974 }
975 
pte_clear(struct mm_struct * mm,unsigned long addr,pte_t * ptep)976 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
977 {
978 	set_pte(ptep, __pte(_PAGE_INVALID));
979 }
980 
981 /*
982  * The following pte modification functions only work if
983  * pte_present() is true. Undefined behaviour if not..
984  */
pte_modify(pte_t pte,pgprot_t newprot)985 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
986 {
987 	pte = clear_pte_bit(pte, __pgprot(~_PAGE_CHG_MASK));
988 	pte = set_pte_bit(pte, newprot);
989 	/*
990 	 * newprot for PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX
991 	 * has the invalid bit set, clear it again for readable, young pages
992 	 */
993 	if ((pte_val(pte) & _PAGE_YOUNG) && (pte_val(pte) & _PAGE_READ))
994 		pte = clear_pte_bit(pte, __pgprot(_PAGE_INVALID));
995 	/*
996 	 * newprot for PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX has the page
997 	 * protection bit set, clear it again for writable, dirty pages
998 	 */
999 	if ((pte_val(pte) & _PAGE_DIRTY) && (pte_val(pte) & _PAGE_WRITE))
1000 		pte = clear_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1001 	return pte;
1002 }
1003 
pte_wrprotect(pte_t pte)1004 static inline pte_t pte_wrprotect(pte_t pte)
1005 {
1006 	pte = clear_pte_bit(pte, __pgprot(_PAGE_WRITE));
1007 	return set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1008 }
1009 
pte_mkwrite(pte_t pte)1010 static inline pte_t pte_mkwrite(pte_t pte)
1011 {
1012 	pte = set_pte_bit(pte, __pgprot(_PAGE_WRITE));
1013 	if (pte_val(pte) & _PAGE_DIRTY)
1014 		pte = clear_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1015 	return pte;
1016 }
1017 
pte_mkclean(pte_t pte)1018 static inline pte_t pte_mkclean(pte_t pte)
1019 {
1020 	pte = clear_pte_bit(pte, __pgprot(_PAGE_DIRTY));
1021 	return set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1022 }
1023 
pte_mkdirty(pte_t pte)1024 static inline pte_t pte_mkdirty(pte_t pte)
1025 {
1026 	pte = set_pte_bit(pte, __pgprot(_PAGE_DIRTY | _PAGE_SOFT_DIRTY));
1027 	if (pte_val(pte) & _PAGE_WRITE)
1028 		pte = clear_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1029 	return pte;
1030 }
1031 
pte_mkold(pte_t pte)1032 static inline pte_t pte_mkold(pte_t pte)
1033 {
1034 	pte = clear_pte_bit(pte, __pgprot(_PAGE_YOUNG));
1035 	return set_pte_bit(pte, __pgprot(_PAGE_INVALID));
1036 }
1037 
pte_mkyoung(pte_t pte)1038 static inline pte_t pte_mkyoung(pte_t pte)
1039 {
1040 	pte = set_pte_bit(pte, __pgprot(_PAGE_YOUNG));
1041 	if (pte_val(pte) & _PAGE_READ)
1042 		pte = clear_pte_bit(pte, __pgprot(_PAGE_INVALID));
1043 	return pte;
1044 }
1045 
pte_mkspecial(pte_t pte)1046 static inline pte_t pte_mkspecial(pte_t pte)
1047 {
1048 	return set_pte_bit(pte, __pgprot(_PAGE_SPECIAL));
1049 }
1050 
1051 #ifdef CONFIG_HUGETLB_PAGE
pte_mkhuge(pte_t pte)1052 static inline pte_t pte_mkhuge(pte_t pte)
1053 {
1054 	return set_pte_bit(pte, __pgprot(_PAGE_LARGE));
1055 }
1056 #endif
1057 
1058 #define IPTE_GLOBAL	0
1059 #define	IPTE_LOCAL	1
1060 
1061 #define IPTE_NODAT	0x400
1062 #define IPTE_GUEST_ASCE	0x800
1063 
__ptep_ipte(unsigned long address,pte_t * ptep,unsigned long opt,unsigned long asce,int local)1064 static __always_inline void __ptep_ipte(unsigned long address, pte_t *ptep,
1065 					unsigned long opt, unsigned long asce,
1066 					int local)
1067 {
1068 	unsigned long pto = __pa(ptep);
1069 
1070 	if (__builtin_constant_p(opt) && opt == 0) {
1071 		/* Invalidation + TLB flush for the pte */
1072 		asm volatile(
1073 			"	ipte	%[r1],%[r2],0,%[m4]"
1074 			: "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address),
1075 			  [m4] "i" (local));
1076 		return;
1077 	}
1078 
1079 	/* Invalidate ptes with options + TLB flush of the ptes */
1080 	opt = opt | (asce & _ASCE_ORIGIN);
1081 	asm volatile(
1082 		"	ipte	%[r1],%[r2],%[r3],%[m4]"
1083 		: [r2] "+a" (address), [r3] "+a" (opt)
1084 		: [r1] "a" (pto), [m4] "i" (local) : "memory");
1085 }
1086 
__ptep_ipte_range(unsigned long address,int nr,pte_t * ptep,int local)1087 static __always_inline void __ptep_ipte_range(unsigned long address, int nr,
1088 					      pte_t *ptep, int local)
1089 {
1090 	unsigned long pto = __pa(ptep);
1091 
1092 	/* Invalidate a range of ptes + TLB flush of the ptes */
1093 	do {
1094 		asm volatile(
1095 			"	ipte %[r1],%[r2],%[r3],%[m4]"
1096 			: [r2] "+a" (address), [r3] "+a" (nr)
1097 			: [r1] "a" (pto), [m4] "i" (local) : "memory");
1098 	} while (nr != 255);
1099 }
1100 
1101 /*
1102  * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
1103  * both clear the TLB for the unmapped pte. The reason is that
1104  * ptep_get_and_clear is used in common code (e.g. change_pte_range)
1105  * to modify an active pte. The sequence is
1106  *   1) ptep_get_and_clear
1107  *   2) set_pte_at
1108  *   3) flush_tlb_range
1109  * On s390 the tlb needs to get flushed with the modification of the pte
1110  * if the pte is active. The only way how this can be implemented is to
1111  * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
1112  * is a nop.
1113  */
1114 pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, pte_t *, pte_t);
1115 pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t);
1116 
1117 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
ptep_test_and_clear_young(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)1118 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
1119 					    unsigned long addr, pte_t *ptep)
1120 {
1121 	pte_t pte = *ptep;
1122 
1123 	pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte));
1124 	return pte_young(pte);
1125 }
1126 
1127 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
ptep_clear_flush_young(struct vm_area_struct * vma,unsigned long address,pte_t * ptep)1128 static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1129 					 unsigned long address, pte_t *ptep)
1130 {
1131 	return ptep_test_and_clear_young(vma, address, ptep);
1132 }
1133 
1134 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
ptep_get_and_clear(struct mm_struct * mm,unsigned long addr,pte_t * ptep)1135 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1136 				       unsigned long addr, pte_t *ptep)
1137 {
1138 	pte_t res;
1139 
1140 	res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
1141 	/* At this point the reference through the mapping is still present */
1142 	if (mm_is_protected(mm) && pte_present(res))
1143 		uv_convert_owned_from_secure(pte_val(res) & PAGE_MASK);
1144 	return res;
1145 }
1146 
1147 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1148 pte_t ptep_modify_prot_start(struct vm_area_struct *, unsigned long, pte_t *);
1149 void ptep_modify_prot_commit(struct vm_area_struct *, unsigned long,
1150 			     pte_t *, pte_t, pte_t);
1151 
1152 #define __HAVE_ARCH_PTEP_CLEAR_FLUSH
ptep_clear_flush(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)1153 static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
1154 				     unsigned long addr, pte_t *ptep)
1155 {
1156 	pte_t res;
1157 
1158 	res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID));
1159 	/* At this point the reference through the mapping is still present */
1160 	if (mm_is_protected(vma->vm_mm) && pte_present(res))
1161 		uv_convert_owned_from_secure(pte_val(res) & PAGE_MASK);
1162 	return res;
1163 }
1164 
1165 /*
1166  * The batched pte unmap code uses ptep_get_and_clear_full to clear the
1167  * ptes. Here an optimization is possible. tlb_gather_mmu flushes all
1168  * tlbs of an mm if it can guarantee that the ptes of the mm_struct
1169  * cannot be accessed while the batched unmap is running. In this case
1170  * full==1 and a simple pte_clear is enough. See tlb.h.
1171  */
1172 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
ptep_get_and_clear_full(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int full)1173 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1174 					    unsigned long addr,
1175 					    pte_t *ptep, int full)
1176 {
1177 	pte_t res;
1178 
1179 	if (full) {
1180 		res = *ptep;
1181 		set_pte(ptep, __pte(_PAGE_INVALID));
1182 	} else {
1183 		res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
1184 	}
1185 	/* At this point the reference through the mapping is still present */
1186 	if (mm_is_protected(mm) && pte_present(res))
1187 		uv_convert_owned_from_secure(pte_val(res) & PAGE_MASK);
1188 	return res;
1189 }
1190 
1191 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
ptep_set_wrprotect(struct mm_struct * mm,unsigned long addr,pte_t * ptep)1192 static inline void ptep_set_wrprotect(struct mm_struct *mm,
1193 				      unsigned long addr, pte_t *ptep)
1194 {
1195 	pte_t pte = *ptep;
1196 
1197 	if (pte_write(pte))
1198 		ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte));
1199 }
1200 
1201 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
ptep_set_access_flags(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t entry,int dirty)1202 static inline int ptep_set_access_flags(struct vm_area_struct *vma,
1203 					unsigned long addr, pte_t *ptep,
1204 					pte_t entry, int dirty)
1205 {
1206 	if (pte_same(*ptep, entry))
1207 		return 0;
1208 	ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);
1209 	return 1;
1210 }
1211 
1212 /*
1213  * Additional functions to handle KVM guest page tables
1214  */
1215 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
1216 		     pte_t *ptep, pte_t entry);
1217 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
1218 void ptep_notify(struct mm_struct *mm, unsigned long addr,
1219 		 pte_t *ptep, unsigned long bits);
1220 int ptep_force_prot(struct mm_struct *mm, unsigned long gaddr,
1221 		    pte_t *ptep, int prot, unsigned long bit);
1222 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
1223 		     pte_t *ptep , int reset);
1224 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
1225 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
1226 		    pte_t *sptep, pte_t *tptep, pte_t pte);
1227 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep);
1228 
1229 bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long address,
1230 			    pte_t *ptep);
1231 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1232 			  unsigned char key, bool nq);
1233 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1234 			       unsigned char key, unsigned char *oldkey,
1235 			       bool nq, bool mr, bool mc);
1236 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr);
1237 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1238 			  unsigned char *key);
1239 
1240 int set_pgste_bits(struct mm_struct *mm, unsigned long addr,
1241 				unsigned long bits, unsigned long value);
1242 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep);
1243 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
1244 			unsigned long *oldpte, unsigned long *oldpgste);
1245 void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr);
1246 void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr);
1247 void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr);
1248 void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr);
1249 
1250 #define pgprot_writecombine	pgprot_writecombine
1251 pgprot_t pgprot_writecombine(pgprot_t prot);
1252 
1253 #define pgprot_writethrough	pgprot_writethrough
1254 pgprot_t pgprot_writethrough(pgprot_t prot);
1255 
1256 /*
1257  * Certain architectures need to do special things when PTEs
1258  * within a page table are directly modified.  Thus, the following
1259  * hook is made available.
1260  */
set_pte_at(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t entry)1261 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
1262 			      pte_t *ptep, pte_t entry)
1263 {
1264 	if (pte_present(entry))
1265 		entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED));
1266 	if (mm_has_pgste(mm))
1267 		ptep_set_pte_at(mm, addr, ptep, entry);
1268 	else
1269 		set_pte(ptep, entry);
1270 }
1271 
1272 /*
1273  * Conversion functions: convert a page and protection to a page entry,
1274  * and a page entry and page directory to the page they refer to.
1275  */
mk_pte_phys(unsigned long physpage,pgprot_t pgprot)1276 static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
1277 {
1278 	pte_t __pte;
1279 
1280 	__pte = __pte(physpage | pgprot_val(pgprot));
1281 	if (!MACHINE_HAS_NX)
1282 		__pte = clear_pte_bit(__pte, __pgprot(_PAGE_NOEXEC));
1283 	return pte_mkyoung(__pte);
1284 }
1285 
mk_pte(struct page * page,pgprot_t pgprot)1286 static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
1287 {
1288 	unsigned long physpage = page_to_phys(page);
1289 	pte_t __pte = mk_pte_phys(physpage, pgprot);
1290 
1291 	if (pte_write(__pte) && PageDirty(page))
1292 		__pte = pte_mkdirty(__pte);
1293 	return __pte;
1294 }
1295 
1296 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
1297 #define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D-1))
1298 #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
1299 #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
1300 
1301 #define p4d_deref(pud) ((unsigned long)__va(p4d_val(pud) & _REGION_ENTRY_ORIGIN))
1302 #define pgd_deref(pgd) ((unsigned long)__va(pgd_val(pgd) & _REGION_ENTRY_ORIGIN))
1303 
pmd_deref(pmd_t pmd)1304 static inline unsigned long pmd_deref(pmd_t pmd)
1305 {
1306 	unsigned long origin_mask;
1307 
1308 	origin_mask = _SEGMENT_ENTRY_ORIGIN;
1309 	if (pmd_large(pmd))
1310 		origin_mask = _SEGMENT_ENTRY_ORIGIN_LARGE;
1311 	return (unsigned long)__va(pmd_val(pmd) & origin_mask);
1312 }
1313 
pmd_pfn(pmd_t pmd)1314 static inline unsigned long pmd_pfn(pmd_t pmd)
1315 {
1316 	return __pa(pmd_deref(pmd)) >> PAGE_SHIFT;
1317 }
1318 
pud_deref(pud_t pud)1319 static inline unsigned long pud_deref(pud_t pud)
1320 {
1321 	unsigned long origin_mask;
1322 
1323 	origin_mask = _REGION_ENTRY_ORIGIN;
1324 	if (pud_large(pud))
1325 		origin_mask = _REGION3_ENTRY_ORIGIN_LARGE;
1326 	return (unsigned long)__va(pud_val(pud) & origin_mask);
1327 }
1328 
pud_pfn(pud_t pud)1329 static inline unsigned long pud_pfn(pud_t pud)
1330 {
1331 	return __pa(pud_deref(pud)) >> PAGE_SHIFT;
1332 }
1333 
1334 /*
1335  * The pgd_offset function *always* adds the index for the top-level
1336  * region/segment table. This is done to get a sequence like the
1337  * following to work:
1338  *	pgdp = pgd_offset(current->mm, addr);
1339  *	pgd = READ_ONCE(*pgdp);
1340  *	p4dp = p4d_offset(&pgd, addr);
1341  *	...
1342  * The subsequent p4d_offset, pud_offset and pmd_offset functions
1343  * only add an index if they dereferenced the pointer.
1344  */
pgd_offset_raw(pgd_t * pgd,unsigned long address)1345 static inline pgd_t *pgd_offset_raw(pgd_t *pgd, unsigned long address)
1346 {
1347 	unsigned long rste;
1348 	unsigned int shift;
1349 
1350 	/* Get the first entry of the top level table */
1351 	rste = pgd_val(*pgd);
1352 	/* Pick up the shift from the table type of the first entry */
1353 	shift = ((rste & _REGION_ENTRY_TYPE_MASK) >> 2) * 11 + 20;
1354 	return pgd + ((address >> shift) & (PTRS_PER_PGD - 1));
1355 }
1356 
1357 #define pgd_offset(mm, address) pgd_offset_raw(READ_ONCE((mm)->pgd), address)
1358 
p4d_offset_lockless(pgd_t * pgdp,pgd_t pgd,unsigned long address)1359 static inline p4d_t *p4d_offset_lockless(pgd_t *pgdp, pgd_t pgd, unsigned long address)
1360 {
1361 	if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R1)
1362 		return (p4d_t *) pgd_deref(pgd) + p4d_index(address);
1363 	return (p4d_t *) pgdp;
1364 }
1365 #define p4d_offset_lockless p4d_offset_lockless
1366 
p4d_offset(pgd_t * pgdp,unsigned long address)1367 static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long address)
1368 {
1369 	return p4d_offset_lockless(pgdp, *pgdp, address);
1370 }
1371 
pud_offset_lockless(p4d_t * p4dp,p4d_t p4d,unsigned long address)1372 static inline pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long address)
1373 {
1374 	if ((p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R2)
1375 		return (pud_t *) p4d_deref(p4d) + pud_index(address);
1376 	return (pud_t *) p4dp;
1377 }
1378 #define pud_offset_lockless pud_offset_lockless
1379 
pud_offset(p4d_t * p4dp,unsigned long address)1380 static inline pud_t *pud_offset(p4d_t *p4dp, unsigned long address)
1381 {
1382 	return pud_offset_lockless(p4dp, *p4dp, address);
1383 }
1384 #define pud_offset pud_offset
1385 
pmd_offset_lockless(pud_t * pudp,pud_t pud,unsigned long address)1386 static inline pmd_t *pmd_offset_lockless(pud_t *pudp, pud_t pud, unsigned long address)
1387 {
1388 	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R3)
1389 		return (pmd_t *) pud_deref(pud) + pmd_index(address);
1390 	return (pmd_t *) pudp;
1391 }
1392 #define pmd_offset_lockless pmd_offset_lockless
1393 
pmd_offset(pud_t * pudp,unsigned long address)1394 static inline pmd_t *pmd_offset(pud_t *pudp, unsigned long address)
1395 {
1396 	return pmd_offset_lockless(pudp, *pudp, address);
1397 }
1398 #define pmd_offset pmd_offset
1399 
pmd_page_vaddr(pmd_t pmd)1400 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
1401 {
1402 	return (unsigned long) pmd_deref(pmd);
1403 }
1404 
gup_fast_permitted(unsigned long start,unsigned long end)1405 static inline bool gup_fast_permitted(unsigned long start, unsigned long end)
1406 {
1407 	return end <= current->mm->context.asce_limit;
1408 }
1409 #define gup_fast_permitted gup_fast_permitted
1410 
1411 #define pfn_pte(pfn, pgprot)	mk_pte_phys(((pfn) << PAGE_SHIFT), (pgprot))
1412 #define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
1413 #define pte_page(x) pfn_to_page(pte_pfn(x))
1414 
1415 #define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
1416 #define pud_page(pud) pfn_to_page(pud_pfn(pud))
1417 #define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d))
1418 #define pgd_page(pgd) pfn_to_page(pgd_pfn(pgd))
1419 
pmd_wrprotect(pmd_t pmd)1420 static inline pmd_t pmd_wrprotect(pmd_t pmd)
1421 {
1422 	pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_WRITE));
1423 	return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1424 }
1425 
pmd_mkwrite(pmd_t pmd)1426 static inline pmd_t pmd_mkwrite(pmd_t pmd)
1427 {
1428 	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_WRITE));
1429 	if (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY)
1430 		pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1431 	return pmd;
1432 }
1433 
pmd_mkclean(pmd_t pmd)1434 static inline pmd_t pmd_mkclean(pmd_t pmd)
1435 {
1436 	pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_DIRTY));
1437 	return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1438 }
1439 
pmd_mkdirty(pmd_t pmd)1440 static inline pmd_t pmd_mkdirty(pmd_t pmd)
1441 {
1442 	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_DIRTY | _SEGMENT_ENTRY_SOFT_DIRTY));
1443 	if (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE)
1444 		pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1445 	return pmd;
1446 }
1447 
pud_wrprotect(pud_t pud)1448 static inline pud_t pud_wrprotect(pud_t pud)
1449 {
1450 	pud = clear_pud_bit(pud, __pgprot(_REGION3_ENTRY_WRITE));
1451 	return set_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1452 }
1453 
pud_mkwrite(pud_t pud)1454 static inline pud_t pud_mkwrite(pud_t pud)
1455 {
1456 	pud = set_pud_bit(pud, __pgprot(_REGION3_ENTRY_WRITE));
1457 	if (pud_val(pud) & _REGION3_ENTRY_DIRTY)
1458 		pud = clear_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1459 	return pud;
1460 }
1461 
pud_mkclean(pud_t pud)1462 static inline pud_t pud_mkclean(pud_t pud)
1463 {
1464 	pud = clear_pud_bit(pud, __pgprot(_REGION3_ENTRY_DIRTY));
1465 	return set_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1466 }
1467 
pud_mkdirty(pud_t pud)1468 static inline pud_t pud_mkdirty(pud_t pud)
1469 {
1470 	pud = set_pud_bit(pud, __pgprot(_REGION3_ENTRY_DIRTY | _REGION3_ENTRY_SOFT_DIRTY));
1471 	if (pud_val(pud) & _REGION3_ENTRY_WRITE)
1472 		pud = clear_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1473 	return pud;
1474 }
1475 
1476 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
massage_pgprot_pmd(pgprot_t pgprot)1477 static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
1478 {
1479 	/*
1480 	 * pgprot is PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW or PAGE_RWX
1481 	 * (see __Pxxx / __Sxxx). Convert to segment table entry format.
1482 	 */
1483 	if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE))
1484 		return pgprot_val(SEGMENT_NONE);
1485 	if (pgprot_val(pgprot) == pgprot_val(PAGE_RO))
1486 		return pgprot_val(SEGMENT_RO);
1487 	if (pgprot_val(pgprot) == pgprot_val(PAGE_RX))
1488 		return pgprot_val(SEGMENT_RX);
1489 	if (pgprot_val(pgprot) == pgprot_val(PAGE_RW))
1490 		return pgprot_val(SEGMENT_RW);
1491 	return pgprot_val(SEGMENT_RWX);
1492 }
1493 
pmd_mkyoung(pmd_t pmd)1494 static inline pmd_t pmd_mkyoung(pmd_t pmd)
1495 {
1496 	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_YOUNG));
1497 	if (pmd_val(pmd) & _SEGMENT_ENTRY_READ)
1498 		pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1499 	return pmd;
1500 }
1501 
pmd_mkold(pmd_t pmd)1502 static inline pmd_t pmd_mkold(pmd_t pmd)
1503 {
1504 	pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_YOUNG));
1505 	return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1506 }
1507 
pmd_modify(pmd_t pmd,pgprot_t newprot)1508 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1509 {
1510 	unsigned long mask;
1511 
1512 	mask  = _SEGMENT_ENTRY_ORIGIN_LARGE;
1513 	mask |= _SEGMENT_ENTRY_DIRTY;
1514 	mask |= _SEGMENT_ENTRY_YOUNG;
1515 	mask |=	_SEGMENT_ENTRY_LARGE;
1516 	mask |= _SEGMENT_ENTRY_SOFT_DIRTY;
1517 	pmd = __pmd(pmd_val(pmd) & mask);
1518 	pmd = set_pmd_bit(pmd, __pgprot(massage_pgprot_pmd(newprot)));
1519 	if (!(pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY))
1520 		pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1521 	if (!(pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG))
1522 		pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1523 	return pmd;
1524 }
1525 
mk_pmd_phys(unsigned long physpage,pgprot_t pgprot)1526 static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
1527 {
1528 	return __pmd(physpage + massage_pgprot_pmd(pgprot));
1529 }
1530 
1531 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLB_PAGE */
1532 
__pmdp_csp(pmd_t * pmdp)1533 static inline void __pmdp_csp(pmd_t *pmdp)
1534 {
1535 	csp((unsigned int *)pmdp + 1, pmd_val(*pmdp),
1536 	    pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
1537 }
1538 
1539 #define IDTE_GLOBAL	0
1540 #define IDTE_LOCAL	1
1541 
1542 #define IDTE_PTOA	0x0800
1543 #define IDTE_NODAT	0x1000
1544 #define IDTE_GUEST_ASCE	0x2000
1545 
__pmdp_idte(unsigned long addr,pmd_t * pmdp,unsigned long opt,unsigned long asce,int local)1546 static __always_inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp,
1547 					unsigned long opt, unsigned long asce,
1548 					int local)
1549 {
1550 	unsigned long sto;
1551 
1552 	sto = __pa(pmdp) - pmd_index(addr) * sizeof(pmd_t);
1553 	if (__builtin_constant_p(opt) && opt == 0) {
1554 		/* flush without guest asce */
1555 		asm volatile(
1556 			"	idte	%[r1],0,%[r2],%[m4]"
1557 			: "+m" (*pmdp)
1558 			: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK)),
1559 			  [m4] "i" (local)
1560 			: "cc" );
1561 	} else {
1562 		/* flush with guest asce */
1563 		asm volatile(
1564 			"	idte	%[r1],%[r3],%[r2],%[m4]"
1565 			: "+m" (*pmdp)
1566 			: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
1567 			  [r3] "a" (asce), [m4] "i" (local)
1568 			: "cc" );
1569 	}
1570 }
1571 
__pudp_idte(unsigned long addr,pud_t * pudp,unsigned long opt,unsigned long asce,int local)1572 static __always_inline void __pudp_idte(unsigned long addr, pud_t *pudp,
1573 					unsigned long opt, unsigned long asce,
1574 					int local)
1575 {
1576 	unsigned long r3o;
1577 
1578 	r3o = __pa(pudp) - pud_index(addr) * sizeof(pud_t);
1579 	r3o |= _ASCE_TYPE_REGION3;
1580 	if (__builtin_constant_p(opt) && opt == 0) {
1581 		/* flush without guest asce */
1582 		asm volatile(
1583 			"	idte	%[r1],0,%[r2],%[m4]"
1584 			: "+m" (*pudp)
1585 			: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK)),
1586 			  [m4] "i" (local)
1587 			: "cc");
1588 	} else {
1589 		/* flush with guest asce */
1590 		asm volatile(
1591 			"	idte	%[r1],%[r3],%[r2],%[m4]"
1592 			: "+m" (*pudp)
1593 			: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
1594 			  [r3] "a" (asce), [m4] "i" (local)
1595 			: "cc" );
1596 	}
1597 }
1598 
1599 pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
1600 pmd_t pmdp_xchg_lazy(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
1601 pud_t pudp_xchg_direct(struct mm_struct *, unsigned long, pud_t *, pud_t);
1602 
1603 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1604 
1605 #define __HAVE_ARCH_PGTABLE_DEPOSIT
1606 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1607 				pgtable_t pgtable);
1608 
1609 #define __HAVE_ARCH_PGTABLE_WITHDRAW
1610 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
1611 
1612 #define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
pmdp_set_access_flags(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp,pmd_t entry,int dirty)1613 static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
1614 					unsigned long addr, pmd_t *pmdp,
1615 					pmd_t entry, int dirty)
1616 {
1617 	VM_BUG_ON(addr & ~HPAGE_MASK);
1618 
1619 	entry = pmd_mkyoung(entry);
1620 	if (dirty)
1621 		entry = pmd_mkdirty(entry);
1622 	if (pmd_val(*pmdp) == pmd_val(entry))
1623 		return 0;
1624 	pmdp_xchg_direct(vma->vm_mm, addr, pmdp, entry);
1625 	return 1;
1626 }
1627 
1628 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
pmdp_test_and_clear_young(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1629 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1630 					    unsigned long addr, pmd_t *pmdp)
1631 {
1632 	pmd_t pmd = *pmdp;
1633 
1634 	pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd_mkold(pmd));
1635 	return pmd_young(pmd);
1636 }
1637 
1638 #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
pmdp_clear_flush_young(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1639 static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
1640 					 unsigned long addr, pmd_t *pmdp)
1641 {
1642 	VM_BUG_ON(addr & ~HPAGE_MASK);
1643 	return pmdp_test_and_clear_young(vma, addr, pmdp);
1644 }
1645 
set_pmd_at(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t entry)1646 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1647 			      pmd_t *pmdp, pmd_t entry)
1648 {
1649 	if (!MACHINE_HAS_NX)
1650 		entry = clear_pmd_bit(entry, __pgprot(_SEGMENT_ENTRY_NOEXEC));
1651 	set_pmd(pmdp, entry);
1652 }
1653 
pmd_mkhuge(pmd_t pmd)1654 static inline pmd_t pmd_mkhuge(pmd_t pmd)
1655 {
1656 	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_LARGE));
1657 	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_YOUNG));
1658 	return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1659 }
1660 
1661 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
pmdp_huge_get_and_clear(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)1662 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1663 					    unsigned long addr, pmd_t *pmdp)
1664 {
1665 	return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1666 }
1667 
1668 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
pmdp_huge_get_and_clear_full(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp,int full)1669 static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,
1670 						 unsigned long addr,
1671 						 pmd_t *pmdp, int full)
1672 {
1673 	if (full) {
1674 		pmd_t pmd = *pmdp;
1675 		set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1676 		return pmd;
1677 	}
1678 	return pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1679 }
1680 
1681 #define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
pmdp_huge_clear_flush(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1682 static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
1683 					  unsigned long addr, pmd_t *pmdp)
1684 {
1685 	return pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp);
1686 }
1687 
1688 #define __HAVE_ARCH_PMDP_INVALIDATE
pmdp_invalidate(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1689 static inline pmd_t pmdp_invalidate(struct vm_area_struct *vma,
1690 				   unsigned long addr, pmd_t *pmdp)
1691 {
1692 	pmd_t pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
1693 
1694 	return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
1695 }
1696 
1697 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
pmdp_set_wrprotect(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)1698 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1699 				      unsigned long addr, pmd_t *pmdp)
1700 {
1701 	pmd_t pmd = *pmdp;
1702 
1703 	if (pmd_write(pmd))
1704 		pmd = pmdp_xchg_lazy(mm, addr, pmdp, pmd_wrprotect(pmd));
1705 }
1706 
pmdp_collapse_flush(struct vm_area_struct * vma,unsigned long address,pmd_t * pmdp)1707 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
1708 					unsigned long address,
1709 					pmd_t *pmdp)
1710 {
1711 	return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
1712 }
1713 #define pmdp_collapse_flush pmdp_collapse_flush
1714 
1715 #define pfn_pmd(pfn, pgprot)	mk_pmd_phys(((pfn) << PAGE_SHIFT), (pgprot))
1716 #define mk_pmd(page, pgprot)	pfn_pmd(page_to_pfn(page), (pgprot))
1717 
pmd_trans_huge(pmd_t pmd)1718 static inline int pmd_trans_huge(pmd_t pmd)
1719 {
1720 	return pmd_val(pmd) & _SEGMENT_ENTRY_LARGE;
1721 }
1722 
1723 #define has_transparent_hugepage has_transparent_hugepage
has_transparent_hugepage(void)1724 static inline int has_transparent_hugepage(void)
1725 {
1726 	return MACHINE_HAS_EDAT1 ? 1 : 0;
1727 }
1728 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1729 
1730 /*
1731  * 64 bit swap entry format:
1732  * A page-table entry has some bits we have to treat in a special way.
1733  * Bits 54 and 63 are used to indicate the page type. Bit 53 marks the pte
1734  * as invalid.
1735  * A swap pte is indicated by bit pattern (pte & 0x201) == 0x200
1736  * |			  offset			|E11XX|type |S0|
1737  * |0000000000111111111122222222223333333333444444444455|55555|55566|66|
1738  * |0123456789012345678901234567890123456789012345678901|23456|78901|23|
1739  *
1740  * Bits 0-51 store the offset.
1741  * Bit 52 (E) is used to remember PG_anon_exclusive.
1742  * Bits 57-61 store the type.
1743  * Bit 62 (S) is used for softdirty tracking.
1744  * Bits 55 and 56 (X) are unused.
1745  */
1746 
1747 #define __SWP_OFFSET_MASK	((1UL << 52) - 1)
1748 #define __SWP_OFFSET_SHIFT	12
1749 #define __SWP_TYPE_MASK		((1UL << 5) - 1)
1750 #define __SWP_TYPE_SHIFT	2
1751 
mk_swap_pte(unsigned long type,unsigned long offset)1752 static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
1753 {
1754 	unsigned long pteval;
1755 
1756 	pteval = _PAGE_INVALID | _PAGE_PROTECT;
1757 	pteval |= (offset & __SWP_OFFSET_MASK) << __SWP_OFFSET_SHIFT;
1758 	pteval |= (type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT;
1759 	return __pte(pteval);
1760 }
1761 
__swp_type(swp_entry_t entry)1762 static inline unsigned long __swp_type(swp_entry_t entry)
1763 {
1764 	return (entry.val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK;
1765 }
1766 
__swp_offset(swp_entry_t entry)1767 static inline unsigned long __swp_offset(swp_entry_t entry)
1768 {
1769 	return (entry.val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK;
1770 }
1771 
__swp_entry(unsigned long type,unsigned long offset)1772 static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset)
1773 {
1774 	return (swp_entry_t) { pte_val(mk_swap_pte(type, offset)) };
1775 }
1776 
1777 #define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
1778 #define __swp_entry_to_pte(x)	((pte_t) { (x).val })
1779 
1780 #define kern_addr_valid(addr)   (1)
1781 
1782 extern int vmem_add_mapping(unsigned long start, unsigned long size);
1783 extern void vmem_remove_mapping(unsigned long start, unsigned long size);
1784 extern int s390_enable_sie(void);
1785 extern int s390_enable_skey(void);
1786 extern void s390_reset_cmma(struct mm_struct *mm);
1787 
1788 /* s390 has a private copy of get unmapped area to deal with cache synonyms */
1789 #define HAVE_ARCH_UNMAPPED_AREA
1790 #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1791 
1792 #define pmd_pgtable(pmd) \
1793 	((pgtable_t)__va(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE))
1794 
1795 #endif /* _S390_PAGE_H */
1796