1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_SWAPOPS_H
3 #define _LINUX_SWAPOPS_H
4 
5 #include <linux/radix-tree.h>
6 #include <linux/bug.h>
7 #include <linux/mm_types.h>
8 
9 #ifdef CONFIG_MMU
10 
11 /*
12  * swapcache pages are stored in the swapper_space radix tree.  We want to
13  * get good packing density in that tree, so the index should be dense in
14  * the low-order bits.
15  *
16  * We arrange the `type' and `offset' fields so that `type' is at the six
17  * high-order bits of the swp_entry_t and `offset' is right-aligned in the
18  * remaining bits.  Although `type' itself needs only five bits, we allow for
19  * shmem/tmpfs to shift it all up a further one bit: see swp_to_radix_entry().
20  *
21  * swp_entry_t's are *never* stored anywhere in their arch-dependent format.
22  */
23 #define SWP_TYPE_SHIFT	(BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT)
24 #define SWP_OFFSET_MASK	((1UL << SWP_TYPE_SHIFT) - 1)
25 
26 /* Clear all flags but only keep swp_entry_t related information */
pte_swp_clear_flags(pte_t pte)27 static inline pte_t pte_swp_clear_flags(pte_t pte)
28 {
29 	if (pte_swp_exclusive(pte))
30 		pte = pte_swp_clear_exclusive(pte);
31 	if (pte_swp_soft_dirty(pte))
32 		pte = pte_swp_clear_soft_dirty(pte);
33 	if (pte_swp_uffd_wp(pte))
34 		pte = pte_swp_clear_uffd_wp(pte);
35 	return pte;
36 }
37 
38 /*
39  * Store a type+offset into a swp_entry_t in an arch-independent format
40  */
swp_entry(unsigned long type,pgoff_t offset)41 static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset)
42 {
43 	swp_entry_t ret;
44 
45 	ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK);
46 	return ret;
47 }
48 
49 /*
50  * Extract the `type' field from a swp_entry_t.  The swp_entry_t is in
51  * arch-independent format
52  */
swp_type(swp_entry_t entry)53 static inline unsigned swp_type(swp_entry_t entry)
54 {
55 	return (entry.val >> SWP_TYPE_SHIFT);
56 }
57 
58 /*
59  * Extract the `offset' field from a swp_entry_t.  The swp_entry_t is in
60  * arch-independent format
61  */
swp_offset(swp_entry_t entry)62 static inline pgoff_t swp_offset(swp_entry_t entry)
63 {
64 	return entry.val & SWP_OFFSET_MASK;
65 }
66 
67 /* check whether a pte points to a swap entry */
is_swap_pte(pte_t pte)68 static inline int is_swap_pte(pte_t pte)
69 {
70 	return !pte_none(pte) && !pte_present(pte);
71 }
72 
73 /*
74  * Convert the arch-dependent pte representation of a swp_entry_t into an
75  * arch-independent swp_entry_t.
76  */
pte_to_swp_entry(pte_t pte)77 static inline swp_entry_t pte_to_swp_entry(pte_t pte)
78 {
79 	swp_entry_t arch_entry;
80 
81 	pte = pte_swp_clear_flags(pte);
82 	arch_entry = __pte_to_swp_entry(pte);
83 	return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
84 }
85 
86 /*
87  * Convert the arch-independent representation of a swp_entry_t into the
88  * arch-dependent pte representation.
89  */
swp_entry_to_pte(swp_entry_t entry)90 static inline pte_t swp_entry_to_pte(swp_entry_t entry)
91 {
92 	swp_entry_t arch_entry;
93 
94 	arch_entry = __swp_entry(swp_type(entry), swp_offset(entry));
95 	return __swp_entry_to_pte(arch_entry);
96 }
97 
radix_to_swp_entry(void * arg)98 static inline swp_entry_t radix_to_swp_entry(void *arg)
99 {
100 	swp_entry_t entry;
101 
102 	entry.val = xa_to_value(arg);
103 	return entry;
104 }
105 
swp_to_radix_entry(swp_entry_t entry)106 static inline void *swp_to_radix_entry(swp_entry_t entry)
107 {
108 	return xa_mk_value(entry.val);
109 }
110 
make_swapin_error_entry(struct page * page)111 static inline swp_entry_t make_swapin_error_entry(struct page *page)
112 {
113 	return swp_entry(SWP_SWAPIN_ERROR, page_to_pfn(page));
114 }
115 
is_swapin_error_entry(swp_entry_t entry)116 static inline int is_swapin_error_entry(swp_entry_t entry)
117 {
118 	return swp_type(entry) == SWP_SWAPIN_ERROR;
119 }
120 
121 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
make_readable_device_private_entry(pgoff_t offset)122 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset)
123 {
124 	return swp_entry(SWP_DEVICE_READ, offset);
125 }
126 
make_writable_device_private_entry(pgoff_t offset)127 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset)
128 {
129 	return swp_entry(SWP_DEVICE_WRITE, offset);
130 }
131 
is_device_private_entry(swp_entry_t entry)132 static inline bool is_device_private_entry(swp_entry_t entry)
133 {
134 	int type = swp_type(entry);
135 	return type == SWP_DEVICE_READ || type == SWP_DEVICE_WRITE;
136 }
137 
is_writable_device_private_entry(swp_entry_t entry)138 static inline bool is_writable_device_private_entry(swp_entry_t entry)
139 {
140 	return unlikely(swp_type(entry) == SWP_DEVICE_WRITE);
141 }
142 
make_readable_device_exclusive_entry(pgoff_t offset)143 static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset)
144 {
145 	return swp_entry(SWP_DEVICE_EXCLUSIVE_READ, offset);
146 }
147 
make_writable_device_exclusive_entry(pgoff_t offset)148 static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset)
149 {
150 	return swp_entry(SWP_DEVICE_EXCLUSIVE_WRITE, offset);
151 }
152 
is_device_exclusive_entry(swp_entry_t entry)153 static inline bool is_device_exclusive_entry(swp_entry_t entry)
154 {
155 	return swp_type(entry) == SWP_DEVICE_EXCLUSIVE_READ ||
156 		swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE;
157 }
158 
is_writable_device_exclusive_entry(swp_entry_t entry)159 static inline bool is_writable_device_exclusive_entry(swp_entry_t entry)
160 {
161 	return unlikely(swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE);
162 }
163 #else /* CONFIG_DEVICE_PRIVATE */
make_readable_device_private_entry(pgoff_t offset)164 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset)
165 {
166 	return swp_entry(0, 0);
167 }
168 
make_writable_device_private_entry(pgoff_t offset)169 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset)
170 {
171 	return swp_entry(0, 0);
172 }
173 
is_device_private_entry(swp_entry_t entry)174 static inline bool is_device_private_entry(swp_entry_t entry)
175 {
176 	return false;
177 }
178 
is_writable_device_private_entry(swp_entry_t entry)179 static inline bool is_writable_device_private_entry(swp_entry_t entry)
180 {
181 	return false;
182 }
183 
make_readable_device_exclusive_entry(pgoff_t offset)184 static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset)
185 {
186 	return swp_entry(0, 0);
187 }
188 
make_writable_device_exclusive_entry(pgoff_t offset)189 static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset)
190 {
191 	return swp_entry(0, 0);
192 }
193 
is_device_exclusive_entry(swp_entry_t entry)194 static inline bool is_device_exclusive_entry(swp_entry_t entry)
195 {
196 	return false;
197 }
198 
is_writable_device_exclusive_entry(swp_entry_t entry)199 static inline bool is_writable_device_exclusive_entry(swp_entry_t entry)
200 {
201 	return false;
202 }
203 #endif /* CONFIG_DEVICE_PRIVATE */
204 
205 #ifdef CONFIG_MIGRATION
is_migration_entry(swp_entry_t entry)206 static inline int is_migration_entry(swp_entry_t entry)
207 {
208 	return unlikely(swp_type(entry) == SWP_MIGRATION_READ ||
209 			swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE ||
210 			swp_type(entry) == SWP_MIGRATION_WRITE);
211 }
212 
is_writable_migration_entry(swp_entry_t entry)213 static inline int is_writable_migration_entry(swp_entry_t entry)
214 {
215 	return unlikely(swp_type(entry) == SWP_MIGRATION_WRITE);
216 }
217 
is_readable_migration_entry(swp_entry_t entry)218 static inline int is_readable_migration_entry(swp_entry_t entry)
219 {
220 	return unlikely(swp_type(entry) == SWP_MIGRATION_READ);
221 }
222 
is_readable_exclusive_migration_entry(swp_entry_t entry)223 static inline int is_readable_exclusive_migration_entry(swp_entry_t entry)
224 {
225 	return unlikely(swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE);
226 }
227 
make_readable_migration_entry(pgoff_t offset)228 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset)
229 {
230 	return swp_entry(SWP_MIGRATION_READ, offset);
231 }
232 
make_readable_exclusive_migration_entry(pgoff_t offset)233 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset)
234 {
235 	return swp_entry(SWP_MIGRATION_READ_EXCLUSIVE, offset);
236 }
237 
make_writable_migration_entry(pgoff_t offset)238 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset)
239 {
240 	return swp_entry(SWP_MIGRATION_WRITE, offset);
241 }
242 
243 extern void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
244 					spinlock_t *ptl);
245 extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
246 					unsigned long address);
247 #ifdef CONFIG_HUGETLB_PAGE
248 extern void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl);
249 extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte);
250 #endif
251 #else
make_readable_migration_entry(pgoff_t offset)252 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset)
253 {
254 	return swp_entry(0, 0);
255 }
256 
make_readable_exclusive_migration_entry(pgoff_t offset)257 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset)
258 {
259 	return swp_entry(0, 0);
260 }
261 
make_writable_migration_entry(pgoff_t offset)262 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset)
263 {
264 	return swp_entry(0, 0);
265 }
266 
is_migration_entry(swp_entry_t swp)267 static inline int is_migration_entry(swp_entry_t swp)
268 {
269 	return 0;
270 }
271 
__migration_entry_wait(struct mm_struct * mm,pte_t * ptep,spinlock_t * ptl)272 static inline void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
273 					spinlock_t *ptl) { }
migration_entry_wait(struct mm_struct * mm,pmd_t * pmd,unsigned long address)274 static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
275 					 unsigned long address) { }
276 #ifdef CONFIG_HUGETLB_PAGE
__migration_entry_wait_huge(pte_t * ptep,spinlock_t * ptl)277 static inline void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl) { }
migration_entry_wait_huge(struct vm_area_struct * vma,pte_t * pte)278 static inline void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) { }
279 #endif
is_writable_migration_entry(swp_entry_t entry)280 static inline int is_writable_migration_entry(swp_entry_t entry)
281 {
282 	return 0;
283 }
is_readable_migration_entry(swp_entry_t entry)284 static inline int is_readable_migration_entry(swp_entry_t entry)
285 {
286 	return 0;
287 }
288 
289 #endif
290 
291 typedef unsigned long pte_marker;
292 
293 #define  PTE_MARKER_UFFD_WP  BIT(0)
294 #define  PTE_MARKER_MASK     (PTE_MARKER_UFFD_WP)
295 
296 #ifdef CONFIG_PTE_MARKER
297 
make_pte_marker_entry(pte_marker marker)298 static inline swp_entry_t make_pte_marker_entry(pte_marker marker)
299 {
300 	return swp_entry(SWP_PTE_MARKER, marker);
301 }
302 
is_pte_marker_entry(swp_entry_t entry)303 static inline bool is_pte_marker_entry(swp_entry_t entry)
304 {
305 	return swp_type(entry) == SWP_PTE_MARKER;
306 }
307 
pte_marker_get(swp_entry_t entry)308 static inline pte_marker pte_marker_get(swp_entry_t entry)
309 {
310 	return swp_offset(entry) & PTE_MARKER_MASK;
311 }
312 
is_pte_marker(pte_t pte)313 static inline bool is_pte_marker(pte_t pte)
314 {
315 	return is_swap_pte(pte) && is_pte_marker_entry(pte_to_swp_entry(pte));
316 }
317 
318 #else /* CONFIG_PTE_MARKER */
319 
make_pte_marker_entry(pte_marker marker)320 static inline swp_entry_t make_pte_marker_entry(pte_marker marker)
321 {
322 	/* This should never be called if !CONFIG_PTE_MARKER */
323 	WARN_ON_ONCE(1);
324 	return swp_entry(0, 0);
325 }
326 
is_pte_marker_entry(swp_entry_t entry)327 static inline bool is_pte_marker_entry(swp_entry_t entry)
328 {
329 	return false;
330 }
331 
pte_marker_get(swp_entry_t entry)332 static inline pte_marker pte_marker_get(swp_entry_t entry)
333 {
334 	return 0;
335 }
336 
is_pte_marker(pte_t pte)337 static inline bool is_pte_marker(pte_t pte)
338 {
339 	return false;
340 }
341 
342 #endif /* CONFIG_PTE_MARKER */
343 
make_pte_marker(pte_marker marker)344 static inline pte_t make_pte_marker(pte_marker marker)
345 {
346 	return swp_entry_to_pte(make_pte_marker_entry(marker));
347 }
348 
349 /*
350  * This is a special version to check pte_none() just to cover the case when
351  * the pte is a pte marker.  It existed because in many cases the pte marker
352  * should be seen as a none pte; it's just that we have stored some information
353  * onto the none pte so it becomes not-none any more.
354  *
355  * It should be used when the pte is file-backed, ram-based and backing
356  * userspace pages, like shmem.  It is not needed upon pgtables that do not
357  * support pte markers at all.  For example, it's not needed on anonymous
358  * memory, kernel-only memory (including when the system is during-boot),
359  * non-ram based generic file-system.  It's fine to be used even there, but the
360  * extra pte marker check will be pure overhead.
361  *
362  * For systems configured with !CONFIG_PTE_MARKER this will be automatically
363  * optimized to pte_none().
364  */
pte_none_mostly(pte_t pte)365 static inline int pte_none_mostly(pte_t pte)
366 {
367 	return pte_none(pte) || is_pte_marker(pte);
368 }
369 
pfn_swap_entry_to_page(swp_entry_t entry)370 static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)
371 {
372 	struct page *p = pfn_to_page(swp_offset(entry));
373 
374 	/*
375 	 * Any use of migration entries may only occur while the
376 	 * corresponding page is locked
377 	 */
378 	BUG_ON(is_migration_entry(entry) && !PageLocked(p));
379 
380 	return p;
381 }
382 
383 /*
384  * A pfn swap entry is a special type of swap entry that always has a pfn stored
385  * in the swap offset. They are used to represent unaddressable device memory
386  * and to restrict access to a page undergoing migration.
387  */
is_pfn_swap_entry(swp_entry_t entry)388 static inline bool is_pfn_swap_entry(swp_entry_t entry)
389 {
390 	return is_migration_entry(entry) || is_device_private_entry(entry) ||
391 	       is_device_exclusive_entry(entry);
392 }
393 
394 struct page_vma_mapped_walk;
395 
396 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
397 extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
398 		struct page *page);
399 
400 extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
401 		struct page *new);
402 
403 extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd);
404 
pmd_to_swp_entry(pmd_t pmd)405 static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd)
406 {
407 	swp_entry_t arch_entry;
408 
409 	if (pmd_swp_soft_dirty(pmd))
410 		pmd = pmd_swp_clear_soft_dirty(pmd);
411 	if (pmd_swp_uffd_wp(pmd))
412 		pmd = pmd_swp_clear_uffd_wp(pmd);
413 	arch_entry = __pmd_to_swp_entry(pmd);
414 	return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
415 }
416 
swp_entry_to_pmd(swp_entry_t entry)417 static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
418 {
419 	swp_entry_t arch_entry;
420 
421 	arch_entry = __swp_entry(swp_type(entry), swp_offset(entry));
422 	return __swp_entry_to_pmd(arch_entry);
423 }
424 
is_pmd_migration_entry(pmd_t pmd)425 static inline int is_pmd_migration_entry(pmd_t pmd)
426 {
427 	return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd));
428 }
429 #else
set_pmd_migration_entry(struct page_vma_mapped_walk * pvmw,struct page * page)430 static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
431 		struct page *page)
432 {
433 	BUILD_BUG();
434 }
435 
remove_migration_pmd(struct page_vma_mapped_walk * pvmw,struct page * new)436 static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
437 		struct page *new)
438 {
439 	BUILD_BUG();
440 }
441 
pmd_migration_entry_wait(struct mm_struct * m,pmd_t * p)442 static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { }
443 
pmd_to_swp_entry(pmd_t pmd)444 static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd)
445 {
446 	return swp_entry(0, 0);
447 }
448 
swp_entry_to_pmd(swp_entry_t entry)449 static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
450 {
451 	return __pmd(0);
452 }
453 
is_pmd_migration_entry(pmd_t pmd)454 static inline int is_pmd_migration_entry(pmd_t pmd)
455 {
456 	return 0;
457 }
458 #endif
459 
460 #ifdef CONFIG_MEMORY_FAILURE
461 
462 extern atomic_long_t num_poisoned_pages __read_mostly;
463 
464 /*
465  * Support for hardware poisoned pages
466  */
make_hwpoison_entry(struct page * page)467 static inline swp_entry_t make_hwpoison_entry(struct page *page)
468 {
469 	BUG_ON(!PageLocked(page));
470 	return swp_entry(SWP_HWPOISON, page_to_pfn(page));
471 }
472 
is_hwpoison_entry(swp_entry_t entry)473 static inline int is_hwpoison_entry(swp_entry_t entry)
474 {
475 	return swp_type(entry) == SWP_HWPOISON;
476 }
477 
hwpoison_entry_to_pfn(swp_entry_t entry)478 static inline unsigned long hwpoison_entry_to_pfn(swp_entry_t entry)
479 {
480 	return swp_offset(entry);
481 }
482 
num_poisoned_pages_inc(void)483 static inline void num_poisoned_pages_inc(void)
484 {
485 	atomic_long_inc(&num_poisoned_pages);
486 }
487 
num_poisoned_pages_dec(void)488 static inline void num_poisoned_pages_dec(void)
489 {
490 	atomic_long_dec(&num_poisoned_pages);
491 }
492 
493 #else
494 
make_hwpoison_entry(struct page * page)495 static inline swp_entry_t make_hwpoison_entry(struct page *page)
496 {
497 	return swp_entry(0, 0);
498 }
499 
is_hwpoison_entry(swp_entry_t swp)500 static inline int is_hwpoison_entry(swp_entry_t swp)
501 {
502 	return 0;
503 }
504 
num_poisoned_pages_inc(void)505 static inline void num_poisoned_pages_inc(void)
506 {
507 }
508 #endif
509 
non_swap_entry(swp_entry_t entry)510 static inline int non_swap_entry(swp_entry_t entry)
511 {
512 	return swp_type(entry) >= MAX_SWAPFILES;
513 }
514 
515 #endif /* CONFIG_MMU */
516 #endif /* _LINUX_SWAPOPS_H */
517