1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Frontswap frontend
4 *
5 * This code provides the generic "frontend" layer to call a matching
6 * "backend" driver implementation of frontswap. See
7 * Documentation/vm/frontswap.rst for more information.
8 *
9 * Copyright (C) 2009-2012 Oracle Corp. All rights reserved.
10 * Author: Dan Magenheimer
11 */
12
13 #include <linux/mman.h>
14 #include <linux/swap.h>
15 #include <linux/swapops.h>
16 #include <linux/security.h>
17 #include <linux/module.h>
18 #include <linux/debugfs.h>
19 #include <linux/frontswap.h>
20 #include <linux/swapfile.h>
21
22 DEFINE_STATIC_KEY_FALSE(frontswap_enabled_key);
23
24 /*
25 * frontswap_ops are added by frontswap_register_ops, and provide the
26 * frontswap "backend" implementation functions. Multiple implementations
27 * may be registered, but implementations can never deregister. This
28 * is a simple singly-linked list of all registered implementations.
29 */
30 static const struct frontswap_ops *frontswap_ops __read_mostly;
31
32 #ifdef CONFIG_DEBUG_FS
33 /*
34 * Counters available via /sys/kernel/debug/frontswap (if debugfs is
35 * properly configured). These are for information only so are not protected
36 * against increment races.
37 */
38 static u64 frontswap_loads;
39 static u64 frontswap_succ_stores;
40 static u64 frontswap_failed_stores;
41 static u64 frontswap_invalidates;
42
inc_frontswap_loads(void)43 static inline void inc_frontswap_loads(void)
44 {
45 data_race(frontswap_loads++);
46 }
inc_frontswap_succ_stores(void)47 static inline void inc_frontswap_succ_stores(void)
48 {
49 data_race(frontswap_succ_stores++);
50 }
inc_frontswap_failed_stores(void)51 static inline void inc_frontswap_failed_stores(void)
52 {
53 data_race(frontswap_failed_stores++);
54 }
inc_frontswap_invalidates(void)55 static inline void inc_frontswap_invalidates(void)
56 {
57 data_race(frontswap_invalidates++);
58 }
59 #else
inc_frontswap_loads(void)60 static inline void inc_frontswap_loads(void) { }
inc_frontswap_succ_stores(void)61 static inline void inc_frontswap_succ_stores(void) { }
inc_frontswap_failed_stores(void)62 static inline void inc_frontswap_failed_stores(void) { }
inc_frontswap_invalidates(void)63 static inline void inc_frontswap_invalidates(void) { }
64 #endif
65
66 /*
67 * Due to the asynchronous nature of the backends loading potentially
68 * _after_ the swap system has been activated, we have chokepoints
69 * on all frontswap functions to not call the backend until the backend
70 * has registered.
71 *
72 * This would not guards us against the user deciding to call swapoff right as
73 * we are calling the backend to initialize (so swapon is in action).
74 * Fortunately for us, the swapon_mutex has been taken by the callee so we are
75 * OK. The other scenario where calls to frontswap_store (called via
76 * swap_writepage) is racing with frontswap_invalidate_area (called via
77 * swapoff) is again guarded by the swap subsystem.
78 *
79 * While no backend is registered all calls to frontswap_[store|load|
80 * invalidate_area|invalidate_page] are ignored or fail.
81 *
82 * The time between the backend being registered and the swap file system
83 * calling the backend (via the frontswap_* functions) is indeterminate as
84 * frontswap_ops is not atomic_t (or a value guarded by a spinlock).
85 * That is OK as we are comfortable missing some of these calls to the newly
86 * registered backend.
87 *
88 * Obviously the opposite (unloading the backend) must be done after all
89 * the frontswap_[store|load|invalidate_area|invalidate_page] start
90 * ignoring or failing the requests. However, there is currently no way
91 * to unload a backend once it is registered.
92 */
93
94 /*
95 * Register operations for frontswap
96 */
frontswap_register_ops(const struct frontswap_ops * ops)97 int frontswap_register_ops(const struct frontswap_ops *ops)
98 {
99 if (frontswap_ops)
100 return -EINVAL;
101
102 frontswap_ops = ops;
103 static_branch_inc(&frontswap_enabled_key);
104 return 0;
105 }
106
107 /*
108 * Called when a swap device is swapon'd.
109 */
frontswap_init(unsigned type,unsigned long * map)110 void frontswap_init(unsigned type, unsigned long *map)
111 {
112 struct swap_info_struct *sis = swap_info[type];
113
114 VM_BUG_ON(sis == NULL);
115
116 /*
117 * p->frontswap is a bitmap that we MUST have to figure out which page
118 * has gone in frontswap. Without it there is no point of continuing.
119 */
120 if (WARN_ON(!map))
121 return;
122 /*
123 * Irregardless of whether the frontswap backend has been loaded
124 * before this function or it will be later, we _MUST_ have the
125 * p->frontswap set to something valid to work properly.
126 */
127 frontswap_map_set(sis, map);
128 frontswap_ops->init(type);
129 }
130
__frontswap_test(struct swap_info_struct * sis,pgoff_t offset)131 static bool __frontswap_test(struct swap_info_struct *sis,
132 pgoff_t offset)
133 {
134 if (sis->frontswap_map)
135 return test_bit(offset, sis->frontswap_map);
136 return false;
137 }
138
__frontswap_set(struct swap_info_struct * sis,pgoff_t offset)139 static inline void __frontswap_set(struct swap_info_struct *sis,
140 pgoff_t offset)
141 {
142 set_bit(offset, sis->frontswap_map);
143 atomic_inc(&sis->frontswap_pages);
144 }
145
__frontswap_clear(struct swap_info_struct * sis,pgoff_t offset)146 static inline void __frontswap_clear(struct swap_info_struct *sis,
147 pgoff_t offset)
148 {
149 clear_bit(offset, sis->frontswap_map);
150 atomic_dec(&sis->frontswap_pages);
151 }
152
153 /*
154 * "Store" data from a page to frontswap and associate it with the page's
155 * swaptype and offset. Page must be locked and in the swap cache.
156 * If frontswap already contains a page with matching swaptype and
157 * offset, the frontswap implementation may either overwrite the data and
158 * return success or invalidate the page from frontswap and return failure.
159 */
__frontswap_store(struct page * page)160 int __frontswap_store(struct page *page)
161 {
162 int ret = -1;
163 swp_entry_t entry = { .val = page_private(page), };
164 int type = swp_type(entry);
165 struct swap_info_struct *sis = swap_info[type];
166 pgoff_t offset = swp_offset(entry);
167
168 VM_BUG_ON(!frontswap_ops);
169 VM_BUG_ON(!PageLocked(page));
170 VM_BUG_ON(sis == NULL);
171
172 /*
173 * If a dup, we must remove the old page first; we can't leave the
174 * old page no matter if the store of the new page succeeds or fails,
175 * and we can't rely on the new page replacing the old page as we may
176 * not store to the same implementation that contains the old page.
177 */
178 if (__frontswap_test(sis, offset)) {
179 __frontswap_clear(sis, offset);
180 frontswap_ops->invalidate_page(type, offset);
181 }
182
183 ret = frontswap_ops->store(type, offset, page);
184 if (ret == 0) {
185 __frontswap_set(sis, offset);
186 inc_frontswap_succ_stores();
187 } else {
188 inc_frontswap_failed_stores();
189 }
190
191 return ret;
192 }
193
194 /*
195 * "Get" data from frontswap associated with swaptype and offset that were
196 * specified when the data was put to frontswap and use it to fill the
197 * specified page with data. Page must be locked and in the swap cache.
198 */
__frontswap_load(struct page * page)199 int __frontswap_load(struct page *page)
200 {
201 int ret = -1;
202 swp_entry_t entry = { .val = page_private(page), };
203 int type = swp_type(entry);
204 struct swap_info_struct *sis = swap_info[type];
205 pgoff_t offset = swp_offset(entry);
206
207 VM_BUG_ON(!frontswap_ops);
208 VM_BUG_ON(!PageLocked(page));
209 VM_BUG_ON(sis == NULL);
210
211 if (!__frontswap_test(sis, offset))
212 return -1;
213
214 /* Try loading from each implementation, until one succeeds. */
215 ret = frontswap_ops->load(type, offset, page);
216 if (ret == 0)
217 inc_frontswap_loads();
218 return ret;
219 }
220
221 /*
222 * Invalidate any data from frontswap associated with the specified swaptype
223 * and offset so that a subsequent "get" will fail.
224 */
__frontswap_invalidate_page(unsigned type,pgoff_t offset)225 void __frontswap_invalidate_page(unsigned type, pgoff_t offset)
226 {
227 struct swap_info_struct *sis = swap_info[type];
228
229 VM_BUG_ON(!frontswap_ops);
230 VM_BUG_ON(sis == NULL);
231
232 if (!__frontswap_test(sis, offset))
233 return;
234
235 frontswap_ops->invalidate_page(type, offset);
236 __frontswap_clear(sis, offset);
237 inc_frontswap_invalidates();
238 }
239
240 /*
241 * Invalidate all data from frontswap associated with all offsets for the
242 * specified swaptype.
243 */
__frontswap_invalidate_area(unsigned type)244 void __frontswap_invalidate_area(unsigned type)
245 {
246 struct swap_info_struct *sis = swap_info[type];
247
248 VM_BUG_ON(!frontswap_ops);
249 VM_BUG_ON(sis == NULL);
250
251 if (sis->frontswap_map == NULL)
252 return;
253
254 frontswap_ops->invalidate_area(type);
255 atomic_set(&sis->frontswap_pages, 0);
256 bitmap_zero(sis->frontswap_map, sis->max);
257 }
258
init_frontswap(void)259 static int __init init_frontswap(void)
260 {
261 #ifdef CONFIG_DEBUG_FS
262 struct dentry *root = debugfs_create_dir("frontswap", NULL);
263 if (root == NULL)
264 return -ENXIO;
265 debugfs_create_u64("loads", 0444, root, &frontswap_loads);
266 debugfs_create_u64("succ_stores", 0444, root, &frontswap_succ_stores);
267 debugfs_create_u64("failed_stores", 0444, root,
268 &frontswap_failed_stores);
269 debugfs_create_u64("invalidates", 0444, root, &frontswap_invalidates);
270 #endif
271 return 0;
272 }
273
274 module_init(init_frontswap);
275