1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * This file contains core hardware tag-based KASAN code.
4 *
5 * Copyright (c) 2020 Google, Inc.
6 * Author: Andrey Konovalov <andreyknvl@google.com>
7 */
8
9 #define pr_fmt(fmt) "kasan: " fmt
10
11 #include <linux/init.h>
12 #include <linux/kasan.h>
13 #include <linux/kernel.h>
14 #include <linux/memory.h>
15 #include <linux/mm.h>
16 #include <linux/static_key.h>
17 #include <linux/string.h>
18 #include <linux/types.h>
19
20 #include "kasan.h"
21
22 enum kasan_arg {
23 KASAN_ARG_DEFAULT,
24 KASAN_ARG_OFF,
25 KASAN_ARG_ON,
26 };
27
28 enum kasan_arg_mode {
29 KASAN_ARG_MODE_DEFAULT,
30 KASAN_ARG_MODE_SYNC,
31 KASAN_ARG_MODE_ASYNC,
32 KASAN_ARG_MODE_ASYMM,
33 };
34
35 enum kasan_arg_vmalloc {
36 KASAN_ARG_VMALLOC_DEFAULT,
37 KASAN_ARG_VMALLOC_OFF,
38 KASAN_ARG_VMALLOC_ON,
39 };
40
41 static enum kasan_arg kasan_arg __ro_after_init;
42 static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
43 static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
44
45 /*
46 * Whether KASAN is enabled at all.
47 * The value remains false until KASAN is initialized by kasan_init_hw_tags().
48 */
49 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
50 EXPORT_SYMBOL(kasan_flag_enabled);
51
52 /*
53 * Whether the selected mode is synchronous, asynchronous, or asymmetric.
54 * Defaults to KASAN_MODE_SYNC.
55 */
56 enum kasan_mode kasan_mode __ro_after_init;
57 EXPORT_SYMBOL_GPL(kasan_mode);
58
59 /* Whether to enable vmalloc tagging. */
60 DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
61
62 /* kasan=off/on */
early_kasan_flag(char * arg)63 static int __init early_kasan_flag(char *arg)
64 {
65 if (!arg)
66 return -EINVAL;
67
68 if (!strcmp(arg, "off"))
69 kasan_arg = KASAN_ARG_OFF;
70 else if (!strcmp(arg, "on"))
71 kasan_arg = KASAN_ARG_ON;
72 else
73 return -EINVAL;
74
75 return 0;
76 }
77 early_param("kasan", early_kasan_flag);
78
79 /* kasan.mode=sync/async/asymm */
early_kasan_mode(char * arg)80 static int __init early_kasan_mode(char *arg)
81 {
82 if (!arg)
83 return -EINVAL;
84
85 if (!strcmp(arg, "sync"))
86 kasan_arg_mode = KASAN_ARG_MODE_SYNC;
87 else if (!strcmp(arg, "async"))
88 kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
89 else if (!strcmp(arg, "asymm"))
90 kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
91 else
92 return -EINVAL;
93
94 return 0;
95 }
96 early_param("kasan.mode", early_kasan_mode);
97
98 /* kasan.vmalloc=off/on */
early_kasan_flag_vmalloc(char * arg)99 static int __init early_kasan_flag_vmalloc(char *arg)
100 {
101 if (!arg)
102 return -EINVAL;
103
104 if (!strcmp(arg, "off"))
105 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF;
106 else if (!strcmp(arg, "on"))
107 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON;
108 else
109 return -EINVAL;
110
111 return 0;
112 }
113 early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
114
kasan_mode_info(void)115 static inline const char *kasan_mode_info(void)
116 {
117 if (kasan_mode == KASAN_MODE_ASYNC)
118 return "async";
119 else if (kasan_mode == KASAN_MODE_ASYMM)
120 return "asymm";
121 else
122 return "sync";
123 }
124
125 /*
126 * kasan_init_hw_tags_cpu() is called for each CPU.
127 * Not marked as __init as a CPU can be hot-plugged after boot.
128 */
kasan_init_hw_tags_cpu(void)129 void kasan_init_hw_tags_cpu(void)
130 {
131 /*
132 * There's no need to check that the hardware is MTE-capable here,
133 * as this function is only called for MTE-capable hardware.
134 */
135
136 /*
137 * If KASAN is disabled via command line, don't initialize it.
138 * When this function is called, kasan_flag_enabled is not yet
139 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead.
140 */
141 if (kasan_arg == KASAN_ARG_OFF)
142 return;
143
144 /*
145 * Enable async or asymm modes only when explicitly requested
146 * through the command line.
147 */
148 kasan_enable_tagging();
149 }
150
151 /* kasan_init_hw_tags() is called once on boot CPU. */
kasan_init_hw_tags(void)152 void __init kasan_init_hw_tags(void)
153 {
154 /* If hardware doesn't support MTE, don't initialize KASAN. */
155 if (!system_supports_mte())
156 return;
157
158 /* If KASAN is disabled via command line, don't initialize it. */
159 if (kasan_arg == KASAN_ARG_OFF)
160 return;
161
162 switch (kasan_arg_mode) {
163 case KASAN_ARG_MODE_DEFAULT:
164 /* Default is specified by kasan_mode definition. */
165 break;
166 case KASAN_ARG_MODE_SYNC:
167 kasan_mode = KASAN_MODE_SYNC;
168 break;
169 case KASAN_ARG_MODE_ASYNC:
170 kasan_mode = KASAN_MODE_ASYNC;
171 break;
172 case KASAN_ARG_MODE_ASYMM:
173 kasan_mode = KASAN_MODE_ASYMM;
174 break;
175 }
176
177 switch (kasan_arg_vmalloc) {
178 case KASAN_ARG_VMALLOC_DEFAULT:
179 /* Default is specified by kasan_flag_vmalloc definition. */
180 break;
181 case KASAN_ARG_VMALLOC_OFF:
182 static_branch_disable(&kasan_flag_vmalloc);
183 break;
184 case KASAN_ARG_VMALLOC_ON:
185 static_branch_enable(&kasan_flag_vmalloc);
186 break;
187 }
188
189 kasan_init_tags();
190
191 /* KASAN is now initialized, enable it. */
192 static_branch_enable(&kasan_flag_enabled);
193
194 pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n",
195 kasan_mode_info(),
196 kasan_vmalloc_enabled() ? "on" : "off",
197 kasan_stack_collection_enabled() ? "on" : "off");
198 }
199
200 #ifdef CONFIG_KASAN_VMALLOC
201
unpoison_vmalloc_pages(const void * addr,u8 tag)202 static void unpoison_vmalloc_pages(const void *addr, u8 tag)
203 {
204 struct vm_struct *area;
205 int i;
206
207 /*
208 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations
209 * (see the comment in __kasan_unpoison_vmalloc), all of the pages
210 * should belong to a single area.
211 */
212 area = find_vm_area((void *)addr);
213 if (WARN_ON(!area))
214 return;
215
216 for (i = 0; i < area->nr_pages; i++) {
217 struct page *page = area->pages[i];
218
219 page_kasan_tag_set(page, tag);
220 }
221 }
222
init_vmalloc_pages(const void * start,unsigned long size)223 static void init_vmalloc_pages(const void *start, unsigned long size)
224 {
225 const void *addr;
226
227 for (addr = start; addr < start + size; addr += PAGE_SIZE) {
228 struct page *page = virt_to_page(addr);
229
230 clear_highpage_kasan_tagged(page);
231 }
232 }
233
__kasan_unpoison_vmalloc(const void * start,unsigned long size,kasan_vmalloc_flags_t flags)234 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
235 kasan_vmalloc_flags_t flags)
236 {
237 u8 tag;
238 unsigned long redzone_start, redzone_size;
239
240 if (!kasan_vmalloc_enabled() || !is_vmalloc_or_module_addr(start)) {
241 if (flags & KASAN_VMALLOC_INIT)
242 init_vmalloc_pages(start, size);
243 return (void *)start;
244 }
245
246 /*
247 * Don't tag non-VM_ALLOC mappings, as:
248 *
249 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only
250 * supports tagging physical memory. Therefore, it can only tag a
251 * single mapping of normal physical pages.
252 * 2. Hardware tag-based KASAN can only tag memory mapped with special
253 * mapping protection bits, see arch_vmap_pgprot_tagged().
254 * As non-VM_ALLOC mappings can be mapped outside of vmalloc code,
255 * providing these bits would require tracking all non-VM_ALLOC
256 * mappers.
257 *
258 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags
259 * the first virtual mapping, which is created by vmalloc().
260 * Tagging the page_alloc memory backing that vmalloc() allocation is
261 * skipped, see ___GFP_SKIP_KASAN_UNPOISON.
262 *
263 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.
264 */
265 if (!(flags & KASAN_VMALLOC_VM_ALLOC)) {
266 WARN_ON(flags & KASAN_VMALLOC_INIT);
267 return (void *)start;
268 }
269
270 /*
271 * Don't tag executable memory.
272 * The kernel doesn't tolerate having the PC register tagged.
273 */
274 if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) {
275 WARN_ON(flags & KASAN_VMALLOC_INIT);
276 return (void *)start;
277 }
278
279 tag = kasan_random_tag();
280 start = set_tag(start, tag);
281
282 /* Unpoison and initialize memory up to size. */
283 kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT);
284
285 /*
286 * Explicitly poison and initialize the in-page vmalloc() redzone.
287 * Unlike software KASAN modes, hardware tag-based KASAN doesn't
288 * unpoison memory when populating shadow for vmalloc() space.
289 */
290 redzone_start = round_up((unsigned long)start + size,
291 KASAN_GRANULE_SIZE);
292 redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start;
293 kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID,
294 flags & KASAN_VMALLOC_INIT);
295
296 /*
297 * Set per-page tag flags to allow accessing physical memory for the
298 * vmalloc() mapping through page_address(vmalloc_to_page()).
299 */
300 unpoison_vmalloc_pages(start, tag);
301
302 return (void *)start;
303 }
304
__kasan_poison_vmalloc(const void * start,unsigned long size)305 void __kasan_poison_vmalloc(const void *start, unsigned long size)
306 {
307 /*
308 * No tagging here.
309 * The physical pages backing the vmalloc() allocation are poisoned
310 * through the usual page_alloc paths.
311 */
312 }
313
314 #endif
315
kasan_enable_tagging(void)316 void kasan_enable_tagging(void)
317 {
318 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
319 hw_enable_tagging_async();
320 else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
321 hw_enable_tagging_asymm();
322 else
323 hw_enable_tagging_sync();
324 }
325
326 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
327
328 EXPORT_SYMBOL_GPL(kasan_enable_tagging);
329
kasan_force_async_fault(void)330 void kasan_force_async_fault(void)
331 {
332 hw_force_async_tag_fault();
333 }
334 EXPORT_SYMBOL_GPL(kasan_force_async_fault);
335
336 #endif
337