1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * This file contains common KASAN code.
4  *
5  * Copyright (c) 2014 Samsung Electronics Co., Ltd.
6  * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
7  *
8  * Some code borrowed from https://github.com/xairy/kasan-prototype by
9  *        Andrey Konovalov <andreyknvl@gmail.com>
10  */
11 
12 #include <linux/export.h>
13 #include <linux/init.h>
14 #include <linux/kasan.h>
15 #include <linux/kernel.h>
16 #include <linux/linkage.h>
17 #include <linux/memblock.h>
18 #include <linux/memory.h>
19 #include <linux/mm.h>
20 #include <linux/module.h>
21 #include <linux/printk.h>
22 #include <linux/sched.h>
23 #include <linux/sched/task_stack.h>
24 #include <linux/slab.h>
25 #include <linux/stacktrace.h>
26 #include <linux/string.h>
27 #include <linux/types.h>
28 #include <linux/bug.h>
29 
30 #include "kasan.h"
31 #include "../slab.h"
32 
kasan_save_stack(gfp_t flags,bool can_alloc)33 depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc)
34 {
35 	unsigned long entries[KASAN_STACK_DEPTH];
36 	unsigned int nr_entries;
37 
38 	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
39 	return __stack_depot_save(entries, nr_entries, flags, can_alloc);
40 }
41 
kasan_set_track(struct kasan_track * track,gfp_t flags)42 void kasan_set_track(struct kasan_track *track, gfp_t flags)
43 {
44 	track->pid = current->pid;
45 	track->stack = kasan_save_stack(flags, true);
46 }
47 
48 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
kasan_enable_current(void)49 void kasan_enable_current(void)
50 {
51 	current->kasan_depth++;
52 }
53 EXPORT_SYMBOL(kasan_enable_current);
54 
kasan_disable_current(void)55 void kasan_disable_current(void)
56 {
57 	current->kasan_depth--;
58 }
59 EXPORT_SYMBOL(kasan_disable_current);
60 
61 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
62 
__kasan_unpoison_range(const void * address,size_t size)63 void __kasan_unpoison_range(const void *address, size_t size)
64 {
65 	kasan_unpoison(address, size, false);
66 }
67 
68 #ifdef CONFIG_KASAN_STACK
69 /* Unpoison the entire stack for a task. */
kasan_unpoison_task_stack(struct task_struct * task)70 void kasan_unpoison_task_stack(struct task_struct *task)
71 {
72 	void *base = task_stack_page(task);
73 
74 	kasan_unpoison(base, THREAD_SIZE, false);
75 }
76 
77 /* Unpoison the stack for the current task beyond a watermark sp value. */
kasan_unpoison_task_stack_below(const void * watermark)78 asmlinkage void kasan_unpoison_task_stack_below(const void *watermark)
79 {
80 	/*
81 	 * Calculate the task stack base address.  Avoid using 'current'
82 	 * because this function is called by early resume code which hasn't
83 	 * yet set up the percpu register (%gs).
84 	 */
85 	void *base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1));
86 
87 	kasan_unpoison(base, watermark - base, false);
88 }
89 #endif /* CONFIG_KASAN_STACK */
90 
91 /*
92  * Only allow cache merging when stack collection is disabled and no metadata
93  * is present.
94  */
__kasan_never_merge(void)95 slab_flags_t __kasan_never_merge(void)
96 {
97 	if (kasan_stack_collection_enabled())
98 		return SLAB_KASAN;
99 	return 0;
100 }
101 
__kasan_unpoison_pages(struct page * page,unsigned int order,bool init)102 void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init)
103 {
104 	u8 tag;
105 	unsigned long i;
106 
107 	if (unlikely(PageHighMem(page)))
108 		return;
109 
110 	tag = kasan_random_tag();
111 	for (i = 0; i < (1 << order); i++)
112 		page_kasan_tag_set(page + i, tag);
113 	kasan_unpoison(page_address(page), PAGE_SIZE << order, init);
114 }
115 
__kasan_poison_pages(struct page * page,unsigned int order,bool init)116 void __kasan_poison_pages(struct page *page, unsigned int order, bool init)
117 {
118 	if (likely(!PageHighMem(page)))
119 		kasan_poison(page_address(page), PAGE_SIZE << order,
120 			     KASAN_PAGE_FREE, init);
121 }
122 
123 /*
124  * Adaptive redzone policy taken from the userspace AddressSanitizer runtime.
125  * For larger allocations larger redzones are used.
126  */
optimal_redzone(unsigned int object_size)127 static inline unsigned int optimal_redzone(unsigned int object_size)
128 {
129 	return
130 		object_size <= 64        - 16   ? 16 :
131 		object_size <= 128       - 32   ? 32 :
132 		object_size <= 512       - 64   ? 64 :
133 		object_size <= 4096      - 128  ? 128 :
134 		object_size <= (1 << 14) - 256  ? 256 :
135 		object_size <= (1 << 15) - 512  ? 512 :
136 		object_size <= (1 << 16) - 1024 ? 1024 : 2048;
137 }
138 
__kasan_cache_create(struct kmem_cache * cache,unsigned int * size,slab_flags_t * flags)139 void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
140 			  slab_flags_t *flags)
141 {
142 	unsigned int ok_size;
143 	unsigned int optimal_size;
144 
145 	/*
146 	 * SLAB_KASAN is used to mark caches as ones that are sanitized by
147 	 * KASAN. Currently this flag is used in two places:
148 	 * 1. In slab_ksize() when calculating the size of the accessible
149 	 *    memory within the object.
150 	 * 2. In slab_common.c to prevent merging of sanitized caches.
151 	 */
152 	*flags |= SLAB_KASAN;
153 
154 	if (!kasan_stack_collection_enabled())
155 		return;
156 
157 	ok_size = *size;
158 
159 	/* Add alloc meta into redzone. */
160 	cache->kasan_info.alloc_meta_offset = *size;
161 	*size += sizeof(struct kasan_alloc_meta);
162 
163 	/*
164 	 * If alloc meta doesn't fit, don't add it.
165 	 * This can only happen with SLAB, as it has KMALLOC_MAX_SIZE equal
166 	 * to KMALLOC_MAX_CACHE_SIZE and doesn't fall back to page_alloc for
167 	 * larger sizes.
168 	 */
169 	if (*size > KMALLOC_MAX_SIZE) {
170 		cache->kasan_info.alloc_meta_offset = 0;
171 		*size = ok_size;
172 		/* Continue, since free meta might still fit. */
173 	}
174 
175 	/* Only the generic mode uses free meta or flexible redzones. */
176 	if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) {
177 		cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;
178 		return;
179 	}
180 
181 	/*
182 	 * Add free meta into redzone when it's not possible to store
183 	 * it in the object. This is the case when:
184 	 * 1. Object is SLAB_TYPESAFE_BY_RCU, which means that it can
185 	 *    be touched after it was freed, or
186 	 * 2. Object has a constructor, which means it's expected to
187 	 *    retain its content until the next allocation, or
188 	 * 3. Object is too small.
189 	 * Otherwise cache->kasan_info.free_meta_offset = 0 is implied.
190 	 */
191 	if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor ||
192 	    cache->object_size < sizeof(struct kasan_free_meta)) {
193 		ok_size = *size;
194 
195 		cache->kasan_info.free_meta_offset = *size;
196 		*size += sizeof(struct kasan_free_meta);
197 
198 		/* If free meta doesn't fit, don't add it. */
199 		if (*size > KMALLOC_MAX_SIZE) {
200 			cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;
201 			*size = ok_size;
202 		}
203 	}
204 
205 	/* Calculate size with optimal redzone. */
206 	optimal_size = cache->object_size + optimal_redzone(cache->object_size);
207 	/* Limit it with KMALLOC_MAX_SIZE (relevant for SLAB only). */
208 	if (optimal_size > KMALLOC_MAX_SIZE)
209 		optimal_size = KMALLOC_MAX_SIZE;
210 	/* Use optimal size if the size with added metas is not large enough. */
211 	if (*size < optimal_size)
212 		*size = optimal_size;
213 }
214 
__kasan_cache_create_kmalloc(struct kmem_cache * cache)215 void __kasan_cache_create_kmalloc(struct kmem_cache *cache)
216 {
217 	cache->kasan_info.is_kmalloc = true;
218 }
219 
__kasan_metadata_size(struct kmem_cache * cache)220 size_t __kasan_metadata_size(struct kmem_cache *cache)
221 {
222 	if (!kasan_stack_collection_enabled())
223 		return 0;
224 	return (cache->kasan_info.alloc_meta_offset ?
225 		sizeof(struct kasan_alloc_meta) : 0) +
226 		(cache->kasan_info.free_meta_offset ?
227 		sizeof(struct kasan_free_meta) : 0);
228 }
229 
kasan_get_alloc_meta(struct kmem_cache * cache,const void * object)230 struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
231 					      const void *object)
232 {
233 	if (!cache->kasan_info.alloc_meta_offset)
234 		return NULL;
235 	return kasan_reset_tag(object) + cache->kasan_info.alloc_meta_offset;
236 }
237 
238 #ifdef CONFIG_KASAN_GENERIC
kasan_get_free_meta(struct kmem_cache * cache,const void * object)239 struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
240 					    const void *object)
241 {
242 	BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32);
243 	if (cache->kasan_info.free_meta_offset == KASAN_NO_FREE_META)
244 		return NULL;
245 	return kasan_reset_tag(object) + cache->kasan_info.free_meta_offset;
246 }
247 #endif
248 
__kasan_poison_slab(struct slab * slab)249 void __kasan_poison_slab(struct slab *slab)
250 {
251 	struct page *page = slab_page(slab);
252 	unsigned long i;
253 
254 	for (i = 0; i < compound_nr(page); i++)
255 		page_kasan_tag_reset(page + i);
256 	kasan_poison(page_address(page), page_size(page),
257 		     KASAN_SLAB_REDZONE, false);
258 }
259 
__kasan_unpoison_object_data(struct kmem_cache * cache,void * object)260 void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object)
261 {
262 	kasan_unpoison(object, cache->object_size, false);
263 }
264 
__kasan_poison_object_data(struct kmem_cache * cache,void * object)265 void __kasan_poison_object_data(struct kmem_cache *cache, void *object)
266 {
267 	kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),
268 			KASAN_SLAB_REDZONE, false);
269 }
270 
271 /*
272  * This function assigns a tag to an object considering the following:
273  * 1. A cache might have a constructor, which might save a pointer to a slab
274  *    object somewhere (e.g. in the object itself). We preassign a tag for
275  *    each object in caches with constructors during slab creation and reuse
276  *    the same tag each time a particular object is allocated.
277  * 2. A cache might be SLAB_TYPESAFE_BY_RCU, which means objects can be
278  *    accessed after being freed. We preassign tags for objects in these
279  *    caches as well.
280  * 3. For SLAB allocator we can't preassign tags randomly since the freelist
281  *    is stored as an array of indexes instead of a linked list. Assign tags
282  *    based on objects indexes, so that objects that are next to each other
283  *    get different tags.
284  */
assign_tag(struct kmem_cache * cache,const void * object,bool init)285 static inline u8 assign_tag(struct kmem_cache *cache,
286 					const void *object, bool init)
287 {
288 	if (IS_ENABLED(CONFIG_KASAN_GENERIC))
289 		return 0xff;
290 
291 	/*
292 	 * If the cache neither has a constructor nor has SLAB_TYPESAFE_BY_RCU
293 	 * set, assign a tag when the object is being allocated (init == false).
294 	 */
295 	if (!cache->ctor && !(cache->flags & SLAB_TYPESAFE_BY_RCU))
296 		return init ? KASAN_TAG_KERNEL : kasan_random_tag();
297 
298 	/* For caches that either have a constructor or SLAB_TYPESAFE_BY_RCU: */
299 #ifdef CONFIG_SLAB
300 	/* For SLAB assign tags based on the object index in the freelist. */
301 	return (u8)obj_to_index(cache, virt_to_slab(object), (void *)object);
302 #else
303 	/*
304 	 * For SLUB assign a random tag during slab creation, otherwise reuse
305 	 * the already assigned tag.
306 	 */
307 	return init ? kasan_random_tag() : get_tag(object);
308 #endif
309 }
310 
__kasan_init_slab_obj(struct kmem_cache * cache,const void * object)311 void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
312 						const void *object)
313 {
314 	struct kasan_alloc_meta *alloc_meta;
315 
316 	if (kasan_stack_collection_enabled()) {
317 		alloc_meta = kasan_get_alloc_meta(cache, object);
318 		if (alloc_meta)
319 			__memset(alloc_meta, 0, sizeof(*alloc_meta));
320 	}
321 
322 	/* Tag is ignored in set_tag() without CONFIG_KASAN_SW/HW_TAGS */
323 	object = set_tag(object, assign_tag(cache, object, true));
324 
325 	return (void *)object;
326 }
327 
____kasan_slab_free(struct kmem_cache * cache,void * object,unsigned long ip,bool quarantine,bool init)328 static inline bool ____kasan_slab_free(struct kmem_cache *cache, void *object,
329 				unsigned long ip, bool quarantine, bool init)
330 {
331 	u8 tag;
332 	void *tagged_object;
333 
334 	if (!kasan_arch_is_ready())
335 		return false;
336 
337 	tag = get_tag(object);
338 	tagged_object = object;
339 	object = kasan_reset_tag(object);
340 
341 	if (is_kfence_address(object))
342 		return false;
343 
344 	if (unlikely(nearest_obj(cache, virt_to_slab(object), object) !=
345 	    object)) {
346 		kasan_report_invalid_free(tagged_object, ip);
347 		return true;
348 	}
349 
350 	/* RCU slabs could be legally used after free within the RCU period */
351 	if (unlikely(cache->flags & SLAB_TYPESAFE_BY_RCU))
352 		return false;
353 
354 	if (!kasan_byte_accessible(tagged_object)) {
355 		kasan_report_invalid_free(tagged_object, ip);
356 		return true;
357 	}
358 
359 	kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),
360 			KASAN_SLAB_FREE, init);
361 
362 	if ((IS_ENABLED(CONFIG_KASAN_GENERIC) && !quarantine))
363 		return false;
364 
365 	if (kasan_stack_collection_enabled())
366 		kasan_set_free_info(cache, object, tag);
367 
368 	return kasan_quarantine_put(cache, object);
369 }
370 
__kasan_slab_free(struct kmem_cache * cache,void * object,unsigned long ip,bool init)371 bool __kasan_slab_free(struct kmem_cache *cache, void *object,
372 				unsigned long ip, bool init)
373 {
374 	return ____kasan_slab_free(cache, object, ip, true, init);
375 }
376 
____kasan_kfree_large(void * ptr,unsigned long ip)377 static inline bool ____kasan_kfree_large(void *ptr, unsigned long ip)
378 {
379 	if (ptr != page_address(virt_to_head_page(ptr))) {
380 		kasan_report_invalid_free(ptr, ip);
381 		return true;
382 	}
383 
384 	if (!kasan_byte_accessible(ptr)) {
385 		kasan_report_invalid_free(ptr, ip);
386 		return true;
387 	}
388 
389 	/*
390 	 * The object will be poisoned by kasan_poison_pages() or
391 	 * kasan_slab_free_mempool().
392 	 */
393 
394 	return false;
395 }
396 
__kasan_kfree_large(void * ptr,unsigned long ip)397 void __kasan_kfree_large(void *ptr, unsigned long ip)
398 {
399 	____kasan_kfree_large(ptr, ip);
400 }
401 
__kasan_slab_free_mempool(void * ptr,unsigned long ip)402 void __kasan_slab_free_mempool(void *ptr, unsigned long ip)
403 {
404 	struct folio *folio;
405 
406 	folio = virt_to_folio(ptr);
407 
408 	/*
409 	 * Even though this function is only called for kmem_cache_alloc and
410 	 * kmalloc backed mempool allocations, those allocations can still be
411 	 * !PageSlab() when the size provided to kmalloc is larger than
412 	 * KMALLOC_MAX_SIZE, and kmalloc falls back onto page_alloc.
413 	 */
414 	if (unlikely(!folio_test_slab(folio))) {
415 		if (____kasan_kfree_large(ptr, ip))
416 			return;
417 		kasan_poison(ptr, folio_size(folio), KASAN_PAGE_FREE, false);
418 	} else {
419 		struct slab *slab = folio_slab(folio);
420 
421 		____kasan_slab_free(slab->slab_cache, ptr, ip, false, false);
422 	}
423 }
424 
set_alloc_info(struct kmem_cache * cache,void * object,gfp_t flags,bool is_kmalloc)425 static void set_alloc_info(struct kmem_cache *cache, void *object,
426 				gfp_t flags, bool is_kmalloc)
427 {
428 	struct kasan_alloc_meta *alloc_meta;
429 
430 	/* Don't save alloc info for kmalloc caches in kasan_slab_alloc(). */
431 	if (cache->kasan_info.is_kmalloc && !is_kmalloc)
432 		return;
433 
434 	alloc_meta = kasan_get_alloc_meta(cache, object);
435 	if (alloc_meta)
436 		kasan_set_track(&alloc_meta->alloc_track, flags);
437 }
438 
__kasan_slab_alloc(struct kmem_cache * cache,void * object,gfp_t flags,bool init)439 void * __must_check __kasan_slab_alloc(struct kmem_cache *cache,
440 					void *object, gfp_t flags, bool init)
441 {
442 	u8 tag;
443 	void *tagged_object;
444 
445 	if (gfpflags_allow_blocking(flags))
446 		kasan_quarantine_reduce();
447 
448 	if (unlikely(object == NULL))
449 		return NULL;
450 
451 	if (is_kfence_address(object))
452 		return (void *)object;
453 
454 	/*
455 	 * Generate and assign random tag for tag-based modes.
456 	 * Tag is ignored in set_tag() for the generic mode.
457 	 */
458 	tag = assign_tag(cache, object, false);
459 	tagged_object = set_tag(object, tag);
460 
461 	/*
462 	 * Unpoison the whole object.
463 	 * For kmalloc() allocations, kasan_kmalloc() will do precise poisoning.
464 	 */
465 	kasan_unpoison(tagged_object, cache->object_size, init);
466 
467 	/* Save alloc info (if possible) for non-kmalloc() allocations. */
468 	if (kasan_stack_collection_enabled())
469 		set_alloc_info(cache, (void *)object, flags, false);
470 
471 	return tagged_object;
472 }
473 
____kasan_kmalloc(struct kmem_cache * cache,const void * object,size_t size,gfp_t flags)474 static inline void *____kasan_kmalloc(struct kmem_cache *cache,
475 				const void *object, size_t size, gfp_t flags)
476 {
477 	unsigned long redzone_start;
478 	unsigned long redzone_end;
479 
480 	if (gfpflags_allow_blocking(flags))
481 		kasan_quarantine_reduce();
482 
483 	if (unlikely(object == NULL))
484 		return NULL;
485 
486 	if (is_kfence_address(kasan_reset_tag(object)))
487 		return (void *)object;
488 
489 	/*
490 	 * The object has already been unpoisoned by kasan_slab_alloc() for
491 	 * kmalloc() or by kasan_krealloc() for krealloc().
492 	 */
493 
494 	/*
495 	 * The redzone has byte-level precision for the generic mode.
496 	 * Partially poison the last object granule to cover the unaligned
497 	 * part of the redzone.
498 	 */
499 	if (IS_ENABLED(CONFIG_KASAN_GENERIC))
500 		kasan_poison_last_granule((void *)object, size);
501 
502 	/* Poison the aligned part of the redzone. */
503 	redzone_start = round_up((unsigned long)(object + size),
504 				KASAN_GRANULE_SIZE);
505 	redzone_end = round_up((unsigned long)(object + cache->object_size),
506 				KASAN_GRANULE_SIZE);
507 	kasan_poison((void *)redzone_start, redzone_end - redzone_start,
508 			   KASAN_SLAB_REDZONE, false);
509 
510 	/*
511 	 * Save alloc info (if possible) for kmalloc() allocations.
512 	 * This also rewrites the alloc info when called from kasan_krealloc().
513 	 */
514 	if (kasan_stack_collection_enabled())
515 		set_alloc_info(cache, (void *)object, flags, true);
516 
517 	/* Keep the tag that was set by kasan_slab_alloc(). */
518 	return (void *)object;
519 }
520 
__kasan_kmalloc(struct kmem_cache * cache,const void * object,size_t size,gfp_t flags)521 void * __must_check __kasan_kmalloc(struct kmem_cache *cache, const void *object,
522 					size_t size, gfp_t flags)
523 {
524 	return ____kasan_kmalloc(cache, object, size, flags);
525 }
526 EXPORT_SYMBOL(__kasan_kmalloc);
527 
__kasan_kmalloc_large(const void * ptr,size_t size,gfp_t flags)528 void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size,
529 						gfp_t flags)
530 {
531 	unsigned long redzone_start;
532 	unsigned long redzone_end;
533 
534 	if (gfpflags_allow_blocking(flags))
535 		kasan_quarantine_reduce();
536 
537 	if (unlikely(ptr == NULL))
538 		return NULL;
539 
540 	/*
541 	 * The object has already been unpoisoned by kasan_unpoison_pages() for
542 	 * alloc_pages() or by kasan_krealloc() for krealloc().
543 	 */
544 
545 	/*
546 	 * The redzone has byte-level precision for the generic mode.
547 	 * Partially poison the last object granule to cover the unaligned
548 	 * part of the redzone.
549 	 */
550 	if (IS_ENABLED(CONFIG_KASAN_GENERIC))
551 		kasan_poison_last_granule(ptr, size);
552 
553 	/* Poison the aligned part of the redzone. */
554 	redzone_start = round_up((unsigned long)(ptr + size),
555 				KASAN_GRANULE_SIZE);
556 	redzone_end = (unsigned long)ptr + page_size(virt_to_page(ptr));
557 	kasan_poison((void *)redzone_start, redzone_end - redzone_start,
558 		     KASAN_PAGE_REDZONE, false);
559 
560 	return (void *)ptr;
561 }
562 
__kasan_krealloc(const void * object,size_t size,gfp_t flags)563 void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flags)
564 {
565 	struct slab *slab;
566 
567 	if (unlikely(object == ZERO_SIZE_PTR))
568 		return (void *)object;
569 
570 	/*
571 	 * Unpoison the object's data.
572 	 * Part of it might already have been unpoisoned, but it's unknown
573 	 * how big that part is.
574 	 */
575 	kasan_unpoison(object, size, false);
576 
577 	slab = virt_to_slab(object);
578 
579 	/* Piggy-back on kmalloc() instrumentation to poison the redzone. */
580 	if (unlikely(!slab))
581 		return __kasan_kmalloc_large(object, size, flags);
582 	else
583 		return ____kasan_kmalloc(slab->slab_cache, object, size, flags);
584 }
585 
__kasan_check_byte(const void * address,unsigned long ip)586 bool __kasan_check_byte(const void *address, unsigned long ip)
587 {
588 	if (!kasan_byte_accessible(address)) {
589 		kasan_report((unsigned long)address, 1, false, ip);
590 		return false;
591 	}
592 	return true;
593 }
594