1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * mm/kmemleak.c
4  *
5  * Copyright (C) 2008 ARM Limited
6  * Written by Catalin Marinas <catalin.marinas@arm.com>
7  *
8  * For more information on the algorithm and kmemleak usage, please see
9  * Documentation/dev-tools/kmemleak.rst.
10  *
11  * Notes on locking
12  * ----------------
13  *
14  * The following locks and mutexes are used by kmemleak:
15  *
16  * - kmemleak_lock (raw_spinlock_t): protects the object_list modifications and
17  *   accesses to the object_tree_root (or object_phys_tree_root). The
18  *   object_list is the main list holding the metadata (struct kmemleak_object)
19  *   for the allocated memory blocks. The object_tree_root and object_phys_tree_root
20  *   are red black trees used to look-up metadata based on a pointer to the
21  *   corresponding memory block. The object_phys_tree_root is for objects
22  *   allocated with physical address. The kmemleak_object structures are
23  *   added to the object_list and object_tree_root (or object_phys_tree_root)
24  *   in the create_object() function called from the kmemleak_alloc() (or
25  *   kmemleak_alloc_phys()) callback and removed in delete_object() called from
26  *   the kmemleak_free() callback
27  * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
28  *   Accesses to the metadata (e.g. count) are protected by this lock. Note
29  *   that some members of this structure may be protected by other means
30  *   (atomic or kmemleak_lock). This lock is also held when scanning the
31  *   corresponding memory block to avoid the kernel freeing it via the
32  *   kmemleak_free() callback. This is less heavyweight than holding a global
33  *   lock like kmemleak_lock during scanning.
34  * - scan_mutex (mutex): ensures that only one thread may scan the memory for
35  *   unreferenced objects at a time. The gray_list contains the objects which
36  *   are already referenced or marked as false positives and need to be
37  *   scanned. This list is only modified during a scanning episode when the
38  *   scan_mutex is held. At the end of a scan, the gray_list is always empty.
39  *   Note that the kmemleak_object.use_count is incremented when an object is
40  *   added to the gray_list and therefore cannot be freed. This mutex also
41  *   prevents multiple users of the "kmemleak" debugfs file together with
42  *   modifications to the memory scanning parameters including the scan_thread
43  *   pointer
44  *
45  * Locks and mutexes are acquired/nested in the following order:
46  *
47  *   scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
48  *
49  * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
50  * regions.
51  *
52  * The kmemleak_object structures have a use_count incremented or decremented
53  * using the get_object()/put_object() functions. When the use_count becomes
54  * 0, this count can no longer be incremented and put_object() schedules the
55  * kmemleak_object freeing via an RCU callback. All calls to the get_object()
56  * function must be protected by rcu_read_lock() to avoid accessing a freed
57  * structure.
58  */
59 
60 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
61 
62 #include <linux/init.h>
63 #include <linux/kernel.h>
64 #include <linux/list.h>
65 #include <linux/sched/signal.h>
66 #include <linux/sched/task.h>
67 #include <linux/sched/task_stack.h>
68 #include <linux/jiffies.h>
69 #include <linux/delay.h>
70 #include <linux/export.h>
71 #include <linux/kthread.h>
72 #include <linux/rbtree.h>
73 #include <linux/fs.h>
74 #include <linux/debugfs.h>
75 #include <linux/seq_file.h>
76 #include <linux/cpumask.h>
77 #include <linux/spinlock.h>
78 #include <linux/module.h>
79 #include <linux/mutex.h>
80 #include <linux/rcupdate.h>
81 #include <linux/stacktrace.h>
82 #include <linux/cache.h>
83 #include <linux/percpu.h>
84 #include <linux/memblock.h>
85 #include <linux/pfn.h>
86 #include <linux/mmzone.h>
87 #include <linux/slab.h>
88 #include <linux/thread_info.h>
89 #include <linux/err.h>
90 #include <linux/uaccess.h>
91 #include <linux/string.h>
92 #include <linux/nodemask.h>
93 #include <linux/mm.h>
94 #include <linux/workqueue.h>
95 #include <linux/crc32.h>
96 
97 #include <asm/sections.h>
98 #include <asm/processor.h>
99 #include <linux/atomic.h>
100 
101 #include <linux/kasan.h>
102 #include <linux/kfence.h>
103 #include <linux/kmemleak.h>
104 #include <linux/memory_hotplug.h>
105 
106 /*
107  * Kmemleak configuration and common defines.
108  */
109 #define MAX_TRACE		16	/* stack trace length */
110 #define MSECS_MIN_AGE		5000	/* minimum object age for reporting */
111 #define SECS_FIRST_SCAN		60	/* delay before the first scan */
112 #define SECS_SCAN_WAIT		600	/* subsequent auto scanning delay */
113 #define MAX_SCAN_SIZE		4096	/* maximum size of a scanned block */
114 
115 #define BYTES_PER_POINTER	sizeof(void *)
116 
117 /* GFP bitmask for kmemleak internal allocations */
118 #define gfp_kmemleak_mask(gfp)	(((gfp) & (GFP_KERNEL | GFP_ATOMIC | \
119 					   __GFP_NOLOCKDEP)) | \
120 				 __GFP_NORETRY | __GFP_NOMEMALLOC | \
121 				 __GFP_NOWARN)
122 
123 /* scanning area inside a memory block */
124 struct kmemleak_scan_area {
125 	struct hlist_node node;
126 	unsigned long start;
127 	size_t size;
128 };
129 
130 #define KMEMLEAK_GREY	0
131 #define KMEMLEAK_BLACK	-1
132 
133 /*
134  * Structure holding the metadata for each allocated memory block.
135  * Modifications to such objects should be made while holding the
136  * object->lock. Insertions or deletions from object_list, gray_list or
137  * rb_node are already protected by the corresponding locks or mutex (see
138  * the notes on locking above). These objects are reference-counted
139  * (use_count) and freed using the RCU mechanism.
140  */
141 struct kmemleak_object {
142 	raw_spinlock_t lock;
143 	unsigned int flags;		/* object status flags */
144 	struct list_head object_list;
145 	struct list_head gray_list;
146 	struct rb_node rb_node;
147 	struct rcu_head rcu;		/* object_list lockless traversal */
148 	/* object usage count; object freed when use_count == 0 */
149 	atomic_t use_count;
150 	unsigned long pointer;
151 	size_t size;
152 	/* pass surplus references to this pointer */
153 	unsigned long excess_ref;
154 	/* minimum number of a pointers found before it is considered leak */
155 	int min_count;
156 	/* the total number of pointers found pointing to this object */
157 	int count;
158 	/* checksum for detecting modified objects */
159 	u32 checksum;
160 	/* memory ranges to be scanned inside an object (empty for all) */
161 	struct hlist_head area_list;
162 	unsigned long trace[MAX_TRACE];
163 	unsigned int trace_len;
164 	unsigned long jiffies;		/* creation timestamp */
165 	pid_t pid;			/* pid of the current task */
166 	char comm[TASK_COMM_LEN];	/* executable name */
167 };
168 
169 /* flag representing the memory block allocation status */
170 #define OBJECT_ALLOCATED	(1 << 0)
171 /* flag set after the first reporting of an unreference object */
172 #define OBJECT_REPORTED		(1 << 1)
173 /* flag set to not scan the object */
174 #define OBJECT_NO_SCAN		(1 << 2)
175 /* flag set to fully scan the object when scan_area allocation failed */
176 #define OBJECT_FULL_SCAN	(1 << 3)
177 /* flag set for object allocated with physical address */
178 #define OBJECT_PHYS		(1 << 4)
179 
180 #define HEX_PREFIX		"    "
181 /* number of bytes to print per line; must be 16 or 32 */
182 #define HEX_ROW_SIZE		16
183 /* number of bytes to print at a time (1, 2, 4, 8) */
184 #define HEX_GROUP_SIZE		1
185 /* include ASCII after the hex output */
186 #define HEX_ASCII		1
187 /* max number of lines to be printed */
188 #define HEX_MAX_LINES		2
189 
190 /* the list of all allocated objects */
191 static LIST_HEAD(object_list);
192 /* the list of gray-colored objects (see color_gray comment below) */
193 static LIST_HEAD(gray_list);
194 /* memory pool allocation */
195 static struct kmemleak_object mem_pool[CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE];
196 static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
197 static LIST_HEAD(mem_pool_free_list);
198 /* search tree for object boundaries */
199 static struct rb_root object_tree_root = RB_ROOT;
200 /* search tree for object (with OBJECT_PHYS flag) boundaries */
201 static struct rb_root object_phys_tree_root = RB_ROOT;
202 /* protecting the access to object_list, object_tree_root (or object_phys_tree_root) */
203 static DEFINE_RAW_SPINLOCK(kmemleak_lock);
204 
205 /* allocation caches for kmemleak internal data */
206 static struct kmem_cache *object_cache;
207 static struct kmem_cache *scan_area_cache;
208 
209 /* set if tracing memory operations is enabled */
210 static int kmemleak_enabled = 1;
211 /* same as above but only for the kmemleak_free() callback */
212 static int kmemleak_free_enabled = 1;
213 /* set in the late_initcall if there were no errors */
214 static int kmemleak_initialized;
215 /* set if a kmemleak warning was issued */
216 static int kmemleak_warning;
217 /* set if a fatal kmemleak error has occurred */
218 static int kmemleak_error;
219 
220 /* minimum and maximum address that may be valid pointers */
221 static unsigned long min_addr = ULONG_MAX;
222 static unsigned long max_addr;
223 
224 static struct task_struct *scan_thread;
225 /* used to avoid reporting of recently allocated objects */
226 static unsigned long jiffies_min_age;
227 static unsigned long jiffies_last_scan;
228 /* delay between automatic memory scannings */
229 static unsigned long jiffies_scan_wait;
230 /* enables or disables the task stacks scanning */
231 static int kmemleak_stack_scan = 1;
232 /* protects the memory scanning, parameters and debug/kmemleak file access */
233 static DEFINE_MUTEX(scan_mutex);
234 /* setting kmemleak=on, will set this var, skipping the disable */
235 static int kmemleak_skip_disable;
236 /* If there are leaks that can be reported */
237 static bool kmemleak_found_leaks;
238 
239 static bool kmemleak_verbose;
240 module_param_named(verbose, kmemleak_verbose, bool, 0600);
241 
242 static void kmemleak_disable(void);
243 
244 /*
245  * Print a warning and dump the stack trace.
246  */
247 #define kmemleak_warn(x...)	do {		\
248 	pr_warn(x);				\
249 	dump_stack();				\
250 	kmemleak_warning = 1;			\
251 } while (0)
252 
253 /*
254  * Macro invoked when a serious kmemleak condition occurred and cannot be
255  * recovered from. Kmemleak will be disabled and further allocation/freeing
256  * tracing no longer available.
257  */
258 #define kmemleak_stop(x...)	do {	\
259 	kmemleak_warn(x);		\
260 	kmemleak_disable();		\
261 } while (0)
262 
263 #define warn_or_seq_printf(seq, fmt, ...)	do {	\
264 	if (seq)					\
265 		seq_printf(seq, fmt, ##__VA_ARGS__);	\
266 	else						\
267 		pr_warn(fmt, ##__VA_ARGS__);		\
268 } while (0)
269 
warn_or_seq_hex_dump(struct seq_file * seq,int prefix_type,int rowsize,int groupsize,const void * buf,size_t len,bool ascii)270 static void warn_or_seq_hex_dump(struct seq_file *seq, int prefix_type,
271 				 int rowsize, int groupsize, const void *buf,
272 				 size_t len, bool ascii)
273 {
274 	if (seq)
275 		seq_hex_dump(seq, HEX_PREFIX, prefix_type, rowsize, groupsize,
276 			     buf, len, ascii);
277 	else
278 		print_hex_dump(KERN_WARNING, pr_fmt(HEX_PREFIX), prefix_type,
279 			       rowsize, groupsize, buf, len, ascii);
280 }
281 
282 /*
283  * Printing of the objects hex dump to the seq file. The number of lines to be
284  * printed is limited to HEX_MAX_LINES to prevent seq file spamming. The
285  * actual number of printed bytes depends on HEX_ROW_SIZE. It must be called
286  * with the object->lock held.
287  */
hex_dump_object(struct seq_file * seq,struct kmemleak_object * object)288 static void hex_dump_object(struct seq_file *seq,
289 			    struct kmemleak_object *object)
290 {
291 	const u8 *ptr = (const u8 *)object->pointer;
292 	size_t len;
293 
294 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))
295 		return;
296 
297 	/* limit the number of lines to HEX_MAX_LINES */
298 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
299 
300 	warn_or_seq_printf(seq, "  hex dump (first %zu bytes):\n", len);
301 	kasan_disable_current();
302 	warn_or_seq_hex_dump(seq, DUMP_PREFIX_NONE, HEX_ROW_SIZE,
303 			     HEX_GROUP_SIZE, kasan_reset_tag((void *)ptr), len, HEX_ASCII);
304 	kasan_enable_current();
305 }
306 
307 /*
308  * Object colors, encoded with count and min_count:
309  * - white - orphan object, not enough references to it (count < min_count)
310  * - gray  - not orphan, not marked as false positive (min_count == 0) or
311  *		sufficient references to it (count >= min_count)
312  * - black - ignore, it doesn't contain references (e.g. text section)
313  *		(min_count == -1). No function defined for this color.
314  * Newly created objects don't have any color assigned (object->count == -1)
315  * before the next memory scan when they become white.
316  */
color_white(const struct kmemleak_object * object)317 static bool color_white(const struct kmemleak_object *object)
318 {
319 	return object->count != KMEMLEAK_BLACK &&
320 		object->count < object->min_count;
321 }
322 
color_gray(const struct kmemleak_object * object)323 static bool color_gray(const struct kmemleak_object *object)
324 {
325 	return object->min_count != KMEMLEAK_BLACK &&
326 		object->count >= object->min_count;
327 }
328 
329 /*
330  * Objects are considered unreferenced only if their color is white, they have
331  * not be deleted and have a minimum age to avoid false positives caused by
332  * pointers temporarily stored in CPU registers.
333  */
unreferenced_object(struct kmemleak_object * object)334 static bool unreferenced_object(struct kmemleak_object *object)
335 {
336 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
337 		time_before_eq(object->jiffies + jiffies_min_age,
338 			       jiffies_last_scan);
339 }
340 
341 /*
342  * Printing of the unreferenced objects information to the seq file. The
343  * print_unreferenced function must be called with the object->lock held.
344  */
print_unreferenced(struct seq_file * seq,struct kmemleak_object * object)345 static void print_unreferenced(struct seq_file *seq,
346 			       struct kmemleak_object *object)
347 {
348 	int i;
349 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
350 
351 	warn_or_seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
352 		   object->pointer, object->size);
353 	warn_or_seq_printf(seq, "  comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
354 		   object->comm, object->pid, object->jiffies,
355 		   msecs_age / 1000, msecs_age % 1000);
356 	hex_dump_object(seq, object);
357 	warn_or_seq_printf(seq, "  backtrace:\n");
358 
359 	for (i = 0; i < object->trace_len; i++) {
360 		void *ptr = (void *)object->trace[i];
361 		warn_or_seq_printf(seq, "    [<%p>] %pS\n", ptr, ptr);
362 	}
363 }
364 
365 /*
366  * Print the kmemleak_object information. This function is used mainly for
367  * debugging special cases when kmemleak operations. It must be called with
368  * the object->lock held.
369  */
dump_object_info(struct kmemleak_object * object)370 static void dump_object_info(struct kmemleak_object *object)
371 {
372 	pr_notice("Object 0x%08lx (size %zu):\n",
373 		  object->pointer, object->size);
374 	pr_notice("  comm \"%s\", pid %d, jiffies %lu\n",
375 		  object->comm, object->pid, object->jiffies);
376 	pr_notice("  min_count = %d\n", object->min_count);
377 	pr_notice("  count = %d\n", object->count);
378 	pr_notice("  flags = 0x%x\n", object->flags);
379 	pr_notice("  checksum = %u\n", object->checksum);
380 	pr_notice("  backtrace:\n");
381 	stack_trace_print(object->trace, object->trace_len, 4);
382 }
383 
384 /*
385  * Look-up a memory block metadata (kmemleak_object) in the object search
386  * tree based on a pointer value. If alias is 0, only values pointing to the
387  * beginning of the memory block are allowed. The kmemleak_lock must be held
388  * when calling this function.
389  */
__lookup_object(unsigned long ptr,int alias,bool is_phys)390 static struct kmemleak_object *__lookup_object(unsigned long ptr, int alias,
391 					       bool is_phys)
392 {
393 	struct rb_node *rb = is_phys ? object_phys_tree_root.rb_node :
394 			     object_tree_root.rb_node;
395 	unsigned long untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
396 
397 	while (rb) {
398 		struct kmemleak_object *object;
399 		unsigned long untagged_objp;
400 
401 		object = rb_entry(rb, struct kmemleak_object, rb_node);
402 		untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);
403 
404 		if (untagged_ptr < untagged_objp)
405 			rb = object->rb_node.rb_left;
406 		else if (untagged_objp + object->size <= untagged_ptr)
407 			rb = object->rb_node.rb_right;
408 		else if (untagged_objp == untagged_ptr || alias)
409 			return object;
410 		else {
411 			kmemleak_warn("Found object by alias at 0x%08lx\n",
412 				      ptr);
413 			dump_object_info(object);
414 			break;
415 		}
416 	}
417 	return NULL;
418 }
419 
420 /* Look-up a kmemleak object which allocated with virtual address. */
lookup_object(unsigned long ptr,int alias)421 static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
422 {
423 	return __lookup_object(ptr, alias, false);
424 }
425 
426 /*
427  * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
428  * that once an object's use_count reached 0, the RCU freeing was already
429  * registered and the object should no longer be used. This function must be
430  * called under the protection of rcu_read_lock().
431  */
get_object(struct kmemleak_object * object)432 static int get_object(struct kmemleak_object *object)
433 {
434 	return atomic_inc_not_zero(&object->use_count);
435 }
436 
437 /*
438  * Memory pool allocation and freeing. kmemleak_lock must not be held.
439  */
mem_pool_alloc(gfp_t gfp)440 static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
441 {
442 	unsigned long flags;
443 	struct kmemleak_object *object;
444 
445 	/* try the slab allocator first */
446 	if (object_cache) {
447 		object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
448 		if (object)
449 			return object;
450 	}
451 
452 	/* slab allocation failed, try the memory pool */
453 	raw_spin_lock_irqsave(&kmemleak_lock, flags);
454 	object = list_first_entry_or_null(&mem_pool_free_list,
455 					  typeof(*object), object_list);
456 	if (object)
457 		list_del(&object->object_list);
458 	else if (mem_pool_free_count)
459 		object = &mem_pool[--mem_pool_free_count];
460 	else
461 		pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
462 	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
463 
464 	return object;
465 }
466 
467 /*
468  * Return the object to either the slab allocator or the memory pool.
469  */
mem_pool_free(struct kmemleak_object * object)470 static void mem_pool_free(struct kmemleak_object *object)
471 {
472 	unsigned long flags;
473 
474 	if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) {
475 		kmem_cache_free(object_cache, object);
476 		return;
477 	}
478 
479 	/* add the object to the memory pool free list */
480 	raw_spin_lock_irqsave(&kmemleak_lock, flags);
481 	list_add(&object->object_list, &mem_pool_free_list);
482 	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
483 }
484 
485 /*
486  * RCU callback to free a kmemleak_object.
487  */
free_object_rcu(struct rcu_head * rcu)488 static void free_object_rcu(struct rcu_head *rcu)
489 {
490 	struct hlist_node *tmp;
491 	struct kmemleak_scan_area *area;
492 	struct kmemleak_object *object =
493 		container_of(rcu, struct kmemleak_object, rcu);
494 
495 	/*
496 	 * Once use_count is 0 (guaranteed by put_object), there is no other
497 	 * code accessing this object, hence no need for locking.
498 	 */
499 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
500 		hlist_del(&area->node);
501 		kmem_cache_free(scan_area_cache, area);
502 	}
503 	mem_pool_free(object);
504 }
505 
506 /*
507  * Decrement the object use_count. Once the count is 0, free the object using
508  * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
509  * delete_object() path, the delayed RCU freeing ensures that there is no
510  * recursive call to the kernel allocator. Lock-less RCU object_list traversal
511  * is also possible.
512  */
put_object(struct kmemleak_object * object)513 static void put_object(struct kmemleak_object *object)
514 {
515 	if (!atomic_dec_and_test(&object->use_count))
516 		return;
517 
518 	/* should only get here after delete_object was called */
519 	WARN_ON(object->flags & OBJECT_ALLOCATED);
520 
521 	/*
522 	 * It may be too early for the RCU callbacks, however, there is no
523 	 * concurrent object_list traversal when !object_cache and all objects
524 	 * came from the memory pool. Free the object directly.
525 	 */
526 	if (object_cache)
527 		call_rcu(&object->rcu, free_object_rcu);
528 	else
529 		free_object_rcu(&object->rcu);
530 }
531 
532 /*
533  * Look up an object in the object search tree and increase its use_count.
534  */
__find_and_get_object(unsigned long ptr,int alias,bool is_phys)535 static struct kmemleak_object *__find_and_get_object(unsigned long ptr, int alias,
536 						     bool is_phys)
537 {
538 	unsigned long flags;
539 	struct kmemleak_object *object;
540 
541 	rcu_read_lock();
542 	raw_spin_lock_irqsave(&kmemleak_lock, flags);
543 	object = __lookup_object(ptr, alias, is_phys);
544 	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
545 
546 	/* check whether the object is still available */
547 	if (object && !get_object(object))
548 		object = NULL;
549 	rcu_read_unlock();
550 
551 	return object;
552 }
553 
554 /* Look up and get an object which allocated with virtual address. */
find_and_get_object(unsigned long ptr,int alias)555 static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
556 {
557 	return __find_and_get_object(ptr, alias, false);
558 }
559 
560 /*
561  * Remove an object from the object_tree_root (or object_phys_tree_root)
562  * and object_list. Must be called with the kmemleak_lock held _if_ kmemleak
563  * is still enabled.
564  */
__remove_object(struct kmemleak_object * object)565 static void __remove_object(struct kmemleak_object *object)
566 {
567 	rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ?
568 				   &object_phys_tree_root :
569 				   &object_tree_root);
570 	list_del_rcu(&object->object_list);
571 }
572 
573 /*
574  * Look up an object in the object search tree and remove it from both
575  * object_tree_root (or object_phys_tree_root) and object_list. The
576  * returned object's use_count should be at least 1, as initially set
577  * by create_object().
578  */
find_and_remove_object(unsigned long ptr,int alias,bool is_phys)579 static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias,
580 						      bool is_phys)
581 {
582 	unsigned long flags;
583 	struct kmemleak_object *object;
584 
585 	raw_spin_lock_irqsave(&kmemleak_lock, flags);
586 	object = __lookup_object(ptr, alias, is_phys);
587 	if (object)
588 		__remove_object(object);
589 	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
590 
591 	return object;
592 }
593 
594 /*
595  * Save stack trace to the given array of MAX_TRACE size.
596  */
__save_stack_trace(unsigned long * trace)597 static int __save_stack_trace(unsigned long *trace)
598 {
599 	return stack_trace_save(trace, MAX_TRACE, 2);
600 }
601 
602 /*
603  * Create the metadata (struct kmemleak_object) corresponding to an allocated
604  * memory block and add it to the object_list and object_tree_root (or
605  * object_phys_tree_root).
606  */
__create_object(unsigned long ptr,size_t size,int min_count,gfp_t gfp,bool is_phys)607 static void __create_object(unsigned long ptr, size_t size,
608 			    int min_count, gfp_t gfp, bool is_phys)
609 {
610 	unsigned long flags;
611 	struct kmemleak_object *object, *parent;
612 	struct rb_node **link, *rb_parent;
613 	unsigned long untagged_ptr;
614 	unsigned long untagged_objp;
615 
616 	object = mem_pool_alloc(gfp);
617 	if (!object) {
618 		pr_warn("Cannot allocate a kmemleak_object structure\n");
619 		kmemleak_disable();
620 		return;
621 	}
622 
623 	INIT_LIST_HEAD(&object->object_list);
624 	INIT_LIST_HEAD(&object->gray_list);
625 	INIT_HLIST_HEAD(&object->area_list);
626 	raw_spin_lock_init(&object->lock);
627 	atomic_set(&object->use_count, 1);
628 	object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0);
629 	object->pointer = ptr;
630 	object->size = kfence_ksize((void *)ptr) ?: size;
631 	object->excess_ref = 0;
632 	object->min_count = min_count;
633 	object->count = 0;			/* white color initially */
634 	object->jiffies = jiffies;
635 	object->checksum = 0;
636 
637 	/* task information */
638 	if (in_hardirq()) {
639 		object->pid = 0;
640 		strncpy(object->comm, "hardirq", sizeof(object->comm));
641 	} else if (in_serving_softirq()) {
642 		object->pid = 0;
643 		strncpy(object->comm, "softirq", sizeof(object->comm));
644 	} else {
645 		object->pid = current->pid;
646 		/*
647 		 * There is a small chance of a race with set_task_comm(),
648 		 * however using get_task_comm() here may cause locking
649 		 * dependency issues with current->alloc_lock. In the worst
650 		 * case, the command line is not correct.
651 		 */
652 		strncpy(object->comm, current->comm, sizeof(object->comm));
653 	}
654 
655 	/* kernel backtrace */
656 	object->trace_len = __save_stack_trace(object->trace);
657 
658 	raw_spin_lock_irqsave(&kmemleak_lock, flags);
659 
660 	untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
661 	/*
662 	 * Only update min_addr and max_addr with object
663 	 * storing virtual address.
664 	 */
665 	if (!is_phys) {
666 		min_addr = min(min_addr, untagged_ptr);
667 		max_addr = max(max_addr, untagged_ptr + size);
668 	}
669 	link = is_phys ? &object_phys_tree_root.rb_node :
670 		&object_tree_root.rb_node;
671 	rb_parent = NULL;
672 	while (*link) {
673 		rb_parent = *link;
674 		parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
675 		untagged_objp = (unsigned long)kasan_reset_tag((void *)parent->pointer);
676 		if (untagged_ptr + size <= untagged_objp)
677 			link = &parent->rb_node.rb_left;
678 		else if (untagged_objp + parent->size <= untagged_ptr)
679 			link = &parent->rb_node.rb_right;
680 		else {
681 			kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
682 				      ptr);
683 			/*
684 			 * No need for parent->lock here since "parent" cannot
685 			 * be freed while the kmemleak_lock is held.
686 			 */
687 			dump_object_info(parent);
688 			kmem_cache_free(object_cache, object);
689 			goto out;
690 		}
691 	}
692 	rb_link_node(&object->rb_node, rb_parent, link);
693 	rb_insert_color(&object->rb_node, is_phys ? &object_phys_tree_root :
694 					  &object_tree_root);
695 
696 	list_add_tail_rcu(&object->object_list, &object_list);
697 out:
698 	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
699 }
700 
701 /* Create kmemleak object which allocated with virtual address. */
create_object(unsigned long ptr,size_t size,int min_count,gfp_t gfp)702 static void create_object(unsigned long ptr, size_t size,
703 			  int min_count, gfp_t gfp)
704 {
705 	__create_object(ptr, size, min_count, gfp, false);
706 }
707 
708 /* Create kmemleak object which allocated with physical address. */
create_object_phys(unsigned long ptr,size_t size,int min_count,gfp_t gfp)709 static void create_object_phys(unsigned long ptr, size_t size,
710 			       int min_count, gfp_t gfp)
711 {
712 	__create_object(ptr, size, min_count, gfp, true);
713 }
714 
715 /*
716  * Mark the object as not allocated and schedule RCU freeing via put_object().
717  */
__delete_object(struct kmemleak_object * object)718 static void __delete_object(struct kmemleak_object *object)
719 {
720 	unsigned long flags;
721 
722 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));
723 	WARN_ON(atomic_read(&object->use_count) < 1);
724 
725 	/*
726 	 * Locking here also ensures that the corresponding memory block
727 	 * cannot be freed when it is being scanned.
728 	 */
729 	raw_spin_lock_irqsave(&object->lock, flags);
730 	object->flags &= ~OBJECT_ALLOCATED;
731 	raw_spin_unlock_irqrestore(&object->lock, flags);
732 	put_object(object);
733 }
734 
735 /*
736  * Look up the metadata (struct kmemleak_object) corresponding to ptr and
737  * delete it.
738  */
delete_object_full(unsigned long ptr)739 static void delete_object_full(unsigned long ptr)
740 {
741 	struct kmemleak_object *object;
742 
743 	object = find_and_remove_object(ptr, 0, false);
744 	if (!object) {
745 #ifdef DEBUG
746 		kmemleak_warn("Freeing unknown object at 0x%08lx\n",
747 			      ptr);
748 #endif
749 		return;
750 	}
751 	__delete_object(object);
752 }
753 
754 /*
755  * Look up the metadata (struct kmemleak_object) corresponding to ptr and
756  * delete it. If the memory block is partially freed, the function may create
757  * additional metadata for the remaining parts of the block.
758  */
delete_object_part(unsigned long ptr,size_t size,bool is_phys)759 static void delete_object_part(unsigned long ptr, size_t size, bool is_phys)
760 {
761 	struct kmemleak_object *object;
762 	unsigned long start, end;
763 
764 	object = find_and_remove_object(ptr, 1, is_phys);
765 	if (!object) {
766 #ifdef DEBUG
767 		kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
768 			      ptr, size);
769 #endif
770 		return;
771 	}
772 
773 	/*
774 	 * Create one or two objects that may result from the memory block
775 	 * split. Note that partial freeing is only done by free_bootmem() and
776 	 * this happens before kmemleak_init() is called.
777 	 */
778 	start = object->pointer;
779 	end = object->pointer + object->size;
780 	if (ptr > start)
781 		__create_object(start, ptr - start, object->min_count,
782 			      GFP_KERNEL, is_phys);
783 	if (ptr + size < end)
784 		__create_object(ptr + size, end - ptr - size, object->min_count,
785 			      GFP_KERNEL, is_phys);
786 
787 	__delete_object(object);
788 }
789 
__paint_it(struct kmemleak_object * object,int color)790 static void __paint_it(struct kmemleak_object *object, int color)
791 {
792 	object->min_count = color;
793 	if (color == KMEMLEAK_BLACK)
794 		object->flags |= OBJECT_NO_SCAN;
795 }
796 
paint_it(struct kmemleak_object * object,int color)797 static void paint_it(struct kmemleak_object *object, int color)
798 {
799 	unsigned long flags;
800 
801 	raw_spin_lock_irqsave(&object->lock, flags);
802 	__paint_it(object, color);
803 	raw_spin_unlock_irqrestore(&object->lock, flags);
804 }
805 
paint_ptr(unsigned long ptr,int color,bool is_phys)806 static void paint_ptr(unsigned long ptr, int color, bool is_phys)
807 {
808 	struct kmemleak_object *object;
809 
810 	object = __find_and_get_object(ptr, 0, is_phys);
811 	if (!object) {
812 		kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
813 			      ptr,
814 			      (color == KMEMLEAK_GREY) ? "Grey" :
815 			      (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
816 		return;
817 	}
818 	paint_it(object, color);
819 	put_object(object);
820 }
821 
822 /*
823  * Mark an object permanently as gray-colored so that it can no longer be
824  * reported as a leak. This is used in general to mark a false positive.
825  */
make_gray_object(unsigned long ptr)826 static void make_gray_object(unsigned long ptr)
827 {
828 	paint_ptr(ptr, KMEMLEAK_GREY, false);
829 }
830 
831 /*
832  * Mark the object as black-colored so that it is ignored from scans and
833  * reporting.
834  */
make_black_object(unsigned long ptr,bool is_phys)835 static void make_black_object(unsigned long ptr, bool is_phys)
836 {
837 	paint_ptr(ptr, KMEMLEAK_BLACK, is_phys);
838 }
839 
840 /*
841  * Add a scanning area to the object. If at least one such area is added,
842  * kmemleak will only scan these ranges rather than the whole memory block.
843  */
add_scan_area(unsigned long ptr,size_t size,gfp_t gfp)844 static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
845 {
846 	unsigned long flags;
847 	struct kmemleak_object *object;
848 	struct kmemleak_scan_area *area = NULL;
849 	unsigned long untagged_ptr;
850 	unsigned long untagged_objp;
851 
852 	object = find_and_get_object(ptr, 1);
853 	if (!object) {
854 		kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
855 			      ptr);
856 		return;
857 	}
858 
859 	untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
860 	untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);
861 
862 	if (scan_area_cache)
863 		area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
864 
865 	raw_spin_lock_irqsave(&object->lock, flags);
866 	if (!area) {
867 		pr_warn_once("Cannot allocate a scan area, scanning the full object\n");
868 		/* mark the object for full scan to avoid false positives */
869 		object->flags |= OBJECT_FULL_SCAN;
870 		goto out_unlock;
871 	}
872 	if (size == SIZE_MAX) {
873 		size = untagged_objp + object->size - untagged_ptr;
874 	} else if (untagged_ptr + size > untagged_objp + object->size) {
875 		kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
876 		dump_object_info(object);
877 		kmem_cache_free(scan_area_cache, area);
878 		goto out_unlock;
879 	}
880 
881 	INIT_HLIST_NODE(&area->node);
882 	area->start = ptr;
883 	area->size = size;
884 
885 	hlist_add_head(&area->node, &object->area_list);
886 out_unlock:
887 	raw_spin_unlock_irqrestore(&object->lock, flags);
888 	put_object(object);
889 }
890 
891 /*
892  * Any surplus references (object already gray) to 'ptr' are passed to
893  * 'excess_ref'. This is used in the vmalloc() case where a pointer to
894  * vm_struct may be used as an alternative reference to the vmalloc'ed object
895  * (see free_thread_stack()).
896  */
object_set_excess_ref(unsigned long ptr,unsigned long excess_ref)897 static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
898 {
899 	unsigned long flags;
900 	struct kmemleak_object *object;
901 
902 	object = find_and_get_object(ptr, 0);
903 	if (!object) {
904 		kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",
905 			      ptr);
906 		return;
907 	}
908 
909 	raw_spin_lock_irqsave(&object->lock, flags);
910 	object->excess_ref = excess_ref;
911 	raw_spin_unlock_irqrestore(&object->lock, flags);
912 	put_object(object);
913 }
914 
915 /*
916  * Set the OBJECT_NO_SCAN flag for the object corresponding to the give
917  * pointer. Such object will not be scanned by kmemleak but references to it
918  * are searched.
919  */
object_no_scan(unsigned long ptr)920 static void object_no_scan(unsigned long ptr)
921 {
922 	unsigned long flags;
923 	struct kmemleak_object *object;
924 
925 	object = find_and_get_object(ptr, 0);
926 	if (!object) {
927 		kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
928 		return;
929 	}
930 
931 	raw_spin_lock_irqsave(&object->lock, flags);
932 	object->flags |= OBJECT_NO_SCAN;
933 	raw_spin_unlock_irqrestore(&object->lock, flags);
934 	put_object(object);
935 }
936 
937 /**
938  * kmemleak_alloc - register a newly allocated object
939  * @ptr:	pointer to beginning of the object
940  * @size:	size of the object
941  * @min_count:	minimum number of references to this object. If during memory
942  *		scanning a number of references less than @min_count is found,
943  *		the object is reported as a memory leak. If @min_count is 0,
944  *		the object is never reported as a leak. If @min_count is -1,
945  *		the object is ignored (not scanned and not reported as a leak)
946  * @gfp:	kmalloc() flags used for kmemleak internal memory allocations
947  *
948  * This function is called from the kernel allocators when a new object
949  * (memory block) is allocated (kmem_cache_alloc, kmalloc etc.).
950  */
kmemleak_alloc(const void * ptr,size_t size,int min_count,gfp_t gfp)951 void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
952 			  gfp_t gfp)
953 {
954 	pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
955 
956 	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
957 		create_object((unsigned long)ptr, size, min_count, gfp);
958 }
959 EXPORT_SYMBOL_GPL(kmemleak_alloc);
960 
961 /**
962  * kmemleak_alloc_percpu - register a newly allocated __percpu object
963  * @ptr:	__percpu pointer to beginning of the object
964  * @size:	size of the object
965  * @gfp:	flags used for kmemleak internal memory allocations
966  *
967  * This function is called from the kernel percpu allocator when a new object
968  * (memory block) is allocated (alloc_percpu).
969  */
kmemleak_alloc_percpu(const void __percpu * ptr,size_t size,gfp_t gfp)970 void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
971 				 gfp_t gfp)
972 {
973 	unsigned int cpu;
974 
975 	pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
976 
977 	/*
978 	 * Percpu allocations are only scanned and not reported as leaks
979 	 * (min_count is set to 0).
980 	 */
981 	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
982 		for_each_possible_cpu(cpu)
983 			create_object((unsigned long)per_cpu_ptr(ptr, cpu),
984 				      size, 0, gfp);
985 }
986 EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
987 
988 /**
989  * kmemleak_vmalloc - register a newly vmalloc'ed object
990  * @area:	pointer to vm_struct
991  * @size:	size of the object
992  * @gfp:	__vmalloc() flags used for kmemleak internal memory allocations
993  *
994  * This function is called from the vmalloc() kernel allocator when a new
995  * object (memory block) is allocated.
996  */
kmemleak_vmalloc(const struct vm_struct * area,size_t size,gfp_t gfp)997 void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp)
998 {
999 	pr_debug("%s(0x%p, %zu)\n", __func__, area, size);
1000 
1001 	/*
1002 	 * A min_count = 2 is needed because vm_struct contains a reference to
1003 	 * the virtual address of the vmalloc'ed block.
1004 	 */
1005 	if (kmemleak_enabled) {
1006 		create_object((unsigned long)area->addr, size, 2, gfp);
1007 		object_set_excess_ref((unsigned long)area,
1008 				      (unsigned long)area->addr);
1009 	}
1010 }
1011 EXPORT_SYMBOL_GPL(kmemleak_vmalloc);
1012 
1013 /**
1014  * kmemleak_free - unregister a previously registered object
1015  * @ptr:	pointer to beginning of the object
1016  *
1017  * This function is called from the kernel allocators when an object (memory
1018  * block) is freed (kmem_cache_free, kfree, vfree etc.).
1019  */
kmemleak_free(const void * ptr)1020 void __ref kmemleak_free(const void *ptr)
1021 {
1022 	pr_debug("%s(0x%p)\n", __func__, ptr);
1023 
1024 	if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1025 		delete_object_full((unsigned long)ptr);
1026 }
1027 EXPORT_SYMBOL_GPL(kmemleak_free);
1028 
1029 /**
1030  * kmemleak_free_part - partially unregister a previously registered object
1031  * @ptr:	pointer to the beginning or inside the object. This also
1032  *		represents the start of the range to be freed
1033  * @size:	size to be unregistered
1034  *
1035  * This function is called when only a part of a memory block is freed
1036  * (usually from the bootmem allocator).
1037  */
kmemleak_free_part(const void * ptr,size_t size)1038 void __ref kmemleak_free_part(const void *ptr, size_t size)
1039 {
1040 	pr_debug("%s(0x%p)\n", __func__, ptr);
1041 
1042 	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1043 		delete_object_part((unsigned long)ptr, size, false);
1044 }
1045 EXPORT_SYMBOL_GPL(kmemleak_free_part);
1046 
1047 /**
1048  * kmemleak_free_percpu - unregister a previously registered __percpu object
1049  * @ptr:	__percpu pointer to beginning of the object
1050  *
1051  * This function is called from the kernel percpu allocator when an object
1052  * (memory block) is freed (free_percpu).
1053  */
kmemleak_free_percpu(const void __percpu * ptr)1054 void __ref kmemleak_free_percpu(const void __percpu *ptr)
1055 {
1056 	unsigned int cpu;
1057 
1058 	pr_debug("%s(0x%p)\n", __func__, ptr);
1059 
1060 	if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1061 		for_each_possible_cpu(cpu)
1062 			delete_object_full((unsigned long)per_cpu_ptr(ptr,
1063 								      cpu));
1064 }
1065 EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
1066 
1067 /**
1068  * kmemleak_update_trace - update object allocation stack trace
1069  * @ptr:	pointer to beginning of the object
1070  *
1071  * Override the object allocation stack trace for cases where the actual
1072  * allocation place is not always useful.
1073  */
kmemleak_update_trace(const void * ptr)1074 void __ref kmemleak_update_trace(const void *ptr)
1075 {
1076 	struct kmemleak_object *object;
1077 	unsigned long flags;
1078 
1079 	pr_debug("%s(0x%p)\n", __func__, ptr);
1080 
1081 	if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
1082 		return;
1083 
1084 	object = find_and_get_object((unsigned long)ptr, 1);
1085 	if (!object) {
1086 #ifdef DEBUG
1087 		kmemleak_warn("Updating stack trace for unknown object at %p\n",
1088 			      ptr);
1089 #endif
1090 		return;
1091 	}
1092 
1093 	raw_spin_lock_irqsave(&object->lock, flags);
1094 	object->trace_len = __save_stack_trace(object->trace);
1095 	raw_spin_unlock_irqrestore(&object->lock, flags);
1096 
1097 	put_object(object);
1098 }
1099 EXPORT_SYMBOL(kmemleak_update_trace);
1100 
1101 /**
1102  * kmemleak_not_leak - mark an allocated object as false positive
1103  * @ptr:	pointer to beginning of the object
1104  *
1105  * Calling this function on an object will cause the memory block to no longer
1106  * be reported as leak and always be scanned.
1107  */
kmemleak_not_leak(const void * ptr)1108 void __ref kmemleak_not_leak(const void *ptr)
1109 {
1110 	pr_debug("%s(0x%p)\n", __func__, ptr);
1111 
1112 	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1113 		make_gray_object((unsigned long)ptr);
1114 }
1115 EXPORT_SYMBOL(kmemleak_not_leak);
1116 
1117 /**
1118  * kmemleak_ignore - ignore an allocated object
1119  * @ptr:	pointer to beginning of the object
1120  *
1121  * Calling this function on an object will cause the memory block to be
1122  * ignored (not scanned and not reported as a leak). This is usually done when
1123  * it is known that the corresponding block is not a leak and does not contain
1124  * any references to other allocated memory blocks.
1125  */
kmemleak_ignore(const void * ptr)1126 void __ref kmemleak_ignore(const void *ptr)
1127 {
1128 	pr_debug("%s(0x%p)\n", __func__, ptr);
1129 
1130 	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1131 		make_black_object((unsigned long)ptr, false);
1132 }
1133 EXPORT_SYMBOL(kmemleak_ignore);
1134 
1135 /**
1136  * kmemleak_scan_area - limit the range to be scanned in an allocated object
1137  * @ptr:	pointer to beginning or inside the object. This also
1138  *		represents the start of the scan area
1139  * @size:	size of the scan area
1140  * @gfp:	kmalloc() flags used for kmemleak internal memory allocations
1141  *
1142  * This function is used when it is known that only certain parts of an object
1143  * contain references to other objects. Kmemleak will only scan these areas
1144  * reducing the number false negatives.
1145  */
kmemleak_scan_area(const void * ptr,size_t size,gfp_t gfp)1146 void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
1147 {
1148 	pr_debug("%s(0x%p)\n", __func__, ptr);
1149 
1150 	if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
1151 		add_scan_area((unsigned long)ptr, size, gfp);
1152 }
1153 EXPORT_SYMBOL(kmemleak_scan_area);
1154 
1155 /**
1156  * kmemleak_no_scan - do not scan an allocated object
1157  * @ptr:	pointer to beginning of the object
1158  *
1159  * This function notifies kmemleak not to scan the given memory block. Useful
1160  * in situations where it is known that the given object does not contain any
1161  * references to other objects. Kmemleak will not scan such objects reducing
1162  * the number of false negatives.
1163  */
kmemleak_no_scan(const void * ptr)1164 void __ref kmemleak_no_scan(const void *ptr)
1165 {
1166 	pr_debug("%s(0x%p)\n", __func__, ptr);
1167 
1168 	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1169 		object_no_scan((unsigned long)ptr);
1170 }
1171 EXPORT_SYMBOL(kmemleak_no_scan);
1172 
1173 /**
1174  * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1175  *			 address argument
1176  * @phys:	physical address of the object
1177  * @size:	size of the object
1178  * @gfp:	kmalloc() flags used for kmemleak internal memory allocations
1179  */
kmemleak_alloc_phys(phys_addr_t phys,size_t size,gfp_t gfp)1180 void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, gfp_t gfp)
1181 {
1182 	pr_debug("%s(0x%pa, %zu)\n", __func__, &phys, size);
1183 
1184 	if (kmemleak_enabled)
1185 		/*
1186 		 * Create object with OBJECT_PHYS flag and
1187 		 * assume min_count 0.
1188 		 */
1189 		create_object_phys((unsigned long)phys, size, 0, gfp);
1190 }
1191 EXPORT_SYMBOL(kmemleak_alloc_phys);
1192 
1193 /**
1194  * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1195  *			     physical address argument
1196  * @phys:	physical address if the beginning or inside an object. This
1197  *		also represents the start of the range to be freed
1198  * @size:	size to be unregistered
1199  */
kmemleak_free_part_phys(phys_addr_t phys,size_t size)1200 void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
1201 {
1202 	pr_debug("%s(0x%pa)\n", __func__, &phys);
1203 
1204 	if (kmemleak_enabled)
1205 		delete_object_part((unsigned long)phys, size, true);
1206 }
1207 EXPORT_SYMBOL(kmemleak_free_part_phys);
1208 
1209 /**
1210  * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1211  *			  address argument
1212  * @phys:	physical address of the object
1213  */
kmemleak_ignore_phys(phys_addr_t phys)1214 void __ref kmemleak_ignore_phys(phys_addr_t phys)
1215 {
1216 	pr_debug("%s(0x%pa)\n", __func__, &phys);
1217 
1218 	if (kmemleak_enabled)
1219 		make_black_object((unsigned long)phys, true);
1220 }
1221 EXPORT_SYMBOL(kmemleak_ignore_phys);
1222 
1223 /*
1224  * Update an object's checksum and return true if it was modified.
1225  */
update_checksum(struct kmemleak_object * object)1226 static bool update_checksum(struct kmemleak_object *object)
1227 {
1228 	u32 old_csum = object->checksum;
1229 
1230 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))
1231 		return false;
1232 
1233 	kasan_disable_current();
1234 	kcsan_disable_current();
1235 	object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);
1236 	kasan_enable_current();
1237 	kcsan_enable_current();
1238 
1239 	return object->checksum != old_csum;
1240 }
1241 
1242 /*
1243  * Update an object's references. object->lock must be held by the caller.
1244  */
update_refs(struct kmemleak_object * object)1245 static void update_refs(struct kmemleak_object *object)
1246 {
1247 	if (!color_white(object)) {
1248 		/* non-orphan, ignored or new */
1249 		return;
1250 	}
1251 
1252 	/*
1253 	 * Increase the object's reference count (number of pointers to the
1254 	 * memory block). If this count reaches the required minimum, the
1255 	 * object's color will become gray and it will be added to the
1256 	 * gray_list.
1257 	 */
1258 	object->count++;
1259 	if (color_gray(object)) {
1260 		/* put_object() called when removing from gray_list */
1261 		WARN_ON(!get_object(object));
1262 		list_add_tail(&object->gray_list, &gray_list);
1263 	}
1264 }
1265 
1266 /*
1267  * Memory scanning is a long process and it needs to be interruptible. This
1268  * function checks whether such interrupt condition occurred.
1269  */
scan_should_stop(void)1270 static int scan_should_stop(void)
1271 {
1272 	if (!kmemleak_enabled)
1273 		return 1;
1274 
1275 	/*
1276 	 * This function may be called from either process or kthread context,
1277 	 * hence the need to check for both stop conditions.
1278 	 */
1279 	if (current->mm)
1280 		return signal_pending(current);
1281 	else
1282 		return kthread_should_stop();
1283 
1284 	return 0;
1285 }
1286 
1287 /*
1288  * Scan a memory block (exclusive range) for valid pointers and add those
1289  * found to the gray list.
1290  */
scan_block(void * _start,void * _end,struct kmemleak_object * scanned)1291 static void scan_block(void *_start, void *_end,
1292 		       struct kmemleak_object *scanned)
1293 {
1294 	unsigned long *ptr;
1295 	unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1296 	unsigned long *end = _end - (BYTES_PER_POINTER - 1);
1297 	unsigned long flags;
1298 	unsigned long untagged_ptr;
1299 
1300 	raw_spin_lock_irqsave(&kmemleak_lock, flags);
1301 	for (ptr = start; ptr < end; ptr++) {
1302 		struct kmemleak_object *object;
1303 		unsigned long pointer;
1304 		unsigned long excess_ref;
1305 
1306 		if (scan_should_stop())
1307 			break;
1308 
1309 		kasan_disable_current();
1310 		pointer = *(unsigned long *)kasan_reset_tag((void *)ptr);
1311 		kasan_enable_current();
1312 
1313 		untagged_ptr = (unsigned long)kasan_reset_tag((void *)pointer);
1314 		if (untagged_ptr < min_addr || untagged_ptr >= max_addr)
1315 			continue;
1316 
1317 		/*
1318 		 * No need for get_object() here since we hold kmemleak_lock.
1319 		 * object->use_count cannot be dropped to 0 while the object
1320 		 * is still present in object_tree_root and object_list
1321 		 * (with updates protected by kmemleak_lock).
1322 		 */
1323 		object = lookup_object(pointer, 1);
1324 		if (!object)
1325 			continue;
1326 		if (object == scanned)
1327 			/* self referenced, ignore */
1328 			continue;
1329 
1330 		/*
1331 		 * Avoid the lockdep recursive warning on object->lock being
1332 		 * previously acquired in scan_object(). These locks are
1333 		 * enclosed by scan_mutex.
1334 		 */
1335 		raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1336 		/* only pass surplus references (object already gray) */
1337 		if (color_gray(object)) {
1338 			excess_ref = object->excess_ref;
1339 			/* no need for update_refs() if object already gray */
1340 		} else {
1341 			excess_ref = 0;
1342 			update_refs(object);
1343 		}
1344 		raw_spin_unlock(&object->lock);
1345 
1346 		if (excess_ref) {
1347 			object = lookup_object(excess_ref, 0);
1348 			if (!object)
1349 				continue;
1350 			if (object == scanned)
1351 				/* circular reference, ignore */
1352 				continue;
1353 			raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1354 			update_refs(object);
1355 			raw_spin_unlock(&object->lock);
1356 		}
1357 	}
1358 	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
1359 }
1360 
1361 /*
1362  * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
1363  */
1364 #ifdef CONFIG_SMP
scan_large_block(void * start,void * end)1365 static void scan_large_block(void *start, void *end)
1366 {
1367 	void *next;
1368 
1369 	while (start < end) {
1370 		next = min(start + MAX_SCAN_SIZE, end);
1371 		scan_block(start, next, NULL);
1372 		start = next;
1373 		cond_resched();
1374 	}
1375 }
1376 #endif
1377 
1378 /*
1379  * Scan a memory block corresponding to a kmemleak_object. A condition is
1380  * that object->use_count >= 1.
1381  */
scan_object(struct kmemleak_object * object)1382 static void scan_object(struct kmemleak_object *object)
1383 {
1384 	struct kmemleak_scan_area *area;
1385 	unsigned long flags;
1386 	void *obj_ptr;
1387 
1388 	/*
1389 	 * Once the object->lock is acquired, the corresponding memory block
1390 	 * cannot be freed (the same lock is acquired in delete_object).
1391 	 */
1392 	raw_spin_lock_irqsave(&object->lock, flags);
1393 	if (object->flags & OBJECT_NO_SCAN)
1394 		goto out;
1395 	if (!(object->flags & OBJECT_ALLOCATED))
1396 		/* already freed object */
1397 		goto out;
1398 
1399 	obj_ptr = object->flags & OBJECT_PHYS ?
1400 		  __va((phys_addr_t)object->pointer) :
1401 		  (void *)object->pointer;
1402 
1403 	if (hlist_empty(&object->area_list) ||
1404 	    object->flags & OBJECT_FULL_SCAN) {
1405 		void *start = obj_ptr;
1406 		void *end = obj_ptr + object->size;
1407 		void *next;
1408 
1409 		do {
1410 			next = min(start + MAX_SCAN_SIZE, end);
1411 			scan_block(start, next, object);
1412 
1413 			start = next;
1414 			if (start >= end)
1415 				break;
1416 
1417 			raw_spin_unlock_irqrestore(&object->lock, flags);
1418 			cond_resched();
1419 			raw_spin_lock_irqsave(&object->lock, flags);
1420 		} while (object->flags & OBJECT_ALLOCATED);
1421 	} else
1422 		hlist_for_each_entry(area, &object->area_list, node)
1423 			scan_block((void *)area->start,
1424 				   (void *)(area->start + area->size),
1425 				   object);
1426 out:
1427 	raw_spin_unlock_irqrestore(&object->lock, flags);
1428 }
1429 
1430 /*
1431  * Scan the objects already referenced (gray objects). More objects will be
1432  * referenced and, if there are no memory leaks, all the objects are scanned.
1433  */
scan_gray_list(void)1434 static void scan_gray_list(void)
1435 {
1436 	struct kmemleak_object *object, *tmp;
1437 
1438 	/*
1439 	 * The list traversal is safe for both tail additions and removals
1440 	 * from inside the loop. The kmemleak objects cannot be freed from
1441 	 * outside the loop because their use_count was incremented.
1442 	 */
1443 	object = list_entry(gray_list.next, typeof(*object), gray_list);
1444 	while (&object->gray_list != &gray_list) {
1445 		cond_resched();
1446 
1447 		/* may add new objects to the list */
1448 		if (!scan_should_stop())
1449 			scan_object(object);
1450 
1451 		tmp = list_entry(object->gray_list.next, typeof(*object),
1452 				 gray_list);
1453 
1454 		/* remove the object from the list and release it */
1455 		list_del(&object->gray_list);
1456 		put_object(object);
1457 
1458 		object = tmp;
1459 	}
1460 	WARN_ON(!list_empty(&gray_list));
1461 }
1462 
1463 /*
1464  * Conditionally call resched() in a object iteration loop while making sure
1465  * that the given object won't go away without RCU read lock by performing a
1466  * get_object() if !pinned.
1467  *
1468  * Return: false if can't do a cond_resched() due to get_object() failure
1469  *	   true otherwise
1470  */
kmemleak_cond_resched(struct kmemleak_object * object,bool pinned)1471 static bool kmemleak_cond_resched(struct kmemleak_object *object, bool pinned)
1472 {
1473 	if (!pinned && !get_object(object))
1474 		return false;
1475 
1476 	rcu_read_unlock();
1477 	cond_resched();
1478 	rcu_read_lock();
1479 	if (!pinned)
1480 		put_object(object);
1481 	return true;
1482 }
1483 
1484 /*
1485  * Scan data sections and all the referenced memory blocks allocated via the
1486  * kernel's standard allocators. This function must be called with the
1487  * scan_mutex held.
1488  */
kmemleak_scan(void)1489 static void kmemleak_scan(void)
1490 {
1491 	struct kmemleak_object *object;
1492 	struct zone *zone;
1493 	int __maybe_unused i;
1494 	int new_leaks = 0;
1495 	int loop_cnt = 0;
1496 
1497 	jiffies_last_scan = jiffies;
1498 
1499 	/* prepare the kmemleak_object's */
1500 	rcu_read_lock();
1501 	list_for_each_entry_rcu(object, &object_list, object_list) {
1502 		bool obj_pinned = false;
1503 
1504 		raw_spin_lock_irq(&object->lock);
1505 #ifdef DEBUG
1506 		/*
1507 		 * With a few exceptions there should be a maximum of
1508 		 * 1 reference to any object at this point.
1509 		 */
1510 		if (atomic_read(&object->use_count) > 1) {
1511 			pr_debug("object->use_count = %d\n",
1512 				 atomic_read(&object->use_count));
1513 			dump_object_info(object);
1514 		}
1515 #endif
1516 
1517 		/* ignore objects outside lowmem (paint them black) */
1518 		if ((object->flags & OBJECT_PHYS) &&
1519 		   !(object->flags & OBJECT_NO_SCAN)) {
1520 			unsigned long phys = object->pointer;
1521 
1522 			if (PHYS_PFN(phys) < min_low_pfn ||
1523 			    PHYS_PFN(phys + object->size) >= max_low_pfn)
1524 				__paint_it(object, KMEMLEAK_BLACK);
1525 		}
1526 
1527 		/* reset the reference count (whiten the object) */
1528 		object->count = 0;
1529 		if (color_gray(object) && get_object(object)) {
1530 			list_add_tail(&object->gray_list, &gray_list);
1531 			obj_pinned = true;
1532 		}
1533 
1534 		raw_spin_unlock_irq(&object->lock);
1535 
1536 		/*
1537 		 * Do a cond_resched() every 64k objects to avoid soft lockup.
1538 		 */
1539 		if (!(++loop_cnt & 0xffff) &&
1540 		    !kmemleak_cond_resched(object, obj_pinned))
1541 			loop_cnt--; /* Try again on next object */
1542 	}
1543 	rcu_read_unlock();
1544 
1545 #ifdef CONFIG_SMP
1546 	/* per-cpu sections scanning */
1547 	for_each_possible_cpu(i)
1548 		scan_large_block(__per_cpu_start + per_cpu_offset(i),
1549 				 __per_cpu_end + per_cpu_offset(i));
1550 #endif
1551 
1552 	/*
1553 	 * Struct page scanning for each node.
1554 	 */
1555 	get_online_mems();
1556 	for_each_populated_zone(zone) {
1557 		unsigned long start_pfn = zone->zone_start_pfn;
1558 		unsigned long end_pfn = zone_end_pfn(zone);
1559 		unsigned long pfn;
1560 
1561 		for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1562 			struct page *page = pfn_to_online_page(pfn);
1563 
1564 			if (!page)
1565 				continue;
1566 
1567 			/* only scan pages belonging to this zone */
1568 			if (page_zone(page) != zone)
1569 				continue;
1570 			/* only scan if page is in use */
1571 			if (page_count(page) == 0)
1572 				continue;
1573 			scan_block(page, page + 1, NULL);
1574 			if (!(pfn & 63))
1575 				cond_resched();
1576 		}
1577 	}
1578 	put_online_mems();
1579 
1580 	/*
1581 	 * Scanning the task stacks (may introduce false negatives).
1582 	 */
1583 	if (kmemleak_stack_scan) {
1584 		struct task_struct *p, *g;
1585 
1586 		rcu_read_lock();
1587 		for_each_process_thread(g, p) {
1588 			void *stack = try_get_task_stack(p);
1589 			if (stack) {
1590 				scan_block(stack, stack + THREAD_SIZE, NULL);
1591 				put_task_stack(p);
1592 			}
1593 		}
1594 		rcu_read_unlock();
1595 	}
1596 
1597 	/*
1598 	 * Scan the objects already referenced from the sections scanned
1599 	 * above.
1600 	 */
1601 	scan_gray_list();
1602 
1603 	/*
1604 	 * Check for new or unreferenced objects modified since the previous
1605 	 * scan and color them gray until the next scan.
1606 	 */
1607 	rcu_read_lock();
1608 	loop_cnt = 0;
1609 	list_for_each_entry_rcu(object, &object_list, object_list) {
1610 		/*
1611 		 * Do a cond_resched() every 64k objects to avoid soft lockup.
1612 		 */
1613 		if (!(++loop_cnt & 0xffff) &&
1614 		    !kmemleak_cond_resched(object, false))
1615 			loop_cnt--;	/* Try again on next object */
1616 
1617 		/*
1618 		 * This is racy but we can save the overhead of lock/unlock
1619 		 * calls. The missed objects, if any, should be caught in
1620 		 * the next scan.
1621 		 */
1622 		if (!color_white(object))
1623 			continue;
1624 		raw_spin_lock_irq(&object->lock);
1625 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1626 		    && update_checksum(object) && get_object(object)) {
1627 			/* color it gray temporarily */
1628 			object->count = object->min_count;
1629 			list_add_tail(&object->gray_list, &gray_list);
1630 		}
1631 		raw_spin_unlock_irq(&object->lock);
1632 	}
1633 	rcu_read_unlock();
1634 
1635 	/*
1636 	 * Re-scan the gray list for modified unreferenced objects.
1637 	 */
1638 	scan_gray_list();
1639 
1640 	/*
1641 	 * If scanning was stopped do not report any new unreferenced objects.
1642 	 */
1643 	if (scan_should_stop())
1644 		return;
1645 
1646 	/*
1647 	 * Scanning result reporting.
1648 	 */
1649 	rcu_read_lock();
1650 	loop_cnt = 0;
1651 	list_for_each_entry_rcu(object, &object_list, object_list) {
1652 		/*
1653 		 * Do a cond_resched() every 64k objects to avoid soft lockup.
1654 		 */
1655 		if (!(++loop_cnt & 0xffff) &&
1656 		    !kmemleak_cond_resched(object, false))
1657 			loop_cnt--;	/* Try again on next object */
1658 
1659 		/*
1660 		 * This is racy but we can save the overhead of lock/unlock
1661 		 * calls. The missed objects, if any, should be caught in
1662 		 * the next scan.
1663 		 */
1664 		if (!color_white(object))
1665 			continue;
1666 		raw_spin_lock_irq(&object->lock);
1667 		if (unreferenced_object(object) &&
1668 		    !(object->flags & OBJECT_REPORTED)) {
1669 			object->flags |= OBJECT_REPORTED;
1670 
1671 			if (kmemleak_verbose)
1672 				print_unreferenced(NULL, object);
1673 
1674 			new_leaks++;
1675 		}
1676 		raw_spin_unlock_irq(&object->lock);
1677 	}
1678 	rcu_read_unlock();
1679 
1680 	if (new_leaks) {
1681 		kmemleak_found_leaks = true;
1682 
1683 		pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
1684 			new_leaks);
1685 	}
1686 
1687 }
1688 
1689 /*
1690  * Thread function performing automatic memory scanning. Unreferenced objects
1691  * at the end of a memory scan are reported but only the first time.
1692  */
kmemleak_scan_thread(void * arg)1693 static int kmemleak_scan_thread(void *arg)
1694 {
1695 	static int first_run = IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN);
1696 
1697 	pr_info("Automatic memory scanning thread started\n");
1698 	set_user_nice(current, 10);
1699 
1700 	/*
1701 	 * Wait before the first scan to allow the system to fully initialize.
1702 	 */
1703 	if (first_run) {
1704 		signed long timeout = msecs_to_jiffies(SECS_FIRST_SCAN * 1000);
1705 		first_run = 0;
1706 		while (timeout && !kthread_should_stop())
1707 			timeout = schedule_timeout_interruptible(timeout);
1708 	}
1709 
1710 	while (!kthread_should_stop()) {
1711 		signed long timeout = READ_ONCE(jiffies_scan_wait);
1712 
1713 		mutex_lock(&scan_mutex);
1714 		kmemleak_scan();
1715 		mutex_unlock(&scan_mutex);
1716 
1717 		/* wait before the next scan */
1718 		while (timeout && !kthread_should_stop())
1719 			timeout = schedule_timeout_interruptible(timeout);
1720 	}
1721 
1722 	pr_info("Automatic memory scanning thread ended\n");
1723 
1724 	return 0;
1725 }
1726 
1727 /*
1728  * Start the automatic memory scanning thread. This function must be called
1729  * with the scan_mutex held.
1730  */
start_scan_thread(void)1731 static void start_scan_thread(void)
1732 {
1733 	if (scan_thread)
1734 		return;
1735 	scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
1736 	if (IS_ERR(scan_thread)) {
1737 		pr_warn("Failed to create the scan thread\n");
1738 		scan_thread = NULL;
1739 	}
1740 }
1741 
1742 /*
1743  * Stop the automatic memory scanning thread.
1744  */
stop_scan_thread(void)1745 static void stop_scan_thread(void)
1746 {
1747 	if (scan_thread) {
1748 		kthread_stop(scan_thread);
1749 		scan_thread = NULL;
1750 	}
1751 }
1752 
1753 /*
1754  * Iterate over the object_list and return the first valid object at or after
1755  * the required position with its use_count incremented. The function triggers
1756  * a memory scanning when the pos argument points to the first position.
1757  */
kmemleak_seq_start(struct seq_file * seq,loff_t * pos)1758 static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
1759 {
1760 	struct kmemleak_object *object;
1761 	loff_t n = *pos;
1762 	int err;
1763 
1764 	err = mutex_lock_interruptible(&scan_mutex);
1765 	if (err < 0)
1766 		return ERR_PTR(err);
1767 
1768 	rcu_read_lock();
1769 	list_for_each_entry_rcu(object, &object_list, object_list) {
1770 		if (n-- > 0)
1771 			continue;
1772 		if (get_object(object))
1773 			goto out;
1774 	}
1775 	object = NULL;
1776 out:
1777 	return object;
1778 }
1779 
1780 /*
1781  * Return the next object in the object_list. The function decrements the
1782  * use_count of the previous object and increases that of the next one.
1783  */
kmemleak_seq_next(struct seq_file * seq,void * v,loff_t * pos)1784 static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1785 {
1786 	struct kmemleak_object *prev_obj = v;
1787 	struct kmemleak_object *next_obj = NULL;
1788 	struct kmemleak_object *obj = prev_obj;
1789 
1790 	++(*pos);
1791 
1792 	list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
1793 		if (get_object(obj)) {
1794 			next_obj = obj;
1795 			break;
1796 		}
1797 	}
1798 
1799 	put_object(prev_obj);
1800 	return next_obj;
1801 }
1802 
1803 /*
1804  * Decrement the use_count of the last object required, if any.
1805  */
kmemleak_seq_stop(struct seq_file * seq,void * v)1806 static void kmemleak_seq_stop(struct seq_file *seq, void *v)
1807 {
1808 	if (!IS_ERR(v)) {
1809 		/*
1810 		 * kmemleak_seq_start may return ERR_PTR if the scan_mutex
1811 		 * waiting was interrupted, so only release it if !IS_ERR.
1812 		 */
1813 		rcu_read_unlock();
1814 		mutex_unlock(&scan_mutex);
1815 		if (v)
1816 			put_object(v);
1817 	}
1818 }
1819 
1820 /*
1821  * Print the information for an unreferenced object to the seq file.
1822  */
kmemleak_seq_show(struct seq_file * seq,void * v)1823 static int kmemleak_seq_show(struct seq_file *seq, void *v)
1824 {
1825 	struct kmemleak_object *object = v;
1826 	unsigned long flags;
1827 
1828 	raw_spin_lock_irqsave(&object->lock, flags);
1829 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
1830 		print_unreferenced(seq, object);
1831 	raw_spin_unlock_irqrestore(&object->lock, flags);
1832 	return 0;
1833 }
1834 
1835 static const struct seq_operations kmemleak_seq_ops = {
1836 	.start = kmemleak_seq_start,
1837 	.next  = kmemleak_seq_next,
1838 	.stop  = kmemleak_seq_stop,
1839 	.show  = kmemleak_seq_show,
1840 };
1841 
kmemleak_open(struct inode * inode,struct file * file)1842 static int kmemleak_open(struct inode *inode, struct file *file)
1843 {
1844 	return seq_open(file, &kmemleak_seq_ops);
1845 }
1846 
dump_str_object_info(const char * str)1847 static int dump_str_object_info(const char *str)
1848 {
1849 	unsigned long flags;
1850 	struct kmemleak_object *object;
1851 	unsigned long addr;
1852 
1853 	if (kstrtoul(str, 0, &addr))
1854 		return -EINVAL;
1855 	object = find_and_get_object(addr, 0);
1856 	if (!object) {
1857 		pr_info("Unknown object at 0x%08lx\n", addr);
1858 		return -EINVAL;
1859 	}
1860 
1861 	raw_spin_lock_irqsave(&object->lock, flags);
1862 	dump_object_info(object);
1863 	raw_spin_unlock_irqrestore(&object->lock, flags);
1864 
1865 	put_object(object);
1866 	return 0;
1867 }
1868 
1869 /*
1870  * We use grey instead of black to ensure we can do future scans on the same
1871  * objects. If we did not do future scans these black objects could
1872  * potentially contain references to newly allocated objects in the future and
1873  * we'd end up with false positives.
1874  */
kmemleak_clear(void)1875 static void kmemleak_clear(void)
1876 {
1877 	struct kmemleak_object *object;
1878 
1879 	rcu_read_lock();
1880 	list_for_each_entry_rcu(object, &object_list, object_list) {
1881 		raw_spin_lock_irq(&object->lock);
1882 		if ((object->flags & OBJECT_REPORTED) &&
1883 		    unreferenced_object(object))
1884 			__paint_it(object, KMEMLEAK_GREY);
1885 		raw_spin_unlock_irq(&object->lock);
1886 	}
1887 	rcu_read_unlock();
1888 
1889 	kmemleak_found_leaks = false;
1890 }
1891 
1892 static void __kmemleak_do_cleanup(void);
1893 
1894 /*
1895  * File write operation to configure kmemleak at run-time. The following
1896  * commands can be written to the /sys/kernel/debug/kmemleak file:
1897  *   off	- disable kmemleak (irreversible)
1898  *   stack=on	- enable the task stacks scanning
1899  *   stack=off	- disable the tasks stacks scanning
1900  *   scan=on	- start the automatic memory scanning thread
1901  *   scan=off	- stop the automatic memory scanning thread
1902  *   scan=...	- set the automatic memory scanning period in seconds (0 to
1903  *		  disable it)
1904  *   scan	- trigger a memory scan
1905  *   clear	- mark all current reported unreferenced kmemleak objects as
1906  *		  grey to ignore printing them, or free all kmemleak objects
1907  *		  if kmemleak has been disabled.
1908  *   dump=...	- dump information about the object found at the given address
1909  */
kmemleak_write(struct file * file,const char __user * user_buf,size_t size,loff_t * ppos)1910 static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
1911 			      size_t size, loff_t *ppos)
1912 {
1913 	char buf[64];
1914 	int buf_size;
1915 	int ret;
1916 
1917 	buf_size = min(size, (sizeof(buf) - 1));
1918 	if (strncpy_from_user(buf, user_buf, buf_size) < 0)
1919 		return -EFAULT;
1920 	buf[buf_size] = 0;
1921 
1922 	ret = mutex_lock_interruptible(&scan_mutex);
1923 	if (ret < 0)
1924 		return ret;
1925 
1926 	if (strncmp(buf, "clear", 5) == 0) {
1927 		if (kmemleak_enabled)
1928 			kmemleak_clear();
1929 		else
1930 			__kmemleak_do_cleanup();
1931 		goto out;
1932 	}
1933 
1934 	if (!kmemleak_enabled) {
1935 		ret = -EPERM;
1936 		goto out;
1937 	}
1938 
1939 	if (strncmp(buf, "off", 3) == 0)
1940 		kmemleak_disable();
1941 	else if (strncmp(buf, "stack=on", 8) == 0)
1942 		kmemleak_stack_scan = 1;
1943 	else if (strncmp(buf, "stack=off", 9) == 0)
1944 		kmemleak_stack_scan = 0;
1945 	else if (strncmp(buf, "scan=on", 7) == 0)
1946 		start_scan_thread();
1947 	else if (strncmp(buf, "scan=off", 8) == 0)
1948 		stop_scan_thread();
1949 	else if (strncmp(buf, "scan=", 5) == 0) {
1950 		unsigned secs;
1951 		unsigned long msecs;
1952 
1953 		ret = kstrtouint(buf + 5, 0, &secs);
1954 		if (ret < 0)
1955 			goto out;
1956 
1957 		msecs = secs * MSEC_PER_SEC;
1958 		if (msecs > UINT_MAX)
1959 			msecs = UINT_MAX;
1960 
1961 		stop_scan_thread();
1962 		if (msecs) {
1963 			WRITE_ONCE(jiffies_scan_wait, msecs_to_jiffies(msecs));
1964 			start_scan_thread();
1965 		}
1966 	} else if (strncmp(buf, "scan", 4) == 0)
1967 		kmemleak_scan();
1968 	else if (strncmp(buf, "dump=", 5) == 0)
1969 		ret = dump_str_object_info(buf + 5);
1970 	else
1971 		ret = -EINVAL;
1972 
1973 out:
1974 	mutex_unlock(&scan_mutex);
1975 	if (ret < 0)
1976 		return ret;
1977 
1978 	/* ignore the rest of the buffer, only one command at a time */
1979 	*ppos += size;
1980 	return size;
1981 }
1982 
1983 static const struct file_operations kmemleak_fops = {
1984 	.owner		= THIS_MODULE,
1985 	.open		= kmemleak_open,
1986 	.read		= seq_read,
1987 	.write		= kmemleak_write,
1988 	.llseek		= seq_lseek,
1989 	.release	= seq_release,
1990 };
1991 
__kmemleak_do_cleanup(void)1992 static void __kmemleak_do_cleanup(void)
1993 {
1994 	struct kmemleak_object *object, *tmp;
1995 
1996 	/*
1997 	 * Kmemleak has already been disabled, no need for RCU list traversal
1998 	 * or kmemleak_lock held.
1999 	 */
2000 	list_for_each_entry_safe(object, tmp, &object_list, object_list) {
2001 		__remove_object(object);
2002 		__delete_object(object);
2003 	}
2004 }
2005 
2006 /*
2007  * Stop the memory scanning thread and free the kmemleak internal objects if
2008  * no previous scan thread (otherwise, kmemleak may still have some useful
2009  * information on memory leaks).
2010  */
kmemleak_do_cleanup(struct work_struct * work)2011 static void kmemleak_do_cleanup(struct work_struct *work)
2012 {
2013 	stop_scan_thread();
2014 
2015 	mutex_lock(&scan_mutex);
2016 	/*
2017 	 * Once it is made sure that kmemleak_scan has stopped, it is safe to no
2018 	 * longer track object freeing. Ordering of the scan thread stopping and
2019 	 * the memory accesses below is guaranteed by the kthread_stop()
2020 	 * function.
2021 	 */
2022 	kmemleak_free_enabled = 0;
2023 	mutex_unlock(&scan_mutex);
2024 
2025 	if (!kmemleak_found_leaks)
2026 		__kmemleak_do_cleanup();
2027 	else
2028 		pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
2029 }
2030 
2031 static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
2032 
2033 /*
2034  * Disable kmemleak. No memory allocation/freeing will be traced once this
2035  * function is called. Disabling kmemleak is an irreversible operation.
2036  */
kmemleak_disable(void)2037 static void kmemleak_disable(void)
2038 {
2039 	/* atomically check whether it was already invoked */
2040 	if (cmpxchg(&kmemleak_error, 0, 1))
2041 		return;
2042 
2043 	/* stop any memory operation tracing */
2044 	kmemleak_enabled = 0;
2045 
2046 	/* check whether it is too early for a kernel thread */
2047 	if (kmemleak_initialized)
2048 		schedule_work(&cleanup_work);
2049 	else
2050 		kmemleak_free_enabled = 0;
2051 
2052 	pr_info("Kernel memory leak detector disabled\n");
2053 }
2054 
2055 /*
2056  * Allow boot-time kmemleak disabling (enabled by default).
2057  */
kmemleak_boot_config(char * str)2058 static int __init kmemleak_boot_config(char *str)
2059 {
2060 	if (!str)
2061 		return -EINVAL;
2062 	if (strcmp(str, "off") == 0)
2063 		kmemleak_disable();
2064 	else if (strcmp(str, "on") == 0)
2065 		kmemleak_skip_disable = 1;
2066 	else
2067 		return -EINVAL;
2068 	return 0;
2069 }
2070 early_param("kmemleak", kmemleak_boot_config);
2071 
2072 /*
2073  * Kmemleak initialization.
2074  */
kmemleak_init(void)2075 void __init kmemleak_init(void)
2076 {
2077 #ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
2078 	if (!kmemleak_skip_disable) {
2079 		kmemleak_disable();
2080 		return;
2081 	}
2082 #endif
2083 
2084 	if (kmemleak_error)
2085 		return;
2086 
2087 	jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
2088 	jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
2089 
2090 	object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
2091 	scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
2092 
2093 	/* register the data/bss sections */
2094 	create_object((unsigned long)_sdata, _edata - _sdata,
2095 		      KMEMLEAK_GREY, GFP_ATOMIC);
2096 	create_object((unsigned long)__bss_start, __bss_stop - __bss_start,
2097 		      KMEMLEAK_GREY, GFP_ATOMIC);
2098 	/* only register .data..ro_after_init if not within .data */
2099 	if (&__start_ro_after_init < &_sdata || &__end_ro_after_init > &_edata)
2100 		create_object((unsigned long)__start_ro_after_init,
2101 			      __end_ro_after_init - __start_ro_after_init,
2102 			      KMEMLEAK_GREY, GFP_ATOMIC);
2103 }
2104 
2105 /*
2106  * Late initialization function.
2107  */
kmemleak_late_init(void)2108 static int __init kmemleak_late_init(void)
2109 {
2110 	kmemleak_initialized = 1;
2111 
2112 	debugfs_create_file("kmemleak", 0644, NULL, NULL, &kmemleak_fops);
2113 
2114 	if (kmemleak_error) {
2115 		/*
2116 		 * Some error occurred and kmemleak was disabled. There is a
2117 		 * small chance that kmemleak_disable() was called immediately
2118 		 * after setting kmemleak_initialized and we may end up with
2119 		 * two clean-up threads but serialized by scan_mutex.
2120 		 */
2121 		schedule_work(&cleanup_work);
2122 		return -ENOMEM;
2123 	}
2124 
2125 	if (IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN)) {
2126 		mutex_lock(&scan_mutex);
2127 		start_scan_thread();
2128 		mutex_unlock(&scan_mutex);
2129 	}
2130 
2131 	pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n",
2132 		mem_pool_free_count);
2133 
2134 	return 0;
2135 }
2136 late_initcall(kmemleak_late_init);
2137