Lines Matching refs:object
296 struct kmemleak_object *object) in hex_dump_object() argument
298 const u8 *ptr = (const u8 *)object->pointer; in hex_dump_object()
301 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in hex_dump_object()
305 len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE); in hex_dump_object()
324 static bool color_white(const struct kmemleak_object *object) in color_white() argument
326 return object->count != KMEMLEAK_BLACK && in color_white()
327 object->count < object->min_count; in color_white()
330 static bool color_gray(const struct kmemleak_object *object) in color_gray() argument
332 return object->min_count != KMEMLEAK_BLACK && in color_gray()
333 object->count >= object->min_count; in color_gray()
341 static bool unreferenced_object(struct kmemleak_object *object) in unreferenced_object() argument
343 return (color_white(object) && object->flags & OBJECT_ALLOCATED) && in unreferenced_object()
344 time_before_eq(object->jiffies + jiffies_min_age, in unreferenced_object()
353 struct kmemleak_object *object) in print_unreferenced() argument
358 unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies); in print_unreferenced()
360 nr_entries = stack_depot_fetch(object->trace_handle, &entries); in print_unreferenced()
362 object->pointer, object->size); in print_unreferenced()
364 object->comm, object->pid, object->jiffies, in print_unreferenced()
366 hex_dump_object(seq, object); in print_unreferenced()
380 static void dump_object_info(struct kmemleak_object *object) in dump_object_info() argument
383 object->pointer, object->size); in dump_object_info()
385 object->comm, object->pid, object->jiffies); in dump_object_info()
386 pr_notice(" min_count = %d\n", object->min_count); in dump_object_info()
387 pr_notice(" count = %d\n", object->count); in dump_object_info()
388 pr_notice(" flags = 0x%x\n", object->flags); in dump_object_info()
389 pr_notice(" checksum = %u\n", object->checksum); in dump_object_info()
391 if (object->trace_handle) in dump_object_info()
392 stack_depot_print(object->trace_handle); in dump_object_info()
409 struct kmemleak_object *object; in __lookup_object() local
412 object = rb_entry(rb, struct kmemleak_object, rb_node); in __lookup_object()
413 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in __lookup_object()
416 rb = object->rb_node.rb_left; in __lookup_object()
417 else if (untagged_objp + object->size <= untagged_ptr) in __lookup_object()
418 rb = object->rb_node.rb_right; in __lookup_object()
420 return object; in __lookup_object()
424 dump_object_info(object); in __lookup_object()
443 static int get_object(struct kmemleak_object *object) in get_object() argument
445 return atomic_inc_not_zero(&object->use_count); in get_object()
454 struct kmemleak_object *object; in mem_pool_alloc() local
458 object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp)); in mem_pool_alloc()
459 if (object) in mem_pool_alloc()
460 return object; in mem_pool_alloc()
465 object = list_first_entry_or_null(&mem_pool_free_list, in mem_pool_alloc()
466 typeof(*object), object_list); in mem_pool_alloc()
467 if (object) in mem_pool_alloc()
468 list_del(&object->object_list); in mem_pool_alloc()
470 object = &mem_pool[--mem_pool_free_count]; in mem_pool_alloc()
475 return object; in mem_pool_alloc()
481 static void mem_pool_free(struct kmemleak_object *object) in mem_pool_free() argument
485 if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) { in mem_pool_free()
486 kmem_cache_free(object_cache, object); in mem_pool_free()
492 list_add(&object->object_list, &mem_pool_free_list); in mem_pool_free()
503 struct kmemleak_object *object = in free_object_rcu() local
510 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) { in free_object_rcu()
514 mem_pool_free(object); in free_object_rcu()
524 static void put_object(struct kmemleak_object *object) in put_object() argument
526 if (!atomic_dec_and_test(&object->use_count)) in put_object()
530 WARN_ON(object->flags & OBJECT_ALLOCATED); in put_object()
538 call_rcu(&object->rcu, free_object_rcu); in put_object()
540 free_object_rcu(&object->rcu); in put_object()
550 struct kmemleak_object *object; in __find_and_get_object() local
554 object = __lookup_object(ptr, alias, is_phys); in __find_and_get_object()
558 if (object && !get_object(object)) in __find_and_get_object()
559 object = NULL; in __find_and_get_object()
562 return object; in __find_and_get_object()
576 static void __remove_object(struct kmemleak_object *object) in __remove_object() argument
578 rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ? in __remove_object()
581 if (!(object->del_state & DELSTATE_NO_DELETE)) in __remove_object()
582 list_del_rcu(&object->object_list); in __remove_object()
583 object->del_state |= DELSTATE_REMOVED; in __remove_object()
596 struct kmemleak_object *object; in find_and_remove_object() local
599 object = __lookup_object(ptr, alias, is_phys); in find_and_remove_object()
600 if (object) in find_and_remove_object()
601 __remove_object(object); in find_and_remove_object()
604 return object; in find_and_remove_object()
635 struct kmemleak_object *object, *parent; in __create_object() local
640 object = mem_pool_alloc(gfp); in __create_object()
641 if (!object) { in __create_object()
647 INIT_LIST_HEAD(&object->object_list); in __create_object()
648 INIT_LIST_HEAD(&object->gray_list); in __create_object()
649 INIT_HLIST_HEAD(&object->area_list); in __create_object()
650 raw_spin_lock_init(&object->lock); in __create_object()
651 atomic_set(&object->use_count, 1); in __create_object()
652 object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0); in __create_object()
653 object->pointer = ptr; in __create_object()
654 object->size = kfence_ksize((void *)ptr) ?: size; in __create_object()
655 object->excess_ref = 0; in __create_object()
656 object->min_count = min_count; in __create_object()
657 object->count = 0; /* white color initially */ in __create_object()
658 object->jiffies = jiffies; in __create_object()
659 object->checksum = 0; in __create_object()
660 object->del_state = 0; in __create_object()
664 object->pid = 0; in __create_object()
665 strncpy(object->comm, "hardirq", sizeof(object->comm)); in __create_object()
667 object->pid = 0; in __create_object()
668 strncpy(object->comm, "softirq", sizeof(object->comm)); in __create_object()
670 object->pid = current->pid; in __create_object()
677 strncpy(object->comm, current->comm, sizeof(object->comm)); in __create_object()
681 object->trace_handle = set_track_prepare(); in __create_object()
713 kmem_cache_free(object_cache, object); in __create_object()
717 rb_link_node(&object->rb_node, rb_parent, link); in __create_object()
718 rb_insert_color(&object->rb_node, is_phys ? &object_phys_tree_root : in __create_object()
720 list_add_tail_rcu(&object->object_list, &object_list); in __create_object()
742 static void __delete_object(struct kmemleak_object *object) in __delete_object() argument
746 WARN_ON(!(object->flags & OBJECT_ALLOCATED)); in __delete_object()
747 WARN_ON(atomic_read(&object->use_count) < 1); in __delete_object()
753 raw_spin_lock_irqsave(&object->lock, flags); in __delete_object()
754 object->flags &= ~OBJECT_ALLOCATED; in __delete_object()
755 raw_spin_unlock_irqrestore(&object->lock, flags); in __delete_object()
756 put_object(object); in __delete_object()
765 struct kmemleak_object *object; in delete_object_full() local
767 object = find_and_remove_object(ptr, 0, false); in delete_object_full()
768 if (!object) { in delete_object_full()
775 __delete_object(object); in delete_object_full()
785 struct kmemleak_object *object; in delete_object_part() local
788 object = find_and_remove_object(ptr, 1, is_phys); in delete_object_part()
789 if (!object) { in delete_object_part()
802 start = object->pointer; in delete_object_part()
803 end = object->pointer + object->size; in delete_object_part()
805 __create_object(start, ptr - start, object->min_count, in delete_object_part()
808 __create_object(ptr + size, end - ptr - size, object->min_count, in delete_object_part()
811 __delete_object(object); in delete_object_part()
814 static void __paint_it(struct kmemleak_object *object, int color) in __paint_it() argument
816 object->min_count = color; in __paint_it()
818 object->flags |= OBJECT_NO_SCAN; in __paint_it()
821 static void paint_it(struct kmemleak_object *object, int color) in paint_it() argument
825 raw_spin_lock_irqsave(&object->lock, flags); in paint_it()
826 __paint_it(object, color); in paint_it()
827 raw_spin_unlock_irqrestore(&object->lock, flags); in paint_it()
832 struct kmemleak_object *object; in paint_ptr() local
834 object = __find_and_get_object(ptr, 0, is_phys); in paint_ptr()
835 if (!object) { in paint_ptr()
842 paint_it(object, color); in paint_ptr()
843 put_object(object); in paint_ptr()
871 struct kmemleak_object *object; in add_scan_area() local
876 object = find_and_get_object(ptr, 1); in add_scan_area()
877 if (!object) { in add_scan_area()
884 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in add_scan_area()
889 raw_spin_lock_irqsave(&object->lock, flags); in add_scan_area()
893 object->flags |= OBJECT_FULL_SCAN; in add_scan_area()
897 size = untagged_objp + object->size - untagged_ptr; in add_scan_area()
898 } else if (untagged_ptr + size > untagged_objp + object->size) { in add_scan_area()
900 dump_object_info(object); in add_scan_area()
909 hlist_add_head(&area->node, &object->area_list); in add_scan_area()
911 raw_spin_unlock_irqrestore(&object->lock, flags); in add_scan_area()
912 put_object(object); in add_scan_area()
924 struct kmemleak_object *object; in object_set_excess_ref() local
926 object = find_and_get_object(ptr, 0); in object_set_excess_ref()
927 if (!object) { in object_set_excess_ref()
933 raw_spin_lock_irqsave(&object->lock, flags); in object_set_excess_ref()
934 object->excess_ref = excess_ref; in object_set_excess_ref()
935 raw_spin_unlock_irqrestore(&object->lock, flags); in object_set_excess_ref()
936 put_object(object); in object_set_excess_ref()
947 struct kmemleak_object *object; in object_no_scan() local
949 object = find_and_get_object(ptr, 0); in object_no_scan()
950 if (!object) { in object_no_scan()
955 raw_spin_lock_irqsave(&object->lock, flags); in object_no_scan()
956 object->flags |= OBJECT_NO_SCAN; in object_no_scan()
957 raw_spin_unlock_irqrestore(&object->lock, flags); in object_no_scan()
958 put_object(object); in object_no_scan()
1100 struct kmemleak_object *object; in kmemleak_update_trace() local
1108 object = find_and_get_object((unsigned long)ptr, 1); in kmemleak_update_trace()
1109 if (!object) { in kmemleak_update_trace()
1117 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_update_trace()
1118 object->trace_handle = set_track_prepare(); in kmemleak_update_trace()
1119 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_update_trace()
1121 put_object(object); in kmemleak_update_trace()
1250 static bool update_checksum(struct kmemleak_object *object) in update_checksum() argument
1252 u32 old_csum = object->checksum; in update_checksum()
1254 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in update_checksum()
1259 object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size); in update_checksum()
1263 return object->checksum != old_csum; in update_checksum()
1269 static void update_refs(struct kmemleak_object *object) in update_refs() argument
1271 if (!color_white(object)) { in update_refs()
1282 object->count++; in update_refs()
1283 if (color_gray(object)) { in update_refs()
1285 WARN_ON(!get_object(object)); in update_refs()
1286 list_add_tail(&object->gray_list, &gray_list); in update_refs()
1326 struct kmemleak_object *object; in scan_block() local
1347 object = lookup_object(pointer, 1); in scan_block()
1348 if (!object) in scan_block()
1350 if (object == scanned) in scan_block()
1359 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in scan_block()
1361 if (color_gray(object)) { in scan_block()
1362 excess_ref = object->excess_ref; in scan_block()
1366 update_refs(object); in scan_block()
1368 raw_spin_unlock(&object->lock); in scan_block()
1371 object = lookup_object(excess_ref, 0); in scan_block()
1372 if (!object) in scan_block()
1374 if (object == scanned) in scan_block()
1377 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in scan_block()
1378 update_refs(object); in scan_block()
1379 raw_spin_unlock(&object->lock); in scan_block()
1406 static void scan_object(struct kmemleak_object *object) in scan_object() argument
1416 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1417 if (object->flags & OBJECT_NO_SCAN) in scan_object()
1419 if (!(object->flags & OBJECT_ALLOCATED)) in scan_object()
1423 obj_ptr = object->flags & OBJECT_PHYS ? in scan_object()
1424 __va((phys_addr_t)object->pointer) : in scan_object()
1425 (void *)object->pointer; in scan_object()
1427 if (hlist_empty(&object->area_list) || in scan_object()
1428 object->flags & OBJECT_FULL_SCAN) { in scan_object()
1430 void *end = obj_ptr + object->size; in scan_object()
1435 scan_block(start, next, object); in scan_object()
1441 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1443 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1444 } while (object->flags & OBJECT_ALLOCATED); in scan_object()
1446 hlist_for_each_entry(area, &object->area_list, node) in scan_object()
1449 object); in scan_object()
1451 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1460 struct kmemleak_object *object, *tmp; in scan_gray_list() local
1467 object = list_entry(gray_list.next, typeof(*object), gray_list); in scan_gray_list()
1468 while (&object->gray_list != &gray_list) { in scan_gray_list()
1473 scan_object(object); in scan_gray_list()
1475 tmp = list_entry(object->gray_list.next, typeof(*object), in scan_gray_list()
1479 list_del(&object->gray_list); in scan_gray_list()
1480 put_object(object); in scan_gray_list()
1482 object = tmp; in scan_gray_list()
1492 static void kmemleak_cond_resched(struct kmemleak_object *object) in kmemleak_cond_resched() argument
1494 if (!get_object(object)) in kmemleak_cond_resched()
1498 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1500 object->del_state |= DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1508 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1509 list_del_rcu(&object->object_list); in kmemleak_cond_resched()
1510 object->del_state &= ~DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1513 put_object(object); in kmemleak_cond_resched()
1523 struct kmemleak_object *object; in kmemleak_scan() local
1532 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_scan()
1533 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1539 if (atomic_read(&object->use_count) > 1) { in kmemleak_scan()
1541 atomic_read(&object->use_count)); in kmemleak_scan()
1542 dump_object_info(object); in kmemleak_scan()
1547 if ((object->flags & OBJECT_PHYS) && in kmemleak_scan()
1548 !(object->flags & OBJECT_NO_SCAN)) { in kmemleak_scan()
1549 unsigned long phys = object->pointer; in kmemleak_scan()
1552 PHYS_PFN(phys + object->size) >= max_low_pfn) in kmemleak_scan()
1553 __paint_it(object, KMEMLEAK_BLACK); in kmemleak_scan()
1557 object->count = 0; in kmemleak_scan()
1558 if (color_gray(object) && get_object(object)) in kmemleak_scan()
1559 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1561 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1564 kmemleak_cond_resched(object); in kmemleak_scan()
1632 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_scan()
1634 kmemleak_cond_resched(object); in kmemleak_scan()
1641 if (!color_white(object)) in kmemleak_scan()
1643 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1644 if (color_white(object) && (object->flags & OBJECT_ALLOCATED) in kmemleak_scan()
1645 && update_checksum(object) && get_object(object)) { in kmemleak_scan()
1647 object->count = object->min_count; in kmemleak_scan()
1648 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1650 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1669 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_scan()
1671 kmemleak_cond_resched(object); in kmemleak_scan()
1678 if (!color_white(object)) in kmemleak_scan()
1680 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1681 if (unreferenced_object(object) && in kmemleak_scan()
1682 !(object->flags & OBJECT_REPORTED)) { in kmemleak_scan()
1683 object->flags |= OBJECT_REPORTED; in kmemleak_scan()
1686 print_unreferenced(NULL, object); in kmemleak_scan()
1690 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1774 struct kmemleak_object *object; in kmemleak_seq_start() local
1783 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_seq_start()
1786 if (get_object(object)) in kmemleak_seq_start()
1789 object = NULL; in kmemleak_seq_start()
1791 return object; in kmemleak_seq_start()
1839 struct kmemleak_object *object = v; in kmemleak_seq_show() local
1842 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_seq_show()
1843 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object)) in kmemleak_seq_show()
1844 print_unreferenced(seq, object); in kmemleak_seq_show()
1845 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_seq_show()
1864 struct kmemleak_object *object; in dump_str_object_info() local
1869 object = find_and_get_object(addr, 0); in dump_str_object_info()
1870 if (!object) { in dump_str_object_info()
1875 raw_spin_lock_irqsave(&object->lock, flags); in dump_str_object_info()
1876 dump_object_info(object); in dump_str_object_info()
1877 raw_spin_unlock_irqrestore(&object->lock, flags); in dump_str_object_info()
1879 put_object(object); in dump_str_object_info()
1891 struct kmemleak_object *object; in kmemleak_clear() local
1894 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_clear()
1895 raw_spin_lock_irq(&object->lock); in kmemleak_clear()
1896 if ((object->flags & OBJECT_REPORTED) && in kmemleak_clear()
1897 unreferenced_object(object)) in kmemleak_clear()
1898 __paint_it(object, KMEMLEAK_GREY); in kmemleak_clear()
1899 raw_spin_unlock_irq(&object->lock); in kmemleak_clear()
2008 struct kmemleak_object *object, *tmp; in __kmemleak_do_cleanup() local
2014 list_for_each_entry_safe(object, tmp, &object_list, object_list) { in __kmemleak_do_cleanup()
2015 __remove_object(object); in __kmemleak_do_cleanup()
2016 __delete_object(object); in __kmemleak_do_cleanup()