289 			    struct kmemleak_object *object)  in hex_dump_object()  argument
291 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
294 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))  in hex_dump_object()
298 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);  in hex_dump_object()
317 static bool color_white(const struct kmemleak_object *object)  in color_white()  argument
319 	return object->count != KMEMLEAK_BLACK &&  in color_white()
320 		object->count < object->min_count;  in color_white()
323 static bool color_gray(const struct kmemleak_object *object)  in color_gray()  argument
325 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
326 		object->count >= object->min_count;  in color_gray()
334 static bool unreferenced_object(struct kmemleak_object *object)  in unreferenced_object()  argument
336 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
337 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
346 			       struct kmemleak_object *object)  in print_unreferenced()  argument
349 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);  in print_unreferenced()
352 		   object->pointer, object->size);  in print_unreferenced()
354 		   object->comm, object->pid, object->jiffies,  in print_unreferenced()
356 	hex_dump_object(seq, object);  in print_unreferenced()
359 	for (i = 0; i < object->trace_len; i++) {  in print_unreferenced()
360 		void *ptr = (void *)object->trace[i];  in print_unreferenced()
370 static void dump_object_info(struct kmemleak_object *object)  in dump_object_info()  argument
373 		  object->pointer, object->size);  in dump_object_info()
375 		  object->comm, object->pid, object->jiffies);  in dump_object_info()
376 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
377 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
378 	pr_notice("  flags = 0x%x\n", object->flags);  in dump_object_info()
379 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
381 	stack_trace_print(object->trace, object->trace_len, 4);  in dump_object_info()
398 		struct kmemleak_object *object;  in __lookup_object()  local
401 		object = rb_entry(rb, struct kmemleak_object, rb_node);  in __lookup_object()
402 		untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in __lookup_object()
405 			rb = object->rb_node.rb_left;  in __lookup_object()
406 		else if (untagged_objp + object->size <= untagged_ptr)  in __lookup_object()
407 			rb = object->rb_node.rb_right;  in __lookup_object()
409 			return object;  in __lookup_object()
413 			dump_object_info(object);  in __lookup_object()
432 static int get_object(struct kmemleak_object *object)  in get_object()  argument
434 	return atomic_inc_not_zero(&object->use_count);  in get_object()
443 	struct kmemleak_object *object;  in mem_pool_alloc()  local
447 		object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));  in mem_pool_alloc()
448 		if (object)  in mem_pool_alloc()
449 			return object;  in mem_pool_alloc()
454 	object = list_first_entry_or_null(&mem_pool_free_list,  in mem_pool_alloc()
455 					  typeof(*object), object_list);  in mem_pool_alloc()
456 	if (object)  in mem_pool_alloc()
457 		list_del(&object->object_list);  in mem_pool_alloc()
459 		object = &mem_pool[--mem_pool_free_count];  in mem_pool_alloc()
464 	return object;  in mem_pool_alloc()
470 static void mem_pool_free(struct kmemleak_object *object)  in mem_pool_free()  argument
474 	if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) {  in mem_pool_free()
475 		kmem_cache_free(object_cache, object);  in mem_pool_free()
481 	list_add(&object->object_list, &mem_pool_free_list);  in mem_pool_free()
492 	struct kmemleak_object *object =  in free_object_rcu()  local
499 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
503 	mem_pool_free(object);  in free_object_rcu()
513 static void put_object(struct kmemleak_object *object)  in put_object()  argument
515 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
519 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
527 		call_rcu(&object->rcu, free_object_rcu);  in put_object()
529 		free_object_rcu(&object->rcu);  in put_object()
539 	struct kmemleak_object *object;  in __find_and_get_object()  local
543 	object = __lookup_object(ptr, alias, is_phys);  in __find_and_get_object()
547 	if (object && !get_object(object))  in __find_and_get_object()
548 		object = NULL;  in __find_and_get_object()
551 	return object;  in __find_and_get_object()
565 static void __remove_object(struct kmemleak_object *object)  in __remove_object()  argument
567 	rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ?  in __remove_object()
570 	list_del_rcu(&object->object_list);  in __remove_object()
583 	struct kmemleak_object *object;  in find_and_remove_object()  local
586 	object = __lookup_object(ptr, alias, is_phys);  in find_and_remove_object()
587 	if (object)  in find_and_remove_object()
588 		__remove_object(object);  in find_and_remove_object()
591 	return object;  in find_and_remove_object()
611 	struct kmemleak_object *object, *parent;  in __create_object()  local
616 	object = mem_pool_alloc(gfp);  in __create_object()
617 	if (!object) {  in __create_object()
623 	INIT_LIST_HEAD(&object->object_list);  in __create_object()
624 	INIT_LIST_HEAD(&object->gray_list);  in __create_object()
625 	INIT_HLIST_HEAD(&object->area_list);  in __create_object()
626 	raw_spin_lock_init(&object->lock);  in __create_object()
627 	atomic_set(&object->use_count, 1);  in __create_object()
628 	object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0);  in __create_object()
629 	object->pointer = ptr;  in __create_object()
630 	object->size = kfence_ksize((void *)ptr) ?: size;  in __create_object()
631 	object->excess_ref = 0;  in __create_object()
632 	object->min_count = min_count;  in __create_object()
633 	object->count = 0;			/* white color initially */  in __create_object()
634 	object->jiffies = jiffies;  in __create_object()
635 	object->checksum = 0;  in __create_object()
639 		object->pid = 0;  in __create_object()
640 		strncpy(object->comm, "hardirq", sizeof(object->comm));  in __create_object()
642 		object->pid = 0;  in __create_object()
643 		strncpy(object->comm, "softirq", sizeof(object->comm));  in __create_object()
645 		object->pid = current->pid;  in __create_object()
652 		strncpy(object->comm, current->comm, sizeof(object->comm));  in __create_object()
656 	object->trace_len = __save_stack_trace(object->trace);  in __create_object()
688 			kmem_cache_free(object_cache, object);  in __create_object()
692 	rb_link_node(&object->rb_node, rb_parent, link);  in __create_object()
693 	rb_insert_color(&object->rb_node, is_phys ? &object_phys_tree_root :  in __create_object()
696 	list_add_tail_rcu(&object->object_list, &object_list);  in __create_object()
718 static void __delete_object(struct kmemleak_object *object)  in __delete_object()  argument
722 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
723 	WARN_ON(atomic_read(&object->use_count) < 1);  in __delete_object()
729 	raw_spin_lock_irqsave(&object->lock, flags);  in __delete_object()
730 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
731 	raw_spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
732 	put_object(object);  in __delete_object()
741 	struct kmemleak_object *object;  in delete_object_full()  local
743 	object = find_and_remove_object(ptr, 0, false);  in delete_object_full()
744 	if (!object) {  in delete_object_full()
751 	__delete_object(object);  in delete_object_full()
761 	struct kmemleak_object *object;  in delete_object_part()  local
764 	object = find_and_remove_object(ptr, 1, is_phys);  in delete_object_part()
765 	if (!object) {  in delete_object_part()
778 	start = object->pointer;  in delete_object_part()
779 	end = object->pointer + object->size;  in delete_object_part()
781 		__create_object(start, ptr - start, object->min_count,  in delete_object_part()
784 		__create_object(ptr + size, end - ptr - size, object->min_count,  in delete_object_part()
787 	__delete_object(object);  in delete_object_part()
790 static void __paint_it(struct kmemleak_object *object, int color)  in __paint_it()  argument
792 	object->min_count = color;  in __paint_it()
794 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
797 static void paint_it(struct kmemleak_object *object, int color)  in paint_it()  argument
801 	raw_spin_lock_irqsave(&object->lock, flags);  in paint_it()
802 	__paint_it(object, color);  in paint_it()
803 	raw_spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
808 	struct kmemleak_object *object;  in paint_ptr()  local
810 	object = __find_and_get_object(ptr, 0, is_phys);  in paint_ptr()
811 	if (!object) {  in paint_ptr()
818 	paint_it(object, color);  in paint_ptr()
819 	put_object(object);  in paint_ptr()
847 	struct kmemleak_object *object;  in add_scan_area()  local
852 	object = find_and_get_object(ptr, 1);  in add_scan_area()
853 	if (!object) {  in add_scan_area()
860 	untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in add_scan_area()
865 	raw_spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
869 		object->flags |= OBJECT_FULL_SCAN;  in add_scan_area()
873 		size = untagged_objp + object->size - untagged_ptr;  in add_scan_area()
874 	} else if (untagged_ptr + size > untagged_objp + object->size) {  in add_scan_area()
876 		dump_object_info(object);  in add_scan_area()
885 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
887 	raw_spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
888 	put_object(object);  in add_scan_area()
900 	struct kmemleak_object *object;  in object_set_excess_ref()  local
902 	object = find_and_get_object(ptr, 0);  in object_set_excess_ref()
903 	if (!object) {  in object_set_excess_ref()
909 	raw_spin_lock_irqsave(&object->lock, flags);  in object_set_excess_ref()
910 	object->excess_ref = excess_ref;  in object_set_excess_ref()
911 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_set_excess_ref()
912 	put_object(object);  in object_set_excess_ref()
923 	struct kmemleak_object *object;  in object_no_scan()  local
925 	object = find_and_get_object(ptr, 0);  in object_no_scan()
926 	if (!object) {  in object_no_scan()
931 	raw_spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
932 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
933 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
934 	put_object(object);  in object_no_scan()
1076 	struct kmemleak_object *object;  in kmemleak_update_trace()  local
1084 	object = find_and_get_object((unsigned long)ptr, 1);  in kmemleak_update_trace()
1085 	if (!object) {  in kmemleak_update_trace()
1093 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1094 	object->trace_len = __save_stack_trace(object->trace);  in kmemleak_update_trace()
1095 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1097 	put_object(object);  in kmemleak_update_trace()
1226 static bool update_checksum(struct kmemleak_object *object)  in update_checksum()  argument
1228 	u32 old_csum = object->checksum;  in update_checksum()
1230 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))  in update_checksum()
1235 	object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);  in update_checksum()
1239 	return object->checksum != old_csum;  in update_checksum()
1245 static void update_refs(struct kmemleak_object *object)  in update_refs()  argument
1247 	if (!color_white(object)) {  in update_refs()
1258 	object->count++;  in update_refs()
1259 	if (color_gray(object)) {  in update_refs()
1261 		WARN_ON(!get_object(object));  in update_refs()
1262 		list_add_tail(&object->gray_list, &gray_list);  in update_refs()
1302 		struct kmemleak_object *object;  in scan_block()  local
1323 		object = lookup_object(pointer, 1);  in scan_block()
1324 		if (!object)  in scan_block()
1326 		if (object == scanned)  in scan_block()
1335 		raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1337 		if (color_gray(object)) {  in scan_block()
1338 			excess_ref = object->excess_ref;  in scan_block()
1342 			update_refs(object);  in scan_block()
1344 		raw_spin_unlock(&object->lock);  in scan_block()
1347 			object = lookup_object(excess_ref, 0);  in scan_block()
1348 			if (!object)  in scan_block()
1350 			if (object == scanned)  in scan_block()
1353 			raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1354 			update_refs(object);  in scan_block()
1355 			raw_spin_unlock(&object->lock);  in scan_block()
1382 static void scan_object(struct kmemleak_object *object)  in scan_object()  argument
1392 	raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1393 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1395 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1399 	obj_ptr = object->flags & OBJECT_PHYS ?  in scan_object()
1400 		  __va((phys_addr_t)object->pointer) :  in scan_object()
1401 		  (void *)object->pointer;  in scan_object()
1403 	if (hlist_empty(&object->area_list) ||  in scan_object()
1404 	    object->flags & OBJECT_FULL_SCAN) {  in scan_object()
1406 		void *end = obj_ptr + object->size;  in scan_object()
1411 			scan_block(start, next, object);  in scan_object()
1417 			raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1419 			raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1420 		} while (object->flags & OBJECT_ALLOCATED);  in scan_object()
1422 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1425 				   object);  in scan_object()
1427 	raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1436 	struct kmemleak_object *object, *tmp;  in scan_gray_list()  local
1443 	object = list_entry(gray_list.next, typeof(*object), gray_list);  in scan_gray_list()
1444 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1449 			scan_object(object);  in scan_gray_list()
1451 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1455 		list_del(&object->gray_list);  in scan_gray_list()
1456 		put_object(object);  in scan_gray_list()
1458 		object = tmp;  in scan_gray_list()
1471 static bool kmemleak_cond_resched(struct kmemleak_object *object, bool pinned)  in kmemleak_cond_resched()  argument
1473 	if (!pinned && !get_object(object))  in kmemleak_cond_resched()
1480 		put_object(object);  in kmemleak_cond_resched()
1491 	struct kmemleak_object *object;  in kmemleak_scan()  local
1501 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1504 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1510 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1512 				 atomic_read(&object->use_count));  in kmemleak_scan()
1513 			dump_object_info(object);  in kmemleak_scan()
1518 		if ((object->flags & OBJECT_PHYS) &&  in kmemleak_scan()
1519 		   !(object->flags & OBJECT_NO_SCAN)) {  in kmemleak_scan()
1520 			unsigned long phys = object->pointer;  in kmemleak_scan()
1523 			    PHYS_PFN(phys + object->size) >= max_low_pfn)  in kmemleak_scan()
1524 				__paint_it(object, KMEMLEAK_BLACK);  in kmemleak_scan()
1528 		object->count = 0;  in kmemleak_scan()
1529 		if (color_gray(object) && get_object(object)) {  in kmemleak_scan()
1530 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1534 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1540 		    !kmemleak_cond_resched(object, obj_pinned))  in kmemleak_scan()
1609 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1614 		    !kmemleak_cond_resched(object, false))  in kmemleak_scan()
1622 		if (!color_white(object))  in kmemleak_scan()
1624 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1625 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1626 		    && update_checksum(object) && get_object(object)) {  in kmemleak_scan()
1628 			object->count = object->min_count;  in kmemleak_scan()
1629 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1631 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1651 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1656 		    !kmemleak_cond_resched(object, false))  in kmemleak_scan()
1664 		if (!color_white(object))  in kmemleak_scan()
1666 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1667 		if (unreferenced_object(object) &&  in kmemleak_scan()
1668 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1669 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1672 				print_unreferenced(NULL, object);  in kmemleak_scan()
1676 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1760 	struct kmemleak_object *object;  in kmemleak_seq_start()  local
1769 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_seq_start()
1772 		if (get_object(object))  in kmemleak_seq_start()
1775 	object = NULL;  in kmemleak_seq_start()
1777 	return object;  in kmemleak_seq_start()
1825 	struct kmemleak_object *object = v;  in kmemleak_seq_show()  local
1828 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1829 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1830 		print_unreferenced(seq, object);  in kmemleak_seq_show()
1831 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1850 	struct kmemleak_object *object;  in dump_str_object_info()  local
1855 	object = find_and_get_object(addr, 0);  in dump_str_object_info()
1856 	if (!object) {  in dump_str_object_info()
1861 	raw_spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1862 	dump_object_info(object);  in dump_str_object_info()
1863 	raw_spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1865 	put_object(object);  in dump_str_object_info()
1877 	struct kmemleak_object *object;  in kmemleak_clear()  local
1880 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_clear()
1881 		raw_spin_lock_irq(&object->lock);  in kmemleak_clear()
1882 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
1883 		    unreferenced_object(object))  in kmemleak_clear()
1884 			__paint_it(object, KMEMLEAK_GREY);  in kmemleak_clear()
1885 		raw_spin_unlock_irq(&object->lock);  in kmemleak_clear()
1994 	struct kmemleak_object *object, *tmp;  in __kmemleak_do_cleanup()  local
2000 	list_for_each_entry_safe(object, tmp, &object_list, object_list) {  in __kmemleak_do_cleanup()
2001 		__remove_object(object);  in __kmemleak_do_cleanup()
2002 		__delete_object(object);  in __kmemleak_do_cleanup()