39  *   Note that the kmemleak_object.use_count is incremented when an object is
47  *   scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
49  * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
110 #define MSECS_MIN_AGE		5000	/* minimum object age for reporting */
136  * object->lock. Insertions or deletions from object_list, gray_list or
143 	unsigned int flags;		/* object status flags */
148 	/* object usage count; object freed when use_count == 0 */
156 	/* the total number of pointers found pointing to this object */
160 	/* memory ranges to be scanned inside an object (empty for all) */
171 /* flag set after the first reporting of an unreference object */
173 /* flag set to not scan the object */
175 /* flag set to fully scan the object when scan_area allocation failed */
177 /* flag set for object allocated with physical address */
198 /* search tree for object boundaries */
200 /* search tree for object (with OBJECT_PHYS flag) boundaries */
286  * with the object->lock held.
289 			    struct kmemleak_object *object)  in hex_dump_object()  argument
291 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
294 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))  in hex_dump_object()
298 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);  in hex_dump_object()
308  * Object colors, encoded with count and min_count:
309  * - white - orphan object, not enough references to it (count < min_count)
314  * Newly created objects don't have any color assigned (object->count == -1)
317 static bool color_white(const struct kmemleak_object *object)  in color_white()  argument
319 	return object->count != KMEMLEAK_BLACK &&  in color_white()
320 		object->count < object->min_count;  in color_white()
323 static bool color_gray(const struct kmemleak_object *object)  in color_gray()  argument
325 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
326 		object->count >= object->min_count;  in color_gray()
334 static bool unreferenced_object(struct kmemleak_object *object)  in unreferenced_object()  argument
336 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
337 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
343  * print_unreferenced function must be called with the object->lock held.
346 			       struct kmemleak_object *object)  in print_unreferenced()  argument
349 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);  in print_unreferenced()
351 	warn_or_seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",  in print_unreferenced()
352 		   object->pointer, object->size);  in print_unreferenced()
354 		   object->comm, object->pid, object->jiffies,  in print_unreferenced()
356 	hex_dump_object(seq, object);  in print_unreferenced()
359 	for (i = 0; i < object->trace_len; i++) {  in print_unreferenced()
360 		void *ptr = (void *)object->trace[i];  in print_unreferenced()
368  * the object->lock held.
370 static void dump_object_info(struct kmemleak_object *object)  in dump_object_info()  argument
372 	pr_notice("Object 0x%08lx (size %zu):\n",  in dump_object_info()
373 		  object->pointer, object->size);  in dump_object_info()
375 		  object->comm, object->pid, object->jiffies);  in dump_object_info()
376 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
377 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
378 	pr_notice("  flags = 0x%x\n", object->flags);  in dump_object_info()
379 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
381 	stack_trace_print(object->trace, object->trace_len, 4);  in dump_object_info()
385  * Look-up a memory block metadata (kmemleak_object) in the object search
398 		struct kmemleak_object *object;  in __lookup_object()  local
401 		object = rb_entry(rb, struct kmemleak_object, rb_node);  in __lookup_object()
402 		untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in __lookup_object()
405 			rb = object->rb_node.rb_left;  in __lookup_object()
406 		else if (untagged_objp + object->size <= untagged_ptr)  in __lookup_object()
407 			rb = object->rb_node.rb_right;  in __lookup_object()
409 			return object;  in __lookup_object()
411 			kmemleak_warn("Found object by alias at 0x%08lx\n",  in __lookup_object()
413 			dump_object_info(object);  in __lookup_object()
420 /* Look-up a kmemleak object which allocated with virtual address. */
427  * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
428  * that once an object's use_count reached 0, the RCU freeing was already
429  * registered and the object should no longer be used. This function must be
432 static int get_object(struct kmemleak_object *object)  in get_object()  argument
434 	return atomic_inc_not_zero(&object->use_count);  in get_object()
443 	struct kmemleak_object *object;  in mem_pool_alloc()  local
447 		object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));  in mem_pool_alloc()
448 		if (object)  in mem_pool_alloc()
449 			return object;  in mem_pool_alloc()
454 	object = list_first_entry_or_null(&mem_pool_free_list,  in mem_pool_alloc()
455 					  typeof(*object), object_list);  in mem_pool_alloc()
456 	if (object)  in mem_pool_alloc()
457 		list_del(&object->object_list);  in mem_pool_alloc()
459 		object = &mem_pool[--mem_pool_free_count];  in mem_pool_alloc()
464 	return object;  in mem_pool_alloc()
468  * Return the object to either the slab allocator or the memory pool.
470 static void mem_pool_free(struct kmemleak_object *object)  in mem_pool_free()  argument
474 	if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) {  in mem_pool_free()
475 		kmem_cache_free(object_cache, object);  in mem_pool_free()
479 	/* add the object to the memory pool free list */  in mem_pool_free()
481 	list_add(&object->object_list, &mem_pool_free_list);  in mem_pool_free()
492 	struct kmemleak_object *object =  in free_object_rcu()  local
497 	 * code accessing this object, hence no need for locking.  in free_object_rcu()
499 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
503 	mem_pool_free(object);  in free_object_rcu()
507  * Decrement the object use_count. Once the count is 0, free the object using
513 static void put_object(struct kmemleak_object *object)  in put_object()  argument
515 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
519 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
524 	 * came from the memory pool. Free the object directly.  in put_object()
527 		call_rcu(&object->rcu, free_object_rcu);  in put_object()
529 		free_object_rcu(&object->rcu);  in put_object()
533  * Look up an object in the object search tree and increase its use_count.
539 	struct kmemleak_object *object;  in __find_and_get_object()  local
543 	object = __lookup_object(ptr, alias, is_phys);  in __find_and_get_object()
546 	/* check whether the object is still available */  in __find_and_get_object()
547 	if (object && !get_object(object))  in __find_and_get_object()
548 		object = NULL;  in __find_and_get_object()
551 	return object;  in __find_and_get_object()
554 /* Look up and get an object which allocated with virtual address. */
561  * Remove an object from the object_tree_root (or object_phys_tree_root)
565 static void __remove_object(struct kmemleak_object *object)  in __remove_object()  argument
567 	rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ?  in __remove_object()
570 	list_del_rcu(&object->object_list);  in __remove_object()
574  * Look up an object in the object search tree and remove it from both
576  * returned object's use_count should be at least 1, as initially set
583 	struct kmemleak_object *object;  in find_and_remove_object()  local
586 	object = __lookup_object(ptr, alias, is_phys);  in find_and_remove_object()
587 	if (object)  in find_and_remove_object()
588 		__remove_object(object);  in find_and_remove_object()
591 	return object;  in find_and_remove_object()
611 	struct kmemleak_object *object, *parent;  in __create_object()  local
616 	object = mem_pool_alloc(gfp);  in __create_object()
617 	if (!object) {  in __create_object()
623 	INIT_LIST_HEAD(&object->object_list);  in __create_object()
624 	INIT_LIST_HEAD(&object->gray_list);  in __create_object()
625 	INIT_HLIST_HEAD(&object->area_list);  in __create_object()
626 	raw_spin_lock_init(&object->lock);  in __create_object()
627 	atomic_set(&object->use_count, 1);  in __create_object()
628 	object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0);  in __create_object()
629 	object->pointer = ptr;  in __create_object()
630 	object->size = kfence_ksize((void *)ptr) ?: size;  in __create_object()
631 	object->excess_ref = 0;  in __create_object()
632 	object->min_count = min_count;  in __create_object()
633 	object->count = 0;			/* white color initially */  in __create_object()
634 	object->jiffies = jiffies;  in __create_object()
635 	object->checksum = 0;  in __create_object()
639 		object->pid = 0;  in __create_object()
640 		strncpy(object->comm, "hardirq", sizeof(object->comm));  in __create_object()
642 		object->pid = 0;  in __create_object()
643 		strncpy(object->comm, "softirq", sizeof(object->comm));  in __create_object()
645 		object->pid = current->pid;  in __create_object()
652 		strncpy(object->comm, current->comm, sizeof(object->comm));  in __create_object()
656 	object->trace_len = __save_stack_trace(object->trace);  in __create_object()
662 	 * Only update min_addr and max_addr with object  in __create_object()
681 			kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",  in __create_object()
688 			kmem_cache_free(object_cache, object);  in __create_object()
692 	rb_link_node(&object->rb_node, rb_parent, link);  in __create_object()
693 	rb_insert_color(&object->rb_node, is_phys ? &object_phys_tree_root :  in __create_object()
696 	list_add_tail_rcu(&object->object_list, &object_list);  in __create_object()
701 /* Create kmemleak object which allocated with virtual address. */
708 /* Create kmemleak object which allocated with physical address. */
716  * Mark the object as not allocated and schedule RCU freeing via put_object().
718 static void __delete_object(struct kmemleak_object *object)  in __delete_object()  argument
722 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
723 	WARN_ON(atomic_read(&object->use_count) < 1);  in __delete_object()
729 	raw_spin_lock_irqsave(&object->lock, flags);  in __delete_object()
730 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
731 	raw_spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
732 	put_object(object);  in __delete_object()
741 	struct kmemleak_object *object;  in delete_object_full()  local
743 	object = find_and_remove_object(ptr, 0, false);  in delete_object_full()
744 	if (!object) {  in delete_object_full()
746 		kmemleak_warn("Freeing unknown object at 0x%08lx\n",  in delete_object_full()
751 	__delete_object(object);  in delete_object_full()
761 	struct kmemleak_object *object;  in delete_object_part()  local
764 	object = find_and_remove_object(ptr, 1, is_phys);  in delete_object_part()
765 	if (!object) {  in delete_object_part()
767 		kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",  in delete_object_part()
778 	start = object->pointer;  in delete_object_part()
779 	end = object->pointer + object->size;  in delete_object_part()
781 		__create_object(start, ptr - start, object->min_count,  in delete_object_part()
784 		__create_object(ptr + size, end - ptr - size, object->min_count,  in delete_object_part()
787 	__delete_object(object);  in delete_object_part()
790 static void __paint_it(struct kmemleak_object *object, int color)  in __paint_it()  argument
792 	object->min_count = color;  in __paint_it()
794 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
797 static void paint_it(struct kmemleak_object *object, int color)  in paint_it()  argument
801 	raw_spin_lock_irqsave(&object->lock, flags);  in paint_it()
802 	__paint_it(object, color);  in paint_it()
803 	raw_spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
808 	struct kmemleak_object *object;  in paint_ptr()  local
810 	object = __find_and_get_object(ptr, 0, is_phys);  in paint_ptr()
811 	if (!object) {  in paint_ptr()
812 		kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",  in paint_ptr()
818 	paint_it(object, color);  in paint_ptr()
819 	put_object(object);  in paint_ptr()
823  * Mark an object permanently as gray-colored so that it can no longer be
832  * Mark the object as black-colored so that it is ignored from scans and
841  * Add a scanning area to the object. If at least one such area is added,
847 	struct kmemleak_object *object;  in add_scan_area()  local
852 	object = find_and_get_object(ptr, 1);  in add_scan_area()
853 	if (!object) {  in add_scan_area()
854 		kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",  in add_scan_area()
860 	untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in add_scan_area()
865 	raw_spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
867 		pr_warn_once("Cannot allocate a scan area, scanning the full object\n");  in add_scan_area()
868 		/* mark the object for full scan to avoid false positives */  in add_scan_area()
869 		object->flags |= OBJECT_FULL_SCAN;  in add_scan_area()
873 		size = untagged_objp + object->size - untagged_ptr;  in add_scan_area()
874 	} else if (untagged_ptr + size > untagged_objp + object->size) {  in add_scan_area()
875 		kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);  in add_scan_area()
876 		dump_object_info(object);  in add_scan_area()
885 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
887 	raw_spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
888 	put_object(object);  in add_scan_area()
892  * Any surplus references (object already gray) to 'ptr' are passed to
894  * vm_struct may be used as an alternative reference to the vmalloc'ed object
900 	struct kmemleak_object *object;  in object_set_excess_ref()  local
902 	object = find_and_get_object(ptr, 0);  in object_set_excess_ref()
903 	if (!object) {  in object_set_excess_ref()
904 		kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",  in object_set_excess_ref()
909 	raw_spin_lock_irqsave(&object->lock, flags);  in object_set_excess_ref()
910 	object->excess_ref = excess_ref;  in object_set_excess_ref()
911 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_set_excess_ref()
912 	put_object(object);  in object_set_excess_ref()
916  * Set the OBJECT_NO_SCAN flag for the object corresponding to the give
917  * pointer. Such object will not be scanned by kmemleak but references to it
923 	struct kmemleak_object *object;  in object_no_scan()  local
925 	object = find_and_get_object(ptr, 0);  in object_no_scan()
926 	if (!object) {  in object_no_scan()
927 		kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);  in object_no_scan()
931 	raw_spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
932 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
933 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
934 	put_object(object);  in object_no_scan()
938  * kmemleak_alloc - register a newly allocated object
939  * @ptr:	pointer to beginning of the object
940  * @size:	size of the object
941  * @min_count:	minimum number of references to this object. If during memory
943  *		the object is reported as a memory leak. If @min_count is 0,
944  *		the object is never reported as a leak. If @min_count is -1,
945  *		the object is ignored (not scanned and not reported as a leak)
948  * This function is called from the kernel allocators when a new object
962  * kmemleak_alloc_percpu - register a newly allocated __percpu object
963  * @ptr:	__percpu pointer to beginning of the object
964  * @size:	size of the object
967  * This function is called from the kernel percpu allocator when a new object
989  * kmemleak_vmalloc - register a newly vmalloc'ed object
991  * @size:	size of the object
995  * object (memory block) is allocated.
1014  * kmemleak_free - unregister a previously registered object
1015  * @ptr:	pointer to beginning of the object
1017  * This function is called from the kernel allocators when an object (memory
1030  * kmemleak_free_part - partially unregister a previously registered object
1031  * @ptr:	pointer to the beginning or inside the object. This also
1048  * kmemleak_free_percpu - unregister a previously registered __percpu object
1049  * @ptr:	__percpu pointer to beginning of the object
1051  * This function is called from the kernel percpu allocator when an object
1068  * kmemleak_update_trace - update object allocation stack trace
1069  * @ptr:	pointer to beginning of the object
1071  * Override the object allocation stack trace for cases where the actual
1076 	struct kmemleak_object *object;  in kmemleak_update_trace()  local
1084 	object = find_and_get_object((unsigned long)ptr, 1);  in kmemleak_update_trace()
1085 	if (!object) {  in kmemleak_update_trace()
1087 		kmemleak_warn("Updating stack trace for unknown object at %p\n",  in kmemleak_update_trace()
1093 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1094 	object->trace_len = __save_stack_trace(object->trace);  in kmemleak_update_trace()
1095 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1097 	put_object(object);  in kmemleak_update_trace()
1102  * kmemleak_not_leak - mark an allocated object as false positive
1103  * @ptr:	pointer to beginning of the object
1105  * Calling this function on an object will cause the memory block to no longer
1118  * kmemleak_ignore - ignore an allocated object
1119  * @ptr:	pointer to beginning of the object
1121  * Calling this function on an object will cause the memory block to be
1136  * kmemleak_scan_area - limit the range to be scanned in an allocated object
1137  * @ptr:	pointer to beginning or inside the object. This also
1142  * This function is used when it is known that only certain parts of an object
1156  * kmemleak_no_scan - do not scan an allocated object
1157  * @ptr:	pointer to beginning of the object
1160  * in situations where it is known that the given object does not contain any
1176  * @phys:	physical address of the object
1177  * @size:	size of the object
1186 		 * Create object with OBJECT_PHYS flag and  in kmemleak_alloc_phys()
1196  * @phys:	physical address if the beginning or inside an object. This
1212  * @phys:	physical address of the object
1224  * Update an object's checksum and return true if it was modified.
1226 static bool update_checksum(struct kmemleak_object *object)  in update_checksum()  argument
1228 	u32 old_csum = object->checksum;  in update_checksum()
1230 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))  in update_checksum()
1235 	object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);  in update_checksum()
1239 	return object->checksum != old_csum;  in update_checksum()
1243  * Update an object's references. object->lock must be held by the caller.
1245 static void update_refs(struct kmemleak_object *object)  in update_refs()  argument
1247 	if (!color_white(object)) {  in update_refs()
1253 	 * Increase the object's reference count (number of pointers to the  in update_refs()
1255 	 * object's color will become gray and it will be added to the  in update_refs()
1258 	object->count++;  in update_refs()
1259 	if (color_gray(object)) {  in update_refs()
1261 		WARN_ON(!get_object(object));  in update_refs()
1262 		list_add_tail(&object->gray_list, &gray_list);  in update_refs()
1302 		struct kmemleak_object *object;  in scan_block()  local
1319 		 * object->use_count cannot be dropped to 0 while the object  in scan_block()
1323 		object = lookup_object(pointer, 1);  in scan_block()
1324 		if (!object)  in scan_block()
1326 		if (object == scanned)  in scan_block()
1331 		 * Avoid the lockdep recursive warning on object->lock being  in scan_block()
1335 		raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1336 		/* only pass surplus references (object already gray) */  in scan_block()
1337 		if (color_gray(object)) {  in scan_block()
1338 			excess_ref = object->excess_ref;  in scan_block()
1339 			/* no need for update_refs() if object already gray */  in scan_block()
1342 			update_refs(object);  in scan_block()
1344 		raw_spin_unlock(&object->lock);  in scan_block()
1347 			object = lookup_object(excess_ref, 0);  in scan_block()
1348 			if (!object)  in scan_block()
1350 			if (object == scanned)  in scan_block()
1353 			raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1354 			update_refs(object);  in scan_block()
1355 			raw_spin_unlock(&object->lock);  in scan_block()
1380  * that object->use_count >= 1.
1382 static void scan_object(struct kmemleak_object *object)  in scan_object()  argument
1389 	 * Once the object->lock is acquired, the corresponding memory block  in scan_object()
1392 	raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1393 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1395 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1396 		/* already freed object */  in scan_object()
1399 	obj_ptr = object->flags & OBJECT_PHYS ?  in scan_object()
1400 		  __va((phys_addr_t)object->pointer) :  in scan_object()
1401 		  (void *)object->pointer;  in scan_object()
1403 	if (hlist_empty(&object->area_list) ||  in scan_object()
1404 	    object->flags & OBJECT_FULL_SCAN) {  in scan_object()
1406 		void *end = obj_ptr + object->size;  in scan_object()
1411 			scan_block(start, next, object);  in scan_object()
1417 			raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1419 			raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1420 		} while (object->flags & OBJECT_ALLOCATED);  in scan_object()
1422 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1425 				   object);  in scan_object()
1427 	raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1436 	struct kmemleak_object *object, *tmp;  in scan_gray_list()  local
1443 	object = list_entry(gray_list.next, typeof(*object), gray_list);  in scan_gray_list()
1444 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1449 			scan_object(object);  in scan_gray_list()
1451 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1454 		/* remove the object from the list and release it */  in scan_gray_list()
1455 		list_del(&object->gray_list);  in scan_gray_list()
1456 		put_object(object);  in scan_gray_list()
1458 		object = tmp;  in scan_gray_list()
1464  * Conditionally call resched() in a object iteration loop while making sure
1465  * that the given object won't go away without RCU read lock by performing a
1471 static bool kmemleak_cond_resched(struct kmemleak_object *object, bool pinned)  in kmemleak_cond_resched()  argument
1473 	if (!pinned && !get_object(object))  in kmemleak_cond_resched()
1480 		put_object(object);  in kmemleak_cond_resched()
1491 	struct kmemleak_object *object;  in kmemleak_scan()  local
1501 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1504 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1508 		 * 1 reference to any object at this point.  in kmemleak_scan()
1510 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1511 			pr_debug("object->use_count = %d\n",  in kmemleak_scan()
1512 				 atomic_read(&object->use_count));  in kmemleak_scan()
1513 			dump_object_info(object);  in kmemleak_scan()
1518 		if ((object->flags & OBJECT_PHYS) &&  in kmemleak_scan()
1519 		   !(object->flags & OBJECT_NO_SCAN)) {  in kmemleak_scan()
1520 			unsigned long phys = object->pointer;  in kmemleak_scan()
1523 			    PHYS_PFN(phys + object->size) >= max_low_pfn)  in kmemleak_scan()
1524 				__paint_it(object, KMEMLEAK_BLACK);  in kmemleak_scan()
1527 		/* reset the reference count (whiten the object) */  in kmemleak_scan()
1528 		object->count = 0;  in kmemleak_scan()
1529 		if (color_gray(object) && get_object(object)) {  in kmemleak_scan()
1530 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1534 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1540 		    !kmemleak_cond_resched(object, obj_pinned))  in kmemleak_scan()
1541 			loop_cnt--; /* Try again on next object */  in kmemleak_scan()
1609 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1614 		    !kmemleak_cond_resched(object, false))  in kmemleak_scan()
1615 			loop_cnt--;	/* Try again on next object */  in kmemleak_scan()
1622 		if (!color_white(object))  in kmemleak_scan()
1624 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1625 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1626 		    && update_checksum(object) && get_object(object)) {  in kmemleak_scan()
1628 			object->count = object->min_count;  in kmemleak_scan()
1629 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1631 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1651 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1656 		    !kmemleak_cond_resched(object, false))  in kmemleak_scan()
1657 			loop_cnt--;	/* Try again on next object */  in kmemleak_scan()
1664 		if (!color_white(object))  in kmemleak_scan()
1666 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1667 		if (unreferenced_object(object) &&  in kmemleak_scan()
1668 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1669 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1672 				print_unreferenced(NULL, object);  in kmemleak_scan()
1676 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1754  * Iterate over the object_list and return the first valid object at or after
1760 	struct kmemleak_object *object;  in kmemleak_seq_start()  local
1769 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_seq_start()
1772 		if (get_object(object))  in kmemleak_seq_start()
1775 	object = NULL;  in kmemleak_seq_start()
1777 	return object;  in kmemleak_seq_start()
1781  * Return the next object in the object_list. The function decrements the
1782  * use_count of the previous object and increases that of the next one.
1804  * Decrement the use_count of the last object required, if any.
1821  * Print the information for an unreferenced object to the seq file.
1825 	struct kmemleak_object *object = v;  in kmemleak_seq_show()  local
1828 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1829 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1830 		print_unreferenced(seq, object);  in kmemleak_seq_show()
1831 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1850 	struct kmemleak_object *object;  in dump_str_object_info()  local
1855 	object = find_and_get_object(addr, 0);  in dump_str_object_info()
1856 	if (!object) {  in dump_str_object_info()
1857 		pr_info("Unknown object at 0x%08lx\n", addr);  in dump_str_object_info()
1861 	raw_spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1862 	dump_object_info(object);  in dump_str_object_info()
1863 	raw_spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1865 	put_object(object);  in dump_str_object_info()
1877 	struct kmemleak_object *object;  in kmemleak_clear()  local
1880 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_clear()
1881 		raw_spin_lock_irq(&object->lock);  in kmemleak_clear()
1882 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
1883 		    unreferenced_object(object))  in kmemleak_clear()
1884 			__paint_it(object, KMEMLEAK_GREY);  in kmemleak_clear()
1885 		raw_spin_unlock_irq(&object->lock);  in kmemleak_clear()
1908  *   dump=...	- dump information about the object found at the given address
1994 	struct kmemleak_object *object, *tmp;  in __kmemleak_do_cleanup()  local
2000 	list_for_each_entry_safe(object, tmp, &object_list, object_list) {  in __kmemleak_do_cleanup()
2001 		__remove_object(object);  in __kmemleak_do_cleanup()
2002 		__delete_object(object);  in __kmemleak_do_cleanup()
2018 	 * longer track object freeing. Ordering of the scan thread stopping and  in kmemleak_do_cleanup()