1 // SPDX-License-Identifier: GPL-2.0-only
9  * Documentation/dev-tools/kmemleak.rst.
12  * ----------------
16  * - kmemleak_lock (raw_spinlock_t): protects the object_list modifications and
19  *   blocks. The object_tree_root is a red black tree used to look-up
25  * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
32  * - scan_mutex (mutex): ensures that only one thread may scan the memory for
45  *   scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
47  * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
129 #define KMEMLEAK_BLACK	-1
134  * object->lock. Insertions or deletions from object_list, gray_list or
136  * the notes on locking above). These objects are reference-counted
152 	/* minimum number of a pointers found before it is considered leak */
163 	pid_t pid;			/* pid of the current task */
188 /* the list of gray-colored objects (see color_gray comment below) */
280  * with the object->lock held.
285 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
289 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);  in hex_dump_object()
300  * - white - orphan object, not enough references to it (count < min_count)
301  * - gray  - not orphan, not marked as false positive (min_count == 0) or
303  * - black - ignore, it doesn't contain references (e.g. text section)
304  *		(min_count == -1). No function defined for this color.
305  * Newly created objects don't have any color assigned (object->count == -1)
310 	return object->count != KMEMLEAK_BLACK &&  in color_white()
311 		object->count < object->min_count;  in color_white()
316 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
317 		object->count >= object->min_count;  in color_gray()
327 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
328 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
334  * print_unreferenced function must be called with the object->lock held.
339 	int i;  in print_unreferenced()  local
340 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);  in print_unreferenced()
343 		   object->pointer, object->size);  in print_unreferenced()
345 		   object->comm, object->pid, object->jiffies,  in print_unreferenced()
350 	for (i = 0; i < object->trace_len; i++) {  in print_unreferenced()
351 		void *ptr = (void *)object->trace[i];  in print_unreferenced()
359  * the object->lock held.
364 		  object->pointer, object->size);  in dump_object_info()
366 		  object->comm, object->pid, object->jiffies);  in dump_object_info()
367 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
368 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
369 	pr_notice("  flags = 0x%x\n", object->flags);  in dump_object_info()
370 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
372 	stack_trace_print(object->trace, object->trace_len, 4);  in dump_object_info()
376  * Look-up a memory block metadata (kmemleak_object) in the object search
388 		if (ptr < object->pointer)  in lookup_object()
389 			rb = object->rb_node.rb_left;  in lookup_object()
390 		else if (object->pointer + object->size <= ptr)  in lookup_object()
391 			rb = object->rb_node.rb_right;  in lookup_object()
392 		else if (object->pointer == ptr || alias)  in lookup_object()
412 	return atomic_inc_not_zero(&object->use_count);  in get_object()
435 		list_del(&object->object_list);  in mem_pool_alloc()
437 		object = &mem_pool[--mem_pool_free_count];  in mem_pool_alloc()
459 	list_add(&object->object_list, &mem_pool_free_list);  in mem_pool_free()
477 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
478 		hlist_del(&area->node);  in free_object_rcu()
486  * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
488  * recursive call to the kernel allocator. Lock-less RCU object_list traversal
493 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
497 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
505 		call_rcu(&object->rcu, free_object_rcu);  in put_object()
507 		free_object_rcu(&object->rcu);  in put_object()
537 	rb_erase(&object->rb_node, &object_tree_root);  in __remove_object()
538 	list_del_rcu(&object->object_list);  in __remove_object()
587 	INIT_LIST_HEAD(&object->object_list);  in create_object()
588 	INIT_LIST_HEAD(&object->gray_list);  in create_object()
589 	INIT_HLIST_HEAD(&object->area_list);  in create_object()
590 	raw_spin_lock_init(&object->lock);  in create_object()
591 	atomic_set(&object->use_count, 1);  in create_object()
592 	object->flags = OBJECT_ALLOCATED;  in create_object()
593 	object->pointer = ptr;  in create_object()
594 	object->size = kfence_ksize((void *)ptr) ?: size;  in create_object()
595 	object->excess_ref = 0;  in create_object()
596 	object->min_count = min_count;  in create_object()
597 	object->count = 0;			/* white color initially */  in create_object()
598 	object->jiffies = jiffies;  in create_object()
599 	object->checksum = 0;  in create_object()
603 		object->pid = 0;  in create_object()
604 		strncpy(object->comm, "hardirq", sizeof(object->comm));  in create_object()
606 		object->pid = 0;  in create_object()
607 		strncpy(object->comm, "softirq", sizeof(object->comm));  in create_object()
609 		object->pid = current->pid;  in create_object()
613 		 * dependency issues with current->alloc_lock. In the worst  in create_object()
616 		strncpy(object->comm, current->comm, sizeof(object->comm));  in create_object()
620 	object->trace_len = __save_stack_trace(object->trace);  in create_object()
632 		if (ptr + size <= parent->pointer)  in create_object()
633 			link = &parent->rb_node.rb_left;  in create_object()
634 		else if (parent->pointer + parent->size <= ptr)  in create_object()
635 			link = &parent->rb_node.rb_right;  in create_object()
640 			 * No need for parent->lock here since "parent" cannot  in create_object()
649 	rb_link_node(&object->rb_node, rb_parent, link);  in create_object()
650 	rb_insert_color(&object->rb_node, &object_tree_root);  in create_object()
652 	list_add_tail_rcu(&object->object_list, &object_list);  in create_object()
665 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
666 	WARN_ON(atomic_read(&object->use_count) < 1);  in __delete_object()
672 	raw_spin_lock_irqsave(&object->lock, flags);  in __delete_object()
673 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
674 	raw_spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
721 	start = object->pointer;  in delete_object_part()
722 	end = object->pointer + object->size;  in delete_object_part()
724 		create_object(start, ptr - start, object->min_count,  in delete_object_part()
727 		create_object(ptr + size, end - ptr - size, object->min_count,  in delete_object_part()
735 	object->min_count = color;  in __paint_it()
737 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
744 	raw_spin_lock_irqsave(&object->lock, flags);  in paint_it()
746 	raw_spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
766  * Mark an object permanently as gray-colored so that it can no longer be
767  * reported as a leak. This is used in general to mark a false positive.
775  * Mark the object as black-colored so that it is ignored from scans and
803 	raw_spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
807 		object->flags |= OBJECT_FULL_SCAN;  in add_scan_area()
811 		size = object->pointer + object->size - ptr;  in add_scan_area()
812 	} else if (ptr + size > object->pointer + object->size) {  in add_scan_area()
819 	INIT_HLIST_NODE(&area->node);  in add_scan_area()
820 	area->start = ptr;  in add_scan_area()
821 	area->size = size;  in add_scan_area()
823 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
825 	raw_spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
847 	raw_spin_lock_irqsave(&object->lock, flags);  in object_set_excess_ref()
848 	object->excess_ref = excess_ref;  in object_set_excess_ref()
849 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_set_excess_ref()
869 	raw_spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
870 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
871 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
876  * kmemleak_alloc - register a newly allocated object
881  *		the object is reported as a memory leak. If @min_count is 0,
882  *		the object is never reported as a leak. If @min_count is -1,
883  *		the object is ignored (not scanned and not reported as a leak)
900  * kmemleak_alloc_percpu - register a newly allocated __percpu object
927  * kmemleak_vmalloc - register a newly vmalloc'ed object
944 		create_object((unsigned long)area->addr, size, 2, gfp);  in kmemleak_vmalloc()
946 				      (unsigned long)area->addr);  in kmemleak_vmalloc()
952  * kmemleak_free - unregister a previously registered object
968  * kmemleak_free_part - partially unregister a previously registered object
986  * kmemleak_free_percpu - unregister a previously registered __percpu object
1006  * kmemleak_update_trace - update object allocation stack trace
1031 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1032 	object->trace_len = __save_stack_trace(object->trace);  in kmemleak_update_trace()
1033 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1040  * kmemleak_not_leak - mark an allocated object as false positive
1044  * be reported as leak and always be scanned.
1056  * kmemleak_ignore - ignore an allocated object
1060  * ignored (not scanned and not reported as a leak). This is usually done when
1061  * it is known that the corresponding block is not a leak and does not contain
1074  * kmemleak_scan_area - limit the range to be scanned in an allocated object
1094  * kmemleak_no_scan - do not scan an allocated object
1112  * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1129  * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1143  * kmemleak_not_leak_phys - similar to kmemleak_not_leak but taking a physical
1155  * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1171 	u32 old_csum = object->checksum;  in update_checksum()
1175 	object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);  in update_checksum()
1179 	return object->checksum != old_csum;  in update_checksum()
1183  * Update an object's references. object->lock must be held by the caller.
1188 		/* non-orphan, ignored or new */  in update_refs()
1198 	object->count++;  in update_refs()
1202 		list_add_tail(&object->gray_list, &gray_list);  in update_refs()
1219 	if (current->mm)  in scan_should_stop()
1220 		return signal_pending(current);  in scan_should_stop()
1236 	unsigned long *end = _end - (BYTES_PER_POINTER - 1);  in scan_block()
1259 		 * object->use_count cannot be dropped to 0 while the object  in scan_block()
1271 		 * Avoid the lockdep recursive warning on object->lock being  in scan_block()
1275 		raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1278 			excess_ref = object->excess_ref;  in scan_block()
1284 		raw_spin_unlock(&object->lock);  in scan_block()
1293 			raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1295 			raw_spin_unlock(&object->lock);  in scan_block()
1320  * that object->use_count >= 1.
1328 	 * Once the object->lock is acquired, the corresponding memory block  in scan_object()
1331 	raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1332 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1334 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1337 	if (hlist_empty(&object->area_list) ||  in scan_object()
1338 	    object->flags & OBJECT_FULL_SCAN) {  in scan_object()
1339 		void *start = (void *)object->pointer;  in scan_object()
1340 		void *end = (void *)(object->pointer + object->size);  in scan_object()
1351 			raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1353 			raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1354 		} while (object->flags & OBJECT_ALLOCATED);  in scan_object()
1356 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1357 			scan_block((void *)area->start,  in scan_object()
1358 				   (void *)(area->start + area->size),  in scan_object()
1361 	raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1378 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1385 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1389 		list_del(&object->gray_list);  in scan_gray_list()
1406 	int i;  in kmemleak_scan()  local
1414 		raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1420 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1421 			pr_debug("object->use_count = %d\n",  in kmemleak_scan()
1422 				 atomic_read(&object->use_count));  in kmemleak_scan()
1427 		object->count = 0;  in kmemleak_scan()
1429 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1431 		raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1436 	/* per-cpu sections scanning */  in kmemleak_scan()
1437 	for_each_possible_cpu(i)  in kmemleak_scan()
1438 		scan_large_block(__per_cpu_start + per_cpu_offset(i),  in kmemleak_scan()
1439 				 __per_cpu_end + per_cpu_offset(i));  in kmemleak_scan()
1446 	for_each_online_node(i) {  in kmemleak_scan()
1447 		unsigned long start_pfn = node_start_pfn(i);  in kmemleak_scan()
1448 		unsigned long end_pfn = node_end_pfn(i);  in kmemleak_scan()
1458 			if (page_to_nid(page) != i)  in kmemleak_scan()
1499 		raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1500 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1503 			object->count = object->min_count;  in kmemleak_scan()
1504 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1506 		raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1511 	 * Re-scan the gray list for modified unreferenced objects.  in kmemleak_scan()
1526 		raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1528 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1529 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1536 		raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1558 	set_user_nice(current, 10);  in kmemleak_scan_thread()
1630 		if (n-- > 0)  in kmemleak_seq_start()
1688 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1689 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1691 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1714 		return -EINVAL;  in dump_str_object_info()
1718 		return -EINVAL;  in dump_str_object_info()
1721 	raw_spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1723 	raw_spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1742 		raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_clear()
1743 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
1746 		raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_clear()
1756  * File write operation to configure kmemleak at run-time. The following
1758  *   off	- disable kmemleak (irreversible)
1759  *   stack=on	- enable the task stacks scanning
1760  *   stack=off	- disable the tasks stacks scanning
1761  *   scan=on	- start the automatic memory scanning thread
1762  *   scan=off	- stop the automatic memory scanning thread
1763  *   scan=...	- set the automatic memory scanning period in seconds (0 to
1765  *   scan	- trigger a memory scan
1766  *   clear	- mark all current reported unreferenced kmemleak objects as
1769  *   dump=...	- dump information about the object found at the given address
1778 	buf_size = min(size, (sizeof(buf) - 1));  in kmemleak_write()
1780 		return -EFAULT;  in kmemleak_write()
1796 		ret = -EPERM;  in kmemleak_write()
1832 		ret = -EINVAL;  in kmemleak_write()
1913 	pr_info("Kernel memory leak detector disabled\n");  in kmemleak_disable()
1917  * Allow boot-time kmemleak disabling (enabled by default).
1922 		return -EINVAL;  in kmemleak_boot_config()
1928 		return -EINVAL;  in kmemleak_boot_config()
1955 	create_object((unsigned long)_sdata, _edata - _sdata,  in kmemleak_init()
1957 	create_object((unsigned long)__bss_start, __bss_stop - __bss_start,  in kmemleak_init()
1962 			      __end_ro_after_init - __start_ro_after_init,  in kmemleak_init()
1980 		 * two clean-up threads but serialized by scan_mutex.  in kmemleak_late_init()
1983 		return -ENOMEM;  in kmemleak_late_init()
1992 	pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n",  in kmemleak_late_init()