1 // SPDX-License-Identifier: GPL-2.0-only
9  * Documentation/dev-tools/kmemleak.rst.
12  * ----------------
16  * - kmemleak_lock (raw_spinlock_t): protects the object_list modifications and
20  *   are red black trees used to look-up metadata based on a pointer to the
27  * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
34  * - scan_mutex (mutex): ensures that only one thread may scan the memory for
47  *   scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
49  * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
123 /* scanning area inside a memory block */
131 #define KMEMLEAK_BLACK	-1
136  * object->lock. Insertions or deletions from object_list, gray_list or
138  * the notes on locking above). These objects are reference-counted
192 /* the list of gray-colored objects (see color_gray comment below) */
286  * with the object->lock held.
291 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
294 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))  in hex_dump_object()
298 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);  in hex_dump_object()
309  * - white - orphan object, not enough references to it (count < min_count)
310  * - gray  - not orphan, not marked as false positive (min_count == 0) or
312  * - black - ignore, it doesn't contain references (e.g. text section)
313  *		(min_count == -1). No function defined for this color.
314  * Newly created objects don't have any color assigned (object->count == -1)
319 	return object->count != KMEMLEAK_BLACK &&  in color_white()
320 		object->count < object->min_count;  in color_white()
325 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
326 		object->count >= object->min_count;  in color_gray()
330  * Objects are considered unreferenced only if their color is white, they have
336 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
337 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
343  * print_unreferenced function must be called with the object->lock held.
349 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);  in print_unreferenced()
352 		   object->pointer, object->size);  in print_unreferenced()
354 		   object->comm, object->pid, object->jiffies,  in print_unreferenced()
359 	for (i = 0; i < object->trace_len; i++) {  in print_unreferenced()
360 		void *ptr = (void *)object->trace[i];  in print_unreferenced()
368  * the object->lock held.
373 		  object->pointer, object->size);  in dump_object_info()
375 		  object->comm, object->pid, object->jiffies);  in dump_object_info()
376 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
377 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
378 	pr_notice("  flags = 0x%x\n", object->flags);  in dump_object_info()
379 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
381 	stack_trace_print(object->trace, object->trace_len, 4);  in dump_object_info()
385  * Look-up a memory block metadata (kmemleak_object) in the object search
402 		untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in __lookup_object()
405 			rb = object->rb_node.rb_left;  in __lookup_object()
406 		else if (untagged_objp + object->size <= untagged_ptr)  in __lookup_object()
407 			rb = object->rb_node.rb_right;  in __lookup_object()
420 /* Look-up a kmemleak object which allocated with virtual address. */
434 	return atomic_inc_not_zero(&object->use_count);  in get_object()
457 		list_del(&object->object_list);  in mem_pool_alloc()
459 		object = &mem_pool[--mem_pool_free_count];  in mem_pool_alloc()
481 	list_add(&object->object_list, &mem_pool_free_list);  in mem_pool_free()
491 	struct kmemleak_scan_area *area;  in free_object_rcu()  local
499 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
500 		hlist_del(&area->node);  in free_object_rcu()
501 		kmem_cache_free(scan_area_cache, area);  in free_object_rcu()
508  * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
510  * recursive call to the kernel allocator. Lock-less RCU object_list traversal
515 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
519 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
527 		call_rcu(&object->rcu, free_object_rcu);  in put_object()
529 		free_object_rcu(&object->rcu);  in put_object()
567 	rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ?  in __remove_object()
570 	list_del_rcu(&object->object_list);  in __remove_object()
623 	INIT_LIST_HEAD(&object->object_list);  in __create_object()
624 	INIT_LIST_HEAD(&object->gray_list);  in __create_object()
625 	INIT_HLIST_HEAD(&object->area_list);  in __create_object()
626 	raw_spin_lock_init(&object->lock);  in __create_object()
627 	atomic_set(&object->use_count, 1);  in __create_object()
628 	object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0);  in __create_object()
629 	object->pointer = ptr;  in __create_object()
630 	object->size = kfence_ksize((void *)ptr) ?: size;  in __create_object()
631 	object->excess_ref = 0;  in __create_object()
632 	object->min_count = min_count;  in __create_object()
633 	object->count = 0;			/* white color initially */  in __create_object()
634 	object->jiffies = jiffies;  in __create_object()
635 	object->checksum = 0;  in __create_object()
639 		object->pid = 0;  in __create_object()
640 		strncpy(object->comm, "hardirq", sizeof(object->comm));  in __create_object()
642 		object->pid = 0;  in __create_object()
643 		strncpy(object->comm, "softirq", sizeof(object->comm));  in __create_object()
645 		object->pid = current->pid;  in __create_object()
649 		 * dependency issues with current->alloc_lock. In the worst  in __create_object()
652 		strncpy(object->comm, current->comm, sizeof(object->comm));  in __create_object()
656 	object->trace_len = __save_stack_trace(object->trace);  in __create_object()
675 		untagged_objp = (unsigned long)kasan_reset_tag((void *)parent->pointer);  in __create_object()
677 			link = &parent->rb_node.rb_left;  in __create_object()
678 		else if (untagged_objp + parent->size <= untagged_ptr)  in __create_object()
679 			link = &parent->rb_node.rb_right;  in __create_object()
684 			 * No need for parent->lock here since "parent" cannot  in __create_object()
692 	rb_link_node(&object->rb_node, rb_parent, link);  in __create_object()
693 	rb_insert_color(&object->rb_node, is_phys ? &object_phys_tree_root :  in __create_object()
696 	list_add_tail_rcu(&object->object_list, &object_list);  in __create_object()
722 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
723 	WARN_ON(atomic_read(&object->use_count) < 1);  in __delete_object()
729 	raw_spin_lock_irqsave(&object->lock, flags);  in __delete_object()
730 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
731 	raw_spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
778 	start = object->pointer;  in delete_object_part()
779 	end = object->pointer + object->size;  in delete_object_part()
781 		__create_object(start, ptr - start, object->min_count,  in delete_object_part()
784 		__create_object(ptr + size, end - ptr - size, object->min_count,  in delete_object_part()
790 static void __paint_it(struct kmemleak_object *object, int color)  in __paint_it()  argument
792 	object->min_count = color;  in __paint_it()
793 	if (color == KMEMLEAK_BLACK)  in __paint_it()
794 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
797 static void paint_it(struct kmemleak_object *object, int color)  in paint_it()  argument
801 	raw_spin_lock_irqsave(&object->lock, flags);  in paint_it()
802 	__paint_it(object, color);  in paint_it()
803 	raw_spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
806 static void paint_ptr(unsigned long ptr, int color, bool is_phys)  in paint_ptr()  argument
812 		kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",  in paint_ptr()
814 			      (color == KMEMLEAK_GREY) ? "Grey" :  in paint_ptr()
815 			      (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");  in paint_ptr()
818 	paint_it(object, color);  in paint_ptr()
823  * Mark an object permanently as gray-colored so that it can no longer be
832  * Mark the object as black-colored so that it is ignored from scans and
841  * Add a scanning area to the object. If at least one such area is added,
848 	struct kmemleak_scan_area *area = NULL;  in add_scan_area()  local
854 		kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",  in add_scan_area()
860 	untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in add_scan_area()
863 		area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));  in add_scan_area()
865 	raw_spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
866 	if (!area) {  in add_scan_area()
867 		pr_warn_once("Cannot allocate a scan area, scanning the full object\n");  in add_scan_area()
869 		object->flags |= OBJECT_FULL_SCAN;  in add_scan_area()
873 		size = untagged_objp + object->size - untagged_ptr;  in add_scan_area()
874 	} else if (untagged_ptr + size > untagged_objp + object->size) {  in add_scan_area()
875 		kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);  in add_scan_area()
877 		kmem_cache_free(scan_area_cache, area);  in add_scan_area()
881 	INIT_HLIST_NODE(&area->node);  in add_scan_area()
882 	area->start = ptr;  in add_scan_area()
883 	area->size = size;  in add_scan_area()
885 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
887 	raw_spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
909 	raw_spin_lock_irqsave(&object->lock, flags);  in object_set_excess_ref()
910 	object->excess_ref = excess_ref;  in object_set_excess_ref()
911 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_set_excess_ref()
931 	raw_spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
932 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
933 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
938  * kmemleak_alloc - register a newly allocated object
944  *		the object is never reported as a leak. If @min_count is -1,
962  * kmemleak_alloc_percpu - register a newly allocated __percpu object
989  * kmemleak_vmalloc - register a newly vmalloc'ed object
990  * @area:	pointer to vm_struct
997 void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp)  in kmemleak_vmalloc()  argument
999 	pr_debug("%s(0x%p, %zu)\n", __func__, area, size);  in kmemleak_vmalloc()
1006 		create_object((unsigned long)area->addr, size, 2, gfp);  in kmemleak_vmalloc()
1007 		object_set_excess_ref((unsigned long)area,  in kmemleak_vmalloc()
1008 				      (unsigned long)area->addr);  in kmemleak_vmalloc()
1014  * kmemleak_free - unregister a previously registered object
1030  * kmemleak_free_part - partially unregister a previously registered object
1048  * kmemleak_free_percpu - unregister a previously registered __percpu object
1068  * kmemleak_update_trace - update object allocation stack trace
1093 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1094 	object->trace_len = __save_stack_trace(object->trace);  in kmemleak_update_trace()
1095 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1102  * kmemleak_not_leak - mark an allocated object as false positive
1118  * kmemleak_ignore - ignore an allocated object
1136  * kmemleak_scan_area - limit the range to be scanned in an allocated object
1138  *		represents the start of the scan area
1139  * @size:	size of the scan area
1156  * kmemleak_no_scan - do not scan an allocated object
1174  * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1194  * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1210  * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1228 	u32 old_csum = object->checksum;  in update_checksum()
1230 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))  in update_checksum()
1235 	object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);  in update_checksum()
1239 	return object->checksum != old_csum;  in update_checksum()
1243  * Update an object's references. object->lock must be held by the caller.
1248 		/* non-orphan, ignored or new */  in update_refs()
1255 	 * object's color will become gray and it will be added to the  in update_refs()
1258 	object->count++;  in update_refs()
1262 		list_add_tail(&object->gray_list, &gray_list);  in update_refs()
1279 	if (current->mm)  in scan_should_stop()
1296 	unsigned long *end = _end - (BYTES_PER_POINTER - 1);  in scan_block()
1319 		 * object->use_count cannot be dropped to 0 while the object  in scan_block()
1331 		 * Avoid the lockdep recursive warning on object->lock being  in scan_block()
1335 		raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1338 			excess_ref = object->excess_ref;  in scan_block()
1344 		raw_spin_unlock(&object->lock);  in scan_block()
1353 			raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1355 			raw_spin_unlock(&object->lock);  in scan_block()
1380  * that object->use_count >= 1.
1384 	struct kmemleak_scan_area *area;  in scan_object()  local
1389 	 * Once the object->lock is acquired, the corresponding memory block  in scan_object()
1392 	raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1393 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1395 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1399 	obj_ptr = object->flags & OBJECT_PHYS ?  in scan_object()
1400 		  __va((phys_addr_t)object->pointer) :  in scan_object()
1401 		  (void *)object->pointer;  in scan_object()
1403 	if (hlist_empty(&object->area_list) ||  in scan_object()
1404 	    object->flags & OBJECT_FULL_SCAN) {  in scan_object()
1406 		void *end = obj_ptr + object->size;  in scan_object()
1417 			raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1419 			raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1420 		} while (object->flags & OBJECT_ALLOCATED);  in scan_object()
1422 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1423 			scan_block((void *)area->start,  in scan_object()
1424 				   (void *)(area->start + area->size),  in scan_object()
1427 	raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1444 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1451 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1455 		list_del(&object->gray_list);  in scan_gray_list()
1504 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1510 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1511 			pr_debug("object->use_count = %d\n",  in kmemleak_scan()
1512 				 atomic_read(&object->use_count));  in kmemleak_scan()
1518 		if ((object->flags & OBJECT_PHYS) &&  in kmemleak_scan()
1519 		   !(object->flags & OBJECT_NO_SCAN)) {  in kmemleak_scan()
1520 			unsigned long phys = object->pointer;  in kmemleak_scan()
1523 			    PHYS_PFN(phys + object->size) >= max_low_pfn)  in kmemleak_scan()
1528 		object->count = 0;  in kmemleak_scan()
1530 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1534 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1541 			loop_cnt--; /* Try again on next object */  in kmemleak_scan()
1546 	/* per-cpu sections scanning */  in kmemleak_scan()
1557 		unsigned long start_pfn = zone->zone_start_pfn;  in kmemleak_scan()
1605 	 * scan and color them gray until the next scan.  in kmemleak_scan()
1615 			loop_cnt--;	/* Try again on next object */  in kmemleak_scan()
1624 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1625 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1627 			/* color it gray temporarily */  in kmemleak_scan()
1628 			object->count = object->min_count;  in kmemleak_scan()
1629 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1631 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1636 	 * Re-scan the gray list for modified unreferenced objects.  in kmemleak_scan()
1657 			loop_cnt--;	/* Try again on next object */  in kmemleak_scan()
1666 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1668 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1669 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1676 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1770 		if (n-- > 0)  in kmemleak_seq_start()
1828 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1829 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1831 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1854 		return -EINVAL;  in dump_str_object_info()
1858 		return -EINVAL;  in dump_str_object_info()
1861 	raw_spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1863 	raw_spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1881 		raw_spin_lock_irq(&object->lock);  in kmemleak_clear()
1882 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
1885 		raw_spin_unlock_irq(&object->lock);  in kmemleak_clear()
1895  * File write operation to configure kmemleak at run-time. The following
1897  *   off	- disable kmemleak (irreversible)
1898  *   stack=on	- enable the task stacks scanning
1899  *   stack=off	- disable the tasks stacks scanning
1900  *   scan=on	- start the automatic memory scanning thread
1901  *   scan=off	- stop the automatic memory scanning thread
1902  *   scan=...	- set the automatic memory scanning period in seconds (0 to
1904  *   scan	- trigger a memory scan
1905  *   clear	- mark all current reported unreferenced kmemleak objects as
1908  *   dump=...	- dump information about the object found at the given address
1917 	buf_size = min(size, (sizeof(buf) - 1));  in kmemleak_write()
1919 		return -EFAULT;  in kmemleak_write()
1935 		ret = -EPERM;  in kmemleak_write()
1971 		ret = -EINVAL;  in kmemleak_write()
2056  * Allow boot-time kmemleak disabling (enabled by default).
2061 		return -EINVAL;  in kmemleak_boot_config()
2067 		return -EINVAL;  in kmemleak_boot_config()
2094 	create_object((unsigned long)_sdata, _edata - _sdata,  in kmemleak_init()
2096 	create_object((unsigned long)__bss_start, __bss_stop - __bss_start,  in kmemleak_init()
2101 			      __end_ro_after_init - __start_ro_after_init,  in kmemleak_init()
2119 		 * two clean-up threads but serialized by scan_mutex.  in kmemleak_late_init()
2122 		return -ENOMEM;  in kmemleak_late_init()