309 			    struct kmemleak_object *object)  in hex_dump_object()  argument
311 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
315 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);  in hex_dump_object()
334 static bool color_white(const struct kmemleak_object *object)  in color_white()  argument
336 	return object->count != KMEMLEAK_BLACK &&  in color_white()
337 		object->count < object->min_count;  in color_white()
340 static bool color_gray(const struct kmemleak_object *object)  in color_gray()  argument
342 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
343 		object->count >= object->min_count;  in color_gray()
351 static bool unreferenced_object(struct kmemleak_object *object)  in unreferenced_object()  argument
353 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
354 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
363 			       struct kmemleak_object *object)  in print_unreferenced()  argument
366 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);  in print_unreferenced()
369 		   object->pointer, object->size);  in print_unreferenced()
371 		   object->comm, object->pid, object->jiffies,  in print_unreferenced()
373 	hex_dump_object(seq, object);  in print_unreferenced()
376 	for (i = 0; i < object->trace_len; i++) {  in print_unreferenced()
377 		void *ptr = (void *)object->trace[i];  in print_unreferenced()
387 static void dump_object_info(struct kmemleak_object *object)  in dump_object_info()  argument
391 	trace.nr_entries = object->trace_len;  in dump_object_info()
392 	trace.entries = object->trace;  in dump_object_info()
395 		  object->pointer, object->size);  in dump_object_info()
397 		  object->comm, object->pid, object->jiffies);  in dump_object_info()
398 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
399 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
400 	pr_notice("  flags = 0x%x\n", object->flags);  in dump_object_info()
401 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
417 		struct kmemleak_object *object =  in lookup_object()  local
419 		if (ptr < object->pointer)  in lookup_object()
420 			rb = object->rb_node.rb_left;  in lookup_object()
421 		else if (object->pointer + object->size <= ptr)  in lookup_object()
422 			rb = object->rb_node.rb_right;  in lookup_object()
423 		else if (object->pointer == ptr || alias)  in lookup_object()
424 			return object;  in lookup_object()
428 			dump_object_info(object);  in lookup_object()
441 static int get_object(struct kmemleak_object *object)  in get_object()  argument
443 	return atomic_inc_not_zero(&object->use_count);  in get_object()
453 	struct kmemleak_object *object =  in free_object_rcu()  local
460 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
464 	kmem_cache_free(object_cache, object);  in free_object_rcu()
474 static void put_object(struct kmemleak_object *object)  in put_object()  argument
476 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
480 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
482 	call_rcu(&object->rcu, free_object_rcu);  in put_object()
491 	struct kmemleak_object *object;  in find_and_get_object()  local
495 	object = lookup_object(ptr, alias);  in find_and_get_object()
499 	if (object && !get_object(object))  in find_and_get_object()
500 		object = NULL;  in find_and_get_object()
503 	return object;  in find_and_get_object()
514 	struct kmemleak_object *object;  in find_and_remove_object()  local
517 	object = lookup_object(ptr, alias);  in find_and_remove_object()
518 	if (object) {  in find_and_remove_object()
519 		rb_erase(&object->rb_node, &object_tree_root);  in find_and_remove_object()
520 		list_del_rcu(&object->object_list);  in find_and_remove_object()
524 	return object;  in find_and_remove_object()
551 	struct kmemleak_object *object, *parent;  in create_object()  local
554 	object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));  in create_object()
555 	if (!object) {  in create_object()
561 	INIT_LIST_HEAD(&object->object_list);  in create_object()
562 	INIT_LIST_HEAD(&object->gray_list);  in create_object()
563 	INIT_HLIST_HEAD(&object->area_list);  in create_object()
564 	spin_lock_init(&object->lock);  in create_object()
565 	atomic_set(&object->use_count, 1);  in create_object()
566 	object->flags = OBJECT_ALLOCATED;  in create_object()
567 	object->pointer = ptr;  in create_object()
568 	object->size = size;  in create_object()
569 	object->excess_ref = 0;  in create_object()
570 	object->min_count = min_count;  in create_object()
571 	object->count = 0;			/* white color initially */  in create_object()
572 	object->jiffies = jiffies;  in create_object()
573 	object->checksum = 0;  in create_object()
577 		object->pid = 0;  in create_object()
578 		strncpy(object->comm, "hardirq", sizeof(object->comm));  in create_object()
580 		object->pid = 0;  in create_object()
581 		strncpy(object->comm, "softirq", sizeof(object->comm));  in create_object()
583 		object->pid = current->pid;  in create_object()
590 		strncpy(object->comm, current->comm, sizeof(object->comm));  in create_object()
594 	object->trace_len = __save_stack_trace(object->trace);  in create_object()
617 			kmem_cache_free(object_cache, object);  in create_object()
618 			object = NULL;  in create_object()
622 	rb_link_node(&object->rb_node, rb_parent, link);  in create_object()
623 	rb_insert_color(&object->rb_node, &object_tree_root);  in create_object()
625 	list_add_tail_rcu(&object->object_list, &object_list);  in create_object()
628 	return object;  in create_object()
634 static void __delete_object(struct kmemleak_object *object)  in __delete_object()  argument
638 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
639 	WARN_ON(atomic_read(&object->use_count) < 1);  in __delete_object()
645 	spin_lock_irqsave(&object->lock, flags);  in __delete_object()
646 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
647 	spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
648 	put_object(object);  in __delete_object()
657 	struct kmemleak_object *object;  in delete_object_full()  local
659 	object = find_and_remove_object(ptr, 0);  in delete_object_full()
660 	if (!object) {  in delete_object_full()
667 	__delete_object(object);  in delete_object_full()
677 	struct kmemleak_object *object;  in delete_object_part()  local
680 	object = find_and_remove_object(ptr, 1);  in delete_object_part()
681 	if (!object) {  in delete_object_part()
696 	start = object->pointer;  in delete_object_part()
697 	end = object->pointer + object->size;  in delete_object_part()
699 		create_object(start, ptr - start, object->min_count,  in delete_object_part()
702 		create_object(ptr + size, end - ptr - size, object->min_count,  in delete_object_part()
705 	__delete_object(object);  in delete_object_part()
708 static void __paint_it(struct kmemleak_object *object, int color)  in __paint_it()  argument
710 	object->min_count = color;  in __paint_it()
712 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
715 static void paint_it(struct kmemleak_object *object, int color)  in paint_it()  argument
719 	spin_lock_irqsave(&object->lock, flags);  in paint_it()
720 	__paint_it(object, color);  in paint_it()
721 	spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
726 	struct kmemleak_object *object;  in paint_ptr()  local
728 	object = find_and_get_object(ptr, 0);  in paint_ptr()
729 	if (!object) {  in paint_ptr()
736 	paint_it(object, color);  in paint_ptr()
737 	put_object(object);  in paint_ptr()
765 	struct kmemleak_object *object;  in add_scan_area()  local
768 	object = find_and_get_object(ptr, 1);  in add_scan_area()
769 	if (!object) {  in add_scan_area()
781 	spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
783 		size = object->pointer + object->size - ptr;  in add_scan_area()
784 	} else if (ptr + size > object->pointer + object->size) {  in add_scan_area()
786 		dump_object_info(object);  in add_scan_area()
795 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
797 	spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
799 	put_object(object);  in add_scan_area()
811 	struct kmemleak_object *object;  in object_set_excess_ref()  local
813 	object = find_and_get_object(ptr, 0);  in object_set_excess_ref()
814 	if (!object) {  in object_set_excess_ref()
820 	spin_lock_irqsave(&object->lock, flags);  in object_set_excess_ref()
821 	object->excess_ref = excess_ref;  in object_set_excess_ref()
822 	spin_unlock_irqrestore(&object->lock, flags);  in object_set_excess_ref()
823 	put_object(object);  in object_set_excess_ref()
834 	struct kmemleak_object *object;  in object_no_scan()  local
836 	object = find_and_get_object(ptr, 0);  in object_no_scan()
837 	if (!object) {  in object_no_scan()
842 	spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
843 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
844 	spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
845 	put_object(object);  in object_no_scan()
890 	struct kmemleak_object *object;  in early_alloc()  local
901 	object = create_object((unsigned long)log->ptr, log->size,  in early_alloc()
903 	if (!object)  in early_alloc()
905 	spin_lock_irqsave(&object->lock, flags);  in early_alloc()
907 		object->trace[i] = log->trace[i];  in early_alloc()
908 	object->trace_len = log->trace_len;  in early_alloc()
909 	spin_unlock_irqrestore(&object->lock, flags);  in early_alloc()
1082 	struct kmemleak_object *object;  in kmemleak_update_trace()  local
1090 	object = find_and_get_object((unsigned long)ptr, 1);  in kmemleak_update_trace()
1091 	if (!object) {  in kmemleak_update_trace()
1099 	spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1100 	object->trace_len = __save_stack_trace(object->trace);  in kmemleak_update_trace()
1101 	spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1103 	put_object(object);  in kmemleak_update_trace()
1245 static bool update_checksum(struct kmemleak_object *object)  in update_checksum()  argument
1247 	u32 old_csum = object->checksum;  in update_checksum()
1250 	object->checksum = crc32(0, (void *)object->pointer, object->size);  in update_checksum()
1253 	return object->checksum != old_csum;  in update_checksum()
1259 static void update_refs(struct kmemleak_object *object)  in update_refs()  argument
1261 	if (!color_white(object)) {  in update_refs()
1272 	object->count++;  in update_refs()
1273 	if (color_gray(object)) {  in update_refs()
1275 		WARN_ON(!get_object(object));  in update_refs()
1276 		list_add_tail(&object->gray_list, &gray_list);  in update_refs()
1315 		struct kmemleak_object *object;  in scan_block()  local
1335 		object = lookup_object(pointer, 1);  in scan_block()
1336 		if (!object)  in scan_block()
1338 		if (object == scanned)  in scan_block()
1347 		spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1349 		if (color_gray(object)) {  in scan_block()
1350 			excess_ref = object->excess_ref;  in scan_block()
1354 			update_refs(object);  in scan_block()
1356 		spin_unlock(&object->lock);  in scan_block()
1359 			object = lookup_object(excess_ref, 0);  in scan_block()
1360 			if (!object)  in scan_block()
1362 			if (object == scanned)  in scan_block()
1365 			spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1366 			update_refs(object);  in scan_block()
1367 			spin_unlock(&object->lock);  in scan_block()
1392 static void scan_object(struct kmemleak_object *object)  in scan_object()  argument
1401 	spin_lock_irqsave(&object->lock, flags);  in scan_object()
1402 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1404 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1407 	if (hlist_empty(&object->area_list)) {  in scan_object()
1408 		void *start = (void *)object->pointer;  in scan_object()
1409 		void *end = (void *)(object->pointer + object->size);  in scan_object()
1414 			scan_block(start, next, object);  in scan_object()
1420 			spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1422 			spin_lock_irqsave(&object->lock, flags);  in scan_object()
1423 		} while (object->flags & OBJECT_ALLOCATED);  in scan_object()
1425 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1428 				   object);  in scan_object()
1430 	spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1439 	struct kmemleak_object *object, *tmp;  in scan_gray_list()  local
1446 	object = list_entry(gray_list.next, typeof(*object), gray_list);  in scan_gray_list()
1447 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1452 			scan_object(object);  in scan_gray_list()
1454 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1458 		list_del(&object->gray_list);  in scan_gray_list()
1459 		put_object(object);  in scan_gray_list()
1461 		object = tmp;  in scan_gray_list()
1474 	struct kmemleak_object *object;  in kmemleak_scan()  local
1482 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1483 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1489 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1491 				 atomic_read(&object->use_count));  in kmemleak_scan()
1492 			dump_object_info(object);  in kmemleak_scan()
1496 		object->count = 0;  in kmemleak_scan()
1497 		if (color_gray(object) && get_object(object))  in kmemleak_scan()
1498 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1500 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1569 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1570 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1571 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1572 		    && update_checksum(object) && get_object(object)) {  in kmemleak_scan()
1574 			object->count = object->min_count;  in kmemleak_scan()
1575 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1577 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1596 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1597 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1598 		if (unreferenced_object(object) &&  in kmemleak_scan()
1599 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1600 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1603 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1687 	struct kmemleak_object *object;  in kmemleak_seq_start()  local
1696 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_seq_start()
1699 		if (get_object(object))  in kmemleak_seq_start()
1702 	object = NULL;  in kmemleak_seq_start()
1704 	return object;  in kmemleak_seq_start()
1752 	struct kmemleak_object *object = v;  in kmemleak_seq_show()  local
1755 	spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1756 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1757 		print_unreferenced(seq, object);  in kmemleak_seq_show()
1758 	spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1777 	struct kmemleak_object *object;  in dump_str_object_info()  local
1782 	object = find_and_get_object(addr, 0);  in dump_str_object_info()
1783 	if (!object) {  in dump_str_object_info()
1788 	spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1789 	dump_object_info(object);  in dump_str_object_info()
1790 	spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1792 	put_object(object);  in dump_str_object_info()
1804 	struct kmemleak_object *object;  in kmemleak_clear()  local
1808 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_clear()
1809 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_clear()
1810 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
1811 		    unreferenced_object(object))  in kmemleak_clear()
1812 			__paint_it(object, KMEMLEAK_GREY);  in kmemleak_clear()
1813 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_clear()
1916 	struct kmemleak_object *object;  in __kmemleak_do_cleanup()  local
1919 	list_for_each_entry_rcu(object, &object_list, object_list)  in __kmemleak_do_cleanup()
1920 		delete_object_full(object->pointer);  in __kmemleak_do_cleanup()