333 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)  in dbg_redzone1()  argument
336 	return (unsigned long long *) (objp + obj_offset(cachep) -  in dbg_redzone1()
340 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)  in dbg_redzone2()  argument
344 		return (unsigned long long *)(objp + cachep->size -  in dbg_redzone2()
347 	return (unsigned long long *) (objp + cachep->size -  in dbg_redzone2()
351 static void **dbg_userword(struct kmem_cache *cachep, void *objp)  in dbg_userword()  argument
354 	return (void **)(objp + cachep->size - BYTES_PER_WORD);  in dbg_userword()
360 #define dbg_redzone1(cachep, objp)	({BUG(); (unsigned long long *)NULL;})  argument
361 #define dbg_redzone2(cachep, objp)	({BUG(); (unsigned long long *)NULL;})  argument
362 #define dbg_userword(cachep, objp)	({BUG(); (void **)NULL;})  argument
553 					struct page *page, void *objp)  in cache_free_pfmemalloc()  argument
563 	free_block(cachep, &objp, 1, page_node, &list);  in cache_free_pfmemalloc()
593 static __always_inline void __free_one(struct array_cache *ac, void *objp)  in __free_one()  argument
597 	    WARN_ON_ONCE(ac->avail > 0 && ac->entry[ac->avail - 1] == objp))  in __free_one()
599 	ac->entry[ac->avail++] = objp;  in __free_one()
617 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)  in cache_free_alien()  argument
763 static int __cache_free_alien(struct kmem_cache *cachep, void *objp,  in __cache_free_alien()  argument
781 		__free_one(ac, objp);  in __cache_free_alien()
787 		free_block(cachep, &objp, 1, page_node, &list);  in __cache_free_alien()
794 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)  in cache_free_alien()  argument
796 	int page_node = page_to_nid(virt_to_page(objp));  in cache_free_alien()
805 	return __cache_free_alien(cachep, objp, node, page_node);  in cache_free_alien()
1434 static void slab_kernel_map(struct kmem_cache *cachep, void *objp, int map)  in slab_kernel_map()  argument
1439 	__kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map);  in slab_kernel_map()
1443 static inline void slab_kernel_map(struct kmem_cache *cachep, void *objp,  in slab_kernel_map()  argument
1489 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)  in print_objinfo()  argument
1496 		       *dbg_redzone1(cachep, objp),  in print_objinfo()
1497 		       *dbg_redzone2(cachep, objp));  in print_objinfo()
1501 		pr_err("Last user: (%pSR)\n", *dbg_userword(cachep, objp));  in print_objinfo()
1502 	realobj = (char *)objp + obj_offset(cachep);  in print_objinfo()
1513 static void check_poison_obj(struct kmem_cache *cachep, void *objp)  in check_poison_obj()  argument
1522 	realobj = (char *)objp + obj_offset(cachep);  in check_poison_obj()
1537 				print_objinfo(cachep, objp, 0);  in check_poison_obj()
1556 		struct page *page = virt_to_head_page(objp);  in check_poison_obj()
1559 		objnr = obj_to_index(cachep, page, objp);  in check_poison_obj()
1561 			objp = index_to_obj(cachep, page, objnr - 1);  in check_poison_obj()
1562 			realobj = (char *)objp + obj_offset(cachep);  in check_poison_obj()
1564 			print_objinfo(cachep, objp, 2);  in check_poison_obj()
1567 			objp = index_to_obj(cachep, page, objnr + 1);  in check_poison_obj()
1568 			realobj = (char *)objp + obj_offset(cachep);  in check_poison_obj()
1570 			print_objinfo(cachep, objp, 2);  in check_poison_obj()
1588 		void *objp = index_to_obj(cachep, page, i);  in slab_destroy_debugcheck()  local
1591 			check_poison_obj(cachep, objp);  in slab_destroy_debugcheck()
1592 			slab_kernel_map(cachep, objp, 1);  in slab_destroy_debugcheck()
1595 			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)  in slab_destroy_debugcheck()
1597 			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)  in slab_destroy_debugcheck()
2335 		void *objp = index_to_obj(cachep, page, i);  in cache_init_objs_debug()  local
2338 			*dbg_userword(cachep, objp) = NULL;  in cache_init_objs_debug()
2341 			*dbg_redzone1(cachep, objp) = RED_INACTIVE;  in cache_init_objs_debug()
2342 			*dbg_redzone2(cachep, objp) = RED_INACTIVE;  in cache_init_objs_debug()
2351 						   objp + obj_offset(cachep));  in cache_init_objs_debug()
2352 			cachep->ctor(objp + obj_offset(cachep));  in cache_init_objs_debug()
2354 				cachep, objp + obj_offset(cachep));  in cache_init_objs_debug()
2358 			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)  in cache_init_objs_debug()
2360 			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)  in cache_init_objs_debug()
2365 			poison_obj(cachep, objp, POISON_FREE);  in cache_init_objs_debug()
2366 			slab_kernel_map(cachep, objp, 0);  in cache_init_objs_debug()
2487 	void *objp;  in cache_init_objs()  local
2501 		objp = index_to_obj(cachep, page, i);  in cache_init_objs()
2502 		objp = kasan_init_slab_obj(cachep, objp);  in cache_init_objs()
2506 			kasan_unpoison_object_data(cachep, objp);  in cache_init_objs()
2507 			cachep->ctor(objp);  in cache_init_objs()
2508 			kasan_poison_object_data(cachep, objp);  in cache_init_objs()
2518 	void *objp;  in slab_get_obj()  local
2520 	objp = index_to_obj(cachep, page, get_free_obj(page, page->active));  in slab_get_obj()
2523 	return objp;  in slab_get_obj()
2527 			struct page *page, void *objp)  in slab_put_obj()  argument
2529 	unsigned int objnr = obj_to_index(cachep, page, objp);  in slab_put_obj()
2537 			       cachep->name, objp);  in slab_put_obj()
2544 		page->freelist = objp + obj_offset(cachep);  in slab_put_obj()
2676 static void kfree_debugcheck(const void *objp)  in kfree_debugcheck()  argument
2678 	if (!virt_addr_valid(objp)) {  in kfree_debugcheck()
2680 		       (unsigned long)objp);  in kfree_debugcheck()
2707 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,  in cache_free_debugcheck()  argument
2713 	BUG_ON(virt_to_cache(objp) != cachep);  in cache_free_debugcheck()
2715 	objp -= obj_offset(cachep);  in cache_free_debugcheck()
2716 	kfree_debugcheck(objp);  in cache_free_debugcheck()
2717 	page = virt_to_head_page(objp);  in cache_free_debugcheck()
2720 		verify_redzone_free(cachep, objp);  in cache_free_debugcheck()
2721 		*dbg_redzone1(cachep, objp) = RED_INACTIVE;  in cache_free_debugcheck()
2722 		*dbg_redzone2(cachep, objp) = RED_INACTIVE;  in cache_free_debugcheck()
2725 		*dbg_userword(cachep, objp) = (void *)caller;  in cache_free_debugcheck()
2727 	objnr = obj_to_index(cachep, page, objp);  in cache_free_debugcheck()
2730 	BUG_ON(objp != index_to_obj(cachep, page, objnr));  in cache_free_debugcheck()
2733 		poison_obj(cachep, objp, POISON_FREE);  in cache_free_debugcheck()
2734 		slab_kernel_map(cachep, objp, 0);  in cache_free_debugcheck()
2736 	return objp;  in cache_free_debugcheck()
2741 #define cache_free_debugcheck(x, objp, z) (objp)  argument
2749 	void *objp;  in fixup_objfreelist_debug()  local
2752 		objp = next - obj_offset(cachep);  in fixup_objfreelist_debug()
2754 		poison_obj(cachep, objp, POISON_FREE);  in fixup_objfreelist_debug()
2771 				void **objp = page->freelist;  in fixup_slab_list()  local
2773 				*objp = *list;  in fixup_slab_list()
2774 				*list = objp;  in fixup_slab_list()
2992 				gfp_t flags, void *objp, unsigned long caller)  in cache_alloc_debugcheck_after()  argument
2995 	if (!objp || is_kfence_address(objp))  in cache_alloc_debugcheck_after()
2996 		return objp;  in cache_alloc_debugcheck_after()
2998 		check_poison_obj(cachep, objp);  in cache_alloc_debugcheck_after()
2999 		slab_kernel_map(cachep, objp, 1);  in cache_alloc_debugcheck_after()
3000 		poison_obj(cachep, objp, POISON_INUSE);  in cache_alloc_debugcheck_after()
3003 		*dbg_userword(cachep, objp) = (void *)caller;  in cache_alloc_debugcheck_after()
3006 		if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||  in cache_alloc_debugcheck_after()
3007 				*dbg_redzone2(cachep, objp) != RED_INACTIVE) {  in cache_alloc_debugcheck_after()
3010 			       objp, *dbg_redzone1(cachep, objp),  in cache_alloc_debugcheck_after()
3011 			       *dbg_redzone2(cachep, objp));  in cache_alloc_debugcheck_after()
3013 		*dbg_redzone1(cachep, objp) = RED_ACTIVE;  in cache_alloc_debugcheck_after()
3014 		*dbg_redzone2(cachep, objp) = RED_ACTIVE;  in cache_alloc_debugcheck_after()
3017 	objp += obj_offset(cachep);  in cache_alloc_debugcheck_after()
3019 		cachep->ctor(objp);  in cache_alloc_debugcheck_after()
3021 	    ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {  in cache_alloc_debugcheck_after()
3023 		       objp, (int)ARCH_SLAB_MINALIGN);  in cache_alloc_debugcheck_after()
3025 	return objp;  in cache_alloc_debugcheck_after()
3028 #define cache_alloc_debugcheck_after(a, b, objp, d) (objp)  argument
3033 	void *objp;  in ____cache_alloc()  local
3041 		objp = ac->entry[--ac->avail];  in ____cache_alloc()
3048 	objp = cache_alloc_refill(cachep, flags);  in ____cache_alloc()
3061 	if (objp)  in ____cache_alloc()
3063 	return objp;  in ____cache_alloc()
3268 	void *objp;  in __do_cache_alloc()  local
3271 		objp = alternate_node_alloc(cache, flags);  in __do_cache_alloc()
3272 		if (objp)  in __do_cache_alloc()
3275 	objp = ____cache_alloc(cache, flags);  in __do_cache_alloc()
3281 	if (!objp)  in __do_cache_alloc()
3282 		objp = ____cache_alloc_node(cache, flags, numa_mem_id());  in __do_cache_alloc()
3285 	return objp;  in __do_cache_alloc()
3301 	void *objp;  in slab_alloc()  local
3310 	objp = kfence_alloc(cachep, orig_size, flags);  in slab_alloc()
3311 	if (unlikely(objp))  in slab_alloc()
3316 	objp = __do_cache_alloc(cachep, flags);  in slab_alloc()
3318 	objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);  in slab_alloc()
3319 	prefetchw(objp);  in slab_alloc()
3323 	slab_post_alloc_hook(cachep, objcg, flags, 1, &objp, init);  in slab_alloc()
3324 	return objp;  in slab_alloc()
3341 		void *objp;  in free_block()  local
3344 		objp = objpp[i];  in free_block()
3346 		page = virt_to_head_page(objp);  in free_block()
3349 		slab_put_obj(cachep, page, objp);  in free_block()
3425 static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,  in __cache_free()  argument
3430 	if (is_kfence_address(objp)) {  in __cache_free()
3431 		kmemleak_free_recursive(objp, cachep->flags);  in __cache_free()
3432 		__kfence_free(objp);  in __cache_free()
3443 		memset(objp, 0, cachep->object_size);  in __cache_free()
3445 	if (kasan_slab_free(cachep, objp, init))  in __cache_free()
3450 		__kcsan_check_access(objp, cachep->object_size,  in __cache_free()
3453 	___cache_free(cachep, objp, caller);  in __cache_free()
3456 void ___cache_free(struct kmem_cache *cachep, void *objp,  in ___cache_free()  argument
3462 	kmemleak_free_recursive(objp, cachep->flags);  in ___cache_free()
3463 	objp = cache_free_debugcheck(cachep, objp, caller);  in ___cache_free()
3464 	memcg_slab_free_hook(cachep, &objp, 1);  in ___cache_free()
3473 	if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))  in ___cache_free()
3484 		struct page *page = virt_to_head_page(objp);  in ___cache_free()
3487 			cache_free_pfmemalloc(cachep, page, objp);  in ___cache_free()
3492 	__free_one(ac, objp);  in ___cache_free()
3540 		void *objp = kfence_alloc(s, s->object_size, flags) ?: __do_cache_alloc(s, flags);  in kmem_cache_alloc_bulk()  local
3542 		if (unlikely(!objp))  in kmem_cache_alloc_bulk()
3544 		p[i] = objp;  in kmem_cache_alloc_bulk()
3664 	void *objp;  in kmem_obj_info()  local
3670 	objp = object - obj_offset(cachep);  in kmem_obj_info()
3672 	page = virt_to_head_page(objp);  in kmem_obj_info()
3673 	objnr = obj_to_index(cachep, page, objp);  in kmem_obj_info()
3674 	objp = index_to_obj(cachep, page, objnr);  in kmem_obj_info()
3675 	kpp->kp_objp = objp;  in kmem_obj_info()
3677 		kpp->kp_ret = *dbg_userword(cachep, objp);  in kmem_obj_info()
3729 void kmem_cache_free(struct kmem_cache *cachep, void *objp)  in kmem_cache_free()  argument
3732 	cachep = cache_from_obj(cachep, objp);  in kmem_cache_free()
3737 	debug_check_no_locks_freed(objp, cachep->object_size);  in kmem_cache_free()
3739 		debug_check_no_obj_freed(objp, cachep->object_size);  in kmem_cache_free()
3740 	__cache_free(cachep, objp, _RET_IP_);  in kmem_cache_free()
3743 	trace_kmem_cache_free(_RET_IP_, objp, cachep->name);  in kmem_cache_free()
3754 		void *objp = p[i];  in kmem_cache_free_bulk()  local
3757 			s = virt_to_cache(objp);  in kmem_cache_free_bulk()
3759 			s = cache_from_obj(orig_s, objp);  in kmem_cache_free_bulk()
3763 		debug_check_no_locks_freed(objp, s->object_size);  in kmem_cache_free_bulk()
3765 			debug_check_no_obj_freed(objp, s->object_size);  in kmem_cache_free_bulk()
3767 		__cache_free(s, objp, _RET_IP_);  in kmem_cache_free_bulk()
3784 void kfree(const void *objp)  in kfree()  argument
3789 	trace_kfree(_RET_IP_, objp);  in kfree()
3791 	if (unlikely(ZERO_OR_NULL_PTR(objp)))  in kfree()
3794 	kfree_debugcheck(objp);  in kfree()
3795 	c = virt_to_cache(objp);  in kfree()
3800 	debug_check_no_locks_freed(objp, c->object_size);  in kfree()
3802 	debug_check_no_obj_freed(objp, c->object_size);  in kfree()
3803 	__cache_free(c, (void *)objp, _RET_IP_);  in kfree()
4236 size_t __ksize(const void *objp)  in __ksize()  argument
4241 	BUG_ON(!objp);  in __ksize()
4242 	if (unlikely(objp == ZERO_SIZE_PTR))  in __ksize()
4245 	c = virt_to_cache(objp);  in __ksize()