333 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)  in dbg_redzone1()  argument
336 	return (unsigned long long *) (objp + obj_offset(cachep) -  in dbg_redzone1()
340 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)  in dbg_redzone2()  argument
344 		return (unsigned long long *)(objp + cachep->size -  in dbg_redzone2()
347 	return (unsigned long long *) (objp + cachep->size -  in dbg_redzone2()
351 static void **dbg_userword(struct kmem_cache *cachep, void *objp)  in dbg_userword()  argument
354 	return (void **)(objp + cachep->size - BYTES_PER_WORD);  in dbg_userword()
360 #define dbg_redzone1(cachep, objp)	({BUG(); (unsigned long long *)NULL;})  argument
361 #define dbg_redzone2(cachep, objp)	({BUG(); (unsigned long long *)NULL;})  argument
362 #define dbg_userword(cachep, objp)	({BUG(); (void **)NULL;})  argument
553 					struct slab *slab, void *objp)  in cache_free_pfmemalloc()  argument
563 	free_block(cachep, &objp, 1, slab_node, &list);  in cache_free_pfmemalloc()
593 static __always_inline void __free_one(struct array_cache *ac, void *objp)  in __free_one()  argument
597 	    WARN_ON_ONCE(ac->avail > 0 && ac->entry[ac->avail - 1] == objp))  in __free_one()
599 	ac->entry[ac->avail++] = objp;  in __free_one()
617 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)  in cache_free_alien()  argument
748 static int __cache_free_alien(struct kmem_cache *cachep, void *objp,  in __cache_free_alien()  argument
766 		__free_one(ac, objp);  in __cache_free_alien()
772 		free_block(cachep, &objp, 1, slab_node, &list);  in __cache_free_alien()
779 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)  in cache_free_alien()  argument
781 	int slab_node = slab_nid(virt_to_slab(objp));  in cache_free_alien()
790 	return __cache_free_alien(cachep, objp, node, slab_node);  in cache_free_alien()
1422 static void slab_kernel_map(struct kmem_cache *cachep, void *objp, int map)  in slab_kernel_map()  argument
1427 	__kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map);  in slab_kernel_map()
1431 static inline void slab_kernel_map(struct kmem_cache *cachep, void *objp,  in slab_kernel_map()  argument
1477 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)  in print_objinfo()  argument
1484 		       *dbg_redzone1(cachep, objp),  in print_objinfo()
1485 		       *dbg_redzone2(cachep, objp));  in print_objinfo()
1489 		pr_err("Last user: (%pSR)\n", *dbg_userword(cachep, objp));  in print_objinfo()
1490 	realobj = (char *)objp + obj_offset(cachep);  in print_objinfo()
1501 static void check_poison_obj(struct kmem_cache *cachep, void *objp)  in check_poison_obj()  argument
1510 	realobj = (char *)objp + obj_offset(cachep);  in check_poison_obj()
1525 				print_objinfo(cachep, objp, 0);  in check_poison_obj()
1544 		struct slab *slab = virt_to_slab(objp);  in check_poison_obj()
1547 		objnr = obj_to_index(cachep, slab, objp);  in check_poison_obj()
1549 			objp = index_to_obj(cachep, slab, objnr - 1);  in check_poison_obj()
1550 			realobj = (char *)objp + obj_offset(cachep);  in check_poison_obj()
1552 			print_objinfo(cachep, objp, 2);  in check_poison_obj()
1555 			objp = index_to_obj(cachep, slab, objnr + 1);  in check_poison_obj()
1556 			realobj = (char *)objp + obj_offset(cachep);  in check_poison_obj()
1558 			print_objinfo(cachep, objp, 2);  in check_poison_obj()
1576 		void *objp = index_to_obj(cachep, slab, i);  in slab_destroy_debugcheck()  local
1579 			check_poison_obj(cachep, objp);  in slab_destroy_debugcheck()
1580 			slab_kernel_map(cachep, objp, 1);  in slab_destroy_debugcheck()
1583 			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)  in slab_destroy_debugcheck()
1585 			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)  in slab_destroy_debugcheck()
2324 		void *objp = index_to_obj(cachep, slab, i);  in cache_init_objs_debug()  local
2327 			*dbg_userword(cachep, objp) = NULL;  in cache_init_objs_debug()
2330 			*dbg_redzone1(cachep, objp) = RED_INACTIVE;  in cache_init_objs_debug()
2331 			*dbg_redzone2(cachep, objp) = RED_INACTIVE;  in cache_init_objs_debug()
2340 						   objp + obj_offset(cachep));  in cache_init_objs_debug()
2341 			cachep->ctor(objp + obj_offset(cachep));  in cache_init_objs_debug()
2343 				cachep, objp + obj_offset(cachep));  in cache_init_objs_debug()
2347 			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)  in cache_init_objs_debug()
2349 			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)  in cache_init_objs_debug()
2354 			poison_obj(cachep, objp, POISON_FREE);  in cache_init_objs_debug()
2355 			slab_kernel_map(cachep, objp, 0);  in cache_init_objs_debug()
2476 	void *objp;  in cache_init_objs()  local
2490 		objp = index_to_obj(cachep, slab, i);  in cache_init_objs()
2491 		objp = kasan_init_slab_obj(cachep, objp);  in cache_init_objs()
2495 			kasan_unpoison_object_data(cachep, objp);  in cache_init_objs()
2496 			cachep->ctor(objp);  in cache_init_objs()
2497 			kasan_poison_object_data(cachep, objp);  in cache_init_objs()
2507 	void *objp;  in slab_get_obj()  local
2509 	objp = index_to_obj(cachep, slab, get_free_obj(slab, slab->active));  in slab_get_obj()
2512 	return objp;  in slab_get_obj()
2516 			struct slab *slab, void *objp)  in slab_put_obj()  argument
2518 	unsigned int objnr = obj_to_index(cachep, slab, objp);  in slab_put_obj()
2526 			       cachep->name, objp);  in slab_put_obj()
2533 		slab->freelist = objp + obj_offset(cachep);  in slab_put_obj()
2654 static void kfree_debugcheck(const void *objp)  in kfree_debugcheck()  argument
2656 	if (!virt_addr_valid(objp)) {  in kfree_debugcheck()
2658 		       (unsigned long)objp);  in kfree_debugcheck()
2685 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,  in cache_free_debugcheck()  argument
2691 	BUG_ON(virt_to_cache(objp) != cachep);  in cache_free_debugcheck()
2693 	objp -= obj_offset(cachep);  in cache_free_debugcheck()
2694 	kfree_debugcheck(objp);  in cache_free_debugcheck()
2695 	slab = virt_to_slab(objp);  in cache_free_debugcheck()
2698 		verify_redzone_free(cachep, objp);  in cache_free_debugcheck()
2699 		*dbg_redzone1(cachep, objp) = RED_INACTIVE;  in cache_free_debugcheck()
2700 		*dbg_redzone2(cachep, objp) = RED_INACTIVE;  in cache_free_debugcheck()
2703 		*dbg_userword(cachep, objp) = (void *)caller;  in cache_free_debugcheck()
2705 	objnr = obj_to_index(cachep, slab, objp);  in cache_free_debugcheck()
2708 	BUG_ON(objp != index_to_obj(cachep, slab, objnr));  in cache_free_debugcheck()
2711 		poison_obj(cachep, objp, POISON_FREE);  in cache_free_debugcheck()
2712 		slab_kernel_map(cachep, objp, 0);  in cache_free_debugcheck()
2714 	return objp;  in cache_free_debugcheck()
2719 #define cache_free_debugcheck(x, objp, z) (objp)  argument
2727 	void *objp;  in fixup_objfreelist_debug()  local
2730 		objp = next - obj_offset(cachep);  in fixup_objfreelist_debug()
2732 		poison_obj(cachep, objp, POISON_FREE);  in fixup_objfreelist_debug()
2749 				void **objp = slab->freelist;  in fixup_slab_list()  local
2751 				*objp = *list;  in fixup_slab_list()
2752 				*list = objp;  in fixup_slab_list()
2964 				gfp_t flags, void *objp, unsigned long caller)  in cache_alloc_debugcheck_after()  argument
2967 	if (!objp || is_kfence_address(objp))  in cache_alloc_debugcheck_after()
2968 		return objp;  in cache_alloc_debugcheck_after()
2970 		check_poison_obj(cachep, objp);  in cache_alloc_debugcheck_after()
2971 		slab_kernel_map(cachep, objp, 1);  in cache_alloc_debugcheck_after()
2972 		poison_obj(cachep, objp, POISON_INUSE);  in cache_alloc_debugcheck_after()
2975 		*dbg_userword(cachep, objp) = (void *)caller;  in cache_alloc_debugcheck_after()
2978 		if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||  in cache_alloc_debugcheck_after()
2979 				*dbg_redzone2(cachep, objp) != RED_INACTIVE) {  in cache_alloc_debugcheck_after()
2982 			       objp, *dbg_redzone1(cachep, objp),  in cache_alloc_debugcheck_after()
2983 			       *dbg_redzone2(cachep, objp));  in cache_alloc_debugcheck_after()
2985 		*dbg_redzone1(cachep, objp) = RED_ACTIVE;  in cache_alloc_debugcheck_after()
2986 		*dbg_redzone2(cachep, objp) = RED_ACTIVE;  in cache_alloc_debugcheck_after()
2989 	objp += obj_offset(cachep);  in cache_alloc_debugcheck_after()
2991 		cachep->ctor(objp);  in cache_alloc_debugcheck_after()
2992 	if ((unsigned long)objp & (arch_slab_minalign() - 1)) {  in cache_alloc_debugcheck_after()
2993 		pr_err("0x%px: not aligned to arch_slab_minalign()=%u\n", objp,  in cache_alloc_debugcheck_after()
2996 	return objp;  in cache_alloc_debugcheck_after()
2999 #define cache_alloc_debugcheck_after(a, b, objp, d) (objp)  argument
3004 	void *objp;  in ____cache_alloc()  local
3012 		objp = ac->entry[--ac->avail];  in ____cache_alloc()
3019 	objp = cache_alloc_refill(cachep, flags);  in ____cache_alloc()
3032 	if (objp)  in ____cache_alloc()
3034 	return objp;  in ____cache_alloc()
3187 	void *objp = NULL;  in __do_cache_alloc()  local
3192 			objp = alternate_node_alloc(cachep, flags);  in __do_cache_alloc()
3193 			if (objp)  in __do_cache_alloc()
3202 		objp = ____cache_alloc(cachep, flags);  in __do_cache_alloc()
3205 		objp = ____cache_alloc(cachep, flags);  in __do_cache_alloc()
3208 		objp = fallback_alloc(cachep, flags);  in __do_cache_alloc()
3216 	if (!objp)  in __do_cache_alloc()
3217 		objp = ____cache_alloc_node(cachep, flags, nodeid);  in __do_cache_alloc()
3219 	return objp;  in __do_cache_alloc()
3236 	void *objp;  in slab_alloc_node()  local
3245 	objp = kfence_alloc(cachep, orig_size, flags);  in slab_alloc_node()
3246 	if (unlikely(objp))  in slab_alloc_node()
3250 	objp = __do_cache_alloc(cachep, flags, nodeid);  in slab_alloc_node()
3252 	objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);  in slab_alloc_node()
3253 	prefetchw(objp);  in slab_alloc_node()
3257 	slab_post_alloc_hook(cachep, objcg, flags, 1, &objp, init);  in slab_alloc_node()
3258 	return objp;  in slab_alloc_node()
3283 		void *objp;  in free_block()  local
3286 		objp = objpp[i];  in free_block()
3288 		slab = virt_to_slab(objp);  in free_block()
3291 		slab_put_obj(cachep, slab, objp);  in free_block()
3367 static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,  in __cache_free()  argument
3372 	memcg_slab_free_hook(cachep, virt_to_slab(objp), &objp, 1);  in __cache_free()
3374 	if (is_kfence_address(objp)) {  in __cache_free()
3375 		kmemleak_free_recursive(objp, cachep->flags);  in __cache_free()
3376 		__kfence_free(objp);  in __cache_free()
3387 		memset(objp, 0, cachep->object_size);  in __cache_free()
3389 	if (kasan_slab_free(cachep, objp, init))  in __cache_free()
3394 		__kcsan_check_access(objp, cachep->object_size,  in __cache_free()
3397 	___cache_free(cachep, objp, caller);  in __cache_free()
3400 void ___cache_free(struct kmem_cache *cachep, void *objp,  in ___cache_free()  argument
3406 	kmemleak_free_recursive(objp, cachep->flags);  in ___cache_free()
3407 	objp = cache_free_debugcheck(cachep, objp, caller);  in ___cache_free()
3416 	if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))  in ___cache_free()
3427 		struct slab *slab = virt_to_slab(objp);  in ___cache_free()
3430 			cache_free_pfmemalloc(cachep, slab, objp);  in ___cache_free()
3435 	__free_one(ac, objp);  in ___cache_free()
3494 		void *objp = kfence_alloc(s, s->object_size, flags) ?:  in kmem_cache_alloc_bulk()  local
3497 		if (unlikely(!objp))  in kmem_cache_alloc_bulk()
3499 		p[i] = objp;  in kmem_cache_alloc_bulk()
3558 	void *objp;  in __kmem_obj_info()  local
3564 	objp = object - obj_offset(cachep);  in __kmem_obj_info()
3566 	slab = virt_to_slab(objp);  in __kmem_obj_info()
3567 	objnr = obj_to_index(cachep, slab, objp);  in __kmem_obj_info()
3568 	objp = index_to_obj(cachep, slab, objnr);  in __kmem_obj_info()
3569 	kpp->kp_objp = objp;  in __kmem_obj_info()
3571 		kpp->kp_ret = *dbg_userword(cachep, objp);  in __kmem_obj_info()
3576 void __do_kmem_cache_free(struct kmem_cache *cachep, void *objp,  in __do_kmem_cache_free()  argument
3582 	debug_check_no_locks_freed(objp, cachep->object_size);  in __do_kmem_cache_free()
3584 		debug_check_no_obj_freed(objp, cachep->object_size);  in __do_kmem_cache_free()
3585 	__cache_free(cachep, objp, caller);  in __do_kmem_cache_free()
3589 void __kmem_cache_free(struct kmem_cache *cachep, void *objp,  in __kmem_cache_free()  argument
3592 	__do_kmem_cache_free(cachep, objp, caller);  in __kmem_cache_free()
3603 void kmem_cache_free(struct kmem_cache *cachep, void *objp)  in kmem_cache_free()  argument
3605 	cachep = cache_from_obj(cachep, objp);  in kmem_cache_free()
3609 	trace_kmem_cache_free(_RET_IP_, objp, cachep);  in kmem_cache_free()
3610 	__do_kmem_cache_free(cachep, objp, _RET_IP_);  in kmem_cache_free()
3619 		void *objp = p[i];  in kmem_cache_free_bulk()  local
3623 			struct folio *folio = virt_to_folio(objp);  in kmem_cache_free_bulk()
3628 				free_large_kmalloc(folio, objp);  in kmem_cache_free_bulk()
3634 			s = cache_from_obj(orig_s, objp);  in kmem_cache_free_bulk()
3640 		debug_check_no_locks_freed(objp, s->object_size);  in kmem_cache_free_bulk()
3642 			debug_check_no_obj_freed(objp, s->object_size);  in kmem_cache_free_bulk()
3644 		__cache_free(s, objp, _RET_IP_);  in kmem_cache_free_bulk()