255 void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,  in kasan_cache_create()  argument
263 	cache->kasan_info.alloc_meta_offset = *size;  in kasan_cache_create()
268 	    (cache->flags & SLAB_TYPESAFE_BY_RCU || cache->ctor ||  in kasan_cache_create()
269 	     cache->object_size < sizeof(struct kasan_free_meta))) {  in kasan_cache_create()
270 		cache->kasan_info.free_meta_offset = *size;  in kasan_cache_create()
274 	redzone_size = optimal_redzone(cache->object_size);  in kasan_cache_create()
275 	redzone_adjust = redzone_size -	(*size - cache->object_size);  in kasan_cache_create()
280 			max(*size, cache->object_size + redzone_size));  in kasan_cache_create()
285 	if (*size <= cache->kasan_info.alloc_meta_offset ||  in kasan_cache_create()
286 			*size <= cache->kasan_info.free_meta_offset) {  in kasan_cache_create()
287 		cache->kasan_info.alloc_meta_offset = 0;  in kasan_cache_create()
288 		cache->kasan_info.free_meta_offset = 0;  in kasan_cache_create()
296 size_t kasan_metadata_size(struct kmem_cache *cache)  in kasan_metadata_size()  argument
298 	return (cache->kasan_info.alloc_meta_offset ?  in kasan_metadata_size()
300 		(cache->kasan_info.free_meta_offset ?  in kasan_metadata_size()
304 struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache,  in get_alloc_info()  argument
307 	return (void *)object + cache->kasan_info.alloc_meta_offset;  in get_alloc_info()
310 struct kasan_free_meta *get_free_info(struct kmem_cache *cache,  in get_free_info()  argument
314 	return (void *)object + cache->kasan_info.free_meta_offset;  in get_free_info()
318 static void kasan_set_free_info(struct kmem_cache *cache,  in kasan_set_free_info()  argument
324 	alloc_meta = get_alloc_info(cache, object);  in kasan_set_free_info()
345 void kasan_unpoison_object_data(struct kmem_cache *cache, void *object)  in kasan_unpoison_object_data()  argument
347 	kasan_unpoison_shadow(object, cache->object_size);  in kasan_unpoison_object_data()
350 void kasan_poison_object_data(struct kmem_cache *cache, void *object)  in kasan_poison_object_data()  argument
353 			round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE),  in kasan_poison_object_data()
371 static u8 assign_tag(struct kmem_cache *cache, const void *object,  in assign_tag()  argument
387 	if (!cache->ctor && !(cache->flags & SLAB_TYPESAFE_BY_RCU))  in assign_tag()
393 	return (u8)obj_to_index(cache, virt_to_page(object), (void *)object);  in assign_tag()
403 void * __must_check kasan_init_slab_obj(struct kmem_cache *cache,  in kasan_init_slab_obj()  argument
408 	if (!(cache->flags & SLAB_KASAN))  in kasan_init_slab_obj()
411 	alloc_info = get_alloc_info(cache, object);  in kasan_init_slab_obj()
416 				assign_tag(cache, object, true, false));  in kasan_init_slab_obj()
436 static bool __kasan_slab_free(struct kmem_cache *cache, void *object,  in __kasan_slab_free()  argument
448 	if (unlikely(nearest_obj(cache, virt_to_head_page(object), object) !=  in __kasan_slab_free()
455 	if (unlikely(cache->flags & SLAB_TYPESAFE_BY_RCU))  in __kasan_slab_free()
464 	rounded_up_size = round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE);  in __kasan_slab_free()
468 			unlikely(!(cache->flags & SLAB_KASAN)))  in __kasan_slab_free()
471 	kasan_set_free_info(cache, object, tag);  in __kasan_slab_free()
473 	quarantine_put(get_free_info(cache, object), cache);  in __kasan_slab_free()
478 bool kasan_slab_free(struct kmem_cache *cache, void *object, unsigned long ip)  in kasan_slab_free()  argument
480 	return __kasan_slab_free(cache, object, ip, true);  in kasan_slab_free()
483 static void *__kasan_kmalloc(struct kmem_cache *cache, const void *object,  in __kasan_kmalloc()  argument
498 	redzone_end = round_up((unsigned long)object + cache->object_size,  in __kasan_kmalloc()
502 		tag = assign_tag(cache, object, false, keep_tag);  in __kasan_kmalloc()
509 	if (cache->flags & SLAB_KASAN)  in __kasan_kmalloc()
510 		set_track(&get_alloc_info(cache, object)->alloc_track, flags);  in __kasan_kmalloc()
515 void * __must_check kasan_slab_alloc(struct kmem_cache *cache, void *object,  in kasan_slab_alloc()  argument
518 	return __kasan_kmalloc(cache, object, cache->object_size, flags, false);  in kasan_slab_alloc()
521 void * __must_check kasan_kmalloc(struct kmem_cache *cache, const void *object,  in kasan_kmalloc()  argument
524 	return __kasan_kmalloc(cache, object, size, flags, true);  in kasan_kmalloc()