24 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
32 	pr_err("BUG: mempool element poison mismatch\n");  in poison_error()
34 	pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : "");  in poison_error()
36 		pr_cont("%x ", *(u8 *)(element + i));  in poison_error()
41 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
43 	u8 *obj = element;  in __check_element()
50 			poison_error(pool, element, size, i);  in __check_element()
57 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
61 		__check_element(pool, element, ksize(element));  in check_element()
65 		void *addr = kmap_atomic((struct page *)element);  in check_element()
72 static void __poison_element(void *element, size_t size)  in __poison_element()  argument
74 	u8 *obj = element;  in __poison_element()
80 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
84 		__poison_element(element, ksize(element));  in poison_element()
88 		void *addr = kmap_atomic((struct page *)element);  in poison_element()
95 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
98 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
103 static __always_inline void kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
106 		kasan_slab_free_mempool(element);  in kasan_poison_element()
108 		kasan_poison_pages(element, (unsigned long)pool->pool_data,  in kasan_poison_element()
112 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
115 		kasan_unpoison_range(element, __ksize(element));  in kasan_unpoison_element()
117 		kasan_unpoison_pages(element, (unsigned long)pool->pool_data,  in kasan_unpoison_element()
121 static __always_inline void add_element(mempool_t *pool, void *element)  in add_element()  argument
124 	poison_element(pool, element);  in add_element()
125 	kasan_poison_element(pool, element);  in add_element()
126 	pool->elements[pool->curr_nr++] = element;  in add_element()
131 	void *element = pool->elements[--pool->curr_nr];  in remove_element()  local
134 	kasan_unpoison_element(pool, element);  in remove_element()
135 	check_element(pool, element);  in remove_element()
136 	return element;  in remove_element()
153 		void *element = remove_element(pool);  in mempool_exit()  local
154 		pool->free(element, pool->pool_data);  in mempool_exit()
199 		void *element;  in mempool_init_node()  local
201 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_init_node()
202 		if (unlikely(!element)) {  in mempool_init_node()
206 		add_element(pool, element);  in mempool_init_node()
218  * @alloc_fn:  user-defined element-allocation function.
219  * @free_fn:   user-defined element-freeing function.
240  * @alloc_fn:  user-defined element-allocation function.
241  * @free_fn:   user-defined element-freeing function.
300 	void *element;  in mempool_resize()  local
310 			element = remove_element(pool);  in mempool_resize()
312 			pool->free(element, pool->pool_data);  in mempool_resize()
341 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
342 		if (!element)  in mempool_resize()
346 			add_element(pool, element);  in mempool_resize()
349 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
361  * mempool_alloc - allocate an element from a specific memory pool
372  * Return: pointer to the allocated element or %NULL on error.
376 	void *element;  in mempool_alloc()  local
392 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc()
393 	if (likely(element != NULL))  in mempool_alloc()
394 		return element;  in mempool_alloc()
398 		element = remove_element(pool);  in mempool_alloc()
406 		kmemleak_update_trace(element);  in mempool_alloc()
407 		return element;  in mempool_alloc()
426 	/* Let's wait for someone else to return an element to @pool */  in mempool_alloc()
444  * mempool_free - return an element to the pool.
445  * @element:   pool element pointer.
451 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
455 	if (unlikely(element == NULL))  in mempool_free()
460 	 * for @element and the following @pool->curr_nr.  This ensures  in mempool_free()
462 	 * allocation of @element.  This is necessary for fringe cases  in mempool_free()
463 	 * where @element was passed to this task without going through  in mempool_free()
481 	 * allocation of @element, any task which decremented curr_nr below  in mempool_free()
484 	 * to min_nr after the allocation of @element, the elements  in mempool_free()
494 			add_element(pool, element);  in mempool_free()
501 	pool->free(element, pool->pool_data);  in mempool_free()
516 void mempool_free_slab(void *element, void *pool_data)  in mempool_free_slab()  argument
519 	kmem_cache_free(mem, element);  in mempool_free_slab()
534 void mempool_kfree(void *element, void *pool_data)  in mempool_kfree()  argument
536 	kfree(element);  in mempool_kfree()
551 void mempool_free_pages(void *element, void *pool_data)  in mempool_free_pages()  argument
554 	__free_pages(element, order);  in mempool_free_pages()