5  *  memory buffer pool support. Such pools are mostly used
24 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
27 	const int nr = pool->curr_nr;  in poison_error()
33 	pr_err("Mempool %p size %zu\n", pool, size);  in poison_error()
41 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
50 			poison_error(pool, element, size, i);  in __check_element()
57 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
60 	if (pool->free == mempool_free_slab || pool->free == mempool_kfree) {  in check_element()
61 		__check_element(pool, element, ksize(element));  in check_element()
62 	} else if (pool->free == mempool_free_pages) {  in check_element()
64 		int order = (int)(long)pool->pool_data;  in check_element()
67 		__check_element(pool, addr, 1UL << (PAGE_SHIFT + order));  in check_element()
80 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
83 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) {  in poison_element()
85 	} else if (pool->alloc == mempool_alloc_pages) {  in poison_element()
87 		int order = (int)(long)pool->pool_data;  in poison_element()
95 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
98 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
103 static __always_inline void kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
105 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_poison_element()
107 	else if (pool->alloc == mempool_alloc_pages)  in kasan_poison_element()
108 		kasan_poison_pages(element, (unsigned long)pool->pool_data,  in kasan_poison_element()
112 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
114 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_unpoison_element()
116 	else if (pool->alloc == mempool_alloc_pages)  in kasan_unpoison_element()
117 		kasan_unpoison_pages(element, (unsigned long)pool->pool_data,  in kasan_unpoison_element()
121 static __always_inline void add_element(mempool_t *pool, void *element)  in add_element()  argument
123 	BUG_ON(pool->curr_nr >= pool->min_nr);  in add_element()
124 	poison_element(pool, element);  in add_element()
125 	kasan_poison_element(pool, element);  in add_element()
126 	pool->elements[pool->curr_nr++] = element;  in add_element()
129 static void *remove_element(mempool_t *pool)  in remove_element()  argument
131 	void *element = pool->elements[--pool->curr_nr];  in remove_element()
133 	BUG_ON(pool->curr_nr < 0);  in remove_element()
134 	kasan_unpoison_element(pool, element);  in remove_element()
135 	check_element(pool, element);  in remove_element()
141  * @pool:      pointer to the memory pool which was initialized with
144  * Free all reserved elements in @pool and @pool itself.  This function
150 void mempool_exit(mempool_t *pool)  in mempool_exit()  argument
152 	while (pool->curr_nr) {  in mempool_exit()
153 		void *element = remove_element(pool);  in mempool_exit()
154 		pool->free(element, pool->pool_data);  in mempool_exit()
156 	kfree(pool->elements);  in mempool_exit()
157 	pool->elements = NULL;  in mempool_exit()
162  * mempool_destroy - deallocate a memory pool
163  * @pool:      pointer to the memory pool which was allocated via
166  * Free all reserved elements in @pool and @pool itself.  This function
169 void mempool_destroy(mempool_t *pool)  in mempool_destroy()  argument
171 	if (unlikely(!pool))  in mempool_destroy()
174 	mempool_exit(pool);  in mempool_destroy()
175 	kfree(pool);  in mempool_destroy()
179 int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init_node()  argument
183 	spin_lock_init(&pool->lock);  in mempool_init_node()
184 	pool->min_nr	= min_nr;  in mempool_init_node()
185 	pool->pool_data = pool_data;  in mempool_init_node()
186 	pool->alloc	= alloc_fn;  in mempool_init_node()
187 	pool->free	= free_fn;  in mempool_init_node()
188 	init_waitqueue_head(&pool->wait);  in mempool_init_node()
190 	pool->elements = kmalloc_array_node(min_nr, sizeof(void *),  in mempool_init_node()
192 	if (!pool->elements)  in mempool_init_node()
198 	while (pool->curr_nr < pool->min_nr) {  in mempool_init_node()
201 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_init_node()
203 			mempool_exit(pool);  in mempool_init_node()
206 		add_element(pool, element);  in mempool_init_node()
214  * mempool_init - initialize a memory pool
215  * @pool:      pointer to the memory pool that should be initialized
217  *             allocated for this pool.
222  * Like mempool_create(), but initializes the pool in (i.e. embedded in another
227 int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init()  argument
230 	return mempool_init_node(pool, min_nr, alloc_fn, free_fn,  in mempool_init()
237  * mempool_create - create a memory pool
239  *             allocated for this pool.
245  * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
250  * Return: pointer to the created memory pool object or %NULL on error.
264 	mempool_t *pool;  in mempool_create_node()  local
266 	pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);  in mempool_create_node()
267 	if (!pool)  in mempool_create_node()
270 	if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,  in mempool_create_node()
272 		kfree(pool);  in mempool_create_node()
276 	return pool;  in mempool_create_node()
281  * mempool_resize - resize an existing memory pool
282  * @pool:       pointer to the memory pool which was allocated via
285  *              allocated for this pool.
287  * This function shrinks/grows the pool. In the case of growing,
288  * it cannot be guaranteed that the pool will be grown to the new
298 int mempool_resize(mempool_t *pool, int new_min_nr)  in mempool_resize()  argument
307 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
308 	if (new_min_nr <= pool->min_nr) {  in mempool_resize()
309 		while (new_min_nr < pool->curr_nr) {  in mempool_resize()
310 			element = remove_element(pool);  in mempool_resize()
311 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
312 			pool->free(element, pool->pool_data);  in mempool_resize()
313 			spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
315 		pool->min_nr = new_min_nr;  in mempool_resize()
318 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
320 	/* Grow the pool */  in mempool_resize()
326 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
327 	if (unlikely(new_min_nr <= pool->min_nr)) {  in mempool_resize()
329 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
333 	memcpy(new_elements, pool->elements,  in mempool_resize()
334 			pool->curr_nr * sizeof(*new_elements));  in mempool_resize()
335 	kfree(pool->elements);  in mempool_resize()
336 	pool->elements = new_elements;  in mempool_resize()
337 	pool->min_nr = new_min_nr;  in mempool_resize()
339 	while (pool->curr_nr < pool->min_nr) {  in mempool_resize()
340 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
341 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
344 		spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
345 		if (pool->curr_nr < pool->min_nr) {  in mempool_resize()
346 			add_element(pool, element);  in mempool_resize()
348 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
349 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
354 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
361  * mempool_alloc - allocate an element from a specific memory pool
362  * @pool:      pointer to the memory pool which was allocated via
374 void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)  in mempool_alloc()  argument
392 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc()
396 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc()
397 	if (likely(pool->curr_nr)) {  in mempool_alloc()
398 		element = remove_element(pool);  in mempool_alloc()
399 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
412 	 * alloc failed with that and @pool was empty, retry immediately.  in mempool_alloc()
415 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
422 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
426 	/* Let's wait for someone else to return an element to @pool */  in mempool_alloc()
428 	prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);  in mempool_alloc()
430 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
438 	finish_wait(&pool->wait, &wait);  in mempool_alloc()
444  * mempool_free - return an element to the pool.
445  * @element:   pool element pointer.
446  * @pool:      pointer to the memory pool which was allocated via
451 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
460 	 * for @element and the following @pool->curr_nr.  This ensures  in mempool_free()
461 	 * that the visible value of @pool->curr_nr is from after the  in mempool_free()
489 	 * pool waking up the waiters.  in mempool_free()
491 	if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {  in mempool_free()
492 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free()
493 		if (likely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
494 			add_element(pool, element);  in mempool_free()
495 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
496 			wake_up(&pool->wait);  in mempool_free()
499 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
501 	pool->free(element, pool->pool_data);  in mempool_free()