17 static int page_pool_init(struct page_pool *pool,  in page_pool_init()  argument
22 	memcpy(&pool->p, params, sizeof(pool->p));  in page_pool_init()
25 	if (pool->p.flags & ~(PP_FLAG_ALL))  in page_pool_init()
28 	if (pool->p.pool_size)  in page_pool_init()
29 		ring_qsize = pool->p.pool_size;  in page_pool_init()
39 	if ((pool->p.dma_dir != DMA_FROM_DEVICE) &&  in page_pool_init()
40 	    (pool->p.dma_dir != DMA_BIDIRECTIONAL))  in page_pool_init()
43 	if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0)  in page_pool_init()
51 	struct page_pool *pool;  in page_pool_create()  local
54 	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid);  in page_pool_create()
55 	if (!pool)  in page_pool_create()
58 	err = page_pool_init(pool, params);  in page_pool_create()
61 		kfree(pool);  in page_pool_create()
64 	return pool;  in page_pool_create()
69 static struct page *__page_pool_get_cached(struct page_pool *pool)  in __page_pool_get_cached()  argument
71 	struct ptr_ring *r = &pool->ring;  in __page_pool_get_cached()
80 		if (likely(pool->alloc.count)) {  in __page_pool_get_cached()
82 			page = pool->alloc.cache[--pool->alloc.count];  in __page_pool_get_cached()
96 			if (pool->alloc.count == PP_ALLOC_CACHE_REFILL)  in __page_pool_get_cached()
98 			pool->alloc.cache[pool->alloc.count++] = page;  in __page_pool_get_cached()
105 	page = ptr_ring_consume(&pool->ring);  in __page_pool_get_cached()
111 static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,  in __page_pool_alloc_pages_slow()  argument
121 	if (pool->p.order)  in __page_pool_alloc_pages_slow()
132 	page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);  in __page_pool_alloc_pages_slow()
136 	if (!(pool->p.flags & PP_FLAG_DMA_MAP))  in __page_pool_alloc_pages_slow()
142 	dma = dma_map_page(pool->p.dev, page, 0,  in __page_pool_alloc_pages_slow()
143 			   (PAGE_SIZE << pool->p.order),  in __page_pool_alloc_pages_slow()
144 			   pool->p.dma_dir);  in __page_pool_alloc_pages_slow()
145 	if (dma_mapping_error(pool->p.dev, dma)) {  in __page_pool_alloc_pages_slow()
159 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)  in page_pool_alloc_pages()  argument
164 	page = __page_pool_get_cached(pool);  in page_pool_alloc_pages()
169 	page = __page_pool_alloc_pages_slow(pool, gfp);  in page_pool_alloc_pages()
175 static void __page_pool_clean_page(struct page_pool *pool,  in __page_pool_clean_page()  argument
178 	if (!(pool->p.flags & PP_FLAG_DMA_MAP))  in __page_pool_clean_page()
182 	dma_unmap_page(pool->p.dev, page_private(page),  in __page_pool_clean_page()
183 		       PAGE_SIZE << pool->p.order, pool->p.dma_dir);  in __page_pool_clean_page()
188 static void __page_pool_return_page(struct page_pool *pool, struct page *page)  in __page_pool_return_page()  argument
190 	__page_pool_clean_page(pool, page);  in __page_pool_return_page()
198 static bool __page_pool_recycle_into_ring(struct page_pool *pool,  in __page_pool_recycle_into_ring()  argument
204 		ret = ptr_ring_produce(&pool->ring, page);  in __page_pool_recycle_into_ring()
206 		ret = ptr_ring_produce_bh(&pool->ring, page);  in __page_pool_recycle_into_ring()
217 				       struct page_pool *pool)  in __page_pool_recycle_direct()  argument
219 	if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE))  in __page_pool_recycle_direct()
223 	pool->alloc.cache[pool->alloc.count++] = page;  in __page_pool_recycle_direct()
227 void __page_pool_put_page(struct page_pool *pool,  in __page_pool_put_page()  argument
240 			if (__page_pool_recycle_direct(page, pool))  in __page_pool_put_page()
243 		if (!__page_pool_recycle_into_ring(pool, page)) {  in __page_pool_put_page()
245 			__page_pool_return_page(pool, page);  in __page_pool_put_page()
262 	__page_pool_clean_page(pool, page);  in __page_pool_put_page()
267 static void __page_pool_empty_ring(struct page_pool *pool)  in __page_pool_empty_ring()  argument
272 	while ((page = ptr_ring_consume_bh(&pool->ring))) {  in __page_pool_empty_ring()
278 		__page_pool_return_page(pool, page);  in __page_pool_empty_ring()
284 	struct page_pool *pool;  in __page_pool_destroy_rcu()  local
286 	pool = container_of(rcu, struct page_pool, rcu);  in __page_pool_destroy_rcu()
288 	WARN(pool->alloc.count, "API usage violation");  in __page_pool_destroy_rcu()
290 	__page_pool_empty_ring(pool);  in __page_pool_destroy_rcu()
291 	ptr_ring_cleanup(&pool->ring, NULL);  in __page_pool_destroy_rcu()
292 	kfree(pool);  in __page_pool_destroy_rcu()
296 void page_pool_destroy(struct page_pool *pool)  in page_pool_destroy()  argument
304 	while (pool->alloc.count) {  in page_pool_destroy()
305 		page = pool->alloc.cache[--pool->alloc.count];  in page_pool_destroy()
306 		__page_pool_return_page(pool, page);  in page_pool_destroy()
312 	__page_pool_empty_ring(pool);  in page_pool_destroy()
315 	call_rcu(&pool->rcu, __page_pool_destroy_rcu);  in page_pool_destroy()