21 static int page_pool_init(struct page_pool *pool,  in page_pool_init()  argument
26 	memcpy(&pool->p, params, sizeof(pool->p));  in page_pool_init()
29 	if (pool->p.flags & ~(PP_FLAG_ALL))  in page_pool_init()
32 	if (pool->p.pool_size)  in page_pool_init()
33 		ring_qsize = pool->p.pool_size;  in page_pool_init()
43 	if ((pool->p.dma_dir != DMA_FROM_DEVICE) &&  in page_pool_init()
44 	    (pool->p.dma_dir != DMA_BIDIRECTIONAL))  in page_pool_init()
47 	if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0)  in page_pool_init()
50 	atomic_set(&pool->pages_state_release_cnt, 0);  in page_pool_init()
53 	refcount_set(&pool->user_cnt, 1);  in page_pool_init()
55 	if (pool->p.flags & PP_FLAG_DMA_MAP)  in page_pool_init()
56 		get_device(pool->p.dev);  in page_pool_init()
63 	struct page_pool *pool;  in page_pool_create()  local
66 	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid);  in page_pool_create()
67 	if (!pool)  in page_pool_create()
70 	err = page_pool_init(pool, params);  in page_pool_create()
73 		kfree(pool);  in page_pool_create()
77 	return pool;  in page_pool_create()
82 static struct page *__page_pool_get_cached(struct page_pool *pool)  in __page_pool_get_cached()  argument
84 	struct ptr_ring *r = &pool->ring;  in __page_pool_get_cached()
90 		if (likely(pool->alloc.count)) {  in __page_pool_get_cached()
92 			page = pool->alloc.cache[--pool->alloc.count];  in __page_pool_get_cached()
108 		pool->alloc.count = __ptr_ring_consume_batched(r,  in __page_pool_get_cached()
109 							pool->alloc.cache,  in __page_pool_get_cached()
117 static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,  in __page_pool_alloc_pages_slow()  argument
127 	if (pool->p.order)  in __page_pool_alloc_pages_slow()
138 	page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);  in __page_pool_alloc_pages_slow()
142 	if (!(pool->p.flags & PP_FLAG_DMA_MAP))  in __page_pool_alloc_pages_slow()
150 	dma = dma_map_page_attrs(pool->p.dev, page, 0,  in __page_pool_alloc_pages_slow()
151 				 (PAGE_SIZE << pool->p.order),  in __page_pool_alloc_pages_slow()
152 				 pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC);  in __page_pool_alloc_pages_slow()
153 	if (dma_mapping_error(pool->p.dev, dma)) {  in __page_pool_alloc_pages_slow()
161 	pool->pages_state_hold_cnt++;  in __page_pool_alloc_pages_slow()
163 	trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt);  in __page_pool_alloc_pages_slow()
172 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)  in page_pool_alloc_pages()  argument
177 	page = __page_pool_get_cached(pool);  in page_pool_alloc_pages()
182 	page = __page_pool_alloc_pages_slow(pool, gfp);  in page_pool_alloc_pages()
192 static s32 page_pool_inflight(struct page_pool *pool)  in page_pool_inflight()  argument
194 	u32 release_cnt = atomic_read(&pool->pages_state_release_cnt);  in page_pool_inflight()
195 	u32 hold_cnt = READ_ONCE(pool->pages_state_hold_cnt);  in page_pool_inflight()
200 	trace_page_pool_inflight(pool, distance, hold_cnt, release_cnt);  in page_pool_inflight()
204 static bool __page_pool_safe_to_destroy(struct page_pool *pool)  in __page_pool_safe_to_destroy()  argument
206 	s32 inflight = page_pool_inflight(pool);  in __page_pool_safe_to_destroy()
215 static void __page_pool_clean_page(struct page_pool *pool,  in __page_pool_clean_page()  argument
220 	if (!(pool->p.flags & PP_FLAG_DMA_MAP))  in __page_pool_clean_page()
225 	dma_unmap_page_attrs(pool->p.dev, dma,  in __page_pool_clean_page()
226 			     PAGE_SIZE << pool->p.order, pool->p.dma_dir,  in __page_pool_clean_page()
230 	atomic_inc(&pool->pages_state_release_cnt);  in __page_pool_clean_page()
231 	trace_page_pool_state_release(pool, page,  in __page_pool_clean_page()
232 			      atomic_read(&pool->pages_state_release_cnt));  in __page_pool_clean_page()
236 void page_pool_unmap_page(struct page_pool *pool, struct page *page)  in page_pool_unmap_page()  argument
241 	__page_pool_clean_page(pool, page);  in page_pool_unmap_page()
246 static void __page_pool_return_page(struct page_pool *pool, struct page *page)  in __page_pool_return_page()  argument
248 	__page_pool_clean_page(pool, page);  in __page_pool_return_page()
257 static bool __page_pool_recycle_into_ring(struct page_pool *pool,  in __page_pool_recycle_into_ring()  argument
263 		ret = ptr_ring_produce(&pool->ring, page);  in __page_pool_recycle_into_ring()
265 		ret = ptr_ring_produce_bh(&pool->ring, page);  in __page_pool_recycle_into_ring()
276 				       struct page_pool *pool)  in __page_pool_recycle_direct()  argument
278 	if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE))  in __page_pool_recycle_direct()
282 	pool->alloc.cache[pool->alloc.count++] = page;  in __page_pool_recycle_direct()
286 void __page_pool_put_page(struct page_pool *pool,  in __page_pool_put_page()  argument
299 			if (__page_pool_recycle_direct(page, pool))  in __page_pool_put_page()
302 		if (!__page_pool_recycle_into_ring(pool, page)) {  in __page_pool_put_page()
304 			__page_pool_return_page(pool, page);  in __page_pool_put_page()
321 	__page_pool_clean_page(pool, page);  in __page_pool_put_page()
326 static void __page_pool_empty_ring(struct page_pool *pool)  in __page_pool_empty_ring()  argument
331 	while ((page = ptr_ring_consume_bh(&pool->ring))) {  in __page_pool_empty_ring()
337 		__page_pool_return_page(pool, page);  in __page_pool_empty_ring()
341 static void __warn_in_flight(struct page_pool *pool)  in __warn_in_flight()  argument
343 	u32 release_cnt = atomic_read(&pool->pages_state_release_cnt);  in __warn_in_flight()
344 	u32 hold_cnt = READ_ONCE(pool->pages_state_hold_cnt);  in __warn_in_flight()
354 void __page_pool_free(struct page_pool *pool)  in __page_pool_free()  argument
357 	if (!page_pool_put(pool))  in __page_pool_free()
360 	WARN(pool->alloc.count, "API usage violation");  in __page_pool_free()
361 	WARN(!ptr_ring_empty(&pool->ring), "ptr_ring is not empty");  in __page_pool_free()
364 	if (!__page_pool_safe_to_destroy(pool))  in __page_pool_free()
365 		__warn_in_flight(pool);  in __page_pool_free()
367 	ptr_ring_cleanup(&pool->ring, NULL);  in __page_pool_free()
369 	if (pool->p.flags & PP_FLAG_DMA_MAP)  in __page_pool_free()
370 		put_device(pool->p.dev);  in __page_pool_free()
372 	kfree(pool);  in __page_pool_free()
379 bool __page_pool_request_shutdown(struct page_pool *pool)  in __page_pool_request_shutdown()  argument
387 	while (pool->alloc.count) {  in __page_pool_request_shutdown()
388 		page = pool->alloc.cache[--pool->alloc.count];  in __page_pool_request_shutdown()
389 		__page_pool_return_page(pool, page);  in __page_pool_request_shutdown()
395 	__page_pool_empty_ring(pool);  in __page_pool_request_shutdown()
397 	return __page_pool_safe_to_destroy(pool);  in __page_pool_request_shutdown()