82 static struct page *__page_pool_get_cached(struct page_pool *pool)  in __page_pool_get_cached()
86 	struct page *page;  in __page_pool_get_cached()  local
92 			page = pool->alloc.cache[--pool->alloc.count];  in __page_pool_get_cached()
93 			return page;  in __page_pool_get_cached()
106 	page = __ptr_ring_consume(r);  in __page_pool_get_cached()
112 	return page;  in __page_pool_get_cached()
117 static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,  in __page_pool_alloc_pages_slow()
120 	struct page *page;  in __page_pool_alloc_pages_slow()  local
138 	page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);  in __page_pool_alloc_pages_slow()
139 	if (!page)  in __page_pool_alloc_pages_slow()
150 	dma = dma_map_page_attrs(pool->p.dev, page, 0,  in __page_pool_alloc_pages_slow()
154 		put_page(page);  in __page_pool_alloc_pages_slow()
157 	page->dma_addr = dma;  in __page_pool_alloc_pages_slow()
163 	trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt);  in __page_pool_alloc_pages_slow()
166 	return page;  in __page_pool_alloc_pages_slow()
172 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)  in page_pool_alloc_pages()
174 	struct page *page;  in page_pool_alloc_pages()  local
177 	page = __page_pool_get_cached(pool);  in page_pool_alloc_pages()
178 	if (page)  in page_pool_alloc_pages()
179 		return page;  in page_pool_alloc_pages()
182 	page = __page_pool_alloc_pages_slow(pool, gfp);  in page_pool_alloc_pages()
183 	return page;  in page_pool_alloc_pages()
216 				   struct page *page)  in __page_pool_clean_page()  argument
223 	dma = page->dma_addr;  in __page_pool_clean_page()
228 	page->dma_addr = 0;  in __page_pool_clean_page()
231 	trace_page_pool_state_release(pool, page,  in __page_pool_clean_page()
236 void page_pool_unmap_page(struct page_pool *pool, struct page *page)  in page_pool_unmap_page()  argument
241 	__page_pool_clean_page(pool, page);  in page_pool_unmap_page()
246 static void __page_pool_return_page(struct page_pool *pool, struct page *page)  in __page_pool_return_page()  argument
248 	__page_pool_clean_page(pool, page);  in __page_pool_return_page()
250 	put_page(page);  in __page_pool_return_page()
258 				   struct page *page)  in __page_pool_recycle_into_ring()  argument
263 		ret = ptr_ring_produce(&pool->ring, page);  in __page_pool_recycle_into_ring()
265 		ret = ptr_ring_produce_bh(&pool->ring, page);  in __page_pool_recycle_into_ring()
275 static bool __page_pool_recycle_direct(struct page *page,  in __page_pool_recycle_direct()  argument
282 	pool->alloc.cache[pool->alloc.count++] = page;  in __page_pool_recycle_direct()
287 			  struct page *page, bool allow_direct)  in __page_pool_put_page()  argument
295 	if (likely(page_ref_count(page) == 1)) {  in __page_pool_put_page()
299 			if (__page_pool_recycle_direct(page, pool))  in __page_pool_put_page()
302 		if (!__page_pool_recycle_into_ring(pool, page)) {  in __page_pool_put_page()
304 			__page_pool_return_page(pool, page);  in __page_pool_put_page()
321 	__page_pool_clean_page(pool, page);  in __page_pool_put_page()
322 	put_page(page);  in __page_pool_put_page()
328 	struct page *page;  in __page_pool_empty_ring()  local
331 	while ((page = ptr_ring_consume_bh(&pool->ring))) {  in __page_pool_empty_ring()
333 		if (!(page_ref_count(page) == 1))  in __page_pool_empty_ring()
335 				__func__, page_ref_count(page));  in __page_pool_empty_ring()
337 		__page_pool_return_page(pool, page);  in __page_pool_empty_ring()
381 	struct page *page;  in __page_pool_request_shutdown()  local
388 		page = pool->alloc.cache[--pool->alloc.count];  in __page_pool_request_shutdown()
389 		__page_pool_return_page(pool, page);  in __page_pool_request_shutdown()