101 static void page_pool_return_page(struct page_pool *pool, struct page *page);
104 static struct page *page_pool_refill_alloc_cache(struct page_pool *pool)  in page_pool_refill_alloc_cache()
107 	struct page *page;  in page_pool_refill_alloc_cache()  local
129 		page = __ptr_ring_consume(r);  in page_pool_refill_alloc_cache()
130 		if (unlikely(!page))  in page_pool_refill_alloc_cache()
133 		if (likely(page_to_nid(page) == pref_nid)) {  in page_pool_refill_alloc_cache()
134 			pool->alloc.cache[pool->alloc.count++] = page;  in page_pool_refill_alloc_cache()
141 			page_pool_return_page(pool, page);  in page_pool_refill_alloc_cache()
142 			page = NULL;  in page_pool_refill_alloc_cache()
149 		page = pool->alloc.cache[--pool->alloc.count];  in page_pool_refill_alloc_cache()
152 	return page;  in page_pool_refill_alloc_cache()
156 static struct page *__page_pool_get_cached(struct page_pool *pool)  in __page_pool_get_cached()
158 	struct page *page;  in __page_pool_get_cached()  local
163 		page = pool->alloc.cache[--pool->alloc.count];  in __page_pool_get_cached()
165 		page = page_pool_refill_alloc_cache(pool);  in __page_pool_get_cached()
168 	return page;  in __page_pool_get_cached()
172 					  struct page *page,  in page_pool_dma_sync_for_device()  argument
176 	dma_sync_single_range_for_device(pool->p.dev, page->dma_addr,  in page_pool_dma_sync_for_device()
183 static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,  in __page_pool_alloc_pages_slow()
186 	struct page *page;  in __page_pool_alloc_pages_slow()  local
205 	page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);  in __page_pool_alloc_pages_slow()
207 	page = alloc_pages(gfp, pool->p.order);  in __page_pool_alloc_pages_slow()
209 	if (!page)  in __page_pool_alloc_pages_slow()
220 	dma = dma_map_page_attrs(pool->p.dev, page, 0,  in __page_pool_alloc_pages_slow()
224 		put_page(page);  in __page_pool_alloc_pages_slow()
227 	page->dma_addr = dma;  in __page_pool_alloc_pages_slow()
230 		page_pool_dma_sync_for_device(pool, page, pool->p.max_len);  in __page_pool_alloc_pages_slow()
236 	trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt);  in __page_pool_alloc_pages_slow()
239 	return page;  in __page_pool_alloc_pages_slow()
245 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)  in page_pool_alloc_pages()
247 	struct page *page;  in page_pool_alloc_pages()  local
250 	page = __page_pool_get_cached(pool);  in page_pool_alloc_pages()
251 	if (page)  in page_pool_alloc_pages()
252 		return page;  in page_pool_alloc_pages()
255 	page = __page_pool_alloc_pages_slow(pool, gfp);  in page_pool_alloc_pages()
256 	return page;  in page_pool_alloc_pages()
284 void page_pool_release_page(struct page_pool *pool, struct page *page)  in page_pool_release_page()  argument
295 	dma = page->dma_addr;  in page_pool_release_page()
301 	page->dma_addr = 0;  in page_pool_release_page()
307 	trace_page_pool_state_release(pool, page, count);  in page_pool_release_page()
312 static void page_pool_return_page(struct page_pool *pool, struct page *page)  in page_pool_return_page()  argument
314 	page_pool_release_page(pool, page);  in page_pool_return_page()
316 	put_page(page);  in page_pool_return_page()
323 static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page)  in page_pool_recycle_in_ring()  argument
328 		ret = ptr_ring_produce(&pool->ring, page);  in page_pool_recycle_in_ring()
330 		ret = ptr_ring_produce_bh(&pool->ring, page);  in page_pool_recycle_in_ring()
340 static bool page_pool_recycle_in_cache(struct page *page,  in page_pool_recycle_in_cache()  argument
347 	pool->alloc.cache[pool->alloc.count++] = page;  in page_pool_recycle_in_cache()
354 static bool pool_page_reusable(struct page_pool *pool, struct page *page)  in pool_page_reusable()  argument
356 	return !page_is_pfmemalloc(page);  in pool_page_reusable()
365 void page_pool_put_page(struct page_pool *pool, struct page *page,  in page_pool_put_page()  argument
374 	if (likely(page_ref_count(page) == 1 &&  in page_pool_put_page()
375 		   pool_page_reusable(pool, page))) {  in page_pool_put_page()
379 			page_pool_dma_sync_for_device(pool, page,  in page_pool_put_page()
383 			if (page_pool_recycle_in_cache(page, pool))  in page_pool_put_page()
386 		if (!page_pool_recycle_in_ring(pool, page)) {  in page_pool_put_page()
388 			page_pool_return_page(pool, page);  in page_pool_put_page()
406 	page_pool_release_page(pool, page);  in page_pool_put_page()
407 	put_page(page);  in page_pool_put_page()
413 	struct page *page;  in page_pool_empty_ring()  local
416 	while ((page = ptr_ring_consume_bh(&pool->ring))) {  in page_pool_empty_ring()
418 		if (!(page_ref_count(page) == 1))  in page_pool_empty_ring()
420 				__func__, page_ref_count(page));  in page_pool_empty_ring()
422 		page_pool_return_page(pool, page);  in page_pool_empty_ring()
441 	struct page *page;  in page_pool_empty_alloc_cache_once()  local
451 		page = pool->alloc.cache[--pool->alloc.count];  in page_pool_empty_alloc_cache_once()
452 		page_pool_return_page(pool, page);  in page_pool_empty_alloc_cache_once()
530 	struct page *page;  in page_pool_update_nid()  local
537 		page = pool->alloc.cache[--pool->alloc.count];  in page_pool_update_nid()
538 		page_pool_return_page(pool, page);  in page_pool_update_nid()