32 #define alloc_stat_inc(pool, __stat)	(pool->alloc_stats.__stat++)  argument
34 #define recycle_stat_inc(pool, __stat)							\  argument
36 		struct page_pool_recycle_stats __percpu *s = pool->recycle_stats;	\
40 #define recycle_stat_add(pool, __stat, val)						\  argument
42 		struct page_pool_recycle_stats __percpu *s = pool->recycle_stats;	\
60 bool page_pool_get_stats(struct page_pool *pool,  in page_pool_get_stats()  argument
69 	stats->alloc_stats.fast += pool->alloc_stats.fast;  in page_pool_get_stats()
70 	stats->alloc_stats.slow += pool->alloc_stats.slow;  in page_pool_get_stats()
71 	stats->alloc_stats.slow_high_order += pool->alloc_stats.slow_high_order;  in page_pool_get_stats()
72 	stats->alloc_stats.empty += pool->alloc_stats.empty;  in page_pool_get_stats()
73 	stats->alloc_stats.refill += pool->alloc_stats.refill;  in page_pool_get_stats()
74 	stats->alloc_stats.waive += pool->alloc_stats.waive;  in page_pool_get_stats()
78 			per_cpu_ptr(pool->recycle_stats, cpu);  in page_pool_get_stats()
131 #define alloc_stat_inc(pool, __stat)  argument
132 #define recycle_stat_inc(pool, __stat)  argument
133 #define recycle_stat_add(pool, __stat, val)  argument
136 static int page_pool_init(struct page_pool *pool,  in page_pool_init()  argument
141 	memcpy(&pool->p, params, sizeof(pool->p));  in page_pool_init()
144 	if (pool->p.flags & ~(PP_FLAG_ALL))  in page_pool_init()
147 	if (pool->p.pool_size)  in page_pool_init()
148 		ring_qsize = pool->p.pool_size;  in page_pool_init()
158 	if (pool->p.flags & PP_FLAG_DMA_MAP) {  in page_pool_init()
159 		if ((pool->p.dma_dir != DMA_FROM_DEVICE) &&  in page_pool_init()
160 		    (pool->p.dma_dir != DMA_BIDIRECTIONAL))  in page_pool_init()
164 	if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) {  in page_pool_init()
168 		if (!(pool->p.flags & PP_FLAG_DMA_MAP))  in page_pool_init()
171 		if (!pool->p.max_len)  in page_pool_init()
174 		/* pool->p.offset has to be set according to the address  in page_pool_init()
180 	    pool->p.flags & PP_FLAG_PAGE_FRAG)  in page_pool_init()
184 	pool->recycle_stats = alloc_percpu(struct page_pool_recycle_stats);  in page_pool_init()
185 	if (!pool->recycle_stats)  in page_pool_init()
189 	if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0)  in page_pool_init()
192 	atomic_set(&pool->pages_state_release_cnt, 0);  in page_pool_init()
195 	refcount_set(&pool->user_cnt, 1);  in page_pool_init()
197 	if (pool->p.flags & PP_FLAG_DMA_MAP)  in page_pool_init()
198 		get_device(pool->p.dev);  in page_pool_init()
205 	struct page_pool *pool;  in page_pool_create()  local
208 	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid);  in page_pool_create()
209 	if (!pool)  in page_pool_create()
212 	err = page_pool_init(pool, params);  in page_pool_create()
215 		kfree(pool);  in page_pool_create()
219 	return pool;  in page_pool_create()
223 static void page_pool_return_page(struct page_pool *pool, struct page *page);
226 static struct page *page_pool_refill_alloc_cache(struct page_pool *pool)  in page_pool_refill_alloc_cache()  argument
228 	struct ptr_ring *r = &pool->ring;  in page_pool_refill_alloc_cache()
234 		alloc_stat_inc(pool, empty);  in page_pool_refill_alloc_cache()
242 	pref_nid = (pool->p.nid == NUMA_NO_NODE) ? numa_mem_id() : pool->p.nid;  in page_pool_refill_alloc_cache()
244 	/* Ignore pool->p.nid setting if !CONFIG_NUMA, helps compiler */  in page_pool_refill_alloc_cache()
255 			pool->alloc.cache[pool->alloc.count++] = page;  in page_pool_refill_alloc_cache()
262 			page_pool_return_page(pool, page);  in page_pool_refill_alloc_cache()
263 			alloc_stat_inc(pool, waive);  in page_pool_refill_alloc_cache()
267 	} while (pool->alloc.count < PP_ALLOC_CACHE_REFILL);  in page_pool_refill_alloc_cache()
270 	if (likely(pool->alloc.count > 0)) {  in page_pool_refill_alloc_cache()
271 		page = pool->alloc.cache[--pool->alloc.count];  in page_pool_refill_alloc_cache()
272 		alloc_stat_inc(pool, refill);  in page_pool_refill_alloc_cache()
279 static struct page *__page_pool_get_cached(struct page_pool *pool)  in __page_pool_get_cached()  argument
284 	if (likely(pool->alloc.count)) {  in __page_pool_get_cached()
286 		page = pool->alloc.cache[--pool->alloc.count];  in __page_pool_get_cached()
287 		alloc_stat_inc(pool, fast);  in __page_pool_get_cached()
289 		page = page_pool_refill_alloc_cache(pool);  in __page_pool_get_cached()
295 static void page_pool_dma_sync_for_device(struct page_pool *pool,  in page_pool_dma_sync_for_device()  argument
301 	dma_sync_size = min(dma_sync_size, pool->p.max_len);  in page_pool_dma_sync_for_device()
302 	dma_sync_single_range_for_device(pool->p.dev, dma_addr,  in page_pool_dma_sync_for_device()
303 					 pool->p.offset, dma_sync_size,  in page_pool_dma_sync_for_device()
304 					 pool->p.dma_dir);  in page_pool_dma_sync_for_device()
307 static bool page_pool_dma_map(struct page_pool *pool, struct page *page)  in page_pool_dma_map()  argument
314 	 * This mapping is kept for lifetime of page, until leaving pool.  in page_pool_dma_map()
316 	dma = dma_map_page_attrs(pool->p.dev, page, 0,  in page_pool_dma_map()
317 				 (PAGE_SIZE << pool->p.order),  in page_pool_dma_map()
318 				 pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC);  in page_pool_dma_map()
319 	if (dma_mapping_error(pool->p.dev, dma))  in page_pool_dma_map()
324 	if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)  in page_pool_dma_map()
325 		page_pool_dma_sync_for_device(pool, page, pool->p.max_len);  in page_pool_dma_map()
330 static void page_pool_set_pp_info(struct page_pool *pool,  in page_pool_set_pp_info()  argument
333 	page->pp = pool;  in page_pool_set_pp_info()
335 	if (pool->p.init_callback)  in page_pool_set_pp_info()
336 		pool->p.init_callback(page, pool->p.init_arg);  in page_pool_set_pp_info()
345 static struct page *__page_pool_alloc_page_order(struct page_pool *pool,  in __page_pool_alloc_page_order()  argument
351 	page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);  in __page_pool_alloc_page_order()
355 	if ((pool->p.flags & PP_FLAG_DMA_MAP) &&  in __page_pool_alloc_page_order()
356 	    unlikely(!page_pool_dma_map(pool, page))) {  in __page_pool_alloc_page_order()
361 	alloc_stat_inc(pool, slow_high_order);  in __page_pool_alloc_page_order()
362 	page_pool_set_pp_info(pool, page);  in __page_pool_alloc_page_order()
365 	pool->pages_state_hold_cnt++;  in __page_pool_alloc_page_order()
366 	trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt);  in __page_pool_alloc_page_order()
372 static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,  in __page_pool_alloc_pages_slow()  argument
376 	unsigned int pp_flags = pool->p.flags;  in __page_pool_alloc_pages_slow()
377 	unsigned int pp_order = pool->p.order;  in __page_pool_alloc_pages_slow()
383 		return __page_pool_alloc_page_order(pool, gfp);  in __page_pool_alloc_pages_slow()
386 	if (unlikely(pool->alloc.count > 0))  in __page_pool_alloc_pages_slow()
387 		return pool->alloc.cache[--pool->alloc.count];  in __page_pool_alloc_pages_slow()
390 	memset(&pool->alloc.cache, 0, sizeof(void *) * bulk);  in __page_pool_alloc_pages_slow()
392 	nr_pages = alloc_pages_bulk_array_node(gfp, pool->p.nid, bulk,  in __page_pool_alloc_pages_slow()
393 					       pool->alloc.cache);  in __page_pool_alloc_pages_slow()
401 		page = pool->alloc.cache[i];  in __page_pool_alloc_pages_slow()
403 		    unlikely(!page_pool_dma_map(pool, page))) {  in __page_pool_alloc_pages_slow()
408 		page_pool_set_pp_info(pool, page);  in __page_pool_alloc_pages_slow()
409 		pool->alloc.cache[pool->alloc.count++] = page;  in __page_pool_alloc_pages_slow()
411 		pool->pages_state_hold_cnt++;  in __page_pool_alloc_pages_slow()
412 		trace_page_pool_state_hold(pool, page,  in __page_pool_alloc_pages_slow()
413 					   pool->pages_state_hold_cnt);  in __page_pool_alloc_pages_slow()
417 	if (likely(pool->alloc.count > 0)) {  in __page_pool_alloc_pages_slow()
418 		page = pool->alloc.cache[--pool->alloc.count];  in __page_pool_alloc_pages_slow()
419 		alloc_stat_inc(pool, slow);  in __page_pool_alloc_pages_slow()
431 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)  in page_pool_alloc_pages()  argument
436 	page = __page_pool_get_cached(pool);  in page_pool_alloc_pages()
441 	page = __page_pool_alloc_pages_slow(pool, gfp);  in page_pool_alloc_pages()
451 static s32 page_pool_inflight(struct page_pool *pool)  in page_pool_inflight()  argument
453 	u32 release_cnt = atomic_read(&pool->pages_state_release_cnt);  in page_pool_inflight()
454 	u32 hold_cnt = READ_ONCE(pool->pages_state_hold_cnt);  in page_pool_inflight()
459 	trace_page_pool_release(pool, inflight, hold_cnt, release_cnt);  in page_pool_inflight()
470 void page_pool_release_page(struct page_pool *pool, struct page *page)  in page_pool_release_page()  argument
475 	if (!(pool->p.flags & PP_FLAG_DMA_MAP))  in page_pool_release_page()
483 	/* When page is unmapped, it cannot be returned to our pool */  in page_pool_release_page()
484 	dma_unmap_page_attrs(pool->p.dev, dma,  in page_pool_release_page()
485 			     PAGE_SIZE << pool->p.order, pool->p.dma_dir,  in page_pool_release_page()
491 	/* This may be the last page returned, releasing the pool, so  in page_pool_release_page()
492 	 * it is not safe to reference pool afterwards.  in page_pool_release_page()
494 	count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt);  in page_pool_release_page()
495 	trace_page_pool_state_release(pool, page, count);  in page_pool_release_page()
500 static void page_pool_return_page(struct page_pool *pool, struct page *page)  in page_pool_return_page()  argument
502 	page_pool_release_page(pool, page);  in page_pool_return_page()
505 	/* An optimization would be to call __free_pages(page, pool->p.order)  in page_pool_return_page()
511 static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page)  in page_pool_recycle_in_ring()  argument
516 		ret = ptr_ring_produce(&pool->ring, page);  in page_pool_recycle_in_ring()
518 		ret = ptr_ring_produce_bh(&pool->ring, page);  in page_pool_recycle_in_ring()
521 		recycle_stat_inc(pool, ring);  in page_pool_recycle_in_ring()
534 				       struct page_pool *pool)  in page_pool_recycle_in_cache()  argument
536 	if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) {  in page_pool_recycle_in_cache()
537 		recycle_stat_inc(pool, cache_full);  in page_pool_recycle_in_cache()
542 	pool->alloc.cache[pool->alloc.count++] = page;  in page_pool_recycle_in_cache()
543 	recycle_stat_inc(pool, cached);  in page_pool_recycle_in_cache()
549  * the configured size min(dma_sync_size, pool->max_len).
554 __page_pool_put_page(struct page_pool *pool, struct page *page,  in __page_pool_put_page()  argument
569 		if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)  in __page_pool_put_page()
570 			page_pool_dma_sync_for_device(pool, page,  in __page_pool_put_page()
574 		    page_pool_recycle_in_cache(page, pool))  in __page_pool_put_page()
593 	recycle_stat_inc(pool, released_refcnt);  in __page_pool_put_page()
595 	page_pool_release_page(pool, page);  in __page_pool_put_page()
601 void page_pool_put_defragged_page(struct page_pool *pool, struct page *page,  in page_pool_put_defragged_page()  argument
604 	page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct);  in page_pool_put_defragged_page()
605 	if (page && !page_pool_recycle_in_ring(pool, page)) {  in page_pool_put_defragged_page()
607 		recycle_stat_inc(pool, ring_full);  in page_pool_put_defragged_page()
608 		page_pool_return_page(pool, page);  in page_pool_put_defragged_page()
614 void page_pool_put_page_bulk(struct page_pool *pool, void **data,  in page_pool_put_page_bulk()  argument
623 		if (!page_pool_is_last_frag(pool, page))  in page_pool_put_page_bulk()
626 		page = __page_pool_put_page(pool, page, -1, false);  in page_pool_put_page_bulk()
636 	page_pool_ring_lock(pool);  in page_pool_put_page_bulk()
638 		if (__ptr_ring_produce(&pool->ring, data[i])) {  in page_pool_put_page_bulk()
640 			recycle_stat_inc(pool, ring_full);  in page_pool_put_page_bulk()
644 	recycle_stat_add(pool, ring, i);  in page_pool_put_page_bulk()
645 	page_pool_ring_unlock(pool);  in page_pool_put_page_bulk()
655 		page_pool_return_page(pool, data[i]);  in page_pool_put_page_bulk()
659 static struct page *page_pool_drain_frag(struct page_pool *pool,  in page_pool_drain_frag()  argument
662 	long drain_count = BIAS_MAX - pool->frag_users;  in page_pool_drain_frag()
669 		if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)  in page_pool_drain_frag()
670 			page_pool_dma_sync_for_device(pool, page, -1);  in page_pool_drain_frag()
675 	page_pool_return_page(pool, page);  in page_pool_drain_frag()
679 static void page_pool_free_frag(struct page_pool *pool)  in page_pool_free_frag()  argument
681 	long drain_count = BIAS_MAX - pool->frag_users;  in page_pool_free_frag()
682 	struct page *page = pool->frag_page;  in page_pool_free_frag()
684 	pool->frag_page = NULL;  in page_pool_free_frag()
689 	page_pool_return_page(pool, page);  in page_pool_free_frag()
692 struct page *page_pool_alloc_frag(struct page_pool *pool,  in page_pool_alloc_frag()  argument
696 	unsigned int max_size = PAGE_SIZE << pool->p.order;  in page_pool_alloc_frag()
697 	struct page *page = pool->frag_page;  in page_pool_alloc_frag()
699 	if (WARN_ON(!(pool->p.flags & PP_FLAG_PAGE_FRAG) ||  in page_pool_alloc_frag()
704 	*offset = pool->frag_offset;  in page_pool_alloc_frag()
707 		page = page_pool_drain_frag(pool, page);  in page_pool_alloc_frag()
709 			alloc_stat_inc(pool, fast);  in page_pool_alloc_frag()
715 		page = page_pool_alloc_pages(pool, gfp);  in page_pool_alloc_frag()
717 			pool->frag_page = NULL;  in page_pool_alloc_frag()
721 		pool->frag_page = page;  in page_pool_alloc_frag()
724 		pool->frag_users = 1;  in page_pool_alloc_frag()
726 		pool->frag_offset = size;  in page_pool_alloc_frag()
731 	pool->frag_users++;  in page_pool_alloc_frag()
732 	pool->frag_offset = *offset + size;  in page_pool_alloc_frag()
733 	alloc_stat_inc(pool, fast);  in page_pool_alloc_frag()
738 static void page_pool_empty_ring(struct page_pool *pool)  in page_pool_empty_ring()  argument
743 	while ((page = ptr_ring_consume_bh(&pool->ring))) {  in page_pool_empty_ring()
749 		page_pool_return_page(pool, page);  in page_pool_empty_ring()
753 static void page_pool_free(struct page_pool *pool)  in page_pool_free()  argument
755 	if (pool->disconnect)  in page_pool_free()
756 		pool->disconnect(pool);  in page_pool_free()
758 	ptr_ring_cleanup(&pool->ring, NULL);  in page_pool_free()
760 	if (pool->p.flags & PP_FLAG_DMA_MAP)  in page_pool_free()
761 		put_device(pool->p.dev);  in page_pool_free()
764 	free_percpu(pool->recycle_stats);  in page_pool_free()
766 	kfree(pool);  in page_pool_free()
769 static void page_pool_empty_alloc_cache_once(struct page_pool *pool)  in page_pool_empty_alloc_cache_once()  argument
773 	if (pool->destroy_cnt)  in page_pool_empty_alloc_cache_once()
780 	while (pool->alloc.count) {  in page_pool_empty_alloc_cache_once()
781 		page = pool->alloc.cache[--pool->alloc.count];  in page_pool_empty_alloc_cache_once()
782 		page_pool_return_page(pool, page);  in page_pool_empty_alloc_cache_once()
786 static void page_pool_scrub(struct page_pool *pool)  in page_pool_scrub()  argument
788 	page_pool_empty_alloc_cache_once(pool);  in page_pool_scrub()
789 	pool->destroy_cnt++;  in page_pool_scrub()
794 	page_pool_empty_ring(pool);  in page_pool_scrub()
797 static int page_pool_release(struct page_pool *pool)  in page_pool_release()  argument
801 	page_pool_scrub(pool);  in page_pool_release()
802 	inflight = page_pool_inflight(pool);  in page_pool_release()
804 		page_pool_free(pool);  in page_pool_release()
812 	struct page_pool *pool = container_of(dwq, typeof(*pool), release_dw);  in page_pool_release_retry()  local
815 	inflight = page_pool_release(pool);  in page_pool_release_retry()
820 	if (time_after_eq(jiffies, pool->defer_warn)) {  in page_pool_release_retry()
821 		int sec = (s32)((u32)jiffies - (u32)pool->defer_start) / HZ;  in page_pool_release_retry()
823 		pr_warn("%s() stalled pool shutdown %d inflight %d sec\n",  in page_pool_release_retry()
825 		pool->defer_warn = jiffies + DEFER_WARN_INTERVAL;  in page_pool_release_retry()
829 	schedule_delayed_work(&pool->release_dw, DEFER_TIME);  in page_pool_release_retry()
832 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),  in page_pool_use_xdp_mem()  argument
835 	refcount_inc(&pool->user_cnt);  in page_pool_use_xdp_mem()
836 	pool->disconnect = disconnect;  in page_pool_use_xdp_mem()
837 	pool->xdp_mem_id = mem->id;  in page_pool_use_xdp_mem()
840 void page_pool_destroy(struct page_pool *pool)  in page_pool_destroy()  argument
842 	if (!pool)  in page_pool_destroy()
845 	if (!page_pool_put(pool))  in page_pool_destroy()
848 	page_pool_free_frag(pool);  in page_pool_destroy()
850 	if (!page_pool_release(pool))  in page_pool_destroy()
853 	pool->defer_start = jiffies;  in page_pool_destroy()
854 	pool->defer_warn  = jiffies + DEFER_WARN_INTERVAL;  in page_pool_destroy()
856 	INIT_DELAYED_WORK(&pool->release_dw, page_pool_release_retry);  in page_pool_destroy()
857 	schedule_delayed_work(&pool->release_dw, DEFER_TIME);  in page_pool_destroy()
862 void page_pool_update_nid(struct page_pool *pool, int new_nid)  in page_pool_update_nid()  argument
866 	trace_page_pool_update_nid(pool, new_nid);  in page_pool_update_nid()
867 	pool->p.nid = new_nid;  in page_pool_update_nid()
869 	/* Flush pool alloc cache, as refill will check NUMA node */  in page_pool_update_nid()
870 	while (pool->alloc.count) {  in page_pool_update_nid()
871 		page = pool->alloc.cache[--pool->alloc.count];  in page_pool_update_nid()
872 		page_pool_return_page(pool, page);  in page_pool_update_nid()
897 	 * The page will be returned to the pool here regardless of the  in page_pool_return_skb_page()