Lines Matching +full:dma +full:- +full:pool
1 /* SPDX-License-Identifier: GPL-2.0
16 #include <linux/dma-direction.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/page-flags.h>
32 #define alloc_stat_inc(pool, __stat) (pool->alloc_stats.__stat++) argument
34 #define recycle_stat_inc(pool, __stat) \ argument
36 struct page_pool_recycle_stats __percpu *s = pool->recycle_stats; \
37 this_cpu_inc(s->__stat); \
40 #define recycle_stat_add(pool, __stat, val) \ argument
42 struct page_pool_recycle_stats __percpu *s = pool->recycle_stats; \
43 this_cpu_add(s->__stat, val); \
60 bool page_pool_get_stats(struct page_pool *pool, in page_pool_get_stats() argument
69 stats->alloc_stats.fast += pool->alloc_stats.fast; in page_pool_get_stats()
70 stats->alloc_stats.slow += pool->alloc_stats.slow; in page_pool_get_stats()
71 stats->alloc_stats.slow_high_order += pool->alloc_stats.slow_high_order; in page_pool_get_stats()
72 stats->alloc_stats.empty += pool->alloc_stats.empty; in page_pool_get_stats()
73 stats->alloc_stats.refill += pool->alloc_stats.refill; in page_pool_get_stats()
74 stats->alloc_stats.waive += pool->alloc_stats.waive; in page_pool_get_stats()
78 per_cpu_ptr(pool->recycle_stats, cpu); in page_pool_get_stats()
80 stats->recycle_stats.cached += pcpu->cached; in page_pool_get_stats()
81 stats->recycle_stats.cache_full += pcpu->cache_full; in page_pool_get_stats()
82 stats->recycle_stats.ring += pcpu->ring; in page_pool_get_stats()
83 stats->recycle_stats.ring_full += pcpu->ring_full; in page_pool_get_stats()
84 stats->recycle_stats.released_refcnt += pcpu->released_refcnt; in page_pool_get_stats()
114 *data++ = pool_stats->alloc_stats.fast; in page_pool_ethtool_stats_get()
115 *data++ = pool_stats->alloc_stats.slow; in page_pool_ethtool_stats_get()
116 *data++ = pool_stats->alloc_stats.slow_high_order; in page_pool_ethtool_stats_get()
117 *data++ = pool_stats->alloc_stats.empty; in page_pool_ethtool_stats_get()
118 *data++ = pool_stats->alloc_stats.refill; in page_pool_ethtool_stats_get()
119 *data++ = pool_stats->alloc_stats.waive; in page_pool_ethtool_stats_get()
120 *data++ = pool_stats->recycle_stats.cached; in page_pool_ethtool_stats_get()
121 *data++ = pool_stats->recycle_stats.cache_full; in page_pool_ethtool_stats_get()
122 *data++ = pool_stats->recycle_stats.ring; in page_pool_ethtool_stats_get()
123 *data++ = pool_stats->recycle_stats.ring_full; in page_pool_ethtool_stats_get()
124 *data++ = pool_stats->recycle_stats.released_refcnt; in page_pool_ethtool_stats_get()
131 #define alloc_stat_inc(pool, __stat) argument
132 #define recycle_stat_inc(pool, __stat) argument
133 #define recycle_stat_add(pool, __stat, val) argument
136 static int page_pool_init(struct page_pool *pool, in page_pool_init() argument
141 memcpy(&pool->p, params, sizeof(pool->p)); in page_pool_init()
144 if (pool->p.flags & ~(PP_FLAG_ALL)) in page_pool_init()
145 return -EINVAL; in page_pool_init()
147 if (pool->p.pool_size) in page_pool_init()
148 ring_qsize = pool->p.pool_size; in page_pool_init()
152 return -E2BIG; in page_pool_init()
154 /* DMA direction is either DMA_FROM_DEVICE or DMA_BIDIRECTIONAL. in page_pool_init()
155 * DMA_BIDIRECTIONAL is for allowing page used for DMA sending, in page_pool_init()
156 * which is the XDP_TX use-case. in page_pool_init()
158 if (pool->p.flags & PP_FLAG_DMA_MAP) { in page_pool_init()
159 if ((pool->p.dma_dir != DMA_FROM_DEVICE) && in page_pool_init()
160 (pool->p.dma_dir != DMA_BIDIRECTIONAL)) in page_pool_init()
161 return -EINVAL; in page_pool_init()
164 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) { in page_pool_init()
165 /* In order to request DMA-sync-for-device the page in page_pool_init()
168 if (!(pool->p.flags & PP_FLAG_DMA_MAP)) in page_pool_init()
169 return -EINVAL; in page_pool_init()
171 if (!pool->p.max_len) in page_pool_init()
172 return -EINVAL; in page_pool_init()
174 /* pool->p.offset has to be set according to the address in page_pool_init()
175 * offset used by the DMA engine to start copying rx data in page_pool_init()
180 pool->p.flags & PP_FLAG_PAGE_FRAG) in page_pool_init()
181 return -EINVAL; in page_pool_init()
184 pool->recycle_stats = alloc_percpu(struct page_pool_recycle_stats); in page_pool_init()
185 if (!pool->recycle_stats) in page_pool_init()
186 return -ENOMEM; in page_pool_init()
189 if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) in page_pool_init()
190 return -ENOMEM; in page_pool_init()
192 atomic_set(&pool->pages_state_release_cnt, 0); in page_pool_init()
195 refcount_set(&pool->user_cnt, 1); in page_pool_init()
197 if (pool->p.flags & PP_FLAG_DMA_MAP) in page_pool_init()
198 get_device(pool->p.dev); in page_pool_init()
205 struct page_pool *pool; in page_pool_create() local
208 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid); in page_pool_create()
209 if (!pool) in page_pool_create()
210 return ERR_PTR(-ENOMEM); in page_pool_create()
212 err = page_pool_init(pool, params); in page_pool_create()
215 kfree(pool); in page_pool_create()
219 return pool; in page_pool_create()
223 static void page_pool_return_page(struct page_pool *pool, struct page *page);
226 static struct page *page_pool_refill_alloc_cache(struct page_pool *pool) in page_pool_refill_alloc_cache() argument
228 struct ptr_ring *r = &pool->ring; in page_pool_refill_alloc_cache()
234 alloc_stat_inc(pool, empty); in page_pool_refill_alloc_cache()
239 * assumes CPU refilling driver RX-ring will also run RX-NAPI. in page_pool_refill_alloc_cache()
242 pref_nid = (pool->p.nid == NUMA_NO_NODE) ? numa_mem_id() : pool->p.nid; in page_pool_refill_alloc_cache()
244 /* Ignore pool->p.nid setting if !CONFIG_NUMA, helps compiler */ in page_pool_refill_alloc_cache()
255 pool->alloc.cache[pool->alloc.count++] = page; in page_pool_refill_alloc_cache()
258 * (1) release 1 page to page-allocator and in page_pool_refill_alloc_cache()
262 page_pool_return_page(pool, page); in page_pool_refill_alloc_cache()
263 alloc_stat_inc(pool, waive); in page_pool_refill_alloc_cache()
267 } while (pool->alloc.count < PP_ALLOC_CACHE_REFILL); in page_pool_refill_alloc_cache()
270 if (likely(pool->alloc.count > 0)) { in page_pool_refill_alloc_cache()
271 page = pool->alloc.cache[--pool->alloc.count]; in page_pool_refill_alloc_cache()
272 alloc_stat_inc(pool, refill); in page_pool_refill_alloc_cache()
279 static struct page *__page_pool_get_cached(struct page_pool *pool) in __page_pool_get_cached() argument
283 /* Caller MUST guarantee safe non-concurrent access, e.g. softirq */ in __page_pool_get_cached()
284 if (likely(pool->alloc.count)) { in __page_pool_get_cached()
285 /* Fast-path */ in __page_pool_get_cached()
286 page = pool->alloc.cache[--pool->alloc.count]; in __page_pool_get_cached()
287 alloc_stat_inc(pool, fast); in __page_pool_get_cached()
289 page = page_pool_refill_alloc_cache(pool); in __page_pool_get_cached()
295 static void page_pool_dma_sync_for_device(struct page_pool *pool, in page_pool_dma_sync_for_device() argument
301 dma_sync_size = min(dma_sync_size, pool->p.max_len); in page_pool_dma_sync_for_device()
302 dma_sync_single_range_for_device(pool->p.dev, dma_addr, in page_pool_dma_sync_for_device()
303 pool->p.offset, dma_sync_size, in page_pool_dma_sync_for_device()
304 pool->p.dma_dir); in page_pool_dma_sync_for_device()
307 static bool page_pool_dma_map(struct page_pool *pool, struct page *page) in page_pool_dma_map() argument
309 dma_addr_t dma; in page_pool_dma_map() local
311 /* Setup DMA mapping: use 'struct page' area for storing DMA-addr in page_pool_dma_map()
313 * into page private data (i.e 32bit cpu with 64bit DMA caps) in page_pool_dma_map()
314 * This mapping is kept for lifetime of page, until leaving pool. in page_pool_dma_map()
316 dma = dma_map_page_attrs(pool->p.dev, page, 0, in page_pool_dma_map()
317 (PAGE_SIZE << pool->p.order), in page_pool_dma_map()
318 pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC); in page_pool_dma_map()
319 if (dma_mapping_error(pool->p.dev, dma)) in page_pool_dma_map()
322 page_pool_set_dma_addr(page, dma); in page_pool_dma_map()
324 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) in page_pool_dma_map()
325 page_pool_dma_sync_for_device(pool, page, pool->p.max_len); in page_pool_dma_map()
330 static void page_pool_set_pp_info(struct page_pool *pool, in page_pool_set_pp_info() argument
333 page->pp = pool; in page_pool_set_pp_info()
334 page->pp_magic |= PP_SIGNATURE; in page_pool_set_pp_info()
335 if (pool->p.init_callback) in page_pool_set_pp_info()
336 pool->p.init_callback(page, pool->p.init_arg); in page_pool_set_pp_info()
341 page->pp_magic = 0; in page_pool_clear_pp_info()
342 page->pp = NULL; in page_pool_clear_pp_info()
345 static struct page *__page_pool_alloc_page_order(struct page_pool *pool, in __page_pool_alloc_page_order() argument
351 page = alloc_pages_node(pool->p.nid, gfp, pool->p.order); in __page_pool_alloc_page_order()
355 if ((pool->p.flags & PP_FLAG_DMA_MAP) && in __page_pool_alloc_page_order()
356 unlikely(!page_pool_dma_map(pool, page))) { in __page_pool_alloc_page_order()
361 alloc_stat_inc(pool, slow_high_order); in __page_pool_alloc_page_order()
362 page_pool_set_pp_info(pool, page); in __page_pool_alloc_page_order()
364 /* Track how many pages are held 'in-flight' */ in __page_pool_alloc_page_order()
365 pool->pages_state_hold_cnt++; in __page_pool_alloc_page_order()
366 trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt); in __page_pool_alloc_page_order()
372 static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, in __page_pool_alloc_pages_slow() argument
376 unsigned int pp_flags = pool->p.flags; in __page_pool_alloc_pages_slow()
377 unsigned int pp_order = pool->p.order; in __page_pool_alloc_pages_slow()
381 /* Don't support bulk alloc for high-order pages */ in __page_pool_alloc_pages_slow()
383 return __page_pool_alloc_page_order(pool, gfp); in __page_pool_alloc_pages_slow()
386 if (unlikely(pool->alloc.count > 0)) in __page_pool_alloc_pages_slow()
387 return pool->alloc.cache[--pool->alloc.count]; in __page_pool_alloc_pages_slow()
390 memset(&pool->alloc.cache, 0, sizeof(void *) * bulk); in __page_pool_alloc_pages_slow()
392 nr_pages = alloc_pages_bulk_array_node(gfp, pool->p.nid, bulk, in __page_pool_alloc_pages_slow()
393 pool->alloc.cache); in __page_pool_alloc_pages_slow()
398 * page element have not been (possibly) DMA mapped. in __page_pool_alloc_pages_slow()
401 page = pool->alloc.cache[i]; in __page_pool_alloc_pages_slow()
403 unlikely(!page_pool_dma_map(pool, page))) { in __page_pool_alloc_pages_slow()
408 page_pool_set_pp_info(pool, page); in __page_pool_alloc_pages_slow()
409 pool->alloc.cache[pool->alloc.count++] = page; in __page_pool_alloc_pages_slow()
410 /* Track how many pages are held 'in-flight' */ in __page_pool_alloc_pages_slow()
411 pool->pages_state_hold_cnt++; in __page_pool_alloc_pages_slow()
412 trace_page_pool_state_hold(pool, page, in __page_pool_alloc_pages_slow()
413 pool->pages_state_hold_cnt); in __page_pool_alloc_pages_slow()
417 if (likely(pool->alloc.count > 0)) { in __page_pool_alloc_pages_slow()
418 page = pool->alloc.cache[--pool->alloc.count]; in __page_pool_alloc_pages_slow()
419 alloc_stat_inc(pool, slow); in __page_pool_alloc_pages_slow()
431 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp) in page_pool_alloc_pages() argument
435 /* Fast-path: Get a page from cache */ in page_pool_alloc_pages()
436 page = __page_pool_get_cached(pool); in page_pool_alloc_pages()
440 /* Slow-path: cache empty, do real allocation */ in page_pool_alloc_pages()
441 page = __page_pool_alloc_pages_slow(pool, gfp); in page_pool_alloc_pages()
449 #define _distance(a, b) (s32)((a) - (b))
451 static s32 page_pool_inflight(struct page_pool *pool) in page_pool_inflight() argument
453 u32 release_cnt = atomic_read(&pool->pages_state_release_cnt); in page_pool_inflight()
454 u32 hold_cnt = READ_ONCE(pool->pages_state_hold_cnt); in page_pool_inflight()
459 trace_page_pool_release(pool, inflight, hold_cnt, release_cnt); in page_pool_inflight()
460 WARN(inflight < 0, "Negative(%d) inflight packet-pages", inflight); in page_pool_inflight()
468 * page-allocator via put_page).
470 void page_pool_release_page(struct page_pool *pool, struct page *page) in page_pool_release_page() argument
472 dma_addr_t dma; in page_pool_release_page() local
475 if (!(pool->p.flags & PP_FLAG_DMA_MAP)) in page_pool_release_page()
481 dma = page_pool_get_dma_addr(page); in page_pool_release_page()
483 /* When page is unmapped, it cannot be returned to our pool */ in page_pool_release_page()
484 dma_unmap_page_attrs(pool->p.dev, dma, in page_pool_release_page()
485 PAGE_SIZE << pool->p.order, pool->p.dma_dir, in page_pool_release_page()
491 /* This may be the last page returned, releasing the pool, so in page_pool_release_page()
492 * it is not safe to reference pool afterwards. in page_pool_release_page()
494 count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt); in page_pool_release_page()
495 trace_page_pool_state_release(pool, page, count); in page_pool_release_page()
500 static void page_pool_return_page(struct page_pool *pool, struct page *page) in page_pool_return_page() argument
502 page_pool_release_page(pool, page); in page_pool_return_page()
505 /* An optimization would be to call __free_pages(page, pool->p.order) in page_pool_return_page()
506 * knowing page is not part of page-cache (thus avoiding a in page_pool_return_page()
511 static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page) in page_pool_recycle_in_ring() argument
516 ret = ptr_ring_produce(&pool->ring, page); in page_pool_recycle_in_ring()
518 ret = ptr_ring_produce_bh(&pool->ring, page); in page_pool_recycle_in_ring()
521 recycle_stat_inc(pool, ring); in page_pool_recycle_in_ring()
529 * alloc side cache. E.g. during RX-NAPI processing for XDP_DROP use-case.
534 struct page_pool *pool) in page_pool_recycle_in_cache() argument
536 if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) { in page_pool_recycle_in_cache()
537 recycle_stat_inc(pool, cache_full); in page_pool_recycle_in_cache()
542 pool->alloc.cache[pool->alloc.count++] = page; in page_pool_recycle_in_cache()
543 recycle_stat_inc(pool, cached); in page_pool_recycle_in_cache()
548 * if PP_FLAG_DMA_SYNC_DEV is set, we'll try to sync the DMA area for
549 * the configured size min(dma_sync_size, pool->max_len).
554 __page_pool_put_page(struct page_pool *pool, struct page *page, in __page_pool_put_page() argument
558 * one-frame-per-page, but have fallbacks that act like the in __page_pool_put_page()
569 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) in __page_pool_put_page()
570 page_pool_dma_sync_for_device(pool, page, in __page_pool_put_page()
574 page_pool_recycle_in_cache(page, pool)) in __page_pool_put_page()
580 /* Fallback/non-XDP mode: API user have elevated refcnt. in __page_pool_put_page()
585 * switching between XDP/non-XDP. in __page_pool_put_page()
587 * In-case page_pool maintains the DMA mapping, API user must in __page_pool_put_page()
589 * case, the DMA is unmapped/released, as driver is likely in __page_pool_put_page()
593 recycle_stat_inc(pool, released_refcnt); in __page_pool_put_page()
595 page_pool_release_page(pool, page); in __page_pool_put_page()
601 void page_pool_put_defragged_page(struct page_pool *pool, struct page *page, in page_pool_put_defragged_page() argument
604 page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct); in page_pool_put_defragged_page()
605 if (page && !page_pool_recycle_in_ring(pool, page)) { in page_pool_put_defragged_page()
607 recycle_stat_inc(pool, ring_full); in page_pool_put_defragged_page()
608 page_pool_return_page(pool, page); in page_pool_put_defragged_page()
614 void page_pool_put_page_bulk(struct page_pool *pool, void **data, in page_pool_put_page_bulk() argument
623 if (!page_pool_is_last_frag(pool, page)) in page_pool_put_page_bulk()
626 page = __page_pool_put_page(pool, page, -1, false); in page_pool_put_page_bulk()
636 page_pool_ring_lock(pool); in page_pool_put_page_bulk()
638 if (__ptr_ring_produce(&pool->ring, data[i])) { in page_pool_put_page_bulk()
640 recycle_stat_inc(pool, ring_full); in page_pool_put_page_bulk()
644 recycle_stat_add(pool, ring, i); in page_pool_put_page_bulk()
645 page_pool_ring_unlock(pool); in page_pool_put_page_bulk()
655 page_pool_return_page(pool, data[i]); in page_pool_put_page_bulk()
659 static struct page *page_pool_drain_frag(struct page_pool *pool, in page_pool_drain_frag() argument
662 long drain_count = BIAS_MAX - pool->frag_users; in page_pool_drain_frag()
669 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) in page_pool_drain_frag()
670 page_pool_dma_sync_for_device(pool, page, -1); in page_pool_drain_frag()
675 page_pool_return_page(pool, page); in page_pool_drain_frag()
679 static void page_pool_free_frag(struct page_pool *pool) in page_pool_free_frag() argument
681 long drain_count = BIAS_MAX - pool->frag_users; in page_pool_free_frag()
682 struct page *page = pool->frag_page; in page_pool_free_frag()
684 pool->frag_page = NULL; in page_pool_free_frag()
689 page_pool_return_page(pool, page); in page_pool_free_frag()
692 struct page *page_pool_alloc_frag(struct page_pool *pool, in page_pool_alloc_frag() argument
696 unsigned int max_size = PAGE_SIZE << pool->p.order; in page_pool_alloc_frag()
697 struct page *page = pool->frag_page; in page_pool_alloc_frag()
699 if (WARN_ON(!(pool->p.flags & PP_FLAG_PAGE_FRAG) || in page_pool_alloc_frag()
704 *offset = pool->frag_offset; in page_pool_alloc_frag()
707 page = page_pool_drain_frag(pool, page); in page_pool_alloc_frag()
709 alloc_stat_inc(pool, fast); in page_pool_alloc_frag()
715 page = page_pool_alloc_pages(pool, gfp); in page_pool_alloc_frag()
717 pool->frag_page = NULL; in page_pool_alloc_frag()
721 pool->frag_page = page; in page_pool_alloc_frag()
724 pool->frag_users = 1; in page_pool_alloc_frag()
726 pool->frag_offset = size; in page_pool_alloc_frag()
731 pool->frag_users++; in page_pool_alloc_frag()
732 pool->frag_offset = *offset + size; in page_pool_alloc_frag()
733 alloc_stat_inc(pool, fast); in page_pool_alloc_frag()
738 static void page_pool_empty_ring(struct page_pool *pool) in page_pool_empty_ring() argument
743 while ((page = ptr_ring_consume_bh(&pool->ring))) { in page_pool_empty_ring()
749 page_pool_return_page(pool, page); in page_pool_empty_ring()
753 static void page_pool_free(struct page_pool *pool) in page_pool_free() argument
755 if (pool->disconnect) in page_pool_free()
756 pool->disconnect(pool); in page_pool_free()
758 ptr_ring_cleanup(&pool->ring, NULL); in page_pool_free()
760 if (pool->p.flags & PP_FLAG_DMA_MAP) in page_pool_free()
761 put_device(pool->p.dev); in page_pool_free()
764 free_percpu(pool->recycle_stats); in page_pool_free()
766 kfree(pool); in page_pool_free()
769 static void page_pool_empty_alloc_cache_once(struct page_pool *pool) in page_pool_empty_alloc_cache_once() argument
773 if (pool->destroy_cnt) in page_pool_empty_alloc_cache_once()
777 * no-longer in use, and page_pool_alloc_pages() cannot be in page_pool_empty_alloc_cache_once()
780 while (pool->alloc.count) { in page_pool_empty_alloc_cache_once()
781 page = pool->alloc.cache[--pool->alloc.count]; in page_pool_empty_alloc_cache_once()
782 page_pool_return_page(pool, page); in page_pool_empty_alloc_cache_once()
786 static void page_pool_scrub(struct page_pool *pool) in page_pool_scrub() argument
788 page_pool_empty_alloc_cache_once(pool); in page_pool_scrub()
789 pool->destroy_cnt++; in page_pool_scrub()
792 * be in-flight. in page_pool_scrub()
794 page_pool_empty_ring(pool); in page_pool_scrub()
797 static int page_pool_release(struct page_pool *pool) in page_pool_release() argument
801 page_pool_scrub(pool); in page_pool_release()
802 inflight = page_pool_inflight(pool); in page_pool_release()
804 page_pool_free(pool); in page_pool_release()
812 struct page_pool *pool = container_of(dwq, typeof(*pool), release_dw); in page_pool_release_retry() local
815 inflight = page_pool_release(pool); in page_pool_release_retry()
820 if (time_after_eq(jiffies, pool->defer_warn)) { in page_pool_release_retry()
821 int sec = (s32)((u32)jiffies - (u32)pool->defer_start) / HZ; in page_pool_release_retry()
823 pr_warn("%s() stalled pool shutdown %d inflight %d sec\n", in page_pool_release_retry()
825 pool->defer_warn = jiffies + DEFER_WARN_INTERVAL; in page_pool_release_retry()
829 schedule_delayed_work(&pool->release_dw, DEFER_TIME); in page_pool_release_retry()
832 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), in page_pool_use_xdp_mem() argument
835 refcount_inc(&pool->user_cnt); in page_pool_use_xdp_mem()
836 pool->disconnect = disconnect; in page_pool_use_xdp_mem()
837 pool->xdp_mem_id = mem->id; in page_pool_use_xdp_mem()
840 void page_pool_destroy(struct page_pool *pool) in page_pool_destroy() argument
842 if (!pool) in page_pool_destroy()
845 if (!page_pool_put(pool)) in page_pool_destroy()
848 page_pool_free_frag(pool); in page_pool_destroy()
850 if (!page_pool_release(pool)) in page_pool_destroy()
853 pool->defer_start = jiffies; in page_pool_destroy()
854 pool->defer_warn = jiffies + DEFER_WARN_INTERVAL; in page_pool_destroy()
856 INIT_DELAYED_WORK(&pool->release_dw, page_pool_release_retry); in page_pool_destroy()
857 schedule_delayed_work(&pool->release_dw, DEFER_TIME); in page_pool_destroy()
862 void page_pool_update_nid(struct page_pool *pool, int new_nid) in page_pool_update_nid() argument
866 trace_page_pool_update_nid(pool, new_nid); in page_pool_update_nid()
867 pool->p.nid = new_nid; in page_pool_update_nid()
869 /* Flush pool alloc cache, as refill will check NUMA node */ in page_pool_update_nid()
870 while (pool->alloc.count) { in page_pool_update_nid()
871 page = pool->alloc.cache[--pool->alloc.count]; in page_pool_update_nid()
872 page_pool_return_page(pool, page); in page_pool_update_nid()
883 /* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation in page_pool_return_skb_page()
890 if (unlikely((page->pp_magic & ~0x3UL) != PP_SIGNATURE)) in page_pool_return_skb_page()
893 pp = page->pp; in page_pool_return_skb_page()
896 * This will *not* work for NIC using a split-page memory model. in page_pool_return_skb_page()
897 * The page will be returned to the pool here regardless of the in page_pool_return_skb_page()