| /Linux-v6.6/include/net/page_pool/ |
| D | types.h | 123 struct page_pool { struct
185 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); argument
186 struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
188 struct page_pool *page_pool_create(const struct page_pool_params *params);
193 void page_pool_unlink_napi(struct page_pool *pool);
194 void page_pool_destroy(struct page_pool *pool);
195 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
197 void page_pool_put_page_bulk(struct page_pool *pool, void **data,
200 static inline void page_pool_unlink_napi(struct page_pool *pool) in page_pool_unlink_napi()
204 static inline void page_pool_destroy(struct page_pool *pool) in page_pool_destroy()
[all …]
|
| D | helpers.h | 44 bool page_pool_get_stats(struct page_pool *pool,
69 static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool) in page_pool_dev_alloc_pages()
76 static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool, in page_pool_dev_alloc_frag()
93 inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) in page_pool_get_dma_dir()
134 static inline bool page_pool_is_last_frag(struct page_pool *pool, in page_pool_is_last_frag()
155 static inline void page_pool_put_page(struct page_pool *pool, in page_pool_put_page()
180 static inline void page_pool_put_full_page(struct page_pool *pool, in page_pool_put_full_page()
194 static inline void page_pool_recycle_direct(struct page_pool *pool, in page_pool_recycle_direct()
227 static inline bool page_pool_put(struct page_pool *pool) in page_pool_put()
232 static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid) in page_pool_nid_changed()
|
| /Linux-v6.6/net/core/ |
| D | page_pool.c | 72 bool page_pool_get_stats(struct page_pool *pool, in page_pool_get_stats()
148 static bool page_pool_producer_lock(struct page_pool *pool) in page_pool_producer_lock()
161 static void page_pool_producer_unlock(struct page_pool *pool, in page_pool_producer_unlock()
171 static int page_pool_init(struct page_pool *pool, in page_pool_init()
242 struct page_pool *page_pool_create(const struct page_pool_params *params) in page_pool_create()
244 struct page_pool *pool; in page_pool_create()
262 static void page_pool_return_page(struct page_pool *pool, struct page *page);
265 static struct page *page_pool_refill_alloc_cache(struct page_pool *pool) in page_pool_refill_alloc_cache()
318 static struct page *__page_pool_get_cached(struct page_pool *pool) in __page_pool_get_cached()
334 static void page_pool_dma_sync_for_device(struct page_pool *pool, in page_pool_dma_sync_for_device()
[all …]
|
| /Linux-v6.6/include/trace/events/ |
| D | page_pool.h | 3 #define TRACE_SYSTEM page_pool
16 TP_PROTO(const struct page_pool *pool,
22 __field(const struct page_pool *, pool)
44 TP_PROTO(const struct page_pool *pool,
50 __field(const struct page_pool *, pool)
69 TP_PROTO(const struct page_pool *pool,
75 __field(const struct page_pool *, pool)
94 TP_PROTO(const struct page_pool *pool, int new_nid),
99 __field(const struct page_pool *, pool)
|
| /Linux-v6.6/Documentation/networking/ |
| D | page_pool.rst | 7 .. kernel-doc:: include/net/page_pool/helpers.h
8 :doc: page_pool allocator
53 .. kernel-doc:: net/core/page_pool.c
56 .. kernel-doc:: include/net/page_pool/types.h
59 .. kernel-doc:: include/net/page_pool/helpers.h
64 .. kernel-doc:: net/core/page_pool.c
105 It takes a pointer to a ``struct page_pool`` and a pointer to a struct
109 statistics about the page_pool.
111 .. kernel-doc:: include/net/page_pool/types.h
130 /* internal DMA mapping in page_pool */
[all …]
|
| /Linux-v6.6/drivers/net/ethernet/apm/xgene/ |
| D | xgene_enet_main.c | 672 struct xgene_enet_desc_ring *buf_pool, *page_pool; in xgene_enet_rx_frame() local
690 page_pool = rx_ring->page_pool; in xgene_enet_rx_frame()
719 xgene_enet_free_pagepool(page_pool, raw_desc, exp_desc); in xgene_enet_rx_frame()
729 slots = page_pool->slots - 1; in xgene_enet_rx_frame()
730 head = page_pool->head; in xgene_enet_rx_frame()
741 page = page_pool->frag_page[head]; in xgene_enet_rx_frame()
747 page_pool->frag_page[head] = NULL; in xgene_enet_rx_frame()
751 page_pool->head = head; in xgene_enet_rx_frame()
764 ret = xgene_enet_refill_pagepool(page_pool, NUM_NXTBUFPOOL); in xgene_enet_rx_frame()
1062 struct xgene_enet_desc_ring *buf_pool, *page_pool; in xgene_enet_delete_desc_rings() local
[all …]
|
| D | xgene_enet_cle.c | 709 if (pdata->rx_ring[idx]->page_pool) { in xgene_cle_set_rss_idt()
710 pool_id = pdata->rx_ring[idx]->page_pool->id; in xgene_cle_set_rss_idt()
786 if (pdata->rx_ring[0]->page_pool) { in xgene_enet_cle_init()
787 pool_id = pdata->rx_ring[0]->page_pool->id; in xgene_enet_cle_init()
|
| /Linux-v6.6/drivers/net/vmxnet3/ |
| D | vmxnet3_xdp.c | 272 page_pool_recycle_direct(rq->page_pool, page); in vmxnet3_run_xdp()
280 page_pool_recycle_direct(rq->page_pool, page); in vmxnet3_run_xdp()
297 page_pool_recycle_direct(rq->page_pool, page); in vmxnet3_run_xdp()
310 page_pool_recycle_direct(rq->page_pool, page); in vmxnet3_build_skb()
335 page = page_pool_alloc_pages(rq->page_pool, GFP_ATOMIC); in vmxnet3_process_xdp_small()
342 xdp_prepare_buff(&xdp, page_address(page), rq->page_pool->p.offset, in vmxnet3_process_xdp_small()
385 rq->page_pool->p.offset, rcd->len, in vmxnet3_process_xdp()
386 page_pool_get_dma_dir(rq->page_pool)); in vmxnet3_process_xdp()
389 xdp_prepare_buff(&xdp, page_address(page), rq->page_pool->p.offset, in vmxnet3_process_xdp()
407 new_data = vmxnet3_pp_get_buff(rq->page_pool, &new_dma_addr, in vmxnet3_process_xdp()
|
| D | vmxnet3_xdp.h | 39 void *vmxnet3_pp_get_buff(struct page_pool *pp, dma_addr_t *dma_addr,
|
| /Linux-v6.6/drivers/net/ethernet/microchip/lan966x/ |
| D | lan966x_fdma.c | 19 page = page_pool_dev_alloc_pages(rx->page_pool); in lan966x_fdma_rx_alloc_page()
34 page_pool_put_full_page(rx->page_pool, in lan966x_fdma_rx_free_pages()
47 page_pool_recycle_direct(rx->page_pool, page); in lan966x_fdma_rx_free_page()
87 rx->page_pool = page_pool_create(&pp_params); in lan966x_fdma_rx_alloc_page_pool()
98 rx->page_pool); in lan966x_fdma_rx_alloc_page_pool()
101 return PTR_ERR_OR_ZERO(rx->page_pool); in lan966x_fdma_rx_alloc_page_pool()
114 return PTR_ERR(rx->page_pool); in lan966x_fdma_rx_alloc()
437 page_pool_recycle_direct(rx->page_pool, in lan966x_fdma_tx_clear_buf()
540 page_pool_recycle_direct(rx->page_pool, page); in lan966x_fdma_rx_get_frame()
910 struct page_pool *page_pool; in lan966x_fdma_reload() local
[all …]
|
| /Linux-v6.6/include/net/ |
| D | xdp_priv.h | 13 struct page_pool *page_pool; member
|
| /Linux-v6.6/block/ |
| D | bounce.c | 32 static mempool_t page_pool; variable
61 ret = mempool_init_page_pool(&page_pool, POOL_SIZE, 0); in init_emergency_pool()
116 mempool_free(bvec->bv_page, &page_pool); in bounce_end_io()
247 bounce_page = mempool_alloc(&page_pool, GFP_NOIO); in __blk_queue_bounce()
|
| /Linux-v6.6/drivers/net/ethernet/marvell/octeontx2/nic/ |
| D | otx2_txrx.h | 123 struct page_pool *page_pool; member
|
| D | otx2_common.c | 533 page = page_pool_alloc_frag(pool->page_pool, &offset, sz, GFP_ATOMIC); in otx2_alloc_pool_buf()
546 if (pool->page_pool) in __otx2_alloc_rbuf()
1215 if (pool->page_pool) { in otx2_free_bufs()
1216 page_pool_put_full_page(pool->page_pool, page, true); in otx2_free_bufs()
1270 page_pool_destroy(pool->page_pool); in otx2_aura_pool_free()
1271 pool->page_pool = NULL; in otx2_aura_pool_free()
1402 pool->page_pool = NULL; in otx2_pool_init()
1412 pool->page_pool = page_pool_create(&pp_params); in otx2_pool_init()
1413 if (IS_ERR(pool->page_pool)) { in otx2_pool_init()
1415 return PTR_ERR(pool->page_pool); in otx2_pool_init()
|
| /Linux-v6.6/drivers/net/ |
| D | veth.c | 69 struct page_pool *page_pool; member
187 if (!priv->rq[i].page_pool) in veth_get_page_pool_stats()
189 page_pool_get_stats(priv->rq[i].page_pool, &pp_stats); in veth_get_page_pool_stats()
757 page = page_pool_dev_alloc_pages(rq->page_pool); in veth_convert_skb_to_xdp_buff()
763 page_pool_put_full_page(rq->page_pool, page, true); in veth_convert_skb_to_xdp_buff()
786 page = page_pool_dev_alloc_pages(rq->page_pool); in veth_convert_skb_to_xdp_buff()
1043 rq->page_pool = page_pool_create(&pp_params); in veth_create_page_pool()
1044 if (IS_ERR(rq->page_pool)) { in veth_create_page_pool()
1045 int err = PTR_ERR(rq->page_pool); in veth_create_page_pool()
1047 rq->page_pool = NULL; in veth_create_page_pool()
[all …]
|
| D | xen-netfront.c | 156 struct page_pool *page_pool; member
281 page = page_pool_alloc_pages(queue->page_pool, in xennet_alloc_one_rx_buffer()
1846 page_pool_destroy(queue->page_pool); in xennet_disconnect_backend()
2172 queue->page_pool = page_pool_create(&pp_params); in xennet_create_page_pool()
2173 if (IS_ERR(queue->page_pool)) { in xennet_create_page_pool()
2174 err = PTR_ERR(queue->page_pool); in xennet_create_page_pool()
2175 queue->page_pool = NULL; in xennet_create_page_pool()
2187 MEM_TYPE_PAGE_POOL, queue->page_pool); in xennet_create_page_pool()
2197 page_pool_destroy(queue->page_pool); in xennet_create_page_pool()
2198 queue->page_pool = NULL; in xennet_create_page_pool()
|
| /Linux-v6.6/drivers/net/ethernet/socionext/ |
| D | netsec.c | 286 struct page_pool *page_pool; member
742 page = page_pool_dev_alloc_pages(dring->page_pool); in netsec_alloc_rx_data()
863 page_pool_get_dma_dir(rx_ring->page_pool); in netsec_xdp_queue_one()
922 page_pool_put_page(dring->page_pool, page, sync, true); in netsec_run_xdp()
932 page_pool_put_page(dring->page_pool, page, sync, true); in netsec_run_xdp()
944 page_pool_put_page(dring->page_pool, page, sync, true); in netsec_run_xdp()
966 dma_dir = page_pool_get_dma_dir(dring->page_pool); in netsec_process_rx()
1041 page_pool_put_page(dring->page_pool, page, pkt_len, in netsec_process_rx()
1215 page_pool_put_full_page(dring->page_pool, page, false); in netsec_uninit_pkt_dring()
1227 page_pool_destroy(dring->page_pool); in netsec_uninit_pkt_dring()
[all …]
|
| /Linux-v6.6/drivers/net/ethernet/ti/ |
| D | cpsw_priv.c | 1106 struct page_pool *pool; in cpsw_fill_rx_channels()
1113 pool = cpsw->page_pool[ch]; in cpsw_fill_rx_channels()
1147 static struct page_pool *cpsw_create_page_pool(struct cpsw_common *cpsw, in cpsw_create_page_pool()
1151 struct page_pool *pool; in cpsw_create_page_pool()
1169 struct page_pool *pool; in cpsw_create_rx_pool()
1177 cpsw->page_pool[ch] = pool; in cpsw_create_rx_pool()
1186 struct page_pool *pool; in cpsw_ndev_create_xdp_rxq()
1189 pool = cpsw->page_pool[ch]; in cpsw_ndev_create_xdp_rxq()
1227 page_pool_destroy(cpsw->page_pool[ch]); in cpsw_destroy_xdp_rxqs()
1228 cpsw->page_pool[ch] = NULL; in cpsw_destroy_xdp_rxqs()
[all …]
|
| /Linux-v6.6/drivers/net/ethernet/engleder/ |
| D | tsnep.h | 129 struct page_pool *page_pool; member
|
| D | tsnep_main.c | 892 page_pool_put_full_page(rx->page_pool, entry->page, in tsnep_rx_ring_cleanup()
900 if (rx->page_pool) in tsnep_rx_ring_cleanup()
901 page_pool_destroy(rx->page_pool); in tsnep_rx_ring_cleanup()
950 rx->page_pool = page_pool_create(&pp_params); in tsnep_rx_ring_create()
951 if (IS_ERR(rx->page_pool)) { in tsnep_rx_ring_create()
952 retval = PTR_ERR(rx->page_pool); in tsnep_rx_ring_create()
953 rx->page_pool = NULL; in tsnep_rx_ring_create()
1018 page_pool_put_full_page(rx->page_pool, *page, false); in tsnep_rx_free_page_buffer()
1032 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool); in tsnep_rx_alloc_page_buffer()
1057 page = page_pool_dev_alloc_pages(rx->page_pool); in tsnep_rx_alloc_buffer()
[all …]
|
| /Linux-v6.6/drivers/net/ethernet/broadcom/bnxt/ |
| D | bnxt_xdp.c | 164 page_pool_recycle_direct(rxr->page_pool, tx_buf->page); in bnxt_tx_int_xdp()
222 page_pool_recycle_direct(rxr->page_pool, page); in bnxt_xdp_buff_frags_free()
323 page_pool_recycle_direct(rxr->page_pool, page); in bnxt_rx_xdp()
477 struct page_pool *pool, struct xdp_buff *xdp, in bnxt_xdp_build_skb()
|
| D | bnxt_xdp.h | 35 u8 num_frags, struct page_pool *pool,
|
| /Linux-v6.6/drivers/net/ethernet/wangxun/libwx/ |
| D | wx_lib.c | 231 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false); in wx_dma_sync_frag()
291 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false); in wx_put_rx_buffer()
378 page = page_pool_dev_alloc_pages(rx_ring->page_pool); in wx_alloc_mapped_page()
2228 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false); in wx_clean_rx_ring()
2243 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false); in wx_clean_rx_ring()
2294 if (rx_ring->page_pool) { in wx_free_rx_resources()
2295 page_pool_destroy(rx_ring->page_pool); in wx_free_rx_resources()
2296 rx_ring->page_pool = NULL; in wx_free_rx_resources()
2447 rx_ring->page_pool = page_pool_create(&pp_params); in wx_alloc_page_pool()
2448 if (IS_ERR(rx_ring->page_pool)) { in wx_alloc_page_pool()
[all …]
|
| /Linux-v6.6/drivers/net/ethernet/marvell/mvpp2/ |
| D | mvpp2_main.c | 104 static struct page_pool *
361 struct page_pool *page_pool) in mvpp2_frag_alloc() argument
363 if (page_pool) in mvpp2_frag_alloc()
364 return page_pool_dev_alloc_pages(page_pool); in mvpp2_frag_alloc()
373 struct page_pool *page_pool, void *data) in mvpp2_frag_free() argument
375 if (page_pool) in mvpp2_frag_free()
376 page_pool_put_full_page(page_pool, virt_to_head_page(data), false); in mvpp2_frag_free()
495 struct page_pool *pp = NULL; in mvpp2_bm_bufs_free()
505 pp = priv->page_pool[bm_pool->id]; in mvpp2_bm_bufs_free()
570 page_pool_destroy(priv->page_pool[bm_pool->id]); in mvpp2_bm_pool_destroy()
[all …]
|
| /Linux-v6.6/drivers/net/ethernet/stmicro/stmmac/ |
| D | stmmac.h | 108 struct page_pool *page_pool; member
|
