| /Linux-v5.15/net/core/ |
| D | xdp.c | 377 void xdp_return_frame(struct xdp_frame *xdpf) in xdp_return_frame() argument
379 __xdp_return(xdpf->data, &xdpf->mem, false, NULL); in xdp_return_frame()
383 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf) in xdp_return_frame_rx_napi() argument
385 __xdp_return(xdpf->data, &xdpf->mem, true, NULL); in xdp_return_frame_rx_napi()
413 void xdp_return_frame_bulk(struct xdp_frame *xdpf, in xdp_return_frame_bulk() argument
416 struct xdp_mem_info *mem = &xdpf->mem; in xdp_return_frame_bulk()
420 __xdp_return(xdpf->data, &xdpf->mem, false, NULL); in xdp_return_frame_bulk()
439 bq->q[bq->count++] = xdpf->data; in xdp_return_frame_bulk()
477 struct xdp_frame *xdpf; in xdp_convert_zc_to_xdp_frame() local
485 if (sizeof(*xdpf) + totsize > PAGE_SIZE) in xdp_convert_zc_to_xdp_frame()
[all …]
|
| /Linux-v5.15/kernel/bpf/ |
| D | cpumap.c | 152 struct xdp_frame *xdpf; in __cpu_map_ring_cleanup() local
154 while ((xdpf = ptr_ring_consume(ring))) in __cpu_map_ring_cleanup()
155 if (WARN_ON_ONCE(xdpf)) in __cpu_map_ring_cleanup()
156 xdp_return_frame(xdpf); in __cpu_map_ring_cleanup()
224 struct xdp_frame *xdpf = frames[i]; in cpu_map_bpf_prog_run_xdp() local
228 rxq.dev = xdpf->dev_rx; in cpu_map_bpf_prog_run_xdp()
229 rxq.mem = xdpf->mem; in cpu_map_bpf_prog_run_xdp()
232 xdp_convert_frame_to_buff(xdpf, &xdp); in cpu_map_bpf_prog_run_xdp()
237 err = xdp_update_frame_from_buff(&xdp, xdpf); in cpu_map_bpf_prog_run_xdp()
239 xdp_return_frame(xdpf); in cpu_map_bpf_prog_run_xdp()
[all …]
|
| D | devmap.c | 334 struct xdp_frame *xdpf = frames[i]; in dev_map_bpf_prog_run() local
338 xdp_convert_frame_to_buff(xdpf, &xdp); in dev_map_bpf_prog_run()
344 err = xdp_update_frame_from_buff(&xdp, xdpf); in dev_map_bpf_prog_run()
346 xdp_return_frame_rx_napi(xdpf); in dev_map_bpf_prog_run()
348 frames[nframes++] = xdpf; in dev_map_bpf_prog_run()
357 xdp_return_frame_rx_napi(xdpf); in dev_map_bpf_prog_run()
376 struct xdp_frame *xdpf = bq->q[i]; in bq_xmit_all() local
378 prefetch(xdpf); in bq_xmit_all()
445 static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, in bq_enqueue() argument
467 bq->q[bq->count++] = xdpf; in bq_enqueue()
[all …]
|
| /Linux-v5.15/include/net/ |
| D | xdp.h | 167 struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
170 struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf,
173 struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf);
233 void xdp_return_frame(struct xdp_frame *xdpf);
234 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
237 void xdp_return_frame_bulk(struct xdp_frame *xdpf,
246 static inline void xdp_release_frame(struct xdp_frame *xdpf) in xdp_release_frame() argument
248 struct xdp_mem_info *mem = &xdpf->mem; in xdp_release_frame()
252 __xdp_release_frame(xdpf->data, mem); in xdp_release_frame()
|
| /Linux-v5.15/drivers/net/ethernet/mellanox/mlx5/core/en/ |
| D | xdp.c | 64 struct xdp_frame *xdpf; in mlx5e_xmit_xdp_buff() local
67 xdpf = xdp_convert_buff_to_frame(xdp); in mlx5e_xmit_xdp_buff()
68 if (unlikely(!xdpf)) in mlx5e_xmit_xdp_buff()
71 xdptxd.data = xdpf->data; in mlx5e_xmit_xdp_buff()
72 xdptxd.len = xdpf->len; in mlx5e_xmit_xdp_buff()
92 xdp_return_frame(xdpf); in mlx5e_xmit_xdp_buff()
97 xdpi.frame.xdpf = xdpf; in mlx5e_xmit_xdp_buff()
108 dma_addr = di->addr + (xdpf->data - (void *)xdpf); in mlx5e_xmit_xdp_buff()
382 xdpi.frame.xdpf->len, DMA_TO_DEVICE); in mlx5e_free_xdpsq_desc()
383 xdp_return_frame_bulk(xdpi.frame.xdpf, bq); in mlx5e_free_xdpsq_desc()
[all …]
|
| /Linux-v5.15/drivers/net/ethernet/intel/ixgbe/ |
| D | ixgbe_xsk.c | 103 struct xdp_frame *xdpf; in ixgbe_run_xdp_zc() local
120 xdpf = xdp_convert_buff_to_frame(xdp); in ixgbe_run_xdp_zc()
121 if (unlikely(!xdpf)) in ixgbe_run_xdp_zc()
123 result = ixgbe_xmit_xdp_ring(adapter, xdpf); in ixgbe_run_xdp_zc()
405 tx_bi->xdpf = NULL; in ixgbe_xmit_zc()
436 xdp_return_frame(tx_bi->xdpf); in ixgbe_clean_xdp_tx_buffer()
463 if (tx_bi->xdpf) in ixgbe_clean_xdp_tx_irq()
468 tx_bi->xdpf = NULL; in ixgbe_clean_xdp_tx_irq()
542 if (tx_bi->xdpf) in ixgbe_xsk_clean_tx_ring()
547 tx_bi->xdpf = NULL; in ixgbe_xsk_clean_tx_ring()
|
| D | ixgbe_txrx_common.h | 16 struct xdp_frame *xdpf);
|
| /Linux-v5.15/drivers/net/ethernet/intel/ice/ |
| D | ice_txrx_lib.c | 274 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); in ice_xmit_xdp_buff() local
276 if (unlikely(!xdpf)) in ice_xmit_xdp_buff()
279 return ice_xmit_xdp_ring(xdpf->data, xdpf->len, xdp_ring); in ice_xmit_xdp_buff()
|
| /Linux-v5.15/drivers/net/ethernet/ti/ |
| D | cpsw_priv.c | 54 struct xdp_frame *xdpf; in cpsw_tx_handler() local
61 xdpf = cpsw_handle_to_xdpf(token); in cpsw_tx_handler()
62 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET; in cpsw_tx_handler()
65 xdp_return_frame(xdpf); in cpsw_tx_handler()
1288 int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf, in cpsw_xdp_tx_frame() argument
1297 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET; in cpsw_xdp_tx_frame()
1304 dma += xdpf->headroom + sizeof(struct xdp_frame); in cpsw_xdp_tx_frame()
1305 ret = cpdma_chan_submit_mapped(txch, cpsw_xdpf_to_handle(xdpf), in cpsw_xdp_tx_frame()
1306 dma, xdpf->len, port); in cpsw_xdp_tx_frame()
1308 if (sizeof(*xmeta) > xdpf->headroom) in cpsw_xdp_tx_frame()
[all …]
|
| D | cpsw_priv.h | 419 static inline void *cpsw_xdpf_to_handle(struct xdp_frame *xdpf) in cpsw_xdpf_to_handle() argument
421 return (void *)((unsigned long)xdpf | BIT(0)); in cpsw_xdpf_to_handle()
440 int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf,
|
| /Linux-v5.15/drivers/net/ethernet/freescale/dpaa/ |
| D | dpaa_eth.c | 1701 xdp_return_frame(swbp->xdpf); in dpaa_cleanup_tx_fd()
2182 struct xdp_frame *new_xdpf, *xdpf = *init_xdpf; in dpaa_a050385_wa_xdpf() local
2196 if (PTR_IS_ALIGNED(xdpf->data, DPAA_FD_DATA_ALIGNMENT) && in dpaa_a050385_wa_xdpf()
2197 xdpf->headroom >= priv->tx_headroom) { in dpaa_a050385_wa_xdpf()
2198 xdpf->headroom = priv->tx_headroom; in dpaa_a050385_wa_xdpf()
2206 aligned_data = PTR_ALIGN_DOWN(xdpf->data, DPAA_FD_DATA_ALIGNMENT); in dpaa_a050385_wa_xdpf()
2207 data_shift = xdpf->data - aligned_data; in dpaa_a050385_wa_xdpf()
2212 if (xdpf->headroom >= data_shift + priv->tx_headroom) { in dpaa_a050385_wa_xdpf()
2213 memmove(aligned_data, xdpf->data, xdpf->len); in dpaa_a050385_wa_xdpf()
2214 xdpf->data = aligned_data; in dpaa_a050385_wa_xdpf()
[all …]
|
| D | dpaa_eth.h | 155 struct xdp_frame *xdpf; member
|
| /Linux-v5.15/drivers/net/ethernet/broadcom/bnxt/ |
| D | bnxt_xdp.c | 60 struct xdp_frame *xdpf) in __bnxt_xmit_xdp_redirect() argument
66 tx_buf->xdpf = xdpf; in __bnxt_xmit_xdp_redirect()
91 xdp_return_frame(tx_buf->xdpf); in bnxt_tx_int_xdp()
93 tx_buf->xdpf = NULL; in bnxt_tx_int_xdp()
|
| /Linux-v5.15/drivers/net/ethernet/sfc/ |
| D | tx.c | 426 struct xdp_frame *xdpf; in efx_xdp_tx_buffers() local
465 xdpf = xdpfs[i]; in efx_xdp_tx_buffers()
473 len = xdpf->len; in efx_xdp_tx_buffers()
477 xdpf->data, len, in efx_xdp_tx_buffers()
484 tx_buffer->xdpf = xdpf; in efx_xdp_tx_buffers()
|
| D | rx.c | 257 struct xdp_frame *xdpf; in efx_do_xdp() local
312 xdpf = xdp_convert_buff_to_frame(&xdp); in efx_do_xdp()
313 err = efx_xdp_tx_buffers(efx, 1, &xdpf, true); in efx_do_xdp()
|
| /Linux-v5.15/drivers/net/ethernet/socionext/ |
| D | netsec.c | 272 struct xdp_frame *xdpf; member
675 bytes += desc->xdpf->len; in netsec_clean_tx_dring()
677 xdp_return_frame_rx_napi(desc->xdpf); in netsec_clean_tx_dring()
679 xdp_return_frame_bulk(desc->xdpf, &bq); in netsec_clean_tx_dring()
821 dring->desc[idx].xdpf = buf; in netsec_set_tx_de()
829 struct xdp_frame *xdpf, bool is_ndo) in netsec_xdp_queue_one() argument
833 struct page *page = virt_to_page(xdpf->data); in netsec_xdp_queue_one()
851 dma_handle = dma_map_single(priv->dev, xdpf->data, xdpf->len, in netsec_xdp_queue_one()
865 dma_handle = page_pool_get_dma_addr(page) + xdpf->headroom + in netsec_xdp_queue_one()
866 sizeof(*xdpf); in netsec_xdp_queue_one()
[all …]
|
| /Linux-v5.15/drivers/net/ethernet/qlogic/qede/ |
| D | qede_fp.c | 306 u16 len, struct page *page, struct xdp_frame *xdpf) in qede_xdp_xmit() argument
333 xdp->xdpf = xdpf; in qede_xdp_xmit()
346 struct xdp_frame *xdpf; in qede_xdp_transmit() local
363 xdpf = frames[i]; in qede_xdp_transmit()
365 mapping = dma_map_single(dmadev, xdpf->data, xdpf->len, in qede_xdp_transmit()
370 if (unlikely(qede_xdp_xmit(xdp_tx, mapping, 0, xdpf->len, in qede_xdp_transmit()
371 NULL, xdpf))) in qede_xdp_transmit()
405 struct xdp_frame *xdpf; in qede_xdp_tx_int() local
413 xdpf = xdp_info->xdpf; in qede_xdp_tx_int()
415 if (xdpf) { in qede_xdp_tx_int()
[all …]
|
| /Linux-v5.15/drivers/net/ethernet/intel/i40e/ |
| D | i40e_xsk.c | 506 xdp_return_frame(tx_bi->xdpf); in i40e_clean_xdp_tx_buffer()
546 if (tx_bi->xdpf) { in i40e_clean_xdp_tx_irq()
548 tx_bi->xdpf = NULL; in i40e_clean_xdp_tx_irq()
645 if (tx_bi->xdpf) in i40e_xsk_clean_tx_ring()
650 tx_bi->xdpf = NULL; in i40e_xsk_clean_tx_ring()
|
| D | i40e_txrx.c | 763 xdp_return_frame(tx_buffer->xdpf); in i40e_unmap_and_free_tx_resource()
971 xdp_return_frame(tx_buf->xdpf); in i40e_clean_tx_irq()
2276 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
2281 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); in i40e_xmit_xdp_tx_ring() local
2283 if (unlikely(!xdpf)) in i40e_xmit_xdp_tx_ring()
2286 return i40e_xmit_xdp_ring(xdpf, xdp_ring); in i40e_xmit_xdp_tx_ring()
3688 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf, in i40e_xmit_xdp_ring() argument
3694 void *data = xdpf->data; in i40e_xmit_xdp_ring()
3695 u32 size = xdpf->len; in i40e_xmit_xdp_ring()
3709 tx_bi->xdpf = xdpf; in i40e_xmit_xdp_ring()
[all …]
|
| /Linux-v5.15/drivers/net/ |
| D | virtio_net.c | 501 struct xdp_frame *xdpf) in __virtnet_xdp_xmit_one() argument
506 if (unlikely(xdpf->headroom < vi->hdr_len)) in __virtnet_xdp_xmit_one()
510 xdpf->data -= vi->hdr_len; in __virtnet_xdp_xmit_one()
512 hdr = xdpf->data; in __virtnet_xdp_xmit_one()
514 xdpf->len += vi->hdr_len; in __virtnet_xdp_xmit_one()
516 sg_init_one(sq->sg, xdpf->data, xdpf->len); in __virtnet_xdp_xmit_one()
518 err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp_to_ptr(xdpf), in __virtnet_xdp_xmit_one()
611 struct xdp_frame *xdpf = frames[i]; in virtnet_xdp_xmit() local
613 if (__virtnet_xdp_xmit_one(vi, sq, xdpf)) in virtnet_xdp_xmit()
739 struct xdp_frame *xdpf; in receive_small() local
[all …]
|
| D | xen-netfront.c | 601 struct xdp_frame *xdpf) in xennet_xdp_xmit_one() argument
608 .page = virt_to_page(xdpf->data), in xennet_xdp_xmit_one()
613 offset_in_page(xdpf->data), in xennet_xdp_xmit_one()
614 xdpf->len); in xennet_xdp_xmit_one()
623 tx_stats->bytes += xdpf->len; in xennet_xdp_xmit_one()
651 struct xdp_frame *xdpf = frames[i]; in xennet_xdp_xmit() local
653 if (!xdpf) in xennet_xdp_xmit()
655 if (xennet_xdp_xmit_one(dev, queue, xdpf)) in xennet_xdp_xmit()
896 struct xdp_frame *xdpf; in xennet_run_xdp() local
910 xdpf = xdp_convert_buff_to_frame(xdp); in xennet_run_xdp()
[all …]
|
| /Linux-v5.15/drivers/net/ethernet/amazon/ena/ |
| D | ena_netdev.c | 241 struct xdp_frame *xdpf, in ena_xdp_tx_map_frame() argument
251 tx_info->xdpf = xdpf; in ena_xdp_tx_map_frame()
252 data = tx_info->xdpf->data; in ena_xdp_tx_map_frame()
253 size = tx_info->xdpf->len; in ena_xdp_tx_map_frame()
297 struct xdp_frame *xdpf, in ena_xdp_xmit_frame() argument
310 rc = ena_xdp_tx_map_frame(xdp_ring, tx_info, xdpf, &ena_tx_ctx); in ena_xdp_xmit_frame()
321 xdpf->len); in ena_xdp_xmit_frame()
335 tx_info->xdpf = NULL; in ena_xdp_xmit_frame()
384 struct xdp_frame *xdpf; in ena_xdp_execute() local
396 xdpf = xdp_convert_buff_to_frame(xdp); in ena_xdp_execute()
[all …]
|
| /Linux-v5.15/drivers/net/ethernet/stmicro/stmmac/ |
| D | stmmac_main.c | 1503 if (tx_q->xdpf[i] && in stmmac_free_tx_buffer()
1506 xdp_return_frame(tx_q->xdpf[i]); in stmmac_free_tx_buffer()
1507 tx_q->xdpf[i] = NULL; in stmmac_free_tx_buffer()
2396 tx_q->xdpf[entry] = NULL; in stmmac_xdp_xmit_zc()
2466 struct xdp_frame *xdpf; in stmmac_tx_clean() local
2473 xdpf = tx_q->xdpf[entry]; in stmmac_tx_clean()
2476 xdpf = NULL; in stmmac_tx_clean()
2479 xdpf = NULL; in stmmac_tx_clean()
2539 if (xdpf && in stmmac_tx_clean()
2541 xdp_return_frame_rx_napi(xdpf); in stmmac_tx_clean()
[all …]
|
| /Linux-v5.15/drivers/net/ethernet/marvell/ |
| D | mvneta.c | 618 struct xdp_frame *xdpf; member
1862 xdp_return_frame_rx_napi(buf->xdpf); in mvneta_txq_bufs_free()
1864 xdp_return_frame_bulk(buf->xdpf, &bq); in mvneta_txq_bufs_free()
2049 struct xdp_frame *xdpf, bool dma_map) in mvneta_xdp_submit_frame() argument
2063 dma_addr = dma_map_single(pp->dev->dev.parent, xdpf->data, in mvneta_xdp_submit_frame()
2064 xdpf->len, DMA_TO_DEVICE); in mvneta_xdp_submit_frame()
2071 struct page *page = virt_to_page(xdpf->data); in mvneta_xdp_submit_frame()
2074 sizeof(*xdpf) + xdpf->headroom; in mvneta_xdp_submit_frame()
2076 xdpf->len, DMA_BIDIRECTIONAL); in mvneta_xdp_submit_frame()
2079 buf->xdpf = xdpf; in mvneta_xdp_submit_frame()
[all …]
|
| /Linux-v5.15/drivers/net/ethernet/intel/igc/ |
| D | igc_main.c | 208 xdp_return_frame(tx_buffer->xdpf); in igc_clean_tx_ring()
2119 struct xdp_frame *xdpf, in igc_xdp_init_tx_buffer() argument
2124 dma = dma_map_single(ring->dev, xdpf->data, xdpf->len, DMA_TO_DEVICE); in igc_xdp_init_tx_buffer()
2131 buffer->xdpf = xdpf; in igc_xdp_init_tx_buffer()
2133 buffer->bytecount = xdpf->len; in igc_xdp_init_tx_buffer()
2136 dma_unmap_len_set(buffer, len, xdpf->len); in igc_xdp_init_tx_buffer()
2143 struct xdp_frame *xdpf) in igc_xdp_init_tx_descriptor() argument
2154 err = igc_xdp_init_tx_buffer(buffer, xdpf, ring); in igc_xdp_init_tx_descriptor()
2195 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); in igc_xdp_xmit_back() local
2201 if (unlikely(!xdpf)) in igc_xdp_xmit_back()
[all …]
|
