| /Linux-v6.1/kernel/bpf/ |
| D | devmap.c | 61 struct bpf_prog *xdp_prog; member
69 struct bpf_prog *xdp_prog; member
218 if (dev->xdp_prog) in dev_map_free()
219 bpf_prog_put(dev->xdp_prog); in dev_map_free()
234 if (dev->xdp_prog) in dev_map_free()
235 bpf_prog_put(dev->xdp_prog); in dev_map_free()
326 static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, in dev_map_bpf_prog_run() argument
342 act = bpf_prog_run_xdp(xdp_prog, &xdp); in dev_map_bpf_prog_run()
352 bpf_warn_invalid_xdp_action(NULL, xdp_prog, act); in dev_map_bpf_prog_run()
355 trace_xdp_exception(dev, xdp_prog, act); in dev_map_bpf_prog_run()
[all …]
|
| /Linux-v6.1/drivers/net/ethernet/broadcom/bnxt/ |
| D | bnxt_xdp.c | 174 struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); in bnxt_xdp_attached() local
176 return !!xdp_prog; in bnxt_xdp_attached()
227 struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); in bnxt_rx_xdp() local
238 if (!xdp_prog) in bnxt_rx_xdp()
248 act = bpf_prog_run_xdp(xdp_prog, &xdp); in bnxt_rx_xdp()
278 trace_xdp_exception(bp->dev, xdp_prog, act); in bnxt_rx_xdp()
305 trace_xdp_exception(bp->dev, xdp_prog, act); in bnxt_rx_xdp()
311 if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) { in bnxt_rx_xdp()
312 trace_xdp_exception(bp->dev, xdp_prog, act); in bnxt_rx_xdp()
320 bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act); in bnxt_rx_xdp()
[all …]
|
| /Linux-v6.1/drivers/net/ethernet/sfc/ |
| D | rx.c | 256 struct bpf_prog *xdp_prog; in efx_do_xdp() local
263 xdp_prog = rcu_dereference_bh(efx->xdp_prog); in efx_do_xdp()
264 if (!xdp_prog) in efx_do_xdp()
294 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); in efx_do_xdp()
320 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
327 err = xdp_do_redirect(efx->net_dev, &xdp, xdp_prog); in efx_do_xdp()
334 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
341 bpf_warn_invalid_xdp_action(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
344 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
348 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
|
| /Linux-v6.1/drivers/net/ethernet/sfc/siena/ |
| D | rx.c | 256 struct bpf_prog *xdp_prog; in efx_do_xdp() local
263 xdp_prog = rcu_dereference_bh(efx->xdp_prog); in efx_do_xdp()
264 if (!xdp_prog) in efx_do_xdp()
294 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); in efx_do_xdp()
320 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
327 err = xdp_do_redirect(efx->net_dev, &xdp, xdp_prog); in efx_do_xdp()
334 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
341 bpf_warn_invalid_xdp_action(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
344 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
348 trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); in efx_do_xdp()
|
| /Linux-v6.1/drivers/net/ethernet/intel/ixgbe/ |
| D | ixgbe_xsk.c | 14 bool xdp_on = READ_ONCE(adapter->xdp_prog); in ixgbe_xsk_pool()
102 struct bpf_prog *xdp_prog; in ixgbe_run_xdp_zc() local
107 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in ixgbe_run_xdp_zc()
108 act = bpf_prog_run_xdp(xdp_prog, xdp); in ixgbe_run_xdp_zc()
111 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ixgbe_run_xdp_zc()
141 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in ixgbe_run_xdp_zc()
146 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in ixgbe_run_xdp_zc()
527 if (!READ_ONCE(adapter->xdp_prog)) in ixgbe_xsk_wakeup()
|
| /Linux-v6.1/drivers/net/ethernet/netronome/nfp/nfd3/ |
| D | xsk.c | 121 struct bpf_prog *xdp_prog; in nfp_nfd3_xsk_rx() local
125 xdp_prog = READ_ONCE(dp->xdp_prog); in nfp_nfd3_xsk_rx()
219 act = bpf_prog_run_xdp(xdp_prog, xrxbuf->xdp); in nfp_nfd3_xsk_rx()
237 if (xdp_do_redirect(dp->netdev, xrxbuf->xdp, xdp_prog)) { in nfp_nfd3_xsk_rx()
245 bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act); in nfp_nfd3_xsk_rx()
248 trace_xdp_exception(dp->netdev, xdp_prog, act); in nfp_nfd3_xsk_rx()
|
| D | dp.c | 532 if (!dp->xdp_prog) { in nfp_nfd3_napi_alloc_one()
547 nfp_net_free_frag(frag, dp->xdp_prog); in nfp_nfd3_napi_alloc_one()
878 struct bpf_prog *xdp_prog; in nfp_nfd3_rx() local
886 xdp_prog = READ_ONCE(dp->xdp_prog); in nfp_nfd3_rx()
887 true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz; in nfp_nfd3_rx()
975 if (xdp_prog && !meta.portid) { in nfp_nfd3_rx()
985 act = bpf_prog_run_xdp(xdp_prog, &xdp); in nfp_nfd3_rx()
1003 xdp_prog, act); in nfp_nfd3_rx()
1006 bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act); in nfp_nfd3_rx()
1009 trace_xdp_exception(dp->netdev, xdp_prog, act); in nfp_nfd3_rx()
[all …]
|
| /Linux-v6.1/drivers/net/ethernet/intel/i40e/ |
| D | i40e_xsk.c | 202 struct bpf_prog *xdp_prog) in i40e_run_xdp_zc() argument
208 act = bpf_prog_run_xdp(xdp_prog, xdp); in i40e_run_xdp_zc()
211 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in i40e_run_xdp_zc()
234 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in i40e_run_xdp_zc()
239 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in i40e_run_xdp_zc()
390 struct bpf_prog *xdp_prog; in i40e_clean_rx_irq_zc() local
397 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in i40e_clean_rx_irq_zc()
435 xdp_res = i40e_run_xdp_zc(rx_ring, bi, xdp_prog); in i40e_clean_rx_irq_zc()
|
| /Linux-v6.1/drivers/net/ethernet/fungible/funeth/ |
| D | funeth_rx.c | 144 struct bpf_prog *xdp_prog; in fun_run_xdp() local
155 xdp_prog = READ_ONCE(q->xdp_prog); in fun_run_xdp()
156 act = bpf_prog_run_xdp(xdp_prog, &xdp); in fun_run_xdp()
177 if (unlikely(xdp_do_redirect(q->netdev, &xdp, xdp_prog))) in fun_run_xdp()
183 bpf_warn_invalid_xdp_action(q->netdev, xdp_prog, act); in fun_run_xdp()
186 trace_xdp_exception(q->netdev, xdp_prog, act); in fun_run_xdp()
609 WRITE_ONCE(q->xdp_prog, prog); in fun_rxq_set_bpf()
725 q->xdp_prog = fp->xdp_prog; in fun_rxq_create_dev()
726 q->headroom = fp->xdp_prog ? FUN_XDP_HEADROOM : FUN_RX_HEADROOM; in fun_rxq_create_dev()
|
| /Linux-v6.1/drivers/net/ |
| D | veth.c | 62 struct bpf_prog __rcu *xdp_prog; member
560 if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog))) in veth_xdp_flush()
590 struct bpf_prog *xdp_prog; in veth_xdp_rcv_one() local
593 xdp_prog = rcu_dereference(rq->xdp_prog); in veth_xdp_rcv_one()
594 if (likely(xdp_prog)) { in veth_xdp_rcv_one()
601 act = bpf_prog_run_xdp(xdp_prog, &xdp); in veth_xdp_rcv_one()
612 trace_xdp_exception(rq->dev, xdp_prog, act); in veth_xdp_rcv_one()
623 if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { in veth_xdp_rcv_one()
632 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); in veth_xdp_rcv_one()
635 trace_xdp_exception(rq->dev, xdp_prog, act); in veth_xdp_rcv_one()
[all …]
|
| D | tun.c | 208 struct bpf_prog __rcu *xdp_prog; member
828 if (rtnl_dereference(tun->xdp_prog)) in tun_attach()
1203 old_prog = rtnl_dereference(tun->xdp_prog); in tun_xdp_set()
1204 rcu_assign_pointer(tun->xdp_prog, prog); in tun_xdp_set()
1617 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog, in tun_xdp_act() argument
1624 err = xdp_do_redirect(tun->dev, xdp, xdp_prog); in tun_xdp_act()
1636 bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act); in tun_xdp_act()
1639 trace_xdp_exception(tun->dev, xdp_prog, act); in tun_xdp_act()
1656 struct bpf_prog *xdp_prog; in tun_build_skb() local
1664 xdp_prog = rcu_dereference(tun->xdp_prog); in tun_build_skb()
[all …]
|
| D | virtio_net.c | 150 struct bpf_prog __rcu *xdp_prog; member
634 struct bpf_prog *xdp_prog; in virtnet_xdp_xmit() local
648 xdp_prog = rcu_access_pointer(rq->xdp_prog); in virtnet_xdp_xmit()
649 if (!xdp_prog) in virtnet_xdp_xmit()
779 struct bpf_prog *xdp_prog; in receive_small() local
802 xdp_prog = NULL; in receive_small()
807 xdp_prog = rcu_dereference(rq->xdp_prog); in receive_small()
808 if (xdp_prog) { in receive_small()
843 act = bpf_prog_run_xdp(xdp_prog, &xdp); in receive_small()
862 trace_xdp_exception(vi->dev, xdp_prog, act); in receive_small()
[all …]
|
| /Linux-v6.1/drivers/net/ethernet/aquantia/atlantic/ |
| D | aq_main.c | 136 prog = READ_ONCE(aq_nic->xdp_prog); in aq_ndev_change_mtu()
228 prog = READ_ONCE(aq_nic->xdp_prog); in aq_ndev_fix_features()
230 aq_nic->xdp_prog && features & NETIF_F_LRO) { in aq_ndev_fix_features()
465 need_update = !!aq_nic->xdp_prog != !!prog; in aq_xdp_setup()
469 old_prog = xchg(&aq_nic->xdp_prog, prog); in aq_xdp_setup()
|
| /Linux-v6.1/drivers/net/ethernet/intel/ice/ |
| D | ice_xsk.c | 600 struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring) in ice_run_xdp_zc() argument
605 act = bpf_prog_run_xdp(xdp_prog, xdp); in ice_run_xdp_zc()
608 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ice_run_xdp_zc()
630 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in ice_run_xdp_zc()
635 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in ice_run_xdp_zc()
654 struct bpf_prog *xdp_prog; in ice_clean_rx_irq_zc() local
661 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in ice_clean_rx_irq_zc()
703 xdp_res = ice_run_xdp_zc(rx_ring, xdp, xdp_prog, xdp_ring); in ice_clean_rx_irq_zc()
|
| D | ice_txrx.c | 457 rx_ring->xdp_prog = NULL; in ice_free_rx_ring()
511 WRITE_ONCE(rx_ring->xdp_prog, rx_ring->vsi->xdp_prog); in ice_setup_rx_ring()
553 struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring) in ice_run_xdp() argument
558 act = bpf_prog_run_xdp(xdp_prog, xdp); in ice_run_xdp()
572 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ice_run_xdp()
577 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); in ice_run_xdp()
581 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); in ice_run_xdp()
1119 struct bpf_prog *xdp_prog = NULL; in ice_clean_rx_irq() local
1129 xdp_prog = READ_ONCE(rx_ring->xdp_prog); in ice_clean_rx_irq()
1130 if (xdp_prog) in ice_clean_rx_irq()
[all …]
|
| /Linux-v6.1/drivers/net/ethernet/stmicro/stmmac/ |
| D | stmmac_xdp.c | 120 need_update = !!priv->xdp_prog != !!prog; in stmmac_xdp_set_prog()
124 old_prog = xchg(&priv->xdp_prog, prog); in stmmac_xdp_set_prog()
|
| /Linux-v6.1/drivers/net/ethernet/mellanox/mlx4/ |
| D | en_rx.c | 438 ring->xdp_prog, in mlx4_en_destroy_rx_ring()
669 struct bpf_prog *xdp_prog; in mlx4_en_process_rx_cq() local
683 xdp_prog = rcu_dereference_bh(ring->xdp_prog); in mlx4_en_process_rx_cq()
769 if (xdp_prog) { in mlx4_en_process_rx_cq()
783 act = bpf_prog_run_xdp(xdp_prog, &xdp); in mlx4_en_process_rx_cq()
796 if (likely(!xdp_do_redirect(dev, &xdp, xdp_prog))) { in mlx4_en_process_rx_cq()
803 trace_xdp_exception(dev, xdp_prog, act); in mlx4_en_process_rx_cq()
812 trace_xdp_exception(dev, xdp_prog, act); in mlx4_en_process_rx_cq()
815 bpf_warn_invalid_xdp_action(dev, xdp_prog, act); in mlx4_en_process_rx_cq()
818 trace_xdp_exception(dev, xdp_prog, act); in mlx4_en_process_rx_cq()
|
| /Linux-v6.1/drivers/net/ethernet/netronome/nfp/nfdk/ |
| D | dp.c | 548 if (!dp->xdp_prog) { in nfp_nfdk_napi_alloc_one()
563 nfp_net_free_frag(frag, dp->xdp_prog); in nfp_nfdk_napi_alloc_one()
995 struct bpf_prog *xdp_prog; in nfp_nfdk_rx() local
1003 xdp_prog = READ_ONCE(dp->xdp_prog); in nfp_nfdk_rx()
1004 true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz; in nfp_nfdk_rx()
1089 if (xdp_prog && !meta.portid) { in nfp_nfdk_rx()
1099 act = bpf_prog_run_xdp(xdp_prog, &xdp); in nfp_nfdk_rx()
1117 xdp_prog, act); in nfp_nfdk_rx()
1120 bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act); in nfp_nfdk_rx()
1123 trace_xdp_exception(dp->netdev, xdp_prog, act); in nfp_nfdk_rx()
[all …]
|
| /Linux-v6.1/drivers/net/ethernet/mellanox/mlx5/core/en/ |
| D | xdp.h | 74 if (priv->channels.params.xdp_prog) in mlx5e_xdp_tx_enable()
80 if (priv->channels.params.xdp_prog) in mlx5e_xdp_tx_disable()
|
| D | params.c | 226 if (params->xdp_prog) in mlx5e_get_linear_rq_headroom()
263 if (params->xdp_prog) in mlx5e_rx_get_linear_stride_sz()
570 if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) in mlx5e_mpwrq_validate_regular()
707 params->xdp_prog); in mlx5e_build_rq_frags_info()
708 if (byte_count > max_mtu || params->xdp_prog) { in mlx5e_build_rq_frags_info()
713 params->xdp_prog); in mlx5e_build_rq_frags_info()
733 if (params->xdp_prog) { in mlx5e_build_rq_frags_info()
1112 if (params->xdp_prog) { in mlx5e_build_icosq_log_wq_sz()
|
| /Linux-v6.1/drivers/net/ethernet/intel/igc/ |
| D | igc_xdp.h | 14 return !!adapter->xdp_prog; in igc_xdp_is_enabled()
|
| /Linux-v6.1/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/ |
| D | pool.c | 116 if (!params->xdp_prog) { in mlx5e_xsk_enable_locked()
176 if (!priv->channels.params.xdp_prog) in mlx5e_xsk_disable_locked()
|
| /Linux-v6.1/drivers/net/ethernet/netronome/nfp/ |
| D | nfp_net_xsk.h | 15 return dp->xdp_prog && dp->xsk_pools[qid]; in nfp_net_has_xsk_pool_slow()
|
| D | nfp_net_dp.c | 21 if (!dp->xdp_prog) { in nfp_net_rx_alloc_one()
36 nfp_net_free_frag(frag, dp->xdp_prog); in nfp_net_rx_alloc_one()
149 nfp_net_free_frag(rx_ring->rxbufs[i].frag, dp->xdp_prog); in nfp_net_rx_ring_bufs_free()
|
| /Linux-v6.1/drivers/net/ethernet/socionext/ |
| D | netsec.c | 294 struct bpf_prog *xdp_prog; member
957 struct bpf_prog *xdp_prog; in netsec_process_rx() local
965 xdp_prog = READ_ONCE(priv->xdp_prog); in netsec_process_rx()
1024 if (xdp_prog) { in netsec_process_rx()
1025 xdp_result = netsec_run_xdp(priv, xdp_prog, &xdp); in netsec_process_rx()
1294 struct bpf_prog *xdp_prog = READ_ONCE(priv->xdp_prog); in netsec_setup_rx_dring() local
1302 .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE, in netsec_setup_rx_dring()
1807 old_prog = xchg(&priv->xdp_prog, prog); in netsec_xdp_setup()
|
