| /Linux-v6.1/include/net/ |
| D | xdp.h | 88 static __always_inline bool xdp_buff_has_frags(struct xdp_buff *xdp) in xdp_buff_has_frags() argument
90 return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS); in xdp_buff_has_frags()
93 static __always_inline void xdp_buff_set_frags_flag(struct xdp_buff *xdp) in xdp_buff_set_frags_flag() argument
95 xdp->flags |= XDP_FLAGS_HAS_FRAGS; in xdp_buff_set_frags_flag()
98 static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp) in xdp_buff_clear_frags_flag() argument
100 xdp->flags &= ~XDP_FLAGS_HAS_FRAGS; in xdp_buff_clear_frags_flag()
103 static __always_inline bool xdp_buff_is_frag_pfmemalloc(struct xdp_buff *xdp) in xdp_buff_is_frag_pfmemalloc() argument
105 return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC); in xdp_buff_is_frag_pfmemalloc()
108 static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp) in xdp_buff_set_frag_pfmemalloc() argument
110 xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC; in xdp_buff_set_frag_pfmemalloc()
[all …]
|
| D | xdp_sock_drv.h | 53 return pool->heads[0].xdp.rxq->napi_id; in xsk_pool_get_napi_id()
73 static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_dma() argument
75 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_dma()
80 static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_frame_dma() argument
82 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_frame_dma()
93 static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) in xsk_buff_alloc_batch() argument
95 return xp_alloc_batch(pool, xdp, max); in xsk_buff_alloc_batch()
103 static inline void xsk_buff_free(struct xdp_buff *xdp) in xsk_buff_free() argument
105 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_free()
110 static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size) in xsk_buff_set_size() argument
[all …]
|
| /Linux-v6.1/drivers/net/ethernet/broadcom/bnxt/ |
| D | bnxt_xdp.c | 28 struct xdp_buff *xdp) in bnxt_xmit_bd() argument
38 if (xdp && xdp_buff_has_frags(xdp)) { in bnxt_xmit_bd()
39 sinfo = xdp_get_shared_info_from_buff(xdp); in bnxt_xmit_bd()
47 if (xdp) in bnxt_xmit_bd()
48 tx_buf->page = virt_to_head_page(xdp->data); in bnxt_xmit_bd()
104 struct xdp_buff *xdp) in __bnxt_xmit_xdp() argument
108 tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp); in __bnxt_xmit_xdp()
181 struct xdp_buff *xdp) in bnxt_xdp_buff_init() argument
199 xdp_init_buff(xdp, buflen, &rxr->xdp_rxq); in bnxt_xdp_buff_init()
200 xdp_prepare_buff(xdp, *data_ptr - offset, offset, *len, false); in bnxt_xdp_buff_init()
[all …]
|
| D | bnxt_xdp.h | 18 struct xdp_buff *xdp);
21 struct xdp_buff xdp, struct page *page, unsigned int *len,
23 int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp);
31 struct xdp_buff *xdp);
33 struct xdp_buff *xdp);
36 struct xdp_buff *xdp,
|
| /Linux-v6.1/samples/bpf/ |
| D | xdp_adjust_tail_kern.c | 70 static __always_inline int send_icmp4_too_big(struct xdp_md *xdp) in send_icmp4_too_big() argument
74 if (bpf_xdp_adjust_head(xdp, 0 - headroom)) in send_icmp4_too_big()
76 void *data = (void *)(long)xdp->data; in send_icmp4_too_big()
77 void *data_end = (void *)(long)xdp->data_end; in send_icmp4_too_big()
120 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
122 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
123 void *data = (void *)(long)xdp->data; in handle_ipv4()
129 if (bpf_xdp_adjust_tail(xdp, 0 - offset)) in handle_ipv4()
131 return send_icmp4_too_big(xdp); in handle_ipv4()
137 int _xdp_icmp(struct xdp_md *xdp) in _xdp_icmp() argument
[all …]
|
| D | xdp_tx_iptunnel_kern.c | 77 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
79 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
80 void *data = (void *)(long)xdp->data; in handle_ipv4()
112 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
152 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
154 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
184 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| /Linux-v6.1/tools/testing/selftests/bpf/progs/ |
| D | test_xdp_do_redirect.c | 16 int xdp_redirect(struct xdp_md *xdp) in xdp_redirect() argument
18 __u32 *metadata = (void *)(long)xdp->data_meta; in xdp_redirect()
19 void *data_end = (void *)(long)xdp->data_end; in xdp_redirect()
20 void *data = (void *)(long)xdp->data; in xdp_redirect()
28 if (xdp->ingress_ifindex != ifindex_in) in xdp_redirect()
42 if (bpf_xdp_adjust_meta(xdp, 4)) in xdp_redirect()
73 int xdp_count_pkts(struct xdp_md *xdp) in xdp_count_pkts() argument
75 void *data = (void *)(long)xdp->data; in xdp_count_pkts()
76 void *data_end = (void *)(long)xdp->data_end; in xdp_count_pkts()
|
| D | test_xdp.c | 78 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
80 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
81 void *data = (void *)(long)xdp->data; in handle_ipv4()
111 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
114 data = (void *)(long)xdp->data; in handle_ipv4()
115 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
151 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
153 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
154 void *data = (void *)(long)xdp->data; in handle_ipv6()
181 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| D | test_xdp_loop.c | 74 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
76 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
77 void *data = (void *)(long)xdp->data; in handle_ipv4()
107 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
110 data = (void *)(long)xdp->data; in handle_ipv4()
111 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
147 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
149 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
150 void *data = (void *)(long)xdp->data; in handle_ipv6()
177 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| D | test_xdp_bpf2bpf.c | 45 int BPF_PROG(trace_on_entry, struct xdp_buff *xdp) in BPF_PROG() argument
48 void *data_end = (void *)(long)xdp->data_end; in BPF_PROG()
49 void *data = (void *)(long)xdp->data; in BPF_PROG()
51 meta.ifindex = xdp->rxq->dev->ifindex; in BPF_PROG()
52 meta.pkt_len = bpf_xdp_get_buff_len((struct xdp_md *)xdp); in BPF_PROG()
53 bpf_xdp_output(xdp, &perf_buf_map, in BPF_PROG()
58 test_result_fentry = xdp->rxq->dev->ifindex; in BPF_PROG()
64 int BPF_PROG(trace_on_exit, struct xdp_buff *xdp, int ret) in BPF_PROG() argument
|
| D | test_xdp_adjust_tail_shrink.c | 13 int _xdp_adjust_tail_shrink(struct xdp_md *xdp) in _xdp_adjust_tail_shrink() argument
15 __u8 *data_end = (void *)(long)xdp->data_end; in _xdp_adjust_tail_shrink()
16 __u8 *data = (void *)(long)xdp->data; in _xdp_adjust_tail_shrink()
19 switch (bpf_xdp_get_buff_len(xdp)) { in _xdp_adjust_tail_shrink()
47 if (bpf_xdp_adjust_tail(xdp, 0 - offset)) in _xdp_adjust_tail_shrink()
|
| D | test_xdp_adjust_tail_grow.c | 6 int _xdp_adjust_tail_grow(struct xdp_md *xdp) in _xdp_adjust_tail_grow() argument
8 void *data_end = (void *)(long)xdp->data_end; in _xdp_adjust_tail_grow()
9 void *data = (void *)(long)xdp->data; in _xdp_adjust_tail_grow()
10 int data_len = bpf_xdp_get_buff_len(xdp); in _xdp_adjust_tail_grow()
32 if (bpf_xdp_adjust_tail(xdp, offset)) in _xdp_adjust_tail_grow()
|
| D | test_xdp_update_frags.c | 14 int xdp_adjust_frags(struct xdp_md *xdp) in xdp_adjust_frags() argument
16 __u8 *data_end = (void *)(long)xdp->data_end; in xdp_adjust_frags()
17 __u8 *data = (void *)(long)xdp->data; in xdp_adjust_frags()
26 err = bpf_xdp_load_bytes(xdp, offset, val, sizeof(val)); in xdp_adjust_frags()
35 err = bpf_xdp_store_bytes(xdp, offset, val, sizeof(val)); in xdp_adjust_frags()
|
| D | test_xdp_context_test_run.c | 6 int xdp_context(struct xdp_md *xdp) in xdp_context() argument
8 void *data = (void *)(long)xdp->data; in xdp_context()
9 __u32 *metadata = (void *)(long)xdp->data_meta; in xdp_context()
15 if (bpf_xdp_adjust_meta(xdp, 4)) in xdp_context()
|
| /Linux-v6.1/drivers/net/ethernet/intel/ixgbe/ |
| D | ixgbe_xsk.c | 99 struct xdp_buff *xdp) in ixgbe_run_xdp_zc() argument
108 act = bpf_prog_run_xdp(xdp_prog, xdp); in ixgbe_run_xdp_zc()
111 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ixgbe_run_xdp_zc()
125 xdpf = xdp_convert_buff_to_frame(xdp); in ixgbe_run_xdp_zc()
168 bi->xdp = xsk_buff_alloc(rx_ring->xsk_pool); in ixgbe_alloc_rx_buffers_zc()
169 if (!bi->xdp) { in ixgbe_alloc_rx_buffers_zc()
174 dma = xsk_buff_xdp_get_dma(bi->xdp); in ixgbe_alloc_rx_buffers_zc()
214 const struct xdp_buff *xdp) in ixgbe_construct_skb_zc() argument
216 unsigned int totalsize = xdp->data_end - xdp->data_meta; in ixgbe_construct_skb_zc()
217 unsigned int metasize = xdp->data - xdp->data_meta; in ixgbe_construct_skb_zc()
[all …]
|
| /Linux-v6.1/include/trace/events/ |
| D | xdp.h | 3 #define TRACE_SYSTEM xdp
31 const struct bpf_prog *xdp, u32 act),
33 TP_ARGS(dev, xdp, act),
42 __entry->prog_id = xdp->aux->id;
92 const struct bpf_prog *xdp,
97 TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index),
123 __entry->prog_id = xdp->aux->id;
142 const struct bpf_prog *xdp,
146 TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index)
151 const struct bpf_prog *xdp,
[all …]
|
| /Linux-v6.1/drivers/net/ethernet/netronome/nfp/ |
| D | nfp_net_xsk.c | 18 struct xdp_buff *xdp) in nfp_net_xsk_rx_bufs_stash() argument
27 rx_ring->xsk_rxbufs[idx].xdp = xdp; in nfp_net_xsk_rx_bufs_stash()
29 xsk_buff_xdp_get_frame_dma(xdp) + headroom; in nfp_net_xsk_rx_bufs_stash()
35 rxbuf->xdp = NULL; in nfp_net_xsk_rx_unstash()
40 if (rxbuf->xdp) in nfp_net_xsk_rx_free()
41 xsk_buff_free(rxbuf->xdp); in nfp_net_xsk_rx_free()
62 struct xdp_buff *xdp; in nfp_net_xsk_rx_ring_fill_freelist() local
67 xdp = xsk_buff_alloc(pool); in nfp_net_xsk_rx_ring_fill_freelist()
68 if (!xdp) in nfp_net_xsk_rx_ring_fill_freelist()
71 nfp_net_xsk_rx_bufs_stash(rx_ring, wr_idx, xdp); in nfp_net_xsk_rx_ring_fill_freelist()
|
| /Linux-v6.1/drivers/net/ethernet/netronome/nfp/nfd3/ |
| D | xsk.c | 35 txbuf->xdp = xrxbuf->xdp; in nfp_nfd3_xsk_tx_xdp()
87 skb_put_data(skb, xrxbuf->xdp->data, pkt_len); in nfp_nfd3_xsk_rx_skb()
105 xrxbuf->xdp->data - xrxbuf->xdp->data_meta); in nfp_nfd3_xsk_rx_skb()
184 xrxbuf->xdp->data += meta_len; in nfp_nfd3_xsk_rx()
185 xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len; in nfp_nfd3_xsk_rx()
186 xdp_set_data_meta_invalid(xrxbuf->xdp); in nfp_nfd3_xsk_rx()
187 xsk_buff_dma_sync_for_cpu(xrxbuf->xdp, r_vec->xsk_pool); in nfp_nfd3_xsk_rx()
188 net_prefetch(xrxbuf->xdp->data); in nfp_nfd3_xsk_rx()
192 xrxbuf->xdp->data - in nfp_nfd3_xsk_rx()
194 xrxbuf->xdp->data, in nfp_nfd3_xsk_rx()
[all …]
|
| /Linux-v6.1/drivers/net/hyperv/ |
| D | netvsc_bpf.c | 25 struct xdp_buff *xdp) in netvsc_run_xdp() argument
35 xdp->data_hard_start = NULL; in netvsc_run_xdp()
56 xdp_init_buff(xdp, PAGE_SIZE, &nvchan->xdp_rxq); in netvsc_run_xdp()
57 xdp_prepare_buff(xdp, page_address(page), NETVSC_XDP_HDRM, len, false); in netvsc_run_xdp()
59 memcpy(xdp->data, data, len); in netvsc_run_xdp()
61 act = bpf_prog_run_xdp(prog, xdp); in netvsc_run_xdp()
73 if (!xdp_do_redirect(ndev, xdp, prog)) { in netvsc_run_xdp()
107 xdp->data_hard_start = NULL; in netvsc_run_xdp()
167 struct netdev_bpf xdp; in netvsc_vf_setxdp() local
178 memset(&xdp, 0, sizeof(xdp)); in netvsc_vf_setxdp()
[all …]
|
| /Linux-v6.1/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/ |
| D | rx.c | 208 static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, struct xdp_buff *xdp) in mlx5e_xsk_construct_skb() argument
210 u32 totallen = xdp->data_end - xdp->data_meta; in mlx5e_xsk_construct_skb()
211 u32 metalen = xdp->data - xdp->data_meta; in mlx5e_xsk_construct_skb()
220 skb_put_data(skb, xdp->data_meta, totallen); in mlx5e_xsk_construct_skb()
236 struct xdp_buff *xdp = wi->alloc_units[page_idx].xsk; in mlx5e_xsk_skb_from_cqe_mpwrq_linear() local
252 xsk_buff_set_size(xdp, cqe_bcnt); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
253 xsk_buff_dma_sync_for_cpu(xdp, rq->xsk_pool); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
254 net_prefetch(xdp->data); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
272 if (likely(prog && mlx5e_xdp_handle(rq, NULL, prog, xdp))) { in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
281 return mlx5e_xsk_construct_skb(rq, xdp); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
[all …]
|
| /Linux-v6.1/tools/testing/selftests/bpf/ |
| D | test_xdp_veth.sh | 64 if ! ip link set dev lo xdp off > /dev/null 2>&1; then
104 xdp_redirect_map.bpf.o $BPF_DIR/progs type xdp \
109 ip link set dev veth1 xdp pinned $BPF_DIR/progs/xdp_redirect_map_0
110 ip link set dev veth2 xdp pinned $BPF_DIR/progs/xdp_redirect_map_1
111 ip link set dev veth3 xdp pinned $BPF_DIR/progs/xdp_redirect_map_2
113 ip -n ${NS1} link set dev veth11 xdp obj xdp_dummy.bpf.o sec xdp
114 ip -n ${NS2} link set dev veth22 xdp obj xdp_tx.bpf.o sec xdp
115 ip -n ${NS3} link set dev veth33 xdp obj xdp_dummy.bpf.o sec xdp
|
| D | test_tcp_check_syncookie_user.c | 82 static int get_map_fd_by_prog_id(int prog_id, bool *xdp) in get_map_fd_by_prog_id() argument
109 *xdp = info.type == BPF_PROG_TYPE_XDP; in get_map_fd_by_prog_id()
120 static int run_test(int server_fd, int results_fd, bool xdp, in run_test() argument
172 if (xdp && value_gen == 0) { in run_test()
230 bool xdp; in main() local
240 results = get_map_fd_by_prog_id(atoi(argv[1]), &xdp); in main()
277 if (run_test(server, results, xdp, in main()
281 if (run_test(server_v6, results, xdp, in main()
285 if (run_test(server_dual, results, xdp, in main()
|
| /Linux-v6.1/net/bpf/ |
| D | test_run.c | 124 struct xdp_test_data *xdp = arg; in xdp_test_run_init_page() local
129 orig_ctx = xdp->orig_ctx; in xdp_test_run_init_page()
139 xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq); in xdp_test_run_init_page()
149 static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx) in xdp_test_run_setup() argument
156 .pool_size = xdp->batch_size, in xdp_test_run_setup()
159 .init_arg = xdp, in xdp_test_run_setup()
162 xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); in xdp_test_run_setup()
163 if (!xdp->frames) in xdp_test_run_setup()
166 xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); in xdp_test_run_setup()
167 if (!xdp->skbs) in xdp_test_run_setup()
[all …]
|
| /Linux-v6.1/drivers/net/ethernet/intel/ice/ |
| D | ice_xsk.c | 447 static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, in ice_fill_rx_descs() argument
454 buffs = xsk_buff_alloc_batch(pool, xdp, count); in ice_fill_rx_descs()
456 dma = xsk_buff_xdp_get_dma(*xdp); in ice_fill_rx_descs()
461 xdp++; in ice_fill_rx_descs()
484 struct xdp_buff **xdp; in __ice_alloc_rx_bufs_zc() local
487 xdp = ice_xdp_buf(rx_ring, ntu); in __ice_alloc_rx_bufs_zc()
490 nb_buffs_extra = ice_fill_rx_descs(rx_ring->xsk_pool, xdp, in __ice_alloc_rx_bufs_zc()
498 xdp = ice_xdp_buf(rx_ring, 0); in __ice_alloc_rx_bufs_zc()
504 nb_buffs = ice_fill_rx_descs(rx_ring->xsk_pool, xdp, rx_desc, count); in __ice_alloc_rx_bufs_zc()
564 ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) in ice_construct_skb_zc() argument
[all …]
|
| /Linux-v6.1/net/core/ |
| D | xdp.c | 376 struct xdp_buff *xdp) in __xdp_return() argument
399 xsk_buff_free(xdp); in __xdp_return()
513 void xdp_return_buff(struct xdp_buff *xdp) in xdp_return_buff() argument
518 if (likely(!xdp_buff_has_frags(xdp))) in xdp_return_buff()
521 sinfo = xdp_get_shared_info_from_buff(xdp); in xdp_return_buff()
525 __xdp_return(page_address(page), &xdp->rxq->mem, true, xdp); in xdp_return_buff()
528 __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp); in xdp_return_buff()
557 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) in xdp_convert_zc_to_xdp_frame() argument
565 metasize = xdp_data_meta_unsupported(xdp) ? 0 : in xdp_convert_zc_to_xdp_frame()
566 xdp->data - xdp->data_meta; in xdp_convert_zc_to_xdp_frame()
[all …]
|
