| /Linux-v5.4/include/net/ |
| D | xdp.h | 94 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
98 struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp) in convert_to_xdp_frame() argument
104 if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) in convert_to_xdp_frame()
105 return xdp_convert_zc_to_xdp_frame(xdp); in convert_to_xdp_frame()
108 headroom = xdp->data - xdp->data_hard_start; in convert_to_xdp_frame()
109 metasize = xdp->data - xdp->data_meta; in convert_to_xdp_frame()
115 xdp_frame = xdp->data_hard_start; in convert_to_xdp_frame()
117 xdp_frame->data = xdp->data; in convert_to_xdp_frame()
118 xdp_frame->len = xdp->data_end - xdp->data; in convert_to_xdp_frame()
123 xdp_frame->mem = xdp->rxq->mem; in convert_to_xdp_frame()
[all …]
|
| /Linux-v5.4/drivers/net/ethernet/broadcom/bnxt/ |
| D | bnxt_xdp.c | 118 struct xdp_buff xdp; in bnxt_rx_xdp() local
136 xdp.data_hard_start = *data_ptr - offset; in bnxt_rx_xdp()
137 xdp.data = *data_ptr; in bnxt_rx_xdp()
138 xdp_set_data_meta_invalid(&xdp); in bnxt_rx_xdp()
139 xdp.data_end = *data_ptr + *len; in bnxt_rx_xdp()
140 xdp.rxq = &rxr->xdp_rxq; in bnxt_rx_xdp()
141 orig_data = xdp.data; in bnxt_rx_xdp()
144 act = bpf_prog_run_xdp(xdp_prog, &xdp); in bnxt_rx_xdp()
154 *len = xdp.data_end - xdp.data; in bnxt_rx_xdp()
155 if (orig_data != xdp.data) { in bnxt_rx_xdp()
[all …]
|
| /Linux-v5.4/samples/bpf/ |
| D | xdp_adjust_tail_kern.c | 67 static __always_inline int send_icmp4_too_big(struct xdp_md *xdp) in send_icmp4_too_big() argument
71 if (bpf_xdp_adjust_head(xdp, 0 - headroom)) in send_icmp4_too_big()
73 void *data = (void *)(long)xdp->data; in send_icmp4_too_big()
74 void *data_end = (void *)(long)xdp->data_end; in send_icmp4_too_big()
117 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
119 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
120 void *data = (void *)(long)xdp->data; in handle_ipv4()
126 if (bpf_xdp_adjust_tail(xdp, 0 - offset)) in handle_ipv4()
128 return send_icmp4_too_big(xdp); in handle_ipv4()
134 int _xdp_icmp(struct xdp_md *xdp) in _xdp_icmp() argument
[all …]
|
| D | xdp_tx_iptunnel_kern.c | 77 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
79 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
80 void *data = (void *)(long)xdp->data; in handle_ipv4()
112 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
152 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
154 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
184 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| D | xdp2skb_meta.sh | 156 call_ip link show dev $device | grep --color=auto xdp
173 call_ip link set dev $device xdp off
204 call_ip link set dev $device xdp obj $file sec $prog
|
| /Linux-v5.4/tools/testing/selftests/bpf/progs/ |
| D | test_xdp.c | 80 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
82 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
83 void *data = (void *)(long)xdp->data; in handle_ipv4()
113 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
116 data = (void *)(long)xdp->data; in handle_ipv4()
117 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
153 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
155 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
156 void *data = (void *)(long)xdp->data; in handle_ipv6()
183 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| D | test_xdp_loop.c | 76 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
78 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
79 void *data = (void *)(long)xdp->data; in handle_ipv4()
109 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
112 data = (void *)(long)xdp->data; in handle_ipv4()
113 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
149 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
151 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
152 void *data = (void *)(long)xdp->data; in handle_ipv6()
179 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| D | test_adjust_tail.c | 15 int _xdp_adjust_tail(struct xdp_md *xdp) in _xdp_adjust_tail() argument
17 void *data_end = (void *)(long)xdp->data_end; in _xdp_adjust_tail()
18 void *data = (void *)(long)xdp->data; in _xdp_adjust_tail()
25 if (bpf_xdp_adjust_tail(xdp, 0 - offset)) in _xdp_adjust_tail()
|
| D | test_xdp_noinline.c | 278 bool encap_v6(struct xdp_md *xdp, struct ctl_value *cval, in encap_v6() argument
289 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in encap_v6()
291 data = (void *)(long)xdp->data; in encap_v6()
292 data_end = (void *)(long)xdp->data_end; in encap_v6()
321 bool encap_v4(struct xdp_md *xdp, struct ctl_value *cval, in encap_v4() argument
337 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in encap_v4()
339 data = (void *)(long)xdp->data; in encap_v4()
340 data_end = (void *)(long)xdp->data_end; in encap_v4()
369 if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct iphdr))) in encap_v4()
375 bool decap_v6(struct xdp_md *xdp, void **data, void **data_end, bool inner_v4) in decap_v6() argument
[all …]
|
| D | xdp_redirect_map.c | 14 int xdp_redirect_map_0(struct xdp_md *xdp) in xdp_redirect_map_0() argument
20 int xdp_redirect_map_1(struct xdp_md *xdp) in xdp_redirect_map_1() argument
26 int xdp_redirect_map_2(struct xdp_md *xdp) in xdp_redirect_map_2() argument
|
| D | test_xdp_redirect.c | 18 int xdp_redirect_to_111(struct xdp_md *xdp) in xdp_redirect_to_111() argument
23 int xdp_redirect_to_222(struct xdp_md *xdp) in xdp_redirect_to_222() argument
|
| /Linux-v5.4/include/trace/events/ |
| D | xdp.h | 3 #define TRACE_SYSTEM xdp
31 const struct bpf_prog *xdp, u32 act),
33 TP_ARGS(dev, xdp, act),
42 __entry->prog_id = xdp->aux->id;
85 const struct bpf_prog *xdp,
89 TP_ARGS(dev, xdp, to_ifindex, err, map, map_index),
102 __entry->prog_id = xdp->aux->id;
120 const struct bpf_prog *xdp,
123 TP_ARGS(dev, xdp, to_ifindex, err, map, map_index)
128 const struct bpf_prog *xdp,
[all …]
|
| /Linux-v5.4/drivers/net/ethernet/intel/ixgbe/ |
| D | ixgbe_xsk.c | 144 struct xdp_buff *xdp) in ixgbe_run_xdp_zc() argument
155 act = bpf_prog_run_xdp(xdp_prog, xdp); in ixgbe_run_xdp_zc()
156 offset = xdp->data - xdp->data_hard_start; in ixgbe_run_xdp_zc()
158 xdp->handle = xsk_umem_adjust_offset(umem, xdp->handle, offset); in ixgbe_run_xdp_zc()
164 xdpf = convert_to_xdp_frame(xdp); in ixgbe_run_xdp_zc()
172 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ixgbe_run_xdp_zc()
396 struct xdp_buff *xdp) in ixgbe_construct_skb_zc() argument
398 unsigned int metasize = xdp->data - xdp->data_meta; in ixgbe_construct_skb_zc()
399 unsigned int datasize = xdp->data_end - xdp->data; in ixgbe_construct_skb_zc()
404 xdp->data_end - xdp->data_hard_start, in ixgbe_construct_skb_zc()
[all …]
|
| /Linux-v5.4/drivers/net/ethernet/intel/i40e/ |
| D | i40e_xsk.c | 191 static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) in i40e_run_xdp_zc() argument
205 act = bpf_prog_run_xdp(xdp_prog, xdp); in i40e_run_xdp_zc()
206 offset = xdp->data - xdp->data_hard_start; in i40e_run_xdp_zc()
208 xdp->handle = xsk_umem_adjust_offset(umem, xdp->handle, offset); in i40e_run_xdp_zc()
215 result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring); in i40e_run_xdp_zc()
218 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in i40e_run_xdp_zc()
488 struct xdp_buff *xdp) in i40e_construct_skb_zc() argument
490 unsigned int metasize = xdp->data - xdp->data_meta; in i40e_construct_skb_zc()
491 unsigned int datasize = xdp->data_end - xdp->data; in i40e_construct_skb_zc()
496 xdp->data_end - xdp->data_hard_start, in i40e_construct_skb_zc()
[all …]
|
| D | i40e_txrx.c | 1996 struct xdp_buff *xdp) in i40e_construct_skb() argument
1998 unsigned int size = xdp->data_end - xdp->data; in i40e_construct_skb()
2008 prefetch(xdp->data); in i40e_construct_skb()
2010 prefetch(xdp->data + L1_CACHE_BYTES); in i40e_construct_skb()
2038 headlen = eth_get_headlen(skb->dev, xdp->data, in i40e_construct_skb()
2042 memcpy(__skb_put(skb, headlen), xdp->data, in i40e_construct_skb()
2077 struct xdp_buff *xdp) in i40e_build_skb() argument
2079 unsigned int metasize = xdp->data - xdp->data_meta; in i40e_build_skb()
2084 SKB_DATA_ALIGN(xdp->data_end - in i40e_build_skb()
2085 xdp->data_hard_start); in i40e_build_skb()
[all …]
|
| /Linux-v5.4/drivers/net/ethernet/mellanox/mlx5/core/en/ |
| D | xdp.c | 60 struct mlx5e_dma_info *di, struct xdp_buff *xdp) in mlx5e_xmit_xdp_buff() argument
67 xdpf = convert_to_xdp_frame(xdp); in mlx5e_xmit_xdp_buff()
74 if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) { in mlx5e_xmit_xdp_buff()
126 struct xdp_buff xdp; in mlx5e_xdp_handle() local
133 xdp.data = va + *rx_headroom; in mlx5e_xdp_handle()
134 xdp_set_data_meta_invalid(&xdp); in mlx5e_xdp_handle()
135 xdp.data_end = xdp.data + *len; in mlx5e_xdp_handle()
136 xdp.data_hard_start = va; in mlx5e_xdp_handle()
138 xdp.handle = di->xsk.handle; in mlx5e_xdp_handle()
139 xdp.rxq = &rq->xdp_rxq; in mlx5e_xdp_handle()
[all …]
|
| /Linux-v5.4/drivers/net/ |
| D | veth.c | 229 struct veth_rq *rq, bool xdp) in veth_forward_skb() argument
231 return __dev_forward_skb(dev, skb) ?: xdp ? in veth_forward_skb()
491 static int veth_xdp_tx(struct net_device *dev, struct xdp_buff *xdp, in veth_xdp_tx() argument
494 struct xdp_frame *frame = convert_to_xdp_frame(xdp); in veth_xdp_tx()
523 struct xdp_buff xdp; in veth_xdp_rcv_one() local
526 xdp.data_hard_start = hard_start; in veth_xdp_rcv_one()
527 xdp.data = frame->data; in veth_xdp_rcv_one()
528 xdp.data_end = frame->data + frame->len; in veth_xdp_rcv_one()
529 xdp.data_meta = frame->data - frame->metasize; in veth_xdp_rcv_one()
530 xdp.rxq = &rq->xdp_rxq; in veth_xdp_rcv_one()
[all …]
|
| /Linux-v5.4/tools/testing/selftests/bpf/ |
| D | test_xdp_veth.sh | 61 if ! ip link set dev lo xdp off > /dev/null 2>&1; then
101 xdp_redirect_map.o $BPF_DIR/progs type xdp \
106 ip link set dev veth1 xdp pinned $BPF_DIR/progs/redirect_map_0
107 ip link set dev veth2 xdp pinned $BPF_DIR/progs/redirect_map_1
108 ip link set dev veth3 xdp pinned $BPF_DIR/progs/redirect_map_2
110 ip -n ns1 link set dev veth11 xdp obj xdp_dummy.o sec xdp_dummy
111 ip -n ns2 link set dev veth22 xdp obj xdp_tx.o sec tx
112 ip -n ns3 link set dev veth33 xdp obj xdp_dummy.o sec xdp_dummy
|
| D | test_tcp_check_syncookie_user.c | 81 static int get_map_fd_by_prog_id(int prog_id, bool *xdp) in get_map_fd_by_prog_id() argument
108 *xdp = info.type == BPF_PROG_TYPE_XDP; in get_map_fd_by_prog_id()
119 static int run_test(int server_fd, int results_fd, bool xdp) in run_test() argument
170 if (xdp && value_gen == 0) { in run_test()
210 bool xdp; in main() local
217 results = get_map_fd_by_prog_id(atoi(argv[1]), &xdp); in main()
242 if (run_test(server, results, xdp)) in main()
245 if (run_test(server_v6, results, xdp)) in main()
|
| D | test_xdp_meta.sh | 17 ip link set dev lo xdp off 2>/dev/null > /dev/null
43 ip netns exec ns1 ip link set dev veth1 xdp obj test_xdp_meta.o sec x
44 ip netns exec ns2 ip link set dev veth2 xdp obj test_xdp_meta.o sec x
|
| /Linux-v5.4/net/xdp/ |
| D | xsk.c | 141 static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) in __xsk_rcv() argument
155 if (unlikely(xdp_data_meta_unsupported(xdp))) { in __xsk_rcv()
156 from_buf = xdp->data; in __xsk_rcv()
159 from_buf = xdp->data_meta; in __xsk_rcv()
160 metalen = xdp->data - xdp->data_meta; in __xsk_rcv()
171 xdp_return_buff(xdp); in __xsk_rcv()
179 static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) in __xsk_rcv_zc() argument
181 int err = xskq_produce_batch_desc(xs->rx, (u64)xdp->handle, len); in __xsk_rcv_zc()
199 int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) in xsk_rcv() argument
206 if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) in xsk_rcv()
[all …]
|
| /Linux-v5.4/net/bpf/ |
| D | test_run.c | 348 struct xdp_buff xdp = {}; in bpf_prog_test_run_xdp() local
360 xdp.data_hard_start = data; in bpf_prog_test_run_xdp()
361 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN; in bpf_prog_test_run_xdp()
362 xdp.data_meta = xdp.data; in bpf_prog_test_run_xdp()
363 xdp.data_end = xdp.data + size; in bpf_prog_test_run_xdp()
366 xdp.rxq = &rxqueue->xdp_rxq; in bpf_prog_test_run_xdp()
368 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration); in bpf_prog_test_run_xdp()
371 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN || in bpf_prog_test_run_xdp()
372 xdp.data_end != xdp.data + size) in bpf_prog_test_run_xdp()
373 size = xdp.data_end - xdp.data; in bpf_prog_test_run_xdp()
[all …]
|
| /Linux-v5.4/net/core/ |
| D | xdp.c | 448 void xdp_return_buff(struct xdp_buff *xdp) in xdp_return_buff() argument
450 __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle); in xdp_return_buff()
500 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) in xdp_convert_zc_to_xdp_frame() argument
508 metasize = xdp_data_meta_unsupported(xdp) ? 0 : in xdp_convert_zc_to_xdp_frame()
509 xdp->data - xdp->data_meta; in xdp_convert_zc_to_xdp_frame()
510 totsize = xdp->data_end - xdp->data + metasize; in xdp_convert_zc_to_xdp_frame()
524 data_to_copy = metasize ? xdp->data_meta : xdp->data; in xdp_convert_zc_to_xdp_frame()
533 xdp_return_buff(xdp); in xdp_convert_zc_to_xdp_frame()
|
| /Linux-v5.4/tools/bpf/bpftool/Documentation/ |
| D | bpftool-net.rst | 29 | *ATTACH_TYPE* := { **xdp** | **xdpgeneric** | **xdpdrv** | **xdpoffload** }
36 Currently, only device driver xdp attachments and tc filter
47 The current output will start with all xdp program attachments, followed by
48 all tc class/qdisc bpf program attachments. Both xdp programs and
62 … **xdp** - try native XDP and fallback to generic XDP if NIC driver does not support it;
102 xdp:
118 "xdp": [{
169 xdp:
179 xdp:
189 xdp:
|
| /Linux-v5.4/drivers/net/ethernet/socionext/ |
| D | netsec.c | 866 static u32 netsec_xdp_xmit_back(struct netsec_priv *priv, struct xdp_buff *xdp) in netsec_xdp_xmit_back() argument
869 struct xdp_frame *xdpf = convert_to_xdp_frame(xdp); in netsec_xdp_xmit_back()
883 struct xdp_buff *xdp) in netsec_run_xdp() argument
889 act = bpf_prog_run_xdp(prog, xdp); in netsec_run_xdp()
896 ret = netsec_xdp_xmit_back(priv, xdp); in netsec_run_xdp()
898 xdp_return_buff(xdp); in netsec_run_xdp()
901 err = xdp_do_redirect(priv->ndev, xdp, prog); in netsec_run_xdp()
906 xdp_return_buff(xdp); in netsec_run_xdp()
917 xdp_return_buff(xdp); in netsec_run_xdp()
948 struct xdp_buff xdp; in netsec_process_rx() local
[all …]
|
