1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
3
4 #include "rx.h"
5 #include "en/xdp.h"
6 #include <net/xdp_sock_drv.h>
7
8 /* RX data path */
9
mlx5e_xsk_construct_skb(struct mlx5e_rq * rq,void * data,u32 cqe_bcnt)10 static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, void *data,
11 u32 cqe_bcnt)
12 {
13 struct sk_buff *skb;
14
15 skb = napi_alloc_skb(rq->cq.napi, cqe_bcnt);
16 if (unlikely(!skb)) {
17 rq->stats->buff_alloc_err++;
18 return NULL;
19 }
20
21 skb_put_data(skb, data, cqe_bcnt);
22
23 return skb;
24 }
25
mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq * rq,struct mlx5e_mpw_info * wi,u16 cqe_bcnt,u32 head_offset,u32 page_idx)26 struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
27 struct mlx5e_mpw_info *wi,
28 u16 cqe_bcnt,
29 u32 head_offset,
30 u32 page_idx)
31 {
32 struct xdp_buff *xdp = wi->umr.dma_info[page_idx].xsk;
33 u32 cqe_bcnt32 = cqe_bcnt;
34
35 /* Check packet size. Note LRO doesn't use linear SKB */
36 if (unlikely(cqe_bcnt > rq->hw_mtu)) {
37 rq->stats->oversize_pkts_sw_drop++;
38 return NULL;
39 }
40
41 /* head_offset is not used in this function, because xdp->data and the
42 * DMA address point directly to the necessary place. Furthermore, in
43 * the current implementation, UMR pages are mapped to XSK frames, so
44 * head_offset should always be 0.
45 */
46 WARN_ON_ONCE(head_offset);
47
48 xdp->data_end = xdp->data + cqe_bcnt32;
49 xdp_set_data_meta_invalid(xdp);
50 xsk_buff_dma_sync_for_cpu(xdp, rq->xsk_pool);
51 net_prefetch(xdp->data);
52
53 /* Possible flows:
54 * - XDP_REDIRECT to XSKMAP:
55 * The page is owned by the userspace from now.
56 * - XDP_TX and other XDP_REDIRECTs:
57 * The page was returned by ZCA and recycled.
58 * - XDP_DROP:
59 * Recycle the page.
60 * - XDP_PASS:
61 * Allocate an SKB, copy the data and recycle the page.
62 *
63 * Pages to be recycled go to the Reuse Ring on MPWQE deallocation. Its
64 * size is the same as the Driver RX Ring's size, and pages for WQEs are
65 * allocated first from the Reuse Ring, so it has enough space.
66 */
67
68 if (likely(mlx5e_xdp_handle(rq, NULL, &cqe_bcnt32, xdp))) {
69 if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)))
70 __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
71 return NULL; /* page/packet was consumed by XDP */
72 }
73
74 /* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the
75 * frame. On SKB allocation failure, NULL is returned.
76 */
77 return mlx5e_xsk_construct_skb(rq, xdp->data, cqe_bcnt32);
78 }
79
mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq * rq,struct mlx5_cqe64 * cqe,struct mlx5e_wqe_frag_info * wi,u32 cqe_bcnt)80 struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
81 struct mlx5_cqe64 *cqe,
82 struct mlx5e_wqe_frag_info *wi,
83 u32 cqe_bcnt)
84 {
85 struct xdp_buff *xdp = wi->di->xsk;
86
87 /* wi->offset is not used in this function, because xdp->data and the
88 * DMA address point directly to the necessary place. Furthermore, the
89 * XSK allocator allocates frames per packet, instead of pages, so
90 * wi->offset should always be 0.
91 */
92 WARN_ON_ONCE(wi->offset);
93
94 xdp->data_end = xdp->data + cqe_bcnt;
95 xdp_set_data_meta_invalid(xdp);
96 xsk_buff_dma_sync_for_cpu(xdp, rq->xsk_pool);
97 net_prefetch(xdp->data);
98
99 if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)) {
100 rq->stats->wqe_err++;
101 return NULL;
102 }
103
104 if (likely(mlx5e_xdp_handle(rq, NULL, &cqe_bcnt, xdp)))
105 return NULL; /* page/packet was consumed by XDP */
106
107 /* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse
108 * will be handled by mlx5e_put_rx_frag.
109 * On SKB allocation failure, NULL is returned.
110 */
111 return mlx5e_xsk_construct_skb(rq, xdp->data, cqe_bcnt);
112 }
113
