1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
3
4 #include "rx.h"
5 #include "en/xdp.h"
6 #include <net/xdp_sock.h>
7
8 /* RX data path */
9
mlx5e_xsk_pages_enough_umem(struct mlx5e_rq * rq,int count)10 bool mlx5e_xsk_pages_enough_umem(struct mlx5e_rq *rq, int count)
11 {
12 /* Check in advance that we have enough frames, instead of allocating
13 * one-by-one, failing and moving frames to the Reuse Ring.
14 */
15 return xsk_umem_has_addrs_rq(rq->umem, count);
16 }
17
mlx5e_xsk_page_alloc_umem(struct mlx5e_rq * rq,struct mlx5e_dma_info * dma_info)18 int mlx5e_xsk_page_alloc_umem(struct mlx5e_rq *rq,
19 struct mlx5e_dma_info *dma_info)
20 {
21 struct xdp_umem *umem = rq->umem;
22 u64 handle;
23
24 if (!xsk_umem_peek_addr_rq(umem, &handle))
25 return -ENOMEM;
26
27 dma_info->xsk.handle = xsk_umem_adjust_offset(umem, handle,
28 rq->buff.umem_headroom);
29 dma_info->xsk.data = xdp_umem_get_data(umem, dma_info->xsk.handle);
30
31 /* No need to add headroom to the DMA address. In striding RQ case, we
32 * just provide pages for UMR, and headroom is counted at the setup
33 * stage when creating a WQE. In non-striding RQ case, headroom is
34 * accounted in mlx5e_alloc_rx_wqe.
35 */
36 dma_info->addr = xdp_umem_get_dma(umem, handle);
37
38 xsk_umem_discard_addr_rq(umem);
39
40 dma_sync_single_for_device(rq->pdev, dma_info->addr, PAGE_SIZE,
41 DMA_BIDIRECTIONAL);
42
43 return 0;
44 }
45
mlx5e_xsk_recycle_frame(struct mlx5e_rq * rq,u64 handle)46 static inline void mlx5e_xsk_recycle_frame(struct mlx5e_rq *rq, u64 handle)
47 {
48 xsk_umem_fq_reuse(rq->umem, handle & rq->umem->chunk_mask);
49 }
50
51 /* XSKRQ uses pages from UMEM, they must not be released. They are returned to
52 * the userspace if possible, and if not, this function is called to reuse them
53 * in the driver.
54 */
mlx5e_xsk_page_release(struct mlx5e_rq * rq,struct mlx5e_dma_info * dma_info)55 void mlx5e_xsk_page_release(struct mlx5e_rq *rq,
56 struct mlx5e_dma_info *dma_info)
57 {
58 mlx5e_xsk_recycle_frame(rq, dma_info->xsk.handle);
59 }
60
61 /* Return a frame back to the hardware to fill in again. It is used by XDP when
62 * the XDP program returns XDP_TX or XDP_REDIRECT not to an XSKMAP.
63 */
mlx5e_xsk_zca_free(struct zero_copy_allocator * zca,unsigned long handle)64 void mlx5e_xsk_zca_free(struct zero_copy_allocator *zca, unsigned long handle)
65 {
66 struct mlx5e_rq *rq = container_of(zca, struct mlx5e_rq, zca);
67
68 mlx5e_xsk_recycle_frame(rq, handle);
69 }
70
mlx5e_xsk_construct_skb(struct mlx5e_rq * rq,void * data,u32 cqe_bcnt)71 static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, void *data,
72 u32 cqe_bcnt)
73 {
74 struct sk_buff *skb;
75
76 skb = napi_alloc_skb(rq->cq.napi, cqe_bcnt);
77 if (unlikely(!skb)) {
78 rq->stats->buff_alloc_err++;
79 return NULL;
80 }
81
82 skb_put_data(skb, data, cqe_bcnt);
83
84 return skb;
85 }
86
mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq * rq,struct mlx5e_mpw_info * wi,u16 cqe_bcnt,u32 head_offset,u32 page_idx)87 struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
88 struct mlx5e_mpw_info *wi,
89 u16 cqe_bcnt,
90 u32 head_offset,
91 u32 page_idx)
92 {
93 struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
94 u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
95 u32 cqe_bcnt32 = cqe_bcnt;
96 void *va, *data;
97 u32 frag_size;
98 bool consumed;
99
100 /* Check packet size. Note LRO doesn't use linear SKB */
101 if (unlikely(cqe_bcnt > rq->hw_mtu)) {
102 rq->stats->oversize_pkts_sw_drop++;
103 return NULL;
104 }
105
106 /* head_offset is not used in this function, because di->xsk.data and
107 * di->addr point directly to the necessary place. Furthermore, in the
108 * current implementation, UMR pages are mapped to XSK frames, so
109 * head_offset should always be 0.
110 */
111 WARN_ON_ONCE(head_offset);
112
113 va = di->xsk.data;
114 data = va + rx_headroom;
115 frag_size = rq->buff.headroom + cqe_bcnt32;
116
117 dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL);
118 prefetch(data);
119
120 rcu_read_lock();
121 consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32, true);
122 rcu_read_unlock();
123
124 /* Possible flows:
125 * - XDP_REDIRECT to XSKMAP:
126 * The page is owned by the userspace from now.
127 * - XDP_TX and other XDP_REDIRECTs:
128 * The page was returned by ZCA and recycled.
129 * - XDP_DROP:
130 * Recycle the page.
131 * - XDP_PASS:
132 * Allocate an SKB, copy the data and recycle the page.
133 *
134 * Pages to be recycled go to the Reuse Ring on MPWQE deallocation. Its
135 * size is the same as the Driver RX Ring's size, and pages for WQEs are
136 * allocated first from the Reuse Ring, so it has enough space.
137 */
138
139 if (likely(consumed)) {
140 if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)))
141 __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
142 return NULL; /* page/packet was consumed by XDP */
143 }
144
145 /* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the
146 * frame. On SKB allocation failure, NULL is returned.
147 */
148 return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt32);
149 }
150
mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq * rq,struct mlx5_cqe64 * cqe,struct mlx5e_wqe_frag_info * wi,u32 cqe_bcnt)151 struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
152 struct mlx5_cqe64 *cqe,
153 struct mlx5e_wqe_frag_info *wi,
154 u32 cqe_bcnt)
155 {
156 struct mlx5e_dma_info *di = wi->di;
157 u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
158 void *va, *data;
159 bool consumed;
160 u32 frag_size;
161
162 /* wi->offset is not used in this function, because di->xsk.data and
163 * di->addr point directly to the necessary place. Furthermore, in the
164 * current implementation, one page = one packet = one frame, so
165 * wi->offset should always be 0.
166 */
167 WARN_ON_ONCE(wi->offset);
168
169 va = di->xsk.data;
170 data = va + rx_headroom;
171 frag_size = rq->buff.headroom + cqe_bcnt;
172
173 dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL);
174 prefetch(data);
175
176 if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)) {
177 rq->stats->wqe_err++;
178 return NULL;
179 }
180
181 rcu_read_lock();
182 consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt, true);
183 rcu_read_unlock();
184
185 if (likely(consumed))
186 return NULL; /* page/packet was consumed by XDP */
187
188 /* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse
189 * will be handled by mlx5e_put_rx_frag.
190 * On SKB allocation failure, NULL is returned.
191 */
192 return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt);
193 }
194
